Fix diff blob excerpt expansion (#35922)

And add comments and tests

go.mod

@@ -117,9 +117,9 @@ require (
 	github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc
 	github.com/yuin/goldmark-meta v1.1.0
 	gitlab.com/gitlab-org/api/client-go v0.142.4
-	golang.org/x/crypto v0.42.0
+	golang.org/x/crypto v0.43.0
 	golang.org/x/image v0.30.0
-	golang.org/x/net v0.44.0
+	golang.org/x/net v0.45.0
 	golang.org/x/oauth2 v0.30.0
 	golang.org/x/sync v0.17.0
 	golang.org/x/sys v0.37.0

go.sum

@@ -840,8 +840,8 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
-golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI=
-golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8=
+golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
+golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -908,8 +908,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
-golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
+golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM=
+golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -987,8 +987,8 @@ golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
-golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ=
-golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA=
+golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
+golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

modules/repository/env.go

@@ -25,6 +25,7 @@ const (
 	EnvKeyID        = "GITEA_KEY_ID" // public key ID
 	EnvDeployKeyID  = "GITEA_DEPLOY_KEY_ID"
 	EnvPRID         = "GITEA_PR_ID"
+	EnvPRIndex      = "GITEA_PR_INDEX" // not used by Gitea at the moment, it is for custom git hooks
 	EnvPushTrigger  = "GITEA_PUSH_TRIGGER"
 	EnvIsInternal   = "GITEA_INTERNAL_PUSH"
 	EnvAppURL       = "GITEA_ROOT_URL"
@@ -50,11 +51,11 @@ func InternalPushingEnvironment(doer *user_model.User, repo *repo_model.Reposito
 
 // PushingEnvironment returns an os environment to allow hooks to work on push
 func PushingEnvironment(doer *user_model.User, repo *repo_model.Repository) []string {
-	return FullPushingEnvironment(doer, doer, repo, repo.Name, 0)
+	return FullPushingEnvironment(doer, doer, repo, repo.Name, 0, 0)
 }
 
 // FullPushingEnvironment returns an os environment to allow hooks to work on push
-func FullPushingEnvironment(author, committer *user_model.User, repo *repo_model.Repository, repoName string, prID int64) []string {
+func FullPushingEnvironment(author, committer *user_model.User, repo *repo_model.Repository, repoName string, prID, prIndex int64) []string {
 	isWiki := "false"
 	if strings.HasSuffix(repoName, ".wiki") {
 		isWiki = "true"
@@ -75,6 +76,7 @@ func FullPushingEnvironment(author, committer *user_model.User, repo *repo_model
 		EnvPusherID+"="+strconv.FormatInt(committer.ID, 10),
 		EnvRepoID+"="+strconv.FormatInt(repo.ID, 10),
 		EnvPRID+"="+strconv.FormatInt(prID, 10),
+		EnvPRIndex+"="+strconv.FormatInt(prIndex, 10),
 		EnvAppURL+"="+setting.AppURL,
 		"SSH_ORIGINAL_COMMAND=gitea-internal",
 	)

routers/api/packages/container/manifest.go

@@ -10,7 +10,6 @@ import (
 	"io"
 	"os"
 	"strings"
-	"time"
 
 	"code.gitea.io/gitea/models/db"
 	packages_model "code.gitea.io/gitea/models/packages"
@@ -260,6 +259,13 @@ func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, met
 		return nil, err
 	}
 
+	// "docker buildx imagetools create" multi-arch operations:
+	// {"type":"oci","is_tagged":false,"platform":"unknown/unknown"}
+	// {"type":"oci","is_tagged":false,"platform":"linux/amd64","layer_creation":["ADD file:9233f6f2237d79659a9521f7e390df217cec49f1a8aa3a12147bbca1956acdb9 in /","CMD [\"/bin/sh\"]"]}
+	// {"type":"oci","is_tagged":false,"platform":"unknown/unknown"}
+	// {"type":"oci","is_tagged":false,"platform":"linux/arm64","layer_creation":["ADD file:df53811312284306901fdaaff0a357a4bf40d631e662fe9ce6d342442e494b6c in /","CMD [\"/bin/sh\"]"]}
+	// {"type":"oci","is_tagged":true,"manifests":[{"platform":"linux/amd64","digest":"sha256:72bb73e706c0dec424d00a1febb21deaf1175a70ead009ad8b159729cfcf5769","size":2819478},{"platform":"linux/arm64","digest":"sha256:9e1426dd084a3221663b85ca1ee99d140c50b153917a5c5604c1f9b78229fd24","size":2716499},{"platform":"unknown/unknown","digest":"sha256:b93f03d0ae11b988243e1b2cd8d29accf5b9670547b7bd8c7d96abecc7283e6e","size":1798},{"platform":"unknown/unknown","digest":"sha256:f034b182ba66366c63a5d195c6dfcd3333c027409c0ac98e55ade36aaa3b2963","size":1798}]}
+
 	_pv := &packages_model.PackageVersion{
 		PackageID:    p.ID,
 		CreatorID:    mci.Creator.ID,
@@ -273,25 +279,16 @@ func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, met
 			log.Error("Error inserting package: %v", err)
 			return nil, err
 		}
-
-		if container_module.IsMediaTypeImageIndex(mci.MediaType) {
-			if pv.CreatedUnix.AsTime().Before(time.Now().Add(-24 * time.Hour)) {
-				if err = packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
-					return nil, err
-				}
-				// keep download count on overwriting
-				_pv.DownloadCount = pv.DownloadCount
-				if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil {
-					if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) {
-						log.Error("Error inserting package: %v", err)
-						return nil, err
-					}
-				}
-			} else {
-				err = packages_model.UpdateVersion(ctx, &packages_model.PackageVersion{ID: pv.ID, MetadataJSON: _pv.MetadataJSON})
-				if err != nil {
-					return nil, err
-				}
+		if err = packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
+			return nil, err
+		}
+		// keep download count on overwriting
+		_pv.DownloadCount = pv.DownloadCount
+		pv, err = packages_model.GetOrInsertVersion(ctx, _pv)
+		if err != nil {
+			if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) {
+				log.Error("Error inserting package: %v", err)
+				return nil, err
 			}
 		}
 	}

routers/web/repo/compare.go

@@ -9,7 +9,6 @@ import (
 	"encoding/csv"
 	"errors"
 	"fmt"
-	"html"
 	"io"
 	"net/http"
 	"net/url"
@@ -957,30 +956,26 @@ func ExcerptBlob(ctx *context.Context) {
 		ctx.HTTPError(http.StatusInternalServerError, "getExcerptLines")
 		return
 	}
-	if idxRight > lastRight {
-		lineText := " "
-		if rightHunkSize > 0 || leftHunkSize > 0 {
-			lineText = fmt.Sprintf("@@ -%d,%d +%d,%d @@\n", idxLeft, leftHunkSize, idxRight, rightHunkSize)
-		}
-		lineText = html.EscapeString(lineText)
-		lineSection := &gitdiff.DiffLine{
-			Type:    gitdiff.DiffLineSection,
-			Content: lineText,
-			SectionInfo: &gitdiff.DiffLineSectionInfo{
-				Path:          filePath,
-				LastLeftIdx:   lastLeft,
-				LastRightIdx:  lastRight,
-				LeftIdx:       idxLeft,
-				RightIdx:      idxRight,
-				LeftHunkSize:  leftHunkSize,
-				RightHunkSize: rightHunkSize,
-			},
-		}
+
+	newLineSection := &gitdiff.DiffLine{
+		Type: gitdiff.DiffLineSection,
+		SectionInfo: &gitdiff.DiffLineSectionInfo{
+			Path:          filePath,
+			LastLeftIdx:   lastLeft,
+			LastRightIdx:  lastRight,
+			LeftIdx:       idxLeft,
+			RightIdx:      idxRight,
+			LeftHunkSize:  leftHunkSize,
+			RightHunkSize: rightHunkSize,
+		},
+	}
+	if newLineSection.GetExpandDirection() != "" {
+		newLineSection.Content = fmt.Sprintf("@@ -%d,%d +%d,%d @@\n", idxLeft, leftHunkSize, idxRight, rightHunkSize)
 		switch direction {
 		case "up":
-			section.Lines = append([]*gitdiff.DiffLine{lineSection}, section.Lines...)
+			section.Lines = append([]*gitdiff.DiffLine{newLineSection}, section.Lines...)
 		case "down":
-			section.Lines = append(section.Lines, lineSection)
+			section.Lines = append(section.Lines, newLineSection)
 		}
 	}
 

routers/web/repo/view_file.go

@@ -92,8 +92,6 @@ func handleFileViewRenderMarkup(ctx *context.Context, filename string, sniffedTy
 		ctx.ServerError("Render", err)
 		return true
 	}
-	// to prevent iframe from loading third-party url
-	ctx.Resp.Header().Add("Content-Security-Policy", "frame-src 'self'")
 	return true
 }
 
@@ -241,14 +239,17 @@ func prepareFileView(ctx *context.Context, entry *git.TreeEntry) {
 	// * IsRenderableXxx: some files are rendered by backend "markup" engine, some are rendered by frontend (pdf, 3d)
 	// * DefaultViewMode: when there is no "display" query parameter, which view mode should be used by default, source or rendered
 
-	utf8Reader := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc), charset.ConvertOpts{})
+	contentReader := io.MultiReader(bytes.NewReader(buf), dataRc)
+	if fInfo.st.IsRepresentableAsText() {
+		contentReader = charset.ToUTF8WithFallbackReader(contentReader, charset.ConvertOpts{})
+	}
 	switch {
 	case fInfo.blobOrLfsSize >= setting.UI.MaxDisplayFileSize:
 		ctx.Data["IsFileTooLarge"] = true
-	case handleFileViewRenderMarkup(ctx, entry.Name(), fInfo.st, buf, utf8Reader):
+	case handleFileViewRenderMarkup(ctx, entry.Name(), fInfo.st, buf, contentReader):
 		// it also sets ctx.Data["FileContent"] and more
 		ctx.Data["IsMarkup"] = true
-	case handleFileViewRenderSource(ctx, entry.Name(), attrs, fInfo, utf8Reader):
+	case handleFileViewRenderSource(ctx, entry.Name(), attrs, fInfo, contentReader):
 		// it also sets ctx.Data["FileContent"] and more
 		ctx.Data["IsDisplayingSource"] = true
 	case handleFileViewRenderImage(ctx, fInfo, buf):

services/gitdiff/gitdiff.go

@@ -82,14 +82,34 @@ type DiffLine struct {
 
 // DiffLineSectionInfo represents diff line section meta data
 type DiffLineSectionInfo struct {
-	Path          string
-	LastLeftIdx   int
-	LastRightIdx  int
-	LeftIdx       int
-	RightIdx      int
+	Path string
+
+	// These line "idx" are 1-based line numbers
+	// Left/Right refer to the left/right side of the diff:
+	//
+	// LastLeftIdx | LastRightIdx
+	// [up/down expander] @@ hunk info @@
+	// LeftIdx     | RightIdx
+
+	LastLeftIdx  int
+	LastRightIdx int
+	LeftIdx      int
+	RightIdx     int
+
+	// Hunk sizes of the hidden lines
 	LeftHunkSize  int
 	RightHunkSize int
 
+	// For example:
+	// 17 | 31
+	// [up/down] @@ -40,23 +54,9 @@ ....
+	// 40 | 54
+	//
+	// In this case:
+	// LastLeftIdx = 17, LastRightIdx = 31
+	// LeftHunkSize = 23, RightHunkSize = 9
+	// LeftIdx = 40, RightIdx = 54
+
 	HiddenCommentIDs []int64 // IDs of hidden comments in this section
 }
 
@@ -158,13 +178,13 @@ func (d *DiffLine) getBlobExcerptQuery() string {
 	return query
 }
 
-func (d *DiffLine) getExpandDirection() string {
+func (d *DiffLine) GetExpandDirection() string {
 	if d.Type != DiffLineSection || d.SectionInfo == nil || d.SectionInfo.LeftIdx-d.SectionInfo.LastLeftIdx <= 1 || d.SectionInfo.RightIdx-d.SectionInfo.LastRightIdx <= 1 {
 		return ""
 	}
 	if d.SectionInfo.LastLeftIdx <= 0 && d.SectionInfo.LastRightIdx <= 0 {
 		return "up"
-	} else if d.SectionInfo.RightIdx-d.SectionInfo.LastRightIdx > BlobExcerptChunkSize && d.SectionInfo.RightHunkSize > 0 {
+	} else if d.SectionInfo.RightIdx-d.SectionInfo.LastRightIdx-1 > BlobExcerptChunkSize && d.SectionInfo.RightHunkSize > 0 {
 		return "updown"
 	} else if d.SectionInfo.LeftHunkSize <= 0 && d.SectionInfo.RightHunkSize <= 0 {
 		return "down"
@@ -202,13 +222,13 @@ func (d *DiffLine) RenderBlobExcerptButtons(fileNameHash string, data *DiffBlobE
 		content += htmlutil.HTMLFormat(`<span class="code-comment-more" data-tooltip-content="%s">%d</span>`, tooltip, len(d.SectionInfo.HiddenCommentIDs))
 	}
 
-	expandDirection := d.getExpandDirection()
-	if expandDirection == "up" || expandDirection == "updown" {
-		content += makeButton("up", "octicon-fold-up")
-	}
+	expandDirection := d.GetExpandDirection()
 	if expandDirection == "updown" || expandDirection == "down" {
 		content += makeButton("down", "octicon-fold-down")
 	}
+	if expandDirection == "up" || expandDirection == "updown" {
+		content += makeButton("up", "octicon-fold-up")
+	}
 	if expandDirection == "single" {
 		content += makeButton("single", "octicon-fold")
 	}

services/gitdiff/gitdiff_test.go

@@ -983,3 +983,126 @@ func TestDiffLine_RenderBlobExcerptButtons(t *testing.T) {
 		})
 	}
 }
+
+func TestDiffLine_GetExpandDirection(t *testing.T) {
+	cases := []struct {
+		name      string
+		diffLine  *DiffLine
+		direction string
+	}{
+		{
+			name:      "NotSectionLine",
+			diffLine:  &DiffLine{Type: DiffLineAdd, SectionInfo: &DiffLineSectionInfo{}},
+			direction: "",
+		},
+		{
+			name:      "NilSectionInfo",
+			diffLine:  &DiffLine{Type: DiffLineSection, SectionInfo: nil},
+			direction: "",
+		},
+		{
+			name: "NoHiddenLines",
+			// last block stops at line 100, next block starts at line 101, so no hidden lines, no expansion.
+			diffLine: &DiffLine{
+				Type: DiffLineSection,
+				SectionInfo: &DiffLineSectionInfo{
+					LastRightIdx: 100,
+					LastLeftIdx:  100,
+					RightIdx:     101,
+					LeftIdx:      101,
+				},
+			},
+			direction: "",
+		},
+		{
+			name: "FileHead",
+			diffLine: &DiffLine{
+				Type: DiffLineSection,
+				SectionInfo: &DiffLineSectionInfo{
+					LastRightIdx: 0, // LastXxxIdx = 0 means this is the first section in the file.
+					LastLeftIdx:  0,
+					RightIdx:     1,
+					LeftIdx:      1,
+				},
+			},
+			direction: "",
+		},
+		{
+			name: "FileHeadHiddenLines",
+			diffLine: &DiffLine{
+				Type: DiffLineSection,
+				SectionInfo: &DiffLineSectionInfo{
+					LastRightIdx: 0,
+					LastLeftIdx:  0,
+					RightIdx:     101,
+					LeftIdx:      101,
+				},
+			},
+			direction: "up",
+		},
+		{
+			name: "HiddenSingleHunk",
+			diffLine: &DiffLine{
+				Type: DiffLineSection,
+				SectionInfo: &DiffLineSectionInfo{
+					LastRightIdx:  100,
+					LastLeftIdx:   100,
+					RightIdx:      102,
+					LeftIdx:       102,
+					RightHunkSize: 1234, // non-zero dummy value
+					LeftHunkSize:  5678, // non-zero dummy value
+				},
+			},
+			direction: "single",
+		},
+		{
+			name: "HiddenSingleFullHunk",
+			// the hidden lines can exactly fit into one hunk
+			diffLine: &DiffLine{
+				Type: DiffLineSection,
+				SectionInfo: &DiffLineSectionInfo{
+					LastRightIdx:  100,
+					LastLeftIdx:   100,
+					RightIdx:      100 + BlobExcerptChunkSize + 1,
+					LeftIdx:       100 + BlobExcerptChunkSize + 1,
+					RightHunkSize: 1234, // non-zero dummy value
+					LeftHunkSize:  5678, // non-zero dummy value
+				},
+			},
+			direction: "single",
+		},
+		{
+			name: "HiddenUpDownHunks",
+			diffLine: &DiffLine{
+				Type: DiffLineSection,
+				SectionInfo: &DiffLineSectionInfo{
+					LastRightIdx:  100,
+					LastLeftIdx:   100,
+					RightIdx:      100 + BlobExcerptChunkSize + 2,
+					LeftIdx:       100 + BlobExcerptChunkSize + 2,
+					RightHunkSize: 1234, // non-zero dummy value
+					LeftHunkSize:  5678, // non-zero dummy value
+				},
+			},
+			direction: "updown",
+		},
+		{
+			name: "FileTail",
+			diffLine: &DiffLine{
+				Type: DiffLineSection,
+				SectionInfo: &DiffLineSectionInfo{
+					LastRightIdx:  100,
+					LastLeftIdx:   100,
+					RightIdx:      102,
+					LeftIdx:       102,
+					RightHunkSize: 0,
+					LeftHunkSize:  0,
+				},
+			},
+			direction: "down",
+		},
+	}
+	for _, c := range cases {
+		assert.Equal(t, c.direction, c.diffLine.GetExpandDirection(), "case %s expected direction: %s", c.name, c.direction)
+	}
+}

services/pull/merge.go

@@ -403,6 +403,7 @@ func doMergeAndPush(ctx context.Context, pr *issues_model.PullRequest, doer *use
 		pr.BaseRepo,
 		pr.BaseRepo.Name,
 		pr.ID,
+		pr.Index,
 	)
 
 	mergeCtx.env = append(mergeCtx.env, repo_module.EnvPushTrigger+"="+string(pushTrigger))

services/pull/update_rebase.go

@@ -80,6 +80,7 @@ func updateHeadByRebaseOnToBase(ctx context.Context, pr *issues_model.PullReques
 			pr.HeadRepo,
 			pr.HeadRepo.Name,
 			pr.ID,
+			pr.Index,
 		)).
 		WithDir(mergeCtx.tmpBasePath).
 		WithStdout(mergeCtx.outbuf).

services/wiki/wiki.go

@@ -223,6 +223,7 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model
 			repo,
 			repo.Name+".wiki",
 			0,
+			0,
 		),
 	}); err != nil {
 		log.Error("Push failed: %v", err)
@@ -341,6 +342,7 @@ func DeleteWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model
 			repo,
 			repo.Name+".wiki",
 			0,
+			0,
 		),
 	}); err != nil {
 		if git.IsErrPushOutOfDate(err) || git.IsErrPushRejected(err) {

tests/gitea-repositories-meta/user30/renderer.git/HEAD

@@ -1 +0,0 @@
-ref: refs/heads/master

tests/gitea-repositories-meta/user30/renderer.git/config

@@ -1,6 +0,0 @@
-[core]
-	repositoryformatversion = 0
-	filemode = true
-	bare = true
-	ignorecase = true
-	precomposeunicode = true

tests/gitea-repositories-meta/user30/renderer.git/description

@@ -1 +0,0 @@
-The repository will be used to test third-party renderer in TestExternalMarkupRenderer

tests/gitea-repositories-meta/user30/renderer.git/hooks/post-receive

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-data=$(cat)
-exitcodes=""
-hookname=$(basename $0)
-GIT_DIR=${GIT_DIR:-$(dirname $0)}
-
-for hook in ${GIT_DIR}/hooks/${hookname}.d/*; do
-test -x "${hook}" && test -f "${hook}" || continue
-echo "${data}" | "${hook}"
-exitcodes="${exitcodes} $?"
-done
-
-for i in ${exitcodes}; do
-[ ${i} -eq 0 ] || exit ${i}
-done

tests/gitea-repositories-meta/user30/renderer.git/hooks/post-receive.d/gitea

@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" post-receive

tests/gitea-repositories-meta/user30/renderer.git/hooks/pre-receive

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-data=$(cat)
-exitcodes=""
-hookname=$(basename $0)
-GIT_DIR=${GIT_DIR:-$(dirname $0)}
-
-for hook in ${GIT_DIR}/hooks/${hookname}.d/*; do
-test -x "${hook}" && test -f "${hook}" || continue
-echo "${data}" | "${hook}"
-exitcodes="${exitcodes} $?"
-done
-
-for i in ${exitcodes}; do
-[ ${i} -eq 0 ] || exit ${i}
-done

tests/gitea-repositories-meta/user30/renderer.git/hooks/pre-receive.d/gitea

@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" pre-receive

tests/gitea-repositories-meta/user30/renderer.git/hooks/update

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-exitcodes=""
-hookname=$(basename $0)
-GIT_DIR=${GIT_DIR:-$(dirname $0)}
-
-for hook in ${GIT_DIR}/hooks/${hookname}.d/*; do
-test -x "${hook}" && test -f "${hook}" || continue
-"${hook}" $1 $2 $3
-exitcodes="${exitcodes} $?"
-done
-
-for i in ${exitcodes}; do
-[ ${i} -eq 0 ] || exit ${i}
-done

tests/gitea-repositories-meta/user30/renderer.git/hooks/update.d/gitea

@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" update $1 $2 $3

tests/gitea-repositories-meta/user30/renderer.git/packed-refs

@@ -1,2 +0,0 @@
-# pack-refs with: peeled fully-peeled sorted 
-c961cc4d1ba6b7ee1ba228a9a02b00b7746d8033 refs/heads/master

tests/integration/api_packages_container_test.go

@@ -28,6 +28,7 @@ import (
 
 	oci "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestPackageContainer(t *testing.T) {
@@ -70,13 +71,12 @@ func TestPackageContainer(t *testing.T) {
 
 	manifestDigest := "sha256:4f10484d1c1bb13e3956b4de1cd42db8e0f14a75be1617b60f2de3cd59c803c6"
 	manifestContent := `{"schemaVersion":2,"mediaType":"` + container_module.ContentTypeDockerDistributionManifestV2 + `","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:4607e093bec406eaadb6f3a340f63400c9d3a7038680744c406903766b938f0d","size":1069},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4","size":32}]}`
-	manifestContentType := container_module.ContentTypeDockerDistributionManifestV2
 
 	untaggedManifestDigest := "sha256:4305f5f5572b9a426b88909b036e52ee3cf3d7b9c1b01fac840e90747f56623d"
 	untaggedManifestContent := `{"schemaVersion":2,"mediaType":"` + oci.MediaTypeImageManifest + `","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:4607e093bec406eaadb6f3a340f63400c9d3a7038680744c406903766b938f0d","size":1069},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4","size":32}]}`
 
-	indexManifestDigest := "sha256:bab112d6efb9e7f221995caaaa880352feb5bd8b1faf52fae8d12c113aa123ec"
-	indexManifestContent := `{"schemaVersion":2,"mediaType":"` + oci.MediaTypeImageIndex + `","manifests":[{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","digest":"` + manifestDigest + `","platform":{"os":"linux","architecture":"arm","variant":"v7"}},{"mediaType":"` + oci.MediaTypeImageManifest + `","digest":"` + untaggedManifestDigest + `","platform":{"os":"linux","architecture":"arm64","variant":"v8"}}]}`
+	indexManifestDigest := "sha256:2c6b5afb967d5de02795ee1d177c3746d005df4b4c2b829385b0d186b3414b6b"
+	indexManifestContent := `{"schemaVersion":2,"mediaType":"` + oci.MediaTypeImageIndex + `","is_tagged":true,"manifests":[{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","digest":"` + manifestDigest + `","platform":{"os":"linux","architecture":"arm","variant":"v7"}},{"mediaType":"` + oci.MediaTypeImageManifest + `","digest":"` + untaggedManifestDigest + `","platform":{"os":"linux","architecture":"arm64","variant":"v8"}}]}`
 
 	anonymousToken := ""
 	userToken := ""
@@ -467,15 +467,16 @@ func TestPackageContainer(t *testing.T) {
 						assert.NoError(t, err)
 						assert.EqualValues(t, 1, pv.DownloadCount)
 
-						// Overwrite existing tag should keep the download count
-						req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/%s", url, tag), strings.NewReader(manifestContent)).
-							AddTokenAuth(userToken).
-							SetHeader("Content-Type", oci.MediaTypeImageManifest)
-						MakeRequest(t, req, http.StatusCreated)
+						t.Run("OverwriteTagKeepDownloadCount", func(t *testing.T) {
+							req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/%s", url, tag), strings.NewReader(manifestContent)).
+								AddTokenAuth(userToken).
+								SetHeader("Content-Type", oci.MediaTypeImageManifest)
+							MakeRequest(t, req, http.StatusCreated)
 
-						pv, err = packages_model.GetVersionByNameAndVersion(t.Context(), user.ID, packages_model.TypeContainer, image, tag)
-						assert.NoError(t, err)
-						assert.EqualValues(t, 1, pv.DownloadCount)
+							pv, err = packages_model.GetVersionByNameAndVersion(t.Context(), user.ID, packages_model.TypeContainer, image, tag)
+							assert.NoError(t, err)
+							assert.EqualValues(t, 1, pv.DownloadCount)
+						})
 					})
 
 					t.Run("HeadManifest", func(t *testing.T) {
@@ -505,7 +506,7 @@ func TestPackageContainer(t *testing.T) {
 						resp := MakeRequest(t, req, http.StatusOK)
 
 						assert.Equal(t, strconv.Itoa(len(manifestContent)), resp.Header().Get("Content-Length"))
-						assert.Equal(t, manifestContentType, resp.Header().Get("Content-Type"))
+						assert.Equal(t, oci.MediaTypeImageManifest, resp.Header().Get("Content-Type")) // the manifest is overwritten by above OverwriteTagKeepDownloadCount
 						assert.Equal(t, manifestDigest, resp.Header().Get("Docker-Content-Digest"))
 						assert.Equal(t, manifestContent, resp.Body.String())
 					})
@@ -599,6 +600,17 @@ func TestPackageContainer(t *testing.T) {
 				assert.True(t, pd.Files[0].File.IsLead)
 				assert.Equal(t, oci.MediaTypeImageIndex, pd.Files[0].Properties.GetByName(container_module.PropertyMediaType))
 				assert.Equal(t, indexManifestDigest, pd.Files[0].Properties.GetByName(container_module.PropertyDigest))
+
+				lastPackageVersionID := pv.ID
+				t.Run("UploadAgain", func(t *testing.T) {
+					req := NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/%s", url, multiTag), strings.NewReader(indexManifestContent)).
+						AddTokenAuth(userToken).
+						SetHeader("Content-Type", oci.MediaTypeImageIndex)
+					MakeRequest(t, req, http.StatusCreated)
+					pv, err := packages_model.GetVersionByNameAndVersion(t.Context(), user.ID, packages_model.TypeContainer, image, multiTag)
+					require.NoError(t, err)
+					assert.NotEqual(t, lastPackageVersionID, pv.ID)
+				})
 			})
 
 			t.Run("HeadBlob", func(t *testing.T) {

tests/integration/markup_external_test.go

@@ -12,6 +12,7 @@ import (
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/external"
 	"code.gitea.io/gitea/modules/setting"
@@ -25,29 +26,45 @@ import (
 func TestExternalMarkupRenderer(t *testing.T) {
 	defer tests.PrepareTestEnv(t)()
 	if !setting.Database.Type.IsSQLite3() {
-		t.Skip()
+		t.Skip("only SQLite3 test config supports external markup renderer")
 		return
 	}
 
+	const binaryContentPrefix = "any prefix text."
+	const binaryContent = binaryContentPrefix + "\xfe\xfe\xfe\x00\xff\xff"
+	detectedEncoding, _ := charset.DetectEncoding([]byte(binaryContent))
+	assert.NotEqual(t, binaryContent, strings.ToValidUTF8(binaryContent, "?"))
+	assert.Equal(t, "ISO-8859-2", detectedEncoding) // even if the binary content can be detected as text encoding, it shouldn't affect the raw rendering
+
 	onGiteaRun(t, func(t *testing.T, _ *url.URL) {
 		user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
 		repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
-		_, err := createFile(user2, repo1, "file.no-sanitizer", "master", `any content`)
+		_, err := createFileInBranch(user2, repo1, createFileInBranchOptions{}, map[string]string{
+			"test.html":         `<div><any attr="val"><script></script></div>`,
+			"html.no-sanitizer": `<script>foo("raw")</script>`,
+			"bin.no-sanitizer":  binaryContent,
+		})
 		require.NoError(t, err)
 
 		t.Run("RenderNoSanitizer", func(t *testing.T) {
-			req := NewRequest(t, "GET", "/user2/repo1/src/branch/master/file.no-sanitizer")
+			req := NewRequest(t, "GET", "/user2/repo1/src/branch/master/html.no-sanitizer")
 			resp := MakeRequest(t, req, http.StatusOK)
-			doc := NewHTMLParser(t, resp.Body)
-			div := doc.Find("div.file-view")
+			div := NewHTMLParser(t, resp.Body).Find("div.file-view")
 			data, err := div.Html()
 			assert.NoError(t, err)
-			assert.Equal(t, `<script>window.alert("hi")</script>`, strings.TrimSpace(data))
+			assert.Equal(t, `<script>foo("raw")</script>`, strings.TrimSpace(data))
+
+			req = NewRequest(t, "GET", "/user2/repo1/src/branch/master/bin.no-sanitizer")
+			resp = MakeRequest(t, req, http.StatusOK)
+			div = NewHTMLParser(t, resp.Body).Find("div.file-view")
+			data, err = div.Html()
+			assert.NoError(t, err)
+			assert.Equal(t, strings.ReplaceAll(binaryContent, "\x00", ""), strings.TrimSpace(data)) // HTML template engine removes the null bytes
 		})
 	})
 
 	t.Run("RenderContentDirectly", func(t *testing.T) {
-		req := NewRequest(t, "GET", "/user30/renderer/src/branch/master/README.html")
+		req := NewRequest(t, "GET", "/user2/repo1/src/branch/master/test.html")
 		resp := MakeRequest(t, req, http.StatusOK)
 		assert.Equal(t, "text/html; charset=utf-8", resp.Header().Get("Content-Type"))
 
@@ -55,18 +72,21 @@ func TestExternalMarkupRenderer(t *testing.T) {
 		div := doc.Find("div.file-view")
 		data, err := div.Html()
 		assert.NoError(t, err)
-		assert.Equal(t, "<div>\n\ttest external renderer\n</div>", strings.TrimSpace(data))
+		// the content is fully sanitized
+		assert.Equal(t, `<div>&lt;script&gt;&lt;/script&gt;</div>`, strings.TrimSpace(data))
 	})
 
-	// above tested "no-sanitizer" mode, then we test iframe mode below
+	// above tested in-page rendering (no iframe), then we test iframe mode below
 	r := markup.GetRendererByFileName("any-file.html").(*external.Renderer)
 	defer test.MockVariableValue(&r.RenderContentMode, setting.RenderContentModeIframe)()
+	assert.True(t, r.NeedPostProcess())
 	r = markup.GetRendererByFileName("any-file.no-sanitizer").(*external.Renderer)
 	defer test.MockVariableValue(&r.RenderContentMode, setting.RenderContentModeIframe)()
+	assert.False(t, r.NeedPostProcess())
 
 	t.Run("RenderContentInIFrame", func(t *testing.T) {
 		t.Run("DefaultSandbox", func(t *testing.T) {
-			req := NewRequest(t, "GET", "/user30/renderer/src/branch/master/README.html")
+			req := NewRequest(t, "GET", "/user2/repo1/src/branch/master/test.html")
 
 			t.Run("ParentPage", func(t *testing.T) {
 				respParent := MakeRequest(t, req, http.StatusOK)
@@ -77,31 +97,42 @@ func TestExternalMarkupRenderer(t *testing.T) {
 
 				// default sandbox on parent page
 				assert.Equal(t, "allow-scripts allow-popups", iframe.AttrOr("sandbox", ""))
-				assert.Equal(t, "/user30/renderer/render/branch/master/README.html", iframe.AttrOr("data-src", ""))
+				assert.Equal(t, "/user2/repo1/render/branch/master/test.html", iframe.AttrOr("data-src", ""))
 			})
 			t.Run("SubPage", func(t *testing.T) {
-				req = NewRequest(t, "GET", "/user30/renderer/render/branch/master/README.html")
+				req = NewRequest(t, "GET", "/user2/repo1/render/branch/master/test.html")
 				respSub := MakeRequest(t, req, http.StatusOK)
 				assert.Equal(t, "text/html; charset=utf-8", respSub.Header().Get("Content-Type"))
 
 				// default sandbox in sub page response
 				assert.Equal(t, "frame-src 'self'; sandbox allow-scripts allow-popups", respSub.Header().Get("Content-Security-Policy"))
-				assert.Equal(t, "<script src=\"/assets/js/external-render-iframe.js\"></script><link rel=\"stylesheet\" href=\"/assets/css/external-render-iframe.css\"><div>\n\ttest external renderer\n</div>\n", respSub.Body.String())
+				// FIXME: actually here is a bug (legacy design problem), the "PostProcess" will escape "<script>" tag, but it indeed is the sanitizer's job
+				assert.Equal(t, `<script src="/assets/js/external-render-iframe.js"></script><link rel="stylesheet" href="/assets/css/external-render-iframe.css"><div><any attr="val">&lt;script&gt;&lt;/script&gt;</any></div>`, respSub.Body.String())
 			})
 		})
 
 		t.Run("NoSanitizerNoSandbox", func(t *testing.T) {
-			req := NewRequest(t, "GET", "/user2/repo1/src/branch/master/file.no-sanitizer")
-			respParent := MakeRequest(t, req, http.StatusOK)
-			iframe := NewHTMLParser(t, respParent.Body).Find("iframe.external-render-iframe")
-			assert.Equal(t, "/user2/repo1/render/branch/master/file.no-sanitizer", iframe.AttrOr("data-src", ""))
+			t.Run("BinaryContent", func(t *testing.T) {
+				req := NewRequest(t, "GET", "/user2/repo1/src/branch/master/bin.no-sanitizer")
+				respParent := MakeRequest(t, req, http.StatusOK)
+				iframe := NewHTMLParser(t, respParent.Body).Find("iframe.external-render-iframe")
+				assert.Equal(t, "/user2/repo1/render/branch/master/bin.no-sanitizer", iframe.AttrOr("data-src", ""))
 
-			req = NewRequest(t, "GET", "/user2/repo1/render/branch/master/file.no-sanitizer")
-			respSub := MakeRequest(t, req, http.StatusOK)
+				req = NewRequest(t, "GET", "/user2/repo1/render/branch/master/bin.no-sanitizer")
+				respSub := MakeRequest(t, req, http.StatusOK)
+				assert.Equal(t, binaryContent, respSub.Body.String()) // raw content should keep the raw bytes (including invalid UTF-8 bytes), and no "external-render-iframe" helpers
 
-			// no sandbox (disabled by RENDER_CONTENT_SANDBOX)
-			assert.Empty(t, iframe.AttrOr("sandbox", ""))
-			assert.Equal(t, "frame-src 'self'", respSub.Header().Get("Content-Security-Policy"))
+				// no sandbox (disabled by RENDER_CONTENT_SANDBOX)
+				assert.Empty(t, iframe.AttrOr("sandbox", ""))
+				assert.Equal(t, "frame-src 'self'", respSub.Header().Get("Content-Security-Policy"))
+			})
+
+			t.Run("HTMLContentWithExternalRenderIframeHelper", func(t *testing.T) {
+				req := NewRequest(t, "GET", "/user2/repo1/render/branch/master/html.no-sanitizer")
+				respSub := MakeRequest(t, req, http.StatusOK)
+				assert.Equal(t, `<script src="/assets/js/external-render-iframe.js"></script><link rel="stylesheet" href="/assets/css/external-render-iframe.css"><script>foo("raw")</script>`, respSub.Body.String())
+				assert.Equal(t, "frame-src 'self'", respSub.Header().Get("Content-Security-Policy"))
+			})
 		})
 	})
 }

tests/sqlite.ini.tmpl

@@ -122,7 +122,7 @@ RENDER_CONTENT_MODE = sanitized
 [markup.no-sanitizer]
 ENABLED = true
 FILE_EXTENSIONS = .no-sanitizer
-RENDER_COMMAND = echo '<script>window.alert("hi")</script>'
+RENDER_COMMAND = go run tools/test-echo.go
 ; This test case is reused, at first it is used to test "no-sanitizer" (sandbox doesn't take effect here)
 ; Then it will be updated and used to test "iframe + sandbox-disabled"
 RENDER_CONTENT_MODE = no-sanitizer