From b97548a2eea26d11e774d69f6829ed38dc95f502 Mon Sep 17 00:00:00 2001 From: silverwind Date: Thu, 24 Nov 2022 07:43:07 +0100 Subject: [PATCH 1/6] Fix table misalignments and tweak webhook and githook lists (#21917) - Fix regression from #21893 which had misaligned a few tables like repo lists and e-mails - Bring githooks list in line with webhooks list for styling - Change webhook list icons to just colored dots, like githook list - Increase size of dot in webhook and githook list from 16 to 22px --- templates/repo/settings/githooks.tmpl | 8 ++++---- templates/repo/settings/webhook/base_list.tmpl | 8 +------- web_src/less/_repository.less | 10 ++++++++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/templates/repo/settings/githooks.tmpl b/templates/repo/settings/githooks.tmpl index 893f52f188c42..9fa396c8d5739 100644 --- a/templates/repo/settings/githooks.tmpl +++ b/templates/repo/settings/githooks.tmpl @@ -13,10 +13,10 @@ {{.locale.Tr "repo.settings.githooks_desc" | Str2html}} {{range .Hooks}} -
- {{svg "octicon-dot-fill"}} - {{.Name}} - +
+ {{svg "octicon-dot-fill" 22}} + {{.Name}} + {{svg "octicon-pencil"}}
diff --git a/templates/repo/settings/webhook/base_list.tmpl b/templates/repo/settings/webhook/base_list.tmpl index 333764a7ebccb..d082f0b008501 100644 --- a/templates/repo/settings/webhook/base_list.tmpl +++ b/templates/repo/settings/webhook/base_list.tmpl @@ -48,13 +48,7 @@
{{range .Webhooks}}
- {{if eq .LastStatus 1}} - {{svg "octicon-check"}} - {{else if eq .LastStatus 2}} - {{svg "octicon-alert"}} - {{else}} - {{svg "octicon-dot-fill"}} - {{end}} + {{svg "octicon-dot-fill" 22}} {{.URL}} {{svg "octicon-pencil"}} {{svg "octicon-trash"}} diff --git a/web_src/less/_repository.less b/web_src/less/_repository.less index 85cb842b72f2d..d0c1e7cce5b4b 100644 --- a/web_src/less/_repository.less +++ b/web_src/less/_repository.less @@ -2612,12 +2612,18 @@ } } + &.webhooks .list > .item:not(:first-child), + &.githooks .list > .item:not(:first-child) { + padding: .25rem 1rem; + margin: 12px -1rem -1rem; + } + .list { > .item { &:not(:first-child) { border-top: 1px solid var(--color-secondary); - padding: .25rem 1rem; - margin: 12px -1rem -1rem; + padding: 1rem; + margin: 16px -1rem -1rem; } > .svg { From b2269dedf20c1186b5348a659ef55c4d7ec69f44 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 24 Nov 2022 11:31:32 +0100 Subject: [PATCH 2/6] Fix typo in sidebar (#21922) --- docs/content/doc/packages/vagrant.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/packages/vagrant.en-us.md b/docs/content/doc/packages/vagrant.en-us.md index e846de1a2a707..1e6d7a772147c 100644 --- a/docs/content/doc/packages/vagrant.en-us.md +++ b/docs/content/doc/packages/vagrant.en-us.md @@ -7,7 +7,7 @@ toc: false menu: sidebar: parent: "packages" - name: "vagrant" + name: "Vagrant" weight: 120 identifier: "vagrant" --- From 26f941fbdac73844a93c902ce6d1144175ff23ed Mon Sep 17 00:00:00 2001 From: Xinyu Zhou Date: Thu, 24 Nov 2022 20:29:43 +0800 Subject: [PATCH 3/6] Fix button in branch list, avoid unexpected page jump before restore branch actually done (#21562) This patch: - Use `
+ {{end}} {{if not $.DisableDownloadSourceArchives}} -
+
+ {{end}} @@ -108,26 +108,30 @@ {{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted)}} -
+
+ {{end}} {{if and (not .IsDeleted) (not $.DisableDownloadSourceArchives)}} -
+
+ {{end}} {{if and $.IsWriter (not $.IsMirror) (not $.Repository.IsArchived) (not .IsProtected)}} {{if .IsDeleted}} - {{svg "octicon-reply"}} + {{else}} - + {{end}} {{end}} From fc7a2d5a959334f7195571a3e1db669ed7667a28 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 24 Nov 2022 15:25:13 +0100 Subject: [PATCH 4/6] Add support for HEAD requests in Maven registry (#21834) Related #18543 Co-authored-by: Lunny Xiao --- modules/context/context.go | 18 +++++-- routers/api/packages/api.go | 1 + routers/api/packages/composer/composer.go | 5 +- routers/api/packages/conan/conan.go | 5 +- routers/api/packages/generic/generic.go | 5 +- routers/api/packages/helm/helm.go | 5 +- routers/api/packages/maven/api.go | 7 +-- routers/api/packages/maven/maven.go | 57 ++++++++++++++++++-- routers/api/packages/npm/npm.go | 10 +++- routers/api/packages/nuget/nuget.go | 10 +++- routers/api/packages/pub/pub.go | 5 +- routers/api/packages/pypi/pypi.go | 5 +- routers/api/packages/rubygems/rubygems.go | 5 +- routers/api/packages/vagrant/vagrant.go | 5 +- routers/api/v1/repo/file.go | 6 ++- routers/common/repo.go | 11 ++-- routers/web/repo/repo.go | 5 +- routers/web/user/package.go | 5 +- tests/integration/api_packages_maven_test.go | 30 ++++++++++- 19 files changed, 161 insertions(+), 39 deletions(-) diff --git a/modules/context/context.go b/modules/context/context.go index 697eb769045a1..47368bb280584 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -349,9 +349,11 @@ func (ctx *Context) RespHeader() http.Header { type ServeHeaderOptions struct { ContentType string // defaults to "application/octet-stream" ContentTypeCharset string + ContentLength *int64 Disposition string // defaults to "attachment" Filename string CacheDuration time.Duration // defaults to 5 minutes + LastModified time.Time } // SetServeHeaders sets necessary content serve headers @@ -369,6 +371,10 @@ func (ctx *Context) SetServeHeaders(opts *ServeHeaderOptions) { header.Set("Content-Type", contentType) header.Set("X-Content-Type-Options", "nosniff") + if opts.ContentLength != nil { + header.Set("Content-Length", strconv.FormatInt(*opts.ContentLength, 10)) + } + if opts.Filename != "" { disposition := opts.Disposition if disposition == "" { @@ -385,14 +391,16 @@ func (ctx *Context) SetServeHeaders(opts *ServeHeaderOptions) { duration = 5 * time.Minute } httpcache.AddCacheControlToHeader(header, duration) + + if !opts.LastModified.IsZero() { + header.Set("Last-Modified", opts.LastModified.UTC().Format(http.TimeFormat)) + } } // ServeContent serves content to http request -func (ctx *Context) ServeContent(name string, r io.ReadSeeker, modTime time.Time) { - ctx.SetServeHeaders(&ServeHeaderOptions{ - Filename: name, - }) - http.ServeContent(ctx.Resp, ctx.Req, name, modTime, r) +func (ctx *Context) ServeContent(r io.ReadSeeker, opts *ServeHeaderOptions) { + ctx.SetServeHeaders(opts) + http.ServeContent(ctx.Resp, ctx.Req, opts.Filename, opts.LastModified, r) } // UploadStream returns the request body or the first form file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 11e7e5d6a67e3..0d8b9ce61eeb9 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -181,6 +181,7 @@ func CommonRoutes(ctx gocontext.Context) *web.Route { r.Group("/maven", func() { r.Put("/*", reqPackageAccess(perm.AccessModeWrite), maven.UploadPackageFile) r.Get("/*", maven.DownloadPackageFile) + r.Head("/*", maven.ProvidePackageFileHeader) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/nuget", func() { r.Group("", func() { // Needs to be unauthenticated for the NuGet client. diff --git a/routers/api/packages/composer/composer.go b/routers/api/packages/composer/composer.go index 92e83dbe79d2d..a19433c6f099d 100644 --- a/routers/api/packages/composer/composer.go +++ b/routers/api/packages/composer/composer.go @@ -184,7 +184,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage creates a new package diff --git a/routers/api/packages/conan/conan.go b/routers/api/packages/conan/conan.go index 188cfce2879a8..e7c891b35f9cf 100644 --- a/routers/api/packages/conan/conan.go +++ b/routers/api/packages/conan/conan.go @@ -477,7 +477,10 @@ func downloadFile(ctx *context.Context, fileFilter container.Set[string], fileKe } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // DeleteRecipeV1 deletes the requested recipe(s) diff --git a/routers/api/packages/generic/generic.go b/routers/api/packages/generic/generic.go index 1bccc6764cf12..1c233da20f6f6 100644 --- a/routers/api/packages/generic/generic.go +++ b/routers/api/packages/generic/generic.go @@ -53,7 +53,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage uploads the specific generic package. diff --git a/routers/api/packages/helm/helm.go b/routers/api/packages/helm/helm.go index 17f0a0d3118c5..11291ca14e069 100644 --- a/routers/api/packages/helm/helm.go +++ b/routers/api/packages/helm/helm.go @@ -138,7 +138,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage creates a new package diff --git a/routers/api/packages/maven/api.go b/routers/api/packages/maven/api.go index b60a317814b4a..4ca541dd6f371 100644 --- a/routers/api/packages/maven/api.go +++ b/routers/api/packages/maven/api.go @@ -6,7 +6,6 @@ package maven import ( "encoding/xml" - "sort" "strings" packages_model "code.gitea.io/gitea/models/packages" @@ -23,12 +22,8 @@ type MetadataResponse struct { Version []string `xml:"versioning>versions>version"` } +// pds is expected to be sorted ascending by CreatedUnix func createMetadataResponse(pds []*packages_model.PackageDescriptor) *MetadataResponse { - sort.Slice(pds, func(i, j int) bool { - // Maven and Gradle order packages by their creation timestamp and not by their version string - return pds[i].Version.CreatedUnix < pds[j].Version.CreatedUnix - }) - var release *packages_model.PackageDescriptor versions := make([]string, 0, len(pds)) diff --git a/routers/api/packages/maven/maven.go b/routers/api/packages/maven/maven.go index de274b2046093..3125062b9201e 100644 --- a/routers/api/packages/maven/maven.go +++ b/routers/api/packages/maven/maven.go @@ -16,6 +16,8 @@ import ( "net/http" "path/filepath" "regexp" + "sort" + "strconv" "strings" packages_model "code.gitea.io/gitea/models/packages" @@ -34,6 +36,10 @@ const ( extensionSHA1 = ".sha1" extensionSHA256 = ".sha256" extensionSHA512 = ".sha512" + extensionPom = ".pom" + extensionJar = ".jar" + contentTypeJar = "application/java-archive" + contentTypeXML = "text/xml" ) var ( @@ -49,6 +55,15 @@ func apiError(ctx *context.Context, status int, obj interface{}) { // DownloadPackageFile serves the content of a package func DownloadPackageFile(ctx *context.Context) { + handlePackageFile(ctx, true) +} + +// ProvidePackageFileHeader provides only the headers describing a package +func ProvidePackageFileHeader(ctx *context.Context) { + handlePackageFile(ctx, false) +} + +func handlePackageFile(ctx *context.Context, serveContent bool) { params, err := extractPathParameters(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) @@ -58,7 +73,7 @@ func DownloadPackageFile(ctx *context.Context) { if params.IsMeta && params.Version == "" { serveMavenMetadata(ctx, params) } else { - servePackageFile(ctx, params) + servePackageFile(ctx, params, serveContent) } } @@ -82,6 +97,11 @@ func serveMavenMetadata(ctx *context.Context, params parameters) { return } + sort.Slice(pds, func(i, j int) bool { + // Maven and Gradle order packages by their creation timestamp and not by their version string + return pds[i].Version.CreatedUnix < pds[j].Version.CreatedUnix + }) + xmlMetadata, err := xml.Marshal(createMetadataResponse(pds)) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -89,6 +109,9 @@ func serveMavenMetadata(ctx *context.Context, params parameters) { } xmlMetadataWithHeader := append([]byte(xml.Header), xmlMetadata...) + latest := pds[len(pds)-1] + ctx.Resp.Header().Set("Last-Modified", latest.Version.CreatedUnix.Format(http.TimeFormat)) + ext := strings.ToLower(filepath.Ext(params.Filename)) if isChecksumExtension(ext) { var hash []byte @@ -110,10 +133,15 @@ func serveMavenMetadata(ctx *context.Context, params parameters) { return } - ctx.PlainTextBytes(http.StatusOK, xmlMetadataWithHeader) + ctx.Resp.Header().Set("Content-Length", strconv.Itoa(len(xmlMetadataWithHeader))) + ctx.Resp.Header().Set("Content-Type", contentTypeXML) + + if _, err := ctx.Resp.Write(xmlMetadataWithHeader); err != nil { + log.Error("write bytes failed: %v", err) + } } -func servePackageFile(ctx *context.Context, params parameters) { +func servePackageFile(ctx *context.Context, params parameters, serveContent bool) { packageName := params.GroupID + "-" + params.ArtifactID pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeMaven, packageName, params.Version) @@ -165,6 +193,23 @@ func servePackageFile(ctx *context.Context, params parameters) { return } + opts := &context.ServeHeaderOptions{ + ContentLength: &pb.Size, + LastModified: pf.CreatedUnix.AsLocalTime(), + } + switch ext { + case extensionJar: + opts.ContentType = contentTypeJar + case extensionPom: + opts.ContentType = contentTypeXML + } + + if !serveContent { + ctx.SetServeHeaders(opts) + ctx.Status(http.StatusOK) + return + } + s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(pb.HashSHA256)) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -177,7 +222,9 @@ func servePackageFile(ctx *context.Context, params parameters) { } } - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + opts.Filename = pf.Name + + ctx.ServeContent(s, opts) } // UploadPackageFile adds a file to the package. If the package does not exist, it gets created. @@ -273,7 +320,7 @@ func UploadPackageFile(ctx *context.Context) { } // If it's the package pom file extract the metadata - if ext == ".pom" { + if ext == extensionPom { pfci.IsLead = true var err error diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index af0e9be56eacd..6c11286a86ceb 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -103,7 +103,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // DownloadPackageFileByName finds the version and serves the contents of a package @@ -146,7 +149,10 @@ func DownloadPackageFileByName(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage creates a new package diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index 442d94243ba37..06aaca596d5d0 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -342,7 +342,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage creates a new package with the metadata contained in the uploaded nupgk file @@ -562,7 +565,10 @@ func DownloadSymbolFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // DeletePackage hard deletes the package diff --git a/routers/api/packages/pub/pub.go b/routers/api/packages/pub/pub.go index 635147b6d0c44..26cb9fbb9a631 100644 --- a/routers/api/packages/pub/pub.go +++ b/routers/api/packages/pub/pub.go @@ -275,5 +275,8 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go index 4853e6658bdc5..76801a92e12bd 100644 --- a/routers/api/packages/pypi/pypi.go +++ b/routers/api/packages/pypi/pypi.go @@ -95,7 +95,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackageFile adds a file to the package. If the package does not exist, it gets created. diff --git a/routers/api/packages/rubygems/rubygems.go b/routers/api/packages/rubygems/rubygems.go index 4adfb15731bac..f0b00f42c3656 100644 --- a/routers/api/packages/rubygems/rubygems.go +++ b/routers/api/packages/rubygems/rubygems.go @@ -192,7 +192,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackageFile adds a file to the package. If the package does not exist, it gets created. diff --git a/routers/api/packages/vagrant/vagrant.go b/routers/api/packages/vagrant/vagrant.go index 31ac56a532c75..746a2b19ba301 100644 --- a/routers/api/packages/vagrant/vagrant.go +++ b/routers/api/packages/vagrant/vagrant.go @@ -239,5 +239,8 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index 6dead81e6d52c..aba5b1f9e7f8a 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -341,7 +341,11 @@ func download(ctx *context.APIContext, archiveName string, archiver *repo_model. return } defer fr.Close() - ctx.ServeContent(downloadName, fr, archiver.CreatedUnix.AsLocalTime()) + + ctx.ServeContent(fr, &context.ServeHeaderOptions{ + Filename: downloadName, + LastModified: archiver.CreatedUnix.AsLocalTime(), + }) } // GetEditorconfig get editor config of a repository diff --git a/routers/common/repo.go b/routers/common/repo.go index f4b813d6b490b..340eb1809f59a 100644 --- a/routers/common/repo.go +++ b/routers/common/repo.go @@ -5,7 +5,6 @@ package common import ( - "fmt" "io" "path" "path/filepath" @@ -52,16 +51,16 @@ func ServeData(ctx *context.Context, filePath string, size int64, reader io.Read buf = buf[:n] } + opts := &context.ServeHeaderOptions{ + Filename: path.Base(filePath), + } + if size >= 0 { - ctx.Resp.Header().Set("Content-Length", fmt.Sprintf("%d", size)) + opts.ContentLength = &size } else { log.Error("ServeData called to serve data: %s with size < 0: %d", filePath, size) } - opts := &context.ServeHeaderOptions{ - Filename: path.Base(filePath), - } - sniffedType := typesniffer.DetectContentType(buf) isPlain := sniffedType.IsText() || ctx.FormBool("render") diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index 7bcca1d02a9a5..17e600182d440 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -426,7 +426,10 @@ func download(ctx *context.Context, archiveName string, archiver *repo_model.Rep } defer fr.Close() - ctx.ServeContent(downloadName, fr, archiver.CreatedUnix.AsLocalTime()) + ctx.ServeContent(fr, &context.ServeHeaderOptions{ + Filename: downloadName, + LastModified: archiver.CreatedUnix.AsLocalTime(), + }) } // InitiateDownload will enqueue an archival request, as needed. It may submit diff --git a/routers/web/user/package.go b/routers/web/user/package.go index 7179e2df97a76..7be37b6a50a69 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -402,5 +402,8 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } diff --git a/tests/integration/api_packages_maven_test.go b/tests/integration/api_packages_maven_test.go index 87d95557ce316..e71e1ff03baf2 100644 --- a/tests/integration/api_packages_maven_test.go +++ b/tests/integration/api_packages_maven_test.go @@ -7,6 +7,7 @@ package integration import ( "fmt" "net/http" + "strconv" "strings" "testing" @@ -39,6 +40,12 @@ func TestPackageMaven(t *testing.T) { MakeRequest(t, req, expectedStatus) } + checkHeaders := func(t *testing.T, h http.Header, contentType string, contentLength int64) { + assert.Equal(t, contentType, h.Get("Content-Type")) + assert.Equal(t, strconv.FormatInt(contentLength, 10), h.Get("Content-Length")) + assert.NotEmpty(t, h.Get("Last-Modified")) + } + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -77,10 +84,18 @@ func TestPackageMaven(t *testing.T) { t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s", root, packageVersion, filename)) + req := NewRequest(t, "HEAD", fmt.Sprintf("%s/%s/%s", root, packageVersion, filename)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) + checkHeaders(t, resp.Header(), "application/java-archive", 4) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s", root, packageVersion, filename)) + req = AddBasicAuthHeader(req, user.Name) + resp = MakeRequest(t, req, http.StatusOK) + + checkHeaders(t, resp.Header(), "application/java-archive", 4) + assert.Equal(t, []byte("test"), resp.Body.Bytes()) pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeMaven) @@ -150,10 +165,18 @@ func TestPackageMaven(t *testing.T) { t.Run("DownloadPOM", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s.pom", root, packageVersion, filename)) + req := NewRequest(t, "HEAD", fmt.Sprintf("%s/%s/%s.pom", root, packageVersion, filename)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) + checkHeaders(t, resp.Header(), "text/xml", int64(len(pomContent))) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s.pom", root, packageVersion, filename)) + req = AddBasicAuthHeader(req, user.Name) + resp = MakeRequest(t, req, http.StatusOK) + + checkHeaders(t, resp.Header(), "text/xml", int64(len(pomContent))) + assert.Equal(t, []byte(pomContent), resp.Body.Bytes()) pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeMaven) @@ -191,6 +214,9 @@ func TestPackageMaven(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) expectedMetadata := `` + "\ncom.giteatest-project1.0.11.0.11.0.1" + + checkHeaders(t, resp.Header(), "text/xml", int64(len(expectedMetadata))) + assert.Equal(t, expectedMetadata, resp.Body.String()) for key, checksum := range map[string]string{ From 9ce5e092f3f0e0048c49419c4ccd886e817a764f Mon Sep 17 00:00:00 2001 From: silverwind Date: Thu, 24 Nov 2022 21:56:57 +0100 Subject: [PATCH 5/6] Fix scroll over mermaid frame (#21925) When starting a scroll while the mouse is over a mermaid diagram, the scroll sometimes propagates to the iframe, preventing the parent page from scrolling. Fix this by disabling scroll inside the iframe. This is not a problem because those frames are never meant to scroll. Bug seems to affect Firefox only. ![scroll](https://user-images.githubusercontent.com/115237/203847578-6831e3c8-9df4-4577-8501-822fb9ea1278.gif) --- web_src/js/markup/mermaid.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/js/markup/mermaid.js b/web_src/js/markup/mermaid.js index 984946045d4f0..efd9abbb56920 100644 --- a/web_src/js/markup/mermaid.js +++ b/web_src/js/markup/mermaid.js @@ -3,7 +3,7 @@ const {mermaidMaxSourceCharacters} = window.config; const iframeCss = ` :root {color-scheme: normal} - body {margin: 0; padding: 0} + body {margin: 0; padding: 0; overflow: hidden} #mermaid {display: block; margin: 0 auto} `; From a1ae83f36ece5e37c738bd796227cf5990ccc030 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 25 Nov 2022 06:47:46 +0100 Subject: [PATCH 6/6] Workaround for container registry push/pull errors (#21862) This PR addresses #19586 I added a mutex to the upload version creation which will prevent the push errors when two requests try to create these database entries. I'm not sure if this should be the final solution for this problem. I added a workaround to allow a reupload of missing blobs. Normally a reupload is skipped because the database knows the blob is already present. The workaround checks if the blob exists on the file system. This should not be needed anymore with the above fix so I marked this code to be removed with Gitea v1.20. Co-authored-by: Lunny Xiao --- modules/packages/content_store.go | 7 ++++ routers/api/packages/container/blob.go | 32 ++++++++++++++++++- routers/api/packages/container/container.go | 28 ++++++++++++++-- routers/api/packages/container/manifest.go | 12 +++++++ .../api_packages_container_test.go | 28 ++++++++++++++++ 5 files changed, 103 insertions(+), 4 deletions(-) diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go index be416ac269350..13763db986480 100644 --- a/modules/packages/content_store.go +++ b/modules/packages/content_store.go @@ -32,6 +32,13 @@ func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) { return s.store.Open(KeyToRelativePath(key)) } +// FIXME: Workaround to be removed in v1.20 +// https://github.com/go-gitea/gitea/issues/19586 +func (s *ContentStore) Has(key BlobHash256Key) error { + _, err := s.store.Stat(KeyToRelativePath(key)) + return err +} + // Save stores a package blob func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error { _, err := s.store.Save(KeyToRelativePath(key), r, size) diff --git a/routers/api/packages/container/blob.go b/routers/api/packages/container/blob.go index df6b7aed92d48..06b450de02d52 100644 --- a/routers/api/packages/container/blob.go +++ b/routers/api/packages/container/blob.go @@ -7,8 +7,11 @@ package container import ( "context" "encoding/hex" + "errors" "fmt" + "os" "strings" + "sync" "code.gitea.io/gitea/models/db" packages_model "code.gitea.io/gitea/models/packages" @@ -16,9 +19,12 @@ import ( "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" container_module "code.gitea.io/gitea/modules/packages/container" + "code.gitea.io/gitea/modules/util" packages_service "code.gitea.io/gitea/services/packages" ) +var uploadVersionMutex sync.Mutex + // saveAsPackageBlob creates a package blob from an upload // The uploaded blob gets stored in a special upload version to link them to the package/image func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_service.PackageInfo) (*packages_model.PackageBlob, error) { @@ -28,6 +34,11 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic contentStore := packages_module.NewContentStore() + var uploadVersion *packages_model.PackageVersion + + // FIXME: Replace usage of mutex with database transaction + // https://github.com/go-gitea/gitea/pull/21862 + uploadVersionMutex.Lock() err := db.WithTx(db.DefaultContext, func(ctx context.Context) error { created := true p := &packages_model.Package{ @@ -68,11 +79,30 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic } } + uploadVersion = pv + + return nil + }) + uploadVersionMutex.Unlock() + if err != nil { + return nil, err + } + + err = db.WithTx(db.DefaultContext, func(ctx context.Context) error { pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb) if err != nil { log.Error("Error inserting package blob: %v", err) return err } + // FIXME: Workaround to be removed in v1.20 + // https://github.com/go-gitea/gitea/issues/19586 + if exists { + err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256)) + if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) { + log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256) + exists = false + } + } if !exists { if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil { log.Error("Error saving package blob in content store: %v", err) @@ -83,7 +113,7 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256)) pf := &packages_model.PackageFile{ - VersionID: pv.ID, + VersionID: uploadVersion.ID, BlobID: pb.ID, Name: filename, LowerName: filename, diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go index 5bc64e1b29f65..7af06a9171c70 100644 --- a/routers/api/packages/container/container.go +++ b/routers/api/packages/container/container.go @@ -10,6 +10,7 @@ import ( "io" "net/http" "net/url" + "os" "regexp" "strconv" "strings" @@ -24,6 +25,7 @@ import ( container_module "code.gitea.io/gitea/modules/packages/container" "code.gitea.io/gitea/modules/packages/container/oci" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/routers/api/packages/helper" packages_service "code.gitea.io/gitea/services/packages" container_service "code.gitea.io/gitea/services/packages/container" @@ -193,7 +195,7 @@ func InitiateUploadBlob(ctx *context.Context) { mount := ctx.FormTrim("mount") from := ctx.FormTrim("from") if mount != "" { - blob, _ := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + blob, _ := workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{ Image: from, Digest: mount, }) @@ -406,7 +408,7 @@ func getBlobFromContext(ctx *context.Context) (*packages_model.PackageFileDescri return nil, container_model.ErrContainerBlobNotExist } - return container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + return workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{ OwnerID: ctx.Package.Owner.ID, Image: ctx.Params("image"), Digest: digest, @@ -548,7 +550,7 @@ func getManifestFromContext(ctx *context.Context) (*packages_model.PackageFileDe return nil, container_model.ErrContainerBlobNotExist } - return container_model.GetContainerBlob(ctx, opts) + return workaroundGetContainerBlob(ctx, opts) } // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#checking-if-content-exists-in-the-registry @@ -688,3 +690,23 @@ func GetTagList(ctx *context.Context) { Tags: tags, }) } + +// FIXME: Workaround to be removed in v1.20 +// https://github.com/go-gitea/gitea/issues/19586 +func workaroundGetContainerBlob(ctx *context.Context, opts *container_model.BlobSearchOptions) (*packages_model.PackageFileDescriptor, error) { + blob, err := container_model.GetContainerBlob(ctx, opts) + if err != nil { + return nil, err + } + + err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(blob.Blob.HashSHA256)) + if err != nil { + if errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist) { + log.Debug("Package registry inconsistent: blob %s does not exist on file system", blob.Blob.HashSHA256) + return nil, container_model.ErrContainerBlobNotExist + } + return nil, err + } + + return blob, nil +} diff --git a/routers/api/packages/container/manifest.go b/routers/api/packages/container/manifest.go index a48b1de3b2802..64c958c2d06c4 100644 --- a/routers/api/packages/container/manifest.go +++ b/routers/api/packages/container/manifest.go @@ -6,8 +6,10 @@ package container import ( "context" + "errors" "fmt" "io" + "os" "strings" "code.gitea.io/gitea/models/db" @@ -19,6 +21,7 @@ import ( packages_module "code.gitea.io/gitea/modules/packages" container_module "code.gitea.io/gitea/modules/packages/container" "code.gitea.io/gitea/modules/packages/container/oci" + "code.gitea.io/gitea/modules/util" packages_service "code.gitea.io/gitea/services/packages" ) @@ -403,6 +406,15 @@ func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *pack log.Error("Error inserting package blob: %v", err) return nil, false, "", err } + // FIXME: Workaround to be removed in v1.20 + // https://github.com/go-gitea/gitea/issues/19586 + if exists { + err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(pb.HashSHA256)) + if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) { + log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256) + exists = false + } + } if !exists { contentStore := packages_module.NewContentStore() if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), buf, buf.Size()); err != nil { diff --git a/tests/integration/api_packages_container_test.go b/tests/integration/api_packages_container_test.go index ba76ee4baa7dd..60cbecd0675bf 100644 --- a/tests/integration/api_packages_container_test.go +++ b/tests/integration/api_packages_container_test.go @@ -6,10 +6,12 @@ package integration import ( "bytes" + "crypto/sha256" "encoding/base64" "fmt" "net/http" "strings" + "sync" "testing" "code.gitea.io/gitea/models/db" @@ -594,6 +596,32 @@ func TestPackageContainer(t *testing.T) { }) } + // https://github.com/go-gitea/gitea/issues/19586 + t.Run("ParallelUpload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf("%sv2/%s/parallel", setting.AppURL, user.Name) + + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + + content := []byte{byte(i)} + digest := fmt.Sprintf("sha256:%x", sha256.Sum256(content)) + + go func() { + defer wg.Done() + + req := NewRequestWithBody(t, "POST", fmt.Sprintf("%s/blobs/uploads?digest=%s", url, digest), bytes.NewReader(content)) + addTokenAuthHeader(req, userToken) + resp := MakeRequest(t, req, http.StatusCreated) + + assert.Equal(t, digest, resp.Header().Get("Docker-Content-Digest")) + }() + } + wg.Wait() + }) + t.Run("OwnerNameChange", func(t *testing.T) { defer tests.PrintCurrentTest(t)()