From 1c6858543ca976933004c21b3056a7301e1729d6 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 25 Feb 2024 16:10:55 +0100 Subject: [PATCH] Integrate alpine `noarch` packages into other architectures index (#29137) Fixes #26691 Revert #24972 The alpine package manager expects `noarch` packages in the index of other architectures too. --------- Co-authored-by: Lauris BH Co-authored-by: Lunny Xiao --- modules/packages/alpine/metadata.go | 2 + routers/api/packages/alpine/alpine.go | 33 +++- services/packages/alpine/repository.go | 56 +++++- tests/integration/api_packages_alpine_test.go | 163 ++++++++++++------ 4 files changed, 190 insertions(+), 64 deletions(-) diff --git a/modules/packages/alpine/metadata.go b/modules/packages/alpine/metadata.go index 582c42610d..c492811744 100644 --- a/modules/packages/alpine/metadata.go +++ b/modules/packages/alpine/metadata.go @@ -34,6 +34,8 @@ const ( RepositoryPackage = "_alpine" RepositoryVersion = "_repository" + + NoArch = "noarch" ) // https://wiki.alpinelinux.org/wiki/Apk_spec diff --git a/routers/api/packages/alpine/alpine.go b/routers/api/packages/alpine/alpine.go index bb14c5163a..3fd8288c01 100644 --- a/routers/api/packages/alpine/alpine.go +++ b/routers/api/packages/alpine/alpine.go @@ -72,7 +72,7 @@ func GetRepositoryFile(ctx *context.Context) { ctx, pv, &packages_service.PackageFileInfo{ - Filename: alpine_service.IndexFilename, + Filename: alpine_service.IndexArchiveFilename, CompositeKey: fmt.Sprintf("%s|%s|%s", ctx.Params("branch"), ctx.Params("repository"), ctx.Params("architecture")), }, ) @@ -182,19 +182,38 @@ func UploadPackageFile(ctx *context.Context) { } func DownloadPackageFile(ctx *context.Context) { - pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ + branch := ctx.Params("branch") + repository := ctx.Params("repository") + architecture := ctx.Params("architecture") + + opts := &packages_model.PackageFileSearchOptions{ OwnerID: ctx.Package.Owner.ID, PackageType: packages_model.TypeAlpine, Query: ctx.Params("filename"), - CompositeKey: fmt.Sprintf("%s|%s|%s", ctx.Params("branch"), ctx.Params("repository"), ctx.Params("architecture")), - }) + CompositeKey: fmt.Sprintf("%s|%s|%s", branch, repository, architecture), + } + pfs, _, err := packages_model.SearchFiles(ctx, opts) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - if len(pfs) != 1 { - apiError(ctx, http.StatusNotFound, nil) - return + if len(pfs) == 0 { + // Try again with architecture 'noarch' + if architecture == alpine_module.NoArch { + apiError(ctx, http.StatusNotFound, nil) + return + } + + opts.CompositeKey = fmt.Sprintf("%s|%s|%s", branch, repository, alpine_module.NoArch) + if pfs, _, err = packages_model.SearchFiles(ctx, opts); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if len(pfs) == 0 { + apiError(ctx, http.StatusNotFound, nil) + return + } } s, u, pf, err := packages_service.GetPackageFileStream(ctx, pfs[0]) diff --git a/services/packages/alpine/repository.go b/services/packages/alpine/repository.go index 104548b421..664ab34559 100644 --- a/services/packages/alpine/repository.go +++ b/services/packages/alpine/repository.go @@ -23,6 +23,7 @@ import ( packages_model "code.gitea.io/gitea/models/packages" alpine_model "code.gitea.io/gitea/models/packages/alpine" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/json" packages_module "code.gitea.io/gitea/modules/packages" alpine_module "code.gitea.io/gitea/modules/packages/alpine" @@ -30,7 +31,10 @@ import ( packages_service "code.gitea.io/gitea/services/packages" ) -const IndexFilename = "APKINDEX.tar.gz" +const ( + IndexFilename = "APKINDEX" + IndexArchiveFilename = IndexFilename + ".tar.gz" +) // GetOrCreateRepositoryVersion gets or creates the internal repository package // The Alpine registry needs multiple index files which are stored in this package. @@ -120,7 +124,22 @@ func BuildSpecificRepositoryFiles(ctx context.Context, ownerID int64, branch, re return err } - return buildPackagesIndex(ctx, ownerID, pv, branch, repository, architecture) + architectures := container.SetOf(architecture) + if architecture == alpine_module.NoArch { + // Update all other architectures too when updating the noarch index + additionalArchitectures, err := alpine_model.GetArchitectures(ctx, ownerID, repository) + if err != nil { + return err + } + architectures.AddMultiple(additionalArchitectures...) + } + + for architecture := range architectures { + if err := buildPackagesIndex(ctx, ownerID, pv, branch, repository, architecture); err != nil { + return err + } + } + return nil } type packageData struct { @@ -133,8 +152,7 @@ type packageData struct { type packageCache = map[*packages_model.PackageFile]*packageData -// https://wiki.alpinelinux.org/wiki/Apk_spec#APKINDEX_Format -func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, branch, repository, architecture string) error { +func searchPackageFiles(ctx context.Context, ownerID int64, branch, repository, architecture string) ([]*packages_model.PackageFile, error) { pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ OwnerID: ownerID, PackageType: packages_model.TypeAlpine, @@ -145,13 +163,30 @@ func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *package alpine_module.PropertyArchitecture: architecture, }, }) + if err != nil { + return nil, err + } + return pfs, nil +} + +// https://wiki.alpinelinux.org/wiki/Apk_spec#APKINDEX_Format +func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, branch, repository, architecture string) error { + pfs, err := searchPackageFiles(ctx, ownerID, branch, repository, architecture) if err != nil { return err } + if architecture != alpine_module.NoArch { + // Add all noarch packages too + noarchFiles, err := searchPackageFiles(ctx, ownerID, branch, repository, alpine_module.NoArch) + if err != nil { + return err + } + pfs = append(pfs, noarchFiles...) + } // Delete the package indices if there are no packages if len(pfs) == 0 { - pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, IndexFilename, fmt.Sprintf("%s|%s|%s", branch, repository, architecture)) + pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, IndexArchiveFilename, fmt.Sprintf("%s|%s|%s", branch, repository, architecture)) if err != nil && !errors.Is(err, util.ErrNotExist) { return err } else if pf == nil { @@ -206,7 +241,7 @@ func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *package fmt.Fprintf(&buf, "C:%s\n", pd.FileMetadata.Checksum) fmt.Fprintf(&buf, "P:%s\n", pd.Package.Name) fmt.Fprintf(&buf, "V:%s\n", pd.Version.Version) - fmt.Fprintf(&buf, "A:%s\n", pd.FileMetadata.Architecture) + fmt.Fprintf(&buf, "A:%s\n", architecture) if pd.VersionMetadata.Description != "" { fmt.Fprintf(&buf, "T:%s\n", pd.VersionMetadata.Description) } @@ -244,7 +279,7 @@ func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *package h := sha1.New() - if err := writeGzipStream(io.MultiWriter(unsignedIndexContent, h), "APKINDEX", buf.Bytes(), true); err != nil { + if err := writeGzipStream(io.MultiWriter(unsignedIndexContent, h), IndexFilename, buf.Bytes(), true); err != nil { return err } @@ -299,13 +334,18 @@ func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *package repoVersion, &packages_service.PackageFileCreationInfo{ PackageFileInfo: packages_service.PackageFileInfo{ - Filename: IndexFilename, + Filename: IndexArchiveFilename, CompositeKey: fmt.Sprintf("%s|%s|%s", branch, repository, architecture), }, Creator: user_model.NewGhostUser(), Data: signedIndexContent, IsLead: false, OverwriteExisting: true, + Properties: map[string]string{ + alpine_module.PropertyBranch: branch, + alpine_module.PropertyRepository: repository, + alpine_module.PropertyArchitecture: architecture, + }, }, ) return err diff --git a/tests/integration/api_packages_alpine_test.go b/tests/integration/api_packages_alpine_test.go index 3cc7178e02..228f497127 100644 --- a/tests/integration/api_packages_alpine_test.go +++ b/tests/integration/api_packages_alpine_test.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" alpine_module "code.gitea.io/gitea/modules/packages/alpine" + alpine_service "code.gitea.io/gitea/services/packages/alpine" "code.gitea.io/gitea/tests" "github.com/stretchr/testify/assert" @@ -59,7 +60,34 @@ Djfa/2q5bH4699v++uMAAAAAAAAAAAAAAAAAAAAAAHbgA/eXQh8AKAAA` content, err := base64.StdEncoding.DecodeString(base64AlpinePackageContent) assert.NoError(t, err) - branches := []string{"v3.16", "v3.17", "v3.18"} + base64AlpinePackageNoArchContent := `H4sIAAAAAAACA9ML9nT30wsKdtQrLU4t0jUzTUo1NDVP0ysqTtQrKE1ioAYwAAIzExMwDQTotCGI +bWhiampuYmRiaGrMYGBoZGZkxKBgwEAHUFpcklikoMAwQkHLB7eoE40P9n5jvx32t7Dy9rq7x19k +66cJPV38t/h+vWe2jdXy+/PzPT0YTF5z39i4cPFptcLa1C1lD0z/XvrNp6In/7nP4PPCF2pZu8uV +z74QXLxpY1XWJuVFysqVf+PdizccFbD6ZL/QPGXd1Ri1fec2XBNuYfK/rFa6wF/h3dK/W12f8mxP +04iP3aCy+vPx7h9S+5M1LLkWr5M/4ezGt3bDW/FjBp/S9hiKP72s/XrJ0vWtO0zr5wa+D/X8XluW +d7BLP7XS3YUhd8WbPPF/NW3691ONJbXsRb69O7BIMZC96uTri+utC/fbie5J+n7zhCxD4Aep/qet +QnlCZyN8MhNdVNlNl7965R1nExrrGvfI/YQZFx8Dg+d9122hZsYd/24WL/L69OWrDAN/y//nS7im +XEive3v7QeTe433TPj/X71+9yHiV6+E9k++3TL8V0Xoq9panhNt23fLgau/pTOvmKx6bV/pS26+Y +5UP4viyuklYeu4/BZl6rLINe1L/uWuUXcH5z7pa2b9+/rp/v/8dFgc1PL3bO3/iVcrI//J/LMU2X +Nzu1IaMmWXnGp7CmyQIR39d0Nai9/+tdPbfjvmsNH88Tu7uVrvNuJE0wjxfePXGv/KHNXD+mnG0t +yTPu+Na0b5WR9O4t0yMd9T5k6ui7hOyU/jL/4dOn6neLwhdrZIZfcl1ectnGvUTurWDo1vY5Gw9k +PTQLVgcA61F+7gAEAAAfiwgAAAAAAAID7VVNa9wwEPXZv2Ig53hHlizbCzkVkobQJtDkB4wl2SvW +lhdbTpP++oyXQGEPLYU2paTvIs3X05PQSNnmjp4+OrJumjfZ3c3V9efL2+T3AhlaqePIOB0Rc50I +VRSlypUoZIJCKJQJPCVvgGWONLGU5H1CCDDRD+4CU57S6zT5j3eCP9Tyv9T/GsuT/scyLxPAt+z/ +aRzjj/J+Fv9HcQZXLriJorPQPAM1i+8tyEzkGZ5PmJ7BMvvQQUt7tx4BPPJH4ccAIpN5Jjj+hSJc +ugZAghDbArco4eH+A+SYq/Sw7wINDi6g89HReRhpMrvVzTzsFZlaV2Hbutmw4zVhmXo2djEe5u1m +c6zNzDikR3mW1a61JepaC0SZHsjsqTsyPoR9GL+GdPbf1iSFtU5Xyu/c4+Q7H04lMfvgI3vT3hsX +5rX40/U9b5CWOA78Mhrq+2ewLjrDp7VNWQbtaF6ZXVWZIhdV09RWOIvU6BqNboSxLSEpkrpQq80x +W1Nla6NavuqtrJQ0sv17D+4L2oD1lwAIAAAfiwgAAAAAAAID7dM/SgNBFAbw6cSAnYXlXsDNm50/ +u1METBeIkEBMK87uzKKEJbB/IN7CxhN4AI/gNcRD6BWciI0WSiBGxO/XvA9mile8L+5P7WrkrfN1 +049dV1XXbNso0FK+zeDzJC4SxqVSqUwkV4IR51KkLFqxHeia1tZhFfY/cR4V7VXlB9QL0b5HnUXD +6fj4bDI5ncXFpS8WTVfFs9GQD5wVxgrvlde5zMmJRKm89KVRmnhmyJYuo5RMj8Ef8EOV36j/6/yx +/5qnxKJ1J8MZJifskD2Zu+fzxfggmT+83F4c3dw/7u1vtf/1ctl+9e+7dwAAAAAAAAAAAAAAAAAA +AACAX/AKARNTyAAoAAA=` + noarchContent, err := base64.StdEncoding.DecodeString(base64AlpinePackageNoArchContent) + assert.NoError(t, err) + + branches := []string{"v3.16", "v3.17"} repositories := []string{"main", "testing"} rootURL := fmt.Sprintf("/api/packages/%s/alpine", user.Name) @@ -139,63 +167,71 @@ Djfa/2q5bH4699v++uMAAAAAAAAAAAAAAAAAAAAAAHbgA/eXQh8AKAAA` }) }) - t.Run("Index", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + readIndexContent := func(r io.Reader) (string, error) { + br := bufio.NewReader(r) - url := fmt.Sprintf("%s/%s/%s/x86_64/APKINDEX.tar.gz", rootURL, branch, repository) + gzr, err := gzip.NewReader(br) + if err != nil { + return "", err + } - req := NewRequest(t, "GET", url) - resp := MakeRequest(t, req, http.StatusOK) - - assert.Condition(t, func() bool { - br := bufio.NewReader(resp.Body) - - gzr, err := gzip.NewReader(br) - assert.NoError(t, err) + for { + gzr.Multistream(false) + tr := tar.NewReader(gzr) for { - gzr.Multistream(false) - - tr := tar.NewReader(gzr) - for { - hd, err := tr.Next() - if err == io.EOF { - break - } - assert.NoError(t, err) - - if hd.Name == "APKINDEX" { - buf, err := io.ReadAll(tr) - assert.NoError(t, err) - - s := string(buf) - - assert.Contains(t, s, "C:Q1/se1PjO94hYXbfpNR1/61hVORIc=\n") - assert.Contains(t, s, "P:"+packageName+"\n") - assert.Contains(t, s, "V:"+packageVersion+"\n") - assert.Contains(t, s, "A:x86_64\n") - assert.Contains(t, s, "T:Gitea Test Package\n") - assert.Contains(t, s, "U:https://gitea.io/\n") - assert.Contains(t, s, "L:MIT\n") - assert.Contains(t, s, "S:1353\n") - assert.Contains(t, s, "I:4096\n") - assert.Contains(t, s, "o:gitea-test\n") - assert.Contains(t, s, "m:KN4CK3R \n") - assert.Contains(t, s, "t:1679498030\n") - - return true - } - } - - err = gzr.Reset(br) + hd, err := tr.Next() if err == io.EOF { break } - assert.NoError(t, err) + if err != nil { + return "", err + } + + if hd.Name == alpine_service.IndexFilename { + buf, err := io.ReadAll(tr) + if err != nil { + return "", err + } + + return string(buf), nil + } } - return false - }) + err = gzr.Reset(br) + if err == io.EOF { + break + } + if err != nil { + return "", err + } + } + + return "", io.EOF + } + + t.Run("Index", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s/x86_64/APKINDEX.tar.gz", rootURL, branch, repository)) + resp := MakeRequest(t, req, http.StatusOK) + + content, err := readIndexContent(resp.Body) + assert.NoError(t, err) + + assert.Contains(t, content, "C:Q1/se1PjO94hYXbfpNR1/61hVORIc=\n") + assert.Contains(t, content, "P:"+packageName+"\n") + assert.Contains(t, content, "V:"+packageVersion+"\n") + assert.Contains(t, content, "A:x86_64\n") + assert.NotContains(t, content, "A:noarch\n") + assert.Contains(t, content, "T:Gitea Test Package\n") + assert.Contains(t, content, "U:https://gitea.io/\n") + assert.Contains(t, content, "L:MIT\n") + assert.Contains(t, content, "S:1353\n") + assert.Contains(t, content, "I:4096\n") + assert.Contains(t, content, "o:gitea-test\n") + assert.Contains(t, content, "m:KN4CK3R \n") + assert.Contains(t, content, "t:1679498030\n") }) t.Run("Download", func(t *testing.T) { @@ -204,6 +240,35 @@ Djfa/2q5bH4699v++uMAAAAAAAAAAAAAAAAAAAAAAHbgA/eXQh8AKAAA` req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s/x86_64/%s-%s.apk", rootURL, branch, repository, packageName, packageVersion)) MakeRequest(t, req, http.StatusOK) }) + + t.Run("NoArch", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/%s/%s", rootURL, branch, repository), bytes.NewReader(noarchContent)). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s/x86_64/APKINDEX.tar.gz", rootURL, branch, repository)) + resp := MakeRequest(t, req, http.StatusOK) + + content, err := readIndexContent(resp.Body) + assert.NoError(t, err) + + assert.Contains(t, content, "C:Q1/se1PjO94hYXbfpNR1/61hVORIc=\n") + assert.Contains(t, content, "A:x86_64\n") + assert.Contains(t, content, "C:Q1kbH5WoIPFccQYyATanaKXd2cJcc=\n") + assert.NotContains(t, content, "A:noarch\n") + + // noarch package should be available with every architecture requested + for _, arch := range []string{alpine_module.NoArch, "x86_64", "my_arch"} { + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s/%s/gitea-noarch-1.4-r0.apk", rootURL, branch, repository, arch)) + MakeRequest(t, req, http.StatusOK) + } + + req = NewRequest(t, "DELETE", fmt.Sprintf("%s/%s/%s/noarch/gitea-noarch-1.4-r0.apk", rootURL, branch, repository)). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusNoContent) + }) }) } }