From df789d962b3d568a77773dd53f98eb37c8bd1f6b Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 5 Feb 2023 11:12:31 +0100 Subject: [PATCH] Add Cargo package registry (#21888) This PR implements a [Cargo registry](https://doc.rust-lang.org/cargo/) to manage Rust packages. This package type was a little bit more complicated because Cargo needs an additional Git repository to store its package index. Screenshots: ![grafik](https://user-images.githubusercontent.com/1666336/203102004-08d812ac-c066-4969-9bda-2fed818554eb.png) ![grafik](https://user-images.githubusercontent.com/1666336/203102141-d9970f14-dca6-4174-b17a-50ba1bd79087.png) ![grafik](https://user-images.githubusercontent.com/1666336/203102244-dc05743b-78b6-4d97-998e-ef76341a978f.png) --------- Co-authored-by: Lunny Xiao --- custom/conf/app.example.ini | 2 + .../doc/advanced/config-cheat-sheet.en-us.md | 1 + docs/content/doc/packages/cargo.en-us.md | 109 ++++++ docs/content/doc/packages/overview.en-us.md | 1 + models/packages/descriptor.go | 3 + models/packages/package.go | 6 + models/packages/package_property.go | 6 + modules/packages/cargo/parser.go | 169 +++++++++ modules/packages/cargo/parser_test.go | 86 +++++ modules/repository/create.go | 1 + modules/setting/packages.go | 2 + options/locale/locale_en-US.ini | 14 + public/img/svg/gitea-cargo.svg | 1 + routers/api/packages/api.go | 15 + routers/api/packages/cargo/cargo.go | 281 +++++++++++++++ routers/api/v1/packages/package.go | 2 +- routers/web/org/setting_packages.go | 20 + routers/web/shared/packages/packages.go | 22 ++ routers/web/user/setting/packages.go | 18 + routers/web/web.go | 8 + services/cron/tasks_basic.go | 4 +- services/forms/package_form.go | 2 +- services/packages/cargo/index.go | 290 +++++++++++++++ services/packages/cleanup/cleanup.go | 154 ++++++++ services/packages/packages.go | 121 +------ templates/org/settings/packages.tmpl | 1 + templates/package/content/cargo.tmpl | 62 ++++ templates/package/metadata/cargo.tmpl | 7 + templates/package/shared/cargo.tmpl | 24 ++ templates/package/view.tmpl | 2 + templates/swagger/v1_json.tmpl | 1 + templates/user/settings/packages.tmpl | 1 + tests/integration/api_packages_cargo_test.go | 341 ++++++++++++++++++ tests/integration/api_packages_test.go | 5 +- web_src/svg/gitea-cargo.svg | 3 + 35 files changed, 1660 insertions(+), 125 deletions(-) create mode 100644 docs/content/doc/packages/cargo.en-us.md create mode 100644 modules/packages/cargo/parser.go create mode 100644 modules/packages/cargo/parser_test.go create mode 100644 public/img/svg/gitea-cargo.svg create mode 100644 routers/api/packages/cargo/cargo.go create mode 100644 services/packages/cargo/index.go create mode 100644 services/packages/cleanup/cleanup.go create mode 100644 templates/package/content/cargo.tmpl create mode 100644 templates/package/metadata/cargo.tmpl create mode 100644 templates/package/shared/cargo.tmpl create mode 100644 tests/integration/api_packages_cargo_test.go create mode 100644 web_src/svg/gitea-cargo.svg diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 04ba2dc67a..0f1f18646b 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2458,6 +2458,8 @@ ROUTER = console ;LIMIT_TOTAL_OWNER_COUNT = -1 ;; Maximum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_TOTAL_OWNER_SIZE = -1 +;; Maximum size of a Cargo upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_CARGO = -1 ;; Maximum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_COMPOSER = -1 ;; Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 9254962dc5..c9116edc66 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -1213,6 +1213,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `CHUNKED_UPLOAD_PATH`: **tmp/package-upload**: Path for chunked uploads. Defaults to `APP_DATA_PATH` + `tmp/package-upload` - `LIMIT_TOTAL_OWNER_COUNT`: **-1**: Maximum count of package versions a single owner can have (`-1` means no limits) - `LIMIT_TOTAL_OWNER_SIZE`: **-1**: Maximum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_CARGO`: **-1**: Maximum size of a Cargo upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_COMPOSER`: **-1**: Maximum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CONAN`: **-1**: Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CONDA`: **-1**: Maximum size of a Conda upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/packages/cargo.en-us.md b/docs/content/doc/packages/cargo.en-us.md new file mode 100644 index 0000000000..1f90d939d1 --- /dev/null +++ b/docs/content/doc/packages/cargo.en-us.md @@ -0,0 +1,109 @@ +--- +date: "2022-11-20T00:00:00+00:00" +title: "Cargo Packages Repository" +slug: "packages/cargo" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Cargo" + weight: 5 + identifier: "cargo" +--- + +# Cargo Packages Repository + +Publish [Cargo](https://doc.rust-lang.org/stable/cargo/) packages for your user or organization. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the Cargo package registry, you need [Rust and Cargo](https://www.rust-lang.org/tools/install). + +Cargo stores informations about the available packages in a package index stored in a git repository. +This repository is needed to work with the registry. +The following section describes how to create it. + +## Index Repository + +Cargo stores informations about the available packages in a package index stored in a git repository. +In Gitea this repository has the special name `_cargo-index`. +After a package was uploaded, its metadata is automatically written to the index. +The content of this repository should not be manually modified. + +The user or organization package settings page allows to create the index repository along with the configuration file. +If needed this action will rewrite the configuration file. +This can be useful if for example the Gitea instance domain was changed. + +If the case arises where the packages stored in Gitea and the information in the index repository are out of sync, the settings page allows to rebuild the index repository. +This action iterates all packages in the registry and writes their information to the index. +If there are lot of packages this process may take some time. + +## Configuring the package registry + +To register the package registry the Cargo configuration must be updated. +Add the following text to the configuration file located in the current users home directory (for example `~/.cargo/config.toml`): + +``` +[registry] +default = "gitea" + +[registries.gitea] +index = "https://gitea.example.com/{owner}/_cargo-index.git" + +[net] +git-fetch-with-cli = true +``` + +| Parameter | Description | +| --------- | ----------- | +| `owner` | The owner of the package. | + +If the registry is private or you want to publish new packages, you have to configure your credentials. +Add the credentials section to the credentials file located in the current users home directory (for example `~/.cargo/credentials.toml`): + +``` +[registries.gitea] +token = "Bearer {token}" +``` + +| Parameter | Description | +| --------- | ----------- | +| `token` | Your [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) | + +## Publish a package + +Publish a package by running the following command in your project: + +```shell +cargo publish +``` + +You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. + +## Install a package + +To install a package from the package registry, execute the following command: + +```shell +cargo add {package_name} +``` + +| Parameter | Description | +| -------------- | ----------- | +| `package_name` | The package name. | + +## Supported commands + +``` +cargo publish +cargo add +cargo install +cargo yank +cargo unyank +cargo search +``` diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md index 9a736c1e56..b3ccb73c19 100644 --- a/docs/content/doc/packages/overview.en-us.md +++ b/docs/content/doc/packages/overview.en-us.md @@ -26,6 +26,7 @@ The following package managers are currently supported: | Name | Language | Package client | | ---- | -------- | -------------- | +| [Cargo]({{< relref "doc/packages/cargo.en-us.md" >}}) | Rust | `cargo` | | [Composer]({{< relref "doc/packages/composer.en-us.md" >}}) | PHP | `composer` | | [Conan]({{< relref "doc/packages/conan.en-us.md" >}}) | C++ | `conan` | | [Conda]({{< relref "doc/packages/conda.en-us.md" >}}) | - | `conda` | diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 3b36ee2266..40010eb720 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -11,6 +11,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/packages/cargo" "code.gitea.io/gitea/modules/packages/composer" "code.gitea.io/gitea/modules/packages/conan" "code.gitea.io/gitea/modules/packages/conda" @@ -129,6 +130,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc var metadata interface{} switch p.Type { + case TypeCargo: + metadata = &cargo.Metadata{} case TypeComposer: metadata = &composer.Metadata{} case TypeConan: diff --git a/models/packages/package.go b/models/packages/package.go index 0015953d81..b6b033cc95 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -30,6 +30,7 @@ type Type string // List of supported packages const ( + TypeCargo Type = "cargo" TypeComposer Type = "composer" TypeConan Type = "conan" TypeConda Type = "conda" @@ -46,6 +47,7 @@ const ( ) var TypeList = []Type{ + TypeCargo, TypeComposer, TypeConan, TypeConda, @@ -64,6 +66,8 @@ var TypeList = []Type{ // Name gets the name of the package type func (pt Type) Name() string { switch pt { + case TypeCargo: + return "Cargo" case TypeComposer: return "Composer" case TypeConan: @@ -97,6 +101,8 @@ func (pt Type) Name() string { // SVGName gets the name of the package type svg image func (pt Type) SVGName() string { switch pt { + case TypeCargo: + return "gitea-cargo" case TypeComposer: return "gitea-composer" case TypeConan: diff --git a/models/packages/package_property.go b/models/packages/package_property.go index 1b7f253d56..e03b12c9df 100644 --- a/models/packages/package_property.go +++ b/models/packages/package_property.go @@ -58,6 +58,12 @@ func GetPropertiesByName(ctx context.Context, refType PropertyType, refID int64, return pps, db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ? AND name = ?", refType, refID, name).Find(&pps) } +// UpdateProperty updates a property +func UpdateProperty(ctx context.Context, pp *PackageProperty) error { + _, err := db.GetEngine(ctx).ID(pp.ID).Update(pp) + return err +} + // DeleteAllProperties deletes all properties of a ref func DeleteAllProperties(ctx context.Context, refType PropertyType, refID int64) error { _, err := db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ?", refType, refID).Delete(&PackageProperty{}) diff --git a/modules/packages/cargo/parser.go b/modules/packages/cargo/parser.go new file mode 100644 index 0000000000..36cd44df84 --- /dev/null +++ b/modules/packages/cargo/parser.go @@ -0,0 +1,169 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cargo + +import ( + "encoding/binary" + "errors" + "io" + "regexp" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" +) + +const PropertyYanked = "cargo.yanked" + +var ( + ErrInvalidName = errors.New("package name is invalid") + ErrInvalidVersion = errors.New("package version is invalid") +) + +// Package represents a Cargo package +type Package struct { + Name string + Version string + Metadata *Metadata + Content io.Reader + ContentSize int64 +} + +// Metadata represents the metadata of a Cargo package +type Metadata struct { + Dependencies []*Dependency `json:"dependencies,omitempty"` + Features map[string][]string `json:"features,omitempty"` + Authors []string `json:"authors,omitempty"` + Description string `json:"description,omitempty"` + DocumentationURL string `json:"documentation_url,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Readme string `json:"readme,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Categories []string `json:"categories,omitempty"` + License string `json:"license,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + Links string `json:"links,omitempty"` +} + +type Dependency struct { + Name string `json:"name"` + Req string `json:"req"` + Features []string `json:"features"` + Optional bool `json:"optional"` + DefaultFeatures bool `json:"default_features"` + Target *string `json:"target"` + Kind string `json:"kind"` + Registry *string `json:"registry"` + Package *string `json:"package"` +} + +var nameMatch = regexp.MustCompile(`\A[a-zA-Z][a-zA-Z0-9-_]{0,63}\z`) + +// ParsePackage reads the metadata and content of a package +func ParsePackage(r io.Reader) (*Package, error) { + var size uint32 + if err := binary.Read(r, binary.LittleEndian, &size); err != nil { + return nil, err + } + + p, err := parsePackage(io.LimitReader(r, int64(size))) + if err != nil { + return nil, err + } + + if err := binary.Read(r, binary.LittleEndian, &size); err != nil { + return nil, err + } + + p.Content = io.LimitReader(r, int64(size)) + p.ContentSize = int64(size) + + return p, nil +} + +func parsePackage(r io.Reader) (*Package, error) { + var meta struct { + Name string `json:"name"` + Vers string `json:"vers"` + Deps []struct { + Name string `json:"name"` + VersionReq string `json:"version_req"` + Features []string `json:"features"` + Optional bool `json:"optional"` + DefaultFeatures bool `json:"default_features"` + Target *string `json:"target"` + Kind string `json:"kind"` + Registry *string `json:"registry"` + ExplicitNameInToml string `json:"explicit_name_in_toml"` + } `json:"deps"` + Features map[string][]string `json:"features"` + Authors []string `json:"authors"` + Description string `json:"description"` + Documentation string `json:"documentation"` + Homepage string `json:"homepage"` + Readme string `json:"readme"` + ReadmeFile string `json:"readme_file"` + Keywords []string `json:"keywords"` + Categories []string `json:"categories"` + License string `json:"license"` + LicenseFile string `json:"license_file"` + Repository string `json:"repository"` + Links string `json:"links"` + } + if err := json.NewDecoder(r).Decode(&meta); err != nil { + return nil, err + } + + if !nameMatch.MatchString(meta.Name) { + return nil, ErrInvalidName + } + + if _, err := version.NewSemver(meta.Vers); err != nil { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(meta.Homepage) { + meta.Homepage = "" + } + if !validation.IsValidURL(meta.Documentation) { + meta.Documentation = "" + } + if !validation.IsValidURL(meta.Repository) { + meta.Repository = "" + } + + dependencies := make([]*Dependency, 0, len(meta.Deps)) + for _, dep := range meta.Deps { + dependencies = append(dependencies, &Dependency{ + Name: dep.Name, + Req: dep.VersionReq, + Features: dep.Features, + Optional: dep.Optional, + DefaultFeatures: dep.DefaultFeatures, + Target: dep.Target, + Kind: dep.Kind, + Registry: dep.Registry, + }) + } + + return &Package{ + Name: meta.Name, + Version: meta.Vers, + Metadata: &Metadata{ + Dependencies: dependencies, + Features: meta.Features, + Authors: meta.Authors, + Description: meta.Description, + DocumentationURL: meta.Documentation, + ProjectURL: meta.Homepage, + Readme: meta.Readme, + Keywords: meta.Keywords, + Categories: meta.Categories, + License: meta.License, + RepositoryURL: meta.Repository, + Links: meta.Links, + }, + }, nil +} diff --git a/modules/packages/cargo/parser_test.go b/modules/packages/cargo/parser_test.go new file mode 100644 index 0000000000..2230a5b499 --- /dev/null +++ b/modules/packages/cargo/parser_test.go @@ -0,0 +1,86 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cargo + +import ( + "bytes" + "encoding/binary" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + description = "Package Description" + author = "KN4CK3R" + homepage = "https://gitea.io/" + license = "MIT" +) + +func TestParsePackage(t *testing.T) { + createPackage := func(name, version string) io.Reader { + metadata := `{ + "name":"` + name + `", + "vers":"` + version + `", + "description":"` + description + `", + "authors": ["` + author + `"], + "deps":[ + { + "name":"dep", + "version_req":"1.0" + } + ], + "homepage":"` + homepage + `", + "license":"` + license + `" +}` + + var buf bytes.Buffer + binary.Write(&buf, binary.LittleEndian, uint32(len(metadata))) + buf.WriteString(metadata) + binary.Write(&buf, binary.LittleEndian, uint32(4)) + buf.WriteString("test") + return &buf + } + + t.Run("InvalidName", func(t *testing.T) { + for _, name := range []string{"", "0test", "-test", "_test", strings.Repeat("a", 65)} { + data := createPackage(name, "1.0.0") + + cp, err := ParsePackage(data) + assert.Nil(t, cp) + assert.ErrorIs(t, err, ErrInvalidName) + } + }) + + t.Run("InvalidVersion", func(t *testing.T) { + for _, version := range []string{"", "1.", "-1.0", "1.0.0/1"} { + data := createPackage("test", version) + + cp, err := ParsePackage(data) + assert.Nil(t, cp) + assert.ErrorIs(t, err, ErrInvalidVersion) + } + }) + + t.Run("Valid", func(t *testing.T) { + data := createPackage("test", "1.0.0") + + cp, err := ParsePackage(data) + assert.NotNil(t, cp) + assert.NoError(t, err) + + assert.Equal(t, "test", cp.Name) + assert.Equal(t, "1.0.0", cp.Version) + assert.Equal(t, description, cp.Metadata.Description) + assert.Equal(t, []string{author}, cp.Metadata.Authors) + assert.Len(t, cp.Metadata.Dependencies, 1) + assert.Equal(t, "dep", cp.Metadata.Dependencies[0].Name) + assert.Equal(t, homepage, cp.Metadata.ProjectURL) + assert.Equal(t, license, cp.Metadata.License) + content, _ := io.ReadAll(cp.Content) + assert.Equal(t, "test", string(content)) + }) +} diff --git a/modules/repository/create.go b/modules/repository/create.go index 7bcda0fe45..b9a72ad573 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -211,6 +211,7 @@ func CreateRepository(doer, u *user_model.User, opts CreateRepoOptions) (*repo_m IsEmpty: !opts.AutoInit, TrustModel: opts.TrustModel, IsMirror: opts.IsMirror, + DefaultBranch: opts.DefaultBranch, } var rollbackRepo *repo_model.Repository diff --git a/modules/setting/packages.go b/modules/setting/packages.go index d0cd80aa03..190c17dd8f 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -25,6 +25,7 @@ var ( LimitTotalOwnerCount int64 LimitTotalOwnerSize int64 + LimitSizeCargo int64 LimitSizeComposer int64 LimitSizeConan int64 LimitSizeConda int64 @@ -65,6 +66,7 @@ func newPackages() { } Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") + Packages.LimitSizeCargo = mustBytes(sec, "LIMIT_SIZE_CARGO") Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER") Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN") Packages.LimitSizeConda = mustBytes(sec, "LIMIT_SIZE_CONDA") diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index f384056613..bc2e8cb91c 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3152,6 +3152,11 @@ versions.on = on versions.view_all = View all dependency.id = ID dependency.version = Version +cargo.registry = Setup this registry in the Cargo configuration file (for example ~/.cargo/config.toml): +cargo.install = To install the package using Cargo, run the following command: +cargo.documentation = For more information on the Cargo registry, see the documentation. +cargo.details.repository_site = Repository Site +cargo.details.documentation_site = Documentation Site composer.registry = Setup this registry in your ~/.composer/config.json file: composer.install = To install the package using Composer, run the following command: composer.documentation = For more information on the Composer registry, see the documentation. @@ -3228,6 +3233,15 @@ settings.delete.description = Deleting a package is permanent and cannot be undo settings.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure? settings.delete.success = The package has been deleted. settings.delete.error = Failed to delete the package. +owner.settings.cargo.title = Cargo Registry Index +owner.settings.cargo.initialize = Initialize Index +owner.settings.cargo.initialize.description = To use the Cargo registry a special index git repository is needed. Here you can (re)create it with the required config. +owner.settings.cargo.initialize.error = Failed to initialize Cargo index: %v +owner.settings.cargo.initialize.success = The Cargo index was successfully created. +owner.settings.cargo.rebuild = Rebuild Index +owner.settings.cargo.rebuild.description = If the index is out of sync with the cargo packages stored you can rebuild it here. +owner.settings.cargo.rebuild.error = Failed to rebuild Cargo index: %v +owner.settings.cargo.rebuild.success = The Cargo index was successfully rebuild. owner.settings.cleanuprules.title = Manage Cleanup Rules owner.settings.cleanuprules.add = Add Cleanup Rule owner.settings.cleanuprules.edit = Edit Cleanup Rule diff --git a/public/img/svg/gitea-cargo.svg b/public/img/svg/gitea-cargo.svg new file mode 100644 index 0000000000..91d53941ca --- /dev/null +++ b/public/img/svg/gitea-cargo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 7a07fea815..8ec9ae9bf7 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/routers/api/packages/cargo" "code.gitea.io/gitea/routers/api/packages/composer" "code.gitea.io/gitea/routers/api/packages/conan" "code.gitea.io/gitea/routers/api/packages/conda" @@ -71,6 +72,20 @@ func CommonRoutes(ctx gocontext.Context) *web.Route { }) r.Group("/{username}", func() { + r.Group("/cargo", func() { + r.Group("/api/v1/crates", func() { + r.Get("", cargo.SearchPackages) + r.Put("/new", reqPackageAccess(perm.AccessModeWrite), cargo.UploadPackage) + r.Group("/{package}", func() { + r.Group("/{version}", func() { + r.Get("/download", cargo.DownloadPackageFile) + r.Delete("/yank", reqPackageAccess(perm.AccessModeWrite), cargo.YankPackage) + r.Put("/unyank", reqPackageAccess(perm.AccessModeWrite), cargo.UnyankPackage) + }) + r.Get("/owners", cargo.ListOwners) + }) + }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/composer", func() { r.Get("/packages.json", composer.ServiceIndex) r.Get("/search.json", composer.SearchPackages) diff --git a/routers/api/packages/cargo/cargo.go b/routers/api/packages/cargo/cargo.go new file mode 100644 index 0000000000..e0bf5da13a --- /dev/null +++ b/routers/api/packages/cargo/cargo.go @@ -0,0 +1,281 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cargo + +import ( + "fmt" + "net/http" + "strconv" + "strings" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + cargo_module "code.gitea.io/gitea/modules/packages/cargo" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/packages/helper" + "code.gitea.io/gitea/services/convert" + packages_service "code.gitea.io/gitea/services/packages" + cargo_service "code.gitea.io/gitea/services/packages/cargo" +) + +// https://doc.rust-lang.org/cargo/reference/registries.html#web-api +type StatusResponse struct { + OK bool `json:"ok"` + Errors []StatusMessage `json:"errors,omitempty"` +} + +type StatusMessage struct { + Message string `json:"detail"` +} + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.JSON(status, StatusResponse{ + OK: false, + Errors: []StatusMessage{ + { + Message: message, + }, + }, + }) + }) +} + +type SearchResult struct { + Crates []*SearchResultCrate `json:"crates"` + Meta SearchResultMeta `json:"meta"` +} + +type SearchResultCrate struct { + Name string `json:"name"` + LatestVersion string `json:"max_version"` + Description string `json:"description"` +} + +type SearchResultMeta struct { + Total int64 `json:"total"` +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#search +func SearchPackages(ctx *context.Context) { + page := ctx.FormInt("page") + if page < 1 { + page = 1 + } + perPage := ctx.FormInt("per_page") + paginator := db.ListOptions{ + Page: page, + PageSize: convert.ToCorrectPageSize(perPage), + } + + pvs, total, err := packages_model.SearchLatestVersions( + ctx, + &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeCargo, + Name: packages_model.SearchValue{Value: ctx.FormTrim("q")}, + IsInternal: util.OptionalBoolFalse, + Paginator: &paginator, + }, + ) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + crates := make([]*SearchResultCrate, 0, len(pvs)) + for _, pd := range pds { + crates = append(crates, &SearchResultCrate{ + Name: pd.Package.Name, + LatestVersion: pd.Version.Version, + Description: pd.Metadata.(*cargo_module.Metadata).Description, + }) + } + + ctx.JSON(http.StatusOK, SearchResult{ + Crates: crates, + Meta: SearchResultMeta{ + Total: total, + }, + }) +} + +type Owners struct { + Users []OwnerUser `json:"users"` +} + +type OwnerUser struct { + ID int64 `json:"id"` + Login string `json:"login"` + Name string `json:"name"` +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#owners-list +func ListOwners(ctx *context.Context) { + ctx.JSON(http.StatusOK, Owners{ + Users: []OwnerUser{ + { + ID: ctx.Package.Owner.ID, + Login: ctx.Package.Owner.Name, + Name: ctx.Package.Owner.DisplayName(), + }, + }, + }) +} + +// DownloadPackageFile serves the content of a package +func DownloadPackageFile(ctx *context.Context) { + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeCargo, + Name: ctx.Params("package"), + Version: ctx.Params("version"), + }, + &packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s-%s.crate", ctx.Params("package"), ctx.Params("version"))), + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#publish +func UploadPackage(ctx *context.Context) { + defer ctx.Req.Body.Close() + + cp, err := cargo_module.ParsePackage(ctx.Req.Body) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + buf, err := packages_module.CreateHashedBufferFromReader(cp.Content, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + if buf.Size() != cp.ContentSize { + apiError(ctx, http.StatusBadRequest, "invalid content size") + return + } + + pv, _, err := packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeCargo, + Name: cp.Name, + Version: cp.Version, + }, + SemverCompatible: true, + Creator: ctx.Doer, + Metadata: cp.Metadata, + VersionProperties: map[string]string{ + cargo_module.PropertyYanked: strconv.FormatBool(false), + }, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s-%s.crate", cp.Name, cp.Version)), + }, + Creator: ctx.Doer, + Data: buf, + IsLead: true, + }, + ) + if err != nil { + switch err { + case packages_model.ErrDuplicatePackageVersion: + apiError(ctx, http.StatusConflict, err) + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + if err := cargo_service.AddOrUpdatePackageIndex(ctx, ctx.Doer, ctx.Package.Owner, pv.PackageID); err != nil { + if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { + log.Error("Rollback creation of package version: %v", err) + } + + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.JSON(http.StatusOK, StatusResponse{OK: true}) +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#yank +func YankPackage(ctx *context.Context) { + yankPackage(ctx, true) +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#unyank +func UnyankPackage(ctx *context.Context) { + yankPackage(ctx, false) +} + +func yankPackage(ctx *context.Context, yank bool) { + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeCargo, ctx.Params("package"), ctx.Params("version")) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, cargo_module.PropertyYanked) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pps) == 0 { + apiError(ctx, http.StatusInternalServerError, "Property not found") + return + } + + pp := pps[0] + pp.Value = strconv.FormatBool(yank) + + if err := packages_model.UpdateProperty(ctx, pp); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if err := cargo_service.AddOrUpdatePackageIndex(ctx, ctx.Doer, ctx.Package.Owner, pv.PackageID); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.JSON(http.StatusOK, StatusResponse{OK: true}) +} diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index 5ffefc4862..2a20d66d1e 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant] + // enum: [cargo, composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant] // - name: q // in: query // description: name filter diff --git a/routers/web/org/setting_packages.go b/routers/web/org/setting_packages.go index 80135ca2d0..21d25bd90a 100644 --- a/routers/web/org/setting_packages.go +++ b/routers/web/org/setting_packages.go @@ -84,3 +84,23 @@ func PackagesRulePreview(ctx *context.Context) { ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview) } + +func InitializeCargoIndex(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.InitializeCargoIndex(ctx, ctx.ContextUser) + + ctx.Redirect(fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name)) +} + +func RebuildCargoIndex(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.RebuildCargoIndex(ctx, ctx.ContextUser) + + ctx.Redirect(fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name)) +} diff --git a/routers/web/shared/packages/packages.go b/routers/web/shared/packages/packages.go index b9aa40bdd2..30c25374d1 100644 --- a/routers/web/shared/packages/packages.go +++ b/routers/web/shared/packages/packages.go @@ -13,9 +13,11 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/services/forms" + cargo_service "code.gitea.io/gitea/services/packages/cargo" container_service "code.gitea.io/gitea/services/packages/container" ) @@ -223,3 +225,23 @@ func getCleanupRuleByContext(ctx *context.Context, owner *user_model.User) *pack return nil } + +func InitializeCargoIndex(ctx *context.Context, owner *user_model.User) { + err := cargo_service.InitializeIndexRepository(ctx, owner, owner) + if err != nil { + log.Error("InitializeIndexRepository failed: %v", err) + ctx.Flash.Error(ctx.Tr("packages.owner.settings.cargo.initialize.error", err)) + } else { + ctx.Flash.Success(ctx.Tr("packages.owner.settings.cargo.initialize.success")) + } +} + +func RebuildCargoIndex(ctx *context.Context, owner *user_model.User) { + err := cargo_service.RebuildIndex(ctx, owner, owner) + if err != nil { + log.Error("RebuildIndex failed: %v", err) + ctx.Flash.Error(ctx.Tr("packages.owner.settings.cargo.rebuild.error", err)) + } else { + ctx.Flash.Success(ctx.Tr("packages.owner.settings.cargo.rebuild.success")) + } +} diff --git a/routers/web/user/setting/packages.go b/routers/web/user/setting/packages.go index f6f7195adf..b3f8a3e41d 100644 --- a/routers/web/user/setting/packages.go +++ b/routers/web/user/setting/packages.go @@ -77,3 +77,21 @@ func PackagesRulePreview(ctx *context.Context) { ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview) } + +func InitializeCargoIndex(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.InitializeCargoIndex(ctx, ctx.Doer) + + ctx.Redirect(setting.AppSubURL + "/user/settings/packages") +} + +func RebuildCargoIndex(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.RebuildCargoIndex(ctx, ctx.Doer) + + ctx.Redirect(setting.AppSubURL + "/user/settings/packages") +} diff --git a/routers/web/web.go b/routers/web/web.go index b7128fc3a9..a024c0ac37 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -468,6 +468,10 @@ func RegisterRoutes(m *web.Route) { m.Get("/preview", user_setting.PackagesRulePreview) }) }) + m.Group("/cargo", func() { + m.Post("/initialize", user_setting.InitializeCargoIndex) + m.Post("/rebuild", user_setting.RebuildCargoIndex) + }) }, packagesEnabled) m.Group("/secrets", func() { m.Get("", user_setting.Secrets) @@ -818,6 +822,10 @@ func RegisterRoutes(m *web.Route) { m.Get("/preview", org.PackagesRulePreview) }) }) + m.Group("/cargo", func() { + m.Post("/initialize", org.InitializeCargoIndex) + m.Post("/rebuild", org.RebuildCargoIndex) + }) }, packagesEnabled) }, func(ctx *context.Context) { ctx.Data["EnableOAuth2"] = setting.OAuth2.Enable diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index aad0e39591..2e6560ec0c 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -16,7 +16,7 @@ import ( "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/migrations" mirror_service "code.gitea.io/gitea/services/mirror" - packages_service "code.gitea.io/gitea/services/packages" + packages_cleanup_service "code.gitea.io/gitea/services/packages/cleanup" repo_service "code.gitea.io/gitea/services/repository" archiver_service "code.gitea.io/gitea/services/repository/archiver" ) @@ -152,7 +152,7 @@ func registerCleanupPackages() { OlderThan: 24 * time.Hour, }, func(ctx context.Context, _ *user_model.User, config Config) error { realConfig := config.(*OlderThanConfig) - return packages_service.Cleanup(ctx, realConfig.OlderThan) + return packages_cleanup_service.Cleanup(ctx, realConfig.OlderThan) }) } diff --git a/services/forms/package_form.go b/services/forms/package_form.go index e78e64ef7e..558ed54b65 100644 --- a/services/forms/package_form.go +++ b/services/forms/package_form.go @@ -15,7 +15,7 @@ import ( type PackageCleanupRuleForm struct { ID int64 Enabled bool - Type string `binding:"Required;In(composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"` + Type string `binding:"Required;In(cargo,composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"` KeepCount int `binding:"In(0,1,5,10,25,50,100)"` KeepPattern string `binding:"RegexPattern"` RemoveDays int `binding:"In(0,7,14,30,60,90,180)"` diff --git a/services/packages/cargo/index.go b/services/packages/cargo/index.go new file mode 100644 index 0000000000..e58a472816 --- /dev/null +++ b/services/packages/cargo/index.go @@ -0,0 +1,290 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cargo + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "path" + "strconv" + "time" + + packages_model "code.gitea.io/gitea/models/packages" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/json" + cargo_module "code.gitea.io/gitea/modules/packages/cargo" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + files_service "code.gitea.io/gitea/services/repository/files" +) + +const ( + IndexRepositoryName = "_cargo-index" + ConfigFileName = "config.json" +) + +// https://doc.rust-lang.org/cargo/reference/registries.html#index-format + +func BuildPackagePath(name string) string { + switch len(name) { + case 0: + panic("Cargo package name can not be empty") + case 1: + return path.Join("1", name) + case 2: + return path.Join("2", name) + case 3: + return path.Join("3", string(name[0]), name) + default: + return path.Join(name[0:2], name[2:4], name) + } +} + +func InitializeIndexRepository(ctx context.Context, doer, owner *user_model.User) error { + repo, err := getOrCreateIndexRepository(ctx, doer, owner) + if err != nil { + return err + } + + if err := createOrUpdateConfigFile(ctx, repo, doer, owner); err != nil { + return fmt.Errorf("createOrUpdateConfigFile: %w", err) + } + + return nil +} + +func RebuildIndex(ctx context.Context, doer, owner *user_model.User) error { + repo, err := getOrCreateIndexRepository(ctx, doer, owner) + if err != nil { + return err + } + + ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeCargo) + if err != nil { + return fmt.Errorf("GetPackagesByType: %w", err) + } + + return alterRepositoryContent( + ctx, + doer, + repo, + "Rebuild Cargo Index", + func(t *files_service.TemporaryUploadRepository) error { + // Remove all existing content but the Cargo config + files, err := t.LsFiles() + if err != nil { + return err + } + for i, file := range files { + if file == ConfigFileName { + files[i] = files[len(files)-1] + files = files[:len(files)-1] + break + } + } + if err := t.RemoveFilesFromIndex(files...); err != nil { + return err + } + + // Add all packages + for _, p := range ps { + if err := addOrUpdatePackageIndex(ctx, t, p); err != nil { + return err + } + } + + return nil + }, + ) +} + +func AddOrUpdatePackageIndex(ctx context.Context, doer, owner *user_model.User, packageID int64) error { + repo, err := getOrCreateIndexRepository(ctx, doer, owner) + if err != nil { + return err + } + + p, err := packages_model.GetPackageByID(ctx, packageID) + if err != nil { + return fmt.Errorf("GetPackageByID[%d]: %w", packageID, err) + } + + return alterRepositoryContent( + ctx, + doer, + repo, + "Update "+p.Name, + func(t *files_service.TemporaryUploadRepository) error { + return addOrUpdatePackageIndex(ctx, t, p) + }, + ) +} + +type IndexVersionEntry struct { + Name string `json:"name"` + Version string `json:"vers"` + Dependencies []*cargo_module.Dependency `json:"deps"` + FileChecksum string `json:"cksum"` + Features map[string][]string `json:"features"` + Yanked bool `json:"yanked"` + Links string `json:"links,omitempty"` +} + +func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + Sort: packages_model.SortVersionAsc, + }) + if err != nil { + return fmt.Errorf("SearchVersions[%s]: %w", p.Name, err) + } + if len(pvs) == 0 { + return nil + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + return fmt.Errorf("GetPackageDescriptors[%s]: %w", p.Name, err) + } + + var b bytes.Buffer + for _, pd := range pds { + metadata := pd.Metadata.(*cargo_module.Metadata) + + dependencies := metadata.Dependencies + if dependencies == nil { + dependencies = make([]*cargo_module.Dependency, 0) + } + + features := metadata.Features + if features == nil { + features = make(map[string][]string) + } + + yanked, _ := strconv.ParseBool(pd.VersionProperties.GetByName(cargo_module.PropertyYanked)) + entry, err := json.Marshal(&IndexVersionEntry{ + Name: pd.Package.Name, + Version: pd.Version.Version, + Dependencies: dependencies, + FileChecksum: pd.Files[0].Blob.HashSHA256, + Features: features, + Yanked: yanked, + Links: metadata.Links, + }) + if err != nil { + return err + } + + b.Write(entry) + b.WriteString("\n") + } + + return writeObjectToIndex(t, BuildPackagePath(pds[0].Package.LowerName), &b) +} + +func getOrCreateIndexRepository(ctx context.Context, doer, owner *user_model.User) (*repo_model.Repository, error) { + repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, owner.Name, IndexRepositoryName) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + repo, err = repo_module.CreateRepository(doer, owner, repo_module.CreateRepoOptions{ + Name: IndexRepositoryName, + }) + if err != nil { + return nil, fmt.Errorf("CreateRepository: %w", err) + } + } else { + return nil, fmt.Errorf("GetRepositoryByOwnerAndName: %w", err) + } + } + + return repo, nil +} + +type Config struct { + DownloadURL string `json:"dl"` + APIURL string `json:"api"` +} + +func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository, doer, owner *user_model.User) error { + return alterRepositoryContent( + ctx, + doer, + repo, + "Initialize Cargo Config", + func(t *files_service.TemporaryUploadRepository) error { + var b bytes.Buffer + err := json.NewEncoder(&b).Encode(Config{ + DownloadURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo/api/v1/crates", + APIURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo", + }) + if err != nil { + return err + } + + return writeObjectToIndex(t, ConfigFileName, &b) + }, + ) +} + +// This is a shorter version of CreateOrUpdateRepoFile which allows to perform multiple actions on a git repository +func alterRepositoryContent(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, commitMessage string, fn func(*files_service.TemporaryUploadRepository) error) error { + t, err := files_service.NewTemporaryUploadRepository(ctx, repo) + if err != nil { + return err + } + defer t.Close() + + var lastCommitID string + if err := t.Clone(repo.DefaultBranch); err != nil { + if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { + return err + } + if err := t.Init(); err != nil { + return err + } + } else { + if err := t.SetDefaultIndex(); err != nil { + return err + } + + commit, err := t.GetBranchCommit(repo.DefaultBranch) + if err != nil { + return err + } + + lastCommitID = commit.ID.String() + } + + if err := fn(t); err != nil { + return err + } + + treeHash, err := t.WriteTree() + if err != nil { + return err + } + + now := time.Now() + commitHash, err := t.CommitTreeWithDate(lastCommitID, doer, doer, treeHash, commitMessage, false, now, now) + if err != nil { + return err + } + + return t.Push(doer, commitHash, repo.DefaultBranch) +} + +func writeObjectToIndex(t *files_service.TemporaryUploadRepository, path string, r io.Reader) error { + hash, err := t.HashObject(r) + if err != nil { + return err + } + + return t.AddObjectToIndex("100644", hash, path) +} diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go new file mode 100644 index 0000000000..2d62a028a4 --- /dev/null +++ b/services/packages/cleanup/cleanup.go @@ -0,0 +1,154 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package container + +import ( + "context" + "fmt" + "time" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/util" + packages_service "code.gitea.io/gitea/services/packages" + cargo_service "code.gitea.io/gitea/services/packages/cargo" + container_service "code.gitea.io/gitea/services/packages/container" +) + +// Cleanup removes expired package data +func Cleanup(taskCtx context.Context, olderThan time.Duration) error { + ctx, committer, err := db.TxContext(taskCtx) + if err != nil { + return err + } + defer committer.Close() + + err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { + select { + case <-taskCtx.Done(): + return db.ErrCancelledf("While processing package cleanup rules") + default: + } + + if err := pcr.CompiledPattern(); err != nil { + return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err) + } + + olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) + + packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err) + } + + for _, p := range packages { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + IsInternal: util.OptionalBoolFalse, + Sort: packages_model.SortCreatedDesc, + Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200), + }) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err) + } + versionDeleted := false + for _, pv := range pvs { + if pcr.Type == packages_model.TypeContainer { + if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) + } else if skip { + log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) + continue + } + } + + toMatch := pv.LowerVersion + if pcr.MatchFullName { + toMatch = p.LowerName + "/" + pv.LowerVersion + } + + if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version) + continue + } + if pv.CreatedUnix.AsLocalTime().After(olderThan) { + log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version) + continue + } + if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version) + continue + } + + log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version) + + if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err) + } + + versionDeleted = true + } + + if versionDeleted { + if pcr.Type == packages_model.TypeCargo { + owner, err := user_model.GetUserByID(ctx, pcr.OwnerID) + if err != nil { + return fmt.Errorf("GetUserByID failed: %w", err) + } + if err := cargo_service.AddOrUpdatePackageIndex(ctx, owner, owner, p.ID); err != nil { + return fmt.Errorf("CleanupRule [%d]: cargo.AddOrUpdatePackageIndex failed: %w", pcr.ID, err) + } + } + } + } + return nil + }) + if err != nil { + return err + } + + if err := container_service.Cleanup(ctx, olderThan); err != nil { + return err + } + + ps, err := packages_model.FindUnreferencedPackages(ctx) + if err != nil { + return err + } + for _, p := range ps { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypePackage, p.ID); err != nil { + return err + } + if err := packages_model.DeletePackageByID(ctx, p.ID); err != nil { + return err + } + } + + pbs, err := packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan) + if err != nil { + return err + } + + for _, pb := range pbs { + if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil { + return err + } + } + + if err := committer.Commit(); err != nil { + return err + } + + contentStore := packages_module.NewContentStore() + for _, pb := range pbs { + if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { + log.Error("Error deleting package blob [%v]: %v", pb.ID, err) + } + } + + return nil +} diff --git a/services/packages/packages.go b/services/packages/packages.go index 9e52cb1450..f502840755 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "strings" - "time" "code.gitea.io/gitea/models/db" packages_model "code.gitea.io/gitea/models/packages" @@ -22,7 +21,6 @@ import ( packages_module "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - container_service "code.gitea.io/gitea/services/packages/container" ) var ( @@ -335,6 +333,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p var typeSpecificSize int64 switch packageType { + case packages_model.TypeCargo: + typeSpecificSize = setting.Packages.LimitSizeCargo case packages_model.TypeComposer: typeSpecificSize = setting.Packages.LimitSizeComposer case packages_model.TypeConan: @@ -448,123 +448,6 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro return packages_model.DeleteFileByID(ctx, pf.ID) } -// Cleanup removes expired package data -func Cleanup(taskCtx context.Context, olderThan time.Duration) error { - ctx, committer, err := db.TxContext(taskCtx) - if err != nil { - return err - } - defer committer.Close() - - err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { - select { - case <-taskCtx.Done(): - return db.ErrCancelledf("While processing package cleanup rules") - default: - } - - if err := pcr.CompiledPattern(); err != nil { - return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err) - } - - olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) - - packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) - if err != nil { - return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err) - } - - for _, p := range packages { - pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ - PackageID: p.ID, - IsInternal: util.OptionalBoolFalse, - Sort: packages_model.SortCreatedDesc, - Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200), - }) - if err != nil { - return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err) - } - for _, pv := range pvs { - if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { - return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) - } else if skip { - log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) - continue - } - - toMatch := pv.LowerVersion - if pcr.MatchFullName { - toMatch = p.LowerName + "/" + pv.LowerVersion - } - - if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { - log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version) - continue - } - if pv.CreatedUnix.AsLocalTime().After(olderThan) { - log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version) - continue - } - if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { - log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version) - continue - } - - log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version) - - if err := DeletePackageVersionAndReferences(ctx, pv); err != nil { - return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err) - } - } - } - return nil - }) - if err != nil { - return err - } - - if err := container_service.Cleanup(ctx, olderThan); err != nil { - return err - } - - ps, err := packages_model.FindUnreferencedPackages(ctx) - if err != nil { - return err - } - for _, p := range ps { - if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypePackage, p.ID); err != nil { - return err - } - if err := packages_model.DeletePackageByID(ctx, p.ID); err != nil { - return err - } - } - - pbs, err := packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan) - if err != nil { - return err - } - - for _, pb := range pbs { - if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil { - return err - } - } - - if err := committer.Commit(); err != nil { - return err - } - - contentStore := packages_module.NewContentStore() - for _, pb := range pbs { - if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { - log.Error("Error deleting package blob [%v]: %v", pb.ID, err) - } - } - - return nil -} - // GetFileStreamByPackageNameAndVersion returns the content of the specific package file func GetFileStreamByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadSeekCloser, *packages_model.PackageFile, error) { log.Trace("Getting package file stream: %v, %v, %s, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pfi.Filename, pfi.CompositeKey) diff --git a/templates/org/settings/packages.tmpl b/templates/org/settings/packages.tmpl index 412a9813c2..04e5d45b50 100644 --- a/templates/org/settings/packages.tmpl +++ b/templates/org/settings/packages.tmpl @@ -7,6 +7,7 @@
{{template "base/alert" .}} {{template "package/shared/cleanup_rules/list" .}} + {{template "package/shared/cargo" .}}
diff --git a/templates/package/content/cargo.tmpl b/templates/package/content/cargo.tmpl new file mode 100644 index 0000000000..54c40a5b0d --- /dev/null +++ b/templates/package/content/cargo.tmpl @@ -0,0 +1,62 @@ +{{if eq .PackageDescriptor.Package.Type "cargo"}} +

{{.locale.Tr "packages.installation"}}

+
+
+
+ + 
[registry]
+default = "gitea"
+
+[registries.gitea]
+index = "{{AppUrl}}{{.PackageDescriptor.Owner.Name}}/_cargo-index.git"
+
+[net]
+git-fetch-with-cli = true
+
+
+ + 
cargo add {{.PackageDescriptor.Package.Name}}@{{.PackageDescriptor.Version.Version}}
+
+
+ +
+
+
+ + {{if or .PackageDescriptor.Metadata.Description .PackageDescriptor.Metadata.Readme}} +

{{.locale.Tr "packages.about"}}

+ {{if .PackageDescriptor.Metadata.Description}}
{{.PackageDescriptor.Metadata.Description}}
{{end}} + {{if .PackageDescriptor.Metadata.Readme}}
{{RenderMarkdownToHtml .PackageDescriptor.Metadata.Readme}}
{{end}} + {{end}} + + {{if .PackageDescriptor.Metadata.Dependencies}} +

{{.locale.Tr "packages.dependencies"}}

+
+ + + + + + + + + {{range .PackageDescriptor.Metadata.Dependencies}} + + + + + {{end}} + +
{{.locale.Tr "packages.dependency.id"}}{{.locale.Tr "packages.dependency.version"}}
{{.Name}}{{.Req}}
+
+ {{end}} + + {{if .PackageDescriptor.Metadata.Keywords}} +

{{.locale.Tr "packages.keywords"}}

+
+ {{range .PackageDescriptor.Metadata.Keywords}} + {{.}} + {{end}} +
+ {{end}} +{{end}} diff --git a/templates/package/metadata/cargo.tmpl b/templates/package/metadata/cargo.tmpl new file mode 100644 index 0000000000..68232d4832 --- /dev/null +++ b/templates/package/metadata/cargo.tmpl @@ -0,0 +1,7 @@ +{{if eq .PackageDescriptor.Package.Type "cargo"}} + {{range .PackageDescriptor.Metadata.Authors}}
{{svg "octicon-person" 16 "mr-3"}} {{.}}
{{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.project_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.RepositoryURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.cargo.details.repository_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.DocumentationURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.cargo.details.documentation_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.License}}
{{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.License}}
{{end}} +{{end}} diff --git a/templates/package/shared/cargo.tmpl b/templates/package/shared/cargo.tmpl new file mode 100644 index 0000000000..b9c2ef4d2b --- /dev/null +++ b/templates/package/shared/cargo.tmpl @@ -0,0 +1,24 @@ +

+ {{.locale.Tr "packages.owner.settings.cargo.title"}} +

+
+
+
+ +
+
+ {{.CsrfTokenHtml}} + +
+
+ +
+
+ {{.CsrfTokenHtml}} + +
+
+ +
+
+
diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index 4611cdb8d8..6209e6cb02 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -19,6 +19,7 @@
+ {{template "package/content/cargo" .}} {{template "package/content/composer" .}} {{template "package/content/conan" .}} {{template "package/content/conda" .}} @@ -43,6 +44,7 @@ {{end}}
{{svg "octicon-calendar" 16 "mr-3"}} {{TimeSinceUnix .PackageDescriptor.Version.CreatedUnix $.locale}}
{{svg "octicon-download" 16 "mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}
+ {{template "package/metadata/cargo" .}} {{template "package/metadata/composer" .}} {{template "package/metadata/conan" .}} {{template "package/metadata/conda" .}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 5a3f2c2bb9..d6e3009b35 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -2100,6 +2100,7 @@ }, { "enum": [ + "cargo", "composer", "conan", "conda", diff --git a/templates/user/settings/packages.tmpl b/templates/user/settings/packages.tmpl index 9aae14f934..472d84327a 100644 --- a/templates/user/settings/packages.tmpl +++ b/templates/user/settings/packages.tmpl @@ -4,6 +4,7 @@
{{template "base/alert" .}} {{template "package/shared/cleanup_rules/list" .}} + {{template "package/shared/cargo" .}}
{{template "base/footer" .}} diff --git a/tests/integration/api_packages_cargo_test.go b/tests/integration/api_packages_cargo_test.go new file mode 100644 index 0000000000..0c542eaf1e --- /dev/null +++ b/tests/integration/api_packages_cargo_test.go @@ -0,0 +1,341 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integration + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "net/http" + neturl "net/url" + "testing" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/json" + cargo_module "code.gitea.io/gitea/modules/packages/cargo" + "code.gitea.io/gitea/modules/setting" + cargo_router "code.gitea.io/gitea/routers/api/packages/cargo" + cargo_service "code.gitea.io/gitea/services/packages/cargo" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/assert" +) + +func TestPackageCargo(t *testing.T) { + onGiteaRun(t, testPackageCargo) +} + +func testPackageCargo(t *testing.T, _ *neturl.URL) { + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + packageName := "cargo-package" + packageVersion := "1.0.3" + packageDescription := "Package Description" + packageAuthor := "KN4CK3R" + packageHomepage := "https://gitea.io/" + packageLicense := "MIT" + + createPackage := func(name, version string) io.Reader { + metadata := `{ + "name":"` + name + `", + "vers":"` + version + `", + "description":"` + packageDescription + `", + "authors": ["` + packageAuthor + `"], + "deps":[ + { + "name":"dep", + "version_req":"1.0", + "registry": "https://gitea.io/user/_cargo-index", + "kind": "normal", + "default_features": true + } + ], + "homepage":"` + packageHomepage + `", + "license":"` + packageLicense + `" +}` + + var buf bytes.Buffer + binary.Write(&buf, binary.LittleEndian, uint32(len(metadata))) + buf.WriteString(metadata) + binary.Write(&buf, binary.LittleEndian, uint32(4)) + buf.WriteString("test") + return &buf + } + + err := cargo_service.InitializeIndexRepository(db.DefaultContext, user, user) + assert.NoError(t, err) + + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, user.Name, cargo_service.IndexRepositoryName) + assert.NotNil(t, repo) + assert.NoError(t, err) + + readGitContent := func(t *testing.T, path string) string { + gitRepo, err := git.OpenRepository(db.DefaultContext, repo.RepoPath()) + assert.NoError(t, err) + defer gitRepo.Close() + + commit, err := gitRepo.GetBranchCommit(repo.DefaultBranch) + assert.NoError(t, err) + + blob, err := commit.GetBlobByPath(path) + assert.NoError(t, err) + + content, err := blob.GetBlobContent() + assert.NoError(t, err) + + return content + } + + root := fmt.Sprintf("%sapi/packages/%s/cargo", setting.AppURL, user.Name) + url := fmt.Sprintf("%s/api/v1/crates", root) + + t.Run("Index", func(t *testing.T) { + t.Run("Config", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + content := readGitContent(t, cargo_service.ConfigFileName) + + var config cargo_service.Config + err := json.Unmarshal([]byte(content), &config) + assert.NoError(t, err) + + assert.Equal(t, url, config.DownloadURL) + assert.Equal(t, root, config.APIURL) + }) + }) + + t.Run("Upload", func(t *testing.T) { + t.Run("InvalidNameOrVersion", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + content := createPackage("0test", "1.0.0") + + req := NewRequestWithBody(t, "PUT", url+"/new", content) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusBadRequest) + + var status cargo_router.StatusResponse + DecodeJSON(t, resp, &status) + assert.False(t, status.OK) + + content = createPackage("test", "-1.0.0") + + req = NewRequestWithBody(t, "PUT", url+"/new", content) + req = AddBasicAuthHeader(req, user.Name) + resp = MakeRequest(t, req, http.StatusBadRequest) + + DecodeJSON(t, resp, &status) + assert.False(t, status.OK) + }) + + t.Run("InvalidContent", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + metadata := `{"name":"test","vers":"1.0.0"}` + + var buf bytes.Buffer + binary.Write(&buf, binary.LittleEndian, uint32(len(metadata))) + buf.WriteString(metadata) + binary.Write(&buf, binary.LittleEndian, uint32(4)) + buf.WriteString("te") + + req := NewRequestWithBody(t, "PUT", url+"/new", &buf) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusBadRequest) + }) + + t.Run("Valid", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequestWithBody(t, "PUT", url+"/new", createPackage(packageName, packageVersion)) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithBody(t, "PUT", url+"/new", createPackage(packageName, packageVersion)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var status cargo_router.StatusResponse + DecodeJSON(t, resp, &status) + assert.True(t, status.OK) + + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeCargo) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.IsType(t, &cargo_module.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, fmt.Sprintf("%s-%s.crate", packageName, packageVersion), pfs[0].Name) + assert.True(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.EqualValues(t, 4, pb.Size) + + req = NewRequestWithBody(t, "PUT", url+"/new", createPackage(packageName, packageVersion)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusConflict) + + t.Run("Index", func(t *testing.T) { + t.Run("Entry", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) + + var entry cargo_service.IndexVersionEntry + err := json.Unmarshal([]byte(content), &entry) + assert.NoError(t, err) + + assert.Equal(t, packageName, entry.Name) + assert.Equal(t, packageVersion, entry.Version) + assert.Equal(t, pb.HashSHA256, entry.FileChecksum) + assert.False(t, entry.Yanked) + assert.Len(t, entry.Dependencies, 1) + dep := entry.Dependencies[0] + assert.Equal(t, "dep", dep.Name) + assert.Equal(t, "1.0", dep.Req) + assert.Equal(t, "normal", dep.Kind) + assert.True(t, dep.DefaultFeatures) + assert.Empty(t, dep.Features) + assert.False(t, dep.Optional) + assert.Nil(t, dep.Target) + assert.NotNil(t, dep.Registry) + assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry) + assert.Nil(t, dep.Package) + }) + + t.Run("Rebuild", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + err := cargo_service.RebuildIndex(db.DefaultContext, user, user) + assert.NoError(t, err) + + _ = readGitContent(t, cargo_service.BuildPackagePath(packageName)) + }) + }) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages.TypeCargo, packageName, packageVersion) + assert.NoError(t, err) + assert.EqualValues(t, 0, pv.DownloadCount) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pv.ID) + assert.NoError(t, err) + assert.Len(t, pfs, 1) + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s/download", url, neturl.PathEscape(packageName), neturl.PathEscape(pv.Version))) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, "test", resp.Body.String()) + + pv, err = packages.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages.TypeCargo, packageName, packageVersion) + assert.NoError(t, err) + assert.EqualValues(t, 1, pv.DownloadCount) + }) + + t.Run("Search", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + cases := []struct { + Query string + Page int + PerPage int + ExpectedTotal int64 + ExpectedResults int + }{ + {"", 0, 0, 1, 1}, + {"", 1, 10, 1, 1}, + {"cargo", 1, 0, 1, 1}, + {"cargo", 1, 10, 1, 1}, + {"cargo", 2, 10, 1, 0}, + {"test", 0, 10, 0, 0}, + } + + for i, c := range cases { + req := NewRequest(t, "GET", fmt.Sprintf("%s?q=%s&page=%d&per_page=%d", url, c.Query, c.Page, c.PerPage)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result cargo_router.SearchResult + DecodeJSON(t, resp, &result) + + assert.Equal(t, c.ExpectedTotal, result.Meta.Total, "case %d: unexpected total hits", i) + assert.Len(t, result.Crates, c.ExpectedResults, "case %d: unexpected result count", i) + } + }) + + t.Run("Yank", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/%s/%s/yank", url, neturl.PathEscape(packageName), neturl.PathEscape(packageVersion))) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var status cargo_router.StatusResponse + DecodeJSON(t, resp, &status) + assert.True(t, status.OK) + + content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) + + var entry cargo_service.IndexVersionEntry + err := json.Unmarshal([]byte(content), &entry) + assert.NoError(t, err) + + assert.True(t, entry.Yanked) + }) + + t.Run("Unyank", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "PUT", fmt.Sprintf("%s/%s/%s/unyank", url, neturl.PathEscape(packageName), neturl.PathEscape(packageVersion))) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var status cargo_router.StatusResponse + DecodeJSON(t, resp, &status) + assert.True(t, status.OK) + + content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) + + var entry cargo_service.IndexVersionEntry + err := json.Unmarshal([]byte(content), &entry) + assert.NoError(t, err) + + assert.False(t, entry.Yanked) + }) + + t.Run("ListOwners", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/owners", url, neturl.PathEscape(packageName))) + resp := MakeRequest(t, req, http.StatusOK) + + var owners cargo_router.Owners + DecodeJSON(t, resp, &owners) + + assert.Len(t, owners.Users, 1) + assert.Equal(t, user.ID, owners.Users[0].ID) + assert.Equal(t, user.Name, owners.Users[0].Login) + assert.Equal(t, user.DisplayName(), owners.Users[0].Name) + }) +} diff --git a/tests/integration/api_packages_test.go b/tests/integration/api_packages_test.go index a6c335eb7e..39852e212c 100644 --- a/tests/integration/api_packages_test.go +++ b/tests/integration/api_packages_test.go @@ -21,6 +21,7 @@ import ( "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" packages_service "code.gitea.io/gitea/services/packages" + packages_cleanup_service "code.gitea.io/gitea/services/packages/cleanup" "code.gitea.io/gitea/tests" "github.com/stretchr/testify/assert" @@ -246,7 +247,7 @@ func TestPackageCleanup(t *testing.T) { _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion) assert.NoError(t, err) - err = packages_service.Cleanup(db.DefaultContext, duration) + err = packages_cleanup_service.Cleanup(db.DefaultContext, duration) assert.NoError(t, err) pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration) @@ -383,7 +384,7 @@ func TestPackageCleanup(t *testing.T) { pcr, err := packages_model.InsertCleanupRule(db.DefaultContext, c.Rule) assert.NoError(t, err) - err = packages_service.Cleanup(db.DefaultContext, duration) + err = packages_cleanup_service.Cleanup(db.DefaultContext, duration) assert.NoError(t, err) for _, v := range c.Versions { diff --git a/web_src/svg/gitea-cargo.svg b/web_src/svg/gitea-cargo.svg new file mode 100644 index 0000000000..dbec107ad0 --- /dev/null +++ b/web_src/svg/gitea-cargo.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file