From 6ba9ff7b4899f1057ac6e41947951da3e43b6918 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 1 Feb 2023 19:30:39 +0100 Subject: [PATCH] Add Conda package registry (#22262) This PR adds a [Conda](https://conda.io/) package registry. --- custom/conf/app.example.ini | 2 + .../doc/advanced/config-cheat-sheet.en-us.md | 1 + docs/content/doc/packages/conda.en-us.md | 85 +++++ docs/content/doc/packages/overview.en-us.md | 1 + go.mod | 2 +- models/packages/conda/search.go | 63 ++++ models/packages/descriptor.go | 3 + models/packages/package.go | 6 + modules/packages/conda/metadata.go | 243 ++++++++++++++ modules/packages/conda/metadata_test.go | 150 +++++++++ modules/setting/packages.go | 2 + options/locale/locale_en-US.ini | 5 + public/img/svg/gitea-conda.svg | 1 + routers/api/packages/api.go | 38 +++ routers/api/packages/conda/conda.go | 306 ++++++++++++++++++ routers/api/v1/packages/package.go | 2 +- services/forms/package_form.go | 2 +- services/packages/packages.go | 2 + templates/package/content/conda.tmpl | 30 ++ templates/package/metadata/conda.tmpl | 6 + templates/package/view.tmpl | 2 + templates/swagger/v1_json.tmpl | 1 + tests/integration/api_packages_conda_test.go | 274 ++++++++++++++++ web_src/svg/gitea-conda.svg | 20 ++ 24 files changed, 1244 insertions(+), 3 deletions(-) create mode 100644 docs/content/doc/packages/conda.en-us.md create mode 100644 models/packages/conda/search.go create mode 100644 modules/packages/conda/metadata.go create mode 100644 modules/packages/conda/metadata_test.go create mode 100644 public/img/svg/gitea-conda.svg create mode 100644 routers/api/packages/conda/conda.go create mode 100644 templates/package/content/conda.tmpl create mode 100644 templates/package/metadata/conda.tmpl create mode 100644 tests/integration/api_packages_conda_test.go create mode 100644 web_src/svg/gitea-conda.svg diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index daf67ef4c6..77efe1417b 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2458,6 +2458,8 @@ ROUTER = console ;LIMIT_SIZE_COMPOSER = -1 ;; Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_CONAN = -1 +;; Maximum size of a Conda upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_CONDA = -1 ;; Maximum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_CONTAINER = -1 ;; Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 4ef630b6be..441bb824ad 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -1214,6 +1214,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `LIMIT_TOTAL_OWNER_SIZE`: **-1**: Maximum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_COMPOSER`: **-1**: Maximum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CONAN`: **-1**: Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_CONDA`: **-1**: Maximum size of a Conda upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CONTAINER`: **-1**: Maximum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_GENERIC`: **-1**: Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_HELM`: **-1**: Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/packages/conda.en-us.md b/docs/content/doc/packages/conda.en-us.md new file mode 100644 index 0000000000..8b82847590 --- /dev/null +++ b/docs/content/doc/packages/conda.en-us.md @@ -0,0 +1,85 @@ +--- +date: "2022-12-28T00:00:00+00:00" +title: "Conda Packages Repository" +slug: "packages/conda" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Conda" + weight: 25 + identifier: "conda" +--- + +# Conda Packages Repository + +Publish [Conda](https://docs.conda.io/en/latest/) packages for your user or organization. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the Conda package registry, you need to use [conda](https://docs.conda.io/projects/conda/en/stable/user-guide/install/index.html). + +## Configuring the package registry + +To register the package registry and provide credentials, edit your `.condarc` file: + +```yaml +channel_alias: https://gitea.example.com/api/packages/{owner}/conda +channels: + - https://gitea.example.com/api/packages/{owner}/conda +default_channels: + - https://gitea.example.com/api/packages/{owner}/conda +``` + +| Placeholder | Description | +| ------------ | ----------- | +| `owner` | The owner of the package. | + +See the [official documentation](https://conda.io/projects/conda/en/latest/user-guide/configuration/use-condarc.html) for explanations of the individual settings. + +If you need to provide credentials, you may embed them as part of the channel url (`https://user:password@gitea.example.com/...`). + +## Publish a package + +To publish a package, perform a HTTP PUT operation with the package content in the request body. + +``` +PUT https://gitea.example.com/api/packages/{owner}/conda/{channel}/{filename} +``` + +| Placeholder | Description | +| ------------ | ----------- | +| `owner` | The owner of the package. | +| `channel` | The [channel](https://conda.io/projects/conda/en/latest/user-guide/concepts/channels.html) of the package. (optional) | +| `filename` | The name of the file. | + +Example request using HTTP Basic authentication: + +```shell +curl --user your_username:your_password_or_token \ + --upload-file path/to/package-1.0.conda \ + https://gitea.example.com/api/packages/testuser/conda/package-1.0.conda +``` + +You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. + +## Install a package + +To install a package from the package registry, execute one of the following commands: + +```shell +conda install {package_name} +conda install {package_name}={package_version} +conda install -c {channel} {package_name} +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `package_name` | The package name. | +| `package_version` | The package version. | +| `channel` | The channel of the package. (optional) | diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md index b12155e14d..9a736c1e56 100644 --- a/docs/content/doc/packages/overview.en-us.md +++ b/docs/content/doc/packages/overview.en-us.md @@ -28,6 +28,7 @@ The following package managers are currently supported: | ---- | -------- | -------------- | | [Composer]({{< relref "doc/packages/composer.en-us.md" >}}) | PHP | `composer` | | [Conan]({{< relref "doc/packages/conan.en-us.md" >}}) | C++ | `conan` | +| [Conda]({{< relref "doc/packages/conda.en-us.md" >}}) | - | `conda` | | [Container]({{< relref "doc/packages/container.en-us.md" >}}) | - | any OCI compliant client | | [Generic]({{< relref "doc/packages/generic.en-us.md" >}}) | - | any HTTP client | | [Helm]({{< relref "doc/packages/helm.en-us.md" >}}) | - | any HTTP client, `cm-push` | diff --git a/go.mod b/go.mod index b3d4040323..a929508e0d 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21 github.com/djherbis/buffer v1.2.0 github.com/djherbis/nio/v3 v3.0.1 + github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 github.com/dustin/go-humanize v1.0.0 github.com/editorconfig/editorconfig-core-go/v2 v2.5.1 github.com/emersion/go-imap v1.2.1 @@ -161,7 +162,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.7.0 // indirect - github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 // indirect github.com/fatih/color v1.13.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect diff --git a/models/packages/conda/search.go b/models/packages/conda/search.go new file mode 100644 index 0000000000..887441e3b2 --- /dev/null +++ b/models/packages/conda/search.go @@ -0,0 +1,63 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conda + +import ( + "context" + "strings" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + conda_module "code.gitea.io/gitea/modules/packages/conda" + + "xorm.io/builder" +) + +type FileSearchOptions struct { + OwnerID int64 + Channel string + Subdir string + Filename string +} + +// SearchFiles gets all files matching the search options +func SearchFiles(ctx context.Context, opts *FileSearchOptions) ([]*packages.PackageFile, error) { + var cond builder.Cond = builder.Eq{ + "package.type": packages.TypeConda, + "package.owner_id": opts.OwnerID, + "package_version.is_internal": false, + } + + if opts.Filename != "" { + cond = cond.And(builder.Eq{ + "package_file.lower_name": strings.ToLower(opts.Filename), + }) + } + + var versionPropsCond builder.Cond = builder.Eq{ + "package_property.ref_type": packages.PropertyTypePackage, + "package_property.name": conda_module.PropertyChannel, + "package_property.value": opts.Channel, + } + + cond = cond.And(builder.In("package.id", builder.Select("package_property.ref_id").Where(versionPropsCond).From("package_property"))) + + var filePropsCond builder.Cond = builder.Eq{ + "package_property.ref_type": packages.PropertyTypeFile, + "package_property.name": conda_module.PropertySubdir, + "package_property.value": opts.Subdir, + } + + cond = cond.And(builder.In("package_file.id", builder.Select("package_property.ref_id").Where(filePropsCond).From("package_property"))) + + sess := db.GetEngine(ctx). + Select("package_file.*"). + Table("package_file"). + Join("INNER", "package_version", "package_version.id = package_file.version_id"). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(cond) + + pfs := make([]*packages.PackageFile, 0, 10) + return pfs, sess.Find(&pfs) +} diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 34f1cad87d..3b36ee2266 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages/composer" "code.gitea.io/gitea/modules/packages/conan" + "code.gitea.io/gitea/modules/packages/conda" "code.gitea.io/gitea/modules/packages/container" "code.gitea.io/gitea/modules/packages/helm" "code.gitea.io/gitea/modules/packages/maven" @@ -132,6 +133,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc metadata = &composer.Metadata{} case TypeConan: metadata = &conan.Metadata{} + case TypeConda: + metadata = &conda.VersionMetadata{} case TypeContainer: metadata = &container.Metadata{} case TypeGeneric: diff --git a/models/packages/package.go b/models/packages/package.go index a804f35de3..0015953d81 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -32,6 +32,7 @@ type Type string const ( TypeComposer Type = "composer" TypeConan Type = "conan" + TypeConda Type = "conda" TypeContainer Type = "container" TypeGeneric Type = "generic" TypeHelm Type = "helm" @@ -47,6 +48,7 @@ const ( var TypeList = []Type{ TypeComposer, TypeConan, + TypeConda, TypeContainer, TypeGeneric, TypeHelm, @@ -66,6 +68,8 @@ func (pt Type) Name() string { return "Composer" case TypeConan: return "Conan" + case TypeConda: + return "Conda" case TypeContainer: return "Container" case TypeGeneric: @@ -97,6 +101,8 @@ func (pt Type) SVGName() string { return "gitea-composer" case TypeConan: return "gitea-conan" + case TypeConda: + return "gitea-conda" case TypeContainer: return "octicon-container" case TypeGeneric: diff --git a/modules/packages/conda/metadata.go b/modules/packages/conda/metadata.go new file mode 100644 index 0000000000..02dbf313ba --- /dev/null +++ b/modules/packages/conda/metadata.go @@ -0,0 +1,243 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conda + +import ( + "archive/tar" + "archive/zip" + "compress/bzip2" + "io" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "github.com/klauspost/compress/zstd" +) + +var ( + ErrInvalidStructure = util.SilentWrap{Message: "package structure is invalid", Err: util.ErrInvalidArgument} + ErrInvalidName = util.SilentWrap{Message: "package name is invalid", Err: util.ErrInvalidArgument} + ErrInvalidVersion = util.SilentWrap{Message: "package version is invalid", Err: util.ErrInvalidArgument} +) + +const ( + PropertyName = "conda.name" + PropertyChannel = "conda.channel" + PropertySubdir = "conda.subdir" + PropertyMetadata = "conda.metdata" +) + +// Package represents a Conda package +type Package struct { + Name string + Version string + Subdir string + VersionMetadata *VersionMetadata + FileMetadata *FileMetadata +} + +// VersionMetadata represents the metadata of a Conda package +type VersionMetadata struct { + Description string `json:"description,omitempty"` + Summary string `json:"summary,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + DocumentationURL string `json:"documentation_url,omitempty"` + License string `json:"license,omitempty"` + LicenseFamily string `json:"license_family,omitempty"` +} + +// FileMetadata represents the metadata of a Conda package file +type FileMetadata struct { + IsCondaPackage bool `json:"is_conda"` + Architecture string `json:"architecture,omitempty"` + NoArch string `json:"noarch,omitempty"` + Build string `json:"build,omitempty"` + BuildNumber int64 `json:"build_number,omitempty"` + Dependencies []string `json:"dependencies,omitempty"` + Platform string `json:"platform,omitempty"` + Timestamp int64 `json:"timestamp,omitempty"` +} + +type index struct { + Name string `json:"name"` + Version string `json:"version"` + Architecture string `json:"arch"` + NoArch string `json:"noarch"` + Build string `json:"build"` + BuildNumber int64 `json:"build_number"` + Dependencies []string `json:"depends"` + License string `json:"license"` + LicenseFamily string `json:"license_family"` + Platform string `json:"platform"` + Subdir string `json:"subdir"` + Timestamp int64 `json:"timestamp"` +} + +type about struct { + Description string `json:"description"` + Summary string `json:"summary"` + ProjectURL string `json:"home"` + RepositoryURL string `json:"dev_url"` + DocumentationURL string `json:"doc_url"` +} + +type ReaderAndReaderAt interface { + io.Reader + io.ReaderAt +} + +// ParsePackageBZ2 parses the Conda package file compressed with bzip2 +func ParsePackageBZ2(r io.Reader) (*Package, error) { + gzr := bzip2.NewReader(r) + + return parsePackageTar(gzr) +} + +// ParsePackageConda parses the Conda package file compressed with zip and zstd +func ParsePackageConda(r io.ReaderAt, size int64) (*Package, error) { + zr, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + for _, file := range zr.File { + if strings.HasPrefix(file.Name, "info-") && strings.HasSuffix(file.Name, ".tar.zst") { + f, err := zr.Open(file.Name) + if err != nil { + return nil, err + } + defer f.Close() + + dec, err := zstd.NewReader(f) + if err != nil { + return nil, err + } + defer dec.Close() + + p, err := parsePackageTar(dec) + if p != nil { + p.FileMetadata.IsCondaPackage = true + } + return p, err + } + } + + return nil, ErrInvalidStructure +} + +func parsePackageTar(r io.Reader) (*Package, error) { + var i *index + var a *about + + tr := tar.NewReader(r) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hdr.Typeflag != tar.TypeReg { + continue + } + + if hdr.Name == "info/index.json" { + if err := json.NewDecoder(tr).Decode(&i); err != nil { + return nil, err + } + + if !checkName(i.Name) { + return nil, ErrInvalidName + } + + if !checkVersion(i.Version) { + return nil, ErrInvalidVersion + } + + if a != nil { + break // stop loop if both files were found + } + } else if hdr.Name == "info/about.json" { + if err := json.NewDecoder(tr).Decode(&a); err != nil { + return nil, err + } + + if !validation.IsValidURL(a.ProjectURL) { + a.ProjectURL = "" + } + if !validation.IsValidURL(a.RepositoryURL) { + a.RepositoryURL = "" + } + if !validation.IsValidURL(a.DocumentationURL) { + a.DocumentationURL = "" + } + + if i != nil { + break // stop loop if both files were found + } + } + } + + if i == nil { + return nil, ErrInvalidStructure + } + if a == nil { + a = &about{} + } + + return &Package{ + Name: i.Name, + Version: i.Version, + Subdir: i.Subdir, + VersionMetadata: &VersionMetadata{ + License: i.License, + LicenseFamily: i.LicenseFamily, + Description: a.Description, + Summary: a.Summary, + ProjectURL: a.ProjectURL, + RepositoryURL: a.RepositoryURL, + DocumentationURL: a.DocumentationURL, + }, + FileMetadata: &FileMetadata{ + Architecture: i.Architecture, + NoArch: i.NoArch, + Build: i.Build, + BuildNumber: i.BuildNumber, + Dependencies: i.Dependencies, + Platform: i.Platform, + Timestamp: i.Timestamp, + }, + }, nil +} + +// https://github.com/conda/conda-build/blob/db9a728a9e4e6cfc895637ca3221117970fc2663/conda_build/metadata.py#L1393 +func checkName(name string) bool { + if name == "" { + return false + } + if name != strings.ToLower(name) { + return false + } + return !checkBadCharacters(name, "!") +} + +// https://github.com/conda/conda-build/blob/db9a728a9e4e6cfc895637ca3221117970fc2663/conda_build/metadata.py#L1403 +func checkVersion(version string) bool { + if version == "" { + return false + } + return !checkBadCharacters(version, "-") +} + +func checkBadCharacters(s, additional string) bool { + if strings.ContainsAny(s, "=@#$%^&*:;\"'\\|<>?/ ") { + return true + } + return strings.ContainsAny(s, additional) +} diff --git a/modules/packages/conda/metadata_test.go b/modules/packages/conda/metadata_test.go new file mode 100644 index 0000000000..2038ca370c --- /dev/null +++ b/modules/packages/conda/metadata_test.go @@ -0,0 +1,150 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conda + +import ( + "archive/tar" + "archive/zip" + "bytes" + "io" + "testing" + + "github.com/dsnet/compress/bzip2" + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/assert" +) + +const ( + packageName = "gitea" + packageVersion = "1.0.1" + description = "Package Description" + projectURL = "https://gitea.io" + repositoryURL = "https://gitea.io/gitea/gitea" + documentationURL = "https://docs.gitea.io" +) + +func TestParsePackage(t *testing.T) { + createArchive := func(files map[string][]byte) *bytes.Buffer { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for filename, content := range files { + hdr := &tar.Header{ + Name: filename, + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write(content) + } + tw.Close() + return &buf + } + + t.Run("MissingIndexFile", func(t *testing.T) { + buf := createArchive(map[string][]byte{"dummy.txt": {}}) + + p, err := parsePackageTar(buf) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidStructure) + }) + + t.Run("MissingAboutFile", func(t *testing.T) { + buf := createArchive(map[string][]byte{"info/index.json": []byte(`{"name":"name","version":"1.0"}`)}) + + p, err := parsePackageTar(buf) + assert.NotNil(t, p) + assert.NoError(t, err) + + assert.Equal(t, "name", p.Name) + assert.Equal(t, "1.0", p.Version) + assert.Empty(t, p.VersionMetadata.ProjectURL) + }) + + t.Run("InvalidName", func(t *testing.T) { + for _, name := range []string{"", "name!", "nAMe"} { + buf := createArchive(map[string][]byte{"info/index.json": []byte(`{"name":"` + name + `","version":"1.0"}`)}) + + p, err := parsePackageTar(buf) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidName) + } + }) + + t.Run("InvalidVersion", func(t *testing.T) { + for _, version := range []string{"", "1.0-2"} { + buf := createArchive(map[string][]byte{"info/index.json": []byte(`{"name":"name","version":"` + version + `"}`)}) + + p, err := parsePackageTar(buf) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidVersion) + } + }) + + t.Run("Valid", func(t *testing.T) { + buf := createArchive(map[string][]byte{ + "info/index.json": []byte(`{"name":"` + packageName + `","version":"` + packageVersion + `","subdir":"linux-64"}`), + "info/about.json": []byte(`{"description":"` + description + `","dev_url":"` + repositoryURL + `","doc_url":"` + documentationURL + `","home":"` + projectURL + `"}`), + }) + + p, err := parsePackageTar(buf) + assert.NotNil(t, p) + assert.NoError(t, err) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, "linux-64", p.Subdir) + assert.Equal(t, description, p.VersionMetadata.Description) + assert.Equal(t, projectURL, p.VersionMetadata.ProjectURL) + assert.Equal(t, repositoryURL, p.VersionMetadata.RepositoryURL) + assert.Equal(t, documentationURL, p.VersionMetadata.DocumentationURL) + }) + + t.Run(".tar.bz2", func(t *testing.T) { + tarArchive := createArchive(map[string][]byte{ + "info/index.json": []byte(`{"name":"` + packageName + `","version":"` + packageVersion + `"}`), + }) + + var buf bytes.Buffer + bw, _ := bzip2.NewWriter(&buf, nil) + io.Copy(bw, tarArchive) + bw.Close() + + br := bytes.NewReader(buf.Bytes()) + + p, err := ParsePackageBZ2(br) + assert.NotNil(t, p) + assert.NoError(t, err) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.False(t, p.FileMetadata.IsCondaPackage) + }) + + t.Run(".conda", func(t *testing.T) { + tarArchive := createArchive(map[string][]byte{ + "info/index.json": []byte(`{"name":"` + packageName + `","version":"` + packageVersion + `"}`), + }) + + var infoBuf bytes.Buffer + zsw, _ := zstd.NewWriter(&infoBuf) + io.Copy(zsw, tarArchive) + zsw.Close() + + var buf bytes.Buffer + zpw := zip.NewWriter(&buf) + w, _ := zpw.Create("info-x.tar.zst") + w.Write(infoBuf.Bytes()) + zpw.Close() + + br := bytes.NewReader(buf.Bytes()) + + p, err := ParsePackageConda(br, int64(br.Len())) + assert.NotNil(t, p) + assert.NoError(t, err) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.True(t, p.FileMetadata.IsCondaPackage) + }) +} diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 120fbb5bda..d0cd80aa03 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -27,6 +27,7 @@ var ( LimitTotalOwnerSize int64 LimitSizeComposer int64 LimitSizeConan int64 + LimitSizeConda int64 LimitSizeContainer int64 LimitSizeGeneric int64 LimitSizeHelm int64 @@ -66,6 +67,7 @@ func newPackages() { Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER") Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN") + Packages.LimitSizeConda = mustBytes(sec, "LIMIT_SIZE_CONDA") Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER") Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC") Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM") diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 8a48a68b17..8465660cc0 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3159,6 +3159,11 @@ conan.details.repository = Repository conan.registry = Setup this registry from the command line: conan.install = To install the package using Conan, run the following command: conan.documentation = For more information on the Conan registry, see the documentation. +conda.registry = Setup this registry as a Conda repository in your .condarc file: +conda.install = To install the package using Conda, run the following command: +conda.documentation = For more information on the Conda registry, see the documentation. +conda.details.repository_site = Repository Site +conda.details.documentation_site = Documentation Site container.details.type = Image Type container.details.platform = Platform container.details.repository_site = Repository Site diff --git a/public/img/svg/gitea-conda.svg b/public/img/svg/gitea-conda.svg new file mode 100644 index 0000000000..cd4817adf2 --- /dev/null +++ b/public/img/svg/gitea-conda.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 78eb5e860b..7a07fea815 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/api/packages/composer" "code.gitea.io/gitea/routers/api/packages/conan" + "code.gitea.io/gitea/routers/api/packages/conda" "code.gitea.io/gitea/routers/api/packages/container" "code.gitea.io/gitea/routers/api/packages/generic" "code.gitea.io/gitea/routers/api/packages/helm" @@ -167,6 +168,43 @@ func CommonRoutes(ctx gocontext.Context) *web.Route { }) }) }, reqPackageAccess(perm.AccessModeRead)) + r.Group("/conda", func() { + var ( + downloadPattern = regexp.MustCompile(`\A(.+/)?(.+)/((?:[^/]+(?:\.tar\.bz2|\.conda))|(?:current_)?repodata\.json(?:\.bz2)?)\z`) + uploadPattern = regexp.MustCompile(`\A(.+/)?([^/]+(?:\.tar\.bz2|\.conda))\z`) + ) + + r.Get("/*", func(ctx *context.Context) { + m := downloadPattern.FindStringSubmatch(ctx.Params("*")) + if len(m) == 0 { + ctx.Status(http.StatusNotFound) + return + } + + ctx.SetParams("channel", strings.TrimSuffix(m[1], "/")) + ctx.SetParams("architecture", m[2]) + ctx.SetParams("filename", m[3]) + + switch m[3] { + case "repodata.json", "repodata.json.bz2", "current_repodata.json", "current_repodata.json.bz2": + conda.EnumeratePackages(ctx) + default: + conda.DownloadPackageFile(ctx) + } + }) + r.Put("/*", reqPackageAccess(perm.AccessModeWrite), func(ctx *context.Context) { + m := uploadPattern.FindStringSubmatch(ctx.Params("*")) + if len(m) == 0 { + ctx.Status(http.StatusNotFound) + return + } + + ctx.SetParams("channel", strings.TrimSuffix(m[1], "/")) + ctx.SetParams("filename", m[2]) + + conda.UploadPackageFile(ctx) + }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/generic", func() { r.Group("/{packagename}/{packageversion}", func() { r.Delete("", reqPackageAccess(perm.AccessModeWrite), generic.DeletePackage) diff --git a/routers/api/packages/conda/conda.go b/routers/api/packages/conda/conda.go new file mode 100644 index 0000000000..2ff619fed4 --- /dev/null +++ b/routers/api/packages/conda/conda.go @@ -0,0 +1,306 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conda + +import ( + "errors" + "fmt" + "io" + "net/http" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + conda_model "code.gitea.io/gitea/models/packages/conda" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + conda_module "code.gitea.io/gitea/modules/packages/conda" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" + + "github.com/dsnet/compress/bzip2" +) + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.JSON(status, struct { + Reason string `json:"reason"` + Message string `json:"message"` + }{ + Reason: http.StatusText(status), + Message: message, + }) + }) +} + +func EnumeratePackages(ctx *context.Context) { + type Info struct { + Subdir string `json:"subdir"` + } + + type PackageInfo struct { + Name string `json:"name"` + Version string `json:"version"` + NoArch string `json:"noarch"` + Subdir string `json:"subdir"` + Timestamp int64 `json:"timestamp"` + Build string `json:"build"` + BuildNumber int64 `json:"build_number"` + Dependencies []string `json:"depends"` + License string `json:"license"` + LicenseFamily string `json:"license_family"` + HashMD5 string `json:"md5"` + HashSHA256 string `json:"sha256"` + Size int64 `json:"size"` + } + + type RepoData struct { + Info Info `json:"info"` + Packages map[string]*PackageInfo `json:"packages"` + PackagesConda map[string]*PackageInfo `json:"packages.conda"` + Removed map[string]*PackageInfo `json:"removed"` + } + + repoData := &RepoData{ + Info: Info{ + Subdir: ctx.Params("architecture"), + }, + Packages: make(map[string]*PackageInfo), + PackagesConda: make(map[string]*PackageInfo), + Removed: make(map[string]*PackageInfo), + } + + pfs, err := conda_model.SearchFiles(ctx, &conda_model.FileSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Channel: ctx.Params("channel"), + Subdir: repoData.Info.Subdir, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if len(pfs) == 0 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + pds := make(map[int64]*packages_model.PackageDescriptor) + + for _, pf := range pfs { + pd, exists := pds[pf.VersionID] + if !exists { + pv, err := packages_model.GetVersionByID(ctx, pf.VersionID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pd, err = packages_model.GetPackageDescriptor(ctx, pv) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pds[pf.VersionID] = pd + } + + var pfd *packages_model.PackageFileDescriptor + for _, d := range pd.Files { + if d.File.ID == pf.ID { + pfd = d + break + } + } + + var fileMetadata *conda_module.FileMetadata + if err := json.Unmarshal([]byte(pfd.Properties.GetByName(conda_module.PropertyMetadata)), &fileMetadata); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + versionMetadata := pd.Metadata.(*conda_module.VersionMetadata) + + pi := &PackageInfo{ + Name: pd.PackageProperties.GetByName(conda_module.PropertyName), + Version: pd.Version.Version, + NoArch: fileMetadata.NoArch, + Subdir: repoData.Info.Subdir, + Timestamp: fileMetadata.Timestamp, + Build: fileMetadata.Build, + BuildNumber: fileMetadata.BuildNumber, + Dependencies: fileMetadata.Dependencies, + License: versionMetadata.License, + LicenseFamily: versionMetadata.LicenseFamily, + HashMD5: pfd.Blob.HashMD5, + HashSHA256: pfd.Blob.HashSHA256, + Size: pfd.Blob.Size, + } + + if fileMetadata.IsCondaPackage { + repoData.PackagesConda[pfd.File.Name] = pi + } else { + repoData.Packages[pfd.File.Name] = pi + } + } + + resp := ctx.Resp + + var w io.Writer = resp + + if strings.HasSuffix(ctx.Params("filename"), ".json") { + resp.Header().Set("Content-Type", "application/json") + } else { + resp.Header().Set("Content-Type", "application/x-bzip2") + + zw, err := bzip2.NewWriter(w, nil) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer zw.Close() + + w = zw + } + + resp.WriteHeader(http.StatusOK) + + if err := json.NewEncoder(w).Encode(repoData); err != nil { + log.Error("JSON encode: %v", err) + } +} + +func UploadPackageFile(ctx *context.Context) { + upload, close, err := ctx.UploadStream() + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if close { + defer upload.Close() + } + + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + var pck *conda_module.Package + if strings.HasSuffix(strings.ToLower(ctx.Params("filename")), ".tar.bz2") { + pck, err = conda_module.ParsePackageBZ2(buf) + } else { + pck, err = conda_module.ParsePackageConda(buf, buf.Size()) + } + if err != nil { + if errors.Is(err, util.ErrInvalidArgument) { + apiError(ctx, http.StatusBadRequest, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + fullName := pck.Name + + channel := ctx.Params("channel") + if channel != "" { + fullName = channel + "/" + pck.Name + } + + extension := ".tar.bz2" + if pck.FileMetadata.IsCondaPackage { + extension = ".conda" + } + + fileMetadataRaw, err := json.Marshal(pck.FileMetadata) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + _, _, err = packages_service.CreatePackageOrAddFileToExisting( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeConda, + Name: fullName, + Version: pck.Version, + }, + SemverCompatible: false, + Creator: ctx.Doer, + Metadata: pck.VersionMetadata, + PackageProperties: map[string]string{ + conda_module.PropertyName: pck.Name, + conda_module.PropertyChannel: channel, + }, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: fmt.Sprintf("%s-%s-%s%s", pck.Name, pck.Version, pck.FileMetadata.Build, extension), + CompositeKey: pck.Subdir, + }, + Creator: ctx.Doer, + Data: buf, + IsLead: true, + Properties: map[string]string{ + conda_module.PropertySubdir: pck.Subdir, + conda_module.PropertyMetadata: string(fileMetadataRaw), + }, + }, + ) + if err != nil { + switch err { + case packages_model.ErrDuplicatePackageFile: + apiError(ctx, http.StatusConflict, err) + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + ctx.Status(http.StatusCreated) +} + +func DownloadPackageFile(ctx *context.Context) { + pfs, err := conda_model.SearchFiles(ctx, &conda_model.FileSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Channel: ctx.Params("channel"), + Subdir: ctx.Params("architecture"), + Filename: ctx.Params("filename"), + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if len(pfs) != 1 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + pf := pfs[0] + + s, _, err := packages_service.GetPackageFileStream(ctx, pf) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) +} diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index 6f9083ba32..5ffefc4862 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [composer, conan, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant] + // enum: [composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant] // - name: q // in: query // description: name filter diff --git a/services/forms/package_form.go b/services/forms/package_form.go index 734bb05dc6..e78e64ef7e 100644 --- a/services/forms/package_form.go +++ b/services/forms/package_form.go @@ -15,7 +15,7 @@ import ( type PackageCleanupRuleForm struct { ID int64 Enabled bool - Type string `binding:"Required;In(composer,conan,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"` + Type string `binding:"Required;In(composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"` KeepCount int `binding:"In(0,1,5,10,25,50,100)"` KeepPattern string `binding:"RegexPattern"` RemoveDays int `binding:"In(0,7,14,30,60,90,180)"` diff --git a/services/packages/packages.go b/services/packages/packages.go index 754dfa7110..9e52cb1450 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -339,6 +339,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p typeSpecificSize = setting.Packages.LimitSizeComposer case packages_model.TypeConan: typeSpecificSize = setting.Packages.LimitSizeConan + case packages_model.TypeConda: + typeSpecificSize = setting.Packages.LimitSizeConda case packages_model.TypeContainer: typeSpecificSize = setting.Packages.LimitSizeContainer case packages_model.TypeGeneric: diff --git a/templates/package/content/conda.tmpl b/templates/package/content/conda.tmpl new file mode 100644 index 0000000000..ecc26bce98 --- /dev/null +++ b/templates/package/content/conda.tmpl @@ -0,0 +1,30 @@ +{{if eq .PackageDescriptor.Package.Type "conda"}} +

{{.locale.Tr "packages.installation"}}

+
+
+
+ + 
channel_alias: {{AppUrl}}api/packages/{{.PackageDescriptor.Owner.Name}}/conda
+channels:
+  - {{AppUrl}}api/packages/{{.PackageDescriptor.Owner.Name}}/conda
+default_channels:
+  - {{AppUrl}}api/packages/{{.PackageDescriptor.Owner.Name}}/conda
+
+
+ + {{$channel := .PackageDescriptor.PackageProperties.GetByName "conda.channel"}} + 
conda install{{if $channel}} -c {{$channel}}{{end}} {{.PackageDescriptor.PackageProperties.GetByName "conda.name"}}={{.PackageDescriptor.Version.Version}}
+
+
+ +
+
+
+ + {{if or .PackageDescriptor.Metadata.Description .PackageDescriptor.Metadata.Summary}} +

{{.locale.Tr "packages.about"}}

+
+ {{if .PackageDescriptor.Metadata.Description}}{{.PackageDescriptor.Metadata.Description}}{{else}}{{.PackageDescriptor.Metadata.Summary}}{{end}} +
+ {{end}} +{{end}} diff --git a/templates/package/metadata/conda.tmpl b/templates/package/metadata/conda.tmpl new file mode 100644 index 0000000000..2201c80356 --- /dev/null +++ b/templates/package/metadata/conda.tmpl @@ -0,0 +1,6 @@ +{{if eq .PackageDescriptor.Package.Type "conda"}} + {{if .PackageDescriptor.Metadata.License}}
{{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.License}}
{{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.project_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.RepositoryURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.conda.details.repository_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.DocumentationURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.conda.details.documentation_site"}}
{{end}} +{{end}} diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index a5b2a2ef68..2222480e7c 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -21,6 +21,7 @@
{{template "package/content/composer" .}} {{template "package/content/conan" .}} + {{template "package/content/conda" .}} {{template "package/content/container" .}} {{template "package/content/generic" .}} {{template "package/content/helm" .}} @@ -44,6 +45,7 @@
{{svg "octicon-download" 16 "mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}
{{template "package/metadata/composer" .}} {{template "package/metadata/conan" .}} + {{template "package/metadata/conda" .}} {{template "package/metadata/container" .}} {{template "package/metadata/generic" .}} {{template "package/metadata/helm" .}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 726b771cfc..5a3f2c2bb9 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -2102,6 +2102,7 @@ "enum": [ "composer", "conan", + "conda", "container", "generic", "helm", diff --git a/tests/integration/api_packages_conda_test.go b/tests/integration/api_packages_conda_test.go new file mode 100644 index 0000000000..daa7dca55f --- /dev/null +++ b/tests/integration/api_packages_conda_test.go @@ -0,0 +1,274 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "archive/tar" + "archive/zip" + "bytes" + "fmt" + "io" + "net/http" + "testing" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + conda_module "code.gitea.io/gitea/modules/packages/conda" + "code.gitea.io/gitea/tests" + + "github.com/dsnet/compress/bzip2" + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/assert" +) + +func TestPackageConda(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + packageName := "test_package" + packageVersion := "1.0.1" + + channel := "test-channel" + root := fmt.Sprintf("/api/packages/%s/conda", user.Name) + + t.Run("Upload", func(t *testing.T) { + tarContent := func() []byte { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + + content := []byte(`{"name":"` + packageName + `","version":"` + packageVersion + `","subdir":"noarch","build":"xxx"}`) + + hdr := &tar.Header{ + Name: "info/index.json", + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write(content) + tw.Close() + return buf.Bytes() + }() + + t.Run(".tar.bz2", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + var buf bytes.Buffer + bw, _ := bzip2.NewWriter(&buf, nil) + io.Copy(bw, bytes.NewReader(tarContent)) + bw.Close() + + filename := fmt.Sprintf("%s-%s.tar.bz2", packageName, packageVersion) + + req := NewRequestWithBody(t, "PUT", root+"/"+filename, bytes.NewReader(buf.Bytes())) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithBody(t, "PUT", root+"/"+filename, bytes.NewReader(buf.Bytes())) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusCreated) + + req = NewRequestWithBody(t, "PUT", root+"/"+filename, bytes.NewReader(buf.Bytes())) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusConflict) + + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeConda) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) + assert.NoError(t, err) + assert.Nil(t, pd.SemVer) + assert.IsType(t, &conda_module.VersionMetadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + assert.Empty(t, pd.PackageProperties.GetByName(conda_module.PropertyChannel)) + }) + + t.Run(".conda", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + var infoBuf bytes.Buffer + zsw, _ := zstd.NewWriter(&infoBuf) + io.Copy(zsw, bytes.NewReader(tarContent)) + zsw.Close() + + var buf bytes.Buffer + zpw := zip.NewWriter(&buf) + w, _ := zpw.Create("info-x.tar.zst") + w.Write(infoBuf.Bytes()) + zpw.Close() + + fullName := channel + "/" + packageName + filename := fmt.Sprintf("%s-%s.conda", packageName, packageVersion) + + req := NewRequestWithBody(t, "PUT", root+"/"+channel+"/"+filename, bytes.NewReader(buf.Bytes())) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithBody(t, "PUT", root+"/"+channel+"/"+filename, bytes.NewReader(buf.Bytes())) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusCreated) + + req = NewRequestWithBody(t, "PUT", root+"/"+channel+"/"+filename, bytes.NewReader(buf.Bytes())) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusConflict) + + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeConda) + assert.NoError(t, err) + assert.Len(t, pvs, 2) + + pds, err := packages.GetPackageDescriptors(db.DefaultContext, pvs) + assert.NoError(t, err) + + assert.Condition(t, func() bool { + for _, pd := range pds { + if pd.Package.Name == fullName { + return true + } + } + return false + }) + + for _, pd := range pds { + if pd.Package.Name == fullName { + assert.Nil(t, pd.SemVer) + assert.IsType(t, &conda_module.VersionMetadata{}, pd.Metadata) + assert.Equal(t, fullName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + assert.Equal(t, channel, pd.PackageProperties.GetByName(conda_module.PropertyChannel)) + } + } + }) + }) + + t.Run("Download", func(t *testing.T) { + t.Run(".tar.bz2", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/noarch/%s-%s-xxx.tar.bz2", root, packageName, packageVersion)) + MakeRequest(t, req, http.StatusOK) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/noarch/%s-%s-xxx.tar.bz2", root, channel, packageName, packageVersion)) + MakeRequest(t, req, http.StatusNotFound) + }) + + t.Run(".conda", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/noarch/%s-%s-xxx.conda", root, packageName, packageVersion)) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/noarch/%s-%s-xxx.conda", root, channel, packageName, packageVersion)) + MakeRequest(t, req, http.StatusOK) + }) + }) + + t.Run("EnumeratePackages", func(t *testing.T) { + type Info struct { + Subdir string `json:"subdir"` + } + + type PackageInfo struct { + Name string `json:"name"` + Version string `json:"version"` + NoArch string `json:"noarch"` + Subdir string `json:"subdir"` + Timestamp int64 `json:"timestamp"` + Build string `json:"build"` + BuildNumber int64 `json:"build_number"` + Dependencies []string `json:"depends"` + License string `json:"license"` + LicenseFamily string `json:"license_family"` + HashMD5 string `json:"md5"` + HashSHA256 string `json:"sha256"` + Size int64 `json:"size"` + } + + type RepoData struct { + Info Info `json:"info"` + Packages map[string]*PackageInfo `json:"packages"` + PackagesConda map[string]*PackageInfo `json:"packages.conda"` + Removed map[string]*PackageInfo `json:"removed"` + } + + req := NewRequest(t, "GET", fmt.Sprintf("%s/noarch/repodata.json", root)) + resp := MakeRequest(t, req, http.StatusOK) + assert.Equal(t, "application/json", resp.Header().Get("Content-Type")) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/noarch/repodata.json.bz2", root)) + resp = MakeRequest(t, req, http.StatusOK) + assert.Equal(t, "application/x-bzip2", resp.Header().Get("Content-Type")) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/noarch/current_repodata.json", root)) + resp = MakeRequest(t, req, http.StatusOK) + assert.Equal(t, "application/json", resp.Header().Get("Content-Type")) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/noarch/current_repodata.json.bz2", root)) + resp = MakeRequest(t, req, http.StatusOK) + assert.Equal(t, "application/x-bzip2", resp.Header().Get("Content-Type")) + + t.Run(".tar.bz2", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages.TypeConda, packageName, packageVersion) + assert.NoError(t, err) + + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pv) + assert.NoError(t, err) + + req := NewRequest(t, "GET", fmt.Sprintf("%s/noarch/repodata.json", root)) + resp := MakeRequest(t, req, http.StatusOK) + + var result RepoData + DecodeJSON(t, resp, &result) + + assert.Equal(t, "noarch", result.Info.Subdir) + assert.Empty(t, result.PackagesConda) + assert.Empty(t, result.Removed) + + filename := fmt.Sprintf("%s-%s-xxx.tar.bz2", packageName, packageVersion) + assert.Contains(t, result.Packages, filename) + packageInfo := result.Packages[filename] + assert.Equal(t, packageName, packageInfo.Name) + assert.Equal(t, packageVersion, packageInfo.Version) + assert.Equal(t, "noarch", packageInfo.Subdir) + assert.Equal(t, "xxx", packageInfo.Build) + assert.Equal(t, pd.Files[0].Blob.HashMD5, packageInfo.HashMD5) + assert.Equal(t, pd.Files[0].Blob.HashSHA256, packageInfo.HashSHA256) + assert.Equal(t, pd.Files[0].Blob.Size, packageInfo.Size) + }) + + t.Run(".conda", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages.TypeConda, channel+"/"+packageName, packageVersion) + assert.NoError(t, err) + + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pv) + assert.NoError(t, err) + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/noarch/repodata.json", root, channel)) + resp := MakeRequest(t, req, http.StatusOK) + + var result RepoData + DecodeJSON(t, resp, &result) + + assert.Equal(t, "noarch", result.Info.Subdir) + assert.Empty(t, result.Packages) + assert.Empty(t, result.Removed) + + filename := fmt.Sprintf("%s-%s-xxx.conda", packageName, packageVersion) + assert.Contains(t, result.PackagesConda, filename) + packageInfo := result.PackagesConda[filename] + assert.Equal(t, packageName, packageInfo.Name) + assert.Equal(t, packageVersion, packageInfo.Version) + assert.Equal(t, "noarch", packageInfo.Subdir) + assert.Equal(t, "xxx", packageInfo.Build) + assert.Equal(t, pd.Files[0].Blob.HashMD5, packageInfo.HashMD5) + assert.Equal(t, pd.Files[0].Blob.HashSHA256, packageInfo.HashSHA256) + assert.Equal(t, pd.Files[0].Blob.Size, packageInfo.Size) + }) + }) +} diff --git a/web_src/svg/gitea-conda.svg b/web_src/svg/gitea-conda.svg new file mode 100644 index 0000000000..c5797c8c10 --- /dev/null +++ b/web_src/svg/gitea-conda.svg @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + +