From fbd4eaceed801e7400ed04a9dadedaf3a25dccb9 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 2 Apr 2023 11:53:37 +0200 Subject: [PATCH] Display image size for multiarch container images (#23821) Fixes #23771 Changes the display of different architectures for multiarch images to show the image size: ![grafik](https://user-images.githubusercontent.com/1666336/228781477-cc76c4d1-4728-434f-8a27-fc008790d924.png) --- models/migrations/migrations.go | 2 + models/migrations/v1_20/v250.go | 135 ++++++++++++++++++ modules/packages/container/metadata.go | 8 +- modules/packages/container/metadata_test.go | 2 +- routers/api/packages/container/manifest.go | 21 ++- templates/package/content/container.tmpl | 28 ++-- templates/package/view.tmpl | 2 + .../api_packages_container_test.go | 22 ++- 8 files changed, 198 insertions(+), 22 deletions(-) create mode 100644 models/migrations/v1_20/v250.go diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index bec406f7bf..ea3619db97 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -477,6 +477,8 @@ var migrations = []Migration{ NewMigration("Add version column to action_runner table", v1_20.AddVersionToActionRunner), // v249 -> v250 NewMigration("Improve Action table indices v3", v1_20.ImproveActionTableIndices), + // v250 -> v251 + NewMigration("Change Container Metadata", v1_20.ChangeContainerMetadataMultiArch), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_20/v250.go b/models/migrations/v1_20/v250.go new file mode 100644 index 0000000000..e05646e5c6 --- /dev/null +++ b/models/migrations/v1_20/v250.go @@ -0,0 +1,135 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_20 //nolint + +import ( + "strings" + + "code.gitea.io/gitea/modules/json" + + "xorm.io/xorm" +) + +func ChangeContainerMetadataMultiArch(x *xorm.Engine) error { + sess := x.NewSession() + defer sess.Close() + + if err := sess.Begin(); err != nil { + return err + } + + type PackageVersion struct { + ID int64 `xorm:"pk"` + MetadataJSON string `xorm:"metadata_json"` + } + + type PackageBlob struct{} + + // Get all relevant packages (manifest list images have a container.manifest.reference property) + + var pvs []*PackageVersion + err := sess. + Table("package_version"). + Select("id, metadata_json"). + Where("id IN (SELECT DISTINCT ref_id FROM package_property WHERE ref_type = 0 AND name = 'container.manifest.reference')"). + Find(&pvs) + if err != nil { + return err + } + + type MetadataOld struct { + Type string `json:"type"` + IsTagged bool `json:"is_tagged"` + Platform string `json:"platform,omitempty"` + Description string `json:"description,omitempty"` + Authors []string `json:"authors,omitempty"` + Licenses string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + DocumentationURL string `json:"documentation_url,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + ImageLayers []string `json:"layer_creation,omitempty"` + MultiArch map[string]string `json:"multiarch,omitempty"` + } + + type Manifest struct { + Platform string `json:"platform"` + Digest string `json:"digest"` + Size int64 `json:"size"` + } + + type MetadataNew struct { + Type string `json:"type"` + IsTagged bool `json:"is_tagged"` + Platform string `json:"platform,omitempty"` + Description string `json:"description,omitempty"` + Authors []string `json:"authors,omitempty"` + Licenses string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + DocumentationURL string `json:"documentation_url,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + ImageLayers []string `json:"layer_creation,omitempty"` + Manifests []*Manifest `json:"manifests,omitempty"` + } + + for _, pv := range pvs { + var old *MetadataOld + if err := json.Unmarshal([]byte(pv.MetadataJSON), &old); err != nil { + return err + } + + // Calculate the size of every contained manifest + + manifests := make([]*Manifest, 0, len(old.MultiArch)) + for platform, digest := range old.MultiArch { + size, err := sess. + Table("package_blob"). + Join("INNER", "package_file", "package_blob.id = package_file.blob_id"). + Join("INNER", "package_version pv", "pv.id = package_file.version_id"). + Join("INNER", "package_version pv2", "pv2.package_id = pv.package_id"). + Where("pv.lower_version = ? AND pv2.id = ?", strings.ToLower(digest), pv.ID). + SumInt(new(PackageBlob), "size") + if err != nil { + return err + } + + manifests = append(manifests, &Manifest{ + Platform: platform, + Digest: digest, + Size: size, + }) + } + + // Convert to new metadata format + + new := &MetadataNew{ + Type: old.Type, + IsTagged: old.IsTagged, + Platform: old.Platform, + Description: old.Description, + Authors: old.Authors, + Licenses: old.Licenses, + ProjectURL: old.ProjectURL, + RepositoryURL: old.RepositoryURL, + DocumentationURL: old.DocumentationURL, + Labels: old.Labels, + ImageLayers: old.ImageLayers, + Manifests: manifests, + } + + metadataJSON, err := json.Marshal(new) + if err != nil { + return err + } + + pv.MetadataJSON = string(metadataJSON) + + if _, err := sess.ID(pv.ID).Update(pv); err != nil { + return err + } + } + + return sess.Commit() +} diff --git a/modules/packages/container/metadata.go b/modules/packages/container/metadata.go index 6f62ab6a54..2a41fb9105 100644 --- a/modules/packages/container/metadata.go +++ b/modules/packages/container/metadata.go @@ -62,7 +62,13 @@ type Metadata struct { DocumentationURL string `json:"documentation_url,omitempty"` Labels map[string]string `json:"labels,omitempty"` ImageLayers []string `json:"layer_creation,omitempty"` - MultiArch map[string]string `json:"multiarch,omitempty"` + Manifests []*Manifest `json:"manifests,omitempty"` +} + +type Manifest struct { + Platform string `json:"platform"` + Digest string `json:"digest"` + Size int64 `json:"size"` } // ParseImageConfig parses the metadata of an image config diff --git a/modules/packages/container/metadata_test.go b/modules/packages/container/metadata_test.go index 5d8d3abfae..48809f4c99 100644 --- a/modules/packages/container/metadata_test.go +++ b/modules/packages/container/metadata_test.go @@ -46,7 +46,7 @@ func TestParseImageConfig(t *testing.T) { }, metadata.Labels, ) - assert.Empty(t, metadata.MultiArch) + assert.Empty(t, metadata.Manifests) configHelm := `{"description":"` + description + `", "home": "` + projectURL + `", "sources": ["` + repositoryURL + `"], "maintainers":[{"name":"` + author + `"}]}` diff --git a/routers/api/packages/container/manifest.go b/routers/api/packages/container/manifest.go index e36c6a851b..1dbd058d6b 100644 --- a/routers/api/packages/container/manifest.go +++ b/routers/api/packages/container/manifest.go @@ -217,7 +217,7 @@ func processImageManifestIndex(mci *manifestCreationInfo, buf *packages_module.H metadata := &container_module.Metadata{ Type: container_module.TypeOCI, - MultiArch: make(map[string]string), + Manifests: make([]*container_module.Manifest, 0, len(index.Manifests)), } for _, manifest := range index.Manifests { @@ -233,7 +233,7 @@ func processImageManifestIndex(mci *manifestCreationInfo, buf *packages_module.H } } - _, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + pfd, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ OwnerID: mci.Owner.ID, Image: mci.Image, Digest: string(manifest.Digest), @@ -246,7 +246,18 @@ func processImageManifestIndex(mci *manifestCreationInfo, buf *packages_module.H return err } - metadata.MultiArch[platform] = string(manifest.Digest) + size, err := packages_model.CalculateFileSize(ctx, &packages_model.PackageFileSearchOptions{ + VersionID: pfd.File.VersionID, + }) + if err != nil { + return err + } + + metadata.Manifests = append(metadata.Manifests, &container_module.Manifest{ + Platform: platform, + Digest: string(manifest.Digest), + Size: size, + }) } pv, err := createPackageAndVersion(ctx, mci, metadata) @@ -369,8 +380,8 @@ func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, met return nil, err } } - for _, digest := range metadata.MultiArch { - if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference, digest); err != nil { + for _, manifest := range metadata.Manifests { + if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference, manifest.Digest); err != nil { log.Error("Error setting package version property: %v", err) return nil, err } diff --git a/templates/package/content/container.tmpl b/templates/package/content/container.tmpl index 0bf749cd70..78c9434386 100644 --- a/templates/package/content/container.tmpl +++ b/templates/package/content/container.tmpl @@ -23,19 +23,27 @@ - {{if .PackageDescriptor.Metadata.MultiArch}} + {{if .PackageDescriptor.Metadata.Manifests}}

{{.locale.Tr "packages.container.multi_arch"}}

-
- {{range $arch, $digest := .PackageDescriptor.Metadata.MultiArch}} -
- - {{if eq $.PackageDescriptor.Metadata.Type "oci"}} - 
docker pull {{$.RegistryHost}}/{{$.PackageDescriptor.Owner.LowerName}}/{{$.PackageDescriptor.Package.LowerName}}@{{$digest}}
+ + + + + + + + + + {{range .PackageDescriptor.Metadata.Manifests}} + + + + + {{end}} - - {{end}} - + +
{{.locale.Tr "packages.container.digest"}}{{.locale.Tr "packages.container.multi_arch"}}{{.locale.Tr "admin.packages.size"}}
{{.Digest}}{{.Platform}}{{FileSize .Size}}
{{end}} {{if .PackageDescriptor.Metadata.Description}} diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index b2a2fb1e5d..beadcf5c1e 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -62,7 +62,9 @@ {{template "package/metadata/rubygems" .}} {{template "package/metadata/swift" .}} {{template "package/metadata/vagrant" .}} + {{if not (and (eq .PackageDescriptor.Package.Type "container") .PackageDescriptor.Metadata.Manifests)}}
{{svg "octicon-database" 16 "gt-mr-3"}} {{FileSize .PackageDescriptor.CalculateBlobSize}}
+ {{end}}
{{if not (eq .PackageDescriptor.Package.Type "container")}}
diff --git a/tests/integration/api_packages_container_test.go b/tests/integration/api_packages_container_test.go index d925fd1647..fe9208bb05 100644 --- a/tests/integration/api_packages_container_test.go +++ b/tests/integration/api_packages_container_test.go @@ -321,7 +321,7 @@ func TestPackageContainer(t *testing.T) { metadata := pd.Metadata.(*container_module.Metadata) assert.Equal(t, container_module.TypeOCI, metadata.Type) assert.Len(t, metadata.ImageLayers, 2) - assert.Empty(t, metadata.MultiArch) + assert.Empty(t, metadata.Manifests) assert.Len(t, pd.Files, 3) for _, pfd := range pd.Files { @@ -462,10 +462,22 @@ func TestPackageContainer(t *testing.T) { assert.IsType(t, &container_module.Metadata{}, pd.Metadata) metadata := pd.Metadata.(*container_module.Metadata) assert.Equal(t, container_module.TypeOCI, metadata.Type) - assert.Contains(t, metadata.MultiArch, "linux/arm/v7") - assert.Equal(t, manifestDigest, metadata.MultiArch["linux/arm/v7"]) - assert.Contains(t, metadata.MultiArch, "linux/arm64/v8") - assert.Equal(t, untaggedManifestDigest, metadata.MultiArch["linux/arm64/v8"]) + assert.Len(t, metadata.Manifests, 2) + assert.Condition(t, func() bool { + for _, m := range metadata.Manifests { + switch m.Platform { + case "linux/arm/v7": + assert.Equal(t, manifestDigest, m.Digest) + assert.EqualValues(t, 1524, m.Size) + case "linux/arm64/v8": + assert.Equal(t, untaggedManifestDigest, m.Digest) + assert.EqualValues(t, 1514, m.Size) + default: + return false + } + } + return true + }) assert.Len(t, pd.Files, 1) assert.True(t, pd.Files[0].File.IsLead)