From 703be6bf307ed19ce8dc8cd311d24aeb6e5b9861 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 25 Nov 2024 11:35:49 -0800 Subject: [PATCH] Add github compatible tarball download API endpoints (#32572) Fix #29654 Fix #32481 --- routers/api/v1/api.go | 2 + routers/api/v1/repo/download.go | 53 +++++++++++++++++++ routers/api/v1/repo/file.go | 15 ++++-- routers/web/repo/repo.go | 14 ++++- services/repository/archiver/archiver.go | 34 +++++++----- services/repository/archiver/archiver_test.go | 25 ++++----- tests/integration/api_repo_archive_test.go | 40 ++++++++++++++ 7 files changed, 152 insertions(+), 31 deletions(-) create mode 100644 routers/api/v1/repo/download.go diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 23f466873b..0079e8dc87 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -1377,6 +1377,8 @@ func Routes() *web.Router { m.Post("", bind(api.UpdateRepoAvatarOption{}), repo.UpdateAvatar) m.Delete("", repo.DeleteAvatar) }, reqAdmin(), reqToken()) + + m.Get("/{ball_type:tarball|zipball|bundle}/*", reqRepoReader(unit.TypeCode), repo.DownloadArchive) }, repoAssignment(), checkTokenPublicOnly()) }, tokenRequiresScopes(auth_model.AccessTokenScopeCategoryRepository)) diff --git a/routers/api/v1/repo/download.go b/routers/api/v1/repo/download.go new file mode 100644 index 0000000000..3620c1465f --- /dev/null +++ b/routers/api/v1/repo/download.go @@ -0,0 +1,53 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repo + +import ( + "fmt" + "net/http" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/services/context" + archiver_service "code.gitea.io/gitea/services/repository/archiver" +) + +func DownloadArchive(ctx *context.APIContext) { + var tp git.ArchiveType + switch ballType := ctx.PathParam("ball_type"); ballType { + case "tarball": + tp = git.TARGZ + case "zipball": + tp = git.ZIP + case "bundle": + tp = git.BUNDLE + default: + ctx.Error(http.StatusBadRequest, "", fmt.Sprintf("Unknown archive type: %s", ballType)) + return + } + + if ctx.Repo.GitRepo == nil { + gitRepo, err := gitrepo.OpenRepository(ctx, ctx.Repo.Repository) + if err != nil { + ctx.Error(http.StatusInternalServerError, "OpenRepository", err) + return + } + ctx.Repo.GitRepo = gitRepo + defer gitRepo.Close() + } + + r, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, ctx.PathParam("*"), tp) + if err != nil { + ctx.ServerError("NewRequest", err) + return + } + + archive, err := r.Await(ctx) + if err != nil { + ctx.ServerError("archive.Await", err) + return + } + + download(ctx, r.GetArchiveName(), archive) +} diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index 05650cc9be..959a4b952a 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -301,7 +301,13 @@ func GetArchive(ctx *context.APIContext) { func archiveDownload(ctx *context.APIContext) { uri := ctx.PathParam("*") - aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri) + ext, tp, err := archiver_service.ParseFileName(uri) + if err != nil { + ctx.Error(http.StatusBadRequest, "ParseFileName", err) + return + } + + aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, strings.TrimSuffix(uri, ext), tp) if err != nil { if errors.Is(err, archiver_service.ErrUnknownArchiveFormat{}) { ctx.Error(http.StatusBadRequest, "unknown archive format", err) @@ -327,9 +333,12 @@ func download(ctx *context.APIContext, archiveName string, archiver *repo_model. // Add nix format link header so tarballs lock correctly: // https://github.com/nixos/nix/blob/56763ff918eb308db23080e560ed2ea3e00c80a7/doc/manual/src/protocols/tarball-fetcher.md - ctx.Resp.Header().Add("Link", fmt.Sprintf(`<%s/archive/%s.tar.gz?rev=%s>; rel="immutable"`, + ctx.Resp.Header().Add("Link", fmt.Sprintf(`<%s/archive/%s.%s?rev=%s>; rel="immutable"`, ctx.Repo.Repository.APIURL(), - archiver.CommitID, archiver.CommitID)) + archiver.CommitID, + archiver.Type.String(), + archiver.CommitID, + )) rPath := archiver.RelativePath() if setting.RepoArchive.Storage.ServeDirect() { diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index b62fd21585..f5e59b0357 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -464,7 +464,12 @@ func RedirectDownload(ctx *context.Context) { // Download an archive of a repository func Download(ctx *context.Context) { uri := ctx.PathParam("*") - aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri) + ext, tp, err := archiver_service.ParseFileName(uri) + if err != nil { + ctx.ServerError("ParseFileName", err) + return + } + aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, strings.TrimSuffix(uri, ext), tp) if err != nil { if errors.Is(err, archiver_service.ErrUnknownArchiveFormat{}) { ctx.Error(http.StatusBadRequest, err.Error()) @@ -523,7 +528,12 @@ func download(ctx *context.Context, archiveName string, archiver *repo_model.Rep // kind of drop it on the floor if this is the case. func InitiateDownload(ctx *context.Context) { uri := ctx.PathParam("*") - aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri) + ext, tp, err := archiver_service.ParseFileName(uri) + if err != nil { + ctx.ServerError("ParseFileName", err) + return + } + aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, strings.TrimSuffix(uri, ext), tp) if err != nil { ctx.ServerError("archiver_service.NewRequest", err) return diff --git a/services/repository/archiver/archiver.go b/services/repository/archiver/archiver.go index c33369d047..e1addbed33 100644 --- a/services/repository/archiver/archiver.go +++ b/services/repository/archiver/archiver.go @@ -67,30 +67,36 @@ func (e RepoRefNotFoundError) Is(err error) bool { return ok } -// NewRequest creates an archival request, based on the URI. The -// resulting ArchiveRequest is suitable for being passed to Await() -// if it's determined that the request still needs to be satisfied. -func NewRequest(repoID int64, repo *git.Repository, uri string) (*ArchiveRequest, error) { - r := &ArchiveRequest{ - RepoID: repoID, - } - - var ext string +func ParseFileName(uri string) (ext string, tp git.ArchiveType, err error) { switch { case strings.HasSuffix(uri, ".zip"): ext = ".zip" - r.Type = git.ZIP + tp = git.ZIP case strings.HasSuffix(uri, ".tar.gz"): ext = ".tar.gz" - r.Type = git.TARGZ + tp = git.TARGZ case strings.HasSuffix(uri, ".bundle"): ext = ".bundle" - r.Type = git.BUNDLE + tp = git.BUNDLE default: - return nil, ErrUnknownArchiveFormat{RequestFormat: uri} + return "", 0, ErrUnknownArchiveFormat{RequestFormat: uri} + } + return ext, tp, nil +} + +// NewRequest creates an archival request, based on the URI. The +// resulting ArchiveRequest is suitable for being passed to Await() +// if it's determined that the request still needs to be satisfied. +func NewRequest(repoID int64, repo *git.Repository, refName string, fileType git.ArchiveType) (*ArchiveRequest, error) { + if fileType < git.ZIP || fileType > git.BUNDLE { + return nil, ErrUnknownArchiveFormat{RequestFormat: fileType.String()} } - r.refName = strings.TrimSuffix(uri, ext) + r := &ArchiveRequest{ + RepoID: repoID, + refName: refName, + Type: fileType, + } // Get corresponding commit. commitID, err := repo.ConvertToGitID(r.refName) diff --git a/services/repository/archiver/archiver_test.go b/services/repository/archiver/archiver_test.go index b3f3ed7bf3..2ab18edf49 100644 --- a/services/repository/archiver/archiver_test.go +++ b/services/repository/archiver/archiver_test.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/services/contexttest" _ "code.gitea.io/gitea/models/actions" @@ -31,47 +32,47 @@ func TestArchive_Basic(t *testing.T) { contexttest.LoadGitRepo(t, ctx) defer ctx.Repo.GitRepo.Close() - bogusReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") + bogusReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit, git.ZIP) assert.NoError(t, err) assert.NotNil(t, bogusReq) assert.EqualValues(t, firstCommit+".zip", bogusReq.GetArchiveName()) // Check a series of bogus requests. // Step 1, valid commit with a bad extension. - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".dilbert") + bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit, 100) assert.Error(t, err) assert.Nil(t, bogusReq) // Step 2, missing commit. - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "dbffff.zip") + bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "dbffff", git.ZIP) assert.Error(t, err) assert.Nil(t, bogusReq) // Step 3, doesn't look like branch/tag/commit. - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "db.zip") + bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "db", git.ZIP) assert.Error(t, err) assert.Nil(t, bogusReq) - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "master.zip") + bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "master", git.ZIP) assert.NoError(t, err) assert.NotNil(t, bogusReq) assert.EqualValues(t, "master.zip", bogusReq.GetArchiveName()) - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "test/archive.zip") + bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "test/archive", git.ZIP) assert.NoError(t, err) assert.NotNil(t, bogusReq) assert.EqualValues(t, "test-archive.zip", bogusReq.GetArchiveName()) // Now two valid requests, firstCommit with valid extensions. - zipReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") + zipReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit, git.ZIP) assert.NoError(t, err) assert.NotNil(t, zipReq) - tgzReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".tar.gz") + tgzReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit, git.TARGZ) assert.NoError(t, err) assert.NotNil(t, tgzReq) - secondReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit+".zip") + secondReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit, git.ZIP) assert.NoError(t, err) assert.NotNil(t, secondReq) @@ -91,7 +92,7 @@ func TestArchive_Basic(t *testing.T) { // Sleep two seconds to make sure the queue doesn't change. time.Sleep(2 * time.Second) - zipReq2, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") + zipReq2, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit, git.ZIP) assert.NoError(t, err) // This zipReq should match what's sitting in the queue, as we haven't // let it release yet. From the consumer's point of view, this looks like @@ -106,12 +107,12 @@ func TestArchive_Basic(t *testing.T) { // Now we'll submit a request and TimedWaitForCompletion twice, before and // after we release it. We should trigger both the timeout and non-timeout // cases. - timedReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit+".tar.gz") + timedReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit, git.TARGZ) assert.NoError(t, err) assert.NotNil(t, timedReq) doArchive(db.DefaultContext, timedReq) - zipReq2, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") + zipReq2, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit, git.ZIP) assert.NoError(t, err) // Now, we're guaranteed to have released the original zipReq from the queue. // Ensure that we don't get handed back the released entry somehow, but they diff --git a/tests/integration/api_repo_archive_test.go b/tests/integration/api_repo_archive_test.go index eecb84d5d1..8589199da3 100644 --- a/tests/integration/api_repo_archive_test.go +++ b/tests/integration/api_repo_archive_test.go @@ -59,3 +59,43 @@ func TestAPIDownloadArchive(t *testing.T) { link, _ = url.Parse(fmt.Sprintf("/api/v1/repos/%s/%s/archive/master", user2.Name, repo.Name)) MakeRequest(t, NewRequest(t, "GET", link.String()).AddTokenAuth(token), http.StatusBadRequest) } + +func TestAPIDownloadArchive2(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + session := loginUser(t, user2.LowerName) + token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeReadRepository) + + link, _ := url.Parse(fmt.Sprintf("/api/v1/repos/%s/%s/zipball/master", user2.Name, repo.Name)) + resp := MakeRequest(t, NewRequest(t, "GET", link.String()).AddTokenAuth(token), http.StatusOK) + bs, err := io.ReadAll(resp.Body) + assert.NoError(t, err) + assert.Len(t, bs, 320) + + link, _ = url.Parse(fmt.Sprintf("/api/v1/repos/%s/%s/tarball/master", user2.Name, repo.Name)) + resp = MakeRequest(t, NewRequest(t, "GET", link.String()).AddTokenAuth(token), http.StatusOK) + bs, err = io.ReadAll(resp.Body) + assert.NoError(t, err) + assert.Len(t, bs, 266) + + // Must return a link to a commit ID as the "immutable" archive link + linkHeaderRe := regexp.MustCompile(`^<(https?://.*/api/v1/repos/user2/repo1/archive/[a-f0-9]+\.tar\.gz.*)>; rel="immutable"$`) + m := linkHeaderRe.FindStringSubmatch(resp.Header().Get("Link")) + assert.NotEmpty(t, m[1]) + resp = MakeRequest(t, NewRequest(t, "GET", m[1]).AddTokenAuth(token), http.StatusOK) + bs2, err := io.ReadAll(resp.Body) + assert.NoError(t, err) + // The locked URL should give the same bytes as the non-locked one + assert.EqualValues(t, bs, bs2) + + link, _ = url.Parse(fmt.Sprintf("/api/v1/repos/%s/%s/bundle/master", user2.Name, repo.Name)) + resp = MakeRequest(t, NewRequest(t, "GET", link.String()).AddTokenAuth(token), http.StatusOK) + bs, err = io.ReadAll(resp.Body) + assert.NoError(t, err) + assert.Len(t, bs, 382) + + link, _ = url.Parse(fmt.Sprintf("/api/v1/repos/%s/%s/archive/master", user2.Name, repo.Name)) + MakeRequest(t, NewRequest(t, "GET", link.String()).AddTokenAuth(token), http.StatusBadRequest) +}