[API] Add pagination to ListBranches (#14524)

* make PaginateUserSlice generic -> PaginateSlice

* Add pagination to ListBranches

* add skip, limit to Repository.GetBranches()

* Move routers/api/v1/utils/utils PaginateSlice -> modules/util/paginate.go

* repo_module.GetBranches paginate

* fix & rename & more logging

* better description

Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: a1012112796 <1012112796@qq.com>
This commit is contained in:
6543 2021-02-03 20:06:13 +01:00 committed by GitHub
parent c295a27d4a
commit 0d1444751f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 239 additions and 87 deletions

View File

@ -57,7 +57,9 @@ func branchAction(t *testing.T, button string) (*HTMLDoc, string) {
htmlDoc := NewHTMLParser(t, resp.Body)
link, exists := htmlDoc.doc.Find(button).Attr("data-url")
assert.True(t, exists, "The template has changed")
if !assert.True(t, exists, "The template has changed") {
t.Skip()
}
req = NewRequestWithValues(t, "POST", link, map[string]string{
"_csrf": getCsrf(t, htmlDoc.doc),
@ -69,7 +71,7 @@ func branchAction(t *testing.T, button string) (*HTMLDoc, string) {
req = NewRequest(t, "GET", "/user2/repo1/branches")
resp = session.MakeRequest(t, req, http.StatusOK)
return NewHTMLParser(t, resp.Body), url.Query()["name"][0]
return NewHTMLParser(t, resp.Body), url.Query().Get("name")
}
func getCsrf(t *testing.T, doc *goquery.Document) string {

View File

@ -554,7 +554,7 @@ func RepoAssignment() func(http.Handler) http.Handler {
}
ctx.Data["Tags"] = tags
brs, err := ctx.Repo.GitRepo.GetBranches()
brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 0)
if err != nil {
ctx.ServerError("GetBranches", err)
return
@ -747,7 +747,7 @@ func RepoRefByType(refType RepoRefType) func(http.Handler) http.Handler {
refName = ctx.Repo.Repository.DefaultBranch
ctx.Repo.BranchName = refName
if !ctx.Repo.GitRepo.IsBranchExist(refName) {
brs, err := ctx.Repo.GitRepo.GetBranches()
brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 0)
if err != nil {
ctx.ServerError("GetBranches", err)
return

View File

@ -78,16 +78,17 @@ func (repo *Repository) GetBranch(branch string) (*Branch, error) {
}
// GetBranchesByPath returns a branch by it's path
func GetBranchesByPath(path string) ([]*Branch, error) {
// if limit = 0 it will not limit
func GetBranchesByPath(path string, skip, limit int) ([]*Branch, int, error) {
gitRepo, err := OpenRepository(path)
if err != nil {
return nil, err
return nil, 0, err
}
defer gitRepo.Close()
brs, err := gitRepo.GetBranches()
brs, countAll, err := gitRepo.GetBranches(skip, limit)
if err != nil {
return nil, err
return nil, 0, err
}
branches := make([]*Branch, len(brs))
@ -99,7 +100,7 @@ func GetBranchesByPath(path string) ([]*Branch, error) {
}
}
return branches, nil
return branches, countAll, nil
}
// DeleteBranchOptions Option(s) for delete branch

View File

@ -25,21 +25,32 @@ func (repo *Repository) IsBranchExist(name string) bool {
return reference.Type() != plumbing.InvalidReference
}
// GetBranches returns all branches of the repository.
func (repo *Repository) GetBranches() ([]string, error) {
// GetBranches returns branches from the repository, skipping skip initial branches and
// returning at most limit branches, or all branches if limit is 0.
func (repo *Repository) GetBranches(skip, limit int) ([]string, int, error) {
var branchNames []string
branches, err := repo.gogitRepo.Branches()
if err != nil {
return nil, err
return nil, 0, err
}
i := 0
count := 0
_ = branches.ForEach(func(branch *plumbing.Reference) error {
count++
if i < skip {
i++
return nil
} else if limit != 0 && count > skip+limit {
return nil
}
branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix))
return nil
})
// TODO: Sort?
return branchNames, nil
return branchNames, count, nil
}

View File

@ -21,14 +21,14 @@ func (repo *Repository) IsBranchExist(name string) bool {
return IsReferenceExist(repo.Path, BranchPrefix+name)
}
// GetBranches returns all branches of the repository.
func (repo *Repository) GetBranches() ([]string, error) {
return callShowRef(repo.Path, BranchPrefix, "--heads")
// GetBranches returns branches from the repository, skipping skip initial branches and
// returning at most limit branches, or all branches if limit is 0.
func (repo *Repository) GetBranches(skip, limit int) ([]string, int, error) {
return callShowRef(repo.Path, BranchPrefix, "--heads", skip, limit)
}
func callShowRef(repoPath, prefix, arg string) ([]string, error) {
var branchNames []string
// callShowRef return refs, if limit = 0 it will not limit
func callShowRef(repoPath, prefix, arg string, skip, limit int) (branchNames []string, countAll int, err error) {
stdoutReader, stdoutWriter := io.Pipe()
defer func() {
_ = stdoutReader.Close()
@ -49,8 +49,21 @@ func callShowRef(repoPath, prefix, arg string) ([]string, error) {
}
}()
i := 0
bufReader := bufio.NewReader(stdoutReader)
for {
for i < skip {
_, isPrefix, err := bufReader.ReadLine()
if err == io.EOF {
return branchNames, i, nil
}
if err != nil {
return nil, 0, err
}
if !isPrefix {
i++
}
}
for limit == 0 || i < skip+limit {
// The output of show-ref is simply a list:
// <sha> SP <ref> LF
_, err := bufReader.ReadSlice(' ')
@ -59,24 +72,39 @@ func callShowRef(repoPath, prefix, arg string) ([]string, error) {
_, err = bufReader.ReadSlice(' ')
}
if err == io.EOF {
return branchNames, nil
return branchNames, i, nil
}
if err != nil {
return nil, err
return nil, 0, err
}
branchName, err := bufReader.ReadString('\n')
if err == io.EOF {
// This shouldn't happen... but we'll tolerate it for the sake of peace
return branchNames, nil
return branchNames, i, nil
}
if err != nil {
return nil, err
return nil, i, err
}
branchName = strings.TrimPrefix(branchName, prefix)
if len(branchName) > 0 {
branchName = branchName[:len(branchName)-1]
}
branchNames = append(branchNames, branchName)
i++
}
// count all refs
for limit != 0 {
_, isPrefix, err := bufReader.ReadLine()
if err == io.EOF {
return branchNames, i, nil
}
if err != nil {
return nil, 0, err
}
if !isPrefix {
i++
}
}
return branchNames, i, nil
}

View File

@ -17,11 +17,26 @@ func TestRepository_GetBranches(t *testing.T) {
assert.NoError(t, err)
defer bareRepo1.Close()
branches, err := bareRepo1.GetBranches()
branches, countAll, err := bareRepo1.GetBranches(0, 2)
assert.NoError(t, err)
assert.Len(t, branches, 2)
assert.EqualValues(t, 3, countAll)
assert.ElementsMatch(t, []string{"branch1", "branch2"}, branches)
branches, countAll, err = bareRepo1.GetBranches(0, 0)
assert.NoError(t, err)
assert.Len(t, branches, 3)
assert.EqualValues(t, 3, countAll)
assert.ElementsMatch(t, []string{"branch1", "branch2", "master"}, branches)
branches, countAll, err = bareRepo1.GetBranches(5, 1)
assert.NoError(t, err)
assert.Len(t, branches, 0)
assert.EqualValues(t, 3, countAll)
assert.ElementsMatch(t, []string{}, branches)
}
func BenchmarkRepository_GetBranches(b *testing.B) {
@ -33,7 +48,7 @@ func BenchmarkRepository_GetBranches(b *testing.B) {
defer bareRepo1.Close()
for i := 0; i < b.N; i++ {
_, err := bareRepo1.GetBranches()
_, _, err := bareRepo1.GetBranches(0, 0)
if err != nil {
b.Fatal(err)
}

View File

@ -13,6 +13,7 @@ func (repo *Repository) IsTagExist(name string) bool {
}
// GetTags returns all tags of the repository.
func (repo *Repository) GetTags() ([]string, error) {
return callShowRef(repo.Path, TagPrefix, "--tags")
func (repo *Repository) GetTags() (tags []string, err error) {
tags, _, err = callShowRef(repo.Path, TagPrefix, "--tags", 0, 0)
return
}

View File

@ -13,6 +13,9 @@ import (
// GetBranch returns a branch by its name
func GetBranch(repo *models.Repository, branch string) (*git.Branch, error) {
if len(branch) == 0 {
return nil, fmt.Errorf("GetBranch: empty string for branch")
}
gitRepo, err := git.OpenRepository(repo.RepoPath())
if err != nil {
return nil, err
@ -22,9 +25,10 @@ func GetBranch(repo *models.Repository, branch string) (*git.Branch, error) {
return gitRepo.GetBranch(branch)
}
// GetBranches returns all the branches of a repository
func GetBranches(repo *models.Repository) ([]*git.Branch, error) {
return git.GetBranchesByPath(repo.RepoPath())
// GetBranches returns branches from the repository, skipping skip initial branches and
// returning at most limit branches, or all branches if limit is 0.
func GetBranches(repo *models.Repository, skip, limit int) ([]*git.Branch, int, error) {
return git.GetBranchesByPath(repo.RepoPath(), skip, limit)
}
// checkBranchName validates branch name with existing repository branches
@ -35,7 +39,7 @@ func checkBranchName(repo *models.Repository, name string) error {
}
defer gitRepo.Close()
branches, err := GetBranches(repo)
branches, _, err := GetBranches(repo, 0, 0)
if err != nil {
return err
}

View File

@ -239,7 +239,7 @@ func adoptRepository(ctx models.DBContext, repoPath string, u *models.User, repo
repo.DefaultBranch = strings.TrimPrefix(repo.DefaultBranch, git.BranchPrefix)
}
branches, _ := gitRepo.GetBranches()
branches, _, _ := gitRepo.GetBranches(0, 0)
found := false
hasDefault := false
hasMaster := false

34
modules/util/paginate.go Normal file
View File

@ -0,0 +1,34 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package util
import "reflect"
// PaginateSlice cut a slice as per pagination options
// if page = 0 it do not paginate
func PaginateSlice(list interface{}, page, pageSize int) interface{} {
if page <= 0 || pageSize <= 0 {
return list
}
if reflect.TypeOf(list).Kind() != reflect.Slice {
return list
}
listValue := reflect.ValueOf(list)
page--
if page*pageSize >= listValue.Len() {
return listValue.Slice(listValue.Len(), listValue.Len()).Interface()
}
listValue = listValue.Slice(page*pageSize, listValue.Len())
if listValue.Len() > pageSize {
return listValue.Slice(0, pageSize).Interface()
}
return listValue.Interface()
}

View File

@ -0,0 +1,47 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package util
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestPaginateSlice(t *testing.T) {
stringSlice := []string{"a", "b", "c", "d", "e"}
result, ok := PaginateSlice(stringSlice, 1, 2).([]string)
assert.True(t, ok)
assert.EqualValues(t, []string{"a", "b"}, result)
result, ok = PaginateSlice(stringSlice, 100, 2).([]string)
assert.True(t, ok)
assert.EqualValues(t, []string{}, result)
result, ok = PaginateSlice(stringSlice, 3, 2).([]string)
assert.True(t, ok)
assert.EqualValues(t, []string{"e"}, result)
result, ok = PaginateSlice(stringSlice, 1, 0).([]string)
assert.True(t, ok)
assert.EqualValues(t, []string{"a", "b", "c", "d", "e"}, result)
result, ok = PaginateSlice(stringSlice, 1, -1).([]string)
assert.True(t, ok)
assert.EqualValues(t, []string{"a", "b", "c", "d", "e"}, result)
type Test struct {
Val int
}
var testVar = []*Test{{Val: 2}, {Val: 3}, {Val: 4}}
testVar, ok = PaginateSlice(testVar, 1, 50).([]*Test)
assert.True(t, ok)
assert.EqualValues(t, []*Test{{Val: 2}, {Val: 3}, {Val: 4}}, testVar)
testVar, ok = PaginateSlice(testVar, 2, 2).([]*Test)
assert.True(t, ok)
assert.EqualValues(t, []*Test{{Val: 4}}, testVar)
}

View File

@ -13,6 +13,7 @@ import (
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/convert"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/routers/api/v1/user"
"code.gitea.io/gitea/routers/api/v1/utils"
@ -28,9 +29,9 @@ func listUserOrgs(ctx *context.APIContext, u *models.User) {
ctx.Error(http.StatusInternalServerError, "GetOrgsByUserID", err)
return
}
maxResults := len(orgs)
orgs = utils.PaginateUserSlice(orgs, listOptions.Page, listOptions.PageSize)
maxResults := len(orgs)
orgs, _ = util.PaginateSlice(orgs, listOptions.Page, listOptions.PageSize).([]*models.User)
apiOrgs := make([]*api.Organization, len(orgs))
for i := range orgs {

View File

@ -17,6 +17,7 @@ import (
repo_module "code.gitea.io/gitea/modules/repository"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/routers/api/v1/utils"
pull_service "code.gitea.io/gitea/services/pull"
repo_service "code.gitea.io/gitea/services/repository"
)
@ -284,11 +285,21 @@ func ListBranches(ctx *context.APIContext) {
// description: name of the repo
// type: string
// required: true
// - name: page
// in: query
// description: page number of results to return (1-based)
// type: integer
// - name: limit
// in: query
// description: page size of results
// type: integer
// responses:
// "200":
// "$ref": "#/responses/BranchList"
branches, err := repo_module.GetBranches(ctx.Repo.Repository)
listOptions := utils.GetListOptions(ctx)
skip, _ := listOptions.GetStartEnd()
branches, totalNumOfBranches, err := repo_module.GetBranches(ctx.Repo.Repository, skip, listOptions.PageSize)
if err != nil {
ctx.Error(http.StatusInternalServerError, "GetBranches", err)
return
@ -313,6 +324,9 @@ func ListBranches(ctx *context.APIContext) {
}
}
ctx.SetLinkHeader(int(totalNumOfBranches), listOptions.PageSize)
ctx.Header().Set("X-Total-Count", fmt.Sprintf("%d", totalNumOfBranches))
ctx.Header().Set("Access-Control-Expose-Headers", "X-Total-Count, Link")
ctx.JSON(http.StatusOK, &apiBranches)
}

View File

@ -66,22 +66,3 @@ func GetListOptions(ctx *context.APIContext) models.ListOptions {
PageSize: convert.ToCorrectPageSize(ctx.QueryInt("limit")),
}
}
// PaginateUserSlice cut a slice of Users as per pagination options
// TODO: make it generic
func PaginateUserSlice(items []*models.User, page, pageSize int) []*models.User {
if page != 0 {
page--
}
if page*pageSize >= len(items) {
return items[len(items):]
}
items = items[page*pageSize:]
if len(items) > pageSize {
return items[:pageSize]
}
return items
}

View File

@ -58,12 +58,14 @@ func Branches(ctx *context.Context) {
page = 1
}
pageSize := ctx.QueryInt("limit")
if pageSize <= 0 || pageSize > git.BranchesRangeSize {
pageSize = git.BranchesRangeSize
limit := ctx.QueryInt("limit")
if limit <= 0 || limit > git.BranchesRangeSize {
limit = git.BranchesRangeSize
}
branches, branchesCount := loadBranches(ctx, page, pageSize)
skip := (page - 1) * limit
log.Debug("Branches: skip: %d limit: %d", skip, limit)
branches, branchesCount := loadBranches(ctx, skip, limit)
if ctx.Written() {
return
}
@ -80,6 +82,7 @@ func DeleteBranchPost(ctx *context.Context) {
defer redirect(ctx)
branchName := ctx.Query("name")
if branchName == ctx.Repo.Repository.DefaultBranch {
log.Debug("DeleteBranch: Can't delete default branch '%s'", branchName)
ctx.Flash.Error(ctx.Tr("repo.branch.default_deletion_failed", branchName))
return
}
@ -92,16 +95,19 @@ func DeleteBranchPost(ctx *context.Context) {
}
if isProtected {
log.Debug("DeleteBranch: Can't delete protected branch '%s'", branchName)
ctx.Flash.Error(ctx.Tr("repo.branch.protected_deletion_failed", branchName))
return
}
if !ctx.Repo.GitRepo.IsBranchExist(branchName) || branchName == ctx.Repo.Repository.DefaultBranch {
if !ctx.Repo.GitRepo.IsBranchExist(branchName) {
log.Debug("DeleteBranch: Can't delete non existing branch '%s'", branchName)
ctx.Flash.Error(ctx.Tr("repo.branch.deletion_failed", branchName))
return
}
if err := deleteBranch(ctx, branchName); err != nil {
log.Error("DeleteBranch: %v", err)
ctx.Flash.Error(ctx.Tr("repo.branch.deletion_failed", branchName))
return
}
@ -129,10 +135,11 @@ func RestoreBranchPost(ctx *context.Context) {
Env: models.PushingEnvironment(ctx.User, ctx.Repo.Repository),
}); err != nil {
if strings.Contains(err.Error(), "already exists") {
log.Debug("RestoreBranch: Can't restore branch '%s', since one with same name already exist", deletedBranch.Name)
ctx.Flash.Error(ctx.Tr("repo.branch.already_exists", deletedBranch.Name))
return
}
log.Error("CreateBranch: %v", err)
log.Error("RestoreBranch: CreateBranch: %v", err)
ctx.Flash.Error(ctx.Tr("repo.branch.restore_failed", deletedBranch.Name))
return
}
@ -148,7 +155,7 @@ func RestoreBranchPost(ctx *context.Context) {
RepoUserName: ctx.Repo.Owner.Name,
RepoName: ctx.Repo.Repository.Name,
}); err != nil {
log.Error("Update: %v", err)
log.Error("RestoreBranch: Update: %v", err)
}
ctx.Flash.Success(ctx.Tr("repo.branch.restore_success", deletedBranch.Name))
@ -196,16 +203,18 @@ func deleteBranch(ctx *context.Context, branchName string) error {
}
// loadBranches loads branches from the repository limited by page & pageSize.
// NOTE: May write to context on error. page & pageSize must be > 0
func loadBranches(ctx *context.Context, page, pageSize int) ([]*Branch, int) {
// NOTE: May write to context on error.
func loadBranches(ctx *context.Context, skip, limit int) ([]*Branch, int) {
defaultBranch, err := repo_module.GetBranch(ctx.Repo.Repository, ctx.Repo.Repository.DefaultBranch)
if err != nil {
log.Error("loadBranches: get default branch: %v", err)
ctx.ServerError("GetDefaultBranch", err)
return nil, 0
}
rawBranches, err := repo_module.GetBranches(ctx.Repo.Repository)
rawBranches, totalNumOfBranches, err := repo_module.GetBranches(ctx.Repo.Repository, skip, limit)
if err != nil {
log.Error("GetBranches: %v", err)
ctx.ServerError("GetBranches", err)
return nil, 0
}
@ -222,32 +231,23 @@ func loadBranches(ctx *context.Context, page, pageSize int) ([]*Branch, int) {
repoIDToGitRepo := map[int64]*git.Repository{}
repoIDToGitRepo[ctx.Repo.Repository.ID] = ctx.Repo.GitRepo
var totalNumOfBranches = len(rawBranches)
var startIndex = (page - 1) * pageSize
if startIndex > totalNumOfBranches {
startIndex = totalNumOfBranches - 1
}
var endIndex = startIndex + pageSize
if endIndex > totalNumOfBranches {
endIndex = totalNumOfBranches - 1
}
var branches []*Branch
for i := startIndex; i < endIndex; i++ {
for i := range rawBranches {
if rawBranches[i].Name == defaultBranch.Name {
// Skip default branch
continue
}
var branch = loadOneBranch(ctx, rawBranches[i], protectedBranches, repoIDToRepo, repoIDToGitRepo)
if branch == nil {
return nil, 0
}
if branch.Name == ctx.Repo.Repository.DefaultBranch {
// Skip default branch
continue
}
branches = append(branches, branch)
}
// Always add the default branch
log.Debug("loadOneBranch: load default: '%s'", defaultBranch.Name)
branches = append(branches, loadOneBranch(ctx, defaultBranch, protectedBranches, repoIDToRepo, repoIDToGitRepo))
if ctx.Repo.CanWrite(models.UnitTypeCode) {
@ -259,12 +259,13 @@ func loadBranches(ctx *context.Context, page, pageSize int) ([]*Branch, int) {
branches = append(branches, deletedBranches...)
}
return branches, len(rawBranches) - 1
return branches, totalNumOfBranches - 1
}
func loadOneBranch(ctx *context.Context, rawBranch *git.Branch, protectedBranches []*models.ProtectedBranch,
repoIDToRepo map[int64]*models.Repository,
repoIDToGitRepo map[int64]*git.Repository) *Branch {
log.Trace("loadOneBranch: '%s'", rawBranch.Name)
commit, err := rawBranch.GetCommit()
if err != nil {

View File

@ -520,7 +520,7 @@ func getBranchesForRepo(user *models.User, repo *models.Repository) (bool, []str
}
defer gitRepo.Close()
branches, err := gitRepo.GetBranches()
branches, _, err := gitRepo.GetBranches(0, 0)
if err != nil {
return false, nil, err
}
@ -541,7 +541,7 @@ func CompareDiff(ctx *context.Context) {
}
if ctx.Data["PageIsComparePull"] == true {
headBranches, err := headGitRepo.GetBranches()
headBranches, _, err := headGitRepo.GetBranches(0, 0)
if err != nil {
ctx.ServerError("GetBranches", err)
return

View File

@ -678,7 +678,7 @@ func RetrieveRepoMetas(ctx *context.Context, repo *models.Repository, isPull boo
return nil
}
brs, err := ctx.Repo.GitRepo.GetBranches()
brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 0)
if err != nil {
ctx.ServerError("GetBranches", err)
return nil

View File

@ -301,7 +301,7 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) {
}
log.Trace("SyncMirrors [repo: %-v]: invalidating mirror branch caches...", m.Repo)
branches, err := repo_module.GetBranches(m.Repo)
branches, _, err := repo_module.GetBranches(m.Repo, 0, 0)
if err != nil {
log.Error("GetBranches: %v", err)
return nil, false

View File

@ -482,7 +482,7 @@ func CloseBranchPulls(doer *models.User, repoID int64, branch string) error {
// CloseRepoBranchesPulls close all pull requests which head branches are in the given repository
func CloseRepoBranchesPulls(doer *models.User, repo *models.Repository) error {
branches, err := git.GetBranchesByPath(repo.RepoPath())
branches, _, err := git.GetBranchesByPath(repo.RepoPath(), 0, 0)
if err != nil {
return err
}

View File

@ -2500,6 +2500,18 @@
"name": "repo",
"in": "path",
"required": true
},
{
"type": "integer",
"description": "page number of results to return (1-based)",
"name": "page",
"in": "query"
},
{
"type": "integer",
"description": "page size of results",
"name": "limit",
"in": "query"
}
],
"responses": {