Support repo license (#24872)

Close #278
Close #24076

## Solutions:
- Use
[google/licenseclassifier](https://github.com/google/licenseclassifier/)
Test result between
[google/licensecheck](https://github.com/google/licensecheck) and
[go-license-detector](https://github.com/go-enry/go-license-detector):
https://github.com/go-gitea/gitea/pull/24872#issuecomment-1560361167
Test result between
[google/licensecheck](https://github.com/google/licensecheck) and
[google/licenseclassifier](https://github.com/google/licenseclassifier/):
https://github.com/go-gitea/gitea/pull/24872#issuecomment-1576092178
- Generate License Convert Name List to avoid import license templates
with same contents
Gitea automatically get latest license data from[
spdx/license-list-data](https://github.com/spdx/license-list-data).
But unfortunately, some license templates have same contents. #20915
[click here to see the
list](https://github.com/go-gitea/gitea/pull/24872#issuecomment-1584141684)
So we will generate a list of these license templates with same contents
and create a new file to save the result when using `make
generate-license`. (Need to decide the save path)
- Save License info into a new table `repo_license`
Can easily support searching repo by license in the future.

## Screen shot
Single License:

![image](https://github.com/go-gitea/gitea/assets/18380374/41260bd7-0b4c-4038-8592-508706cffa9f)

Multiple Licenses:

![image](https://github.com/go-gitea/gitea/assets/18380374/34ce2f73-7e18-446b-9b96-ecc4fb61bd70)

Triggers:
- [x] Push commit to default branch
- [x] Create repo
- [x] Mirror repo
- [x] When Default Branch is changed, licenses should be updated

Todo:
- [x] Save Licenses info in to DB when there's a change to license file
in the commit
- [x] DB Migration
- [x] A nominal test?
- [x] Select which library to
use(https://github.com/go-gitea/gitea/pull/24872#issuecomment-1560361167)
- [x] API Support
- [x] Add repo license table
- ~Select license in settings if there are several licenses(Not
recommended)~
- License board(later, not in this PR)

![image](https://github.com/go-gitea/gitea/assets/18380374/2c3c3bf8-bcc2-4c6d-8ce0-81d1a9733878)

---------

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: Denys Konovalov <kontakt@denyskon.de>
Co-authored-by: delvh <dev.lh@web.de>
Co-authored-by: KN4CK3R <admin@oldschoolhack.me>
Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: 6543 <m.huber@kithara.com>
Co-authored-by: a1012112796 <1012112796@qq.com>
Co-authored-by: techknowlogick <techknowlogick@gitea.com>
This commit is contained in:
yp05327 2024-10-02 04:25:08 +09:00 committed by GitHub
parent f4b8f6fc40
commit 70b7df0e5e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
47 changed files with 906 additions and 22 deletions

File diff suppressed because one or more lines are too long

View File

@ -1,3 +1,6 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build ignore //go:build ignore
package main package main
@ -5,6 +8,8 @@ package main
import ( import (
"archive/tar" "archive/tar"
"compress/gzip" "compress/gzip"
"crypto/md5"
"encoding/hex"
"flag" "flag"
"fmt" "fmt"
"io" "io"
@ -15,6 +20,8 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"code.gitea.io/gitea/build/license"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/util"
) )
@ -77,7 +84,7 @@ func main() {
} }
tr := tar.NewReader(gz) tr := tar.NewReader(gz)
aliasesFiles := make(map[string][]string)
for { for {
hdr, err := tr.Next() hdr, err := tr.Next()
@ -97,26 +104,73 @@ func main() {
continue continue
} }
if strings.HasPrefix(filepath.Base(hdr.Name), "README") { fileBaseName := filepath.Base(hdr.Name)
licenseName := strings.TrimSuffix(fileBaseName, ".txt")
if strings.HasPrefix(fileBaseName, "README") {
continue continue
} }
if strings.HasPrefix(filepath.Base(hdr.Name), "deprecated_") { if strings.HasPrefix(fileBaseName, "deprecated_") {
continue continue
} }
out, err := os.Create(path.Join(destination, strings.TrimSuffix(filepath.Base(hdr.Name), ".txt"))) out, err := os.Create(path.Join(destination, licenseName))
if err != nil { if err != nil {
log.Fatalf("Failed to create new file. %s", err) log.Fatalf("Failed to create new file. %s", err)
} }
defer out.Close() defer out.Close()
if _, err := io.Copy(out, tr); err != nil { // some license files have same content, so we need to detect these files and create a convert map into a json file
// Later we use this convert map to avoid adding same license content with different license name
h := md5.New()
// calculate md5 and write file in the same time
r := io.TeeReader(tr, h)
if _, err := io.Copy(out, r); err != nil {
log.Fatalf("Failed to write new file. %s", err) log.Fatalf("Failed to write new file. %s", err)
} else { } else {
fmt.Printf("Written %s\n", out.Name()) fmt.Printf("Written %s\n", out.Name())
md5 := hex.EncodeToString(h.Sum(nil))
aliasesFiles[md5] = append(aliasesFiles[md5], licenseName)
} }
} }
// generate convert license name map
licenseAliases := make(map[string]string)
for _, fileNames := range aliasesFiles {
if len(fileNames) > 1 {
licenseName := license.GetLicenseNameFromAliases(fileNames)
if licenseName == "" {
// license name should not be empty as expected
// if it is empty, we need to rewrite the logic of GetLicenseNameFromAliases
log.Fatalf("GetLicenseNameFromAliases: license name is empty")
}
for _, fileName := range fileNames {
licenseAliases[fileName] = licenseName
}
}
}
// save convert license name map to file
b, err := json.Marshal(licenseAliases)
if err != nil {
log.Fatalf("Failed to create json bytes. %s", err)
}
licenseAliasesDestination := filepath.Join(destination, "etc", "license-aliases.json")
if err := os.MkdirAll(filepath.Dir(licenseAliasesDestination), 0o755); err != nil {
log.Fatalf("Failed to create directory for license aliases json file. %s", err)
}
f, err := os.Create(licenseAliasesDestination)
if err != nil {
log.Fatalf("Failed to create license aliases json file. %s", err)
}
defer f.Close()
if _, err = f.Write(b); err != nil {
log.Fatalf("Failed to write license aliases json file. %s", err)
}
fmt.Println("Done") fmt.Println("Done")
} }

View File

@ -0,0 +1,41 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package license
import "strings"
func GetLicenseNameFromAliases(fnl []string) string {
if len(fnl) == 0 {
return ""
}
shortestItem := func(list []string) string {
s := list[0]
for _, l := range list[1:] {
if len(l) < len(s) {
s = l
}
}
return s
}
allHasPrefix := func(list []string, s string) bool {
for _, l := range list {
if !strings.HasPrefix(l, s) {
return false
}
}
return true
}
sl := shortestItem(fnl)
slv := strings.Split(sl, "-")
var result string
for i := len(slv); i >= 0; i-- {
result = strings.Join(slv[:i], "-")
if allHasPrefix(fnl, result) {
return result
}
}
return ""
}

View File

@ -0,0 +1,39 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package license
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetLicenseNameFromAliases(t *testing.T) {
tests := []struct {
target string
inputs []string
}{
{
// real case which you can find in license-aliases.json
target: "AGPL-1.0",
inputs: []string{
"AGPL-1.0-only",
"AGPL-1.0-or-late",
},
},
{
target: "",
inputs: []string{
"APSL-1.0",
"AGPL-1.0-only",
"AGPL-1.0-or-late",
},
},
}
for _, tt := range tests {
result := GetLicenseNameFromAliases(tt.inputs)
assert.Equal(t, result, tt.target)
}
}

1
go.mod
View File

@ -68,6 +68,7 @@ require (
github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85
github.com/golang-jwt/jwt/v5 v5.2.1 github.com/golang-jwt/jwt/v5 v5.2.1
github.com/google/go-github/v61 v61.0.0 github.com/google/go-github/v61 v61.0.0
github.com/google/licenseclassifier/v2 v2.0.0
github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8 github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8
github.com/google/uuid v1.6.0 github.com/google/uuid v1.6.0
github.com/gorilla/feeds v1.2.0 github.com/gorilla/feeds v1.2.0

3
go.sum
View File

@ -441,6 +441,8 @@ github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/licenseclassifier/v2 v2.0.0 h1:1Y57HHILNf4m0ABuMVb6xk4vAJYEUO0gDxNpog0pyeA=
github.com/google/licenseclassifier/v2 v2.0.0/go.mod h1:cOjbdH0kyC9R22sdQbYsFkto4NGCAc+ZSwbeThazEtM=
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8 h1:ASJ/LAqdCHOyMYI+dwNxn7Rd8FscNkMyTr1KZU1JI/M= github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8 h1:ASJ/LAqdCHOyMYI+dwNxn7Rd8FscNkMyTr1KZU1JI/M=
github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
@ -735,6 +737,7 @@ github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jN
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516/go.mod h1:Yow6lPLSAXx2ifx470yD/nUe22Dv5vBvxK/UK9UUTVs= github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516/go.mod h1:Yow6lPLSAXx2ifx470yD/nUe22Dv5vBvxK/UK9UUTVs=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=

View File

@ -0,0 +1 @@
[] # empty

View File

@ -26,7 +26,7 @@
fork_id: 0 fork_id: 0
is_template: false is_template: false
template_id: 0 template_id: 0
size: 7597 size: 8478
is_fsck_enabled: true is_fsck_enabled: true
close_issues_via_commit_in_any_branch: false close_issues_via_commit_in_any_branch: false

View File

@ -500,7 +500,7 @@ var migrations = []Migration{
// v259 -> v260 // v259 -> v260
NewMigration("Convert scoped access tokens", v1_20.ConvertScopedAccessTokens), NewMigration("Convert scoped access tokens", v1_20.ConvertScopedAccessTokens),
// Gitea 1.20.0 ends at 260 // Gitea 1.20.0 ends at v260
// v260 -> v261 // v260 -> v261
NewMigration("Drop custom_labels column of action_runner table", v1_21.DropCustomLabelsColumnOfActionRunner), NewMigration("Drop custom_labels column of action_runner table", v1_21.DropCustomLabelsColumnOfActionRunner),
@ -601,6 +601,8 @@ var migrations = []Migration{
NewMigration("Add metadata column for comment table", v1_23.AddCommentMetaDataColumn), NewMigration("Add metadata column for comment table", v1_23.AddCommentMetaDataColumn),
// v304 -> v305 // v304 -> v305
NewMigration("Add index for release sha1", v1_23.AddIndexForReleaseSha1), NewMigration("Add index for release sha1", v1_23.AddIndexForReleaseSha1),
// v305 -> v306
NewMigration("Add Repository Licenses", v1_23.AddRepositoryLicenses),
} }
// GetCurrentDBVersion returns the current db version // GetCurrentDBVersion returns the current db version

View File

@ -0,0 +1,23 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_23 //nolint
import (
"code.gitea.io/gitea/modules/timeutil"
"xorm.io/xorm"
)
func AddRepositoryLicenses(x *xorm.Engine) error {
type RepoLicense struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"UNIQUE(s) NOT NULL"`
CommitID string
License string `xorm:"VARCHAR(255) UNIQUE(s) NOT NULL"`
CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX UPDATED"`
}
return x.Sync(new(RepoLicense))
}

120
models/repo/license.go Normal file
View File

@ -0,0 +1,120 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package repo
import (
"context"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/timeutil"
)
func init() {
db.RegisterModel(new(RepoLicense))
}
type RepoLicense struct { //revive:disable-line:exported
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"UNIQUE(s) NOT NULL"`
CommitID string
License string `xorm:"VARCHAR(255) UNIQUE(s) NOT NULL"`
CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX UPDATED"`
}
// RepoLicenseList defines a list of repo licenses
type RepoLicenseList []*RepoLicense //revive:disable-line:exported
func (rll RepoLicenseList) StringList() []string {
var licenses []string
for _, rl := range rll {
licenses = append(licenses, rl.License)
}
return licenses
}
// GetRepoLicenses returns the license statistics for a repository
func GetRepoLicenses(ctx context.Context, repo *Repository) (RepoLicenseList, error) {
licenses := make(RepoLicenseList, 0)
if err := db.GetEngine(ctx).Where("`repo_id` = ?", repo.ID).Asc("`license`").Find(&licenses); err != nil {
return nil, err
}
return licenses, nil
}
// UpdateRepoLicenses updates the license statistics for repository
func UpdateRepoLicenses(ctx context.Context, repo *Repository, commitID string, licenses []string) error {
oldLicenses, err := GetRepoLicenses(ctx, repo)
if err != nil {
return err
}
for _, license := range licenses {
upd := false
for _, o := range oldLicenses {
// Update already existing license
if o.License == license {
if _, err := db.GetEngine(ctx).ID(o.ID).Cols("`commit_id`").Update(o); err != nil {
return err
}
upd = true
break
}
}
// Insert new license
if !upd {
if err := db.Insert(ctx, &RepoLicense{
RepoID: repo.ID,
CommitID: commitID,
License: license,
}); err != nil {
return err
}
}
}
// Delete old licenses
licenseToDelete := make([]int64, 0, len(oldLicenses))
for _, o := range oldLicenses {
if o.CommitID != commitID {
licenseToDelete = append(licenseToDelete, o.ID)
}
}
if len(licenseToDelete) > 0 {
if _, err := db.GetEngine(ctx).In("`id`", licenseToDelete).Delete(&RepoLicense{}); err != nil {
return err
}
}
return nil
}
// CopyLicense Copy originalRepo license information to destRepo (use for forked repo)
func CopyLicense(ctx context.Context, originalRepo, destRepo *Repository) error {
repoLicenses, err := GetRepoLicenses(ctx, originalRepo)
if err != nil {
return err
}
if len(repoLicenses) > 0 {
newRepoLicenses := make(RepoLicenseList, 0, len(repoLicenses))
for _, rl := range repoLicenses {
newRepoLicense := &RepoLicense{
RepoID: destRepo.ID,
CommitID: rl.CommitID,
License: rl.License,
}
newRepoLicenses = append(newRepoLicenses, newRepoLicense)
}
if err := db.Insert(ctx, &newRepoLicenses); err != nil {
return err
}
}
return nil
}
// CleanRepoLicenses will remove all license record of the repo
func CleanRepoLicenses(ctx context.Context, repo *Repository) error {
return db.DeleteBeans(ctx, &RepoLicense{
RepoID: repo.ID,
})
}

View File

@ -23,7 +23,7 @@ type LicenseValues struct {
func GetLicense(name string, values *LicenseValues) ([]byte, error) { func GetLicense(name string, values *LicenseValues) ([]byte, error) {
data, err := options.License(name) data, err := options.License(name)
if err != nil { if err != nil {
return nil, fmt.Errorf("GetRepoInitFile[%s]: %w", name, err) return nil, fmt.Errorf("GetLicense[%s]: %w", name, err)
} }
return fillLicensePlaceholder(name, values, data), nil return fillLicensePlaceholder(name, values, data), nil
} }

View File

@ -114,6 +114,7 @@ type Repository struct {
MirrorUpdated time.Time `json:"mirror_updated,omitempty"` MirrorUpdated time.Time `json:"mirror_updated,omitempty"`
RepoTransfer *RepoTransfer `json:"repo_transfer"` RepoTransfer *RepoTransfer `json:"repo_transfer"`
Topics []string `json:"topics"` Topics []string `json:"topics"`
Licenses []string `json:"licenses"`
} }
// CreateRepoOption options when creating repository // CreateRepoOption options when creating repository

View File

@ -0,0 +1 @@
{"AGPL-1.0-only":"AGPL-1.0","AGPL-1.0-or-later":"AGPL-1.0","AGPL-3.0-only":"AGPL-3.0","AGPL-3.0-or-later":"AGPL-3.0","CAL-1.0":"CAL-1.0","CAL-1.0-Combined-Work-Exception":"CAL-1.0","GFDL-1.1-invariants-only":"GFDL-1.1","GFDL-1.1-invariants-or-later":"GFDL-1.1","GFDL-1.1-no-invariants-only":"GFDL-1.1","GFDL-1.1-no-invariants-or-later":"GFDL-1.1","GFDL-1.1-only":"GFDL-1.1","GFDL-1.1-or-later":"GFDL-1.1","GFDL-1.2-invariants-only":"GFDL-1.2","GFDL-1.2-invariants-or-later":"GFDL-1.2","GFDL-1.2-no-invariants-only":"GFDL-1.2","GFDL-1.2-no-invariants-or-later":"GFDL-1.2","GFDL-1.2-only":"GFDL-1.2","GFDL-1.2-or-later":"GFDL-1.2","GFDL-1.3-invariants-only":"GFDL-1.3","GFDL-1.3-invariants-or-later":"GFDL-1.3","GFDL-1.3-no-invariants-only":"GFDL-1.3","GFDL-1.3-no-invariants-or-later":"GFDL-1.3","GFDL-1.3-only":"GFDL-1.3","GFDL-1.3-or-later":"GFDL-1.3","GPL-1.0-only":"GPL-1.0","GPL-1.0-or-later":"GPL-1.0","GPL-2.0-only":"GPL-2.0","GPL-2.0-or-later":"GPL-2.0","GPL-3.0-only":"GPL-3.0","GPL-3.0-or-later":"GPL-3.0","LGPL-2.0-only":"LGPL-2.0","LGPL-2.0-or-later":"LGPL-2.0","LGPL-2.1-only":"LGPL-2.1","LGPL-2.1-or-later":"LGPL-2.1","LGPL-3.0-only":"LGPL-3.0","LGPL-3.0-or-later":"LGPL-3.0","MPL-2.0":"MPL-2.0","MPL-2.0-no-copyleft-exception":"MPL-2.0","OFL-1.0":"OFL-1.0","OFL-1.0-RFN":"OFL-1.0","OFL-1.0-no-RFN":"OFL-1.0","OFL-1.1":"OFL-1.1","OFL-1.1-RFN":"OFL-1.1","OFL-1.1-no-RFN":"OFL-1.1"}

View File

@ -1040,6 +1040,7 @@ issue_labels_helper = Select an issue label set.
license = License license = License
license_helper = Select a license file. license_helper = Select a license file.
license_helper_desc = A license governs what others can and can't do with your code. Not sure which one is right for your project? See <a target="_blank" rel="noopener noreferrer" href="%s">Choose a license.</a> license_helper_desc = A license governs what others can and can't do with your code. Not sure which one is right for your project? See <a target="_blank" rel="noopener noreferrer" href="%s">Choose a license.</a>
multiple_licenses = Multiple Licenses
object_format = Object Format object_format = Object Format
object_format_helper = Object format of the repository. Cannot be changed later. SHA1 is most compatible. object_format_helper = Object format of the repository. Cannot be changed later. SHA1 is most compatible.
readme = README readme = README
@ -2942,6 +2943,7 @@ dashboard.start_schedule_tasks = Start actions schedule tasks
dashboard.sync_branch.started = Branches Sync started dashboard.sync_branch.started = Branches Sync started
dashboard.sync_tag.started = Tags Sync started dashboard.sync_tag.started = Tags Sync started
dashboard.rebuild_issue_indexer = Rebuild issue indexer dashboard.rebuild_issue_indexer = Rebuild issue indexer
dashboard.sync_repo_licenses = Sync repo licenses
users.user_manage_panel = User Account Management users.user_manage_panel = User Account Management
users.new_account = Create User Account users.new_account = Create User Account

View File

@ -1327,6 +1327,7 @@ func Routes() *web.Router {
m.Get("/issue_config", context.ReferencesGitRepo(), repo.GetIssueConfig) m.Get("/issue_config", context.ReferencesGitRepo(), repo.GetIssueConfig)
m.Get("/issue_config/validate", context.ReferencesGitRepo(), repo.ValidateIssueConfig) m.Get("/issue_config/validate", context.ReferencesGitRepo(), repo.ValidateIssueConfig)
m.Get("/languages", reqRepoReader(unit.TypeCode), repo.GetLanguages) m.Get("/languages", reqRepoReader(unit.TypeCode), repo.GetLanguages)
m.Get("/licenses", reqRepoReader(unit.TypeCode), repo.GetLicenses)
m.Get("/activities/feeds", repo.ListRepoActivityFeeds) m.Get("/activities/feeds", repo.ListRepoActivityFeeds)
m.Get("/new_pin_allowed", repo.AreNewIssuePinsAllowed) m.Get("/new_pin_allowed", repo.AreNewIssuePinsAllowed)
m.Group("/avatar", func() { m.Group("/avatar", func() {

View File

@ -0,0 +1,51 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package repo
import (
"net/http"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/services/context"
)
// GetLicenses returns licenses
func GetLicenses(ctx *context.APIContext) {
// swagger:operation GET /repos/{owner}/{repo}/licenses repository repoGetLicenses
// ---
// summary: Get repo licenses
// produces:
// - application/json
// parameters:
// - name: owner
// in: path
// description: owner of the repo
// type: string
// required: true
// - name: repo
// in: path
// description: name of the repo
// type: string
// required: true
// responses:
// "404":
// "$ref": "#/responses/notFound"
// "200":
// "$ref": "#/responses/LicensesList"
licenses, err := repo_model.GetRepoLicenses(ctx, ctx.Repo.Repository)
if err != nil {
log.Error("GetRepoLicenses failed: %v", err)
ctx.InternalServerError(err)
return
}
resp := make([]string, len(licenses))
for i := range licenses {
resp[i] = licenses[i].License
}
ctx.JSON(http.StatusOK, resp)
}

View File

@ -731,6 +731,7 @@ func updateBasicProperties(ctx *context.APIContext, opts api.EditRepoOption) err
} }
// Default branch only updated if changed and exist or the repository is empty // Default branch only updated if changed and exist or the repository is empty
updateRepoLicense := false
if opts.DefaultBranch != nil && repo.DefaultBranch != *opts.DefaultBranch && (repo.IsEmpty || ctx.Repo.GitRepo.IsBranchExist(*opts.DefaultBranch)) { if opts.DefaultBranch != nil && repo.DefaultBranch != *opts.DefaultBranch && (repo.IsEmpty || ctx.Repo.GitRepo.IsBranchExist(*opts.DefaultBranch)) {
if !repo.IsEmpty { if !repo.IsEmpty {
if err := gitrepo.SetDefaultBranch(ctx, ctx.Repo.Repository, *opts.DefaultBranch); err != nil { if err := gitrepo.SetDefaultBranch(ctx, ctx.Repo.Repository, *opts.DefaultBranch); err != nil {
@ -739,6 +740,7 @@ func updateBasicProperties(ctx *context.APIContext, opts api.EditRepoOption) err
return err return err
} }
} }
updateRepoLicense = true
} }
repo.DefaultBranch = *opts.DefaultBranch repo.DefaultBranch = *opts.DefaultBranch
} }
@ -748,6 +750,15 @@ func updateBasicProperties(ctx *context.APIContext, opts api.EditRepoOption) err
return err return err
} }
if updateRepoLicense {
if err := repo_service.AddRepoToLicenseUpdaterQueue(&repo_service.LicenseUpdaterOptions{
RepoID: ctx.Repo.Repository.ID,
}); err != nil {
ctx.Error(http.StatusInternalServerError, "AddRepoToLicenseUpdaterQueue", err)
return err
}
}
log.Trace("Repository basic settings updated: %s/%s", owner.Name, repo.Name) log.Trace("Repository basic settings updated: %s/%s", owner.Name, repo.Name)
return nil return nil
} }

View File

@ -359,6 +359,13 @@ type swaggerLanguageStatistics struct {
Body map[string]int64 `json:"body"` Body map[string]int64 `json:"body"`
} }
// LicensesList
// swagger:response LicensesList
type swaggerLicensesList struct {
// in: body
Body []string `json:"body"`
}
// CombinedStatus // CombinedStatus
// swagger:response CombinedStatus // swagger:response CombinedStatus
type swaggerCombinedStatus struct { type swaggerCombinedStatus struct {

View File

@ -172,6 +172,8 @@ func InitWebInstalled(ctx context.Context) {
actions_service.Init() actions_service.Init()
mustInit(repo_service.InitLicenseClassifier)
// Finally start up the cron // Finally start up the cron
cron.NewContext(ctx) cron.NewContext(ctx)
} }

View File

@ -12,6 +12,7 @@ import (
"code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/private"
gitea_context "code.gitea.io/gitea/services/context" gitea_context "code.gitea.io/gitea/services/context"
repo_service "code.gitea.io/gitea/services/repository"
) )
// SetDefaultBranch updates the default branch // SetDefaultBranch updates the default branch
@ -36,5 +37,15 @@ func SetDefaultBranch(ctx *gitea_context.PrivateContext) {
}) })
return return
} }
if err := repo_service.AddRepoToLicenseUpdaterQueue(&repo_service.LicenseUpdaterOptions{
RepoID: ctx.Repo.Repository.ID,
}); err != nil {
ctx.JSON(http.StatusInternalServerError, private.Response{
Err: fmt.Sprintf("Unable to set default branch on repository: %s/%s Error: %v", ownerName, repoName, err),
})
return
}
ctx.PlainText(http.StatusOK, "success") ctx.PlainText(http.StatusOK, "success")
} }

View File

@ -278,11 +278,20 @@ func HookPostReceive(ctx *gitea_context.PrivateContext) {
branch := refFullName.BranchName() branch := refFullName.BranchName()
if branch == baseRepo.DefaultBranch {
if err := repo_service.AddRepoToLicenseUpdaterQueue(&repo_service.LicenseUpdaterOptions{
RepoID: repo.ID,
}); err != nil {
ctx.JSON(http.StatusInternalServerError, private.Response{Err: err.Error()})
return
}
// If our branch is the default branch of an unforked repo - there's no PR to create or refer to // If our branch is the default branch of an unforked repo - there's no PR to create or refer to
if !repo.IsFork && branch == baseRepo.DefaultBranch { if !repo.IsFork {
results = append(results, private.HookPostReceiveBranchResult{}) results = append(results, private.HookPostReceiveBranchResult{})
continue continue
} }
}
pr, err := issues_model.GetUnmergedPullRequest(ctx, repo.ID, baseRepo.ID, branch, baseRepo.DefaultBranch, issues_model.PullRequestFlowGithub) pr, err := issues_model.GetUnmergedPullRequest(ctx, repo.ID, baseRepo.ID, branch, baseRepo.DefaultBranch, issues_model.PullRequestFlowGithub)
if err != nil && !issues_model.IsErrPullRequestNotExist(err) { if err != nil && !issues_model.IsErrPullRequestNotExist(err) {

View File

@ -89,7 +89,7 @@ func Branches(ctx *context.Context) {
pager := context.NewPagination(int(branchesCount), pageSize, page, 5) pager := context.NewPagination(int(branchesCount), pageSize, page, 5)
pager.SetDefaultParams(ctx) pager.SetDefaultParams(ctx)
ctx.Data["Page"] = pager ctx.Data["Page"] = pager
ctx.Data["LicenseFileName"] = repo_service.LicenseFileName
ctx.HTML(http.StatusOK, tplBranch) ctx.HTML(http.StatusOK, tplBranch)
} }

View File

@ -29,7 +29,7 @@ import (
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/services/context" "code.gitea.io/gitea/services/context"
"code.gitea.io/gitea/services/gitdiff" "code.gitea.io/gitea/services/gitdiff"
git_service "code.gitea.io/gitea/services/repository" repo_service "code.gitea.io/gitea/services/repository"
) )
const ( const (
@ -101,7 +101,7 @@ func Commits(ctx *context.Context) {
pager := context.NewPagination(int(commitsCount), pageSize, page, 5) pager := context.NewPagination(int(commitsCount), pageSize, page, 5)
pager.SetDefaultParams(ctx) pager.SetDefaultParams(ctx)
ctx.Data["Page"] = pager ctx.Data["Page"] = pager
ctx.Data["LicenseFileName"] = repo_service.LicenseFileName
ctx.HTML(http.StatusOK, tplCommits) ctx.HTML(http.StatusOK, tplCommits)
} }
@ -218,6 +218,8 @@ func SearchCommits(ctx *context.Context) {
} }
ctx.Data["Username"] = ctx.Repo.Owner.Name ctx.Data["Username"] = ctx.Repo.Owner.Name
ctx.Data["Reponame"] = ctx.Repo.Repository.Name ctx.Data["Reponame"] = ctx.Repo.Repository.Name
ctx.Data["RefName"] = ctx.Repo.RefName
ctx.Data["LicenseFileName"] = repo_service.LicenseFileName
ctx.HTML(http.StatusOK, tplCommits) ctx.HTML(http.StatusOK, tplCommits)
} }
@ -263,12 +265,12 @@ func FileHistory(ctx *context.Context) {
pager := context.NewPagination(int(commitsCount), setting.Git.CommitsRangeSize, page, 5) pager := context.NewPagination(int(commitsCount), setting.Git.CommitsRangeSize, page, 5)
pager.SetDefaultParams(ctx) pager.SetDefaultParams(ctx)
ctx.Data["Page"] = pager ctx.Data["Page"] = pager
ctx.Data["LicenseFileName"] = repo_service.LicenseFileName
ctx.HTML(http.StatusOK, tplCommits) ctx.HTML(http.StatusOK, tplCommits)
} }
func LoadBranchesAndTags(ctx *context.Context) { func LoadBranchesAndTags(ctx *context.Context) {
response, err := git_service.LoadBranchesAndTags(ctx, ctx.Repo, ctx.PathParam("sha")) response, err := repo_service.LoadBranchesAndTags(ctx, ctx.Repo, ctx.PathParam("sha"))
if err == nil { if err == nil {
ctx.JSON(http.StatusOK, response) ctx.JSON(http.StatusOK, response)
return return

View File

@ -289,7 +289,6 @@ func releasesOrTagsFeed(ctx *context.Context, isReleasesOnly bool, formatType st
// SingleRelease renders a single release's page // SingleRelease renders a single release's page
func SingleRelease(ctx *context.Context) { func SingleRelease(ctx *context.Context) {
ctx.Data["PageIsReleaseList"] = true ctx.Data["PageIsReleaseList"] = true
ctx.Data["DefaultBranch"] = ctx.Repo.Repository.DefaultBranch
writeAccess := ctx.Repo.CanWrite(unit.TypeReleases) writeAccess := ctx.Repo.CanWrite(unit.TypeReleases)
ctx.Data["CanCreateRelease"] = writeAccess && !ctx.Repo.Repository.IsArchived ctx.Data["CanCreateRelease"] = writeAccess && !ctx.Repo.Repository.IsArchived

View File

@ -51,6 +51,7 @@ import (
"code.gitea.io/gitea/routers/web/feed" "code.gitea.io/gitea/routers/web/feed"
"code.gitea.io/gitea/services/context" "code.gitea.io/gitea/services/context"
issue_service "code.gitea.io/gitea/services/issue" issue_service "code.gitea.io/gitea/services/issue"
repo_service "code.gitea.io/gitea/services/repository"
files_service "code.gitea.io/gitea/services/repository/files" files_service "code.gitea.io/gitea/services/repository/files"
"github.com/nektos/act/pkg/model" "github.com/nektos/act/pkg/model"
@ -1077,6 +1078,7 @@ func renderHomeCode(ctx *context.Context) {
ctx.Data["TreeLink"] = treeLink ctx.Data["TreeLink"] = treeLink
ctx.Data["TreeNames"] = treeNames ctx.Data["TreeNames"] = treeNames
ctx.Data["BranchLink"] = branchLink ctx.Data["BranchLink"] = branchLink
ctx.Data["LicenseFileName"] = repo_service.LicenseFileName
ctx.HTML(http.StatusOK, tplRepoHome) ctx.HTML(http.StatusOK, tplRepoHome)
} }

View File

@ -404,6 +404,13 @@ func repoAssignment(ctx *Context, repo *repo_model.Repository) {
ctx.Data["PushMirrors"] = pushMirrors ctx.Data["PushMirrors"] = pushMirrors
ctx.Data["RepoName"] = ctx.Repo.Repository.Name ctx.Data["RepoName"] = ctx.Repo.Repository.Name
ctx.Data["IsEmptyRepo"] = ctx.Repo.Repository.IsEmpty ctx.Data["IsEmptyRepo"] = ctx.Repo.Repository.IsEmpty
repoLicenses, err := repo_model.GetRepoLicenses(ctx, ctx.Repo.Repository)
if err != nil {
ctx.ServerError("GetRepoLicenses", err)
return
}
ctx.Data["DetectedRepoLicenses"] = repoLicenses.StringList()
} }
// RepoAssignment returns a middleware to handle repository assignment // RepoAssignment returns a middleware to handle repository assignment

View File

@ -175,6 +175,11 @@ func innerToRepo(ctx context.Context, repo *repo_model.Repository, permissionInR
language = repo.PrimaryLanguage.Language language = repo.PrimaryLanguage.Language
} }
repoLicenses, err := repo_model.GetRepoLicenses(ctx, repo)
if err != nil {
return nil
}
repoAPIURL := repo.APIURL() repoAPIURL := repo.APIURL()
return &api.Repository{ return &api.Repository{
@ -238,6 +243,7 @@ func innerToRepo(ctx context.Context, repo *repo_model.Repository, permissionInR
RepoTransfer: transfer, RepoTransfer: transfer,
Topics: repo.Topics, Topics: repo.Topics,
ObjectFormatName: repo.ObjectFormatName, ObjectFormatName: repo.ObjectFormatName,
Licenses: repoLicenses.StringList(),
} }
} }

View File

@ -156,6 +156,16 @@ func registerCleanupPackages() {
}) })
} }
func registerSyncRepoLicenses() {
RegisterTaskFatal("sync_repo_licenses", &BaseConfig{
Enabled: false,
RunAtStart: false,
Schedule: "@annually",
}, func(ctx context.Context, _ *user_model.User, config Config) error {
return repo_service.SyncRepoLicenses(ctx)
})
}
func initBasicTasks() { func initBasicTasks() {
if setting.Mirror.Enabled { if setting.Mirror.Enabled {
registerUpdateMirrorTask() registerUpdateMirrorTask()
@ -172,4 +182,5 @@ func initBasicTasks() {
if setting.Packages.Enabled { if setting.Packages.Enabled {
registerCleanupPackages() registerCleanupPackages()
} }
registerSyncRepoLicenses()
} }

View File

@ -26,6 +26,7 @@ import (
"code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/optional"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/modules/test"
repo_service "code.gitea.io/gitea/services/repository"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
@ -302,6 +303,8 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) {
toRepoName := "migrated" toRepoName := "migrated"
uploader := NewGiteaLocalUploader(context.Background(), fromRepoOwner, fromRepoOwner.Name, toRepoName) uploader := NewGiteaLocalUploader(context.Background(), fromRepoOwner, fromRepoOwner.Name, toRepoName)
uploader.gitServiceType = structs.GiteaService uploader.gitServiceType = structs.GiteaService
assert.NoError(t, repo_service.Init(context.Background()))
assert.NoError(t, uploader.CreateRepo(&base.Repository{ assert.NoError(t, uploader.CreateRepo(&base.Repository{
Description: "description", Description: "description",
OriginalURL: fromRepo.RepoPath(), OriginalURL: fromRepo.RepoPath(),

View File

@ -24,6 +24,7 @@ import (
"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/util"
notify_service "code.gitea.io/gitea/services/notify" notify_service "code.gitea.io/gitea/services/notify"
repo_service "code.gitea.io/gitea/services/repository"
) )
// gitShortEmptySha Git short empty SHA // gitShortEmptySha Git short empty SHA
@ -559,6 +560,14 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
} }
} }
// Update License
if err = repo_service.AddRepoToLicenseUpdaterQueue(&repo_service.LicenseUpdaterOptions{
RepoID: m.Repo.ID,
}); err != nil {
log.Error("SyncMirrors [repo: %-v]: unable to add repo to license updater queue: %v", m.Repo, err)
return false
}
log.Trace("SyncMirrors [repo: %-v]: Successfully updated", m.Repo) log.Trace("SyncMirrors [repo: %-v]: Successfully updated", m.Repo)
return true return true

View File

@ -612,6 +612,14 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, gitR
return err return err
} }
if !repo.IsEmpty {
if err := AddRepoToLicenseUpdaterQueue(&LicenseUpdaterOptions{
RepoID: repo.ID,
}); err != nil {
log.Error("AddRepoToLicenseUpdaterQueue: %v", err)
}
}
notify_service.ChangeDefaultBranch(ctx, repo) notify_service.ChangeDefaultBranch(ctx, repo)
return nil return nil

View File

@ -303,6 +303,25 @@ func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opt
rollbackRepo.OwnerID = u.ID rollbackRepo.OwnerID = u.ID
return fmt.Errorf("CreateRepository(git update-server-info): %w", err) return fmt.Errorf("CreateRepository(git update-server-info): %w", err)
} }
// update licenses
var licenses []string
if len(opts.License) > 0 {
licenses = append(licenses, ConvertLicenseName(opts.License))
stdout, _, err := git.NewCommand(ctx, "rev-parse", "HEAD").
SetDescription(fmt.Sprintf("CreateRepository(git rev-parse HEAD): %s", repoPath)).
RunStdString(&git.RunOpts{Dir: repoPath})
if err != nil {
log.Error("CreateRepository(git rev-parse HEAD) in %v: Stdout: %s\nError: %v", repo, stdout, err)
rollbackRepo = repo
rollbackRepo.OwnerID = u.ID
return fmt.Errorf("CreateRepository(git rev-parse HEAD): %w", err)
}
if err := repo_model.UpdateRepoLicenses(ctx, repo, stdout, licenses); err != nil {
return err
}
}
return nil return nil
}); err != nil { }); err != nil {
if rollbackRepo != nil { if rollbackRepo != nil {

View File

@ -140,6 +140,7 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID
&git_model.Branch{RepoID: repoID}, &git_model.Branch{RepoID: repoID},
&git_model.LFSLock{RepoID: repoID}, &git_model.LFSLock{RepoID: repoID},
&repo_model.LanguageStat{RepoID: repoID}, &repo_model.LanguageStat{RepoID: repoID},
&repo_model.RepoLicense{RepoID: repoID},
&issues_model.Milestone{RepoID: repoID}, &issues_model.Milestone{RepoID: repoID},
&repo_model.Mirror{RepoID: repoID}, &repo_model.Mirror{RepoID: repoID},
&activities_model.Notification{RepoID: repoID}, &activities_model.Notification{RepoID: repoID},

View File

@ -198,6 +198,9 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork
if err := repo_model.CopyLanguageStat(ctx, opts.BaseRepo, repo); err != nil { if err := repo_model.CopyLanguageStat(ctx, opts.BaseRepo, repo); err != nil {
log.Error("Copy language stat from oldRepo failed: %v", err) log.Error("Copy language stat from oldRepo failed: %v", err)
} }
if err := repo_model.CopyLicense(ctx, opts.BaseRepo, repo); err != nil {
return nil, err
}
gitRepo, err := gitrepo.OpenRepository(ctx, repo) gitRepo, err := gitrepo.OpenRepository(ctx, repo)
if err != nil { if err != nil {

View File

@ -0,0 +1,205 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package repository
import (
"context"
"fmt"
"io"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/queue"
licenseclassifier "github.com/google/licenseclassifier/v2"
)
var (
classifier *licenseclassifier.Classifier
LicenseFileName = "LICENSE"
licenseAliases map[string]string
// licenseUpdaterQueue represents a queue to handle update repo licenses
licenseUpdaterQueue *queue.WorkerPoolQueue[*LicenseUpdaterOptions]
)
func AddRepoToLicenseUpdaterQueue(opts *LicenseUpdaterOptions) error {
if opts == nil {
return nil
}
return licenseUpdaterQueue.Push(opts)
}
func loadLicenseAliases() error {
if licenseAliases != nil {
return nil
}
data, err := options.AssetFS().ReadFile("license", "etc", "license-aliases.json")
if err != nil {
return err
}
err = json.Unmarshal(data, &licenseAliases)
if err != nil {
return err
}
return nil
}
func ConvertLicenseName(name string) string {
if err := loadLicenseAliases(); err != nil {
return name
}
v, ok := licenseAliases[name]
if ok {
return v
}
return name
}
func InitLicenseClassifier() error {
// threshold should be 0.84~0.86 or the test will be failed
classifier = licenseclassifier.NewClassifier(.85)
licenseFiles, err := options.AssetFS().ListFiles("license", true)
if err != nil {
return err
}
existLicense := make(container.Set[string])
if len(licenseFiles) > 0 {
for _, licenseFile := range licenseFiles {
licenseName := ConvertLicenseName(licenseFile)
if existLicense.Contains(licenseName) {
continue
}
existLicense.Add(licenseName)
data, err := options.License(licenseFile)
if err != nil {
return err
}
classifier.AddContent("License", licenseFile, licenseName, data)
}
}
return nil
}
type LicenseUpdaterOptions struct {
RepoID int64
}
func repoLicenseUpdater(items ...*LicenseUpdaterOptions) []*LicenseUpdaterOptions {
ctx := graceful.GetManager().ShutdownContext()
for _, opts := range items {
repo, err := repo_model.GetRepositoryByID(ctx, opts.RepoID)
if err != nil {
log.Error("repoLicenseUpdater [%d] failed: GetRepositoryByID: %v", opts.RepoID, err)
continue
}
if repo.IsEmpty {
continue
}
gitRepo, err := gitrepo.OpenRepository(ctx, repo)
if err != nil {
log.Error("repoLicenseUpdater [%d] failed: OpenRepository: %v", opts.RepoID, err)
continue
}
defer gitRepo.Close()
commit, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
if err != nil {
log.Error("repoLicenseUpdater [%d] failed: GetBranchCommit: %v", opts.RepoID, err)
continue
}
if err = UpdateRepoLicenses(ctx, repo, commit); err != nil {
log.Error("repoLicenseUpdater [%d] failed: updateRepoLicenses: %v", opts.RepoID, err)
}
}
return nil
}
func SyncRepoLicenses(ctx context.Context) error {
log.Trace("Doing: SyncRepoLicenses")
if err := db.Iterate(
ctx,
nil,
func(ctx context.Context, repo *repo_model.Repository) error {
select {
case <-ctx.Done():
return db.ErrCancelledf("before sync repo licenses for %s", repo.FullName())
default:
}
return AddRepoToLicenseUpdaterQueue(&LicenseUpdaterOptions{RepoID: repo.ID})
},
); err != nil {
log.Trace("Error: SyncRepoLicenses: %v", err)
return err
}
log.Trace("Finished: SyncReposLicenses")
return nil
}
// UpdateRepoLicenses will update repository licenses col if license file exists
func UpdateRepoLicenses(ctx context.Context, repo *repo_model.Repository, commit *git.Commit) error {
if commit == nil {
return nil
}
b, err := commit.GetBlobByPath(LicenseFileName)
if err != nil && !git.IsErrNotExist(err) {
return fmt.Errorf("GetBlobByPath: %w", err)
}
if git.IsErrNotExist(err) {
return repo_model.CleanRepoLicenses(ctx, repo)
}
licenses := make([]string, 0)
if b != nil {
r, err := b.DataAsync()
if err != nil {
return err
}
defer r.Close()
licenses, err = detectLicense(r)
if err != nil {
return fmt.Errorf("detectLicense: %w", err)
}
}
return repo_model.UpdateRepoLicenses(ctx, repo, commit.ID.String(), licenses)
}
// detectLicense returns the licenses detected by the given content buff
func detectLicense(r io.Reader) ([]string, error) {
if r == nil {
return nil, nil
}
matches, err := classifier.MatchFrom(r)
if err != nil {
return nil, err
}
if len(matches.Matches) > 0 {
results := make(container.Set[string], len(matches.Matches))
for _, r := range matches.Matches {
if r.MatchType == "License" && !results.Contains(r.Variant) {
results.Add(r.Variant)
}
}
return results.Values(), nil
}
return nil, nil
}

View File

@ -0,0 +1,73 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package repository
import (
"fmt"
"strings"
"testing"
repo_module "code.gitea.io/gitea/modules/repository"
"github.com/stretchr/testify/assert"
)
func Test_detectLicense(t *testing.T) {
type DetectLicenseTest struct {
name string
arg string
want []string
}
tests := []DetectLicenseTest{
{
name: "empty",
arg: "",
want: nil,
},
{
name: "no detected license",
arg: "Copyright (c) 2023 Gitea",
want: nil,
},
}
repo_module.LoadRepoConfig()
err := loadLicenseAliases()
assert.NoError(t, err)
for _, licenseName := range repo_module.Licenses {
license, err := repo_module.GetLicense(licenseName, &repo_module.LicenseValues{
Owner: "Gitea",
Email: "teabot@gitea.io",
Repo: "gitea",
Year: "2024",
})
assert.NoError(t, err)
tests = append(tests, DetectLicenseTest{
name: fmt.Sprintf("single license test: %s", licenseName),
arg: string(license),
want: []string{ConvertLicenseName(licenseName)},
})
}
err = InitLicenseClassifier()
assert.NoError(t, err)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
license, err := detectLicense(strings.NewReader(tt.arg))
assert.NoError(t, err)
assert.Equal(t, tt.want, license)
})
}
result, err := detectLicense(strings.NewReader(tests[2].arg + tests[3].arg + tests[4].arg))
assert.NoError(t, err)
t.Run("multiple licenses test", func(t *testing.T) {
assert.Equal(t, 3, len(result))
assert.Contains(t, result, tests[2].want[0])
assert.Contains(t, result, tests[3].want[0])
assert.Contains(t, result, tests[4].want[0])
})
}

View File

@ -172,6 +172,11 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
return repo, fmt.Errorf("StoreMissingLfsObjectsInRepository: %w", err) return repo, fmt.Errorf("StoreMissingLfsObjectsInRepository: %w", err)
} }
} }
// Update repo license
if err := AddRepoToLicenseUpdaterQueue(&LicenseUpdaterOptions{RepoID: repo.ID}); err != nil {
log.Error("Failed to add repo to license updater queue: %v", err)
}
} }
ctx, committer, err := db.TxContext(ctx) ctx, committer, err := db.TxContext(ctx)

View File

@ -18,6 +18,7 @@ import (
user_model "code.gitea.io/gitea/models/user" user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/queue"
repo_module "code.gitea.io/gitea/modules/repository" repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
@ -96,6 +97,12 @@ func PushCreateRepo(ctx context.Context, authUser, owner *user_model.User, repoN
// Init start repository service // Init start repository service
func Init(ctx context.Context) error { func Init(ctx context.Context) error {
licenseUpdaterQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "repo_license_updater", repoLicenseUpdater)
if licenseUpdaterQueue == nil {
return fmt.Errorf("unable to create repo_license_updater queue")
}
go graceful.GetManager().RunWithCancel(licenseUpdaterQueue)
if err := repo_module.LoadRepoConfig(); err != nil { if err := repo_module.LoadRepoConfig(); err != nil {
return err return err
} }

View File

@ -13,7 +13,12 @@
{{svg "octicon-tag"}} <b>{{ctx.Locale.PrettyNumber .NumTags}}</b> {{ctx.Locale.TrN .NumTags "repo.tag" "repo.tags"}} {{svg "octicon-tag"}} <b>{{ctx.Locale.PrettyNumber .NumTags}}</b> {{ctx.Locale.TrN .NumTags "repo.tag" "repo.tags"}}
</a> </a>
{{end}} {{end}}
<span class="item not-mobile" {{if not (eq .Repository.Size 0)}}data-tooltip-content="{{.Repository.SizeDetailsString}}"{{end}}> {{if .DetectedRepoLicenses}}
<a class="item muted" href="{{.RepoLink}}/src/{{.Repository.DefaultBranch}}/{{PathEscapeSegments .LicenseFileName}}" data-tooltip-placement="top" data-tooltip-content="{{StringUtils.Join .DetectedRepoLicenses ", "}}">
{{svg "octicon-law"}} <b>{{if eq (len .DetectedRepoLicenses) 1}}{{index .DetectedRepoLicenses 0}}{{else}}{{ctx.Locale.Tr "repo.multiple_licenses"}}{{end}}</b>
</a>
{{end}}
<span class="item not-mobile" {{if not (eq .Repository.Size 0)}}data-tooltip-placement="top" data-tooltip-content="{{.Repository.SizeDetailsString}}"{{end}}>
{{$fileSizeFormatted := FileSize .Repository.Size}}{{/* the formatted string is always "{val} {unit}" */}} {{$fileSizeFormatted := FileSize .Repository.Size}}{{/* the formatted string is always "{val} {unit}" */}}
{{$fileSizeFields := StringUtils.Split $fileSizeFormatted " "}} {{$fileSizeFields := StringUtils.Split $fileSizeFormatted " "}}
{{svg "octicon-database"}} <b>{{ctx.Locale.PrettyNumber (index $fileSizeFields 0)}}</b> {{index $fileSizeFields 1}} {{svg "octicon-database"}} <b>{{ctx.Locale.PrettyNumber (index $fileSizeFields 0)}}</b> {{index $fileSizeFields 1}}

View File

@ -10640,6 +10640,42 @@
} }
} }
}, },
"/repos/{owner}/{repo}/licenses": {
"get": {
"produces": [
"application/json"
],
"tags": [
"repository"
],
"summary": "Get repo licenses",
"operationId": "repoGetLicenses",
"parameters": [
{
"type": "string",
"description": "owner of the repo",
"name": "owner",
"in": "path",
"required": true
},
{
"type": "string",
"description": "name of the repo",
"name": "repo",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"$ref": "#/responses/LicensesList"
},
"404": {
"$ref": "#/responses/notFound"
}
}
}
},
"/repos/{owner}/{repo}/media/{filepath}": { "/repos/{owner}/{repo}/media/{filepath}": {
"get": { "get": {
"produces": [ "produces": [
@ -24142,6 +24178,13 @@
"type": "string", "type": "string",
"x-go-name": "LanguagesURL" "x-go-name": "LanguagesURL"
}, },
"licenses": {
"type": "array",
"items": {
"type": "string"
},
"x-go-name": "Licenses"
},
"link": { "link": {
"type": "string", "type": "string",
"x-go-name": "Link" "x-go-name": "Link"
@ -25717,6 +25760,15 @@
} }
} }
}, },
"LicensesList": {
"description": "LicensesList",
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
"MarkdownRender": { "MarkdownRender": {
"description": "MarkdownRender is a rendered markdown document", "description": "MarkdownRender is a rendered markdown document",
"schema": { "schema": {

View File

@ -0,0 +1,2 @@
x•ŽM
Â0F]çÙ 2“d&ñ*ù™`Á¶¶Æ…··Wð[>Þƒ¯®ó< ëÐ<C3AB>Æ®j!Æ&=Õ *Êž1*¢@””TS*X3WÞu©cé.êK<C3AA>Ð90E¦Äž”$h+µ„fòg<ÖÝ~_@ÞE{=,! €÷m»Ôu¾YŒ Ž„B´g8fz|ú_erkö9U]Þj~2]<¼

View File

@ -1 +1 @@
65f1bf27bc3bf70f64657658635e66094edbcb4d 90c1019714259b24fb81711d4416ac0f18667dfa

View File

@ -304,11 +304,11 @@ func TestAPICron(t *testing.T) {
AddTokenAuth(token) AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK) resp := MakeRequest(t, req, http.StatusOK)
assert.Equal(t, "28", resp.Header().Get("X-Total-Count")) assert.Equal(t, "29", resp.Header().Get("X-Total-Count"))
var crons []api.Cron var crons []api.Cron
DecodeJSON(t, resp, &crons) DecodeJSON(t, resp, &crons)
assert.Len(t, crons, 28) assert.Len(t, crons, 29)
}) })
t.Run("Execute", func(t *testing.T) { t.Run("Execute", func(t *testing.T) {

View File

@ -0,0 +1,80 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package integration
import (
"fmt"
"net/http"
"net/url"
"testing"
"time"
auth_model "code.gitea.io/gitea/models/auth"
api "code.gitea.io/gitea/modules/structs"
"github.com/stretchr/testify/assert"
)
var testLicenseContent = `
Copyright (c) 2024 Gitea
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
`
func TestAPIRepoLicense(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
session := loginUser(t, "user2")
// Request editor page
req := NewRequest(t, "GET", "/user2/repo1/_new/master/")
resp := session.MakeRequest(t, req, http.StatusOK)
doc := NewHTMLParser(t, resp.Body)
lastCommit := doc.GetInputValueByName("last_commit")
assert.NotEmpty(t, lastCommit)
// Save new file to master branch
req = NewRequestWithValues(t, "POST", "/user2/repo1/_new/master/", map[string]string{
"_csrf": doc.GetCSRF(),
"last_commit": lastCommit,
"tree_path": "LICENSE",
"content": testLicenseContent,
"commit_choice": "direct",
})
session.MakeRequest(t, req, http.StatusSeeOther)
// let gitea update repo license
time.Sleep(time.Second)
checkRepoLicense(t, "user2", "repo1", []string{"BSD-2-Clause"})
// Change default branch
token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository)
branchName := "DefaultBranch"
req = NewRequestWithJSON(t, "PATCH", "/api/v1/repos/user2/repo1", api.EditRepoOption{
DefaultBranch: &branchName,
}).AddTokenAuth(token)
session.MakeRequest(t, req, http.StatusOK)
// let gitea update repo license
time.Sleep(time.Second)
checkRepoLicense(t, "user2", "repo1", []string{"MIT"})
})
}
func checkRepoLicense(t *testing.T, owner, repo string, expected []string) {
reqURL := fmt.Sprintf("/api/v1/repos/%s/%s/licenses", owner, repo)
req := NewRequest(t, "GET", reqURL)
resp := MakeRequest(t, req, http.StatusOK)
var licenses []string
DecodeJSON(t, resp, &licenses)
assert.ElementsMatch(t, expected, licenses, 0)
}