From 63d7cbcef13b4eb719a5749a1a79361461c77135 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 7 Sep 2021 17:49:36 +0200 Subject: [PATCH] Make mirror feature more configurable (#16957) Rename`[repository]` `DISABLE_MIRRORS` to `[mirror]` `DISABLE_NEW_PULL` and add `ENABLED` and `DISABLE_NEW_PUSH` with the below meanings: - `ENABLED`: **true**: Enables the mirror functionality. Set to **false** to disable all mirrors. - `DISABLE_NEW_PULL`: **false**: Disable the creation of **new** mirrors. Pre-existing mirrors remain valid. - `DISABLE_NEW_PUSH`: **false**: Disable the creation of **new** push mirrors. Pre-existing mirrors remain valid. Co-authored-by: zeripath Co-authored-by: delvh --- custom/conf/app.example.ini | 9 ++- .../doc/advanced/config-cheat-sheet.en-us.md | 4 +- integrations/api_settings_test.go | 2 +- modules/setting/mirror.go | 59 +++++++++++++++ modules/setting/repository.go | 2 - modules/setting/setting.go | 18 +---- routers/api/v1/repo/migrate.go | 4 +- routers/api/v1/repo/mirror.go | 6 ++ routers/api/v1/settings/settings.go | 2 +- routers/web/org/home.go | 2 +- routers/web/repo/migrate.go | 15 ++-- routers/web/repo/setting.go | 23 +++++- services/mirror/mirror.go | 13 ++++ templates/org/home.tmpl | 2 +- templates/repo/migrate/options.tmpl | 2 +- templates/repo/settings/options.tmpl | 74 ++++++++++--------- 16 files changed, 162 insertions(+), 75 deletions(-) create mode 100644 modules/setting/mirror.go diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 40d5742eff..2637e0ec7a 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -800,9 +800,6 @@ PATH = ;; Prefix archive files by placing them in a directory named after the repository ;PREFIX_ARCHIVE_FILES = true ;; -;; Disable the creation of new mirrors. Pre-existing mirrors remain valid. -;DISABLE_MIRRORS = false -;; ;; Disable migrating feature. ;DISABLE_MIGRATIONS = false ;; @@ -1945,6 +1942,12 @@ PATH = ;[mirror] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Enables the mirror functionality. Set to **false** to disable all mirrors. +;ENABLED = true +;; Disable the creation of **new** pull mirrors. Pre-existing mirrors remain valid. Will be ignored if `mirror.ENABLED` is `false`. +;DISABLE_NEW_PULL = false +;; Disable the creation of **new** push mirrors. Pre-existing mirrors remain valid. Will be ignored if `mirror.ENABLED` is `false`. +;DISABLE_NEW_PUSH = false ;; Default interval as a duration between each check ;DEFAULT_INTERVAL = 8h ;; Min interval as a duration must be > 1m diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 242189f5b0..c63932cdc7 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -73,7 +73,6 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`. - `DISABLED_REPO_UNITS`: **_empty_**: Comma separated list of globally disabled repo units. Allowed values: \[repo.issues, repo.ext_issues, repo.pulls, repo.wiki, repo.ext_wiki, repo.projects\] - `DEFAULT_REPO_UNITS`: **repo.code,repo.releases,repo.issues,repo.pulls,repo.wiki,repo.projects**: Comma separated list of default repo units. Allowed values: \[repo.code, repo.releases, repo.issues, repo.pulls, repo.wiki, repo.projects\]. Note: Code and Releases can currently not be deactivated. If you specify default repo units you should still list them for future compatibility. External wiki and issue tracker can't be enabled by default as it requires additional settings. Disabled repo units will not be added to new repositories regardless if it is in the default list. - `PREFIX_ARCHIVE_FILES`: **true**: Prefix archive files by placing them in a directory named after the repository. -- `DISABLE_MIRRORS`: **false**: Disable the creation of **new** mirrors. Pre-existing mirrors remain valid. - `DISABLE_MIGRATIONS`: **false**: Disable migrating feature. - `DISABLE_STARS`: **false**: Disable stars feature. - `DEFAULT_BRANCH`: **master**: Default branch name of all repositories. @@ -955,6 +954,9 @@ Task queue configuration has been moved to `queue.task`. However, the below conf ## Mirror (`mirror`) +- `ENABLED`: **true**: Enables the mirror functionality. Set to **false** to disable all mirrors. +- `DISABLE_NEW_PULL`: **false**: Disable the creation of **new** pull mirrors. Pre-existing mirrors remain valid. Will be ignored if `mirror.ENABLED` is `false`. +- `DISABLE_NEW_PUSH`: **false**: Disable the creation of **new** push mirrors. Pre-existing mirrors remain valid. Will be ignored if `mirror.ENABLED` is `false`. - `DEFAULT_INTERVAL`: **8h**: Default interval between each check - `MIN_INTERVAL`: **10m**: Minimum interval for checking. (Must be >1m). diff --git a/integrations/api_settings_test.go b/integrations/api_settings_test.go index fdd8b02d62..10318cec7f 100644 --- a/integrations/api_settings_test.go +++ b/integrations/api_settings_test.go @@ -43,7 +43,7 @@ func TestAPIExposedSettings(t *testing.T) { DecodeJSON(t, resp, &repo) assert.EqualValues(t, &api.GeneralRepoSettings{ - MirrorsDisabled: setting.Repository.DisableMirrors, + MirrorsDisabled: !setting.Mirror.Enabled, HTTPGitDisabled: setting.Repository.DisableHTTPGit, MigrationsDisabled: setting.Repository.DisableMigrations, TimeTrackingDisabled: false, diff --git a/modules/setting/mirror.go b/modules/setting/mirror.go new file mode 100644 index 0000000000..b0d5b2a4dc --- /dev/null +++ b/modules/setting/mirror.go @@ -0,0 +1,59 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "time" + + "code.gitea.io/gitea/modules/log" +) + +var ( + // Mirror settings + Mirror = struct { + Enabled bool + DisableNewPull bool + DisableNewPush bool + DefaultInterval time.Duration + MinInterval time.Duration + }{ + Enabled: true, + DisableNewPull: false, + DisableNewPush: false, + MinInterval: 10 * time.Minute, + DefaultInterval: 8 * time.Hour, + } +) + +func newMirror() { + // Handle old configuration through `[repository]` `DISABLE_MIRRORS` + // - please note this was badly named and only disabled the creation of new pull mirrors + if Cfg.Section("repository").Key("DISABLE_MIRRORS").MustBool(false) { + log.Warn("Deprecated DISABLE_MIRRORS config is used, please change your config and use the options within the [mirror] section") + // TODO: enable on v1.17.0: log.Error("Deprecated fallback used, will be removed in v1.18.0") + Mirror.DisableNewPull = true + } + if err := Cfg.Section("mirror").MapTo(&Mirror); err != nil { + log.Fatal("Failed to map Mirror settings: %v", err) + } + + if !Mirror.Enabled { + Mirror.DisableNewPull = true + Mirror.DisableNewPush = true + } + + if Mirror.MinInterval.Minutes() < 1 { + log.Warn("Mirror.MinInterval is too low, set to 1 minute") + Mirror.MinInterval = 1 * time.Minute + } + if Mirror.DefaultInterval < Mirror.MinInterval { + if time.Hour*8 < Mirror.MinInterval { + Mirror.DefaultInterval = Mirror.MinInterval + } else { + Mirror.DefaultInterval = time.Hour * 8 + } + log.Warn("Mirror.DefaultInterval is less than Mirror.MinInterval, set to %s", Mirror.DefaultInterval.String()) + } +} diff --git a/modules/setting/repository.go b/modules/setting/repository.go index f981809427..de57eb9140 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -41,7 +41,6 @@ var ( DisabledRepoUnits []string DefaultRepoUnits []string PrefixArchiveFiles bool - DisableMirrors bool DisableMigrations bool DisableStars bool `ini:"DISABLE_STARS"` DefaultBranch string @@ -155,7 +154,6 @@ var ( DisabledRepoUnits: []string{}, DefaultRepoUnits: []string{}, PrefixArchiveFiles: true, - DisableMirrors: false, DisableMigrations: false, DisableStars: false, DefaultBranch: "master", diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 441bceda20..ee8934a24c 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -348,12 +348,6 @@ var ( ManifestData string - // Mirror settings - Mirror struct { - DefaultInterval time.Duration - MinInterval time.Duration - } - // API settings API = struct { EnableSwagger bool @@ -938,17 +932,7 @@ func NewContext() { newGit() - sec = Cfg.Section("mirror") - Mirror.MinInterval = sec.Key("MIN_INTERVAL").MustDuration(10 * time.Minute) - Mirror.DefaultInterval = sec.Key("DEFAULT_INTERVAL").MustDuration(8 * time.Hour) - if Mirror.MinInterval.Minutes() < 1 { - log.Warn("Mirror.MinInterval is too low") - Mirror.MinInterval = 1 * time.Minute - } - if Mirror.DefaultInterval < Mirror.MinInterval { - log.Warn("Mirror.DefaultInterval is less than Mirror.MinInterval") - Mirror.DefaultInterval = time.Hour * 8 - } + newMirror() Langs = Cfg.Section("i18n").Key("LANGS").Strings(",") if len(Langs) == 0 { diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index de33a3645b..87ceb547c6 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -108,8 +108,8 @@ func Migrate(ctx *context.APIContext) { gitServiceType := convert.ToGitServiceType(form.Service) - if form.Mirror && setting.Repository.DisableMirrors { - ctx.Error(http.StatusForbidden, "MirrorsGlobalDisabled", fmt.Errorf("the site administrator has disabled mirrors")) + if form.Mirror && setting.Mirror.DisableNewPull { + ctx.Error(http.StatusForbidden, "MirrorsGlobalDisabled", fmt.Errorf("the site administrator has disabled the creation of new pull mirrors")) return } diff --git a/routers/api/v1/repo/mirror.go b/routers/api/v1/repo/mirror.go index 82da680e55..67d0c7ee1c 100644 --- a/routers/api/v1/repo/mirror.go +++ b/routers/api/v1/repo/mirror.go @@ -9,6 +9,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" mirror_service "code.gitea.io/gitea/services/mirror" ) @@ -42,6 +43,11 @@ func MirrorSync(ctx *context.APIContext) { ctx.Error(http.StatusForbidden, "MirrorSync", "Must have write access") } + if !setting.Mirror.Enabled { + ctx.Error(http.StatusBadRequest, "MirrorSync", "Mirror feature is disabled") + return + } + mirror_service.StartToMirror(repo.ID) ctx.Status(http.StatusOK) diff --git a/routers/api/v1/settings/settings.go b/routers/api/v1/settings/settings.go index ca2d28fb8b..36f93893a8 100644 --- a/routers/api/v1/settings/settings.go +++ b/routers/api/v1/settings/settings.go @@ -58,7 +58,7 @@ func GetGeneralRepoSettings(ctx *context.APIContext) { // "200": // "$ref": "#/responses/GeneralRepoSettings" ctx.JSON(http.StatusOK, api.GeneralRepoSettings{ - MirrorsDisabled: setting.Repository.DisableMirrors, + MirrorsDisabled: !setting.Mirror.Enabled, HTTPGitDisabled: setting.Repository.DisableHTTPGit, MigrationsDisabled: setting.Repository.DisableMigrations, StarsDisabled: setting.Repository.DisableStars, diff --git a/routers/web/org/home.go b/routers/web/org/home.go index 0c1f381d30..f682dc5cb6 100644 --- a/routers/web/org/home.go +++ b/routers/web/org/home.go @@ -141,7 +141,7 @@ func Home(ctx *context.Context) { ctx.Data["Members"] = members ctx.Data["Teams"] = org.Teams - ctx.Data["DisabledMirrors"] = setting.Repository.DisableMirrors + ctx.Data["DisableNewPullMirrors"] = setting.Mirror.DisableNewPull pager := context.NewPagination(int(count), setting.UI.User.RepoPagingNum, page, 5) pager.SetDefaultParams(ctx) diff --git a/routers/web/repo/migrate.go b/routers/web/repo/migrate.go index 3d710d0448..b2e6fa890b 100644 --- a/routers/web/repo/migrate.go +++ b/routers/web/repo/migrate.go @@ -152,9 +152,12 @@ func MigratePost(ctx *context.Context) { return } - serviceType := structs.GitServiceType(form.Service) + if form.Mirror && setting.Mirror.DisableNewPull { + ctx.Error(http.StatusBadRequest, "MigratePost: the site administrator has disabled creation of new mirrors") + return + } - setMigrationContextData(ctx, serviceType) + setMigrationContextData(ctx, form.Service) ctxUser := checkContextUser(ctx, form.UID) if ctx.Written() { @@ -162,7 +165,7 @@ func MigratePost(ctx *context.Context) { } ctx.Data["ContextUser"] = ctxUser - tpl := base.TplName("repo/migrate/" + serviceType.Name()) + tpl := base.TplName("repo/migrate/" + form.Service.Name()) if ctx.HasError() { ctx.HTML(http.StatusOK, tpl) @@ -198,12 +201,12 @@ func MigratePost(ctx *context.Context) { var opts = migrations.MigrateOptions{ OriginalURL: form.CloneAddr, - GitServiceType: serviceType, + GitServiceType: form.Service, CloneAddr: remoteAddr, RepoName: form.RepoName, Description: form.Description, Private: form.Private || setting.Repository.ForcePrivate, - Mirror: form.Mirror && !setting.Repository.DisableMirrors, + Mirror: form.Mirror, LFS: form.LFS, LFSEndpoint: form.LFSEndpoint, AuthUsername: form.AuthUsername, @@ -246,7 +249,7 @@ func setMigrationContextData(ctx *context.Context, serviceType structs.GitServic ctx.Data["LFSActive"] = setting.LFS.StartServer ctx.Data["IsForcedPrivate"] = setting.Repository.ForcePrivate - ctx.Data["DisableMirrors"] = setting.Repository.DisableMirrors + ctx.Data["DisableNewPullMirrors"] = setting.Mirror.DisableNewPull // Plain git should be first ctx.Data["Services"] = append([]structs.GitServiceType{structs.PlainGitService}, structs.SupportedFullGitService...) diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 136e08cb47..624c01814e 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -52,7 +52,8 @@ func Settings(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("repo.settings") ctx.Data["PageIsSettingsOptions"] = true ctx.Data["ForcePrivate"] = setting.Repository.ForcePrivate - ctx.Data["DisabledMirrors"] = setting.Repository.DisableMirrors + ctx.Data["MirrorsEnabled"] = setting.Mirror.Enabled + ctx.Data["DisableNewPushMirrors"] = setting.Mirror.DisableNewPush ctx.Data["DefaultMirrorInterval"] = setting.Mirror.DefaultInterval signing, _ := models.SigningKey(ctx.Repo.Repository.RepoPath()) @@ -144,7 +145,7 @@ func SettingsPost(ctx *context.Context) { ctx.Redirect(repo.Link() + "/settings") case "mirror": - if !repo.IsMirror { + if !setting.Mirror.Enabled || !repo.IsMirror { ctx.NotFound("", nil) return } @@ -220,7 +221,7 @@ func SettingsPost(ctx *context.Context) { ctx.Redirect(repo.Link() + "/settings") case "mirror-sync": - if !repo.IsMirror { + if !setting.Mirror.Enabled || !repo.IsMirror { ctx.NotFound("", nil) return } @@ -231,6 +232,11 @@ func SettingsPost(ctx *context.Context) { ctx.Redirect(repo.Link() + "/settings") case "push-mirror-sync": + if !setting.Mirror.Enabled { + ctx.NotFound("", nil) + return + } + m, err := selectPushMirrorByForm(form, repo) if err != nil { ctx.NotFound("", nil) @@ -243,6 +249,11 @@ func SettingsPost(ctx *context.Context) { ctx.Redirect(repo.Link() + "/settings") case "push-mirror-remove": + if !setting.Mirror.Enabled { + ctx.NotFound("", nil) + return + } + // This section doesn't require repo_name/RepoName to be set in the form, don't show it // as an error on the UI for this action ctx.Data["Err_RepoName"] = nil @@ -267,6 +278,11 @@ func SettingsPost(ctx *context.Context) { ctx.Redirect(repo.Link() + "/settings") case "push-mirror-add": + if setting.Mirror.DisableNewPush { + ctx.NotFound("", nil) + return + } + // This section doesn't require repo_name/RepoName to be set in the form, don't show it // as an error on the UI for this action ctx.Data["Err_RepoName"] = nil @@ -698,6 +714,7 @@ func SettingsPost(ctx *context.Context) { log.Trace("Repository was archived: %s/%s", ctx.Repo.Owner.Name, repo.Name) ctx.Redirect(ctx.Repo.RepoLink + "/settings") + case "unarchive": if !ctx.Repo.IsOwner() { ctx.Error(http.StatusForbidden) diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 1e30c919e6..7a3e37d993 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -22,6 +22,10 @@ var mirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength) // Update checks and updates mirror repositories. func Update(ctx context.Context) error { + if !setting.Mirror.Enabled { + log.Warn("Mirror feature disabled, but cron job enabled: skip update") + return nil + } log.Trace("Doing: Update") handler := func(idx int, bean interface{}) error { @@ -89,15 +93,24 @@ func syncMirrors(ctx context.Context) { // InitSyncMirrors initializes a go routine to sync the mirrors func InitSyncMirrors() { + if !setting.Mirror.Enabled { + return + } go graceful.GetManager().RunWithShutdownContext(syncMirrors) } // StartToMirror adds repoID to mirror queue func StartToMirror(repoID int64) { + if !setting.Mirror.Enabled { + return + } go mirrorQueue.Add(fmt.Sprintf("pull %d", repoID)) } // AddPushMirrorToQueue adds the push mirror to the queue func AddPushMirrorToQueue(mirrorID int64) { + if !setting.Mirror.Enabled { + return + } go mirrorQueue.Add(fmt.Sprintf("push %d", mirrorID)) } diff --git a/templates/org/home.tmpl b/templates/org/home.tmpl index 5e0a53aa56..153de30ca3 100644 --- a/templates/org/home.tmpl +++ b/templates/org/home.tmpl @@ -26,7 +26,7 @@
{{if .CanCreateOrgRepo}}
- {{if not .DisabledMirrors}} + {{if not .DisableNewPullMirrors}} {{.i18n.Tr "new_migrate"}} {{end}} {{.i18n.Tr "new_repo"}} diff --git a/templates/repo/migrate/options.tmpl b/templates/repo/migrate/options.tmpl index bca773a53f..8aefaf68f1 100644 --- a/templates/repo/migrate/options.tmpl +++ b/templates/repo/migrate/options.tmpl @@ -1,7 +1,7 @@
- {{if .DisableMirrors}} + {{if .DisableNewPullMirrors}} {{else}} diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 4054158544..688966d523 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -70,7 +70,7 @@
- {{if or .Repository.IsMirror (not .DisabledMirrors)}} + {{if .MirrorsEnabled}}

{{.i18n.Tr "repo.settings.mirror_settings"}}

@@ -191,42 +191,44 @@ {{$.i18n.Tr "repo.settings.mirror_settings.push_mirror.none"}} {{end}} - - -
- {{.CsrfTokenHtml}} - -
- - -

{{.i18n.Tr "repo.mirror_address_desc"}}

-
-
- - {{.i18n.Tr "repo.need_auth"}} - -
-
- - -
- -
- - -
+ {{if (not .DisableNewPushMirrors)}} + + + + {{.CsrfTokenHtml}} + +
+ + +

{{.i18n.Tr "repo.mirror_address_desc"}}

-
-
- - -
-
- -
-
- - +
+ + {{.i18n.Tr "repo.need_auth"}} + +
+
+ + +
+ +
+ + +
+
+
+
+ + +
+
+ +
+ + + + {{end}}