From 51530a1451510f36cc19a71b89fc39e28a365333 Mon Sep 17 00:00:00 2001 From: Bruno Sofiato Date: Mon, 11 Nov 2024 18:28:03 -0300 Subject: [PATCH] Adjustments to remove dangling repository locks Signed-off-by: Bruno Sofiato --- custom/conf/app.example.ini | 3 + modules/git/command.go | 37 ++++++++++- modules/git/repo_cleanup.go | 60 ++++++++++++++++++ modules/git/repo_cleanup_test.go | 101 +++++++++++++++++++++++++++++++ modules/setting/repository.go | 5 +- 5 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 modules/git/repo_cleanup.go create mode 100644 modules/git/repo_cleanup_test.go diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index ef5684237d..fa8381e80d 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1039,6 +1039,9 @@ LEVEL = Info ;; Allow fork repositories without maximum number limit ;ALLOW_FORK_WITHOUT_MAXIMUM_LIMIT = true +;; The elapsed time for dangling repository lock to be removed +;DANGLING_LOCK_THRESHOLD = 1h + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;[repository.editor] diff --git a/modules/git/command.go b/modules/git/command.go index 22cb275ab2..90b392b932 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -269,8 +269,43 @@ func CommonCmdServEnvs() []string { var ErrBrokenCommand = errors.New("git command is broken") -// Run runs the command with the RunOpts func (c *Command) Run(opts *RunOpts) error { + // Check if there is some dangling locks older than the given threshold + if err := ForciblyUnlockRepositoryIfNeeded(c.parentContext, opts.Dir); err != nil { + log.Error("Error while trying to unlock repository: %v", err) + return err + } + // Execute the git command + if err := c.doRun(opts); err != nil { + unlockUponCrashing(c.parentContext, err, opts.Dir) + return err + } + return nil +} + +func unlockUponCrashing(ctx context.Context, originalError error, repoDir string) { + if hasGitProcessCrashed(originalError) { + log.Warn("The git process has crashed. Attempting to forcbily unlock the underlying repo at %s", repoDir) + if err := ForciblyUnlockRepository(ctx, repoDir); err != nil { + log.Error("Error while trying to unlock repository at %v", err) + } + } +} + +func hasGitProcessCrashed(err error) bool { + if exitError, ok := err.(*exec.ExitError); ok { + if runtime.GOOS == "windows" { + log.Warn("Cannot realiably detected if the git process has crashed in windows. Assuming it hasn't [exitCode: %s, pid: %s]", exitError.ExitCode(), exitError.Pid()) + return false + } + return exitError.ExitCode() > 128 + } + log.Debug("The given error is not an ExitError [err: %v]. Assuming it the git process hasn't crashed", err) + return false +} + +// Run runs the command with the RunOpts +func (c *Command) doRun(opts *RunOpts) error { if len(c.brokenArgs) != 0 { log.Error("git command is broken: %s, broken args: %s", c.String(), strings.Join(c.brokenArgs, " ")) return ErrBrokenCommand diff --git a/modules/git/repo_cleanup.go b/modules/git/repo_cleanup.go new file mode 100644 index 0000000000..6fdc47fcd8 --- /dev/null +++ b/modules/git/repo_cleanup.go @@ -0,0 +1,60 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "context" + "os" + "path/filepath" + "strings" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +func ForciblyUnlockRepository(ctx context.Context, repoPath string) error { + return cleanLocksIfNeeded(repoPath, time.Now()) +} + +func ForciblyUnlockRepositoryIfNeeded(ctx context.Context, repoPath string) error { + lockThreshold := time.Now().Add(-1 * setting.Repository.DanglingLockThreshold) + return cleanLocksIfNeeded(repoPath, lockThreshold) +} + +func cleanLocksIfNeeded(repoPath string, threshold time.Time) error { + if repoPath == "" { + return nil + } + log.Trace("Checking if repository %s is locked [lock threshold is %s]", repoPath, threshold) + return filepath.Walk(repoPath, func(filePath string, fileInfo os.FileInfo, err error) error { + if err != nil { + return err + } + if err := cleanLockIfNeeded(filePath, fileInfo, threshold); err != nil { + log.Error("Failed to remove lock file %s: %v", filePath, err) + return err + } + return nil + }) +} + +func cleanLockIfNeeded(filePath string, fileInfo os.FileInfo, threshold time.Time) error { + if isLock(fileInfo) { + if fileInfo.ModTime().Before(threshold) { + if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) { + return err + } + log.Info("Lock file %s has been removed since its older than %s [timestamp: %s]", filePath, threshold, fileInfo.ModTime()) + return nil + } + log.Warn("Cannot exclude lock file %s because it is younger than the threshold %s [timestamp: %s]", filePath, threshold, fileInfo.ModTime()) + return nil + } + return nil +} + +func isLock(lockFile os.FileInfo) bool { + return !lockFile.IsDir() && strings.HasSuffix(lockFile.Name(), ".lock") +} diff --git a/modules/git/repo_cleanup_test.go b/modules/git/repo_cleanup_test.go new file mode 100644 index 0000000000..769a3a3ed5 --- /dev/null +++ b/modules/git/repo_cleanup_test.go @@ -0,0 +1,101 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "context" + "os" + "os/exec" + "runtime" + "testing" + "time" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + "github.com/stretchr/testify/assert" +) + +// This test mimics a repository having dangling locks. If the locks are older than the threshold, they should be +// removed. Otherwise, they'll remain and the command will fail. + +func TestMaintainExistentLock(t *testing.T) { + if runtime.GOOS != "linux" { + // Need to use touch to change the last access time of the lock files + t.Skip("Skipping test on non-linux OS") + } + + shouldRemainLocked := func(lockFiles []string, err error) { + assert.Error(t, err) + for _, lockFile := range lockFiles { + assert.FileExists(t, lockFile) + } + } + + shouldBeUnlocked := func(lockFiles []string, err error) { + assert.NoError(t, err) + for _, lockFile := range lockFiles { + assert.NoFileExists(t, lockFile) + } + } + + t.Run("2 days lock file (1 hour threshold)", func(t *testing.T) { + doTestLockCleanup(t, "2 days", time.Hour, shouldBeUnlocked) + }) + + t.Run("1 hour lock file (1 hour threshold)", func(t *testing.T) { + doTestLockCleanup(t, "1 hour", time.Hour, shouldBeUnlocked) + }) + + t.Run("1 minutes lock file (1 hour threshold)", func(t *testing.T) { + doTestLockCleanup(t, "1 minutes", time.Hour, shouldRemainLocked) + }) + + t.Run("1 hour lock file (2 hour threshold)", func(t *testing.T) { + doTestLockCleanup(t, "1 hour", 2*time.Hour, shouldRemainLocked) + }) +} + +func doTestLockCleanup(t *testing.T, lockAge string, threshold time.Duration, expectedResult func(lockFiles []string, err error)) { + defer test.MockVariableValue(&setting.Repository, setting.Repository)() + + setting.Repository.DanglingLockThreshold = threshold + + if tmpDir, err := os.MkdirTemp("", "cleanup-after-crash"); err != nil { + t.Fatal(err) + } else { + defer os.RemoveAll(tmpDir) + + if err := os.CopyFS(tmpDir, os.DirFS("../../tests/gitea-repositories-meta/org3/repo3.git")); err != nil { + t.Fatal(err) + } + + lockFiles := lockFilesFor(tmpDir) + + os.MkdirAll(tmpDir+"/objects/info/commit-graphs", os.ModeSticky|os.ModePerm) + + for _, lockFile := range lockFiles { + createLockFiles(t, lockFile, lockAge) + } + + cmd := NewCommand(context.Background(), "fetch") + _, _, cmdErr := cmd.RunStdString(&RunOpts{Dir: tmpDir}) + + expectedResult(lockFiles, cmdErr) + } +} + +func lockFilesFor(path string) []string { + return []string{ + path + "/config.lock", + path + "/HEAD.lock", + path + "/objects/info/commit-graphs/commit-graph-chain.lock", + } +} + +func createLockFiles(t *testing.T, file, lockAge string) { + cmd := exec.Command("touch", "-m", "-a", "-d", "-"+lockAge, file) + if err := cmd.Run(); err != nil { + t.Error(err) + } +} diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 14cf5805c0..e325faafbb 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -8,6 +8,7 @@ import ( "path" "path/filepath" "strings" + "time" "code.gitea.io/gitea/modules/log" ) @@ -53,7 +54,7 @@ var ( AllowDeleteOfUnadoptedRepositories bool DisableDownloadSourceArchives bool AllowForkWithoutMaximumLimit bool - + DanglingLockThreshold time.Duration // Repository editor settings Editor struct { LineWrapExtensions []string @@ -283,6 +284,8 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { Repository.GoGetCloneURLProtocol = sec.Key("GO_GET_CLONE_URL_PROTOCOL").MustString("https") Repository.MaxCreationLimit = sec.Key("MAX_CREATION_LIMIT").MustInt(-1) Repository.DefaultBranch = sec.Key("DEFAULT_BRANCH").MustString(Repository.DefaultBranch) + Repository.DanglingLockThreshold = sec.Key("DANGLING_LOCK_THRESHOLD").MustDuration(time.Hour) + RepoRootPath = sec.Key("ROOT").MustString(path.Join(AppDataPath, "gitea-repositories")) if !filepath.IsAbs(RepoRootPath) { RepoRootPath = filepath.Join(AppWorkPath, RepoRootPath)