Backport Migration Option AuthToken (#13101)

fix #13085

Backport #12862
Backport Parts of #12672

modules/migrations/gitea.go

@@ -93,12 +93,15 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
 	}
 
 	var remoteAddr = repo.CloneURL
-	if len(opts.AuthUsername) > 0 {
+	if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
 		u, err := url.Parse(repo.CloneURL)
 		if err != nil {
 			return err
 		}
 		u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
+		if len(opts.AuthToken) > 0 {
+			u.User = url.UserPassword("oauth2", opts.AuthToken)
+		}
 		remoteAddr = u.String()
 	}
 

modules/migrations/gitea_test.go

@@ -26,7 +26,7 @@ func TestGiteaUploadRepo(t *testing.T) {
 	user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
 
 	var (
-		downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder")
+		downloader = NewGithubDownloaderV3("", "", "", "go-xorm", "builder")
 		repoName   = "builder-" + time.Now().Format("2006-01-02-15-04-05")
 		uploader   = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName)
 	)

modules/migrations/github.go

@@ -60,7 +60,7 @@ func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Download
 
 	log.Trace("Create github downloader: %s/%s", oldOwner, oldName)
 
-	return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil
+	return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil
 }
 
 // GitServiceType returns the type of git service
@@ -81,7 +81,7 @@ type GithubDownloaderV3 struct {
 }
 
 // NewGithubDownloaderV3 creates a github Downloader via github v3 API
-func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *GithubDownloaderV3 {
+func NewGithubDownloaderV3(userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
 	var downloader = GithubDownloaderV3{
 		userName:  userName,
 		password:  password,
@@ -90,15 +90,7 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith
 		repoName:  repoName,
 	}
 
-	var client *http.Client
-	if userName != "" {
-		if password == "" {
-			ts := oauth2.StaticTokenSource(
-				&oauth2.Token{AccessToken: userName},
-			)
-			client = oauth2.NewClient(downloader.ctx, ts)
-		} else {
-			client = &http.Client{
+	client := &http.Client{
 		Transport: &http.Transport{
 			Proxy: func(req *http.Request) (*url.URL, error) {
 				req.SetBasicAuth(userName, password)
@@ -106,7 +98,11 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith
 			},
 		},
 	}
-		}
+	if token != "" {
+		ts := oauth2.StaticTokenSource(
+			&oauth2.Token{AccessToken: token},
+		)
+		client = oauth2.NewClient(downloader.ctx, ts)
 	}
 	downloader.client = github.NewClient(client)
 	return &downloader

modules/migrations/github_test.go

@@ -64,7 +64,7 @@ func assertLabelEqual(t *testing.T, name, color, description string, label *base
 
 func TestGitHubDownloadRepo(t *testing.T) {
 	GithubLimitRateRemaining = 3 //Wait at 3 remaining since we could have 3 CI in //
-	downloader := NewGithubDownloaderV3(os.Getenv("GITHUB_READ_TOKEN"), "", "go-gitea", "test_repo")
+	downloader := NewGithubDownloaderV3("", "", os.Getenv("GITHUB_READ_TOKEN"), "go-gitea", "test_repo")
 	err := downloader.RefreshRate()
 	assert.NoError(t, err)
 

modules/migrations/gitlab.go

@@ -56,10 +56,11 @@ func (f *GitlabDownloaderFactory) New(opts base.MigrateOptions) (base.Downloader
 
 	baseURL := u.Scheme + "://" + u.Host
 	repoNameSpace := strings.TrimPrefix(u.Path, "/")
+	repoNameSpace = strings.TrimSuffix(repoNameSpace, ".git")
 
 	log.Trace("Create gitlab downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace)
 
-	return NewGitlabDownloader(baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword), nil
+	return NewGitlabDownloader(baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword, opts.AuthToken), nil
 }
 
 // GitServiceType returns the type of git service
@@ -85,16 +86,13 @@ type GitlabDownloader struct {
 // NewGitlabDownloader creates a gitlab Downloader via gitlab API
 //   Use either a username/password, personal token entered into the username field, or anonymous/public access
 //   Note: Public access only allows very basic access
-func NewGitlabDownloader(baseURL, repoPath, username, password string) *GitlabDownloader {
-	var gitlabClient *gitlab.Client
-	var err error
-	if username != "" {
-		if password == "" {
-			gitlabClient, err = gitlab.NewClient(username, gitlab.WithBaseURL(baseURL))
-		} else {
+func NewGitlabDownloader(baseURL, repoPath, username, password, token string) *GitlabDownloader {
+	gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL))
+	// Only use basic auth if token is blank and password is NOT
+	// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
+	if token == "" && password != "" {
 		gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL))
 	}
-	}
 
 	if err != nil {
 		log.Trace("Error logging into gitlab: %v", err)

modules/migrations/gitlab_test.go

@@ -27,7 +27,7 @@ func TestGitlabDownloadRepo(t *testing.T) {
 		t.Skipf("Can't access test repo, skipping %s", t.Name())
 	}
 
-	downloader := NewGitlabDownloader("https://gitlab.com", "gitea/test_repo", gitlabPersonalAccessToken, "")
+	downloader := NewGitlabDownloader("https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken)
 	if downloader == nil {
 		t.Fatal("NewGitlabDownloader is nil")
 	}

modules/migrations/migrate.go

@@ -36,6 +36,12 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
 		theFactory base.DownloaderFactory
 	)
 
+	// determine if user is token
+	if len(opts.AuthUsername) != 0 && len(opts.AuthPassword) == 0 {
+		opts.AuthToken = opts.AuthUsername
+		opts.AuthUsername = ""
+	}
+
 	for _, factory := range factories {
 		if match, err := factory.Match(opts); err != nil {
 			return nil, err

modules/structs/repo.go

@@ -213,6 +213,7 @@ type MigrateRepoOption struct {
 	CloneAddr    string `json:"clone_addr" binding:"Required"`
 	AuthUsername string `json:"auth_username"`
 	AuthPassword string `json:"auth_password"`
+	AuthToken    string `json:"auth_token"`
 	// required: true
 	UID int `json:"uid" binding:"Required"`
 	// required: true