add test coverage for original author conversion during migrations (#18506)

* add test coverage for original author conversion during migrations

And create a function to factorize a code snippet that is repeated
five times and would otherwise be more difficult to test and maintain
consistently.

Signed-off-by: Loïc Dachary <loic@dachary.org>

* fix variable scope and int64 formatting

* add missing calls to remapExternalUser and fix misplaced %d

Co-authored-by: Loïc Dachary <loic@dachary.org>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>

models/fixtures/external_login_user.yml

@@ -0,0 +1 @@
+[] # empty

models/issue.go

@@ -2329,3 +2329,20 @@ func deleteIssuesByRepoID(sess db.Engine, repoID int64) (attachmentPaths []strin
 
 	return
 }
+
+// RemapExternalUser ExternalUserRemappable interface
+func (issue *Issue) RemapExternalUser(externalName string, externalID, userID int64) error {
+	issue.OriginalAuthor = externalName
+	issue.OriginalAuthorID = externalID
+	issue.PosterID = userID
+	return nil
+}
+
+// GetUserID ExternalUserRemappable interface
+func (issue *Issue) GetUserID() int64 { return issue.PosterID }
+
+// GetExternalName ExternalUserRemappable interface
+func (issue *Issue) GetExternalName() string { return issue.OriginalAuthor }
+
+// GetExternalID ExternalUserRemappable interface
+func (issue *Issue) GetExternalID() int64 { return issue.OriginalAuthorID }

models/issue_comment.go

@@ -1464,3 +1464,20 @@ func commitBranchCheck(gitRepo *git.Repository, startCommit *git.Commit, endComm
 	}
 	return nil
 }
+
+// RemapExternalUser ExternalUserRemappable interface
+func (c *Comment) RemapExternalUser(externalName string, externalID, userID int64) error {
+	c.OriginalAuthor = externalName
+	c.OriginalAuthorID = externalID
+	c.PosterID = userID
+	return nil
+}
+
+// GetUserID ExternalUserRemappable interface
+func (c *Comment) GetUserID() int64 { return c.PosterID }
+
+// GetExternalName ExternalUserRemappable interface
+func (c *Comment) GetExternalName() string { return c.OriginalAuthor }
+
+// GetExternalID ExternalUserRemappable interface
+func (c *Comment) GetExternalID() int64 { return c.OriginalAuthorID }

models/issue_reaction.go

@@ -343,3 +343,20 @@ func (list ReactionList) GetMoreUserCount() int {
 	}
 	return len(list) - setting.UI.ReactionMaxUserNum
 }
+
+// RemapExternalUser ExternalUserRemappable interface
+func (r *Reaction) RemapExternalUser(externalName string, externalID, userID int64) error {
+	r.OriginalAuthor = externalName
+	r.OriginalAuthorID = externalID
+	r.UserID = userID
+	return nil
+}
+
+// GetUserID ExternalUserRemappable interface
+func (r *Reaction) GetUserID() int64 { return r.UserID }
+
+// GetExternalName ExternalUserRemappable interface
+func (r *Reaction) GetExternalName() string { return r.OriginalAuthor }
+
+// GetExternalID ExternalUserRemappable interface
+func (r *Reaction) GetExternalID() int64 { return r.OriginalAuthorID }

models/release.go

@@ -456,3 +456,20 @@ func SaveOrUpdateTag(repo *repo_model.Repository, newRel *Release) error {
 	}
 	return nil
 }
+
+// RemapExternalUser ExternalUserRemappable interface
+func (r *Release) RemapExternalUser(externalName string, externalID, userID int64) error {
+	r.OriginalAuthor = externalName
+	r.OriginalAuthorID = externalID
+	r.PublisherID = userID
+	return nil
+}
+
+// UserID ExternalUserRemappable interface
+func (r *Release) GetUserID() int64 { return r.PublisherID }
+
+// ExternalName ExternalUserRemappable interface
+func (r *Release) GetExternalName() string { return r.OriginalAuthor }
+
+// ExternalID ExternalUserRemappable interface
+func (r *Release) GetExternalID() int64 { return r.OriginalAuthorID }

models/review.go

@@ -987,3 +987,20 @@ func (r *Review) HTMLURL() string {
 	}
 	return comment.HTMLURL()
 }
+
+// RemapExternalUser ExternalUserRemappable interface
+func (r *Review) RemapExternalUser(externalName string, externalID, userID int64) error {
+	r.OriginalAuthor = externalName
+	r.OriginalAuthorID = externalID
+	r.ReviewerID = userID
+	return nil
+}
+
+// GetUserID ExternalUserRemappable interface
+func (r *Review) GetUserID() int64 { return r.ReviewerID }
+
+// GetExternalName ExternalUserRemappable interface
+func (r *Review) GetExternalName() string { return r.OriginalAuthor }
+
+// GetExternalID ExternalUserRemappable interface
+func (r *Review) GetExternalID() int64 { return r.OriginalAuthorID }

models/user/external_login_user.go

@@ -68,6 +68,17 @@ type ExternalLoginUser struct {
 	ExpiresAt         time.Time
 }
 
+type ExternalUserMigrated interface {
+	GetExternalName() string
+	GetExternalID() int64
+}
+
+type ExternalUserRemappable interface {
+	GetUserID() int64
+	RemapExternalUser(externalName string, externalID, userID int64) error
+	ExternalUserMigrated
+}
+
 func init() {
 	db.RegisterModel(new(ExternalLoginUser))
 }

modules/migration/comment.go

@@ -18,3 +18,9 @@ type Comment struct {
 	Content     string
 	Reactions   []*Reaction
 }
+
+// GetExternalName ExternalUserMigrated interface
+func (c *Comment) GetExternalName() string { return c.PosterName }
+
+// ExternalID ExternalUserMigrated interface
+func (c *Comment) GetExternalID() int64 { return c.PosterID }

modules/migration/issue.go

@@ -46,3 +46,9 @@ type Issue struct {
 	Assignees   []string     `json:"assignees"`
 	Context     IssueContext `yaml:"-"`
 }
+
+// GetExternalName ExternalUserMigrated interface
+func (i *Issue) GetExternalName() string { return i.PosterName }
+
+// GetExternalID ExternalUserMigrated interface
+func (i *Issue) GetExternalID() int64 { return i.PosterID }

modules/migration/pullrequest.go

@@ -61,3 +61,9 @@ type PullRequestBranch struct {
 func (p PullRequestBranch) RepoPath() string {
 	return fmt.Sprintf("%s/%s", p.OwnerName, p.RepoName)
 }
+
+// GetExternalName ExternalUserMigrated interface
+func (p *PullRequest) GetExternalName() string { return p.PosterName }
+
+// ExternalID ExternalUserMigrated interface
+func (p *PullRequest) GetExternalID() int64 { return p.PosterID }

modules/migration/reaction.go

@@ -10,3 +10,9 @@ type Reaction struct {
 	UserName string `yaml:"user_name" json:"user_name"`
 	Content  string `json:"content"`
 }
+
+// GetExternalName ExternalUserMigrated interface
+func (r *Reaction) GetExternalName() string { return r.UserName }
+
+// GetExternalID ExternalUserMigrated interface
+func (r *Reaction) GetExternalID() int64 { return r.UserID }

modules/migration/release.go

@@ -38,3 +38,9 @@ type Release struct {
 	Created         time.Time
 	Published       time.Time
 }
+
+// GetExternalName ExternalUserMigrated interface
+func (r *Release) GetExternalName() string { return r.PublisherName }
+
+// GetExternalID ExternalUserMigrated interface
+func (r *Release) GetExternalID() int64 { return r.PublisherID }

modules/migration/review.go

@@ -28,6 +28,12 @@ type Review struct {
 	Comments     []*ReviewComment
 }
 
+// GetExternalName ExternalUserMigrated interface
+func (r *Review) GetExternalName() string { return r.ReviewerName }
+
+// ExternalID ExternalUserMigrated interface
+func (r *Review) GetExternalID() int64 { return r.ReviewerID }
+
 // ReviewComment represents a review comment
 type ReviewComment struct {
 	ID        int64

services/migrations/gitea_uploader.go

@@ -254,25 +254,8 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
 			CreatedUnix:  timeutil.TimeStamp(release.Created.Unix()),
 		}
 
-		userid, ok := g.userMap[release.PublisherID]
+		if err := g.remapExternalUser(release, &rel); err != nil {
-		tp := g.gitServiceType.Name()
+			return err
-		if !ok && tp != "" {
-			var err error
-			userid, err = user_model.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", release.PublisherID))
-			if err != nil {
-				log.Error("GetUserIDByExternalUserID: %v", err)
-			}
-			if userid > 0 {
-				g.userMap[release.PublisherID] = userid
-			}
-		}
-
-		if userid > 0 {
-			rel.PublisherID = userid
-		} else {
-			rel.PublisherID = g.doer.ID
-			rel.OriginalAuthor = release.PublisherName
-			rel.OriginalAuthorID = release.PublisherID
 		}
 
 		// calc NumCommits if no draft
@@ -394,25 +377,8 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
 			UpdatedUnix: timeutil.TimeStamp(issue.Updated.Unix()),
 		}
 
-		userid, ok := g.userMap[issue.PosterID]
+		if err := g.remapExternalUser(issue, &is); err != nil {
-		tp := g.gitServiceType.Name()
+			return err
-		if !ok && tp != "" {
-			var err error
-			userid, err = user_model.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", issue.PosterID))
-			if err != nil {
-				log.Error("GetUserIDByExternalUserID: %v", err)
-			}
-			if userid > 0 {
-				g.userMap[issue.PosterID] = userid
-			}
-		}
-
-		if userid > 0 {
-			is.PosterID = userid
-		} else {
-			is.PosterID = g.doer.ID
-			is.OriginalAuthor = issue.PosterName
-			is.OriginalAuthorID = issue.PosterID
 		}
 
 		if issue.Closed != nil {
@@ -420,27 +386,12 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
 		}
 		// add reactions
 		for _, reaction := range issue.Reactions {
-			userid, ok := g.userMap[reaction.UserID]
-			if !ok && tp != "" {
-				var err error
-				userid, err = user_model.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
-				if err != nil {
-					log.Error("GetUserIDByExternalUserID: %v", err)
-				}
-				if userid > 0 {
-					g.userMap[reaction.UserID] = userid
-				}
-			}
 			res := models.Reaction{
 				Type:        reaction.Content,
 				CreatedUnix: timeutil.TimeStampNow(),
 			}
-			if userid > 0 {
+			if err := g.remapExternalUser(reaction, &res); err != nil {
-				res.UserID = userid
+				return err
-			} else {
-				res.UserID = g.doer.ID
-				res.OriginalAuthorID = reaction.UserID
-				res.OriginalAuthor = reaction.UserName
 			}
 			is.Reactions = append(is.Reactions, &res)
 		}
@@ -477,19 +428,6 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
 			issue = issueInter.(*models.Issue)
 		}
 
-		userid, ok := g.userMap[comment.PosterID]
-		tp := g.gitServiceType.Name()
-		if !ok && tp != "" {
-			var err error
-			userid, err = user_model.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", comment.PosterID))
-			if err != nil {
-				log.Error("GetUserIDByExternalUserID: %v", err)
-			}
-			if userid > 0 {
-				g.userMap[comment.PosterID] = userid
-			}
-		}
-
 		if comment.Created.IsZero() {
 			comment.Created = time.Unix(int64(issue.CreatedUnix), 0)
 		}
@@ -505,37 +443,18 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
 			UpdatedUnix: timeutil.TimeStamp(comment.Updated.Unix()),
 		}
 
-		if userid > 0 {
+		if err := g.remapExternalUser(comment, &cm); err != nil {
-			cm.PosterID = userid
+			return err
-		} else {
-			cm.PosterID = g.doer.ID
-			cm.OriginalAuthor = comment.PosterName
-			cm.OriginalAuthorID = comment.PosterID
 		}
 
 		// add reactions
 		for _, reaction := range comment.Reactions {
-			userid, ok := g.userMap[reaction.UserID]
-			if !ok && tp != "" {
-				var err error
-				userid, err = user_model.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
-				if err != nil {
-					log.Error("GetUserIDByExternalUserID: %v", err)
-				}
-				if userid > 0 {
-					g.userMap[reaction.UserID] = userid
-				}
-			}
 			res := models.Reaction{
 				Type:        reaction.Content,
 				CreatedUnix: timeutil.TimeStampNow(),
 			}
-			if userid > 0 {
+			if err := g.remapExternalUser(reaction, &res); err != nil {
-				res.UserID = userid
+				return err
-			} else {
-				res.UserID = g.doer.ID
-				res.OriginalAuthorID = reaction.UserID
-				res.OriginalAuthor = reaction.UserName
 			}
 			cm.Reactions = append(cm.Reactions, &res)
 		}
@@ -558,25 +477,8 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error
 			return err
 		}
 
-		userid, ok := g.userMap[pr.PosterID]
+		if err := g.remapExternalUser(pr, gpr.Issue); err != nil {
-		tp := g.gitServiceType.Name()
+			return err
-		if !ok && tp != "" {
-			var err error
-			userid, err = user_model.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
-			if err != nil {
-				log.Error("GetUserIDByExternalUserID: %v", err)
-			}
-			if userid > 0 {
-				g.userMap[pr.PosterID] = userid
-			}
-		}
-
-		if userid > 0 {
-			gpr.Issue.PosterID = userid
-		} else {
-			gpr.Issue.PosterID = g.doer.ID
-			gpr.Issue.OriginalAuthor = pr.PosterName
-			gpr.Issue.OriginalAuthorID = pr.PosterID
 		}
 
 		gprs = append(gprs, gpr)
@@ -736,51 +638,18 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 		UpdatedUnix: timeutil.TimeStamp(pr.Updated.Unix()),
 	}
 
-	tp := g.gitServiceType.Name()
+	if err := g.remapExternalUser(pr, &issue); err != nil {
-
+		return nil, err
-	userid, ok := g.userMap[pr.PosterID]
-	if !ok && tp != "" {
-		var err error
-		userid, err = user_model.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
-		if err != nil {
-			log.Error("GetUserIDByExternalUserID: %v", err)
-		}
-		if userid > 0 {
-			g.userMap[pr.PosterID] = userid
-		}
-	}
-
-	if userid > 0 {
-		issue.PosterID = userid
-	} else {
-		issue.PosterID = g.doer.ID
-		issue.OriginalAuthor = pr.PosterName
-		issue.OriginalAuthorID = pr.PosterID
 	}
 
 	// add reactions
 	for _, reaction := range pr.Reactions {
-		userid, ok := g.userMap[reaction.UserID]
-		if !ok && tp != "" {
-			var err error
-			userid, err = user_model.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
-			if err != nil {
-				log.Error("GetUserIDByExternalUserID: %v", err)
-			}
-			if userid > 0 {
-				g.userMap[reaction.UserID] = userid
-			}
-		}
 		res := models.Reaction{
 			Type:        reaction.Content,
 			CreatedUnix: timeutil.TimeStampNow(),
 		}
-		if userid > 0 {
+		if err := g.remapExternalUser(reaction, &res); err != nil {
-			res.UserID = userid
+			return nil, err
-		} else {
-			res.UserID = g.doer.ID
-			res.OriginalAuthorID = reaction.UserID
-			res.OriginalAuthor = reaction.UserName
 		}
 		issue.Reactions = append(issue.Reactions, &res)
 	}
@@ -843,19 +712,6 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
 			issue = issueInter.(*models.Issue)
 		}
 
-		userid, ok := g.userMap[review.ReviewerID]
-		tp := g.gitServiceType.Name()
-		if !ok && tp != "" {
-			var err error
-			userid, err = user_model.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", review.ReviewerID))
-			if err != nil {
-				log.Error("GetUserIDByExternalUserID: %v", err)
-			}
-			if userid > 0 {
-				g.userMap[review.ReviewerID] = userid
-			}
-		}
-
 		if review.CreatedAt.IsZero() {
 			review.CreatedAt = time.Unix(int64(issue.CreatedUnix), 0)
 		}
@@ -869,12 +725,8 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
 			UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
 		}
 
-		if userid > 0 {
+		if err := g.remapExternalUser(review, &cm); err != nil {
-			cm.ReviewerID = userid
+			return err
-		} else {
-			cm.ReviewerID = g.doer.ID
-			cm.OriginalAuthor = review.ReviewerName
-			cm.OriginalAuthorID = review.ReviewerID
 		}
 
 		// get pr
@@ -926,7 +778,6 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
 
 			c := models.Comment{
 				Type:        models.CommentTypeCode,
-				PosterID:    comment.PosterID,
 				IssueID:     issue.ID,
 				Content:     comment.Content,
 				Line:        int64(line + comment.Position - 1),
@@ -937,12 +788,8 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
 				UpdatedUnix: timeutil.TimeStamp(comment.UpdatedAt.Unix()),
 			}
 
-			if userid > 0 {
+			if err := g.remapExternalUser(review, &c); err != nil {
-				c.PosterID = userid
+				return err
-			} else {
-				c.PosterID = g.doer.ID
-				c.OriginalAuthor = review.ReviewerName
-				c.OriginalAuthorID = review.ReviewerID
 			}
 
 			cm.Comments = append(cm.Comments, &c)
@@ -983,3 +830,24 @@ func (g *GiteaLocalUploader) Finish() error {
 	g.repo.Status = repo_model.RepositoryReady
 	return repo_model.UpdateRepositoryCols(g.repo, "status")
 }
+
+func (g *GiteaLocalUploader) remapExternalUser(source user_model.ExternalUserMigrated, target user_model.ExternalUserRemappable) (err error) {
+	userid, ok := g.userMap[source.GetExternalID()]
+	tp := g.gitServiceType.Name()
+	if !ok && tp != "" {
+		userid, err = user_model.GetUserIDByExternalUserID(tp, fmt.Sprintf("%d", source.GetExternalID()))
+		if err != nil {
+			log.Error("GetUserIDByExternalUserID: %v", err)
+		}
+		if userid > 0 {
+			g.userMap[source.GetExternalID()] = userid
+		}
+	}
+
+	if userid > 0 {
+		err = target.RemapExternalUser("", 0, userid)
+	} else {
+		err = target.RemapExternalUser(source.GetExternalName(), source.GetExternalID(), g.doer.ID)
+	}
+	return
+}

services/migrations/gitea_uploader_test.go

@@ -7,6 +7,7 @@ package migrations
 
 import (
 	"context"
+	"strconv"
 	"testing"
 	"time"
 
@@ -115,3 +116,50 @@ func TestGiteaUploadRepo(t *testing.T) {
 	assert.NoError(t, pulls[0].Issue.LoadDiscussComments())
 	assert.Len(t, pulls[0].Issue.Comments, 2)
 }
+
+func TestGiteaUploadRemapExternalUser(t *testing.T) {
+	unittest.PrepareTestEnv(t)
+	doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}).(*user_model.User)
+
+	repoName := "migrated"
+	uploader := NewGiteaLocalUploader(context.Background(), doer, doer.Name, repoName)
+	uploader.gitServiceType = structs.GiteaService
+
+	externalID := int64(1234567)
+	externalName := "url.or.something"
+	source := base.Release{
+		PublisherID:   externalID,
+		PublisherName: externalName,
+	}
+
+	//
+	// When there is no user linked to the external ID, the migrated data is authored
+	// by the doer
+	//
+	target := models.Release{}
+	err := uploader.remapExternalUser(&source, &target)
+	assert.NoError(t, err)
+	assert.EqualValues(t, doer.ID, target.GetUserID())
+
+	//
+	// Link the external ID to an existing user
+	//
+	linkedUser := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}).(*user_model.User)
+	externalLoginUser := &user_model.ExternalLoginUser{
+		ExternalID:    strconv.FormatInt(externalID, 10),
+		UserID:        linkedUser.ID,
+		LoginSourceID: 0,
+		Provider:      structs.GiteaService.Name(),
+	}
+	err = user_model.LinkExternalToUser(linkedUser, externalLoginUser)
+	assert.NoError(t, err)
+
+	//
+	// When a user is linked to the external ID, it becomes the author of
+	// the migrated data
+	//
+	target = models.Release{}
+	err = uploader.remapExternalUser(&source, &target)
+	assert.NoError(t, err)
+	assert.EqualValues(t, target.GetUserID(), linkedUser.ID)
+}