From cee5f7c5e2bfe3132d7089a6bffc1a4bfc392e21 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 22 Aug 2021 00:47:45 +0200 Subject: [PATCH] Add migrate from OneDev (#16356) * Use context to simplify logic. * Added migration from OneDev. This PR adds [OneDev](https://code.onedev.io/) as migration source. Supported: - [x] Milestones - [x] Issues - [x] Pull Requests - [x] Comments - [x] Reviews - [x] Labels --- modules/convert/utils.go | 2 + modules/migrations/base/downloader.go | 8 +- modules/migrations/base/issue.go | 20 + modules/migrations/base/null_downloader.go | 4 +- modules/migrations/base/pullrequest.go | 2 +- modules/migrations/base/retry_downloader.go | 4 +- modules/migrations/gitea_downloader.go | 20 +- modules/migrations/gitea_downloader_test.go | 4 +- modules/migrations/gitea_uploader.go | 3 + modules/migrations/github.go | 20 +- modules/migrations/github_test.go | 8 +- modules/migrations/gitlab.go | 69 ++- modules/migrations/gitlab_test.go | 32 +- modules/migrations/gogs.go | 6 +- modules/migrations/gogs_test.go | 2 +- modules/migrations/migrate.go | 18 +- modules/migrations/onedev.go | 619 ++++++++++++++++++++ modules/migrations/onedev_test.go | 169 ++++++ modules/migrations/restore.go | 10 +- modules/structs/repo.go | 4 + options/locale/locale_en-US.ini | 1 + public/img/svg/gitea-onedev.svg | 1 + templates/repo/migrate/onedev.tmpl | 117 ++++ web_src/svg/gitea-onedev.svg | 42 ++ 24 files changed, 1093 insertions(+), 92 deletions(-) create mode 100644 modules/migrations/onedev.go create mode 100644 modules/migrations/onedev_test.go create mode 100644 public/img/svg/gitea-onedev.svg create mode 100644 templates/repo/migrate/onedev.tmpl create mode 100644 web_src/svg/gitea-onedev.svg diff --git a/modules/convert/utils.go b/modules/convert/utils.go index 69de306689e0..a0463d7b1006 100644 --- a/modules/convert/utils.go +++ b/modules/convert/utils.go @@ -33,6 +33,8 @@ func ToGitServiceType(value string) structs.GitServiceType { return structs.GitlabService case "gogs": return structs.GogsService + case "onedev": + return structs.OneDevService default: return structs.PlainGitService } diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index 71c8f3eaf978..3c581b869900 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -13,9 +13,9 @@ import ( // GetCommentOptions represents an options for get comment type GetCommentOptions struct { - IssueNumber int64 - Page int - PageSize int + Context IssueContext + Page int + PageSize int } // Downloader downloads the site repo information @@ -30,7 +30,7 @@ type Downloader interface { GetComments(opts GetCommentOptions) ([]*Comment, bool, error) SupportGetRepoComments() bool GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) - GetReviews(pullRequestNumber int64) ([]*Review, error) + GetReviews(pullRequestContext IssueContext) ([]*Review, error) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) } diff --git a/modules/migrations/base/issue.go b/modules/migrations/base/issue.go index 8b1b4612444d..7addd1336a28 100644 --- a/modules/migrations/base/issue.go +++ b/modules/migrations/base/issue.go @@ -7,6 +7,25 @@ package base import "time" +// IssueContext is used to map between local and foreign issue/PR ids. +type IssueContext interface { + LocalID() int64 + ForeignID() int64 +} + +// BasicIssueContext is a 1:1 mapping between local and foreign ids. +type BasicIssueContext int64 + +// LocalID gets the local id. +func (c BasicIssueContext) LocalID() int64 { + return int64(c) +} + +// ForeignID gets the foreign id. +func (c BasicIssueContext) ForeignID() int64 { + return int64(c) +} + // Issue is a standard issue information type Issue struct { Number int64 @@ -25,4 +44,5 @@ type Issue struct { Labels []*Label Reactions []*Reaction Assignees []string + Context IssueContext `yaml:"-"` } diff --git a/modules/migrations/base/null_downloader.go b/modules/migrations/base/null_downloader.go index 53a536709d1f..c64d0e263379 100644 --- a/modules/migrations/base/null_downloader.go +++ b/modules/migrations/base/null_downloader.go @@ -50,7 +50,7 @@ func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { return nil, false, &ErrNotSupported{Entity: "Issues"} } -// GetComments returns comments according issueNumber +// GetComments returns comments according the options func (n NullDownloader) GetComments(GetCommentOptions) ([]*Comment, bool, error) { return nil, false, &ErrNotSupported{Entity: "Comments"} } @@ -61,7 +61,7 @@ func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool } // GetReviews returns pull requests review -func (n NullDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) { +func (n NullDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) { return nil, &ErrNotSupported{Entity: "Reviews"} } diff --git a/modules/migrations/base/pullrequest.go b/modules/migrations/base/pullrequest.go index 6411137d0aa3..84b302d18fad 100644 --- a/modules/migrations/base/pullrequest.go +++ b/modules/migrations/base/pullrequest.go @@ -13,7 +13,6 @@ import ( // PullRequest defines a standard pull request information type PullRequest struct { Number int64 - OriginalNumber int64 `yaml:"original_number"` Title string PosterName string `yaml:"poster_name"` PosterID int64 `yaml:"poster_id"` @@ -34,6 +33,7 @@ type PullRequest struct { Assignees []string IsLocked bool `yaml:"is_locked"` Reactions []*Reaction + Context IssueContext `yaml:"-"` } // IsForkPullRequest returns true if the pull request from a forked repository but not the same repository diff --git a/modules/migrations/base/retry_downloader.go b/modules/migrations/base/retry_downloader.go index e6c80038f181..623bfc86b526 100644 --- a/modules/migrations/base/retry_downloader.go +++ b/modules/migrations/base/retry_downloader.go @@ -182,14 +182,14 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo } // GetReviews returns pull requests reviews -func (d *RetryDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) { +func (d *RetryDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) { var ( reviews []*Review err error ) err = d.retry(func() error { - reviews, err = d.Downloader.GetReviews(pullRequestNumber) + reviews, err = d.Downloader.GetReviews(pullRequestContext) return err }) diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go index 23ede93a4208..b947ee74a422 100644 --- a/modules/migrations/gitea_downloader.go +++ b/modules/migrations/gitea_downloader.go @@ -444,6 +444,7 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err Labels: labels, Assignees: assignees, IsLocked: issue.IsLocked, + Context: base.BasicIssueContext(issue.Index), }) } @@ -466,26 +467,26 @@ func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comm default: } - comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.IssueNumber, gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ + comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ // PageSize: g.maxPerPage, // Page: i, }}) if err != nil { - return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.IssueNumber, err) + return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.Context.ForeignID(), err) } for _, comment := range comments { reactions, err := g.getCommentReactions(comment.ID) if err != nil { - log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.IssueNumber, comment.ID, g.repoOwner, g.repoName, err) + log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err) if err2 := models.CreateRepositoryNotice( - fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.IssueNumber, comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { + fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { log.Error("create repository notice failed: ", err2) } } allComments = append(allComments, &base.Comment{ - IssueIndex: opts.IssueNumber, + IssueIndex: opts.Context.LocalID(), PosterID: comment.Poster.ID, PosterName: comment.Poster.UserName, PosterEmail: comment.Poster.Email, @@ -615,6 +616,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques RepoName: g.repoName, OwnerName: g.repoOwner, }, + Context: base.BasicIssueContext(pr.Index), }) } @@ -626,7 +628,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques } // GetReviews returns pull requests review -func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) { +func (g *GiteaDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) { if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil { log.Info("GiteaDownloader: instance to old, skip GetReviews") return nil, nil @@ -642,7 +644,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) { default: } - prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, index, gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{ + prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, context.ForeignID(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{ Page: i, PageSize: g.maxPerPage, }}) @@ -652,7 +654,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) { for _, pr := range prl { - rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, index, pr.ID) + rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, context.ForeignID(), pr.ID) if err != nil { return nil, err } @@ -678,7 +680,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) { allReviews = append(allReviews, &base.Review{ ID: pr.ID, - IssueIndex: index, + IssueIndex: context.LocalID(), ReviewerID: pr.Reviewer.ID, ReviewerName: pr.Reviewer.UserName, Official: pr.Official, diff --git a/modules/migrations/gitea_downloader_test.go b/modules/migrations/gitea_downloader_test.go index 7ce8aa0e0bd9..71bdecaeaddd 100644 --- a/modules/migrations/gitea_downloader_test.go +++ b/modules/migrations/gitea_downloader_test.go @@ -199,7 +199,7 @@ func TestGiteaDownloadRepo(t *testing.T) { }, issues) comments, _, err := downloader.GetComments(base.GetCommentOptions{ - IssueNumber: 4, + Context: base.BasicIssueContext(4), }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ @@ -265,7 +265,7 @@ func TestGiteaDownloadRepo(t *testing.T) { PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch", }, prs[1]) - reviews, err := downloader.GetReviews(7) + reviews, err := downloader.GetReviews(base.BasicIssueContext(7)) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index 01fb9bda8af4..c77ace797bc0 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -609,6 +609,9 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR // download patch file err := func() error { + if pr.PatchURL == "" { + return nil + } // pr.PatchURL maybe a local file ret, err := uri.Open(pr.PatchURL) if err != nil { diff --git a/modules/migrations/github.go b/modules/migrations/github.go index f6063b06613e..54af10d116aa 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -428,6 +428,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, Closed: issue.ClosedAt, IsLocked: issue.GetLocked(), Assignees: assignees, + Context: base.BasicIssueContext(*issue.Number), }) } @@ -441,15 +442,15 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool { // GetComments returns comments according issueNumber func (g *GithubDownloaderV3) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - if opts.IssueNumber > 0 { - comments, err := g.getComments(opts.IssueNumber) + if opts.Context != nil { + comments, err := g.getComments(opts.Context) return comments, false, err } return g.GetAllComments(opts.Page, opts.PageSize) } -func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, error) { +func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*base.Comment, error) { var ( allComments = make([]*base.Comment, 0, g.maxPerPage) created = "created" @@ -464,7 +465,7 @@ func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, er } for { g.sleep() - comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt) + comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } @@ -495,7 +496,7 @@ func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, er } allComments = append(allComments, &base.Comment{ - IssueIndex: issueNumber, + IssueIndex: issueContext.LocalID(), PosterID: comment.GetUser().GetID(), PosterName: comment.GetUser().GetLogin(), PosterEmail: comment.GetUser().GetEmail(), @@ -661,6 +662,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq }, PatchURL: pr.GetPatchURL(), Reactions: reactions, + Context: base.BasicIssueContext(*pr.Number), }) } @@ -724,28 +726,28 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques } // GetReviews returns pull requests review -func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { +func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Review, error) { var allReviews = make([]*base.Review, 0, g.maxPerPage) opt := &github.ListOptions{ PerPage: g.maxPerPage, } for { g.sleep() - reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(pullRequestNumber), opt) + reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } g.rate = &resp.Rate for _, review := range reviews { r := convertGithubReview(review) - r.IssueIndex = pullRequestNumber + r.IssueIndex = context.LocalID() // retrieve all review comments opt2 := &github.ListOptions{ PerPage: g.maxPerPage, } for { g.sleep() - reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(pullRequestNumber), review.GetID(), opt2) + reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } diff --git a/modules/migrations/github_test.go b/modules/migrations/github_test.go index 16d79d200cae..4a53f20a76dd 100644 --- a/modules/migrations/github_test.go +++ b/modules/migrations/github_test.go @@ -216,7 +216,7 @@ func TestGitHubDownloadRepo(t *testing.T) { // downloader.GetComments() comments, _, err := downloader.GetComments(base.GetCommentOptions{ - IssueNumber: 2, + Context: base.BasicIssueContext(2), }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ @@ -286,6 +286,7 @@ func TestGitHubDownloadRepo(t *testing.T) { Merged: true, MergedTime: timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)), MergeCommitSHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2", + Context: base.BasicIssueContext(3), }, { Number: 4, @@ -332,10 +333,11 @@ func TestGitHubDownloadRepo(t *testing.T) { Content: "+1", }, }, + Context: base.BasicIssueContext(4), }, }, prs) - reviews, err := downloader.GetReviews(3) + reviews, err := downloader.GetReviews(base.BasicIssueContext(3)) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -367,7 +369,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, reviews) - reviews, err = downloader.GetReviews(4) + reviews, err = downloader.GetReviews(base.BasicIssueContext(4)) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go index 28e9eac63c00..d5bf2d2d762d 100644 --- a/modules/migrations/gitlab.go +++ b/modules/migrations/gitlab.go @@ -63,17 +63,14 @@ func (f *GitlabDownloaderFactory) GitServiceType() structs.GitServiceType { // from gitlab via go-gitlab // - issueCount is incremented in GetIssues() to ensure PR and Issue numbers do not overlap, // because Gitlab has individual Issue and Pull Request numbers. -// - issueSeen, working alongside issueCount, is checked in GetComments() to see whether we -// need to fetch the Issue or PR comments, as Gitlab stores them separately. type GitlabDownloader struct { base.NullDownloader - ctx context.Context - client *gitlab.Client - repoID int - repoName string - issueCount int64 - fetchPRcomments bool - maxPerPage int + ctx context.Context + client *gitlab.Client + repoID int + repoName string + issueCount int64 + maxPerPage int } // NewGitlabDownloader creates a gitlab Downloader via gitlab API @@ -364,6 +361,20 @@ func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) { return releases, nil } +type gitlabIssueContext struct { + foreignID int64 + localID int64 + IsMergeRequest bool +} + +func (c gitlabIssueContext) LocalID() int64 { + return c.localID +} + +func (c gitlabIssueContext) ForeignID() int64 { + return c.foreignID +} + // GetIssues returns issues according start and limit // Note: issue label description and colors are not supported by the go-gitlab library at this time func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { @@ -433,6 +444,11 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er Closed: issue.ClosedAt, IsLocked: issue.DiscussionLocked, Updated: *issue.UpdatedAt, + Context: gitlabIssueContext{ + foreignID: int64(issue.IID), + localID: int64(issue.IID), + IsMergeRequest: false, + }, }) // increment issueCount, to be used in GetPullRequests() @@ -445,27 +461,26 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er // GetComments returns comments according issueNumber // TODO: figure out how to transfer comment reactions func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - var issueNumber = opts.IssueNumber + context, ok := opts.Context.(gitlabIssueContext) + if !ok { + return nil, false, fmt.Errorf("unexpected context: %+v", opts.Context) + } + var allComments = make([]*base.Comment, 0, g.maxPerPage) var page = 1 - var realIssueNumber int64 for { var comments []*gitlab.Discussion var resp *gitlab.Response var err error - // fetchPRcomments decides whether to fetch Issue or PR comments - if !g.fetchPRcomments { - realIssueNumber = issueNumber - comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListIssueDiscussionsOptions{ + if !context.IsMergeRequest { + comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(context.ForeignID()), &gitlab.ListIssueDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) } else { - // If this is a PR, we need to figure out the Gitlab/original PR ID to be passed below - realIssueNumber = issueNumber - g.issueCount - comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListMergeRequestDiscussionsOptions{ + comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(context.ForeignID()), &gitlab.ListMergeRequestDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) @@ -479,7 +494,7 @@ func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com if !comment.IndividualNote { for _, note := range comment.Notes { allComments = append(allComments, &base.Comment{ - IssueIndex: realIssueNumber, + IssueIndex: context.LocalID(), PosterID: int64(note.Author.ID), PosterName: note.Author.Username, PosterEmail: note.Author.Email, @@ -490,7 +505,7 @@ func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com } else { c := comment.Notes[0] allComments = append(allComments, &base.Comment{ - IssueIndex: realIssueNumber, + IssueIndex: context.LocalID(), PosterID: int64(c.Author.ID), PosterName: c.Author.Username, PosterEmail: c.Author.Email, @@ -521,9 +536,6 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque }, } - // Set fetchPRcomments to true here, so PR comments are fetched instead of Issue comments - g.fetchPRcomments = true - var allPRs = make([]*base.PullRequest, 0, perPage) prs, _, err := g.client.MergeRequests.ListProjectMergeRequests(g.repoID, opt, nil, gitlab.WithContext(g.ctx)) @@ -587,7 +599,6 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque allPRs = append(allPRs, &base.PullRequest{ Title: pr.Title, Number: newPRNumber, - OriginalNumber: int64(pr.IID), PosterName: pr.Author.Username, PosterID: int64(pr.Author.ID), Content: pr.Description, @@ -615,6 +626,11 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque OwnerName: pr.Author.Username, }, PatchURL: pr.WebURL + ".patch", + Context: gitlabIssueContext{ + foreignID: int64(pr.IID), + localID: newPRNumber, + IsMergeRequest: true, + }, }) } @@ -622,8 +638,8 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests review -func (g *GitlabDownloader) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { - approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(pullRequestNumber), gitlab.WithContext(g.ctx)) +func (g *GitlabDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) { + approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(context.ForeignID()), gitlab.WithContext(g.ctx)) if err != nil { if resp != nil && resp.StatusCode == 404 { log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error())) @@ -635,6 +651,7 @@ func (g *GitlabDownloader) GetReviews(pullRequestNumber int64) ([]*base.Review, var reviews = make([]*base.Review, 0, len(approvals.ApprovedBy)) for _, user := range approvals.ApprovedBy { reviews = append(reviews, &base.Review{ + IssueIndex: context.LocalID(), ReviewerID: int64(user.User.ID), ReviewerName: user.User.Username, CreatedAt: *approvals.UpdatedAt, diff --git a/modules/migrations/gitlab_test.go b/modules/migrations/gitlab_test.go index 8fd915e0f51b..c3ee8118c55d 100644 --- a/modules/migrations/gitlab_test.go +++ b/modules/migrations/gitlab_test.go @@ -210,7 +210,11 @@ func TestGitlabDownloadRepo(t *testing.T) { }, issues) comments, _, err := downloader.GetComments(base.GetCommentOptions{ - IssueNumber: 2, + Context: gitlabIssueContext{ + foreignID: 2, + localID: 2, + IsMergeRequest: false, + }, }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ @@ -252,15 +256,14 @@ func TestGitlabDownloadRepo(t *testing.T) { assert.NoError(t, err) assertPullRequestsEqual(t, []*base.PullRequest{ { - Number: 4, - OriginalNumber: 2, - Title: "Test branch", - Content: "do not merge this PR", - Milestone: "1.0.0", - PosterID: 1241334, - PosterName: "lafriks", - State: "opened", - Created: time.Date(2019, 11, 28, 15, 56, 54, 104000000, time.UTC), + Number: 4, + Title: "Test branch", + Content: "do not merge this PR", + Milestone: "1.0.0", + PosterID: 1241334, + PosterName: "lafriks", + State: "opened", + Created: time.Date(2019, 11, 28, 15, 56, 54, 104000000, time.UTC), Labels: []*base.Label{ { Name: "bug", @@ -293,10 +296,15 @@ func TestGitlabDownloadRepo(t *testing.T) { Merged: false, MergedTime: nil, MergeCommitSHA: "", + Context: gitlabIssueContext{ + foreignID: 2, + localID: 4, + IsMergeRequest: true, + }, }, }, prs) - rvs, err := downloader.GetReviews(1) + rvs, err := downloader.GetReviews(base.BasicIssueContext(1)) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -313,7 +321,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, rvs) - rvs, err = downloader.GetReviews(2) + rvs, err = downloader.GetReviews(base.BasicIssueContext(2)) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/modules/migrations/gogs.go b/modules/migrations/gogs.go index 2c7fa76146ad..06c944278b43 100644 --- a/modules/migrations/gogs.go +++ b/modules/migrations/gogs.go @@ -228,10 +228,9 @@ func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, // GetComments returns comments according issueNumber func (g *GogsDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - var issueNumber = opts.IssueNumber var allComments = make([]*base.Comment, 0, 100) - comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, issueNumber) + comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID()) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } @@ -240,7 +239,7 @@ func (g *GogsDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comme continue } allComments = append(allComments, &base.Comment{ - IssueIndex: issueNumber, + IssueIndex: opts.Context.LocalID(), PosterID: comment.Poster.ID, PosterName: comment.Poster.Login, PosterEmail: comment.Poster.Email, @@ -304,6 +303,7 @@ func convertGogsIssue(issue *gogs.Issue) *base.Issue { Updated: issue.Updated, Labels: labels, Closed: closed, + Context: base.BasicIssueContext(issue.Index), } } diff --git a/modules/migrations/gogs_test.go b/modules/migrations/gogs_test.go index e5bd634c550a..8816fab44fbc 100644 --- a/modules/migrations/gogs_test.go +++ b/modules/migrations/gogs_test.go @@ -112,7 +112,7 @@ func TestGogsDownloadRepo(t *testing.T) { // downloader.GetComments() comments, _, err := downloader.GetComments(base.GetCommentOptions{ - IssueNumber: 1, + Context: base.BasicIssueContext(1), }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 0a507d9c3341..7d5aa9670b1e 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -318,7 +318,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts for _, issue := range issues { log.Trace("migrating issue %d's comments", issue.Number) comments, _, err := downloader.GetComments(base.GetCommentOptions{ - IssueNumber: issue.Number, + Context: issue.Context, }) if err != nil { if !base.IsErrNotSupported(err) { @@ -376,7 +376,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts for _, pr := range prs { log.Trace("migrating pull request %d's comments", pr.Number) comments, _, err := downloader.GetComments(base.GetCommentOptions{ - IssueNumber: pr.Number, + Context: pr.Context, }) if err != nil { if !base.IsErrNotSupported(err) { @@ -404,14 +404,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts // migrate reviews var allReviews = make([]*base.Review, 0, reviewBatchSize) for _, pr := range prs { - number := pr.Number - - // on gitlab migrations pull number change - if pr.OriginalNumber > 0 { - number = pr.OriginalNumber - } - - reviews, err := downloader.GetReviews(number) + reviews, err := downloader.GetReviews(pr.Context) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -419,11 +412,6 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts log.Warn("migrating reviews is not supported, ignored") break } - if pr.OriginalNumber > 0 { - for i := range reviews { - reviews[i].IssueIndex = pr.Number - } - } allReviews = append(allReviews, reviews...) diff --git a/modules/migrations/onedev.go b/modules/migrations/onedev.go new file mode 100644 index 000000000000..e60265895fbc --- /dev/null +++ b/modules/migrations/onedev.go @@ -0,0 +1,619 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/structs" +) + +var ( + _ base.Downloader = &OneDevDownloader{} + _ base.DownloaderFactory = &OneDevDownloaderFactory{} +) + +func init() { + RegisterDownloaderFactory(&OneDevDownloaderFactory{}) +} + +// OneDevDownloaderFactory defines a downloader factory +type OneDevDownloaderFactory struct { +} + +// New returns a downloader related to this factory according MigrateOptions +func (f *OneDevDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) { + u, err := url.Parse(opts.CloneAddr) + if err != nil { + return nil, err + } + + repoName := "" + + fields := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(fields) == 2 && fields[0] == "projects" { + repoName = fields[1] + } else if len(fields) == 1 { + repoName = fields[0] + } else { + return nil, fmt.Errorf("invalid path: %s", u.Path) + } + + u.Path = "" + u.Fragment = "" + + log.Trace("Create onedev downloader. BaseURL: %v RepoName: %s", u, repoName) + + return NewOneDevDownloader(ctx, u, opts.AuthUsername, opts.AuthPassword, repoName), nil +} + +// GitServiceType returns the type of git service +func (f *OneDevDownloaderFactory) GitServiceType() structs.GitServiceType { + return structs.OneDevService +} + +type onedevUser struct { + ID int64 `json:"id"` + Name string `json:"name"` + Email string `json:"email"` +} + +// OneDevDownloader implements a Downloader interface to get repository informations +// from OneDev +type OneDevDownloader struct { + base.NullDownloader + ctx context.Context + client *http.Client + baseURL *url.URL + repoName string + repoID int64 + maxIssueIndex int64 + userMap map[int64]*onedevUser + milestoneMap map[int64]string +} + +// SetContext set context +func (d *OneDevDownloader) SetContext(ctx context.Context) { + d.ctx = ctx +} + +// NewOneDevDownloader creates a new downloader +func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader { + var downloader = &OneDevDownloader{ + ctx: ctx, + baseURL: baseURL, + repoName: repoName, + client: &http.Client{ + Transport: &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + if len(username) > 0 && len(password) > 0 { + req.SetBasicAuth(username, password) + } + return nil, nil + }, + }, + }, + userMap: make(map[int64]*onedevUser), + milestoneMap: make(map[int64]string), + } + + return downloader +} + +func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result interface{}) error { + u, err := d.baseURL.Parse(endpoint) + if err != nil { + return err + } + + if parameter != nil { + query := u.Query() + for k, v := range parameter { + query.Set(k, v) + } + u.RawQuery = query.Encode() + } + + req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil) + if err != nil { + return err + } + + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + decoder := json.NewDecoder(resp.Body) + return decoder.Decode(&result) +} + +// GetRepoInfo returns repository information +func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) { + info := make([]struct { + ID int64 `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + }, 0, 1) + + err := d.callAPI( + "/api/projects", + map[string]string{ + "query": `"Name" is "` + d.repoName + `"`, + "offset": "0", + "count": "1", + }, + &info, + ) + if err != nil { + return nil, err + } + if len(info) != 1 { + return nil, fmt.Errorf("Project %s not found", d.repoName) + } + + d.repoID = info[0].ID + + cloneURL, err := d.baseURL.Parse(info[0].Name) + if err != nil { + return nil, err + } + originalURL, err := d.baseURL.Parse("/projects/" + info[0].Name) + if err != nil { + return nil, err + } + + return &base.Repository{ + Name: info[0].Name, + Description: info[0].Description, + CloneURL: cloneURL.String(), + OriginalURL: originalURL.String(), + }, nil +} + +// GetMilestones returns milestones +func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) { + rawMilestones := make([]struct { + ID int64 `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + DueDate *time.Time `json:"dueDate"` + Closed bool `json:"closed"` + }, 0, 100) + + endpoint := fmt.Sprintf("/api/projects/%d/milestones", d.repoID) + + var milestones = make([]*base.Milestone, 0, 100) + offset := 0 + for { + err := d.callAPI( + endpoint, + map[string]string{ + "offset": strconv.Itoa(offset), + "count": "100", + }, + &rawMilestones, + ) + if err != nil { + return nil, err + } + if len(rawMilestones) == 0 { + break + } + offset += 100 + + for _, milestone := range rawMilestones { + d.milestoneMap[milestone.ID] = milestone.Name + closed := milestone.DueDate + if !milestone.Closed { + closed = nil + } + + milestones = append(milestones, &base.Milestone{ + Title: milestone.Name, + Description: milestone.Description, + Deadline: milestone.DueDate, + Closed: closed, + }) + } + } + return milestones, nil +} + +// GetLabels returns labels +func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) { + return []*base.Label{ + { + Name: "Bug", + Color: "f64e60", + }, + { + Name: "Build Failure", + Color: "f64e60", + }, + { + Name: "Discussion", + Color: "8950fc", + }, + { + Name: "Improvement", + Color: "1bc5bd", + }, + { + Name: "New Feature", + Color: "1bc5bd", + }, + { + Name: "Support Request", + Color: "8950fc", + }, + }, nil +} + +type onedevIssueContext struct { + foreignID int64 + localID int64 + IsPullRequest bool +} + +func (c onedevIssueContext) LocalID() int64 { + return c.localID +} + +func (c onedevIssueContext) ForeignID() int64 { + return c.foreignID +} + +// GetIssues returns issues +func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { + rawIssues := make([]struct { + ID int64 `json:"id"` + Number int64 `json:"number"` + State string `json:"state"` + Title string `json:"title"` + Description string `json:"description"` + MilestoneID int64 `json:"milestoneId"` + SubmitterID int64 `json:"submitterId"` + SubmitDate time.Time `json:"submitDate"` + }, 0, perPage) + + err := d.callAPI( + "/api/issues", + map[string]string{ + "query": `"Project" is "` + d.repoName + `"`, + "offset": strconv.Itoa((page - 1) * perPage), + "count": strconv.Itoa(perPage), + }, + &rawIssues, + ) + if err != nil { + return nil, false, err + } + + issues := make([]*base.Issue, 0, len(rawIssues)) + for _, issue := range rawIssues { + fields := make([]struct { + Name string `json:"name"` + Value string `json:"value"` + }, 0, 10) + err := d.callAPI( + fmt.Sprintf("/api/issues/%d/fields", issue.ID), + nil, + &fields, + ) + if err != nil { + return nil, false, err + } + + var label *base.Label + for _, field := range fields { + if field.Name == "Type" { + label = &base.Label{Name: field.Value} + break + } + } + + state := strings.ToLower(issue.State) + if state == "released" { + state = "closed" + } + poster := d.tryGetUser(issue.SubmitterID) + issues = append(issues, &base.Issue{ + Title: issue.Title, + Number: issue.Number, + PosterName: poster.Name, + PosterEmail: poster.Email, + Content: issue.Description, + Milestone: d.milestoneMap[issue.MilestoneID], + State: state, + Created: issue.SubmitDate, + Updated: issue.SubmitDate, + Labels: []*base.Label{label}, + Context: onedevIssueContext{ + foreignID: issue.ID, + localID: issue.Number, + IsPullRequest: false, + }, + }) + + if d.maxIssueIndex < issue.Number { + d.maxIssueIndex = issue.Number + } + } + + return issues, len(issues) == 0, nil +} + +// GetComments returns comments +func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + context, ok := opts.Context.(onedevIssueContext) + if !ok { + return nil, false, fmt.Errorf("unexpected comment context: %+v", opts.Context) + } + + rawComments := make([]struct { + Date time.Time `json:"date"` + UserID int64 `json:"userId"` + Content string `json:"content"` + }, 0, 100) + + var endpoint string + if context.IsPullRequest { + endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", context.ForeignID()) + } else { + endpoint = fmt.Sprintf("/api/issues/%d/comments", context.ForeignID()) + } + + err := d.callAPI( + endpoint, + nil, + &rawComments, + ) + if err != nil { + return nil, false, err + } + + rawChanges := make([]struct { + Date time.Time `json:"date"` + UserID int64 `json:"userId"` + Data map[string]interface{} `json:"data"` + }, 0, 100) + + if context.IsPullRequest { + endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", context.ForeignID()) + } else { + endpoint = fmt.Sprintf("/api/issues/%d/changes", context.ForeignID()) + } + + err = d.callAPI( + endpoint, + nil, + &rawChanges, + ) + if err != nil { + return nil, false, err + } + + comments := make([]*base.Comment, 0, len(rawComments)+len(rawChanges)) + for _, comment := range rawComments { + if len(comment.Content) == 0 { + continue + } + poster := d.tryGetUser(comment.UserID) + comments = append(comments, &base.Comment{ + IssueIndex: context.LocalID(), + PosterID: poster.ID, + PosterName: poster.Name, + PosterEmail: poster.Email, + Content: comment.Content, + Created: comment.Date, + Updated: comment.Date, + }) + } + for _, change := range rawChanges { + contentV, ok := change.Data["content"] + if !ok { + contentV, ok = change.Data["comment"] + if !ok { + continue + } + } + content, ok := contentV.(string) + if !ok || len(content) == 0 { + continue + } + + poster := d.tryGetUser(change.UserID) + comments = append(comments, &base.Comment{ + IssueIndex: context.LocalID(), + PosterID: poster.ID, + PosterName: poster.Name, + PosterEmail: poster.Email, + Content: content, + Created: change.Date, + Updated: change.Date, + }) + } + + return comments, true, nil +} + +// GetPullRequests returns pull requests +func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { + rawPullRequests := make([]struct { + ID int64 `json:"id"` + Number int64 `json:"number"` + Title string `json:"title"` + SubmitterID int64 `json:"submitterId"` + SubmitDate time.Time `json:"submitDate"` + Description string `json:"description"` + TargetBranch string `json:"targetBranch"` + SourceBranch string `json:"sourceBranch"` + BaseCommitHash string `json:"baseCommitHash"` + CloseInfo *struct { + Date *time.Time `json:"date"` + Status string `json:"status"` + } + }, 0, perPage) + + err := d.callAPI( + "/api/pull-requests", + map[string]string{ + "query": `"Target Project" is "` + d.repoName + `"`, + "offset": strconv.Itoa((page - 1) * perPage), + "count": strconv.Itoa(perPage), + }, + &rawPullRequests, + ) + if err != nil { + return nil, false, err + } + + pullRequests := make([]*base.PullRequest, 0, len(rawPullRequests)) + for _, pr := range rawPullRequests { + var mergePreview struct { + TargetHeadCommitHash string `json:"targetHeadCommitHash"` + HeadCommitHash string `json:"headCommitHash"` + MergeStrategy string `json:"mergeStrategy"` + MergeCommitHash string `json:"mergeCommitHash"` + } + err := d.callAPI( + fmt.Sprintf("/api/pull-requests/%d/merge-preview", pr.ID), + nil, + &mergePreview, + ) + if err != nil { + return nil, false, err + } + + state := "open" + merged := false + var closeTime *time.Time + var mergedTime *time.Time + if pr.CloseInfo != nil { + state = "closed" + closeTime = pr.CloseInfo.Date + if pr.CloseInfo.Status == "MERGED" { // "DISCARDED" + merged = true + mergedTime = pr.CloseInfo.Date + } + } + poster := d.tryGetUser(pr.SubmitterID) + + number := pr.Number + d.maxIssueIndex + pullRequests = append(pullRequests, &base.PullRequest{ + Title: pr.Title, + Number: number, + PosterName: poster.Name, + PosterID: poster.ID, + Content: pr.Description, + State: state, + Created: pr.SubmitDate, + Updated: pr.SubmitDate, + Closed: closeTime, + Merged: merged, + MergedTime: mergedTime, + Head: base.PullRequestBranch{ + Ref: pr.SourceBranch, + SHA: mergePreview.HeadCommitHash, + RepoName: d.repoName, + }, + Base: base.PullRequestBranch{ + Ref: pr.TargetBranch, + SHA: mergePreview.TargetHeadCommitHash, + RepoName: d.repoName, + }, + Context: onedevIssueContext{ + foreignID: pr.ID, + localID: number, + IsPullRequest: true, + }, + }) + } + + return pullRequests, len(pullRequests) == 0, nil +} + +// GetReviews returns pull requests reviews +func (d *OneDevDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) { + rawReviews := make([]struct { + ID int64 `json:"id"` + UserID int64 `json:"userId"` + Result *struct { + Commit string `json:"commit"` + Approved bool `json:"approved"` + Comment string `json:"comment"` + } + }, 0, 100) + + err := d.callAPI( + fmt.Sprintf("/api/pull-requests/%d/reviews", context.ForeignID()), + nil, + &rawReviews, + ) + if err != nil { + return nil, err + } + + var reviews = make([]*base.Review, 0, len(rawReviews)) + for _, review := range rawReviews { + state := base.ReviewStatePending + content := "" + if review.Result != nil { + if len(review.Result.Comment) > 0 { + state = base.ReviewStateCommented + content = review.Result.Comment + } + if review.Result.Approved { + state = base.ReviewStateApproved + } + } + + poster := d.tryGetUser(review.UserID) + reviews = append(reviews, &base.Review{ + IssueIndex: context.LocalID(), + ReviewerID: poster.ID, + ReviewerName: poster.Name, + Content: content, + State: state, + }) + } + + return reviews, nil +} + +// GetTopics return repository topics +func (d *OneDevDownloader) GetTopics() ([]string, error) { + return []string{}, nil +} + +func (d *OneDevDownloader) tryGetUser(userID int64) *onedevUser { + user, ok := d.userMap[userID] + if !ok { + err := d.callAPI( + fmt.Sprintf("/api/users/%d", userID), + nil, + &user, + ) + if err != nil { + user = &onedevUser{ + Name: fmt.Sprintf("User %d", userID), + } + } + d.userMap[userID] = user + } + + return user +} diff --git a/modules/migrations/onedev_test.go b/modules/migrations/onedev_test.go new file mode 100644 index 000000000000..48d56c3e22b3 --- /dev/null +++ b/modules/migrations/onedev_test.go @@ -0,0 +1,169 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "context" + "fmt" + "net/http" + "net/url" + "testing" + "time" + + "code.gitea.io/gitea/modules/migrations/base" + + "github.com/stretchr/testify/assert" +) + +func TestOneDevDownloadRepo(t *testing.T) { + resp, err := http.Get("https://code.onedev.io/projects/go-gitea-test_repo") + if err != nil || resp.StatusCode != 200 { + t.Skipf("Can't access test repo, skipping %s", t.Name()) + } + + u, _ := url.Parse("https://code.onedev.io") + downloader := NewOneDevDownloader(context.Background(), u, "", "", "go-gitea-test_repo") + if err != nil { + t.Fatal(fmt.Sprintf("NewOneDevDownloader is nil: %v", err)) + } + repo, err := downloader.GetRepoInfo() + assert.NoError(t, err) + assert.EqualValues(t, &base.Repository{ + Name: "go-gitea-test_repo", + Owner: "", + Description: "Test repository for testing migration from OneDev to gitea", + CloneURL: "https://code.onedev.io/go-gitea-test_repo", + OriginalURL: "https://code.onedev.io/projects/go-gitea-test_repo", + }, repo) + + milestones, err := downloader.GetMilestones() + assert.NoError(t, err) + assert.Len(t, milestones, 2) + deadline := time.Unix(1620086400, 0) + assert.EqualValues(t, []*base.Milestone{ + { + Title: "1.0.0", + Deadline: &deadline, + Closed: &deadline, + }, + { + Title: "1.1.0", + Description: "next things?", + }, + }, milestones) + + labels, err := downloader.GetLabels() + assert.NoError(t, err) + assert.Len(t, labels, 6) + + issues, isEnd, err := downloader.GetIssues(1, 2) + assert.NoError(t, err) + assert.Len(t, issues, 2) + assert.False(t, isEnd) + assert.EqualValues(t, []*base.Issue{ + { + Number: 4, + Title: "Hi there", + Content: "an issue not assigned to a milestone", + PosterName: "User 336", + State: "open", + Created: time.Unix(1628549776, 734000000), + Updated: time.Unix(1628549776, 734000000), + Labels: []*base.Label{ + { + Name: "Improvement", + }, + }, + Context: onedevIssueContext{ + foreignID: 398, + localID: 4, + IsPullRequest: false, + }, + }, + { + Number: 3, + Title: "Add an awesome feature", + Content: "just another issue to test against", + PosterName: "User 336", + State: "open", + Milestone: "1.1.0", + Created: time.Unix(1628549749, 878000000), + Updated: time.Unix(1628549749, 878000000), + Labels: []*base.Label{ + { + Name: "New Feature", + }, + }, + Context: onedevIssueContext{ + foreignID: 397, + localID: 3, + IsPullRequest: false, + }, + }, + }, issues) + + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Context: onedevIssueContext{ + foreignID: 398, + localID: 4, + IsPullRequest: false, + }, + }) + assert.NoError(t, err) + assert.Len(t, comments, 1) + assert.EqualValues(t, []*base.Comment{ + { + IssueIndex: 4, + PosterName: "User 336", + Created: time.Unix(1628549791, 128000000), + Updated: time.Unix(1628549791, 128000000), + Content: "it has a comment\r\n\r\nEDIT: that got edited", + }, + }, comments) + + prs, _, err := downloader.GetPullRequests(1, 1) + assert.NoError(t, err) + assert.Len(t, prs, 1) + assert.EqualValues(t, []*base.PullRequest{ + { + Number: 5, + Title: "Pull to add a new file", + Content: "just do some git stuff", + PosterName: "User 336", + State: "open", + Created: time.Unix(1628550076, 25000000), + Updated: time.Unix(1628550076, 25000000), + Head: base.PullRequestBranch{ + Ref: "branch-for-a-pull", + SHA: "343deffe3526b9bc84e873743ff7f6e6d8b827c0", + RepoName: "go-gitea-test_repo", + }, + Base: base.PullRequestBranch{ + Ref: "master", + SHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2", + RepoName: "go-gitea-test_repo", + }, + Context: onedevIssueContext{ + foreignID: 186, + localID: 5, + IsPullRequest: true, + }, + }, + }, prs) + + rvs, err := downloader.GetReviews(onedevIssueContext{ + foreignID: 186, + localID: 5, + }) + assert.NoError(t, err) + assert.Len(t, rvs, 1) + assert.EqualValues(t, []*base.Review{ + { + IssueIndex: 5, + ReviewerName: "User 317", + State: "PENDING", + }, + }, rvs) +} diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 6177f80cbbca..6287d601c2a0 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -208,13 +208,16 @@ func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, if err != nil { return nil, false, err } + for _, issue := range issues { + issue.Context = base.BasicIssueContext(issue.Number) + } return issues, true, nil } // GetComments returns comments according issueNumber func (r *RepositoryRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { var comments = make([]*base.Comment, 0, 10) - p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", opts.IssueNumber)) + p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", opts.Context.ForeignID())) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { @@ -258,14 +261,15 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq } for _, pr := range pulls { pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL) + pr.Context = base.BasicIssueContext(pr.Number) } return pulls, true, nil } // GetReviews returns pull requests review -func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { +func (r *RepositoryRestorer) GetReviews(context base.IssueContext) ([]*base.Review, error) { var reviews = make([]*base.Review, 0, 10) - p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", pullRequestNumber)) + p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", context.ForeignID())) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 2089f4d69cd0..313a982f43d6 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -248,6 +248,7 @@ const ( GiteaService // 3 gitea service GitlabService // 4 gitlab service GogsService // 5 gogs service + OneDevService // 6 onedev service ) // Name represents the service type's name @@ -267,6 +268,8 @@ func (gt GitServiceType) Title() string { return "GitLab" case GogsService: return "Gogs" + case OneDevService: + return "OneDev" case PlainGitService: return "Git" } @@ -322,5 +325,6 @@ var ( GitlabService, GiteaService, GogsService, + OneDevService, } ) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 4715afcd3e54..3eb38257768a 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -904,6 +904,7 @@ migrate.git.description = Migrating or Mirroring git data from Git services migrate.gitlab.description = Migrating data from GitLab.com or Self-Hosted gitlab server. migrate.gitea.description = Migrating data from Gitea.com or Self-Hosted Gitea server. migrate.gogs.description = Migrating data from notabug.org or other Self-Hosted Gogs server. +migrate.onedev.description = Migrating data from code.onedev.io or Self-Hosted OneDev server. migrate.migrating_git = Migrating Git Data migrate.migrating_topics = Migrating Topics migrate.migrating_milestones = Migrating Milestones diff --git a/public/img/svg/gitea-onedev.svg b/public/img/svg/gitea-onedev.svg new file mode 100644 index 000000000000..1f0d1d8363b4 --- /dev/null +++ b/public/img/svg/gitea-onedev.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/templates/repo/migrate/onedev.tmpl b/templates/repo/migrate/onedev.tmpl new file mode 100644 index 000000000000..def366f9d8da --- /dev/null +++ b/templates/repo/migrate/onedev.tmpl @@ -0,0 +1,117 @@ +{{template "base/head" .}} +
+
+
+
+ {{.CsrfTokenHtml}} +

+ {{.i18n.Tr "repo.migrate.migrate" .service.Title}} + +

+
+ {{template "base/alert" .}} +
+ + + + {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} + +
+ +
+ + +
+ +
+ + +
+ + {{template "repo/migrate/options" .}} + +
+
+ +
+ + +
+
+ + +
+
+
+ +
+ + +
+
+ + +
+
+
+ +
+ +
+ +
+ + + {{avatar .ContextUser 28 "mini"}} + {{.ContextUser.ShortName 40}} + + {{svg "octicon-triangle-down" 14 "dropdown icon"}} +
+
+ {{avatar .SignedUser 28 "mini"}} + {{.SignedUser.ShortName 40}} +
+ {{range .Orgs}} +
+ {{avatar . 28 "mini"}} + {{.ShortName 40}} +
+ {{end}} +
+
+
+ +
+ + +
+
+ +
+ {{if .IsForcedPrivate}} + + + {{else}} + + + {{end}} +
+
+
+ + +
+ +
+ + + {{.i18n.Tr "cancel"}} +
+
+
+
+
+
+{{template "base/footer" .}} diff --git a/web_src/svg/gitea-onedev.svg b/web_src/svg/gitea-onedev.svg new file mode 100644 index 000000000000..490c22fc8e2f --- /dev/null +++ b/web_src/svg/gitea-onedev.svg @@ -0,0 +1,42 @@ + + + + + + + + + + + \ No newline at end of file