forked from gitea/gitea
* Add logging to long migrations Also fix v136 to not use models Signed-off-by: Andrew Thornton <art27@cantab.net>
This commit is contained in:
parent
795b6865af
commit
d78c31c216
|
@ -8,8 +8,10 @@ import (
|
||||||
"crypto/md5"
|
"crypto/md5"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
"math"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
@ -26,8 +28,19 @@ func renameExistingUserAvatarName(x *xorm.Engine) error {
|
||||||
LowerName string `xorm:"UNIQUE NOT NULL"`
|
LowerName string `xorm:"UNIQUE NOT NULL"`
|
||||||
Avatar string
|
Avatar string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ticker := time.NewTicker(5 * time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
count, err := x.Count(new(User))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Info("%d User Avatar(s) to migrate ...", count)
|
||||||
|
|
||||||
deleteList := make(map[string]struct{})
|
deleteList := make(map[string]struct{})
|
||||||
start := 0
|
start := 0
|
||||||
|
migrated := 0
|
||||||
for {
|
for {
|
||||||
if err := sess.Begin(); err != nil {
|
if err := sess.Begin(); err != nil {
|
||||||
return fmt.Errorf("session.Begin: %v", err)
|
return fmt.Errorf("session.Begin: %v", err)
|
||||||
|
@ -73,6 +86,19 @@ func renameExistingUserAvatarName(x *xorm.Engine) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
deleteList[filepath.Join(setting.AvatarUploadPath, oldAvatar)] = struct{}{}
|
deleteList[filepath.Join(setting.AvatarUploadPath, oldAvatar)] = struct{}{}
|
||||||
|
migrated++
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
log.Info(
|
||||||
|
"%d/%d (%2.0f%%) User Avatar(s) migrated (%d old avatars to be deleted) in %d batches. %d Remaining ...",
|
||||||
|
migrated,
|
||||||
|
count,
|
||||||
|
float64(migrated)/float64(count)*100,
|
||||||
|
len(deleteList),
|
||||||
|
int(math.Ceil(float64(migrated)/float64(50))),
|
||||||
|
count-int64(migrated))
|
||||||
|
default:
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if err := sess.Commit(); err != nil {
|
if err := sess.Commit(); err != nil {
|
||||||
_ = sess.Rollback()
|
_ = sess.Rollback()
|
||||||
|
@ -80,11 +106,28 @@ func renameExistingUserAvatarName(x *xorm.Engine) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
deleteCount := len(deleteList)
|
||||||
|
log.Info("Deleting %d old avatars ...", deleteCount)
|
||||||
|
i := 0
|
||||||
for file := range deleteList {
|
for file := range deleteList {
|
||||||
if err := os.Remove(file); err != nil {
|
if err := os.Remove(file); err != nil {
|
||||||
log.Warn("os.Remove: %v", err)
|
log.Warn("os.Remove: %v", err)
|
||||||
}
|
}
|
||||||
|
i++
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
log.Info(
|
||||||
|
"%d/%d (%2.0f%%) Old User Avatar(s) deleted. %d Remaining ...",
|
||||||
|
i,
|
||||||
|
deleteCount,
|
||||||
|
float64(i)/float64(deleteCount)*100,
|
||||||
|
deleteCount-i)
|
||||||
|
default:
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("Completed migrating %d User Avatar(s) and deleting %d Old Avatars", count, deleteCount)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,8 +6,10 @@ package migrations
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"code.gitea.io/gitea/modules/git"
|
"code.gitea.io/gitea/modules/git"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
@ -43,17 +45,27 @@ func fixMergeBase(x *xorm.Engine) error {
|
||||||
limit = 50
|
limit = 50
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ticker := time.NewTicker(5 * time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
count, err := x.Count(new(PullRequest))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Info("%d Pull Request(s) to migrate ...", count)
|
||||||
|
|
||||||
i := 0
|
i := 0
|
||||||
|
start := 0
|
||||||
for {
|
for {
|
||||||
prs := make([]PullRequest, 0, 50)
|
prs := make([]PullRequest, 0, 50)
|
||||||
if err := x.Limit(limit, i).Asc("id").Find(&prs); err != nil {
|
if err := x.Limit(limit, start).Asc("id").Find(&prs); err != nil {
|
||||||
return fmt.Errorf("Find: %v", err)
|
return fmt.Errorf("Find: %v", err)
|
||||||
}
|
}
|
||||||
if len(prs) == 0 {
|
if len(prs) == 0 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
i += len(prs)
|
start += 50
|
||||||
for _, pr := range prs {
|
for _, pr := range prs {
|
||||||
baseRepo := &Repository{ID: pr.BaseRepoID}
|
baseRepo := &Repository{ID: pr.BaseRepoID}
|
||||||
has, err := x.Table("repository").Get(baseRepo)
|
has, err := x.Table("repository").Get(baseRepo)
|
||||||
|
@ -102,8 +114,14 @@ func fixMergeBase(x *xorm.Engine) error {
|
||||||
}
|
}
|
||||||
pr.MergeBase = strings.TrimSpace(pr.MergeBase)
|
pr.MergeBase = strings.TrimSpace(pr.MergeBase)
|
||||||
x.ID(pr.ID).Cols("merge_base").Update(pr)
|
x.ID(pr.ID).Cols("merge_base").Update(pr)
|
||||||
|
i++
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
log.Info("%d/%d (%2.0f%%) Pull Request(s) migrated in %d batches. %d PRs Remaining ...", i, count, float64(i)/float64(count)*100, int(math.Ceil(float64(i)/float64(limit))), count-int64(i))
|
||||||
|
default:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
log.Info("Completed migrating %d Pull Request(s) in: %d batches", count, int(math.Ceil(float64(i)/float64(limit))))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,8 +6,10 @@ package migrations
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"code.gitea.io/gitea/modules/git"
|
"code.gitea.io/gitea/modules/git"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
@ -43,17 +45,26 @@ func refixMergeBase(x *xorm.Engine) error {
|
||||||
limit = 50
|
limit = 50
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ticker := time.NewTicker(5 * time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
count, err := x.Where("has_merged = ?", true).Count(new(PullRequest))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Info("%d Merged Pull Request(s) to migrate ...", count)
|
||||||
|
|
||||||
i := 0
|
i := 0
|
||||||
|
start := 0
|
||||||
for {
|
for {
|
||||||
prs := make([]PullRequest, 0, 50)
|
prs := make([]PullRequest, 0, 50)
|
||||||
if err := x.Limit(limit, i).Asc("id").Where("has_merged = ?", true).Find(&prs); err != nil {
|
if err := x.Limit(limit, start).Asc("id").Where("has_merged = ?", true).Find(&prs); err != nil {
|
||||||
return fmt.Errorf("Find: %v", err)
|
return fmt.Errorf("Find: %v", err)
|
||||||
}
|
}
|
||||||
if len(prs) == 0 {
|
if len(prs) == 0 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
i += len(prs)
|
start += 50
|
||||||
for _, pr := range prs {
|
for _, pr := range prs {
|
||||||
baseRepo := &Repository{ID: pr.BaseRepoID}
|
baseRepo := &Repository{ID: pr.BaseRepoID}
|
||||||
has, err := x.Table("repository").Get(baseRepo)
|
has, err := x.Table("repository").Get(baseRepo)
|
||||||
|
@ -90,7 +101,15 @@ func refixMergeBase(x *xorm.Engine) error {
|
||||||
}
|
}
|
||||||
pr.MergeBase = strings.TrimSpace(pr.MergeBase)
|
pr.MergeBase = strings.TrimSpace(pr.MergeBase)
|
||||||
x.ID(pr.ID).Cols("merge_base").Update(pr)
|
x.ID(pr.ID).Cols("merge_base").Update(pr)
|
||||||
|
i++
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
log.Info("%d/%d (%2.0f%%) Pull Request(s) migrated in %d batches. %d PRs Remaining ...", i, count, float64(i)/float64(count)*100, int(math.Ceil(float64(i)/float64(limit))), count-int64(i))
|
||||||
|
default:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("Completed migrating %d Pull Request(s) in: %d batches", count, int(math.Ceil(float64(i)/float64(limit))))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,25 +6,60 @@ package migrations
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"code.gitea.io/gitea/models"
|
"code.gitea.io/gitea/models"
|
||||||
|
"code.gitea.io/gitea/modules/git"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
pull_service "code.gitea.io/gitea/services/pull"
|
|
||||||
|
|
||||||
"xorm.io/xorm"
|
"xorm.io/xorm"
|
||||||
)
|
)
|
||||||
|
|
||||||
func addCommitDivergenceToPulls(x *xorm.Engine) error {
|
func addCommitDivergenceToPulls(x *xorm.Engine) error {
|
||||||
|
type Repository struct {
|
||||||
|
ID int64 `xorm:"pk autoincr"`
|
||||||
|
OwnerID int64 `xorm:"UNIQUE(s) index"`
|
||||||
|
OwnerName string
|
||||||
|
LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"`
|
||||||
|
Name string `xorm:"INDEX NOT NULL"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type PullRequest struct {
|
||||||
|
ID int64 `xorm:"pk autoincr"`
|
||||||
|
|
||||||
|
CommitsAhead int
|
||||||
|
CommitsBehind int
|
||||||
|
|
||||||
|
BaseRepoID int64 `xorm:"INDEX"`
|
||||||
|
BaseBranch string
|
||||||
|
|
||||||
|
HasMerged bool `xorm:"INDEX"`
|
||||||
|
MergedCommitID string `xorm:"VARCHAR(40)"`
|
||||||
|
}
|
||||||
|
|
||||||
if err := x.Sync2(new(models.PullRequest)); err != nil {
|
if err := x.Sync2(new(models.PullRequest)); err != nil {
|
||||||
return fmt.Errorf("Sync2: %v", err)
|
return fmt.Errorf("Sync2: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var last int
|
last := 0
|
||||||
|
migrated := 0
|
||||||
|
|
||||||
batchSize := setting.Database.IterateBufferSize
|
batchSize := setting.Database.IterateBufferSize
|
||||||
sess := x.NewSession()
|
sess := x.NewSession()
|
||||||
defer sess.Close()
|
defer sess.Close()
|
||||||
|
|
||||||
|
ticker := time.NewTicker(5 * time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
count, err := sess.Where("has_merged = ?", false).Count(new(PullRequest))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Info("%d Unmerged Pull Request(s) to migrate ...", count)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
if err := sess.Begin(); err != nil {
|
if err := sess.Begin(); err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -37,27 +72,53 @@ func addCommitDivergenceToPulls(x *xorm.Engine) error {
|
||||||
if len(results) == 0 {
|
if len(results) == 0 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
last += len(results)
|
last += batchSize
|
||||||
|
|
||||||
for _, pr := range results {
|
for _, pr := range results {
|
||||||
divergence, err := pull_service.GetDiverging(pr)
|
baseRepo := &Repository{ID: pr.BaseRepoID}
|
||||||
|
has, err := x.Table("repository").Get(baseRepo)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Unable to get base repo %d %v", pr.BaseRepoID, err)
|
||||||
|
}
|
||||||
|
if !has {
|
||||||
|
log.Error("Missing base repo with id %d for PR ID %d", pr.BaseRepoID, pr.ID)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
userPath := filepath.Join(setting.RepoRootPath, strings.ToLower(baseRepo.OwnerName))
|
||||||
|
repoPath := filepath.Join(userPath, strings.ToLower(baseRepo.Name)+".git")
|
||||||
|
|
||||||
|
gitRefName := fmt.Sprintf("refs/pull/%d/head", pr.Index)
|
||||||
|
|
||||||
|
divergence, err := git.GetDivergingCommits(repoPath, pr.BaseBranch, gitRefName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn("Could not recalculate Divergence for pull: %d", pr.ID)
|
log.Warn("Could not recalculate Divergence for pull: %d", pr.ID)
|
||||||
pr.CommitsAhead = 0
|
pr.CommitsAhead = 0
|
||||||
pr.CommitsBehind = 0
|
pr.CommitsBehind = 0
|
||||||
}
|
}
|
||||||
if divergence != nil {
|
|
||||||
pr.CommitsAhead = divergence.Ahead
|
pr.CommitsAhead = divergence.Ahead
|
||||||
pr.CommitsBehind = divergence.Behind
|
pr.CommitsBehind = divergence.Behind
|
||||||
}
|
|
||||||
if _, err = sess.ID(pr.ID).Cols("commits_ahead", "commits_behind").Update(pr); err != nil {
|
if _, err = sess.ID(pr.ID).Cols("commits_ahead", "commits_behind").Update(pr); err != nil {
|
||||||
return fmt.Errorf("Update Cols: %v", err)
|
return fmt.Errorf("Update Cols: %v", err)
|
||||||
}
|
}
|
||||||
|
migrated++
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := sess.Commit(); err != nil {
|
if err := sess.Commit(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
log.Info(
|
||||||
|
"%d/%d (%2.0f%%) Pull Request(s) migrated in %d batches. %d PRs Remaining ...",
|
||||||
|
migrated,
|
||||||
|
count,
|
||||||
|
float64(migrated)/float64(count)*100,
|
||||||
|
int(math.Ceil(float64(migrated)/float64(batchSize))),
|
||||||
|
count-int64(migrated))
|
||||||
|
default:
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
log.Info("Completed migrating %d Pull Request(s) in: %d batches", count, int(math.Ceil(float64(migrated)/float64(batchSize))))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue