From 4aba8a6a5ff96f0995ada7e183c7864f8f5bc05c Mon Sep 17 00:00:00 2001 From: a1012112796 <1012112796@qq.com> Date: Thu, 29 Jun 2023 06:41:02 +0800 Subject: [PATCH] Split lfs size from repository size (#22900) releated to #21820 - Split `Size` in repository table as two new colunms, one is `GitSize` for git size, the other is `LFSSize` for lfs data. still store full size in `Size` colunm. - Show full size on ui, but show each of them by a `title`; example: ![image](https://user-images.githubusercontent.com/25342410/218636251-e200f085-d7e7-4a25-9ff1-b586a63e07a9.png) - Return full size in api response. --------- Signed-off-by: a1012112796 <1012112796@qq.com> Co-authored-by: Lunny Xiao Co-authored-by: silverwind Co-authored-by: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Co-authored-by: Giteabot --- models/db/search.go | 4 +++ models/migrations/migrations.go | 2 ++ models/migrations/v1_21/v263.go | 41 ++++++++++++++++++++++++++++ models/repo/repo.go | 39 ++++++++++++++++++++++++++ models/repo/update.go | 8 ++++-- modules/repository/create.go | 2 +- options/locale/locale_en-US.ini | 1 + routers/web/explore/repo.go | 8 ++++++ templates/admin/repo/list.tmpl | 11 ++++++-- templates/repo/settings/options.tmpl | 2 +- templates/repo/sub_menu.tmpl | 2 +- templates/user/settings/repos.tmpl | 2 +- 12 files changed, 112 insertions(+), 10 deletions(-) create mode 100644 models/migrations/v1_21/v263.go diff --git a/models/db/search.go b/models/db/search.go index 105cb64c4151..aa577f08e043 100644 --- a/models/db/search.go +++ b/models/db/search.go @@ -20,6 +20,10 @@ const ( SearchOrderByNewest SearchOrderBy = "created_unix DESC" SearchOrderBySize SearchOrderBy = "size ASC" SearchOrderBySizeReverse SearchOrderBy = "size DESC" + SearchOrderByGitSize SearchOrderBy = "git_size ASC" + SearchOrderByGitSizeReverse SearchOrderBy = "git_size DESC" + SearchOrderByLFSSize SearchOrderBy = "lfs_size ASC" + SearchOrderByLFSSizeReverse SearchOrderBy = "lfs_size DESC" SearchOrderByID SearchOrderBy = "id ASC" SearchOrderByIDReverse SearchOrderBy = "id DESC" SearchOrderByStars SearchOrderBy = "num_stars ASC" diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 30a0b6e7eb73..3e9b348e63cf 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -507,6 +507,8 @@ var migrations = []Migration{ NewMigration("Add variable table", v1_21.CreateVariableTable), // v262 -> v263 NewMigration("Add TriggerEvent to action_run table", v1_21.AddTriggerEventToActionRun), + // v263 -> v264 + NewMigration("Add git_size and lfs_size columns to repository table", v1_21.AddGitSizeAndLFSSizeToRepositoryTable), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_21/v263.go b/models/migrations/v1_21/v263.go new file mode 100644 index 000000000000..88a5cb92b49f --- /dev/null +++ b/models/migrations/v1_21/v263.go @@ -0,0 +1,41 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_21 //nolint + +import ( + "fmt" + + "xorm.io/xorm" +) + +// AddGitSizeAndLFSSizeToRepositoryTable: add GitSize and LFSSize columns to Repository +func AddGitSizeAndLFSSizeToRepositoryTable(x *xorm.Engine) error { + type Repository struct { + GitSize int64 `xorm:"NOT NULL DEFAULT 0"` + LFSSize int64 `xorm:"NOT NULL DEFAULT 0"` + } + + sess := x.NewSession() + defer sess.Close() + + if err := sess.Begin(); err != nil { + return err + } + + if err := sess.Sync2(new(Repository)); err != nil { + return fmt.Errorf("Sync2: %w", err) + } + + _, err := sess.Exec(`UPDATE repository SET lfs_size=(SELECT SUM(size) FROM lfs_meta_object WHERE lfs_meta_object.repository_id=repository.ID) WHERE EXISTS (SELECT 1 FROM lfs_meta_object WHERE lfs_meta_object.repository_id=repository.ID)`) + if err != nil { + return err + } + + _, err = sess.Exec(`UPDATE repository SET git_size = size - lfs_size`) + if err != nil { + return err + } + + return sess.Commit() +} diff --git a/models/repo/repo.go b/models/repo/repo.go index d3e6daa95b53..b7c02057c272 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" @@ -163,6 +164,8 @@ type Repository struct { IsTemplate bool `xorm:"INDEX NOT NULL DEFAULT false"` TemplateID int64 `xorm:"INDEX"` Size int64 `xorm:"NOT NULL DEFAULT 0"` + GitSize int64 `xorm:"NOT NULL DEFAULT 0"` + LFSSize int64 `xorm:"NOT NULL DEFAULT 0"` CodeIndexerStatus *RepoIndexerStatus `xorm:"-"` StatsIndexerStatus *RepoIndexerStatus `xorm:"-"` IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true"` @@ -196,6 +199,42 @@ func (repo *Repository) SanitizedOriginalURL() string { return u.String() } +// text representations to be returned in SizeDetail.Name +const ( + SizeDetailNameGit = "git" + SizeDetailNameLFS = "lfs" +) + +type SizeDetail struct { + Name string + Size int64 +} + +// SizeDetails forms a struct with various size details about repository +func (repo *Repository) SizeDetails() []SizeDetail { + sizeDetails := []SizeDetail{ + { + Name: SizeDetailNameGit, + Size: repo.GitSize, + }, + { + Name: SizeDetailNameLFS, + Size: repo.LFSSize, + }, + } + return sizeDetails +} + +// SizeDetailsString returns a concatenation of all repository size details as a string +func (repo *Repository) SizeDetailsString() string { + var str strings.Builder + sizeDetails := repo.SizeDetails() + for _, detail := range sizeDetails { + str.WriteString(fmt.Sprintf("%s: %s, ", detail.Name, base.FileSize(detail.Size))) + } + return strings.TrimSuffix(str.String(), ", ") +} + func (repo *Repository) LogString() string { if repo == nil { return "" diff --git a/models/repo/update.go b/models/repo/update.go index 4894e0a1b9c5..c4fba32ad205 100644 --- a/models/repo/update.go +++ b/models/repo/update.go @@ -185,9 +185,11 @@ func ChangeRepositoryName(doer *user_model.User, repo *Repository, newRepoName s } // UpdateRepoSize updates the repository size, calculating it using getDirectorySize -func UpdateRepoSize(ctx context.Context, repoID, size int64) error { - _, err := db.GetEngine(ctx).ID(repoID).Cols("size").NoAutoTime().Update(&Repository{ - Size: size, +func UpdateRepoSize(ctx context.Context, repoID, gitSize, lfsSize int64) error { + _, err := db.GetEngine(ctx).ID(repoID).Cols("size", "git_size", "lfs_size").NoAutoTime().Update(&Repository{ + Size: gitSize + lfsSize, + GitSize: gitSize, + LFSSize: lfsSize, }) return err } diff --git a/modules/repository/create.go b/modules/repository/create.go index 0558d7f1c006..e8a1b8ba2bf8 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -330,7 +330,7 @@ func UpdateRepoSize(ctx context.Context, repo *repo_model.Repository) error { return fmt.Errorf("updateSize: GetLFSMetaObjects: %w", err) } - return repo_model.UpdateRepoSize(ctx, repo.ID, size+lfsSize) + return repo_model.UpdateRepoSize(ctx, repo.ID, size, lfsSize) } // CheckDaemonExportOK creates/removes git-daemon-export-ok for git-daemon... diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 4d9d9bc60e66..6de025805115 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2797,6 +2797,7 @@ repos.stars = Stars repos.forks = Forks repos.issues = Issues repos.size = Size +repos.lfs_size = LFS Size packages.package_manage_panel = Package Management packages.total_size = Total Size: %s diff --git a/routers/web/explore/repo.go b/routers/web/explore/repo.go index be5ad1b015b7..e5f7977abd01 100644 --- a/routers/web/explore/repo.go +++ b/routers/web/explore/repo.go @@ -73,6 +73,14 @@ func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) { orderBy = db.SearchOrderBySizeReverse case "size": orderBy = db.SearchOrderBySize + case "reversegitsize": + orderBy = db.SearchOrderByGitSizeReverse + case "gitsize": + orderBy = db.SearchOrderByGitSize + case "reverselfssize": + orderBy = db.SearchOrderByLFSSizeReverse + case "lfssize": + orderBy = db.SearchOrderByLFSSize case "moststars": orderBy = db.SearchOrderByStarsReverse case "feweststars": diff --git a/templates/admin/repo/list.tmpl b/templates/admin/repo/list.tmpl index f485784d0c68..3cd8f08f51b1 100644 --- a/templates/admin/repo/list.tmpl +++ b/templates/admin/repo/list.tmpl @@ -29,9 +29,13 @@ {{SortArrow "mostforks" "fewestforks" $.SortType false}} {{.locale.Tr "admin.repos.issues"}} - + {{.locale.Tr "admin.repos.size"}} - {{SortArrow "size" "reversesize" $.SortType false}} + {{SortArrow "gitsize" "reversegitsize" $.SortType false}} + + + {{.locale.Tr "admin.repos.lfs_size"}} + {{SortArrow "lfssize" "reverselfssize" $.SortType false}} {{.locale.Tr "admin.auths.updated"}} {{.locale.Tr "admin.users.created"}} @@ -80,7 +84,8 @@ {{.NumStars}} {{.NumForks}} {{.NumIssues}} - {{FileSize .Size}} + {{FileSize .GitSize}} + {{FileSize .LFSSize}} {{DateTime "short" .UpdatedUnix}} {{DateTime "short" .CreatedUnix}} {{svg "octicon-trash"}} diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 18de3dac0e99..8c394d5bf590 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -14,7 +14,7 @@
- {{FileSize .Repository.Size}} + {{FileSize .Repository.Size}}
diff --git a/templates/repo/sub_menu.tmpl b/templates/repo/sub_menu.tmpl index bfe5a20b164d..fbff7cfd2912 100644 --- a/templates/repo/sub_menu.tmpl +++ b/templates/repo/sub_menu.tmpl @@ -14,7 +14,7 @@ {{svg "octicon-tag"}} {{.NumTags}} {{.locale.TrN .NumTags "repo.tag" "repo.tags"}}
{{end}} -
+
{{$fileSizeFormatted := FileSize .Repository.Size}}{{/* the formatted string is always "{val} {unit}" */}} {{$fileSizeFields := StringUtils.Split $fileSizeFormatted " "}} {{svg "octicon-database"}} {{.locale.PrettyNumber (index $fileSizeFields 0)}} {{index $fileSizeFields 1}} diff --git a/templates/user/settings/repos.tmpl b/templates/user/settings/repos.tmpl index d2164d199a08..963f889144f1 100644 --- a/templates/user/settings/repos.tmpl +++ b/templates/user/settings/repos.tmpl @@ -24,7 +24,7 @@ {{svg "octicon-repo"}} {{end}} {{$repo.OwnerName}}/{{$repo.Name}} - {{FileSize $repo.Size}} + {{FileSize $repo.Size}} {{if $repo.IsFork}} {{$.locale.Tr "repo.forked_from"}} {{$repo.BaseRepo.OwnerName}}/{{$repo.BaseRepo.Name}}