forked from gitea/gitea
1
0
Fork 0

Fix bugs in LFS meta garbage collection (#26122)

This PR

- Fix #26093. Replace `time.Time` with `timeutil.TimeStamp`
- Fix #26135. Add missing `xorm:"extends"` to `CountLFSMetaObject` for
LFS meta object query
- Add a unit test for LFS meta object garbage collection
This commit is contained in:
Zettat123 2023-07-26 15:02:53 +08:00 committed by GitHub
parent dcd3a63128
commit f3ed0ef692
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 71 additions and 7 deletions

View File

@ -6,7 +6,6 @@ package git
import ( import (
"context" "context"
"fmt" "fmt"
"time"
"code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/perm" "code.gitea.io/gitea/models/perm"
@ -370,8 +369,8 @@ func IterateRepositoryIDsWithLFSMetaObjects(ctx context.Context, f func(ctx cont
// IterateLFSMetaObjectsForRepoOptions provides options for IterateLFSMetaObjectsForRepo // IterateLFSMetaObjectsForRepoOptions provides options for IterateLFSMetaObjectsForRepo
type IterateLFSMetaObjectsForRepoOptions struct { type IterateLFSMetaObjectsForRepoOptions struct {
OlderThan time.Time OlderThan timeutil.TimeStamp
UpdatedLessRecentlyThan time.Time UpdatedLessRecentlyThan timeutil.TimeStamp
OrderByUpdated bool OrderByUpdated bool
LoopFunctionAlwaysUpdates bool LoopFunctionAlwaysUpdates bool
} }
@ -382,8 +381,8 @@ func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(cont
batchSize := setting.Database.IterateBufferSize batchSize := setting.Database.IterateBufferSize
engine := db.GetEngine(ctx) engine := db.GetEngine(ctx)
type CountLFSMetaObject struct { type CountLFSMetaObject struct {
Count int64 Count int64
LFSMetaObject LFSMetaObject `xorm:"extends"`
} }
id := int64(0) id := int64(0)

View File

@ -15,6 +15,7 @@ import (
"code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
) )
// GarbageCollectLFSMetaObjectsOptions provides options for GarbageCollectLFSMetaObjects function // GarbageCollectLFSMetaObjectsOptions provides options for GarbageCollectLFSMetaObjects function
@ -120,8 +121,8 @@ func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.R
// //
// It is likely that a week is potentially excessive but it should definitely be enough that any // It is likely that a week is potentially excessive but it should definitely be enough that any
// unassociated LFS object is genuinely unassociated. // unassociated LFS object is genuinely unassociated.
OlderThan: opts.OlderThan, OlderThan: timeutil.TimeStamp(opts.OlderThan.Unix()),
UpdatedLessRecentlyThan: opts.UpdatedLessRecentlyThan, UpdatedLessRecentlyThan: timeutil.TimeStamp(opts.UpdatedLessRecentlyThan.Unix()),
OrderByUpdated: true, OrderByUpdated: true,
LoopFunctionAlwaysUpdates: true, LoopFunctionAlwaysUpdates: true,
}) })

View File

@ -0,0 +1,64 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package repository
import (
"bytes"
"context"
"testing"
"time"
"code.gitea.io/gitea/models/db"
git_model "code.gitea.io/gitea/models/git"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"github.com/stretchr/testify/assert"
)
func TestGarbageCollectLFSMetaObjects(t *testing.T) {
assert.NoError(t, unittest.PrepareTestDatabase())
setting.LFS.StartServer = true
err := storage.Init()
assert.NoError(t, err)
repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "repo1")
assert.NoError(t, err)
// add lfs object
lfsContent := []byte("gitea1")
lfsOid := storeObjectInRepo(t, repo.ID, &lfsContent)
// gc
err = GarbageCollectLFSMetaObjects(context.Background(), GarbageCollectLFSMetaObjectsOptions{
AutoFix: true,
OlderThan: time.Now().Add(7 * 24 * time.Hour).Add(5 * 24 * time.Hour),
UpdatedLessRecentlyThan: time.Now().Add(7 * 24 * time.Hour).Add(3 * 24 * time.Hour),
})
assert.NoError(t, err)
// lfs meta has been deleted
_, err = git_model.GetLFSMetaObjectByOid(db.DefaultContext, repo.ID, lfsOid)
assert.ErrorIs(t, err, git_model.ErrLFSObjectNotExist)
}
func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string {
pointer, err := lfs.GeneratePointer(bytes.NewReader(*content))
assert.NoError(t, err)
_, err = git_model.NewLFSMetaObject(db.DefaultContext, &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: repositoryID})
assert.NoError(t, err)
contentStore := lfs.NewContentStore()
exist, err := contentStore.Exists(pointer)
assert.NoError(t, err)
if !exist {
err := contentStore.Put(pointer, bytes.NewReader(*content))
assert.NoError(t, err)
}
return pointer.Oid
}