Fix bugs in LFS meta garbage collection (#26122)
This PR - Fix #26093. Replace `time.Time` with `timeutil.TimeStamp` - Fix #26135. Add missing `xorm:"extends"` to `CountLFSMetaObject` for LFS meta object query - Add a unit test for LFS meta object garbage collection
This commit is contained in:
parent
dcd3a63128
commit
f3ed0ef692
|
@ -6,7 +6,6 @@ package git
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
|
||||||
|
|
||||||
"code.gitea.io/gitea/models/db"
|
"code.gitea.io/gitea/models/db"
|
||||||
"code.gitea.io/gitea/models/perm"
|
"code.gitea.io/gitea/models/perm"
|
||||||
|
@ -370,8 +369,8 @@ func IterateRepositoryIDsWithLFSMetaObjects(ctx context.Context, f func(ctx cont
|
||||||
|
|
||||||
// IterateLFSMetaObjectsForRepoOptions provides options for IterateLFSMetaObjectsForRepo
|
// IterateLFSMetaObjectsForRepoOptions provides options for IterateLFSMetaObjectsForRepo
|
||||||
type IterateLFSMetaObjectsForRepoOptions struct {
|
type IterateLFSMetaObjectsForRepoOptions struct {
|
||||||
OlderThan time.Time
|
OlderThan timeutil.TimeStamp
|
||||||
UpdatedLessRecentlyThan time.Time
|
UpdatedLessRecentlyThan timeutil.TimeStamp
|
||||||
OrderByUpdated bool
|
OrderByUpdated bool
|
||||||
LoopFunctionAlwaysUpdates bool
|
LoopFunctionAlwaysUpdates bool
|
||||||
}
|
}
|
||||||
|
@ -382,8 +381,8 @@ func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(cont
|
||||||
batchSize := setting.Database.IterateBufferSize
|
batchSize := setting.Database.IterateBufferSize
|
||||||
engine := db.GetEngine(ctx)
|
engine := db.GetEngine(ctx)
|
||||||
type CountLFSMetaObject struct {
|
type CountLFSMetaObject struct {
|
||||||
Count int64
|
Count int64
|
||||||
LFSMetaObject
|
LFSMetaObject `xorm:"extends"`
|
||||||
}
|
}
|
||||||
|
|
||||||
id := int64(0)
|
id := int64(0)
|
||||||
|
|
|
@ -15,6 +15,7 @@ import (
|
||||||
"code.gitea.io/gitea/modules/lfs"
|
"code.gitea.io/gitea/modules/lfs"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GarbageCollectLFSMetaObjectsOptions provides options for GarbageCollectLFSMetaObjects function
|
// GarbageCollectLFSMetaObjectsOptions provides options for GarbageCollectLFSMetaObjects function
|
||||||
|
@ -120,8 +121,8 @@ func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.R
|
||||||
//
|
//
|
||||||
// It is likely that a week is potentially excessive but it should definitely be enough that any
|
// It is likely that a week is potentially excessive but it should definitely be enough that any
|
||||||
// unassociated LFS object is genuinely unassociated.
|
// unassociated LFS object is genuinely unassociated.
|
||||||
OlderThan: opts.OlderThan,
|
OlderThan: timeutil.TimeStamp(opts.OlderThan.Unix()),
|
||||||
UpdatedLessRecentlyThan: opts.UpdatedLessRecentlyThan,
|
UpdatedLessRecentlyThan: timeutil.TimeStamp(opts.UpdatedLessRecentlyThan.Unix()),
|
||||||
OrderByUpdated: true,
|
OrderByUpdated: true,
|
||||||
LoopFunctionAlwaysUpdates: true,
|
LoopFunctionAlwaysUpdates: true,
|
||||||
})
|
})
|
||||||
|
|
|
@ -0,0 +1,64 @@
|
||||||
|
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package repository
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
git_model "code.gitea.io/gitea/models/git"
|
||||||
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
|
"code.gitea.io/gitea/models/unittest"
|
||||||
|
"code.gitea.io/gitea/modules/lfs"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/storage"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGarbageCollectLFSMetaObjects(t *testing.T) {
|
||||||
|
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||||
|
|
||||||
|
setting.LFS.StartServer = true
|
||||||
|
err := storage.Init()
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "repo1")
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
// add lfs object
|
||||||
|
lfsContent := []byte("gitea1")
|
||||||
|
lfsOid := storeObjectInRepo(t, repo.ID, &lfsContent)
|
||||||
|
|
||||||
|
// gc
|
||||||
|
err = GarbageCollectLFSMetaObjects(context.Background(), GarbageCollectLFSMetaObjectsOptions{
|
||||||
|
AutoFix: true,
|
||||||
|
OlderThan: time.Now().Add(7 * 24 * time.Hour).Add(5 * 24 * time.Hour),
|
||||||
|
UpdatedLessRecentlyThan: time.Now().Add(7 * 24 * time.Hour).Add(3 * 24 * time.Hour),
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
// lfs meta has been deleted
|
||||||
|
_, err = git_model.GetLFSMetaObjectByOid(db.DefaultContext, repo.ID, lfsOid)
|
||||||
|
assert.ErrorIs(t, err, git_model.ErrLFSObjectNotExist)
|
||||||
|
}
|
||||||
|
|
||||||
|
func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string {
|
||||||
|
pointer, err := lfs.GeneratePointer(bytes.NewReader(*content))
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = git_model.NewLFSMetaObject(db.DefaultContext, &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: repositoryID})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
contentStore := lfs.NewContentStore()
|
||||||
|
exist, err := contentStore.Exists(pointer)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
if !exist {
|
||||||
|
err := contentStore.Put(pointer, bytes.NewReader(*content))
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
return pointer.Oid
|
||||||
|
}
|
Loading…
Reference in New Issue