Improve mirror iterator (#18928)

* Improve mirror iterator

* fix test
This commit is contained in:
Lunny Xiao 2022-03-01 03:41:06 +08:00 committed by GitHub
parent 59959ab222
commit b75ad7b87f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 19 additions and 19 deletions

View File

@ -120,11 +120,12 @@ func DeleteMirrorByRepoID(repoID int64) error {
} }
// MirrorsIterate iterates all mirror repositories. // MirrorsIterate iterates all mirror repositories.
func MirrorsIterate(f func(idx int, bean interface{}) error) error { func MirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
return db.GetEngine(db.DefaultContext). return db.GetEngine(db.DefaultContext).
Where("next_update_unix<=?", time.Now().Unix()). Where("next_update_unix<=?", time.Now().Unix()).
And("next_update_unix!=0"). And("next_update_unix!=0").
OrderBy("updated_unix ASC"). OrderBy("updated_unix ASC").
Limit(limit).
Iterate(new(Mirror), f) Iterate(new(Mirror), f)
} }

View File

@ -101,10 +101,11 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) {
} }
// PushMirrorsIterate iterates all push-mirror repositories. // PushMirrorsIterate iterates all push-mirror repositories.
func PushMirrorsIterate(f func(idx int, bean interface{}) error) error { func PushMirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
return db.GetEngine(db.DefaultContext). return db.GetEngine(db.DefaultContext).
Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()). Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()).
And("`interval` != 0"). And("`interval` != 0").
OrderBy("last_update ASC"). OrderBy("last_update ASC").
Limit(limit).
Iterate(new(PushMirror), f) Iterate(new(PushMirror), f)
} }

View File

@ -40,7 +40,7 @@ func TestPushMirrorsIterate(t *testing.T) {
time.Sleep(1 * time.Millisecond) time.Sleep(1 * time.Millisecond)
PushMirrorsIterate(func(idx int, bean interface{}) error { PushMirrorsIterate(1, func(idx int, bean interface{}) error {
m, ok := bean.(*PushMirror) m, ok := bean.(*PushMirror)
assert.True(t, ok) assert.True(t, ok)
assert.Equal(t, "test-1", m.RemoteName) assert.Equal(t, "test-1", m.RemoteName)

View File

@ -55,9 +55,7 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error {
} }
log.Trace("Doing: Update") log.Trace("Doing: Update")
requested := 0 handler := func(idx int, bean interface{}) error {
handler := func(idx int, bean interface{}, limit int) error {
var item SyncRequest var item SyncRequest
var repo *repo_model.Repository var repo *repo_model.Repository
if m, ok := bean.(*repo_model.Mirror); ok { if m, ok := bean.(*repo_model.Mirror); ok {
@ -104,35 +102,35 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error {
} }
return err return err
} }
requested++
if limit > 0 && requested > limit {
return errLimit
}
return nil return nil
} }
pullMirrorsRequested := 0 pullMirrorsRequested := 0
if pullLimit != 0 { if pullLimit != 0 {
requested = 0 if err := repo_model.MirrorsIterate(pullLimit, func(idx int, bean interface{}) error {
if err := repo_model.MirrorsIterate(func(idx int, bean interface{}) error { if err := handler(idx, bean); err != nil {
return handler(idx, bean, pullLimit) return err
}
pullMirrorsRequested++
return nil
}); err != nil && err != errLimit { }); err != nil && err != errLimit {
log.Error("MirrorsIterate: %v", err) log.Error("MirrorsIterate: %v", err)
return err return err
} }
pullMirrorsRequested, requested = requested, 0
} }
pushMirrorsRequested := 0 pushMirrorsRequested := 0
if pushLimit != 0 { if pushLimit != 0 {
requested = 0 if err := repo_model.PushMirrorsIterate(pushLimit, func(idx int, bean interface{}) error {
if err := repo_model.PushMirrorsIterate(func(idx int, bean interface{}) error { if err := handler(idx, bean); err != nil {
return handler(idx, bean, pushLimit) return err
}
pushMirrorsRequested++
return nil
}); err != nil && err != errLimit { }); err != nil && err != errLimit {
log.Error("PushMirrorsIterate: %v", err) log.Error("PushMirrorsIterate: %v", err)
return err return err
} }
pushMirrorsRequested, requested = requested, 0
} }
log.Trace("Finished: Update: %d pull mirrors and %d push mirrors queued", pullMirrorsRequested, pushMirrorsRequested) log.Trace("Finished: Update: %d pull mirrors and %d push mirrors queued", pullMirrorsRequested, pushMirrorsRequested)
return nil return nil