mirror of
https://github.com/go-gitea/gitea.git
synced 2024-09-05 23:34:19 -04:00
Add migrate repo archiver and packages storage support on command line (#20757)
* Add migrate repo archiver and packages storage support on command line * Fix typo * Use stdCtx * Use packageblob and fix command description * Add migrate packages unit tests * Fix comment year * Fix the migrate storage command line description * Update cmd/migrate_storage.go Co-authored-by: zeripath <art27@cantab.net> * Update cmd/migrate_storage.go Co-authored-by: zeripath <art27@cantab.net> * Update cmd/migrate_storage.go Co-authored-by: zeripath <art27@cantab.net> * Fix test Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: zeripath <art27@cantab.net>
This commit is contained in:
parent
86c85c19b6
commit
1f146090ec
23
cmd/main_test.go
Normal file
23
cmd/main_test.go
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/models/unittest"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
setting.SetCustomPathAndConf("", "", "")
|
||||||
|
setting.LoadForTest()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
unittest.MainTest(m, &unittest.TestOptions{
|
||||||
|
GiteaRootPath: "..",
|
||||||
|
})
|
||||||
|
}
|
@ -12,9 +12,11 @@ import (
|
|||||||
"code.gitea.io/gitea/models/db"
|
"code.gitea.io/gitea/models/db"
|
||||||
git_model "code.gitea.io/gitea/models/git"
|
git_model "code.gitea.io/gitea/models/git"
|
||||||
"code.gitea.io/gitea/models/migrations"
|
"code.gitea.io/gitea/models/migrations"
|
||||||
|
packages_model "code.gitea.io/gitea/models/packages"
|
||||||
repo_model "code.gitea.io/gitea/models/repo"
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
user_model "code.gitea.io/gitea/models/user"
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
packages_module "code.gitea.io/gitea/modules/packages"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
"code.gitea.io/gitea/modules/storage"
|
"code.gitea.io/gitea/modules/storage"
|
||||||
|
|
||||||
@ -25,13 +27,13 @@ import (
|
|||||||
var CmdMigrateStorage = cli.Command{
|
var CmdMigrateStorage = cli.Command{
|
||||||
Name: "migrate-storage",
|
Name: "migrate-storage",
|
||||||
Usage: "Migrate the storage",
|
Usage: "Migrate the storage",
|
||||||
Description: "This is a command for migrating storage.",
|
Description: "Copies stored files from storage configured in app.ini to parameter-configured storage",
|
||||||
Action: runMigrateStorage,
|
Action: runMigrateStorage,
|
||||||
Flags: []cli.Flag{
|
Flags: []cli.Flag{
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "type, t",
|
Name: "type, t",
|
||||||
Value: "",
|
Value: "",
|
||||||
Usage: "Kinds of files to migrate, currently only 'attachments' is supported",
|
Usage: "Type of stored files to copy. Allowed types: 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages'",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "storage, s",
|
Name: "storage, s",
|
||||||
@ -80,34 +82,53 @@ var CmdMigrateStorage = cli.Command{
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
func migrateAttachments(dstStorage storage.ObjectStorage) error {
|
func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
||||||
return repo_model.IterateAttachment(func(attach *repo_model.Attachment) error {
|
return db.IterateObjects(ctx, func(attach *repo_model.Attachment) error {
|
||||||
_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
|
_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
|
||||||
return err
|
return err
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func migrateLFS(dstStorage storage.ObjectStorage) error {
|
func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
||||||
return git_model.IterateLFS(func(mo *git_model.LFSMetaObject) error {
|
return db.IterateObjects(ctx, func(mo *git_model.LFSMetaObject) error {
|
||||||
_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
|
_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
|
||||||
return err
|
return err
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func migrateAvatars(dstStorage storage.ObjectStorage) error {
|
func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
||||||
return user_model.IterateUser(func(user *user_model.User) error {
|
return db.IterateObjects(ctx, func(user *user_model.User) error {
|
||||||
_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
|
_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
|
||||||
return err
|
return err
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func migrateRepoAvatars(dstStorage storage.ObjectStorage) error {
|
func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
||||||
return repo_model.IterateRepository(func(repo *repo_model.Repository) error {
|
return db.IterateObjects(ctx, func(repo *repo_model.Repository) error {
|
||||||
_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
|
_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
|
||||||
return err
|
return err
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
||||||
|
return db.IterateObjects(ctx, func(archiver *repo_model.RepoArchiver) error {
|
||||||
|
p, err := archiver.RelativePath()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = storage.Copy(dstStorage, p, storage.RepoArchives, p)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
||||||
|
return db.IterateObjects(ctx, func(pb *packages_model.PackageBlob) error {
|
||||||
|
p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256))
|
||||||
|
_, err := storage.Copy(dstStorage, p, storage.Packages, p)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func runMigrateStorage(ctx *cli.Context) error {
|
func runMigrateStorage(ctx *cli.Context) error {
|
||||||
stdCtx, cancel := installSignals()
|
stdCtx, cancel := installSignals()
|
||||||
defer cancel()
|
defer cancel()
|
||||||
@ -127,8 +148,6 @@ func runMigrateStorage(ctx *cli.Context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
goCtx := context.Background()
|
|
||||||
|
|
||||||
if err := storage.Init(); err != nil {
|
if err := storage.Init(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -145,13 +164,13 @@ func runMigrateStorage(ctx *cli.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
dstStorage, err = storage.NewLocalStorage(
|
dstStorage, err = storage.NewLocalStorage(
|
||||||
goCtx,
|
stdCtx,
|
||||||
storage.LocalStorageConfig{
|
storage.LocalStorageConfig{
|
||||||
Path: p,
|
Path: p,
|
||||||
})
|
})
|
||||||
case string(storage.MinioStorageType):
|
case string(storage.MinioStorageType):
|
||||||
dstStorage, err = storage.NewMinioStorage(
|
dstStorage, err = storage.NewMinioStorage(
|
||||||
goCtx,
|
stdCtx,
|
||||||
storage.MinioStorageConfig{
|
storage.MinioStorageConfig{
|
||||||
Endpoint: ctx.String("minio-endpoint"),
|
Endpoint: ctx.String("minio-endpoint"),
|
||||||
AccessKeyID: ctx.String("minio-access-key-id"),
|
AccessKeyID: ctx.String("minio-access-key-id"),
|
||||||
@ -162,35 +181,29 @@ func runMigrateStorage(ctx *cli.Context) error {
|
|||||||
UseSSL: ctx.Bool("minio-use-ssl"),
|
UseSSL: ctx.Bool("minio-use-ssl"),
|
||||||
})
|
})
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("Unsupported storage type: %s", ctx.String("storage"))
|
return fmt.Errorf("unsupported storage type: %s", ctx.String("storage"))
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
migratedMethods := map[string]func(context.Context, storage.ObjectStorage) error{
|
||||||
|
"attachments": migrateAttachments,
|
||||||
|
"lfs": migrateLFS,
|
||||||
|
"avatars": migrateAvatars,
|
||||||
|
"repo-avatars": migrateRepoAvatars,
|
||||||
|
"repo-archivers": migrateRepoArchivers,
|
||||||
|
"packages": migratePackages,
|
||||||
|
}
|
||||||
|
|
||||||
tp := strings.ToLower(ctx.String("type"))
|
tp := strings.ToLower(ctx.String("type"))
|
||||||
switch tp {
|
if m, ok := migratedMethods[tp]; ok {
|
||||||
case "attachments":
|
if err := m(stdCtx, dstStorage); err != nil {
|
||||||
if err := migrateAttachments(dstStorage); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
case "lfs":
|
log.Info("%s files have successfully been copied to the new storage.", tp)
|
||||||
if err := migrateLFS(dstStorage); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
case "avatars":
|
|
||||||
if err := migrateAvatars(dstStorage); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
case "repo-avatars":
|
|
||||||
if err := migrateRepoAvatars(dstStorage); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("Unsupported storage: %s", ctx.String("type"))
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Warn("All files have been copied to the new placement but old files are still on the original placement.")
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("unsupported storage: %s", ctx.String("type"))
|
||||||
}
|
}
|
||||||
|
74
cmd/migrate_storage_test.go
Normal file
74
cmd/migrate_storage_test.go
Normal file
@ -0,0 +1,74 @@
|
|||||||
|
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/models/packages"
|
||||||
|
"code.gitea.io/gitea/models/unittest"
|
||||||
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
|
packages_module "code.gitea.io/gitea/modules/packages"
|
||||||
|
"code.gitea.io/gitea/modules/storage"
|
||||||
|
packages_service "code.gitea.io/gitea/services/packages"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMigratePackages(t *testing.T) {
|
||||||
|
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||||
|
|
||||||
|
creator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
|
||||||
|
|
||||||
|
content := "package main\n\nfunc main() {\nfmt.Println(\"hi\")\n}\n"
|
||||||
|
buf, err := packages_module.CreateHashedBufferFromReader(strings.NewReader(content), 1024)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
defer buf.Close()
|
||||||
|
|
||||||
|
v, f, err := packages_service.CreatePackageAndAddFile(&packages_service.PackageCreationInfo{
|
||||||
|
PackageInfo: packages_service.PackageInfo{
|
||||||
|
Owner: creator,
|
||||||
|
PackageType: packages.TypeGeneric,
|
||||||
|
Name: "test",
|
||||||
|
Version: "1.0.0",
|
||||||
|
},
|
||||||
|
Creator: creator,
|
||||||
|
SemverCompatible: true,
|
||||||
|
VersionProperties: map[string]string{},
|
||||||
|
}, &packages_service.PackageFileCreationInfo{
|
||||||
|
PackageFileInfo: packages_service.PackageFileInfo{
|
||||||
|
Filename: "a.go",
|
||||||
|
},
|
||||||
|
Data: buf,
|
||||||
|
IsLead: true,
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.NotNil(t, v)
|
||||||
|
assert.NotNil(t, f)
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
p, err := os.MkdirTemp(os.TempDir(), "migrated_packages")
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
dstStorage, err := storage.NewLocalStorage(
|
||||||
|
ctx,
|
||||||
|
storage.LocalStorageConfig{
|
||||||
|
Path: p,
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
err = migratePackages(ctx, dstStorage)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
entries, err := os.ReadDir(p)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.EqualValues(t, 2, len(entries))
|
||||||
|
assert.EqualValues(t, "01", entries[0].Name())
|
||||||
|
assert.EqualValues(t, "tmp", entries[1].Name())
|
||||||
|
}
|
34
models/db/iterate.go
Normal file
34
models/db/iterate.go
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package db
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
)
|
||||||
|
|
||||||
|
// IterateObjects iterate all the Bean object
|
||||||
|
func IterateObjects[Object any](ctx context.Context, f func(repo *Object) error) error {
|
||||||
|
var start int
|
||||||
|
batchSize := setting.Database.IterateBufferSize
|
||||||
|
sess := GetEngine(ctx)
|
||||||
|
for {
|
||||||
|
repos := make([]*Object, 0, batchSize)
|
||||||
|
if err := sess.Limit(batchSize, start).Find(&repos); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(repos) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
start += len(repos)
|
||||||
|
|
||||||
|
for _, repo := range repos {
|
||||||
|
if err := f(repo); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -278,29 +278,6 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int6
|
|||||||
return committer.Commit()
|
return committer.Commit()
|
||||||
}
|
}
|
||||||
|
|
||||||
// IterateLFS iterates lfs object
|
|
||||||
func IterateLFS(f func(mo *LFSMetaObject) error) error {
|
|
||||||
var start int
|
|
||||||
const batchSize = 100
|
|
||||||
e := db.GetEngine(db.DefaultContext)
|
|
||||||
for {
|
|
||||||
mos := make([]*LFSMetaObject, 0, batchSize)
|
|
||||||
if err := e.Limit(batchSize, start).Find(&mos); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(mos) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
start += len(mos)
|
|
||||||
|
|
||||||
for _, mo := range mos {
|
|
||||||
if err := f(mo); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// CopyLFS copies LFS data from one repo to another
|
// CopyLFS copies LFS data from one repo to another
|
||||||
func CopyLFS(ctx context.Context, newRepo, oldRepo *repo_model.Repository) error {
|
func CopyLFS(ctx context.Context, newRepo, oldRepo *repo_model.Repository) error {
|
||||||
var lfsObjects []*LFSMetaObject
|
var lfsObjects []*LFSMetaObject
|
||||||
|
@ -226,28 +226,6 @@ func DeleteAttachmentsByRelease(releaseID int64) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// IterateAttachment iterates attachments; it should not be used when Gitea is servicing users.
|
|
||||||
func IterateAttachment(f func(attach *Attachment) error) error {
|
|
||||||
var start int
|
|
||||||
const batchSize = 100
|
|
||||||
for {
|
|
||||||
attachments := make([]*Attachment, 0, batchSize)
|
|
||||||
if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&attachments); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(attachments) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
start += len(attachments)
|
|
||||||
|
|
||||||
for _, attach := range attachments {
|
|
||||||
if err := f(attach); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// CountOrphanedAttachments returns the number of bad attachments
|
// CountOrphanedAttachments returns the number of bad attachments
|
||||||
func CountOrphanedAttachments() (int64, error) {
|
func CountOrphanedAttachments() (int64, error) {
|
||||||
return db.GetEngine(db.DefaultContext).Where("(issue_id > 0 and issue_id not in (select id from issue)) or (release_id > 0 and release_id not in (select id from `release`))").
|
return db.GetEngine(db.DefaultContext).Where("(issue_id > 0 and issue_id not in (select id from issue)) or (release_id > 0 and release_id not in (select id from `release`))").
|
||||||
|
@ -15,36 +15,12 @@ import (
|
|||||||
"code.gitea.io/gitea/models/unit"
|
"code.gitea.io/gitea/models/unit"
|
||||||
user_model "code.gitea.io/gitea/models/user"
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
"code.gitea.io/gitea/modules/container"
|
"code.gitea.io/gitea/modules/container"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
|
||||||
"code.gitea.io/gitea/modules/structs"
|
"code.gitea.io/gitea/modules/structs"
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/util"
|
||||||
|
|
||||||
"xorm.io/builder"
|
"xorm.io/builder"
|
||||||
)
|
)
|
||||||
|
|
||||||
// IterateRepository iterate repositories
|
|
||||||
func IterateRepository(f func(repo *Repository) error) error {
|
|
||||||
var start int
|
|
||||||
batchSize := setting.Database.IterateBufferSize
|
|
||||||
sess := db.GetEngine(db.DefaultContext)
|
|
||||||
for {
|
|
||||||
repos := make([]*Repository, 0, batchSize)
|
|
||||||
if err := sess.Limit(batchSize, start).Find(&repos); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(repos) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
start += len(repos)
|
|
||||||
|
|
||||||
for _, repo := range repos {
|
|
||||||
if err := f(repo); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// FindReposMapByIDs find repos as map
|
// FindReposMapByIDs find repos as map
|
||||||
func FindReposMapByIDs(repoIDs []int64, res map[int64]*Repository) error {
|
func FindReposMapByIDs(repoIDs []int64, res map[int64]*Repository) error {
|
||||||
return db.GetEngine(db.DefaultContext).In("id", repoIDs).Find(&res)
|
return db.GetEngine(db.DefaultContext).In("id", repoIDs).Find(&res)
|
||||||
|
@ -9,7 +9,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"code.gitea.io/gitea/models/db"
|
"code.gitea.io/gitea/models/db"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
|
||||||
"code.gitea.io/gitea/modules/structs"
|
"code.gitea.io/gitea/modules/structs"
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/util"
|
||||||
|
|
||||||
@ -125,28 +124,6 @@ func SearchUsers(opts *SearchUserOptions) (users []*User, _ int64, _ error) {
|
|||||||
return users, count, sessQuery.Find(&users)
|
return users, count, sessQuery.Find(&users)
|
||||||
}
|
}
|
||||||
|
|
||||||
// IterateUser iterate users
|
|
||||||
func IterateUser(f func(user *User) error) error {
|
|
||||||
var start int
|
|
||||||
batchSize := setting.Database.IterateBufferSize
|
|
||||||
for {
|
|
||||||
users := make([]*User, 0, batchSize)
|
|
||||||
if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&users); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(users) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
start += len(users)
|
|
||||||
|
|
||||||
for _, user := range users {
|
|
||||||
if err := f(user); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// BuildCanSeeUserCondition creates a condition which can be used to restrict results to users/orgs the actor can see
|
// BuildCanSeeUserCondition creates a condition which can be used to restrict results to users/orgs the actor can see
|
||||||
func BuildCanSeeUserCondition(actor *User) builder.Cond {
|
func BuildCanSeeUserCondition(actor *User) builder.Cond {
|
||||||
if actor != nil {
|
if actor != nil {
|
||||||
|
@ -27,21 +27,21 @@ func NewContentStore() *ContentStore {
|
|||||||
|
|
||||||
// Get gets a package blob
|
// Get gets a package blob
|
||||||
func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) {
|
func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) {
|
||||||
return s.store.Open(keyToRelativePath(key))
|
return s.store.Open(KeyToRelativePath(key))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save stores a package blob
|
// Save stores a package blob
|
||||||
func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error {
|
func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error {
|
||||||
_, err := s.store.Save(keyToRelativePath(key), r, size)
|
_, err := s.store.Save(KeyToRelativePath(key), r, size)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete deletes a package blob
|
// Delete deletes a package blob
|
||||||
func (s *ContentStore) Delete(key BlobHash256Key) error {
|
func (s *ContentStore) Delete(key BlobHash256Key) error {
|
||||||
return s.store.Delete(keyToRelativePath(key))
|
return s.store.Delete(KeyToRelativePath(key))
|
||||||
}
|
}
|
||||||
|
|
||||||
// keyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000...
|
// KeyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000...
|
||||||
func keyToRelativePath(key BlobHash256Key) string {
|
func KeyToRelativePath(key BlobHash256Key) string {
|
||||||
return path.Join(string(key)[0:2], string(key)[2:4], string(key))
|
return path.Join(string(key)[0:2], string(key)[2:4], string(key))
|
||||||
}
|
}
|
||||||
|
@ -9,6 +9,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
user_model "code.gitea.io/gitea/models/user"
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
"code.gitea.io/gitea/modules/context"
|
"code.gitea.io/gitea/modules/context"
|
||||||
"code.gitea.io/gitea/modules/json"
|
"code.gitea.io/gitea/modules/json"
|
||||||
@ -59,7 +60,7 @@ func SendEmail(ctx *context.PrivateContext) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
err := user_model.IterateUser(func(user *user_model.User) error {
|
err := db.IterateObjects(ctx, func(user *user_model.User) error {
|
||||||
if len(user.Email) > 0 && user.IsActive {
|
if len(user.Email) > 0 && user.IsActive {
|
||||||
emails = append(emails, user.Email)
|
emails = append(emails, user.Email)
|
||||||
}
|
}
|
||||||
|
@ -96,7 +96,7 @@ func DeleteAvatar(repo *repo_model.Repository) error {
|
|||||||
|
|
||||||
// RemoveRandomAvatars removes the randomly generated avatars that were created for repositories
|
// RemoveRandomAvatars removes the randomly generated avatars that were created for repositories
|
||||||
func RemoveRandomAvatars(ctx context.Context) error {
|
func RemoveRandomAvatars(ctx context.Context) error {
|
||||||
return repo_model.IterateRepository(func(repository *repo_model.Repository) error {
|
return db.IterateObjects(ctx, func(repository *repo_model.Repository) error {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return db.ErrCancelledf("before random avatars removed for %s", repository.FullName())
|
return db.ErrCancelledf("before random avatars removed for %s", repository.FullName())
|
||||||
|
Loading…
Reference in New Issue
Block a user