From 0c6ae61229bce9d9ad3d359cee927464968a2dd1 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 8 Aug 2023 02:46:10 +0200 Subject: [PATCH] Allow package cleanup from admin page (#25307) Until now expired package data gets deleted daily by a cronjob. The admin page shows the size of all packages and the size of unreferenced data. The users (#25035, #20631) expect the deletion of this data if they run the cronjob from the admin page but the job only deletes data older than 24h. This PR adds a new button which deletes all expired data. ![grafik](https://github.com/go-gitea/gitea/assets/1666336/b3e35d73-9496-4fa7-a20c-e5d30b1f6850) --------- Co-authored-by: silverwind --- options/locale/locale_en-US.ini | 1 + routers/web/admin/packages.go | 12 ++++++++++++ routers/web/web.go | 1 + services/cron/tasks_basic.go | 2 +- services/packages/cleanup/cleanup.go | 26 ++++++++++++++++++++++---- templates/admin/packages/list.tmpl | 6 ++++++ tests/integration/api_packages_test.go | 4 ++-- 7 files changed, 45 insertions(+), 7 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 64bc0c7cc1..f946aff10c 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2833,6 +2833,7 @@ repos.lfs_size = LFS Size packages.package_manage_panel = Package Management packages.total_size = Total Size: %s packages.unreferenced_size = Unreferenced Size: %s +packages.cleanup = Clean up expired data packages.owner = Owner packages.creator = Creator packages.name = Name diff --git a/routers/web/admin/packages.go b/routers/web/admin/packages.go index 8e4b8a373e..8d4c29813e 100644 --- a/routers/web/admin/packages.go +++ b/routers/web/admin/packages.go @@ -6,6 +6,7 @@ package admin import ( "net/http" "net/url" + "time" "code.gitea.io/gitea/models/db" packages_model "code.gitea.io/gitea/models/packages" @@ -14,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" packages_service "code.gitea.io/gitea/services/packages" + packages_cleanup_service "code.gitea.io/gitea/services/packages/cleanup" ) const ( @@ -99,3 +101,13 @@ func DeletePackageVersion(ctx *context.Context) { ctx.Flash.Success(ctx.Tr("packages.settings.delete.success")) ctx.JSONRedirect(setting.AppSubURL + "/admin/packages?page=" + url.QueryEscape(ctx.FormString("page")) + "&q=" + url.QueryEscape(ctx.FormString("q")) + "&type=" + url.QueryEscape(ctx.FormString("type"))) } + +func CleanupExpiredData(ctx *context.Context) { + if err := packages_cleanup_service.CleanupExpiredData(ctx, time.Duration(0)); err != nil { + ctx.ServerError("CleanupExpiredData", err) + return + } + + ctx.Flash.Success(ctx.Tr("packages.cleanup.success")) + ctx.Redirect(setting.AppSubURL + "/admin/packages") +} diff --git a/routers/web/web.go b/routers/web/web.go index aa3d830f94..2c2309e827 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -597,6 +597,7 @@ func registerRoutes(m *web.Route) { m.Group("/packages", func() { m.Get("", admin.Packages) m.Post("/delete", admin.DeletePackageVersion) + m.Post("/cleanup", admin.CleanupExpiredData) }, packagesEnabled) m.Group("/hooks", func() { diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index 2e6560ec0c..2a213ae515 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -152,7 +152,7 @@ func registerCleanupPackages() { OlderThan: 24 * time.Hour, }, func(ctx context.Context, _ *user_model.User, config Config) error { realConfig := config.(*OlderThanConfig) - return packages_cleanup_service.Cleanup(ctx, realConfig.OlderThan) + return packages_cleanup_service.CleanupTask(ctx, realConfig.OlderThan) }) } diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go index 43fbc1ad9b..77bcfb1942 100644 --- a/services/packages/cleanup/cleanup.go +++ b/services/packages/cleanup/cleanup.go @@ -20,9 +20,17 @@ import ( debian_service "code.gitea.io/gitea/services/packages/debian" ) -// Cleanup removes expired package data -func Cleanup(taskCtx context.Context, olderThan time.Duration) error { - ctx, committer, err := db.TxContext(taskCtx) +// Task method to execute cleanup rules and cleanup expired package data +func CleanupTask(ctx context.Context, olderThan time.Duration) error { + if err := ExecuteCleanupRules(ctx); err != nil { + return err + } + + return CleanupExpiredData(ctx, olderThan) +} + +func ExecuteCleanupRules(outerCtx context.Context) error { + ctx, committer, err := db.TxContext(outerCtx) if err != nil { return err } @@ -30,7 +38,7 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { select { - case <-taskCtx.Done(): + case <-outerCtx.Done(): return db.ErrCancelledf("While processing package cleanup rules") default: } @@ -122,6 +130,16 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { return err } + return committer.Commit() +} + +func CleanupExpiredData(outerCtx context.Context, olderThan time.Duration) error { + ctx, committer, err := db.TxContext(outerCtx) + if err != nil { + return err + } + defer committer.Close() + if err := container_service.Cleanup(ctx, olderThan); err != nil { return err } diff --git a/templates/admin/packages/list.tmpl b/templates/admin/packages/list.tmpl index 9aa1d933f6..4cf30f58e6 100644 --- a/templates/admin/packages/list.tmpl +++ b/templates/admin/packages/list.tmpl @@ -4,6 +4,12 @@ {{.locale.Tr "admin.packages.package_manage_panel"}} ({{.locale.Tr "admin.total" .TotalCount}}, {{.locale.Tr "admin.packages.total_size" (FileSize .TotalBlobSize)}}, {{.locale.Tr "admin.packages.unreferenced_size" (FileSize .TotalUnreferencedBlobSize)}}) +
+
+ {{.CsrfTokenHtml}} + +
+
diff --git a/tests/integration/api_packages_test.go b/tests/integration/api_packages_test.go index cd981e9c73..e530b2c1ad 100644 --- a/tests/integration/api_packages_test.go +++ b/tests/integration/api_packages_test.go @@ -475,7 +475,7 @@ func TestPackageCleanup(t *testing.T) { _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, user.ID, packages_model.TypeContainer, "cleanup-test", container_model.UploadVersion) assert.NoError(t, err) - err = packages_cleanup_service.Cleanup(db.DefaultContext, duration) + err = packages_cleanup_service.CleanupTask(db.DefaultContext, duration) assert.NoError(t, err) pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration) @@ -610,7 +610,7 @@ func TestPackageCleanup(t *testing.T) { pcr, err := packages_model.InsertCleanupRule(db.DefaultContext, c.Rule) assert.NoError(t, err) - err = packages_cleanup_service.Cleanup(db.DefaultContext, duration) + err = packages_cleanup_service.CleanupTask(db.DefaultContext, duration) assert.NoError(t, err) for _, v := range c.Versions {