mirror of
https://github.com/go-gitea/gitea.git
synced 2024-09-06 23:44:20 -04:00
Fix container blob mount (#22226)
This commit is contained in:
parent
1aba53db62
commit
3510d7e33a
@ -25,6 +25,7 @@ type BlobSearchOptions struct {
|
|||||||
Digest string
|
Digest string
|
||||||
Tag string
|
Tag string
|
||||||
IsManifest bool
|
IsManifest bool
|
||||||
|
Repository string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (opts *BlobSearchOptions) toConds() builder.Cond {
|
func (opts *BlobSearchOptions) toConds() builder.Cond {
|
||||||
@ -53,6 +54,15 @@ func (opts *BlobSearchOptions) toConds() builder.Cond {
|
|||||||
|
|
||||||
cond = cond.And(builder.In("package_file.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property")))
|
cond = cond.And(builder.In("package_file.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property")))
|
||||||
}
|
}
|
||||||
|
if opts.Repository != "" {
|
||||||
|
var propsCond builder.Cond = builder.Eq{
|
||||||
|
"package_property.ref_type": packages.PropertyTypePackage,
|
||||||
|
"package_property.name": container_module.PropertyRepository,
|
||||||
|
"package_property.value": opts.Repository,
|
||||||
|
}
|
||||||
|
|
||||||
|
cond = cond.And(builder.In("package.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property")))
|
||||||
|
}
|
||||||
|
|
||||||
return cond
|
return cond
|
||||||
}
|
}
|
||||||
|
@ -33,6 +33,60 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
|
|||||||
|
|
||||||
contentStore := packages_module.NewContentStore()
|
contentStore := packages_module.NewContentStore()
|
||||||
|
|
||||||
|
uploadVersion, err := getOrCreateUploadVersion(pi)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = db.WithTx(db.DefaultContext, func(ctx context.Context) error {
|
||||||
|
pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error inserting package blob: %v", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// FIXME: Workaround to be removed in v1.20
|
||||||
|
// https://github.com/go-gitea/gitea/issues/19586
|
||||||
|
if exists {
|
||||||
|
err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256))
|
||||||
|
if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
|
||||||
|
log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
|
||||||
|
exists = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil {
|
||||||
|
log.Error("Error saving package blob in content store: %v", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return createFileForBlob(ctx, uploadVersion, pb)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if !exists {
|
||||||
|
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
|
||||||
|
log.Error("Error deleting package blob from content store: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return pb, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// mountBlob mounts the specific blob to a different package
|
||||||
|
func mountBlob(pi *packages_service.PackageInfo, pb *packages_model.PackageBlob) error {
|
||||||
|
uploadVersion, err := getOrCreateUploadVersion(pi)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return db.WithTx(db.DefaultContext, func(ctx context.Context) error {
|
||||||
|
return createFileForBlob(ctx, uploadVersion, pb)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func getOrCreateUploadVersion(pi *packages_service.PackageInfo) (*packages_model.PackageVersion, error) {
|
||||||
var uploadVersion *packages_model.PackageVersion
|
var uploadVersion *packages_model.PackageVersion
|
||||||
|
|
||||||
// FIXME: Replace usage of mutex with database transaction
|
// FIXME: Replace usage of mutex with database transaction
|
||||||
@ -83,41 +137,21 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
|
|||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
uploadVersionMutex.Unlock()
|
uploadVersionMutex.Unlock()
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = db.WithTx(db.DefaultContext, func(ctx context.Context) error {
|
return uploadVersion, err
|
||||||
pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb)
|
}
|
||||||
if err != nil {
|
|
||||||
log.Error("Error inserting package blob: %v", err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// FIXME: Workaround to be removed in v1.20
|
|
||||||
// https://github.com/go-gitea/gitea/issues/19586
|
|
||||||
if exists {
|
|
||||||
err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256))
|
|
||||||
if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
|
|
||||||
log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
|
|
||||||
exists = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !exists {
|
|
||||||
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil {
|
|
||||||
log.Error("Error saving package blob in content store: %v", err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
func createFileForBlob(ctx context.Context, pv *packages_model.PackageVersion, pb *packages_model.PackageBlob) error {
|
||||||
filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256))
|
filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256))
|
||||||
|
|
||||||
pf := &packages_model.PackageFile{
|
pf := &packages_model.PackageFile{
|
||||||
VersionID: uploadVersion.ID,
|
VersionID: pv.ID,
|
||||||
BlobID: pb.ID,
|
BlobID: pb.ID,
|
||||||
Name: filename,
|
Name: filename,
|
||||||
LowerName: filename,
|
LowerName: filename,
|
||||||
CompositeKey: packages_model.EmptyFileKey,
|
CompositeKey: packages_model.EmptyFileKey,
|
||||||
}
|
}
|
||||||
|
var err error
|
||||||
if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
|
if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
|
||||||
if err == packages_model.ErrDuplicatePackageFile {
|
if err == packages_model.ErrDuplicatePackageFile {
|
||||||
return nil
|
return nil
|
||||||
@ -132,17 +166,6 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
|
|||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
if !exists {
|
|
||||||
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
|
|
||||||
log.Error("Error deleting package blob from content store: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return pb, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func deleteBlob(ownerID int64, image, digest string) error {
|
func deleteBlob(ownerID int64, image, digest string) error {
|
||||||
|
@ -195,10 +195,15 @@ func InitiateUploadBlob(ctx *context.Context) {
|
|||||||
from := ctx.FormTrim("from")
|
from := ctx.FormTrim("from")
|
||||||
if mount != "" {
|
if mount != "" {
|
||||||
blob, _ := workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{
|
blob, _ := workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{
|
||||||
Image: from,
|
Repository: from,
|
||||||
Digest: mount,
|
Digest: mount,
|
||||||
})
|
})
|
||||||
if blob != nil {
|
if blob != nil {
|
||||||
|
if err := mountBlob(&packages_service.PackageInfo{Owner: ctx.Package.Owner, Name: image}, blob.Blob); err != nil {
|
||||||
|
apiError(ctx, http.StatusInternalServerError, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
setResponseHeaders(ctx.Resp, &containerHeaders{
|
setResponseHeaders(ctx.Resp, &containerHeaders{
|
||||||
Location: fmt.Sprintf("/v2/%s/%s/blobs/%s", ctx.Package.Owner.LowerName, image, mount),
|
Location: fmt.Sprintf("/v2/%s/%s/blobs/%s", ctx.Package.Owner.LowerName, image, mount),
|
||||||
ContentDigest: mount,
|
ContentDigest: mount,
|
||||||
|
@ -51,7 +51,7 @@
|
|||||||
{{.locale.Tr "packages.settings.delete"}}
|
{{.locale.Tr "packages.settings.delete"}}
|
||||||
</div>
|
</div>
|
||||||
<div class="content">
|
<div class="content">
|
||||||
<div class="ui warning message text left">
|
<div class="ui warning message text left word-break">
|
||||||
{{.locale.Tr "packages.settings.delete.notice" .PackageDescriptor.Package.Name .PackageDescriptor.Version.Version}}
|
{{.locale.Tr "packages.settings.delete.notice" .PackageDescriptor.Package.Name .PackageDescriptor.Version.Version}}
|
||||||
</div>
|
</div>
|
||||||
<form class="ui form" action="{{.Link}}" method="post">
|
<form class="ui form" action="{{.Link}}" method="post">
|
||||||
|
@ -256,6 +256,32 @@ func TestPackageContainer(t *testing.T) {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("UploadBlob/Mount", func(t *testing.T) {
|
||||||
|
defer tests.PrintCurrentTest(t)()
|
||||||
|
|
||||||
|
req := NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, unknownDigest))
|
||||||
|
addTokenAuthHeader(req, userToken)
|
||||||
|
MakeRequest(t, req, http.StatusAccepted)
|
||||||
|
|
||||||
|
req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, blobDigest))
|
||||||
|
addTokenAuthHeader(req, userToken)
|
||||||
|
resp := MakeRequest(t, req, http.StatusCreated)
|
||||||
|
|
||||||
|
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location"))
|
||||||
|
assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest"))
|
||||||
|
|
||||||
|
req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s&from=%s", url, unknownDigest, "unknown/image"))
|
||||||
|
addTokenAuthHeader(req, userToken)
|
||||||
|
MakeRequest(t, req, http.StatusAccepted)
|
||||||
|
|
||||||
|
req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s&from=%s/%s", url, blobDigest, user.Name, image))
|
||||||
|
addTokenAuthHeader(req, userToken)
|
||||||
|
resp = MakeRequest(t, req, http.StatusCreated)
|
||||||
|
|
||||||
|
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location"))
|
||||||
|
assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest"))
|
||||||
|
})
|
||||||
|
|
||||||
for _, tag := range tags {
|
for _, tag := range tags {
|
||||||
t.Run(fmt.Sprintf("[Tag:%s]", tag), func(t *testing.T) {
|
t.Run(fmt.Sprintf("[Tag:%s]", tag), func(t *testing.T) {
|
||||||
t.Run("UploadManifest", func(t *testing.T) {
|
t.Run("UploadManifest", func(t *testing.T) {
|
||||||
@ -444,21 +470,6 @@ func TestPackageContainer(t *testing.T) {
|
|||||||
assert.Equal(t, indexManifestDigest, pd.Files[0].Properties.GetByName(container_module.PropertyDigest))
|
assert.Equal(t, indexManifestDigest, pd.Files[0].Properties.GetByName(container_module.PropertyDigest))
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("UploadBlob/Mount", func(t *testing.T) {
|
|
||||||
defer tests.PrintCurrentTest(t)()
|
|
||||||
|
|
||||||
req := NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, unknownDigest))
|
|
||||||
addTokenAuthHeader(req, userToken)
|
|
||||||
MakeRequest(t, req, http.StatusAccepted)
|
|
||||||
|
|
||||||
req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, blobDigest))
|
|
||||||
addTokenAuthHeader(req, userToken)
|
|
||||||
resp := MakeRequest(t, req, http.StatusCreated)
|
|
||||||
|
|
||||||
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location"))
|
|
||||||
assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest"))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("HeadBlob", func(t *testing.T) {
|
t.Run("HeadBlob", func(t *testing.T) {
|
||||||
defer tests.PrintCurrentTest(t)()
|
defer tests.PrintCurrentTest(t)()
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user