1
0
mirror of https://github.com/go-gitea/gitea.git synced 2024-12-04 14:46:57 -05:00
gitea/vendor/github.com/minio/minio-go/v7/api-bucket-notification.go
Lunny Xiao 62e6c9bc6c
Add a storage layer for attachments (#11387)
* Add a storage layer for attachments

* Fix some bug

* fix test

* Fix copyright head and lint

* Fix bug

* Add setting for minio and flags for migrate-storage

* Add documents

* fix lint

* Add test for minio store type on attachments

* fix test

* fix test

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Add warning when storage migrated successfully

* Fix drone

* fix test

* rebase

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* remove log on xorm

* Fi download bug

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* Add URL function to serve attachments directly from S3/Minio

* Add ability to enable/disable redirection in attachment configuration

* Fix typo

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* don't change unrelated files

* Fix lint

* Fix build

* update go.mod and go.sum

* Use github.com/minio/minio-go/v6

* Remove unused function

* Upgrade minio to v7 and some other improvements

* fix lint

* Fix go mod

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
Co-authored-by: Tyler <tystuyfzand@gmail.com>
2020-08-18 12:23:45 +08:00

256 lines
7.7 KiB
Go
Vendored

/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2017-2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"bufio"
"bytes"
"context"
"encoding/xml"
"net/http"
"net/url"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/minio/minio-go/v7/pkg/notification"
"github.com/minio/minio-go/v7/pkg/s3utils"
)
// SetBucketNotification saves a new bucket notification with a context to control cancellations and timeouts.
func (c Client) SetBucketNotification(ctx context.Context, bucketName string, config notification.Configuration) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
}
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("notification", "")
notifBytes, err := xml.Marshal(&config)
if err != nil {
return err
}
notifBuffer := bytes.NewReader(notifBytes)
reqMetadata := requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentBody: notifBuffer,
contentLength: int64(len(notifBytes)),
contentMD5Base64: sumMD5Base64(notifBytes),
contentSHA256Hex: sum256Hex(notifBytes),
}
// Execute PUT to upload a new bucket notification.
resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata)
defer closeResponse(resp)
if err != nil {
return err
}
if resp != nil {
if resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp, bucketName, "")
}
}
return nil
}
// RemoveAllBucketNotification - Remove bucket notification clears all previously specified config
func (c Client) RemoveAllBucketNotification(ctx context.Context, bucketName string) error {
return c.SetBucketNotification(ctx, bucketName, notification.Configuration{})
}
// GetBucketNotification returns current bucket notification configuration
func (c Client) GetBucketNotification(ctx context.Context, bucketName string) (bucketNotification notification.Configuration, err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return notification.Configuration{}, err
}
return c.getBucketNotification(ctx, bucketName)
}
// Request server for notification rules.
func (c Client) getBucketNotification(ctx context.Context, bucketName string) (notification.Configuration, error) {
urlValues := make(url.Values)
urlValues.Set("notification", "")
// Execute GET on bucket to list objects.
resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
})
defer closeResponse(resp)
if err != nil {
return notification.Configuration{}, err
}
return processBucketNotificationResponse(bucketName, resp)
}
// processes the GetNotification http response from the server.
func processBucketNotificationResponse(bucketName string, resp *http.Response) (notification.Configuration, error) {
if resp.StatusCode != http.StatusOK {
errResponse := httpRespToErrorResponse(resp, bucketName, "")
return notification.Configuration{}, errResponse
}
var bucketNotification notification.Configuration
err := xmlDecoder(resp.Body, &bucketNotification)
if err != nil {
return notification.Configuration{}, err
}
return bucketNotification, nil
}
// ListenNotification listen for all events, this is a MinIO specific API
func (c Client) ListenNotification(ctx context.Context, prefix, suffix string, events []string) <-chan notification.Info {
return c.ListenBucketNotification(ctx, "", prefix, suffix, events)
}
// ListenBucketNotification listen for bucket events, this is a MinIO specific API
func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix, suffix string, events []string) <-chan notification.Info {
notificationInfoCh := make(chan notification.Info, 1)
const notificationCapacity = 4 * 1024 * 1024
notificationEventBuffer := make([]byte, notificationCapacity)
// Only success, start a routine to start reading line by line.
go func(notificationInfoCh chan<- notification.Info) {
defer close(notificationInfoCh)
// Validate the bucket name.
if bucketName != "" {
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
select {
case notificationInfoCh <- notification.Info{
Err: err,
}:
case <-ctx.Done():
}
return
}
}
// Check ARN partition to verify if listening bucket is supported
if s3utils.IsAmazonEndpoint(*c.endpointURL) || s3utils.IsGoogleEndpoint(*c.endpointURL) {
select {
case notificationInfoCh <- notification.Info{
Err: errAPINotSupported("Listening for bucket notification is specific only to `minio` server endpoints"),
}:
case <-ctx.Done():
}
return
}
// Continuously run and listen on bucket notification.
// Create a done channel to control 'ListObjects' go routine.
retryDoneCh := make(chan struct{}, 1)
// Indicate to our routine to exit cleanly upon return.
defer close(retryDoneCh)
// Prepare urlValues to pass into the request on every loop
urlValues := make(url.Values)
urlValues.Set("prefix", prefix)
urlValues.Set("suffix", suffix)
urlValues["events"] = events
// Wait on the jitter retry loop.
for range c.newRetryTimerContinous(time.Second, time.Second*30, MaxJitter, retryDoneCh) {
// Execute GET on bucket to list objects.
resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
})
if err != nil {
select {
case notificationInfoCh <- notification.Info{
Err: err,
}:
case <-ctx.Done():
}
return
}
// Validate http response, upon error return quickly.
if resp.StatusCode != http.StatusOK {
errResponse := httpRespToErrorResponse(resp, bucketName, "")
select {
case notificationInfoCh <- notification.Info{
Err: errResponse,
}:
case <-ctx.Done():
}
return
}
// Initialize a new bufio scanner, to read line by line.
bio := bufio.NewScanner(resp.Body)
// Use a higher buffer to support unexpected
// caching done by proxies
bio.Buffer(notificationEventBuffer, notificationCapacity)
var json = jsoniter.ConfigCompatibleWithStandardLibrary
// Unmarshal each line, returns marshaled values.
for bio.Scan() {
var notificationInfo notification.Info
if err = json.Unmarshal(bio.Bytes(), &notificationInfo); err != nil {
// Unexpected error during json unmarshal, send
// the error to caller for actionable as needed.
select {
case notificationInfoCh <- notification.Info{
Err: err,
}:
case <-ctx.Done():
return
}
closeResponse(resp)
continue
}
// Send notificationInfo
select {
case notificationInfoCh <- notificationInfo:
case <-ctx.Done():
closeResponse(resp)
return
}
}
if err = bio.Err(); err != nil {
select {
case notificationInfoCh <- notification.Info{
Err: err,
}:
case <-ctx.Done():
return
}
}
// Close current connection before looping further.
closeResponse(resp)
}
}(notificationInfoCh)
// Returns the notification info channel, for caller to start reading from.
return notificationInfoCh
}