feat: add support for a credentials chain for minio access (#31051)

We wanted to be able to use the IAM role provided by the EC2 instance
metadata in order to access S3 via the Minio configuration. To do this,
a new credentials chain is added that will check the following locations
for credentials when an access key is not provided. In priority order,
they are:

1. MINIO_ prefixed environment variables
2. AWS_ prefixed environment variables
3. a minio credentials file
4. an aws credentials file
5. EC2 instance metadata

custom/conf/app.example.ini

@@ -1872,7 +1872,10 @@ LEVEL = Info
 ;; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
 ;MINIO_ENDPOINT = localhost:9000
 ;;
-;; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
+;; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`.
+;; If not provided and STORAGE_TYPE is `minio`, will search for credentials in known
+;; environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files
+;; (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata.
 ;MINIO_ACCESS_KEY_ID =
 ;;
 ;; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
@@ -2573,7 +2576,10 @@ LEVEL = Info
 ;; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
 ;MINIO_ENDPOINT = localhost:9000
 ;;
-;; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
+;; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`.
+;; If not provided and STORAGE_TYPE is `minio`, will search for credentials in known
+;; environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files
+;; (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata.
 ;MINIO_ACCESS_KEY_ID =
 ;;
 ;; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`

docs/content/administration/config-cheat-sheet.en-us.md

@@ -843,7 +843,7 @@ Default templates for project board view:
 - `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
 - `PATH`: **attachments**: Path to store attachments only available when STORAGE_TYPE is `local`, relative paths will be resolved to `${AppDataPath}/${attachment.PATH}`.
 - `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when STORAGE_TYPE is `minio`
-- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
+- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`. If not provided and STORAGE_TYPE is `minio`, will search for credentials in known environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata.
 - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
 - `MINIO_BUCKET`: **gitea**: Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
 - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when STORAGE_TYPE is `minio`
@@ -1274,7 +1274,7 @@ is `data/lfs` and the default of `MINIO_BASE_PATH` is `lfs/`.
 - `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
 - `PATH`: **./data/lfs**: Where to store LFS files, only available when `STORAGE_TYPE` is `local`. If not set it fall back to deprecated LFS_CONTENT_PATH value in [server] section.
 - `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio`
-- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
+- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`. If not provided and STORAGE_TYPE is `minio`, will search for credentials in known environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata.
 - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio`
 - `MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `STORAGE_TYPE` is `minio`
 - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio`
@@ -1290,7 +1290,7 @@ Default storage configuration for attachments, lfs, avatars, repo-avatars, repo-
 - `STORAGE_TYPE`: **local**: Storage type, `local` for local disk or `minio` for s3 compatible object storage service.
 - `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
 - `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio`
-- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
+- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`. If not provided and STORAGE_TYPE is `minio`, will search for credentials in known environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata.
 - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio`
 - `MINIO_BUCKET`: **gitea**: Minio bucket to store the data only available when `STORAGE_TYPE` is `minio`
 - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio`
@@ -1305,7 +1305,10 @@ The recommended storage configuration for minio like below:
 STORAGE_TYPE = minio
 ; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
 MINIO_ENDPOINT = localhost:9000
-; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
+; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`.
+; If not provided and STORAGE_TYPE is `minio`, will search for credentials in known
+; environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files
+; (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata.
 MINIO_ACCESS_KEY_ID =
 ; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
 MINIO_SECRET_ACCESS_KEY =
@@ -1354,7 +1357,10 @@ STORAGE_TYPE = my_minio
 STORAGE_TYPE = minio
 ; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
 MINIO_ENDPOINT = localhost:9000
-; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
+; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`.
+; If not provided and STORAGE_TYPE is `minio`, will search for credentials in known
+; environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files
+; (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata.
 MINIO_ACCESS_KEY_ID =
 ; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
 MINIO_SECRET_ACCESS_KEY =
@@ -1380,7 +1386,7 @@ is `data/repo-archive` and the default of `MINIO_BASE_PATH` is `repo-archive/`.
 - `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
 - `PATH`: **./data/repo-archive**: Where to store archive files, only available when `STORAGE_TYPE` is `local`.
 - `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio`
-- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
+- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`. If not provided and STORAGE_TYPE is `minio`, will search for credentials in known environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata.
 - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio`
 - `MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `STORAGE_TYPE` is `minio`
 - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio`

modules/storage/minio.go

@@ -97,7 +97,7 @@ func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage,
 	}
 
 	minioClient, err := minio.New(config.Endpoint, &minio.Options{
-		Creds:        credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""),
+		Creds:        buildMinioCredentials(config, credentials.DefaultIAMRoleEndpoint),
 		Secure:       config.UseSSL,
 		Transport:    &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}},
 		Region:       config.Location,
@@ -164,6 +164,35 @@ func (m *MinioStorage) buildMinioDirPrefix(p string) string {
 	return p
 }
 
+func buildMinioCredentials(config setting.MinioStorageConfig, iamEndpoint string) *credentials.Credentials {
+	// If static credentials are provided, use those
+	if config.AccessKeyID != "" {
+		return credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, "")
+	}
+
+	// Otherwise, fallback to a credentials chain for S3 access
+	chain := []credentials.Provider{
+		// configure based upon MINIO_ prefixed environment variables
+		&credentials.EnvMinio{},
+		// configure based upon AWS_ prefixed environment variables
+		&credentials.EnvAWS{},
+		// read credentials from MINIO_SHARED_CREDENTIALS_FILE
+		// environment variable, or default json config files
+		&credentials.FileMinioClient{},
+		// read credentials from AWS_SHARED_CREDENTIALS_FILE
+		// environment variable, or default credentials file
+		&credentials.FileAWSCredentials{},
+		// read IAM role from EC2 metadata endpoint if available
+		&credentials.IAM{
+			Endpoint: iamEndpoint,
+			Client: &http.Client{
+				Transport: http.DefaultTransport,
+			},
+		},
+	}
+	return credentials.NewChainCredentials(chain)
+}
+
 // Open opens a file
 func (m *MinioStorage) Open(path string) (Object, error) {
 	opts := minio.GetObjectOptions{}

modules/storage/minio_test.go

@@ -6,6 +6,7 @@ package storage
 import (
 	"context"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"testing"
 
@@ -92,3 +93,106 @@ func TestS3StorageBadRequest(t *testing.T) {
 	_, err := NewStorage(setting.MinioStorageType, cfg)
 	assert.ErrorContains(t, err, message)
 }
+
+func TestMinioCredentials(t *testing.T) {
+	const (
+		ExpectedAccessKey       = "ExampleAccessKeyID"
+		ExpectedSecretAccessKey = "ExampleSecretAccessKeyID"
+		// Use a FakeEndpoint for IAM credentials to avoid logging any
+		// potential real IAM credentials when running in EC2.
+		FakeEndpoint = "http://localhost"
+	)
+
+	t.Run("Static Credentials", func(t *testing.T) {
+		cfg := setting.MinioStorageConfig{
+			AccessKeyID:     ExpectedAccessKey,
+			SecretAccessKey: ExpectedSecretAccessKey,
+		}
+		creds := buildMinioCredentials(cfg, FakeEndpoint)
+		v, err := creds.Get()
+
+		assert.NoError(t, err)
+		assert.Equal(t, ExpectedAccessKey, v.AccessKeyID)
+		assert.Equal(t, ExpectedSecretAccessKey, v.SecretAccessKey)
+	})
+
+	t.Run("Chain", func(t *testing.T) {
+		cfg := setting.MinioStorageConfig{}
+
+		t.Run("EnvMinio", func(t *testing.T) {
+			t.Setenv("MINIO_ACCESS_KEY", ExpectedAccessKey+"Minio")
+			t.Setenv("MINIO_SECRET_KEY", ExpectedSecretAccessKey+"Minio")
+
+			creds := buildMinioCredentials(cfg, FakeEndpoint)
+			v, err := creds.Get()
+
+			assert.NoError(t, err)
+			assert.Equal(t, ExpectedAccessKey+"Minio", v.AccessKeyID)
+			assert.Equal(t, ExpectedSecretAccessKey+"Minio", v.SecretAccessKey)
+		})
+
+		t.Run("EnvAWS", func(t *testing.T) {
+			t.Setenv("AWS_ACCESS_KEY", ExpectedAccessKey+"AWS")
+			t.Setenv("AWS_SECRET_KEY", ExpectedSecretAccessKey+"AWS")
+
+			creds := buildMinioCredentials(cfg, FakeEndpoint)
+			v, err := creds.Get()
+
+			assert.NoError(t, err)
+			assert.Equal(t, ExpectedAccessKey+"AWS", v.AccessKeyID)
+			assert.Equal(t, ExpectedSecretAccessKey+"AWS", v.SecretAccessKey)
+		})
+
+		t.Run("FileMinio", func(t *testing.T) {
+			t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/minio.json")
+			// prevent loading any actual credentials files from the user
+			t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/fake")
+
+			creds := buildMinioCredentials(cfg, FakeEndpoint)
+			v, err := creds.Get()
+
+			assert.NoError(t, err)
+			assert.Equal(t, ExpectedAccessKey+"MinioFile", v.AccessKeyID)
+			assert.Equal(t, ExpectedSecretAccessKey+"MinioFile", v.SecretAccessKey)
+		})
+
+		t.Run("FileAWS", func(t *testing.T) {
+			// prevent loading any actual credentials files from the user
+			t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/fake.json")
+			t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/aws_credentials")
+
+			creds := buildMinioCredentials(cfg, FakeEndpoint)
+			v, err := creds.Get()
+
+			assert.NoError(t, err)
+			assert.Equal(t, ExpectedAccessKey+"AWSFile", v.AccessKeyID)
+			assert.Equal(t, ExpectedSecretAccessKey+"AWSFile", v.SecretAccessKey)
+		})
+
+		t.Run("IAM", func(t *testing.T) {
+			// prevent loading any actual credentials files from the user
+			t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/fake.json")
+			t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/fake")
+
+			// Spawn a server to emulate the EC2 Instance Metadata
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				// The client will actually make 3 requests here,
+				// first will be to get the IMDSv2 token, second to
+				// get the role, and third for the actual
+				// credentials. However, we can return credentials
+				// every request since we're not emulating a full
+				// IMDSv2 flow.
+				w.Write([]byte(`{"Code":"Success","AccessKeyId":"ExampleAccessKeyIDIAM","SecretAccessKey":"ExampleSecretAccessKeyIDIAM"}`))
+			}))
+			defer server.Close()
+
+			// Use the provided EC2 Instance Metadata server
+			creds := buildMinioCredentials(cfg, server.URL)
+			v, err := creds.Get()
+
+			assert.NoError(t, err)
+			assert.Equal(t, ExpectedAccessKey+"IAM", v.AccessKeyID)
+			assert.Equal(t, ExpectedSecretAccessKey+"IAM", v.SecretAccessKey)
+		})
+	})
+}

modules/storage/testdata/aws_credentials

@@ -0,0 +1,3 @@
+[default]
+aws_access_key_id=ExampleAccessKeyIDAWSFile
+aws_secret_access_key=ExampleSecretAccessKeyIDAWSFile

modules/storage/testdata/minio.json

@@ -0,0 +1,12 @@
+{
+        "version": "10",
+        "aliases": {
+                "s3": {
+                        "url": "https://s3.amazonaws.com",
+                        "accessKey": "ExampleAccessKeyIDMinioFile",
+                        "secretKey": "ExampleSecretAccessKeyIDMinioFile",
+                        "api": "S3v4",
+                        "path": "dns"
+                }
+        }
+}