aniani/gitea
1
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2024-12-04 14:46:57 -05:00
Code

Make environment-to-ini support loading key value from file (#24832)

Browse Source 
Replace #19857

Close #19856
Close #10311
Close #10123

Major changes:

1. Move a lot of code from `environment-to-ini.go` to `config_env.go` to
make them testable.
2. Add `__FILE` support
3. Update documents
4. Add tests
This commit is contained in:
wxiaoguang 2023-05-24 11:37:22 +08:00 committed by GitHub
parent 1aa9107fea
commit c21605951b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 276 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
contrib/environment-to-ini/environment-to-ini.go
View File

@ -5,8 +5,6 @@ package main
import (
"os"
"regexp"
"strconv"
"strings"
"code.gitea.io/gitea/modules/log"
@ -14,7 +12,7 @@ import (
"code.gitea.io/gitea/modules/util"
"github.com/urfave/cli"
ini "gopkg.in/ini.v1"
"gopkg.in/ini.v1"
)
// EnvironmentPrefix environment variables prefixed with this represent ini values to write
@ -32,6 +30,10 @@ func main() {
will be mapped to the ini section "[section_name]" and the key
"KEY_NAME" with the value as provided.
Environment variables of the form "GITEA__SECTION_NAME__KEY_NAME__FILE"
will be mapped to the ini section "[section_name]" and the key
"KEY_NAME" with the value loaded from the specified file.
Environment variables are usually restricted to a reduced character
set "0-9A-Z_" - in order to allow the setting of sections with
characters outside of that set, they should be escaped as following:
@ -96,11 +98,11 @@ func runEnvironmentToIni(c *cli.Context) error {
setting.SetCustomPathAndConf(providedCustom, providedConf, providedWorkPath)
cfg := ini.Empty()
isFile, err := util.IsFile(setting.CustomConf)
confFileExists, err := util.IsFile(setting.CustomConf)
if err != nil {
log.Fatal("Unable to check if %s is a file. Error: %v", setting.CustomConf, err)
}
if isFile {
if confFileExists {
if err := cfg.Append(setting.CustomConf); err != nil {
log.Fatal("Failed to load custom conf '%s': %v", setting.CustomConf, err)
}
@ -109,47 +111,11 @@ func runEnvironmentToIni(c *cli.Context) error {
}
cfg.NameMapper = ini.SnackCase
changed := false
prefixGitea := c.String("prefix") + "__"
suffixFile := "__FILE"
changed := setting.EnvironmentToConfig(cfg, prefixGitea, suffixFile, os.Environ())
prefix := c.String("prefix") + "__"
for _, kv := range os.Environ() {
idx := strings.IndexByte(kv, '=')
if idx < 0 {
continue
}
eKey := kv[:idx]
value := kv[idx+1:]
if !strings.HasPrefix(eKey, prefix) {
continue
}
eKey = eKey[len(prefix):]
sectionName, keyName := DecodeSectionKey(eKey)
if len(keyName) == 0 {
continue
}
section, err := cfg.GetSection(sectionName)
if err != nil {
section, err = cfg.NewSection(sectionName)
if err != nil {
log.Error("Error creating section: %s : %v", sectionName, err)
continue
}
}
key := section.Key(keyName)
if key == nil {
key, err = section.NewKey(keyName, value)
if err != nil {
log.Error("Error creating key: %s in section: %s with value: %s : %v", keyName, sectionName, value, err)
continue
}
}
oldValue := key.Value()
if !changed && oldValue != value {
changed = true
}
key.SetValue(value)
}
// try to save the config file
destination := c.String("out")
if len(destination) == 0 {
destination = setting.CustomConf
@ -161,6 +127,8 @@ func runEnvironmentToIni(c *cli.Context) error {
return err
}
}
// clear Gitea's specific environment variables if requested
if c.Bool("clear") {
for _, kv := range os.Environ() {
idx := strings.IndexByte(kv, '=')
@ -168,69 +136,11 @@ func runEnvironmentToIni(c *cli.Context) error {
continue
}
eKey := kv[:idx]
if strings.HasPrefix(eKey, prefix) {
if strings.HasPrefix(eKey, prefixGitea) {
_ = os.Unsetenv(eKey)
}
}
}
return nil
}
const escapeRegexpString = "_0[xX](([0-9a-fA-F][0-9a-fA-F])+)_"
var escapeRegex = regexp.MustCompile(escapeRegexpString)
// DecodeSectionKey will decode a portable string encoded Section__Key pair
// Portable strings are considered to be of the form [A-Z0-9_]*
// We will encode a disallowed value as the UTF8 byte string preceded by _0X and
// followed by _. E.g. _0X2C_ for a '-' and _0X2E_ for '.'
// Section and Key are separated by a plain '__'.
// The entire section can be encoded as a UTF8 byte string
func DecodeSectionKey(encoded string) (string, string) {
section := ""
key := ""
inKey := false
last := 0
escapeStringIndices := escapeRegex.FindAllStringIndex(encoded, -1)
for _, unescapeIdx := range escapeStringIndices {
preceding := encoded[last:unescapeIdx[0]]
if !inKey {
if splitter := strings.Index(preceding, "__"); splitter > -1 {
section += preceding[:splitter]
inKey = true
key += preceding[splitter+2:]
} else {
section += preceding
}
} else {
key += preceding
}
toDecode := encoded[unescapeIdx[0]+3 : unescapeIdx[1]-1]
decodedBytes := make([]byte, len(toDecode)/2)
for i := 0; i < len(toDecode)/2; i++ {
// Can ignore error here as we know these should be hexadecimal from the regexp
byteInt, _ := strconv.ParseInt(toDecode[2*i:2*i+2], 16, 0)
decodedBytes[i] = byte(byteInt)
}
if inKey {
key += string(decodedBytes)
} else {
section += string(decodedBytes)
}
last = unescapeIdx[1]
}
remaining := encoded[last:]
if !inKey {
if splitter := strings.Index(remaining, "__"); splitter > -1 {
section += remaining[:splitter]
key += remaining[splitter+2:]
} else {
section += remaining
}
} else {
key += remaining
}
section = strings.ToLower(section)
return section, key
}

13
docs/content/doc/installation/with-docker-rootless.en-us.md
View File

@ -286,9 +286,18 @@ docker-compose up -d
## Managing Deployments With Environment Variables
In addition to the environment variables above, any settings in `app.ini` can be set or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`. These settings are applied each time the docker container starts. Full information [here](https://github.com/go-gitea/gitea/tree/main/contrib/environment-to-ini).
In addition to the environment variables above, any settings in `app.ini` can be set
or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`.
These settings are applied each time the docker container starts.
Full information [here](https://github.com/go-gitea/gitea/tree/main/contrib/environment-to-ini).
These environment variables can be passed to the docker container in `docker-compose.yml`. The following example will enable an smtp mail server if the required env variables `GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host or in a `.env` file in the same directory as `docker-compose.yml`:
These environment variables can be passed to the docker container in `docker-compose.yml`.
The following example will enable a smtp mail server if the required env variables
`GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host
or in a `.env` file in the same directory as `docker-compose.yml`.
The settings can be also set or overridden with the content of a file by defining an environment variable of the form:
`GITEA__section_name__KEY_NAME__FILE` that points to a file.
```bash
...

13
docs/content/doc/installation/with-docker.en-us.md
View File

@ -287,9 +287,18 @@ docker-compose up -d
## Managing Deployments With Environment Variables
In addition to the environment variables above, any settings in `app.ini` can be set or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`. These settings are applied each time the docker container starts. Full information [here](https://github.com/go-gitea/gitea/tree/master/contrib/environment-to-ini).
In addition to the environment variables above, any settings in `app.ini` can be set
or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`.
These settings are applied each time the docker container starts.
Full information [here](https://github.com/go-gitea/gitea/tree/master/contrib/environment-to-ini).
These environment variables can be passed to the docker container in `docker-compose.yml`. The following example will enable an smtp mail server if the required env variables `GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host or in a `.env` file in the same directory as `docker-compose.yml`:
These environment variables can be passed to the docker container in `docker-compose.yml`.
The following example will enable an smtp mail server if the required env variables
`GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host
or in a `.env` file in the same directory as `docker-compose.yml`.
The settings can be also set or overridden with the content of a file by defining an environment variable of the form:
`GITEA__section_name__KEY_NAME__FILE` that points to a file.
```bash
...

142
modules/setting/config_env.go Normal file
View File

@ -0,0 +1,142 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package setting
import (
"os"
"regexp"
"strconv"
"strings"
"code.gitea.io/gitea/modules/log"
"gopkg.in/ini.v1"
)
const escapeRegexpString = "_0[xX](([0-9a-fA-F][0-9a-fA-F])+)_"
var escapeRegex = regexp.MustCompile(escapeRegexpString)
// decodeEnvSectionKey will decode a portable string encoded Section__Key pair
// Portable strings are considered to be of the form [A-Z0-9_]*
// We will encode a disallowed value as the UTF8 byte string preceded by _0X and
// followed by _. E.g. _0X2C_ for a '-' and _0X2E_ for '.'
// Section and Key are separated by a plain '__'.
// The entire section can be encoded as a UTF8 byte string
func decodeEnvSectionKey(encoded string) (ok bool, section, key string) {
inKey := false
last := 0
escapeStringIndices := escapeRegex.FindAllStringIndex(encoded, -1)
for _, unescapeIdx := range escapeStringIndices {
preceding := encoded[last:unescapeIdx[0]]
if !inKey {
if splitter := strings.Index(preceding, "__"); splitter > -1 {
section += preceding[:splitter]
inKey = true
key += preceding[splitter+2:]
} else {
section += preceding
}
} else {
key += preceding
}
toDecode := encoded[unescapeIdx[0]+3 : unescapeIdx[1]-1]
decodedBytes := make([]byte, len(toDecode)/2)
for i := 0; i < len(toDecode)/2; i++ {
// Can ignore error here as we know these should be hexadecimal from the regexp
byteInt, _ := strconv.ParseInt(toDecode[2*i:2*i+2], 16, 0)
decodedBytes[i] = byte(byteInt)
}
if inKey {
key += string(decodedBytes)
} else {
section += string(decodedBytes)
}
last = unescapeIdx[1]
}
remaining := encoded[last:]
if !inKey {
if splitter := strings.Index(remaining, "__"); splitter > -1 {
section += remaining[:splitter]
key += remaining[splitter+2:]
} else {
section += remaining
}
} else {
key += remaining
}
section = strings.ToLower(section)
ok = section != "" && key != ""
if !ok {
section = ""
key = ""
}
return ok, section, key
}
// decodeEnvironmentKey decode the environment key to section and key
// The environment key is in the form of GITEA__SECTION__KEY or GITEA__SECTION__KEY__FILE
func decodeEnvironmentKey(prefixGitea, suffixFile, envKey string) (ok bool, section, key string, useFileValue bool) {
if !strings.HasPrefix(envKey, prefixGitea) {
return false, "", "", false
}
if strings.HasSuffix(envKey, suffixFile) {
useFileValue = true
envKey = envKey[:len(envKey)-len(suffixFile)]
}
ok, section, key = decodeEnvSectionKey(envKey[len(prefixGitea):])
return ok, section, key, useFileValue
}
func EnvironmentToConfig(cfg *ini.File, prefixGitea, suffixFile string, envs []string) (changed bool) {
for _, kv := range envs {
idx := strings.IndexByte(kv, '=')
if idx < 0 {
continue
}
// parse the environment variable to config section name and key name
envKey := kv[:idx]
envValue := kv[idx+1:]
ok, sectionName, keyName, useFileValue := decodeEnvironmentKey(prefixGitea, suffixFile, envKey)
if !ok {
continue
}
// use environment value as config value, or read the file content as value if the key indicates a file
keyValue := envValue
if useFileValue {
fileContent, err := os.ReadFile(envValue)
if err != nil {
log.Error("Error reading file for %s : %v", envKey, envValue, err)
continue
}
keyValue = string(fileContent)
}
// try to set the config value if necessary
section, err := cfg.GetSection(sectionName)
if err != nil {
section, err = cfg.NewSection(sectionName)
if err != nil {
log.Error("Error creating section: %s : %v", sectionName, err)
continue
}
}
key := section.Key(keyName)
if key == nil {
key, err = section.NewKey(keyName, keyValue)
if err != nil {
log.Error("Error creating key: %s in section: %s with value: %s : %v", keyName, sectionName, keyValue, err)
continue
}
}
oldValue := key.Value()
if !changed && oldValue != keyValue {
changed = true
}
key.SetValue(keyValue)
}
return changed
}

97
modules/setting/config_env_test.go Normal file
View File

@ -0,0 +1,97 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package setting
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"gopkg.in/ini.v1"
)
func TestDecodeEnvSectionKey(t *testing.T) {
ok, section, key := decodeEnvSectionKey("SEC__KEY")
assert.True(t, ok)
assert.Equal(t, "sec", section)
assert.Equal(t, "KEY", key)
ok, section, key = decodeEnvSectionKey("sec__key")
assert.True(t, ok)
assert.Equal(t, "sec", section)
assert.Equal(t, "key", key)
ok, section, key = decodeEnvSectionKey("LOG_0x2E_CONSOLE__STDERR")
assert.True(t, ok)
assert.Equal(t, "log.console", section)
assert.Equal(t, "STDERR", key)
ok, section, key = decodeEnvSectionKey("SEC")
assert.False(t, ok)
assert.Equal(t, "", section)
assert.Equal(t, "", key)
}
func TestDecodeEnvironmentKey(t *testing.T) {
prefix := "GITEA__"
suffix := "__FILE"
ok, section, key, file := decodeEnvironmentKey(prefix, suffix, "SEC__KEY")
assert.False(t, ok)
assert.Equal(t, "", section)
assert.Equal(t, "", key)
assert.False(t, file)
ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC")
assert.False(t, ok)
assert.Equal(t, "", section)
assert.Equal(t, "", key)
assert.False(t, file)
ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY")
assert.True(t, ok)
assert.Equal(t, "sec", section)
assert.Equal(t, "KEY", key)
assert.False(t, file)
// with "__FILE" suffix, it doesn't support to write "[sec].FILE" to config (no such key FILE is used in Gitea)
// but it could be fixed in the future by adding a new suffix like "__VALUE" (no such key VALUE is used in Gitea either)
ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__FILE")
assert.False(t, ok)
assert.Equal(t, "", section)
assert.Equal(t, "", key)
assert.True(t, file)
ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY__FILE")
assert.True(t, ok)
assert.Equal(t, "sec", section)
assert.Equal(t, "KEY", key)
assert.True(t, file)
}
func TestEnvironmentToConfig(t *testing.T) {
cfg := ini.Empty()
changed := EnvironmentToConfig(cfg, "GITEA__", "__FILE", nil)
assert.False(t, changed)
cfg, err := ini.Load([]byte(`
[sec]
key = old
`))
assert.NoError(t, err)
changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key=new"})
assert.True(t, changed)
assert.Equal(t, "new", cfg.Section("sec").Key("key").String())
changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key=new"})
assert.False(t, changed)
tmpFile := t.TempDir() + "/the-file"
_ = os.WriteFile(tmpFile, []byte("value-from-file"), 0o644)
changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key__FILE=" + tmpFile})
assert.True(t, changed)
assert.Equal(t, "value-from-file", cfg.Section("sec").Key("key").String())
}