Make environment-to-ini support loading key value from file (#24832)

Replace #19857

Close #19856
Close #10311
Close #10123

Major changes:

1. Move a lot of code from `environment-to-ini.go` to `config_env.go` to
make them testable.
2. Add `__FILE` support
3. Update documents
4. Add tests

contrib/environment-to-ini/environment-to-ini.go

@@ -5,8 +5,6 @@ package main
 
 import (
 	"os"
-	"regexp"
-	"strconv"
 	"strings"
 
 	"code.gitea.io/gitea/modules/log"
@@ -14,7 +12,7 @@ import (
 	"code.gitea.io/gitea/modules/util"
 
 	"github.com/urfave/cli"
-	ini "gopkg.in/ini.v1"
+	"gopkg.in/ini.v1"
 )
 
 // EnvironmentPrefix environment variables prefixed with this represent ini values to write
@@ -32,6 +30,10 @@ func main() {
 	will be mapped to the ini section "[section_name]" and the key
 	"KEY_NAME" with the value as provided.
 
+	Environment variables of the form "GITEA__SECTION_NAME__KEY_NAME__FILE"
+	will be mapped to the ini section "[section_name]" and the key
+	"KEY_NAME" with the value loaded from the specified file.
+
 	Environment variables are usually restricted to a reduced character
 	set "0-9A-Z_" - in order to allow the setting of sections with
 	characters outside of that set, they should be escaped as following:
@@ -96,11 +98,11 @@ func runEnvironmentToIni(c *cli.Context) error {
 	setting.SetCustomPathAndConf(providedCustom, providedConf, providedWorkPath)
 
 	cfg := ini.Empty()
-	isFile, err := util.IsFile(setting.CustomConf)
+	confFileExists, err := util.IsFile(setting.CustomConf)
 	if err != nil {
 		log.Fatal("Unable to check if %s is a file. Error: %v", setting.CustomConf, err)
 	}
-	if isFile {
+	if confFileExists {
 		if err := cfg.Append(setting.CustomConf); err != nil {
 			log.Fatal("Failed to load custom conf '%s': %v", setting.CustomConf, err)
 		}
@@ -109,47 +111,11 @@ func runEnvironmentToIni(c *cli.Context) error {
 	}
 	cfg.NameMapper = ini.SnackCase
 
-	changed := false
+	prefixGitea := c.String("prefix") + "__"
+	suffixFile := "__FILE"
+	changed := setting.EnvironmentToConfig(cfg, prefixGitea, suffixFile, os.Environ())
 
-	prefix := c.String("prefix") + "__"
+	// try to save the config file
-
-	for _, kv := range os.Environ() {
-		idx := strings.IndexByte(kv, '=')
-		if idx < 0 {
-			continue
-		}
-		eKey := kv[:idx]
-		value := kv[idx+1:]
-		if !strings.HasPrefix(eKey, prefix) {
-			continue
-		}
-		eKey = eKey[len(prefix):]
-		sectionName, keyName := DecodeSectionKey(eKey)
-		if len(keyName) == 0 {
-			continue
-		}
-		section, err := cfg.GetSection(sectionName)
-		if err != nil {
-			section, err = cfg.NewSection(sectionName)
-			if err != nil {
-				log.Error("Error creating section: %s : %v", sectionName, err)
-				continue
-			}
-		}
-		key := section.Key(keyName)
-		if key == nil {
-			key, err = section.NewKey(keyName, value)
-			if err != nil {
-				log.Error("Error creating key: %s in section: %s with value: %s : %v", keyName, sectionName, value, err)
-				continue
-			}
-		}
-		oldValue := key.Value()
-		if !changed && oldValue != value {
-			changed = true
-		}
-		key.SetValue(value)
-	}
 	destination := c.String("out")
 	if len(destination) == 0 {
 		destination = setting.CustomConf
@@ -161,6 +127,8 @@ func runEnvironmentToIni(c *cli.Context) error {
 			return err
 		}
 	}
+
+	// clear Gitea's specific environment variables if requested
 	if c.Bool("clear") {
 		for _, kv := range os.Environ() {
 			idx := strings.IndexByte(kv, '=')
@@ -168,69 +136,11 @@ func runEnvironmentToIni(c *cli.Context) error {
 				continue
 			}
 			eKey := kv[:idx]
-			if strings.HasPrefix(eKey, prefix) {
+			if strings.HasPrefix(eKey, prefixGitea) {
 				_ = os.Unsetenv(eKey)
 			}
 		}
 	}
+
 	return nil
 }
-
-const escapeRegexpString = "_0[xX](([0-9a-fA-F][0-9a-fA-F])+)_"
-
-var escapeRegex = regexp.MustCompile(escapeRegexpString)
-
-// DecodeSectionKey will decode a portable string encoded Section__Key pair
-// Portable strings are considered to be of the form [A-Z0-9_]*
-// We will encode a disallowed value as the UTF8 byte string preceded by _0X and
-// followed by _. E.g. _0X2C_ for a '-' and _0X2E_ for '.'
-// Section and Key are separated by a plain '__'.
-// The entire section can be encoded as a UTF8 byte string
-func DecodeSectionKey(encoded string) (string, string) {
-	section := ""
-	key := ""
-
-	inKey := false
-	last := 0
-	escapeStringIndices := escapeRegex.FindAllStringIndex(encoded, -1)
-	for _, unescapeIdx := range escapeStringIndices {
-		preceding := encoded[last:unescapeIdx[0]]
-		if !inKey {
-			if splitter := strings.Index(preceding, "__"); splitter > -1 {
-				section += preceding[:splitter]
-				inKey = true
-				key += preceding[splitter+2:]
-			} else {
-				section += preceding
-			}
-		} else {
-			key += preceding
-		}
-		toDecode := encoded[unescapeIdx[0]+3 : unescapeIdx[1]-1]
-		decodedBytes := make([]byte, len(toDecode)/2)
-		for i := 0; i < len(toDecode)/2; i++ {
-			// Can ignore error here as we know these should be hexadecimal from the regexp
-			byteInt, _ := strconv.ParseInt(toDecode[2*i:2*i+2], 16, 0)
-			decodedBytes[i] = byte(byteInt)
-		}
-		if inKey {
-			key += string(decodedBytes)
-		} else {
-			section += string(decodedBytes)
-		}
-		last = unescapeIdx[1]
-	}
-	remaining := encoded[last:]
-	if !inKey {
-		if splitter := strings.Index(remaining, "__"); splitter > -1 {
-			section += remaining[:splitter]
-			key += remaining[splitter+2:]
-		} else {
-			section += remaining
-		}
-	} else {
-		key += remaining
-	}
-	section = strings.ToLower(section)
-	return section, key
-}

docs/content/doc/installation/with-docker-rootless.en-us.md

@@ -286,9 +286,18 @@ docker-compose up -d
 
 ## Managing Deployments With Environment Variables
 
-In addition to the environment variables above, any settings in `app.ini` can be set or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`. These settings are applied each time the docker container starts. Full information [here](https://github.com/go-gitea/gitea/tree/main/contrib/environment-to-ini).
+In addition to the environment variables above, any settings in `app.ini` can be set
+or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`.
+These settings are applied each time the docker container starts.
+Full information [here](https://github.com/go-gitea/gitea/tree/main/contrib/environment-to-ini).
 
-These environment variables can be passed to the docker container in `docker-compose.yml`. The following example will enable an smtp mail server if the required env variables `GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host or in a `.env` file in the same directory as `docker-compose.yml`:
+These environment variables can be passed to the docker container in `docker-compose.yml`.
+The following example will enable a smtp mail server if the required env variables
+`GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host
+or in a `.env` file in the same directory as `docker-compose.yml`.
+
+The settings can be also set or overridden with the content of a file by defining an environment variable of the form:
+`GITEA__section_name__KEY_NAME__FILE` that points to a file.
 
 ```bash
 ...

docs/content/doc/installation/with-docker.en-us.md

@@ -287,9 +287,18 @@ docker-compose up -d
 
 ## Managing Deployments With Environment Variables
 
-In addition to the environment variables above, any settings in `app.ini` can be set or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`. These settings are applied each time the docker container starts. Full information [here](https://github.com/go-gitea/gitea/tree/master/contrib/environment-to-ini).
+In addition to the environment variables above, any settings in `app.ini` can be set
+or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`.
+These settings are applied each time the docker container starts.
+Full information [here](https://github.com/go-gitea/gitea/tree/master/contrib/environment-to-ini).
 
-These environment variables can be passed to the docker container in `docker-compose.yml`. The following example will enable an smtp mail server if the required env variables `GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host or in a `.env` file in the same directory as `docker-compose.yml`:
+These environment variables can be passed to the docker container in `docker-compose.yml`.
+The following example will enable an smtp mail server if the required env variables
+`GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host
+or in a `.env` file in the same directory as `docker-compose.yml`.
+
+The settings can be also set or overridden with the content of a file by defining an environment variable of the form:
+`GITEA__section_name__KEY_NAME__FILE` that points to a file.
 
 ```bash
 ...

modules/setting/config_env.go

@@ -0,0 +1,142 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package setting
+
+import (
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+
+	"gopkg.in/ini.v1"
+)
+
+const escapeRegexpString = "_0[xX](([0-9a-fA-F][0-9a-fA-F])+)_"
+
+var escapeRegex = regexp.MustCompile(escapeRegexpString)
+
+// decodeEnvSectionKey will decode a portable string encoded Section__Key pair
+// Portable strings are considered to be of the form [A-Z0-9_]*
+// We will encode a disallowed value as the UTF8 byte string preceded by _0X and
+// followed by _. E.g. _0X2C_ for a '-' and _0X2E_ for '.'
+// Section and Key are separated by a plain '__'.
+// The entire section can be encoded as a UTF8 byte string
+func decodeEnvSectionKey(encoded string) (ok bool, section, key string) {
+	inKey := false
+	last := 0
+	escapeStringIndices := escapeRegex.FindAllStringIndex(encoded, -1)
+	for _, unescapeIdx := range escapeStringIndices {
+		preceding := encoded[last:unescapeIdx[0]]
+		if !inKey {
+			if splitter := strings.Index(preceding, "__"); splitter > -1 {
+				section += preceding[:splitter]
+				inKey = true
+				key += preceding[splitter+2:]
+			} else {
+				section += preceding
+			}
+		} else {
+			key += preceding
+		}
+		toDecode := encoded[unescapeIdx[0]+3 : unescapeIdx[1]-1]
+		decodedBytes := make([]byte, len(toDecode)/2)
+		for i := 0; i < len(toDecode)/2; i++ {
+			// Can ignore error here as we know these should be hexadecimal from the regexp
+			byteInt, _ := strconv.ParseInt(toDecode[2*i:2*i+2], 16, 0)
+			decodedBytes[i] = byte(byteInt)
+		}
+		if inKey {
+			key += string(decodedBytes)
+		} else {
+			section += string(decodedBytes)
+		}
+		last = unescapeIdx[1]
+	}
+	remaining := encoded[last:]
+	if !inKey {
+		if splitter := strings.Index(remaining, "__"); splitter > -1 {
+			section += remaining[:splitter]
+			key += remaining[splitter+2:]
+		} else {
+			section += remaining
+		}
+	} else {
+		key += remaining
+	}
+	section = strings.ToLower(section)
+	ok = section != "" && key != ""
+	if !ok {
+		section = ""
+		key = ""
+	}
+	return ok, section, key
+}
+
+// decodeEnvironmentKey decode the environment key to section and key
+// The environment key is in the form of GITEA__SECTION__KEY or GITEA__SECTION__KEY__FILE
+func decodeEnvironmentKey(prefixGitea, suffixFile, envKey string) (ok bool, section, key string, useFileValue bool) {
+	if !strings.HasPrefix(envKey, prefixGitea) {
+		return false, "", "", false
+	}
+	if strings.HasSuffix(envKey, suffixFile) {
+		useFileValue = true
+		envKey = envKey[:len(envKey)-len(suffixFile)]
+	}
+	ok, section, key = decodeEnvSectionKey(envKey[len(prefixGitea):])
+	return ok, section, key, useFileValue
+}
+
+func EnvironmentToConfig(cfg *ini.File, prefixGitea, suffixFile string, envs []string) (changed bool) {
+	for _, kv := range envs {
+		idx := strings.IndexByte(kv, '=')
+		if idx < 0 {
+			continue
+		}
+
+		// parse the environment variable to config section name and key name
+		envKey := kv[:idx]
+		envValue := kv[idx+1:]
+		ok, sectionName, keyName, useFileValue := decodeEnvironmentKey(prefixGitea, suffixFile, envKey)
+		if !ok {
+			continue
+		}
+
+		// use environment value as config value, or read the file content as value if the key indicates a file
+		keyValue := envValue
+		if useFileValue {
+			fileContent, err := os.ReadFile(envValue)
+			if err != nil {
+				log.Error("Error reading file for %s : %v", envKey, envValue, err)
+				continue
+			}
+			keyValue = string(fileContent)
+		}
+
+		// try to set the config value if necessary
+		section, err := cfg.GetSection(sectionName)
+		if err != nil {
+			section, err = cfg.NewSection(sectionName)
+			if err != nil {
+				log.Error("Error creating section: %s : %v", sectionName, err)
+				continue
+			}
+		}
+		key := section.Key(keyName)
+		if key == nil {
+			key, err = section.NewKey(keyName, keyValue)
+			if err != nil {
+				log.Error("Error creating key: %s in section: %s with value: %s : %v", keyName, sectionName, keyValue, err)
+				continue
+			}
+		}
+		oldValue := key.Value()
+		if !changed && oldValue != keyValue {
+			changed = true
+		}
+		key.SetValue(keyValue)
+	}
+	return changed
+}

modules/setting/config_env_test.go

@@ -0,0 +1,97 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package setting
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/ini.v1"
+)
+
+func TestDecodeEnvSectionKey(t *testing.T) {
+	ok, section, key := decodeEnvSectionKey("SEC__KEY")
+	assert.True(t, ok)
+	assert.Equal(t, "sec", section)
+	assert.Equal(t, "KEY", key)
+
+	ok, section, key = decodeEnvSectionKey("sec__key")
+	assert.True(t, ok)
+	assert.Equal(t, "sec", section)
+	assert.Equal(t, "key", key)
+
+	ok, section, key = decodeEnvSectionKey("LOG_0x2E_CONSOLE__STDERR")
+	assert.True(t, ok)
+	assert.Equal(t, "log.console", section)
+	assert.Equal(t, "STDERR", key)
+
+	ok, section, key = decodeEnvSectionKey("SEC")
+	assert.False(t, ok)
+	assert.Equal(t, "", section)
+	assert.Equal(t, "", key)
+}
+
+func TestDecodeEnvironmentKey(t *testing.T) {
+	prefix := "GITEA__"
+	suffix := "__FILE"
+
+	ok, section, key, file := decodeEnvironmentKey(prefix, suffix, "SEC__KEY")
+	assert.False(t, ok)
+	assert.Equal(t, "", section)
+	assert.Equal(t, "", key)
+	assert.False(t, file)
+
+	ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC")
+	assert.False(t, ok)
+	assert.Equal(t, "", section)
+	assert.Equal(t, "", key)
+	assert.False(t, file)
+
+	ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY")
+	assert.True(t, ok)
+	assert.Equal(t, "sec", section)
+	assert.Equal(t, "KEY", key)
+	assert.False(t, file)
+
+	// with "__FILE" suffix, it doesn't support to write "[sec].FILE" to config (no such key FILE is used in Gitea)
+	// but it could be fixed in the future by adding a new suffix like "__VALUE" (no such key VALUE is used in Gitea either)
+	ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__FILE")
+	assert.False(t, ok)
+	assert.Equal(t, "", section)
+	assert.Equal(t, "", key)
+	assert.True(t, file)
+
+	ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY__FILE")
+	assert.True(t, ok)
+	assert.Equal(t, "sec", section)
+	assert.Equal(t, "KEY", key)
+	assert.True(t, file)
+}
+
+func TestEnvironmentToConfig(t *testing.T) {
+	cfg := ini.Empty()
+
+	changed := EnvironmentToConfig(cfg, "GITEA__", "__FILE", nil)
+	assert.False(t, changed)
+
+	cfg, err := ini.Load([]byte(`
+[sec]
+key = old
+`))
+	assert.NoError(t, err)
+
+	changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key=new"})
+	assert.True(t, changed)
+	assert.Equal(t, "new", cfg.Section("sec").Key("key").String())
+
+	changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key=new"})
+	assert.False(t, changed)
+
+	tmpFile := t.TempDir() + "/the-file"
+	_ = os.WriteFile(tmpFile, []byte("value-from-file"), 0o644)
+	changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key__FILE=" + tmpFile})
+	assert.True(t, changed)
+	assert.Equal(t, "value-from-file", cfg.Section("sec").Key("key").String())
+}