Configuration: Support environment expansion in configuration

cmd/loki/main.go

@@ -28,12 +28,13 @@ func init() {
 }
 
 type Config struct {
-	loki.Config  `yaml:",inline"`
-	printVersion bool
-	verifyConfig bool
-	printConfig  bool
-	logConfig    bool
-	configFile   string
+	loki.Config     `yaml:",inline"`
+	printVersion    bool
+	verifyConfig    bool
+	printConfig     bool
+	logConfig       bool
+	configFile      string
+	configExpandEnv bool
 }
 
 func (c *Config) RegisterFlags(f *flag.FlagSet) {
@@ -43,6 +44,7 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&c.logConfig, "log-config-reverse-order", false, "Dump the entire Loki config object at Info log "+
 		"level with the order reversed, reversing the order makes viewing the entries easier in Grafana.")
 	f.StringVar(&c.configFile, "config.file", "", "yaml file to load")
+	f.BoolVar(&c.configExpandEnv, "config.expand-env", false, "Expands ${var} in config according to the values of the environment variables.")
 	c.Config.RegisterFlags(f)
 }
 

cmd/promtail/main.go

@@ -28,12 +28,13 @@ func init() {
 }
 
 type Config struct {
-	config.Config `yaml:",inline"`
-	printVersion  bool
-	printConfig   bool
-	logConfig     bool
-	dryRun        bool
-	configFile    string
+	config.Config   `yaml:",inline"`
+	printVersion    bool
+	printConfig     bool
+	logConfig       bool
+	dryRun          bool
+	configFile      string
+	configExpandEnv bool
 }
 
 func (c *Config) RegisterFlags(f *flag.FlagSet) {
@@ -43,6 +44,7 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) {
 		"level with the order reversed, reversing the order makes viewing the entries easier in Grafana.")
 	f.BoolVar(&c.dryRun, "dry-run", false, "Start Promtail but print entries instead of sending them to Loki.")
 	f.StringVar(&c.configFile, "config.file", "", "yaml file to load")
+	f.BoolVar(&c.configExpandEnv, "config.expand-env", false, "Expands ${var} in config according to the values of the environment variables.")
 	c.Config.RegisterFlags(f)
 }
 

docs/sources/clients/promtail/configuration.md

@@ -80,7 +80,30 @@ For more detailed information on configuring how to discover and scrape logs fro
 targets, see [Scraping](../scraping/). For more information on transforming logs
 from scraped targets, see [Pipelines](../pipelines/).
 
-Generic placeholders are defined as follows:
+### Use environment variables in the configuration
+
+You can use environment variable references in the config file to set values that need to be configurable during deployment.
+To do this, use:
+
+```
+${VAR}
+```
+
+Where VAR is the name of the environment variable.
+
+Each variable reference is replaced at startup by the value of the environment variable.
+The replacement is case-sensitive and occurs before the YAML file is parsed.
+References to undefined variables are replaced by empty strings unless you specify a default value or custom error text.
+
+To specify a default value, use:
+
+```
+${VAR:default_value}
+```
+
+Where default_value is the value to use if the environment variable is undefined.
+
+### Generic placeholders:
 
 - `<boolean>`: a boolean that can take the values `true` or `false`
 - `<int>`: any integer matching the regular expression `[1-9]+[0-9]*`
@@ -93,7 +116,7 @@ Generic placeholders are defined as follows:
 - `<string>`: a regular string
 - `<secret>`: a regular string that is a secret, such as a password
 
-Supported contents and default values of `config.yaml`:
+### Supported contents and default values of `config.yaml`:
 
 ```yaml
 # Configures the server for Promtail.

docs/sources/configuration/_index.md

@@ -64,7 +64,30 @@ command line. The file is written in [YAML format](https://en.wikipedia.org/wiki
 defined by the scheme below. Brackets indicate that a parameter is optional. For
 non-list parameters the value is set to the specified default.
 
-Generic placeholders are defined as follows:
+### Use environment variables in the configuration
+
+You can use environment variable references in the config file to set values that need to be configurable during deployment.
+To do this, use:
+
+```
+${VAR}
+```
+
+Where VAR is the name of the environment variable.
+
+Each variable reference is replaced at startup by the value of the environment variable.
+The replacement is case-sensitive and occurs before the YAML file is parsed.
+References to undefined variables are replaced by empty strings unless you specify a default value or custom error text.
+
+To specify a default value, use:
+
+```
+${VAR:default_value}
+```
+
+Where default_value is the value to use if the environment variable is undefined.
+
+### Generic placeholders:
 
 - `<boolean>` : a boolean that can take the values `true` or `false`
 - `<int>` : any integer matching the regular expression `[1-9]+[0-9]*`
@@ -76,7 +99,7 @@ Generic placeholders are defined as follows:
 - `<string>` : a regular string
 - `<secret>` : a regular string that is a secret, such as a password
 
-Supported contents and default values of `loki.yaml`:
+### Supported contents and default values of `loki.yaml`:
 
 ```yaml
 # The module to run Loki with. Supported values
@@ -1850,3 +1873,4 @@ multi_kv_config:
     mirror-enabled: false
     primary: consul
 ```
+### Generic placeholders

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/docker/docker v17.12.0-ce-rc1.0.20201009160326-9c15e82f19b0+incompatible
 	github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 // indirect
 	github.com/docker/go-plugins-helpers v0.0.0-20181025120712-1e6269c305b8
+	github.com/drone/envsubst v1.0.2
 	github.com/dustin/go-humanize v1.0.0
 	github.com/fatih/color v1.9.0
 	github.com/fluent/fluent-bit-go v0.0.0-20190925192703-ea13c021720c

go.sum

@@ -327,6 +327,8 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/drone/envsubst v1.0.2 h1:dpYLMAspQHW0a8dZpLRKe9jCNvIGZPhCPrycZzIHdqo=
+github.com/drone/envsubst v1.0.2/go.mod h1:bkZbnc/2vh1M12Ecn7EYScpI4YGYU0etwLJICOWi8Z0=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=

pkg/cfg/files.go

@@ -6,7 +6,9 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"strconv"
 
+	"github.com/drone/envsubst"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 )
@@ -36,13 +38,19 @@ func dJSON(y []byte) Source {
 }
 
 // YAML returns a Source that opens the supplied `.yaml` file and loads it.
-func YAML(f string) Source {
+func YAML(f string, envSubst bool) Source {
 	return func(dst Cloneable) error {
 		y, err := ioutil.ReadFile(f)
 		if err != nil {
 			return err
 		}
-
+		if envSubst {
+			s, err := envsubst.EvalEnv(string(y))
+			if err != nil {
+				return err
+			}
+			y = []byte(s)
+		}
 		err = dYAML(y)(dst)
 		return errors.Wrap(err, f)
 	}
@@ -82,8 +90,13 @@ func YAMLFlag(args []string, name string) Source {
 		if f == nil || f.Value.String() == "" {
 			return nil
 		}
+		expandEnv := false
+		expandEnvFlag := freshFlags.Lookup("config.expand-env")
+		if expandEnvFlag != nil {
+			expandEnv, _ = strconv.ParseBool(expandEnvFlag.Value.String()) // Can ignore error as false returned
+		}
 
-		return YAML(f.Value.String())(dst)
+		return YAML(f.Value.String(), expandEnv)(dst)
 
 	}
 }

pkg/logcli/query/query.go

@@ -178,7 +178,7 @@ func (q *Query) DoLocalQuery(out output.LogOutput, statistics bool, orgID string
 	if q.LocalConfig == "" {
 		return errors.New("no supplied config file")
 	}
-	if err := cfg.YAML(q.LocalConfig)(&conf); err != nil {
+	if err := cfg.YAML(q.LocalConfig, false)(&conf); err != nil {
 		return err
 	}