Use viper bindings for all env vars

README.md

@@ -19,14 +19,14 @@ Git as Terraform backend? Seriously? I know, might sound like a stupid idea at f
       - [Standalone Terraform HTTP Backend Mode](#standalone-terraform-http-backend-mode)
     - [Wrappers CLI](#wrappers-cli)
     - [Configuration](#configuration)
-    - [Git Credentials](#git-credentials)
-    - [State Encryption](#state-encryption)
-      - [`sops`](#sops)
-        - [PGP](#pgp)
-        - [AWS KMS](#aws-kms)
-        - [GCP KMS](#gcp-kms)
-        - [Hashicorp Vault](#hashicorp-vault)
-      - [AES256](#aes256)
+      - [Git Credentials](#git-credentials)
+      - [State Encryption](#state-encryption)
+        - [`sops`](#sops)
+          - [PGP](#pgp)
+          - [AWS KMS](#aws-kms)
+          - [GCP KMS](#gcp-kms)
+          - [Hashicorp Vault](#hashicorp-vault)
+        - [AES256](#aes256)
     - [Running backend remotely](#running-backend-remotely)
     - [TLS](#tls)
     - [Basic HTTP Authentication](#basic-http-authentication)
@@ -109,15 +109,26 @@ This mode is explained in more depth in the [wrapper CLI](#wrappers-cli) section
 
 #### Hashicorp Configuration Language (HCL) Mode
 
-You could also create a `terraform-backend-git.hcl` config file and put it next to your `*.tf` code:
+You could also create a `terraform-backend-git.hcl` and `terraform-backend-git.secret.hcl` config files and put it next to your `*.tf` code.
 
+
+*terraform-backend-git.hcl*
 ```hcl
 git.repository = "https://github.com/my-org/tf-state"
 git.ref = "main"
 git.state = "my/state.json"
+
+encryption.provider = "aes"
+```
+
+*terraform-backend-git.secret.hcl*
+```
+encryption.aes.passprase = "<some-strong-passphrase>"
 ```
 
-You can also specify custom path to the `hcl` config file using `--config` arg.
+If you use `terraform-backend-git.secret.hcl` it's essential that you DO NOT commit it to repository. Simpliest way to do it is to add `terraform-backend-git.secret.hcl` to `.gitignore` file.
+
+You can also specify custom path to the `hcl` config file using `--config` arg. Multiple files can be passed by using `--config` arg multiple times.
 
 You can also have a mixed setup, where some parts of configuration comes from `terraform-backend-git.hcl` and some - from CLI arguments or even environment variables (see details below).
 
@@ -184,7 +195,8 @@ Initially it is meant to only support `git` as a storage, hence the name of it i
 
 ### Configuration
 
-CLI | `terraform-backend-git.hcl` | Environment Variable | TF HTTP backend config | Description
+#### Common configuration
+CLI | `terraform-backend-git.[secret].hcl` | Environment Variable | TF HTTP backend config | Description
 --- | --- | --- | --- | ---
 `--repository` | `git.repository` | `TF_BACKEND_GIT_GIT_REPOSITORY` |`repository` | Required; Which repository to use for storing TF state?
 `--ref` | `git.ref` | `TF_BACKEND_GIT_GIT_REF` |`ref` | Optional; Which branch to use in that `repository`? Default: `master`.
@@ -193,27 +205,27 @@ CLI | `terraform-backend-git.hcl` | Environment Variable | TF HTTP backend confi
 `--address` | `address` | `TF_BACKEND_GIT_ADDRESS` | - | Optional; Local binding address and port to listen for HTTP requests. Only change the port, **do not change the address to `0.0.0.0` before you read [Running backend remotely](#running-backend-remotely)**. Default: `127.0.0.1:6061`.
 `--access-logs` | `accessLogs` | `TF_BACKEND_GIT_ACCESSLOGS` | - | Optional; Set to `true` to enable HTTP access logs on backend. Default: `false`.
 
-### Git Credentials
+#### Git Credentials
 
-Both HTTP and SSH protocols are supported. As of now, any sensitive configuration is only supported via environment variables.
+Both HTTP and SSH protocols are supported. Be sure not to commit any sensitive configuration to your respository.
 
-Variable | Description
---- | ---
-`GIT_USERNAME` | Specify username for Git, only required for HTTP protocol.
-`GIT_PASSWORD`/`GITHUB_TOKEN` | Git password or token for HTTP protocol. In case of token you still have to specify `GIT_USERNAME`.
-`SSH_AUTH_SOCK` | `ssh-agent` socket.
-`SSH_PRIVATE_KEY` | Path to SSH key for Git access.
-`StrictHostKeyChecking` | Optional; If set to `no`, will not require strict host key checking. Somewhat more secure way of using Git in automation is to use `ssh -T -oStrictHostKeyChecking=accept-new [email protected]` before starting any automation.
+`terraform-backend-git.secret.hcl` | Environment Variable | Description
+--- | --- | ---
+`git.username` | `GIT_USERNAME` | Specify username for Git, only required for HTTP protocol.
+`git.password` / `git.github_token`| `GIT_PASSWORD` / `GITHUB_TOKEN` | Git password or token for HTTP protocol. In case of token you still have to specify `GIT_USERNAME`.
+ `-` | `SSH_AUTH_SOCK` | `ssh-agent` socket.
+`git.ssh_private_key` | `SSH_PRIVATE_KEY` | Path to SSH key for Git access.
+`git.strict_host_key_checking` | `StrictHostKeyChecking` / `STRICT_HOST_KEY_CHECKING` | Optional; If set to `no`, will not require strict host key checking. Somewhat more secure way of using Git in automation is to use `ssh -T -oStrictHostKeyChecking=accept-new [email protected]` before starting any automation.
 
 Backend will determine which protocol you are using based on the `repository` URL.
 
 For SSH, it will see if `ssh-agent` is running by looking into `SSH_AUTH_SOCK` variable, and if not - it will need a private key. It will try to use `~/.ssh/id_rsa` unless you explicitly specify a different path via `SSH_PRIVATE_KEY`.
 
 Unfortunately `go-git` will not mimic real Git client and will not automatically pickup credentials from the environment, so this custom credentials resolver chain has been implemented since I'm lazy to research the "right" original Git client approach. It is recommended to use Git Credentials Helpers (aka `ASKPASS`).
 
-### State Encryption
+#### State Encryption
 
-To enable encryption set the env var `TF_BACKEND_HTTP_ENCRYPTION_PROVIDER` to one of the following values:
+To enable encryption set the env var `TF_BACKEND_HTTP_ENCRYPTION_PROVIDER` or `encryption.provider` setting to one of the following values:
 
 - `sops`
 - `aes`
@@ -227,29 +239,29 @@ We are using [`sops`](https://github.com/mozilla/sops) as encryption abstraction
 
 Before we integrated with `sops` - we had a basic AES256 encryption via static passphrase. It is no longer recommended, although might be useful in some limited scenarios. Basic AES256 encryption is using one shared key, and it encrypts entire JSON state file that it can no longer be read as JSON. `sops` supports various encryption-as-service providers such as AWS KMS and Hashicorp Vault Transit - meaning encryption can be safely performed without revealing private key to the encryption clients. That means keys can be easily rotated, access can be easily revoked and generally it dramatically reduces chances of the key leaks.
 
-#### `sops`
+##### `sops`
 
-`sops` supports [Shamir's Secret Sharing](https://github.com/mozilla/sops#214key-groups). You can configure multiple backends at once - each will be used to encrypt a part of the key. You can set `TF_BACKEND_HTTP_SOPS_SHAMIR_THRESHOLD` if you want to use a specific threshold - by default, all keys used for encryption will be required for decryption.
+`sops` supports [Shamir's Secret Sharing](https://github.com/mozilla/sops#214key-groups). You can configure multiple backends at once - each will be used to encrypt a part of the key. You can set `TF_BACKEND_HTTP_SOPS_SHAMIR_THRESHOLD` / `encryption.sops.shamir_threshold` if you want to use a specific threshold - by default, all keys used for encryption will be required for decryption.
 
-##### PGP
+###### PGP
 
-Use `TF_BACKEND_HTTP_SOPS_PGP_FP` to provide a comma separated PGP key fingerprints. Keys must be added to a local `gpg` in order to encrypt. Private part of the key must be present in order for decrypt.
+Use `TF_BACKEND_HTTP_SOPS_PGP_FP` / `encryption.sops.gpg.key_ids`  to provide a comma separated PGP key fingerprints. Keys must be added to a local `gpg` in order to encrypt. Private part of the key must be present in order for decrypt.
 
-##### AWS KMS
+###### AWS KMS
 
-Use `TF_BACKEND_HTTP_SOPS_AWS_KMS_ARNS` to provide a comma separated list of KMS ARNs. AWS SDK will use standard [credentials provider chain](https://docs.aws.amazon.com/sdk-for-go/api/aws/credentials/) in order to automatically discover local credentials in standard `AWS_*` environment variables or `~/.aws`. You can optionally use `TF_BACKEND_HTTP_SOPS_AWS_PROFILE` to point it to a specific shared profile. You can also provide additional KMS encryption context using `TF_BACKEND_HTTP_SOPS_AWS_KMS_CONTEXT` - it is a comma separated list of `key=value` pairs.
+Use `TF_BACKEND_HTTP_SOPS_AWS_KMS_ARNS` / `encryption.sops.aws.key_arns` to provide a comma separated list of KMS ARNs. AWS SDK will use standard [credentials provider chain](https://docs.aws.amazon.com/sdk-for-go/api/aws/credentials/) in order to automatically discover local credentials in standard `AWS_*` environment variables or `~/.aws`. You can optionally use `TF_BACKEND_HTTP_SOPS_AWS_PROFILE` / `encryption.sops.aws.profile` to point it to a specific shared profile. You can also provide additional KMS encryption context using `TF_BACKEND_HTTP_SOPS_AWS_KMS_CONTEXT` / `encryption.sops.aws.kms_context` - it is a comma separated list of `key=value` pairs.
 
-##### GCP KMS
+###### GCP KMS
 
-Use `TF_BACKEND_HTTP_SOPS_GCP_KMS_KEYS` to provide a comma separated list of GCP KMS IDs. Read [Encrypting using GCP KMS](https://github.com/getsops/sops#encrypting-using-gcp-kms) for further details.
+Use `TF_BACKEND_HTTP_SOPS_GCP_KMS_KEYS` / `encryption.sops.gcp.key` to provide a comma separated list of GCP KMS IDs. Read [Encrypting using GCP KMS](https://github.com/getsops/sops#encrypting-using-gcp-kms) for further details.
 
-##### Hashicorp Vault
+###### Hashicorp Vault
 
-Use `TF_BACKEND_HTTP_SOPS_HC_VAULT_URIS` to point it to the Vault Transit keys. It is a comma separated list of URLs in a form of `${VAULT_ADDR}/v1/transit/keys/key`, where `transit` is a name of Vault Transit mount and `key` is the name of the key in that mount. Under the hood Vault SDK is using standard credentials resolver to automatically discover Vault credentials in the environment, meaning you can either use `vault login` or set `VAULT_TOKEN` environment variable.
+Use `TF_BACKEND_HTTP_SOPS_HC_VAULT_URIS` / `encryption.sops.hc_vault.uris` to point it to the Vault Transit keys. It is a comma separated list of URLs in a form of `${VAULT_ADDR}/v1/transit/keys/key`, where `transit` is a name of Vault Transit mount and `key` is the name of the key in that mount. Under the hood Vault SDK is using standard credentials resolver to automatically discover Vault credentials in the environment, meaning you can either use `vault login` or set `VAULT_TOKEN` environment variable.
 
-#### AES256
+##### AES256
 
-To enable state encryption, you can use `TF_BACKEND_HTTP_ENCRYPTION_PASSPHRASE` environment variable to set a passphrase. Backend will encrypt and decrypt (using AES256, server-side) all state files transparently before storing them in Git. If it fails to decrypt the file obtained from Git, it will assume encryption was not previously enabled and return it as-is. Note this doesn't encrypt the traffic at REST, as Terraform doesn't support any sort of encryption for HTTP backend. Traffic between Terraform and this backend stays unencrypted at all times.
+To enable state encryption, you can use `TF_BACKEND_HTTP_ENCRYPTION_PASSPHRASE` / `encryption.aes.passprase` environment variable / config file setting to set a passphrase. Backend will encrypt and decrypt (using AES256, server-side) all state files transparently before storing them in Git. If it fails to decrypt the file obtained from Git, it will assume encryption was not previously enabled and return it as-is. Note this doesn't encrypt the traffic at REST, as Terraform doesn't support any sort of encryption for HTTP backend. Traffic between Terraform and this backend stays unencrypted at all times.
 
 ### Running backend remotely
 
@@ -263,11 +275,11 @@ If you are absolutely sure you want to run this backend in remote standalone mod
 
 ### TLS
 
-You can set `TF_BACKEND_GIT_HTTPS_CERT` and `TF_BACKEND_GIT_HTTPS_KEY` pointing to your cert and a key files. This will make HTTP backend to start in TLS mode. If you are using self-signed certificate - you can also set `TF_BACKEND_GIT_HTTPS_SKIP_VERIFICATION=true` in a wrapper mode and that will enable `skip_cert_verification` in the terraform config (or configure it yourself for standalone mode).
+You can set `TF_BACKEND_GIT_HTTPS_CERT` / `server.https_cert` and `TF_BACKEND_GIT_HTTPS_KEY` / `server.https_key` pointing to your cert and a key files. This will make HTTP backend to start in TLS mode. If you are using self-signed certificate - you can also set `TF_BACKEND_GIT_HTTPS_SKIP_VERIFICATION=true` / `server.skip_https_verification` in a wrapper mode and that will enable `skip_cert_verification` in the terraform config (or configure it yourself for standalone mode).
 
 ### Basic HTTP Authentication
 
-You can use `TF_BACKEND_GIT_HTTP_USERNAME` and `TF_BACKEND_GIT_HTTP_PASSWORD` environment variables to add an extra layer of protection. In `wrapper` mode, same environment variables will be used to render `*.auto.tf` config for Terraform, but if you are using backend in standalone mode - you will have to tell these credentials to the Terraform explicitly:
+You can use `TF_BACKEND_GIT_HTTP_USERNAME` / `server.http_username` and `TF_BACKEND_GIT_HTTP_PASSWORD` / `server.http_password` environment variables / config file settings to add an extra layer of protection. In `wrapper` mode, same environment variables will be used to render `*.auto.tf` config for Terraform, but if you are using backend in standalone mode - you will have to tell these credentials to the Terraform explicitly:
 
 ```terraform
 terraform {

backend/crypt.go

@@ -2,7 +2,8 @@ package backend
 
 import (
 	"fmt"
-	"os"
+
+	"github.com/spf13/viper"
 
 	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
@@ -11,7 +12,8 @@ import (
 )
 
 func getEncryptionProvider() (crypt.EncryptionProvider, error) {
-	provider, enabled := os.LookupEnv("TF_BACKEND_HTTP_ENCRYPTION_PROVIDER")
+	provider := viper.GetString("encryption.provider")
+	enabled := (provider != "")
 	if enabled {
 		if !slices.Contains(maps.Keys(crypt.EncryptionProviders), provider) {
 			return nil, fmt.Errorf("Unknown encryption provider %q", provider)
@@ -20,7 +22,7 @@ func getEncryptionProvider() (crypt.EncryptionProvider, error) {
 	}
 
 	// For backward compatibility
-	_, aesEnabled := os.LookupEnv("TF_BACKEND_HTTP_ENCRYPTION_PASSPHRASE")
+	aesEnabled := viper.InConfig("aes.passprase")
 	if aesEnabled {
 		return crypt.EncryptionProviders["aes"], nil
 	}
@@ -47,3 +49,8 @@ func decryptIfEnabled(state []byte) ([]byte, error) {
 	}
 	return state, nil
 }
+
+func init() {
+	viper.BindEnv("encryption.provider", "TF_BACKEND_HTTP_ENCRYPTION_PROVIDER")
+	viper.BindEnv("encryption.aes.passprase", "TF_BACKEND_HTTP_ENCRYPTION_PASSPHRASE")
+}

cmd/git_backend.go

@@ -47,21 +47,20 @@ terraform {
 			log.Fatal(err)
 		}
 
-		_, okHttpCert := os.LookupEnv("TF_BACKEND_GIT_HTTPS_CERT")
-		_, okHttpKey := os.LookupEnv("TF_BACKEND_GIT_HTTPS_KEY")
+		httpCert := viper.GetString("server.https_cert")
+		httpKey := viper.GetString("server.https_key")
 		protocol := "http"
-		if okHttpCert && okHttpKey {
+		if httpCert != "" && httpKey != "" {
 			protocol = "https"
 		}
 
-		skipHttpsVerification, okSkipHttpsVerification := os.LookupEnv("TF_BACKEND_GIT_HTTPS_SKIP_VERIFICATION")
-		if !okSkipHttpsVerification {
+		username := viper.GetString("server.http_username")
+		password := viper.GetString("server.http_password")
+		skipHttpsVerification := viper.GetString("server.skip_https_verification")
+		if skipHttpsVerification == "" {
 			skipHttpsVerification = "false"
 		}
 
-		username, _ := os.LookupEnv("TF_BACKEND_GIT_HTTP_USERNAME")
-		password, _ := os.LookupEnv("TF_BACKEND_GIT_HTTP_PASSWORD")
-
 		addr := strings.Split(viper.GetString("address"), ":")
 		p := map[string]string{
 			"port":                  addr[len(addr)-1],
@@ -111,4 +110,10 @@ func init() {
 	viper.BindPFlag("git.dir", gitBackendCmd.PersistentFlags().Lookup("dir"))
 
 	discovery.RegisterBackend(gitBackendCmd)
+
+	viper.BindEnv("server.https_cert", "TF_BACKEND_GIT_HTTPS_CERT")
+	viper.BindEnv("server.https_key", "TF_BACKEND_GIT_HTTPS_KEY")
+	viper.BindEnv("server.skip_https_verification", "TF_BACKEND_GIT_HTTPS_SKIP_VERIFICATION")
+	viper.BindEnv("server.http_username", "TF_BACKEND_GIT_HTTP_USERNAME")
+	viper.BindEnv("server.http_password", "TF_BACKEND_GIT_HTTP_PASSWORD")
 }

cmd/root.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 
 	"github.com/mitchellh/go-homedir"
@@ -16,7 +17,7 @@ import (
 	"github.com/plumber-cd/terraform-backend-git/server"
 )
 
-var cfgFile string
+var cfgFiles []string
 
 // rootCmd main command that just starts the server and keeps listening on port until terminated
 var rootCmd = &cobra.Command{
@@ -60,7 +61,7 @@ func init() {
 
 	cobra.OnInitialize(initConfig)
 
-	rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file (default is terraform-backend-git.hcl)")
+	rootCmd.PersistentFlags().StringArrayVarP(&cfgFiles, "config", "c", []string{}, "config file, can be multiple. (default is [terraform-backend-git.hcl, terraform-backend-git.secret.hcl])")
 
 	rootCmd.PersistentFlags().StringP("address", "a", "127.0.0.1:6061", "Specify the listen address")
 	viper.BindPFlag("address", rootCmd.PersistentFlags().Lookup("address"))
@@ -74,22 +75,27 @@ func init() {
 
 func initConfig() {
 	viper.SetConfigType("hcl")
-	viper.SetConfigName("terraform-backend-git")
 
-	if cfgFile != "" {
-		viper.SetConfigFile(cfgFile)
+	if len(cfgFiles) > 0 {
+		for i := 0; i < len(cfgFiles); i++ {
+			addViperConfigPath(cfgFiles[i])
+		}
 	} else {
 		home, err := homedir.Dir()
 		if err != nil {
 			log.Fatal(err)
 		}
-		viper.AddConfigPath(home)
+
+		addViperConfigPath(filepath.Join(home, "terraform-backend-git.hcl"))
+		addViperConfigPath(filepath.Join(home, "terraform-backend-git.secret.hcl"))
 
 		cwd, err := os.Getwd()
 		if err != nil {
 			log.Fatal(err)
 		}
-		viper.AddConfigPath(cwd)
+
+		addViperConfigPath(filepath.Join(cwd, "terraform-backend-git.hcl"))
+		addViperConfigPath(filepath.Join(cwd, "terraform-backend-git.secret.hcl"))
 	}
 
 	viper.AutomaticEnv()
@@ -100,3 +106,23 @@ func initConfig() {
 		log.Println("Using config file:", viper.ConfigFileUsed())
 	}
 }
+
+func addViperConfigPath(path string) error {
+	_, err := os.Stat(path)
+	if err != nil {
+
+		return err
+	}
+
+	// fmt.Println("Adding config", path)
+
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	viper.MergeConfig(f)
+
+	return nil
+}

crypt/aes.go

@@ -6,7 +6,8 @@ import (
 	"crypto/rand"
 	"errors"
 	"io"
-	"os"
+
+	"github.com/spf13/viper"
 )
 
 func init() {
@@ -21,8 +22,8 @@ type AESEncryptionProvider struct{}
 
 // getEncryptionPassphrase should check all possible config sources and return a state backend encryption key.
 func getEncryptionPassphrase() (string, error) {
-	passphrase, ok := os.LookupEnv("TF_BACKEND_HTTP_ENCRYPTION_PASSPHRASE")
-	if !ok {
+	passphrase := viper.GetString("encryption.aes.passprase")
+	if passphrase == "" {
 		return "", ErrEncryptionPassphraseNotSet
 	}
 	return passphrase, nil

crypt/sops.go

@@ -3,7 +3,6 @@ package crypt
 import (
 	"fmt"
 	"log"
-	"os"
 	"strconv"
 
 	sops "go.mozilla.org/sops/v3"
@@ -14,6 +13,7 @@ import (
 	"go.mozilla.org/sops/v3/version"
 
 	sc "github.com/plumber-cd/terraform-backend-git/crypt/sops"
+	"github.com/spf13/viper"
 )
 
 func init() {
@@ -43,7 +43,7 @@ func (p *SOPSEncryptionProvider) Encrypt(data []byte) ([]byte, error) {
 		},
 	}
 
-	if shamirThreshold, ok := os.LookupEnv("TF_BACKEND_HTTP_SOPS_SHAMIR_THRESHOLD"); ok {
+	if shamirThreshold := viper.GetString("encryption.sops.shamir_threshold"); shamirThreshold != "" {
 		st, err := strconv.Atoi(shamirThreshold)
 		if err != nil {
 			return nil, err
@@ -93,3 +93,7 @@ func (p *SOPSEncryptionProvider) Decrypt(data []byte) ([]byte, error) {
 	outputStore := &sopsjson.Store{}
 	return outputStore.EmitPlainFile(tree.Branches)
 }
+
+func init() {
+	viper.BindEnv("sops.shamir_threshold", "TF_BACKEND_HTTP_SOPS_SHAMIR_THRESHOLD")
+}