Add Consul & BoltDB datasource support

data.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -14,6 +15,7 @@ import (
 	"time"
 
 	"github.com/blang/vfs"
+	"github.com/hairyhenderson/gomplate/libkv"
 	"github.com/hairyhenderson/gomplate/vault"
 )
 
@@ -42,6 +44,10 @@ func init() {
 	addSourceReader("https", readHTTP)
 	addSourceReader("file", readFile)
 	addSourceReader("vault", readVault)
+	addSourceReader("consul", readLibKV)
+	addSourceReader("consul+http", readLibKV)
+	addSourceReader("consul+https", readLibKV)
+	addSourceReader("boltdb", readLibKV)
 }
 
 var sourceReaders map[string]func(*Source, ...string) ([]byte, error)
@@ -85,6 +91,7 @@ type Source struct {
 	FS     vfs.Filesystem // used for file: URLs, nil otherwise
 	HC     *http.Client   // used for http[s]: URLs, nil otherwise
 	VC     *vault.Client  //used for vault: URLs, nil otherwise
+	KV     *libkv.LibKV   // used for consul:, etcd:, zookeeper: & boltdb: URLs, nil otherwise
 	Header http.Header    // used for http[s]: URLs, nil otherwise
 }
 
@@ -98,7 +105,7 @@ func NewSource(alias string, URL *url.URL) (s *Source) {
 		Ext:   ext,
 	}
 
-	if ext != "" {
+	if ext != "" && URL.Scheme != "boltdb" {
 		mediatype := mime.TypeByExtension(ext)
 		t, params, err := mime.ParseMediaType(mediatype)
 		if err != nil {
@@ -194,6 +201,9 @@ func (d *Data) Datasource(alias string, args ...string) interface{} {
 	if source.Type == "application/toml" {
 		return ty.TOML(s)
 	}
+	if source.Type == "text/plain" {
+		return s
+	}
 	log.Fatalf("Datasources of type %s not yet supported", source.Type)
 	return nil
 }
@@ -326,6 +336,36 @@ func readVault(source *Source, args ...string) ([]byte, error) {
 	return data, nil
 }
 
+func readLibKV(source *Source, args ...string) ([]byte, error) {
+	if source.KV == nil {
+		source.KV = libkv.New(source.URL)
+		err := source.KV.Login()
+		addCleanupHook(source.KV.Logout)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	p := source.URL.Path
+
+	if source.URL.Scheme == "boltdb" {
+		if len(args) != 1 {
+			return nil, errors.New("missing key")
+		}
+		p = args[0]
+	} else if len(args) == 1 {
+		p = p + "/" + args[0]
+	}
+
+	data, err := source.KV.Read(p)
+	if err != nil {
+		return nil, err
+	}
+	source.Type = "text/plain"
+
+	return data, nil
+}
+
 func parseHeaderArgs(headerArgs []string) map[string]http.Header {
 	headers := make(map[string]http.Header)
 	for _, v := range headerArgs {

docs/content/functions/general.md

@@ -511,6 +511,51 @@ $ gomplate -d foo=https://httpbin.org/get -H 'foo=Foo: bar' -i '{{(datasource "f
 bar
 ```
 
+##### Usage with Consul data
+
+There are three URL schemes which can be used to retrieve data from [Hashicorp Consul](https://consul.io/).
+The `consul://` (or `consul+http://`) scheme can optionally be used with a hostname and port to specify a server (e.g. `consul://localhost:8500`).
+By default this will be contacted by HTTP, but the `$CONSUL_HTTP_SSL` can be used to switch to HTTPS mode. Alternatively
+the `consul+https://` scheme can be used.
+
+If the server address isn't included the variable `$CONSUL_HTTP_ADDR` will be checked, otherwise `localhost:8500` will be used.
+
+The following environment variables can be used:
+
+| name | usage |
+| -- | -- |
+| `CONSUL_HTTP_ADDR` | Hostname and optional port for connecting to Consul. Defaults to localhost and port 8500. |
+| `CONSUL_TIMEOUT` | Timeout (in seconds) when communicating to Consul. Defaults to 10 seconds. |
+| `CONSUL_HTTP_TOKEN` | The Consul token to use when connecting to the server. |
+| `CONSUL_HTTP_AUTH` | Should be specified as <username>:<password>. Used to authenticate to the server. |
+| `CONSUL_HTTP_SSL` | Switch to HTTPS mode if set to a true value. It accepts 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False. Alternatively use the `consul+https://` scheme. |
+| `CONSUL_TLS_SERVER_NAME` | The server name to use as the SNI host when connecting to Consul via TLS. |
+| `CONSUL_CACERT` | If specified points to a CA file for verifying Consul server using TLS. |
+| `CONSUL_CAPATH` | If specified points to a directory of CA files for verifying Consul server using TLS. |
+| `CONSUL_CLIENT_CERT` | Client certificate file for certificate authentication. Both a certificate and key are required. |
+| `CONSUL_CLIENT_KEY` | Client key file for certificate authentication. Both a certificate and key are required. |
+| `CONSUL_HTTP_SSL_VERIFY` | Disable Consul TLS certificate checking. It accepts 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False. |
+
+If a path is included it is used as a prefix for all uses of the datasource.
+
+##### Usage with BoldDB data
+
+[BoldDB](https://github.com/boltdb/bolt) is a simple local key/value store used by many Go tools.
+
+It can be accessed using the `boltdb://` scheme in addition to the full path to the database file
+and the bucket name specified using the #fragment identifier (e.g. `boltdb:////tmp/database.db#bucket).
+
+As access is vi [libkv](https://github.com/docker/libkv) the first 8 bytes of all values is used as an
+incrementing last modified index value. Therefore all values must be at least 9 bytes long, with the first
+8 being ignored.
+
+The following environment variables can be used:
+
+| name | usage |
+| -- | -- |
+| `BOLTDB_TIMEOUT` | Timeout (in seconds) to wait for a lock on the database file when opening. |
+| `BOLTDB_PERSIST` | If set keep the database open instead of closing after each read. It accepts 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False. |
+
 ##### Usage with Vault data
 
 The special `vault://` URL scheme can be used to retrieve data from [Hashicorp

docs/content/index.md

@@ -9,8 +9,8 @@ menu:
 
 A [Go template](https://golang.org/pkg/text/template/)-based CLI tool. `gomplate` can be used as an alternative to
 [`envsubst`](https://www.gnu.org/software/gettext/manual/html_node/envsubst-Invocation.html) but also supports
-additional template datasources such as: JSON, YAML, AWS EC2 metadata, and
-[Hashicorp Vault](https://www.vaultproject.io/) secrets.
+additional template datasources such as: JSON, YAML, AWS EC2 metadata, [BoldDB](https://github.com/boltdb/bolt),
+[Hashicorp Consul](https://www.consul.io/) and [Hashicorp Vault](https://www.vaultproject.io/) secrets.
 
 I really like `envsubst` for use as a super-minimalist template processor. But its simplicity is also its biggest flaw: it's all-or-nothing with shell-like variables.
 

glide.yaml

@@ -18,6 +18,7 @@ import:
   - codec
 - package: github.com/hairyhenderson/toml
   version: support-map-interface-keys
+- package: github.com/docker/libkv
 testImport:
 - package: github.com/stretchr/testify
   version: ^1.1.4

libkv/libkv.go

@@ -0,0 +1,185 @@
+package libkv
+
+import (
+	"crypto/tls"
+	"errors"
+	"log"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/blang/vfs"
+	"github.com/docker/libkv"
+	"github.com/docker/libkv/store"
+	"github.com/docker/libkv/store/boltdb"
+	"github.com/docker/libkv/store/consul"
+	"github.com/hairyhenderson/gomplate/env"
+	consulapi "github.com/hashicorp/consul/api"
+)
+
+// logFatal is defined so log.Fatal calls can be overridden for testing
+var logFatal = log.Fatal
+
+// LibKV -
+type LibKV struct {
+	store store.Store
+	fs    vfs.Filesystem
+}
+
+type SetupDetails struct {
+	sourceType store.Backend
+	client     string
+	options    *store.Config
+}
+
+// New - instantiate a new
+func New(url *url.URL) *LibKV {
+	var setup *SetupDetails
+	var err error
+
+	if url.Scheme == "consul" || url.Scheme == "consul+http" {
+		setup, err = setupConsul(url, false)
+		if err != nil {
+			logFatal("consul setup error", err)
+		}
+	}
+	if url.Scheme == "consul+https" {
+		setup, err = setupConsul(url, true)
+		if err != nil {
+			logFatal("consul setup error", err)
+		}
+	}
+	if url.Scheme == "boltdb" {
+		setup, err = setupBoltDB(url, false)
+		if err != nil {
+			logFatal("boltdb setup error", err)
+		}
+	}
+
+	if setup.client == "" {
+		logFatal("missing client location")
+	}
+
+	kv, err := libkv.NewStore(
+		setup.sourceType,
+		[]string{setup.client},
+		setup.options,
+	)
+	if err != nil {
+		logFatal("Cannot create store", err)
+	}
+
+	return &LibKV{kv, nil}
+}
+
+func setupConsul(url *url.URL, enableTLS bool) (*SetupDetails, error) {
+	setup := &SetupDetails{}
+	consul.Register()
+	setup.sourceType = store.CONSUL
+	setup.client = env.Getenv("CONSUL_HTTP_ADDR", "localhost:8500")
+	setup.options = &store.Config{}
+	if timeout := env.Getenv("CONSUL_TIMEOUT", ""); timeout != "" {
+		num, err := strconv.ParseInt(timeout, 10, 16)
+		if err != nil {
+			return nil, err
+		}
+		setup.options.ConnectionTimeout = time.Duration(num) * time.Second
+	}
+	if ssl := env.Getenv("CONSUL_HTTP_SSL", ""); ssl != "" {
+		enabled, err := strconv.ParseBool(ssl)
+		if err != nil {
+			return nil, err
+		}
+		enableTLS = enabled
+	}
+	if enableTLS {
+		config, err := setupTLS("CONSUL")
+		if err != nil {
+			return nil, err
+		}
+		setup.options.TLS = config
+	}
+	return setup, nil
+}
+
+func setupBoltDB(url *url.URL, enableTLS bool) (*SetupDetails, error) {
+	setup := &SetupDetails{}
+	boltdb.Register()
+	setup.sourceType = store.BOLTDB
+	setup.client = url.Path
+	setup.options = &store.Config{}
+	setup.options.Bucket = url.Fragment
+	if setup.options.Bucket == "" {
+		return nil, errors.New("missing bucket")
+	}
+	if timeout := env.Getenv("BOLTDB_TIMEOUT", ""); timeout != "" {
+		num, err := strconv.ParseInt(timeout, 10, 16)
+		if err != nil {
+			return nil, err
+		}
+		setup.options.ConnectionTimeout = time.Duration(num) * time.Second
+	}
+	if persist := env.Getenv("BOLTDB_PERSIST", ""); persist != "" {
+		enabled, err := strconv.ParseBool(persist)
+		if err != nil {
+			return nil, err
+		}
+		setup.options.PersistConnection = enabled
+	}
+	return setup, nil
+}
+
+func setupTLS(prefix string) (*tls.Config, error) {
+	tlsConfig := &consulapi.TLSConfig{}
+
+	if v := env.Getenv(prefix+"_TLS_SERVER_NAME", ""); v != "" {
+		tlsConfig.Address = v
+	}
+	if v := env.Getenv(prefix+"_CACERT", ""); v != "" {
+		tlsConfig.CAFile = v
+	}
+	if v := env.Getenv(prefix+"_CAPATH", ""); v != "" {
+		tlsConfig.CAPath = v
+	}
+	if v := env.Getenv(prefix+"_CLIENT_CERT", ""); v != "" {
+		tlsConfig.CertFile = v
+	}
+	if v := env.Getenv(prefix+"_CLIENT_KEY", ""); v != "" {
+		tlsConfig.KeyFile = v
+	}
+	if v := env.Getenv(prefix+"_HTTP_SSL_VERIFY", ""); v != "" {
+		verify, err := strconv.ParseBool(v)
+		if err != nil {
+			return nil, err
+		}
+		if !verify {
+			tlsConfig.InsecureSkipVerify = true
+		}
+	}
+
+	config, err := consulapi.SetupTLSConfig(tlsConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return config, nil
+}
+
+// Login -
+func (kv *LibKV) Login() error {
+	return nil
+}
+
+// Logout -
+func (kv *LibKV) Logout() {
+}
+
+// Read -
+func (kv *LibKV) Read(path string) ([]byte, error) {
+	data, err := kv.store.Get(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return data.Value, nil
+}

test/integration/Dockerfile

@@ -1,6 +1,7 @@
 FROM alpine:edge
 
 ENV VAULT_VER 0.7.0
+ENV CONSUL_VER 0.9.0
 RUN apk add --no-cache \
     curl \
     bash \
@@ -10,6 +11,10 @@ RUN apk add --no-cache \
   && unzip /tmp/vault.zip \
   && mv vault /bin/vault \
   && rm /tmp/vault.zip \
+  && curl -L -o /tmp/consul.zip https://releases.hashicorp.com/consul/${CONSUL_VER}/consul_${CONSUL_VER}_linux_amd64.zip \
+  && unzip /tmp/consul.zip \
+  && mv consul /bin/consul \
+  && rm /tmp/consul.zip \
   && apk del curl
 
 RUN mkdir /lib64 \
@@ -20,5 +25,6 @@ COPY mirror /bin/mirror
 COPY *.sh /tests/
 COPY *.bash /tests/
 COPY *.bats /tests/
+COPY *.db /test/integration/
 
 CMD ["/tests/test.sh"]