Change configuration package to influxdata/config

We are unifying the way that we handle configuration across the products
into the influxdata/config package. This provides the same API as
BurntSushi/toml package used previously, but uses influxdata/toml under
the hood (which is a fork of naoina/toml). The underlying toml parser
has been changed because Telegraf uses specific features of this parser
that cannot be easily replicated with the BurntSushi parser.
Furthermore, our fork of naoina/toml provides support for maps and
pointers within structs and toml documentation facilities[1].

[1] This is accessible by adding a "doc" struct tag with a comment
describing a particular field of a config struct. When marshalling that
struct as TOML, the "doc" struct tag will be placed appropriately to
document that field in the resultant TOML output.

Godeps

@@ -11,9 +11,12 @@ github.com/golang/snappy 5979233c5d6225d4a8e438cdd0b411888449ddab
 github.com/hashicorp/go-msgpack fa3f63826f7c23912c15263591e65d54d080b458
 github.com/hashicorp/raft 8fd9a2fdfd154f4b393aa24cff91e3c317efe839
 github.com/hashicorp/raft-boltdb d1e82c1ec3f15ee991f7cc7ffd5b67ff6f5bbaee
+github.com/influxdata/config 45e0b9a7d1805982fca48b9c753c3b2541aea2bd
+github.com/influxdata/toml 51291e16df98ff89f93f3b949ff023ca1ad599b4
 github.com/influxdata/usage-client 475977e68d79883d9c8d67131c84e4241523f452
 github.com/jwilder/encoding 07d88d4f35eec497617bee0c7bfe651a796dae13
 github.com/kimor79/gollectd 61d0deeb4ffcc167b2a1baa8efd72365692811bc
+github.com/naoina/go-stringutil 6b638e95a32d0c1131db0e7fe83775cbea4a0d0b
 github.com/paulbellamy/ratecounter 5a11f585a31379765c190c033b6ad39956584447
 github.com/peterh/liner ad1edfd30321d8f006ccf05f1e0524adeb943060
 github.com/rakyll/statik 274df120e9065bdd08eb1120e0375e3dc1ae8465

cluster/config.go

@@ -3,7 +3,7 @@ package cluster
 import (
 	"time"
 
-	"github.com/influxdata/influxdb/toml"
+	"github.com/influxdata/config"
 )
 
 const (
@@ -23,19 +23,19 @@ const (
 
 // Config represents the configuration for the clustering service.
 type Config struct {
-	ForceRemoteShardMapping   bool          `toml:"force-remote-mapping"`
-	WriteTimeout              toml.Duration `toml:"write-timeout"`
-	ShardWriterTimeout        toml.Duration `toml:"shard-writer-timeout"`
-	MaxRemoteWriteConnections int           `toml:"max-remote-write-connections"`
-	ShardMapperTimeout        toml.Duration `toml:"shard-mapper-timeout"`
+	ForceRemoteShardMapping   bool            `toml:"force-remote-mapping"`
+	WriteTimeout              config.Duration `toml:"write-timeout"`
+	ShardWriterTimeout        config.Duration `toml:"shard-writer-timeout"`
+	MaxRemoteWriteConnections int             `toml:"max-remote-write-connections"`
+	ShardMapperTimeout        config.Duration `toml:"shard-mapper-timeout"`
 }
 
 // NewConfig returns an instance of Config with defaults.
 func NewConfig() Config {
 	return Config{
-		WriteTimeout:              toml.Duration(DefaultWriteTimeout),
-		ShardWriterTimeout:        toml.Duration(DefaultShardWriterTimeout),
-		ShardMapperTimeout:        toml.Duration(DefaultShardMapperTimeout),
+		WriteTimeout:              config.Duration(DefaultWriteTimeout),
+		ShardWriterTimeout:        config.Duration(DefaultShardWriterTimeout),
+		ShardMapperTimeout:        config.Duration(DefaultShardMapperTimeout),
 		MaxRemoteWriteConnections: DefaultMaxRemoteWriteConnections,
 	}
 }

cluster/config_test.go

@@ -4,14 +4,14 @@ import (
 	"testing"
 	"time"
 
-	"github.com/BurntSushi/toml"
+	"github.com/influxdata/config"
 	"github.com/influxdata/influxdb/cluster"
 )
 
 func TestConfig_Parse(t *testing.T) {
 	// Parse configuration.
 	var c cluster.Config
-	if _, err := toml.Decode(`
+	if err := config.Decode(`
 shard-writer-timeout = "10s"
 write-timeout = "20s"
 `, &c); err != nil {

cluster/shard_writer_test.go

@@ -6,9 +6,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/influxdata/config"
 	"github.com/influxdata/influxdb/cluster"
 	"github.com/influxdata/influxdb/models"
-	"github.com/influxdata/influxdb/toml"
 )
 
 // Ensure the shard writer can successfully write a single request.
@@ -195,7 +195,7 @@ func TestShardWriter_Write_ErrReadTimeout(t *testing.T) {
 func TestShardWriter_Write_PoolMax(t *testing.T) {
 	ts := newTestWriteService(writeShardSlow)
 	s := cluster.NewService(cluster.Config{
-		ShardWriterTimeout: toml.Duration(100 * time.Millisecond),
+		ShardWriterTimeout: config.Duration(100 * time.Millisecond),
 	})
 	s.Listener = ts.muxln
 	s.TSDBStore = &ts.TSDBStore

cmd/influxd/run/command.go

@@ -13,7 +13,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/BurntSushi/toml"
+	cfg "github.com/influxdata/config"
 )
 
 const logo = `
@@ -212,9 +212,11 @@ func (cmd *Command) ParseConfig(path string) (*Config, error) {
 	}
 
 	log.Printf("Using configuration at: %s\n", path)
+	l := log.New(cmd.Stderr, "[config] ", log.LstdFlags)
 
 	config := NewConfig()
-	if _, err := toml.DecodeFile(path, &config); err != nil {
+	cfg.SetLogger(l)
+	if err := cfg.DecodeFile(path, config); err != nil {
 		return nil, err
 	}
 

cmd/influxd/run/config.go

@@ -39,6 +39,18 @@ const (
 
 // Config represents the configuration format for the influxd binary.
 type Config struct {
+	// Server reporting
+	ReportingDisabled bool `toml:"reporting-disabled"`
+
+	// BindAddress is the address that all TCP services use (Raft, Snapshot, Cluster, etc.)
+	BindAddress string `toml:"bind-address"`
+
+	// Hostname is the hostname portion to use when registering local
+	// addresses.  This hostname must be resolvable from other nodes.
+	Hostname string `toml:"hostname"`
+
+	Join string `toml:"join"`
+
 	Meta       *meta.Config      `toml:"meta"`
 	Data       tsdb.Config       `toml:"data"`
 	Cluster    cluster.Config    `toml:"cluster"`
@@ -49,25 +61,13 @@ type Config struct {
 	Monitor    monitor.Config    `toml:"monitor"`
 	Subscriber subscriber.Config `toml:"subscriber"`
 	HTTPD      httpd.Config      `toml:"http"`
-	Graphites  []graphite.Config `toml:"graphite"`
+	Graphites  []graphite.Config `toml:"graphite" doc:"List of graphite listen endpoints"`
 	Collectd   collectd.Config   `toml:"collectd"`
 	OpenTSDB   opentsdb.Config   `toml:"opentsdb"`
 	UDPs       []udp.Config      `toml:"udp"`
 
 	ContinuousQuery continuous_querier.Config `toml:"continuous_queries"`
 	HintedHandoff   hh.Config                 `toml:"hinted-handoff"`
-
-	// Server reporting
-	ReportingDisabled bool `toml:"reporting-disabled"`
-
-	// BindAddress is the address that all TCP services use (Raft, Snapshot, Cluster, etc.)
-	BindAddress string `toml:"bind-address"`
-
-	// Hostname is the hostname portion to use when registering local
-	// addresses.  This hostname must be resolvable from other nodes.
-	Hostname string `toml:"hostname"`
-
-	Join string `toml:"join"`
 }
 
 // NewConfig returns an instance of Config with reasonable defaults.
@@ -90,30 +90,14 @@ func NewConfig() *Config {
 	c.HintedHandoff = hh.NewConfig()
 	c.BindAddress = DefaultBindAddress
 
-	// All ARRAY attributes have to be init after toml decode
-	// See: https://github.com/BurntSushi/toml/pull/68
-	// Those attributes will be initialized in Config.InitTableAttrs method
-	// Concerned Attributes:
-	//  * `c.Graphites`
-	//  * `c.UDPs`
-
+	c.UDPs = []udp.Config{udp.NewConfig()}
+	c.Graphites = []graphite.Config{graphite.NewConfig()}
 	return c
 }
 
-// InitTableAttrs initialises all ARRAY attributes if empty
-func (c *Config) InitTableAttrs() {
-	if len(c.UDPs) == 0 {
-		c.UDPs = []udp.Config{udp.NewConfig()}
-	}
-	if len(c.Graphites) == 0 {
-		c.Graphites = []graphite.Config{graphite.NewConfig()}
-	}
-}
-
 // NewDemoConfig returns the config that runs when no config is specified.
 func NewDemoConfig() (*Config, error) {
 	c := NewConfig()
-	c.InitTableAttrs()
 
 	var homeDir string
 	// By default, store meta and data files in current users home directory

cmd/influxd/run/config_command.go

@@ -7,6 +7,7 @@ import (
 	"os"
 
 	"github.com/BurntSushi/toml"
+	cfg "github.com/influxdata/config"
 )
 
 // PrintConfigCommand represents the command executed by "influxd config".
@@ -51,7 +52,7 @@ func (cmd *PrintConfigCommand) Run(args ...string) error {
 		return fmt.Errorf("%s. To generate a valid configuration file run `influxd config > influxdb.generated.conf`", err)
 	}
 
-	toml.NewEncoder(cmd.Stdout).Encode(config)
+	cfg.NewEncoder(cmd.Stdout).Encode(config)
 	fmt.Fprint(cmd.Stdout, "\n")
 
 	return nil
@@ -68,7 +69,6 @@ func (cmd *PrintConfigCommand) parseConfig(path string) (*Config, error) {
 	if _, err := toml.DecodeFile(path, &config); err != nil {
 		return nil, err
 	}
-	config.InitTableAttrs()
 	return config, nil
 }
 

cmd/influxd/run/server_helpers_test.go

@@ -18,11 +18,11 @@ import (
 	"testing"
 	"time"
 
+	toml "github.com/influxdata/config"
 	"github.com/influxdata/influxdb/client/v2"
 	"github.com/influxdata/influxdb/cmd/influxd/run"
 	"github.com/influxdata/influxdb/services/httpd"
 	"github.com/influxdata/influxdb/services/meta"
-	"github.com/influxdata/influxdb/toml"
 )
 
 const emptyResults = `{"results":[{}]}`

monitor/config.go

@@ -3,7 +3,7 @@ package monitor
 import (
 	"time"
 
-	"github.com/influxdata/influxdb/toml"
+	"github.com/influxdata/config"
 )
 
 const (
@@ -20,16 +20,16 @@ const (
 
 // Config represents the configuration for the monitor service.
 type Config struct {
-	StoreEnabled  bool          `toml:"store-enabled"`
-	StoreDatabase string        `toml:"store-database"`
-	StoreInterval toml.Duration `toml:"store-interval"`
+	StoreEnabled  bool            `toml:"store-enabled"`
+	StoreDatabase string          `toml:"store-database"`
+	StoreInterval config.Duration `toml:"store-interval"`
 }
 
 // NewConfig returns an instance of Config with defaults.
 func NewConfig() Config {
 	return Config{
 		StoreEnabled:  true,
 		StoreDatabase: DefaultStoreDatabase,
-		StoreInterval: toml.Duration(DefaultStoreInterval),
+		StoreInterval: config.Duration(DefaultStoreInterval),
 	}
 }

monitor/config_test.go

@@ -4,14 +4,14 @@ import (
 	"testing"
 	"time"
 
-	"github.com/BurntSushi/toml"
+	"github.com/influxdata/config"
 	"github.com/influxdata/influxdb/monitor"
 )
 
 func TestConfig_Parse(t *testing.T) {
 	// Parse configuration.
 	var c monitor.Config
-	if _, err := toml.Decode(`
+	if err := config.Decode(`
 store-enabled=true
 store-database="the_db"
 store-interval="10m"

services/admin/config_test.go

@@ -3,14 +3,14 @@ package admin_test
 import (
 	"testing"
 
-	"github.com/BurntSushi/toml"
+	"github.com/influxdata/config"
 	"github.com/influxdata/influxdb/services/admin"
 )
 
 func TestConfig_Parse(t *testing.T) {
 	// Parse configuration.
 	var c admin.Config
-	if _, err := toml.Decode(`
+	if err := config.Decode(`
 enabled = true
 bind-address = ":8083"
 https-enabled = true

services/collectd/config.go

@@ -3,7 +3,7 @@ package collectd
 import (
 	"time"
 
-	"github.com/influxdata/influxdb/toml"
+	"github.com/influxdata/config"
 )
 
 const (
@@ -23,7 +23,7 @@ const (
 	DefaultBatchPending = 10
 
 	// DefaultBatchDuration is the default batch timeout duration.
-	DefaultBatchDuration = toml.Duration(10 * time.Second)
+	DefaultBatchDuration = config.Duration(10 * time.Second)
 
 	// DefaultTypesDB is the default location of the collectd types db file.
 	DefaultTypesDB = "/usr/share/collectd/types.db"
@@ -44,15 +44,15 @@ const (
 
 // Config represents a configuration for the collectd service.
 type Config struct {
-	Enabled         bool          `toml:"enabled"`
-	BindAddress     string        `toml:"bind-address"`
-	Database        string        `toml:"database"`
-	RetentionPolicy string        `toml:"retention-policy"`
-	BatchSize       int           `toml:"batch-size"`
-	BatchPending    int           `toml:"batch-pending"`
-	BatchDuration   toml.Duration `toml:"batch-timeout"`
-	ReadBuffer      int           `toml:"read-buffer"`
-	TypesDB         string        `toml:"typesdb"`
+	Enabled         bool            `toml:"enabled"`
+	BindAddress     string          `toml:"bind-address"`
+	Database        string          `toml:"database"`
+	RetentionPolicy string          `toml:"retention-policy"`
+	BatchSize       int             `toml:"batch-size"`
+	BatchPending    int             `toml:"batch-pending"`
+	BatchDuration   config.Duration `toml:"batch-timeout"`
+	ReadBuffer      int             `toml:"read-buffer"`
+	TypesDB         string          `toml:"typesdb"`
 }
 
 // NewConfig returns a new instance of Config with defaults.

services/collectd/config_test.go

@@ -3,14 +3,14 @@ package collectd_test
 import (
 	"testing"
 
-	"github.com/BurntSushi/toml"
+	"github.com/influxdata/config"
 	"github.com/influxdata/influxdb/services/collectd"
 )
 
 func TestConfig_Parse(t *testing.T) {
 	// Parse configuration.
 	var c collectd.Config
-	if _, err := toml.Decode(`
+	if err := config.Decode(`
 enabled = true
 bind-address = ":9000"
 database = "xxx"

services/collectd/service_test.go

@@ -9,10 +9,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/influxdata/config"
 	"github.com/influxdata/influxdb/cluster"
 	"github.com/influxdata/influxdb/models"
 	"github.com/influxdata/influxdb/services/meta"
-	"github.com/influxdata/influxdb/toml"
 )
 
 // Test that the service checks / creates the target database on startup.
@@ -190,7 +190,7 @@ func newTestService(batchSize int, batchDuration time.Duration) *testService {
 			BindAddress:   "127.0.0.1:0",
 			Database:      "collectd_test",
 			BatchSize:     batchSize,
-			BatchDuration: toml.Duration(batchDuration),
+			BatchDuration: config.Duration(batchDuration),
 		}),
 	}
 	s.Service.PointsWriter = &s.PointsWriter

services/continuous_querier/config.go

@@ -3,7 +3,7 @@ package continuous_querier
 import (
 	"time"
 
-	"github.com/influxdata/influxdb/toml"
+	"github.com/influxdata/config"
 )
 
 // Default values for aspects of interval computation.
@@ -23,14 +23,14 @@ type Config struct {
 	// of the interval for running continuous queries. If you only aggregate continuous queries
 	// every minute, this should be set to 1 minute. The default is set to '1s' so the interval
 	// is compatible with most aggregations.
-	RunInterval toml.Duration `toml:"run-interval"`
+	RunInterval config.Duration `toml:"run-interval"`
 }
 
 // NewConfig returns a new instance of Config with defaults.
 func NewConfig() Config {
 	return Config{
 		LogEnabled:  true,
 		Enabled:     true,
-		RunInterval: toml.Duration(DefaultRunInterval),
+		RunInterval: config.Duration(DefaultRunInterval),
 	}
 }