Config: use map for services connection settings. Using maps we could…

… guarantee services uniqueness from config file level.

doc/usage-en.md

@@ -5,6 +5,7 @@ Index of content:
 - [Requirements](#requirements)
 - [Quick start](#quick-start)
 - [YAML configuration](#yaml-configuration-settings)
+- [YAML configuration example](#yaml-configuration-file-example)
 - [Bootstrap and Uninstall modes](#bootstrap-and-uninstall-modes)
 - [Security considerations](#security-considerations)
 - [Troubleshooting](#troubleshooting)
@@ -65,8 +66,8 @@ used.
 - **api_key**: API key for accessing to Weaponry service. Default value: "". *Needed only for Weaponry clients.*
 
 
-- **services**: list of services to which pgSCV should connect and monitor. Defining `services` automatically disables 
-auto-discovery. Empty by default, looking for services using auto-discovery.
+- **services**: dictionary of services to which pgSCV should connect and monitor. Defining `services` automatically disables 
+auto-discovery. Empty by default, looking for services using auto-discovery. See [example](#yaml-configuration-file-example) for details. 
   - **service_type**: type of the service, must be one of `postgres`, `pgbouncer`.
   - **conninfo**: connection string or DSN for connecting to service.
 
@@ -94,17 +95,20 @@ auto-discovery. Empty by default, looking for services using auto-discovery.
 
 - **disable_collectors**: list of [collectors](./collectors.md) which should be disabled. Default value: [] (all collectors are enabled).
 
-YAML configuration file example:
+### YAML Configuration file example
+Complete YAML configuration file example:
 ```
 listen_address: 127.0.0.1:9890
 autoupdate: off
 no_track_mode: false
 send_metrics_url: https://push.weaponry.io
 api_key: 12345678-abcd-1234-abcd-123456789012
 services:
-  - service_type: "postgres"
+  "postgres:5432":
+    service_type: "postgres"
     conninfo: "postgres://[email protected]:5432/postgres"
-  - service_type: "pgbouncer"
+  "pgbouncer:6432": 
+    service_type: "pgbouncer"
     conninfo: "postgres://[email protected]:6432/pgbouncer"
 defaults:
     postgres_username: "monitoring"

internal/pgscv/config.go

@@ -24,18 +24,18 @@ const (
 
 // Config defines application's configuration.
 type Config struct {
-	BinaryPath           string                // full path of the program, required for auto-update procedure
-	BinaryVersion        string                // version of the program, required for auto-update procedure
-	AutoUpdate           string                `yaml:"autoupdate"`       // controls auto-update procedure
-	NoTrackMode          bool                  `yaml:"no_track_mode"`    // controls tracking sensitive information (query texts, etc)
-	ListenAddress        string                `yaml:"listen_address"`   // Network address and port where the application should listen on
-	SendMetricsURL       string                `yaml:"send_metrics_url"` // URL of Weaponry service metric gateway
-	SendMetricsInterval  time.Duration         // Metric send interval
-	APIKey               string                `yaml:"api_key"`  // API key for accessing to Weaponry
-	ServicesConnSettings []service.ConnSetting `yaml:"services"` // Slice of connection settings for exact services
-	Defaults             map[string]string     `yaml:"defaults"` // Defaults
-	Filters              filter.Filters        `yaml:"filters"`
-	DisableCollectors    []string              `yaml:"disable_collectors"` // List of collectors which should be disabled.
+	BinaryPath            string                // full path of the program, required for auto-update procedure
+	BinaryVersion         string                // version of the program, required for auto-update procedure
+	AutoUpdate            string                `yaml:"autoupdate"`       // controls auto-update procedure
+	NoTrackMode           bool                  `yaml:"no_track_mode"`    // controls tracking sensitive information (query texts, etc)
+	ListenAddress         string                `yaml:"listen_address"`   // Network address and port where the application should listen on
+	SendMetricsURL        string                `yaml:"send_metrics_url"` // URL of Weaponry service metric gateway
+	SendMetricsInterval   time.Duration         // Metric send interval
+	APIKey                string                `yaml:"api_key"`  // API key for accessing to Weaponry
+	ServicesConnsSettings service.ConnsSettings `yaml:"services"` // All connections settings for exact services
+	Defaults              map[string]string     `yaml:"defaults"` // Defaults
+	Filters               filter.Filters        `yaml:"filters"`
+	DisableCollectors     []string              `yaml:"disable_collectors"` // List of collectors which should be disabled.
 }
 
 // NewConfig creates new config based on config file or return default config of config is not exists.
@@ -110,16 +110,19 @@ func (c *Config) Validate() error {
 
 	// User might specify its own set of services which he would like to monitor. This services should be validated and
 	// invalid should be rejected. Validation is performed using pgx.ParseConfig method which does all dirty work.
-	if c.ServicesConnSettings != nil {
-		if len(c.ServicesConnSettings) != 0 {
-			for _, s := range c.ServicesConnSettings {
+	if c.ServicesConnsSettings != nil {
+		if len(c.ServicesConnsSettings) != 0 {
+			for k, s := range c.ServicesConnsSettings {
+				if k == "" {
+					return fmt.Errorf("empty service specified")
+				}
 				if s.ServiceType == "" {
-					return fmt.Errorf("service_type is not specified for %s", s.Conninfo)
+					return fmt.Errorf("empty service_type for %s", k)
 				}
 
 				_, err := pgx.ParseConfig(s.Conninfo)
 				if err != nil {
-					return fmt.Errorf("invalid conninfo: %s", err)
+					return fmt.Errorf("invalid conninfo for %s: %s", k, err)
 				}
 			}
 		}

internal/pgscv/config_test.go

@@ -53,9 +53,9 @@ func TestNewConfig(t *testing.T) {
 			want: &Config{
 				ListenAddress: "127.0.0.1:8080",
 				Defaults:      map[string]string{},
-				ServicesConnSettings: []service.ConnSetting{
-					{ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 port=5432 dbname=pgscv_fixtures user=pgscv"},
-					{ServiceType: model.ServiceTypePgbouncer, Conninfo: "host=127.0.0.1 port=6432 dbname=pgbouncer user=pgscv"},
+				ServicesConnsSettings: service.ConnsSettings{
+					"postgres:5432":  {ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 port=5432 dbname=pgscv_fixtures user=pgscv"},
+					"pgbouncer:6432": {ServiceType: model.ServiceTypePgbouncer, Conninfo: "host=127.0.0.1 port=6432 dbname=pgbouncer user=pgscv"},
 				},
 			},
 		},
@@ -132,9 +132,9 @@ func TestConfig_Validate(t *testing.T) {
 		{
 			name:  "valid config with specified services",
 			valid: true,
-			in: &Config{ListenAddress: "127.0.0.1:8080", ServicesConnSettings: []service.ConnSetting{
-				{ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 dbname=pgscv_fixtures user=pgscv"},
-				{ServiceType: model.ServiceTypePgbouncer, Conninfo: "host=127.0.0.1 port=6432 dbname=pgbouncer user=pgscv"},
+			in: &Config{ListenAddress: "127.0.0.1:8080", ServicesConnsSettings: service.ConnsSettings{
+				"postgres:5432":  {ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 dbname=pgscv_fixtures user=pgscv"},
+				"pgbouncer:6432": {ServiceType: model.ServiceTypePgbouncer, Conninfo: "host=127.0.0.1 port=6432 dbname=pgbouncer user=pgscv"},
 			}},
 		},
 		{
@@ -150,15 +150,22 @@ func TestConfig_Validate(t *testing.T) {
 		{
 			name:  "invalid config with specified services: empty service type",
 			valid: false,
-			in: &Config{ListenAddress: "127.0.0.1:8080", ServicesConnSettings: []service.ConnSetting{
-				{ServiceType: "", Conninfo: "host=127.0.0.1 dbname=pgscv_fixtures user=pgscv"},
+			in: &Config{ListenAddress: "127.0.0.1:8080", ServicesConnsSettings: service.ConnsSettings{
+				"": {ServiceType: "postgres", Conninfo: "host=127.0.0.1 dbname=pgscv_fixtures user=pgscv"},
+			}},
+		},
+		{
+			name:  "invalid config with specified services: empty service type",
+			valid: false,
+			in: &Config{ListenAddress: "127.0.0.1:8080", ServicesConnsSettings: service.ConnsSettings{
+				"test": {ServiceType: "", Conninfo: "host=127.0.0.1 dbname=pgscv_fixtures user=pgscv"},
 			}},
 		},
 		{
 			name:  "invalid config with specified services: invalid conninfo",
 			valid: false,
-			in: &Config{ListenAddress: "127.0.0.1:8080", ServicesConnSettings: []service.ConnSetting{
-				{ServiceType: model.ServiceTypePostgresql, Conninfo: "invalid"},
+			in: &Config{ListenAddress: "127.0.0.1:8080", ServicesConnsSettings: service.ConnsSettings{
+				"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "invalid"},
 			}},
 		},
 		{

internal/pgscv/pgscv.go

@@ -29,15 +29,15 @@ func Start(ctx context.Context, config *Config) error {
 	serviceConfig := service.Config{
 		NoTrackMode:        config.NoTrackMode,
 		ConnDefaults:       config.Defaults,
-		ConnSettings:       config.ServicesConnSettings,
+		ConnsSettings:      config.ServicesConnsSettings,
 		Filters:            config.Filters,
 		DisabledCollectors: config.DisableCollectors,
 	}
 
 	var wg sync.WaitGroup
 	ctx, cancel := context.WithCancel(ctx)
 
-	if config.ServicesConnSettings == nil {
+	if config.ServicesConnsSettings == nil {
 		// run background discovery, the service repo will be fulfilled at first iteration
 		wg.Add(1)
 		go func() {

internal/pgscv/pgscv_test.go

@@ -115,8 +115,8 @@ func Test_runSendMetricsLoop(t *testing.T) {
 	}
 	repo := service.NewRepository()
 	repo.AddServicesFromConfig(service.Config{
-		ConnSettings: nil,
-		ConnDefaults: nil,
+		ConnsSettings: nil,
+		ConnDefaults:  nil,
 	})
 
 	// Run sending metrics to test server.

internal/pgscv/testdata/pgscv-services-example.yaml

@@ -1,6 +1,8 @@
 listen_address: "127.0.0.1:8080"
 services:
-  - service_type: "postgres"
+  "postgres:5432":
+    service_type: "postgres"
     conninfo: "host=127.0.0.1 port=5432 dbname=pgscv_fixtures user=pgscv"
-  - service_type: "pgbouncer"
+  "pgbouncer:6432":
+    service_type: "pgbouncer"
     conninfo: "host=127.0.0.1 port=6432 dbname=pgbouncer user=pgscv"

internal/service/service.go

@@ -54,7 +54,7 @@ type Config struct {
 	RuntimeMode        int
 	NoTrackMode        bool
 	ConnDefaults       map[string]string `yaml:"defaults"` // Defaults
-	ConnSettings       []ConnSetting
+	ConnsSettings      ConnsSettings
 	Filters            map[string]filter.Filter
 	DisabledCollectors []string
 }
@@ -74,6 +74,9 @@ type ConnSetting struct {
 	Conninfo string `yaml:"conninfo"`
 }
 
+// ConnsSettings defines a set of all connection settings of exact services.
+type ConnsSettings map[string]ConnSetting
+
 // connectionParams is the set of parameters that may be required when constructing connection string.
 // For example, this struct describes the postmaster.pid representation https://www.postgresql.org/docs/current/storage-file-layout.html
 type connectionParams struct {
@@ -134,6 +137,7 @@ func (repo *Repository) StartBackgroundDiscovery(ctx context.Context, config Con
 /* Private methods of Repository */
 
 // addService adds service to the repo.
+// TODO: refactor to remove id argument, because it is already in the 's' argument.
 func (repo *Repository) addService(id string, s Service) {
 	repo.Lock()
 	repo.Services[id] = s
@@ -209,15 +213,15 @@ func (repo *Repository) addServicesFromConfig(config Config) {
 	log.Info("registered new service [system:0]")
 
 	// Sanity check, but basically should be always passed.
-	if config.ConnSettings == nil {
+	if config.ConnsSettings == nil {
 		log.Warn("connection settings for service are not defined, do nothing")
 		return
 	}
 
 	// Check all passed connection settings and try to connect using them. In case of success, create a 'Service' instance
 	// in the repo.
-	for _, cs := range config.ConnSettings {
-		// *ConnConfig struct will be used for
+	for k, cs := range config.ConnsSettings {
+		// each ConnSetting struct is used for
 		//   1) doing connection;
 		//   2) getting connection properties to define service-specific parameters.
 		pgconfig, err := pgx.ParseConfig(cs.Conninfo)
@@ -236,14 +240,13 @@ func (repo *Repository) addServicesFromConfig(config Config) {
 
 		// Connection was successful, create 'Service' struct with service-related properties and add it to service repo.
 		s := Service{
-			ServiceID:    cs.ServiceType + ":" + strconv.Itoa(int(pgconfig.Port)),
+			ServiceID:    k,
 			ConnSettings: cs,
 			Collector:    nil,
 		}
 
-		// Add "host", because user might manually specify services with the same port (but the are running on different hosts).
-		var key = strings.Join([]string{cs.ServiceType, pgconfig.Host, strconv.Itoa(int(pgconfig.Port))}, ":")
-		repo.addService(key, s)
+		// Use entry key as ServiceID unique identifier.
+		repo.addService(k, s)
 		log.Infof("registered new service [%s]", s.ServiceID)
 		log.Debugf("new service available through: %s@%s:%d/%s", pgconfig.User, pgconfig.Host, pgconfig.Port, pgconfig.Database)
 	}

internal/service/service_test.go

@@ -86,8 +86,8 @@ func TestRepository_addServicesFromConfig(t *testing.T) {
 	}{
 		{
 			name: "valid",
-			config: Config{ConnSettings: []ConnSetting{
-				{ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 port=5432 user=pgscv dbname=pgscv_fixtures"},
+			config: Config{ConnsSettings: ConnsSettings{
+				"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 port=5432 user=pgscv dbname=pgscv_fixtures"},
 			}},
 			expected: 2,
 		},
@@ -98,12 +98,12 @@ func TestRepository_addServicesFromConfig(t *testing.T) {
 		},
 		{
 			name:     "invalid service",
-			config:   Config{ConnSettings: []ConnSetting{{ServiceType: model.ServiceTypePostgresql, Conninfo: "invalid conninfo"}}},
+			config:   Config{ConnsSettings: ConnsSettings{"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "invalid conninfo"}}},
 			expected: 1,
 		},
 		{
 			name:     "unavailable service",
-			config:   Config{ConnSettings: []ConnSetting{{ServiceType: model.ServiceTypePostgresql, Conninfo: "port=1"}}},
+			config:   Config{ConnsSettings: ConnsSettings{"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "port=1"}}},
 			expected: 1,
 		},
 	}
@@ -149,8 +149,8 @@ func TestRepository_setupServices(t *testing.T) {
 		{
 			name: "valid",
 			config: Config{
-				ConnSettings: []ConnSetting{
-					{ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 port=5432 user=pgscv dbname=pgscv_fixtures"},
+				ConnsSettings: ConnsSettings{
+					"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 port=5432 user=pgscv dbname=pgscv_fixtures"},
 				},
 			},
 			expected: 2,
@@ -163,7 +163,7 @@ func TestRepository_setupServices(t *testing.T) {
 		assert.Equal(t, tc.expected, r.totalServices())
 
 		assert.NoError(t, r.setupServices(tc.config))
-		s := r.GetService("postgres:127.0.0.1:5432")
+		s := r.GetService("test")
 		assert.NotNil(t, s.Collector)
 
 		prometheus.Unregister(s.Collector)