Metricbeat: MongoDB TLS connection support (#7401)

Add support for TLS in MongoDB metricbeat module. A common builder is also added for MongoDB metricsets.

Fixes #6619

CHANGELOG.asciidoc

@@ -284,6 +284,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Add postgresql statement metricset. {issue}7048[7048] {pull}7060[7060]
 - Update `state_container` metricset to support latest `kube-state-metrics` version. {pull}7216[7216]
 - Collect accumulated docker network metrics and mark old ones as deprecated. {pull}7253[7253]
+- Add TLS support to MongoDB module. {pull}7401[7401]
 
 *Packetbeat*
 

metricbeat/docs/modules/mongodb.asciidoc

@@ -77,13 +77,30 @@ metricbeat.modules:
   # password configuration options.
   hosts: ["localhost:27017"]
 
+  # Optional SSL. By default is off.
+  #ssl.enabled: true
+
+  # Mode of verification of server certificate ('none' or 'full')
+  #ssl.verification_mode: 'full'
+
+  # List of root certificates for TLS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
   # Username to use when connecting to MongoDB. Empty by default.
   #username: user
 
   # Password to use when connecting to MongoDB. Empty by default.
   #password: pass
 ----
 
+This module supports TLS connection when using `ssl` config field, as described in <<configuration-ssl>>. It also supports the options described in <<module-http-config-options>>.
+
 [float]
 === Metricsets
 

metricbeat/metricbeat.reference.yml

@@ -427,6 +427,21 @@ metricbeat.modules:
   # password configuration options.
   hosts: ["localhost:27017"]
 
+  # Optional SSL. By default is off.
+  #ssl.enabled: true
+
+  # Mode of verification of server certificate ('none' or 'full')
+  #ssl.verification_mode: 'full'
+
+  # List of root certificates for TLS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
   # Username to use when connecting to MongoDB. Empty by default.
   #username: user
 

metricbeat/module/mongodb/_meta/config.reference.yml

@@ -10,6 +10,21 @@
   # password configuration options.
   hosts: ["localhost:27017"]
 
+  # Optional SSL. By default is off.
+  #ssl.enabled: true
+
+  # Mode of verification of server certificate ('none' or 'full')
+  #ssl.verification_mode: 'full'
+
+  # List of root certificates for TLS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
   # Username to use when connecting to MongoDB. Empty by default.
   #username: user
 

metricbeat/module/mongodb/_meta/config.yml

@@ -11,6 +11,21 @@
   # password configuration options.
   hosts: ["localhost:27017"]
 
+  # Optional SSL. By default is off.
+  #ssl.enabled: true
+
+  # Mode of verification of server certificate ('none' or 'full')
+  #ssl.verification_mode: 'full'
+
+  # List of root certificates for TLS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
   # Username to use when connecting to MongoDB. Empty by default.
   #username: user
 

metricbeat/module/mongodb/_meta/fields.yml

@@ -4,6 +4,7 @@
     Metrics collected from MongoDB servers.
   short_config: false
   release: ga
+  settings: ["ssl"]
   fields:
     - name: mongodb
       type: group

metricbeat/module/mongodb/collstats/collstats.go

@@ -24,8 +24,6 @@ import (
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/metricbeat/mb"
 	"github.com/elastic/beats/metricbeat/module/mongodb"
-
-	"gopkg.in/mgo.v2"
 )
 
 var debugf = logp.MakeDebug("mongodb.collstats")
@@ -42,24 +40,18 @@ func init() {
 // additional entries. These variables can be used to persist data or configuration between
 // multiple fetch calls.
 type MetricSet struct {
-	mb.BaseMetricSet
-	dialInfo *mgo.DialInfo
+	*mongodb.MetricSet
 }
 
-// New create a new instance of the MetricSet
+// New creates a new instance of the MetricSet
 // Part of new is also setting up the configuration by processing additional
 // configuration entries if needed.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
-	dialInfo, err := mgo.ParseURL(base.HostData().URI)
+	ms, err := mongodb.NewMetricSet(base)
 	if err != nil {
 		return nil, err
 	}
-	dialInfo.Timeout = base.Module().Config().Timeout
-
-	return &MetricSet{
-		BaseMetricSet: base,
-		dialInfo:      dialInfo,
-	}, nil
+	return &MetricSet{ms}, nil
 }
 
 // Fetch methods implements the data gathering and data conversion to the right format
@@ -70,7 +62,7 @@ func (m *MetricSet) Fetch() ([]common.MapStr, error) {
 	var events []common.MapStr
 
 	// instantiate direct connections to each of the configured Mongo hosts
-	mongoSession, err := mongodb.NewDirectSession(m.dialInfo)
+	mongoSession, err := mongodb.NewDirectSession(m.DialInfo)
 	if err != nil {
 		return nil, err
 	}

metricbeat/module/mongodb/dbstats/dbstats.go

@@ -20,8 +20,6 @@ package dbstats
 import (
 	"errors"
 
-	"gopkg.in/mgo.v2"
-
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/metricbeat/mb"
@@ -44,24 +42,18 @@ func init() {
 // additional entries. These variables can be used to persist data or configuration between
 // multiple fetch calls.
 type MetricSet struct {
-	mb.BaseMetricSet
-	dialInfo *mgo.DialInfo
+	*mongodb.MetricSet
 }
 
 // New creates a new instance of the MetricSet
 // Part of new is also setting up the configuration by processing additional
 // configuration entries if needed.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
-	dialInfo, err := mgo.ParseURL(base.HostData().URI)
+	ms, err := mongodb.NewMetricSet(base)
 	if err != nil {
 		return nil, err
 	}
-	dialInfo.Timeout = base.Module().Config().Timeout
-
-	return &MetricSet{
-		BaseMetricSet: base,
-		dialInfo:      dialInfo,
-	}, nil
+	return &MetricSet{ms}, nil
 }
 
 // Fetch methods implements the data gathering and data conversion to the right format
@@ -72,7 +64,7 @@ func (m *MetricSet) Fetch() ([]common.MapStr, error) {
 	var events []common.MapStr
 
 	// instantiate direct connections to each of the configured Mongo hosts
-	mongoSession, err := mongodb.NewDirectSession(m.dialInfo)
+	mongoSession, err := mongodb.NewDirectSession(m.DialInfo)
 	if err != nil {
 		return nil, err
 	}

metricbeat/module/mongodb/metricset.go

@@ -0,0 +1,76 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mongodb
+
+import (
+	"crypto/tls"
+	"net"
+
+	"gopkg.in/mgo.v2"
+
+	"github.com/elastic/beats/libbeat/common/transport/tlscommon"
+	"github.com/elastic/beats/libbeat/logp"
+	"github.com/elastic/beats/metricbeat/mb"
+)
+
+// ModuleConfig contains the common configuration for this module
+type ModuleConfig struct {
+	TLS *tlscommon.Config `config:"ssl"`
+}
+
+// MetricSet type defines all fields of the MetricSet
+type MetricSet struct {
+	mb.BaseMetricSet
+	DialInfo *mgo.DialInfo
+}
+
+// NewMetricSet creates a new instance of the MetricSet
+func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
+	var config ModuleConfig
+	err := base.Module().UnpackConfig(&config)
+	if err != nil {
+		return nil, err
+	}
+
+	dialInfo, err := mgo.ParseURL(base.HostData().URI)
+	if err != nil {
+		return nil, err
+	}
+	dialInfo.Timeout = base.Module().Config().Timeout
+
+	if config.TLS.IsEnabled() {
+		tlsConfig, err := tlscommon.LoadTLSConfig(config.TLS)
+		if err != nil {
+			return nil, err
+		}
+
+		dialInfo.DialServer = func(addr *mgo.ServerAddr) (net.Conn, error) {
+			hostname, _, err := net.SplitHostPort(base.HostData().Host)
+			if err != nil {
+				logp.Warn("Failed to obtain hostname from `%s`: %s", hostname, err)
+				hostname = ""
+			}
+			return tls.Dial("tcp", addr.String(), tlsConfig.BuildModuleConfig(hostname))
+		}
+	}
+
+	return &MetricSet{
+		BaseMetricSet: base,
+		DialInfo:      dialInfo,
+	}, nil
+}

metricbeat/module/mongodb/status/status.go

@@ -23,7 +23,6 @@ import (
 	"github.com/elastic/beats/metricbeat/mb"
 	"github.com/elastic/beats/metricbeat/module/mongodb"
 
-	"gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
 )
 
@@ -47,24 +46,18 @@ func init() {
 // additional entries. These variables can be used to persist data or configuration between
 // multiple fetch calls.
 type MetricSet struct {
-	mb.BaseMetricSet
-	dialInfo *mgo.DialInfo
+	*mongodb.MetricSet
 }
 
 // New creates a new instance of the MetricSet
 // Part of new is also setting up the configuration by processing additional
 // configuration entries if needed.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
-	dialInfo, err := mgo.ParseURL(base.HostData().URI)
+	ms, err := mongodb.NewMetricSet(base)
 	if err != nil {
 		return nil, err
 	}
-	dialInfo.Timeout = base.Module().Config().Timeout
-
-	return &MetricSet{
-		BaseMetricSet: base,
-		dialInfo:      dialInfo,
-	}, nil
+	return &MetricSet{ms}, nil
 }
 
 // Fetch methods implements the data gathering and data conversion to the right format
@@ -73,7 +66,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 func (m *MetricSet) Fetch() (common.MapStr, error) {
 
 	// instantiate direct connections to each of the configured Mongo hosts
-	mongoSession, err := mongodb.NewDirectSession(m.dialInfo)
+	mongoSession, err := mongodb.NewDirectSession(m.DialInfo)
 	if err != nil {
 		return nil, err
 	}

metricbeat/modules.d/mongodb.yml.disabled

@@ -14,6 +14,21 @@
   # password configuration options.
   hosts: ["localhost:27017"]
 
+  # Optional SSL. By default is off.
+  #ssl.enabled: true
+
+  # Mode of verification of server certificate ('none' or 'full')
+  #ssl.verification_mode: 'full'
+
+  # List of root certificates for TLS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
   # Username to use when connecting to MongoDB. Empty by default.
   #username: user