Add TLS Metadata to HTTP/TCP monitors. +mapval enhancements (#7545)

* We now write certificate PEM data, and chain validation times to events
* Enhanced test helpers (including mapval IsDefs) to handle new testing scenarios
* Improved mapval handling of type mismatches to help with the tests here
* Move CertToPEMString to libbeat to share between hb and pb
* Incorporate @urso 's PR feedback using registration for equality types

CHANGELOG.asciidoc

@@ -51,6 +51,8 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 
 *Heartbeat*
 
+- Added support for extra TLS/x509 metadata. {pull}7944[7944]
+
 *Metricbeat*
 
 - Fix golang.heap.gc.cpu_fraction type from long to float in Golang module. {pull}7789[7789]

heartbeat/docs/fields.asciidoc

@@ -898,6 +898,33 @@ TLS layer related fields.
 
 
 
+*`tls.certificates`*::
++
+--
+type: keyword
+
+List of PEM encoded x509 certificates.
+
+--
+
+*`tls.certificate_not_valid_before`*::
++
+--
+type: date
+
+Earliest time at which the connection's certificates are valid.
+
+--
+
+*`tls.certificate_not_valid_after`*::
++
+--
+type: date
+
+Latest time at which the connection's certificates are valid.
+
+--
+
 [float]
 == rtt fields
 

heartbeat/hbtest/hbtestutil.go

@@ -18,14 +18,20 @@
 package hbtest
 
 import (
+	"crypto/x509"
 	"io"
+	"io/ioutil"
 	"net/http"
+	"net/http/httptest"
 	"net/url"
+	"os"
 	"strconv"
+	"testing"
 
-	"net/http/httptest"
+	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/libbeat/common/mapval"
+	"github.com/elastic/beats/libbeat/common/x509util"
 )
 
 // HelloWorldBody is the body of the HelloWorldHandler.
@@ -58,6 +64,19 @@ func ServerPort(server *httptest.Server) (uint16, error) {
 	return uint16(p), nil
 }
 
+// TLSChecks validates the given x509 cert at the given position.
+func TLSChecks(chainIndex, certIndex int, certificate *x509.Certificate) mapval.Validator {
+	certPEMString := x509util.CertToPEMString(certificate)
+	return mapval.MustCompile(mapval.Map{
+		"tls": mapval.Map{
+			"rtt.handshake.us":             mapval.IsDuration,
+			"certificates":                 mapval.Slice{certPEMString},
+			"certificate_not_valid_before": certificate.NotBefore,
+			"certificate_not_valid_after":  certificate.NotAfter,
+		},
+	})
+}
+
 // MonitorChecks creates a skima.Validator that represents the "monitor" field present
 // in all heartbeat events.
 func MonitorChecks(id string, host string, ip string, scheme string, status string) mapval.Validator {
@@ -101,3 +120,17 @@ func RespondingTCPChecks(port uint16) mapval.Validator {
 		mapval.MustCompile(mapval.Map{"tcp.rtt.connect.us": mapval.IsDuration}),
 	)
 }
+
+// CertToTempFile takes a certificate and returns an *os.File with a PEM encoded
+// x.509 representation of that cert. Note that this takes tls.Certificate
+// objects from a server like httptest. This doesn't take x509 certs.
+// We never parse the x509 data in this case, we just transpose the bytes.
+// This is a little confusing, but is actually less work and less code.
+func CertToTempFile(t *testing.T, cert *x509.Certificate) *os.File {
+	// Write the certificate to a tempFile. Heartbeat would normally read certs from
+	// disk, not memory, so this little bit of extra work is worthwhile
+	certFile, err := ioutil.TempFile("", "sslcert")
+	require.NoError(t, err)
+	certFile.WriteString(x509util.CertToPEMString(cert))
+	return certFile
+}

heartbeat/monitors/active/dialchain/_meta/fields.yml

@@ -32,6 +32,15 @@
       description: >
         TLS layer related fields.
       fields:
+        - name: certificates
+          type: keyword
+          description: List of PEM encoded x509 certificates.
+        - name: certificate_not_valid_before
+          type: date
+          description: Earliest time at which the connection's certificates are valid.
+        - name: certificate_not_valid_after
+          type: date
+          description: Latest time at which the connection's certificates are valid.
         - name: rtt
           type: group
           description: >

heartbeat/monitors/active/dialchain/tls.go

@@ -18,11 +18,14 @@
 package dialchain
 
 import (
+	cryptoTLS "crypto/tls"
+	"fmt"
 	"net"
 	"time"
 
 	"github.com/elastic/beats/heartbeat/look"
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/common/x509util"
 	"github.com/elastic/beats/libbeat/outputs/transport"
 )
 
@@ -49,10 +52,43 @@ func TLSLayer(cfg *transport.TLSConfig, to time.Duration) Layer {
 		}
 
 		return afterDial(dialer, func(conn net.Conn) (net.Conn, error) {
-			// TODO: extract TLS connection parameters from connection object.
+			tlsConn, ok := conn.(*cryptoTLS.Conn)
+			if !ok {
+				panic(fmt.Sprintf("TLS afterDial received a non-tls connection %t. This should never happen", conn))
+			}
 
+			// TODO: extract TLS connection parameters from connection object.
 			timer.stop()
 			event.Put("tls.rtt.handshake", look.RTT(timer.duration()))
+
+			var certs []string
+
+			// Pointers because we need a nil value
+			var chainNotValidBefore *time.Time
+			var chainNotValidAfter *time.Time
+
+			// Here we compute the minimal bounds during which this certificate chain is valid
+			// To do this correctly, we take the maximum NotBefore and the minimum NotAfter.
+			// This *should* always wind up being the terminal cert in the chain, but we should
+			// compute this correctly.
+			for _, chain := range tlsConn.ConnectionState().VerifiedChains {
+				for _, cert := range chain {
+					certs = append(certs, x509util.CertToPEMString(cert))
+
+					if chainNotValidBefore == nil || chainNotValidBefore.Before(cert.NotBefore) {
+						chainNotValidBefore = &cert.NotBefore
+					}
+
+					if chainNotValidAfter == nil || chainNotValidAfter.After(cert.NotAfter) {
+						chainNotValidAfter = &cert.NotAfter
+					}
+				}
+			}
+
+			event.Put("tls.certificate_not_valid_before", *chainNotValidBefore)
+			event.Put("tls.certificate_not_valid_after", *chainNotValidAfter)
+			event.Put("tls.certificates", certs)
+
 			return conn, nil
 		}), nil
 	}

heartbeat/monitors/active/http/http_test.go

@@ -18,9 +18,11 @@
 package http
 
 import (
+	"crypto/x509"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -34,9 +36,24 @@ import (
 	"github.com/elastic/beats/libbeat/testing/mapvaltest"
 )
 
-func testRequest(t *testing.T, url string) beat.Event {
-	config := common.NewConfig()
-	config.SetString("urls", 0, url)
+func testRequest(t *testing.T, testURL string) beat.Event {
+	return testTLSRequest(t, testURL, "")
+}
+
+// testTLSRequest tests the given request. certPath is optional, if given
+// an empty string no cert will be set.
+func testTLSRequest(t *testing.T, testURL string, certPath string) beat.Event {
+	configSrc := map[string]interface{}{
+		"urls":    testURL,
+		"timeout": "1s",
+	}
+
+	if certPath != "" {
+		configSrc["ssl.certificate_authorities"] = certPath
+	}
+
+	config, err := common.NewConfigFrom(configSrc)
+	require.NoError(t, err)
 
 	jobs, err := create(monitors.Info{}, config)
 	require.NoError(t, err)
@@ -52,7 +69,6 @@ func testRequest(t *testing.T, url string) beat.Event {
 func checkServer(t *testing.T, handlerFunc http.HandlerFunc) (*httptest.Server, beat.Event) {
 	server := httptest.NewServer(handlerFunc)
 	defer server.Close()
-
 	event := testRequest(t, server.URL)
 
 	return server, event
@@ -195,6 +211,34 @@ func TestDownStatuses(t *testing.T) {
 	}
 }
 
+func TestHTTPSServer(t *testing.T) {
+	server := httptest.NewTLSServer(hbtest.HelloWorldHandler(http.StatusOK))
+	port, err := hbtest.ServerPort(server)
+	require.NoError(t, err)
+
+	// Parse the cert so we can test against it.
+	cert, err := x509.ParseCertificate(server.TLS.Certificates[0].Certificate[0])
+	require.NoError(t, err)
+
+	// Write the cert to a tempfile so heartbeat can use it in its config.
+	certFile := hbtest.CertToTempFile(t, cert)
+	require.NoError(t, certFile.Close())
+	defer os.Remove(certFile.Name())
+
+	event := testTLSRequest(t, server.URL, certFile.Name())
+
+	mapvaltest.Test(
+		t,
+		mapval.Strict(mapval.Compose(
+			hbtest.MonitorChecks("http@"+server.URL, server.URL, "127.0.0.1", "https", "up"),
+			hbtest.RespondingTCPChecks(port),
+			hbtest.TLSChecks(0, 0, cert),
+			respondingHTTPChecks(server.URL, http.StatusOK),
+		)),
+		event.Fields,
+	)
+}
+
 func TestConnRefusedJob(t *testing.T) {
 	ip := "127.0.0.1"
 	port, err := btesting.AvailableTCP4Port()