Add certificate TLS verification mode

CHANGELOG.next.asciidoc

@@ -19,6 +19,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Make error message about locked data path actionable. {pull}18667[18667]
 - Ensure dynamic template names are unique for the same field. {pull}18849[18849]
 - Remove the deprecated `xpack.monitoring.*` settings. Going forward only `monitoring.*` settings may be used. {issue}9424[9424] {pull}18608[18608]
+- Added `certificate` TLS verification mode to ignore server name mismatch. {issue}12283[12283] {pull}20293[20293]
 
 *Auditbeat*
 

NOTICE.txt

@@ -12135,11 +12135,11 @@ Contents of probable licence file $GOMODCACHE/github.com/oklog/[email protected]/LICEN
 
 --------------------------------------------------------------------------------
 Dependency : github.com/pierrre/gotestcover
-Version: v0.0.0-20160113212533-7b94f124d338
+Version: v0.0.0-20160517101806-924dca7d15f0
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/pierrre/[email protected]20160113212533-7b94f124d338/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/pierrre/[email protected]20160517101806-924dca7d15f0/LICENSE:
 
 Copyright (C) 2015 Pierre Durand
 

go.mod

@@ -123,7 +123,7 @@ require (
 	github.com/oklog/ulid v1.3.1
 	github.com/opencontainers/go-digest v1.0.0-rc1.0.20190228220655-ac19fd6e7483 // indirect
 	github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 // indirect
-	github.com/pierrre/gotestcover v0.0.0-20160113212533-7b94f124d338
+	github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/poy/eachers v0.0.0-20181020210610-23942921fe77 // indirect

go.sum

@@ -569,8 +569,8 @@ github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOTh
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg=
 github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
-github.com/pierrre/gotestcover v0.0.0-20160113212533-7b94f124d338 h1:/VAZ3an4jHXs+61iNHugNR1mG25MSpaxtMnwOJVEAQM=
-github.com/pierrre/gotestcover v0.0.0-20160113212533-7b94f124d338/go.mod h1:4xpMLz7RBWyB+ElzHu8Llua96TRCB3YwX+l5EP1wmHk=
+github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0 h1:i5VIxp6QB8oWZ8IkK8zrDgeT6ORGIUeiN+61iETwJbI=
+github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0/go.mod h1:4xpMLz7RBWyB+ElzHu8Llua96TRCB3YwX+l5EP1wmHk=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1-0.20170505043639-c605e284fe17/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

libbeat/common/transport/tlscommon/ca_pinning.go

@@ -32,27 +32,21 @@ var ErrCAPinMissmatch = errors.New("provided CA certificate pins doesn't match a
 // TLS connection is used.
 type verifyPeerCertFunc func([][]byte, [][]*x509.Certificate) error
 
-// MakeCAPinCallback loops through the verified chains and will try to match the certificates pin.
+// verifyCAPin loops through the verified chains and will try to match the certificates pin.
 //
 // NOTE: Defining a PIN to check certificates is not a replacement for the normal TLS validations it's
 // an additional validation. In fact if you set `InsecureSkipVerify` to true and a PIN, the
 // verifiedChains variable will be empty and the added validation will fail.
-func MakeCAPinCallback(hashes []string) func([][]byte, [][]*x509.Certificate) error {
-	return func(_ [][]byte, verifiedChains [][]*x509.Certificate) error {
-		// The chain of trust has been already established before the call to the VerifyPeerCertificate
-		// function, after we go through the chain to make sure we have at least a certificate certificate
-		//	that match the provided pin.
-		for _, chain := range verifiedChains {
-			for _, certificate := range chain {
-				h := Fingerprint(certificate)
-				if matches(hashes, h) {
-					return nil
-				}
+func verifyCAPin(hashes []string, verifiedChains [][]*x509.Certificate) error {
+	for _, chain := range verifiedChains {
+		for _, certificate := range chain {
+			h := Fingerprint(certificate)
+			if matches(hashes, h) {
+				return nil
 			}
 		}
-
-		return ErrCAPinMissmatch
 	}
+	return ErrCAPinMissmatch
 }
 
 // Fingerprint takes a certificate and create a hash of the DER encoded public key.

libbeat/common/transport/tlscommon/testdata/ca.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIC/zCCAeegAwIBAgIJAIVZ8xw3LMNkMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV
+BAMMC21vcmVsbG8ub3ZoMB4XDTE5MDgwOTA5MzQwMFoXDTI5MDgwNjA5MzQwMFow
+FjEUMBIGA1UEAwwLbW9yZWxsby5vdmgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCoM2HYyuTTlu41SlgVO0Hdx7eUQevGSKO6pjPjN49/KKY1z/3DoKzr
+seWaGOjiWUAqx/GHX8AsR9ToVoKGBbSNeDxT33pt3I9aCnnOPTt3yDIOlr4ZWnKq
+NnNHwfydsMBfBAYgdU/L506KuNHJQ18Zey5+A0roTWyHUT48mQBsjetXg77RfDMB
+MYVOWETfl70GKAaAlVGZfJHCkfBzYnPcEjqtcuU/7d27WZrSMhXifzHAEmm0KPER
+EWdo4UHTK23wLY6dvkp2O5i0bKHv+PuLpqYrm7R7SWGhhwD651n5S5W20FHDow+d
+js0yW2gqYsZZN6S1uAsJ8rdYAEPhK9J9AgMBAAGjUDBOMB0GA1UdDgQWBBQ6Lsen
+0HbE+7M6iV9r8n5rZrbl4jAfBgNVHSMEGDAWgBQ6Lsen0HbE+7M6iV9r8n5rZrbl
+4jAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAgrLJnK4s/OVnh8CRk
+GmikP+ZxhDs4k1nlr7+rTYkU0huoHK8p802w4zd74szYsHpo8kON/zSmFD7JpU4L
+o2kseENqMsgrCPhF3+TDwf/Li43pbK162iAq8ZEpYnSXbQsRyP+Tz0lzoEoli6o7
+6KVn4VNookLMyhGIAOmhfbNm0jG+B2zz+bvoTAe9CiDfvq1k0fnuKFzRtRsj09NJ
+FNMhSc02N4EDrGpL5CYmEXjPZS3lUsoYPwbYlmUt3Bzuf5hI0mDHCt3BYKH1vFI4
+W8/h9wwGn/yytsH21dkj41KEQK6N65gT9i0fBBiubuS2H1SVMMJ/J7PUqol278Ar
+zGpS
+-----END CERTIFICATE-----

libbeat/common/transport/tlscommon/testdata/tls.crt

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDnzCCAoegAwIBAgIRAKtKtQKtGFIUneRz5r1FnUMwDQYJKoZIhvcNAQELBQAw
+FjEUMBIGA1UEAwwLbW9yZWxsby5vdmgwHhcNMTkwODA5MDkzOTIyWhcNMTkxMTA3
+MDkzOTIyWjBOMRkwFwYDVQQKExBFbGFzdGljc2VhcmNoIENBMTEwLwYDVQQDEyhl
+bGFzdGljc2VhcmNoLXNhbXBsZS1lcy1odHRwLmRlZmF1bHQuc3ZjMIIBIjANBgkq
+hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAq6HRcrfV1kHnXv5Z+ImkgKDvxCezI3/p
+yiR0jSv6L7+bblHzzsqkPnz3aaIPJJ2G4sdwaIhl5rJdOvCj48It8OtRidZjzuJH
+hN2RpN2Ii5WX4D1u18CrjEQrRUzs/vuwpyP0zWx0yP3lp88fy8kfWHj8cE06KZ3c
+jq1fTRjEDv/N6xofqBSIHPsnvOVIP0Sp9bJkw5yO0H3oBfrqP0N2mjnwQknclz30
+t/LoXHcRrZTOH42pgG5ODZslqLNgKLXQHzRcglzNQPwYKYHigBiy+xsHxbIIXe1n
+R70PYKXisA0bhHTiV1Sa77dqQRdSkm0JzrNg58lHZYA1sVKTh0nRMQIDAQABo4Gv
+MIGsMA4GA1UdDwEB/wQEAwIFoDAMBgNVHRMBAf8EAjAAMB8GA1UdIwQYMBaAFDou
+x6fQdsT7szqJX2vyfmtmtuXiMGsGA1UdEQRkMGKCKGVsYXN0aWNzZWFyY2gtc2Ft
+cGxlLWVzLWh0dHAuZGVmYXVsdC5zdmOCNmVsYXN0aWNzZWFyY2gtc2FtcGxlLWVz
+LWh0dHAuZGVmYXVsdC5zdmMuY2x1c3Rlci5sb2NhbDANBgkqhkiG9w0BAQsFAAOC
+AQEAL0EBOx2vPXJSIjv8t0S2HkbCSerdDvGSNtkOrTizBtL7EwRSec6nes6OaWo6
+JYVNCP0Y+a4jQQrD9MkFKniKxluvLgbsHHsCnQC5tI5iwaOIZe+33pVyNksTc3CC
+l2s6Imqpvt6S3GyuWhcwWhwi3pK0ce9RqoO7GONHZmyuOaHGm1OxPeXJQYu7gTKg
+3hMjnNAzLOF1oOIrPKnkxfP4jdOrQE1oKk9QR7ScIKLVHJTJoogCM50I7yD7HnMT
+itkHwZhk5ptdA29P/OAcZheO5NOGlWJ6OeQl35A9SxgB3DSRTFORoEBfwPZB4ZLC
+zODbmFEr7N0FzCN6hU8PjcLLhg==
+-----END CERTIFICATE-----

libbeat/common/transport/tlscommon/testdata/unsigned_tls.crt

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDmjCCAoKgAwIBAgIfVNT1201IZeL6eZ5nBDNfdg7z5Rx3pSWKx48R5xEUMzAN
+BgkqhkiG9w0BAQUFADBmMQkwBwYDVQQGEwAxCTAHBgNVBAoMADEJMAcGA1UECwwA
+MRgwFgYDVQQDDA93d3cuZXhhbXBsZS5jb20xDzANBgkqhkiG9w0BCQEWADEYMBYG
+A1UEAwwPd3d3LmV4YW1wbGUuY29tMB4XDTIwMDcyMzIzNTE1NloXDTMwMDcyNDIz
+NTE1NlowTDEJMAcGA1UEBhMAMQkwBwYDVQQKDAAxCTAHBgNVBAsMADEYMBYGA1UE
+AwwPd3d3LmV4YW1wbGUuY29tMQ8wDQYJKoZIhvcNAQkBFgAwggEiMA0GCSqGSIb3
+DQEBAQUAA4IBDwAwggEKAoIBAQDUM6FCJj36941WQVrIKVjHCNKf0bdGiinfxGgL
+4SaUywGUo35mp70SFSpEcl3HE5B62Nab3axZ7N3oYeCD5iCJGPI0JWE3/gPdn5ao
+2xsGr1sKS+453dkmpDBEnTHNo7HjmvZIDIEzKHDW1QnfeeSGef9TKtVsnoDhGp+u
+mMndqBBUEXE/4tIrFuKZLQjxlchw6JQ6fpjmXxZKRCgXJq18/x9jfJnduYpb/DOc
+bXfQKZCbJeQdlZO9yxwwmzetZ/7kRZ774qvYtcHs+RVH5tPob1J/xgEoVpE4XAgp
+IrYrYCA159ejRJfb5Zs9Hx0AbatzFzTrHzod+jhfDpCh/NX3AgMBAAGjTzBNMB0G
+A1UdDgQWBBSuVtBMQ/Q6YHXDi6FQxOGzp+U5pTAfBgNVHSMEGDAWgBSuVtBMQ/Q6
+YHXDi6FQxOGzp+U5pTALBgNVHREEBDACggAwDQYJKoZIhvcNAQEFBQADggEBADNC
+AZZUgG4uXpDEIcWKT7gI8G+lbQJjIYciCNtqJsSpxOyN1Vs6tt8FXZBrVjxCa+Ik
+TpBZ0OxhY7Ry3veqVoeh9o8ASM8mvFE7y/CjZHtqxh5Q/Q1O5/UuMVy4ilT4hzEb
+jXvoH+gLCVxPcaV4cfqfWEWoW3RwfG+NtBq7ZnCl5o7ATDjDl1qe9sZ1rvIq7mLb
+Lk7lvNjqZU1PBRj6riW84Tv+yZc2kytqu61l8+NmphKwrKUgVUcbY37knmNIF2tB
+pl742yDqYtSu3ODWFtjNw2CZRGhTOcJMXasBFpjch0dz3uM++As0n9r63cNDssDi
+GQ6OHiviqMYraJMVFsc=
+-----END CERTIFICATE-----

libbeat/common/transport/tlscommon/tls_config.go

@@ -20,6 +20,9 @@ package tlscommon
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"time"
+
+	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
@@ -68,6 +71,10 @@ type TLSConfig struct {
 	// CASha256 is the CA certificate pin, this is used to validate the CA that will be used to trust
 	// the server certificate.
 	CASha256 []string
+
+	// time returns the current time as the number of seconds since the epoch.
+	// If time is nil, TLS uses time.Now.
+	time func() time.Time
 }
 
 // ToConfig generates a tls.Config object. Note, you must use BuildModuleConfig to generate a config with
@@ -78,18 +85,16 @@ func (c *TLSConfig) ToConfig() *tls.Config {
 	}
 
 	minVersion, maxVersion := extractMinMaxVersion(c.Versions)
+
+	// When we are using the CAsha256 pin to validate the CA used to validate the chain,
+	// or when we are using 'certificate' TLS verification mode, we add a custom callback
+	verifyPeerCertFn := makeVerifyPeerCertificate(c)
+
 	insecure := c.Verification != VerifyFull
-	if insecure {
+	if c.Verification == VerifyNone {
 		logp.NewLogger("tls").Warn("SSL/TLS verifications disabled.")
 	}
 
-	// When we are usign the CAsha256 pin to validate the CA used to validate the chain
-	// we add a custom callback.
-	var verifyPeerCertFn verifyPeerCertFunc
-	if len(c.CASha256) > 0 {
-		verifyPeerCertFn = MakeCAPinCallback(c.CASha256)
-	}
-
 	return &tls.Config{
 		MinVersion:            minVersion,
 		MaxVersion:            maxVersion,
@@ -102,6 +107,7 @@ func (c *TLSConfig) ToConfig() *tls.Config {
 		Renegotiation:         c.Renegotiation,
 		ClientAuth:            c.ClientAuth,
 		VerifyPeerCertificate: verifyPeerCertFn,
+		Time:                  c.time,
 	}
 }
 
@@ -116,3 +122,79 @@ func (c *TLSConfig) BuildModuleConfig(host string) *tls.Config {
 	config.ServerName = host
 	return config
 }
+
+// makeVerifyPeerCertificate creates the verification combination of checking certificate pins and skipping host name validation depending on the config
+func makeVerifyPeerCertificate(cfg *TLSConfig) verifyPeerCertFunc {
+	pin := len(cfg.CASha256) > 0
+	skipHostName := cfg.Verification == VerifyCertificate
+
+	if pin && !skipHostName {
+		return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+			return verifyCAPin(cfg.CASha256, verifiedChains)
+		}
+	}
+
+	if pin && skipHostName {
+		return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+			_, _, err := verifyCertificateExceptServerName(rawCerts, cfg)
+			if err != nil {
+				return err
+			}
+			return verifyCAPin(cfg.CASha256, verifiedChains)
+		}
+	}
+
+	if !pin && skipHostName {
+		return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+			_, _, err := verifyCertificateExceptServerName(rawCerts, cfg)
+			return err
+		}
+	}
+
+	return nil
+}
+
+// verifyCertificateExceptServerName is a TLS Certificate verification utility method that verifies that the provided
+// certificate chain is valid and is signed by one of the root CAs in the provided tls.Config. It is intended to be
+// as similar as possible to the default verify, but does not verify that the provided certificate matches the
+// ServerName in the tls.Config.
+func verifyCertificateExceptServerName(
+	rawCerts [][]byte,
+	c *TLSConfig,
+) ([]*x509.Certificate, [][]*x509.Certificate, error) {
+	// this is where we're a bit suboptimal, as we have to re-parse the certificates that have been presented
+	// during the handshake.
+	// the verification code here is taken from verifyServerCertificate in crypto/tls/handshake_client.go:824
+	certs := make([]*x509.Certificate, len(rawCerts))
+	for i, asn1Data := range rawCerts {
+		cert, err := x509.ParseCertificate(asn1Data)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "tls: failed to parse certificate from server")
+		}
+		certs[i] = cert
+	}
+
+	var t time.Time
+	if c.time != nil {
+		t = c.time()
+	} else {
+		t = time.Now()
+	}
+
+	// DNSName omitted in VerifyOptions in order to skip ServerName verification
+	opts := x509.VerifyOptions{
+		Roots:         c.RootCAs,
+		CurrentTime:   t,
+		Intermediates: x509.NewCertPool(),
+	}
+
+	for _, cert := range certs[1:] {
+		opts.Intermediates.AddCert(cert)
+	}
+
+	headCert := certs[0]
+
+	// defer to the default verification performed
+	chains, err := headCert.Verify(opts)
+	return certs, chains, err
+}

libbeat/common/transport/tlscommon/tls_config_test.go

@@ -0,0 +1,110 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package tlscommon
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// This does not actually test that it ignores the server name because no part of the func even consumes the server name
+func Test_verifyCertificateExceptServerName(t *testing.T) {
+
+	tests := []struct {
+		name    string
+		ca      string
+		chain   string
+		cert    string
+		time    func() time.Time
+		wantErr bool
+	}{
+		{
+			name: "happy path",
+			// a CA for morello.ovh valid from August 9 2019 to 2029
+			ca: "ca.crt",
+			// a cert signed by morello.ovh that expired in nov 2019
+			cert: "tls.crt",
+			time: func() time.Time {
+				layout := "2006-01-02"
+				t, _ := time.Parse(layout, "2019-10-01")
+				return t
+			},
+			wantErr: false,
+		},
+		{
+			name: "cert not signed by CA",
+			ca:   "ca.crt",
+			// a self-signed cert for www.example.com valid from July 23 2020 to 2030
+			cert: "unsigned_tls.crt",
+			time: func() time.Time {
+				layout := "2006-01-02"
+				t, _ := time.Parse(layout, "2020-07-24")
+				return t
+			},
+			wantErr: true,
+		},
+		{
+			name:    "cert expired",
+			ca:      "ca.crt",
+			cert:    "tls.crt",
+			wantErr: true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := &TLSConfig{time: tc.time}
+			// load the CA
+			if tc.ca != "" {
+				ca := loadFileBytes(tc.ca)
+				caCertPool := x509.NewCertPool()
+				caCertPool.AppendCertsFromPEM(ca)
+				cfg.RootCAs = caCertPool
+			}
+
+			// load the cert
+			rawCerts := [][]byte{}
+			if tc.cert != "" {
+				pemCert := loadFileBytes(tc.cert)
+				block, _ := pem.Decode(pemCert)
+				rawCerts = append(rawCerts, block.Bytes)
+			}
+
+			_, _, got := verifyCertificateExceptServerName(rawCerts, cfg)
+			if tc.wantErr {
+				assert.Error(t, got)
+			} else {
+				assert.NoError(t, got)
+			}
+		})
+	}
+}
+
+func loadFileBytes(fileName string) []byte {
+	contents, err := ioutil.ReadFile(filepath.Join("testdata", fileName))
+	if err != nil {
+		panic(err)
+	}
+	return contents
+}