diff --git a/libbeat/common/transport/httpcommon/httpcommon.go b/libbeat/common/transport/httpcommon/httpcommon.go index 4368b0e12cb0..6ba839b718c7 100644 --- a/libbeat/common/transport/httpcommon/httpcommon.go +++ b/libbeat/common/transport/httpcommon/httpcommon.go @@ -21,215 +21,11 @@ package httpcommon import ( "net/http" - "time" - "go.elastic.co/apm/module/apmhttp" - "golang.org/x/net/http2" - - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" - "github.com/elastic/beats/v7/libbeat/logp" -) - -// HTTPTransportSettings provides common HTTP settings for HTTP clients. -type HTTPTransportSettings struct { - // TLS provides ssl/tls setup settings - TLS *tlscommon.Config `config:"ssl" yaml:"ssl,omitempty" json:"ssl,omitempty"` - - // Timeout configures the `(http.Transport).Timeout`. - Timeout time.Duration `config:"timeout" yaml:"timeout,omitempty" json:"timeout,omitempty"` - - Proxy HTTPClientProxySettings `config:",inline" yaml:",inline"` - - // TODO: Add more settings: - // - DisableKeepAlive - // - MaxIdleConns - // - IdleConnTimeout - // - ResponseHeaderTimeout - // - ConnectionTimeout (currently 'Timeout' is used for both) -} - -// WithKeepaliveSettings options can be used to modify the Keepalive -type WithKeepaliveSettings struct { - Disable bool - MaxIdleConns int - MaxIdleConnsPerHost int - IdleConnTimeout time.Duration -} - -var _ httpTransportOption = WithKeepaliveSettings{} - -const defaultHTTPTimeout = 90 * time.Second - -type ( - // TransportOption are applied to the http.RoundTripper to be build - // from HTTPTransportSettings. - TransportOption interface{ sealTransportOption() } - - extraSettings struct { - logger *logp.Logger - http2 bool - } - - dialerOption interface { - TransportOption - baseDialer() transport.Dialer - } - dialerModOption interface { - TransportOption - applyDialer(*HTTPTransportSettings, transport.Dialer) transport.Dialer - } - httpTransportOption interface { - TransportOption - applyTransport(*HTTPTransportSettings, *http.Transport) - } - roundTripperOption interface { - TransportOption - applyRoundTripper(*HTTPTransportSettings, http.RoundTripper) http.RoundTripper - } - extraOption interface { - TransportOption - applyExtra(*extraSettings) - } ) -type baseDialerFunc func() transport.Dialer - -var _ dialerOption = baseDialerFunc(nil) - -func (baseDialerFunc) sealTransportOption() {} -func (fn baseDialerFunc) baseDialer() transport.Dialer { - return fn() -} - -type dialerOptFunc func(transport.Dialer) transport.Dialer - -var _ dialerModOption = dialerOptFunc(nil) - -func (dialerOptFunc) sealTransportOption() {} -func (fn dialerOptFunc) applyDialer(_ *HTTPTransportSettings, d transport.Dialer) transport.Dialer { - return fn(d) - -} - -type transportOptFunc func(*HTTPTransportSettings, *http.Transport) - -var _ httpTransportOption = transportOptFunc(nil) - -func (transportOptFunc) sealTransportOption() {} -func (fn transportOptFunc) applyTransport(s *HTTPTransportSettings, t *http.Transport) { - fn(s, t) -} - -type rtOptFunc func(http.RoundTripper) http.RoundTripper - -var _ roundTripperOption = rtOptFunc(nil) - -func (rtOptFunc) sealTransportOption() {} -func (fn rtOptFunc) applyRoundTripper(_ *HTTPTransportSettings, rt http.RoundTripper) http.RoundTripper { - return fn(rt) -} - -type extraOptionFunc func(*extraSettings) - -func (extraOptionFunc) sealTransportOption() {} -func (fn extraOptionFunc) applyExtra(s *extraSettings) { fn(s) } - -// DefaultHTTPTransportSettings returns the default HTTP transport setting. -func DefaultHTTPTransportSettings() HTTPTransportSettings { - return HTTPTransportSettings{ - Proxy: DefaultHTTPClientProxySettings(), - Timeout: defaultHTTPTimeout, - } -} - -// Unpack reads a config object into the settings. -func (settings *HTTPTransportSettings) Unpack(cfg *common.Config) error { - tmp := struct { - TLS *tlscommon.Config `config:"ssl"` - Timeout time.Duration `config:"timeout"` - }{Timeout: settings.Timeout} - - if err := cfg.Unpack(&tmp); err != nil { - return err - } - - var proxy HTTPClientProxySettings - if err := cfg.Unpack(&proxy); err != nil { - return err - } - - _, err := tlscommon.LoadTLSConfig(tmp.TLS) - if err != nil { - return err - } - - *settings = HTTPTransportSettings{ - TLS: tmp.TLS, - Timeout: tmp.Timeout, - Proxy: proxy, - } - return nil -} - -// RoundTripper creates a http.RoundTripper for use with http.Client. -// -// The dialers will registers with stats if given. Stats is used to collect metrics for io errors, -// bytes in, and bytes out. -func (settings *HTTPTransportSettings) RoundTripper(opts ...TransportOption) (http.RoundTripper, error) { - var dialer transport.Dialer - - var extra extraSettings - for _, opt := range opts { - if opt, ok := opt.(extraOption); ok { - opt.applyExtra(&extra) - } - } - - for _, opt := range opts { - if dialOpt, ok := opt.(dialerOption); ok { - dialer = dialOpt.baseDialer() - } - } - - if dialer == nil { - dialer = transport.NetDialer(settings.Timeout) - } - - tls, err := tlscommon.LoadTLSConfig(settings.TLS) - if err != nil { - return nil, err - } - - tlsDialer := transport.TLSDialer(dialer, tls, settings.Timeout) - for _, opt := range opts { - if dialOpt, ok := opt.(dialerModOption); ok { - dialer = dialOpt.applyDialer(settings, dialer) - tlsDialer = dialOpt.applyDialer(settings, tlsDialer) - } - } - - if logger := extra.logger; logger != nil { - dialer = transport.LoggingDialer(dialer, logger) - tlsDialer = transport.LoggingDialer(tlsDialer, logger) - } - - var rt http.RoundTripper - if extra.http2 { - rt, err = settings.http2RoundTripper(tls, dialer, tlsDialer, opts...) - } else { - rt, err = settings.httpRoundTripper(tls, dialer, tlsDialer, opts...) - } - - for _, opt := range opts { - if rtOpt, ok := opt.(roundTripperOption); ok { - rt = rtOpt.applyRoundTripper(settings, rt) - } - } - return rt, nil -} - func (settings *HTTPTransportSettings) httpRoundTripper( tls *tlscommon.TLSConfig, dialer, tlsDialer transport.Dialer, @@ -257,128 +53,3 @@ func (settings *HTTPTransportSettings) httpRoundTripper( return t, nil } - -func (settings *HTTPTransportSettings) http2RoundTripper( - tls *tlscommon.TLSConfig, - dialer, tlsDialer transport.Dialer, - opts ...TransportOption, -) (*http2.Transport, error) { - t1, err := settings.httpRoundTripper(tls, dialer, tlsDialer, opts...) - if err != nil { - return nil, err - } - - t2, err := http2.ConfigureTransports(t1) - if err != nil { - return nil, err - } - - t2.AllowHTTP = true - return t2, nil -} - -// Client creates a new http.Client with configured Transport. The transport is -// instrumented using apmhttp.WrapRoundTripper. -func (settings HTTPTransportSettings) Client(opts ...TransportOption) (*http.Client, error) { - rt, err := settings.RoundTripper(opts...) - if err != nil { - return nil, err - } - - return &http.Client{Transport: rt, Timeout: settings.Timeout}, nil -} - -func (opts WithKeepaliveSettings) sealTransportOption() {} -func (opts WithKeepaliveSettings) applyTransport(_ *HTTPTransportSettings, t *http.Transport) { - t.DisableKeepAlives = opts.Disable - if opts.IdleConnTimeout != 0 { - t.IdleConnTimeout = opts.IdleConnTimeout - } - if opts.MaxIdleConns != 0 { - t.MaxIdleConns = opts.MaxIdleConns - } - if opts.MaxIdleConnsPerHost != 0 { - t.MaxIdleConnsPerHost = opts.MaxIdleConnsPerHost - } -} - -// WithBaseDialer configures the dialer used for TCP and TLS connections. -func WithBaseDialer(d transport.Dialer) TransportOption { - return baseDialerFunc(func() transport.Dialer { - return d - }) -} - -// WithIOStats instruments the RoundTripper dialers with the given statser, such -// that bytes in, bytes out, and errors can be monitored. -func WithIOStats(stats transport.IOStatser) TransportOption { - return dialerOptFunc(func(d transport.Dialer) transport.Dialer { - if stats == nil { - return d - } - return transport.StatsDialer(d, stats) - }) -} - -// WithTransportFunc register a custom function that is used to apply -// custom changes to the net.Transport, when the Client is build. -func WithTransportFunc(fn func(*http.Transport)) TransportOption { - return transportOptFunc(func(_ *HTTPTransportSettings, t *http.Transport) { - fn(t) - }) -} - -// WithHTTP2Only will ensure that a HTTP 2 only roundtripper is created. -func WithHTTP2Only(b bool) TransportOption { - return extraOptionFunc(func(settings *extraSettings) { - settings.http2 = b - }) -} - -// WithForceAttemptHTTP2 sets the `http.Tansport.ForceAttemptHTTP2` field. -func WithForceAttemptHTTP2(b bool) TransportOption { - return transportOptFunc(func(settings *HTTPTransportSettings, t *http.Transport) { - t.ForceAttemptHTTP2 = b - }) -} - -// WithNOProxy disables the configured proxy. Proxy environment variables -// like HTTP_PROXY and HTTPS_PROXY will have no affect. -func WithNOProxy() TransportOption { - return transportOptFunc(func(s *HTTPTransportSettings, t *http.Transport) { - t.Proxy = nil - }) -} - -// WithoutProxyEnvironmentVariables disables support for the HTTP_PROXY, HTTPS_PROXY and -// NO_PROXY envionrment variables. Explicitely configured proxy URLs will still applied. -func WithoutProxyEnvironmentVariables() TransportOption { - return transportOptFunc(func(settings *HTTPTransportSettings, t *http.Transport) { - if settings.Proxy.Disable || settings.Proxy.URL == nil { - t.Proxy = nil - } - }) -} - -// WithModRoundtripper allows customization of the roundtipper. -func WithModRoundtripper(w func(http.RoundTripper) http.RoundTripper) TransportOption { - return rtOptFunc(w) -} - -var withAPMHTTPRountTripper = WithModRoundtripper(func(rt http.RoundTripper) http.RoundTripper { - return apmhttp.WrapRoundTripper(rt) -}) - -// WithAPMHTTPInstrumentation insruments the HTTP client via apmhttp.WrapRoundTripper. -// Custom APM round tripper wrappers can be configured via WithModRoundtripper. -func WithAPMHTTPInstrumentation() TransportOption { - return withAPMHTTPRountTripper -} - -// WithLogger sets the internal logger that will be used to log dial or TCP level errors. -// Logging at the connection level will only happen if the logger has been set. -func WithLogger(logger *logp.Logger) TransportOption { - return extraOptionFunc(func(s *extraSettings) { - s.logger = logger - }) -} diff --git a/libbeat/common/transport/httpcommon/httpcommon_common.go b/libbeat/common/transport/httpcommon/httpcommon_common.go new file mode 100644 index 000000000000..fed0c8c48330 --- /dev/null +++ b/libbeat/common/transport/httpcommon/httpcommon_common.go @@ -0,0 +1,354 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package httpcommon + +import ( + "net/http" + "time" + + "go.elastic.co/apm/module/apmhttp" + "golang.org/x/net/http2" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/logp" +) + +// HTTPTransportSettings provides common HTTP settings for HTTP clients. +type HTTPTransportSettings struct { + // TLS provides ssl/tls setup settings + TLS *tlscommon.Config `config:"ssl" yaml:"ssl,omitempty" json:"ssl,omitempty"` + + // Timeout configures the `(http.Transport).Timeout`. + Timeout time.Duration `config:"timeout" yaml:"timeout,omitempty" json:"timeout,omitempty"` + + Proxy HTTPClientProxySettings `config:",inline" yaml:",inline"` + + // TODO: Add more settings: + // - DisableKeepAlive + // - MaxIdleConns + // - IdleConnTimeout + // - ResponseHeaderTimeout + // - ConnectionTimeout (currently 'Timeout' is used for both) +} + +// WithKeepaliveSettings options can be used to modify the Keepalive +type WithKeepaliveSettings struct { + Disable bool + MaxIdleConns int + MaxIdleConnsPerHost int + IdleConnTimeout time.Duration +} + +var _ httpTransportOption = WithKeepaliveSettings{} + +const defaultHTTPTimeout = 90 * time.Second + +type ( + // TransportOption are applied to the http.RoundTripper to be build + // from HTTPTransportSettings. + TransportOption interface{ sealTransportOption() } + + extraSettings struct { + logger *logp.Logger + http2 bool + } + + dialerOption interface { + TransportOption + baseDialer() transport.Dialer + } + dialerModOption interface { + TransportOption + applyDialer(*HTTPTransportSettings, transport.Dialer) transport.Dialer + } + httpTransportOption interface { + TransportOption + applyTransport(*HTTPTransportSettings, *http.Transport) + } + roundTripperOption interface { + TransportOption + applyRoundTripper(*HTTPTransportSettings, http.RoundTripper) http.RoundTripper + } + extraOption interface { + TransportOption + applyExtra(*extraSettings) + } +) + +type baseDialerFunc func() transport.Dialer + +var _ dialerOption = baseDialerFunc(nil) + +func (baseDialerFunc) sealTransportOption() {} +func (fn baseDialerFunc) baseDialer() transport.Dialer { + return fn() +} + +type dialerOptFunc func(transport.Dialer) transport.Dialer + +var _ dialerModOption = dialerOptFunc(nil) + +func (dialerOptFunc) sealTransportOption() {} +func (fn dialerOptFunc) applyDialer(_ *HTTPTransportSettings, d transport.Dialer) transport.Dialer { + return fn(d) + +} + +type transportOptFunc func(*HTTPTransportSettings, *http.Transport) + +var _ httpTransportOption = transportOptFunc(nil) + +func (transportOptFunc) sealTransportOption() {} +func (fn transportOptFunc) applyTransport(s *HTTPTransportSettings, t *http.Transport) { + fn(s, t) +} + +type rtOptFunc func(http.RoundTripper) http.RoundTripper + +var _ roundTripperOption = rtOptFunc(nil) + +func (rtOptFunc) sealTransportOption() {} +func (fn rtOptFunc) applyRoundTripper(_ *HTTPTransportSettings, rt http.RoundTripper) http.RoundTripper { + return fn(rt) +} + +type extraOptionFunc func(*extraSettings) + +func (extraOptionFunc) sealTransportOption() {} +func (fn extraOptionFunc) applyExtra(s *extraSettings) { fn(s) } + +// DefaultHTTPTransportSettings returns the default HTTP transport setting. +func DefaultHTTPTransportSettings() HTTPTransportSettings { + return HTTPTransportSettings{ + Proxy: DefaultHTTPClientProxySettings(), + Timeout: defaultHTTPTimeout, + } +} + +// Unpack reads a config object into the settings. +func (settings *HTTPTransportSettings) Unpack(cfg *common.Config) error { + tmp := struct { + TLS *tlscommon.Config `config:"ssl"` + Timeout time.Duration `config:"timeout"` + }{Timeout: settings.Timeout} + + if err := cfg.Unpack(&tmp); err != nil { + return err + } + + var proxy HTTPClientProxySettings + if err := cfg.Unpack(&proxy); err != nil { + return err + } + + _, err := tlscommon.LoadTLSConfig(tmp.TLS) + if err != nil { + return err + } + + *settings = HTTPTransportSettings{ + TLS: tmp.TLS, + Timeout: tmp.Timeout, + Proxy: proxy, + } + return nil +} + +// RoundTripper creates a http.RoundTripper for use with http.Client. +// +// The dialers will registers with stats if given. Stats is used to collect metrics for io errors, +// bytes in, and bytes out. +func (settings *HTTPTransportSettings) RoundTripper(opts ...TransportOption) (http.RoundTripper, error) { + var dialer transport.Dialer + + var extra extraSettings + for _, opt := range opts { + if opt, ok := opt.(extraOption); ok { + opt.applyExtra(&extra) + } + } + + for _, opt := range opts { + if dialOpt, ok := opt.(dialerOption); ok { + dialer = dialOpt.baseDialer() + } + } + + if dialer == nil { + dialer = transport.NetDialer(settings.Timeout) + } + + tls, err := tlscommon.LoadTLSConfig(settings.TLS) + if err != nil { + return nil, err + } + + tlsDialer := transport.TLSDialer(dialer, tls, settings.Timeout) + for _, opt := range opts { + if dialOpt, ok := opt.(dialerModOption); ok { + dialer = dialOpt.applyDialer(settings, dialer) + tlsDialer = dialOpt.applyDialer(settings, tlsDialer) + } + } + + if logger := extra.logger; logger != nil { + dialer = transport.LoggingDialer(dialer, logger) + tlsDialer = transport.LoggingDialer(tlsDialer, logger) + } + + var rt http.RoundTripper + if extra.http2 { + rt, err = settings.http2RoundTripper(tls, dialer, tlsDialer, opts...) + } else { + rt, err = settings.httpRoundTripper(tls, dialer, tlsDialer, opts...) + } + + for _, opt := range opts { + if rtOpt, ok := opt.(roundTripperOption); ok { + rt = rtOpt.applyRoundTripper(settings, rt) + } + } + return rt, nil +} + +func (settings *HTTPTransportSettings) http2RoundTripper( + tls *tlscommon.TLSConfig, + dialer, tlsDialer transport.Dialer, + opts ...TransportOption, +) (*http2.Transport, error) { + t1, err := settings.httpRoundTripper(tls, dialer, tlsDialer, opts...) + if err != nil { + return nil, err + } + + t2, err := http2.ConfigureTransports(t1) + if err != nil { + return nil, err + } + + t2.AllowHTTP = true + return t2, nil +} + +// Client creates a new http.Client with configured Transport. The transport is +// instrumented using apmhttp.WrapRoundTripper. +func (settings HTTPTransportSettings) Client(opts ...TransportOption) (*http.Client, error) { + rt, err := settings.RoundTripper(opts...) + if err != nil { + return nil, err + } + + return &http.Client{Transport: rt, Timeout: settings.Timeout}, nil +} + +func (opts WithKeepaliveSettings) sealTransportOption() {} +func (opts WithKeepaliveSettings) applyTransport(_ *HTTPTransportSettings, t *http.Transport) { + t.DisableKeepAlives = opts.Disable + if opts.IdleConnTimeout != 0 { + t.IdleConnTimeout = opts.IdleConnTimeout + } + if opts.MaxIdleConns != 0 { + t.MaxIdleConns = opts.MaxIdleConns + } + if opts.MaxIdleConnsPerHost != 0 { + t.MaxIdleConnsPerHost = opts.MaxIdleConnsPerHost + } +} + +// WithBaseDialer configures the dialer used for TCP and TLS connections. +func WithBaseDialer(d transport.Dialer) TransportOption { + return baseDialerFunc(func() transport.Dialer { + return d + }) +} + +// WithIOStats instruments the RoundTripper dialers with the given statser, such +// that bytes in, bytes out, and errors can be monitored. +func WithIOStats(stats transport.IOStatser) TransportOption { + return dialerOptFunc(func(d transport.Dialer) transport.Dialer { + if stats == nil { + return d + } + return transport.StatsDialer(d, stats) + }) +} + +// WithTransportFunc register a custom function that is used to apply +// custom changes to the net.Transport, when the Client is build. +func WithTransportFunc(fn func(*http.Transport)) TransportOption { + return transportOptFunc(func(_ *HTTPTransportSettings, t *http.Transport) { + fn(t) + }) +} + +// WithHTTP2Only will ensure that a HTTP 2 only roundtripper is created. +func WithHTTP2Only(b bool) TransportOption { + return extraOptionFunc(func(settings *extraSettings) { + settings.http2 = b + }) +} + +// WithForceAttemptHTTP2 sets the `http.Tansport.ForceAttemptHTTP2` field. +func WithForceAttemptHTTP2(b bool) TransportOption { + return transportOptFunc(func(settings *HTTPTransportSettings, t *http.Transport) { + t.ForceAttemptHTTP2 = b + }) +} + +// WithNOProxy disables the configured proxy. Proxy environment variables +// like HTTP_PROXY and HTTPS_PROXY will have no affect. +func WithNOProxy() TransportOption { + return transportOptFunc(func(s *HTTPTransportSettings, t *http.Transport) { + t.Proxy = nil + }) +} + +// WithoutProxyEnvironmentVariables disables support for the HTTP_PROXY, HTTPS_PROXY and +// NO_PROXY envionrment variables. Explicitely configured proxy URLs will still applied. +func WithoutProxyEnvironmentVariables() TransportOption { + return transportOptFunc(func(settings *HTTPTransportSettings, t *http.Transport) { + if settings.Proxy.Disable || settings.Proxy.URL == nil { + t.Proxy = nil + } + }) +} + +// WithModRoundtripper allows customization of the roundtipper. +func WithModRoundtripper(w func(http.RoundTripper) http.RoundTripper) TransportOption { + return rtOptFunc(w) +} + +var withAPMHTTPRountTripper = WithModRoundtripper(func(rt http.RoundTripper) http.RoundTripper { + return apmhttp.WrapRoundTripper(rt) +}) + +// WithAPMHTTPInstrumentation insruments the HTTP client via apmhttp.WrapRoundTripper. +// Custom APM round tripper wrappers can be configured via WithModRoundtripper. +func WithAPMHTTPInstrumentation() TransportOption { + return withAPMHTTPRountTripper +} + +// WithLogger sets the internal logger that will be used to log dial or TCP level errors. +// Logging at the connection level will only happen if the logger has been set. +func WithLogger(logger *logp.Logger) TransportOption { + return extraOptionFunc(func(s *extraSettings) { + s.logger = logger + }) +} diff --git a/libbeat/common/transport/httpcommon/httpcommon_legacy.go b/libbeat/common/transport/httpcommon/httpcommon_legacy.go index 00293d3e6a89..c3e1e2e31a7b 100644 --- a/libbeat/common/transport/httpcommon/httpcommon_legacy.go +++ b/libbeat/common/transport/httpcommon/httpcommon_legacy.go @@ -21,215 +21,11 @@ package httpcommon import ( "net/http" - "time" - "go.elastic.co/apm/module/apmhttp" - "golang.org/x/net/http2" - - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" - "github.com/elastic/beats/v7/libbeat/logp" -) - -// HTTPTransportSettings provides common HTTP settings for HTTP clients. -type HTTPTransportSettings struct { - // TLS provides ssl/tls setup settings - TLS *tlscommon.Config `config:"ssl" yaml:"ssl,omitempty" json:"ssl,omitempty"` - - // Timeout configures the `(http.Transport).Timeout`. - Timeout time.Duration `config:"timeout" yaml:"timeout,omitempty" json:"timeout,omitempty"` - - Proxy HTTPClientProxySettings `config:",inline" yaml:",inline"` - - // TODO: Add more settings: - // - DisableKeepAlive - // - MaxIdleConns - // - IdleConnTimeout - // - ResponseHeaderTimeout - // - ConnectionTimeout (currently 'Timeout' is used for both) -} - -// WithKeepaliveSettings options can be used to modify the Keepalive -type WithKeepaliveSettings struct { - Disable bool - MaxIdleConns int - MaxIdleConnsPerHost int - IdleConnTimeout time.Duration -} - -var _ httpTransportOption = WithKeepaliveSettings{} - -const defaultHTTPTimeout = 90 * time.Second - -type ( - // TransportOption are applied to the http.RoundTripper to be build - // from HTTPTransportSettings. - TransportOption interface{ sealTransportOption() } - - extraSettings struct { - logger *logp.Logger - http2 bool - } - - dialerOption interface { - TransportOption - baseDialer() transport.Dialer - } - dialerModOption interface { - TransportOption - applyDialer(*HTTPTransportSettings, transport.Dialer) transport.Dialer - } - httpTransportOption interface { - TransportOption - applyTransport(*HTTPTransportSettings, *http.Transport) - } - roundTripperOption interface { - TransportOption - applyRoundTripper(*HTTPTransportSettings, http.RoundTripper) http.RoundTripper - } - extraOption interface { - TransportOption - applyExtra(*extraSettings) - } ) -type baseDialerFunc func() transport.Dialer - -var _ dialerOption = baseDialerFunc(nil) - -func (baseDialerFunc) sealTransportOption() {} -func (fn baseDialerFunc) baseDialer() transport.Dialer { - return fn() -} - -type dialerOptFunc func(transport.Dialer) transport.Dialer - -var _ dialerModOption = dialerOptFunc(nil) - -func (dialerOptFunc) sealTransportOption() {} -func (fn dialerOptFunc) applyDialer(_ *HTTPTransportSettings, d transport.Dialer) transport.Dialer { - return fn(d) - -} - -type transportOptFunc func(*HTTPTransportSettings, *http.Transport) - -var _ httpTransportOption = transportOptFunc(nil) - -func (transportOptFunc) sealTransportOption() {} -func (fn transportOptFunc) applyTransport(s *HTTPTransportSettings, t *http.Transport) { - fn(s, t) -} - -type rtOptFunc func(http.RoundTripper) http.RoundTripper - -var _ roundTripperOption = rtOptFunc(nil) - -func (rtOptFunc) sealTransportOption() {} -func (fn rtOptFunc) applyRoundTripper(_ *HTTPTransportSettings, rt http.RoundTripper) http.RoundTripper { - return fn(rt) -} - -type extraOptionFunc func(*extraSettings) - -func (extraOptionFunc) sealTransportOption() {} -func (fn extraOptionFunc) applyExtra(s *extraSettings) { fn(s) } - -// DefaultHTTPTransportSettings returns the default HTTP transport setting. -func DefaultHTTPTransportSettings() HTTPTransportSettings { - return HTTPTransportSettings{ - Proxy: DefaultHTTPClientProxySettings(), - Timeout: defaultHTTPTimeout, - } -} - -// Unpack reads a config object into the settings. -func (settings *HTTPTransportSettings) Unpack(cfg *common.Config) error { - tmp := struct { - TLS *tlscommon.Config `config:"ssl"` - Timeout time.Duration `config:"timeout"` - }{Timeout: settings.Timeout} - - if err := cfg.Unpack(&tmp); err != nil { - return err - } - - var proxy HTTPClientProxySettings - if err := cfg.Unpack(&proxy); err != nil { - return err - } - - _, err := tlscommon.LoadTLSConfig(tmp.TLS) - if err != nil { - return err - } - - *settings = HTTPTransportSettings{ - TLS: tmp.TLS, - Timeout: tmp.Timeout, - Proxy: proxy, - } - return nil -} - -// RoundTripper creates a http.RoundTripper for use with http.Client. -// -// The dialers will registers with stats if given. Stats is used to collect metrics for io errors, -// bytes in, and bytes out. -func (settings *HTTPTransportSettings) RoundTripper(opts ...TransportOption) (http.RoundTripper, error) { - var dialer transport.Dialer - - var extra extraSettings - for _, opt := range opts { - if opt, ok := opt.(extraOption); ok { - opt.applyExtra(&extra) - } - } - - for _, opt := range opts { - if dialOpt, ok := opt.(dialerOption); ok { - dialer = dialOpt.baseDialer() - } - } - - if dialer == nil { - dialer = transport.NetDialer(settings.Timeout) - } - - tls, err := tlscommon.LoadTLSConfig(settings.TLS) - if err != nil { - return nil, err - } - - tlsDialer := transport.TLSDialer(dialer, tls, settings.Timeout) - for _, opt := range opts { - if dialOpt, ok := opt.(dialerModOption); ok { - dialer = dialOpt.applyDialer(settings, dialer) - tlsDialer = dialOpt.applyDialer(settings, tlsDialer) - } - } - - if logger := extra.logger; logger != nil { - dialer = transport.LoggingDialer(dialer, logger) - tlsDialer = transport.LoggingDialer(tlsDialer, logger) - } - - var rt http.RoundTripper - if extra.http2 { - rt, err = settings.http2RoundTripper(tls, dialer, tlsDialer, opts...) - } else { - rt, err = settings.httpRoundTripper(tls, dialer, tlsDialer, opts...) - } - - for _, opt := range opts { - if rtOpt, ok := opt.(roundTripperOption); ok { - rt = rtOpt.applyRoundTripper(settings, rt) - } - } - return rt, nil -} - func (settings *HTTPTransportSettings) httpRoundTripper( tls *tlscommon.TLSConfig, dialer, tlsDialer transport.Dialer, @@ -256,128 +52,3 @@ func (settings *HTTPTransportSettings) httpRoundTripper( return t, nil } - -func (settings *HTTPTransportSettings) http2RoundTripper( - tls *tlscommon.TLSConfig, - dialer, tlsDialer transport.Dialer, - opts ...TransportOption, -) (*http2.Transport, error) { - t1, err := settings.httpRoundTripper(tls, dialer, tlsDialer, opts...) - if err != nil { - return nil, err - } - - t2, err := http2.ConfigureTransports(t1) - if err != nil { - return nil, err - } - - t2.AllowHTTP = true - return t2, nil -} - -// Client creates a new http.Client with configured Transport. The transport is -// instrumented using apmhttp.WrapRoundTripper. -func (settings HTTPTransportSettings) Client(opts ...TransportOption) (*http.Client, error) { - rt, err := settings.RoundTripper(opts...) - if err != nil { - return nil, err - } - - return &http.Client{Transport: rt, Timeout: settings.Timeout}, nil -} - -func (opts WithKeepaliveSettings) sealTransportOption() {} -func (opts WithKeepaliveSettings) applyTransport(_ *HTTPTransportSettings, t *http.Transport) { - t.DisableKeepAlives = opts.Disable - if opts.IdleConnTimeout != 0 { - t.IdleConnTimeout = opts.IdleConnTimeout - } - if opts.MaxIdleConns != 0 { - t.MaxIdleConns = opts.MaxIdleConns - } - if opts.MaxIdleConnsPerHost != 0 { - t.MaxIdleConnsPerHost = opts.MaxIdleConnsPerHost - } -} - -// WithBaseDialer configures the dialer used for TCP and TLS connections. -func WithBaseDialer(d transport.Dialer) TransportOption { - return baseDialerFunc(func() transport.Dialer { - return d - }) -} - -// WithIOStats instruments the RoundTripper dialers with the given statser, such -// that bytes in, bytes out, and errors can be monitored. -func WithIOStats(stats transport.IOStatser) TransportOption { - return dialerOptFunc(func(d transport.Dialer) transport.Dialer { - if stats == nil { - return d - } - return transport.StatsDialer(d, stats) - }) -} - -// WithTransportFunc register a custom function that is used to apply -// custom changes to the net.Transport, when the Client is build. -func WithTransportFunc(fn func(*http.Transport)) TransportOption { - return transportOptFunc(func(_ *HTTPTransportSettings, t *http.Transport) { - fn(t) - }) -} - -// WithHTTP2Only will ensure that a HTTP 2 only roundtripper is created. -func WithHTTP2Only(b bool) TransportOption { - return extraOptionFunc(func(settings *extraSettings) { - settings.http2 = b - }) -} - -// WithForceAttemptHTTP2 sets the `http.Tansport.ForceAttemptHTTP2` field. -func WithForceAttemptHTTP2(b bool) TransportOption { - return transportOptFunc(func(settings *HTTPTransportSettings, t *http.Transport) { - t.ForceAttemptHTTP2 = b - }) -} - -// WithNOProxy disables the configured proxy. Proxy environment variables -// like HTTP_PROXY and HTTPS_PROXY will have no affect. -func WithNOProxy() TransportOption { - return transportOptFunc(func(s *HTTPTransportSettings, t *http.Transport) { - t.Proxy = nil - }) -} - -// WithoutProxyEnvironmentVariables disables support for the HTTP_PROXY, HTTPS_PROXY and -// NO_PROXY envionrment variables. Explicitely configured proxy URLs will still applied. -func WithoutProxyEnvironmentVariables() TransportOption { - return transportOptFunc(func(settings *HTTPTransportSettings, t *http.Transport) { - if settings.Proxy.Disable || settings.Proxy.URL == nil { - t.Proxy = nil - } - }) -} - -// WithModRoundtripper allows customization of the roundtipper. -func WithModRoundtripper(w func(http.RoundTripper) http.RoundTripper) TransportOption { - return rtOptFunc(w) -} - -var withAPMHTTPRountTripper = WithModRoundtripper(func(rt http.RoundTripper) http.RoundTripper { - return apmhttp.WrapRoundTripper(rt) -}) - -// WithAPMHTTPInstrumentation insruments the HTTP client via apmhttp.WrapRoundTripper. -// Custom APM round tripper wrappers can be configured via WithModRoundtripper. -func WithAPMHTTPInstrumentation() TransportOption { - return withAPMHTTPRountTripper -} - -// WithLogger sets the internal logger that will be used to log dial or TCP level errors. -// Logging at the connection level will only happen if the logger has been set. -func WithLogger(logger *logp.Logger) TransportOption { - return extraOptionFunc(func(s *extraSettings) { - s.logger = logger - }) -} diff --git a/libbeat/common/transport/tls.go b/libbeat/common/transport/tls.go index 99dacd61461c..b32268497567 100644 --- a/libbeat/common/transport/tls.go +++ b/libbeat/common/transport/tls.go @@ -21,7 +21,6 @@ package transport import ( "crypto/tls" - "errors" "fmt" "net" "sync" @@ -31,10 +30,6 @@ import ( "github.com/elastic/beats/v7/libbeat/testing" ) -func TLSDialer(forward Dialer, config *tlscommon.TLSConfig, timeout time.Duration) Dialer { - return TestTLSDialer(testing.NullDriver, forward, config, timeout) -} - func TestTLSDialer( d testing.Driver, forward Dialer, @@ -131,86 +126,3 @@ func TestTLSDialerH2( return tlsDialWith(d, forward, network, address, timeout, tlsConfig, config) }), nil } - -func tlsDialWith( - d testing.Driver, - dialer Dialer, - network, address string, - timeout time.Duration, - tlsConfig *tls.Config, - config *tlscommon.TLSConfig, -) (net.Conn, error) { - socket, err := dialer.Dial(network, address) - if err != nil { - return nil, err - } - - conn := tls.Client(socket, tlsConfig) - - withTimeout := timeout > 0 - if withTimeout { - if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil { - d.Fatal("timeout", err) - _ = conn.Close() - return nil, err - } - } - - if tlsConfig.InsecureSkipVerify { - d.Warn("security", "server's certificate chain verification is disabled") - } else { - d.Info("security", "server's certificate chain verification is enabled") - } - - err = conn.Handshake() - d.Fatal("handshake", err) - if err != nil { - _ = conn.Close() - return nil, err - } - - // remove timeout if handshake was subject to timeout: - if withTimeout { - conn.SetDeadline(time.Time{}) - } - - if err := postVerifyTLSConnection(d, conn, config); err != nil { - _ = conn.Close() - return nil, err - } - - return conn, nil -} - -func postVerifyTLSConnection(d testing.Driver, conn *tls.Conn, config *tlscommon.TLSConfig) error { - st := conn.ConnectionState() - - if !st.HandshakeComplete { - err := errors.New("incomplete handshake") - d.Fatal("incomplete handshake", err) - return err - } - - d.Info("TLS version", fmt.Sprintf("%v", tlscommon.TLSVersion(st.Version))) - - // no more checks if no extra configs available - if config == nil { - return nil - } - - versions := config.Versions - if versions == nil { - versions = tlscommon.TLSDefaultVersions - } - versionOK := false - for _, version := range versions { - versionOK = versionOK || st.Version == uint16(version) - } - if !versionOK { - err := fmt.Errorf("tls version %v not configured", tlscommon.TLSVersion(st.Version)) - d.Fatal("TLS version", err) - return err - } - - return nil -} diff --git a/libbeat/common/transport/tls_common.go b/libbeat/common/transport/tls_common.go new file mode 100644 index 000000000000..232b42f9c094 --- /dev/null +++ b/libbeat/common/transport/tls_common.go @@ -0,0 +1,116 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package transport + +import ( + "crypto/tls" + "errors" + "fmt" + "net" + "time" + + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/testing" +) + +func TLSDialer(forward Dialer, config *tlscommon.TLSConfig, timeout time.Duration) Dialer { + return TestTLSDialer(testing.NullDriver, forward, config, timeout) +} + +func tlsDialWith( + d testing.Driver, + dialer Dialer, + network, address string, + timeout time.Duration, + tlsConfig *tls.Config, + config *tlscommon.TLSConfig, +) (net.Conn, error) { + socket, err := dialer.Dial(network, address) + if err != nil { + return nil, err + } + + conn := tls.Client(socket, tlsConfig) + + withTimeout := timeout > 0 + if withTimeout { + if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil { + d.Fatal("timeout", err) + _ = conn.Close() + return nil, err + } + } + + if tlsConfig.InsecureSkipVerify { + d.Warn("security", "server's certificate chain verification is disabled") + } else { + d.Info("security", "server's certificate chain verification is enabled") + } + + err = conn.Handshake() + d.Fatal("handshake", err) + if err != nil { + _ = conn.Close() + return nil, err + } + + // remove timeout if handshake was subject to timeout: + if withTimeout { + conn.SetDeadline(time.Time{}) + } + + if err := postVerifyTLSConnection(d, conn, config); err != nil { + _ = conn.Close() + return nil, err + } + + return conn, nil +} + +func postVerifyTLSConnection(d testing.Driver, conn *tls.Conn, config *tlscommon.TLSConfig) error { + st := conn.ConnectionState() + + if !st.HandshakeComplete { + err := errors.New("incomplete handshake") + d.Fatal("incomplete handshake", err) + return err + } + + d.Info("TLS version", fmt.Sprintf("%v", tlscommon.TLSVersion(st.Version))) + + // no more checks if no extra configs available + if config == nil { + return nil + } + + versions := config.Versions + if versions == nil { + versions = tlscommon.TLSDefaultVersions + } + versionOK := false + for _, version := range versions { + versionOK = versionOK || st.Version == uint16(version) + } + if !versionOK { + err := fmt.Errorf("tls version %v not configured", tlscommon.TLSVersion(st.Version)) + d.Fatal("TLS version", err) + return err + } + + return nil +} diff --git a/libbeat/common/transport/tls_legacy.go b/libbeat/common/transport/tls_legacy.go index 3fbb49248d21..533a3cf8e56f 100644 --- a/libbeat/common/transport/tls_legacy.go +++ b/libbeat/common/transport/tls_legacy.go @@ -21,7 +21,6 @@ package transport import ( "crypto/tls" - "errors" "fmt" "net" "sync" @@ -31,10 +30,6 @@ import ( "github.com/elastic/beats/v7/libbeat/testing" ) -func TLSDialer(forward Dialer, config *tlscommon.TLSConfig, timeout time.Duration) Dialer { - return TestTLSDialer(testing.NullDriver, forward, config, timeout) -} - func TestTLSDialer( d testing.Driver, forward Dialer, @@ -74,86 +69,3 @@ func TestTLSDialer( return tlsDialWith(d, forward, network, address, timeout, tlsConfig, config) }) } - -func tlsDialWith( - d testing.Driver, - dialer Dialer, - network, address string, - timeout time.Duration, - tlsConfig *tls.Config, - config *tlscommon.TLSConfig, -) (net.Conn, error) { - socket, err := dialer.Dial(network, address) - if err != nil { - return nil, err - } - - conn := tls.Client(socket, tlsConfig) - - withTimeout := timeout > 0 - if withTimeout { - if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil { - d.Fatal("timeout", err) - _ = conn.Close() - return nil, err - } - } - - if tlsConfig.InsecureSkipVerify { - d.Warn("security", "server's certificate chain verification is disabled") - } else { - d.Info("security", "server's certificate chain verification is enabled") - } - - err = conn.Handshake() - d.Fatal("handshake", err) - if err != nil { - _ = conn.Close() - return nil, err - } - - // remove timeout if handshake was subject to timeout: - if withTimeout { - conn.SetDeadline(time.Time{}) - } - - if err := postVerifyTLSConnection(d, conn, config); err != nil { - _ = conn.Close() - return nil, err - } - - return conn, nil -} - -func postVerifyTLSConnection(d testing.Driver, conn *tls.Conn, config *tlscommon.TLSConfig) error { - st := conn.ConnectionState() - - if !st.HandshakeComplete { - err := errors.New("incomplete handshake") - d.Fatal("incomplete handshake", err) - return err - } - - d.Info("TLS version", fmt.Sprintf("%v", tlscommon.TLSVersion(st.Version))) - - // no more checks if no extra configs available - if config == nil { - return nil - } - - versions := config.Versions - if versions == nil { - versions = tlscommon.TLSDefaultVersions - } - versionOK := false - for _, version := range versions { - versionOK = versionOK || st.Version == uint16(version) - } - if !versionOK { - err := fmt.Errorf("tls version %v not configured", tlscommon.TLSVersion(st.Version)) - d.Fatal("TLS version", err) - return err - } - - return nil -} diff --git a/libbeat/common/transport/tlscommon/config.go b/libbeat/common/transport/tlscommon/config.go index bce85d6a66fd..04fd1088e4a7 100644 --- a/libbeat/common/transport/tlscommon/config.go +++ b/libbeat/common/transport/tlscommon/config.go @@ -104,8 +104,3 @@ func (c *Config) Validate() error { return c.Certificate.Validate() } - -// IsEnabled returns true if the `enable` field is set to true in the yaml. -func (c *Config) IsEnabled() bool { - return c != nil && (c.Enabled == nil || *c.Enabled) -} diff --git a/libbeat/common/transport/tlscommon/config_common.go b/libbeat/common/transport/tlscommon/config_common.go new file mode 100644 index 000000000000..b61c26fb4956 --- /dev/null +++ b/libbeat/common/transport/tlscommon/config_common.go @@ -0,0 +1,23 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tlscommon + +// IsEnabled returns true if the `enable` field is set to true in the yaml. +func (c *Config) IsEnabled() bool { + return c != nil && (c.Enabled == nil || *c.Enabled) +} diff --git a/libbeat/common/transport/tlscommon/config_legacy.go b/libbeat/common/transport/tlscommon/config_legacy.go index c14e05d79e5f..333dbc340dc8 100644 --- a/libbeat/common/transport/tlscommon/config_legacy.go +++ b/libbeat/common/transport/tlscommon/config_legacy.go @@ -100,8 +100,3 @@ func LoadTLSConfig(config *Config) (*TLSConfig, error) { func (c *Config) Validate() error { return c.Certificate.Validate() } - -// IsEnabled returns true if the `enable` field is set to true in the yaml. -func (c *Config) IsEnabled() bool { - return c != nil && (c.Enabled == nil || *c.Enabled) -} diff --git a/libbeat/common/transport/tlscommon/server_config.go b/libbeat/common/transport/tlscommon/server_config.go index a7e6940f365a..a37de7ef7d76 100644 --- a/libbeat/common/transport/tlscommon/server_config.go +++ b/libbeat/common/transport/tlscommon/server_config.go @@ -23,8 +23,6 @@ import ( "crypto/tls" "github.com/joeshaw/multierror" - - "github.com/elastic/beats/v7/libbeat/common" ) // ServerConfig defines the user configurable tls options for any TCP based service. @@ -94,41 +92,3 @@ func LoadTLSServerConfig(config *ServerConfig) (*TLSConfig, error) { CASha256: config.CASha256, }, nil } - -// Unpack unpacks the TLS Server configuration. -func (c *ServerConfig) Unpack(cfg common.Config) error { - const clientAuthKey = "client_authentication" - const ca = "certificate_authorities" - - // When we have explicitely defined the `certificate_authorities` in the configuration we default - // to `required` for the `client_authentication`, when CA is not defined we should set to `none`. - if cfg.HasField(ca) && !cfg.HasField(clientAuthKey) { - cfg.SetString(clientAuthKey, -1, "required") - } - type serverCfg ServerConfig - var sCfg serverCfg - if err := cfg.Unpack(&sCfg); err != nil { - return err - } - *c = ServerConfig(sCfg) - return nil -} - -// Validate values the TLSConfig struct making sure certificate sure we have both a certificate and -// a key. -func (c *ServerConfig) Validate() error { - if c.IsEnabled() { - // c.Certificate.Validate() ensures that both a certificate and key - // are specified, or neither are specified. For server-side TLS we - // require both to be specified. - if c.Certificate.Certificate == "" { - return ErrCertificateUnspecified - } - } - return c.Certificate.Validate() -} - -// IsEnabled returns true if the `enable` field is set to true in the yaml. -func (c *ServerConfig) IsEnabled() bool { - return c != nil && (c.Enabled == nil || *c.Enabled) -} diff --git a/libbeat/common/transport/tlscommon/server_config_common.go b/libbeat/common/transport/tlscommon/server_config_common.go new file mode 100644 index 000000000000..cce8d3236806 --- /dev/null +++ b/libbeat/common/transport/tlscommon/server_config_common.go @@ -0,0 +1,60 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tlscommon + +import ( + "github.com/elastic/beats/v7/libbeat/common" +) + +// Unpack unpacks the TLS Server configuration. +func (c *ServerConfig) Unpack(cfg common.Config) error { + const clientAuthKey = "client_authentication" + const ca = "certificate_authorities" + + // When we have explicitely defined the `certificate_authorities` in the configuration we default + // to `required` for the `client_authentication`, when CA is not defined we should set to `none`. + if cfg.HasField(ca) && !cfg.HasField(clientAuthKey) { + cfg.SetString(clientAuthKey, -1, "required") + } + type serverCfg ServerConfig + var sCfg serverCfg + if err := cfg.Unpack(&sCfg); err != nil { + return err + } + *c = ServerConfig(sCfg) + return nil +} + +// Validate values the TLSConfig struct making sure certificate sure we have both a certificate and +// a key. +func (c *ServerConfig) Validate() error { + if c.IsEnabled() { + // c.Certificate.Validate() ensures that both a certificate and key + // are specified, or neither are specified. For server-side TLS we + // require both to be specified. + if c.Certificate.Certificate == "" { + return ErrCertificateUnspecified + } + } + return c.Certificate.Validate() +} + +// IsEnabled returns true if the `enable` field is set to true in the yaml. +func (c *ServerConfig) IsEnabled() bool { + return c != nil && (c.Enabled == nil || *c.Enabled) +} diff --git a/libbeat/common/transport/tlscommon/server_config_legacy.go b/libbeat/common/transport/tlscommon/server_config_legacy.go index eebd90672ddf..89a45adc94ec 100644 --- a/libbeat/common/transport/tlscommon/server_config_legacy.go +++ b/libbeat/common/transport/tlscommon/server_config_legacy.go @@ -23,8 +23,6 @@ import ( "crypto/tls" "github.com/joeshaw/multierror" - - "github.com/elastic/beats/v7/libbeat/common" ) // ServerConfig defines the user configurable tls options for any TCP based service. @@ -92,41 +90,3 @@ func LoadTLSServerConfig(config *ServerConfig) (*TLSConfig, error) { ClientAuth: tls.ClientAuthType(config.ClientAuth), }, nil } - -// Unpack unpacks the TLS Server configuration. -func (c *ServerConfig) Unpack(cfg common.Config) error { - const clientAuthKey = "client_authentication" - const ca = "certificate_authorities" - - // When we have explicitely defined the `certificate_authorities` in the configuration we default - // to `required` for the `client_authentication`, when CA is not defined we should set to `none`. - if cfg.HasField(ca) && !cfg.HasField(clientAuthKey) { - cfg.SetString(clientAuthKey, -1, "required") - } - type serverCfg ServerConfig - var sCfg serverCfg - if err := cfg.Unpack(&sCfg); err != nil { - return err - } - *c = ServerConfig(sCfg) - return nil -} - -// Validate values the TLSConfig struct making sure certificate sure we have both a certificate and -// a key. -func (c *ServerConfig) Validate() error { - if c.IsEnabled() { - // c.Certificate.Validate() ensures that both a certificate and key - // are specified, or neither are specified. For server-side TLS we - // require both to be specified. - if c.Certificate.Certificate == "" { - return ErrCertificateUnspecified - } - } - return c.Certificate.Validate() -} - -// IsEnabled returns true if the `enable` field is set to true in the yaml. -func (c *ServerConfig) IsEnabled() bool { - return c != nil && (c.Enabled == nil || *c.Enabled) -} diff --git a/libbeat/common/transport/tlscommon/tls.go b/libbeat/common/transport/tlscommon/tls.go index 531879d556f8..ec3fd60815fc 100644 --- a/libbeat/common/transport/tlscommon/tls.go +++ b/libbeat/common/transport/tlscommon/tls.go @@ -20,199 +20,16 @@ package tlscommon import ( - "bytes" - "crypto/tls" - "crypto/x509" - "encoding/pem" - "errors" - "fmt" - "io" "io/ioutil" "os" "strings" - - "github.com/elastic/beats/v7/libbeat/logp" ) -const logSelector = "tls" - -// LoadCertificate will load a certificate from disk and return a tls.Certificate or error -func LoadCertificate(config *CertificateConfig) (*tls.Certificate, error) { - if err := config.Validate(); err != nil { - return nil, err - } - - certificate := config.Certificate - key := config.Key - if certificate == "" { - return nil, nil - } - - log := logp.NewLogger(logSelector) - - certPEM, err := ReadPEMFile(log, certificate, config.Passphrase) - if err != nil { - log.Errorf("Failed reading certificate file %v: %+v", certificate, err) - return nil, fmt.Errorf("%v %v", err, certificate) - } - - keyPEM, err := ReadPEMFile(log, key, config.Passphrase) - if err != nil { - log.Errorf("Failed reading key file %v: %+v", key, err) - return nil, fmt.Errorf("%v %v", err, key) - } - - cert, err := tls.X509KeyPair(certPEM, keyPEM) - if err != nil { - log.Errorf("Failed loading client certificate %+v", err) - return nil, err - } - - log.Debugf("Loading certificate: %v and key %v", certificate, key) - return &cert, nil -} - -// ReadPEMFile reads a PEM formatted string either from disk or passed as a plain text starting with a "-" -// and decrypt it with the provided password and return the raw content. -func ReadPEMFile(log *logp.Logger, s, passphrase string) ([]byte, error) { - pass := []byte(passphrase) - var blocks []*pem.Block - - r, err := NewPEMReader(s) - if err != nil { - return nil, err - } - defer r.Close() - - content, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - - for len(content) > 0 { - var block *pem.Block - - block, content = pem.Decode(content) - if block == nil { - if len(blocks) == 0 { - return nil, errors.New("no pem file") - } - break - } - - if x509.IsEncryptedPEMBlock(block) { - var buffer []byte - var err error - if len(pass) == 0 { - err = errors.New("No passphrase available") - } else { - // Note, decrypting pem might succeed even with wrong password, but - // only noise will be stored in buffer in this case. - buffer, err = x509.DecryptPEMBlock(block, pass) - } - - if err != nil { - log.Errorf("Dropping encrypted pem '%v' block read from %v. %+v", - block.Type, r, err) - continue - } - - // DEK-Info contains encryption info. Remove header to mark block as - // unencrypted. - delete(block.Headers, "DEK-Info") - block.Bytes = buffer - } - blocks = append(blocks, block) - } - - if len(blocks) == 0 { - return nil, errors.New("no PEM blocks") - } - - // re-encode available, decrypted blocks - buffer := bytes.NewBuffer(nil) - for _, block := range blocks { - err := pem.Encode(buffer, block) - if err != nil { - return nil, err - } - } - return buffer.Bytes(), nil -} - -// LoadCertificateAuthorities read the slice of CAcert and return a Certpool. -func LoadCertificateAuthorities(CAs []string) (*x509.CertPool, []error) { - errors := []error{} - - if len(CAs) == 0 { - return nil, nil - } - - log := logp.NewLogger(logSelector) - roots := x509.NewCertPool() - for _, s := range CAs { - r, err := NewPEMReader(s) - if err != nil { - log.Errorf("Failed reading CA certificate: %+v", err) - errors = append(errors, fmt.Errorf("%v reading %v", err, r)) - continue - } - defer r.Close() - - pemData, err := ioutil.ReadAll(r) - if err != nil { - log.Errorf("Failed reading CA certificate: %+v", err) - errors = append(errors, fmt.Errorf("%v reading %v", err, r)) - continue - } - - if ok := roots.AppendCertsFromPEM(pemData); !ok { - log.Error("Failed to add CA to the cert pool, CA is not a valid PEM document") - errors = append(errors, fmt.Errorf("%v adding %v to the list of known CAs", ErrNotACertificate, r)) - continue - } - log.Debugf("Successfully loaded CA certificate: %v", r) - } - - return roots, errors -} - -func extractMinMaxVersion(versions []TLSVersion) (uint16, uint16) { - if len(versions) == 0 { - versions = TLSDefaultVersions - } - - minVersion := uint16(0xffff) - maxVersion := uint16(0) - for _, version := range versions { - v := uint16(version) - if v < minVersion { - minVersion = v - } - if v > maxVersion { - maxVersion = v - } - } - - return minVersion, maxVersion -} - -// ResolveTLSVersion takes the integer representation and return the name. -func ResolveTLSVersion(v uint16) string { - return TLSVersion(v).String() -} - // ResolveCipherSuite takes the integer representation and return the cipher name. func ResolveCipherSuite(cipher uint16) string { return CipherSuite(cipher).String() } -// PEMReader allows to read a certificate in PEM format either through the disk or from a string. -type PEMReader struct { - reader io.ReadCloser - debugStr string -} - // NewPEMReader returns a new PEMReader. func NewPEMReader(certificate string) (*PEMReader, error) { if IsPEMString(certificate) { @@ -225,24 +42,3 @@ func NewPEMReader(certificate string) (*PEMReader, error) { } return &PEMReader{reader: r, debugStr: certificate}, nil } - -// Close closes the target io.ReadCloser. -func (p *PEMReader) Close() error { - return p.reader.Close() -} - -// Read read bytes from the io.ReadCloser. -func (p *PEMReader) Read(b []byte) (n int, err error) { - return p.reader.Read(b) -} - -func (p *PEMReader) String() string { - return p.debugStr -} - -// IsPEMString returns true if the provided string match a PEM formatted certificate. try to pem decode to validate. -func IsPEMString(s string) bool { - // Trim the certificates to make sure we tolerate any yaml weirdness, we assume that the string starts - // with "-" and let further validation verifies the PEM format. - return strings.HasPrefix(strings.TrimSpace(s), "-") -} diff --git a/libbeat/common/transport/tlscommon/tls_common.go b/libbeat/common/transport/tlscommon/tls_common.go new file mode 100644 index 000000000000..e2cb40aceb76 --- /dev/null +++ b/libbeat/common/transport/tlscommon/tls_common.go @@ -0,0 +1,227 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tlscommon + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "io" + "io/ioutil" + "strings" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +const logSelector = "tls" + +// LoadCertificate will load a certificate from disk and return a tls.Certificate or error +func LoadCertificate(config *CertificateConfig) (*tls.Certificate, error) { + if err := config.Validate(); err != nil { + return nil, err + } + + certificate := config.Certificate + key := config.Key + if certificate == "" { + return nil, nil + } + + log := logp.NewLogger(logSelector) + + certPEM, err := ReadPEMFile(log, certificate, config.Passphrase) + if err != nil { + log.Errorf("Failed reading certificate file %v: %+v", certificate, err) + return nil, fmt.Errorf("%v %v", err, certificate) + } + + keyPEM, err := ReadPEMFile(log, key, config.Passphrase) + if err != nil { + log.Errorf("Failed reading key file %v: %+v", key, err) + return nil, fmt.Errorf("%v %v", err, key) + } + + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + log.Errorf("Failed loading client certificate %+v", err) + return nil, err + } + + log.Debugf("Loading certificate: %v and key %v", certificate, key) + return &cert, nil +} + +// ReadPEMFile reads a PEM formatted string either from disk or passed as a plain text starting with a "-" +// and decrypt it with the provided password and return the raw content. +func ReadPEMFile(log *logp.Logger, s, passphrase string) ([]byte, error) { + pass := []byte(passphrase) + var blocks []*pem.Block + + r, err := NewPEMReader(s) + if err != nil { + return nil, err + } + defer r.Close() + + content, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + for len(content) > 0 { + var block *pem.Block + + block, content = pem.Decode(content) + if block == nil { + if len(blocks) == 0 { + return nil, errors.New("no pem file") + } + break + } + + if x509.IsEncryptedPEMBlock(block) { + var buffer []byte + var err error + if len(pass) == 0 { + err = errors.New("No passphrase available") + } else { + // Note, decrypting pem might succeed even with wrong password, but + // only noise will be stored in buffer in this case. + buffer, err = x509.DecryptPEMBlock(block, pass) + } + + if err != nil { + log.Errorf("Dropping encrypted pem '%v' block read from %v. %+v", + block.Type, r, err) + continue + } + + // DEK-Info contains encryption info. Remove header to mark block as + // unencrypted. + delete(block.Headers, "DEK-Info") + block.Bytes = buffer + } + blocks = append(blocks, block) + } + + if len(blocks) == 0 { + return nil, errors.New("no PEM blocks") + } + + // re-encode available, decrypted blocks + buffer := bytes.NewBuffer(nil) + for _, block := range blocks { + err := pem.Encode(buffer, block) + if err != nil { + return nil, err + } + } + return buffer.Bytes(), nil +} + +// LoadCertificateAuthorities read the slice of CAcert and return a Certpool. +func LoadCertificateAuthorities(CAs []string) (*x509.CertPool, []error) { + errors := []error{} + + if len(CAs) == 0 { + return nil, nil + } + + log := logp.NewLogger(logSelector) + roots := x509.NewCertPool() + for _, s := range CAs { + r, err := NewPEMReader(s) + if err != nil { + log.Errorf("Failed reading CA certificate: %+v", err) + errors = append(errors, fmt.Errorf("%v reading %v", err, r)) + continue + } + defer r.Close() + + pemData, err := ioutil.ReadAll(r) + if err != nil { + log.Errorf("Failed reading CA certificate: %+v", err) + errors = append(errors, fmt.Errorf("%v reading %v", err, r)) + continue + } + + if ok := roots.AppendCertsFromPEM(pemData); !ok { + log.Error("Failed to add CA to the cert pool, CA is not a valid PEM document") + errors = append(errors, fmt.Errorf("%v adding %v to the list of known CAs", ErrNotACertificate, r)) + continue + } + log.Debugf("Successfully loaded CA certificate: %v", r) + } + + return roots, errors +} + +func extractMinMaxVersion(versions []TLSVersion) (uint16, uint16) { + if len(versions) == 0 { + versions = TLSDefaultVersions + } + + minVersion := uint16(0xffff) + maxVersion := uint16(0) + for _, version := range versions { + v := uint16(version) + if v < minVersion { + minVersion = v + } + if v > maxVersion { + maxVersion = v + } + } + + return minVersion, maxVersion +} + +// ResolveTLSVersion takes the integer representation and return the name. +func ResolveTLSVersion(v uint16) string { + return TLSVersion(v).String() +} + +// PEMReader allows to read a certificate in PEM format either through the disk or from a string. +type PEMReader struct { + reader io.ReadCloser + debugStr string +} + +// Close closes the target io.ReadCloser. +func (p *PEMReader) Close() error { + return p.reader.Close() +} + +// Read read bytes from the io.ReadCloser. +func (p *PEMReader) Read(b []byte) (n int, err error) { + return p.reader.Read(b) +} + +func (p *PEMReader) String() string { + return p.debugStr +} + +// IsPEMString returns true if the provided string match a PEM formatted certificate. try to pem decode to validate. +func IsPEMString(s string) bool { + // Trim the certificates to make sure we tolerate any yaml weirdness, we assume that the string starts + // with "-" and let further validation verifies the PEM format. + return strings.HasPrefix(strings.TrimSpace(s), "-") +} diff --git a/libbeat/common/transport/tlscommon/tls_config_legacy.go b/libbeat/common/transport/tlscommon/tls_config_legacy.go index 84ace6feff77..d82a72aa6425 100644 --- a/libbeat/common/transport/tlscommon/tls_config_legacy.go +++ b/libbeat/common/transport/tlscommon/tls_config_legacy.go @@ -125,6 +125,20 @@ func (c *TLSConfig) BuildModuleClientConfig(host string) *tls.Config { return config } +// BuildServerConfig takes the TLSConfig and transform it into a `tls.Config` for server side objects. +func (c *TLSConfig) BuildServerConfig(host string) *tls.Config { + if c == nil { + // use default TLS settings, if config is empty. + return &tls.Config{ + ServerName: host, + } + } + + config := c.ToConfig() + config.ServerName = host + return config +} + // BuildModuleConfig takes the TLSConfig and transform it into a `tls.Config`. func (c *TLSConfig) BuildModuleConfig(host string) *tls.Config { if c == nil { diff --git a/libbeat/common/transport/tlscommon/tls_legacy.go b/libbeat/common/transport/tlscommon/tls_legacy.go index dcfb730ac303..ec97ab8d8ba1 100644 --- a/libbeat/common/transport/tlscommon/tls_legacy.go +++ b/libbeat/common/transport/tlscommon/tls_legacy.go @@ -20,199 +20,16 @@ package tlscommon import ( - "bytes" - "crypto/tls" - "crypto/x509" - "encoding/pem" - "errors" - "fmt" - "io" "io/ioutil" "os" "strings" - - "github.com/elastic/beats/v7/libbeat/logp" ) -const logSelector = "tls" - -// LoadCertificate will load a certificate from disk and return a tls.Certificate or error -func LoadCertificate(config *CertificateConfig) (*tls.Certificate, error) { - if err := config.Validate(); err != nil { - return nil, err - } - - certificate := config.Certificate - key := config.Key - if certificate == "" { - return nil, nil - } - - log := logp.NewLogger(logSelector) - - certPEM, err := ReadPEMFile(log, certificate, config.Passphrase) - if err != nil { - log.Errorf("Failed reading certificate file %v: %+v", certificate, err) - return nil, fmt.Errorf("%v %v", err, certificate) - } - - keyPEM, err := ReadPEMFile(log, key, config.Passphrase) - if err != nil { - log.Errorf("Failed reading key file %v: %+v", key, err) - return nil, fmt.Errorf("%v %v", err, key) - } - - cert, err := tls.X509KeyPair(certPEM, keyPEM) - if err != nil { - log.Errorf("Failed loading client certificate %+v", err) - return nil, err - } - - log.Debugf("Loading certificate: %v and key %v", certificate, key) - return &cert, nil -} - -// ReadPEMFile reads a PEM formatted string either from disk or passed as a plain text starting with a "-" -// and decrypt it with the provided password and return the raw content. -func ReadPEMFile(log *logp.Logger, s, passphrase string) ([]byte, error) { - pass := []byte(passphrase) - var blocks []*pem.Block - - r, err := NewPEMReader(s) - if err != nil { - return nil, err - } - defer r.Close() - - content, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - - for len(content) > 0 { - var block *pem.Block - - block, content = pem.Decode(content) - if block == nil { - if len(blocks) == 0 { - return nil, errors.New("no pem file") - } - break - } - - if x509.IsEncryptedPEMBlock(block) { - var buffer []byte - var err error - if len(pass) == 0 { - err = errors.New("No passphrase available") - } else { - // Note, decrypting pem might succeed even with wrong password, but - // only noise will be stored in buffer in this case. - buffer, err = x509.DecryptPEMBlock(block, pass) - } - - if err != nil { - log.Errorf("Dropping encrypted pem '%v' block read from %v. %+v", - block.Type, r, err) - continue - } - - // DEK-Info contains encryption info. Remove header to mark block as - // unencrypted. - delete(block.Headers, "DEK-Info") - block.Bytes = buffer - } - blocks = append(blocks, block) - } - - if len(blocks) == 0 { - return nil, errors.New("no PEM blocks") - } - - // re-encode available, decrypted blocks - buffer := bytes.NewBuffer(nil) - for _, block := range blocks { - err := pem.Encode(buffer, block) - if err != nil { - return nil, err - } - } - return buffer.Bytes(), nil -} - -// LoadCertificateAuthorities read the slice of CAcert and return a Certpool. -func LoadCertificateAuthorities(CAs []string) (*x509.CertPool, []error) { - errors := []error{} - - if len(CAs) == 0 { - return nil, nil - } - - log := logp.NewLogger(logSelector) - roots := x509.NewCertPool() - for _, s := range CAs { - r, err := NewPEMReader(s) - if err != nil { - log.Errorf("Failed reading CA certificate: %+v", err) - errors = append(errors, fmt.Errorf("%v reading %v", err, r)) - continue - } - defer r.Close() - - pemData, err := ioutil.ReadAll(r) - if err != nil { - log.Errorf("Failed reading CA certificate: %+v", err) - errors = append(errors, fmt.Errorf("%v reading %v", err, r)) - continue - } - - if ok := roots.AppendCertsFromPEM(pemData); !ok { - log.Error("Failed to add CA to the cert pool, CA is not a valid PEM document") - errors = append(errors, fmt.Errorf("%v adding %v to the list of known CAs", ErrNotACertificate, r)) - continue - } - log.Debugf("Successfully loaded CA certificate: %v", r) - } - - return roots, errors -} - -func extractMinMaxVersion(versions []TLSVersion) (uint16, uint16) { - if len(versions) == 0 { - versions = TLSDefaultVersions - } - - minVersion := uint16(0xffff) - maxVersion := uint16(0) - for _, version := range versions { - v := uint16(version) - if v < minVersion { - minVersion = v - } - if v > maxVersion { - maxVersion = v - } - } - - return minVersion, maxVersion -} - -// ResolveTLSVersion takes the integer representation and return the name. -func ResolveTLSVersion(v uint16) string { - return TLSVersion(v).String() -} - // ResolveCipherSuite takes the integer representation and return the cipher name. func ResolveCipherSuite(cipher uint16) string { return tlsCipherSuite(cipher).String() } -// PEMReader allows to read a certificate in PEM format either through the disk or from a string. -type PEMReader struct { - reader io.ReadCloser - debugStr string -} - // NewPEMReader returns a new PEMReader. func NewPEMReader(certificate string) (*PEMReader, error) { if IsPEMString(certificate) { @@ -227,24 +44,3 @@ func NewPEMReader(certificate string) (*PEMReader, error) { } return &PEMReader{reader: r, debugStr: certificate}, nil } - -// Close closes the target io.ReadCloser. -func (p *PEMReader) Close() error { - return p.reader.Close() -} - -// Read read bytes from the io.ReadCloser. -func (p *PEMReader) Read(b []byte) (n int, err error) { - return p.reader.Read(b) -} - -func (p *PEMReader) String() string { - return p.debugStr -} - -// IsPEMString returns true if the provided string match a PEM formatted certificate. try to pem decode to validate. -func IsPEMString(s string) bool { - // Trim the certificates to make sure we tolerate any yaml weirdness, we assume that the string starts - // with "-" and let further validation verifies the PEM format. - return strings.HasPrefix(strings.TrimSpace(s), "-") -}