From 199e27f24a4a5d6e7811e01fd5e450aeccc136c6 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Fri, 26 Nov 2021 12:45:46 +0530 Subject: [PATCH 01/50] Add authentication mechanism for cometd input for Salesforce connector --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/include/list.go | 1 + x-pack/filebeat/input/cometd/config.go | 27 +++ x-pack/filebeat/input/cometd/config_auth.go | 57 ++++++ x-pack/filebeat/input/cometd/input.go | 192 ++++++++++++++++++++ 5 files changed, 278 insertions(+) create mode 100644 x-pack/filebeat/input/cometd/config.go create mode 100644 x-pack/filebeat/input/cometd/config_auth.go create mode 100644 x-pack/filebeat/input/cometd/input.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a4f34cb48ba6..685a3ad18776 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -348,6 +348,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support in aws-s3 input for s3 notification from SNS to SQS. {pull}28800[28800] - Add support in aws-s3 input for custom script parsing of s3 notifications. {pull}28946[28946] - Improve error handling in aws-s3 input for malformed s3 notifications. {issue}28828[28828] {pull}28946[28946] +- Add commetD input to retrieve logs from Salesforce Streaming API. {pull}#[#] *Heartbeat* diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index adfb028469c5..92686aaa22c8 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -11,6 +11,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/filebeat/input/awscloudwatch" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/awss3" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/azureeventhub" + _ "github.com/elastic/beats/v7/x-pack/filebeat/input/cometd" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/gcppubsub" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/activemq" diff --git a/x-pack/filebeat/input/cometd/config.go b/x-pack/filebeat/input/cometd/config.go new file mode 100644 index 000000000000..67bd449a3bac --- /dev/null +++ b/x-pack/filebeat/input/cometd/config.go @@ -0,0 +1,27 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cometd + +import "fmt" + +type config struct { + ChannelName string `config:"channel_name" validate:"required"` + Auth *authConfig `config:"auth"` + // Overrides the default Pub/Sub service address and disables TLS. For testing. + AlternativeHost string `config:"alternative_host"` +} + +func (c *config) Validate() error { + if c.ChannelName == "" { + return fmt.Errorf("no channel name was configured or detected") + } + return nil +} + +func defaultConfig() config { + var c config + c.ChannelName = "cometd-channel" + return c +} diff --git a/x-pack/filebeat/input/cometd/config_auth.go b/x-pack/filebeat/input/cometd/config_auth.go new file mode 100644 index 000000000000..f362fda76f29 --- /dev/null +++ b/x-pack/filebeat/input/cometd/config_auth.go @@ -0,0 +1,57 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cometd + +import ( + "context" + "errors" + "fmt" + "net/http" + + "golang.org/x/oauth2" +) + +const authStyleInParams = 1 + +type authConfig struct { + OAuth2 *oAuth2Config `config:"oauth2"` +} + +type oAuth2Config struct { + // common oauth fields + ClientID string `config:"client.id"` + ClientSecret string `config:"client.secret"` + User string `config:"user"` + Password string `config:"password"` + EndpointParams map[string][]string `config:"endpoint_params"` + Scopes []string `config:"scopes"` + TokenURL string `config:"token_url"` +} + +// Client wraps the given http.Client and returns a new one that will use the oauth authentication. +func (o *oAuth2Config) client(ctx context.Context, client *http.Client) (*http.Client, error) { + ctx = context.WithValue(ctx, oauth2.HTTPClient, client) + conf := &oauth2.Config{ + ClientID: o.ClientID, + ClientSecret: o.ClientSecret, + Endpoint: oauth2.Endpoint{ + TokenURL: o.TokenURL, + AuthStyle: authStyleInParams, + }, + } + token, err := conf.PasswordCredentialsToken(ctx, o.User, o.Password) + if err != nil { + return nil, fmt.Errorf("oauth2 client: error loading credentials using user and password: %w", err) + } + return conf.Client(ctx, token), nil +} + +// Validate checks if oauth2 config is valid. +func (o *oAuth2Config) Validate() error { + if o.TokenURL == "" || o.ClientID == "" || o.ClientSecret == "" || o.User == "" || o.Password == "" { + return errors.New("both token_url and client credentials must be provided") + } + return nil +} diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go new file mode 100644 index 000000000000..ea770d834779 --- /dev/null +++ b/x-pack/filebeat/input/cometd/input.go @@ -0,0 +1,192 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// A lot of code in this file would be updated as a part of data collection +// mechanism implementation. +// Please do not review it, as the current code would test the authentication +// mechanism only. + +package cometd + +import ( + "context" + "io/ioutil" + "net/http" + "sync" + "time" + + retryablehttp "github.com/hashicorp/go-retryablehttp" + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/filebeat/channel" + "github.com/elastic/beats/v7/filebeat/input" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/atomic" + "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" + "github.com/elastic/beats/v7/libbeat/logp" +) + +const ( + inputName = "cometd" +) + +type cometdInput struct { + config + + log *logp.Logger + inputCtx context.Context // Wraps the Done channel from parent input.Context. + + workerCtx context.Context // Worker goroutine context. It's cancelled when the input stops or the worker exits. + workerCancel context.CancelFunc // Used to signal that the worker should stop. + workerOnce sync.Once // Guarantees that the worker goroutine is only started once. + workerWg sync.WaitGroup // Waits on pubsub worker goroutine. + + ackedCount *atomic.Uint32 // Total number of successfully ACKed pubsub messages. + Transport httpcommon.HTTPTransportSettings `config:",inline"` + Retry retryConfig `config:"retry"` +} + +func init() { + err := input.Register(inputName, NewInput) + if err != nil { + panic(errors.Wrapf(err, "failed to register %v input", inputName)) + } +} + +// NewInput creates a new CometD Pub/Sub input that consumes events from +// a topic subscription. +func NewInput( + cfg *common.Config, + connector channel.Connector, + inputContext input.Context, +) (inp input.Input, err error) { + // Extract and validate the input's configuration. + conf := defaultConfig() + if err = cfg.Unpack(&conf); err != nil { + return nil, err + } + + logger := logp.NewLogger("cometd").With( + "pubsub_channel", conf.ChannelName) + + // Wrap input.Context's Done channel with a context.Context. This goroutine + // stops with the parent closes the Done channel. + inputCtx, cancelInputCtx := context.WithCancel(context.Background()) + go func() { + defer cancelInputCtx() + select { + case <-inputContext.Done: + case <-inputCtx.Done(): + } + }() + + // If the input ever needs to be made restartable, then context would need + // to be recreated with each restart. + workerCtx, workerCancel := context.WithCancel(inputCtx) + + in := &cometdInput{ + config: conf, + log: logger, + inputCtx: inputCtx, + workerCtx: workerCtx, + workerCancel: workerCancel, + ackedCount: atomic.NewUint32(0), + } + + in.log.Info("Initialized cometD input.") + return in, nil +} + +// Run starts the pubsub input worker then returns. Only the first invocation +// will ever start the pubsub worker. +func (in *cometdInput) Run() { + in.workerOnce.Do(func() { + in.workerWg.Add(1) + go func() { + in.log.Info("Pub/Sub input worker has started.") + defer in.log.Info("Pub/Sub input worker has stopped.") + defer in.workerWg.Done() + defer in.workerCancel() + if err := in.run(); err != nil { + in.log.Error(err) + return + } + }() + }) +} + +// Stop stops the pubsub input and waits for it to fully stop. +func (in *cometdInput) Stop() { + in.workerCancel() + in.workerWg.Wait() +} + +// Wait is an alias for Stop. +func (in *cometdInput) Wait() { + in.Stop() +} + +func (in *cometdInput) run() error { + ctx, cancel := context.WithCancel(in.workerCtx) + defer cancel() + + client, err := in.newPubsubClient(ctx) + if err != nil { + return err + } + + in.log.Debug("client successfully created") + + // For testing http client + // Start of the test code for authentication and client creation + baseUrl := "" + req, err := http.NewRequest("GET", baseUrl, nil) + if err != nil { + in.log.Errorf("Error on request generation: %v", err.Error()) + } + resp, err := client.Do(req) + if err != nil { + in.log.Errorf("Error on response: %v", err) + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + in.log.Errorf("Error while reading the response bytes: %v", err) + } + + in.log.Infof("Response body: %v", string(body)) + // End of the test code for authentication and client creation + return nil +} + +type retryConfig struct { + MaxAttempts *int `config:"max_attempts"` + WaitMin *time.Duration `config:"wait_min"` + WaitMax *time.Duration `config:"wait_max"` +} + +func (in *cometdInput) newPubsubClient(ctx context.Context) (*http.Client, error) { + + // Make retryable HTTP client + netHTTPClient, err := in.Transport.Client( + httpcommon.WithAPMHTTPInstrumentation(), + httpcommon.WithKeepaliveSettings{Disable: true}, + ) + if err != nil { + return nil, err + } + + client := &retryablehttp.Client{ + HTTPClient: netHTTPClient, + CheckRetry: retryablehttp.DefaultRetryPolicy, + Backoff: retryablehttp.DefaultBackoff, + } + + authClient, err := in.config.Auth.OAuth2.client(ctx, client.StandardClient()) + if err != nil { + return nil, err + } + return authClient, nil +} From 006763337222fbc4b3a1309d720206b7460e3832 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 1 Dec 2021 17:08:46 +0530 Subject: [PATCH 02/50] update based on comments: clear empty line and add error.Errorf with context --- x-pack/filebeat/input/cometd/input.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index ea770d834779..9904e4d3789e 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -11,6 +11,7 @@ package cometd import ( "context" + "fmt" "io/ioutil" "net/http" "sync" @@ -64,7 +65,7 @@ func NewInput( // Extract and validate the input's configuration. conf := defaultConfig() if err = cfg.Unpack(&conf); err != nil { - return nil, err + return nil, fmt.Errorf("cometd config: unpacking of config failed: %v", err) } logger := logp.NewLogger("cometd").With( @@ -133,7 +134,7 @@ func (in *cometdInput) run() error { client, err := in.newPubsubClient(ctx) if err != nil { - return err + return fmt.Errorf("cometd client: error creating pub-sub client: %v", err) } in.log.Debug("client successfully created") @@ -168,14 +169,13 @@ type retryConfig struct { } func (in *cometdInput) newPubsubClient(ctx context.Context) (*http.Client, error) { - // Make retryable HTTP client netHTTPClient, err := in.Transport.Client( httpcommon.WithAPMHTTPInstrumentation(), httpcommon.WithKeepaliveSettings{Disable: true}, ) if err != nil { - return nil, err + return nil, fmt.Errorf("cometd client: error on newHTTPClient: %v", err) } client := &retryablehttp.Client{ @@ -186,7 +186,7 @@ func (in *cometdInput) newPubsubClient(ctx context.Context) (*http.Client, error authClient, err := in.config.Auth.OAuth2.client(ctx, client.StandardClient()) if err != nil { - return nil, err + return nil, fmt.Errorf("cometd client: error on authClient: client not created: %v", err) } return authClient, nil } From 321463b79b49814a4e036271fa3db27d02a3d242 Mon Sep 17 00:00:00 2001 From: yug-crest Date: Wed, 8 Dec 2021 15:48:39 +0530 Subject: [PATCH 03/50] Add data collection mechanism for Salesforce cometd input --- x-pack/filebeat/input/cometd/config.go | 52 +- x-pack/filebeat/input/cometd/config_test.go | 19 + x-pack/filebeat/input/cometd/input.go | 634 ++++++++++++++------ x-pack/filebeat/input/cometd/input_test.go | 19 + 4 files changed, 505 insertions(+), 219 deletions(-) create mode 100644 x-pack/filebeat/input/cometd/config_test.go create mode 100644 x-pack/filebeat/input/cometd/input_test.go diff --git a/x-pack/filebeat/input/cometd/config.go b/x-pack/filebeat/input/cometd/config.go index 67bd449a3bac..6e32f1fa16aa 100644 --- a/x-pack/filebeat/input/cometd/config.go +++ b/x-pack/filebeat/input/cometd/config.go @@ -1,27 +1,25 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package cometd - -import "fmt" - -type config struct { - ChannelName string `config:"channel_name" validate:"required"` - Auth *authConfig `config:"auth"` - // Overrides the default Pub/Sub service address and disables TLS. For testing. - AlternativeHost string `config:"alternative_host"` -} - -func (c *config) Validate() error { - if c.ChannelName == "" { - return fmt.Errorf("no channel name was configured or detected") - } - return nil -} - -func defaultConfig() config { - var c config - c.ChannelName = "cometd-channel" - return c -} +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cometd + +import "fmt" + +type config struct { + ChannelName string `config:"channel_name" validate:"required"` + Auth *authConfig `config:"auth"` +} + +func (c *config) Validate() error { + if c.ChannelName == "" { + return fmt.Errorf("no channel name was configured or detected") + } + return nil +} + +func defaultConfig() config { + var c config + c.ChannelName = "cometd-channel" + return c +} diff --git a/x-pack/filebeat/input/cometd/config_test.go b/x-pack/filebeat/input/cometd/config_test.go new file mode 100644 index 000000000000..b59ac403126a --- /dev/null +++ b/x-pack/filebeat/input/cometd/config_test.go @@ -0,0 +1,19 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cometd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Validate that it finds the application default credentials and does +// not trigger a config validation error because credentials were not +// set in the config. +func TestConfigValidate(t *testing.T) { + c := defaultConfig() + assert.NoError(t, c.Validate()) +} diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index 9904e4d3789e..8aac1dc14e34 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -1,192 +1,442 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -// A lot of code in this file would be updated as a part of data collection -// mechanism implementation. -// Please do not review it, as the current code would test the authentication -// mechanism only. - -package cometd - -import ( - "context" - "fmt" - "io/ioutil" - "net/http" - "sync" - "time" - - retryablehttp "github.com/hashicorp/go-retryablehttp" - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/filebeat/channel" - "github.com/elastic/beats/v7/filebeat/input" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/atomic" - "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" - "github.com/elastic/beats/v7/libbeat/logp" -) - -const ( - inputName = "cometd" -) - -type cometdInput struct { - config - - log *logp.Logger - inputCtx context.Context // Wraps the Done channel from parent input.Context. - - workerCtx context.Context // Worker goroutine context. It's cancelled when the input stops or the worker exits. - workerCancel context.CancelFunc // Used to signal that the worker should stop. - workerOnce sync.Once // Guarantees that the worker goroutine is only started once. - workerWg sync.WaitGroup // Waits on pubsub worker goroutine. - - ackedCount *atomic.Uint32 // Total number of successfully ACKed pubsub messages. - Transport httpcommon.HTTPTransportSettings `config:",inline"` - Retry retryConfig `config:"retry"` -} - -func init() { - err := input.Register(inputName, NewInput) - if err != nil { - panic(errors.Wrapf(err, "failed to register %v input", inputName)) - } -} - -// NewInput creates a new CometD Pub/Sub input that consumes events from -// a topic subscription. -func NewInput( - cfg *common.Config, - connector channel.Connector, - inputContext input.Context, -) (inp input.Input, err error) { - // Extract and validate the input's configuration. - conf := defaultConfig() - if err = cfg.Unpack(&conf); err != nil { - return nil, fmt.Errorf("cometd config: unpacking of config failed: %v", err) - } - - logger := logp.NewLogger("cometd").With( - "pubsub_channel", conf.ChannelName) - - // Wrap input.Context's Done channel with a context.Context. This goroutine - // stops with the parent closes the Done channel. - inputCtx, cancelInputCtx := context.WithCancel(context.Background()) - go func() { - defer cancelInputCtx() - select { - case <-inputContext.Done: - case <-inputCtx.Done(): - } - }() - - // If the input ever needs to be made restartable, then context would need - // to be recreated with each restart. - workerCtx, workerCancel := context.WithCancel(inputCtx) - - in := &cometdInput{ - config: conf, - log: logger, - inputCtx: inputCtx, - workerCtx: workerCtx, - workerCancel: workerCancel, - ackedCount: atomic.NewUint32(0), - } - - in.log.Info("Initialized cometD input.") - return in, nil -} - -// Run starts the pubsub input worker then returns. Only the first invocation -// will ever start the pubsub worker. -func (in *cometdInput) Run() { - in.workerOnce.Do(func() { - in.workerWg.Add(1) - go func() { - in.log.Info("Pub/Sub input worker has started.") - defer in.log.Info("Pub/Sub input worker has stopped.") - defer in.workerWg.Done() - defer in.workerCancel() - if err := in.run(); err != nil { - in.log.Error(err) - return - } - }() - }) -} - -// Stop stops the pubsub input and waits for it to fully stop. -func (in *cometdInput) Stop() { - in.workerCancel() - in.workerWg.Wait() -} - -// Wait is an alias for Stop. -func (in *cometdInput) Wait() { - in.Stop() -} - -func (in *cometdInput) run() error { - ctx, cancel := context.WithCancel(in.workerCtx) - defer cancel() - - client, err := in.newPubsubClient(ctx) - if err != nil { - return fmt.Errorf("cometd client: error creating pub-sub client: %v", err) - } - - in.log.Debug("client successfully created") - - // For testing http client - // Start of the test code for authentication and client creation - baseUrl := "" - req, err := http.NewRequest("GET", baseUrl, nil) - if err != nil { - in.log.Errorf("Error on request generation: %v", err.Error()) - } - resp, err := client.Do(req) - if err != nil { - in.log.Errorf("Error on response: %v", err) - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - in.log.Errorf("Error while reading the response bytes: %v", err) - } - - in.log.Infof("Response body: %v", string(body)) - // End of the test code for authentication and client creation - return nil -} - -type retryConfig struct { - MaxAttempts *int `config:"max_attempts"` - WaitMin *time.Duration `config:"wait_min"` - WaitMax *time.Duration `config:"wait_max"` -} - -func (in *cometdInput) newPubsubClient(ctx context.Context) (*http.Client, error) { - // Make retryable HTTP client - netHTTPClient, err := in.Transport.Client( - httpcommon.WithAPMHTTPInstrumentation(), - httpcommon.WithKeepaliveSettings{Disable: true}, - ) - if err != nil { - return nil, fmt.Errorf("cometd client: error on newHTTPClient: %v", err) - } - - client := &retryablehttp.Client{ - HTTPClient: netHTTPClient, - CheckRetry: retryablehttp.DefaultRetryPolicy, - Backoff: retryablehttp.DefaultBackoff, - } - - authClient, err := in.config.Auth.OAuth2.client(ctx, client.StandardClient()) - if err != nil { - return nil, fmt.Errorf("cometd client: error on authClient: client not created: %v", err) - } - return authClient, nil -} +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cometd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "sync" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/filebeat/channel" + "github.com/elastic/beats/v7/filebeat/input" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/atomic" + "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" + "github.com/elastic/beats/v7/libbeat/logp" +) + +const ( + cometdVersion = "38.0" + inputName = "cometd" + + // Replay accepts the following values + // -2: replay all events from past 24 hrs + // -1: start at current + // >= 0: start from this event number + Replay = -1 +) + +var ( + out chan TriggerEvent + status = Status{[]string{}, "", false} + wg sync.WaitGroup +) + +// Run starts the input worker then returns. Only the first invocation +// will ever start the worker. +func (in *cometdInput) Run() { + in.workerOnce.Do(func() { + in.workerWg.Add(1) + go func() { + in.log.Info("Input worker has started.") + defer in.log.Info("Input worker has stopped.") + defer in.workerWg.Done() + defer in.workerCancel() + if err := in.run(); err != nil { + in.log.Error(err) + return + } + }() + }) +} + +func (in *cometdInput) run() error { + ctx, cancel := context.WithCancel(in.workerCtx) + defer cancel() + b := Bayeux{} + creds, err := in.config.Auth.OAuth2.GetSalesforceCredentials() + if err != nil { + return fmt.Errorf("error while getting Salesforce credentials: %v", err) + } + out, err = b.TopicToChannel(ctx, creds, in.config.ChannelName, in.log) + if err != nil { + return fmt.Errorf("failed to subscribe to channel: %v", err) + } + + var event Event + for { + select { + case e := <-out: + if !e.Successful { + if e.Data.Object == nil { + return nil + } + msg, err := json.Marshal(e.Data.Object) + if err != nil { + return fmt.Errorf("JSON error: %v", err) + } + err = json.Unmarshal(e.Data.Object, &event) + if err != nil { + return fmt.Errorf("error while parsing JSON: %v", err) + } + if ok := in.outlet.OnEvent(makeEvent(event.EventId, string(msg))); !ok { + in.log.Debug("OnEvent returned false. Stopping input worker.") + cancel() + return fmt.Errorf("error ingesting data to elasticsearch") + } + } + } + } +} + +func init() { + err := input.Register(inputName, NewInput) + if err != nil { + panic(errors.Wrapf(err, "failed to register %v input", inputName)) + } +} + +// NewInput creates a new CometD input that consumes events from +// a topic subscription. +func NewInput( + cfg *common.Config, + connector channel.Connector, + inputContext input.Context, +) (inp input.Input, err error) { + // Extract and validate the input's configuration. + conf := defaultConfig() + if err = cfg.Unpack(&conf); err != nil { + return nil, err + } + + logger := logp.NewLogger("cometd").With( + "pubsub_channel", conf.ChannelName) + + // Wrap input.Context's Done channel with a context.Context. This goroutine + // stops with the parent closes the Done channel. + inputCtx, cancelInputCtx := context.WithCancel(context.Background()) + go func() { + defer cancelInputCtx() + select { + case <-inputContext.Done: + case <-inputCtx.Done(): + } + }() + + // If the input ever needs to be made restartable, then context would need + // to be recreated with each restart. + workerCtx, workerCancel := context.WithCancel(inputCtx) + + in := &cometdInput{ + config: conf, + log: logger, + inputCtx: inputCtx, + workerCtx: workerCtx, + workerCancel: workerCancel, + ackedCount: atomic.NewUint32(0), + } + + // Build outlet for events. + in.outlet, err = connector.Connect(cfg) + if err != nil { + return nil, err + } + in.log.Infof("Initialized %s input.", inputName) + return in, nil +} + +// Stop stops the input and waits for it to fully stop. +func (in *cometdInput) Stop() { + close(out) + in.workerCancel() + in.workerWg.Wait() +} + +// Wait is an alias for Stop. +func (in *cometdInput) Wait() { + in.Stop() +} + +type cometdInput struct { + config + + log *logp.Logger + outlet channel.Outleter // Output of received messages. + inputCtx context.Context // Wraps the Done channel from parent input.Context. + + workerCtx context.Context // Worker goroutine context. It's cancelled when the input stops or the worker exits. + workerCancel context.CancelFunc // Used to signal that the worker should stop. + workerOnce sync.Once // Guarantees that the worker goroutine is only started once. + workerWg sync.WaitGroup // Waits on worker goroutine. + + ackedCount *atomic.Uint32 // Total number of successfully ACKed messages. + Transport httpcommon.HTTPTransportSettings `config:",inline"` + Retry retryConfig `config:"retry"` +} + +// TriggerEvent describes an event received from Bayeaux Endpoint +type TriggerEvent struct { + Channel string `json:"channel"` + ClientID string `json:"clientId"` + Data struct { + Event struct { + CreatedDate time.Time `json:"createdDate"` + ReplayID int `json:"replayId"` + Type string `json:"type"` + } `json:"event"` + Object json.RawMessage `json:"payload"` + } `json:"data,omitempty"` + Successful bool `json:"successful,omitempty"` +} + +// Status is the state of success and subscribed channels +type Status struct { + channels []string + clientID string + connected bool +} + +type BayeuxHandshake []struct { + ClientID string `json:"clientId"` + Channel string `json:"channel"` + Ext struct { + Replay bool `json:"replay"` + } `json:"ext"` + MinimumVersion string `json:"minimumVersion"` + Successful bool `json:"successful"` + SupportedConnectionTypes []string `json:"supportedConnectionTypes"` + Version string `json:"version"` +} + +type Subscription struct { + ClientID string `json:"clientId"` + Channel string `json:"channel"` + Subscription string `json:"subscription"` + Successful bool `json:"successful"` +} + +type Credentials struct { + AccessToken string `json:"access_token"` + InstanceURL string `json:"instance_url"` + IssuedAt string `json:"issued_at"` + ID string `json:"id"` + TokenType string `json:"token_type"` + Signature string `json:"signature"` +} + +type clientIDAndCookies struct { + clientID string + cookies []*http.Cookie +} + +// Bayeux struct allow for centralized storage of creds, ids, and cookies +type Bayeux struct { + creds Credentials + id clientIDAndCookies +} + +type retryConfig struct { + MaxAttempts *int `config:"max_attempts"` + WaitMin *time.Duration `config:"wait_min"` + WaitMax *time.Duration `config:"wait_max"` +} + +type Event struct { + EventId string `json:"EventIdentifier"` +} + +func (c Credentials) bayeuxUrl() string { + return c.InstanceURL + "/cometd/" + cometdVersion +} + +// Call is the base function for making bayeux requests +func (b *Bayeux) call(body string, route string) (resp *http.Response, e error) { + var jsonStr = []byte(body) + req, err := http.NewRequest("POST", route, bytes.NewBuffer(jsonStr)) + if err != nil { + return nil, fmt.Errorf("bad Call request: %v", err) + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", b.creds.AccessToken)) + // Passing back cookies is required though undocumented in Salesforce API + // We were unable to get process working without passing cookies back to SF server. + // SF Reference: https://developer.salesforce.com/docs/atlas.en-us.api_streaming.meta/api_streaming/intro_client_specs.htm + for _, cookie := range b.id.cookies { + req.AddCookie(cookie) + } + + client := &http.Client{} + resp, err = client.Do(req) + if err == io.EOF { + return nil, fmt.Errorf("bad bayeuxCall io.EOF: %v", err) + } else if err != nil { + return nil, fmt.Errorf("unknown error: %v", err) + } + return resp, e +} + +func (b *Bayeux) getClientID() error { + handshake := `{"channel": "/meta/handshake", "supportedConnectionTypes": ["long-polling"], "version": "1.0"}` + // Stub out clientIDAndCookies for first bayeuxCall + resp, err := b.call(handshake, b.creds.bayeuxUrl()) + if err != nil { + return fmt.Errorf("cannot get client id: %v", err) + } + defer resp.Body.Close() + + decoder := json.NewDecoder(resp.Body) + var h BayeuxHandshake + if err := decoder.Decode(&h); err == io.EOF { + return fmt.Errorf("reached end of response: %v", err) + } else if err != nil { + return fmt.Errorf("error while reading response: %v", err) + } + creds := clientIDAndCookies{h[0].ClientID, resp.Cookies()} + b.id = creds + return nil +} + +func (b *Bayeux) subscribe(topic string, Replay int, log *logp.Logger) (Subscription, error) { + handshake := fmt.Sprintf(`{ + "channel": "/meta/subscribe", + "subscription": "%s", + "clientId": "%s", + "ext": { + "replay": {"%s": "%d"} + } + }`, topic, b.id.clientID, topic, Replay) + resp, err := b.call(handshake, b.creds.bayeuxUrl()) + if err != nil { + return Subscription{}, fmt.Errorf("error while subscribing: %v", err) + } + + defer resp.Body.Close() + + // Read the content + var content []byte + if resp.Body != nil { + content, err = ioutil.ReadAll(resp.Body) + if err != nil { + return Subscription{}, fmt.Errorf("error while reading content: %v", err) + } + } + // Restore the io.ReadCloser to its original state + resp.Body = ioutil.NopCloser(bytes.NewBuffer(content)) + + if resp.StatusCode > 299 { + return Subscription{}, fmt.Errorf("received non 2XX response: HTTP_CODE %v", resp.StatusCode) + } + decoder := json.NewDecoder(resp.Body) + var h []Subscription + if err := decoder.Decode(&h); err == io.EOF { + return Subscription{}, fmt.Errorf("reached end of response: %v", err) + } else if err != nil { + return Subscription{}, fmt.Errorf("error while reading response: %v", err) + } + sub := h[0] + status.connected = sub.Successful + status.clientID = sub.ClientID + status.channels = append(status.channels, topic) + log.Infof("Established connection(s): %+v", status) + return sub, nil +} + +func (b *Bayeux) connect(log *logp.Logger) (chan TriggerEvent, error) { + out = make(chan TriggerEvent) + go func() { + for { + postBody := fmt.Sprintf(`{"channel": "/meta/connect", "connectionType": "long-polling", "clientId": "%s"} `, b.id.clientID) + resp, err := b.call(postBody, b.creds.bayeuxUrl()) + if err != nil { + log.Warnf("Cannot connect to bayeux %s, trying again...", err) + } else { + // Read the content + var b []byte + if resp.Body != nil { + b, err = ioutil.ReadAll(resp.Body) + } + if err != nil { + return + } + // Restore the io.ReadCloser to its original state + resp.Body = ioutil.NopCloser(bytes.NewBuffer(b)) + var x []TriggerEvent + decoder := json.NewDecoder(resp.Body) + if err := decoder.Decode(&x); err != nil && err != io.EOF { + return + } + for _, e := range x { + out <- e + } + } + } + }() + return out, nil +} + +func (o *oAuth2Config) GetSalesforceCredentials() (Credentials, error) { + route := o.TokenURL + params := url.Values{"grant_type": {"password"}, + "client_id": {o.ClientID}, + "client_secret": {o.ClientSecret}, + "username": {o.User}, + "password": {o.Password}} + res, err := http.PostForm(route, params) + if err != nil { + return Credentials{}, fmt.Errorf("error while sending http request: %v", err) + } + decoder := json.NewDecoder(res.Body) + var creds Credentials + if err := decoder.Decode(&creds); err == io.EOF { + return Credentials{}, fmt.Errorf("reached end of response: %v", err) + } else if err != nil { + return Credentials{}, fmt.Errorf("error while reading response: %v", err) + } else if creds.AccessToken == "" { + return Credentials{}, fmt.Errorf("unable to fetch access token") + } + return creds, nil +} + +func (b *Bayeux) TopicToChannel(ctx context.Context, creds Credentials, topic string, log *logp.Logger) (chan TriggerEvent, error) { + b.creds = creds + err := b.getClientID() + if err != nil { + return make(chan TriggerEvent), fmt.Errorf("error while getting client ID: %v", err) + } + b.subscribe(topic, Replay, log) + c, err := b.connect(log) + if err != nil { + return make(chan TriggerEvent), fmt.Errorf("error while creating a connection: %v", err) + } + wg.Add(1) + return c, nil +} + +func makeEvent(id string, body string) beat.Event { + event := beat.Event{ + Timestamp: time.Now().UTC(), + Fields: common.MapStr{ + "event": common.MapStr{ + "id": id, + "created": time.Now().UTC(), + }, + "message": body, + }, + Private: body, + } + event.SetID(id) + + return event +} diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go new file mode 100644 index 000000000000..172b06e0b74b --- /dev/null +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -0,0 +1,19 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cometd + +import ( + "testing" + + "github.com/elastic/beats/v7/filebeat/input/inputtest" + "github.com/elastic/beats/v7/libbeat/common" +) + +func TestNewInputDone(t *testing.T) { + config := common.MapStr{ + "channel_name": "some-channel", + } + inputtest.AssertNotStartedInputCanBeDone(t, NewInput, &config) +} From 32a17909916d3dbb76d56581c69bdc466b90fd46 Mon Sep 17 00:00:00 2001 From: yug-crest Date: Thu, 16 Dec 2021 18:53:59 +0530 Subject: [PATCH 04/50] Update the approach of the channel --- x-pack/filebeat/input/cometd/input.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index 8aac1dc14e34..011d0ec3f1c6 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -39,7 +39,6 @@ const ( ) var ( - out chan TriggerEvent status = Status{[]string{}, "", false} wg sync.WaitGroup ) @@ -70,7 +69,7 @@ func (in *cometdInput) run() error { if err != nil { return fmt.Errorf("error while getting Salesforce credentials: %v", err) } - out, err = b.TopicToChannel(ctx, creds, in.config.ChannelName, in.log) + in.out, err = b.TopicToChannel(ctx, creds, in.config.ChannelName, in.log, in.out) if err != nil { return fmt.Errorf("failed to subscribe to channel: %v", err) } @@ -78,7 +77,7 @@ func (in *cometdInput) run() error { var event Event for { select { - case e := <-out: + case e := <-in.out: if !e.Successful { if e.Data.Object == nil { return nil @@ -93,6 +92,7 @@ func (in *cometdInput) run() error { } if ok := in.outlet.OnEvent(makeEvent(event.EventId, string(msg))); !ok { in.log.Debug("OnEvent returned false. Stopping input worker.") + close(in.out) cancel() return fmt.Errorf("error ingesting data to elasticsearch") } @@ -148,6 +148,9 @@ func NewInput( ackedCount: atomic.NewUint32(0), } + // Creating a new channel for cometd input + in.out = make(chan TriggerEvent) + // Build outlet for events. in.outlet, err = connector.Connect(cfg) if err != nil { @@ -159,7 +162,7 @@ func NewInput( // Stop stops the input and waits for it to fully stop. func (in *cometdInput) Stop() { - close(out) + close(in.out) in.workerCancel() in.workerWg.Wait() } @@ -184,6 +187,7 @@ type cometdInput struct { ackedCount *atomic.Uint32 // Total number of successfully ACKed messages. Transport httpcommon.HTTPTransportSettings `config:",inline"` Retry retryConfig `config:"retry"` + out chan TriggerEvent } // TriggerEvent describes an event received from Bayeaux Endpoint @@ -353,8 +357,7 @@ func (b *Bayeux) subscribe(topic string, Replay int, log *logp.Logger) (Subscrip return sub, nil } -func (b *Bayeux) connect(log *logp.Logger) (chan TriggerEvent, error) { - out = make(chan TriggerEvent) +func (b *Bayeux) connect(out chan TriggerEvent, log *logp.Logger) (chan TriggerEvent, error) { go func() { for { postBody := fmt.Sprintf(`{"channel": "/meta/connect", "connectionType": "long-polling", "clientId": "%s"} `, b.id.clientID) @@ -409,14 +412,14 @@ func (o *oAuth2Config) GetSalesforceCredentials() (Credentials, error) { return creds, nil } -func (b *Bayeux) TopicToChannel(ctx context.Context, creds Credentials, topic string, log *logp.Logger) (chan TriggerEvent, error) { +func (b *Bayeux) TopicToChannel(ctx context.Context, creds Credentials, topic string, log *logp.Logger, out chan TriggerEvent) (chan TriggerEvent, error) { b.creds = creds err := b.getClientID() if err != nil { return make(chan TriggerEvent), fmt.Errorf("error while getting client ID: %v", err) } b.subscribe(topic, Replay, log) - c, err := b.connect(log) + c, err := b.connect(out, log) if err != nil { return make(chan TriggerEvent), fmt.Errorf("error while creating a connection: %v", err) } From 3855505fa6cc6501a69c6850e7b602a7a561e261 Mon Sep 17 00:00:00 2001 From: yug-crest Date: Thu, 16 Dec 2021 18:57:27 +0530 Subject: [PATCH 05/50] Add asciidoc for cometd input --- .../docs/inputs/input-cometd.asciidoc | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 x-pack/filebeat/docs/inputs/input-cometd.asciidoc diff --git a/x-pack/filebeat/docs/inputs/input-cometd.asciidoc b/x-pack/filebeat/docs/inputs/input-cometd.asciidoc new file mode 100644 index 000000000000..66929ab8e49e --- /dev/null +++ b/x-pack/filebeat/docs/inputs/input-cometd.asciidoc @@ -0,0 +1,65 @@ +[role="xpack"] + +:type: cometd + +[id="{beatname_lc}-input-{type}"] +=== CometD input + +++++ +CometD +++++ + +Use the `cometd` input to stream the real-time events from a Salesforce generic subscription Push Topic. + +This input can, for example, be used to receive Login and Logout events that are generated when users log in or out of the Salesforce instance. + +Example configuration: + +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: cometd + channel_name: /event/MyEventStream + auth.oauth2: + client.id: my-client-id + client.secret: my-client-secret + token_url: https://login.salesforce.com/services/oauth2/token + user: my.email@mail.com + password: my-password +---- + +==== Configuration options + +The `cometd` input supports the following configuration options. + +[float] +==== `channel_name` + +Salesforce generic subscription Push Topic name. Required. + +[float] +==== `auth.oauth2.client.id` + +The client ID used as part of the authentication flow. Required. + +[float] +==== `auth.oauth2.client.secret` + +The client secret used as part of the authentication flow. Required. + +[float] +==== `auth.oauth2.token_url` + +The endpoint that will be used to generate the token during the oauth2 flow. Required. + +[float] +==== `auth.oauth2.user` + +The user used as part of the authentication flow. It is required for authentication - grant type password. Required. + +[float] +==== `auth.oauth2.password` + +The password used as part of the authentication flow. It is required for authentication - grant type password. Required. + +:type!: From 6e585ade4227fc9183bb2c21ebbcce9ef33328e7 Mon Sep 17 00:00:00 2001 From: yug-crest Date: Fri, 24 Dec 2021 18:49:45 +0530 Subject: [PATCH 06/50] Resolve review comments --- x-pack/filebeat/input/cometd/config_auth.go | 49 ++++-- x-pack/filebeat/input/cometd/input.go | 172 ++++++++------------ 2 files changed, 103 insertions(+), 118 deletions(-) diff --git a/x-pack/filebeat/input/cometd/config_auth.go b/x-pack/filebeat/input/cometd/config_auth.go index f362fda76f29..e4dafad3c5a8 100644 --- a/x-pack/filebeat/input/cometd/config_auth.go +++ b/x-pack/filebeat/input/cometd/config_auth.go @@ -5,16 +5,14 @@ package cometd import ( - "context" + "encoding/json" "errors" "fmt" + "io" "net/http" - - "golang.org/x/oauth2" + "net/url" ) -const authStyleInParams = 1 - type authConfig struct { OAuth2 *oAuth2Config `config:"oauth2"` } @@ -30,22 +28,37 @@ type oAuth2Config struct { TokenURL string `config:"token_url"` } +type credentials struct { + AccessToken string `json:"access_token"` + InstanceURL string `json:"instance_url"` + IssuedAt string `json:"issued_at"` + ID string `json:"id"` + TokenType string `json:"token_type"` + Signature string `json:"signature"` +} + // Client wraps the given http.Client and returns a new one that will use the oauth authentication. -func (o *oAuth2Config) client(ctx context.Context, client *http.Client) (*http.Client, error) { - ctx = context.WithValue(ctx, oauth2.HTTPClient, client) - conf := &oauth2.Config{ - ClientID: o.ClientID, - ClientSecret: o.ClientSecret, - Endpoint: oauth2.Endpoint{ - TokenURL: o.TokenURL, - AuthStyle: authStyleInParams, - }, - } - token, err := conf.PasswordCredentialsToken(ctx, o.User, o.Password) +func (o *oAuth2Config) client() (credentials, error) { + route := o.TokenURL + params := url.Values{"grant_type": {"password"}, + "client_id": {o.ClientID}, + "client_secret": {o.ClientSecret}, + "username": {o.User}, + "password": {o.Password}} + response, err := http.PostForm(route, params) if err != nil { - return nil, fmt.Errorf("oauth2 client: error loading credentials using user and password: %w", err) + return credentials{}, fmt.Errorf("error while sending http request: %v", err) + } + decoder := json.NewDecoder(response.Body) + var creds credentials + if err := decoder.Decode(&creds); err == io.EOF { + return credentials{}, fmt.Errorf("reached end of response: %v", err) + } else if err != nil { + return credentials{}, fmt.Errorf("error while reading response: %v", err) + } else if creds.AccessToken == "" { + return credentials{}, fmt.Errorf("unable to fetch access token") } - return conf.Client(ctx, token), nil + return creds, nil } // Validate checks if oauth2 config is valid. diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index 011d0ec3f1c6..0a14c6607b08 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -12,7 +12,6 @@ import ( "io" "io/ioutil" "net/http" - "net/url" "sync" "time" @@ -28,9 +27,11 @@ import ( ) const ( - cometdVersion = "38.0" - inputName = "cometd" - + cometdVersion = "38.0" + inputName = "cometd" + subscribe_channel = "/meta/subscribe" + connetion_type = "long-polling" + connect_channel = "/meta/connect" // Replay accepts the following values // -2: replay all events from past 24 hrs // -1: start at current @@ -39,8 +40,8 @@ const ( ) var ( - status = Status{[]string{}, "", false} - wg sync.WaitGroup + st = status{[]string{}, "", false} + wg sync.WaitGroup ) // Run starts the input worker then returns. Only the first invocation @@ -64,8 +65,8 @@ func (in *cometdInput) Run() { func (in *cometdInput) run() error { ctx, cancel := context.WithCancel(in.workerCtx) defer cancel() - b := Bayeux{} - creds, err := in.config.Auth.OAuth2.GetSalesforceCredentials() + b := bayeux{} + creds, err := in.config.Auth.OAuth2.client() if err != nil { return fmt.Errorf("error while getting Salesforce credentials: %v", err) } @@ -75,30 +76,33 @@ func (in *cometdInput) run() error { } var event Event - for { - select { - case e := <-in.out: - if !e.Successful { - if e.Data.Object == nil { - return nil - } - msg, err := json.Marshal(e.Data.Object) - if err != nil { - return fmt.Errorf("JSON error: %v", err) - } - err = json.Unmarshal(e.Data.Object, &event) - if err != nil { - return fmt.Errorf("error while parsing JSON: %v", err) - } - if ok := in.outlet.OnEvent(makeEvent(event.EventId, string(msg))); !ok { - in.log.Debug("OnEvent returned false. Stopping input worker.") - close(in.out) - cancel() - return fmt.Errorf("error ingesting data to elasticsearch") - } + for e := range in.out { + if !e.Successful { + // To handle the last response where the object received was empty + if e.Data.Object == nil { + return nil + } + + // Convert json.RawMessage response to []byte + msg, err := json.Marshal(e.Data.Object) + if err != nil { + return fmt.Errorf("JSON error: %v", err) + } + + // Extract event IDs from json.RawMessage + err = json.Unmarshal(e.Data.Object, &event) + if err != nil { + return fmt.Errorf("error while parsing JSON: %v", err) + } + if ok := in.outlet.OnEvent(makeEvent(event.EventId, string(msg))); !ok { + in.log.Debug("OnEvent returned false. Stopping input worker.") + close(in.out) + cancel() + return fmt.Errorf("error ingesting data to elasticsearch") } } } + return nil } func init() { @@ -149,7 +153,7 @@ func NewInput( } // Creating a new channel for cometd input - in.out = make(chan TriggerEvent) + in.out = make(chan triggerEvent) // Build outlet for events. in.outlet, err = connector.Connect(cfg) @@ -187,11 +191,11 @@ type cometdInput struct { ackedCount *atomic.Uint32 // Total number of successfully ACKed messages. Transport httpcommon.HTTPTransportSettings `config:",inline"` Retry retryConfig `config:"retry"` - out chan TriggerEvent + out chan triggerEvent } -// TriggerEvent describes an event received from Bayeaux Endpoint -type TriggerEvent struct { +// triggerEvent describes an event received from Bayeaux Endpoint +type triggerEvent struct { Channel string `json:"channel"` ClientID string `json:"clientId"` Data struct { @@ -205,14 +209,14 @@ type TriggerEvent struct { Successful bool `json:"successful,omitempty"` } -// Status is the state of success and subscribed channels -type Status struct { +// status is the state of success and subscribed channels +type status struct { channels []string clientID string connected bool } -type BayeuxHandshake []struct { +type bayeuxHandshake []struct { ClientID string `json:"clientId"` Channel string `json:"channel"` Ext struct { @@ -224,31 +228,23 @@ type BayeuxHandshake []struct { Version string `json:"version"` } -type Subscription struct { +type subscription struct { ClientID string `json:"clientId"` Channel string `json:"channel"` Subscription string `json:"subscription"` Successful bool `json:"successful"` } -type Credentials struct { - AccessToken string `json:"access_token"` - InstanceURL string `json:"instance_url"` - IssuedAt string `json:"issued_at"` - ID string `json:"id"` - TokenType string `json:"token_type"` - Signature string `json:"signature"` -} - type clientIDAndCookies struct { clientID string cookies []*http.Cookie } -// Bayeux struct allow for centralized storage of creds, ids, and cookies -type Bayeux struct { - creds Credentials - id clientIDAndCookies +// bayeux struct allow for centralized storage of creds, ids, and cookies +type bayeux struct { + creds credentials + id clientIDAndCookies + client *http.Client } type retryConfig struct { @@ -261,12 +257,12 @@ type Event struct { EventId string `json:"EventIdentifier"` } -func (c Credentials) bayeuxUrl() string { +func (c credentials) bayeuxUrl() string { return c.InstanceURL + "/cometd/" + cometdVersion } // Call is the base function for making bayeux requests -func (b *Bayeux) call(body string, route string) (resp *http.Response, e error) { +func (b *bayeux) call(body string, route string) (*http.Response, error) { var jsonStr = []byte(body) req, err := http.NewRequest("POST", route, bytes.NewBuffer(jsonStr)) if err != nil { @@ -275,23 +271,21 @@ func (b *Bayeux) call(body string, route string) (resp *http.Response, e error) req.Header.Add("Content-Type", "application/json") req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", b.creds.AccessToken)) // Passing back cookies is required though undocumented in Salesforce API - // We were unable to get process working without passing cookies back to SF server. // SF Reference: https://developer.salesforce.com/docs/atlas.en-us.api_streaming.meta/api_streaming/intro_client_specs.htm for _, cookie := range b.id.cookies { req.AddCookie(cookie) } - client := &http.Client{} - resp, err = client.Do(req) + resp, err := b.client.Do(req) if err == io.EOF { return nil, fmt.Errorf("bad bayeuxCall io.EOF: %v", err) } else if err != nil { return nil, fmt.Errorf("unknown error: %v", err) } - return resp, e + return resp, nil } -func (b *Bayeux) getClientID() error { +func (b *bayeux) getClientID() error { handshake := `{"channel": "/meta/handshake", "supportedConnectionTypes": ["long-polling"], "version": "1.0"}` // Stub out clientIDAndCookies for first bayeuxCall resp, err := b.call(handshake, b.creds.bayeuxUrl()) @@ -301,7 +295,7 @@ func (b *Bayeux) getClientID() error { defer resp.Body.Close() decoder := json.NewDecoder(resp.Body) - var h BayeuxHandshake + var h bayeuxHandshake if err := decoder.Decode(&h); err == io.EOF { return fmt.Errorf("reached end of response: %v", err) } else if err != nil { @@ -312,18 +306,18 @@ func (b *Bayeux) getClientID() error { return nil } -func (b *Bayeux) subscribe(topic string, Replay int, log *logp.Logger) (Subscription, error) { +func (b *bayeux) subscribe(topic string, Replay int, log *logp.Logger) (subscription, error) { handshake := fmt.Sprintf(`{ - "channel": "/meta/subscribe", + "channel": "%s", "subscription": "%s", "clientId": "%s", "ext": { "replay": {"%s": "%d"} } - }`, topic, b.id.clientID, topic, Replay) + }`, subscribe_channel, topic, b.id.clientID, topic, Replay) resp, err := b.call(handshake, b.creds.bayeuxUrl()) if err != nil { - return Subscription{}, fmt.Errorf("error while subscribing: %v", err) + return subscription{}, fmt.Errorf("error while subscribing: %v", err) } defer resp.Body.Close() @@ -333,34 +327,34 @@ func (b *Bayeux) subscribe(topic string, Replay int, log *logp.Logger) (Subscrip if resp.Body != nil { content, err = ioutil.ReadAll(resp.Body) if err != nil { - return Subscription{}, fmt.Errorf("error while reading content: %v", err) + return subscription{}, fmt.Errorf("error while reading content: %v", err) } } // Restore the io.ReadCloser to its original state resp.Body = ioutil.NopCloser(bytes.NewBuffer(content)) if resp.StatusCode > 299 { - return Subscription{}, fmt.Errorf("received non 2XX response: HTTP_CODE %v", resp.StatusCode) + return subscription{}, fmt.Errorf("received non 2XX response: HTTP_CODE %v", resp.StatusCode) } decoder := json.NewDecoder(resp.Body) - var h []Subscription + var h []subscription if err := decoder.Decode(&h); err == io.EOF { - return Subscription{}, fmt.Errorf("reached end of response: %v", err) + return subscription{}, fmt.Errorf("reached end of response: %v", err) } else if err != nil { - return Subscription{}, fmt.Errorf("error while reading response: %v", err) + return subscription{}, fmt.Errorf("error while reading response: %v", err) } sub := h[0] - status.connected = sub.Successful - status.clientID = sub.ClientID - status.channels = append(status.channels, topic) - log.Infof("Established connection(s): %+v", status) + st.connected = sub.Successful + st.clientID = sub.ClientID + st.channels = append(st.channels, topic) + log.Infof("Established connection(s): %+v", st) return sub, nil } -func (b *Bayeux) connect(out chan TriggerEvent, log *logp.Logger) (chan TriggerEvent, error) { +func (b *bayeux) connect(out chan triggerEvent, log *logp.Logger) (chan triggerEvent, error) { go func() { for { - postBody := fmt.Sprintf(`{"channel": "/meta/connect", "connectionType": "long-polling", "clientId": "%s"} `, b.id.clientID) + postBody := fmt.Sprintf(`{"channel": "%s", "connectionType": "%s", "clientId": "%s"} `, connect_channel, connetion_type, b.id.clientID) resp, err := b.call(postBody, b.creds.bayeuxUrl()) if err != nil { log.Warnf("Cannot connect to bayeux %s, trying again...", err) @@ -375,7 +369,7 @@ func (b *Bayeux) connect(out chan TriggerEvent, log *logp.Logger) (chan TriggerE } // Restore the io.ReadCloser to its original state resp.Body = ioutil.NopCloser(bytes.NewBuffer(b)) - var x []TriggerEvent + var x []triggerEvent decoder := json.NewDecoder(resp.Body) if err := decoder.Decode(&x); err != nil && err != io.EOF { return @@ -389,39 +383,17 @@ func (b *Bayeux) connect(out chan TriggerEvent, log *logp.Logger) (chan TriggerE return out, nil } -func (o *oAuth2Config) GetSalesforceCredentials() (Credentials, error) { - route := o.TokenURL - params := url.Values{"grant_type": {"password"}, - "client_id": {o.ClientID}, - "client_secret": {o.ClientSecret}, - "username": {o.User}, - "password": {o.Password}} - res, err := http.PostForm(route, params) - if err != nil { - return Credentials{}, fmt.Errorf("error while sending http request: %v", err) - } - decoder := json.NewDecoder(res.Body) - var creds Credentials - if err := decoder.Decode(&creds); err == io.EOF { - return Credentials{}, fmt.Errorf("reached end of response: %v", err) - } else if err != nil { - return Credentials{}, fmt.Errorf("error while reading response: %v", err) - } else if creds.AccessToken == "" { - return Credentials{}, fmt.Errorf("unable to fetch access token") - } - return creds, nil -} - -func (b *Bayeux) TopicToChannel(ctx context.Context, creds Credentials, topic string, log *logp.Logger, out chan TriggerEvent) (chan TriggerEvent, error) { +func (b *bayeux) TopicToChannel(ctx context.Context, creds credentials, topic string, log *logp.Logger, out chan triggerEvent) (chan triggerEvent, error) { b.creds = creds + b.client = &http.Client{} err := b.getClientID() if err != nil { - return make(chan TriggerEvent), fmt.Errorf("error while getting client ID: %v", err) + return make(chan triggerEvent), fmt.Errorf("error while getting client ID: %v", err) } b.subscribe(topic, Replay, log) c, err := b.connect(out, log) if err != nil { - return make(chan TriggerEvent), fmt.Errorf("error while creating a connection: %v", err) + return make(chan triggerEvent), fmt.Errorf("error while creating a connection: %v", err) } wg.Add(1) return c, nil From d67702d7b486bb724f17080c520a844be0bad408 Mon Sep 17 00:00:00 2001 From: yug-crest Date: Thu, 13 Jan 2022 14:07:12 +0530 Subject: [PATCH 07/50] Update cometd input to import forked dependency --- x-pack/filebeat/input/cometd/config_auth.go | 50 +--- x-pack/filebeat/input/cometd/input.go | 249 ++------------------ 2 files changed, 23 insertions(+), 276 deletions(-) diff --git a/x-pack/filebeat/input/cometd/config_auth.go b/x-pack/filebeat/input/cometd/config_auth.go index e4dafad3c5a8..cf2b88a40c2e 100644 --- a/x-pack/filebeat/input/cometd/config_auth.go +++ b/x-pack/filebeat/input/cometd/config_auth.go @@ -5,12 +5,7 @@ package cometd import ( - "encoding/json" "errors" - "fmt" - "io" - "net/http" - "net/url" ) type authConfig struct { @@ -19,46 +14,11 @@ type authConfig struct { type oAuth2Config struct { // common oauth fields - ClientID string `config:"client.id"` - ClientSecret string `config:"client.secret"` - User string `config:"user"` - Password string `config:"password"` - EndpointParams map[string][]string `config:"endpoint_params"` - Scopes []string `config:"scopes"` - TokenURL string `config:"token_url"` -} - -type credentials struct { - AccessToken string `json:"access_token"` - InstanceURL string `json:"instance_url"` - IssuedAt string `json:"issued_at"` - ID string `json:"id"` - TokenType string `json:"token_type"` - Signature string `json:"signature"` -} - -// Client wraps the given http.Client and returns a new one that will use the oauth authentication. -func (o *oAuth2Config) client() (credentials, error) { - route := o.TokenURL - params := url.Values{"grant_type": {"password"}, - "client_id": {o.ClientID}, - "client_secret": {o.ClientSecret}, - "username": {o.User}, - "password": {o.Password}} - response, err := http.PostForm(route, params) - if err != nil { - return credentials{}, fmt.Errorf("error while sending http request: %v", err) - } - decoder := json.NewDecoder(response.Body) - var creds credentials - if err := decoder.Decode(&creds); err == io.EOF { - return credentials{}, fmt.Errorf("reached end of response: %v", err) - } else if err != nil { - return credentials{}, fmt.Errorf("error while reading response: %v", err) - } else if creds.AccessToken == "" { - return credentials{}, fmt.Errorf("unable to fetch access token") - } - return creds, nil + ClientID string `config:"client.id"` + ClientSecret string `config:"client.secret"` + User string `config:"user"` + Password string `config:"password"` + TokenURL string `config:"token_url"` } // Validate checks if oauth2 config is valid. diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index 0a14c6607b08..4b8c90ef6d6e 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -5,13 +5,10 @@ package cometd import ( - "bytes" "context" "encoding/json" "fmt" - "io" - "io/ioutil" - "net/http" + "os" "sync" "time" @@ -24,24 +21,12 @@ import ( "github.com/elastic/beats/v7/libbeat/common/atomic" "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" "github.com/elastic/beats/v7/libbeat/logp" -) -const ( - cometdVersion = "38.0" - inputName = "cometd" - subscribe_channel = "/meta/subscribe" - connetion_type = "long-polling" - connect_channel = "/meta/connect" - // Replay accepts the following values - // -2: replay all events from past 24 hrs - // -1: start at current - // >= 0: start from this event number - Replay = -1 + bay "github.com/kush-elastic/bayeux" ) -var ( - st = status{[]string{}, "", false} - wg sync.WaitGroup +const ( + inputName = "cometd" ) // Run starts the input worker then returns. Only the first invocation @@ -63,41 +48,33 @@ func (in *cometdInput) Run() { } func (in *cometdInput) run() error { - ctx, cancel := context.WithCancel(in.workerCtx) - defer cancel() - b := bayeux{} - creds, err := in.config.Auth.OAuth2.client() - if err != nil { - return fmt.Errorf("error while getting Salesforce credentials: %v", err) - } - in.out, err = b.TopicToChannel(ctx, creds, in.config.ChannelName, in.log, in.out) - if err != nil { - return fmt.Errorf("failed to subscribe to channel: %v", err) - } + + b := bay.Bayeux{} + creds := bay.GetSalesforceCredentials() + in.out = b.Channel(in.out, "-1", creds, in.config.ChannelName) var event Event for e := range in.out { if !e.Successful { // To handle the last response where the object received was empty - if e.Data.Object == nil { + if e.Data.Payload == nil { return nil } // Convert json.RawMessage response to []byte - msg, err := json.Marshal(e.Data.Object) + msg, err := json.Marshal(e.Data.Payload) if err != nil { return fmt.Errorf("JSON error: %v", err) } // Extract event IDs from json.RawMessage - err = json.Unmarshal(e.Data.Object, &event) + err = json.Unmarshal(e.Data.Payload, &event) if err != nil { return fmt.Errorf("error while parsing JSON: %v", err) } if ok := in.outlet.OnEvent(makeEvent(event.EventId, string(msg))); !ok { in.log.Debug("OnEvent returned false. Stopping input worker.") close(in.out) - cancel() return fmt.Errorf("error ingesting data to elasticsearch") } } @@ -125,6 +102,11 @@ func NewInput( return nil, err } + os.Setenv("SALESFORCE_CONSUMER_KEY", conf.Auth.OAuth2.ClientID) + os.Setenv("SALESFORCE_CONSUMER_SECRET", conf.Auth.OAuth2.ClientSecret) + os.Setenv("SALESFORCE_USER", conf.Auth.OAuth2.User) + os.Setenv("SALESFORCE_PASSWORD", conf.Auth.OAuth2.Password) + logger := logp.NewLogger("cometd").With( "pubsub_channel", conf.ChannelName) @@ -153,7 +135,7 @@ func NewInput( } // Creating a new channel for cometd input - in.out = make(chan triggerEvent) + in.out = make(chan bay.TriggerEvent) // Build outlet for events. in.outlet, err = connector.Connect(cfg) @@ -191,60 +173,7 @@ type cometdInput struct { ackedCount *atomic.Uint32 // Total number of successfully ACKed messages. Transport httpcommon.HTTPTransportSettings `config:",inline"` Retry retryConfig `config:"retry"` - out chan triggerEvent -} - -// triggerEvent describes an event received from Bayeaux Endpoint -type triggerEvent struct { - Channel string `json:"channel"` - ClientID string `json:"clientId"` - Data struct { - Event struct { - CreatedDate time.Time `json:"createdDate"` - ReplayID int `json:"replayId"` - Type string `json:"type"` - } `json:"event"` - Object json.RawMessage `json:"payload"` - } `json:"data,omitempty"` - Successful bool `json:"successful,omitempty"` -} - -// status is the state of success and subscribed channels -type status struct { - channels []string - clientID string - connected bool -} - -type bayeuxHandshake []struct { - ClientID string `json:"clientId"` - Channel string `json:"channel"` - Ext struct { - Replay bool `json:"replay"` - } `json:"ext"` - MinimumVersion string `json:"minimumVersion"` - Successful bool `json:"successful"` - SupportedConnectionTypes []string `json:"supportedConnectionTypes"` - Version string `json:"version"` -} - -type subscription struct { - ClientID string `json:"clientId"` - Channel string `json:"channel"` - Subscription string `json:"subscription"` - Successful bool `json:"successful"` -} - -type clientIDAndCookies struct { - clientID string - cookies []*http.Cookie -} - -// bayeux struct allow for centralized storage of creds, ids, and cookies -type bayeux struct { - creds credentials - id clientIDAndCookies - client *http.Client + out chan bay.TriggerEvent } type retryConfig struct { @@ -257,148 +186,6 @@ type Event struct { EventId string `json:"EventIdentifier"` } -func (c credentials) bayeuxUrl() string { - return c.InstanceURL + "/cometd/" + cometdVersion -} - -// Call is the base function for making bayeux requests -func (b *bayeux) call(body string, route string) (*http.Response, error) { - var jsonStr = []byte(body) - req, err := http.NewRequest("POST", route, bytes.NewBuffer(jsonStr)) - if err != nil { - return nil, fmt.Errorf("bad Call request: %v", err) - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", b.creds.AccessToken)) - // Passing back cookies is required though undocumented in Salesforce API - // SF Reference: https://developer.salesforce.com/docs/atlas.en-us.api_streaming.meta/api_streaming/intro_client_specs.htm - for _, cookie := range b.id.cookies { - req.AddCookie(cookie) - } - - resp, err := b.client.Do(req) - if err == io.EOF { - return nil, fmt.Errorf("bad bayeuxCall io.EOF: %v", err) - } else if err != nil { - return nil, fmt.Errorf("unknown error: %v", err) - } - return resp, nil -} - -func (b *bayeux) getClientID() error { - handshake := `{"channel": "/meta/handshake", "supportedConnectionTypes": ["long-polling"], "version": "1.0"}` - // Stub out clientIDAndCookies for first bayeuxCall - resp, err := b.call(handshake, b.creds.bayeuxUrl()) - if err != nil { - return fmt.Errorf("cannot get client id: %v", err) - } - defer resp.Body.Close() - - decoder := json.NewDecoder(resp.Body) - var h bayeuxHandshake - if err := decoder.Decode(&h); err == io.EOF { - return fmt.Errorf("reached end of response: %v", err) - } else if err != nil { - return fmt.Errorf("error while reading response: %v", err) - } - creds := clientIDAndCookies{h[0].ClientID, resp.Cookies()} - b.id = creds - return nil -} - -func (b *bayeux) subscribe(topic string, Replay int, log *logp.Logger) (subscription, error) { - handshake := fmt.Sprintf(`{ - "channel": "%s", - "subscription": "%s", - "clientId": "%s", - "ext": { - "replay": {"%s": "%d"} - } - }`, subscribe_channel, topic, b.id.clientID, topic, Replay) - resp, err := b.call(handshake, b.creds.bayeuxUrl()) - if err != nil { - return subscription{}, fmt.Errorf("error while subscribing: %v", err) - } - - defer resp.Body.Close() - - // Read the content - var content []byte - if resp.Body != nil { - content, err = ioutil.ReadAll(resp.Body) - if err != nil { - return subscription{}, fmt.Errorf("error while reading content: %v", err) - } - } - // Restore the io.ReadCloser to its original state - resp.Body = ioutil.NopCloser(bytes.NewBuffer(content)) - - if resp.StatusCode > 299 { - return subscription{}, fmt.Errorf("received non 2XX response: HTTP_CODE %v", resp.StatusCode) - } - decoder := json.NewDecoder(resp.Body) - var h []subscription - if err := decoder.Decode(&h); err == io.EOF { - return subscription{}, fmt.Errorf("reached end of response: %v", err) - } else if err != nil { - return subscription{}, fmt.Errorf("error while reading response: %v", err) - } - sub := h[0] - st.connected = sub.Successful - st.clientID = sub.ClientID - st.channels = append(st.channels, topic) - log.Infof("Established connection(s): %+v", st) - return sub, nil -} - -func (b *bayeux) connect(out chan triggerEvent, log *logp.Logger) (chan triggerEvent, error) { - go func() { - for { - postBody := fmt.Sprintf(`{"channel": "%s", "connectionType": "%s", "clientId": "%s"} `, connect_channel, connetion_type, b.id.clientID) - resp, err := b.call(postBody, b.creds.bayeuxUrl()) - if err != nil { - log.Warnf("Cannot connect to bayeux %s, trying again...", err) - } else { - // Read the content - var b []byte - if resp.Body != nil { - b, err = ioutil.ReadAll(resp.Body) - } - if err != nil { - return - } - // Restore the io.ReadCloser to its original state - resp.Body = ioutil.NopCloser(bytes.NewBuffer(b)) - var x []triggerEvent - decoder := json.NewDecoder(resp.Body) - if err := decoder.Decode(&x); err != nil && err != io.EOF { - return - } - for _, e := range x { - out <- e - } - } - } - }() - return out, nil -} - -func (b *bayeux) TopicToChannel(ctx context.Context, creds credentials, topic string, log *logp.Logger, out chan triggerEvent) (chan triggerEvent, error) { - b.creds = creds - b.client = &http.Client{} - err := b.getClientID() - if err != nil { - return make(chan triggerEvent), fmt.Errorf("error while getting client ID: %v", err) - } - b.subscribe(topic, Replay, log) - c, err := b.connect(out, log) - if err != nil { - return make(chan triggerEvent), fmt.Errorf("error while creating a connection: %v", err) - } - wg.Add(1) - return c, nil -} - func makeEvent(id string, body string) beat.Event { event := beat.Event{ Timestamp: time.Now().UTC(), From 8a24338867febf602c1cc6b77a1f27985b66642b Mon Sep 17 00:00:00 2001 From: yug-elastic Date: Fri, 28 Jan 2022 18:46:05 +0530 Subject: [PATCH 08/50] Temorarily remove changelog entry --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8f24381d583d..3e59467780b5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -354,7 +354,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support in aws-s3 input for s3 notification from SNS to SQS. {pull}28800[28800] - Add support in aws-s3 input for custom script parsing of s3 notifications. {pull}28946[28946] - Improve error handling in aws-s3 input for malformed s3 notifications. {issue}28828[28828] {pull}28946[28946] -- Add commetD input to retrieve logs from Salesforce Streaming API. {pull}#[#] - Add support for parsers on journald input {pull}29070[29070] - Add support in httpjson input for oAuth2ProviderDefault of password grant_type. {pull}29087[29087] From ab2a42fe9c0addded6b225bdd870ffeedd3564a9 Mon Sep 17 00:00:00 2001 From: yug-elastic Date: Fri, 28 Jan 2022 19:56:40 +0530 Subject: [PATCH 09/50] Run mage check and update --- x-pack/filebeat/input/cometd/config.go | 50 +-- x-pack/filebeat/input/cometd/config_test.go | 38 +- x-pack/filebeat/input/cometd/input.go | 408 ++++++++++---------- x-pack/filebeat/input/cometd/input_test.go | 38 +- 4 files changed, 267 insertions(+), 267 deletions(-) diff --git a/x-pack/filebeat/input/cometd/config.go b/x-pack/filebeat/input/cometd/config.go index 6e32f1fa16aa..0b196cf55162 100644 --- a/x-pack/filebeat/input/cometd/config.go +++ b/x-pack/filebeat/input/cometd/config.go @@ -1,25 +1,25 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package cometd - -import "fmt" - -type config struct { - ChannelName string `config:"channel_name" validate:"required"` - Auth *authConfig `config:"auth"` -} - -func (c *config) Validate() error { - if c.ChannelName == "" { - return fmt.Errorf("no channel name was configured or detected") - } - return nil -} - -func defaultConfig() config { - var c config - c.ChannelName = "cometd-channel" - return c -} +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cometd + +import "fmt" + +type config struct { + ChannelName string `config:"channel_name" validate:"required"` + Auth *authConfig `config:"auth"` +} + +func (c *config) Validate() error { + if c.ChannelName == "" { + return fmt.Errorf("no channel name was configured or detected") + } + return nil +} + +func defaultConfig() config { + var c config + c.ChannelName = "cometd-channel" + return c +} diff --git a/x-pack/filebeat/input/cometd/config_test.go b/x-pack/filebeat/input/cometd/config_test.go index b59ac403126a..9af56d07d3f6 100644 --- a/x-pack/filebeat/input/cometd/config_test.go +++ b/x-pack/filebeat/input/cometd/config_test.go @@ -1,19 +1,19 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package cometd - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -// Validate that it finds the application default credentials and does -// not trigger a config validation error because credentials were not -// set in the config. -func TestConfigValidate(t *testing.T) { - c := defaultConfig() - assert.NoError(t, c.Validate()) -} +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cometd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Validate that it finds the application default credentials and does +// not trigger a config validation error because credentials were not +// set in the config. +func TestConfigValidate(t *testing.T) { + c := defaultConfig() + assert.NoError(t, c.Validate()) +} diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index 4b8c90ef6d6e..c1de6db2cc59 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -1,204 +1,204 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package cometd - -import ( - "context" - "encoding/json" - "fmt" - "os" - "sync" - "time" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/filebeat/channel" - "github.com/elastic/beats/v7/filebeat/input" - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/atomic" - "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" - "github.com/elastic/beats/v7/libbeat/logp" - - bay "github.com/kush-elastic/bayeux" -) - -const ( - inputName = "cometd" -) - -// Run starts the input worker then returns. Only the first invocation -// will ever start the worker. -func (in *cometdInput) Run() { - in.workerOnce.Do(func() { - in.workerWg.Add(1) - go func() { - in.log.Info("Input worker has started.") - defer in.log.Info("Input worker has stopped.") - defer in.workerWg.Done() - defer in.workerCancel() - if err := in.run(); err != nil { - in.log.Error(err) - return - } - }() - }) -} - -func (in *cometdInput) run() error { - - b := bay.Bayeux{} - creds := bay.GetSalesforceCredentials() - in.out = b.Channel(in.out, "-1", creds, in.config.ChannelName) - - var event Event - for e := range in.out { - if !e.Successful { - // To handle the last response where the object received was empty - if e.Data.Payload == nil { - return nil - } - - // Convert json.RawMessage response to []byte - msg, err := json.Marshal(e.Data.Payload) - if err != nil { - return fmt.Errorf("JSON error: %v", err) - } - - // Extract event IDs from json.RawMessage - err = json.Unmarshal(e.Data.Payload, &event) - if err != nil { - return fmt.Errorf("error while parsing JSON: %v", err) - } - if ok := in.outlet.OnEvent(makeEvent(event.EventId, string(msg))); !ok { - in.log.Debug("OnEvent returned false. Stopping input worker.") - close(in.out) - return fmt.Errorf("error ingesting data to elasticsearch") - } - } - } - return nil -} - -func init() { - err := input.Register(inputName, NewInput) - if err != nil { - panic(errors.Wrapf(err, "failed to register %v input", inputName)) - } -} - -// NewInput creates a new CometD input that consumes events from -// a topic subscription. -func NewInput( - cfg *common.Config, - connector channel.Connector, - inputContext input.Context, -) (inp input.Input, err error) { - // Extract and validate the input's configuration. - conf := defaultConfig() - if err = cfg.Unpack(&conf); err != nil { - return nil, err - } - - os.Setenv("SALESFORCE_CONSUMER_KEY", conf.Auth.OAuth2.ClientID) - os.Setenv("SALESFORCE_CONSUMER_SECRET", conf.Auth.OAuth2.ClientSecret) - os.Setenv("SALESFORCE_USER", conf.Auth.OAuth2.User) - os.Setenv("SALESFORCE_PASSWORD", conf.Auth.OAuth2.Password) - - logger := logp.NewLogger("cometd").With( - "pubsub_channel", conf.ChannelName) - - // Wrap input.Context's Done channel with a context.Context. This goroutine - // stops with the parent closes the Done channel. - inputCtx, cancelInputCtx := context.WithCancel(context.Background()) - go func() { - defer cancelInputCtx() - select { - case <-inputContext.Done: - case <-inputCtx.Done(): - } - }() - - // If the input ever needs to be made restartable, then context would need - // to be recreated with each restart. - workerCtx, workerCancel := context.WithCancel(inputCtx) - - in := &cometdInput{ - config: conf, - log: logger, - inputCtx: inputCtx, - workerCtx: workerCtx, - workerCancel: workerCancel, - ackedCount: atomic.NewUint32(0), - } - - // Creating a new channel for cometd input - in.out = make(chan bay.TriggerEvent) - - // Build outlet for events. - in.outlet, err = connector.Connect(cfg) - if err != nil { - return nil, err - } - in.log.Infof("Initialized %s input.", inputName) - return in, nil -} - -// Stop stops the input and waits for it to fully stop. -func (in *cometdInput) Stop() { - close(in.out) - in.workerCancel() - in.workerWg.Wait() -} - -// Wait is an alias for Stop. -func (in *cometdInput) Wait() { - in.Stop() -} - -type cometdInput struct { - config - - log *logp.Logger - outlet channel.Outleter // Output of received messages. - inputCtx context.Context // Wraps the Done channel from parent input.Context. - - workerCtx context.Context // Worker goroutine context. It's cancelled when the input stops or the worker exits. - workerCancel context.CancelFunc // Used to signal that the worker should stop. - workerOnce sync.Once // Guarantees that the worker goroutine is only started once. - workerWg sync.WaitGroup // Waits on worker goroutine. - - ackedCount *atomic.Uint32 // Total number of successfully ACKed messages. - Transport httpcommon.HTTPTransportSettings `config:",inline"` - Retry retryConfig `config:"retry"` - out chan bay.TriggerEvent -} - -type retryConfig struct { - MaxAttempts *int `config:"max_attempts"` - WaitMin *time.Duration `config:"wait_min"` - WaitMax *time.Duration `config:"wait_max"` -} - -type Event struct { - EventId string `json:"EventIdentifier"` -} - -func makeEvent(id string, body string) beat.Event { - event := beat.Event{ - Timestamp: time.Now().UTC(), - Fields: common.MapStr{ - "event": common.MapStr{ - "id": id, - "created": time.Now().UTC(), - }, - "message": body, - }, - Private: body, - } - event.SetID(id) - - return event -} +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cometd + +import ( + "context" + "encoding/json" + "fmt" + "os" + "sync" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/filebeat/channel" + "github.com/elastic/beats/v7/filebeat/input" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/atomic" + "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" + "github.com/elastic/beats/v7/libbeat/logp" + + bay "github.com/kush-elastic/bayeux" +) + +const ( + inputName = "cometd" +) + +// Run starts the input worker then returns. Only the first invocation +// will ever start the worker. +func (in *cometdInput) Run() { + in.workerOnce.Do(func() { + in.workerWg.Add(1) + go func() { + in.log.Info("Input worker has started.") + defer in.log.Info("Input worker has stopped.") + defer in.workerWg.Done() + defer in.workerCancel() + if err := in.run(); err != nil { + in.log.Error(err) + return + } + }() + }) +} + +func (in *cometdInput) run() error { + + b := bay.Bayeux{} + creds := bay.GetSalesforceCredentials() + in.out = b.Channel(in.out, "-1", creds, in.config.ChannelName) + + var event Event + for e := range in.out { + if !e.Successful { + // To handle the last response where the object received was empty + if e.Data.Payload == nil { + return nil + } + + // Convert json.RawMessage response to []byte + msg, err := json.Marshal(e.Data.Payload) + if err != nil { + return fmt.Errorf("JSON error: %v", err) + } + + // Extract event IDs from json.RawMessage + err = json.Unmarshal(e.Data.Payload, &event) + if err != nil { + return fmt.Errorf("error while parsing JSON: %v", err) + } + if ok := in.outlet.OnEvent(makeEvent(event.EventId, string(msg))); !ok { + in.log.Debug("OnEvent returned false. Stopping input worker.") + close(in.out) + return fmt.Errorf("error ingesting data to elasticsearch") + } + } + } + return nil +} + +func init() { + err := input.Register(inputName, NewInput) + if err != nil { + panic(errors.Wrapf(err, "failed to register %v input", inputName)) + } +} + +// NewInput creates a new CometD input that consumes events from +// a topic subscription. +func NewInput( + cfg *common.Config, + connector channel.Connector, + inputContext input.Context, +) (inp input.Input, err error) { + // Extract and validate the input's configuration. + conf := defaultConfig() + if err = cfg.Unpack(&conf); err != nil { + return nil, err + } + + os.Setenv("SALESFORCE_CONSUMER_KEY", conf.Auth.OAuth2.ClientID) + os.Setenv("SALESFORCE_CONSUMER_SECRET", conf.Auth.OAuth2.ClientSecret) + os.Setenv("SALESFORCE_USER", conf.Auth.OAuth2.User) + os.Setenv("SALESFORCE_PASSWORD", conf.Auth.OAuth2.Password) + + logger := logp.NewLogger("cometd").With( + "pubsub_channel", conf.ChannelName) + + // Wrap input.Context's Done channel with a context.Context. This goroutine + // stops with the parent closes the Done channel. + inputCtx, cancelInputCtx := context.WithCancel(context.Background()) + go func() { + defer cancelInputCtx() + select { + case <-inputContext.Done: + case <-inputCtx.Done(): + } + }() + + // If the input ever needs to be made restartable, then context would need + // to be recreated with each restart. + workerCtx, workerCancel := context.WithCancel(inputCtx) + + in := &cometdInput{ + config: conf, + log: logger, + inputCtx: inputCtx, + workerCtx: workerCtx, + workerCancel: workerCancel, + ackedCount: atomic.NewUint32(0), + } + + // Creating a new channel for cometd input + in.out = make(chan bay.TriggerEvent) + + // Build outlet for events. + in.outlet, err = connector.Connect(cfg) + if err != nil { + return nil, err + } + in.log.Infof("Initialized %s input.", inputName) + return in, nil +} + +// Stop stops the input and waits for it to fully stop. +func (in *cometdInput) Stop() { + close(in.out) + in.workerCancel() + in.workerWg.Wait() +} + +// Wait is an alias for Stop. +func (in *cometdInput) Wait() { + in.Stop() +} + +type cometdInput struct { + config + + log *logp.Logger + outlet channel.Outleter // Output of received messages. + inputCtx context.Context // Wraps the Done channel from parent input.Context. + + workerCtx context.Context // Worker goroutine context. It's cancelled when the input stops or the worker exits. + workerCancel context.CancelFunc // Used to signal that the worker should stop. + workerOnce sync.Once // Guarantees that the worker goroutine is only started once. + workerWg sync.WaitGroup // Waits on worker goroutine. + + ackedCount *atomic.Uint32 // Total number of successfully ACKed messages. + Transport httpcommon.HTTPTransportSettings `config:",inline"` + Retry retryConfig `config:"retry"` + out chan bay.TriggerEvent +} + +type retryConfig struct { + MaxAttempts *int `config:"max_attempts"` + WaitMin *time.Duration `config:"wait_min"` + WaitMax *time.Duration `config:"wait_max"` +} + +type Event struct { + EventId string `json:"EventIdentifier"` +} + +func makeEvent(id string, body string) beat.Event { + event := beat.Event{ + Timestamp: time.Now().UTC(), + Fields: common.MapStr{ + "event": common.MapStr{ + "id": id, + "created": time.Now().UTC(), + }, + "message": body, + }, + Private: body, + } + event.SetID(id) + + return event +} diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index 172b06e0b74b..ec8887373773 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -1,19 +1,19 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package cometd - -import ( - "testing" - - "github.com/elastic/beats/v7/filebeat/input/inputtest" - "github.com/elastic/beats/v7/libbeat/common" -) - -func TestNewInputDone(t *testing.T) { - config := common.MapStr{ - "channel_name": "some-channel", - } - inputtest.AssertNotStartedInputCanBeDone(t, NewInput, &config) -} +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cometd + +import ( + "testing" + + "github.com/elastic/beats/v7/filebeat/input/inputtest" + "github.com/elastic/beats/v7/libbeat/common" +) + +func TestNewInputDone(t *testing.T) { + config := common.MapStr{ + "channel_name": "some-channel", + } + inputtest.AssertNotStartedInputCanBeDone(t, NewInput, &config) +} From 958c51c7cde44fe015f3593a4645d69bab65c6c0 Mon Sep 17 00:00:00 2001 From: yug-elastic Date: Sun, 30 Jan 2022 08:00:35 +0530 Subject: [PATCH 10/50] Update go.mod and go.sum --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index 5330c1dc46dc..8303010d5238 100644 --- a/go.mod +++ b/go.mod @@ -249,6 +249,7 @@ require ( github.com/jstemmer/go-junit-report v0.9.1 // indirect github.com/karrick/godirwalk v1.15.6 // indirect github.com/klauspost/compress v1.13.6 // indirect + github.com/kush-elastic/bayeux v1.0.1 // indirect github.com/markbates/pkger v0.17.0 // indirect github.com/mattn/go-isatty v0.0.12 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect diff --git a/go.sum b/go.sum index 64389aae72c6..20ca45e80cf0 100644 --- a/go.sum +++ b/go.sum @@ -1087,6 +1087,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kush-elastic/bayeux v1.0.1 h1:tW9t3mR1DkMQGgA9PqGn9zf8n2py6POBwVHGKOjgmCs= +github.com/kush-elastic/bayeux v1.0.1/go.mod h1:Irc+xWtpooZvkHTIXCIc+YgkFruHpq5C2ndvw1Geqwg= github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= From 7ff87d4c9df7e526886c38eeee322129e6aa4631 Mon Sep 17 00:00:00 2001 From: yug-elastic Date: Tue, 1 Feb 2022 15:30:14 +0530 Subject: [PATCH 11/50] Update go.mod and NOTICE.txt --- NOTICE.txt | 33 ++++++++++++++++++++++++++++++++- go.mod | 3 ++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 8f660fe88437..8ab8591313b8 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,5 +1,5 @@ Elastic Beats -Copyright 2014-2021 Elasticsearch BV +Copyright 2014-2022 Elasticsearch BV This product includes software developed by The Apache Software Foundation (http://www.apache.org/). @@ -12408,6 +12408,37 @@ freely, subject to the following restrictions: distribution. +-------------------------------------------------------------------------------- +Dependency : github.com/kush-elastic/bayeux +Version: v1.0.1 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/kush-elastic/bayeux@v1.0.1/LICENSE: + +MIT License + +Copyright (c) 2017 Zander Hill + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/lib/pq Version: v1.10.3 diff --git a/go.mod b/go.mod index 8303010d5238..e5f520d6965c 100644 --- a/go.mod +++ b/go.mod @@ -192,6 +192,8 @@ require ( kernel.org/pub/linux/libs/security/libcap/cap v1.2.57 ) +require github.com/kush-elastic/bayeux v1.0.1 + require ( code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f // indirect github.com/Azure/azure-amqp-common-go/v3 v3.2.1 // indirect @@ -249,7 +251,6 @@ require ( github.com/jstemmer/go-junit-report v0.9.1 // indirect github.com/karrick/godirwalk v1.15.6 // indirect github.com/klauspost/compress v1.13.6 // indirect - github.com/kush-elastic/bayeux v1.0.1 // indirect github.com/markbates/pkger v0.17.0 // indirect github.com/mattn/go-isatty v0.0.12 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect From c658cd8aab9485c21c425b0d807ee12e1261f83a Mon Sep 17 00:00:00 2001 From: yug-elastic Date: Wed, 16 Mar 2022 12:51:13 +0530 Subject: [PATCH 12/50] Make changes with respect to ownership transfer --- x-pack/filebeat/input/cometd/input.go | 3 ++- x-pack/filebeat/input/cometd/input_test.go | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index c1de6db2cc59..cef169302b96 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -22,7 +22,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" "github.com/elastic/beats/v7/libbeat/logp" - bay "github.com/kush-elastic/bayeux" + bay "github.com/elastic/bayeux" ) const ( @@ -106,6 +106,7 @@ func NewInput( os.Setenv("SALESFORCE_CONSUMER_SECRET", conf.Auth.OAuth2.ClientSecret) os.Setenv("SALESFORCE_USER", conf.Auth.OAuth2.User) os.Setenv("SALESFORCE_PASSWORD", conf.Auth.OAuth2.Password) + os.Setenv("SALESFORCE_TOKEN_URL", conf.Auth.OAuth2.TokenURL) logger := logp.NewLogger("cometd").With( "pubsub_channel", conf.ChannelName) diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index ec8887373773..e56dd397d254 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -13,7 +13,12 @@ import ( func TestNewInputDone(t *testing.T) { config := common.MapStr{ - "channel_name": "some-channel", + "channel_name": "cometd-channel", + "auth.oauth2.client.id": "DEMOCLIENTID", + "auth.oauth2.client.secret": "DEMOCLIENTSECRET", + "auth.oauth2.user": "salesforce_user", + "auth.oauth2.password": "P@$$w0₹D", + "auth.oauth2.token_url": "https://login.salesforce.com/services/oauth2/token", } inputtest.AssertNotStartedInputCanBeDone(t, NewInput, &config) } From c05d8ba5b3409a836e5cc847765e14f3f2046868 Mon Sep 17 00:00:00 2001 From: yug-elastic Date: Wed, 16 Mar 2022 14:10:05 +0530 Subject: [PATCH 13/50] Update go.mod and go.sum --- NOTICE.txt | 90 +++++++++++++++++++++++++++--------------------------- go.mod | 2 +- go.sum | 4 +-- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 4fa5b64c2eef..8e8dc381bc64 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12409,12 +12409,12 @@ freely, subject to the following restrictions: -------------------------------------------------------------------------------- -Dependency : github.com/kush-elastic/bayeux +Dependency : github.com/elastic/bayeux Version: v1.0.1 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/kush-elastic/bayeux@v1.0.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/bayeux@v1.0.1/LICENSE: MIT License @@ -20561,28 +20561,28 @@ Licence type (autodetected): MIT Contents of probable licence file $GOMODCACHE/github.com/!azure/go-amqp@v0.16.0/LICENSE: - MIT License - - Copyright (C) 2017 Kale Blankenship - Portions Copyright (C) Microsoft Corporation - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE + MIT License + + Copyright (C) 2017 Kale Blankenship + Portions Copyright (C) Microsoft Corporation + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE -------------------------------------------------------------------------------- @@ -22291,27 +22291,27 @@ Licence type (autodetected): MIT Contents of probable licence file $GOMODCACHE/github.com/akavel/rsrc@v0.8.0/LICENSE.txt: -The MIT License (MIT) - -Copyright (c) 2013-2017 The rsrc Authors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +The MIT License (MIT) + +Copyright (c) 2013-2017 The rsrc Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. -------------------------------------------------------------------------------- diff --git a/go.mod b/go.mod index 910ab23c9330..d495cccd4bd0 100644 --- a/go.mod +++ b/go.mod @@ -192,7 +192,6 @@ require ( kernel.org/pub/linux/libs/security/libcap/cap v1.2.57 ) -require github.com/kush-elastic/bayeux v1.0.1 require github.com/shirou/gopsutil/v3 v3.21.12 require ( @@ -219,6 +218,7 @@ require ( github.com/docker/distribution v2.8.1+incompatible // indirect github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/elastic/bayeux v1.0.1 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/fearful-symmetry/gomsr v0.0.1 // indirect github.com/go-logr/logr v1.2.0 // indirect diff --git a/go.sum b/go.sum index 3104f3196f81..602999fdac44 100644 --- a/go.sum +++ b/go.sum @@ -510,6 +510,8 @@ github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7j github.com/eclipse/paho.mqtt.golang v1.3.5 h1:sWtmgNxYM9P2sP+xEItMozsR3w0cqZFlqnNN1bdl41Y= github.com/eclipse/paho.mqtt.golang v1.3.5/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elastic/bayeux v1.0.1 h1:siXwhEy+SJYHWiwq2XuAF4FKr7soVCdbQdx7W+as+0Y= +github.com/elastic/bayeux v1.0.1/go.mod h1:ET5dBf91w0cukiCqUHtlEaGx32f75SBU5IZyx+jNfjw= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo= @@ -1110,8 +1112,6 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kush-elastic/bayeux v1.0.1 h1:tW9t3mR1DkMQGgA9PqGn9zf8n2py6POBwVHGKOjgmCs= -github.com/kush-elastic/bayeux v1.0.1/go.mod h1:Irc+xWtpooZvkHTIXCIc+YgkFruHpq5C2ndvw1Geqwg= github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= From 798142bdf4aeb1324b527d57f4392db8ea95bf8a Mon Sep 17 00:00:00 2001 From: yug-elastic Date: Fri, 25 Mar 2022 12:26:59 +0530 Subject: [PATCH 14/50] Add two unit test cases and nit --- x-pack/filebeat/input/cometd/config_test.go | 6 ++++++ x-pack/filebeat/input/cometd/input.go | 1 - x-pack/filebeat/input/cometd/input_test.go | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/cometd/config_test.go b/x-pack/filebeat/input/cometd/config_test.go index 9af56d07d3f6..4a2adbe991f4 100644 --- a/x-pack/filebeat/input/cometd/config_test.go +++ b/x-pack/filebeat/input/cometd/config_test.go @@ -17,3 +17,9 @@ func TestConfigValidate(t *testing.T) { c := defaultConfig() assert.NoError(t, c.Validate()) } + +func TestConfigValidateFailure(t *testing.T) { + var c config + c.ChannelName = "" + assert.Error(t, c.Validate()) +} diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index cef169302b96..1ed1df45420a 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -48,7 +48,6 @@ func (in *cometdInput) Run() { } func (in *cometdInput) run() error { - b := bay.Bayeux{} creds := bay.GetSalesforceCredentials() in.out = b.Channel(in.out, "-1", creds, in.config.ChannelName) diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index e56dd397d254..b287bd82be09 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -6,9 +6,12 @@ package cometd import ( "testing" + "time" "github.com/elastic/beats/v7/filebeat/input/inputtest" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/stretchr/testify/assert" ) func TestNewInputDone(t *testing.T) { @@ -22,3 +25,18 @@ func TestNewInputDone(t *testing.T) { } inputtest.AssertNotStartedInputCanBeDone(t, NewInput, &config) } + +func TestMakeEventFailure(t *testing.T) { + event := beat.Event{ + Timestamp: time.Now().UTC(), + Fields: common.MapStr{ + "event": common.MapStr{ + "id": "DEMOID", + "created": time.Now().UTC(), + }, + "message": "DEMOBODYFAIL", + }, + Private: "DEMOBODYFAIL", + } + assert.NotEqual(t, event, makeEvent("DEMOID", "DEMOBODY")) +} From a3fe48bee6a5b6238d5cc9cb8c8c3dc6ac0b986a Mon Sep 17 00:00:00 2001 From: yug-elastic Date: Sun, 27 Mar 2022 14:42:18 +0530 Subject: [PATCH 15/50] Add unit test cases and integration tests --- .../input/cometd/cometd_integration_test.go | 22 ++++++++++ x-pack/filebeat/input/cometd/config_test.go | 40 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 x-pack/filebeat/input/cometd/cometd_integration_test.go diff --git a/x-pack/filebeat/input/cometd/cometd_integration_test.go b/x-pack/filebeat/input/cometd/cometd_integration_test.go new file mode 100644 index 000000000000..30c282af586e --- /dev/null +++ b/x-pack/filebeat/input/cometd/cometd_integration_test.go @@ -0,0 +1,22 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cometd + +import ( + "testing" + + "github.com/elastic/beats/v7/filebeat/input" + "github.com/stretchr/testify/require" +) + +func TestInput(t *testing.T) { + err := input.Register(inputName, NewInput) + require.Error(t, err) + + var cometd cometdInput + cometd.Run() + + makeEvent("test", "test") +} diff --git a/x-pack/filebeat/input/cometd/config_test.go b/x-pack/filebeat/input/cometd/config_test.go index 4a2adbe991f4..e85b1d750588 100644 --- a/x-pack/filebeat/input/cometd/config_test.go +++ b/x-pack/filebeat/input/cometd/config_test.go @@ -23,3 +23,43 @@ func TestConfigValidateFailure(t *testing.T) { c.ChannelName = "" assert.Error(t, c.Validate()) } + +func TestConfigAuthValidate(t *testing.T) { + var o oAuth2Config + o.ClientID = "DEMOCLIENTID" + o.ClientSecret = "DEMOCLIENTSECRET" + o.User = "salesforce_user" + o.Password = "P@$$w0₹D" + o.TokenURL = "https://login.salesforce.com/services/oauth2/token" + + assert.NoError(t, o.Validate()) +} + +func TestConfigAuthValidateFailure_MissingTokenURL(t *testing.T) { + var o oAuth2Config + o.ClientID = "DEMOCLIENTID" + o.ClientSecret = "DEMOCLIENTSECRET" + o.User = "salesforce_user" + o.Password = "P@$$w0₹D" + + assert.Error(t, o.Validate()) +} + +func TestConfigAuthValidateFailure_MissingClientCredentials(t *testing.T) { + var o oAuth2Config + o.ClientSecret = "DEMOCLIENTSECRET" + o.User = "salesforce_user" + o.Password = "P@$$w0₹D" + o.TokenURL = "https://login.salesforce.com/services/oauth2/token" + + assert.Error(t, o.Validate()) +} + +func TestConfigAuthValidateFailure_MissingUsernamePassword(t *testing.T) { + var o oAuth2Config + o.ClientID = "DEMOCLIENTID" + o.ClientSecret = "DEMOCLIENTSECRET" + o.TokenURL = "https://login.salesforce.com/services/oauth2/token" + + assert.Error(t, o.Validate()) +} From 497d16ddd73bf628411dcb88358a727cdb5fbd81 Mon Sep 17 00:00:00 2001 From: yug-elastic Date: Fri, 1 Apr 2022 18:45:41 +0530 Subject: [PATCH 16/50] Address review comments --- x-pack/filebeat/input/cometd/config_auth.go | 13 +++++++-- x-pack/filebeat/input/cometd/input.go | 29 ++++++--------------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/x-pack/filebeat/input/cometd/config_auth.go b/x-pack/filebeat/input/cometd/config_auth.go index cf2b88a40c2e..fe96583664f2 100644 --- a/x-pack/filebeat/input/cometd/config_auth.go +++ b/x-pack/filebeat/input/cometd/config_auth.go @@ -23,8 +23,17 @@ type oAuth2Config struct { // Validate checks if oauth2 config is valid. func (o *oAuth2Config) Validate() error { - if o.TokenURL == "" || o.ClientID == "" || o.ClientSecret == "" || o.User == "" || o.Password == "" { - return errors.New("both token_url and client credentials must be provided") + if o.TokenURL == "" { + return errors.New("token_url must be provided") + } + if o.ClientID == "" { + return errors.New("client.id must be provided") + } + if o.ClientSecret == "" { + return errors.New("client.secret must be provided") + } + if o.User == "" || o.Password == "" { + return errors.New("both user and password must be provided") } return nil } diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index 1ed1df45420a..bed3819869c9 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -19,7 +19,6 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/atomic" - "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" "github.com/elastic/beats/v7/libbeat/logp" bay "github.com/elastic/bayeux" @@ -39,6 +38,8 @@ func (in *cometdInput) Run() { defer in.log.Info("Input worker has stopped.") defer in.workerWg.Done() defer in.workerCancel() + in.b = bay.Bayeux{} + in.creds = bay.GetSalesforceCredentials() if err := in.run(); err != nil { in.log.Error(err) return @@ -48,9 +49,7 @@ func (in *cometdInput) Run() { } func (in *cometdInput) run() error { - b := bay.Bayeux{} - creds := bay.GetSalesforceCredentials() - in.out = b.Channel(in.out, "-1", creds, in.config.ChannelName) + in.out = in.b.Channel(in.out, "-1", in.creds, in.config.ChannelName) var event Event for e := range in.out { @@ -61,7 +60,7 @@ func (in *cometdInput) run() error { } // Convert json.RawMessage response to []byte - msg, err := json.Marshal(e.Data.Payload) + msg, err := e.Data.Payload.MarshalJSON() if err != nil { return fmt.Errorf("JSON error: %v", err) } @@ -113,13 +112,7 @@ func NewInput( // Wrap input.Context's Done channel with a context.Context. This goroutine // stops with the parent closes the Done channel. inputCtx, cancelInputCtx := context.WithCancel(context.Background()) - go func() { - defer cancelInputCtx() - select { - case <-inputContext.Done: - case <-inputCtx.Done(): - } - }() + defer cancelInputCtx() // If the input ever needs to be made restartable, then context would need // to be recreated with each restart. @@ -170,16 +163,10 @@ type cometdInput struct { workerOnce sync.Once // Guarantees that the worker goroutine is only started once. workerWg sync.WaitGroup // Waits on worker goroutine. - ackedCount *atomic.Uint32 // Total number of successfully ACKed messages. - Transport httpcommon.HTTPTransportSettings `config:",inline"` - Retry retryConfig `config:"retry"` + ackedCount *atomic.Uint32 // Total number of successfully ACKed messages. out chan bay.TriggerEvent -} - -type retryConfig struct { - MaxAttempts *int `config:"max_attempts"` - WaitMin *time.Duration `config:"wait_min"` - WaitMax *time.Duration `config:"wait_max"` + b bay.Bayeux + creds bay.Credentials } type Event struct { From f7135cc9c1981af46aa3add809b51170d1d87dd9 Mon Sep 17 00:00:00 2001 From: yug-elastic Date: Tue, 5 Apr 2022 00:29:21 +0530 Subject: [PATCH 17/50] Resolve some linting issues --- x-pack/filebeat/input/cometd/config_test.go | 40 ++++++++++++--------- x-pack/filebeat/input/cometd/input.go | 4 +-- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/x-pack/filebeat/input/cometd/config_test.go b/x-pack/filebeat/input/cometd/config_test.go index e85b1d750588..ac9e8912c4bc 100644 --- a/x-pack/filebeat/input/cometd/config_test.go +++ b/x-pack/filebeat/input/cometd/config_test.go @@ -10,6 +10,14 @@ import ( "github.com/stretchr/testify/assert" ) +const ( + demoClientID = "DEMOCLIENTID" + demoClientSecret = "DEMOCLIENTSECRET" + salesforceUser = "salesforce_user" + password = "P@$$w0₹D" + tokenURL = "https://login.salesforce.com/services/oauth2/token" +) + // Validate that it finds the application default credentials and does // not trigger a config validation error because credentials were not // set in the config. @@ -26,40 +34,40 @@ func TestConfigValidateFailure(t *testing.T) { func TestConfigAuthValidate(t *testing.T) { var o oAuth2Config - o.ClientID = "DEMOCLIENTID" - o.ClientSecret = "DEMOCLIENTSECRET" - o.User = "salesforce_user" - o.Password = "P@$$w0₹D" - o.TokenURL = "https://login.salesforce.com/services/oauth2/token" + o.ClientID = demoClientID + o.ClientSecret = demoClientSecret + o.User = salesforceUser + o.Password = password + o.TokenURL = tokenURL assert.NoError(t, o.Validate()) } func TestConfigAuthValidateFailure_MissingTokenURL(t *testing.T) { var o oAuth2Config - o.ClientID = "DEMOCLIENTID" - o.ClientSecret = "DEMOCLIENTSECRET" - o.User = "salesforce_user" - o.Password = "P@$$w0₹D" + o.ClientID = demoClientID + o.ClientSecret = demoClientSecret + o.User = salesforceUser + o.Password = password assert.Error(t, o.Validate()) } func TestConfigAuthValidateFailure_MissingClientCredentials(t *testing.T) { var o oAuth2Config - o.ClientSecret = "DEMOCLIENTSECRET" - o.User = "salesforce_user" - o.Password = "P@$$w0₹D" - o.TokenURL = "https://login.salesforce.com/services/oauth2/token" + o.ClientSecret = demoClientSecret + o.User = salesforceUser + o.Password = password + o.TokenURL = tokenURL assert.Error(t, o.Validate()) } func TestConfigAuthValidateFailure_MissingUsernamePassword(t *testing.T) { var o oAuth2Config - o.ClientID = "DEMOCLIENTID" - o.ClientSecret = "DEMOCLIENTSECRET" - o.TokenURL = "https://login.salesforce.com/services/oauth2/token" + o.ClientID = demoClientID + o.ClientSecret = demoClientSecret + o.TokenURL = tokenURL assert.Error(t, o.Validate()) } diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index bed3819869c9..a3c8427bdc46 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -62,13 +62,13 @@ func (in *cometdInput) run() error { // Convert json.RawMessage response to []byte msg, err := e.Data.Payload.MarshalJSON() if err != nil { - return fmt.Errorf("JSON error: %v", err) + return fmt.Errorf("JSON error: %w", err) } // Extract event IDs from json.RawMessage err = json.Unmarshal(e.Data.Payload, &event) if err != nil { - return fmt.Errorf("error while parsing JSON: %v", err) + return fmt.Errorf("error while parsing JSON: %w", err) } if ok := in.outlet.OnEvent(makeEvent(event.EventId, string(msg))); !ok { in.log.Debug("OnEvent returned false. Stopping input worker.") From 76268f03da188ab480908797fef4a9bb2de85e1f Mon Sep 17 00:00:00 2001 From: yug-elastic Date: Tue, 5 Apr 2022 12:46:18 +0530 Subject: [PATCH 18/50] Resolve some linting issues --- x-pack/filebeat/input/cometd/config_test.go | 8 ++++---- x-pack/filebeat/input/cometd/input.go | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/x-pack/filebeat/input/cometd/config_test.go b/x-pack/filebeat/input/cometd/config_test.go index ac9e8912c4bc..8750880181e6 100644 --- a/x-pack/filebeat/input/cometd/config_test.go +++ b/x-pack/filebeat/input/cometd/config_test.go @@ -14,7 +14,7 @@ const ( demoClientID = "DEMOCLIENTID" demoClientSecret = "DEMOCLIENTSECRET" salesforceUser = "salesforce_user" - password = "P@$$w0₹D" + pwd = "P@$$w0₹D" tokenURL = "https://login.salesforce.com/services/oauth2/token" ) @@ -37,7 +37,7 @@ func TestConfigAuthValidate(t *testing.T) { o.ClientID = demoClientID o.ClientSecret = demoClientSecret o.User = salesforceUser - o.Password = password + o.Password = pwd o.TokenURL = tokenURL assert.NoError(t, o.Validate()) @@ -48,7 +48,7 @@ func TestConfigAuthValidateFailure_MissingTokenURL(t *testing.T) { o.ClientID = demoClientID o.ClientSecret = demoClientSecret o.User = salesforceUser - o.Password = password + o.Password = pwd assert.Error(t, o.Validate()) } @@ -57,7 +57,7 @@ func TestConfigAuthValidateFailure_MissingClientCredentials(t *testing.T) { var o oAuth2Config o.ClientSecret = demoClientSecret o.User = salesforceUser - o.Password = password + o.Password = pwd o.TokenURL = tokenURL assert.Error(t, o.Validate()) diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index a3c8427bdc46..36e6776e61fe 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -12,8 +12,6 @@ import ( "sync" "time" - "github.com/pkg/errors" - "github.com/elastic/beats/v7/filebeat/channel" "github.com/elastic/beats/v7/filebeat/input" "github.com/elastic/beats/v7/libbeat/beat" @@ -83,7 +81,7 @@ func (in *cometdInput) run() error { func init() { err := input.Register(inputName, NewInput) if err != nil { - panic(errors.Wrapf(err, "failed to register %v input", inputName)) + panic(fmt.Errorf("failed to register %v input: %w", inputName, err)) } } From d114bc4fa58440a536d877c269a669c3ca254d68 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 6 Apr 2022 02:16:53 +0530 Subject: [PATCH 19/50] Add unit test for input --- x-pack/filebeat/input/cometd/client_mocked.go | 60 ++++++++ x-pack/filebeat/input/cometd/input_test.go | 136 ++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 x-pack/filebeat/input/cometd/client_mocked.go diff --git a/x-pack/filebeat/input/cometd/client_mocked.go b/x-pack/filebeat/input/cometd/client_mocked.go new file mode 100644 index 000000000000..95ffef839ba8 --- /dev/null +++ b/x-pack/filebeat/input/cometd/client_mocked.go @@ -0,0 +1,60 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cometd + +import ( + "github.com/elastic/beats/v7/filebeat/channel" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" +) + +type mockedConnector struct { + connectWithError error + outlet channel.Outleter +} + +var _ channel.Connector = new(mockedConnector) + +func (m *mockedConnector) Connect(c *common.Config) (channel.Outleter, error) { + return m.ConnectWith(c, beat.ClientConfig{}) +} + +func (m *mockedConnector) ConnectWith(*common.Config, beat.ClientConfig) (channel.Outleter, error) { + if m.connectWithError != nil { + return nil, m.connectWithError + } + return m.outlet, nil +} + +type mockedOutleter struct { + onEventHandler func(event beat.Event) bool +} + +var _ channel.Outleter = new(mockedOutleter) + +func (m mockedOutleter) Close() error { + panic("implement me") +} + +func (m mockedOutleter) Done() <-chan struct{} { + panic("implement me") +} + +func (m mockedOutleter) OnEvent(event beat.Event) bool { + return m.onEventHandler(event) +} diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index b287bd82be09..54d71e384ece 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -5,13 +5,23 @@ package cometd import ( + "io/ioutil" + "net/http" + "net/http/httptest" "testing" "time" + bay "github.com/elastic/bayeux" + finput "github.com/elastic/beats/v7/filebeat/input" "github.com/elastic/beats/v7/filebeat/input/inputtest" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + serverURL string ) func TestNewInputDone(t *testing.T) { @@ -40,3 +50,129 @@ func TestMakeEventFailure(t *testing.T) { } assert.NotEqual(t, event, makeEvent("DEMOID", "DEMOBODY")) } + +func TestNewInput_Run(t *testing.T) { + eventsCh := make(chan beat.Event) + + outlet := &mockedOutleter{ + onEventHandler: func(event beat.Event) bool { + eventsCh <- event + return true + }, + } + connector := &mockedConnector{ + outlet: outlet, + } + var inputContext finput.Context + + var msg bay.TriggerEvent + msg.Data.Event.ReplayID = 1234 + msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) + msg.Channel = "first-channel" + + config := map[string]interface{}{ + "channel_name": "first-channel", + "auth.oauth2.client.id": "client.id", + "auth.oauth2.client.secret": "client.secret", + "auth.oauth2.user": "user", + "auth.oauth2.password": "password", + } + + r := http.HandlerFunc(oauth2Handler) + server := httptest.NewServer(r) + serverURL = server.URL + config["auth.oauth2.token_url"] = server.URL + "/token" + + cfg := common.MustNewConfigFrom(config) + + input, err := NewInput(cfg, connector, inputContext) + require.NoError(t, err) + require.NotNil(t, input) + + input.Run() + for _, event := range []beat.Event{<-eventsCh} { + require.NoError(t, err) + assertEventMatches(t, msg, event) + } + server.Close() +} + +func oauth2TokenHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + _ = r.ParseForm() + switch { + case r.Method != "POST": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong method"}`)) + default: + response := `{"instance_url": "` + serverURL + `", "expires_in": "60", "access_token": "abcd"}` + _, _ = w.Write([]byte(response)) + } +} + +func oauth2ClientIdHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + _ = r.ParseForm() + switch { + case r.Method != "POST": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong method"}`)) + default: + _, _ = w.Write([]byte(`[{"ext":{"replay":true,"payload.format":true},"minimumVersion":"1.0","clientId":"94b112sp7ph1c9s41mycpzik4rkj3","supportedConnectionTypes":["long-polling"],"channel":"/meta/handshake","version":"1.0","successful":true}]`)) + } +} + +func oauth2SubscribeHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + _ = r.ParseForm() + switch { + case r.Method != "POST": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong method"}`)) + default: + _, _ = w.Write([]byte(`[{"clientId": "94b112sp7ph1c9s41mycpzik4rkj3", "channel": "/meta/subscribe", "subscription": "/event/LoginEventStream", "successful":true}]`)) + } +} + +func oauth2EventHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + _ = r.ParseForm() + switch { + case r.Method != "POST": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong method"}`)) + default: + _, _ = w.Write([]byte(`[{"data": {"payload": {"CountryIso": "IN"}, "event": {"replayId":1234}}, "channel": "/event/LoginEventStream"}]`)) + } +} + +func oauth2Handler(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/token" { + oauth2TokenHandler(w, r) + return + } + body, _ := ioutil.ReadAll(r.Body) + if r.URL.Path == "/cometd/38.0" && string(body) == `{"channel": "/meta/handshake", "supportedConnectionTypes": ["long-polling"], "version": "1.0"}` { + oauth2ClientIdHandler(w, r) + return + } else if r.URL.Path == "/cometd/38.0" && string(body) == `{"channel": "/meta/connect", "connectionType": "long-polling", "clientId": "94b112sp7ph1c9s41mycpzik4rkj3"} ` { + oauth2EventHandler(w, r) + return + } else if r.URL.Path == "/cometd/38.0" && string(body) == `{ + "channel": "/meta/subscribe", + "subscription": "first-channel", + "clientId": "94b112sp7ph1c9s41mycpzik4rkj3", + "ext": { + "replay": {"first-channel": "-1"} + } + }` { + oauth2SubscribeHandler(w, r) + return + } +} + +func assertEventMatches(t *testing.T, expected bay.TriggerEvent, got beat.Event) { + message, err := got.GetValue("message") + require.NoError(t, err) + require.Equal(t, string(expected.Data.Payload), message) +} From a17f2610bf9218b56883728c17d454ef2c794397 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 6 Apr 2022 12:41:16 +0530 Subject: [PATCH 20/50] Add integration test --- filebeat/docker-compose.yml | 19 ++++ go.mod | 2 +- go.sum | 2 + .../docker/cometd/files/config.yml | 27 ++++++ x-pack/filebeat/filebeat.yml | 23 +++-- .../input/cometd/cometd_integration_test.go | 97 ++++++++++++++++++- x-pack/filebeat/input/cometd/input.go | 2 +- 7 files changed, 158 insertions(+), 14 deletions(-) create mode 100644 testing/environments/docker/cometd/files/config.yml diff --git a/filebeat/docker-compose.yml b/filebeat/docker-compose.yml index a73f0bc39d6b..0ce8e391c73a 100644 --- a/filebeat/docker-compose.yml +++ b/filebeat/docker-compose.yml @@ -35,6 +35,7 @@ services: kibana: { condition: service_healthy } mosquitto: { condition: service_healthy } redis: { condition: service_healthy } + cometd: { condition: service_healthy } elasticsearch: extends: @@ -68,3 +69,21 @@ services: redis: build: ${PWD}/input/redis/_meta + + cometd: + image: docker.elastic.co/observability/stream:v0.6.1 + hostname: cometd + ports: + - 8080:8080 + volumes: + - ${ES_BEATS}/testing/environments/docker/cometd/files/:/files:ro + environment: + PORT: "8080" + healthcheck: + test: ["CMD-SHELL", "curl -X POST http://127.0.0.1:8080/token"] + retries: 300 + interval: 1s + command: + - http-server + - --addr=:8080 + - --config=/files/config.yml diff --git a/go.mod b/go.mod index e86abb34c531..acbb785e1237 100644 --- a/go.mod +++ b/go.mod @@ -220,7 +220,7 @@ require ( github.com/docker/distribution v2.8.1+incompatible // indirect github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/elastic/bayeux v1.0.1 // indirect + github.com/elastic/bayeux v1.0.2 // indirect github.com/envoyproxy/go-control-plane v0.10.1 // indirect github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect diff --git a/go.sum b/go.sum index 5c0ec71b642e..cf8e60231b4c 100644 --- a/go.sum +++ b/go.sum @@ -565,6 +565,8 @@ github.com/eclipse/paho.mqtt.golang v1.3.5/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+J github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/bayeux v1.0.1 h1:siXwhEy+SJYHWiwq2XuAF4FKr7soVCdbQdx7W+as+0Y= github.com/elastic/bayeux v1.0.1/go.mod h1:ET5dBf91w0cukiCqUHtlEaGx32f75SBU5IZyx+jNfjw= +github.com/elastic/bayeux v1.0.2 h1:bdDIH9DkXkzv2PDa/Z8vGiHcqNWv+FmouGFYWLgnw60= +github.com/elastic/bayeux v1.0.2/go.mod h1:ET5dBf91w0cukiCqUHtlEaGx32f75SBU5IZyx+jNfjw= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo= diff --git a/testing/environments/docker/cometd/files/config.yml b/testing/environments/docker/cometd/files/config.yml new file mode 100644 index 000000000000..96a2646dab0e --- /dev/null +++ b/testing/environments/docker/cometd/files/config.yml @@ -0,0 +1,27 @@ +rules: + - path: /token + methods: ["POST"] + responses: + - status_code: 200 + body: | + {"instance_url": "http://127.0.0.1:8080", "expires_in": "60", "access_token": "abcd"} + - path: /cometd/38.0 + methods: ["POST"] + request_body: '{"channel": "/meta/handshake", "supportedConnectionTypes": ["long-polling"], "version": "1.0"}' + responses: + - status_code: 200 + body: | + [{"ext":{"replay":true,"payload.format":true},"minimumVersion":"1.0","clientId":"94b112sp7ph1c9s41mycpzik4rkj3","supportedConnectionTypes":["long-polling"],"channel":"/meta/handshake","version":"1.0","successful":true}] + - path: /cometd/38.0 + methods: ["POST"] + request_body: '{"channel": "/meta/connect", "connectionType": "long-polling", "clientId": "94b112sp7ph1c9s41mycpzik4rkj3"} ' + responses: + - status_code: 200 + body: | + [{"data": {"payload": {"CountryIso": "IN"}, "event": {"replayId":1234}}, "channel": "/event/LoginEventStream"}] + - path: /cometd/38.0 + methods: ["POST"] + responses: + - status_code: 200 + body: | + [{"clientId": "94b112sp7ph1c9s41mycpzik4rkj3", "channel": "/meta/subscribe", "subscription": "/event/LoginEventStream", "successful":true}] diff --git a/x-pack/filebeat/filebeat.yml b/x-pack/filebeat/filebeat.yml index fb67b14d67ba..f224563892a4 100644 --- a/x-pack/filebeat/filebeat.yml +++ b/x-pack/filebeat/filebeat.yml @@ -13,7 +13,15 @@ # ============================== Filebeat inputs =============================== filebeat.inputs: - +- type: cometd + enabled: true + channel_name: /event/LoginEventStream + auth.oauth2: + client.id: 3MVG9pRzvMkjMb6m1I.fz3f7zBuH4xdKCJcM9B5XLgxXh2AFTmQmr8JMn1v9_AoKIW7gwf2FIGqRwKZVEMR_C + client.secret: E9FE118633BC7C55D0AD395204F19C1D4529D09CB7231754AD2F2CA668400619 + token_url: https://login.salesforce.com/services/oauth2/token + user: kush.rana@elastic.co + password: Crest@123 # Each - is an input. Most options can be set at the input level, so # you can use different inputs for various configurations. # Below are the input specific configurations. @@ -130,11 +138,12 @@ setup.kibana: # ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. - +output.console: + pretty: true # ---------------------------- Elasticsearch Output ---------------------------- -output.elasticsearch: +# output.elasticsearch: # Array of hosts to connect to. - hosts: ["localhost:9200"] + # hosts: ["localhost:9200"] # Protocol - either `http` (default) or `https`. #protocol: "https" @@ -163,9 +172,9 @@ output.elasticsearch: processors: - add_host_metadata: when.not.contains.tags: forwarded - - add_cloud_metadata: ~ - - add_docker_metadata: ~ - - add_kubernetes_metadata: ~ + # - add_cloud_metadata: ~ + # - add_docker_metadata: ~ + # - add_kubernetes_metadata: ~ # ================================== Logging =================================== diff --git a/x-pack/filebeat/input/cometd/cometd_integration_test.go b/x-pack/filebeat/input/cometd/cometd_integration_test.go index 30c282af586e..c8f909af20d9 100644 --- a/x-pack/filebeat/input/cometd/cometd_integration_test.go +++ b/x-pack/filebeat/input/cometd/cometd_integration_test.go @@ -5,18 +5,105 @@ package cometd import ( + "fmt" + "sync" "testing" + bay "github.com/elastic/bayeux" + "github.com/elastic/beats/v7/filebeat/channel" "github.com/elastic/beats/v7/filebeat/input" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/stretchr/testify/require" ) +type eventCaptor struct { + c chan struct{} + closeOnce sync.Once + closed bool + events chan beat.Event +} + +func newEventCaptor(events chan beat.Event) channel.Outleter { + return &eventCaptor{ + c: make(chan struct{}), + events: events, + } +} + +func (ec *eventCaptor) OnEvent(event beat.Event) bool { + ec.events <- event + return true +} + +func (ec *eventCaptor) Close() error { + ec.closeOnce.Do(func() { + ec.closed = true + close(ec.c) + }) + return nil +} + +func (ec *eventCaptor) Done() <-chan struct{} { + return ec.c +} + func TestInput(t *testing.T) { - err := input.Register(inputName, NewInput) - require.Error(t, err) + logp.TestingSetup(logp.WithSelectors("cometd input", "cometd")) + + // Setup the input config. + config := common.MustNewConfigFrom(common.MapStr{ + "channel_name": "channel_name", + "auth.oauth2.client.id": "client.id", + "auth.oauth2.client.secret": "client.secret", + "auth.oauth2.user": "user", + "auth.oauth2.password": "password", + "auth.oauth2.token_url": "http://127.0.0.1:8080/token", + }) + + // Route input events through our captor instead of sending through ES. + eventsCh := make(chan beat.Event) + defer close(eventsCh) + + captor := newEventCaptor(eventsCh) + defer captor.Close() + + connector := channel.ConnectorFunc(func(_ *common.Config, _ beat.ClientConfig) (channel.Outleter, error) { + return channel.SubOutlet(captor), nil + }) + + // Mock the context. + inputContext := input.Context{ + Done: make(chan struct{}), + BeatDone: make(chan struct{}), + } + + // Setup the input + input, err := NewInput(config, connector, inputContext) + if err != nil { + fmt.Println("error: ", err) + } + + require.NoError(t, err) + require.NotNil(t, input) + + // Run the input. + input.Run() + + verifiedCh := make(chan struct{}) + defer close(verifiedCh) - var cometd cometdInput - cometd.Run() + var msg bay.TriggerEvent + msg.Data.Event.ReplayID = 1234 + msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) + msg.Channel = "first-channel" - makeEvent("test", "test") + for _, event := range []beat.Event{<-eventsCh} { + require.NoError(t, err) + message, err := event.GetValue("message") + fmt.Println(message) + require.NoError(t, err) + require.Equal(t, string(msg.Data.Payload), message) + } } diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index 36e6776e61fe..63db0ecf74ee 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -139,8 +139,8 @@ func NewInput( // Stop stops the input and waits for it to fully stop. func (in *cometdInput) Stop() { - close(in.out) in.workerCancel() + close(in.out) in.workerWg.Wait() } From 48c335ce5519558473f12630f2f7e9a934548fd1 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 6 Apr 2022 13:01:04 +0530 Subject: [PATCH 21/50] Lint issues --- x-pack/filebeat/filebeat.yml | 22 +++++-------------- .../input/cometd/cometd_integration_test.go | 5 ----- x-pack/filebeat/input/cometd/input_test.go | 14 ++++++------ 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/x-pack/filebeat/filebeat.yml b/x-pack/filebeat/filebeat.yml index f224563892a4..347152e32b16 100644 --- a/x-pack/filebeat/filebeat.yml +++ b/x-pack/filebeat/filebeat.yml @@ -13,15 +13,6 @@ # ============================== Filebeat inputs =============================== filebeat.inputs: -- type: cometd - enabled: true - channel_name: /event/LoginEventStream - auth.oauth2: - client.id: 3MVG9pRzvMkjMb6m1I.fz3f7zBuH4xdKCJcM9B5XLgxXh2AFTmQmr8JMn1v9_AoKIW7gwf2FIGqRwKZVEMR_C - client.secret: E9FE118633BC7C55D0AD395204F19C1D4529D09CB7231754AD2F2CA668400619 - token_url: https://login.salesforce.com/services/oauth2/token - user: kush.rana@elastic.co - password: Crest@123 # Each - is an input. Most options can be set at the input level, so # you can use different inputs for various configurations. # Below are the input specific configurations. @@ -138,12 +129,11 @@ setup.kibana: # ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -output.console: - pretty: true + # ---------------------------- Elasticsearch Output ---------------------------- -# output.elasticsearch: +output.elasticsearch: # Array of hosts to connect to. - # hosts: ["localhost:9200"] + hosts: ["localhost:9200"] # Protocol - either `http` (default) or `https`. #protocol: "https" @@ -172,9 +162,9 @@ output.console: processors: - add_host_metadata: when.not.contains.tags: forwarded - # - add_cloud_metadata: ~ - # - add_docker_metadata: ~ - # - add_kubernetes_metadata: ~ + - add_cloud_metadata: ~ + - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ # ================================== Logging =================================== diff --git a/x-pack/filebeat/input/cometd/cometd_integration_test.go b/x-pack/filebeat/input/cometd/cometd_integration_test.go index c8f909af20d9..ea868d31eb7f 100644 --- a/x-pack/filebeat/input/cometd/cometd_integration_test.go +++ b/x-pack/filebeat/input/cometd/cometd_integration_test.go @@ -81,10 +81,6 @@ func TestInput(t *testing.T) { // Setup the input input, err := NewInput(config, connector, inputContext) - if err != nil { - fmt.Println("error: ", err) - } - require.NoError(t, err) require.NotNil(t, input) @@ -102,7 +98,6 @@ func TestInput(t *testing.T) { for _, event := range []beat.Event{<-eventsCh} { require.NoError(t, err) message, err := event.GetValue("message") - fmt.Println(message) require.NoError(t, err) require.Equal(t, string(msg.Data.Payload), message) } diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index 54d71e384ece..52b1efb24b63 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -101,7 +101,7 @@ func oauth2TokenHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "application/json") _ = r.ParseForm() switch { - case r.Method != "POST": + case r.Method != http.MethodPost: w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(`{"error":"wrong method"}`)) default: @@ -114,7 +114,7 @@ func oauth2ClientIdHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "application/json") _ = r.ParseForm() switch { - case r.Method != "POST": + case r.Method != http.MethodPost: w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(`{"error":"wrong method"}`)) default: @@ -126,7 +126,7 @@ func oauth2SubscribeHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "application/json") _ = r.ParseForm() switch { - case r.Method != "POST": + case r.Method != http.MethodPost: w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(`{"error":"wrong method"}`)) default: @@ -138,7 +138,7 @@ func oauth2EventHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "application/json") _ = r.ParseForm() switch { - case r.Method != "POST": + case r.Method != http.MethodPost: w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(`{"error":"wrong method"}`)) default: @@ -152,13 +152,13 @@ func oauth2Handler(w http.ResponseWriter, r *http.Request) { return } body, _ := ioutil.ReadAll(r.Body) - if r.URL.Path == "/cometd/38.0" && string(body) == `{"channel": "/meta/handshake", "supportedConnectionTypes": ["long-polling"], "version": "1.0"}` { + if string(body) == `{"channel": "/meta/handshake", "supportedConnectionTypes": ["long-polling"], "version": "1.0"}` { oauth2ClientIdHandler(w, r) return - } else if r.URL.Path == "/cometd/38.0" && string(body) == `{"channel": "/meta/connect", "connectionType": "long-polling", "clientId": "94b112sp7ph1c9s41mycpzik4rkj3"} ` { + } else if string(body) == `{"channel": "/meta/connect", "connectionType": "long-polling", "clientId": "94b112sp7ph1c9s41mycpzik4rkj3"} ` { oauth2EventHandler(w, r) return - } else if r.URL.Path == "/cometd/38.0" && string(body) == `{ + } else if string(body) == `{ "channel": "/meta/subscribe", "subscription": "first-channel", "clientId": "94b112sp7ph1c9s41mycpzik4rkj3", From 4bc6bb8236819ef665eba5a7cca2f610a9f91537 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 6 Apr 2022 13:06:27 +0530 Subject: [PATCH 22/50] Lint --- x-pack/filebeat/input/cometd/cometd_integration_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/filebeat/input/cometd/cometd_integration_test.go b/x-pack/filebeat/input/cometd/cometd_integration_test.go index ea868d31eb7f..907a45b3975c 100644 --- a/x-pack/filebeat/input/cometd/cometd_integration_test.go +++ b/x-pack/filebeat/input/cometd/cometd_integration_test.go @@ -5,7 +5,6 @@ package cometd import ( - "fmt" "sync" "testing" From 5300d56d1bbe15dc7910b9d91a93cfc5b7252439 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 6 Apr 2022 15:18:05 +0530 Subject: [PATCH 23/50] Lint --- filebeat/docker-compose.yml | 5 ----- x-pack/filebeat/filebeat.yml | 1 + .../input/cometd/cometd_integration_test.go | 8 +++++--- x-pack/filebeat/input/cometd/config_test.go | 4 ++-- x-pack/filebeat/input/cometd/input_test.go | 19 ++++++++++++------- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/filebeat/docker-compose.yml b/filebeat/docker-compose.yml index 0ce8e391c73a..95dd4488639a 100644 --- a/filebeat/docker-compose.yml +++ b/filebeat/docker-compose.yml @@ -35,7 +35,6 @@ services: kibana: { condition: service_healthy } mosquitto: { condition: service_healthy } redis: { condition: service_healthy } - cometd: { condition: service_healthy } elasticsearch: extends: @@ -79,10 +78,6 @@ services: - ${ES_BEATS}/testing/environments/docker/cometd/files/:/files:ro environment: PORT: "8080" - healthcheck: - test: ["CMD-SHELL", "curl -X POST http://127.0.0.1:8080/token"] - retries: 300 - interval: 1s command: - http-server - --addr=:8080 diff --git a/x-pack/filebeat/filebeat.yml b/x-pack/filebeat/filebeat.yml index 347152e32b16..fb67b14d67ba 100644 --- a/x-pack/filebeat/filebeat.yml +++ b/x-pack/filebeat/filebeat.yml @@ -13,6 +13,7 @@ # ============================== Filebeat inputs =============================== filebeat.inputs: + # Each - is an input. Most options can be set at the input level, so # you can use different inputs for various configurations. # Below are the input specific configurations. diff --git a/x-pack/filebeat/input/cometd/cometd_integration_test.go b/x-pack/filebeat/input/cometd/cometd_integration_test.go index 907a45b3975c..45bccb04ed25 100644 --- a/x-pack/filebeat/input/cometd/cometd_integration_test.go +++ b/x-pack/filebeat/input/cometd/cometd_integration_test.go @@ -8,13 +8,15 @@ import ( "sync" "testing" - bay "github.com/elastic/bayeux" + "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/filebeat/channel" "github.com/elastic/beats/v7/filebeat/input" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" - "github.com/stretchr/testify/require" + + bay "github.com/elastic/bayeux" ) type eventCaptor struct { @@ -49,7 +51,7 @@ func (ec *eventCaptor) Done() <-chan struct{} { } func TestInput(t *testing.T) { - logp.TestingSetup(logp.WithSelectors("cometd input", "cometd")) + logp.TestingSetup(logp.WithSelectors("cometd input", "cometd")) //nolint:errcheck // Bad linter! no need to test this. // Setup the input config. config := common.MustNewConfigFrom(common.MapStr{ diff --git a/x-pack/filebeat/input/cometd/config_test.go b/x-pack/filebeat/input/cometd/config_test.go index 8750880181e6..6de088b8e3bb 100644 --- a/x-pack/filebeat/input/cometd/config_test.go +++ b/x-pack/filebeat/input/cometd/config_test.go @@ -14,8 +14,8 @@ const ( demoClientID = "DEMOCLIENTID" demoClientSecret = "DEMOCLIENTSECRET" salesforceUser = "salesforce_user" - pwd = "P@$$w0₹D" - tokenURL = "https://login.salesforce.com/services/oauth2/token" + pwd = "P@$$w0₹D" //nolint:gosec // Bad linter! The pwd is used as testing purpose. + tokenURL = "https://login.salesforce.com/services/oauth2/token" //nolint:gosec // Bad linter! The tokenURL is used as testing purpose. ) // Validate that it finds the application default credentials and does diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index 52b1efb24b63..c289863c0593 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -11,13 +11,18 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + bay "github.com/elastic/bayeux" finput "github.com/elastic/beats/v7/filebeat/input" "github.com/elastic/beats/v7/filebeat/input/inputtest" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" +) + +const ( + channelName = "first-channel" ) var ( @@ -26,12 +31,12 @@ var ( func TestNewInputDone(t *testing.T) { config := common.MapStr{ - "channel_name": "cometd-channel", + "channel_name": channelName, "auth.oauth2.client.id": "DEMOCLIENTID", "auth.oauth2.client.secret": "DEMOCLIENTSECRET", "auth.oauth2.user": "salesforce_user", - "auth.oauth2.password": "P@$$w0₹D", - "auth.oauth2.token_url": "https://login.salesforce.com/services/oauth2/token", + "auth.oauth2.password": "pwd", + "auth.oauth2.token_url": "https://example.com/token", } inputtest.AssertNotStartedInputCanBeDone(t, NewInput, &config) } @@ -68,10 +73,10 @@ func TestNewInput_Run(t *testing.T) { var msg bay.TriggerEvent msg.Data.Event.ReplayID = 1234 msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) - msg.Channel = "first-channel" + msg.Channel = channelName config := map[string]interface{}{ - "channel_name": "first-channel", + "channel_name": channelName, "auth.oauth2.client.id": "client.id", "auth.oauth2.client.secret": "client.secret", "auth.oauth2.user": "user", From 2af933fc0502be6f0cde1de38953d4b03887f369 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 6 Apr 2022 18:12:45 +0530 Subject: [PATCH 24/50] update bayeux v1.0.3 --- go.mod | 2 +- go.sum | 6 ++--- x-pack/filebeat/input/cometd/client_mocked.go | 19 +++----------- x-pack/filebeat/input/cometd/input.go | 25 ++++++++++++------- 4 files changed, 22 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index acbb785e1237..21ce67eda2ce 100644 --- a/go.mod +++ b/go.mod @@ -189,6 +189,7 @@ require ( ) require ( + github.com/elastic/bayeux v1.0.3 github.com/elastic/elastic-agent-libs v0.1.1 github.com/shirou/gopsutil/v3 v3.21.12 ) @@ -220,7 +221,6 @@ require ( github.com/docker/distribution v2.8.1+incompatible // indirect github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/elastic/bayeux v1.0.2 // indirect github.com/envoyproxy/go-control-plane v0.10.1 // indirect github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect diff --git a/go.sum b/go.sum index cf8e60231b4c..44d764317865 100644 --- a/go.sum +++ b/go.sum @@ -563,10 +563,8 @@ github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7j github.com/eclipse/paho.mqtt.golang v1.3.5 h1:sWtmgNxYM9P2sP+xEItMozsR3w0cqZFlqnNN1bdl41Y= github.com/eclipse/paho.mqtt.golang v1.3.5/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elastic/bayeux v1.0.1 h1:siXwhEy+SJYHWiwq2XuAF4FKr7soVCdbQdx7W+as+0Y= -github.com/elastic/bayeux v1.0.1/go.mod h1:ET5dBf91w0cukiCqUHtlEaGx32f75SBU5IZyx+jNfjw= -github.com/elastic/bayeux v1.0.2 h1:bdDIH9DkXkzv2PDa/Z8vGiHcqNWv+FmouGFYWLgnw60= -github.com/elastic/bayeux v1.0.2/go.mod h1:ET5dBf91w0cukiCqUHtlEaGx32f75SBU5IZyx+jNfjw= +github.com/elastic/bayeux v1.0.3 h1:1ovbe7dTDVqWFmtNrRfaMwb9hheCNEaMg+ymvohRGrM= +github.com/elastic/bayeux v1.0.3/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo= diff --git a/x-pack/filebeat/input/cometd/client_mocked.go b/x-pack/filebeat/input/cometd/client_mocked.go index 95ffef839ba8..a4123b77bf80 100644 --- a/x-pack/filebeat/input/cometd/client_mocked.go +++ b/x-pack/filebeat/input/cometd/client_mocked.go @@ -1,19 +1,6 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. package cometd diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index 63db0ecf74ee..30b384840eb1 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -8,7 +8,6 @@ import ( "context" "encoding/json" "fmt" - "os" "sync" "time" @@ -29,6 +28,7 @@ const ( // Run starts the input worker then returns. Only the first invocation // will ever start the worker. func (in *cometdInput) Run() { + var err error in.workerOnce.Do(func() { in.workerWg.Add(1) go func() { @@ -37,7 +37,11 @@ func (in *cometdInput) Run() { defer in.workerWg.Done() defer in.workerCancel() in.b = bay.Bayeux{} - in.creds = bay.GetSalesforceCredentials() + in.creds, err = bay.GetSalesforceCredentials(in.authParams) + if err != nil { + in.log.Error("not able to get access token", err) + return + } if err := in.run(); err != nil { in.log.Error(err) return @@ -47,7 +51,7 @@ func (in *cometdInput) Run() { } func (in *cometdInput) run() error { - in.out = in.b.Channel(in.out, "-1", in.creds, in.config.ChannelName) + in.out = in.b.Channel(in.out, "-1", *in.creds, in.config.ChannelName) var event Event for e := range in.out { @@ -98,11 +102,12 @@ func NewInput( return nil, err } - os.Setenv("SALESFORCE_CONSUMER_KEY", conf.Auth.OAuth2.ClientID) - os.Setenv("SALESFORCE_CONSUMER_SECRET", conf.Auth.OAuth2.ClientSecret) - os.Setenv("SALESFORCE_USER", conf.Auth.OAuth2.User) - os.Setenv("SALESFORCE_PASSWORD", conf.Auth.OAuth2.Password) - os.Setenv("SALESFORCE_TOKEN_URL", conf.Auth.OAuth2.TokenURL) + var authParams bay.AuthenticationParameters + authParams.ClientID = conf.Auth.OAuth2.ClientID + authParams.ClientSecret = conf.Auth.OAuth2.ClientSecret + authParams.Username = conf.Auth.OAuth2.User + authParams.Password = conf.Auth.OAuth2.Password + authParams.TokenURL = conf.Auth.OAuth2.TokenURL logger := logp.NewLogger("cometd").With( "pubsub_channel", conf.ChannelName) @@ -123,6 +128,7 @@ func NewInput( workerCtx: workerCtx, workerCancel: workerCancel, ackedCount: atomic.NewUint32(0), + authParams: authParams, } // Creating a new channel for cometd input @@ -164,7 +170,8 @@ type cometdInput struct { ackedCount *atomic.Uint32 // Total number of successfully ACKed messages. out chan bay.TriggerEvent b bay.Bayeux - creds bay.Credentials + creds *bay.Credentials + authParams bay.AuthenticationParameters } type Event struct { From a58177f7b2c6eb307573541bbfa20da7e0e53dbf Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 6 Apr 2022 20:41:42 +0530 Subject: [PATCH 25/50] Add NOTICE.txt --- NOTICE.txt | 178 ++++++++++++++++++++++------------------------------- 1 file changed, 74 insertions(+), 104 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index ff9d39e989f7..17474c242fbf 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -5805,6 +5805,37 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/bayeux +Version: v1.0.3 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/bayeux@v1.0.3/LICENSE: + +MIT License + +Copyright (c) 2017 Zander Hill + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-client/v7 Version: v7.0.0-20210727140539-f0905d9377f6 @@ -12490,67 +12521,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/kardianos/service -Version: v1.2.1-0.20210728001519-a323c3813bc7 -Licence type (autodetected): Zlib --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/kardianos/service@v1.2.1-0.20210728001519-a323c3813bc7/LICENSE: - -Copyright (c) 2015 Daniel Theophanes - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. - - --------------------------------------------------------------------------------- -Dependency : github.com/elastic/bayeux -Version: v1.0.1 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/elastic/bayeux@v1.0.1/LICENSE: - -MIT License - -Copyright (c) 2017 Zander Hill - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/lib/pq Version: v1.10.3 @@ -20425,28 +20395,28 @@ Licence type (autodetected): MIT Contents of probable licence file $GOMODCACHE/github.com/!azure/go-amqp@v0.16.0/LICENSE: - MIT License - - Copyright (C) 2017 Kale Blankenship - Portions Copyright (C) Microsoft Corporation - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE + MIT License + + Copyright (C) 2017 Kale Blankenship + Portions Copyright (C) Microsoft Corporation + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE -------------------------------------------------------------------------------- @@ -22124,27 +22094,27 @@ Licence type (autodetected): MIT Contents of probable licence file $GOMODCACHE/github.com/akavel/rsrc@v0.8.0/LICENSE.txt: -The MIT License (MIT) - -Copyright (c) 2013-2017 The rsrc Authors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +The MIT License (MIT) + +Copyright (c) 2013-2017 The rsrc Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. -------------------------------------------------------------------------------- From ce5f45416706db2022f27bbd17bba95f82a067f8 Mon Sep 17 00:00:00 2001 From: yug-elastic Date: Thu, 7 Apr 2022 15:13:50 +0530 Subject: [PATCH 26/50] Add negative test cases --- x-pack/filebeat/input/cometd/input_test.go | 96 ++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index c289863c0593..584ab2d4a0b9 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -5,6 +5,7 @@ package cometd import ( + "context" "io/ioutil" "net/http" "net/http/httptest" @@ -19,6 +20,8 @@ import ( "github.com/elastic/beats/v7/filebeat/input/inputtest" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/atomic" + "github.com/elastic/beats/v7/libbeat/logp" ) const ( @@ -102,6 +105,99 @@ func TestNewInput_Run(t *testing.T) { server.Close() } +func TestNewInput_Run_Wait(t *testing.T) { + eventsCh := make(chan beat.Event) + + outlet := &mockedOutleter{ + onEventHandler: func(event beat.Event) bool { + eventsCh <- event + return true + }, + } + connector := &mockedConnector{ + outlet: outlet, + } + var inputContext finput.Context + + var msg bay.TriggerEvent + msg.Data.Event.ReplayID = 1234 + msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) + msg.Channel = channelName + + config := map[string]interface{}{ + "channel_name": channelName, + "auth.oauth2.client.id": "client.id", + "auth.oauth2.client.secret": "client.secret", + "auth.oauth2.user": "user", + "auth.oauth2.password": "password", + } + + r := http.HandlerFunc(oauth2Handler) + server := httptest.NewServer(r) + serverURL = server.URL + config["auth.oauth2.token_url"] = server.URL + "/token" + + cfg := common.MustNewConfigFrom(config) + + input, err := NewInput(cfg, connector, inputContext) + require.NoError(t, err) + require.NotNil(t, input) + + input.Run() + + go func() { + time.Sleep(100 * time.Millisecond) // let input.Stop() be executed. + for range eventsCh { + } + }() + + input.Wait() +} + +func TestStop(t *testing.T) { + conf := defaultConfig() + logger := logp.NewLogger("test") + authParams := bay.AuthenticationParameters{} + inputCtx, cancelInputCtx := context.WithCancel(context.Background()) + workerCtx, workerCancel := context.WithCancel(inputCtx) + defer cancelInputCtx() + + input := &cometdInput{ + config: conf, + log: logger, + inputCtx: inputCtx, + workerCtx: workerCtx, + workerCancel: workerCancel, + ackedCount: atomic.NewUint32(0), + authParams: authParams, + } + input.out = make(chan bay.TriggerEvent) + + input.Stop() +} + +func TestWait(t *testing.T) { + conf := defaultConfig() + logger := logp.NewLogger("test") + authParams := bay.AuthenticationParameters{} + inputCtx, cancelInputCtx := context.WithCancel(context.Background()) + workerCtx, workerCancel := context.WithCancel(inputCtx) + defer cancelInputCtx() + + input := &cometdInput{ + config: conf, + log: logger, + inputCtx: inputCtx, + workerCtx: workerCtx, + workerCancel: workerCancel, + ackedCount: atomic.NewUint32(0), + authParams: authParams, + } + input.out = make(chan bay.TriggerEvent) + + input.Wait() +} + func oauth2TokenHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "application/json") _ = r.ParseForm() From 7a26efc372a85dedb60ea01a5a52435053e903d6 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 13 Apr 2022 14:47:19 +0530 Subject: [PATCH 27/50] requested changes --- NOTICE.txt | 90 +++++------ filebeat/docker-compose.yml | 14 -- go.mod | 2 +- go.sum | 2 + testing/environments/docker/cometd/Dockerfile | 8 + .../docker/cometd/files/config.yml | 2 +- x-pack/filebeat/docker-compose.yml | 6 + .../input/cometd/cometd_integration_test.go | 29 ++-- x-pack/filebeat/input/cometd/input.go | 35 +++-- x-pack/filebeat/input/cometd/input_test.go | 146 ++++++++++++++---- 10 files changed, 215 insertions(+), 119 deletions(-) create mode 100644 testing/environments/docker/cometd/Dockerfile diff --git a/NOTICE.txt b/NOTICE.txt index 17474c242fbf..754c8847bc88 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -5807,11 +5807,11 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -------------------------------------------------------------------------------- Dependency : github.com/elastic/bayeux -Version: v1.0.3 +Version: v1.0.4 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/bayeux@v1.0.3/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/bayeux@v1.0.4/LICENSE: MIT License @@ -20395,28 +20395,28 @@ Licence type (autodetected): MIT Contents of probable licence file $GOMODCACHE/github.com/!azure/go-amqp@v0.16.0/LICENSE: - MIT License - - Copyright (C) 2017 Kale Blankenship - Portions Copyright (C) Microsoft Corporation - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE + MIT License + + Copyright (C) 2017 Kale Blankenship + Portions Copyright (C) Microsoft Corporation + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE -------------------------------------------------------------------------------- @@ -22094,27 +22094,27 @@ Licence type (autodetected): MIT Contents of probable licence file $GOMODCACHE/github.com/akavel/rsrc@v0.8.0/LICENSE.txt: -The MIT License (MIT) - -Copyright (c) 2013-2017 The rsrc Authors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +The MIT License (MIT) + +Copyright (c) 2013-2017 The rsrc Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. -------------------------------------------------------------------------------- diff --git a/filebeat/docker-compose.yml b/filebeat/docker-compose.yml index 95dd4488639a..a73f0bc39d6b 100644 --- a/filebeat/docker-compose.yml +++ b/filebeat/docker-compose.yml @@ -68,17 +68,3 @@ services: redis: build: ${PWD}/input/redis/_meta - - cometd: - image: docker.elastic.co/observability/stream:v0.6.1 - hostname: cometd - ports: - - 8080:8080 - volumes: - - ${ES_BEATS}/testing/environments/docker/cometd/files/:/files:ro - environment: - PORT: "8080" - command: - - http-server - - --addr=:8080 - - --config=/files/config.yml diff --git a/go.mod b/go.mod index 21ce67eda2ce..e5a014fbd63e 100644 --- a/go.mod +++ b/go.mod @@ -189,7 +189,7 @@ require ( ) require ( - github.com/elastic/bayeux v1.0.3 + github.com/elastic/bayeux v1.0.4 github.com/elastic/elastic-agent-libs v0.1.1 github.com/shirou/gopsutil/v3 v3.21.12 ) diff --git a/go.sum b/go.sum index 44d764317865..18ac4e938223 100644 --- a/go.sum +++ b/go.sum @@ -565,6 +565,8 @@ github.com/eclipse/paho.mqtt.golang v1.3.5/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+J github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/bayeux v1.0.3 h1:1ovbe7dTDVqWFmtNrRfaMwb9hheCNEaMg+ymvohRGrM= github.com/elastic/bayeux v1.0.3/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= +github.com/elastic/bayeux v1.0.4 h1:tIPINegYnFdLuyEKdd31rYbQsrMH3ItwZak88lAxvV0= +github.com/elastic/bayeux v1.0.4/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo= diff --git a/testing/environments/docker/cometd/Dockerfile b/testing/environments/docker/cometd/Dockerfile new file mode 100644 index 000000000000..44f19504af2a --- /dev/null +++ b/testing/environments/docker/cometd/Dockerfile @@ -0,0 +1,8 @@ +FROM docker.elastic.co/observability/stream:v0.6.1 + +RUN apt update && \ + apt -y install curl +ENV PORT="8080" +COPY files /files +HEALTHCHECK --interval=1s --retries=600 CMD curl -X POST http://localhost:8080/token +CMD [ "http-server", "--addr=:8080", "--config=/files/config.yml" ] diff --git a/testing/environments/docker/cometd/files/config.yml b/testing/environments/docker/cometd/files/config.yml index 96a2646dab0e..fd02ac72a195 100644 --- a/testing/environments/docker/cometd/files/config.yml +++ b/testing/environments/docker/cometd/files/config.yml @@ -4,7 +4,7 @@ rules: responses: - status_code: 200 body: | - {"instance_url": "http://127.0.0.1:8080", "expires_in": "60", "access_token": "abcd"} + {"instance_url": "http://cometd:8080", "expires_in": "60", "access_token": "abcd"} - path: /cometd/38.0 methods: ["POST"] request_body: '{"channel": "/meta/handshake", "supportedConnectionTypes": ["long-polling"], "version": "1.0"}' diff --git a/x-pack/filebeat/docker-compose.yml b/x-pack/filebeat/docker-compose.yml index c53bb1ca983c..f0170efce624 100644 --- a/x-pack/filebeat/docker-compose.yml +++ b/x-pack/filebeat/docker-compose.yml @@ -23,6 +23,7 @@ services: image: busybox depends_on: elasticsearch: { condition: service_healthy } + cometd: { condition: service_healthy } elasticsearch: extends: @@ -33,3 +34,8 @@ services: retries: 300 interval: 1s + cometd: + build: ${ES_BEATS}/testing/environments/docker/cometd + hostname: cometd + ports: + - 8080:8080 diff --git a/x-pack/filebeat/input/cometd/cometd_integration_test.go b/x-pack/filebeat/input/cometd/cometd_integration_test.go index 45bccb04ed25..b0461e2a5ec0 100644 --- a/x-pack/filebeat/input/cometd/cometd_integration_test.go +++ b/x-pack/filebeat/input/cometd/cometd_integration_test.go @@ -2,6 +2,9 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build integration +// +build integration + package cometd import ( @@ -55,12 +58,12 @@ func TestInput(t *testing.T) { // Setup the input config. config := common.MustNewConfigFrom(common.MapStr{ - "channel_name": "channel_name", + "channel_name": "channel_name1", "auth.oauth2.client.id": "client.id", "auth.oauth2.client.secret": "client.secret", "auth.oauth2.user": "user", "auth.oauth2.password": "password", - "auth.oauth2.token_url": "http://127.0.0.1:8080/token", + "auth.oauth2.token_url": "http://cometd:8080/token", }) // Route input events through our captor instead of sending through ES. @@ -85,21 +88,19 @@ func TestInput(t *testing.T) { require.NoError(t, err) require.NotNil(t, input) + var msg bay.MaybeMsg + msg.Msg.Data.Event.ReplayID = 1234 + msg.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) + msg.Msg.Channel = "channel_name1" + // Run the input. input.Run() - verifiedCh := make(chan struct{}) - defer close(verifiedCh) + event := <-eventsCh - var msg bay.TriggerEvent - msg.Data.Event.ReplayID = 1234 - msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) - msg.Channel = "first-channel" + val, err := event.GetValue("message") + require.NoError(t, err) + require.Equal(t, string(msg.Msg.Data.Payload), val) - for _, event := range []beat.Event{<-eventsCh} { - require.NoError(t, err) - message, err := event.GetValue("message") - require.NoError(t, err) - require.Equal(t, string(msg.Data.Payload), message) - } + input.Stop() } diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index 30b384840eb1..a8aaccb139f8 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -15,7 +15,6 @@ import ( "github.com/elastic/beats/v7/filebeat/input" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/atomic" "github.com/elastic/beats/v7/libbeat/logp" bay "github.com/elastic/bayeux" @@ -51,30 +50,31 @@ func (in *cometdInput) Run() { } func (in *cometdInput) run() error { - in.out = in.b.Channel(in.out, "-1", *in.creds, in.config.ChannelName) - - var event Event - for e := range in.out { - if !e.Successful { + in.msgCh = in.b.Channel(in.workerCtx, in.msgCh, "-1", *in.creds, in.config.ChannelName) + for e := range in.msgCh { + if e.Failed() { + return fmt.Errorf("error collecting events: %w", e.Err) + } + if !e.Msg.Successful { + var event Event // To handle the last response where the object received was empty - if e.Data.Payload == nil { + if e.Msg.Data.Payload == nil { return nil } // Convert json.RawMessage response to []byte - msg, err := e.Data.Payload.MarshalJSON() + msg, err := e.Msg.Data.Payload.MarshalJSON() if err != nil { return fmt.Errorf("JSON error: %w", err) } // Extract event IDs from json.RawMessage - err = json.Unmarshal(e.Data.Payload, &event) + err = json.Unmarshal(e.Msg.Data.Payload, &event) if err != nil { return fmt.Errorf("error while parsing JSON: %w", err) } if ok := in.outlet.OnEvent(makeEvent(event.EventId, string(msg))); !ok { in.log.Debug("OnEvent returned false. Stopping input worker.") - close(in.out) return fmt.Errorf("error ingesting data to elasticsearch") } } @@ -115,7 +115,10 @@ func NewInput( // Wrap input.Context's Done channel with a context.Context. This goroutine // stops with the parent closes the Done channel. inputCtx, cancelInputCtx := context.WithCancel(context.Background()) - defer cancelInputCtx() + go func() { + <-inputContext.Done + cancelInputCtx() + }() // If the input ever needs to be made restartable, then context would need // to be recreated with each restart. @@ -127,12 +130,11 @@ func NewInput( inputCtx: inputCtx, workerCtx: workerCtx, workerCancel: workerCancel, - ackedCount: atomic.NewUint32(0), authParams: authParams, } - // Creating a new channel for cometd input - in.out = make(chan bay.TriggerEvent) + // Creating a new channel for cometd input. + in.msgCh = make(chan bay.MaybeMsg) // Build outlet for events. in.outlet, err = connector.Connect(cfg) @@ -141,12 +143,12 @@ func NewInput( } in.log.Infof("Initialized %s input.", inputName) return in, nil + } // Stop stops the input and waits for it to fully stop. func (in *cometdInput) Stop() { in.workerCancel() - close(in.out) in.workerWg.Wait() } @@ -167,8 +169,7 @@ type cometdInput struct { workerOnce sync.Once // Guarantees that the worker goroutine is only started once. workerWg sync.WaitGroup // Waits on worker goroutine. - ackedCount *atomic.Uint32 // Total number of successfully ACKed messages. - out chan bay.TriggerEvent + msgCh chan bay.MaybeMsg b bay.Bayeux creds *bay.Credentials authParams bay.AuthenticationParameters diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index 584ab2d4a0b9..09d6b96b84f0 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -6,6 +6,7 @@ package cometd import ( "context" + "fmt" "io/ioutil" "net/http" "net/http/httptest" @@ -20,12 +21,12 @@ import ( "github.com/elastic/beats/v7/filebeat/input/inputtest" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/atomic" "github.com/elastic/beats/v7/libbeat/logp" ) const ( - channelName = "first-channel" + firstChannel = "channel_name1" + secondChannel = "channel_name2" ) var ( @@ -34,7 +35,7 @@ var ( func TestNewInputDone(t *testing.T) { config := common.MapStr{ - "channel_name": channelName, + "channel_name": firstChannel, "auth.oauth2.client.id": "DEMOCLIENTID", "auth.oauth2.client.secret": "DEMOCLIENTSECRET", "auth.oauth2.user": "salesforce_user", @@ -73,13 +74,13 @@ func TestNewInput_Run(t *testing.T) { } var inputContext finput.Context - var msg bay.TriggerEvent - msg.Data.Event.ReplayID = 1234 - msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) - msg.Channel = channelName + var expected bay.MaybeMsg + expected.Msg.Data.Event.ReplayID = 1234 + expected.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) + expected.Msg.Channel = firstChannel config := map[string]interface{}{ - "channel_name": channelName, + "channel_name": firstChannel, "auth.oauth2.client.id": "client.id", "auth.oauth2.client.secret": "client.secret", "auth.oauth2.user": "user", @@ -88,6 +89,8 @@ func TestNewInput_Run(t *testing.T) { r := http.HandlerFunc(oauth2Handler) server := httptest.NewServer(r) + defer server.Close() + serverURL = server.URL config["auth.oauth2.token_url"] = server.URL + "/token" @@ -98,13 +101,88 @@ func TestNewInput_Run(t *testing.T) { require.NotNil(t, input) input.Run() - for _, event := range []beat.Event{<-eventsCh} { - require.NoError(t, err) - assertEventMatches(t, msg, event) + defer input.Stop() + event := <-eventsCh + message, err := event.GetValue("message") + require.NoError(t, err) + require.Equal(t, string(expected.Msg.Data.Payload), message) +} + +func TestNewMultiInput_Run(t *testing.T) { + eventsCh := make(chan beat.Event) + defer close(eventsCh) + + outlet := &mockedOutleter{ + onEventHandler: func(event beat.Event) bool { + eventsCh <- event + return true + }, + } + connector := &mockedConnector{ + outlet: outlet, + } + + var expected bay.MaybeMsg + expected.Msg.Data.Event.ReplayID = 1234 + expected.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) + expected.Msg.Channel = firstChannel + + config1 := map[string]interface{}{ + "channel_name": firstChannel, + "auth.oauth2.client.id": "client.id", + "auth.oauth2.client.secret": "client.secret", + "auth.oauth2.user": "user", + "auth.oauth2.password": "password", + } + config2 := map[string]interface{}{ + "channel_name": secondChannel, + "auth.oauth2.client.id": "client.id", + "auth.oauth2.client.secret": "client.secret", + "auth.oauth2.user": "user", + "auth.oauth2.password": "password", } + + // create Server + r := http.HandlerFunc(oauth2Handler) + server := httptest.NewServer(r) + serverURL = server.URL + config1["auth.oauth2.token_url"] = serverURL + "/token" + config2["auth.oauth2.token_url"] = serverURL + "/token" + + // get common config + cfg1 := common.MustNewConfigFrom(config1) + cfg2 := common.MustNewConfigFrom(config2) + + var inputContext finput.Context + + // intialize inputs + input1, err := NewInput(cfg1, connector, inputContext) + require.NoError(t, err) + require.NotNil(t, input1) + + input2, err := NewInput(cfg2, connector, inputContext) + require.NoError(t, err) + require.NotNil(t, input2) + + // run input + input1.Run() + defer input1.Stop() + + event1 := <-eventsCh + assertEventMatches(t, expected, event1) + + // run input + input2.Run() + defer input2.Stop() + + event2 := <-eventsCh + assertEventMatches(t, expected, event2) + + // close server server.Close() } +// TestNewInput_Run_Wait to test input wait func TestNewInput_Run_Wait(t *testing.T) { eventsCh := make(chan beat.Event) @@ -119,13 +197,13 @@ func TestNewInput_Run_Wait(t *testing.T) { } var inputContext finput.Context - var msg bay.TriggerEvent - msg.Data.Event.ReplayID = 1234 - msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) - msg.Channel = channelName + var msg bay.MaybeMsg + msg.Msg.Data.Event.ReplayID = 1234 + msg.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) + msg.Msg.Channel = firstChannel config := map[string]interface{}{ - "channel_name": channelName, + "channel_name": firstChannel, "auth.oauth2.client.id": "client.id", "auth.oauth2.client.secret": "client.secret", "auth.oauth2.user": "user", @@ -135,7 +213,7 @@ func TestNewInput_Run_Wait(t *testing.T) { r := http.HandlerFunc(oauth2Handler) server := httptest.NewServer(r) serverURL = server.URL - config["auth.oauth2.token_url"] = server.URL + "/token" + config["auth.oauth2.token_url"] = serverURL + "/token" cfg := common.MustNewConfigFrom(config) @@ -143,15 +221,16 @@ func TestNewInput_Run_Wait(t *testing.T) { require.NoError(t, err) require.NotNil(t, input) + // run input input.Run() go func() { time.Sleep(100 * time.Millisecond) // let input.Stop() be executed. - for range eventsCh { - } + input.Wait() }() - input.Wait() + for range []beat.Event{<-eventsCh} { + } } func TestStop(t *testing.T) { @@ -168,12 +247,16 @@ func TestStop(t *testing.T) { inputCtx: inputCtx, workerCtx: workerCtx, workerCancel: workerCancel, - ackedCount: atomic.NewUint32(0), authParams: authParams, } - input.out = make(chan bay.TriggerEvent) + input.msgCh = make(chan bay.MaybeMsg) input.Stop() + select { + case <-workerCtx.Done(): + default: + require.NoError(t, fmt.Errorf("input is not stopped.")) + } } func TestWait(t *testing.T) { @@ -190,12 +273,21 @@ func TestWait(t *testing.T) { inputCtx: inputCtx, workerCtx: workerCtx, workerCancel: workerCancel, - ackedCount: atomic.NewUint32(0), authParams: authParams, } - input.out = make(chan bay.TriggerEvent) + input.msgCh = make(chan bay.MaybeMsg) + + go func() { + time.Sleep(1000 * time.Millisecond) + input.Wait() + }() - input.Wait() + time.Sleep(1000 * time.Millisecond) // let input.Stop() be executed. + select { + case <-workerCtx.Done(): + default: + require.NoError(t, fmt.Errorf("input is not stopped.")) + } } func oauth2TokenHandler(w http.ResponseWriter, r *http.Request) { @@ -272,8 +364,8 @@ func oauth2Handler(w http.ResponseWriter, r *http.Request) { } } -func assertEventMatches(t *testing.T, expected bay.TriggerEvent, got beat.Event) { +func assertEventMatches(t *testing.T, expected bay.MaybeMsg, got beat.Event) { message, err := got.GetValue("message") require.NoError(t, err) - require.Equal(t, string(expected.Data.Payload), message) + require.Equal(t, string(expected.Msg.Data.Payload), message) } From abd6b7096405f1ecfa102632f98be243f969b024 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 13 Apr 2022 15:01:28 +0530 Subject: [PATCH 28/50] Resolve CI lint errors --- x-pack/filebeat/input/cometd/input_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index 09d6b96b84f0..3f0640f4d729 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -155,7 +155,7 @@ func TestNewMultiInput_Run(t *testing.T) { var inputContext finput.Context - // intialize inputs + // initialize inputs input1, err := NewInput(cfg1, connector, inputContext) require.NoError(t, err) require.NotNil(t, input1) From 1024be9183e3e7162ea9efeeb0a76fb4678dc5d0 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 13 Apr 2022 15:08:31 +0530 Subject: [PATCH 29/50] Update go.sum --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 18ac4e938223..f14a125a9653 100644 --- a/go.sum +++ b/go.sum @@ -563,8 +563,6 @@ github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7j github.com/eclipse/paho.mqtt.golang v1.3.5 h1:sWtmgNxYM9P2sP+xEItMozsR3w0cqZFlqnNN1bdl41Y= github.com/eclipse/paho.mqtt.golang v1.3.5/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elastic/bayeux v1.0.3 h1:1ovbe7dTDVqWFmtNrRfaMwb9hheCNEaMg+ymvohRGrM= -github.com/elastic/bayeux v1.0.3/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/bayeux v1.0.4 h1:tIPINegYnFdLuyEKdd31rYbQsrMH3ItwZak88lAxvV0= github.com/elastic/bayeux v1.0.4/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= From d8940a933fc78bf8541814466f9a43dd57488edd Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 13 Apr 2022 15:29:44 +0530 Subject: [PATCH 30/50] Update NOTICE.txt --- NOTICE.txt | 86 +++++++++++++++++++++++++++--------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 754c8847bc88..27ab51b8df4d 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -20395,28 +20395,28 @@ Licence type (autodetected): MIT Contents of probable licence file $GOMODCACHE/github.com/!azure/go-amqp@v0.16.0/LICENSE: - MIT License - - Copyright (C) 2017 Kale Blankenship - Portions Copyright (C) Microsoft Corporation - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE + MIT License + + Copyright (C) 2017 Kale Blankenship + Portions Copyright (C) Microsoft Corporation + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE -------------------------------------------------------------------------------- @@ -22094,27 +22094,27 @@ Licence type (autodetected): MIT Contents of probable licence file $GOMODCACHE/github.com/akavel/rsrc@v0.8.0/LICENSE.txt: -The MIT License (MIT) - -Copyright (c) 2013-2017 The rsrc Authors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +The MIT License (MIT) + +Copyright (c) 2013-2017 The rsrc Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. -------------------------------------------------------------------------------- From efa7df0904a166be3fa302fdd8345ee353e9b702 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 13 Apr 2022 18:03:16 +0530 Subject: [PATCH 31/50] Update unit tests --- x-pack/filebeat/input/cometd/input_test.go | 509 +++++++++++---------- 1 file changed, 255 insertions(+), 254 deletions(-) diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index 3f0640f4d729..48d09af556da 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -33,261 +33,262 @@ var ( serverURL string ) -func TestNewInputDone(t *testing.T) { - config := common.MapStr{ - "channel_name": firstChannel, - "auth.oauth2.client.id": "DEMOCLIENTID", - "auth.oauth2.client.secret": "DEMOCLIENTSECRET", - "auth.oauth2.user": "salesforce_user", - "auth.oauth2.password": "pwd", - "auth.oauth2.token_url": "https://example.com/token", - } - inputtest.AssertNotStartedInputCanBeDone(t, NewInput, &config) -} - -func TestMakeEventFailure(t *testing.T) { - event := beat.Event{ - Timestamp: time.Now().UTC(), - Fields: common.MapStr{ - "event": common.MapStr{ - "id": "DEMOID", - "created": time.Now().UTC(), +func TestInput(t *testing.T) { + t.Run("Check input Done", func(t *testing.T) { + config := common.MapStr{ + "channel_name": firstChannel, + "auth.oauth2.client.id": "DEMOCLIENTID", + "auth.oauth2.client.secret": "DEMOCLIENTSECRET", + "auth.oauth2.user": "salesforce_user", + "auth.oauth2.password": "pwd", + "auth.oauth2.token_url": "https://example.com/token", + } + inputtest.AssertNotStartedInputCanBeDone(t, NewInput, &config) + }) + + t.Run("Check make event failure", func(t *testing.T) { + event := beat.Event{ + Timestamp: time.Now().UTC(), + Fields: common.MapStr{ + "event": common.MapStr{ + "id": "DEMOID", + "created": time.Now().UTC(), + }, + "message": "DEMOBODYFAIL", }, - "message": "DEMOBODYFAIL", - }, - Private: "DEMOBODYFAIL", - } - assert.NotEqual(t, event, makeEvent("DEMOID", "DEMOBODY")) -} - -func TestNewInput_Run(t *testing.T) { - eventsCh := make(chan beat.Event) - - outlet := &mockedOutleter{ - onEventHandler: func(event beat.Event) bool { - eventsCh <- event - return true - }, - } - connector := &mockedConnector{ - outlet: outlet, - } - var inputContext finput.Context - - var expected bay.MaybeMsg - expected.Msg.Data.Event.ReplayID = 1234 - expected.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) - expected.Msg.Channel = firstChannel - - config := map[string]interface{}{ - "channel_name": firstChannel, - "auth.oauth2.client.id": "client.id", - "auth.oauth2.client.secret": "client.secret", - "auth.oauth2.user": "user", - "auth.oauth2.password": "password", - } - - r := http.HandlerFunc(oauth2Handler) - server := httptest.NewServer(r) - defer server.Close() - - serverURL = server.URL - config["auth.oauth2.token_url"] = server.URL + "/token" - - cfg := common.MustNewConfigFrom(config) - - input, err := NewInput(cfg, connector, inputContext) - require.NoError(t, err) - require.NotNil(t, input) - - input.Run() - defer input.Stop() - event := <-eventsCh - message, err := event.GetValue("message") - require.NoError(t, err) - require.Equal(t, string(expected.Msg.Data.Payload), message) -} - -func TestNewMultiInput_Run(t *testing.T) { - eventsCh := make(chan beat.Event) - defer close(eventsCh) - - outlet := &mockedOutleter{ - onEventHandler: func(event beat.Event) bool { - eventsCh <- event - return true - }, - } - connector := &mockedConnector{ - outlet: outlet, - } - - var expected bay.MaybeMsg - expected.Msg.Data.Event.ReplayID = 1234 - expected.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) - expected.Msg.Channel = firstChannel - - config1 := map[string]interface{}{ - "channel_name": firstChannel, - "auth.oauth2.client.id": "client.id", - "auth.oauth2.client.secret": "client.secret", - "auth.oauth2.user": "user", - "auth.oauth2.password": "password", - } - config2 := map[string]interface{}{ - "channel_name": secondChannel, - "auth.oauth2.client.id": "client.id", - "auth.oauth2.client.secret": "client.secret", - "auth.oauth2.user": "user", - "auth.oauth2.password": "password", - } - - // create Server - r := http.HandlerFunc(oauth2Handler) - server := httptest.NewServer(r) - serverURL = server.URL - config1["auth.oauth2.token_url"] = serverURL + "/token" - config2["auth.oauth2.token_url"] = serverURL + "/token" - - // get common config - cfg1 := common.MustNewConfigFrom(config1) - cfg2 := common.MustNewConfigFrom(config2) - - var inputContext finput.Context - - // initialize inputs - input1, err := NewInput(cfg1, connector, inputContext) - require.NoError(t, err) - require.NotNil(t, input1) - - input2, err := NewInput(cfg2, connector, inputContext) - require.NoError(t, err) - require.NotNil(t, input2) - - // run input - input1.Run() - defer input1.Stop() - - event1 := <-eventsCh - assertEventMatches(t, expected, event1) - - // run input - input2.Run() - defer input2.Stop() - - event2 := <-eventsCh - assertEventMatches(t, expected, event2) - - // close server - server.Close() -} - -// TestNewInput_Run_Wait to test input wait -func TestNewInput_Run_Wait(t *testing.T) { - eventsCh := make(chan beat.Event) - - outlet := &mockedOutleter{ - onEventHandler: func(event beat.Event) bool { - eventsCh <- event - return true - }, - } - connector := &mockedConnector{ - outlet: outlet, - } - var inputContext finput.Context - - var msg bay.MaybeMsg - msg.Msg.Data.Event.ReplayID = 1234 - msg.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) - msg.Msg.Channel = firstChannel - - config := map[string]interface{}{ - "channel_name": firstChannel, - "auth.oauth2.client.id": "client.id", - "auth.oauth2.client.secret": "client.secret", - "auth.oauth2.user": "user", - "auth.oauth2.password": "password", - } - - r := http.HandlerFunc(oauth2Handler) - server := httptest.NewServer(r) - serverURL = server.URL - config["auth.oauth2.token_url"] = serverURL + "/token" - - cfg := common.MustNewConfigFrom(config) - - input, err := NewInput(cfg, connector, inputContext) - require.NoError(t, err) - require.NotNil(t, input) - - // run input - input.Run() - - go func() { - time.Sleep(100 * time.Millisecond) // let input.Stop() be executed. - input.Wait() - }() - - for range []beat.Event{<-eventsCh} { - } -} - -func TestStop(t *testing.T) { - conf := defaultConfig() - logger := logp.NewLogger("test") - authParams := bay.AuthenticationParameters{} - inputCtx, cancelInputCtx := context.WithCancel(context.Background()) - workerCtx, workerCancel := context.WithCancel(inputCtx) - defer cancelInputCtx() - - input := &cometdInput{ - config: conf, - log: logger, - inputCtx: inputCtx, - workerCtx: workerCtx, - workerCancel: workerCancel, - authParams: authParams, - } - input.msgCh = make(chan bay.MaybeMsg) - - input.Stop() - select { - case <-workerCtx.Done(): - default: - require.NoError(t, fmt.Errorf("input is not stopped.")) - } -} - -func TestWait(t *testing.T) { - conf := defaultConfig() - logger := logp.NewLogger("test") - authParams := bay.AuthenticationParameters{} - inputCtx, cancelInputCtx := context.WithCancel(context.Background()) - workerCtx, workerCancel := context.WithCancel(inputCtx) - defer cancelInputCtx() - - input := &cometdInput{ - config: conf, - log: logger, - inputCtx: inputCtx, - workerCtx: workerCtx, - workerCancel: workerCancel, - authParams: authParams, - } - input.msgCh = make(chan bay.MaybeMsg) - - go func() { - time.Sleep(1000 * time.Millisecond) - input.Wait() - }() - - time.Sleep(1000 * time.Millisecond) // let input.Stop() be executed. - select { - case <-workerCtx.Done(): - default: - require.NoError(t, fmt.Errorf("input is not stopped.")) - } + Private: "DEMOBODYFAIL", + } + assert.NotEqual(t, event, makeEvent("DEMOID", "DEMOBODY")) + }) + + t.Run("Check new single input", func(t *testing.T) { + eventsCh := make(chan beat.Event) + + outlet := &mockedOutleter{ + onEventHandler: func(event beat.Event) bool { + eventsCh <- event + return true + }, + } + connector := &mockedConnector{ + outlet: outlet, + } + var inputContext finput.Context + + var expected bay.MaybeMsg + expected.Msg.Data.Event.ReplayID = 1234 + expected.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) + expected.Msg.Channel = firstChannel + + config := map[string]interface{}{ + "channel_name": firstChannel, + "auth.oauth2.client.id": "client.id", + "auth.oauth2.client.secret": "client.secret", + "auth.oauth2.user": "user", + "auth.oauth2.password": "password", + } + + r := http.HandlerFunc(oauth2Handler) + server := httptest.NewServer(r) + defer server.Close() + + serverURL = server.URL + config["auth.oauth2.token_url"] = server.URL + "/token" + + cfg := common.MustNewConfigFrom(config) + + input, err := NewInput(cfg, connector, inputContext) + require.NoError(t, err) + require.NotNil(t, input) + + input.Run() + defer input.Stop() + event := <-eventsCh + message, err := event.GetValue("message") + require.NoError(t, err) + require.Equal(t, string(expected.Msg.Data.Payload), message) + }) + + t.Run("Check new multi inputs", func(t *testing.T) { + eventsCh := make(chan beat.Event) + defer close(eventsCh) + + outlet := &mockedOutleter{ + onEventHandler: func(event beat.Event) bool { + eventsCh <- event + return true + }, + } + connector := &mockedConnector{ + outlet: outlet, + } + + var expected bay.MaybeMsg + expected.Msg.Data.Event.ReplayID = 1234 + expected.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) + expected.Msg.Channel = firstChannel + + config1 := map[string]interface{}{ + "channel_name": firstChannel, + "auth.oauth2.client.id": "client.id", + "auth.oauth2.client.secret": "client.secret", + "auth.oauth2.user": "user", + "auth.oauth2.password": "password", + } + config2 := map[string]interface{}{ + "channel_name": secondChannel, + "auth.oauth2.client.id": "client.id", + "auth.oauth2.client.secret": "client.secret", + "auth.oauth2.user": "user", + "auth.oauth2.password": "password", + } + + // create Server + r := http.HandlerFunc(oauth2Handler) + server := httptest.NewServer(r) + serverURL = server.URL + config1["auth.oauth2.token_url"] = serverURL + "/token" + config2["auth.oauth2.token_url"] = serverURL + "/token" + + // get common config + cfg1 := common.MustNewConfigFrom(config1) + cfg2 := common.MustNewConfigFrom(config2) + + var inputContext finput.Context + + // initialize inputs + input1, err := NewInput(cfg1, connector, inputContext) + require.NoError(t, err) + require.NotNil(t, input1) + + input2, err := NewInput(cfg2, connector, inputContext) + require.NoError(t, err) + require.NotNil(t, input2) + + // run input + input1.Run() + defer input1.Stop() + + event1 := <-eventsCh + assertEventMatches(t, expected, event1) + + // run input + input2.Run() + defer input2.Stop() + + event2 := <-eventsCh + assertEventMatches(t, expected, event2) + + // close server + server.Close() + }) + + t.Run("Check gracefully shutdown input", func(t *testing.T) { + eventsCh := make(chan beat.Event) + + outlet := &mockedOutleter{ + onEventHandler: func(event beat.Event) bool { + eventsCh <- event + return true + }, + } + connector := &mockedConnector{ + outlet: outlet, + } + var inputContext finput.Context + + var msg bay.MaybeMsg + msg.Msg.Data.Event.ReplayID = 1234 + msg.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) + msg.Msg.Channel = firstChannel + + config := map[string]interface{}{ + "channel_name": firstChannel, + "auth.oauth2.client.id": "client.id", + "auth.oauth2.client.secret": "client.secret", + "auth.oauth2.user": "user", + "auth.oauth2.password": "password", + } + + r := http.HandlerFunc(oauth2Handler) + server := httptest.NewServer(r) + serverURL = server.URL + config["auth.oauth2.token_url"] = serverURL + "/token" + + cfg := common.MustNewConfigFrom(config) + + input, err := NewInput(cfg, connector, inputContext) + require.NoError(t, err) + require.NotNil(t, input) + + // run input + input.Run() + + go func() { + time.Sleep(100 * time.Millisecond) // let input.Stop() be executed. + input.Wait() + }() + + for range []beat.Event{<-eventsCh} { + } + }) + + t.Run("Check input stop", func(t *testing.T) { + conf := defaultConfig() + logger := logp.NewLogger("test") + authParams := bay.AuthenticationParameters{} + inputCtx, cancelInputCtx := context.WithCancel(context.Background()) + workerCtx, workerCancel := context.WithCancel(inputCtx) + defer cancelInputCtx() + + input := &cometdInput{ + config: conf, + log: logger, + inputCtx: inputCtx, + workerCtx: workerCtx, + workerCancel: workerCancel, + authParams: authParams, + } + input.msgCh = make(chan bay.MaybeMsg) + + input.Stop() + select { + case <-workerCtx.Done(): + default: + require.NoError(t, fmt.Errorf("input is not stopped.")) + } + }) + + t.Run("Check input wait", func(t *testing.T) { + conf := defaultConfig() + logger := logp.NewLogger("test") + authParams := bay.AuthenticationParameters{} + inputCtx, cancelInputCtx := context.WithCancel(context.Background()) + workerCtx, workerCancel := context.WithCancel(inputCtx) + defer cancelInputCtx() + + input := &cometdInput{ + config: conf, + log: logger, + inputCtx: inputCtx, + workerCtx: workerCtx, + workerCancel: workerCancel, + authParams: authParams, + } + input.msgCh = make(chan bay.MaybeMsg) + + go func() { + time.Sleep(1000 * time.Millisecond) + input.Wait() + }() + + time.Sleep(1000 * time.Millisecond) // let input.Stop() be executed. + select { + case <-workerCtx.Done(): + default: + require.NoError(t, fmt.Errorf("input is not stopped.")) + } + }) } func oauth2TokenHandler(w http.ResponseWriter, r *http.Request) { From b03af9fd66ed52bc3e80892ef27772197eb9a3de Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 13 Apr 2022 19:29:48 +0530 Subject: [PATCH 32/50] Update unit test name --- x-pack/filebeat/input/cometd/input_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index 48d09af556da..58cd52f4b6f9 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -33,7 +33,7 @@ var ( serverURL string ) -func TestInput(t *testing.T) { +func TestNewInput(t *testing.T) { t.Run("Check input Done", func(t *testing.T) { config := common.MapStr{ "channel_name": firstChannel, From ea1cd768abfa188a2f7548b0255ba368eac8660d Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Thu, 14 Apr 2022 19:17:39 +0530 Subject: [PATCH 33/50] Update tests --- x-pack/filebeat/input/cometd/input_test.go | 6 +- x-pack/filebeat/input/cometd/test.out | 88 ++++++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 x-pack/filebeat/input/cometd/test.out diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index 58cd52f4b6f9..bc89af8962fe 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -225,11 +225,15 @@ func TestNewInput(t *testing.T) { input.Run() go func() { - time.Sleep(100 * time.Millisecond) // let input.Stop() be executed. + time.Sleep(time.Second) // let input.Stop() be executed. input.Wait() }() + start := time.Now() for range []beat.Event{<-eventsCh} { + if time.Since(start) > time.Second { + require.Fail(t, "Channel was not closed after Wait()") + } } }) diff --git a/x-pack/filebeat/input/cometd/test.out b/x-pack/filebeat/input/cometd/test.out new file mode 100644 index 000000000000..e9bb41a76404 --- /dev/null +++ b/x-pack/filebeat/input/cometd/test.out @@ -0,0 +1,88 @@ +channel closed +panic: test timed out after 30s + +goroutine 41 [running]: +testing.(*M).startAlarm.func1() + /usr/local/go/src/testing/testing.go:1788 +0x8e +created by time.goFunc + /usr/local/go/src/time/sleep.go:180 +0x31 + +goroutine 1 [chan receive]: +testing.(*T).Run(0xc0000ae9c0, {0x115ca75, 0x806033}, 0x11945f0) + /usr/local/go/src/testing/testing.go:1307 +0x375 +testing.runTests.func1(0xc0000ae9c0) + /usr/local/go/src/testing/testing.go:1598 +0x6e +testing.tRunner(0xc0000ae9c0, 0xc00063fd18) + /usr/local/go/src/testing/testing.go:1259 +0x102 +testing.runTests(0xc0005dd580, {0x1a5db80, 0x7, 0x7}, {0x82088d, 0x115e89c, 0x1c11b00}) + /usr/local/go/src/testing/testing.go:1596 +0x43f +testing.(*M).Run(0xc0005dd580) + /usr/local/go/src/testing/testing.go:1504 +0x51d +main.main() + _testmain.go:55 +0x14b + +goroutine 50 [chan receive]: +testing.(*T).Run(0xc0000aeb60, {0x1164e82, 0xfc11d58024ad}, 0x11945c8) + /usr/local/go/src/testing/testing.go:1307 +0x375 +github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.TestNewInput(0x0) + /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input_test.go:64 +0x6f +testing.tRunner(0xc0000aeb60, 0x11945f0) + /usr/local/go/src/testing/testing.go:1259 +0x102 +created by testing.(*T).Run + /usr/local/go/src/testing/testing.go:1306 +0x35a + +goroutine 60 [semacquire]: +sync.runtime_Semacquire(0x126d701) + /usr/local/go/src/runtime/sema.go:56 +0x25 +sync.(*WaitGroup).Wait(0xc0005ee080) + /usr/local/go/src/sync/waitgroup.go:130 +0x71 +github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.(*cometdInput).Stop(0xc00027a2c0) + /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input.go:152 +0x30 +github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.TestNewInput.func3(0xd40010) + /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input_test.go:110 +0x6cc +testing.tRunner(0xc0000af520, 0x11945c8) + /usr/local/go/src/testing/testing.go:1259 +0x102 +created by testing.(*T).Run + /usr/local/go/src/testing/testing.go:1306 +0x35a + +goroutine 61 [IO wait]: +internal/poll.runtime_pollWait(0x7fa9d9969928, 0x72) + /usr/local/go/src/runtime/netpoll.go:229 +0x89 +internal/poll.(*pollDesc).wait(0xc0005dd680, 0x203000, 0x0) + /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 +internal/poll.(*pollDesc).waitRead(...) + /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0005dd680) + /usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c +net.(*netFD).accept(0xc0005dd680) + /usr/local/go/src/net/fd_unix.go:173 +0x35 +net.(*TCPListener).accept(0xc00000e5a0) + /usr/local/go/src/net/tcpsock_posix.go:140 +0x28 +net.(*TCPListener).Accept(0xc00000e5a0) + /usr/local/go/src/net/tcpsock.go:262 +0x3d +net/http.(*Server).Serve(0xc0002ae000, {0x127cb68, 0xc00000e5a0}) + /usr/local/go/src/net/http/server.go:3001 +0x394 +net/http/httptest.(*Server).goServe.func1() + /usr/local/go/src/net/http/httptest/server.go:308 +0x6a +created by net/http/httptest.(*Server).goServe + /usr/local/go/src/net/http/httptest/server.go:306 +0x6f + +goroutine 68 [chan receive (nil chan)]: +github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.NewInput.func1() + /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input.go:119 +0x28 +created by github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.NewInput + /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input.go:118 +0x2ff + +goroutine 69 [chan send]: +github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.TestNewInput.func3.1({{0x3581a67, 0xed9ea1c45, 0x0}, 0xc000124c90, 0xc000124bd0, {0x100ee00, 0xc0001e2070}, 0x0}) + /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input_test.go:69 +0x88 +github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.mockedOutleter.OnEvent(...) + /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/client_mocked.go:46 +github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.(*cometdInput).run(0xc00027a2c0) + /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input.go:76 +0x3af +github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.(*cometdInput).Run.func1.1() + /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input.go:44 +0x2fa +created by github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.(*cometdInput).Run.func1 + /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input.go:33 +0x93 +FAIL github.com/elastic/beats/v7/x-pack/filebeat/input/cometd 30.043s +FAIL From 73c028f80211f148f8aaa2be68c1c274119e6712 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Thu, 14 Apr 2022 19:18:18 +0530 Subject: [PATCH 34/50] Remove test.out --- x-pack/filebeat/input/cometd/test.out | 88 --------------------------- 1 file changed, 88 deletions(-) delete mode 100644 x-pack/filebeat/input/cometd/test.out diff --git a/x-pack/filebeat/input/cometd/test.out b/x-pack/filebeat/input/cometd/test.out deleted file mode 100644 index e9bb41a76404..000000000000 --- a/x-pack/filebeat/input/cometd/test.out +++ /dev/null @@ -1,88 +0,0 @@ -channel closed -panic: test timed out after 30s - -goroutine 41 [running]: -testing.(*M).startAlarm.func1() - /usr/local/go/src/testing/testing.go:1788 +0x8e -created by time.goFunc - /usr/local/go/src/time/sleep.go:180 +0x31 - -goroutine 1 [chan receive]: -testing.(*T).Run(0xc0000ae9c0, {0x115ca75, 0x806033}, 0x11945f0) - /usr/local/go/src/testing/testing.go:1307 +0x375 -testing.runTests.func1(0xc0000ae9c0) - /usr/local/go/src/testing/testing.go:1598 +0x6e -testing.tRunner(0xc0000ae9c0, 0xc00063fd18) - /usr/local/go/src/testing/testing.go:1259 +0x102 -testing.runTests(0xc0005dd580, {0x1a5db80, 0x7, 0x7}, {0x82088d, 0x115e89c, 0x1c11b00}) - /usr/local/go/src/testing/testing.go:1596 +0x43f -testing.(*M).Run(0xc0005dd580) - /usr/local/go/src/testing/testing.go:1504 +0x51d -main.main() - _testmain.go:55 +0x14b - -goroutine 50 [chan receive]: -testing.(*T).Run(0xc0000aeb60, {0x1164e82, 0xfc11d58024ad}, 0x11945c8) - /usr/local/go/src/testing/testing.go:1307 +0x375 -github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.TestNewInput(0x0) - /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input_test.go:64 +0x6f -testing.tRunner(0xc0000aeb60, 0x11945f0) - /usr/local/go/src/testing/testing.go:1259 +0x102 -created by testing.(*T).Run - /usr/local/go/src/testing/testing.go:1306 +0x35a - -goroutine 60 [semacquire]: -sync.runtime_Semacquire(0x126d701) - /usr/local/go/src/runtime/sema.go:56 +0x25 -sync.(*WaitGroup).Wait(0xc0005ee080) - /usr/local/go/src/sync/waitgroup.go:130 +0x71 -github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.(*cometdInput).Stop(0xc00027a2c0) - /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input.go:152 +0x30 -github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.TestNewInput.func3(0xd40010) - /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input_test.go:110 +0x6cc -testing.tRunner(0xc0000af520, 0x11945c8) - /usr/local/go/src/testing/testing.go:1259 +0x102 -created by testing.(*T).Run - /usr/local/go/src/testing/testing.go:1306 +0x35a - -goroutine 61 [IO wait]: -internal/poll.runtime_pollWait(0x7fa9d9969928, 0x72) - /usr/local/go/src/runtime/netpoll.go:229 +0x89 -internal/poll.(*pollDesc).wait(0xc0005dd680, 0x203000, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Accept(0xc0005dd680) - /usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c -net.(*netFD).accept(0xc0005dd680) - /usr/local/go/src/net/fd_unix.go:173 +0x35 -net.(*TCPListener).accept(0xc00000e5a0) - /usr/local/go/src/net/tcpsock_posix.go:140 +0x28 -net.(*TCPListener).Accept(0xc00000e5a0) - /usr/local/go/src/net/tcpsock.go:262 +0x3d -net/http.(*Server).Serve(0xc0002ae000, {0x127cb68, 0xc00000e5a0}) - /usr/local/go/src/net/http/server.go:3001 +0x394 -net/http/httptest.(*Server).goServe.func1() - /usr/local/go/src/net/http/httptest/server.go:308 +0x6a -created by net/http/httptest.(*Server).goServe - /usr/local/go/src/net/http/httptest/server.go:306 +0x6f - -goroutine 68 [chan receive (nil chan)]: -github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.NewInput.func1() - /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input.go:119 +0x28 -created by github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.NewInput - /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input.go:118 +0x2ff - -goroutine 69 [chan send]: -github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.TestNewInput.func3.1({{0x3581a67, 0xed9ea1c45, 0x0}, 0xc000124c90, 0xc000124bd0, {0x100ee00, 0xc0001e2070}, 0x0}) - /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input_test.go:69 +0x88 -github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.mockedOutleter.OnEvent(...) - /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/client_mocked.go:46 -github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.(*cometdInput).run(0xc00027a2c0) - /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input.go:76 +0x3af -github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.(*cometdInput).Run.func1.1() - /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input.go:44 +0x2fa -created by github.com/elastic/beats/v7/x-pack/filebeat/input/cometd.(*cometdInput).Run.func1 - /usr/local/go/src/github.com/elastic/kush-elastic/beats/x-pack/filebeat/input/cometd/input.go:33 +0x93 -FAIL github.com/elastic/beats/v7/x-pack/filebeat/input/cometd 30.043s -FAIL From 0f337e1262c3b5df1f8d9469d48fb56d9bb538cb Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Thu, 14 Apr 2022 14:47:04 -0400 Subject: [PATCH 35/50] Use time.After to eliminate possible flaky test --- x-pack/filebeat/input/cometd/input_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index bc89af8962fe..219054b62164 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -282,14 +282,12 @@ func TestNewInput(t *testing.T) { input.msgCh = make(chan bay.MaybeMsg) go func() { - time.Sleep(1000 * time.Millisecond) input.Wait() }() - time.Sleep(1000 * time.Millisecond) // let input.Stop() be executed. select { case <-workerCtx.Done(): - default: + case <-time.After(time.Second): // let input.Stop() be executed. require.NoError(t, fmt.Errorf("input is not stopped.")) } }) From 14f07a2643678d1c07c1594d638d6c9c384038fd Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Tue, 19 Apr 2022 12:52:33 +0530 Subject: [PATCH 36/50] Add buffer of 1 in message channel --- x-pack/filebeat/input/cometd/input.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index a8aaccb139f8..35bf1a15977d 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -38,7 +38,7 @@ func (in *cometdInput) Run() { in.b = bay.Bayeux{} in.creds, err = bay.GetSalesforceCredentials(in.authParams) if err != nil { - in.log.Error("not able to get access token", err) + in.log.Error("not able to get access token: ", err) return } if err := in.run(); err != nil { @@ -134,7 +134,7 @@ func NewInput( } // Creating a new channel for cometd input. - in.msgCh = make(chan bay.MaybeMsg) + in.msgCh = make(chan bay.MaybeMsg, 1) // Build outlet for events. in.outlet, err = connector.Connect(cfg) From 4d7e486e4fec942ecdc485a38b9572989e2444f8 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Tue, 19 Apr 2022 19:39:24 +0530 Subject: [PATCH 37/50] Update tests based on bayeux client status watcher changes --- x-pack/filebeat/input/cometd/input.go | 18 +- x-pack/filebeat/input/cometd/input_test.go | 576 ++++++++++++--------- 2 files changed, 353 insertions(+), 241 deletions(-) diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index 35bf1a15977d..2d3ebb6178f8 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -35,7 +35,6 @@ func (in *cometdInput) Run() { defer in.log.Info("Input worker has stopped.") defer in.workerWg.Done() defer in.workerCancel() - in.b = bay.Bayeux{} in.creds, err = bay.GetSalesforceCredentials(in.authParams) if err != nil { in.log.Error("not able to get access token: ", err) @@ -89,11 +88,10 @@ func init() { } } -// NewInput creates a new CometD input that consumes events from -// a topic subscription. -func NewInput( +func newInput( cfg *common.Config, connector channel.Connector, + bayeuxClient func() bay.Client, inputContext input.Context, ) (inp input.Input, err error) { // Extract and validate the input's configuration. @@ -133,6 +131,9 @@ func NewInput( authParams: authParams, } + client := bayeuxClient() + in.b = client.BayOb + // Creating a new channel for cometd input. in.msgCh = make(chan bay.MaybeMsg, 1) @@ -143,7 +144,16 @@ func NewInput( } in.log.Infof("Initialized %s input.", inputName) return in, nil +} +// NewInput creates a new CometD input that consumes events from +// a topic subscription. +func NewInput( + cfg *common.Config, + connector channel.Connector, + inputContext input.Context, +) (inp input.Input, err error) { + return newInput(cfg, connector, bay.NewClient, inputContext) } // Stop stops the input and waits for it to fully stop. diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index 219054b62164..d85de8e81949 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -10,6 +10,8 @@ import ( "io/ioutil" "net/http" "net/http/httptest" + "sync" + "sync/atomic" "testing" "time" @@ -31,266 +33,315 @@ const ( var ( serverURL string + called1 uint64 + called2 uint64 + clientId uint64 ) -func TestNewInput(t *testing.T) { - t.Run("Check input Done", func(t *testing.T) { - config := common.MapStr{ - "channel_name": firstChannel, - "auth.oauth2.client.id": "DEMOCLIENTID", - "auth.oauth2.client.secret": "DEMOCLIENTSECRET", - "auth.oauth2.user": "salesforce_user", - "auth.oauth2.password": "pwd", - "auth.oauth2.token_url": "https://example.com/token", - } - inputtest.AssertNotStartedInputCanBeDone(t, NewInput, &config) - }) - - t.Run("Check make event failure", func(t *testing.T) { - event := beat.Event{ - Timestamp: time.Now().UTC(), - Fields: common.MapStr{ - "event": common.MapStr{ - "id": "DEMOID", - "created": time.Now().UTC(), - }, - "message": "DEMOBODYFAIL", +func TestInputDone(t *testing.T) { + config := common.MapStr{ + "channel_name": firstChannel, + "auth.oauth2.client.id": "DEMOCLIENTID", + "auth.oauth2.client.secret": "DEMOCLIENTSECRET", + "auth.oauth2.user": "salesforce_user", + "auth.oauth2.password": "pwd", + "auth.oauth2.token_url": "https://example.com/token", + } + inputtest.AssertNotStartedInputCanBeDone(t, NewInput, &config) +} + +func TestMakeEventFailure(t *testing.T) { + event := beat.Event{ + Timestamp: time.Now().UTC(), + Fields: common.MapStr{ + "event": common.MapStr{ + "id": "DEMOID", + "created": time.Now().UTC(), }, - Private: "DEMOBODYFAIL", - } - assert.NotEqual(t, event, makeEvent("DEMOID", "DEMOBODY")) - }) + "message": "DEMOBODYFAIL", + }, + Private: "DEMOBODYFAIL", + } + assert.NotEqual(t, event, makeEvent("DEMOID", "DEMOBODY")) +} - t.Run("Check new single input", func(t *testing.T) { - eventsCh := make(chan beat.Event) +func TestSingleInput(t *testing.T) { + eventsCh := make(chan beat.Event) - outlet := &mockedOutleter{ - onEventHandler: func(event beat.Event) bool { - eventsCh <- event - return true - }, - } - connector := &mockedConnector{ - outlet: outlet, - } - var inputContext finput.Context - - var expected bay.MaybeMsg - expected.Msg.Data.Event.ReplayID = 1234 - expected.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) - expected.Msg.Channel = firstChannel - - config := map[string]interface{}{ - "channel_name": firstChannel, - "auth.oauth2.client.id": "client.id", - "auth.oauth2.client.secret": "client.secret", - "auth.oauth2.user": "user", - "auth.oauth2.password": "password", - } + outlet := &mockedOutleter{ + onEventHandler: func(event beat.Event) bool { + eventsCh <- event + return true + }, + } + connector := &mockedConnector{ + outlet: outlet, + } + var inputContext finput.Context + + var expected bay.MaybeMsg + expected.Msg.Data.Event.ReplayID = 1234 + expected.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) + expected.Msg.Channel = firstChannel + + config := map[string]interface{}{ + "channel_name": firstChannel, + "auth.oauth2.client.id": "client.id", + "auth.oauth2.client.secret": "client.secret", + "auth.oauth2.user": "user", + "auth.oauth2.password": "password", + } - r := http.HandlerFunc(oauth2Handler) - server := httptest.NewServer(r) - defer server.Close() + r := http.HandlerFunc(oauth2Handler) + server := httptest.NewServer(r) + defer server.Close() - serverURL = server.URL - config["auth.oauth2.token_url"] = server.URL + "/token" + serverURL = server.URL + config["auth.oauth2.token_url"] = server.URL + "/token" - cfg := common.MustNewConfigFrom(config) + cfg := common.MustNewConfigFrom(config) - input, err := NewInput(cfg, connector, inputContext) - require.NoError(t, err) - require.NotNil(t, input) + input, err := NewInput(cfg, connector, inputContext) + require.NoError(t, err) + require.NotNil(t, input) - input.Run() - defer input.Stop() - event := <-eventsCh - message, err := event.GetValue("message") - require.NoError(t, err) - require.Equal(t, string(expected.Msg.Data.Payload), message) - }) - - t.Run("Check new multi inputs", func(t *testing.T) { - eventsCh := make(chan beat.Event) - defer close(eventsCh) - - outlet := &mockedOutleter{ - onEventHandler: func(event beat.Event) bool { - eventsCh <- event - return true - }, - } - connector := &mockedConnector{ - outlet: outlet, - } + input.Run() - var expected bay.MaybeMsg - expected.Msg.Data.Event.ReplayID = 1234 - expected.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) - expected.Msg.Channel = firstChannel - - config1 := map[string]interface{}{ - "channel_name": firstChannel, - "auth.oauth2.client.id": "client.id", - "auth.oauth2.client.secret": "client.secret", - "auth.oauth2.user": "user", - "auth.oauth2.password": "password", - } - config2 := map[string]interface{}{ - "channel_name": secondChannel, - "auth.oauth2.client.id": "client.id", - "auth.oauth2.client.secret": "client.secret", - "auth.oauth2.user": "user", - "auth.oauth2.password": "password", - } + event := <-eventsCh + assertEventMatches(t, expected, event) + input.Stop() + atomic.StoreUint64(&called1, 0) + atomic.StoreUint64(&clientId, 0) +} - // create Server - r := http.HandlerFunc(oauth2Handler) - server := httptest.NewServer(r) - serverURL = server.URL - config1["auth.oauth2.token_url"] = serverURL + "/token" - config2["auth.oauth2.token_url"] = serverURL + "/token" +func TestInputStop_Wait(t *testing.T) { + eventsCh := make(chan beat.Event) - // get common config - cfg1 := common.MustNewConfigFrom(config1) - cfg2 := common.MustNewConfigFrom(config2) + const numMessages = 1 - var inputContext finput.Context + var eventProcessing sync.WaitGroup + eventProcessing.Add(numMessages) - // initialize inputs - input1, err := NewInput(cfg1, connector, inputContext) - require.NoError(t, err) - require.NotNil(t, input1) + outlet := &mockedOutleter{ + onEventHandler: func(event beat.Event) bool { + eventProcessing.Done() + eventsCh <- event + return true + }, + } + connector := &mockedConnector{ + outlet: outlet, + } + var inputContext finput.Context + + var expected bay.MaybeMsg + expected.Msg.Data.Event.ReplayID = 1234 + expected.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) + expected.Msg.Channel = firstChannel + + config := map[string]interface{}{ + "channel_name": firstChannel, + "auth.oauth2.client.id": "client.id", + "auth.oauth2.client.secret": "client.secret", + "auth.oauth2.user": "user", + "auth.oauth2.password": "password", + } - input2, err := NewInput(cfg2, connector, inputContext) - require.NoError(t, err) - require.NotNil(t, input2) + r := http.HandlerFunc(oauth2Handler) + server := httptest.NewServer(r) + defer server.Close() - // run input - input1.Run() - defer input1.Stop() + serverURL = server.URL + config["auth.oauth2.token_url"] = server.URL + "/token" - event1 := <-eventsCh - assertEventMatches(t, expected, event1) + cfg := common.MustNewConfigFrom(config) - // run input - input2.Run() - defer input2.Stop() + bayClient := bay.Client{} - event2 := <-eventsCh - assertEventMatches(t, expected, event2) + newBayeuxMockClient := func() bay.Client { + b := bay.Bayeux{} + bayClient = bay.Client{ + BayOb: b, + } + return bayClient + } - // close server - server.Close() - }) + input, err := newInput(cfg, connector, newBayeuxMockClient, inputContext) + require.NoError(t, err) + require.NotNil(t, input) - t.Run("Check gracefully shutdown input", func(t *testing.T) { - eventsCh := make(chan beat.Event) + require.Equal(t, 0, bayClient.BayOb.GetConnectedCount()) + input.Run() + eventProcessing.Wait() + require.Equal(t, 1, bayClient.BayOb.GetConnectedCount()) - outlet := &mockedOutleter{ - onEventHandler: func(event beat.Event) bool { - eventsCh <- event - return true - }, - } - connector := &mockedConnector{ - outlet: outlet, - } - var inputContext finput.Context - - var msg bay.MaybeMsg - msg.Msg.Data.Event.ReplayID = 1234 - msg.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) - msg.Msg.Channel = firstChannel - - config := map[string]interface{}{ - "channel_name": firstChannel, - "auth.oauth2.client.id": "client.id", - "auth.oauth2.client.secret": "client.secret", - "auth.oauth2.user": "user", - "auth.oauth2.password": "password", + go func() { + time.Sleep(100 * time.Millisecond) // let input.Stop() be executed. + for range eventsCh { } + }() - r := http.HandlerFunc(oauth2Handler) - server := httptest.NewServer(r) - serverURL = server.URL - config["auth.oauth2.token_url"] = serverURL + "/token" + input.Wait() + require.Equal(t, 0, bayClient.BayOb.GetConnectedCount()) + atomic.StoreUint64(&called1, 0) + atomic.StoreUint64(&clientId, 0) +} - cfg := common.MustNewConfigFrom(config) +func TestMultiInput(t *testing.T) { + eventsCh := make(chan beat.Event) - input, err := NewInput(cfg, connector, inputContext) - require.NoError(t, err) - require.NotNil(t, input) + const numMessages = 2 - // run input - input.Run() + var eventProcessing sync.WaitGroup + eventProcessing.Add(numMessages) - go func() { - time.Sleep(time.Second) // let input.Stop() be executed. - input.Wait() - }() + outlet := &mockedOutleter{ + onEventHandler: func(event beat.Event) bool { + eventProcessing.Done() + eventsCh <- event + return true + }, + } + connector := &mockedConnector{ + outlet: outlet, + } - start := time.Now() - for range []beat.Event{<-eventsCh} { - if time.Since(start) > time.Second { - require.Fail(t, "Channel was not closed after Wait()") - } - } - }) - - t.Run("Check input stop", func(t *testing.T) { - conf := defaultConfig() - logger := logp.NewLogger("test") - authParams := bay.AuthenticationParameters{} - inputCtx, cancelInputCtx := context.WithCancel(context.Background()) - workerCtx, workerCancel := context.WithCancel(inputCtx) - defer cancelInputCtx() - - input := &cometdInput{ - config: conf, - log: logger, - inputCtx: inputCtx, - workerCtx: workerCtx, - workerCancel: workerCancel, - authParams: authParams, - } - input.msgCh = make(chan bay.MaybeMsg) + var expected1 bay.MaybeMsg + expected1.Msg.Data.Event.ReplayID = 1234 + expected1.Msg.Data.Payload = []byte(`{"CountryIso": "IN"}`) + expected1.Msg.Channel = firstChannel + + var expected2 bay.MaybeMsg + expected2.Msg.Data.Event.ReplayID = 1234 + expected2.Msg.Data.Payload = []byte(`{"CountryIso": "US"}`) + expected2.Msg.Channel = firstChannel + + config1 := map[string]interface{}{ + "channel_name": firstChannel, + "auth.oauth2.client.id": "client.id", + "auth.oauth2.client.secret": "client.secret", + "auth.oauth2.user": "user", + "auth.oauth2.password": "password", + } + config2 := map[string]interface{}{ + "channel_name": secondChannel, + "auth.oauth2.client.id": "client.id", + "auth.oauth2.client.secret": "client.secret", + "auth.oauth2.user": "user", + "auth.oauth2.password": "password", + } - input.Stop() - select { - case <-workerCtx.Done(): - default: - require.NoError(t, fmt.Errorf("input is not stopped.")) - } - }) - - t.Run("Check input wait", func(t *testing.T) { - conf := defaultConfig() - logger := logp.NewLogger("test") - authParams := bay.AuthenticationParameters{} - inputCtx, cancelInputCtx := context.WithCancel(context.Background()) - workerCtx, workerCancel := context.WithCancel(inputCtx) - defer cancelInputCtx() - - input := &cometdInput{ - config: conf, - log: logger, - inputCtx: inputCtx, - workerCtx: workerCtx, - workerCancel: workerCancel, - authParams: authParams, - } - input.msgCh = make(chan bay.MaybeMsg) + // create Server + r := http.HandlerFunc(oauth2Handler) + server := httptest.NewServer(r) + serverURL = server.URL + config1["auth.oauth2.token_url"] = serverURL + "/token" + config2["auth.oauth2.token_url"] = serverURL + "/token" - go func() { - input.Wait() - }() + // get common config + cfg1 := common.MustNewConfigFrom(config1) + cfg2 := common.MustNewConfigFrom(config2) - select { - case <-workerCtx.Done(): - case <-time.After(time.Second): // let input.Stop() be executed. - require.NoError(t, fmt.Errorf("input is not stopped.")) + var inputContext finput.Context + + bayClient := bay.Client{} + + newBayeuxMockClient := func() bay.Client { + b := bay.Bayeux{} + bayClient = bay.Client{ + BayOb: b, } - }) + return bayClient + } + + // initialize inputs + input1, err := newInput(cfg1, connector, newBayeuxMockClient, inputContext) + require.NoError(t, err) + require.NotNil(t, input1) + + input2, err := newInput(cfg2, connector, newBayeuxMockClient, inputContext) + require.NoError(t, err) + require.NotNil(t, input2) + + require.Equal(t, 0, bayClient.BayOb.GetConnectedCount()) + // run input + input1.Run() + + go func() { + time.Sleep(200 * time.Millisecond) // let input.Stop() be executed. + event := <-eventsCh + assertEventMatches(t, expected1, event) + }() + + // run input + input2.Run() + eventProcessing.Wait() + require.Equal(t, 2, bayClient.BayOb.GetConnectedCount()) + + go func() { + time.Sleep(100 * time.Millisecond) // let input.Stop() be executed. + event := <-eventsCh + assertEventMatches(t, expected2, event) + }() + + input1.Wait() + input2.Wait() + + require.Equal(t, 0, bayClient.BayOb.GetConnectedCount()) +} + +func TestStop(t *testing.T) { + conf := defaultConfig() + logger := logp.NewLogger("test") + authParams := bay.AuthenticationParameters{} + inputCtx, cancelInputCtx := context.WithCancel(context.Background()) + workerCtx, workerCancel := context.WithCancel(inputCtx) + defer cancelInputCtx() + input := &cometdInput{ + config: conf, + log: logger, + inputCtx: inputCtx, + workerCtx: workerCtx, + workerCancel: workerCancel, + authParams: authParams, + } + input.msgCh = make(chan bay.MaybeMsg) + + input.Stop() + select { + case <-workerCtx.Done(): + default: + require.NoError(t, fmt.Errorf("input is not stopped.")) + } +} + +func TestWait(t *testing.T) { + conf := defaultConfig() + logger := logp.NewLogger("test") + authParams := bay.AuthenticationParameters{} + inputCtx, cancelInputCtx := context.WithCancel(context.Background()) + workerCtx, workerCancel := context.WithCancel(inputCtx) + defer cancelInputCtx() + + input := &cometdInput{ + config: conf, + log: logger, + inputCtx: inputCtx, + workerCtx: workerCtx, + workerCancel: workerCancel, + authParams: authParams, + } + input.msgCh = make(chan bay.MaybeMsg) + + go func() { + input.Wait() + }() + + select { + case <-workerCtx.Done(): + case <-time.After(time.Second): // let input.Stop() be executed. + require.NoError(t, fmt.Errorf("input is not stopped.")) + } } func oauth2TokenHandler(w http.ResponseWriter, r *http.Request) { @@ -314,11 +365,19 @@ func oauth2ClientIdHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(`{"error":"wrong method"}`)) default: - _, _ = w.Write([]byte(`[{"ext":{"replay":true,"payload.format":true},"minimumVersion":"1.0","clientId":"94b112sp7ph1c9s41mycpzik4rkj3","supportedConnectionTypes":["long-polling"],"channel":"/meta/handshake","version":"1.0","successful":true}]`)) + switch clientId { + case 0: + atomic.StoreUint64(&clientId, 1) + _, _ = w.Write([]byte(`[{"ext":{"replay":true,"payload.format":true},"minimumVersion":"1.0","clientId":"client_id1","supportedConnectionTypes":["long-polling"],"channel":"/meta/handshake","version":"1.0","successful":true}]`)) + case 1: + atomic.StoreUint64(&clientId, 0) + _, _ = w.Write([]byte(`[{"ext":{"replay":true,"payload.format":true},"minimumVersion":"1.0","clientId":"client_id2","supportedConnectionTypes":["long-polling"],"channel":"/meta/handshake","version":"1.0","successful":true}]`)) + default: + } } } -func oauth2SubscribeHandler(w http.ResponseWriter, r *http.Request) { +func oauth2SubscribeHandlerChannel1(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "application/json") _ = r.ParseForm() switch { @@ -326,11 +385,11 @@ func oauth2SubscribeHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(`{"error":"wrong method"}`)) default: - _, _ = w.Write([]byte(`[{"clientId": "94b112sp7ph1c9s41mycpzik4rkj3", "channel": "/meta/subscribe", "subscription": "/event/LoginEventStream", "successful":true}]`)) + _, _ = w.Write([]byte(`[{"clientId": "client_id1", "channel": "/meta/subscribe", "subscription": "channel_name1", "successful":true}]`)) } } -func oauth2EventHandler(w http.ResponseWriter, r *http.Request) { +func oauth2SubscribeHandlerChannel2(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "application/json") _ = r.ParseForm() switch { @@ -338,7 +397,37 @@ func oauth2EventHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(`{"error":"wrong method"}`)) default: - _, _ = w.Write([]byte(`[{"data": {"payload": {"CountryIso": "IN"}, "event": {"replayId":1234}}, "channel": "/event/LoginEventStream"}]`)) + _, _ = w.Write([]byte(`[{"clientId": "client_id2", "channel": "/meta/subscribe", "subscription": "channel_name2", "successful":true}]`)) + } +} + +func oauth2EventHandlerChannel1(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + _ = r.ParseForm() + switch { + case r.Method != http.MethodPost: + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong method"}`)) + default: + if called1 < 1 { + atomic.AddUint64(&called1, 1) + _, _ = w.Write([]byte(`[{"data": {"payload": {"CountryIso": "IN"}, "event": {"replayId":1234}}, "channel": "channel_name1"}]`)) + } + } +} + +func oauth2EventHandlerChannel2(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + _ = r.ParseForm() + switch { + case r.Method != http.MethodPost: + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong method"}`)) + default: + if called2 < 1 { + atomic.AddUint64(&called2, 1) + _, _ = w.Write([]byte(`[{"data": {"payload": {"CountryIso": "US"}, "event": {"replayId":1234}}, "channel": "channel_name2"}]`)) + } } } @@ -351,18 +440,31 @@ func oauth2Handler(w http.ResponseWriter, r *http.Request) { if string(body) == `{"channel": "/meta/handshake", "supportedConnectionTypes": ["long-polling"], "version": "1.0"}` { oauth2ClientIdHandler(w, r) return - } else if string(body) == `{"channel": "/meta/connect", "connectionType": "long-polling", "clientId": "94b112sp7ph1c9s41mycpzik4rkj3"} ` { - oauth2EventHandler(w, r) + } else if string(body) == `{"channel": "/meta/connect", "connectionType": "long-polling", "clientId": "client_id1"} ` { + oauth2EventHandlerChannel1(w, r) + return + } else if string(body) == `{"channel": "/meta/connect", "connectionType": "long-polling", "clientId": "client_id2"} ` { + oauth2EventHandlerChannel2(w, r) + return + } else if string(body) == `{ + "channel": "/meta/subscribe", + "subscription": "channel_name1", + "clientId": "client_id1", + "ext": { + "replay": {"channel_name1": "-1"} + } + }` { + oauth2SubscribeHandlerChannel1(w, r) return } else if string(body) == `{ "channel": "/meta/subscribe", - "subscription": "first-channel", - "clientId": "94b112sp7ph1c9s41mycpzik4rkj3", + "subscription": "channel_name2", + "clientId": "client_id2", "ext": { - "replay": {"first-channel": "-1"} + "replay": {"channel_name2": "-1"} } }` { - oauth2SubscribeHandler(w, r) + oauth2SubscribeHandlerChannel2(w, r) return } } From 292bd7ee2aaa6f44a512c722f07369a1f2dca3bf Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Thu, 21 Apr 2022 17:53:22 +0530 Subject: [PATCH 38/50] incorporate bayeux changes --- NOTICE.txt | 4 +- go.mod | 2 +- go.sum | 4 +- x-pack/filebeat/input/cometd/input.go | 19 ++----- x-pack/filebeat/input/cometd/input_test.go | 58 ++++++++-------------- 5 files changed, 31 insertions(+), 56 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index afc9220b3e3d..0445ff2fc569 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -5807,11 +5807,11 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -------------------------------------------------------------------------------- Dependency : github.com/elastic/bayeux -Version: v1.0.4 +Version: v1.0.5 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/bayeux@v1.0.4/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/bayeux@v1.0.5/LICENSE: MIT License diff --git a/go.mod b/go.mod index b467c2c26f33..d95c3e99406f 100644 --- a/go.mod +++ b/go.mod @@ -186,7 +186,7 @@ require ( ) require ( - github.com/elastic/bayeux v1.0.4 + github.com/elastic/bayeux v1.0.5 github.com/elastic/elastic-agent-libs v0.1.1 github.com/shirou/gopsutil/v3 v3.21.12 go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0 diff --git a/go.sum b/go.sum index db0cca70df2b..d2ce77fed342 100644 --- a/go.sum +++ b/go.sum @@ -561,8 +561,8 @@ github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7j github.com/eclipse/paho.mqtt.golang v1.3.5 h1:sWtmgNxYM9P2sP+xEItMozsR3w0cqZFlqnNN1bdl41Y= github.com/eclipse/paho.mqtt.golang v1.3.5/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elastic/bayeux v1.0.4 h1:tIPINegYnFdLuyEKdd31rYbQsrMH3ItwZak88lAxvV0= -github.com/elastic/bayeux v1.0.4/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= +github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= +github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo= diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index 2d3ebb6178f8..1072f1bf33c5 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -35,6 +35,7 @@ func (in *cometdInput) Run() { defer in.log.Info("Input worker has stopped.") defer in.workerWg.Done() defer in.workerCancel() + in.b = bay.Bayeux{} in.creds, err = bay.GetSalesforceCredentials(in.authParams) if err != nil { in.log.Error("not able to get access token: ", err) @@ -88,10 +89,11 @@ func init() { } } -func newInput( +// NewInput creates a new CometD input that consumes events from +// a topic subscription. +func NewInput( cfg *common.Config, connector channel.Connector, - bayeuxClient func() bay.Client, inputContext input.Context, ) (inp input.Input, err error) { // Extract and validate the input's configuration. @@ -131,9 +133,6 @@ func newInput( authParams: authParams, } - client := bayeuxClient() - in.b = client.BayOb - // Creating a new channel for cometd input. in.msgCh = make(chan bay.MaybeMsg, 1) @@ -146,16 +145,6 @@ func newInput( return in, nil } -// NewInput creates a new CometD input that consumes events from -// a topic subscription. -func NewInput( - cfg *common.Config, - connector channel.Connector, - inputContext input.Context, -) (inp input.Input, err error) { - return newInput(cfg, connector, bay.NewClient, inputContext) -} - // Stop stops the input and waits for it to fully stop. func (in *cometdInput) Stop() { in.workerCancel() diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index d85de8e81949..d2fe3e0f766f 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -111,6 +111,7 @@ func TestSingleInput(t *testing.T) { assertEventMatches(t, expected, event) input.Stop() atomic.StoreUint64(&called1, 0) + atomic.StoreUint64(&called2, 0) atomic.StoreUint64(&clientId, 0) } @@ -156,24 +157,14 @@ func TestInputStop_Wait(t *testing.T) { cfg := common.MustNewConfigFrom(config) - bayClient := bay.Client{} - - newBayeuxMockClient := func() bay.Client { - b := bay.Bayeux{} - bayClient = bay.Client{ - BayOb: b, - } - return bayClient - } - - input, err := newInput(cfg, connector, newBayeuxMockClient, inputContext) + input, err := NewInput(cfg, connector, inputContext) require.NoError(t, err) require.NotNil(t, input) - require.Equal(t, 0, bayClient.BayOb.GetConnectedCount()) + require.Equal(t, 0, bay.GetConnectedCount()) input.Run() eventProcessing.Wait() - require.Equal(t, 1, bayClient.BayOb.GetConnectedCount()) + require.Equal(t, 1, bay.GetConnectedCount()) go func() { time.Sleep(100 * time.Millisecond) // let input.Stop() be executed. @@ -182,8 +173,9 @@ func TestInputStop_Wait(t *testing.T) { }() input.Wait() - require.Equal(t, 0, bayClient.BayOb.GetConnectedCount()) + require.Equal(t, 0, bay.GetConnectedCount()) atomic.StoreUint64(&called1, 0) + atomic.StoreUint64(&called2, 0) atomic.StoreUint64(&clientId, 0) } @@ -214,7 +206,7 @@ func TestMultiInput(t *testing.T) { var expected2 bay.MaybeMsg expected2.Msg.Data.Event.ReplayID = 1234 expected2.Msg.Data.Payload = []byte(`{"CountryIso": "US"}`) - expected2.Msg.Channel = firstChannel + expected2.Msg.Channel = secondChannel config1 := map[string]interface{}{ "channel_name": firstChannel, @@ -244,42 +236,32 @@ func TestMultiInput(t *testing.T) { var inputContext finput.Context - bayClient := bay.Client{} - - newBayeuxMockClient := func() bay.Client { - b := bay.Bayeux{} - bayClient = bay.Client{ - BayOb: b, - } - return bayClient - } - // initialize inputs - input1, err := newInput(cfg1, connector, newBayeuxMockClient, inputContext) + input1, err := NewInput(cfg1, connector, inputContext) require.NoError(t, err) require.NotNil(t, input1) - input2, err := newInput(cfg2, connector, newBayeuxMockClient, inputContext) + input2, err := NewInput(cfg2, connector, inputContext) require.NoError(t, err) require.NotNil(t, input2) - require.Equal(t, 0, bayClient.BayOb.GetConnectedCount()) + require.Equal(t, 0, bay.GetConnectedCount()) // run input input1.Run() + // run input + input2.Run() + eventProcessing.Wait() + require.Equal(t, 2, bay.GetConnectedCount()) + go func() { - time.Sleep(200 * time.Millisecond) // let input.Stop() be executed. + time.Sleep(time.Second) // let input.Stop() be executed. event := <-eventsCh assertEventMatches(t, expected1, event) }() - // run input - input2.Run() - eventProcessing.Wait() - require.Equal(t, 2, bayClient.BayOb.GetConnectedCount()) - go func() { - time.Sleep(100 * time.Millisecond) // let input.Stop() be executed. + time.Sleep(2 * time.Second) // let input.Stop() be executed. event := <-eventsCh assertEventMatches(t, expected2, event) }() @@ -287,7 +269,11 @@ func TestMultiInput(t *testing.T) { input1.Wait() input2.Wait() - require.Equal(t, 0, bayClient.BayOb.GetConnectedCount()) + require.Equal(t, 0, bay.GetConnectedCount()) + + atomic.StoreUint64(&called1, 0) + atomic.StoreUint64(&called2, 0) + atomic.StoreUint64(&clientId, 0) } func TestStop(t *testing.T) { From 09f81e66da924583a8dd6d58893c142a156d5975 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Thu, 21 Apr 2022 19:13:08 +0530 Subject: [PATCH 39/50] Update unit tests --- x-pack/filebeat/input/cometd/input_test.go | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index d2fe3e0f766f..64187e080053 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -66,6 +66,9 @@ func TestMakeEventFailure(t *testing.T) { } func TestSingleInput(t *testing.T) { + defer atomic.StoreUint64(&called1, 0) + defer atomic.StoreUint64(&called2, 0) + defer atomic.StoreUint64(&clientId, 0) eventsCh := make(chan beat.Event) outlet := &mockedOutleter{ @@ -110,12 +113,12 @@ func TestSingleInput(t *testing.T) { event := <-eventsCh assertEventMatches(t, expected, event) input.Stop() - atomic.StoreUint64(&called1, 0) - atomic.StoreUint64(&called2, 0) - atomic.StoreUint64(&clientId, 0) } func TestInputStop_Wait(t *testing.T) { + defer atomic.StoreUint64(&called1, 0) + defer atomic.StoreUint64(&called2, 0) + defer atomic.StoreUint64(&clientId, 0) eventsCh := make(chan beat.Event) const numMessages = 1 @@ -174,12 +177,12 @@ func TestInputStop_Wait(t *testing.T) { input.Wait() require.Equal(t, 0, bay.GetConnectedCount()) - atomic.StoreUint64(&called1, 0) - atomic.StoreUint64(&called2, 0) - atomic.StoreUint64(&clientId, 0) } func TestMultiInput(t *testing.T) { + defer atomic.StoreUint64(&called1, 0) + defer atomic.StoreUint64(&called2, 0) + defer atomic.StoreUint64(&clientId, 0) eventsCh := make(chan beat.Event) const numMessages = 2 @@ -226,6 +229,7 @@ func TestMultiInput(t *testing.T) { // create Server r := http.HandlerFunc(oauth2Handler) server := httptest.NewServer(r) + defer server.Close() serverURL = server.URL config1["auth.oauth2.token_url"] = serverURL + "/token" config2["auth.oauth2.token_url"] = serverURL + "/token" @@ -255,13 +259,13 @@ func TestMultiInput(t *testing.T) { require.Equal(t, 2, bay.GetConnectedCount()) go func() { - time.Sleep(time.Second) // let input.Stop() be executed. + time.Sleep(4 * time.Second) event := <-eventsCh assertEventMatches(t, expected1, event) }() go func() { - time.Sleep(2 * time.Second) // let input.Stop() be executed. + time.Sleep(5 * time.Second) event := <-eventsCh assertEventMatches(t, expected2, event) }() @@ -270,10 +274,6 @@ func TestMultiInput(t *testing.T) { input2.Wait() require.Equal(t, 0, bay.GetConnectedCount()) - - atomic.StoreUint64(&called1, 0) - atomic.StoreUint64(&called2, 0) - atomic.StoreUint64(&clientId, 0) } func TestStop(t *testing.T) { From 80021a09b8a3c2c32bea103263949dd5f8276fa5 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Fri, 22 Apr 2022 10:17:01 +0530 Subject: [PATCH 40/50] resolve unit tests --- x-pack/filebeat/input/cometd/input_test.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index 64187e080053..b87b7545d9a5 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -185,14 +185,8 @@ func TestMultiInput(t *testing.T) { defer atomic.StoreUint64(&clientId, 0) eventsCh := make(chan beat.Event) - const numMessages = 2 - - var eventProcessing sync.WaitGroup - eventProcessing.Add(numMessages) - outlet := &mockedOutleter{ onEventHandler: func(event beat.Event) bool { - eventProcessing.Done() eventsCh <- event return true }, @@ -255,8 +249,6 @@ func TestMultiInput(t *testing.T) { // run input input2.Run() - eventProcessing.Wait() - require.Equal(t, 2, bay.GetConnectedCount()) go func() { time.Sleep(4 * time.Second) @@ -272,8 +264,6 @@ func TestMultiInput(t *testing.T) { input1.Wait() input2.Wait() - - require.Equal(t, 0, bay.GetConnectedCount()) } func TestStop(t *testing.T) { From b5cee6675718099e468e14228ba601fffd04c2c0 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Fri, 22 Apr 2022 18:33:43 +0530 Subject: [PATCH 41/50] additional tests with multiple inputs, added channel_name filed in comman cometd fields --- x-pack/filebeat/input/cometd/input.go | 7 +++-- x-pack/filebeat/input/cometd/input_test.go | 32 ++++++++++++---------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index 1072f1bf33c5..e3af2c53533e 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -73,7 +73,7 @@ func (in *cometdInput) run() error { if err != nil { return fmt.Errorf("error while parsing JSON: %w", err) } - if ok := in.outlet.OnEvent(makeEvent(event.EventId, string(msg))); !ok { + if ok := in.outlet.OnEvent(makeEvent(event.EventId, e.Msg.Channel, string(msg))); !ok { in.log.Debug("OnEvent returned false. Stopping input worker.") return fmt.Errorf("error ingesting data to elasticsearch") } @@ -178,7 +178,7 @@ type Event struct { EventId string `json:"EventIdentifier"` } -func makeEvent(id string, body string) beat.Event { +func makeEvent(id string, channel string, body string) beat.Event { event := beat.Event{ Timestamp: time.Now().UTC(), Fields: common.MapStr{ @@ -187,6 +187,9 @@ func makeEvent(id string, body string) beat.Event { "created": time.Now().UTC(), }, "message": body, + "cometd": common.MapStr{ + "channel_name": channel, + }, }, Private: body, } diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index b87b7545d9a5..43fbab7bac02 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -59,10 +59,13 @@ func TestMakeEventFailure(t *testing.T) { "created": time.Now().UTC(), }, "message": "DEMOBODYFAIL", + "cometd": common.MapStr{ + "channel_name": "DEMOCHANNEL", + }, }, Private: "DEMOBODYFAIL", } - assert.NotEqual(t, event, makeEvent("DEMOID", "DEMOBODY")) + assert.NotEqual(t, event, makeEvent("DEMOCHANNEL", "DEMOID", "DEMOBODY")) } func TestSingleInput(t *testing.T) { @@ -70,6 +73,7 @@ func TestSingleInput(t *testing.T) { defer atomic.StoreUint64(&called2, 0) defer atomic.StoreUint64(&clientId, 0) eventsCh := make(chan beat.Event) + defer close(eventsCh) outlet := &mockedOutleter{ onEventHandler: func(event beat.Event) bool { @@ -120,6 +124,7 @@ func TestInputStop_Wait(t *testing.T) { defer atomic.StoreUint64(&called2, 0) defer atomic.StoreUint64(&clientId, 0) eventsCh := make(chan beat.Event) + defer close(eventsCh) const numMessages = 1 @@ -184,6 +189,7 @@ func TestMultiInput(t *testing.T) { defer atomic.StoreUint64(&called2, 0) defer atomic.StoreUint64(&clientId, 0) eventsCh := make(chan beat.Event) + defer close(eventsCh) outlet := &mockedOutleter{ onEventHandler: func(event beat.Event) bool { @@ -246,24 +252,22 @@ func TestMultiInput(t *testing.T) { require.Equal(t, 0, bay.GetConnectedCount()) // run input input1.Run() + defer input1.Stop() // run input input2.Run() + defer input2.Stop() - go func() { - time.Sleep(4 * time.Second) - event := <-eventsCh - assertEventMatches(t, expected1, event) - }() - - go func() { - time.Sleep(5 * time.Second) - event := <-eventsCh - assertEventMatches(t, expected2, event) - }() + for _, event := range []beat.Event{<-eventsCh, <-eventsCh} { + channel, err := event.GetValue("cometd.channel_name") + require.NoError(t, err) - input1.Wait() - input2.Wait() + if channel == "channel_name1" { + assertEventMatches(t, expected1, event) + } else { + assertEventMatches(t, expected2, event) + } + } } func TestStop(t *testing.T) { From 94663b686c222d35739b5a6f990799a0be87677d Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Tue, 26 Apr 2022 20:08:20 +0530 Subject: [PATCH 42/50] nit: Event struct -> event struct --- x-pack/filebeat/input/cometd/input.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index e3af2c53533e..aec4864eb596 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -56,7 +56,7 @@ func (in *cometdInput) run() error { return fmt.Errorf("error collecting events: %w", e.Err) } if !e.Msg.Successful { - var event Event + var event event // To handle the last response where the object received was empty if e.Msg.Data.Payload == nil { return nil @@ -174,12 +174,12 @@ type cometdInput struct { authParams bay.AuthenticationParameters } -type Event struct { +type event struct { EventId string `json:"EventIdentifier"` } func makeEvent(id string, channel string, body string) beat.Event { - event := beat.Event{ + e := beat.Event{ Timestamp: time.Now().UTC(), Fields: common.MapStr{ "event": common.MapStr{ @@ -193,7 +193,7 @@ func makeEvent(id string, channel string, body string) beat.Event { }, Private: body, } - event.SetID(id) + e.SetID(id) - return event + return e } From ab365e883add31b26377608013a1bb4625def212 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 27 Apr 2022 00:04:54 +0530 Subject: [PATCH 43/50] test case improvement --- x-pack/filebeat/input/cometd/input_test.go | 25 +++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index 43fbab7bac02..84c7c050c3a5 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -174,14 +174,21 @@ func TestInputStop_Wait(t *testing.T) { eventProcessing.Wait() require.Equal(t, 1, bay.GetConnectedCount()) + var wg sync.WaitGroup + wg.Add(1) go func() { - time.Sleep(100 * time.Millisecond) // let input.Stop() be executed. - for range eventsCh { - } + require.Equal(t, 1, bay.GetConnectedCount()) // current open channels count should be 1 + event := <-eventsCh + assertEventMatches(t, expected, event) // wait for single event + wg.Done() + time.Sleep(100 * time.Millisecond) // let input.Stop() be executed. + require.Equal(t, 0, bay.GetConnectedCount()) // current open channels count should be 0 }() + time.Sleep(100 * time.Millisecond) // let input.Stop() be executed. + + wg.Wait() input.Wait() - require.Equal(t, 0, bay.GetConnectedCount()) } func TestMultiInput(t *testing.T) { @@ -191,8 +198,14 @@ func TestMultiInput(t *testing.T) { eventsCh := make(chan beat.Event) defer close(eventsCh) + const numMessages = 2 + + var eventProcessing sync.WaitGroup + eventProcessing.Add(numMessages) + outlet := &mockedOutleter{ onEventHandler: func(event beat.Event) bool { + eventProcessing.Done() eventsCh <- event return true }, @@ -258,6 +271,8 @@ func TestMultiInput(t *testing.T) { input2.Run() defer input2.Stop() + eventProcessing.Wait() + for _, event := range []beat.Event{<-eventsCh, <-eventsCh} { channel, err := event.GetValue("cometd.channel_name") require.NoError(t, err) @@ -290,7 +305,7 @@ func TestStop(t *testing.T) { input.Stop() select { case <-workerCtx.Done(): - default: + case <-time.After(time.Second): // let input.Stop() be executed. require.NoError(t, fmt.Errorf("input is not stopped.")) } } From c1a80a65e804a5a8e057c32fe04150580442b4ff Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 27 Apr 2022 07:15:28 +0530 Subject: [PATCH 44/50] update looger Error -> Errorw to print formatted errors --- x-pack/filebeat/input/cometd/input.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index aec4864eb596..51d6c6d3843b 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -38,7 +38,7 @@ func (in *cometdInput) Run() { in.b = bay.Bayeux{} in.creds, err = bay.GetSalesforceCredentials(in.authParams) if err != nil { - in.log.Error("not able to get access token: ", err) + in.log.Errorw("not able to get access token", "error", err) return } if err := in.run(); err != nil { @@ -109,7 +109,7 @@ func NewInput( authParams.Password = conf.Auth.OAuth2.Password authParams.TokenURL = conf.Auth.OAuth2.TokenURL - logger := logp.NewLogger("cometd").With( + logger := logp.NewLogger(inputName).With( "pubsub_channel", conf.ChannelName) // Wrap input.Context's Done channel with a context.Context. This goroutine From 7ce08864c63836b240079bc76df20ce39c943a1d Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 27 Apr 2022 07:32:33 +0530 Subject: [PATCH 45/50] update docs with reference links --- x-pack/filebeat/docs/inputs/input-cometd.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/docs/inputs/input-cometd.asciidoc b/x-pack/filebeat/docs/inputs/input-cometd.asciidoc index 66929ab8e49e..d4542944cd04 100644 --- a/x-pack/filebeat/docs/inputs/input-cometd.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-cometd.asciidoc @@ -9,7 +9,7 @@ CometD ++++ -Use the `cometd` input to stream the real-time events from a Salesforce generic subscription Push Topic. +Use the `https://docs.cometd.org/[cometd]` input to stream the real-time events from a https://resources.docs.salesforce.com/sfdc/pdf/api_streaming.pdf[Salesforce generic subscription Push Topic]. This input can, for example, be used to receive Login and Logout events that are generated when users log in or out of the Salesforce instance. From 66dbbfe243895c2324898ccb48cce3c8148ade9d Mon Sep 17 00:00:00 2001 From: kush-elastc Date: Thu, 28 Apr 2022 12:09:04 +0530 Subject: [PATCH 46/50] Add CHANGELOG entry and update docs --- CHANGELOG.asciidoc | 1 + filebeat/docs/filebeat-options.asciidoc | 3 +++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 44fcb439400c..585e8aa5edf0 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -5784,6 +5784,7 @@ https://github.com/elastic/beats/compare/v6.2.3\...v6.3.0[View commits] - Add support human friendly size for the UDP input. {pull}6886[6886] - Add Syslog input to ingest RFC3164 Events via TCP and UDP {pull}6842[6842] - Remove the undefined `username` option from the Redis input and clarify the documentation. {pull}6662[6662] +- Add CometD input {pull}30089[30089] *Heartbeat* diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc index 1c4e9d3dd44c..d95c3b07c37a 100644 --- a/filebeat/docs/filebeat-options.asciidoc +++ b/filebeat/docs/filebeat-options.asciidoc @@ -66,6 +66,7 @@ You can configure {beatname_uc} to use the following inputs: * <<{beatname_lc}-input-aws-s3>> * <<{beatname_lc}-input-azure-eventhub>> * <<{beatname_lc}-input-cloudfoundry>> +* <<{beatname_lc}-input-cometd>> * <<{beatname_lc}-input-container>> * <<{beatname_lc}-input-filestream>> * <<{beatname_lc}-input-gcp-pubsub>> @@ -94,6 +95,8 @@ include::../../x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc[] include::../../x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc[] +include::../../x-pack/filebeat/docs/inputs/input-cometd.asciidoc[] + include::inputs/input-container.asciidoc[] include::inputs/input-filestream.asciidoc[] From 6d5aaa8459a7fe44dfa5583dcd5fbdbe5ebc5808 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Fri, 29 Apr 2022 12:14:52 +0530 Subject: [PATCH 47/50] Use mapstr.M instead of common.MapStr --- x-pack/filebeat/input/cometd/config_auth.go | 12 +++++------- x-pack/filebeat/input/cometd/input.go | 7 ++++--- x-pack/filebeat/input/cometd/input_test.go | 9 +++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/x-pack/filebeat/input/cometd/config_auth.go b/x-pack/filebeat/input/cometd/config_auth.go index fe96583664f2..378680864c14 100644 --- a/x-pack/filebeat/input/cometd/config_auth.go +++ b/x-pack/filebeat/input/cometd/config_auth.go @@ -4,9 +4,7 @@ package cometd -import ( - "errors" -) +import "fmt" type authConfig struct { OAuth2 *oAuth2Config `config:"oauth2"` @@ -24,16 +22,16 @@ type oAuth2Config struct { // Validate checks if oauth2 config is valid. func (o *oAuth2Config) Validate() error { if o.TokenURL == "" { - return errors.New("token_url must be provided") + return fmt.Errorf("token_url must be provided") } if o.ClientID == "" { - return errors.New("client.id must be provided") + return fmt.Errorf("client.id must be provided") } if o.ClientSecret == "" { - return errors.New("client.secret must be provided") + return fmt.Errorf("client.secret must be provided") } if o.User == "" || o.Password == "" { - return errors.New("both user and password must be provided") + return fmt.Errorf("both user and password must be provided") } return nil } diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index 51d6c6d3843b..647ec23fa957 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -16,6 +16,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/elastic-agent-libs/mapstr" bay "github.com/elastic/bayeux" ) @@ -181,13 +182,13 @@ type event struct { func makeEvent(id string, channel string, body string) beat.Event { e := beat.Event{ Timestamp: time.Now().UTC(), - Fields: common.MapStr{ - "event": common.MapStr{ + Fields: mapstr.M{ + "event": mapstr.M{ "id": id, "created": time.Now().UTC(), }, "message": body, - "cometd": common.MapStr{ + "cometd": mapstr.M{ "channel_name": channel, }, }, diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index 84c7c050c3a5..575cd7687eec 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -24,6 +24,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/elastic-agent-libs/mapstr" ) const ( @@ -39,7 +40,7 @@ var ( ) func TestInputDone(t *testing.T) { - config := common.MapStr{ + config := mapstr.M{ "channel_name": firstChannel, "auth.oauth2.client.id": "DEMOCLIENTID", "auth.oauth2.client.secret": "DEMOCLIENTSECRET", @@ -53,13 +54,13 @@ func TestInputDone(t *testing.T) { func TestMakeEventFailure(t *testing.T) { event := beat.Event{ Timestamp: time.Now().UTC(), - Fields: common.MapStr{ - "event": common.MapStr{ + Fields: mapstr.M{ + "event": mapstr.M{ "id": "DEMOID", "created": time.Now().UTC(), }, "message": "DEMOBODYFAIL", - "cometd": common.MapStr{ + "cometd": mapstr.M{ "channel_name": "DEMOCHANNEL", }, }, From 3dba2fbd18ac304b64db19efaeea2754eb3306fb Mon Sep 17 00:00:00 2001 From: kush-elastc Date: Fri, 29 Apr 2022 14:21:05 +0530 Subject: [PATCH 48/50] Changes based on Replace common.Config and friends with config.C --- x-pack/filebeat/input/cometd/client_mocked.go | 6 +++--- .../filebeat/input/cometd/cometd_integration_test.go | 5 +++-- x-pack/filebeat/input/cometd/input.go | 4 ++-- x-pack/filebeat/input/cometd/input_test.go | 10 +++++----- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/x-pack/filebeat/input/cometd/client_mocked.go b/x-pack/filebeat/input/cometd/client_mocked.go index a4123b77bf80..6079cb1f174e 100644 --- a/x-pack/filebeat/input/cometd/client_mocked.go +++ b/x-pack/filebeat/input/cometd/client_mocked.go @@ -7,7 +7,7 @@ package cometd import ( "github.com/elastic/beats/v7/filebeat/channel" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common" + conf "github.com/elastic/elastic-agent-libs/config" ) type mockedConnector struct { @@ -17,11 +17,11 @@ type mockedConnector struct { var _ channel.Connector = new(mockedConnector) -func (m *mockedConnector) Connect(c *common.Config) (channel.Outleter, error) { +func (m *mockedConnector) Connect(c *conf.C) (channel.Outleter, error) { return m.ConnectWith(c, beat.ClientConfig{}) } -func (m *mockedConnector) ConnectWith(*common.Config, beat.ClientConfig) (channel.Outleter, error) { +func (m *mockedConnector) ConnectWith(*conf.C, beat.ClientConfig) (channel.Outleter, error) { if m.connectWithError != nil { return nil, m.connectWithError } diff --git a/x-pack/filebeat/input/cometd/cometd_integration_test.go b/x-pack/filebeat/input/cometd/cometd_integration_test.go index b0461e2a5ec0..612af627576f 100644 --- a/x-pack/filebeat/input/cometd/cometd_integration_test.go +++ b/x-pack/filebeat/input/cometd/cometd_integration_test.go @@ -20,6 +20,7 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" bay "github.com/elastic/bayeux" + conf "github.com/elastic/elastic-agent-libs/config" ) type eventCaptor struct { @@ -57,7 +58,7 @@ func TestInput(t *testing.T) { logp.TestingSetup(logp.WithSelectors("cometd input", "cometd")) //nolint:errcheck // Bad linter! no need to test this. // Setup the input config. - config := common.MustNewConfigFrom(common.MapStr{ + config := conf.MustNewConfigFrom(conf.MapStr{ "channel_name": "channel_name1", "auth.oauth2.client.id": "client.id", "auth.oauth2.client.secret": "client.secret", @@ -73,7 +74,7 @@ func TestInput(t *testing.T) { captor := newEventCaptor(eventsCh) defer captor.Close() - connector := channel.ConnectorFunc(func(_ *common.Config, _ beat.ClientConfig) (channel.Outleter, error) { + connector := channel.ConnectorFunc(func(_ *conf.C, _ beat.ClientConfig) (channel.Outleter, error) { return channel.SubOutlet(captor), nil }) diff --git a/x-pack/filebeat/input/cometd/input.go b/x-pack/filebeat/input/cometd/input.go index 647ec23fa957..0ad0084a8f77 100644 --- a/x-pack/filebeat/input/cometd/input.go +++ b/x-pack/filebeat/input/cometd/input.go @@ -14,11 +14,11 @@ import ( "github.com/elastic/beats/v7/filebeat/channel" "github.com/elastic/beats/v7/filebeat/input" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/elastic-agent-libs/mapstr" bay "github.com/elastic/bayeux" + conf "github.com/elastic/elastic-agent-libs/config" ) const ( @@ -93,7 +93,7 @@ func init() { // NewInput creates a new CometD input that consumes events from // a topic subscription. func NewInput( - cfg *common.Config, + cfg *conf.C, connector channel.Connector, inputContext input.Context, ) (inp input.Input, err error) { diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index 575cd7687eec..5b6b5999a89d 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -22,8 +22,8 @@ import ( finput "github.com/elastic/beats/v7/filebeat/input" "github.com/elastic/beats/v7/filebeat/input/inputtest" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" + conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -107,7 +107,7 @@ func TestSingleInput(t *testing.T) { serverURL = server.URL config["auth.oauth2.token_url"] = server.URL + "/token" - cfg := common.MustNewConfigFrom(config) + cfg := conf.MustNewConfigFrom(config) input, err := NewInput(cfg, connector, inputContext) require.NoError(t, err) @@ -164,7 +164,7 @@ func TestInputStop_Wait(t *testing.T) { serverURL = server.URL config["auth.oauth2.token_url"] = server.URL + "/token" - cfg := common.MustNewConfigFrom(config) + cfg := conf.MustNewConfigFrom(config) input, err := NewInput(cfg, connector, inputContext) require.NoError(t, err) @@ -249,8 +249,8 @@ func TestMultiInput(t *testing.T) { config2["auth.oauth2.token_url"] = serverURL + "/token" // get common config - cfg1 := common.MustNewConfigFrom(config1) - cfg2 := common.MustNewConfigFrom(config2) + cfg1 := conf.MustNewConfigFrom(config1) + cfg2 := conf.MustNewConfigFrom(config2) var inputContext finput.Context From e427fededfdbf4f925d6ab0382c3e527c130a6f5 Mon Sep 17 00:00:00 2001 From: kush-elastc Date: Fri, 29 Apr 2022 14:45:45 +0530 Subject: [PATCH 49/50] resolve make check error --- x-pack/filebeat/input/cometd/cometd_integration_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/filebeat/input/cometd/cometd_integration_test.go b/x-pack/filebeat/input/cometd/cometd_integration_test.go index 612af627576f..f5caa298c5b9 100644 --- a/x-pack/filebeat/input/cometd/cometd_integration_test.go +++ b/x-pack/filebeat/input/cometd/cometd_integration_test.go @@ -16,7 +16,6 @@ import ( "github.com/elastic/beats/v7/filebeat/channel" "github.com/elastic/beats/v7/filebeat/input" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" bay "github.com/elastic/bayeux" From f13db1237b06fce1d12ff8720756f5c5f0fa8e97 Mon Sep 17 00:00:00 2001 From: kush-elastc Date: Fri, 29 Apr 2022 16:21:00 +0530 Subject: [PATCH 50/50] resolve comman config errors --- x-pack/filebeat/input/cometd/cometd_integration_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/cometd/cometd_integration_test.go b/x-pack/filebeat/input/cometd/cometd_integration_test.go index f5caa298c5b9..8b6bbf60c897 100644 --- a/x-pack/filebeat/input/cometd/cometd_integration_test.go +++ b/x-pack/filebeat/input/cometd/cometd_integration_test.go @@ -20,6 +20,7 @@ import ( bay "github.com/elastic/bayeux" conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" ) type eventCaptor struct { @@ -57,7 +58,7 @@ func TestInput(t *testing.T) { logp.TestingSetup(logp.WithSelectors("cometd input", "cometd")) //nolint:errcheck // Bad linter! no need to test this. // Setup the input config. - config := conf.MustNewConfigFrom(conf.MapStr{ + config := conf.MustNewConfigFrom(mapstr.M{ "channel_name": "channel_name1", "auth.oauth2.client.id": "client.id", "auth.oauth2.client.secret": "client.secret",