diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 32751d0a5bcc..8f47070b9677 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -61,14 +61,6 @@ Example configurations with authentication: url: http://localhost ---- -["source","yaml",subs="attributes"] ----- -{beatname_lc}.inputs: -- type: httpjson - api_key: 12345678901234567890abcdef - url: http://localhost ----- - ["source","yaml",subs="attributes"] ---- {beatname_lc}.inputs: @@ -294,24 +286,19 @@ the `oauth2` section is missing. The `provider` setting can be used to configure supported oauth2 providers. Each supported provider will require specific settings. It is not set by default. - -NOTE: Supported providers are: `azure`, `google`. +Supported providers are: `azure`, `google`. [float] ==== `oauth2.client.id` The `client.id` setting is used as part of the authentication flow. It is always required -except if using `google` as provider. - -NOTE: Required for providers: `default`, `azure`. +except if using `google` as provider. Required for providers: `default`, `azure`. [float] ==== `oauth2.client.secret` The `client.secret` setting is used as part of the authentication flow. It is always required -except if using `google` as provider. - -NOTE: Required for providers: `default`, `azure`. +except if using `google` as provider. Required for providers: `default`, `azure`. [float] ==== `oauth2.scopes` @@ -325,16 +312,14 @@ It is optional for all providers. The `token_url` setting specifies the endpoint that will be used to generate the tokens during the oauth2 flow. It is required if no provider is specified. -NOTE: For `azure` provider, a default `token_url` will be used if none provided, -but it can be set to override the default one. +NOTE: For `azure` provider either `token_url` or `azure.tenant_id` is required. [float] ==== `oauth2.endpoint_params` The `endpoint_params` setting specifies a set of values that will be sent on each request to the `token_url`. Each param key can have multiple values. - -NOTE: Can be set for all providers except `google`. +Can be set for all providers except `google`. ["source","yaml",subs="attributes"] ---- @@ -355,8 +340,8 @@ The `azure.tenant_id` is used for authentication when using `azure` provider. Since it is used in the process to generate the `token_url`, it can't be used in combination with it. It is not required. -NOTE: For information about where to find it, you can refer to -https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal +For information about where to find it, you can refer to +https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal. [float] ==== `oauth2.azure.resource` @@ -370,7 +355,7 @@ It is not required. The `google.credentials_file` setting specifies the credentials file for Google. NOTE: Only one of the credentials settings can be set at once. If none is provided, loading -default credentials from the environment will be attempted via ADC. For for information about +default credentials from the environment will be attempted via ADC. For more information about how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. [float] @@ -379,7 +364,7 @@ how to provide Google credentials, please refer to https://cloud.google.com/docs The `google.credentials_json` setting allows to write your credentials information as raw JSON. NOTE: Only one of the credentials settings can be set at once. If none is provided, loading -default credentials from the environment will be attempted via ADC. For for information about +default credentials from the environment will be attempted via ADC. For more information about how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. [float] @@ -388,7 +373,7 @@ how to provide Google credentials, please refer to https://cloud.google.com/docs The `google.jwt_file` setting specifies the JWT Account Key file for Google. NOTE: Only one of the credentials settings can be set at once. If none is provided, loading -default credentials from the environment will be attempted via ADC. For for information about +default credentials from the environment will be attempted via ADC. For more information about how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. [id="{beatname_lc}-input-{type}-common-options"] diff --git a/x-pack/filebeat/input/httpjson/config_oauth.go b/x-pack/filebeat/input/httpjson/config_oauth.go index a5bd0f378a48..b9bdb45668c8 100644 --- a/x-pack/filebeat/input/httpjson/config_oauth.go +++ b/x-pack/filebeat/input/httpjson/config_oauth.go @@ -69,23 +69,24 @@ func (o *OAuth2) Client(ctx context.Context, client *http.Client) (*http.Client, ctx = context.WithValue(ctx, oauth2.HTTPClient, client) switch o.GetProvider() { + case OAuth2ProviderAzure, OAuth2ProviderDefault: + creds := clientcredentials.Config{ + ClientID: o.ClientID, + ClientSecret: o.ClientSecret, + TokenURL: o.GetTokenURL(), + Scopes: o.Scopes, + EndpointParams: o.GetEndpointParams(), + } + return creds.Client(ctx), nil case OAuth2ProviderGoogle: creds, err := google.CredentialsFromJSON(ctx, o.GoogleCredentialsJSON, o.Scopes...) if err != nil { return nil, fmt.Errorf("oauth2 client: error loading credentials: %w", err) } return oauth2.NewClient(ctx, creds.TokenSource), nil + default: + return nil, errors.New("oauth2 client: unknown provider") } - - creds := clientcredentials.Config{ - ClientID: o.ClientID, - ClientSecret: o.ClientSecret, - TokenURL: o.GetTokenURL(), - Scopes: o.Scopes, - EndpointParams: o.GetEndpointParams(), - } - - return creds.Client(ctx), nil } // GetTokenURL returns the TokenURL. diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index 5ad7b0a5e6a7..78432527f606 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -109,310 +109,260 @@ func TestConfigValidationCase7(t *testing.T) { } } -func TestConfigValidationCase8(t *testing.T) { - const expectedErr = "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config" - m := map[string]interface{}{ - "api_key": "an_api_key", - "oauth2": map[string]interface{}{ - "token_url": "localhost", - "client": map[string]interface{}{ - "id": "a_client_id", - "secret": "a_client_secret", +func TestConfigOauth2Validation(t *testing.T) { + cases := []struct { + name string + expectedErr string + input map[string]interface{} + setup func() + teardown func() + }{ + { + name: "can't set oauth2 and api_key together", + expectedErr: "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config", + input: map[string]interface{}{ + "api_key": "an_api_key", + "oauth2": map[string]interface{}{ + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + "url": "localhost", }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase9(t *testing.T) { - const expectedErr = "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config" - m := map[string]interface{}{ - "authentication_scheme": "an_api_key", - "oauth2": map[string]interface{}{ - "token_url": "localhost", - "client": map[string]interface{}{ - "id": "a_client_id", - "secret": "a_client_secret", + { + name: "can't set oauth2 and authentication_scheme", + expectedErr: "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config", + input: map[string]interface{}{ + "authentication_scheme": "a_scheme", + "oauth2": map[string]interface{}{ + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + "url": "localhost", }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase10(t *testing.T) { - const expectedErr = "invalid configuration: both token_url and client credentials must be provided accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase11(t *testing.T) { - const expectedErr = "invalid configuration: unknown provider \"unknown\" accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "unknown", + { + name: "token_url and client credentials must be set", + expectedErr: "invalid configuration: both token_url and client credentials must be provided accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{}, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase12(t *testing.T) { - const expectedErr = "invalid configuration: at least one of token_url or tenant_id must be provided accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "azure", + { + name: "must fail with an unknown provider", + expectedErr: "invalid configuration: unknown provider \"unknown\" accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "unknown", + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase13(t *testing.T) { - const expectedErr = "invalid configuration: only one of token_url and tenant_id can be used accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "azure", - "azure.tenant_id": "a_tenant_id", - "token_url": "localhost", + { + name: "azure must have either tenant_id or token_url", + expectedErr: "invalid configuration: at least one of token_url or tenant_id must be provided accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase14(t *testing.T) { - const expectedErr = "invalid configuration: client credentials must be provided accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "azure", - "azure.tenant_id": "a_tenant_id", + { + name: "azure must have only one of token_url and tenant_id", + expectedErr: "invalid configuration: only one of token_url and tenant_id can be used accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + "azure.tenant_id": "a_tenant_id", + "token_url": "localhost", + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase15(t *testing.T) { - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "azure", - "azure": map[string]interface{}{ - "tenant_id": "a_tenant_id", + { + name: "azure must have client credentials set", + expectedErr: "invalid configuration: client credentials must be provided accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + "azure.tenant_id": "a_tenant_id", + }, + "url": "localhost", }, - "client.id": "a_client_id", - "client.secret": "a_client_secret", }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err != nil { - t.Fatalf("Configuration validation failed. no error expected but got %q", err) - } -} - -func TestConfigValidationCase16(t *testing.T) { - const expectedErr = "invalid configuration: none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", - "azure": map[string]interface{}{ - "tenant_id": "a_tenant_id", + { + name: "azure config is valid", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + "azure": map[string]interface{}{ + "tenant_id": "a_tenant_id", + }, + "client.id": "a_client_id", + "client.secret": "a_client_secret", + }, + "url": "localhost", }, - "client.id": "a_client_id", - "client.secret": "a_client_secret", - "token_url": "localhost", }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase17(t *testing.T) { - // we change the default function to force a failure - findDefaultGoogleCredentials = func(context.Context, ...string) (*google.Credentials, error) { - return nil, errors.New("failed") - } - - defer func() { findDefaultGoogleCredentials = google.FindDefaultCredentials }() - - const expectedErr = "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", + { + name: "google can't have token_url or client credentials set", + expectedErr: "invalid configuration: none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "azure": map[string]interface{}{ + "tenant_id": "a_tenant_id", + }, + "client.id": "a_client_id", + "client.secret": "a_client_secret", + "token_url": "localhost", + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase18(t *testing.T) { - const expectedErr = "invalid configuration: the file \"./wrong\" cannot be found accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_file": "./wrong", + { + name: "google must fail if no ADC available", + expectedErr: "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + }, + "url": "localhost", + }, + setup: func() { + // we change the default function to force a failure + findDefaultGoogleCredentials = func(context.Context, ...string) (*google.Credentials, error) { + return nil, errors.New("failed") + } + }, + teardown: func() { findDefaultGoogleCredentials = google.FindDefaultCredentials }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase19(t *testing.T) { - os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./wrong") - const expectedErr = "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", + { + name: "google must fail if credentials file not found", + expectedErr: "invalid configuration: the file \"./wrong\" cannot be found accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./wrong", + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase20(t *testing.T) { - os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./testdata/credentials.json") - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", + { + name: "google must fail if ADC is wrongly set", + expectedErr: "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + }, + "url": "localhost", + }, + setup: func() { os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./wrong") }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err != nil { - t.Fatalf("Configuration validation failed. no error expected but got %q", err) - } -} - -func TestConfigValidationCase21(t *testing.T) { - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_file": "./testdata/credentials.json", + { + name: "google must work if ADC is set up", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + }, + "url": "localhost", + }, + setup: func() { os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./testdata/credentials.json") }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err != nil { - t.Fatalf("Configuration validation failed. no error expected but got %q", err) - } -} - -func TestConfigValidationCase22(t *testing.T) { - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", - "google.jwt_file": "./testdata/credentials.json", + { + name: "google must work if credentials_file is correct", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./testdata/credentials.json", + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err != nil { - t.Fatalf("Configuration validation failed. no error expected but got %q", err) - } -} - -func TestConfigValidationCase23(t *testing.T) { - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_json": []byte(`{ - "type": "service_account", - "project_id": "foo", - "private_key_id": "x", - "client_email": "foo@bar.com", - "client_id": "0" - }`), + { + name: "google must work if jwt_file is correct", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.jwt_file": "./testdata/credentials.json", + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err != nil { - t.Fatalf("Configuration validation failed. no error expected but got %q", err) - } -} - -func TestConfigValidationCase24(t *testing.T) { - const expectedErr = "invalid configuration: google.credentials_json must be valid JSON accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_json": []byte(`invalid`), + { + name: "google must work if credentials_json is correct", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_json": []byte(`{ + "type": "service_account", + "project_id": "foo", + "private_key_id": "x", + "client_email": "foo@bar.com", + "client_id": "0" + }`), + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase25(t *testing.T) { - const expectedErr = "invalid configuration: the file \"./testdata/invalid_credentials.json\" does not contain valid JSON accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_file": "./testdata/invalid_credentials.json", + { + name: "google must fail if credentials_json is not a valid JSON", + expectedErr: "invalid configuration: google.credentials_json must be valid JSON accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_json": []byte(`invalid`), + }, + "url": "localhost", + }, + }, + { + name: "google must fail if the provided credentials file is not a valid JSON", + expectedErr: "invalid configuration: the file \"./testdata/invalid_credentials.json\" does not contain valid JSON accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./testdata/invalid_credentials.json", + }, + "url": "localhost", + }, }, - "url": "localhost", } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + if c.setup != nil { + c.setup() + } + + if c.teardown != nil { + defer c.teardown() + } + + cfg := common.MustNewConfigFrom(c.input) + conf := defaultConfig() + err := cfg.Unpack(&conf) + + switch { + case c.expectedErr == "": + if err != nil { + t.Fatalf("Configuration validation failed. no error expected but got %q", err) + } + + case c.expectedErr != "": + if err == nil || err.Error() != c.expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", c.expectedErr, err) + } + } + }) } }