From 2d3aa03e4ac54f8bbbd88a2365666e95b8c2bbb3 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 10 Apr 2020 12:08:07 +0200 Subject: [PATCH 1/5] add option to select a different azure env --- filebeat/docs/modules/azure.asciidoc | 7 ++++++ .../docs/inputs/input-azure-eventhub.asciidoc | 11 +++++--- x-pack/filebeat/filebeat.reference.yml | 12 ++++++--- x-pack/filebeat/input/azureeventhub/config.go | 2 ++ x-pack/filebeat/input/azureeventhub/eph.go | 25 ++++++++++++++++++- x-pack/filebeat/module/azure/_meta/config.yml | 12 ++++++--- .../filebeat/module/azure/_meta/docs.asciidoc | 7 ++++++ .../activitylogs/config/azure-eventhub.yml | 1 + .../module/azure/activitylogs/manifest.yml | 1 + .../azure/auditlogs/config/azure-eventhub.yml | 1 + .../module/azure/auditlogs/manifest.yml | 1 + .../signinlogs/config/azure-eventhub.yml | 1 + .../module/azure/signinlogs/manifest.yml | 1 + x-pack/filebeat/modules.d/azure.yml.disabled | 12 ++++++--- 14 files changed, 78 insertions(+), 16 deletions(-) diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc index 5d52e33beace..8fa93a584b21 100644 --- a/filebeat/docs/modules/azure.asciidoc +++ b/filebeat/docs/modules/azure.asciidoc @@ -43,6 +43,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -52,6 +53,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + override_resource_manager_endpoint: "" signinlogs: enabled: false @@ -61,6 +63,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + override_resource_manager_endpoint: "" ``` @@ -90,6 +93,10 @@ The name of the storage account the state/offsets will be stored and updated. _string_ The storage account key, this key will be used to authorize access to data in your storage account. +`override_resource_manager_endpoint` :: +_string_ +Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. + include::../include/what-happens.asciidoc[] include::../include/gs-link.asciidoc[] diff --git a/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc b/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc index 15b628169cea..f5954de0c52a 100644 --- a/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc @@ -28,6 +28,8 @@ Example configuration: storage_account: "azureeph" storage_account_key: "....." storage_account_container: "" + override_resource_manager_endpoint: "" + ---- ==== Configuration options @@ -36,7 +38,7 @@ The `azure-eventhub` input supports the following configuration: ==== `eventhub` -The name of the eventhub users would like to read from. +The name of the eventhub users would like to read from, field required. ==== `consumer_group` @@ -50,14 +52,17 @@ A Blob Storage account is required in order to store/retrieve/update the offset ==== `storage_account` -The name of the storage account. +The name of the storage account. Required. ==== `storage_account_key` -The storage account key, this key will be used to authorize access to data in your storage account. +The storage account key, this key will be used to authorize access to data in your storage account, option is required. ==== `storage_account_container` Optional, the name of the storage account container you would like to store the offset information in. +==== `override_resource_manager_endpoint` + +Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index d42397123fb4..1674d6d5e60e 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -288,16 +288,18 @@ filebeat.modules: activitylogs: enabled: true var: - # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub + # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub eventhub: "insights-operational-logs" - # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module + # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module consumer_group: "$Default" # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string connection_string: "" - # the name of the storage account the state/offsets will be stored and updated. + # the name of the storage account the state/offsets will be stored and updated storage_account: "" - #The storage account key, this key will be used to authorize access to data in your storage account. + # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + # by default the azure public environment is used, to override, users can provide a specific resource manager endpoint + override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -307,6 +309,7 @@ filebeat.modules: # connection_string: "" # storage_account: "" # storage_account_key: "" + # override_resource_manager_endpoint: "" signinlogs: enabled: false # var: @@ -315,6 +318,7 @@ filebeat.modules: # connection_string: "" # storage_account: "" # storage_account_key: "" + # override_resource_manager_endpoint: "" #--------------------------------- CEF Module --------------------------------- - module: cef diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go index b567b25e35b2..bfd60b99a3ff 100644 --- a/x-pack/filebeat/input/azureeventhub/config.go +++ b/x-pack/filebeat/input/azureeventhub/config.go @@ -17,6 +17,8 @@ type azureInputConfig struct { SAName string `config:"storage_account"` SAKey string `config:"storage_account_key"` SAContainer string `config:"storage_account_container"` + // by default the azure public environment is used, to override, users can provide a specific resource manager endpoint + OverrideEnvironment string `config:"override_resource_manager_endpoint"` } const ephContainerName = "filebeat" diff --git a/x-pack/filebeat/input/azureeventhub/eph.go b/x-pack/filebeat/input/azureeventhub/eph.go index 8848483c8be8..fa63b8f46f3b 100644 --- a/x-pack/filebeat/input/azureeventhub/eph.go +++ b/x-pack/filebeat/input/azureeventhub/eph.go @@ -23,7 +23,11 @@ func (a *azureInput) runWithEPH() error { if err != nil { return err } - leaserCheckpointer, err := storage.NewStorageLeaserCheckpointer(cred, a.config.SAName, a.config.SAContainer, azure.PublicCloud) + env, err := getAzureEnvironment(a.config.OverrideEnvironment) + if err != nil { + return err + } + leaserCheckpointer, err := storage.NewStorageLeaserCheckpointer(cred, a.config.SAName, a.config.SAContainer, env) if err != nil { return err } @@ -74,3 +78,22 @@ func (a *azureInput) runWithEPH() error { } return nil } + +func getAzureEnvironment(overrideResManager string) (azure.Environment, error) { + // if no overrride is set then the azure public cloud is used + if overrideResManager == "" { + return azure.PublicCloud, nil + } + // users can select from one of the already defined azure cloud envs + var environments = map[string]azure.Environment{ + azure.ChinaCloud.ResourceManagerEndpoint: azure.ChinaCloud, + azure.GermanCloud.ResourceManagerEndpoint: azure.GermanCloud, + azure.PublicCloud.ResourceManagerEndpoint: azure.PublicCloud, + azure.USGovernmentCloud.ResourceManagerEndpoint: azure.USGovernmentCloud, + } + if env, ok := environments[overrideResManager]; ok { + return env, nil + } + // can retrieve hybrid env from the resource manager endpoint + return azure.EnvironmentFromURL(overrideResManager) +} diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml index 7509037c28ee..557c96664b86 100644 --- a/x-pack/filebeat/module/azure/_meta/config.yml +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -3,16 +3,18 @@ activitylogs: enabled: true var: - # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub + # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub eventhub: "insights-operational-logs" - # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module + # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module consumer_group: "$Default" # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string connection_string: "" - # the name of the storage account the state/offsets will be stored and updated. + # the name of the storage account the state/offsets will be stored and updated storage_account: "" - #The storage account key, this key will be used to authorize access to data in your storage account. + # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + # by default the azure public environment is used, to override, users can provide a specific resource manager endpoint + override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -22,6 +24,7 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" + # override_resource_manager_endpoint: "" signinlogs: enabled: false # var: @@ -30,3 +33,4 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" + # override_resource_manager_endpoint: "" diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc index 5bf7bb576d00..fe5b3235ab3d 100644 --- a/x-pack/filebeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -38,6 +38,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -47,6 +48,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + override_resource_manager_endpoint: "" signinlogs: enabled: false @@ -56,6 +58,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + override_resource_manager_endpoint: "" ``` @@ -85,6 +88,10 @@ The name of the storage account the state/offsets will be stored and updated. _string_ The storage account key, this key will be used to authorize access to data in your storage account. +`override_resource_manager_endpoint` :: +_string_ +Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. + include::../include/what-happens.asciidoc[] include::../include/gs-link.asciidoc[] diff --git a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml index b89bebb30f92..e4f8da96b8a1 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml @@ -4,3 +4,4 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} +override_resource_manager_endpoint: {{ .override_resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/activitylogs/manifest.yml b/x-pack/filebeat/module/azure/activitylogs/manifest.yml index 7375b6e42a4b..e7078b075f33 100644 --- a/x-pack/filebeat/module/azure/activitylogs/manifest.yml +++ b/x-pack/filebeat/module/azure/activitylogs/manifest.yml @@ -10,6 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key + - name: override_resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml index 01796611504f..b01a4d7f8edf 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml @@ -4,4 +4,5 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} +override_resource_manager_endpoint: {{ .override_resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/auditlogs/manifest.yml b/x-pack/filebeat/module/azure/auditlogs/manifest.yml index d6cd469718b9..0f193cf6b0db 100644 --- a/x-pack/filebeat/module/azure/auditlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/auditlogs/manifest.yml @@ -10,6 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key + - name: override_resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml index b89bebb30f92..e4f8da96b8a1 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml @@ -4,3 +4,4 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} +override_resource_manager_endpoint: {{ .override_resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/signinlogs/manifest.yml b/x-pack/filebeat/module/azure/signinlogs/manifest.yml index f68109af4a1e..7f82837828b9 100644 --- a/x-pack/filebeat/module/azure/signinlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/signinlogs/manifest.yml @@ -10,6 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key + - name: override_resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled index c8003fbcf96b..25beff244792 100644 --- a/x-pack/filebeat/modules.d/azure.yml.disabled +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -6,16 +6,18 @@ activitylogs: enabled: true var: - # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub + # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub eventhub: "insights-operational-logs" - # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module + # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module consumer_group: "$Default" # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string connection_string: "" - # the name of the storage account the state/offsets will be stored and updated. + # the name of the storage account the state/offsets will be stored and updated storage_account: "" - #The storage account key, this key will be used to authorize access to data in your storage account. + # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + # by default the azure public environment is used, to override, users can provide a specific resource manager endpoint + override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -25,6 +27,7 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" + # override_resource_manager_endpoint: "" signinlogs: enabled: false # var: @@ -33,3 +36,4 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" + # override_resource_manager_endpoint: "" From f1dc7da81c7e20b84fb9dcb9231158aed146a125 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 10 Apr 2020 12:19:39 +0200 Subject: [PATCH 2/5] changelog --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/azureeventhub/eph.go | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 55de0e08f3b2..ac4d631e1c42 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -250,6 +250,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add dashboard for Google Cloud Audit and AWS CloudTrail. {pull}17379[17379] - Improve ECS categorization field mappings for mysql module. {issue}16172[16172] {pull}17491[17491] - Release Google Cloud module as GA. {pull}17511[17511] +- Add config option to select a different azure cloud env in the azure-eventhub input and azure module. {issue}17649[17649] {pull}17659[17659] *Heartbeat* diff --git a/x-pack/filebeat/input/azureeventhub/eph.go b/x-pack/filebeat/input/azureeventhub/eph.go index fa63b8f46f3b..bab54a45223b 100644 --- a/x-pack/filebeat/input/azureeventhub/eph.go +++ b/x-pack/filebeat/input/azureeventhub/eph.go @@ -16,6 +16,14 @@ import ( "github.com/Azure/go-autorest/autorest/azure" ) +// users can select from one of the already defined azure cloud envs +var environments = map[string]azure.Environment{ + azure.ChinaCloud.ResourceManagerEndpoint: azure.ChinaCloud, + azure.GermanCloud.ResourceManagerEndpoint: azure.GermanCloud, + azure.PublicCloud.ResourceManagerEndpoint: azure.PublicCloud, + azure.USGovernmentCloud.ResourceManagerEndpoint: azure.USGovernmentCloud, +} + // runWithEPH will consume ingested events using the Event Processor Host (EPH) https://github.com/Azure/azure-event-hubs-go#event-processor-host, https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-event-processor-host func (a *azureInput) runWithEPH() error { // create a new Azure Storage Leaser / Checkpointer @@ -84,13 +92,6 @@ func getAzureEnvironment(overrideResManager string) (azure.Environment, error) { if overrideResManager == "" { return azure.PublicCloud, nil } - // users can select from one of the already defined azure cloud envs - var environments = map[string]azure.Environment{ - azure.ChinaCloud.ResourceManagerEndpoint: azure.ChinaCloud, - azure.GermanCloud.ResourceManagerEndpoint: azure.GermanCloud, - azure.PublicCloud.ResourceManagerEndpoint: azure.PublicCloud, - azure.USGovernmentCloud.ResourceManagerEndpoint: azure.USGovernmentCloud, - } if env, ok := environments[overrideResManager]; ok { return env, nil } From 33dd8f335797e571f634fc989a9ebf1e3f7a88bf Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 20 Apr 2020 13:48:56 +0200 Subject: [PATCH 3/5] rename --- filebeat/docs/modules/azure.asciidoc | 14 ++++++++++---- .../docs/inputs/input-azure-eventhub.asciidoc | 10 ++++++++-- x-pack/filebeat/filebeat.reference.yml | 4 ---- x-pack/filebeat/input/azureeventhub/config.go | 2 +- x-pack/filebeat/module/azure/_meta/config.yml | 4 ---- x-pack/filebeat/module/azure/_meta/docs.asciidoc | 14 ++++++++++---- .../azure/activitylogs/config/azure-eventhub.yml | 2 +- .../module/azure/activitylogs/manifest.yml | 2 +- .../azure/auditlogs/config/azure-eventhub.yml | 2 +- .../filebeat/module/azure/auditlogs/manifest.yml | 2 +- .../azure/signinlogs/config/azure-eventhub.yml | 2 +- .../filebeat/module/azure/signinlogs/manifest.yml | 2 +- x-pack/filebeat/modules.d/azure.yml.disabled | 4 ---- 13 files changed, 35 insertions(+), 29 deletions(-) diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc index 8fa93a584b21..b194b7c320ca 100644 --- a/filebeat/docs/modules/azure.asciidoc +++ b/filebeat/docs/modules/azure.asciidoc @@ -43,7 +43,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" - override_resource_manager_endpoint: "" + resource_manager_endpoint: "" auditlogs: enabled: false @@ -53,7 +53,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" - override_resource_manager_endpoint: "" + resource_manager_endpoint: "" signinlogs: enabled: false @@ -63,7 +63,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" - override_resource_manager_endpoint: "" + resource_manager_endpoint: "" ``` @@ -93,9 +93,15 @@ The name of the storage account the state/offsets will be stored and updated. _string_ The storage account key, this key will be used to authorize access to data in your storage account. -`override_resource_manager_endpoint` :: +`resource_manager_endpoint` :: _string_ Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. +Ex: +https://management.chinacloudapi.cn/ for azure ChinaCloud +https://management.microsoftazure.de/ for azure GermanCloud +https://management.azure.com/ for azure PublicCloud +https://management.usgovcloudapi.net/ for azure USGovernmentCloud +Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints. include::../include/what-happens.asciidoc[] diff --git a/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc b/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc index f5954de0c52a..ac91fb476d66 100644 --- a/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc @@ -28,7 +28,7 @@ Example configuration: storage_account: "azureeph" storage_account_key: "....." storage_account_container: "" - override_resource_manager_endpoint: "" + resource_manager_endpoint: "" ---- @@ -62,7 +62,13 @@ The storage account key, this key will be used to authorize access to data in yo Optional, the name of the storage account container you would like to store the offset information in. -==== `override_resource_manager_endpoint` +==== `resource_manager_endpoint` Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. +Ex: +https://management.chinacloudapi.cn/ for azure ChinaCloud +https://management.microsoftazure.de/ for azure GermanCloud +https://management.azure.com/ for azure PublicCloud +https://management.usgovcloudapi.net/ for azure USGovernmentCloud +Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints. diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 1674d6d5e60e..7a522fdd41e1 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -298,8 +298,6 @@ filebeat.modules: storage_account: "" # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" - # by default the azure public environment is used, to override, users can provide a specific resource manager endpoint - override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -309,7 +307,6 @@ filebeat.modules: # connection_string: "" # storage_account: "" # storage_account_key: "" - # override_resource_manager_endpoint: "" signinlogs: enabled: false # var: @@ -318,7 +315,6 @@ filebeat.modules: # connection_string: "" # storage_account: "" # storage_account_key: "" - # override_resource_manager_endpoint: "" #--------------------------------- CEF Module --------------------------------- - module: cef diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go index bfd60b99a3ff..0521d3a76e68 100644 --- a/x-pack/filebeat/input/azureeventhub/config.go +++ b/x-pack/filebeat/input/azureeventhub/config.go @@ -18,7 +18,7 @@ type azureInputConfig struct { SAKey string `config:"storage_account_key"` SAContainer string `config:"storage_account_container"` // by default the azure public environment is used, to override, users can provide a specific resource manager endpoint - OverrideEnvironment string `config:"override_resource_manager_endpoint"` + OverrideEnvironment string `config:"resource_manager_endpoint"` } const ephContainerName = "filebeat" diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml index 557c96664b86..ab7f477b8bb7 100644 --- a/x-pack/filebeat/module/azure/_meta/config.yml +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -13,8 +13,6 @@ storage_account: "" # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" - # by default the azure public environment is used, to override, users can provide a specific resource manager endpoint - override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -24,7 +22,6 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" - # override_resource_manager_endpoint: "" signinlogs: enabled: false # var: @@ -33,4 +30,3 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" - # override_resource_manager_endpoint: "" diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc index fe5b3235ab3d..eea82995532a 100644 --- a/x-pack/filebeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -38,7 +38,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" - override_resource_manager_endpoint: "" + resource_manager_endpoint: "" auditlogs: enabled: false @@ -48,7 +48,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" - override_resource_manager_endpoint: "" + resource_manager_endpoint: "" signinlogs: enabled: false @@ -58,7 +58,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" - override_resource_manager_endpoint: "" + resource_manager_endpoint: "" ``` @@ -88,9 +88,15 @@ The name of the storage account the state/offsets will be stored and updated. _string_ The storage account key, this key will be used to authorize access to data in your storage account. -`override_resource_manager_endpoint` :: +`resource_manager_endpoint` :: _string_ Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. +Ex: +https://management.chinacloudapi.cn/ for azure ChinaCloud +https://management.microsoftazure.de/ for azure GermanCloud +https://management.azure.com/ for azure PublicCloud +https://management.usgovcloudapi.net/ for azure USGovernmentCloud +Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints. include::../include/what-happens.asciidoc[] diff --git a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml index e4f8da96b8a1..9b747e1092d7 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml @@ -4,4 +4,4 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} -override_resource_manager_endpoint: {{ .override_resource_manager_endpoint }} +resource_manager_endpoint: {{ .resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/activitylogs/manifest.yml b/x-pack/filebeat/module/azure/activitylogs/manifest.yml index e7078b075f33..4d5c20a72710 100644 --- a/x-pack/filebeat/module/azure/activitylogs/manifest.yml +++ b/x-pack/filebeat/module/azure/activitylogs/manifest.yml @@ -10,7 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key - - name: override_resource_manager_endpoint + - name: resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml index b01a4d7f8edf..3c2ea50cf8bf 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml @@ -4,5 +4,5 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} -override_resource_manager_endpoint: {{ .override_resource_manager_endpoint }} +resource_manager_endpoint: {{ .resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/auditlogs/manifest.yml b/x-pack/filebeat/module/azure/auditlogs/manifest.yml index 0f193cf6b0db..095371bff16c 100644 --- a/x-pack/filebeat/module/azure/auditlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/auditlogs/manifest.yml @@ -10,7 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key - - name: override_resource_manager_endpoint + - name: resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml index e4f8da96b8a1..9b747e1092d7 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml @@ -4,4 +4,4 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} -override_resource_manager_endpoint: {{ .override_resource_manager_endpoint }} +resource_manager_endpoint: {{ .resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/signinlogs/manifest.yml b/x-pack/filebeat/module/azure/signinlogs/manifest.yml index 7f82837828b9..97fddae51e90 100644 --- a/x-pack/filebeat/module/azure/signinlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/signinlogs/manifest.yml @@ -10,7 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key - - name: override_resource_manager_endpoint + - name: resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled index 25beff244792..0c7eb3d6e010 100644 --- a/x-pack/filebeat/modules.d/azure.yml.disabled +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -16,8 +16,6 @@ storage_account: "" # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" - # by default the azure public environment is used, to override, users can provide a specific resource manager endpoint - override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -27,7 +25,6 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" - # override_resource_manager_endpoint: "" signinlogs: enabled: false # var: @@ -36,4 +33,3 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" - # override_resource_manager_endpoint: "" From 2fafce9521f3c0431314a616b88e063dbfc1caf9 Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 20 Apr 2020 14:14:17 +0200 Subject: [PATCH 4/5] add test --- x-pack/filebeat/input/azureeventhub/eph_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/x-pack/filebeat/input/azureeventhub/eph_test.go b/x-pack/filebeat/input/azureeventhub/eph_test.go index 3a0ac99db7d6..be6db60b9fcb 100644 --- a/x-pack/filebeat/input/azureeventhub/eph_test.go +++ b/x-pack/filebeat/input/azureeventhub/eph_test.go @@ -5,6 +5,7 @@ package azureeventhub import ( + "github.com/Azure/go-autorest/autorest/azure" "testing" "github.com/stretchr/testify/assert" @@ -26,3 +27,17 @@ func TestRunWithEPH(t *testing.T) { err := input.runWithEPH() assert.Error(t, err, '7') } + +func TestGetAzureEnvironment(t *testing.T) { + resMan := "" + env, err := getAzureEnvironment(resMan) + assert.NoError(t, err) + assert.Equal(t, env, azure.PublicCloud) + resMan = "https://management.microsoftazure.de/" + env, err = getAzureEnvironment(resMan) + assert.NoError(t, err) + assert.Equal(t, env, azure.GermanCloud) + resMan = "http://management.invalidhybrid.com/" + env, err = getAzureEnvironment(resMan) + assert.Errorf(t, err, "invalid character 'F' looking for beginning of value") +} From c244182bf5462e3445e7dc7bbe03103d51090c2a Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 20 Apr 2020 15:00:56 +0200 Subject: [PATCH 5/5] mage fmt update --- x-pack/filebeat/input/azureeventhub/eph_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/azureeventhub/eph_test.go b/x-pack/filebeat/input/azureeventhub/eph_test.go index be6db60b9fcb..b48499eb7c4d 100644 --- a/x-pack/filebeat/input/azureeventhub/eph_test.go +++ b/x-pack/filebeat/input/azureeventhub/eph_test.go @@ -5,9 +5,10 @@ package azureeventhub import ( - "github.com/Azure/go-autorest/autorest/azure" "testing" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/stretchr/testify/assert" )