From 835c3c7cee4bef9aea55e831541d22803125f292 Mon Sep 17 00:00:00 2001 From: Giuseppe Santoro Date: Tue, 13 Jun 2023 14:28:44 +0100 Subject: [PATCH] Allow routing for integrations that are not input packages (#6340) * fix system test for kubernetes integration for k8s v1.27.0 * minor changes from formatting * syslong changes * revert a merge conflict * add permissions to selected list of datastreams * update PR id * fix merge conflict with main * Increment minor version instead of bugfix version * added comments for new settings * update activemq version --------- Co-authored-by: Felix Barnsteiner --- packages/activemq/changelog.yml | 5 +++++ packages/activemq/data_stream/log/manifest.yml | 3 +++ packages/activemq/manifest.yml | 2 +- packages/auditd/changelog.yml | 5 +++++ packages/auditd/data_stream/log/manifest.yml | 3 +++ packages/auditd/manifest.yml | 2 +- packages/aws/changelog.yml | 5 +++++ packages/aws/data_stream/cloudwatch_logs/manifest.yml | 4 +++- packages/aws/data_stream/ec2_logs/manifest.yml | 3 +++ packages/aws/manifest.yml | 2 +- packages/docker/changelog.yml | 5 +++++ packages/docker/data_stream/container_logs/manifest.yml | 3 +++ packages/docker/manifest.yml | 2 +- packages/kafka/changelog.yml | 5 +++++ packages/kafka/data_stream/log/manifest.yml | 3 +++ packages/kafka/manifest.yml | 2 +- packages/kubernetes/changelog.yml | 7 ++++++- .../kubernetes/data_stream/container_logs/manifest.yml | 3 +++ packages/kubernetes/manifest.yml | 2 +- packages/system/changelog.yml | 5 +++++ packages/system/data_stream/syslog/manifest.yml | 3 +++ packages/system/manifest.yml | 2 +- 22 files changed, 67 insertions(+), 9 deletions(-) diff --git a/packages/activemq/changelog.yml b/packages/activemq/changelog.yml index 97e4816e018..917af247b36 100644 --- a/packages/activemq/changelog.yml +++ b/packages/activemq/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.10.0" + changes: + - description: Add permissions to reroute events to logs-*-* for log datastream + type: enhancement + link: https://github.com/elastic/integrations/pull/6340 - version: "0.9.2" changes: - description: Added dimension fields for topic datastream for TSDB enablement diff --git a/packages/activemq/data_stream/log/manifest.yml b/packages/activemq/data_stream/log/manifest.yml index 7e987a5b24a..b87ac34b72d 100644 --- a/packages/activemq/data_stream/log/manifest.yml +++ b/packages/activemq/data_stream/log/manifest.yml @@ -39,3 +39,6 @@ streams: template_path: log.yml.hbs title: ActiveMQ log description: Collect ActiveMQ log logs using log input +# Ensures agents have permissions to write data to `logs-*-*` +elasticsearch.dynamic_dataset: true +elasticsearch.dynamic_namespace: true diff --git a/packages/activemq/manifest.yml b/packages/activemq/manifest.yml index 49a7a8155db..1ab222815e9 100644 --- a/packages/activemq/manifest.yml +++ b/packages/activemq/manifest.yml @@ -1,6 +1,6 @@ name: activemq title: ActiveMQ -version: "0.9.2" +version: "0.10.0" description: Collect logs and metrics from ActiveMQ instances with Elastic Agent. type: integration icons: diff --git a/packages/auditd/changelog.yml b/packages/auditd/changelog.yml index 7968829d212..e6e20c8207e 100644 --- a/packages/auditd/changelog.yml +++ b/packages/auditd/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.9.0" + changes: + - description: Add permissions to reroute events to logs-*-* for log datastream. + type: enhancement + link: https://github.com/elastic/integrations/pull/6340 - version: "3.8.0" changes: - description: Update package to ECS 8.8.0. diff --git a/packages/auditd/data_stream/log/manifest.yml b/packages/auditd/data_stream/log/manifest.yml index 285926867fe..cd819a63321 100644 --- a/packages/auditd/data_stream/log/manifest.yml +++ b/packages/auditd/data_stream/log/manifest.yml @@ -39,3 +39,6 @@ streams: template_path: log.yml.hbs title: Auditd logs description: Collect Auditd logs using log input +# Ensures agents have permissions to write data to `logs-*-*` +elasticsearch.dynamic_dataset: true +elasticsearch.dynamic_namespace: true diff --git a/packages/auditd/manifest.yml b/packages/auditd/manifest.yml index 9121830d2aa..69e192d9f5f 100644 --- a/packages/auditd/manifest.yml +++ b/packages/auditd/manifest.yml @@ -1,6 +1,6 @@ name: auditd title: Auditd Logs -version: "3.8.0" +version: "3.9.0" description: Collect logs from Linux audit daemon with Elastic Agent. type: integration icons: diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index e2f6e888065..70da1d63dca 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.44.0" + changes: + - description: Add permissions to reroute events to logs-*-* for cloudwatch_logs and ec2_logs datastream. + type: enhancement + link: https://github.com/elastic/integrations/pull/6340 - version: "1.43.2" changes: - description: Add documentation for latency parameter diff --git a/packages/aws/data_stream/cloudwatch_logs/manifest.yml b/packages/aws/data_stream/cloudwatch_logs/manifest.yml index 04bd2936e3a..2eb4e4266e5 100644 --- a/packages/aws/data_stream/cloudwatch_logs/manifest.yml +++ b/packages/aws/data_stream/cloudwatch_logs/manifest.yml @@ -196,4 +196,6 @@ streams: title: Dataset name description: > Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). - +# Ensures agents have permissions to write data to `logs-*-*` +elasticsearch.dynamic_dataset: true +elasticsearch.dynamic_namespace: true diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml index 8f1bb76d097..a47435b294f 100644 --- a/packages/aws/data_stream/ec2_logs/manifest.yml +++ b/packages/aws/data_stream/ec2_logs/manifest.yml @@ -187,3 +187,6 @@ streams: type: bool multi: false default: false +# Ensures agents have permissions to write data to `logs-*-*` +elasticsearch.dynamic_dataset: true +elasticsearch.dynamic_namespace: true diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 2fe2ea375ee..2952211c6fa 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 1.43.2 +version: 1.44.0 license: basic description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration diff --git a/packages/docker/changelog.yml b/packages/docker/changelog.yml index ab10408ce0c..0c65088b474 100644 --- a/packages/docker/changelog.yml +++ b/packages/docker/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.6.0" + changes: + - description: Add permissions to reroute events to logs-*-* for container_logs datastream + type: enhancement + link: https://github.com/elastic/integrations/pull/6340 - version: "2.5.1" changes: - description: Set dimension fields to metrics data streams, except for event. diff --git a/packages/docker/data_stream/container_logs/manifest.yml b/packages/docker/data_stream/container_logs/manifest.yml index b7d749b71b0..3b7e5531f19 100644 --- a/packages/docker/data_stream/container_logs/manifest.yml +++ b/packages/docker/data_stream/container_logs/manifest.yml @@ -47,3 +47,6 @@ streams: show_user: false description: >- Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. +# Ensures agents have permissions to write data to `logs-*-*` +elasticsearch.dynamic_dataset: true +elasticsearch.dynamic_namespace: true diff --git a/packages/docker/manifest.yml b/packages/docker/manifest.yml index 85d9d281c91..3ccaa718df0 100644 --- a/packages/docker/manifest.yml +++ b/packages/docker/manifest.yml @@ -1,6 +1,6 @@ name: docker title: Docker -version: 2.5.1 +version: 2.6.0 release: ga description: Collect metrics and logs from Docker instances with Elastic Agent. type: integration diff --git a/packages/kafka/changelog.yml b/packages/kafka/changelog.yml index 7d28f483743..20981a0310b 100644 --- a/packages/kafka/changelog.yml +++ b/packages/kafka/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.7.0" + changes: + - description: Add permissions to reroute events to logs-*-* for log datastream + type: enhancement + link: https://github.com/elastic/integrations/pull/6340 - version: "1.6.0" changes: - description: Rename ownership from obs-service-integrations to obs-infraobs-integrations diff --git a/packages/kafka/data_stream/log/manifest.yml b/packages/kafka/data_stream/log/manifest.yml index b5c67cf4819..cd4e39e0eca 100644 --- a/packages/kafka/data_stream/log/manifest.yml +++ b/packages/kafka/data_stream/log/manifest.yml @@ -48,3 +48,6 @@ streams: template_path: log.yml.hbs title: Kafka log logs (log) description: Collect Kafka log logs using log input +# Ensures agents have permissions to write data to `logs-*-*` +elasticsearch.dynamic_dataset: true +elasticsearch.dynamic_namespace: true \ No newline at end of file diff --git a/packages/kafka/manifest.yml b/packages/kafka/manifest.yml index bdc3ee4140f..86be01636f1 100644 --- a/packages/kafka/manifest.yml +++ b/packages/kafka/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: kafka title: Kafka -version: "1.6.0" +version: "1.7.0" license: basic description: Collect logs and metrics from Kafka servers with Elastic Agent. type: integration diff --git a/packages/kubernetes/changelog.yml b/packages/kubernetes/changelog.yml index e2f53b6e51c..18117d3ecfa 100644 --- a/packages/kubernetes/changelog.yml +++ b/packages/kubernetes/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.42.0" + changes: + - description: Add permissions to reroute events to logs-*-* for container_logs datastream + type: enhancement + link: https://github.com/elastic/integrations/pull/6340 - version: "1.41.0" changes: - description: Enable time series data streams for the metrics datasets. This improves storage usage and query performance. For more details, see https://www.elastic.co/guide/en/elasticsearch/reference/current/tsds.html @@ -6,7 +11,7 @@ link: https://github.com/elastic/integrations/pull/6332 - version: "1.40.0" changes: - - description: Fix system tests for kubernetes integration for k8s v1.27.0 + - description: Fix system tests for kubernetes integration for k8s v1.27.0 type: bugfix link: https://github.com/elastic/integrations/pull/6310 - version: "1.40.0-beta.2" diff --git a/packages/kubernetes/data_stream/container_logs/manifest.yml b/packages/kubernetes/data_stream/container_logs/manifest.yml index 0f15117c872..22a687d180b 100644 --- a/packages/kubernetes/data_stream/container_logs/manifest.yml +++ b/packages/kubernetes/data_stream/container_logs/manifest.yml @@ -76,3 +76,6 @@ streams: type: yaml default: "" +# Ensures agents have permissions to write data to `logs-*-*` +elasticsearch.dynamic_dataset: true +elasticsearch.dynamic_namespace: true \ No newline at end of file diff --git a/packages/kubernetes/manifest.yml b/packages/kubernetes/manifest.yml index e723e2ff4e2..173bc702634 100644 --- a/packages/kubernetes/manifest.yml +++ b/packages/kubernetes/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: kubernetes title: Kubernetes -version: 1.41.0 +version: 1.42.0 license: basic description: Collect logs and metrics from Kubernetes clusters with Elastic Agent. type: integration diff --git a/packages/system/changelog.yml b/packages/system/changelog.yml index f73e617d27b..d6d69ce0813 100644 --- a/packages/system/changelog.yml +++ b/packages/system/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.33.0" + changes: + - description: Add permissions to reroute events to logs-*-* for syslog datastream + type: enhancement + link: https://github.com/elastic/integrations/pull/6340 - version: "1.32.0-beta.2" changes: - description: Enable TSDS for network data_streams diff --git a/packages/system/data_stream/syslog/manifest.yml b/packages/system/data_stream/syslog/manifest.yml index 185db44694a..f7374ee8f52 100644 --- a/packages/system/data_stream/syslog/manifest.yml +++ b/packages/system/data_stream/syslog/manifest.yml @@ -37,3 +37,6 @@ streams: template_path: log.yml.hbs title: System syslog logs (log) description: Collect System syslog logs using log input +# Ensures agents have permissions to write data to `logs-*-*` +elasticsearch.dynamic_dataset: true +elasticsearch.dynamic_namespace: true diff --git a/packages/system/manifest.yml b/packages/system/manifest.yml index 1d603febb86..5525fdd27c3 100644 --- a/packages/system/manifest.yml +++ b/packages/system/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: system title: System -version: 1.32.0-beta.2 +version: 1.33.0 license: basic description: Collect system logs and metrics from your servers with Elastic Agent (TSDB Beta). type: integration