Allow routing for integrations that are not input packages (#6340)

* fix system test for kubernetes integration for k8s v1.27.0 * minor changes from formatting * syslong changes * revert a merge conflict * add permissions to selected list of datastreams * update PR id * fix merge conflict with main * Increment minor version instead of bugfix version * added comments for new settings * update activemq version --------- Co-authored-by: Felix Barnsteiner <[email protected]>
elastic · Jun 15, 2023 · b3c3d18 · b3c3d18
1 parent 0f8d402
commit b3c3d18
Show file tree

Hide file tree

Showing 22 changed files with 67 additions and 9 deletions.
diff --git a/packages/activemq/changelog.yml b/packages/activemq/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.10.0"
+  changes:
+    - description: Add permissions to reroute events to logs-*-* for log datastream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6340
 - version: "0.9.2"
   changes:
     - description: Added dimension fields for topic datastream for TSDB enablement

diff --git a/packages/activemq/data_stream/log/manifest.yml b/packages/activemq/data_stream/log/manifest.yml
@@ -39,3 +39,6 @@ streams:
     template_path: log.yml.hbs
     title: ActiveMQ log
     description: Collect ActiveMQ log logs using log input
+# Ensures agents have permissions to write data to `logs-*-*`
+elasticsearch.dynamic_dataset: true
+elasticsearch.dynamic_namespace: true
diff --git a/packages/activemq/manifest.yml b/packages/activemq/manifest.yml
@@ -1,6 +1,6 @@
 name: activemq
 title: ActiveMQ
-version: "0.9.2"
+version: "0.10.0"
 description: Collect logs and metrics from ActiveMQ instances with Elastic Agent.
 type: integration
 icons:

diff --git a/packages/auditd/changelog.yml b/packages/auditd/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.9.0"
+  changes:
+    - description: Add permissions to reroute events to logs-*-* for log datastream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6340
 - version: "3.8.0"
   changes:
     - description: Update package to ECS 8.8.0.

diff --git a/packages/auditd/data_stream/log/manifest.yml b/packages/auditd/data_stream/log/manifest.yml
@@ -39,3 +39,6 @@ streams:
     template_path: log.yml.hbs
     title: Auditd logs
     description: Collect Auditd logs using log input
+# Ensures agents have permissions to write data to `logs-*-*`
+elasticsearch.dynamic_dataset: true
+elasticsearch.dynamic_namespace: true
diff --git a/packages/auditd/manifest.yml b/packages/auditd/manifest.yml
@@ -1,6 +1,6 @@
 name: auditd
 title: Auditd Logs
-version: "3.8.0"
+version: "3.9.0"
 description: Collect logs from Linux audit daemon with Elastic Agent.
 type: integration
 icons:

diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.44.0"
+  changes:
+    - description: Add permissions to reroute events to logs-*-* for cloudwatch_logs and ec2_logs datastream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6340
 - version: "1.43.2"
   changes:
     - description: Add documentation for latency parameter

diff --git a/packages/aws/data_stream/cloudwatch_logs/manifest.yml b/packages/aws/data_stream/cloudwatch_logs/manifest.yml
@@ -196,4 +196,6 @@ streams:
         title: Dataset name
         description: >
           Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html).
-
+# Ensures agents have permissions to write data to `logs-*-*`
+elasticsearch.dynamic_dataset: true
+elasticsearch.dynamic_namespace: true
diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml
@@ -187,3 +187,6 @@ streams:
         type: bool
         multi: false
         default: false
+# Ensures agents have permissions to write data to `logs-*-*`
+elasticsearch.dynamic_dataset: true
+elasticsearch.dynamic_namespace: true
diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: aws
 title: AWS
-version: 1.43.2
+version: 1.44.0
 license: basic
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration

diff --git a/packages/docker/changelog.yml b/packages/docker/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.6.0"
+  changes:
+    - description: Add permissions to reroute events to logs-*-* for container_logs datastream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6340
 - version: "2.5.1"
   changes:
     - description: Set dimension fields to metrics data streams, except for event.

diff --git a/packages/docker/data_stream/container_logs/manifest.yml b/packages/docker/data_stream/container_logs/manifest.yml
@@ -47,3 +47,6 @@ streams:
         show_user: false
         description: >-
           Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+# Ensures agents have permissions to write data to `logs-*-*`
+elasticsearch.dynamic_dataset: true
+elasticsearch.dynamic_namespace: true
diff --git a/packages/docker/manifest.yml b/packages/docker/manifest.yml
@@ -1,6 +1,6 @@
 name: docker
 title: Docker
-version: 2.5.1
+version: 2.6.0
 release: ga
 description: Collect metrics and logs from Docker instances with Elastic Agent.
 type: integration

diff --git a/packages/kafka/changelog.yml b/packages/kafka/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.7.0"
+  changes:
+    - description: Add permissions to reroute events to logs-*-* for log datastream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6340
 - version: "1.6.0"
   changes:
     - description: Rename ownership from obs-service-integrations to obs-infraobs-integrations

diff --git a/packages/kafka/data_stream/log/manifest.yml b/packages/kafka/data_stream/log/manifest.yml
@@ -48,3 +48,6 @@ streams:
     template_path: log.yml.hbs
     title: Kafka log logs (log)
     description: Collect Kafka log logs using log input
+# Ensures agents have permissions to write data to `logs-*-*`
+elasticsearch.dynamic_dataset: true
+elasticsearch.dynamic_namespace: true
diff --git a/packages/kafka/manifest.yml b/packages/kafka/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: kafka
 title: Kafka
-version: "1.6.0"
+version: "1.7.0"
 license: basic
 description: Collect logs and metrics from Kafka servers with Elastic Agent.
 type: integration

diff --git a/packages/kubernetes/changelog.yml b/packages/kubernetes/changelog.yml
@@ -1,12 +1,17 @@
 # newer versions go on top
+- version: "1.42.0"
+  changes:
+    - description: Add permissions to reroute events to logs-*-* for container_logs datastream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6340
 - version: "1.41.0"
   changes:
     - description: Enable time series data streams for the metrics datasets. This improves storage usage and query performance. For more details, see https://www.elastic.co/guide/en/elasticsearch/reference/current/tsds.html
       type: enhancement
       link: https://github.com/elastic/integrations/pull/6332
 - version: "1.40.0"
   changes:
-    - description: Fix system tests for kubernetes integration for k8s v1.27.0 
+    - description: Fix system tests for kubernetes integration for k8s v1.27.0
       type: bugfix
       link: https://github.com/elastic/integrations/pull/6310
 - version: "1.40.0-beta.2"

diff --git a/packages/kubernetes/data_stream/container_logs/manifest.yml b/packages/kubernetes/data_stream/container_logs/manifest.yml
@@ -76,3 +76,6 @@ streams:
 
         type: yaml
         default: ""
+# Ensures agents have permissions to write data to `logs-*-*`
+elasticsearch.dynamic_dataset: true
+elasticsearch.dynamic_namespace: true
diff --git a/packages/kubernetes/manifest.yml b/packages/kubernetes/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: kubernetes
 title: Kubernetes
-version: 1.41.0
+version: 1.42.0
 license: basic
 description: Collect logs and metrics from Kubernetes clusters with Elastic Agent.
 type: integration

diff --git a/packages/system/changelog.yml b/packages/system/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.33.0"
+  changes:
+    - description: Add permissions to reroute events to logs-*-* for syslog datastream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6340
 - version: "1.32.0-beta.2"
   changes:
     - description: Enable TSDS for network data_streams

diff --git a/packages/system/data_stream/syslog/manifest.yml b/packages/system/data_stream/syslog/manifest.yml
@@ -37,3 +37,6 @@ streams:
     template_path: log.yml.hbs
     title: System syslog logs (log)
     description: Collect System syslog logs using log input
+# Ensures agents have permissions to write data to `logs-*-*`
+elasticsearch.dynamic_dataset: true
+elasticsearch.dynamic_namespace: true
diff --git a/packages/system/manifest.yml b/packages/system/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: system
 title: System
-version: 1.32.0-beta.2
+version: 1.33.0
 license: basic
 description: Collect system logs and metrics from your servers with Elastic Agent (TSDB Beta).
 type: integration