Added dimension fields for partition datastream

packages/kafka/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.5.5"
+  changes:
+    - description: Add  dimension fields for partition datastream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/5962
 - version: "1.5.4"
   changes:
     - description: Add  dimension mapping for consumergroup datastream.

packages/kafka/data_stream/partition/fields/agent.yml

@@ -25,6 +25,7 @@
       ignore_above: 1024
       description: Instance ID of the host machine.
       example: i-1234567890abcdef0
+      dimension: true
     - name: instance.name
       level: extended
       type: keyword
@@ -42,6 +43,7 @@
       ignore_above: 1024
       description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
       example: aws
+      dimension: true
     - name: region
       level: extended
       type: keyword
@@ -51,6 +53,7 @@
     - name: project.id
       type: keyword
       description: Name of the project in Google Cloud.
+      dimension: true
     - name: image.id
       type: keyword
       description: Image ID for the cloud instance.
@@ -67,6 +70,7 @@
       type: keyword
       ignore_above: 1024
       description: Unique container id.
+      dimension: true
     - name: image.name
       level: extended
       type: keyword
@@ -134,6 +138,7 @@
       level: core
       type: keyword
       ignore_above: 1024
+      dimension: true
       description: 'Name of the host.
 
         It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.'
@@ -196,3 +201,11 @@
       description: >
         OS codename, if any.
 
+- name: agent
+  title: Agent
+  type: group
+  fields:
+    - name: id
+      type: keyword
+      ignore_above: 1024
+      dimension: true

packages/kafka/data_stream/partition/fields/ecs.yml

@@ -2,6 +2,7 @@
   name: ecs.version
 - external: ecs
   name: service.address
+  dimension: true
 - external: ecs
   name: service.type
 - external: ecs

packages/kafka/data_stream/partition/fields/package-fields.yml

@@ -10,10 +10,14 @@
             Broker id
         - name: address
           type: keyword
+          #Reason to add as dimension field: Multiple brokers may exist in a Kafka cluster.
+          dimension: true
           description: |
             Broker advertised address
     - name: topic.name
       type: keyword
+      #Reason to add as dimension field: Multiple values of topics exist. 
+      dimension: true
       description: |
         Topic name
     - name: topic.error.code
@@ -26,7 +30,11 @@
         Partition id.
     - name: partition.topic_id
       type: keyword
+      #Reason to add as dimension field: Multiple records exist for the same kafka.partition.id, kafka.topic.name. Observed kafka.partition.topic_id is formed by combining kafka.partition.id with kafka.topic.name.
+      dimension: true
       description: Unique id of the partition in the topic.
     - name: partition.topic_broker_id
       type: keyword
+      #Reason to add as dimension field: For future use. 
+      dimension: true
       description: Unique id of the partition in the topic and the broker.

packages/kafka/docs/README.md

@@ -381,6 +381,7 @@ An example event for `partition` looks as following:
 | Field | Description | Type | Metric Type |
 |---|---|---|---|
 | @timestamp | Event timestamp. | date |  |
+| agent.id |  | keyword |  |
 | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword |  |
 | cloud.availability_zone | Availability zone in which this host is running. | keyword |  |
 | cloud.image.id | Image ID for the cloud instance. | keyword |  |

packages/kafka/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: kafka
 title: Kafka
-version: 1.5.4
+version: 1.5.5
 license: basic
 description: Collect logs and metrics from Kafka servers with Elastic Agent.
 type: integration