[prometheus] Eliminate labels fingerprint (#9785)

* remove collector labels fingerprint

* remove query labels fingerprint

* remove remote_write labels fingerprint

* update docs

* bump package version

* fix PR link

* bump kibana version

* fingerprint metrics_names for remote_write

* bump minor version

* build readme

* add metrics_names_fingerprint field

* update changelog description

packages/prometheus/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.17.0"
+  changes:
+    - description: Remove labels fingerprint, leading to better TSDB performance and storage reduction.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9785
 - version: "1.16.0"
   changes:
     - description: Add global filter on data_stream.dataset to improve performance.

packages/prometheus/data_stream/collector/fields/fields.yml

@@ -2,14 +2,10 @@
   type: group
   fields:
     - name: labels.*
-      type: object
-      object_type: keyword
-      description: |
-        Prometheus metric labels
-    - name: labels_fingerprint
       type: keyword
       dimension: true
-      description: Autogenerated ID representing the fingerprint of labels object
+      description: |
+        Prometheus metric labels
 - name: prometheus.metrics.*
   type: object
   object_type: double

packages/prometheus/data_stream/query/elasticsearch/ingest_pipeline/default.yml

@@ -8,10 +8,6 @@ processors:
           field: "prometheus.labels.query_name"
           value: "{{_ingest._key}}"
           description: Add query_name as a label to cover cases, when query returns a single value with no labels
-  - fingerprint:
-      fields: ["prometheus.labels"]
-      target_field: "prometheus.labels_fingerprint"
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

packages/prometheus/data_stream/query/fields/fields.yml

@@ -2,14 +2,10 @@
   type: group
   fields:
     - name: labels.*
-      type: object
-      object_type: keyword
-      description: |
-        Prometheus metric labels
-    - name: labels_fingerprint
       type: keyword
       dimension: true
-      description: Autogenerated ID representing the fingerprint of labels object and includes query name
+      description: |
+        Prometheus metric labels
 - name: prometheus.query.*
   type: object
   object_type: double

packages/prometheus/data_stream/remote_write/elasticsearch/ingest_pipeline/default.yml

@@ -23,8 +23,8 @@ processors:
           value: ["{{_ingest._key}}"]
           description: Add all keys of the 'prometheus' object to the earlier created field, it includes all metric names and key "labels"
   - fingerprint:
-      fields: ["prometheus.labels"]
-      target_field: "prometheus.labels_fingerprint"
+      fields: ["prometheus.labels.metrics_names"]
+      target_field: "prometheus.metrics_names_fingerprint"
       ignore_missing: true
   - remove:
       field: "prometheus.labels.metrics_names"

packages/prometheus/data_stream/remote_write/fields/fields.yml

@@ -2,14 +2,14 @@
   type: group
   fields:
     - name: labels.*
-      type: object
-      object_type: keyword
+      type: keyword
+      dimension: true
       description: |
         Prometheus metric labels
-    - name: labels_fingerprint
+    - name: metrics_names_fingerprint
       type: keyword
       dimension: true
-      description: Autogenerated ID representing the fingerprint of all labels and the list of metrics names
+      description: Autogenerated ID representing the fingerprint of the list of metrics names
     - name: metrics.*
       type: object
       object_type: double

packages/prometheus/docs/README.md

@@ -223,8 +223,7 @@ The fields reported are:
 | prometheus.\*.histogram | Prometheus histogram metric | object |  |
 | prometheus.\*.rate | Prometheus rated counter metric | object | gauge |
 | prometheus.\*.value | Prometheus gauge metric | object | gauge |
-| prometheus.labels.\* | Prometheus metric labels | object |  |
-| prometheus.labels_fingerprint | Autogenerated ID representing the fingerprint of labels object | keyword |  |
+| prometheus.labels.\* | Prometheus metric labels | keyword |  |
 | prometheus.metrics.\* | Prometheus metric | object | gauge |
 | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword |  |
 | service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword |  |
@@ -437,9 +436,9 @@ The fields reported are:
 | prometheus.\*.histogram | Prometheus histogram metric | object |  |
 | prometheus.\*.rate | Prometheus rated counter metric | object | gauge |
 | prometheus.\*.value | Prometheus gauge metric | object | gauge |
-| prometheus.labels.\* | Prometheus metric labels | object |  |
-| prometheus.labels_fingerprint | Autogenerated ID representing the fingerprint of all labels and the list of metrics names | keyword |  |
+| prometheus.labels.\* | Prometheus metric labels | keyword |  |
 | prometheus.metrics.\* | Prometheus metric | object | gauge |
+| prometheus.metrics_names_fingerprint | Autogenerated ID representing the fingerprint of the list of metrics names | keyword |  |
 | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword |  |
 | service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword |  |
 
@@ -660,8 +659,7 @@ The fields reported are:
 | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword |  |
 | host.os.version | Operating system version as a raw string. | keyword |  |
 | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |  |
-| prometheus.labels.\* | Prometheus metric labels | object |  |
-| prometheus.labels_fingerprint | Autogenerated ID representing the fingerprint of labels object and includes query name | keyword |  |
+| prometheus.labels.\* | Prometheus metric labels | keyword |  |
 | prometheus.query.\* | Prometheus value resulted from PromQL | object | gauge |
 | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword |  |
 | service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword |  |

packages/prometheus/manifest.yml

@@ -1,15 +1,15 @@
 format_version: 2.10.0
 name: prometheus
 title: Prometheus
-version: 1.16.0
+version: 1.17.0
 description: Collect metrics from Prometheus servers with Elastic Agent.
 type: integration
 categories:
   - observability
   - monitoring
   - containers
 conditions:
-  kibana.version: "^8.12.1"
+  kibana.version: "^8.14.0"
 screenshots:
   - src: /img/prometheus-server-overview.png
     title: Metricbeat Prometheus Overview