Add new fields, delete deprecated ones and update dashboards for apiserver and controllermanager

packages/kubernetes/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.22.0"
+  changes:
+    - description: Update apiserver and controllermanaged deprecated fields and dashboards
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3825
 - version: "1.21.2"
   changes:
     - description: add container ID and pod name as part of Kubernetes Cotainer Logs filestream input

packages/kubernetes/data_stream/apiserver/fields/fields.yml

@@ -1,11 +1,6 @@
 - name: kubernetes.apiserver
   type: group
   fields:
-    - name: request.client
-      dimension: true
-      type: keyword
-      description: |
-        Client executing requests
     - name: request.resource
       dimension: true
       type: keyword
@@ -101,53 +96,48 @@
           type: double
           metric_type: gauge
           description: Seconds since the process started
-    - name: http
+    - name: watch
       type: group
       fields:
-        - name: request.duration.us.percentile.*
+        - name: events.size.bytes.bucket.*
           type: object
-          description: Request duration microseconds percentiles
-        - name: request.duration.us.sum
-          type: double
-          metric_type: counter
-          unit: micros
-          description: Request duration microseconds cumulative sum
-        - name: request.duration.us.count
-          type: long
-          metric_type: counter
-          unit: micros
-          description: Request count for duration
-        - name: request.size.bytes.percentile.*
-          type: object
-          description: Request size percentiles
-        - name: request.size.bytes.sum
+          object_type: long
+          description: Watch event size distribution in bytes
+        - name: events.size.bytes.sum
           type: long
           format: bytes
           unit: byte
           metric_type: counter
-          description: Request size cumulative sum
-        - name: request.size.bytes.count
+          description: Sum of watch events sizes in bytes
+        - name: events.size.bytes.count
           type: long
-          unit: byte
           metric_type: counter
-          description: Request count for size
-        - name: response.size.bytes.percentile.*
+          description: Number of watch events
+        - name: events.kind
+          type: keyword
+          dimension: true
+          description: >
+            Resource kind of the watch event
+
+    - name: response
+      type: group
+      fields:
+        - name: size.bytes.bucket.*
           type: object
-          description: Response size percentiles
-        - name: response.size.bytes.sum
+          object_type: long
+          description: >
+            Response size distribution in bytes for each group, version, verb, resource, subresource, scope and component.
+
+        - name: size.bytes.sum
           type: long
           format: bytes
           unit: byte
           metric_type: counter
-          description: Response size cumulative sum
-        - name: response.size.bytes.count
+          description: Sum of responses sizes in bytes
+        - name: size.bytes.count
           type: long
           metric_type: counter
-          description: Response count
-        - name: request.count
-          type: long
-          metric_type: counter
-          description: Request count for response
+          description: Number of responses to requests
     - name: client.request.count
       type: long
       metric_type: counter
@@ -159,17 +149,6 @@
           type: long
           metric_type: counter
           description: Number of requests
-        - name: latency.sum
-          type: long
-          metric_type: counter
-          description: Requests latency, sum of latencies in microseconds
-        - name: latency.count
-          type: long
-          metric_type: counter
-          description: Request latency, number of requests
-        - name: latency.bucket.*
-          type: object
-          description: Request latency histogram buckets
         - name: duration.us.sum
           type: long
           metric_type: counter

packages/kubernetes/data_stream/controllermanager/fields/fields.yml

@@ -1,11 +1,18 @@
 - name: kubernetes.controllermanager
   type: group
   fields:
-    - name: handler
+    - name: url
       dimension: true
       type: keyword
-      description: |
-        Request handler
+      description: >
+        Request url
+
+    - name: verb
+      dimension: true
+      type: keyword
+      description: >
+        Request verb
+
     - name: code
       dimension: true
       type: keyword
@@ -54,63 +61,35 @@
           type: long
           metric_type: gauge
           description: Number of open file descriptors
+        - name: fds.max.count
+          type: long
+          metric_type: gauge
+          description: Maximum number of open file descriptors
         - name: started.sec
           type: double
           metric_type: gauge
           description: Seconds since the process started
-    - name: http
+    - name: client
       type: group
       fields:
-        - name: request.duration.us.percentile.*
-          type: object
-          description: Request duration microseconds percentiles
-        - name: request.duration.us.sum
-          type: double
-          unit: micros
-          metric_type: counter
-          description: Request duration microseconds cumulative sum
-        - name: request.duration.us.count
-          type: long
-          unit: micros
-          metric_type: counter
-          description: Request count for duration
-        - name: request.size.bytes.percentile.*
-          type: object
-          description: Request size percentiles
-        - name: request.size.bytes.sum
-          type: long
-          format: bytes
-          unit: byte
-          metric_type: counter
-          description: Request size cumulative sum
-        - name: request.size.bytes.count
+        - name: request.count
           type: long
-          unit: byte
           metric_type: counter
-          description: Request count for size
-        - name: response.size.bytes.percentile.*
+          description: >
+            Number of requests as client
+
+        - name: request.duration.sec.bucket.*
           type: object
-          description: Response size percentiles
-        - name: response.size.bytes.sum
+          object_type: long
+          description: Response latency distribution, histogram buckets
+        - name: request.duration.sec.sum
           type: long
-          format: bytes
-          unit: byte
-          metric_type: counter
-          description: Response size cumulative sum
-        - name: response.size.bytes.count
-          type: long
-          unit: byte
           metric_type: counter
-          description: Response count
-        - name: request.count
+          description: Request duration, sum in seconds
+        - name: request.duration.sec.count
           type: long
           metric_type: counter
-          description: Request count for response
-    - name: client.request.count
-      type: long
-      metric_type: counter
-      description: |
-        Number of requests as client
+          description: Request duration, number of operations
     - name: workqueue
       type: group
       fields:
@@ -134,6 +113,18 @@
           type: long
           metric_type: counter
           description: Workqueue number of retries
+        - name: work.duration.sec.sum
+          type: long
+          metric_type: counter
+          description: Processing time of workqueue item, sum in seconds
+        - name: work.duration.sec.count
+          type: long
+          metric_type: counter
+          description: Processing time of workqueue item, number of operations
+        - name: work.duration.sec.bucket.*
+          type: object
+          object_type: long
+          description: Processing time of workqueue item, histogram buckets
     - name: node.collector
       type: group
       fields:

packages/kubernetes/docs/kube-apiserver.md

@@ -142,22 +142,11 @@ An example event for `apiserver` looks as following:
 | kubernetes.apiserver.audit.rejected.count | Number of audit rejected events | long |  | counter |
 | kubernetes.apiserver.client.request.count | Number of requests as client | long |  | counter |
 | kubernetes.apiserver.etcd.object.count | Number of kubernetes objects at etcd | long |  | gauge |
-| kubernetes.apiserver.http.request.count | Request count for response | long |  | counter |
-| kubernetes.apiserver.http.request.duration.us.count | Request count for duration | long | micros | counter |
-| kubernetes.apiserver.http.request.duration.us.percentile.\* | Request duration microseconds percentiles | object |  |  |
-| kubernetes.apiserver.http.request.duration.us.sum | Request duration microseconds cumulative sum | double | micros | counter |
-| kubernetes.apiserver.http.request.size.bytes.count | Request count for size | long | byte | counter |
-| kubernetes.apiserver.http.request.size.bytes.percentile.\* | Request size percentiles | object |  |  |
-| kubernetes.apiserver.http.request.size.bytes.sum | Request size cumulative sum | long | byte | counter |
-| kubernetes.apiserver.http.response.size.bytes.count | Response count | long |  | counter |
-| kubernetes.apiserver.http.response.size.bytes.percentile.\* | Response size percentiles | object |  |  |
-| kubernetes.apiserver.http.response.size.bytes.sum | Response size cumulative sum | long | byte | counter |
 | kubernetes.apiserver.process.cpu.sec | CPU seconds | double |  | counter |
 | kubernetes.apiserver.process.fds.open.count | Number of open file descriptors | long |  | gauge |
 | kubernetes.apiserver.process.memory.resident.bytes | Bytes in resident memory | long | byte | gauge |
 | kubernetes.apiserver.process.memory.virtual.bytes | Bytes in virtual memory | long | byte | gauge |
 | kubernetes.apiserver.process.started.sec | Seconds since the process started | double |  | gauge |
-| kubernetes.apiserver.request.client | Client executing requests | keyword |  |  |
 | kubernetes.apiserver.request.code | HTTP code | keyword |  |  |
 | kubernetes.apiserver.request.component | Component handling the request | keyword |  |  |
 | kubernetes.apiserver.request.content_type | Request HTTP content type | keyword |  |  |
@@ -171,16 +160,20 @@ An example event for `apiserver` looks as following:
 | kubernetes.apiserver.request.handler | Request handler | keyword |  |  |
 | kubernetes.apiserver.request.host | Request host | keyword |  |  |
 | kubernetes.apiserver.request.kind | Kind of request | keyword |  |  |
-| kubernetes.apiserver.request.latency.bucket.\* | Request latency histogram buckets | object |  |  |
-| kubernetes.apiserver.request.latency.count | Request latency, number of requests | long |  | counter |
-| kubernetes.apiserver.request.latency.sum | Requests latency, sum of latencies in microseconds | long |  | counter |
 | kubernetes.apiserver.request.longrunning.count | Number of requests active long running requests | long |  | counter |
 | kubernetes.apiserver.request.method | HTTP method | keyword |  |  |
 | kubernetes.apiserver.request.resource | Requested resource | keyword |  |  |
 | kubernetes.apiserver.request.scope | Request scope (cluster, namespace, resource) | keyword |  |  |
 | kubernetes.apiserver.request.subresource | Requested subresource | keyword |  |  |
 | kubernetes.apiserver.request.verb | HTTP verb | keyword |  |  |
 | kubernetes.apiserver.request.version | version for the group | keyword |  |  |
+| kubernetes.apiserver.response.size.bytes.bucket.\* | Response size distribution in bytes for each group, version, verb, resource, subresource, scope and component. | object |  |  |
+| kubernetes.apiserver.response.size.bytes.count | Number of responses to requests | long |  | counter |
+| kubernetes.apiserver.response.size.bytes.sum | Sum of responses sizes in bytes | long | byte | counter |
+| kubernetes.apiserver.watch.events.kind | Resource kind of the watch event | keyword |  |  |
+| kubernetes.apiserver.watch.events.size.bytes.bucket.\* | Watch event size distribution in bytes | object |  |  |
+| kubernetes.apiserver.watch.events.size.bytes.count | Number of watch events | long |  | counter |
+| kubernetes.apiserver.watch.events.size.bytes.sum | Sum of watch events sizes in bytes | long | byte | counter |
 | orchestrator.cluster.name | Name of the cluster. | keyword |  |  |
 | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword |  |  |
 | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword |  |  |

packages/kubernetes/docs/kube-controller-manager.md

@@ -148,19 +148,11 @@ An example event for `controllermanager` looks as following:
 | kubernetes.container.image | Kubernetes container image | keyword |  |  |
 | kubernetes.container.name | Kubernetes container name | keyword |  |  |
 | kubernetes.controllermanager.client.request.count | Number of requests as client | long |  | counter |
+| kubernetes.controllermanager.client.request.duration.sec.bucket.\* | Response latency distribution, histogram buckets | object |  |  |
+| kubernetes.controllermanager.client.request.duration.sec.count | Request duration, number of operations | long |  | counter |
+| kubernetes.controllermanager.client.request.duration.sec.sum | Request duration, sum in seconds | long |  | counter |
 | kubernetes.controllermanager.code | HTTP code | keyword |  |  |
-| kubernetes.controllermanager.handler | Request handler | keyword |  |  |
 | kubernetes.controllermanager.host | Request host | keyword |  |  |
-| kubernetes.controllermanager.http.request.count | Request count for response | long |  | counter |
-| kubernetes.controllermanager.http.request.duration.us.count | Request count for duration | long | micros | counter |
-| kubernetes.controllermanager.http.request.duration.us.percentile.\* | Request duration microseconds percentiles | object |  |  |
-| kubernetes.controllermanager.http.request.duration.us.sum | Request duration microseconds cumulative sum | double | micros | counter |
-| kubernetes.controllermanager.http.request.size.bytes.count | Request count for size | long | byte | counter |
-| kubernetes.controllermanager.http.request.size.bytes.percentile.\* | Request size percentiles | object |  |  |
-| kubernetes.controllermanager.http.request.size.bytes.sum | Request size cumulative sum | long | byte | counter |
-| kubernetes.controllermanager.http.response.size.bytes.count | Response count | long | byte | counter |
-| kubernetes.controllermanager.http.response.size.bytes.percentile.\* | Response size percentiles | object |  |  |
-| kubernetes.controllermanager.http.response.size.bytes.sum | Response size cumulative sum | long | byte | counter |
 | kubernetes.controllermanager.leader.is_master | Whether the node is master | boolean |  |  |
 | kubernetes.controllermanager.method | HTTP method | keyword |  |  |
 | kubernetes.controllermanager.name | Name for the resource | keyword |  |  |
@@ -169,15 +161,21 @@ An example event for `controllermanager` looks as following:
 | kubernetes.controllermanager.node.collector.health.pct | Percentage of healthy nodes | long |  | gauge |
 | kubernetes.controllermanager.node.collector.unhealthy.count | Number of unhealthy nodes | long |  | gauge |
 | kubernetes.controllermanager.process.cpu.sec | CPU seconds | double |  | counter |
+| kubernetes.controllermanager.process.fds.max.count | Maximum number of open file descriptors | long |  | gauge |
 | kubernetes.controllermanager.process.fds.open.count | Number of open file descriptors | long |  | gauge |
 | kubernetes.controllermanager.process.memory.resident.bytes | Bytes in resident memory | long | byte | gauge |
 | kubernetes.controllermanager.process.memory.virtual.bytes | Bytes in virtual memory | long | byte | gauge |
 | kubernetes.controllermanager.process.started.sec | Seconds since the process started | double |  | gauge |
+| kubernetes.controllermanager.url | Request url | keyword |  |  |
+| kubernetes.controllermanager.verb | Request verb | keyword |  |  |
 | kubernetes.controllermanager.workqueue.adds.count | Workqueue add count | long |  | counter |
 | kubernetes.controllermanager.workqueue.depth.count | Workqueue depth count | long |  | gauge |
 | kubernetes.controllermanager.workqueue.longestrunning.sec | Longest running processors | double |  | gauge |
 | kubernetes.controllermanager.workqueue.retries.count | Workqueue number of retries | long |  | counter |
 | kubernetes.controllermanager.workqueue.unfinished.sec | Unfinished processors | double |  | gauge |
+| kubernetes.controllermanager.workqueue.work.duration.sec.bucket.\* | Processing time of workqueue item, histogram buckets | object |  |  |
+| kubernetes.controllermanager.workqueue.work.duration.sec.count | Processing time of workqueue item, number of operations | long |  | counter |
+| kubernetes.controllermanager.workqueue.work.duration.sec.sum | Processing time of workqueue item, sum in seconds | long |  | counter |
 | kubernetes.controllermanager.zone | Infrastructure zone | keyword |  |  |
 | kubernetes.deployment.name | Kubernetes deployment name | keyword |  |  |
 | kubernetes.labels.\* | Kubernetes labels map | object |  |  |