From 2004252e5eed6d37b2614d13a48d35cb6acd8a83 Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Wed, 28 Aug 2024 11:44:46 +0300 Subject: [PATCH 01/28] update event --- .../data_stream/datastore/sample_event.json | 94 ++++++++----------- 1 file changed, 40 insertions(+), 54 deletions(-) diff --git a/packages/vsphere/data_stream/datastore/sample_event.json b/packages/vsphere/data_stream/datastore/sample_event.json index ed9113d9b8c..609fa779b2f 100644 --- a/packages/vsphere/data_stream/datastore/sample_event.json +++ b/packages/vsphere/data_stream/datastore/sample_event.json @@ -1,78 +1,64 @@ { - "@timestamp": "2023-06-29T08:03:30.114Z", - "agent": { - "ephemeral_id": "8b019ff3-cbda-41fa-b1ff-974d482b9694", - "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", - "name": "docker-fleet-agent", - "type": "metricbeat", - "version": "8.8.1" - }, - "data_stream": { - "dataset": "vsphere.datastore", - "namespace": "ep", - "type": "metrics" - }, - "ecs": { - "version": "8.11.0" - }, - "elastic_agent": { - "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", - "snapshot": false, - "version": "8.8.1" - }, + "@timestamp": "2017-10-12T08:05:34.853Z", "event": { - "agent_id_status": "verified", "dataset": "vsphere.datastore", - "duration": 23155458, - "ingested": "2023-06-29T08:03:31Z", + "duration": 115000, "module": "vsphere" }, - "host": { - "architecture": "aarch64", - "containerized": false, - "hostname": "docker-fleet-agent", - "id": "d08b346fbb8f49f5a2bb1a477f8ceb54", - "ip": [ - "172.23.0.7" - ], - "mac": [ - "02-42-AC-17-00-07" - ], - "name": "docker-fleet-agent", - "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.10.104-linuxkit", - "name": "Ubuntu", - "platform": "ubuntu", - "type": "linux", - "version": "20.04.6 LTS (Focal Fossa)" - } - }, "metricset": { "name": "datastore", "period": 10000 }, "service": { - "address": "https://elastic-package-service_vsphere-metrics_1:8989/sdk", + "address": "127.0.0.1:33365", "type": "vsphere" }, "vsphere": { "datastore": { + "iops": 0, + "host": { + "count": 1, + "names": [ + "DC3_H0" + ] + }, + "status": "green", + "vm": { + "count": 6, + "names": [ + "DC3_H0_VM0" + ] + }, "capacity": { "free": { - "bytes": 47869427712 + "bytes": 37120094208 }, "total": { - "bytes": 62725623808 + "bytes": 74686664704 }, "used": { - "bytes": 14856196096, - "pct": 0.237 - } + "bytes": 37566570496, + "pct": 0.502988996026061 + }, + "read": { + "bytes": 0, + "latency": { + "total": { + "ms": 0 + } + } + }, + "write": { + "bytes": 337000, + "latency": { + "total": { + "ms": 0 + } + } + } }, - "fstype": "OTHER", + "fstype": "local", "name": "LocalDS_0" } } -} \ No newline at end of file + } \ No newline at end of file From a2c25910350fa1f0cea66c05f12ca175e35dcea3 Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Wed, 28 Aug 2024 11:46:40 +0300 Subject: [PATCH 02/28] update fields.yml --- .../data_stream/datastore/fields/fields.yml | 62 +++++++++++++++++-- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/packages/vsphere/data_stream/datastore/fields/fields.yml b/packages/vsphere/data_stream/datastore/fields/fields.yml index d92231a60fe..e687ed7d634 100644 --- a/packages/vsphere/data_stream/datastore/fields/fields.yml +++ b/packages/vsphere/data_stream/datastore/fields/fields.yml @@ -6,33 +6,83 @@ type: keyword # Reason to add as a dimension field: to uniquely identify the datastore dimension: true - description: Datastore name + description: Datastore name. - name: fstype type: keyword # Reason to add as a dimension field: in case datastore name is duplicated across different type. dimension: true - description: Filesystem type + description: Filesystem type. - name: capacity.total.bytes type: long metric_type: gauge unit: byte - description: Total bytes of the datastore + description: Total bytes of the datastore. format: bytes - name: capacity.free.bytes type: long metric_type: gauge unit: byte - description: Free bytes of the datastore + description: Free bytes of the datastore. format: bytes - name: capacity.used.bytes type: long metric_type: gauge unit: byte - description: Used bytes of the datastore + description: Used bytes of the datastore. format: bytes - name: capacity.used.pct type: scaled_float metric_type: gauge unit: percent - description: Used percent of the datastore + description: Percentage of datastore capacity used. format: percent + - name: fstype + type: keyword + description: > + Filesystem type. + - name: host.count + type: long + description: > + Number of hosts. + - name: host.names + type: keyword + description: > + List of all the host names. + - name: iops + type: long + description: > + Storage I/O Control aggregated Input/Output Operations Per Second. + - name: name + type: keyword + description: > + Datastore name. + - name: read.bytes + type: long + description: > + Rate of reading data from the datastore. + format: bytes + - name: read.latency.total.ms + type: long + description: > + Average amount of time for a read operation from the datastore in milliseconds. + - name: status + type: keyword + description: > + Status of the datastore. + - name: vm.count + type: long + description: > + Number of VMs. + - name: vm.names + type: keyword + description: > + List of all the VM names. + - name: write.bytes + type: long + description: > + Rate of writing data to the datastore. + format: bytes + - name: write.latency.total.ms + type: long + description: > + Average amount of time for a write operation from the datastore in milliseconds. From d81254b170c173a793d75f2357b2bc960c079e03 Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Wed, 28 Aug 2024 12:02:34 +0300 Subject: [PATCH 03/28] add changelog entry --- packages/vsphere/changelog.yml | 5 + .../data_stream/datastore/fields/fields.yml | 8 -- packages/vsphere/docs/README.md | 112 +++++++++--------- packages/vsphere/manifest.yml | 2 +- 4 files changed, 60 insertions(+), 67 deletions(-) diff --git a/packages/vsphere/changelog.yml b/packages/vsphere/changelog.yml index 3ae4efab7dc..48fab3be924 100644 --- a/packages/vsphere/changelog.yml +++ b/packages/vsphere/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.14.0" + changes: + - description: Add additional metrics for datastore datastream + type: enhancement + link: https://github.com/elastic/integrations/pull/10911 - version: "1.13.0" changes: - description: ECS version updated to 8.11.0. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. diff --git a/packages/vsphere/data_stream/datastore/fields/fields.yml b/packages/vsphere/data_stream/datastore/fields/fields.yml index e687ed7d634..38e295f1842 100644 --- a/packages/vsphere/data_stream/datastore/fields/fields.yml +++ b/packages/vsphere/data_stream/datastore/fields/fields.yml @@ -36,10 +36,6 @@ unit: percent description: Percentage of datastore capacity used. format: percent - - name: fstype - type: keyword - description: > - Filesystem type. - name: host.count type: long description: > @@ -52,10 +48,6 @@ type: long description: > Storage I/O Control aggregated Input/Output Operations Per Second. - - name: name - type: keyword - description: > - Datastore name. - name: read.bytes type: long description: > diff --git a/packages/vsphere/docs/README.md b/packages/vsphere/docs/README.md index 741fad087f5..4052e1631b8 100644 --- a/packages/vsphere/docs/README.md +++ b/packages/vsphere/docs/README.md @@ -299,79 +299,65 @@ An example event for `datastore` looks as following: ```json { - "@timestamp": "2023-06-29T08:03:30.114Z", - "agent": { - "ephemeral_id": "8b019ff3-cbda-41fa-b1ff-974d482b9694", - "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", - "name": "docker-fleet-agent", - "type": "metricbeat", - "version": "8.8.1" - }, - "data_stream": { - "dataset": "vsphere.datastore", - "namespace": "ep", - "type": "metrics" - }, - "ecs": { - "version": "8.11.0" - }, - "elastic_agent": { - "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", - "snapshot": false, - "version": "8.8.1" - }, + "@timestamp": "2017-10-12T08:05:34.853Z", "event": { - "agent_id_status": "verified", "dataset": "vsphere.datastore", - "duration": 23155458, - "ingested": "2023-06-29T08:03:31Z", + "duration": 115000, "module": "vsphere" }, - "host": { - "architecture": "aarch64", - "containerized": false, - "hostname": "docker-fleet-agent", - "id": "d08b346fbb8f49f5a2bb1a477f8ceb54", - "ip": [ - "172.23.0.7" - ], - "mac": [ - "02-42-AC-17-00-07" - ], - "name": "docker-fleet-agent", - "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.10.104-linuxkit", - "name": "Ubuntu", - "platform": "ubuntu", - "type": "linux", - "version": "20.04.6 LTS (Focal Fossa)" - } - }, "metricset": { "name": "datastore", "period": 10000 }, "service": { - "address": "https://elastic-package-service_vsphere-metrics_1:8989/sdk", + "address": "127.0.0.1:33365", "type": "vsphere" }, "vsphere": { "datastore": { + "iops": 0, + "host": { + "count": 1, + "names": [ + "DC3_H0" + ] + }, + "status": "green", + "vm": { + "count": 6, + "names": [ + "DC3_H0_VM0" + ] + }, "capacity": { "free": { - "bytes": 47869427712 + "bytes": 37120094208 }, "total": { - "bytes": 62725623808 + "bytes": 74686664704 }, "used": { - "bytes": 14856196096, - "pct": 0.237 + "bytes": 37566570496, + "pct": 0.502988996026061 + }, + "read": { + "bytes": 0, + "latency": { + "total": { + "ms": 0 + } + } + }, + "write": { + "bytes": 337000, + "latency": { + "total": { + "ms": 0 + } + } } }, - "fstype": "OTHER", + "fstype": "local", "name": "LocalDS_0" } } @@ -404,12 +390,22 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | host.os.build | OS build information. | keyword | | | | host.os.codename | OS codename, if any. | keyword | | | | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | -| vsphere.datastore.capacity.free.bytes | Free bytes of the datastore | long | byte | gauge | -| vsphere.datastore.capacity.total.bytes | Total bytes of the datastore | long | byte | gauge | -| vsphere.datastore.capacity.used.bytes | Used bytes of the datastore | long | byte | gauge | -| vsphere.datastore.capacity.used.pct | Used percent of the datastore | scaled_float | percent | gauge | -| vsphere.datastore.fstype | Filesystem type | keyword | | | -| vsphere.datastore.name | Datastore name | keyword | | | +| vsphere.datastore.capacity.free.bytes | Free bytes of the datastore. | long | byte | gauge | +| vsphere.datastore.capacity.total.bytes | Total bytes of the datastore. | long | byte | gauge | +| vsphere.datastore.capacity.used.bytes | Used bytes of the datastore. | long | byte | gauge | +| vsphere.datastore.capacity.used.pct | Percentage of datastore capacity used. | scaled_float | percent | gauge | +| vsphere.datastore.fstype | Filesystem type. | keyword | | | +| vsphere.datastore.host.count | Number of hosts. | long | | | +| vsphere.datastore.host.names | List of all the host names. | keyword | | | +| vsphere.datastore.iops | Storage I/O Control aggregated Input/Output Operations Per Second. | long | | | +| vsphere.datastore.name | Datastore name. | keyword | | | +| vsphere.datastore.read.bytes | Rate of reading data from the datastore. | long | | | +| vsphere.datastore.read.latency.total.ms | Average amount of time for a read operation from the datastore in milliseconds. | long | | | +| vsphere.datastore.status | Status of the datastore. | keyword | | | +| vsphere.datastore.vm.count | Number of VMs. | long | | | +| vsphere.datastore.vm.names | List of all the VM names. | keyword | | | +| vsphere.datastore.write.bytes | Rate of writing data to the datastore. | long | | | +| vsphere.datastore.write.latency.total.ms | Average amount of time for a write operation from the datastore in milliseconds. | long | | | ## Logs diff --git a/packages/vsphere/manifest.yml b/packages/vsphere/manifest.yml index 8d9bb1e5bf1..ea08140b424 100644 --- a/packages/vsphere/manifest.yml +++ b/packages/vsphere/manifest.yml @@ -1,7 +1,7 @@ title: VMware vSphere format_version: "3.0.2" name: vsphere -version: "1.13.0" +version: "1.14.0" description: This Elastic integration collects metrics and logs from vSphere/vCenter servers type: integration categories: From e9147ab0e4633c978eb853a1e18c7d109a1aa6a4 Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Wed, 28 Aug 2024 12:05:37 +0300 Subject: [PATCH 04/28] update sample event --- .../data_stream/datastore/sample_event.json | 48 ++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/packages/vsphere/data_stream/datastore/sample_event.json b/packages/vsphere/data_stream/datastore/sample_event.json index 609fa779b2f..88b778dbac9 100644 --- a/packages/vsphere/data_stream/datastore/sample_event.json +++ b/packages/vsphere/data_stream/datastore/sample_event.json @@ -1,10 +1,54 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "event": { + "agent": { + "ephemeral_id": "8b019ff3-cbda-41fa-b1ff-974d482b9694", + "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.8.1" + }, + "data_stream": { "dataset": "vsphere.datastore", - "duration": 115000, + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", + "snapshot": false, + "version": "8.8.1" + }, + "event": { + "agent_id_status": "verified", + "dataset": "vsphere.datastore", + "duration": 23155458, + "ingested": "2023-06-29T08:03:31Z", "module": "vsphere" }, + "host": { + "architecture": "aarch64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "d08b346fbb8f49f5a2bb1a477f8ceb54", + "ip": [ + "172.23.0.7" + ], + "mac": [ + "02-42-AC-17-00-07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.104-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.6 LTS (Focal Fossa)" + } + }, "metricset": { "name": "datastore", "period": 10000 From 2eb74a8c1d492d270d31b1f48c6a6229797ee899 Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Wed, 28 Aug 2024 12:24:47 +0300 Subject: [PATCH 05/28] elastic-package build --- .../data_stream/datastore/fields/fields.yml | 10 ++++ .../data_stream/datastore/sample_event.json | 38 +++++++-------- packages/vsphere/docs/README.md | 46 ++++++++++++++++++- 3 files changed, 74 insertions(+), 20 deletions(-) diff --git a/packages/vsphere/data_stream/datastore/fields/fields.yml b/packages/vsphere/data_stream/datastore/fields/fields.yml index 38e295f1842..e379886229d 100644 --- a/packages/vsphere/data_stream/datastore/fields/fields.yml +++ b/packages/vsphere/data_stream/datastore/fields/fields.yml @@ -40,41 +40,51 @@ type: long description: > Number of hosts. + - name: host.names type: keyword description: > List of all the host names. + - name: iops type: long description: > Storage I/O Control aggregated Input/Output Operations Per Second. + - name: read.bytes type: long description: > Rate of reading data from the datastore. + format: bytes - name: read.latency.total.ms type: long description: > Average amount of time for a read operation from the datastore in milliseconds. + - name: status type: keyword description: > Status of the datastore. + - name: vm.count type: long description: > Number of VMs. + - name: vm.names type: keyword description: > List of all the VM names. + - name: write.bytes type: long description: > Rate of writing data to the datastore. + format: bytes - name: write.latency.total.ms type: long description: > Average amount of time for a write operation from the datastore in milliseconds. + diff --git a/packages/vsphere/data_stream/datastore/sample_event.json b/packages/vsphere/data_stream/datastore/sample_event.json index 88b778dbac9..6a496794b59 100644 --- a/packages/vsphere/data_stream/datastore/sample_event.json +++ b/packages/vsphere/data_stream/datastore/sample_event.json @@ -20,7 +20,7 @@ "snapshot": false, "version": "8.8.1" }, - "event": { + "event": { "agent_id_status": "verified", "dataset": "vsphere.datastore", "duration": 23155458, @@ -61,17 +61,17 @@ "datastore": { "iops": 0, "host": { - "count": 1, - "names": [ - "DC3_H0" - ] + "count": 1, + "names": [ + "DC3_H0" + ] }, "status": "green", "vm": { - "count": 6, - "names": [ - "DC3_H0_VM0" - ] + "count": 6, + "names": [ + "DC3_H0_VM0" + ] }, "capacity": { "free": { @@ -87,22 +87,22 @@ "read": { "bytes": 0, "latency": { - "total": { - "ms": 0 - } + "total": { + "ms": 0 + } } - }, - "write": { + }, + "write": { "bytes": 337000, "latency": { - "total": { - "ms": 0 - } + "total": { + "ms": 0 + } } - } + } }, "fstype": "local", "name": "LocalDS_0" } } - } \ No newline at end of file +} \ No newline at end of file diff --git a/packages/vsphere/docs/README.md b/packages/vsphere/docs/README.md index 4052e1631b8..3bdade70c45 100644 --- a/packages/vsphere/docs/README.md +++ b/packages/vsphere/docs/README.md @@ -300,11 +300,55 @@ An example event for `datastore` looks as following: ```json { "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "ephemeral_id": "8b019ff3-cbda-41fa-b1ff-974d482b9694", + "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.8.1" + }, + "data_stream": { + "dataset": "vsphere.datastore", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", + "snapshot": false, + "version": "8.8.1" + }, "event": { + "agent_id_status": "verified", "dataset": "vsphere.datastore", - "duration": 115000, + "duration": 23155458, + "ingested": "2023-06-29T08:03:31Z", "module": "vsphere" }, + "host": { + "architecture": "aarch64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "d08b346fbb8f49f5a2bb1a477f8ceb54", + "ip": [ + "172.23.0.7" + ], + "mac": [ + "02-42-AC-17-00-07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.104-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.6 LTS (Focal Fossa)" + } + }, "metricset": { "name": "datastore", "period": 10000 From 11a135b99238de88564c3fcdadc6759754af37f3 Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Wed, 28 Aug 2024 14:34:51 +0300 Subject: [PATCH 06/28] fix sample event --- .../data_stream/datastore/fields/fields.yml | 10 ----- .../data_stream/datastore/sample_event.json | 32 ++++++++-------- packages/vsphere/docs/README.md | 37 ++++++++++--------- 3 files changed, 35 insertions(+), 44 deletions(-) diff --git a/packages/vsphere/data_stream/datastore/fields/fields.yml b/packages/vsphere/data_stream/datastore/fields/fields.yml index e379886229d..38e295f1842 100644 --- a/packages/vsphere/data_stream/datastore/fields/fields.yml +++ b/packages/vsphere/data_stream/datastore/fields/fields.yml @@ -40,51 +40,41 @@ type: long description: > Number of hosts. - - name: host.names type: keyword description: > List of all the host names. - - name: iops type: long description: > Storage I/O Control aggregated Input/Output Operations Per Second. - - name: read.bytes type: long description: > Rate of reading data from the datastore. - format: bytes - name: read.latency.total.ms type: long description: > Average amount of time for a read operation from the datastore in milliseconds. - - name: status type: keyword description: > Status of the datastore. - - name: vm.count type: long description: > Number of VMs. - - name: vm.names type: keyword description: > List of all the VM names. - - name: write.bytes type: long description: > Rate of writing data to the datastore. - format: bytes - name: write.latency.total.ms type: long description: > Average amount of time for a write operation from the datastore in milliseconds. - diff --git a/packages/vsphere/data_stream/datastore/sample_event.json b/packages/vsphere/data_stream/datastore/sample_event.json index 6a496794b59..9210e065845 100644 --- a/packages/vsphere/data_stream/datastore/sample_event.json +++ b/packages/vsphere/data_stream/datastore/sample_event.json @@ -83,26 +83,26 @@ "used": { "bytes": 37566570496, "pct": 0.502988996026061 - }, - "read": { - "bytes": 0, - "latency": { - "total": { - "ms": 0 - } + } + }, + "read": { + "bytes": 0, + "latency": { + "total": { + "ms": 0 } - }, - "write": { - "bytes": 337000, - "latency": { - "total": { - "ms": 0 - } + } + }, + "write": { + "bytes": 337000, + "latency": { + "total": { + "ms": 0 } } + } }, "fstype": "local", "name": "LocalDS_0" } - } -} \ No newline at end of file +} diff --git a/packages/vsphere/docs/README.md b/packages/vsphere/docs/README.md index 3bdade70c45..143425f3a38 100644 --- a/packages/vsphere/docs/README.md +++ b/packages/vsphere/docs/README.md @@ -383,29 +383,30 @@ An example event for `datastore` looks as following: "used": { "bytes": 37566570496, "pct": 0.502988996026061 - }, - "read": { - "bytes": 0, - "latency": { - "total": { - "ms": 0 - } - } - }, - "write": { - "bytes": 337000, - "latency": { - "total": { - "ms": 0 - } + } + }, + "read": { + "bytes": 0, + "latency": { + "total": { + "ms": 0 } } }, - "fstype": "local", - "name": "LocalDS_0" - } + "write": { + "bytes": 337000, + "latency": { + "total": { + "ms": 0 + } + } + } + }, + "fstype": "local", + "name": "LocalDS_0" } } + ``` **ECS Field Reference** From 8ce790b01249d4ac06922558927b57f5b0b01862 Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Wed, 28 Aug 2024 14:53:07 +0300 Subject: [PATCH 07/28] fix vsphere name and fstype --- .../vsphere/data_stream/datastore/fields/fields.yml | 10 ++++++++++ .../vsphere/data_stream/datastore/sample_event.json | 2 +- packages/vsphere/docs/README.md | 8 ++++---- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/vsphere/data_stream/datastore/fields/fields.yml b/packages/vsphere/data_stream/datastore/fields/fields.yml index 38e295f1842..e379886229d 100644 --- a/packages/vsphere/data_stream/datastore/fields/fields.yml +++ b/packages/vsphere/data_stream/datastore/fields/fields.yml @@ -40,41 +40,51 @@ type: long description: > Number of hosts. + - name: host.names type: keyword description: > List of all the host names. + - name: iops type: long description: > Storage I/O Control aggregated Input/Output Operations Per Second. + - name: read.bytes type: long description: > Rate of reading data from the datastore. + format: bytes - name: read.latency.total.ms type: long description: > Average amount of time for a read operation from the datastore in milliseconds. + - name: status type: keyword description: > Status of the datastore. + - name: vm.count type: long description: > Number of VMs. + - name: vm.names type: keyword description: > List of all the VM names. + - name: write.bytes type: long description: > Rate of writing data to the datastore. + format: bytes - name: write.latency.total.ms type: long description: > Average amount of time for a write operation from the datastore in milliseconds. + diff --git a/packages/vsphere/data_stream/datastore/sample_event.json b/packages/vsphere/data_stream/datastore/sample_event.json index 9210e065845..1c182717bd7 100644 --- a/packages/vsphere/data_stream/datastore/sample_event.json +++ b/packages/vsphere/data_stream/datastore/sample_event.json @@ -100,9 +100,9 @@ "ms": 0 } } - } }, "fstype": "local", "name": "LocalDS_0" } + } } diff --git a/packages/vsphere/docs/README.md b/packages/vsphere/docs/README.md index 143425f3a38..21138a035bb 100644 --- a/packages/vsphere/docs/README.md +++ b/packages/vsphere/docs/README.md @@ -400,10 +400,10 @@ An example event for `datastore` looks as following: "ms": 0 } } - } - }, - "fstype": "local", - "name": "LocalDS_0" + }, + "fstype": "local", + "name": "LocalDS_0" + } } } From 321a5e2b59c038427b9b3d9946cdfda5e04c59ec Mon Sep 17 00:00:00 2001 From: Lucian Date: Mon, 2 Sep 2024 12:53:29 +0300 Subject: [PATCH 08/28] follow changelog guidelines --- packages/vsphere/changelog.yml | 2 +- packages/vsphere/manifest.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/vsphere/changelog.yml b/packages/vsphere/changelog.yml index 48fab3be924..93202d7313a 100644 --- a/packages/vsphere/changelog.yml +++ b/packages/vsphere/changelog.yml @@ -1,5 +1,5 @@ # newer versions go on top -- version: "1.14.0" +- version: "1.14.0-next" changes: - description: Add additional metrics for datastore datastream type: enhancement diff --git a/packages/vsphere/manifest.yml b/packages/vsphere/manifest.yml index ea08140b424..8d9bb1e5bf1 100644 --- a/packages/vsphere/manifest.yml +++ b/packages/vsphere/manifest.yml @@ -1,7 +1,7 @@ title: VMware vSphere format_version: "3.0.2" name: vsphere -version: "1.14.0" +version: "1.13.0" description: This Elastic integration collects metrics and logs from vSphere/vCenter servers type: integration categories: From 0cbd1bc92ae083ed829d89046a603748f5c79b08 Mon Sep 17 00:00:00 2001 From: Lucian Date: Mon, 2 Sep 2024 12:55:37 +0300 Subject: [PATCH 09/28] update fields --- .../vsphere/data_stream/datastore/fields/fields.yml | 12 +----------- packages/vsphere/docs/README.md | 2 +- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/packages/vsphere/data_stream/datastore/fields/fields.yml b/packages/vsphere/data_stream/datastore/fields/fields.yml index e379886229d..a234cd034c6 100644 --- a/packages/vsphere/data_stream/datastore/fields/fields.yml +++ b/packages/vsphere/data_stream/datastore/fields/fields.yml @@ -39,52 +39,42 @@ - name: host.count type: long description: > - Number of hosts. - + Number of hosts associated with the datastore. - name: host.names type: keyword description: > List of all the host names. - - name: iops type: long description: > Storage I/O Control aggregated Input/Output Operations Per Second. - - name: read.bytes type: long description: > Rate of reading data from the datastore. - format: bytes - name: read.latency.total.ms type: long description: > Average amount of time for a read operation from the datastore in milliseconds. - - name: status type: keyword description: > Status of the datastore. - - name: vm.count type: long description: > Number of VMs. - - name: vm.names type: keyword description: > List of all the VM names. - - name: write.bytes type: long description: > Rate of writing data to the datastore. - format: bytes - name: write.latency.total.ms type: long description: > Average amount of time for a write operation from the datastore in milliseconds. - diff --git a/packages/vsphere/docs/README.md b/packages/vsphere/docs/README.md index 21138a035bb..c9fe3e74199 100644 --- a/packages/vsphere/docs/README.md +++ b/packages/vsphere/docs/README.md @@ -440,7 +440,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | vsphere.datastore.capacity.used.bytes | Used bytes of the datastore. | long | byte | gauge | | vsphere.datastore.capacity.used.pct | Percentage of datastore capacity used. | scaled_float | percent | gauge | | vsphere.datastore.fstype | Filesystem type. | keyword | | | -| vsphere.datastore.host.count | Number of hosts. | long | | | +| vsphere.datastore.host.count | Number of hosts associated with the datastore. | long | | | | vsphere.datastore.host.names | List of all the host names. | keyword | | | | vsphere.datastore.iops | Storage I/O Control aggregated Input/Output Operations Per Second. | long | | | | vsphere.datastore.name | Datastore name. | keyword | | | From 067710889ec5a28012a42bdedfa6c784719b8eb5 Mon Sep 17 00:00:00 2001 From: Lucian Date: Mon, 2 Sep 2024 12:57:49 +0300 Subject: [PATCH 10/28] better field descriptions --- packages/vsphere/data_stream/datastore/fields/fields.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/vsphere/data_stream/datastore/fields/fields.yml b/packages/vsphere/data_stream/datastore/fields/fields.yml index a234cd034c6..a457b6ea9b3 100644 --- a/packages/vsphere/data_stream/datastore/fields/fields.yml +++ b/packages/vsphere/data_stream/datastore/fields/fields.yml @@ -43,7 +43,7 @@ - name: host.names type: keyword description: > - List of all the host names. + List of all the host names associated with the datastore. - name: iops type: long description: > @@ -64,11 +64,11 @@ - name: vm.count type: long description: > - Number of VMs. + Number of VMs associated with the datastore. - name: vm.names type: keyword description: > - List of all the VM names. + List of all the VM names associated with the datastore. - name: write.bytes type: long description: > From 3b9812219f0c9309df20814f3131c32f0131a06c Mon Sep 17 00:00:00 2001 From: Lucian Date: Mon, 2 Sep 2024 13:06:05 +0300 Subject: [PATCH 11/28] update sample event --- .../data_stream/datastore/sample_event.json | 100 ++++++++---------- 1 file changed, 46 insertions(+), 54 deletions(-) diff --git a/packages/vsphere/data_stream/datastore/sample_event.json b/packages/vsphere/data_stream/datastore/sample_event.json index 1c182717bd7..356a2f665a0 100644 --- a/packages/vsphere/data_stream/datastore/sample_event.json +++ b/packages/vsphere/data_stream/datastore/sample_event.json @@ -1,48 +1,50 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", + "@timestamp": "2024-09-02T10:04:25.122Z", "agent": { - "ephemeral_id": "8b019ff3-cbda-41fa-b1ff-974d482b9694", - "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", - "name": "docker-fleet-agent", + "ephemeral_id": "4da294a3-ad54-47f4-92c7-544e1356a0d8", + "id": "b01ab3cf-51ad-4c4d-87bd-fc2d4aa59d8a", + "name": "elastic-agent-76236", "type": "metricbeat", - "version": "8.8.1" + "version": "8.16.0" }, "data_stream": { "dataset": "vsphere.datastore", - "namespace": "ep", + "namespace": "86691", "type": "metrics" }, "ecs": { "version": "8.11.0" }, "elastic_agent": { - "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", - "snapshot": false, - "version": "8.8.1" + "id": "b01ab3cf-51ad-4c4d-87bd-fc2d4aa59d8a", + "snapshot": true, + "version": "8.16.0" }, "event": { "agent_id_status": "verified", "dataset": "vsphere.datastore", - "duration": 23155458, - "ingested": "2023-06-29T08:03:31Z", + "duration": 190147614, + "ingested": "2024-09-02T10:04:27Z", "module": "vsphere" }, "host": { - "architecture": "aarch64", - "containerized": false, - "hostname": "docker-fleet-agent", - "id": "d08b346fbb8f49f5a2bb1a477f8ceb54", + "architecture": "x86_64", + "containerized": true, + "hostname": "elastic-agent-76236", + "id": "e744630f9d4f43dc818e497d221bd0b2", "ip": [ - "172.23.0.7" + "172.18.0.4", + "172.21.0.2" ], "mac": [ - "02-42-AC-17-00-07" + "02-42-AC-12-00-04", + "02-42-AC-15-00-02" ], - "name": "docker-fleet-agent", + "name": "elastic-agent-76236", "os": { "codename": "focal", "family": "debian", - "kernel": "5.10.104-linuxkit", + "kernel": "5.15.153.1-microsoft-standard-WSL2", "name": "Ubuntu", "platform": "ubuntu", "type": "linux", @@ -54,55 +56,45 @@ "period": 10000 }, "service": { - "address": "127.0.0.1:33365", + "address": "https://svc-vsphere-metrics:8989/sdk", "type": "vsphere" }, "vsphere": { "datastore": { - "iops": 0, - "host": { - "count": 1, - "names": [ - "DC3_H0" - ] - }, - "status": "green", - "vm": { - "count": 6, - "names": [ - "DC3_H0_VM0" - ] - }, "capacity": { "free": { - "bytes": 37120094208 + "bytes": 10952166604800 }, "total": { - "bytes": 74686664704 + "bytes": 10995116277760 }, "used": { - "bytes": 37566570496, - "pct": 0.502988996026061 + "bytes": 42949672960, + "pct": 0.004 } }, + "fstype": "OTHER", + "host": { + "count": 1, + "names": "DC0_H0" + }, + "name": "LocalDS_0", "read": { - "bytes": 0, - "latency": { - "total": { - "ms": 0 - } - } + "bytes": 1024 }, - "write": { - "bytes": 337000, - "latency": { - "total": { - "ms": 0 - } - } + "status": "green", + "vm": { + "count": 4, + "names": [ + "DC0_C0_RP0_VM0", + "DC0_C0_RP0_VM1", + "DC0_H0_VM0", + "DC0_H0_VM1" + ] }, - "fstype": "local", - "name": "LocalDS_0" + "write": { + "bytes": 450560 + } } } -} +} \ No newline at end of file From 38b42aacde565df91543d1ea083da1892919558c Mon Sep 17 00:00:00 2001 From: Lucian Date: Mon, 2 Sep 2024 13:19:17 +0300 Subject: [PATCH 12/28] package build --- packages/vsphere/docs/README.md | 105 +++++++++++++++----------------- 1 file changed, 48 insertions(+), 57 deletions(-) diff --git a/packages/vsphere/docs/README.md b/packages/vsphere/docs/README.md index c9fe3e74199..8fbbc9db060 100644 --- a/packages/vsphere/docs/README.md +++ b/packages/vsphere/docs/README.md @@ -299,50 +299,52 @@ An example event for `datastore` looks as following: ```json { - "@timestamp": "2017-10-12T08:05:34.853Z", + "@timestamp": "2024-09-02T10:04:25.122Z", "agent": { - "ephemeral_id": "8b019ff3-cbda-41fa-b1ff-974d482b9694", - "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", - "name": "docker-fleet-agent", + "ephemeral_id": "4da294a3-ad54-47f4-92c7-544e1356a0d8", + "id": "b01ab3cf-51ad-4c4d-87bd-fc2d4aa59d8a", + "name": "elastic-agent-76236", "type": "metricbeat", - "version": "8.8.1" + "version": "8.16.0" }, "data_stream": { "dataset": "vsphere.datastore", - "namespace": "ep", + "namespace": "86691", "type": "metrics" }, "ecs": { "version": "8.11.0" }, "elastic_agent": { - "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", - "snapshot": false, - "version": "8.8.1" + "id": "b01ab3cf-51ad-4c4d-87bd-fc2d4aa59d8a", + "snapshot": true, + "version": "8.16.0" }, "event": { "agent_id_status": "verified", "dataset": "vsphere.datastore", - "duration": 23155458, - "ingested": "2023-06-29T08:03:31Z", + "duration": 190147614, + "ingested": "2024-09-02T10:04:27Z", "module": "vsphere" }, "host": { - "architecture": "aarch64", - "containerized": false, - "hostname": "docker-fleet-agent", - "id": "d08b346fbb8f49f5a2bb1a477f8ceb54", + "architecture": "x86_64", + "containerized": true, + "hostname": "elastic-agent-76236", + "id": "e744630f9d4f43dc818e497d221bd0b2", "ip": [ - "172.23.0.7" + "172.18.0.4", + "172.21.0.2" ], "mac": [ - "02-42-AC-17-00-07" + "02-42-AC-12-00-04", + "02-42-AC-15-00-02" ], - "name": "docker-fleet-agent", + "name": "elastic-agent-76236", "os": { "codename": "focal", "family": "debian", - "kernel": "5.10.104-linuxkit", + "kernel": "5.15.153.1-microsoft-standard-WSL2", "name": "Ubuntu", "platform": "ubuntu", "type": "linux", @@ -354,59 +356,48 @@ An example event for `datastore` looks as following: "period": 10000 }, "service": { - "address": "127.0.0.1:33365", + "address": "https://svc-vsphere-metrics:8989/sdk", "type": "vsphere" }, "vsphere": { "datastore": { - "iops": 0, - "host": { - "count": 1, - "names": [ - "DC3_H0" - ] - }, - "status": "green", - "vm": { - "count": 6, - "names": [ - "DC3_H0_VM0" - ] - }, "capacity": { "free": { - "bytes": 37120094208 + "bytes": 10952166604800 }, "total": { - "bytes": 74686664704 + "bytes": 10995116277760 }, "used": { - "bytes": 37566570496, - "pct": 0.502988996026061 + "bytes": 42949672960, + "pct": 0.004 } }, + "fstype": "OTHER", + "host": { + "count": 1, + "names": "DC0_H0" + }, + "name": "LocalDS_0", "read": { - "bytes": 0, - "latency": { - "total": { - "ms": 0 - } - } + "bytes": 1024 }, - "write": { - "bytes": 337000, - "latency": { - "total": { - "ms": 0 - } - } + "status": "green", + "vm": { + "count": 4, + "names": [ + "DC0_C0_RP0_VM0", + "DC0_C0_RP0_VM1", + "DC0_H0_VM0", + "DC0_H0_VM1" + ] }, - "fstype": "local", - "name": "LocalDS_0" + "write": { + "bytes": 450560 + } } } } - ``` **ECS Field Reference** @@ -441,14 +432,14 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | vsphere.datastore.capacity.used.pct | Percentage of datastore capacity used. | scaled_float | percent | gauge | | vsphere.datastore.fstype | Filesystem type. | keyword | | | | vsphere.datastore.host.count | Number of hosts associated with the datastore. | long | | | -| vsphere.datastore.host.names | List of all the host names. | keyword | | | +| vsphere.datastore.host.names | List of all the host names associated with the datastore. | keyword | | | | vsphere.datastore.iops | Storage I/O Control aggregated Input/Output Operations Per Second. | long | | | | vsphere.datastore.name | Datastore name. | keyword | | | | vsphere.datastore.read.bytes | Rate of reading data from the datastore. | long | | | | vsphere.datastore.read.latency.total.ms | Average amount of time for a read operation from the datastore in milliseconds. | long | | | | vsphere.datastore.status | Status of the datastore. | keyword | | | -| vsphere.datastore.vm.count | Number of VMs. | long | | | -| vsphere.datastore.vm.names | List of all the VM names. | keyword | | | +| vsphere.datastore.vm.count | Number of VMs associated with the datastore. | long | | | +| vsphere.datastore.vm.names | List of all the VM names associated with the datastore. | keyword | | | | vsphere.datastore.write.bytes | Rate of writing data to the datastore. | long | | | | vsphere.datastore.write.latency.total.ms | Average amount of time for a write operation from the datastore in milliseconds. | long | | | From 2cb93db5ddddefa7c446c1cba02cce135ba8aed6 Mon Sep 17 00:00:00 2001 From: Lucian Date: Tue, 3 Sep 2024 15:14:35 +0300 Subject: [PATCH 13/28] add metric types --- packages/vsphere/_dev/deploy/docker/Dockerfile | 4 +--- .../data_stream/datastore/fields/fields.yml | 11 +++++++++++ packages/vsphere/docs/README.md | 14 +++++++------- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/vsphere/_dev/deploy/docker/Dockerfile b/packages/vsphere/_dev/deploy/docker/Dockerfile index ce3f58eb072..3f491ee48f4 100644 --- a/packages/vsphere/_dev/deploy/docker/Dockerfile +++ b/packages/vsphere/_dev/deploy/docker/Dockerfile @@ -1,6 +1,4 @@ -ARG VSPHERE_GOLANG_VERSION - -FROM golang:${VSPHERE_GOLANG_VERSION}-alpine +FROM golang:1.20-alpine RUN apk add --no-cache curl git RUN go install github.com/vmware/govmomi/vcsim@v0.30.4 diff --git a/packages/vsphere/data_stream/datastore/fields/fields.yml b/packages/vsphere/data_stream/datastore/fields/fields.yml index a457b6ea9b3..19600b73558 100644 --- a/packages/vsphere/data_stream/datastore/fields/fields.yml +++ b/packages/vsphere/data_stream/datastore/fields/fields.yml @@ -38,6 +38,7 @@ format: percent - name: host.count type: long + metric_type: gauge description: > Number of hosts associated with the datastore. - name: host.names @@ -46,15 +47,20 @@ List of all the host names associated with the datastore. - name: iops type: long + metric_type: gauge description: > Storage I/O Control aggregated Input/Output Operations Per Second. - name: read.bytes type: long + metric_type: gauge + unit: byte description: > Rate of reading data from the datastore. format: bytes - name: read.latency.total.ms type: long + metric_type: gauge + unit: ms description: > Average amount of time for a read operation from the datastore in milliseconds. - name: status @@ -63,6 +69,7 @@ Status of the datastore. - name: vm.count type: long + metric_type: gauge description: > Number of VMs associated with the datastore. - name: vm.names @@ -71,10 +78,14 @@ List of all the VM names associated with the datastore. - name: write.bytes type: long + metric_type: gauge + unit: byte description: > Rate of writing data to the datastore. format: bytes - name: write.latency.total.ms + metric_type: gauge + unit: ms type: long description: > Average amount of time for a write operation from the datastore in milliseconds. diff --git a/packages/vsphere/docs/README.md b/packages/vsphere/docs/README.md index 8fbbc9db060..95516f06336 100644 --- a/packages/vsphere/docs/README.md +++ b/packages/vsphere/docs/README.md @@ -431,17 +431,17 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | vsphere.datastore.capacity.used.bytes | Used bytes of the datastore. | long | byte | gauge | | vsphere.datastore.capacity.used.pct | Percentage of datastore capacity used. | scaled_float | percent | gauge | | vsphere.datastore.fstype | Filesystem type. | keyword | | | -| vsphere.datastore.host.count | Number of hosts associated with the datastore. | long | | | +| vsphere.datastore.host.count | Number of hosts associated with the datastore. | long | | gauge | | vsphere.datastore.host.names | List of all the host names associated with the datastore. | keyword | | | -| vsphere.datastore.iops | Storage I/O Control aggregated Input/Output Operations Per Second. | long | | | +| vsphere.datastore.iops | Storage I/O Control aggregated Input/Output Operations Per Second. | long | | gauge | | vsphere.datastore.name | Datastore name. | keyword | | | -| vsphere.datastore.read.bytes | Rate of reading data from the datastore. | long | | | -| vsphere.datastore.read.latency.total.ms | Average amount of time for a read operation from the datastore in milliseconds. | long | | | +| vsphere.datastore.read.bytes | Rate of reading data from the datastore. | long | byte | gauge | +| vsphere.datastore.read.latency.total.ms | Average amount of time for a read operation from the datastore in milliseconds. | long | ms | gauge | | vsphere.datastore.status | Status of the datastore. | keyword | | | -| vsphere.datastore.vm.count | Number of VMs associated with the datastore. | long | | | +| vsphere.datastore.vm.count | Number of VMs associated with the datastore. | long | | gauge | | vsphere.datastore.vm.names | List of all the VM names associated with the datastore. | keyword | | | -| vsphere.datastore.write.bytes | Rate of writing data to the datastore. | long | | | -| vsphere.datastore.write.latency.total.ms | Average amount of time for a write operation from the datastore in milliseconds. | long | | | +| vsphere.datastore.write.bytes | Rate of writing data to the datastore. | long | byte | gauge | +| vsphere.datastore.write.latency.total.ms | Average amount of time for a write operation from the datastore in milliseconds. | long | ms | gauge | ## Logs From 9f6b19f46365deb4f343c7b6cc265e8a8cc290fe Mon Sep 17 00:00:00 2001 From: Lucian Date: Tue, 3 Sep 2024 22:42:11 +0300 Subject: [PATCH 14/28] revert dockerfile --- packages/vsphere/_dev/deploy/docker/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/vsphere/_dev/deploy/docker/Dockerfile b/packages/vsphere/_dev/deploy/docker/Dockerfile index 3f491ee48f4..21e8fe9882d 100644 --- a/packages/vsphere/_dev/deploy/docker/Dockerfile +++ b/packages/vsphere/_dev/deploy/docker/Dockerfile @@ -1,7 +1,9 @@ -FROM golang:1.20-alpine +ARG VSPHERE_GOLANG_VERSION + +FROM golang:${VSPHERE_GOLANG_VERSION}-alpine RUN apk add --no-cache curl git RUN go install github.com/vmware/govmomi/vcsim@v0.30.4 HEALTHCHECK --interval=1s --retries=60 --timeout=10s CMD curl http://localhost:8989/ -CMD vcsim -l :8989 +CMD vcsim -l :8989 \ No newline at end of file From 73366a77311ac0b6b7731e71891b3842adcaa544 Mon Sep 17 00:00:00 2001 From: Lucian Date: Tue, 3 Sep 2024 22:43:32 +0300 Subject: [PATCH 15/28] add dockerfile newline --- packages/vsphere/_dev/deploy/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vsphere/_dev/deploy/docker/Dockerfile b/packages/vsphere/_dev/deploy/docker/Dockerfile index 21e8fe9882d..ce3f58eb072 100644 --- a/packages/vsphere/_dev/deploy/docker/Dockerfile +++ b/packages/vsphere/_dev/deploy/docker/Dockerfile @@ -6,4 +6,4 @@ RUN apk add --no-cache curl git RUN go install github.com/vmware/govmomi/vcsim@v0.30.4 HEALTHCHECK --interval=1s --retries=60 --timeout=10s CMD curl http://localhost:8989/ -CMD vcsim -l :8989 \ No newline at end of file +CMD vcsim -l :8989 From 6a47c43f671db6df60aad4c989872c490570abe4 Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Wed, 18 Sep 2024 11:17:56 +0300 Subject: [PATCH 16/28] add period --- .../data_stream/datastore/manifest.yml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/vsphere/data_stream/datastore/manifest.yml b/packages/vsphere/data_stream/datastore/manifest.yml index 9930155855d..ca5db37a649 100644 --- a/packages/vsphere/data_stream/datastore/manifest.yml +++ b/packages/vsphere/data_stream/datastore/manifest.yml @@ -4,5 +4,27 @@ streams: - input: vsphere/metrics title: vSphere datastore metrics description: vSphere datastore metrics + vars: + - name: period + type: text + title: Period + description: The vSphere performance API allows for collecting host and datastore metrics at various intervals, including real-time (every 20 seconds), 5 minutes, or longer durations. Properly configuring the "period" parameter is crucial to manage usage restrictions and ensure accurate data collection. For more information on optimizing your data collection setup, refer to the [Data Collection Intervals](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-247646EA-A04B-411A-8DD4-62A3DCFCF49B.html) and [Data Collection Levels](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-25800DE4-68E5-41CC-82D9-8811E27924BC.html) documentation. + default: 20s + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - vsphere-host + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details. elasticsearch: index_mode: "time_series" From 57aad0bed859b0f72a0f9aa6471b6562777f7f0d Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Wed, 18 Sep 2024 11:26:55 +0300 Subject: [PATCH 17/28] add triggered alarms --- packages/vsphere/data_stream/datastore/fields/fields.yml | 5 +++++ packages/vsphere/docs/README.md | 1 + 2 files changed, 6 insertions(+) diff --git a/packages/vsphere/data_stream/datastore/fields/fields.yml b/packages/vsphere/data_stream/datastore/fields/fields.yml index 19600b73558..47d0f57ea60 100644 --- a/packages/vsphere/data_stream/datastore/fields/fields.yml +++ b/packages/vsphere/data_stream/datastore/fields/fields.yml @@ -67,6 +67,11 @@ type: keyword description: > Status of the datastore. + - name: triggerd_alarms.* + type: object + object_type: keyword + description: > + List of all the triggerd alarms. - name: vm.count type: long metric_type: gauge diff --git a/packages/vsphere/docs/README.md b/packages/vsphere/docs/README.md index 95516f06336..fc93fcb07f7 100644 --- a/packages/vsphere/docs/README.md +++ b/packages/vsphere/docs/README.md @@ -438,6 +438,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | vsphere.datastore.read.bytes | Rate of reading data from the datastore. | long | byte | gauge | | vsphere.datastore.read.latency.total.ms | Average amount of time for a read operation from the datastore in milliseconds. | long | ms | gauge | | vsphere.datastore.status | Status of the datastore. | keyword | | | +| vsphere.datastore.triggerd_alarms.\* | List of all the triggerd alarms. | object | | | | vsphere.datastore.vm.count | Number of VMs associated with the datastore. | long | | gauge | | vsphere.datastore.vm.names | List of all the VM names associated with the datastore. | keyword | | | | vsphere.datastore.write.bytes | Rate of writing data to the datastore. | long | byte | gauge | From 4c457aa07c3c09192f17c113b9c88167b4e52a44 Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Wed, 18 Sep 2024 11:40:22 +0300 Subject: [PATCH 18/28] add disk capacity fields --- .../data_stream/datastore/fields/fields.yml | 29 +++++++++++++++++++ .../data_stream/datastore/sample_event.json | 19 +++++++++--- packages/vsphere/docs/README.md | 22 +++++++++++--- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/packages/vsphere/data_stream/datastore/fields/fields.yml b/packages/vsphere/data_stream/datastore/fields/fields.yml index 47d0f57ea60..6498c82132e 100644 --- a/packages/vsphere/data_stream/datastore/fields/fields.yml +++ b/packages/vsphere/data_stream/datastore/fields/fields.yml @@ -12,6 +12,24 @@ # Reason to add as a dimension field: in case datastore name is duplicated across different type. dimension: true description: Filesystem type. + - name: disk.capacity.bytes + type: long + description: > + Configured size of the datastore. + + format: bytes + - name: disk.capacity.usage.bytes + type: long + description: > + The amount of storage capacity currently being consumed by datastore. + + format: bytes + - name: disk.provisioned.bytes + type: long + description: > + Amount of storage set aside for use by a datastore. + + format: bytes - name: capacity.total.bytes type: long metric_type: gauge @@ -41,21 +59,25 @@ metric_type: gauge description: > Number of hosts associated with the datastore. + - name: host.names type: keyword description: > List of all the host names associated with the datastore. + - name: iops type: long metric_type: gauge description: > Storage I/O Control aggregated Input/Output Operations Per Second. + - name: read.bytes type: long metric_type: gauge unit: byte description: > Rate of reading data from the datastore. + format: bytes - name: read.latency.total.ms type: long @@ -63,30 +85,36 @@ unit: ms description: > Average amount of time for a read operation from the datastore in milliseconds. + - name: status type: keyword description: > Status of the datastore. + - name: triggerd_alarms.* type: object object_type: keyword description: > List of all the triggerd alarms. + - name: vm.count type: long metric_type: gauge description: > Number of VMs associated with the datastore. + - name: vm.names type: keyword description: > List of all the VM names associated with the datastore. + - name: write.bytes type: long metric_type: gauge unit: byte description: > Rate of writing data to the datastore. + format: bytes - name: write.latency.total.ms metric_type: gauge @@ -94,3 +122,4 @@ type: long description: > Average amount of time for a write operation from the datastore in milliseconds. + diff --git a/packages/vsphere/data_stream/datastore/sample_event.json b/packages/vsphere/data_stream/datastore/sample_event.json index 356a2f665a0..f48886186e3 100644 --- a/packages/vsphere/data_stream/datastore/sample_event.json +++ b/packages/vsphere/data_stream/datastore/sample_event.json @@ -61,16 +61,27 @@ }, "vsphere": { "datastore": { + "disk": { + "capacity": { + "usage": { + "bytes": 520505786368 + }, + "bytes": 1610344300544 + }, + "provisioned": { + "bytes": 520505786368 + } + }, "capacity": { "free": { - "bytes": 10952166604800 + "bytes": 37120094208 }, "total": { - "bytes": 10995116277760 + "bytes": 74686664704 }, "used": { - "bytes": 42949672960, - "pct": 0.004 + "bytes": 37566570496, + "pct": 0.502988996026061 } }, "fstype": "OTHER", diff --git a/packages/vsphere/docs/README.md b/packages/vsphere/docs/README.md index fc93fcb07f7..966453cc600 100644 --- a/packages/vsphere/docs/README.md +++ b/packages/vsphere/docs/README.md @@ -361,16 +361,27 @@ An example event for `datastore` looks as following: }, "vsphere": { "datastore": { + "disk": { + "capacity": { + "usage": { + "bytes": 520505786368 + }, + "bytes": 1610344300544 + }, + "provisioned": { + "bytes": 520505786368 + } + }, "capacity": { "free": { - "bytes": 10952166604800 + "bytes": 37120094208 }, "total": { - "bytes": 10995116277760 + "bytes": 74686664704 }, "used": { - "bytes": 42949672960, - "pct": 0.004 + "bytes": 37566570496, + "pct": 0.502988996026061 } }, "fstype": "OTHER", @@ -430,6 +441,9 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | vsphere.datastore.capacity.total.bytes | Total bytes of the datastore. | long | byte | gauge | | vsphere.datastore.capacity.used.bytes | Used bytes of the datastore. | long | byte | gauge | | vsphere.datastore.capacity.used.pct | Percentage of datastore capacity used. | scaled_float | percent | gauge | +| vsphere.datastore.disk.capacity.bytes | Configured size of the datastore. | long | | | +| vsphere.datastore.disk.capacity.usage.bytes | The amount of storage capacity currently being consumed by datastore. | long | | | +| vsphere.datastore.disk.provisioned.bytes | Amount of storage set aside for use by a datastore. | long | | | | vsphere.datastore.fstype | Filesystem type. | keyword | | | | vsphere.datastore.host.count | Number of hosts associated with the datastore. | long | | gauge | | vsphere.datastore.host.names | List of all the host names associated with the datastore. | keyword | | | From 7bb821b68d7c05d01fc4e3d4d002a7e81a73d6df Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Wed, 18 Sep 2024 11:42:46 +0300 Subject: [PATCH 19/28] add metric types --- .../data_stream/datastore/fields/fields.yml | 17 +++-------------- packages/vsphere/docs/README.md | 6 +++--- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/packages/vsphere/data_stream/datastore/fields/fields.yml b/packages/vsphere/data_stream/datastore/fields/fields.yml index 6498c82132e..45fcbccf7ef 100644 --- a/packages/vsphere/data_stream/datastore/fields/fields.yml +++ b/packages/vsphere/data_stream/datastore/fields/fields.yml @@ -14,21 +14,21 @@ description: Filesystem type. - name: disk.capacity.bytes type: long + metric_type: gauge description: > Configured size of the datastore. - format: bytes - name: disk.capacity.usage.bytes type: long + metric_type: gauge description: > The amount of storage capacity currently being consumed by datastore. - format: bytes - name: disk.provisioned.bytes type: long + metric_type: gauge description: > Amount of storage set aside for use by a datastore. - format: bytes - name: capacity.total.bytes type: long @@ -59,25 +59,21 @@ metric_type: gauge description: > Number of hosts associated with the datastore. - - name: host.names type: keyword description: > List of all the host names associated with the datastore. - - name: iops type: long metric_type: gauge description: > Storage I/O Control aggregated Input/Output Operations Per Second. - - name: read.bytes type: long metric_type: gauge unit: byte description: > Rate of reading data from the datastore. - format: bytes - name: read.latency.total.ms type: long @@ -85,36 +81,30 @@ unit: ms description: > Average amount of time for a read operation from the datastore in milliseconds. - - name: status type: keyword description: > Status of the datastore. - - name: triggerd_alarms.* type: object object_type: keyword description: > List of all the triggerd alarms. - - name: vm.count type: long metric_type: gauge description: > Number of VMs associated with the datastore. - - name: vm.names type: keyword description: > List of all the VM names associated with the datastore. - - name: write.bytes type: long metric_type: gauge unit: byte description: > Rate of writing data to the datastore. - format: bytes - name: write.latency.total.ms metric_type: gauge @@ -122,4 +112,3 @@ type: long description: > Average amount of time for a write operation from the datastore in milliseconds. - diff --git a/packages/vsphere/docs/README.md b/packages/vsphere/docs/README.md index 966453cc600..cc10f939c19 100644 --- a/packages/vsphere/docs/README.md +++ b/packages/vsphere/docs/README.md @@ -441,9 +441,9 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | vsphere.datastore.capacity.total.bytes | Total bytes of the datastore. | long | byte | gauge | | vsphere.datastore.capacity.used.bytes | Used bytes of the datastore. | long | byte | gauge | | vsphere.datastore.capacity.used.pct | Percentage of datastore capacity used. | scaled_float | percent | gauge | -| vsphere.datastore.disk.capacity.bytes | Configured size of the datastore. | long | | | -| vsphere.datastore.disk.capacity.usage.bytes | The amount of storage capacity currently being consumed by datastore. | long | | | -| vsphere.datastore.disk.provisioned.bytes | Amount of storage set aside for use by a datastore. | long | | | +| vsphere.datastore.disk.capacity.bytes | Configured size of the datastore. | long | | gauge | +| vsphere.datastore.disk.capacity.usage.bytes | The amount of storage capacity currently being consumed by datastore. | long | | gauge | +| vsphere.datastore.disk.provisioned.bytes | Amount of storage set aside for use by a datastore. | long | | gauge | | vsphere.datastore.fstype | Filesystem type. | keyword | | | | vsphere.datastore.host.count | Number of hosts associated with the datastore. | long | | gauge | | vsphere.datastore.host.names | List of all the host names associated with the datastore. | keyword | | | From 7cad1389cf38fb2fd231aa1ea6c7162e81c15c48 Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Wed, 18 Sep 2024 12:18:05 +0300 Subject: [PATCH 20/28] update tag --- packages/vsphere/data_stream/datastore/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vsphere/data_stream/datastore/manifest.yml b/packages/vsphere/data_stream/datastore/manifest.yml index ca5db37a649..e2eedfa706c 100644 --- a/packages/vsphere/data_stream/datastore/manifest.yml +++ b/packages/vsphere/data_stream/datastore/manifest.yml @@ -17,7 +17,7 @@ streams: required: true show_user: false default: - - vsphere-host + - vsphere-datastore - name: processors type: yaml title: Processors From 5333a1155f7dbe6f85c311ca0a01677f2141f325 Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Thu, 19 Sep 2024 13:10:19 +0300 Subject: [PATCH 21/28] update version to 1.15.0-next --- packages/vsphere/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vsphere/changelog.yml b/packages/vsphere/changelog.yml index 509edfc8686..168dfd5301d 100644 --- a/packages/vsphere/changelog.yml +++ b/packages/vsphere/changelog.yml @@ -1,5 +1,5 @@ # newer versions go on top -- version: "1.14.0-next" +- version: "1.15.0-next" changes: - description: Add additional metrics for datastore datastream type: enhancement From 73a1b4714225610bcf3461d7ab17ff13fbbc25aa Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Thu, 19 Sep 2024 13:12:42 +0300 Subject: [PATCH 22/28] update kibana version to 8.15.2 --- packages/vsphere/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vsphere/manifest.yml b/packages/vsphere/manifest.yml index ea08140b424..497b180b7cd 100644 --- a/packages/vsphere/manifest.yml +++ b/packages/vsphere/manifest.yml @@ -9,7 +9,7 @@ categories: - virtualization conditions: kibana: - version: "^8.13.0" + version: "^8.15.2" elastic: subscription: basic screenshots: From 4afc1867e96f91af0c7b415a14147b3e9dc5411e Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Thu, 19 Sep 2024 13:13:36 +0300 Subject: [PATCH 23/28] remove global period --- packages/vsphere/manifest.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/vsphere/manifest.yml b/packages/vsphere/manifest.yml index 497b180b7cd..7e6e074eaf5 100644 --- a/packages/vsphere/manifest.yml +++ b/packages/vsphere/manifest.yml @@ -67,10 +67,6 @@ policy_templates: show_user: true default: "pass" description: vSphere account password (not root VMAMI account) - - name: period - type: text - title: Period - default: 10s - name: insecure required: true show_user: true From ac6260429cd469cbc7f3cdc7de98ebf4cd14f35d Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Thu, 19 Sep 2024 13:18:07 +0300 Subject: [PATCH 24/28] remove latency and iops fields --- .../data_stream/datastore/fields/fields.yml | 21 ++----------------- packages/vsphere/docs/README.md | 5 +---- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/packages/vsphere/data_stream/datastore/fields/fields.yml b/packages/vsphere/data_stream/datastore/fields/fields.yml index 45fcbccf7ef..d6353109707 100644 --- a/packages/vsphere/data_stream/datastore/fields/fields.yml +++ b/packages/vsphere/data_stream/datastore/fields/fields.yml @@ -63,11 +63,6 @@ type: keyword description: > List of all the host names associated with the datastore. - - name: iops - type: long - metric_type: gauge - description: > - Storage I/O Control aggregated Input/Output Operations Per Second. - name: read.bytes type: long metric_type: gauge @@ -75,17 +70,11 @@ description: > Rate of reading data from the datastore. format: bytes - - name: read.latency.total.ms - type: long - metric_type: gauge - unit: ms - description: > - Average amount of time for a read operation from the datastore in milliseconds. - name: status type: keyword description: > Status of the datastore. - - name: triggerd_alarms.* + - name: triggered_alarms.* type: object object_type: keyword description: > @@ -105,10 +94,4 @@ unit: byte description: > Rate of writing data to the datastore. - format: bytes - - name: write.latency.total.ms - metric_type: gauge - unit: ms - type: long - description: > - Average amount of time for a write operation from the datastore in milliseconds. + format: bytes \ No newline at end of file diff --git a/packages/vsphere/docs/README.md b/packages/vsphere/docs/README.md index cc10f939c19..fe0bec76595 100644 --- a/packages/vsphere/docs/README.md +++ b/packages/vsphere/docs/README.md @@ -447,16 +447,13 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | vsphere.datastore.fstype | Filesystem type. | keyword | | | | vsphere.datastore.host.count | Number of hosts associated with the datastore. | long | | gauge | | vsphere.datastore.host.names | List of all the host names associated with the datastore. | keyword | | | -| vsphere.datastore.iops | Storage I/O Control aggregated Input/Output Operations Per Second. | long | | gauge | | vsphere.datastore.name | Datastore name. | keyword | | | | vsphere.datastore.read.bytes | Rate of reading data from the datastore. | long | byte | gauge | -| vsphere.datastore.read.latency.total.ms | Average amount of time for a read operation from the datastore in milliseconds. | long | ms | gauge | | vsphere.datastore.status | Status of the datastore. | keyword | | | -| vsphere.datastore.triggerd_alarms.\* | List of all the triggerd alarms. | object | | | +| vsphere.datastore.triggered_alarms.\* | List of all the triggerd alarms. | object | | | | vsphere.datastore.vm.count | Number of VMs associated with the datastore. | long | | gauge | | vsphere.datastore.vm.names | List of all the VM names associated with the datastore. | keyword | | | | vsphere.datastore.write.bytes | Rate of writing data to the datastore. | long | byte | gauge | -| vsphere.datastore.write.latency.total.ms | Average amount of time for a write operation from the datastore in milliseconds. | long | ms | gauge | ## Logs From 9c7ae05553dcc7b216fd21bc3f977d9d9cbd9450 Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Thu, 19 Sep 2024 13:27:30 +0300 Subject: [PATCH 25/28] update triggered alarms to group --- .../data_stream/datastore/fields/fields.yml | 30 +++++++++++++++---- packages/vsphere/docs/README.md | 7 ++++- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/packages/vsphere/data_stream/datastore/fields/fields.yml b/packages/vsphere/data_stream/datastore/fields/fields.yml index d6353109707..1eb689a7383 100644 --- a/packages/vsphere/data_stream/datastore/fields/fields.yml +++ b/packages/vsphere/data_stream/datastore/fields/fields.yml @@ -74,11 +74,6 @@ type: keyword description: > Status of the datastore. - - name: triggered_alarms.* - type: object - object_type: keyword - description: > - List of all the triggerd alarms. - name: vm.count type: long metric_type: gauge @@ -94,4 +89,27 @@ unit: byte description: > Rate of writing data to the datastore. - format: bytes \ No newline at end of file + format: bytes + - name: triggered_alarms + type: group + description: > + List of all the triggered alarms. + fields: + - name: status + type: keyword + description: Status of the alarm. + - name: triggered_time + type: date + description: Time when the alarm was triggered. + - name: description + type: keyword + description: Description of the alarm. + - name: entity_name + type: keyword + description: Name of the entity associated with the alarm. + - name: name + type: keyword + description: Name of the alarm. + - name: id + type: keyword + description: Unique identifier for the alarm. \ No newline at end of file diff --git a/packages/vsphere/docs/README.md b/packages/vsphere/docs/README.md index fe0bec76595..aea17420cbb 100644 --- a/packages/vsphere/docs/README.md +++ b/packages/vsphere/docs/README.md @@ -450,7 +450,12 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | vsphere.datastore.name | Datastore name. | keyword | | | | vsphere.datastore.read.bytes | Rate of reading data from the datastore. | long | byte | gauge | | vsphere.datastore.status | Status of the datastore. | keyword | | | -| vsphere.datastore.triggered_alarms.\* | List of all the triggerd alarms. | object | | | +| vsphere.datastore.triggered_alarms.description | Description of the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.entity_name | Name of the entity associated with the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.id | Unique identifier for the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.name | Name of the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.status | Status of the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.triggered_time | Time when the alarm was triggered. | date | | | | vsphere.datastore.vm.count | Number of VMs associated with the datastore. | long | | gauge | | vsphere.datastore.vm.names | List of all the VM names associated with the datastore. | keyword | | | | vsphere.datastore.write.bytes | Rate of writing data to the datastore. | long | byte | gauge | From 1bdac8d9f739801e6a9f9a47cdd7fca0449d1ca4 Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Thu, 19 Sep 2024 13:39:07 +0300 Subject: [PATCH 26/28] add tags in stream.yml.hbs --- .../data_stream/datastore/agent/stream/stream.yml.hbs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/vsphere/data_stream/datastore/agent/stream/stream.yml.hbs b/packages/vsphere/data_stream/datastore/agent/stream/stream.yml.hbs index 322880bb3bb..fdda4f6f2a3 100644 --- a/packages/vsphere/data_stream/datastore/agent/stream/stream.yml.hbs +++ b/packages/vsphere/data_stream/datastore/agent/stream/stream.yml.hbs @@ -8,7 +8,13 @@ username: {{username}} password: {{password}} # If insecure is true, don't verify the server's certificate chain insecure: {{insecure}} + +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} + {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} From 47317a7bce676d8d54fb751645a232764b8070f9 Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Thu, 19 Sep 2024 13:47:41 +0300 Subject: [PATCH 27/28] add system tests --- .../_dev/test/pipeline/test-datastore.json | 55 ++++++++++++++++++ .../test-datastore.json-expected.json | 58 +++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json create mode 100644 packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json-expected.json diff --git a/packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json b/packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json new file mode 100644 index 00000000000..b44197b3212 --- /dev/null +++ b/packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json @@ -0,0 +1,55 @@ +{ + "events": [ + { + "vsphere": { + "datastore": { + "disk": { + "capacity": { + "usage": { + "bytes": 520505786368 + }, + "bytes": 1610344300544 + }, + "provisioned": { + "bytes": 520505786368 + } + }, + "capacity": { + "free": { + "bytes": 37120094208 + }, + "total": { + "bytes": 74686664704 + }, + "used": { + "bytes": 37566570496, + "pct": 0.502988996026061 + } + }, + "fstype": "OTHER", + "host": { + "count": 1, + "names": "DC0_H0" + }, + "name": "LocalDS_0", + "read": { + "bytes": 1024 + }, + "status": "green", + "vm": { + "count": 4, + "names": [ + "DC0_C0_RP0_VM0", + "DC0_C0_RP0_VM1", + "DC0_H0_VM0", + "DC0_H0_VM1" + ] + }, + "write": { + "bytes": 450560 + } + } + } + } + ] +} \ No newline at end of file diff --git a/packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json-expected.json b/packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json-expected.json new file mode 100644 index 00000000000..6e1ac127741 --- /dev/null +++ b/packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json-expected.json @@ -0,0 +1,58 @@ +{ + "expected": [ + { + "ecs": { + "version": "8.11.0" + }, + "vsphere": { + "datastore": { + "capacity": { + "free": { + "bytes": 37120094208 + }, + "total": { + "bytes": 74686664704 + }, + "used": { + "bytes": 37566570496, + "pct": 0.502988996026061 + } + }, + "disk": { + "capacity": { + "bytes": 1610344300544, + "usage": { + "bytes": 520505786368 + } + }, + "provisioned": { + "bytes": 520505786368 + } + }, + "fstype": "OTHER", + "host": { + "count": 1, + "names": "DC0_H0" + }, + "name": "LocalDS_0", + "read": { + "bytes": 1024 + }, + "status": "green", + "vm": { + "count": 4, + "names": [ + "DC0_C0_RP0_VM0", + "DC0_C0_RP0_VM1", + "DC0_H0_VM0", + "DC0_H0_VM1" + ] + }, + "write": { + "bytes": 450560 + } + } + } + } + ] +} \ No newline at end of file From 9c15224b38e6df3e2e16abda67ed9b3bbca93fde Mon Sep 17 00:00:00 2001 From: Lucian Ioan Date: Thu, 19 Sep 2024 13:50:28 +0300 Subject: [PATCH 28/28] update readme --- packages/vsphere/docs/README.md | 93 +++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 29 deletions(-) diff --git a/packages/vsphere/docs/README.md b/packages/vsphere/docs/README.md index 5ea5bf7633d..36dcaa727ff 100644 --- a/packages/vsphere/docs/README.md +++ b/packages/vsphere/docs/README.md @@ -172,50 +172,52 @@ An example event for `datastore` looks as following: ```json { - "@timestamp": "2023-06-29T08:03:30.114Z", + "@timestamp": "2024-09-02T10:04:25.122Z", "agent": { - "ephemeral_id": "8b019ff3-cbda-41fa-b1ff-974d482b9694", - "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", - "name": "docker-fleet-agent", + "ephemeral_id": "4da294a3-ad54-47f4-92c7-544e1356a0d8", + "id": "b01ab3cf-51ad-4c4d-87bd-fc2d4aa59d8a", + "name": "elastic-agent-76236", "type": "metricbeat", - "version": "8.8.1" + "version": "8.16.0" }, "data_stream": { "dataset": "vsphere.datastore", - "namespace": "ep", + "namespace": "86691", "type": "metrics" }, "ecs": { "version": "8.11.0" }, "elastic_agent": { - "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", - "snapshot": false, - "version": "8.8.1" + "id": "b01ab3cf-51ad-4c4d-87bd-fc2d4aa59d8a", + "snapshot": true, + "version": "8.16.0" }, "event": { "agent_id_status": "verified", "dataset": "vsphere.datastore", - "duration": 23155458, - "ingested": "2023-06-29T08:03:31Z", + "duration": 190147614, + "ingested": "2024-09-02T10:04:27Z", "module": "vsphere" }, "host": { - "architecture": "aarch64", - "containerized": false, - "hostname": "docker-fleet-agent", - "id": "d08b346fbb8f49f5a2bb1a477f8ceb54", + "architecture": "x86_64", + "containerized": true, + "hostname": "elastic-agent-76236", + "id": "e744630f9d4f43dc818e497d221bd0b2", "ip": [ - "172.23.0.7" + "172.18.0.4", + "172.21.0.2" ], "mac": [ - "02-42-AC-17-00-07" + "02-42-AC-12-00-04", + "02-42-AC-15-00-02" ], - "name": "docker-fleet-agent", + "name": "elastic-agent-76236", "os": { "codename": "focal", "family": "debian", - "kernel": "5.10.104-linuxkit", + "kernel": "5.15.153.1-microsoft-standard-WSL2", "name": "Ubuntu", "platform": "ubuntu", "type": "linux", @@ -227,25 +229,56 @@ An example event for `datastore` looks as following: "period": 10000 }, "service": { - "address": "https://elastic-package-service_vsphere-metrics_1:8989/sdk", + "address": "https://svc-vsphere-metrics:8989/sdk", "type": "vsphere" }, "vsphere": { "datastore": { + "disk": { + "capacity": { + "usage": { + "bytes": 520505786368 + }, + "bytes": 1610344300544 + }, + "provisioned": { + "bytes": 520505786368 + } + }, "capacity": { "free": { - "bytes": 47869427712 + "bytes": 37120094208 }, "total": { - "bytes": 62725623808 + "bytes": 74686664704 }, "used": { - "bytes": 14856196096, - "pct": 0.237 + "bytes": 37566570496, + "pct": 0.502988996026061 } }, "fstype": "OTHER", - "name": "LocalDS_0" + "host": { + "count": 1, + "names": "DC0_H0" + }, + "name": "LocalDS_0", + "read": { + "bytes": 1024 + }, + "status": "green", + "vm": { + "count": 4, + "names": [ + "DC0_C0_RP0_VM0", + "DC0_C0_RP0_VM1", + "DC0_H0_VM0", + "DC0_H0_VM1" + ] + }, + "write": { + "bytes": 450560 + } } } } @@ -287,16 +320,18 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | vsphere.datastore.fstype | Filesystem type. | keyword | | | | vsphere.datastore.host.count | Number of hosts associated with the datastore. | long | | gauge | | vsphere.datastore.host.names | List of all the host names associated with the datastore. | keyword | | | -| vsphere.datastore.iops | Storage I/O Control aggregated Input/Output Operations Per Second. | long | | gauge | | vsphere.datastore.name | Datastore name. | keyword | | | | vsphere.datastore.read.bytes | Rate of reading data from the datastore. | long | byte | gauge | -| vsphere.datastore.read.latency.total.ms | Average amount of time for a read operation from the datastore in milliseconds. | long | ms | gauge | | vsphere.datastore.status | Status of the datastore. | keyword | | | -| vsphere.datastore.triggerd_alarms.\* | List of all the triggerd alarms. | object | | | +| vsphere.datastore.triggered_alarms.description | Description of the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.entity_name | Name of the entity associated with the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.id | Unique identifier for the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.name | Name of the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.status | Status of the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.triggered_time | Time when the alarm was triggered. | date | | | | vsphere.datastore.vm.count | Number of VMs associated with the datastore. | long | | gauge | | vsphere.datastore.vm.names | List of all the VM names associated with the datastore. | keyword | | | | vsphere.datastore.write.bytes | Rate of writing data to the datastore. | long | byte | gauge | -| vsphere.datastore.write.latency.total.ms | Average amount of time for a write operation from the datastore in milliseconds. | long | ms | gauge | ### Host