From 03670c2b55e960d94f43213f7d4f11c98ec94168 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 11 Oct 2022 18:28:27 +0200 Subject: [PATCH 1/9] Add system tests for logstash package (cherry picked from commit 4df6b991a6db0cb851598c34063363d90f1c0749) --- .../_dev/test/system/test-default-config.yml | 5 + .../data_stream/log/sample_event.json | 21 +- .../_dev/test/system/test-default-config.yml | 8 + .../data_stream/node/fields/fields.yml | 2 + .../data_stream/node/sample_event.json | 207 ++++++------ .../_dev/test/system/test-default-config.yml | 8 + .../data_stream/node_stats/fields/fields.yml | 170 ++++++---- .../node_stats/fields/package-fields.yml | 6 +- .../data_stream/node_stats/sample_event.json | 295 ++++++++++++++---- .../_dev/test/system/test-default-config.yml | 5 + .../data_stream/slowlog/sample_event.json | 27 +- 11 files changed, 507 insertions(+), 247 deletions(-) create mode 100644 packages/logstash/data_stream/log/_dev/test/system/test-default-config.yml create mode 100644 packages/logstash/data_stream/node/_dev/test/system/test-default-config.yml create mode 100644 packages/logstash/data_stream/node_stats/_dev/test/system/test-default-config.yml create mode 100644 packages/logstash/data_stream/slowlog/_dev/test/system/test-default-config.yml diff --git a/packages/logstash/data_stream/log/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/log/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..6363f89d6c1 --- /dev/null +++ b/packages/logstash/data_stream/log/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +input: logfile +data_stream: + vars: + paths: + - "{{SERVICE_LOGS_DIR}}/logstash/logstash-json.log" diff --git a/packages/logstash/data_stream/log/sample_event.json b/packages/logstash/data_stream/log/sample_event.json index 7746611f36f..8a72e5e68e2 100644 --- a/packages/logstash/data_stream/log/sample_event.json +++ b/packages/logstash/data_stream/log/sample_event.json @@ -1,8 +1,8 @@ { - "@timestamp": "2022-09-13T19:08:48.030Z", + "@timestamp": "2022-10-11T14:03:32.641Z", "agent": { - "ephemeral_id": "c028b260-d373-4eab-978c-08bc42b9dc7c", - "id": "aeed8481-2d37-45f5-989f-daa7c2173ca0", + "ephemeral_id": "e1088945-f8e4-466b-83fd-f636ffbf9bdf", + "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606", "name": "docker-fleet-agent", "type": "filebeat", "version": "8.5.0" @@ -16,33 +16,34 @@ "version": "1.10.0" }, "elastic_agent": { - "id": "aeed8481-2d37-45f5-989f-daa7c2173ca0", + "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606", "snapshot": true, "version": "8.5.0" }, "event": { "agent_id_status": "verified", + "created": "2022-10-11T14:03:32.641Z", "dataset": "logstash.log", - "ingested": "2022-09-13T19:08:58Z", + "ingested": "2022-10-11T14:03:44Z", "kind": "event", "type": "info" }, "host": { "architecture": "x86_64", - "containerized": true, + "containerized": false, "hostname": "docker-fleet-agent", - "id": "8127511256f0493fa1abf625ca3e0609", + "id": "b6bc6723e51b43959ce07f0c3105c72d", "ip": [ - "172.20.0.8" + "192.168.0.7" ], "mac": [ - "02-42-AC-14-00-08" + "02-42-C0-A8-00-07" ], "name": "docker-fleet-agent", "os": { "codename": "focal", "family": "debian", - "kernel": "5.10.47-linuxkit", + "kernel": "5.10.124-linuxkit", "name": "Ubuntu", "platform": "ubuntu", "type": "linux", diff --git a/packages/logstash/data_stream/node/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/node/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..aa617605b1d --- /dev/null +++ b/packages/logstash/data_stream/node/_dev/test/system/test-default-config.yml @@ -0,0 +1,8 @@ +type: logstash/metrics +dataset: logstash.stack_monitoring.node +vars: + hosts: + - "http://elastic-package-service-logstash-1:9600" + username: elastic + password: changeme +data_stream: ~ diff --git a/packages/logstash/data_stream/node/fields/fields.yml b/packages/logstash/data_stream/node/fields/fields.yml index 4e71f8e6ce1..50bdc5f8114 100644 --- a/packages/logstash/data_stream/node/fields/fields.yml +++ b/packages/logstash/data_stream/node/fields/fields.yml @@ -1,6 +1,8 @@ - name: logstash type: group fields: + - name: cluster.id + type: keyword - name: elasticsearch.cluster.id type: keyword - name: node diff --git a/packages/logstash/data_stream/node/sample_event.json b/packages/logstash/data_stream/node/sample_event.json index 6584ddcde97..c5682eb6c54 100644 --- a/packages/logstash/data_stream/node/sample_event.json +++ b/packages/logstash/data_stream/node/sample_event.json @@ -1,101 +1,144 @@ { - "process": { - "pid": 1 - }, + "@timestamp": "2022-10-11T14:04:44.089Z", "agent": { - "hostname": "docker-fleet-agent", + "ephemeral_id": "1a1ca75b-a20f-4ae4-82a9-4e269c855a5d", + "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606", "name": "docker-fleet-agent", - "id": "0c223a58-fac1-457d-84d2-13b4cc188cd8", - "ephemeral_id": "14484f41-a26f-44c9-adf0-fc0f1495b4f3", "type": "metricbeat", - "version": "7.15.0" + "version": "8.5.0" + }, + "data_stream": { + "dataset": "logstash.stack_monitoring.node", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" }, "elastic_agent": { - "id": "0c223a58-fac1-457d-84d2-13b4cc188cd8", - "version": "7.15.0", - "snapshot": true + "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606", + "snapshot": true, + "version": "8.5.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "logstash.stack_monitoring.node", + "duration": 131377542, + "ingested": "2022-10-11T14:04:45Z", + "module": "logstash" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "b6bc6723e51b43959ce07f0c3105c72d", + "ip": [ + "192.168.0.7" + ], + "mac": [ + "02-42-C0-A8-00-07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.124-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" + } }, "logstash": { + "cluster": { + "id": "U8DCOXCFQHWlaKczNT4LNQ" + }, + "elasticsearch": { + "cluster": { + "id": "U8DCOXCFQHWlaKczNT4LNQ" + } + }, "node": { - "host": "2cb47f6e0eab", - "version": "8.0.0", + "host": "17a6005cfeaa", + "id": "7d7ee953-cf82-4d1d-91e0-1714346531de", "jvm": { - "version": "11.0.5" + "version": "17.0.4" }, - "id": "4cc683ce-3ddc-46e3-bea3-aefbf37bc082", "state": { "pipeline": { - "hash": "3000c3abf87d4dfa4a57aaf6af0a1f5bee2e0fc1c48a8e8636e2a33d7d2e91dd", - "ephemeral_id": "afb1a50a-95f0-484a-b7d7-e683ddddc75a", + "batch_size": 125, + "ephemeral_id": "3d2aff1f-dde1-4c56-9560-14f3b092f894", + "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", + "id": "pipeline-with-persisted-queue", "representation": { "graph": { "edges": [ { - "from": "1bf3a9cc73ceb7c3a9cbe885df249b23f3496c52a342a6d513153cc865d78182", - "id": "b3db599ec6ae0b9493158bd7024dcd922c8a3e76295c37fef0da440086bf3f8c", + "from": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", + "id": "9ed824e4f189b461c111ae27c17644c3c5f6d7c3c2bb213cbc7cc067cbd68fe6", "to": "__QUEUE__", "type": "plain" }, { - "type": "plain", - "from": "71b91bc85b66ab25c5fb16e63db4dd7111c183f96d1f18e19078051ed5fc74f7", - "id": "9db20a77b3e1eb91229a50bd33388425d59725f9093e076a37e6565f8d5a20ad", - "to": "__QUEUE__" + "from": "__QUEUE__", + "id": "cb33f8fb7611e31a2c1751b74cdedf5b8cdb96ea46b812a2541e2db4f13dca10", + "to": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", + "type": "plain" }, { - "id": "9b2bc571e978746fb9b55b83521a6603c3c940144cde0e3f4296298cea6585cf", - "to": "a339cb309b29181703c6adf321da3d639f5b60713de5a1e5519ebfea069556d8", - "type": "plain", - "from": "__QUEUE__" + "from": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", + "id": "63ef166c45b87a40f31e0a6def175f10460b6b0ed656e70968eb52b1c454ab16", + "to": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", + "type": "plain" } ], "vertices": [ { - "config_name": "beats", + "config_name": "java_generator", "explicit_id": false, - "id": "1bf3a9cc73ceb7c3a9cbe885df249b23f3496c52a342a6d513153cc865d78182", + "id": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", "meta": { "source": { - "line": 2, - "protocol": "file", "column": 3, - "id": "/usr/share/logstash/pipeline/default.conf" + "id": "/usr/share/logstash/pipeline/persisted-queue.conf", + "line": 2, + "protocol": "file" } }, "plugin_type": "input", "type": "plugin" }, { - "plugin_type": "input", - "type": "plugin", - "config_name": "beats", "explicit_id": false, - "id": "71b91bc85b66ab25c5fb16e63db4dd7111c183f96d1f18e19078051ed5fc74f7", + "id": "__QUEUE__", + "meta": null, + "type": "queue" + }, + { + "config_name": "sleep", + "explicit_id": false, + "id": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", "meta": { "source": { - "protocol": "file", "column": 3, - "id": "/usr/share/logstash/pipeline/default.conf", - "line": 7 + "id": "/usr/share/logstash/pipeline/persisted-queue.conf", + "line": 8, + "protocol": "file" } - } - }, - { - "explicit_id": false, - "id": "__QUEUE__", - "meta": null, - "type": "queue" + }, + "plugin_type": "filter", + "type": "plugin" }, { "config_name": "elasticsearch", "explicit_id": false, - "id": "a339cb309b29181703c6adf321da3d639f5b60713de5a1e5519ebfea069556d8", + "id": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", "meta": { "source": { - "id": "/usr/share/logstash/pipeline/default.conf", - "line": 17, - "protocol": "file", - "column": 3 + "column": 3, + "id": "/usr/share/logstash/pipeline/persisted-queue.conf", + "line": 15, + "protocol": "file" } }, "plugin_type": "output", @@ -103,65 +146,29 @@ } ] }, + "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", "type": "lir", - "version": "0.0.0", - "hash": "3000c3abf87d4dfa4a57aaf6af0a1f5bee2e0fc1c48a8e8636e2a33d7d2e91dd" + "version": "0.0.0" }, - "batch_size": 125, - "workers": 12, - "id": "main" + "workers": 7 } - } + }, + "version": "8.5.0" } }, - "@timestamp": "2021-09-02T17:31:04.592Z", - "ecs": { - "version": "1.10.0" + "metricset": { + "name": "node", + "period": 10000 }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "logstash.node" + "process": { + "pid": 1 }, "service": { - "hostname": "45943bf17069", - "address": "http://logstash:9600/_node", + "address": "http://elastic-package-service-logstash-1:9600/_node", + "hostname": "17a6005cfeaa", + "id": "7d7ee953-cf82-4d1d-91e0-1714346531de", "name": "logstash", - "id": "8cfe1a39-ac50-439d-8bf2-93198aa26c0d", "type": "logstash", - "version": "8.0.0" - }, - "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "family": "redhat", - "type": "linux", - "version": "7 (Core)", - "platform": "centos" - }, - "ip": [ - "172.25.0.4" - ], - "containerized": true, - "name": "docker-fleet-agent", - "id": "1292624d19b2cee1a317ad634c9a8358", - "mac": [ - "02:42:ac:19:00:04" - ], - "architecture": "x86_64" - }, - "metricset": { - "period": 10000, - "name": "node" - }, - "event": { - "duration": 13519531, - "agent_id_status": "verified", - "ingested": "2021-09-02T17:31:05.607256453Z", - "module": "logstash", - "dataset": "logstash.node" + "version": "8.5.0" } } \ No newline at end of file diff --git a/packages/logstash/data_stream/node_stats/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/node_stats/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..f6f7967f71a --- /dev/null +++ b/packages/logstash/data_stream/node_stats/_dev/test/system/test-default-config.yml @@ -0,0 +1,8 @@ +type: logstash/metrics +dataset: logstash.stack_monitoring.node_stats +vars: + hosts: + - "http://elastic-package-service-logstash-1:9600" + username: elastic + password: changeme +data_stream: ~ diff --git a/packages/logstash/data_stream/node_stats/fields/fields.yml b/packages/logstash/data_stream/node_stats/fields/fields.yml index 11430915fac..6209ef6f820 100644 --- a/packages/logstash/data_stream/node_stats/fields/fields.yml +++ b/packages/logstash/data_stream/node_stats/fields/fields.yml @@ -30,6 +30,28 @@ type: long - name: heap_max_in_bytes type: long + - name: heap_used_percent + type: long + - name: gc + type: group + fields: + - name: collectors + type: group + fields: + - name: old + type: group + fields: + - name: collection_count + type: long + - name: collection_time_in_millis + type: long + - name: young + type: group + fields: + - name: collection_count + type: long + - name: collection_time_in_millis + type: long - name: events type: group fields: @@ -54,31 +76,59 @@ type: keyword - name: version type: keyword + - name: ephemeral_id + type: keyword + - name: host + type: keyword + - name: http_address + type: keyword + - name: name + type: keyword + - name: snapshot + type: boolean + - name: status + type: keyword + - name: pipeline + type: group + fields: + - name: batch_size + type: long + - name: workers + type: long - name: os type: group fields: - name: cpu type: group fields: + - name: percent + type: double - name: load_average type: group fields: - name: 15m - type: long + type: half_float - name: 1m - type: long + type: half_float - name: 5m - type: long + type: half_float - name: cgroup type: group fields: - - name: cpuacct.usage_nanos - type: long + - name: cpuacct + type: nested + fields: + - name: usage_nanos + type: long - name: cpu type: group fields: + - name: control_group + type: text + - name: cfs_quota_micros + type: long - name: stat - type: group + type: nested fields: - name: number_of_elapsed_periods type: long @@ -86,19 +136,61 @@ type: long - name: number_of_times_throttled type: long - - name: process.cpu.percent - type: double + - name: process + type: group + fields: + - name: cpu.percent + type: double + - name: max_file_descriptors + type: long + - name: open_file_descriptors + type: long - name: queue.events_count type: long - name: pipelines type: nested fields: + - name: id + type: keyword + description: id + - name: hash + type: keyword + - name: ephemeral_id + type: keyword + - name: reloads + type: group + fields: + - name: failures + type: long + - name: successes + type: long + - name: queue + type: group + fields: + - name: events_count + type: long + - name: type + type: keyword + - name: queue_size_in_bytes + type: long + - name: max_queue_size_in_bytes + type: long + - name: events + type: group + fields: + - name: in + type: long + - name: out + type: long + - name: filtered + type: long + - name: duration_in_millis + type: long + - name: queue_push_duration_in_millis + type: long - name: vertices type: nested fields: - - name: id - type: keyword - description: id - name: long_counters.name type: keyword - name: long_counters @@ -119,64 +211,10 @@ - name: queue_push_duration_in_millis type: long description: queue_push_duration_in_millis - - name: pipelines.ephemeral_id - type: keyword - - name: pipelines.id - type: keyword - - name: pipelines.hash - type: keyword - - name: pipelines.reloads + - name: reloads type: group fields: - name: failures type: long - name: successes type: long - - name: pipelines.queue - type: group - fields: - - name: events_count - type: long - - name: type - type: keyword - - name: queue_size_in_bytes - type: long - - name: max_queue_size_in_bytes - type: long - - name: pipelines.events - type: group - fields: - - name: in - type: long - - name: out - type: long - - name: filtered - type: long - - name: duration_in_millis - type: long - - name: queue_push_duration_in_millis - type: long - - name: pipelines.vertices - type: nested - - name: pipelines.vertices.id - type: keyword - description: id - - name: pipelines.vertices.long_counters - type: nested - - name: pipelines.vertices.long_counters.name - type: keyword - - name: pipelines.vertices.long_counters.value - type: long - - name: pipelines.vertices.duration_in_millis - type: long - - name: pipelines.vertices.events_in - type: long - - name: pipelines.vertices.pipeline_ephemeral_id - type: keyword - description: pipeline_ephemeral_id - - name: pipelines.vertices.events_out - type: long - description: events_out - - name: pipelines.vertices.queue_push_duration_in_millis - type: long - description: queue_push_duration_in_millis diff --git a/packages/logstash/data_stream/node_stats/fields/package-fields.yml b/packages/logstash/data_stream/node_stats/fields/package-fields.yml index 92ec0ae3e20..3d471cb573e 100644 --- a/packages/logstash/data_stream/node_stats/fields/package-fields.yml +++ b/packages/logstash/data_stream/node_stats/fields/package-fields.yml @@ -1,9 +1,9 @@ -- name: cluster_uuid +- name: logstash.cluster.id type: alias path: logstash.elasticsearch.cluster.id - name: timestamp type: alias - path: '@timestamp' + path: "@timestamp" - name: logstash_stats type: group fields: @@ -11,7 +11,7 @@ type: nested - name: timestamp type: alias - path: '@timestamp' + path: "@timestamp" - name: jvm type: group fields: diff --git a/packages/logstash/data_stream/node_stats/sample_event.json b/packages/logstash/data_stream/node_stats/sample_event.json index b52906c3f75..a1ae07f57fd 100644 --- a/packages/logstash/data_stream/node_stats/sample_event.json +++ b/packages/logstash/data_stream/node_stats/sample_event.json @@ -1,76 +1,261 @@ { + "@timestamp": "2022-10-11T14:05:39.791Z", "agent": { - "hostname": "docker-fleet-agent", + "ephemeral_id": "1a1ca75b-a20f-4ae4-82a9-4e269c855a5d", + "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606", "name": "docker-fleet-agent", - "id": "0c223a58-fac1-457d-84d2-13b4cc188cd8", "type": "metricbeat", - "ephemeral_id": "14484f41-a26f-44c9-adf0-fc0f1495b4f3", - "version": "7.15.0" - }, - "elastic_agent": { - "id": "0c223a58-fac1-457d-84d2-13b4cc188cd8", - "version": "7.15.0", - "snapshot": true + "version": "8.5.0" }, - "logstash": { - "node": { - "stats": { - "events": { - "filtered": 0, - "in": 0, - "out": 0 - } - } - } + "data_stream": { + "dataset": "logstash.stack_monitoring.node_stats", + "namespace": "ep", + "type": "metrics" }, - "@timestamp": "2021-09-02T17:29:14.596Z", "ecs": { - "version": "1.10.0" + "version": "8.0.0" }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "logstash.node_stats" + "elastic_agent": { + "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606", + "snapshot": true, + "version": "8.5.0" }, - "service": { - "hostname": "45943bf17069", - "address": "http://logstash:9600/_node/stats", - "name": "logstash", - "id": "8cfe1a39-ac50-439d-8bf2-93198aa26c0d", - "type": "logstash", - "version": "8.0.0" + "event": { + "agent_id_status": "verified", + "dataset": "logstash.stack_monitoring.node_stats", + "duration": 125822375, + "ingested": "2022-10-11T14:05:40Z", + "module": "logstash" }, "host": { + "architecture": "x86_64", + "containerized": false, "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "type": "linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, - "containerized": true, + "id": "b6bc6723e51b43959ce07f0c3105c72d", "ip": [ - "172.25.0.4" + "192.168.0.7" ], - "name": "docker-fleet-agent", - "id": "1292624d19b2cee1a317ad634c9a8358", "mac": [ - "02:42:ac:19:00:04" + "02-42-C0-A8-00-07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.124-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" + } + }, + "logstash": { + "cluster": { + "id": "U8DCOXCFQHWlaKczNT4LNQ" + }, + "elasticsearch": { + "cluster": { + "id": "U8DCOXCFQHWlaKczNT4LNQ" + } + }, + "node": { + "stats": { + "events": { + "duration_in_millis": 322, + "filtered": 132, + "in": 593, + "out": 132 + }, + "jvm": { + "gc": { + "collectors": { + "old": { + "collection_count": 0, + "collection_time_in_millis": 0 + }, + "young": { + "collection_count": 25, + "collection_time_in_millis": 269 + } + } + }, + "mem": { + "heap_max_in_bytes": 3137339390, + "heap_used_in_bytes": 208271008, + "heap_used_percent": 6 + }, + "uptime_in_millis": 17121 + }, + "logstash": { + "ephemeral_id": "59ea6513-500d-4b0b-8d54-a32d94631b1f", + "host": "ee237ad022ba", + "http_address": "0.0.0.0:9600", + "name": "ee237ad022ba", + "pipeline": { + "batch_size": 125, + "workers": 7 + }, + "snapshot": true, + "status": "green", + "uuid": "cb4f884e-d57b-43a3-bec6-7b3ec1adcbb9", + "version": "8.5.0" + }, + "os": { + "cgroup": { + "cpu": { + "control_group": "", + "stat": null + }, + "cpuacct": null + }, + "cpu": { + "load_average": { + "15m": 2.17, + "1m": 3.32, + "5m": 2.32 + }, + "percent": 0 + } + }, + "pipelines": [ + { + "ephemeral_id": "0eff59ef-d130-4753-bd4e-289341a84c1a", + "events": { + "duration_in_millis": 199, + "filtered": 86, + "in": 92, + "out": 86, + "queue_push_duration_in_millis": 4 + }, + "hash": "0542fa70daa36dc3e858ea099f125cc8c9e451ebbfe8ea8867e52f9764da0a35", + "id": "pipeline-with-memory-queue", + "queue": { + "events_count": 0, + "max_queue_size_in_bytes": 0, + "queue_size_in_bytes": 0, + "type": "memory" + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "vertices": [ + { + "events_out": 92, + "id": "4c5941552cdaa72ebc285557c697a7150c359ee3eacf9b5664c4b1048e26153b", + "pipeline_ephemeral_id": "0eff59ef-d130-4753-bd4e-289341a84c1a", + "queue_push_duration_in_millis": 4 + }, + { + "cluster_uuid": "U8DCOXCFQHWlaKczNT4LNQ", + "duration_in_millis": 197, + "events_in": 86, + "events_out": 86, + "id": "635a080aacc8700059852859da284a9cb92cb78a6d7112fbf55e441e51b6658a", + "long_counters": [ + { + "name": "bulk_requests.successes", + "value": 15 + }, + { + "name": "bulk_requests.responses.200", + "value": 15 + }, + { + "name": "documents.successes", + "value": 86 + } + ], + "pipeline_ephemeral_id": "0eff59ef-d130-4753-bd4e-289341a84c1a" + } + ] + }, + { + "ephemeral_id": "5ba3b3b3-4d82-4877-b96e-f327335bf1e1", + "events": { + "duration_in_millis": 0, + "filtered": 0, + "in": 456, + "out": 0, + "queue_push_duration_in_millis": 52 + }, + "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", + "id": "pipeline-with-persisted-queue", + "queue": { + "capacity": { + "max_queue_size_in_bytes": 1073741824, + "max_unread_events": 0, + "page_capacity_in_bytes": 67108864, + "queue_size_in_bytes": 139404 + }, + "data": { + "free_space_in_bytes": 170819031040, + "path": "/usr/share/logstash/data/queue/pipeline-with-persisted-queue", + "storage_type": "overlay" + }, + "events": 0, + "events_count": 0, + "max_queue_size_in_bytes": 1073741824, + "queue_size_in_bytes": 139404, + "type": "persisted" + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "vertices": [ + { + "events_out": 456, + "id": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", + "pipeline_ephemeral_id": "5ba3b3b3-4d82-4877-b96e-f327335bf1e1", + "queue_push_duration_in_millis": 52 + }, + { + "duration_in_millis": 0, + "events_in": 375, + "events_out": 0, + "id": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", + "pipeline_ephemeral_id": "5ba3b3b3-4d82-4877-b96e-f327335bf1e1" + }, + { + "cluster_uuid": "U8DCOXCFQHWlaKczNT4LNQ", + "duration_in_millis": 0, + "events_in": 0, + "events_out": 0, + "id": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", + "pipeline_ephemeral_id": "5ba3b3b3-4d82-4877-b96e-f327335bf1e1" + } + ] + } + ], + "process": { + "cpu": { + "percent": 3 + }, + "max_file_descriptors": 1048576, + "open_file_descriptors": 86 + }, + "queue": { + "events_count": 0 + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "timestamp": "2022-10-11T14:05:39.916Z" + } + } }, "metricset": { - "period": 10000, - "name": "node_stats" + "name": "node_stats", + "period": 10000 }, - "event": { - "duration": 18621194, - "agent_id_status": "verified", - "ingested": "2021-09-02T17:29:15.608149964Z", - "module": "logstash", - "dataset": "logstash.node_stats" + "service": { + "address": "http://elastic-package-service-logstash-1:9600/_node/stats", + "hostname": "ee237ad022ba", + "id": "", + "name": "logstash", + "type": "logstash", + "version": "8.5.0" } } \ No newline at end of file diff --git a/packages/logstash/data_stream/slowlog/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/slowlog/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..f27fb3744e3 --- /dev/null +++ b/packages/logstash/data_stream/slowlog/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +input: logfile +data_stream: + vars: + paths: + - "{{SERVICE_LOGS_DIR}}/logstash/logstash-slowlog-json.log" diff --git a/packages/logstash/data_stream/slowlog/sample_event.json b/packages/logstash/data_stream/slowlog/sample_event.json index d8e1aca69b8..0f9f48f7468 100644 --- a/packages/logstash/data_stream/slowlog/sample_event.json +++ b/packages/logstash/data_stream/slowlog/sample_event.json @@ -1,8 +1,8 @@ { - "@timestamp": "2022-09-13T19:09:45.759Z", + "@timestamp": "2022-10-11T14:06:31.846Z", "agent": { - "ephemeral_id": "f3343539-9dd4-4db6-9284-e5d738fd2228", - "id": "aeed8481-2d37-45f5-989f-daa7c2173ca0", + "ephemeral_id": "445bd68e-3789-4085-bf0a-0e871b82ce72", + "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606", "name": "docker-fleet-agent", "type": "filebeat", "version": "8.5.0" @@ -16,34 +16,35 @@ "version": "1.10.0" }, "elastic_agent": { - "id": "aeed8481-2d37-45f5-989f-daa7c2173ca0", + "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606", "snapshot": true, "version": "8.5.0" }, "event": { "agent_id_status": "verified", + "created": "2022-10-11T14:06:31.846Z", "dataset": "logstash.slowlog", - "duration": 417800, - "ingested": "2022-09-13T19:09:55Z", + "duration": 287417, + "ingested": "2022-10-11T14:06:43Z", "kind": "event", "type": "info" }, "host": { "architecture": "x86_64", - "containerized": true, + "containerized": false, "hostname": "docker-fleet-agent", - "id": "8127511256f0493fa1abf625ca3e0609", + "id": "b6bc6723e51b43959ce07f0c3105c72d", "ip": [ - "172.20.0.8" + "192.168.0.7" ], "mac": [ - "02-42-AC-14-00-08" + "02-42-C0-A8-00-07" ], "name": "docker-fleet-agent", "os": { "codename": "focal", "family": "debian", - "kernel": "5.10.47-linuxkit", + "kernel": "5.10.124-linuxkit", "name": "Ubuntu", "platform": "ubuntu", "type": "linux", @@ -62,7 +63,7 @@ }, "logstash": { "slowlog": { - "event": "{\"hostname\":\"128aab07d8dc\",\"message\":\"Hello world!\",\"sequence\":0,\"@timestamp\":\"2022-09-13T19:09:43.252725100Z\",\"thread_number\":0,\"@version\":\"1\"}", + "event": "{\"message\":\"Hello world!\",\"sequence\":0,\"@version\":\"1\",\"@timestamp\":\"2022-10-11T14:06:29.325781334Z\",\"thread_number\":0,\"hostname\":\"da53f01a03cb\"}", "module": "slowlog.logstash.filters.sleep", "plugin_name": "sleep", "plugin_params_object": { @@ -71,7 +72,7 @@ "time": 1 }, "plugin_type": "filters", - "thread": "[pipeline-with-persisted-queue]\u003eworker6", + "thread": "[pipeline-with-persisted-queue]\u003eworker1", "took_in_millis": 0 } } From 72dfc90410c9cf7d6ea9f7bebeacb333785d8ae0 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 11 Oct 2022 20:47:26 +0200 Subject: [PATCH 2/9] Update README --- packages/logstash/docs/README.md | 502 +++++++++++++++++++++---------- 1 file changed, 347 insertions(+), 155 deletions(-) diff --git a/packages/logstash/docs/README.md b/packages/logstash/docs/README.md index 6c42d1bbe2c..2b868af0797 100644 --- a/packages/logstash/docs/README.md +++ b/packages/logstash/docs/README.md @@ -30,79 +30,264 @@ An example event for `node_stats` looks as following: ```json { + "@timestamp": "2022-10-11T14:05:39.791Z", "agent": { - "hostname": "docker-fleet-agent", + "ephemeral_id": "1a1ca75b-a20f-4ae4-82a9-4e269c855a5d", + "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606", "name": "docker-fleet-agent", - "id": "0c223a58-fac1-457d-84d2-13b4cc188cd8", "type": "metricbeat", - "ephemeral_id": "14484f41-a26f-44c9-adf0-fc0f1495b4f3", - "version": "7.15.0" - }, - "elastic_agent": { - "id": "0c223a58-fac1-457d-84d2-13b4cc188cd8", - "version": "7.15.0", - "snapshot": true + "version": "8.5.0" }, - "logstash": { - "node": { - "stats": { - "events": { - "filtered": 0, - "in": 0, - "out": 0 - } - } - } + "data_stream": { + "dataset": "logstash.stack_monitoring.node_stats", + "namespace": "ep", + "type": "metrics" }, - "@timestamp": "2021-09-02T17:29:14.596Z", "ecs": { - "version": "1.10.0" + "version": "8.0.0" }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "logstash.node_stats" + "elastic_agent": { + "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606", + "snapshot": true, + "version": "8.5.0" }, - "service": { - "hostname": "45943bf17069", - "address": "http://logstash:9600/_node/stats", - "name": "logstash", - "id": "8cfe1a39-ac50-439d-8bf2-93198aa26c0d", - "type": "logstash", - "version": "8.0.0" + "event": { + "agent_id_status": "verified", + "dataset": "logstash.stack_monitoring.node_stats", + "duration": 125822375, + "ingested": "2022-10-11T14:05:40Z", + "module": "logstash" }, "host": { + "architecture": "x86_64", + "containerized": false, "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "type": "linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, - "containerized": true, + "id": "b6bc6723e51b43959ce07f0c3105c72d", "ip": [ - "172.25.0.4" + "192.168.0.7" ], - "name": "docker-fleet-agent", - "id": "1292624d19b2cee1a317ad634c9a8358", "mac": [ - "02:42:ac:19:00:04" + "02-42-C0-A8-00-07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.124-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" + } + }, + "logstash": { + "cluster": { + "id": "U8DCOXCFQHWlaKczNT4LNQ" + }, + "elasticsearch": { + "cluster": { + "id": "U8DCOXCFQHWlaKczNT4LNQ" + } + }, + "node": { + "stats": { + "events": { + "duration_in_millis": 322, + "filtered": 132, + "in": 593, + "out": 132 + }, + "jvm": { + "gc": { + "collectors": { + "old": { + "collection_count": 0, + "collection_time_in_millis": 0 + }, + "young": { + "collection_count": 25, + "collection_time_in_millis": 269 + } + } + }, + "mem": { + "heap_max_in_bytes": 3137339390, + "heap_used_in_bytes": 208271008, + "heap_used_percent": 6 + }, + "uptime_in_millis": 17121 + }, + "logstash": { + "ephemeral_id": "59ea6513-500d-4b0b-8d54-a32d94631b1f", + "host": "ee237ad022ba", + "http_address": "0.0.0.0:9600", + "name": "ee237ad022ba", + "pipeline": { + "batch_size": 125, + "workers": 7 + }, + "snapshot": true, + "status": "green", + "uuid": "cb4f884e-d57b-43a3-bec6-7b3ec1adcbb9", + "version": "8.5.0" + }, + "os": { + "cgroup": { + "cpu": { + "control_group": "", + "stat": null + }, + "cpuacct": null + }, + "cpu": { + "load_average": { + "15m": 2.17, + "1m": 3.32, + "5m": 2.32 + }, + "percent": 0 + } + }, + "pipelines": [ + { + "ephemeral_id": "0eff59ef-d130-4753-bd4e-289341a84c1a", + "events": { + "duration_in_millis": 199, + "filtered": 86, + "in": 92, + "out": 86, + "queue_push_duration_in_millis": 4 + }, + "hash": "0542fa70daa36dc3e858ea099f125cc8c9e451ebbfe8ea8867e52f9764da0a35", + "id": "pipeline-with-memory-queue", + "queue": { + "events_count": 0, + "max_queue_size_in_bytes": 0, + "queue_size_in_bytes": 0, + "type": "memory" + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "vertices": [ + { + "events_out": 92, + "id": "4c5941552cdaa72ebc285557c697a7150c359ee3eacf9b5664c4b1048e26153b", + "pipeline_ephemeral_id": "0eff59ef-d130-4753-bd4e-289341a84c1a", + "queue_push_duration_in_millis": 4 + }, + { + "cluster_uuid": "U8DCOXCFQHWlaKczNT4LNQ", + "duration_in_millis": 197, + "events_in": 86, + "events_out": 86, + "id": "635a080aacc8700059852859da284a9cb92cb78a6d7112fbf55e441e51b6658a", + "long_counters": [ + { + "name": "bulk_requests.successes", + "value": 15 + }, + { + "name": "bulk_requests.responses.200", + "value": 15 + }, + { + "name": "documents.successes", + "value": 86 + } + ], + "pipeline_ephemeral_id": "0eff59ef-d130-4753-bd4e-289341a84c1a" + } + ] + }, + { + "ephemeral_id": "5ba3b3b3-4d82-4877-b96e-f327335bf1e1", + "events": { + "duration_in_millis": 0, + "filtered": 0, + "in": 456, + "out": 0, + "queue_push_duration_in_millis": 52 + }, + "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", + "id": "pipeline-with-persisted-queue", + "queue": { + "capacity": { + "max_queue_size_in_bytes": 1073741824, + "max_unread_events": 0, + "page_capacity_in_bytes": 67108864, + "queue_size_in_bytes": 139404 + }, + "data": { + "free_space_in_bytes": 170819031040, + "path": "/usr/share/logstash/data/queue/pipeline-with-persisted-queue", + "storage_type": "overlay" + }, + "events": 0, + "events_count": 0, + "max_queue_size_in_bytes": 1073741824, + "queue_size_in_bytes": 139404, + "type": "persisted" + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "vertices": [ + { + "events_out": 456, + "id": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", + "pipeline_ephemeral_id": "5ba3b3b3-4d82-4877-b96e-f327335bf1e1", + "queue_push_duration_in_millis": 52 + }, + { + "duration_in_millis": 0, + "events_in": 375, + "events_out": 0, + "id": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", + "pipeline_ephemeral_id": "5ba3b3b3-4d82-4877-b96e-f327335bf1e1" + }, + { + "cluster_uuid": "U8DCOXCFQHWlaKczNT4LNQ", + "duration_in_millis": 0, + "events_in": 0, + "events_out": 0, + "id": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", + "pipeline_ephemeral_id": "5ba3b3b3-4d82-4877-b96e-f327335bf1e1" + } + ] + } + ], + "process": { + "cpu": { + "percent": 3 + }, + "max_file_descriptors": 1048576, + "open_file_descriptors": 86 + }, + "queue": { + "events_count": 0 + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "timestamp": "2022-10-11T14:05:39.916Z" + } + } }, "metricset": { - "period": 10000, - "name": "node_stats" + "name": "node_stats", + "period": 10000 }, - "event": { - "duration": 18621194, - "agent_id_status": "verified", - "ingested": "2021-09-02T17:29:15.608149964Z", - "module": "logstash", - "dataset": "logstash.node_stats" + "service": { + "address": "http://elastic-package-service-logstash-1:9600/_node/stats", + "hostname": "ee237ad022ba", + "id": "", + "name": "logstash", + "type": "logstash", + "version": "8.5.0" } } ``` @@ -161,103 +346,146 @@ An example event for `node` looks as following: ```json { - "process": { - "pid": 1 - }, + "@timestamp": "2022-10-11T14:04:44.089Z", "agent": { - "hostname": "docker-fleet-agent", + "ephemeral_id": "1a1ca75b-a20f-4ae4-82a9-4e269c855a5d", + "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606", "name": "docker-fleet-agent", - "id": "0c223a58-fac1-457d-84d2-13b4cc188cd8", - "ephemeral_id": "14484f41-a26f-44c9-adf0-fc0f1495b4f3", "type": "metricbeat", - "version": "7.15.0" + "version": "8.5.0" + }, + "data_stream": { + "dataset": "logstash.stack_monitoring.node", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" }, "elastic_agent": { - "id": "0c223a58-fac1-457d-84d2-13b4cc188cd8", - "version": "7.15.0", - "snapshot": true + "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606", + "snapshot": true, + "version": "8.5.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "logstash.stack_monitoring.node", + "duration": 131377542, + "ingested": "2022-10-11T14:04:45Z", + "module": "logstash" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "b6bc6723e51b43959ce07f0c3105c72d", + "ip": [ + "192.168.0.7" + ], + "mac": [ + "02-42-C0-A8-00-07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.124-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" + } }, "logstash": { + "cluster": { + "id": "U8DCOXCFQHWlaKczNT4LNQ" + }, + "elasticsearch": { + "cluster": { + "id": "U8DCOXCFQHWlaKczNT4LNQ" + } + }, "node": { - "host": "2cb47f6e0eab", - "version": "8.0.0", + "host": "17a6005cfeaa", + "id": "7d7ee953-cf82-4d1d-91e0-1714346531de", "jvm": { - "version": "11.0.5" + "version": "17.0.4" }, - "id": "4cc683ce-3ddc-46e3-bea3-aefbf37bc082", "state": { "pipeline": { - "hash": "3000c3abf87d4dfa4a57aaf6af0a1f5bee2e0fc1c48a8e8636e2a33d7d2e91dd", - "ephemeral_id": "afb1a50a-95f0-484a-b7d7-e683ddddc75a", + "batch_size": 125, + "ephemeral_id": "3d2aff1f-dde1-4c56-9560-14f3b092f894", + "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", + "id": "pipeline-with-persisted-queue", "representation": { "graph": { "edges": [ { - "from": "1bf3a9cc73ceb7c3a9cbe885df249b23f3496c52a342a6d513153cc865d78182", - "id": "b3db599ec6ae0b9493158bd7024dcd922c8a3e76295c37fef0da440086bf3f8c", + "from": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", + "id": "9ed824e4f189b461c111ae27c17644c3c5f6d7c3c2bb213cbc7cc067cbd68fe6", "to": "__QUEUE__", "type": "plain" }, { - "type": "plain", - "from": "71b91bc85b66ab25c5fb16e63db4dd7111c183f96d1f18e19078051ed5fc74f7", - "id": "9db20a77b3e1eb91229a50bd33388425d59725f9093e076a37e6565f8d5a20ad", - "to": "__QUEUE__" + "from": "__QUEUE__", + "id": "cb33f8fb7611e31a2c1751b74cdedf5b8cdb96ea46b812a2541e2db4f13dca10", + "to": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", + "type": "plain" }, { - "id": "9b2bc571e978746fb9b55b83521a6603c3c940144cde0e3f4296298cea6585cf", - "to": "a339cb309b29181703c6adf321da3d639f5b60713de5a1e5519ebfea069556d8", - "type": "plain", - "from": "__QUEUE__" + "from": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", + "id": "63ef166c45b87a40f31e0a6def175f10460b6b0ed656e70968eb52b1c454ab16", + "to": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", + "type": "plain" } ], "vertices": [ { - "config_name": "beats", + "config_name": "java_generator", "explicit_id": false, - "id": "1bf3a9cc73ceb7c3a9cbe885df249b23f3496c52a342a6d513153cc865d78182", + "id": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", "meta": { "source": { - "line": 2, - "protocol": "file", "column": 3, - "id": "/usr/share/logstash/pipeline/default.conf" + "id": "/usr/share/logstash/pipeline/persisted-queue.conf", + "line": 2, + "protocol": "file" } }, "plugin_type": "input", "type": "plugin" }, { - "plugin_type": "input", - "type": "plugin", - "config_name": "beats", "explicit_id": false, - "id": "71b91bc85b66ab25c5fb16e63db4dd7111c183f96d1f18e19078051ed5fc74f7", + "id": "__QUEUE__", + "meta": null, + "type": "queue" + }, + { + "config_name": "sleep", + "explicit_id": false, + "id": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", "meta": { "source": { - "protocol": "file", "column": 3, - "id": "/usr/share/logstash/pipeline/default.conf", - "line": 7 + "id": "/usr/share/logstash/pipeline/persisted-queue.conf", + "line": 8, + "protocol": "file" } - } - }, - { - "explicit_id": false, - "id": "__QUEUE__", - "meta": null, - "type": "queue" + }, + "plugin_type": "filter", + "type": "plugin" }, { "config_name": "elasticsearch", "explicit_id": false, - "id": "a339cb309b29181703c6adf321da3d639f5b60713de5a1e5519ebfea069556d8", + "id": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", "meta": { "source": { - "id": "/usr/share/logstash/pipeline/default.conf", - "line": 17, - "protocol": "file", - "column": 3 + "column": 3, + "id": "/usr/share/logstash/pipeline/persisted-queue.conf", + "line": 15, + "protocol": "file" } }, "plugin_type": "output", @@ -265,66 +493,30 @@ An example event for `node` looks as following: } ] }, + "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", "type": "lir", - "version": "0.0.0", - "hash": "3000c3abf87d4dfa4a57aaf6af0a1f5bee2e0fc1c48a8e8636e2a33d7d2e91dd" + "version": "0.0.0" }, - "batch_size": 125, - "workers": 12, - "id": "main" + "workers": 7 } - } + }, + "version": "8.5.0" } }, - "@timestamp": "2021-09-02T17:31:04.592Z", - "ecs": { - "version": "1.10.0" + "metricset": { + "name": "node", + "period": 10000 }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "logstash.node" + "process": { + "pid": 1 }, "service": { - "hostname": "45943bf17069", - "address": "http://logstash:9600/_node", + "address": "http://elastic-package-service-logstash-1:9600/_node", + "hostname": "17a6005cfeaa", + "id": "7d7ee953-cf82-4d1d-91e0-1714346531de", "name": "logstash", - "id": "8cfe1a39-ac50-439d-8bf2-93198aa26c0d", "type": "logstash", - "version": "8.0.0" - }, - "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "family": "redhat", - "type": "linux", - "version": "7 (Core)", - "platform": "centos" - }, - "ip": [ - "172.25.0.4" - ], - "containerized": true, - "name": "docker-fleet-agent", - "id": "1292624d19b2cee1a317ad634c9a8358", - "mac": [ - "02:42:ac:19:00:04" - ], - "architecture": "x86_64" - }, - "metricset": { - "period": 10000, - "name": "node" - }, - "event": { - "duration": 13519531, - "agent_id_status": "verified", - "ingested": "2021-09-02T17:31:05.607256453Z", - "module": "logstash", - "dataset": "logstash.node" + "version": "8.5.0" } } ``` From 6f69d52c265b8049faca0a739870c9976b8911d7 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 12 Oct 2022 13:43:51 +0200 Subject: [PATCH 3/9] Test .env file --- packages/kibana/_dev/deploy/docker/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/kibana/_dev/deploy/docker/docker-compose.yml b/packages/kibana/_dev/deploy/docker/docker-compose.yml index c1254a9765e..fa6a1dacabd 100644 --- a/packages/kibana/_dev/deploy/docker/docker-compose.yml +++ b/packages/kibana/_dev/deploy/docker/docker-compose.yml @@ -1,6 +1,8 @@ version: "2.3" services: elasticsearch: + env_file: + - ./.env image: "docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}" environment: - "ES_JAVA_OPTS=-Xms1g -Xmx1g" From 9b9429da4c0ec8eae6c0f001001778b4726b105c Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 12 Oct 2022 14:49:10 +0200 Subject: [PATCH 4/9] Set default env var value --- packages/kibana/_dev/deploy/docker/docker-compose.yml | 2 -- packages/logstash/_dev/deploy/docker/docker-compose.yml | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/kibana/_dev/deploy/docker/docker-compose.yml b/packages/kibana/_dev/deploy/docker/docker-compose.yml index fa6a1dacabd..c1254a9765e 100644 --- a/packages/kibana/_dev/deploy/docker/docker-compose.yml +++ b/packages/kibana/_dev/deploy/docker/docker-compose.yml @@ -1,8 +1,6 @@ version: "2.3" services: elasticsearch: - env_file: - - ./.env image: "docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}" environment: - "ES_JAVA_OPTS=-Xms1g -Xmx1g" diff --git a/packages/logstash/_dev/deploy/docker/docker-compose.yml b/packages/logstash/_dev/deploy/docker/docker-compose.yml index ee35207e259..39ec613d739 100644 --- a/packages/logstash/_dev/deploy/docker/docker-compose.yml +++ b/packages/logstash/_dev/deploy/docker/docker-compose.yml @@ -1,7 +1,7 @@ -version: '2.3' +version: "2.3" services: logstash: - image: "docker.elastic.co/logstash/logstash:${ELASTIC_VERSION}" + image: "docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-8.5.0-SNAPSHOT}" volumes: - "./pipeline:/usr/share/logstash/pipeline" - "./config:/usr/share/logstash/config" From 3fdd75c9cec17999d2cdbd5a1d5a89df9ffac927 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 12 Oct 2022 16:17:29 +0200 Subject: [PATCH 5/9] Fix alias mapping --- .../data_stream/node_stats/fields/package-fields.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/logstash/data_stream/node_stats/fields/package-fields.yml b/packages/logstash/data_stream/node_stats/fields/package-fields.yml index 3d471cb573e..b2b7dba18e8 100644 --- a/packages/logstash/data_stream/node_stats/fields/package-fields.yml +++ b/packages/logstash/data_stream/node_stats/fields/package-fields.yml @@ -55,7 +55,7 @@ type: group fields: - name: stat - type: group + type: nested fields: - name: number_of_elapsed_periods type: alias @@ -81,9 +81,12 @@ - name: cgroup type: group fields: - - name: cpuacct.usage_nanos - type: alias - path: logstash.node.stats.os.cgroup.cpuacct.usage_nanos + - name: cpuacct + type: nested + fields: + - name: usage_nanos + type: alias + path: logstash.node.stats.os.cgroup.cpuacct.usage_nanos - name: process.cpu.percent type: alias path: logstash.node.stats.process.cpu.percent From 8bb09944169b40074d775a267ac476fe83875445 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 12 Oct 2022 18:08:37 +0200 Subject: [PATCH 6/9] Fix mapping and use Hostname var on test config --- .../_dev/deploy/docker/docker-compose.yml | 3 +- .../_dev/test/system/test-default-config.yml | 2 +- .../_dev/test/system/test-default-config.yml | 2 +- .../node/fields/package-fields.yml | 86 +------------------ .../_dev/test/system/test-default-config.yml | 2 +- .../data_stream/node_stats/fields/fields.yml | 4 +- .../node_stats/fields/package-fields.yml | 13 +-- .../_dev/test/system/test-default-config.yml | 2 +- 8 files changed, 11 insertions(+), 103 deletions(-) diff --git a/packages/logstash/_dev/deploy/docker/docker-compose.yml b/packages/logstash/_dev/deploy/docker/docker-compose.yml index 39ec613d739..6b0fd5d508c 100644 --- a/packages/logstash/_dev/deploy/docker/docker-compose.yml +++ b/packages/logstash/_dev/deploy/docker/docker-compose.yml @@ -1,10 +1,11 @@ version: "2.3" services: logstash: + user: root image: "docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-8.5.0-SNAPSHOT}" volumes: - "./pipeline:/usr/share/logstash/pipeline" - "./config:/usr/share/logstash/config" - - ${SERVICE_LOGS_DIR}/logstash:/usr/share/logstash/logs + - ${SERVICE_LOGS_DIR}:/usr/share/logstash/logs:rw ports: - "127.0.0.1:9600:9600" diff --git a/packages/logstash/data_stream/log/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/log/_dev/test/system/test-default-config.yml index 6363f89d6c1..1f44a2a92a0 100644 --- a/packages/logstash/data_stream/log/_dev/test/system/test-default-config.yml +++ b/packages/logstash/data_stream/log/_dev/test/system/test-default-config.yml @@ -2,4 +2,4 @@ input: logfile data_stream: vars: paths: - - "{{SERVICE_LOGS_DIR}}/logstash/logstash-json.log" + - "{{SERVICE_LOGS_DIR}}/logstash-json.log" diff --git a/packages/logstash/data_stream/node/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/node/_dev/test/system/test-default-config.yml index aa617605b1d..8869a1e437e 100644 --- a/packages/logstash/data_stream/node/_dev/test/system/test-default-config.yml +++ b/packages/logstash/data_stream/node/_dev/test/system/test-default-config.yml @@ -2,7 +2,7 @@ type: logstash/metrics dataset: logstash.stack_monitoring.node vars: hosts: - - "http://elastic-package-service-logstash-1:9600" + - "http://{{Hostname}}:9600" username: elastic password: changeme data_stream: ~ diff --git a/packages/logstash/data_stream/node/fields/package-fields.yml b/packages/logstash/data_stream/node/fields/package-fields.yml index f7af108fdca..05ed19bc52f 100644 --- a/packages/logstash/data_stream/node/fields/package-fields.yml +++ b/packages/logstash/data_stream/node/fields/package-fields.yml @@ -3,91 +3,7 @@ path: logstash.elasticsearch.cluster.id - name: timestamp type: alias - path: '@timestamp' -- name: logstash_stats - type: group - fields: - - name: timestamp - type: alias - path: '@timestamp' - - name: jvm - type: group - fields: - - name: mem - type: group - fields: - - name: heap_used_in_bytes - type: alias - path: logstash.node.stats.jvm.mem.heap_used_in_bytes - - name: heap_max_in_bytes - type: alias - path: logstash.node.stats.jvm.mem.heap_max_in_bytes - - name: uptime_in_millis - type: alias - path: logstash.node.stats.jvm.uptime_in_millis - - name: events - type: group - fields: - - name: in - type: alias - path: logstash.node.stats.events.in - - name: out - type: alias - path: logstash.node.stats.events.out - - name: duration_in_millis - type: alias - path: logstash.node.stats.events.duration_in_millis - - name: logstash - type: group - fields: - - name: uuid - type: alias - path: logstash.node.stats.logstash.uuid - - name: version - type: alias - path: logstash.node.stats.logstash.version - - name: os - type: group - fields: - - name: cpu - type: group - fields: - - name: stat - type: group - fields: - - name: number_of_elapsed_periods - type: alias - path: logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods - - name: time_throttled_nanos - type: alias - path: logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos - - name: number_of_times_throttled - type: alias - path: logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled - - name: load_average - type: group - fields: - - name: 15m - type: alias - path: logstash.node.stats.os.cpu.load_average.15m - - name: 1m - type: alias - path: logstash.node.stats.os.cpu.load_average.1m - - name: 5m - type: alias - path: logstash.node.stats.os.cpu.load_average.5m - - name: cgroup - type: group - fields: - - name: cpuacct.usage_nanos - type: alias - path: logstash.node.stats.os.cgroup.cpuacct.usage_nanos - - name: process.cpu.percent - type: alias - path: logstash.node.stats.process.cpu.percent - - name: queue.events_count - type: alias - path: logstash.node.stats.queue.events_count + path: "@timestamp" - name: logstash_state type: group fields: diff --git a/packages/logstash/data_stream/node_stats/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/node_stats/_dev/test/system/test-default-config.yml index f6f7967f71a..38dfc0c028a 100644 --- a/packages/logstash/data_stream/node_stats/_dev/test/system/test-default-config.yml +++ b/packages/logstash/data_stream/node_stats/_dev/test/system/test-default-config.yml @@ -2,7 +2,7 @@ type: logstash/metrics dataset: logstash.stack_monitoring.node_stats vars: hosts: - - "http://elastic-package-service-logstash-1:9600" + - "http://{{Hostname}}:9600" username: elastic password: changeme data_stream: ~ diff --git a/packages/logstash/data_stream/node_stats/fields/fields.yml b/packages/logstash/data_stream/node_stats/fields/fields.yml index 6209ef6f820..458a3ea4a8b 100644 --- a/packages/logstash/data_stream/node_stats/fields/fields.yml +++ b/packages/logstash/data_stream/node_stats/fields/fields.yml @@ -116,7 +116,7 @@ type: group fields: - name: cpuacct - type: nested + type: object fields: - name: usage_nanos type: long @@ -128,7 +128,7 @@ - name: cfs_quota_micros type: long - name: stat - type: nested + type: object fields: - name: number_of_elapsed_periods type: long diff --git a/packages/logstash/data_stream/node_stats/fields/package-fields.yml b/packages/logstash/data_stream/node_stats/fields/package-fields.yml index b2b7dba18e8..b446c06e0ef 100644 --- a/packages/logstash/data_stream/node_stats/fields/package-fields.yml +++ b/packages/logstash/data_stream/node_stats/fields/package-fields.yml @@ -55,7 +55,7 @@ type: group fields: - name: stat - type: nested + type: object fields: - name: number_of_elapsed_periods type: alias @@ -82,7 +82,7 @@ type: group fields: - name: cpuacct - type: nested + type: object fields: - name: usage_nanos type: alias @@ -93,12 +93,3 @@ - name: queue.events_count type: alias path: logstash.node.stats.queue.events_count -- name: logstash_state - type: group - fields: - - name: pipeline.id - type: alias - path: logstash.node.state.pipeline.id - - name: pipeline.hash - type: alias - path: logstash.node.state.pipeline.hash diff --git a/packages/logstash/data_stream/slowlog/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/slowlog/_dev/test/system/test-default-config.yml index f27fb3744e3..1869dd61299 100644 --- a/packages/logstash/data_stream/slowlog/_dev/test/system/test-default-config.yml +++ b/packages/logstash/data_stream/slowlog/_dev/test/system/test-default-config.yml @@ -2,4 +2,4 @@ input: logfile data_stream: vars: paths: - - "{{SERVICE_LOGS_DIR}}/logstash/logstash-slowlog-json.log" + - "{{SERVICE_LOGS_DIR}}/logstash-slowlog-json.log" From 3b4ffe8c04c2d02cc55b6114304252697e8f85df Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 12 Oct 2022 21:32:55 +0200 Subject: [PATCH 7/9] Add missing filed to node_stats mapping --- packages/logstash/data_stream/node_stats/fields/fields.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/logstash/data_stream/node_stats/fields/fields.yml b/packages/logstash/data_stream/node_stats/fields/fields.yml index 458a3ea4a8b..3a7837659fd 100644 --- a/packages/logstash/data_stream/node_stats/fields/fields.yml +++ b/packages/logstash/data_stream/node_stats/fields/fields.yml @@ -118,6 +118,8 @@ - name: cpuacct type: object fields: + - name: control_group + type: text - name: usage_nanos type: long - name: cpu From ec8d0b56eabcfccf4ed9091b643002babae63edc Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Fri, 14 Oct 2022 10:33:27 +0200 Subject: [PATCH 8/9] Fix docker-compose file --- packages/logstash/_dev/deploy/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/logstash/_dev/deploy/docker/docker-compose.yml b/packages/logstash/_dev/deploy/docker/docker-compose.yml index 6b0fd5d508c..fee6d006a26 100644 --- a/packages/logstash/_dev/deploy/docker/docker-compose.yml +++ b/packages/logstash/_dev/deploy/docker/docker-compose.yml @@ -6,6 +6,6 @@ services: volumes: - "./pipeline:/usr/share/logstash/pipeline" - "./config:/usr/share/logstash/config" - - ${SERVICE_LOGS_DIR}:/usr/share/logstash/logs:rw + - ${SERVICE_LOGS_DIR}:/usr/share/logstash/logs ports: - "127.0.0.1:9600:9600" From 30fc344a09bec4e213eeb01fa6e18a073da50661 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Fri, 14 Oct 2022 13:03:38 +0200 Subject: [PATCH 9/9] Fix test config and replaces .env with variantes.yml --- packages/logstash/_dev/deploy/docker/.env | 1 - packages/logstash/_dev/deploy/variants.yml | 4 ++++ .../data_stream/node/_dev/test/system/test-default-config.yml | 2 -- .../node_stats/_dev/test/system/test-default-config.yml | 2 -- 4 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 packages/logstash/_dev/deploy/docker/.env create mode 100644 packages/logstash/_dev/deploy/variants.yml diff --git a/packages/logstash/_dev/deploy/docker/.env b/packages/logstash/_dev/deploy/docker/.env deleted file mode 100644 index 0130e89172e..00000000000 --- a/packages/logstash/_dev/deploy/docker/.env +++ /dev/null @@ -1 +0,0 @@ -ELASTIC_VERSION=8.5.0-SNAPSHOT diff --git a/packages/logstash/_dev/deploy/variants.yml b/packages/logstash/_dev/deploy/variants.yml new file mode 100644 index 00000000000..c4785b200da --- /dev/null +++ b/packages/logstash/_dev/deploy/variants.yml @@ -0,0 +1,4 @@ +variants: + logstash_8.5.0: + ELASTIC_VERSION: 8.5.0-SNAPSHOT +default: logstash_8.5.0 diff --git a/packages/logstash/data_stream/node/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/node/_dev/test/system/test-default-config.yml index 8869a1e437e..cf83c82866c 100644 --- a/packages/logstash/data_stream/node/_dev/test/system/test-default-config.yml +++ b/packages/logstash/data_stream/node/_dev/test/system/test-default-config.yml @@ -3,6 +3,4 @@ dataset: logstash.stack_monitoring.node vars: hosts: - "http://{{Hostname}}:9600" - username: elastic - password: changeme data_stream: ~ diff --git a/packages/logstash/data_stream/node_stats/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/node_stats/_dev/test/system/test-default-config.yml index 38dfc0c028a..9360c0654b6 100644 --- a/packages/logstash/data_stream/node_stats/_dev/test/system/test-default-config.yml +++ b/packages/logstash/data_stream/node_stats/_dev/test/system/test-default-config.yml @@ -3,6 +3,4 @@ dataset: logstash.stack_monitoring.node_stats vars: hosts: - "http://{{Hostname}}:9600" - username: elastic - password: changeme data_stream: ~