From 83484e42ec7b199fbdf387e731f116af5e12a2b6 Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Tue, 12 Dec 2023 20:56:33 +0100 Subject: [PATCH 1/4] [beat] Set cluster UUID in state metricset correctly --- metricbeat/module/beat/state/data.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/metricbeat/module/beat/state/data.go b/metricbeat/module/beat/state/data.go index b555c84bd402..5d8231046043 100644 --- a/metricbeat/module/beat/state/data.go +++ b/metricbeat/module/beat/state/data.go @@ -77,22 +77,19 @@ func eventMapping(r mb.ReporterV2, info beat.Info, content []byte, isXpack bool) return fmt.Errorf("failure parsing Beat's State API response: %w", err) } - event.MetricSetFields, _ = schema.Apply(data) - clusterUUID := getMonitoringClusterUUID(data) if clusterUUID == "" { if isOutputES(data) { clusterUUID = getClusterUUID(data) - if clusterUUID != "" { - event.ModuleFields.Put("elasticsearch.cluster.id", clusterUUID) - if event.MetricSetFields != nil { - event.MetricSetFields.Put("cluster.uuid", clusterUUID) - } + if clusterUUID == "" { + return nil } } } + event.ModuleFields.Put("elasticsearch.cluster.id", clusterUUID) + event.MetricSetFields, _ = schema.Apply(data) if event.MetricSetFields != nil { From 10ec882e87b0cb2586a63a389bba99f90704e3dd Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Wed, 13 Dec 2023 11:12:52 +0100 Subject: [PATCH 2/4] Add test --- .../beat/state/_meta/test/uuid_es_output.json | 43 +++++++++ .../test/uuid_es_output_pre_connect.json | 43 +++++++++ .../_meta/test/uuid_monitoring_config.json | 52 +++++++++++ .../_meta/test/uuid_no_monitoring_config.json | 52 +++++++++++ metricbeat/module/beat/state/data_test.go | 89 +++++++++++++++++++ 5 files changed, 279 insertions(+) create mode 100644 metricbeat/module/beat/state/_meta/test/uuid_es_output.json create mode 100644 metricbeat/module/beat/state/_meta/test/uuid_es_output_pre_connect.json create mode 100644 metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json create mode 100644 metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json diff --git a/metricbeat/module/beat/state/_meta/test/uuid_es_output.json b/metricbeat/module/beat/state/_meta/test/uuid_es_output.json new file mode 100644 index 000000000000..3cabff1ad069 --- /dev/null +++ b/metricbeat/module/beat/state/_meta/test/uuid_es_output.json @@ -0,0 +1,43 @@ +{ + "beat": { + "name": "Shaunaks-MBP-2" + }, + "host": { + "architecture": "x86_64", + "hostname": "Shaunaks-MBP-2", + "id": "EF6274EA-462F-5316-A14A-850E7BFD8126", + "os": { + "build": "18F132", + "family": "darwin", + "kernel": "18.6.0", + "name": "Mac OS X", + "platform": "darwin", + "version": "10.14.5" + } + }, + "management": { + "enabled": false + }, + "module": { + "count": 3, + "names": [ + "system" + ] + }, + "output": { + "name": "elasticsearch" + }, + "outputs": { + "elasticsearch": { + "cluster_uuid": "uuid_from_es_output" + } + }, + "queue": { + "name": "mem" + }, + "service": { + "id": "1f0c187b-f2ef-4950-b9cc-dd6864b9191a", + "name": "metricbeat", + "version": "8.0.0" + } +} diff --git a/metricbeat/module/beat/state/_meta/test/uuid_es_output_pre_connect.json b/metricbeat/module/beat/state/_meta/test/uuid_es_output_pre_connect.json new file mode 100644 index 000000000000..557b4163d359 --- /dev/null +++ b/metricbeat/module/beat/state/_meta/test/uuid_es_output_pre_connect.json @@ -0,0 +1,43 @@ +{ + "beat": { + "name": "Shaunaks-MBP-2" + }, + "host": { + "architecture": "x86_64", + "hostname": "Shaunaks-MBP-2", + "id": "EF6274EA-462F-5316-A14A-850E7BFD8126", + "os": { + "build": "18F132", + "family": "darwin", + "kernel": "18.6.0", + "name": "Mac OS X", + "platform": "darwin", + "version": "10.14.5" + } + }, + "management": { + "enabled": false + }, + "module": { + "count": 3, + "names": [ + "system" + ] + }, + "output": { + "name": "elasticsearch" + }, + "outputs": { + "elasticsearch": { + "cluster_uuid": "" + } + }, + "queue": { + "name": "mem" + }, + "service": { + "id": "1f0c187b-f2ef-4950-b9cc-dd6864b9191a", + "name": "metricbeat", + "version": "8.0.0" + } +} diff --git a/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json b/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json new file mode 100644 index 000000000000..d05e311f7599 --- /dev/null +++ b/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json @@ -0,0 +1,52 @@ +{ + "beat": { + "name": "MacBook-Pro" + }, + "host": { + "architecture": "arm64", + "hostname": "MacBook-Pro", + "id": "B0C1E948-241E-53FD-A6A3-0D0F352403AF", + "os": { + "build": "23B92", + "family": "darwin", + "kernel": "23.1.0", + "name": "macOS", + "platform": "darwin", + "version": "14.1.2" + } + }, + "input": { + "count": 1, + "names": [ + "log" + ] + }, + "management": { + "enabled": false + }, + "module": { + "count": 0, + "names": [] + }, + "monitoring": { + "cluster_uuid": "uuid_from_monitoring_config" + }, + "output": { + "batch_size": 2048, + "clients": 1, + "name": "logstash" + }, + "outputs": { + "elasticsearch": { + "cluster_uuid": "" + } + }, + "queue": { + "name": "mem" + }, + "service": { + "id": "8d6af17b-cf55-4029-bd22-64cd538acbd0", + "name": "filebeat", + "version": "8.13.0" + } +} \ No newline at end of file diff --git a/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json b/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json new file mode 100644 index 000000000000..891292a0b07a --- /dev/null +++ b/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json @@ -0,0 +1,52 @@ +{ + "beat": { + "name": "MacBook-Pro" + }, + "host": { + "architecture": "arm64", + "hostname": "MacBook-Pro", + "id": "B0C1E948-241E-53FD-A6A3-0D0F352403AF", + "os": { + "build": "23B92", + "family": "darwin", + "kernel": "23.1.0", + "name": "macOS", + "platform": "darwin", + "version": "14.1.2" + } + }, + "input": { + "count": 1, + "names": [ + "log" + ] + }, + "management": { + "enabled": false + }, + "module": { + "count": 0, + "names": [] + }, + "monitoring": { + "cluster_uuid": "" + }, + "output": { + "batch_size": 2048, + "clients": 1, + "name": "logstash" + }, + "outputs": { + "elasticsearch": { + "cluster_uuid": "" + } + }, + "queue": { + "name": "mem" + }, + "service": { + "id": "8d6af17b-cf55-4029-bd22-64cd538acbd0", + "name": "filebeat", + "version": "8.13.0" + } +} \ No newline at end of file diff --git a/metricbeat/module/beat/state/data_test.go b/metricbeat/module/beat/state/data_test.go index 6123b7539b7e..0167f4fb181d 100644 --- a/metricbeat/module/beat/state/data_test.go +++ b/metricbeat/module/beat/state/data_test.go @@ -53,3 +53,92 @@ func TestEventMapping(t *testing.T) { require.Equal(t, 0, len(reporter.GetErrors()), f) } } + +func TestUuidFromEsOutput(t *testing.T) { + reporter := &mbtest.CapturingReporterV2{} + + info := beat.Info{ + UUID: "1234", + Beat: "testbeat", + } + + input, err := ioutil.ReadFile("./_meta/test/uuid_es_output.json") + require.NoError(t, err) + + err = eventMapping(reporter, info, input, true) + require.NoError(t, err) + require.True(t, len(reporter.GetEvents()) >= 1) + require.Equal(t, 0, len(reporter.GetErrors())) + + event := reporter.GetEvents()[0] + + uuid, err := event.ModuleFields.GetValue("elasticsearch.cluster.id") + require.NoError(t, err) + + require.Equal(t, "uuid_from_es_output", uuid) +} + +func TestNoEventIfEsOutputButNoUuidYet(t *testing.T) { + reporter := &mbtest.CapturingReporterV2{} + + info := beat.Info{ + UUID: "1234", + Beat: "testbeat", + } + + input, err := ioutil.ReadFile("./_meta/test/uuid_es_output_pre_connect.json") + require.NoError(t, err) + + err = eventMapping(reporter, info, input, true) + require.NoError(t, err) + require.Equal(t, 0, len(reporter.GetEvents())) + require.Equal(t, 0, len(reporter.GetErrors())) +} + +func TestUuidFromMonitoringConfig(t *testing.T) { + reporter := &mbtest.CapturingReporterV2{} + + info := beat.Info{ + UUID: "1234", + Beat: "testbeat", + } + + input, err := ioutil.ReadFile("./_meta/test/uuid_monitoring_config.json") + require.NoError(t, err) + + err = eventMapping(reporter, info, input, true) + require.NoError(t, err) + require.True(t, len(reporter.GetEvents()) >= 1) + require.Equal(t, 0, len(reporter.GetErrors())) + + event := reporter.GetEvents()[0] + + uuid, err := event.ModuleFields.GetValue("elasticsearch.cluster.id") + require.NoError(t, err) + + require.Equal(t, "uuid_from_monitoring_config", uuid) +} + +func TestNoUuidInMonitoringConfig(t *testing.T) { + reporter := &mbtest.CapturingReporterV2{} + + info := beat.Info{ + UUID: "1234", + Beat: "testbeat", + } + + input, err := ioutil.ReadFile("./_meta/test/uuid_no_monitoring_config.json") + require.NoError(t, err) + + err = eventMapping(reporter, info, input, true) + require.NoError(t, err) + require.True(t, len(reporter.GetEvents()) >= 1) + require.Equal(t, 0, len(reporter.GetErrors())) + + event := reporter.GetEvents()[0] + + uuid, err := event.ModuleFields.GetValue("elasticsearch.cluster.id") + require.NoError(t, err) + + require.Equal(t, "", uuid) +} From 82fe57982821bb5924fa2ccfc0667bd81b92993a Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Wed, 13 Dec 2023 11:28:09 +0100 Subject: [PATCH 3/4] Fix file ending with new line --- .../module/beat/state/_meta/test/uuid_monitoring_config.json | 2 +- .../module/beat/state/_meta/test/uuid_no_monitoring_config.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json b/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json index d05e311f7599..1464e65b6402 100644 --- a/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json +++ b/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json @@ -49,4 +49,4 @@ "name": "filebeat", "version": "8.13.0" } -} \ No newline at end of file +} diff --git a/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json b/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json index 891292a0b07a..c204e26d2491 100644 --- a/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json +++ b/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json @@ -49,4 +49,4 @@ "name": "filebeat", "version": "8.13.0" } -} \ No newline at end of file +} From e5347d5a5c2fed03fdd464f1c10a4cdb1cd06f5d Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Wed, 13 Dec 2023 11:35:43 +0100 Subject: [PATCH 4/4] Fix spacing error --- metricbeat/module/beat/state/data_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/beat/state/data_test.go b/metricbeat/module/beat/state/data_test.go index 0167f4fb181d..4d94a3f24da9 100644 --- a/metricbeat/module/beat/state/data_test.go +++ b/metricbeat/module/beat/state/data_test.go @@ -91,7 +91,7 @@ func TestNoEventIfEsOutputButNoUuidYet(t *testing.T) { err = eventMapping(reporter, info, input, true) require.NoError(t, err) - require.Equal(t, 0, len(reporter.GetEvents())) + require.Equal(t, 0, len(reporter.GetEvents())) require.Equal(t, 0, len(reporter.GetErrors())) }