From 6f6e57c877b3a18985bb1a7432abb9cd560a1438 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Sat, 20 Feb 2021 11:34:22 +0800 Subject: [PATCH] Add service name to dataset field (#4674) (#4847) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Conflicts: # changelogs/head.asciidoc Co-authored-by: Juan Álvarez --- datastreams/servicename.go | 1 - model/error.go | 3 ++- model/error_test.go | 8 +++--- model/metricset.go | 8 +++--- model/metricset_test.go | 10 +++---- model/profile.go | 3 ++- model/profile_test.go | 2 +- model/span.go | 4 ++- model/span_test.go | 6 ++--- model/transaction.go | 4 ++- model/transaction_test.go | 2 +- .../jaeger_sampling_rate.approved.json | 6 ++--- .../metadata_jaeger-no-language.approved.json | 2 +- .../metadata_jaeger-version.approved.json | 2 +- .../metadata_jaeger.approved.json | 2 +- .../span_jaeger_custom.approved.json | 2 +- .../span_jaeger_db.approved.json | 2 +- .../span_jaeger_http.approved.json | 14 +++++----- ...span_jaeger_http_status_code.approved.json | 2 +- ...an_jaeger_https_default_port.approved.json | 2 +- .../span_jaeger_messaging.approved.json | 2 +- ...pan_jaeger_subtype_component.approved.json | 2 +- .../transaction_jaeger_custom.approved.json | 2 +- .../transaction_jaeger_full.approved.json | 14 +++++----- .../transaction_jaeger_no_attrs.approved.json | 2 +- ...action_jaeger_type_component.approved.json | 2 +- ...action_jaeger_type_messaging.approved.json | 2 +- ...nsaction_jaeger_type_request.approved.json | 2 +- ...n_jaeger_type_request_result.approved.json | 2 +- .../testIntakeIntegrationErrors.approved.json | 10 +++---- .../testIntakeIntegrationEvents.approved.json | 8 +++--- ...ntakeIntegrationInvalidEvent.approved.json | 2 +- ...eIntegrationInvalidJSONEvent.approved.json | 2 +- ...ntegrationMetadataNullValues.approved.json | 2 +- ...tIntakeIntegrationMetricsets.approved.json | 6 ++--- ...akeIntegrationMinimalService.approved.json | 4 +-- ...ntegrationOptionalTimestamps.approved.json | 6 ++--- ...stIntakeIntegrationRumErrors.approved.json | 2 +- ...keIntegrationRumTransactions.approved.json | 4 +-- .../testIntakeIntegrationSpans.approved.json | 12 ++++----- ...ntakeIntegrationTransactions.approved.json | 8 +++--- .../testIntakeRUMV3Errors.approved.json | 2 +- .../testIntakeRUMV3Events.approved.json | 26 +++++++++---------- .../TestDataStreamsEnabled/true.approved.json | 2 +- 44 files changed, 109 insertions(+), 102 deletions(-) diff --git a/datastreams/servicename.go b/datastreams/servicename.go index d6208237f6e..a72703a5c1c 100644 --- a/datastreams/servicename.go +++ b/datastreams/servicename.go @@ -25,7 +25,6 @@ import "strings" // Concretely, this function will lowercase the string and replace any // reserved characters with "_". // -// TODO: use when Fleet supports variables in data streams (see #4492) func NormalizeServiceName(s string) string { s = strings.ToLower(s) s = strings.Map(replaceReservedRune, s) diff --git a/model/error.go b/model/error.go index 3c10dfda6b8..0865eec4b29 100644 --- a/model/error.go +++ b/model/error.go @@ -121,7 +121,8 @@ func (e *Error) Transform(ctx context.Context, cfg *transform.Config) []beat.Eve // By storing errors in a "logs" data stream, they can be viewed in the Logs app // in Kibana. fields[datastreams.TypeField] = datastreams.LogsType - fields[datastreams.DatasetField] = ErrorsDataset + dataset := fmt.Sprintf("%s.%s", ErrorsDataset, datastreams.NormalizeServiceName(e.Metadata.Service.Name)) + fields[datastreams.DatasetField] = dataset } // first set the generic metadata (order is relevant) diff --git a/model/error_test.go b/model/error_test.go index 50d33b264fe..ebcc27dd199 100644 --- a/model/error_test.go +++ b/model/error_test.go @@ -308,7 +308,7 @@ func TestEvents(t *testing.T) { Transformable: &Error{Timestamp: timestamp, Metadata: md}, Output: common.MapStr{ "data_stream.type": "logs", - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.myservice", "agent": common.MapStr{"name": "go", "version": "1.0"}, "service": common.MapStr{"name": "myservice", "version": "1.0"}, "error": common.MapStr{ @@ -323,7 +323,7 @@ func TestEvents(t *testing.T) { Transformable: &Error{Timestamp: timestamp, Metadata: md, TransactionSampled: &sampledFalse}, Output: common.MapStr{ "data_stream.type": "logs", - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.myservice", "transaction": common.MapStr{"sampled": false}, "agent": common.MapStr{"name": "go", "version": "1.0"}, "service": common.MapStr{"name": "myservice", "version": "1.0"}, @@ -339,7 +339,7 @@ func TestEvents(t *testing.T) { Transformable: &Error{Timestamp: timestamp, Metadata: md, TransactionType: &transactionType}, Output: common.MapStr{ "data_stream.type": "logs", - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.myservice", "transaction": common.MapStr{"type": "request"}, "error": common.MapStr{ "grouping_key": "d41d8cd98f00b204e9800998ecf8427e", @@ -370,7 +370,7 @@ func TestEvents(t *testing.T) { Output: common.MapStr{ "data_stream.type": "logs", - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.myservice", "labels": common.MapStr{"key": true, "label": 101}, "service": common.MapStr{"name": "myservice", "version": "1.0"}, "agent": common.MapStr{"name": "go", "version": "1.0"}, diff --git a/model/metricset.go b/model/metricset.go index 4a8c94e6b3a..5019962247d 100644 --- a/model/metricset.go +++ b/model/metricset.go @@ -19,6 +19,7 @@ package model import ( "context" + "fmt" "time" "github.com/elastic/beats/v7/libbeat/beat" @@ -199,15 +200,16 @@ func (me *Metricset) Transform(ctx context.Context, cfg *transform.Config) []bea fields["processor"] = metricsetProcessorEntry if cfg.DataStreams { + dataset := AppMetricsDataset // Metrics are stored in "metrics" data streams. if isInternal { // Metrics that include well-defined transaction/span fields // (i.e. breakdown metrics, transaction and span metrics) will // be stored separately from application and runtime metrics. - fields[datastreams.DatasetField] = InternalMetricsDataset - } else { - fields[datastreams.DatasetField] = AppMetricsDataset + dataset = InternalMetricsDataset } + dataset += fmt.Sprintf(".%s", datastreams.NormalizeServiceName(me.Metadata.Service.Name)) + fields[datastreams.DatasetField] = dataset fields[datastreams.TypeField] = datastreams.MetricsType } diff --git a/model/metricset_test.go b/model/metricset_test.go index 79b8bbb87db..130f009a209 100644 --- a/model/metricset_test.go +++ b/model/metricset_test.go @@ -63,7 +63,7 @@ func TestTransform(t *testing.T) { Output: []common.MapStr{ { "data_stream.type": "metrics", - "data_stream.dataset": "apm.app", + "data_stream.dataset": "apm.app.myservice", "processor": common.MapStr{"event": "metric", "name": "metric"}, "service": common.MapStr{ "name": "myservice", @@ -91,7 +91,7 @@ func TestTransform(t *testing.T) { Output: []common.MapStr{ { "data_stream.type": "metrics", - "data_stream.dataset": "apm.app", + "data_stream.dataset": "apm.app.myservice", "processor": common.MapStr{"event": "metric", "name": "metric"}, "service": common.MapStr{"name": "myservice"}, "labels": common.MapStr{"a_b": "a.b.value"}, @@ -116,7 +116,7 @@ func TestTransform(t *testing.T) { Output: []common.MapStr{ { "data_stream.type": "metrics", - "data_stream.dataset": "apm.internal", + "data_stream.dataset": "apm.internal.myservice", "processor": common.MapStr{"event": "metric", "name": "metric"}, "service": common.MapStr{"name": "myservice"}, "transaction": common.MapStr{"type": trType, "name": trName}, @@ -154,7 +154,7 @@ func TestTransform(t *testing.T) { Output: []common.MapStr{ { "data_stream.type": "metrics", - "data_stream.dataset": "apm.internal", + "data_stream.dataset": "apm.internal.myservice", "processor": common.MapStr{"event": "metric", "name": "metric"}, "service": common.MapStr{"name": "myservice"}, "event": common.MapStr{"outcome": eventOutcome}, @@ -197,7 +197,7 @@ func TestTransform(t *testing.T) { Output: []common.MapStr{ { "data_stream.type": "metrics", - "data_stream.dataset": "apm.internal", + "data_stream.dataset": "apm.internal.myservice", "processor": common.MapStr{"event": "metric", "name": "metric"}, "service": common.MapStr{"name": "myservice"}, "span": common.MapStr{"type": spType, "subtype": spSubtype, diff --git a/model/profile.go b/model/profile.go index 1c3be53bfa2..c779fdd7aee 100644 --- a/model/profile.go +++ b/model/profile.go @@ -128,7 +128,8 @@ func (pp PprofProfile) Transform(ctx context.Context, cfg *transform.Config) []b } if cfg.DataStreams { event.Fields[datastreams.TypeField] = datastreams.MetricsType - event.Fields[datastreams.DatasetField] = ProfilesDataset + dataset := fmt.Sprintf("%s.%s", ProfilesDataset, datastreams.NormalizeServiceName(pp.Metadata.Service.Name)) + event.Fields[datastreams.DatasetField] = dataset } var profileLabels common.MapStr if len(sample.Label) > 0 { diff --git a/model/profile_test.go b/model/profile_test.go index f4a0b038330..f5c1d520fae 100644 --- a/model/profile_test.go +++ b/model/profile_test.go @@ -98,7 +98,7 @@ func TestPprofProfileTransform(t *testing.T) { Timestamp: timestamp, Fields: common.MapStr{ "data_stream.type": "metrics", - "data_stream.dataset": "apm.profiling", + "data_stream.dataset": "apm.profiling.myservice", "processor": common.MapStr{"event": "profile", "name": "profile"}, "service": common.MapStr{ "name": "myService", diff --git a/model/span.go b/model/span.go index 2e3c0377f00..896b07e4dc6 100644 --- a/model/span.go +++ b/model/span.go @@ -19,6 +19,7 @@ package model import ( "context" + "fmt" "net" "time" @@ -198,7 +199,8 @@ func (e *Span) Transform(ctx context.Context, cfg *transform.Config) []beat.Even if cfg.DataStreams { // Spans are stored in a "traces" data stream along with transactions. fields[datastreams.TypeField] = datastreams.TracesType - fields[datastreams.DatasetField] = TracesDataset + dataset := fmt.Sprintf("%s.%s", TracesDataset, datastreams.NormalizeServiceName(e.Metadata.Service.Name)) + fields[datastreams.DatasetField] = dataset } // first set the generic metadata diff --git a/model/span_test.go b/model/span_test.go index 6e6ef285cf1..64789fac59c 100644 --- a/model/span_test.go +++ b/model/span_test.go @@ -59,7 +59,7 @@ func TestSpanTransform(t *testing.T) { Span: Span{Timestamp: timestamp, Metadata: metadata}, Output: common.MapStr{ "data_stream.type": "traces", - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.myservice", "processor": common.MapStr{"event": "span", "name": "transaction"}, "service": common.MapStr{"name": serviceName, "environment": env, "version": serviceVersion}, "span": common.MapStr{ @@ -77,7 +77,7 @@ func TestSpanTransform(t *testing.T) { Span: Span{Timestamp: timestamp, Metadata: metadata, Outcome: "success"}, Output: common.MapStr{ "data_stream.type": "traces", - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.myservice", "processor": common.MapStr{"event": "span", "name": "transaction"}, "service": common.MapStr{"name": serviceName, "environment": env, "version": serviceVersion}, "span": common.MapStr{ @@ -126,7 +126,7 @@ func TestSpanTransform(t *testing.T) { }, Output: common.MapStr{ "data_stream.type": "traces", - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.myservice", "span": common.MapStr{ "id": hexID, "duration": common.MapStr{"us": 1200}, diff --git a/model/transaction.go b/model/transaction.go index 3a491524eac..6ca97a839a8 100644 --- a/model/transaction.go +++ b/model/transaction.go @@ -19,6 +19,7 @@ package model import ( "context" + "fmt" "time" "github.com/elastic/beats/v7/libbeat/beat" @@ -121,7 +122,8 @@ func (e *Transaction) Transform(_ context.Context, cfg *transform.Config) []beat if cfg.DataStreams { // Transactions are stored in a "traces" data stream along with spans. fields[datastreams.TypeField] = datastreams.TracesType - fields[datastreams.DatasetField] = TracesDataset + dataset := fmt.Sprintf("%s.%s", TracesDataset, datastreams.NormalizeServiceName(e.Metadata.Service.Name)) + fields[datastreams.DatasetField] = dataset } // first set generic metadata (order is relevant) diff --git a/model/transaction_test.go b/model/transaction_test.go index 5fa32b34468..31dc11c080b 100644 --- a/model/transaction_test.go +++ b/model/transaction_test.go @@ -182,7 +182,7 @@ func TestEventsTransformWithMetadata(t *testing.T) { require.Len(t, events, 1) assert.Equal(t, events[0].Fields, common.MapStr{ "data_stream.type": "traces", - "data_stream.dataset": "apm", + "data_stream.dataset": "apm." + serviceName, "user": common.MapStr{"id": "123", "name": "jane"}, "client": common.MapStr{"ip": ip}, "source": common.MapStr{"ip": ip}, diff --git a/processor/otel/test_approved/jaeger_sampling_rate.approved.json b/processor/otel/test_approved/jaeger_sampling_rate.approved.json index 28bfb67a1b4..672b5722af6 100644 --- a/processor/otel/test_approved/jaeger_sampling_rate.approved.json +++ b/processor/otel/test_approved/jaeger_sampling_rate.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -46,7 +46,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -90,7 +90,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/otel/test_approved/metadata_jaeger-no-language.approved.json b/processor/otel/test_approved/metadata_jaeger-no-language.approved.json index 0dc34aa4121..1ab9045e498 100644 --- a/processor/otel/test_approved/metadata_jaeger-no-language.approved.json +++ b/processor/otel/test_approved/metadata_jaeger-no-language.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "3.4.12" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/otel/test_approved/metadata_jaeger-version.approved.json b/processor/otel/test_approved/metadata_jaeger-version.approved.json index a648c2fad35..e51b27c7dd6 100644 --- a/processor/otel/test_approved/metadata_jaeger-version.approved.json +++ b/processor/otel/test_approved/metadata_jaeger-version.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger/PHP", "version": "3.4.12" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/otel/test_approved/metadata_jaeger.approved.json b/processor/otel/test_approved/metadata_jaeger.approved.json index 1a9a8aa2057..432db874f60 100644 --- a/processor/otel/test_approved/metadata_jaeger.approved.json +++ b/processor/otel/test_approved/metadata_jaeger.approved.json @@ -7,7 +7,7 @@ "name": "Jaeger/C++", "version": "3.2.1" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.foo", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/otel/test_approved/span_jaeger_custom.approved.json b/processor/otel/test_approved/span_jaeger_custom.approved.json index 5ceab2067b1..ba8adfa6bfe 100644 --- a/processor/otel/test_approved/span_jaeger_custom.approved.json +++ b/processor/otel/test_approved/span_jaeger_custom.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/otel/test_approved/span_jaeger_db.approved.json b/processor/otel/test_approved/span_jaeger_db.approved.json index 6c1a5f48212..39be0278b28 100644 --- a/processor/otel/test_approved/span_jaeger_db.approved.json +++ b/processor/otel/test_approved/span_jaeger_db.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "destination": { "address": "db", diff --git a/processor/otel/test_approved/span_jaeger_http.approved.json b/processor/otel/test_approved/span_jaeger_http.approved.json index 6ce3d7864a3..976f6da05a1 100644 --- a/processor/otel/test_approved/span_jaeger_http.approved.json +++ b/processor/otel/test_approved/span_jaeger_http.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "destination": { "address": "foo.bar.com", @@ -80,7 +80,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.unknown", "data_stream.type": "logs", "error": { "exception": [ @@ -141,7 +141,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.unknown", "data_stream.type": "logs", "error": { "grouping_key": "23b7ac1bdf1ca957f9f581cfadee467c", @@ -197,7 +197,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.unknown", "data_stream.type": "logs", "error": { "exception": [ @@ -255,7 +255,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.unknown", "data_stream.type": "logs", "error": { "exception": [ @@ -313,7 +313,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.unknown", "data_stream.type": "logs", "error": { "exception": [ @@ -371,7 +371,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.unknown", "data_stream.type": "logs", "error": { "grouping_key": "c9221918248f05433f6b81c46a666aee", diff --git a/processor/otel/test_approved/span_jaeger_http_status_code.approved.json b/processor/otel/test_approved/span_jaeger_http_status_code.approved.json index 9e41b864067..26cc09c28eb 100644 --- a/processor/otel/test_approved/span_jaeger_http_status_code.approved.json +++ b/processor/otel/test_approved/span_jaeger_http_status_code.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "destination": { "address": "foo.bar.com", diff --git a/processor/otel/test_approved/span_jaeger_https_default_port.approved.json b/processor/otel/test_approved/span_jaeger_https_default_port.approved.json index 77127b6392b..ad21552a76f 100644 --- a/processor/otel/test_approved/span_jaeger_https_default_port.approved.json +++ b/processor/otel/test_approved/span_jaeger_https_default_port.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "destination": { "address": "foo.bar.com", diff --git a/processor/otel/test_approved/span_jaeger_messaging.approved.json b/processor/otel/test_approved/span_jaeger_messaging.approved.json index 1c1a27dee3e..8f043d35258 100644 --- a/processor/otel/test_approved/span_jaeger_messaging.approved.json +++ b/processor/otel/test_approved/span_jaeger_messaging.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "destination": { "address": "mq", diff --git a/processor/otel/test_approved/span_jaeger_subtype_component.approved.json b/processor/otel/test_approved/span_jaeger_subtype_component.approved.json index bab6c03e440..f6741349620 100644 --- a/processor/otel/test_approved/span_jaeger_subtype_component.approved.json +++ b/processor/otel/test_approved/span_jaeger_subtype_component.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/otel/test_approved/transaction_jaeger_custom.approved.json b/processor/otel/test_approved/transaction_jaeger_custom.approved.json index 5e82aabde88..ce93b5aa208 100644 --- a/processor/otel/test_approved/transaction_jaeger_custom.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_custom.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/otel/test_approved/transaction_jaeger_full.approved.json b/processor/otel/test_approved/transaction_jaeger_full.approved.json index a7299ac5dea..8410ffce440 100644 --- a/processor/otel/test_approved/transaction_jaeger_full.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_full.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "event": { "outcome": "failure" @@ -75,7 +75,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.unknown", "data_stream.type": "logs", "error": { "exception": [ @@ -142,7 +142,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.unknown", "data_stream.type": "logs", "error": { "grouping_key": "23b7ac1bdf1ca957f9f581cfadee467c", @@ -204,7 +204,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.unknown", "data_stream.type": "logs", "error": { "exception": [ @@ -268,7 +268,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.unknown", "data_stream.type": "logs", "error": { "exception": [ @@ -332,7 +332,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.unknown", "data_stream.type": "logs", "error": { "exception": [ @@ -396,7 +396,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.unknown", "data_stream.type": "logs", "error": { "grouping_key": "c9221918248f05433f6b81c46a666aee", diff --git a/processor/otel/test_approved/transaction_jaeger_no_attrs.approved.json b/processor/otel/test_approved/transaction_jaeger_no_attrs.approved.json index def1b95ac71..e07547abab9 100644 --- a/processor/otel/test_approved/transaction_jaeger_no_attrs.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_no_attrs.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "event": { "outcome": "failure" diff --git a/processor/otel/test_approved/transaction_jaeger_type_component.approved.json b/processor/otel/test_approved/transaction_jaeger_type_component.approved.json index def3ffc0f73..b2122d431ea 100644 --- a/processor/otel/test_approved/transaction_jaeger_type_component.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_type_component.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json b/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json index 5867df8200e..31bd074272d 100644 --- a/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/otel/test_approved/transaction_jaeger_type_request.approved.json b/processor/otel/test_approved/transaction_jaeger_type_request.approved.json index a7384f462ca..7ec6ece460c 100644 --- a/processor/otel/test_approved/transaction_jaeger_type_request.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_type_request.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "event": { "outcome": "failure" diff --git a/processor/otel/test_approved/transaction_jaeger_type_request_result.approved.json b/processor/otel/test_approved/transaction_jaeger_type_request_result.approved.json index f0307e59e43..1e0d1e234e5 100644 --- a/processor/otel/test_approved/transaction_jaeger_type_request_result.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_type_request_result.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.unknown", "data_stream.type": "traces", "event": { "outcome": "success" diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationErrors.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationErrors.approved.json index d02a858f54b..28cb7e0501c 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationErrors.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationErrors.approved.json @@ -36,7 +36,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.service1", "data_stream.type": "logs", "error": { "culprit": "my.module.function_name", @@ -383,7 +383,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.1234_service_12a3", "data_stream.type": "logs", "error": { "grouping_key": "dc8dd667f7036ec5f0bae87bf2188243", @@ -492,7 +492,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.1234_service_12a3", "data_stream.type": "logs", "error": { "exception": [ @@ -597,7 +597,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.service1", "data_stream.type": "logs", "error": { "exception": [ @@ -708,7 +708,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.1234_service_12a3", "data_stream.type": "logs", "error": { "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json index b8bdea2433b..d44e28bd1f4 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json @@ -13,7 +13,7 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.experimental_java", "data_stream.type": "traces", "event": { "outcome": "success" @@ -196,7 +196,7 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.1234_service_12a3", "data_stream.type": "traces", "event": { "outcome": "success" @@ -343,7 +343,7 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.internal", + "data_stream.dataset": "apm.internal.1234_service_12a3", "data_stream.type": "metrics", "dotted": { "float": { @@ -469,7 +469,7 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.service1", "data_stream.type": "logs", "error": { "culprit": "opbeans.controllers.DTInterceptor.preHandle(DTInterceptor.java:73)", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidEvent.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidEvent.approved.json index 3c0be4e4581..f659fa936a9 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidEvent.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidEvent.approved.json @@ -6,7 +6,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.1234_service_12a3", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidJSONEvent.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidJSONEvent.approved.json index 3c0be4e4581..f659fa936a9 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidJSONEvent.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidJSONEvent.approved.json @@ -6,7 +6,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.1234_service_12a3", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationMetadataNullValues.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationMetadataNullValues.approved.json index c15059bb4b6..28a23968ebf 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationMetadataNullValues.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationMetadataNullValues.approved.json @@ -6,7 +6,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.1234_service_12a3", "data_stream.type": "logs", "error": { "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationMetricsets.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationMetricsets.approved.json index ca9c0d4fe7b..3737d299e03 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationMetricsets.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationMetricsets.approved.json @@ -7,7 +7,7 @@ "version": "3.14.0" }, "byte_counter": 1, - "data_stream.dataset": "apm.internal", + "data_stream.dataset": "apm.internal.1234_service_12a3", "data_stream.type": "metrics", "dotted": { "float": { @@ -99,7 +99,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.app", + "data_stream.dataset": "apm.app.1234_service_12a3", "data_stream.type": "metrics", "go": { "memstats": { @@ -145,7 +145,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.app", + "data_stream.dataset": "apm.app.1234_service_12a3", "data_stream.type": "metrics", "host": { "ip": "192.0.0.1" diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationMinimalService.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationMinimalService.approved.json index 36770bde9be..a1532e7cbad 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationMinimalService.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationMinimalService.approved.json @@ -6,7 +6,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.app", + "data_stream.dataset": "apm.app.1234_service_12a3", "data_stream.type": "metrics", "go": { "memstats": { @@ -37,7 +37,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.1234_service_12a3", "data_stream.type": "logs", "error": { "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationOptionalTimestamps.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationOptionalTimestamps.approved.json index db7425b4784..959e3e3c64f 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationOptionalTimestamps.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationOptionalTimestamps.approved.json @@ -6,7 +6,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.backendspans", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -83,7 +83,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.backendspans", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -165,7 +165,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.app", + "data_stream.dataset": "apm.app.backendspans", "data_stream.type": "metrics", "host": { "architecture": "x64", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationRumErrors.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationRumErrors.approved.json index 502f1b0ba89..943b273fc39 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationRumErrors.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationRumErrors.approved.json @@ -9,7 +9,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.apm_agent_js", "data_stream.type": "logs", "error": { "culprit": "test/e2e/general-usecase/bundle.js.map", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationRumTransactions.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationRumTransactions.approved.json index e7c7de01031..79051b7e7da 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationRumTransactions.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationRumTransactions.approved.json @@ -9,7 +9,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.apm_agent_js", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -77,7 +77,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.apm_agent_js", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json index 52f6cc1e36b..fc4db25284e 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json @@ -37,7 +37,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.backendspans", "data_stream.type": "traces", "event": { "outcome": "success" @@ -157,7 +157,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.backendspans", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -278,7 +278,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.backendspans", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -403,7 +403,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.backendspans", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -525,7 +525,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.backendspans", "data_stream.type": "traces", "destination": { "address": "0:0::0:1", @@ -729,7 +729,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.backendspans", "data_stream.type": "traces", "destination": { "address": "0:0::0:1", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json index f13793d2d78..e28295adac5 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json @@ -32,7 +32,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.1234_service_12a3", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -154,7 +154,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.1234_service_12a3", "data_stream.type": "traces", "event": { "outcome": "success" @@ -367,7 +367,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.service1", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -512,7 +512,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.1234_service_12a3", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/stream/test_approved_es_documents/testIntakeRUMV3Errors.approved.json b/processor/stream/test_approved_es_documents/testIntakeRUMV3Errors.approved.json index e58bbaba770..9d4b28ce273 100644 --- a/processor/stream/test_approved_es_documents/testIntakeRUMV3Errors.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeRUMV3Errors.approved.json @@ -9,7 +9,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.error", + "data_stream.dataset": "apm.error.apm_a_rum_test_e2e_general_usecase", "data_stream.type": "logs", "error": { "culprit": "test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret", diff --git a/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json b/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json index 4711169427a..6cf3af3b8c7 100644 --- a/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json @@ -9,7 +9,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", "data_stream.type": "traces", "event": { "outcome": "success" @@ -157,7 +157,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -228,7 +228,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -299,7 +299,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", "data_stream.type": "traces", "destination": { "address": "localhost", @@ -391,7 +391,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -461,7 +461,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", "data_stream.type": "traces", "destination": { "address": "localhost", @@ -553,7 +553,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", "data_stream.type": "traces", "destination": { "address": "localhost", @@ -645,7 +645,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", "data_stream.type": "traces", "destination": { "address": "localhost", @@ -738,7 +738,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", "data_stream.type": "traces", "event": { "outcome": "success" @@ -831,7 +831,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.internal", + "data_stream.dataset": "apm.internal.apm_a_rum_test_e2e_general_usecase", "data_stream.type": "metrics", "labels": { "testTagKey": "testTagValue" @@ -888,7 +888,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.internal", + "data_stream.dataset": "apm.internal.apm_a_rum_test_e2e_general_usecase", "data_stream.type": "metrics", "labels": { "testTagKey": "testTagValue" @@ -945,7 +945,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.internal", + "data_stream.dataset": "apm.internal.apm_a_rum_test_e2e_general_usecase", "data_stream.type": "metrics", "labels": { "testTagKey": "testTagValue" @@ -1002,7 +1002,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.internal", + "data_stream.dataset": "apm.internal.apm_a_rum_test_e2e_general_usecase", "data_stream.type": "metrics", "labels": { "tag1": "value1", diff --git a/systemtest/approvals/TestDataStreamsEnabled/true.approved.json b/systemtest/approvals/TestDataStreamsEnabled/true.approved.json index 28ca86e8ed5..c9a6191a011 100644 --- a/systemtest/approvals/TestDataStreamsEnabled/true.approved.json +++ b/systemtest/approvals/TestDataStreamsEnabled/true.approved.json @@ -6,7 +6,7 @@ "name": "go", "version": "0.0.0" }, - "data_stream.dataset": "apm", + "data_stream.dataset": "apm.systemtest", "data_stream.namespace": "default", "data_stream.type": "traces", "ecs": {