From 7982c3ccec415c73e277039972fcffc6e93b6383 Mon Sep 17 00:00:00 2001 From: Constanca Manteigas Date: Mon, 13 Jan 2025 18:20:25 +0100 Subject: [PATCH 1/9] Add translator for cloudwatch --- .../cloudwatchencodingextension/Makefile | 1 + .../cloudwatchencodingextension/config.go | 17 ++ .../config_test.go | 1 + .../cloudwatchencodingextension/doc.go | 2 + .../cloudwatchencodingextension/extension.go | 45 ++++++ .../extension_test.go | 1 + .../cloudwatchencodingextension/factory.go | 15 ++ .../factory_test.go | 1 + .../generated_component_test.go | 49 ++++++ .../generated_package_test.go | 12 ++ .../cloudwatchencodingextension/go.mod | 57 +++++++ .../internal/metadata/generated_status.go | 16 ++ .../cloudwatchencodingextension/metadata.yaml | 9 ++ pkg/translator/cloudwatch/Makefile | 1 + pkg/translator/cloudwatch/cwlogs_to_logs.go | 97 ++++++++++++ .../cloudwatch/cwlogs_to_logs_test.go | 66 ++++++++ .../cloudwatch/cwmetrics_to_metrics.go | 145 ++++++++++++++++++ .../cloudwatch/cwmetrics_to_metrics_test.go | 57 +++++++ pkg/translator/cloudwatch/go.mod | 42 +++++ pkg/translator/cloudwatch/go.sum | 109 +++++++++++++ pkg/translator/cloudwatch/metadata.yaml | 3 + pkg/translator/cloudwatch/package_test.go | 11 ++ .../cloudwatch/testdata/log/invalid_records | 4 + .../cloudwatch/testdata/log/multiple_records | 2 + .../testdata/log/multiple_resources | 6 + .../cloudwatch/testdata/log/single_record | 1 + .../testdata/log/some_invalid_records | 3 + .../testdata/metric/invalid_records | 4 + .../testdata/metric/multiple_records | 127 +++++++++++++++ .../cloudwatch/testdata/metric/single_record | 1 + .../testdata/metric/some_invalid_records | 92 +++++++++++ 31 files changed, 997 insertions(+) create mode 100644 extension/encoding/cloudwatchencodingextension/Makefile create mode 100644 extension/encoding/cloudwatchencodingextension/config.go create mode 100644 extension/encoding/cloudwatchencodingextension/config_test.go create mode 100644 extension/encoding/cloudwatchencodingextension/doc.go create mode 100644 extension/encoding/cloudwatchencodingextension/extension.go create mode 100644 extension/encoding/cloudwatchencodingextension/extension_test.go create mode 100644 extension/encoding/cloudwatchencodingextension/factory.go create mode 100644 extension/encoding/cloudwatchencodingextension/factory_test.go create mode 100644 extension/encoding/cloudwatchencodingextension/generated_component_test.go create mode 100644 extension/encoding/cloudwatchencodingextension/generated_package_test.go create mode 100644 extension/encoding/cloudwatchencodingextension/go.mod create mode 100644 extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go create mode 100644 extension/encoding/cloudwatchencodingextension/metadata.yaml create mode 100644 pkg/translator/cloudwatch/Makefile create mode 100644 pkg/translator/cloudwatch/cwlogs_to_logs.go create mode 100644 pkg/translator/cloudwatch/cwlogs_to_logs_test.go create mode 100644 pkg/translator/cloudwatch/cwmetrics_to_metrics.go create mode 100644 pkg/translator/cloudwatch/cwmetrics_to_metrics_test.go create mode 100644 pkg/translator/cloudwatch/go.mod create mode 100644 pkg/translator/cloudwatch/go.sum create mode 100644 pkg/translator/cloudwatch/metadata.yaml create mode 100644 pkg/translator/cloudwatch/package_test.go create mode 100644 pkg/translator/cloudwatch/testdata/log/invalid_records create mode 100644 pkg/translator/cloudwatch/testdata/log/multiple_records create mode 100644 pkg/translator/cloudwatch/testdata/log/multiple_resources create mode 100644 pkg/translator/cloudwatch/testdata/log/single_record create mode 100644 pkg/translator/cloudwatch/testdata/log/some_invalid_records create mode 100644 pkg/translator/cloudwatch/testdata/metric/invalid_records create mode 100644 pkg/translator/cloudwatch/testdata/metric/multiple_records create mode 100644 pkg/translator/cloudwatch/testdata/metric/single_record create mode 100644 pkg/translator/cloudwatch/testdata/metric/some_invalid_records diff --git a/extension/encoding/cloudwatchencodingextension/Makefile b/extension/encoding/cloudwatchencodingextension/Makefile new file mode 100644 index 000000000000..bdd863a203be --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/Makefile @@ -0,0 +1 @@ +include ../../../Makefile.Common diff --git a/extension/encoding/cloudwatchencodingextension/config.go b/extension/encoding/cloudwatchencodingextension/config.go new file mode 100644 index 000000000000..db74532e917e --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/config.go @@ -0,0 +1,17 @@ +package cloudwatchencodingextension + +import ( + "go.opentelemetry.io/collector/component" +) + +type Config struct{} + +const defaultDelimiter = "\n" + +func createDefaultConfig() component.Config { + return &Config{} +} + +func (c *Config) Validate() error { + return nil +} diff --git a/extension/encoding/cloudwatchencodingextension/config_test.go b/extension/encoding/cloudwatchencodingextension/config_test.go new file mode 100644 index 000000000000..634cf7233288 --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/config_test.go @@ -0,0 +1 @@ +package cloudwatchencodingextension diff --git a/extension/encoding/cloudwatchencodingextension/doc.go b/extension/encoding/cloudwatchencodingextension/doc.go new file mode 100644 index 000000000000..c4ceff17eda5 --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/doc.go @@ -0,0 +1,2 @@ +//go:generate mdatagen metadata.yaml +package cloudwatchencodingextension diff --git a/extension/encoding/cloudwatchencodingextension/extension.go b/extension/encoding/cloudwatchencodingextension/extension.go new file mode 100644 index 000000000000..bd9bb3998cbb --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/extension.go @@ -0,0 +1,45 @@ +package cloudwatchencodingextension + +import ( + "context" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/extension" + plog "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.uber.org/zap" +) + +var ( + _ encoding.LogsUnmarshalerExtension = (*cloudwatchExtension)(nil) + _ encoding.MetricsUnmarshalerExtension = (*cloudwatchExtension)(nil) +) + +type cloudwatchExtension struct { + config *Config + logger *zap.Logger +} + +func createExtension(_ context.Context, settings extension.Settings, config component.Config) (extension.Extension, error) { + return &cloudwatchExtension{ + config: config.(*Config), + logger: settings.Logger, + }, nil +} + +func (c *cloudwatchExtension) Start(_ context.Context, _ component.Host) error { + return nil +} + +func (c *cloudwatchExtension) Shutdown(_ context.Context) error { + return nil +} + +func (c *cloudwatchExtension) UnmarshalLogs(data []byte) (plog.Logs, error) { + return cloudwatch.UnmarshalLogs(data, c.logger) +} + +func (c *cloudwatchExtension) UnmarshalMetrics(data []byte) (pmetric.Metrics, error) { + return cloudwatch.UnmarshalMetrics(data, c.logger) +} diff --git a/extension/encoding/cloudwatchencodingextension/extension_test.go b/extension/encoding/cloudwatchencodingextension/extension_test.go new file mode 100644 index 000000000000..634cf7233288 --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/extension_test.go @@ -0,0 +1 @@ +package cloudwatchencodingextension diff --git a/extension/encoding/cloudwatchencodingextension/factory.go b/extension/encoding/cloudwatchencodingextension/factory.go new file mode 100644 index 000000000000..8327968d2653 --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/factory.go @@ -0,0 +1,15 @@ +package cloudwatchencodingextension + +import ( + "cloudwatchencodingextension/internal/metadata" + "go.opentelemetry.io/collector/extension" +) + +func NewFactory() extension.Factory { + return extension.NewFactory( + metadata.Type, + createDefaultConfig, + createExtension, + metadata.ExtensionStability, + ) +} diff --git a/extension/encoding/cloudwatchencodingextension/factory_test.go b/extension/encoding/cloudwatchencodingextension/factory_test.go new file mode 100644 index 000000000000..634cf7233288 --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/factory_test.go @@ -0,0 +1 @@ +package cloudwatchencodingextension diff --git a/extension/encoding/cloudwatchencodingextension/generated_component_test.go b/extension/encoding/cloudwatchencodingextension/generated_component_test.go new file mode 100644 index 000000000000..ca9003d4db78 --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/generated_component_test.go @@ -0,0 +1,49 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package cloudwatchencodingextension + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/extension/extensiontest" +) + +func TestComponentFactoryType(t *testing.T) { + require.Equal(t, "cloudwatch_encoding", NewFactory().Type().String()) +} + +func TestComponentConfigStruct(t *testing.T) { + require.NoError(t, componenttest.CheckConfigStruct(NewFactory().CreateDefaultConfig())) +} + +func TestComponentLifecycle(t *testing.T) { + factory := NewFactory() + + cm, err := confmaptest.LoadConf("metadata.yaml") + require.NoError(t, err) + cfg := factory.CreateDefaultConfig() + sub, err := cm.Sub("tests::config") + require.NoError(t, err) + require.NoError(t, sub.Unmarshal(&cfg)) + t.Run("shutdown", func(t *testing.T) { + e, err := factory.Create(context.Background(), extensiontest.NewNopSettings(), cfg) + require.NoError(t, err) + err = e.Shutdown(context.Background()) + require.NoError(t, err) + }) + t.Run("lifecycle", func(t *testing.T) { + firstExt, err := factory.Create(context.Background(), extensiontest.NewNopSettings(), cfg) + require.NoError(t, err) + require.NoError(t, firstExt.Start(context.Background(), componenttest.NewNopHost())) + require.NoError(t, firstExt.Shutdown(context.Background())) + + secondExt, err := factory.Create(context.Background(), extensiontest.NewNopSettings(), cfg) + require.NoError(t, err) + require.NoError(t, secondExt.Start(context.Background(), componenttest.NewNopHost())) + require.NoError(t, secondExt.Shutdown(context.Background())) + }) +} diff --git a/extension/encoding/cloudwatchencodingextension/generated_package_test.go b/extension/encoding/cloudwatchencodingextension/generated_package_test.go new file mode 100644 index 000000000000..6a98f31ff6c3 --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/generated_package_test.go @@ -0,0 +1,12 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package cloudwatchencodingextension + +import ( + "go.uber.org/goleak" + "testing" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/extension/encoding/cloudwatchencodingextension/go.mod b/extension/encoding/cloudwatchencodingextension/go.mod new file mode 100644 index 000000000000..0880f8342a6d --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/go.mod @@ -0,0 +1,57 @@ +module github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension + +go 1.23 + +require ( + github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding v0.117.0 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch v0.0.0-00010101000000-000000000000 + github.com/stretchr/testify v1.10.0 + go.opentelemetry.io/collector/component v0.117.0 + go.opentelemetry.io/collector/component/componenttest v0.117.0 + go.opentelemetry.io/collector/confmap v1.23.0 + go.opentelemetry.io/collector/extension v0.117.0 + go.opentelemetry.io/collector/extension/extensiontest v0.117.0 + go.opentelemetry.io/collector/pdata v1.23.0 + go.uber.org/goleak v1.3.0 + go.uber.org/zap v1.27.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.2 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.117.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.117.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.117.0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.117.0 // indirect + go.opentelemetry.io/collector/semconv v0.117.0 // indirect + go.opentelemetry.io/otel v1.32.0 // indirect + go.opentelemetry.io/otel/metric v1.32.0 // indirect + go.opentelemetry.io/otel/sdk v1.32.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect + go.opentelemetry.io/otel/trace v1.32.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect + google.golang.org/grpc v1.69.2 // indirect + google.golang.org/protobuf v1.36.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch => ../../../pkg/translator/cloudwatch diff --git a/extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go b/extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go new file mode 100644 index 000000000000..d64ea26b7694 --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go @@ -0,0 +1,16 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" +) + +var ( + Type = component.MustNewType("cloudwatch_encoding") + ScopeName = "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" +) + +const ( + ExtensionStability = component.StabilityLevelAlpha +) diff --git a/extension/encoding/cloudwatchencodingextension/metadata.yaml b/extension/encoding/cloudwatchencodingextension/metadata.yaml new file mode 100644 index 000000000000..1b902fc6ce83 --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/metadata.yaml @@ -0,0 +1,9 @@ +type: cloudwatch_encoding + +status: + class: extension + stability: + alpha: [extension] + distributions: [] + codeowners: + active: [] diff --git a/pkg/translator/cloudwatch/Makefile b/pkg/translator/cloudwatch/Makefile new file mode 100644 index 000000000000..bdd863a203be --- /dev/null +++ b/pkg/translator/cloudwatch/Makefile @@ -0,0 +1 @@ +include ../../../Makefile.Common diff --git a/pkg/translator/cloudwatch/cwlogs_to_logs.go b/pkg/translator/cloudwatch/cwlogs_to_logs.go new file mode 100644 index 000000000000..d2ba5668eedd --- /dev/null +++ b/pkg/translator/cloudwatch/cwlogs_to_logs.go @@ -0,0 +1,97 @@ +package cloudwatch + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil" + + conventions "go.opentelemetry.io/collector/semconv/v1.6.1" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.uber.org/zap" + "io" + "time" +) + +type cloudwatchLog struct { + MessageType string `json:"messageType"` + Owner string `json:"owner"` + LogGroup string `json:"logGroup"` + LogStream string `json:"logStream"` + SubscriptionFilters []string `json:"subscriptionFilters"` + LogEvents []struct { + ID string `json:"id"` + Timestamp int64 `json:"timestamp"` + Message string `json:"message"` + } `json:"logEvents"` +} + +const ( + attributeAWSCloudWatchLogGroupName = "aws.cloudwatch.log_group_name" + attributeAWSCloudWatchLogStreamName = "aws.cloudwatch.log_stream_name" +) + +var ( + errInvalidLogRecord = errors.New("no resource logs were obtained from the compressed record") +) + +// isLogValid validates that the cloudwatch log has been unmarshalled correctly +func isLogValid(log cloudwatchLog) bool { + return log.Owner != "" && log.LogGroup != "" && log.LogStream != "" +} + +func UnmarshalLogs(compressedRecord []byte, logger *zap.Logger) (plog.Logs, error) { + r, err := gzip.NewReader(bytes.NewReader(compressedRecord)) + if err != nil { + return plog.Logs{}, fmt.Errorf("failed to create gzip reader from compressed record: %w", err) + } + decoder := json.NewDecoder(r) + + logs := plog.NewLogs() + for datumIndex := 0; ; datumIndex++ { + var log cloudwatchLog + if err = decoder.Decode(&log); err != nil { + if errors.Is(err, io.EOF) { + break + } + logger.Error( + "Unable to unmarshal input", + zap.Int("datum_index", datumIndex), + zap.Error(err), + ) + continue + } + if !isLogValid(log) { + logger.Error( + "Invalid cloudwatch log", + zap.Int("datum_index", datumIndex), + ) + continue + } + + rl := logs.ResourceLogs().AppendEmpty() + resourceAttrs := rl.Resource().Attributes() + resourceAttrs.PutStr(conventions.AttributeCloudAccountID, log.Owner) + resourceAttrs.PutStr(attributeAWSCloudWatchLogGroupName, log.LogGroup) + resourceAttrs.PutStr(attributeAWSCloudWatchLogStreamName, log.LogStream) + + logRecords := rl.ScopeLogs().AppendEmpty().LogRecords() + for _, event := range log.LogEvents { + logRecord := logRecords.AppendEmpty() + // pcommon.Timestamp is a time specified as UNIX Epoch time in nanoseconds + // but timestamp in cloudwatch logs are in milliseconds. + logRecord.SetTimestamp(pcommon.Timestamp(event.Timestamp * int64(time.Millisecond))) + logRecord.Body().SetStr(event.Message) + } + } + + if logs.ResourceLogs().Len() == 0 { + return logs, errInvalidLogRecord + } + pdatautil.GroupByResourceLogs(logs.ResourceLogs()) + return logs, nil +} diff --git a/pkg/translator/cloudwatch/cwlogs_to_logs_test.go b/pkg/translator/cloudwatch/cwlogs_to_logs_test.go new file mode 100644 index 000000000000..8b9653f54e36 --- /dev/null +++ b/pkg/translator/cloudwatch/cwlogs_to_logs_test.go @@ -0,0 +1,66 @@ +package cloudwatch + +import ( + "bytes" + "compress/gzip" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "os" + "path/filepath" + "testing" +) + +func TestUnmarshalLogs(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + filename string + resourceCount int + logCount int + err error + }{ + "WithMultipleRecords": { + filename: "multiple_records", + resourceCount: 1, + logCount: 2, + }, + "WithSingleRecord": { + filename: "single_record", + resourceCount: 1, + logCount: 1, + }, + "WithInvalidRecords": { + filename: "invalid_records", + err: errInvalidLogRecord, + }, + "WithSomeInvalidRecords": { + filename: "some_invalid_records", + resourceCount: 1, + logCount: 2, + }, + "WithMultipleResources": { + filename: "multiple_resources", + resourceCount: 3, + logCount: 6, + }, + } + for _, test := range tests { + t.Run(test.filename, func(t *testing.T) { + record, err := os.ReadFile(filepath.Join("testdata/log", test.filename)) + require.NoError(t, err) + + var buf bytes.Buffer + g := gzip.NewWriter(&buf) + _, err = g.Write(record) + require.NoError(t, err) + err = g.Close() + require.NoError(t, err) + compressedRecord := buf.Bytes() + + logs, err := UnmarshalLogs(compressedRecord, zap.NewNop()) + require.Equal(t, test.err, err) + require.Equal(t, test.resourceCount, logs.ResourceLogs().Len()) + require.Equal(t, test.logCount, logs.LogRecordCount()) + }) + } +} diff --git a/pkg/translator/cloudwatch/cwmetrics_to_metrics.go b/pkg/translator/cloudwatch/cwmetrics_to_metrics.go new file mode 100644 index 000000000000..43e4028127ec --- /dev/null +++ b/pkg/translator/cloudwatch/cwmetrics_to_metrics.go @@ -0,0 +1,145 @@ +package cloudwatch // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch" + +import ( + "bytes" + "encoding/json" + "errors" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.uber.org/zap" + "io" + "strings" + "time" + + expmetrics "github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics" + conventions "go.opentelemetry.io/collector/semconv/v1.27.0" +) + +// The cloudwatchMetric is the format for the CloudWatch metric stream records. +// +// More details can be found at: +// https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-formats-json.html +type cloudwatchMetric struct { + MetricStreamName string `json:"metric_stream_name"` + AccountID string `json:"account_id"` + Region string `json:"region"` + Namespace string `json:"namespace"` + MetricName string `json:"metric_name"` + Dimensions map[string]string `json:"dimensions"` + Timestamp int64 `json:"timestamp"` + Value *cloudwatchMetricValue `json:"value"` + Unit string `json:"unit"` +} + +// The cloudwatchMetricValue is the actual values of the CloudWatch metric. +type cloudwatchMetricValue struct { + Max float64 `json:"max"` + Min float64 `json:"min"` + Sum float64 `json:"sum"` + Count float64 `json:"count"` +} + +const ( + attributeAWSCloudWatchMetricStreamName = "aws.cloudwatch.metric_stream_name" + dimensionInstanceID = "InstanceId" + namespaceDelimiter = "/" +) + +var ( + errInvalidMetricRecord = errors.New("no resource metrics were obtained from the record") +) + +// isMetricValid validates that the cloudwatch metric has been unmarshalled correctly +func isMetricValid(metric cloudwatchMetric) bool { + return metric.MetricName != "" && metric.Namespace != "" && metric.Unit != "" && metric.Value != nil +} + +// setResourceAttributes sets attributes on a pcommon.Resource from a cwMetric. +func setResourceAttributes(m cloudwatchMetric, resource pcommon.Resource) { + attributes := resource.Attributes() + attributes.PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAWS) + attributes.PutStr(conventions.AttributeCloudAccountID, m.AccountID) + attributes.PutStr(conventions.AttributeCloudRegion, m.Region) + serviceNamespace, serviceName := toServiceAttributes(m.Namespace) + if serviceNamespace != "" { + attributes.PutStr(conventions.AttributeServiceNamespace, serviceNamespace) + } + attributes.PutStr(conventions.AttributeServiceName, serviceName) + attributes.PutStr(attributeAWSCloudWatchMetricStreamName, m.MetricStreamName) +} + +// toServiceAttributes splits the CloudWatch namespace into service namespace/name +// if prepended by AWS/. Otherwise, it returns the CloudWatch namespace as the +// service name with an empty service namespace +func toServiceAttributes(namespace string) (serviceNamespace, serviceName string) { + index := strings.Index(namespace, namespaceDelimiter) + if index != -1 && strings.EqualFold(namespace[:index], conventions.AttributeCloudProviderAWS) { + return namespace[:index], namespace[index+1:] + } + return "", namespace +} + +// setResourceAttributes sets attributes on a metric data point from a cloudwatchMetric. +func setDataPointAttributes(m cloudwatchMetric, dp pmetric.SummaryDataPoint) { + attrs := dp.Attributes() + for k, v := range m.Dimensions { + switch k { + case dimensionInstanceID: + attrs.PutStr(conventions.AttributeServiceInstanceID, v) + default: + attrs.PutStr(k, v) + } + } +} + +func UnmarshalMetrics(record []byte, logger *zap.Logger) (pmetric.Metrics, error) { + decoder := json.NewDecoder(bytes.NewReader(record)) + metrics := pmetric.NewMetrics() + for datumIndex := 0; ; datumIndex++ { + var cwMetric cloudwatchMetric + if err := decoder.Decode(&cwMetric); err != nil { + if errors.Is(err, io.EOF) { + break + } + logger.Error( + "Unable to unmarshal input", + zap.Int("datum_index", datumIndex), + zap.Error(err), + ) + continue + } + if !isMetricValid(cwMetric) { + logger.Error( + "Invalid cloudwatch cwMetric", + zap.Int("datum_index", datumIndex), + ) + continue + } + + rm := metrics.ResourceMetrics().AppendEmpty() + setResourceAttributes(cwMetric, rm.Resource()) + + metric := rm.ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() + metric.SetName(cwMetric.MetricName) + metric.SetUnit(cwMetric.Unit) + + dp := metric.SetEmptySummary().DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.NewTimestampFromTime(time.UnixMilli(cwMetric.Timestamp))) + setDataPointAttributes(cwMetric, dp) + dp.SetCount(uint64(cwMetric.Value.Count)) + dp.SetSum(cwMetric.Value.Sum) + minQ := dp.QuantileValues().AppendEmpty() + minQ.SetQuantile(0) + minQ.SetValue(cwMetric.Value.Min) + maxQ := dp.QuantileValues().AppendEmpty() + maxQ.SetQuantile(1) + maxQ.SetValue(cwMetric.Value.Max) + } + + if metrics.MetricCount() == 0 { + return metrics, errInvalidMetricRecord + } + + metrics = expmetrics.Merge(pmetric.NewMetrics(), metrics) + return metrics, nil +} diff --git a/pkg/translator/cloudwatch/cwmetrics_to_metrics_test.go b/pkg/translator/cloudwatch/cwmetrics_to_metrics_test.go new file mode 100644 index 000000000000..ee5f9ee3c82d --- /dev/null +++ b/pkg/translator/cloudwatch/cwmetrics_to_metrics_test.go @@ -0,0 +1,57 @@ +package cloudwatch + +import ( + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "os" + "path/filepath" + "testing" +) + +func TestUnmarshalMetrics(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + filename string + resourceCount int + metricCount int + datapointCount int + err error + }{ + "WithMultipleRecords": { + filename: "multiple_records", + resourceCount: 6, + metricCount: 33, + datapointCount: 127, + }, + "WithSingleRecord": { + filename: "single_record", + resourceCount: 1, + metricCount: 1, + datapointCount: 1, + }, + "WithInvalidRecords": { + filename: "invalid_records", + err: errInvalidMetricRecord, + }, + "WithSomeInvalidRecords": { + filename: "some_invalid_records", + resourceCount: 5, + metricCount: 36, + datapointCount: 88, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + record, err := os.ReadFile(filepath.Join("testdata/metric", testCase.filename)) + require.NoError(t, err) + + metrics, err := UnmarshalMetrics(record, zap.NewNop()) + + require.Equal(t, testCase.err, err) + require.Equal(t, testCase.resourceCount, metrics.ResourceMetrics().Len()) + require.Equal(t, testCase.metricCount, metrics.MetricCount()) + require.Equal(t, testCase.datapointCount, metrics.DataPointCount()) + }) + } +} diff --git a/pkg/translator/cloudwatch/go.mod b/pkg/translator/cloudwatch/go.mod new file mode 100644 index 000000000000..819e57722b55 --- /dev/null +++ b/pkg/translator/cloudwatch/go.mod @@ -0,0 +1,42 @@ +module github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch + +go 1.23 + +require ( + github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.117.0 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0 + github.com/stretchr/testify v1.10.0 + go.opentelemetry.io/collector/pdata v1.23.0 + go.opentelemetry.io/collector/semconv v0.117.0 + go.uber.org/goleak v1.3.0 + go.uber.org/zap v1.27.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.117.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect + google.golang.org/grpc v1.69.2 // indirect + google.golang.org/protobuf v1.36.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../pdatautil + +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics => ../../../internal/exp/metrics + +retract ( + v0.76.2 + v0.76.1 + v0.65.0 +) diff --git a/pkg/translator/cloudwatch/go.sum b/pkg/translator/cloudwatch/go.sum new file mode 100644 index 000000000000..2688b869e30d --- /dev/null +++ b/pkg/translator/cloudwatch/go.sum @@ -0,0 +1,109 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0 h1:SjZHtgph6f675yK/p+GTo8I7VlIAjJFod3SMNgfpEAg= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0/go.mod h1:e4cIzlw2iGmMCgHKyJHLf+PF99A1Y6GvNC0+E3x1dA4= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.117.0 h1:tOJFUIZaAU4zm5CilqZN1/AuKQa7diTrcEhgQIYly6k= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.117.0/go.mod h1:PJ2FGCS+Hw+tlHUNNWVHNo3IXtEsb9RKgl/ssSi3Z98= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.117.0 h1:/wMNk8w1UEHKpKoNk1jA2aifHgfGZE+WelGNrCf0CJ0= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.117.0/go.mod h1:ESyMNHmgZYh8Ouhr2veecTMK6sB8gQ8u2s3dsy9Og6k= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector/pdata v1.23.0 h1:tEk0dkfB8RdSukoOMfEa8duB938gfZowdfRkrJxGDrw= +go.opentelemetry.io/collector/pdata v1.23.0/go.mod h1:I2jggpBMiO8A+7TXhzNpcJZkJtvi1cU0iVNIi+6bc+o= +go.opentelemetry.io/collector/semconv v0.117.0 h1:SavOvSbHPVD/QdAnXlI/cMca+yxCNyXStY1mQzerHs4= +go.opentelemetry.io/collector/semconv v0.117.0/go.mod h1:N6XE8Q0JKgBN2fAhkUQtqK9LT7rEGR6+Wu/Rtbal1iI= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 h1:XVhgTWWV3kGQlwJHR3upFWZeTsei6Oks1apkZSeonIE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= +google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU= +google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/translator/cloudwatch/metadata.yaml b/pkg/translator/cloudwatch/metadata.yaml new file mode 100644 index 000000000000..47c8adf2f35a --- /dev/null +++ b/pkg/translator/cloudwatch/metadata.yaml @@ -0,0 +1,3 @@ +status: + codeowners: + active: [] diff --git a/pkg/translator/cloudwatch/package_test.go b/pkg/translator/cloudwatch/package_test.go new file mode 100644 index 000000000000..aef2b34f0690 --- /dev/null +++ b/pkg/translator/cloudwatch/package_test.go @@ -0,0 +1,11 @@ +package cloudwatch + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/pkg/translator/cloudwatch/testdata/log/invalid_records b/pkg/translator/cloudwatch/testdata/log/invalid_records new file mode 100644 index 000000000000..d2672dbd26d3 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/log/invalid_records @@ -0,0 +1,4 @@ +{"CHANGE":-0.09,"PRICE":4.96,"TICKER_SYMBOL":"KIN","SECTOR":"ENERGY"} +{"CHANGE":-1.47,"PRICE":134.74,"TICKER_SYMBOL":"DFG","SECTOR":"TECHNOLOGY"} +{"CHANGE":1.96,"PRICE":57.53,"TICKER_SYMBOL":"SAC","SECTOR":"ENERGY"} +{"CHANGE":0.04,"PRICE":32.84,"TICKER_SYMBOL":"PJN","SECTOR":"RETAIL"} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/log/multiple_records b/pkg/translator/cloudwatch/testdata/log/multiple_records new file mode 100644 index 000000000000..18a1888d99a8 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/log/multiple_records @@ -0,0 +1,2 @@ +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695232","timestamp":1725544035523,"message":"Hello world, here is our first log message!"}]} +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695233","timestamp":1725554035523,"message":"Hello world, here is our second log message!"}]} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/log/multiple_resources b/pkg/translator/cloudwatch/testdata/log/multiple_resources new file mode 100644 index 000000000000..0115614f1b9b --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/log/multiple_resources @@ -0,0 +1,6 @@ +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695232","timestamp":1725544035523,"message":"Hello world, here is our first log message!"}]} +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695233","timestamp":1725554035523,"message":"Hello world, here is our second log message!"}]} +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test2","logStream":"test1","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695234","timestamp":1725564035523,"message":"Hello world, here is our third log message!"}]} +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test2","logStream":"test2","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695235","timestamp":1725574035523,"message":"Hello world, here is our fourth log message!"}]} +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test2","logStream":"test1","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695236","timestamp":1725584035523,"message":"Hello world, here is our fifth log message!"}]} +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test2","logStream":"test2","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695237","timestamp":1725594035523,"message":"Hello world, here is our sixth log message!"}]} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/log/single_record b/pkg/translator/cloudwatch/testdata/log/single_record new file mode 100644 index 000000000000..b35e166e47da --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/log/single_record @@ -0,0 +1 @@ +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695232","timestamp":1725544035523,"message":"Hello world, here is our first log message!"}]} diff --git a/pkg/translator/cloudwatch/testdata/log/some_invalid_records b/pkg/translator/cloudwatch/testdata/log/some_invalid_records new file mode 100644 index 000000000000..4026ad877a43 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/log/some_invalid_records @@ -0,0 +1,3 @@ +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695232","timestamp":1725544035523,"message":"Hello world, here is our first log message!"}]} +{"CHANGE":1.96,"PRICE":57.53,"TICKER_SYMBOL":"SAC","SECTOR":"ENERGY"} +{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695233","timestamp":1725554035523,"message":"Hello world, here is our second log message!"}]} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/metric/invalid_records b/pkg/translator/cloudwatch/testdata/metric/invalid_records new file mode 100644 index 000000000000..d2672dbd26d3 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/metric/invalid_records @@ -0,0 +1,4 @@ +{"CHANGE":-0.09,"PRICE":4.96,"TICKER_SYMBOL":"KIN","SECTOR":"ENERGY"} +{"CHANGE":-1.47,"PRICE":134.74,"TICKER_SYMBOL":"DFG","SECTOR":"TECHNOLOGY"} +{"CHANGE":1.96,"PRICE":57.53,"TICKER_SYMBOL":"SAC","SECTOR":"ENERGY"} +{"CHANGE":0.04,"PRICE":32.84,"TICKER_SYMBOL":"PJN","SECTOR":"RETAIL"} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/metric/multiple_records b/pkg/translator/cloudwatch/testdata/metric/multiple_records new file mode 100644 index 000000000000..20fa9b367fe7 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/metric/multiple_records @@ -0,0 +1,127 @@ +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ConnectionEstablishedCount","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ErrorPortAllocation","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":477184.0,"count":92.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916720000,"value":{"max":40.0,"min":40.0,"sum":40.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":59.98,"min":59.98,"sum":59.98,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"MetricUpdate","dimensions":{},"timestamp":1643916720000,"value":{"max":47.0,"min":41.0,"sum":88.0,"count":2.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916720000,"value":{"max":96768.0,"min":96768.0,"sum":96768.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"MetricUpdate","dimensions":{"MetricStreamName":"MyMetricStream"},"timestamp":1643916720000,"value":{"max":47.0,"min":41.0,"sum":88.0,"count":2.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUSurplusCreditsCharged","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":0.00133333333333333,"min":0.00133333333333333,"sum":0.00133333333333333,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916720000,"value":{"max":0.006538,"min":0.006538,"sum":0.006538,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916720000,"value":{"max":0.028416,"min":0.028416,"sum":0.028416,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.08,"count":92.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916720000,"value":{"max":223744.0,"min":223744.0,"sum":223744.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUCreditUsage","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.018548,"min":0.018548,"sum":0.018548,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ActiveConnectionCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"OnDemand","Service":"Fargate","Type":"Resource"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ConnectionAttemptCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ErrorPortAllocation","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":60.145003,"min":60.145003,"sum":60.145003,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"PublishErrorRate","dimensions":{"MetricStreamName":"MyMetricStream"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"BurstBalance","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":100.0,"min":100.0,"sum":100.0,"count":1.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":34.0,"min":34.0,"sum":34.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"Spot","Service":"Fargate","Type":"Resource"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Logs","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeMetricFilters","Service":"Logs","Type":"API"},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ErrorPortAllocation","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Logs","metric_name":"CallCount","dimensions":{},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeInstanceStatus","Service":"EC2","Type":"API"},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":92.0,"min":92.0,"sum":92.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeMetricFilters","Service":"Logs","Type":"API"},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":60.07399,"min":60.07399,"sum":60.07399,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ConnectionEstablishedCount","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"PublishErrorRate","dimensions":{},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ConnectionAttemptCount","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ActiveConnectionCount","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"ListGroups","Service":"Resource Groups","Type":"API"},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":7.0,"min":7.0,"sum":7.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"MetricUpdate","dimensions":{},"timestamp":1643916780000,"value":{"max":48.0,"min":36.0,"sum":84.0,"count":2.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":178688.0,"min":178688.0,"sum":178688.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"MetricUpdate","dimensions":{"MetricStreamName":"MyMetricStream"},"timestamp":1643916780000,"value":{"max":48.0,"min":36.0,"sum":84.0,"count":2.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.018855,"min":0.018855,"sum":0.018855,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.004314,"min":0.004314,"sum":0.004314,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":40448.0,"min":40448.0,"sum":40448.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"OnDemand","Service":"Fargate","Type":"Resource"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} diff --git a/pkg/translator/cloudwatch/testdata/metric/single_record b/pkg/translator/cloudwatch/testdata/metric/single_record new file mode 100644 index 000000000000..6d2fd8e1b658 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/metric/single_record @@ -0,0 +1 @@ +{"metric_stream_name": "MyMetricStream","account_id": "1234567890","region": "us-east-1","namespace": "AWS/EC2","metric_name": "DiskWriteOps","dimensions": {"InstanceId": "i-123456789012"},"timestamp": 1611929698000,"value": {"count": 3.0,"sum": 20.0,"max": 18.0,"min": 0.0},"unit": "Seconds"} diff --git a/pkg/translator/cloudwatch/testdata/metric/some_invalid_records b/pkg/translator/cloudwatch/testdata/metric/some_invalid_records new file mode 100644 index 000000000000..d1598692b12c --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/metric/some_invalid_records @@ -0,0 +1,92 @@ +{"CHANGE":-0.09,"PRICE":4.96,"TICKER_SYMBOL":"KIN","SECTOR":"ENERGY"} +{"CHANGE":-1.47,"PRICE":134.74,"TICKER_SYMBOL":"DFG","SECTOR":"TECHNOLOGY"} +{"CHANGE":1.96,"PRICE":57.53,"TICKER_SYMBOL":"SAC","SECTOR":"ENERGY"} +{"CHANGE":0.04,"PRICE":32.84,"TICKER_SYMBOL":"PJN","SECTOR":"RETAIL"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":60.058357,"min":60.058357,"sum":60.058357,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadOps","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsOut","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":189.0,"min":187.0,"sum":376.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUUtilization","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":2.225,"min":2.208333333333333,"sum":4.433333333333334,"count":2.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSByteBalance%","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":99.0,"min":99.0,"sum":198.0,"count":2.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadBytes","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUUtilization","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":2.225,"min":2.1,"sum":8.675,"count":4.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteOps","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":37.0,"min":12.0,"sum":49.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"BurstBalance","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":100.0,"min":100.0,"sum":100.0,"count":1.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":7.0,"min":7.0,"sum":7.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteBytes","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":246272.0,"min":49664.0,"sum":552448.0,"count":4.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteBytes","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":246272.0,"min":49664.0,"sum":295936.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"Spot","Service":"Fargate","Type":"Resource"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsOut","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":189.0,"min":167.0,"sum":729.0,"count":4.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkOut","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":33477.0,"min":28026.0,"sum":127491.0,"count":4.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadOps","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":4.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Logs","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeMetricFilters","Service":"Logs","Type":"API"},"timestamp":1643916600000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Logs","metric_name":"CallCount","dimensions":{},"timestamp":1643916600000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadOps","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkIn","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":96546.0,"min":75023.0,"sum":330899.0,"count":4.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":172.0,"min":172.0,"sum":172.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeMetricFilters","Service":"Logs","Type":"API"},"timestamp":1643916600000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":4.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkOut","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":33477.0,"min":33411.0,"sum":66888.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":60.107249,"min":60.107249,"sum":60.107249,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSByteBalance%","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":100.0,"min":100.0,"sum":200.0,"count":2.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSByteBalance%","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":100.0,"min":99.0,"sum":398.0,"count":4.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"Standard/OnDemand","Resource":"vCPU","Service":"EC2","Type":"Resource"},"timestamp":1643916600000,"value":{"max":6.0,"min":6.0,"sum":6.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadBytes","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteOps","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":50.0,"min":7.0,"sum":106.0,"count":4.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsIn","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":210.0,"min":206.0,"sum":416.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsIn","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":205.0,"min":182.0,"sum":387.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsOut","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":186.0,"min":167.0,"sum":353.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":10.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteOps","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":50.0,"min":7.0,"sum":57.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":892416.0,"count":172.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":12.0,"min":12.0,"sum":12.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":59.98,"min":59.98,"sum":59.98,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkIn","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":75359.0,"min":75023.0,"sum":150382.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":49664.0,"min":49664.0,"sum":49664.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadBytes","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":5.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":5.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkIn","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":96546.0,"min":72132.0,"sum":252649.0,"count":3.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUUtilization","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":2.1416666666666666,"min":2.1,"sum":4.241666666666667,"count":2.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSIOBalance%","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":100.0,"min":99.0,"sum":497.0,"count":5.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsIn","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":210.0,"min":182.0,"sum":991.0,"count":5.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkOut","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":32577.0,"min":28026.0,"sum":60603.0,"count":2.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteBytes","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":223744.0,"min":78336.0,"sum":480256.0,"count":3.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":0.00133333333333333,"min":0.00133333333333333,"sum":0.00133333333333333,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.004637,"min":0.004637,"sum":0.004637,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.00977,"min":0.00977,"sum":0.00977,"count":1.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.08,"count":172.0},"unit":"Seconds"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSIOBalance%","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":99.0,"min":99.0,"sum":297.0,"count":3.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSIOBalance%","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":100.0,"min":100.0,"sum":200.0,"count":2.0},"unit":"Percent"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":78336.0,"min":78336.0,"sum":78336.0,"count":1.0},"unit":"Bytes"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"OnDemand","Service":"Fargate","Type":"Resource"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} +{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} From 4ac64fe78f49f9727b3b1726183048c89a5c8b98 Mon Sep 17 00:00:00 2001 From: Constanca Manteigas Date: Tue, 14 Jan 2025 16:58:55 +0100 Subject: [PATCH 2/9] - Add translator for cloudwatch - Add firehose encoding --- .../cloudwatchencodingextension/config.go | 17 -- .../config_test.go | 1 - .../cloudwatchencodingextension/extension.go | 45 ---- .../extension_test.go | 1 - .../factory_test.go | 1 - .../Makefile | 0 .../firehoseencodingextension/config.go | 53 ++++ .../firehoseencodingextension/config_test.go | 58 +++++ .../doc.go | 2 +- .../firehoseencodingextension/extension.go | 209 +++++++++++++++ .../extension_test.go | 238 ++++++++++++++++++ .../factory.go | 4 +- .../firehoseencodingextension/factory_test.go | 15 ++ .../generated_component_test.go | 4 +- .../generated_package_test.go | 2 +- .../go.mod | 7 +- .../internal/metadata/generated_status.go | 2 +- .../metadata.yaml | 2 +- .../testdata/config.yaml | 9 + .../testdata/logs/empty_log.json | 10 + .../testdata/logs/invalid_log.json | 18 ++ .../logs/multiple_resources_expected.json | 94 +++++++ .../testdata/logs/one_log.json | 19 ++ .../testdata/logs/one_log_expected.json | 45 ++++ .../testdata/logs/some_invalid_log.json | 26 ++ .../testdata/logs/two_logs.json | 24 ++ .../testdata/logs/two_logs_expected.json | 53 ++++ .../testdata/logs/two_resources.json | 34 +++ .../testdata/metrics/empty_record.json | 1 + .../testdata/metrics/invalid_metric.json | 5 + .../testdata/metrics/multiple_resources.json | 44 ++++ .../metrics/multiple_resources_expected.json | 182 ++++++++++++++ .../testdata/metrics/one_metric.json | 23 ++ .../testdata/metrics/one_metric_expected.json | 93 +++++++ .../metrics/some_invalid_metrics.json | 26 ++ .../testdata/metrics/two_metrics.json | 44 ++++ .../metrics/two_metrics_expected.json | 123 +++++++++ .../cloudwatch/cwlogs_to_log_record.go | 60 +++++ .../cloudwatch/cwlogs_to_log_record_test.go | 79 ++++++ pkg/translator/cloudwatch/cwlogs_to_logs.go | 97 ------- .../cloudwatch/cwlogs_to_logs_test.go | 66 ----- .../cloudwatch/cwmetrics_to_metrics.go | 116 +++++---- .../cloudwatch/cwmetrics_to_metrics_test.go | 83 +++--- pkg/translator/cloudwatch/go.mod | 3 + pkg/translator/cloudwatch/go.sum | 6 + .../cloudwatch/testdata/log/empty_record.json | 1 + .../testdata/log/invalid_record.json | 5 + .../cloudwatch/testdata/log/invalid_records | 4 - .../cloudwatch/testdata/log/multiple_records | 2 - .../testdata/log/multiple_records.json | 10 + .../log/multiple_records_expected.json | 34 +++ .../testdata/log/multiple_resources | 6 - .../cloudwatch/testdata/log/single_record | 1 - .../testdata/log/single_record.json | 6 + .../testdata/log/single_record_expected.json | 26 ++ .../testdata/log/some_invalid_record.json | 9 + .../testdata/log/some_invalid_records | 3 - .../testdata/metric/empty_record.json | 1 + .../testdata/metric/invalid_record.json | 5 + .../testdata/metric/invalid_records | 4 - .../testdata/metric/multiple_records | 127 ---------- .../testdata/metric/multiple_records.json | 44 ++++ .../metric/multiple_records_expected.json | 123 +++++++++ .../testdata/metric/multiple_resources.json | 44 ++++ .../metric/multiple_resources_expected.json | 182 ++++++++++++++ .../cloudwatch/testdata/metric/single_record | 1 - .../testdata/metric/single_record.json | 23 ++ .../metric/single_record_expected.json | 93 +++++++ .../testdata/metric/some_invalid_record.json | 26 ++ .../testdata/metric/some_invalid_records | 92 ------- 70 files changed, 2360 insertions(+), 556 deletions(-) delete mode 100644 extension/encoding/cloudwatchencodingextension/config.go delete mode 100644 extension/encoding/cloudwatchencodingextension/config_test.go delete mode 100644 extension/encoding/cloudwatchencodingextension/extension.go delete mode 100644 extension/encoding/cloudwatchencodingextension/extension_test.go delete mode 100644 extension/encoding/cloudwatchencodingextension/factory_test.go rename extension/encoding/{cloudwatchencodingextension => firehoseencodingextension}/Makefile (100%) create mode 100644 extension/encoding/firehoseencodingextension/config.go create mode 100644 extension/encoding/firehoseencodingextension/config_test.go rename extension/encoding/{cloudwatchencodingextension => firehoseencodingextension}/doc.go (50%) create mode 100644 extension/encoding/firehoseencodingextension/extension.go create mode 100644 extension/encoding/firehoseencodingextension/extension_test.go rename extension/encoding/{cloudwatchencodingextension => firehoseencodingextension}/factory.go (57%) create mode 100644 extension/encoding/firehoseencodingextension/factory_test.go rename extension/encoding/{cloudwatchencodingextension => firehoseencodingextension}/generated_component_test.go (93%) rename extension/encoding/{cloudwatchencodingextension => firehoseencodingextension}/generated_package_test.go (80%) rename extension/encoding/{cloudwatchencodingextension => firehoseencodingextension}/go.mod (93%) rename extension/encoding/{cloudwatchencodingextension => firehoseencodingextension}/internal/metadata/generated_status.go (84%) rename extension/encoding/{cloudwatchencodingextension => firehoseencodingextension}/metadata.yaml (81%) create mode 100644 extension/encoding/firehoseencodingextension/testdata/config.yaml create mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/empty_log.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/invalid_log.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/multiple_resources_expected.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/one_log.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/one_log_expected.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/some_invalid_log.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/two_logs.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/two_logs_expected.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/two_resources.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/empty_record.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/invalid_metric.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources_expected.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/one_metric.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/one_metric_expected.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/some_invalid_metrics.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics.json create mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics_expected.json create mode 100644 pkg/translator/cloudwatch/cwlogs_to_log_record.go create mode 100644 pkg/translator/cloudwatch/cwlogs_to_log_record_test.go delete mode 100644 pkg/translator/cloudwatch/cwlogs_to_logs.go delete mode 100644 pkg/translator/cloudwatch/cwlogs_to_logs_test.go create mode 100644 pkg/translator/cloudwatch/testdata/log/empty_record.json create mode 100644 pkg/translator/cloudwatch/testdata/log/invalid_record.json delete mode 100644 pkg/translator/cloudwatch/testdata/log/invalid_records delete mode 100644 pkg/translator/cloudwatch/testdata/log/multiple_records create mode 100644 pkg/translator/cloudwatch/testdata/log/multiple_records.json create mode 100644 pkg/translator/cloudwatch/testdata/log/multiple_records_expected.json delete mode 100644 pkg/translator/cloudwatch/testdata/log/multiple_resources delete mode 100644 pkg/translator/cloudwatch/testdata/log/single_record create mode 100644 pkg/translator/cloudwatch/testdata/log/single_record.json create mode 100644 pkg/translator/cloudwatch/testdata/log/single_record_expected.json create mode 100644 pkg/translator/cloudwatch/testdata/log/some_invalid_record.json delete mode 100644 pkg/translator/cloudwatch/testdata/log/some_invalid_records create mode 100644 pkg/translator/cloudwatch/testdata/metric/empty_record.json create mode 100644 pkg/translator/cloudwatch/testdata/metric/invalid_record.json delete mode 100644 pkg/translator/cloudwatch/testdata/metric/invalid_records delete mode 100644 pkg/translator/cloudwatch/testdata/metric/multiple_records create mode 100644 pkg/translator/cloudwatch/testdata/metric/multiple_records.json create mode 100644 pkg/translator/cloudwatch/testdata/metric/multiple_records_expected.json create mode 100644 pkg/translator/cloudwatch/testdata/metric/multiple_resources.json create mode 100644 pkg/translator/cloudwatch/testdata/metric/multiple_resources_expected.json delete mode 100644 pkg/translator/cloudwatch/testdata/metric/single_record create mode 100644 pkg/translator/cloudwatch/testdata/metric/single_record.json create mode 100644 pkg/translator/cloudwatch/testdata/metric/single_record_expected.json create mode 100644 pkg/translator/cloudwatch/testdata/metric/some_invalid_record.json delete mode 100644 pkg/translator/cloudwatch/testdata/metric/some_invalid_records diff --git a/extension/encoding/cloudwatchencodingextension/config.go b/extension/encoding/cloudwatchencodingextension/config.go deleted file mode 100644 index db74532e917e..000000000000 --- a/extension/encoding/cloudwatchencodingextension/config.go +++ /dev/null @@ -1,17 +0,0 @@ -package cloudwatchencodingextension - -import ( - "go.opentelemetry.io/collector/component" -) - -type Config struct{} - -const defaultDelimiter = "\n" - -func createDefaultConfig() component.Config { - return &Config{} -} - -func (c *Config) Validate() error { - return nil -} diff --git a/extension/encoding/cloudwatchencodingextension/config_test.go b/extension/encoding/cloudwatchencodingextension/config_test.go deleted file mode 100644 index 634cf7233288..000000000000 --- a/extension/encoding/cloudwatchencodingextension/config_test.go +++ /dev/null @@ -1 +0,0 @@ -package cloudwatchencodingextension diff --git a/extension/encoding/cloudwatchencodingextension/extension.go b/extension/encoding/cloudwatchencodingextension/extension.go deleted file mode 100644 index bd9bb3998cbb..000000000000 --- a/extension/encoding/cloudwatchencodingextension/extension.go +++ /dev/null @@ -1,45 +0,0 @@ -package cloudwatchencodingextension - -import ( - "context" - "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding" - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/extension" - plog "go.opentelemetry.io/collector/pdata/plog" - "go.opentelemetry.io/collector/pdata/pmetric" - "go.uber.org/zap" -) - -var ( - _ encoding.LogsUnmarshalerExtension = (*cloudwatchExtension)(nil) - _ encoding.MetricsUnmarshalerExtension = (*cloudwatchExtension)(nil) -) - -type cloudwatchExtension struct { - config *Config - logger *zap.Logger -} - -func createExtension(_ context.Context, settings extension.Settings, config component.Config) (extension.Extension, error) { - return &cloudwatchExtension{ - config: config.(*Config), - logger: settings.Logger, - }, nil -} - -func (c *cloudwatchExtension) Start(_ context.Context, _ component.Host) error { - return nil -} - -func (c *cloudwatchExtension) Shutdown(_ context.Context) error { - return nil -} - -func (c *cloudwatchExtension) UnmarshalLogs(data []byte) (plog.Logs, error) { - return cloudwatch.UnmarshalLogs(data, c.logger) -} - -func (c *cloudwatchExtension) UnmarshalMetrics(data []byte) (pmetric.Metrics, error) { - return cloudwatch.UnmarshalMetrics(data, c.logger) -} diff --git a/extension/encoding/cloudwatchencodingextension/extension_test.go b/extension/encoding/cloudwatchencodingextension/extension_test.go deleted file mode 100644 index 634cf7233288..000000000000 --- a/extension/encoding/cloudwatchencodingextension/extension_test.go +++ /dev/null @@ -1 +0,0 @@ -package cloudwatchencodingextension diff --git a/extension/encoding/cloudwatchencodingextension/factory_test.go b/extension/encoding/cloudwatchencodingextension/factory_test.go deleted file mode 100644 index 634cf7233288..000000000000 --- a/extension/encoding/cloudwatchencodingextension/factory_test.go +++ /dev/null @@ -1 +0,0 @@ -package cloudwatchencodingextension diff --git a/extension/encoding/cloudwatchencodingextension/Makefile b/extension/encoding/firehoseencodingextension/Makefile similarity index 100% rename from extension/encoding/cloudwatchencodingextension/Makefile rename to extension/encoding/firehoseencodingextension/Makefile diff --git a/extension/encoding/firehoseencodingextension/config.go b/extension/encoding/firehoseencodingextension/config.go new file mode 100644 index 000000000000..0ab666fcc949 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/config.go @@ -0,0 +1,53 @@ +package firehoseencodingextension + +import ( + "fmt" + "go.opentelemetry.io/collector/component" +) + +type contentEncoding string + +const ( + NoEncoding = "" + GZipEncoded = "gzip" +) + +type Config struct { + Logs `mapstructure:"logs"` + Metrics `mapstructure:"metrics"` +} + +type Logs struct { + LogsEncoding contentEncoding `mapstructure:"content_encoding"` +} + +type Metrics struct { + MetricsEncoding contentEncoding `mapstructure:"content_encoding"` +} + +func createDefaultConfig() component.Config { + return &Config{ + Metrics: Metrics{ + MetricsEncoding: NoEncoding, + }, + Logs: Logs{ + LogsEncoding: GZipEncoded, + }, + } +} + +func (c *Config) Validate() error { + switch c.MetricsEncoding { + case NoEncoding: + case GZipEncoded: + default: + return fmt.Errorf("unknown metrics encoding %s", c.MetricsEncoding) + } + switch c.LogsEncoding { + case NoEncoding: + case GZipEncoded: + default: + return fmt.Errorf("unknown logs encoding %s", c.LogsEncoding) + } + return nil +} diff --git a/extension/encoding/firehoseencodingextension/config_test.go b/extension/encoding/firehoseencodingextension/config_test.go new file mode 100644 index 000000000000..e068de93568f --- /dev/null +++ b/extension/encoding/firehoseencodingextension/config_test.go @@ -0,0 +1,58 @@ +package firehoseencodingextension + +import ( + "errors" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension/internal/metadata" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap/confmaptest" + "path/filepath" + "testing" +) + +func TestLoadConfig(t *testing.T) { + t.Parallel() + + defaultCfg := createDefaultConfig().(*Config) + + tests := []struct { + id component.ID + expected component.Config + expectedErr error + }{ + { + id: component.NewID(metadata.Type), + expected: defaultCfg, + }, + { + id: component.NewIDWithName(metadata.Type, "metrics_invalid_encoding"), + expectedErr: errors.New("unknown metrics encoding invalid"), + }, + { + id: component.NewIDWithName(metadata.Type, "logs_invalid_encoding"), + expectedErr: errors.New("unknown logs encoding invalid"), + }, + } + + for _, tt := range tests { + t.Run(tt.id.String(), func(t *testing.T) { + cm, err := confmaptest.LoadConf(filepath.Join("testdata", "config.yaml")) + require.NoError(t, err) + + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + sub, err := cm.Sub(tt.id.String()) + require.NoError(t, err) + require.NoError(t, sub.Unmarshal(cfg)) + + if tt.expectedErr != nil { + require.Equal(t, tt.expectedErr, component.ValidateConfig(cfg)) + return + } + + require.NoError(t, component.ValidateConfig(cfg)) + require.Equal(t, tt.expected, cfg) + }) + } + +} diff --git a/extension/encoding/cloudwatchencodingextension/doc.go b/extension/encoding/firehoseencodingextension/doc.go similarity index 50% rename from extension/encoding/cloudwatchencodingextension/doc.go rename to extension/encoding/firehoseencodingextension/doc.go index c4ceff17eda5..c981b29efef9 100644 --- a/extension/encoding/cloudwatchencodingextension/doc.go +++ b/extension/encoding/firehoseencodingextension/doc.go @@ -1,2 +1,2 @@ //go:generate mdatagen metadata.yaml -package cloudwatchencodingextension +package firehoseencodingextension diff --git a/extension/encoding/firehoseencodingextension/extension.go b/extension/encoding/firehoseencodingextension/extension.go new file mode 100644 index 000000000000..33619d5ef566 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/extension.go @@ -0,0 +1,209 @@ +package firehoseencodingextension + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding" + expmetrics "github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics" + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch" + "github.com/tidwall/gjson" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/extension" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" + conventions "go.opentelemetry.io/collector/semconv/v1.6.1" + "go.uber.org/zap" + "io" +) + +var ( + _ encoding.LogsUnmarshalerExtension = (*cloudwatchExtension)(nil) + _ encoding.MetricsUnmarshalerExtension = (*cloudwatchExtension)(nil) +) + +type cloudwatchExtension struct { + config *Config + logger *zap.Logger +} + +type payload struct { + RequestID string `json:"requestId"` + Timestamp int64 `json:"timestamp"` + Records []record `json:"records"` +} + +type record struct { + Data string `json:"data"` +} + +type cloudwatchLog struct { + Owner string `json:"owner"` + LogGroup string `json:"logGroup"` + LogStream string `json:"logStream"` + + // LogEvents should have the logs data. + // It is left as raw, so the translator + // for cloudwatch log can unmarshal it. + LogEvents json.RawMessage `json:"logEvents"` +} + +const ( + attributeAWSCloudWatchLogGroupName = "aws.cloudwatch.log_group_name" + attributeAWSCloudWatchLogStreamName = "aws.cloudwatch.log_stream_name" +) + +func createExtension(_ context.Context, settings extension.Settings, config component.Config) (extension.Extension, error) { + return &cloudwatchExtension{ + config: config.(*Config), + logger: settings.Logger, + }, nil +} + +func (c *cloudwatchExtension) Start(_ context.Context, _ component.Host) error { + return nil +} + +func (c *cloudwatchExtension) Shutdown(_ context.Context) error { + return nil +} + +func decompress(decoded []byte, encoding contentEncoding) ([]byte, error) { + switch encoding { + case NoEncoding: + return decoded, nil + case GZipEncoded: + reader, err := gzip.NewReader(bytes.NewReader(decoded)) + if err != nil { + return nil, fmt.Errorf("failed to create gzip reader: %w", err) + } + defer reader.Close() + + decompressed, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("failed to read from gzip reader: %w", err) + } + return decompressed, nil + default: + // not possible, prevented by config.Validate + return nil, nil + } +} + +func getRecordsData(buf []byte, encoding contentEncoding) ([][]byte, error) { + var p payload + if err := json.Unmarshal(buf, &p); err != nil { + return nil, fmt.Errorf("failed to unmarshall firehose payload: %w", err) + } + + var records [][]byte + for i, rec := range p.Records { + decoded, err := base64.StdEncoding.DecodeString(rec.Data) + if err != nil { + return nil, fmt.Errorf("failed to base64 decode data from record [%d]: %w", i, err) + } + decompressed, err := decompress(decoded, encoding) + if err != nil { + return nil, fmt.Errorf("failed to decompress decoded data from record [%d]: %w", i, err) + } + records = append(records, decompressed) + } + + return records, nil +} + +func isCloudwatchLogValid(log cloudwatchLog) (bool, error) { + if log.Owner == "" { + return false, errors.New("cloudwatch log from firehose record data is missing owner field") + } + if log.LogGroup == "" { + return false, errors.New("cloudwatch log from firehose record data is missing log group field") + } + if log.LogStream == "" { + return false, errors.New("cloudwatch log from firehose record data is missing log stream field") + } + return true, nil +} + +func addLog(log cloudwatchLog, logs plog.Logs) error { + // Log events is a list of cloudwatch logs. + // It needs to be sent to the translator + // as a bytes array that has a new line + // splitting all logs. + result := gjson.ParseBytes(log.LogEvents) + var buf bytes.Buffer + result.ForEach(func(_, value gjson.Result) bool { + buf.WriteString(value.Raw) + buf.WriteByte('\n') + return true + }) + + logRecords, err := cloudwatch.UnmarshalLogs(buf.Bytes()) + if err != nil { + return err + } + + rl := logs.ResourceLogs().AppendEmpty() + lrs := rl.ScopeLogs().AppendEmpty().LogRecords() + logRecords.CopyTo(lrs) + resourceAttrs := rl.Resource().Attributes() + resourceAttrs.PutStr(conventions.AttributeCloudAccountID, log.Owner) + resourceAttrs.PutStr(attributeAWSCloudWatchLogGroupName, log.LogGroup) + resourceAttrs.PutStr(attributeAWSCloudWatchLogStreamName, log.LogStream) + return nil +} + +func (c *cloudwatchExtension) UnmarshalLogs(buf []byte) (plog.Logs, error) { + allData, e := getRecordsData(buf, c.config.LogsEncoding) + if e != nil { + return plog.Logs{}, e + } + + logs := plog.NewLogs() + for i, data := range allData { + var log cloudwatchLog + if err := json.Unmarshal(data, &log); err != nil { + return plog.Logs{}, fmt.Errorf("failed to unmarshall cloudwatch log from record data [%d]: %w", i, err) + } + if valid, err := isCloudwatchLogValid(log); !valid { + return plog.Logs{}, fmt.Errorf("cloudwatch log from record data [%d] is invalid: %w", i, err) + } + if err := addLog(log, logs); err != nil { + return plog.Logs{}, fmt.Errorf("failed to add cloudwatch log from record data [%d]: %w", i, err) + } + } + + if logs.ResourceLogs().Len() == 0 { + return logs, errors.New("no resource logs could be obtained from the firehose records") + } + pdatautil.GroupByResourceLogs(logs.ResourceLogs()) + return logs, nil +} + +func (c *cloudwatchExtension) UnmarshalMetrics(buf []byte) (pmetric.Metrics, error) { + allData, e := getRecordsData(buf, c.config.MetricsEncoding) + if e != nil { + return pmetric.Metrics{}, e + } + + metrics := pmetric.NewMetrics() + for i, data := range allData { + recordMetrics, err := cloudwatch.UnmarshalMetrics(data) + if err != nil { + return pmetric.Metrics{}, fmt.Errorf("failed to unmarshall metrics from record data [%d]: %w", i, err) + } + metrics = expmetrics.Merge(metrics, recordMetrics) + } + + if metrics.MetricCount() == 0 { + return metrics, errors.New("no resource metrics could be obtained from the record") + } + + metrics = expmetrics.Merge(pmetric.NewMetrics(), metrics) + return metrics, nil +} diff --git a/extension/encoding/firehoseencodingextension/extension_test.go b/extension/encoding/firehoseencodingextension/extension_test.go new file mode 100644 index 000000000000..2181bcd6d124 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/extension_test.go @@ -0,0 +1,238 @@ +package firehoseencodingextension + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.uber.org/zap" + "os" + "path/filepath" + "strings" + "testing" +) + +func compressAndEncode(content []byte) (string, error) { + var compressed bytes.Buffer + gzipWriter := gzip.NewWriter(&compressed) + defer gzipWriter.Close() + _, err := gzipWriter.Write(content) + if err != nil { + return "", err + } + gzipWriter.Close() + encoded := base64.StdEncoding.EncodeToString(compressed.Bytes()) + return encoded, nil +} + +func TestUnmarshalLogs_Records(t *testing.T) { + t.Parallel() + + extension := cloudwatchExtension{ + config: &Config{ + Metrics: Metrics{ + MetricsEncoding: NoEncoding, + }, + Logs: Logs{ + LogsEncoding: GZipEncoded, + }, + }, + logger: zap.NewNop(), + } + + tests := map[string]struct { + filename string + err error + }{ + "OneLogRecord": { + filename: "one_log", + }, + "TwoLogRecords": { + filename: "two_logs", + }, + "InvalidRecord": { + filename: "invalid_log", + err: fmt.Errorf("failed to add cloudwatch log from record data [0]: %w", + fmt.Errorf("cloudwatch log from datum [0] is invalid: %w", + errors.New("cloudwatch log is missing timestamp field"), + ), + ), + }, + "SomeInvalidRecord": { + filename: "some_invalid_log", + err: fmt.Errorf("failed to add cloudwatch log from record data [0]: %w", + fmt.Errorf("cloudwatch log from datum [1] is invalid: %w", + errors.New("cloudwatch log is missing timestamp field"), + ), + ), + }, + "EmptyLogEvents": { + filename: "empty_log", + err: fmt.Errorf("failed to add cloudwatch log from record data [0]: %w", + errors.New("no log records could be obtained from the record"), + ), + }, + } + + unmarshaller := plog.JSONUnmarshaler{} + for name, test := range tests { + t.Run(name, func(t *testing.T) { + content, err := os.ReadFile(filepath.Join("testdata/logs", test.filename+".json")) + require.NoError(t, err) + + encoded, err := compressAndEncode(content) + require.NoError(t, err) + + // put the data in the format firehose expects + // it to be + p := payload{Records: []record{{Data: encoded}}} + allData, err := json.Marshal(p) + require.NoError(t, err) + + result, err := extension.UnmarshalLogs(allData) + require.Equal(t, test.err, err) + if err != nil { + return + } + + content, err = os.ReadFile(filepath.Join("testdata/logs", test.filename+"_expected.json")) + require.NoError(t, err) + expected, err := unmarshaller.UnmarshalLogs(content) + require.NoError(t, err) + + require.Equal(t, expected, result) + }) + } +} + +func TestUnmarshalLogs_MultipleResources(t *testing.T) { + t.Parallel() + + extension := cloudwatchExtension{ + config: &Config{ + Metrics: Metrics{ + MetricsEncoding: NoEncoding, + }, + Logs: Logs{ + LogsEncoding: GZipEncoded, + }, + }, + logger: zap.NewNop(), + } + + filename1 := "one_log" + content1, err := os.ReadFile(filepath.Join("testdata/logs", filename1+".json")) + require.NoError(t, err) + encoded1, err := compressAndEncode(content1) + require.NoError(t, err) + + filename2 := "two_logs" + content2, err := os.ReadFile(filepath.Join("testdata/logs", filename2+".json")) + require.NoError(t, err) + encoded2, err := compressAndEncode(content2) + require.NoError(t, err) + + // put the data in the format firehose expects it to be + p := payload{Records: []record{{Data: encoded1}, {Data: encoded2}}} + allData, err := json.Marshal(p) + require.NoError(t, err) + + result, err := extension.UnmarshalLogs(allData) + require.NoError(t, err) + + content, err := os.ReadFile("testdata/logs/multiple_resources_expected.json") + require.NoError(t, err) + unmarshaller := plog.JSONUnmarshaler{} + expected, err := unmarshaller.UnmarshalLogs(content) + require.NoError(t, err) + + require.Equal(t, expected, result) +} + +func TestUnmarshalMetrics(t *testing.T) { + t.Parallel() + + extension := cloudwatchExtension{ + config: &Config{ + Metrics: Metrics{ + MetricsEncoding: NoEncoding, + }, + Logs: Logs{ + LogsEncoding: GZipEncoded, + }, + }, + logger: zap.NewNop(), + } + + tests := map[string]struct { + filename string + errPrefix string + }{ + "OneMetric": { + filename: "one_metric", + }, + "TwoMetrics": { + filename: "two_metrics", + }, + "MultipleResources": { + filename: "multiple_resources", + }, + "InvalidRecord": { + filename: "invalid_metric", + errPrefix: "failed to unmarshall metrics from record data [0]:", + }, + "SomeInvalidRecord": { + filename: "some_invalid_metrics", + errPrefix: "failed to unmarshall metrics from record data [1]:", + }, + "EmptyRecord": { + filename: "empty_record", + errPrefix: "no resource metrics could be obtained from the record", + }, + } + + unmarshaller := pmetric.JSONUnmarshaler{} + for name, test := range tests { + t.Run(name, func(t *testing.T) { + content, err := os.ReadFile(filepath.Join("testdata/metrics", test.filename+".json")) + require.NoError(t, err) + + // for each metric, encode base64 + var records []record + var buf bytes.Buffer + gjson.ParseBytes(content).ForEach(func(_, value gjson.Result) bool { + data, e := json.Marshal(value.Value()) + require.NoError(t, e) + encoded := base64.StdEncoding.EncodeToString(data) + _, err = buf.WriteString(encoded + "\n") + require.NoError(t, err) + records = append(records, record{Data: encoded}) + return true + }) + + p := payload{Records: records} + allData, err := json.Marshal(p) + require.NoError(t, err) + + result, err := extension.UnmarshalMetrics(allData) + if err != nil { + require.NotEmpty(t, test.errPrefix) + require.True(t, strings.HasPrefix(err.Error(), test.errPrefix)) + return + } + + content, err = os.ReadFile(filepath.Join("testdata/metrics", test.filename+"_expected.json")) + require.NoError(t, err) + expected, err := unmarshaller.UnmarshalMetrics(content) + require.NoError(t, err) + + require.Equal(t, expected, result) + }) + } +} diff --git a/extension/encoding/cloudwatchencodingextension/factory.go b/extension/encoding/firehoseencodingextension/factory.go similarity index 57% rename from extension/encoding/cloudwatchencodingextension/factory.go rename to extension/encoding/firehoseencodingextension/factory.go index 8327968d2653..c3fd23fed14e 100644 --- a/extension/encoding/cloudwatchencodingextension/factory.go +++ b/extension/encoding/firehoseencodingextension/factory.go @@ -1,7 +1,7 @@ -package cloudwatchencodingextension +package firehoseencodingextension import ( - "cloudwatchencodingextension/internal/metadata" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension/internal/metadata" "go.opentelemetry.io/collector/extension" ) diff --git a/extension/encoding/firehoseencodingextension/factory_test.go b/extension/encoding/firehoseencodingextension/factory_test.go new file mode 100644 index 000000000000..2bb6ea1790d9 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/factory_test.go @@ -0,0 +1,15 @@ +package firehoseencodingextension + +import ( + "context" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/extension/extensiontest" + "testing" +) + +func TestCreateExtension(t *testing.T) { + cfg := createDefaultConfig().(*Config) + ext, err := createExtension(context.Background(), extensiontest.NewNopSettings(), cfg) + require.NoError(t, err) + require.NotNil(t, ext) +} diff --git a/extension/encoding/cloudwatchencodingextension/generated_component_test.go b/extension/encoding/firehoseencodingextension/generated_component_test.go similarity index 93% rename from extension/encoding/cloudwatchencodingextension/generated_component_test.go rename to extension/encoding/firehoseencodingextension/generated_component_test.go index ca9003d4db78..5d4451c68c5d 100644 --- a/extension/encoding/cloudwatchencodingextension/generated_component_test.go +++ b/extension/encoding/firehoseencodingextension/generated_component_test.go @@ -1,6 +1,6 @@ // Code generated by mdatagen. DO NOT EDIT. -package cloudwatchencodingextension +package firehoseencodingextension import ( "context" @@ -13,7 +13,7 @@ import ( ) func TestComponentFactoryType(t *testing.T) { - require.Equal(t, "cloudwatch_encoding", NewFactory().Type().String()) + require.Equal(t, "firehose_encoding", NewFactory().Type().String()) } func TestComponentConfigStruct(t *testing.T) { diff --git a/extension/encoding/cloudwatchencodingextension/generated_package_test.go b/extension/encoding/firehoseencodingextension/generated_package_test.go similarity index 80% rename from extension/encoding/cloudwatchencodingextension/generated_package_test.go rename to extension/encoding/firehoseencodingextension/generated_package_test.go index 6a98f31ff6c3..575f3c8e6b14 100644 --- a/extension/encoding/cloudwatchencodingextension/generated_package_test.go +++ b/extension/encoding/firehoseencodingextension/generated_package_test.go @@ -1,6 +1,6 @@ // Code generated by mdatagen. DO NOT EDIT. -package cloudwatchencodingextension +package firehoseencodingextension import ( "go.uber.org/goleak" diff --git a/extension/encoding/cloudwatchencodingextension/go.mod b/extension/encoding/firehoseencodingextension/go.mod similarity index 93% rename from extension/encoding/cloudwatchencodingextension/go.mod rename to extension/encoding/firehoseencodingextension/go.mod index 0880f8342a6d..0a18fc111493 100644 --- a/extension/encoding/cloudwatchencodingextension/go.mod +++ b/extension/encoding/firehoseencodingextension/go.mod @@ -4,14 +4,17 @@ go 1.23 require ( github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding v0.117.0 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch v0.0.0-00010101000000-000000000000 github.com/stretchr/testify v1.10.0 + github.com/tidwall/gjson v1.18.0 go.opentelemetry.io/collector/component v0.117.0 go.opentelemetry.io/collector/component/componenttest v0.117.0 go.opentelemetry.io/collector/confmap v1.23.0 go.opentelemetry.io/collector/extension v0.117.0 go.opentelemetry.io/collector/extension/extensiontest v0.117.0 go.opentelemetry.io/collector/pdata v1.23.0 + go.opentelemetry.io/collector/semconv v0.117.0 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 ) @@ -33,12 +36,12 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.117.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.117.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.117.0 // indirect go.opentelemetry.io/collector/pdata/pprofile v0.117.0 // indirect - go.opentelemetry.io/collector/semconv v0.117.0 // indirect go.opentelemetry.io/otel v1.32.0 // indirect go.opentelemetry.io/otel/metric v1.32.0 // indirect go.opentelemetry.io/otel/sdk v1.32.0 // indirect diff --git a/extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go b/extension/encoding/firehoseencodingextension/internal/metadata/generated_status.go similarity index 84% rename from extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go rename to extension/encoding/firehoseencodingextension/internal/metadata/generated_status.go index d64ea26b7694..dc1b03ab839f 100644 --- a/extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go +++ b/extension/encoding/firehoseencodingextension/internal/metadata/generated_status.go @@ -7,7 +7,7 @@ import ( ) var ( - Type = component.MustNewType("cloudwatch_encoding") + Type = component.MustNewType("firehose_encoding") ScopeName = "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" ) diff --git a/extension/encoding/cloudwatchencodingextension/metadata.yaml b/extension/encoding/firehoseencodingextension/metadata.yaml similarity index 81% rename from extension/encoding/cloudwatchencodingextension/metadata.yaml rename to extension/encoding/firehoseencodingextension/metadata.yaml index 1b902fc6ce83..c6c4cc5dae73 100644 --- a/extension/encoding/cloudwatchencodingextension/metadata.yaml +++ b/extension/encoding/firehoseencodingextension/metadata.yaml @@ -1,4 +1,4 @@ -type: cloudwatch_encoding +type: firehose_encoding status: class: extension diff --git a/extension/encoding/firehoseencodingextension/testdata/config.yaml b/extension/encoding/firehoseencodingextension/testdata/config.yaml new file mode 100644 index 000000000000..be3dbb1578d5 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/config.yaml @@ -0,0 +1,9 @@ +firehose_encoding: + +firehose_encoding/logs_invalid_encoding: + logs: + content_encoding: invalid + +firehose_encoding/metrics_invalid_encoding: + metrics: + content_encoding: invalid \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/empty_log.json b/extension/encoding/firehoseencodingextension/testdata/logs/empty_log.json new file mode 100644 index 000000000000..7c35990d2bf2 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/logs/empty_log.json @@ -0,0 +1,10 @@ +{ + "messageType":"DATA_MESSAGE", + "owner":"1234567890", + "logGroup":"test", + "logStream":"test-log-stream", + "subscriptionFilters":[ + "test logs filter" + ], + "logEvents":[] +} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/invalid_log.json b/extension/encoding/firehoseencodingextension/testdata/logs/invalid_log.json new file mode 100644 index 000000000000..b71ac98d123a --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/logs/invalid_log.json @@ -0,0 +1,18 @@ +{ + "messageType":"DATA_MESSAGE", + "owner":"1234567890", + "logGroup":"test", + "logStream":"test-log-stream", + "subscriptionFilters":[ + "test logs filter" + ], + "logEvents":[ + { + "id":"111", + "message":"it should fail without timestamp", + "extractedFields":{ + "action":"REJECT" + } + } + ] +} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/multiple_resources_expected.json b/extension/encoding/firehoseencodingextension/testdata/logs/multiple_resources_expected.json new file mode 100644 index 000000000000..0ff798dc63ae --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/logs/multiple_resources_expected.json @@ -0,0 +1,94 @@ +{ + "resourceLogs":[ + { + "resource":{ + "attributes":[ + { + "key":"cloud.account.id", + "value":{ + "stringValue":"1234567890" + } + }, + { + "key":"aws.cloudwatch.log_group_name", + "value":{ + "stringValue":"vpc-flow-logs" + } + }, + { + "key":"aws.cloudwatch.log_stream_name", + "value":{ + "stringValue":"test-log-stream" + } + } + ] + }, + "scopeLogs":[ + { + "scope":{ + + }, + "logRecords":[ + { + "timeUnixNano":"1646096348000000000", + "body":{ + "stringValue":"REJECT" + }, + "traceId":"", + "spanId":"" + } + ] + } + ] + }, + { + "resource":{ + "attributes":[ + { + "key":"cloud.account.id", + "value":{ + "stringValue":"1234567890" + } + }, + { + "key":"aws.cloudwatch.log_group_name", + "value":{ + "stringValue":"vpc-flow-logs" + } + }, + { + "key":"aws.cloudwatch.log_stream_name", + "value":{ + "stringValue":"test-log-stream-2" + } + } + ] + }, + "scopeLogs":[ + { + "scope":{ + + }, + "logRecords":[ + { + "timeUnixNano":"1646096348000000000", + "body":{ + "stringValue":"REJECT" + }, + "traceId":"", + "spanId":"" + }, + { + "timeUnixNano":"1725544035523000000", + "body":{ + "stringValue":"Hello world!" + }, + "traceId":"", + "spanId":"" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/one_log.json b/extension/encoding/firehoseencodingextension/testdata/logs/one_log.json new file mode 100644 index 000000000000..6f958db9e0f9 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/logs/one_log.json @@ -0,0 +1,19 @@ +{ + "messageType":"DATA_MESSAGE", + "owner":"1234567890", + "logGroup":"vpc-flow-logs", + "logStream":"test-log-stream", + "subscriptionFilters":[ + "vpc flow logs filter" + ], + "logEvents":[ + { + "id":"111", + "timestamp":1646096348000, + "message":"REJECT", + "extractedFields":{ + "action":"REJECT" + } + } + ] +} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/one_log_expected.json b/extension/encoding/firehoseencodingextension/testdata/logs/one_log_expected.json new file mode 100644 index 000000000000..05fd75da83e4 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/logs/one_log_expected.json @@ -0,0 +1,45 @@ +{ + "resourceLogs":[ + { + "resource":{ + "attributes":[ + { + "key":"cloud.account.id", + "value":{ + "stringValue":"1234567890" + } + }, + { + "key":"aws.cloudwatch.log_group_name", + "value":{ + "stringValue":"vpc-flow-logs" + } + }, + { + "key":"aws.cloudwatch.log_stream_name", + "value":{ + "stringValue":"test-log-stream" + } + } + ] + }, + "scopeLogs":[ + { + "scope":{ + + }, + "logRecords":[ + { + "timeUnixNano":"1646096348000000000", + "body":{ + "stringValue":"REJECT" + }, + "traceId":"", + "spanId":"" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/some_invalid_log.json b/extension/encoding/firehoseencodingextension/testdata/logs/some_invalid_log.json new file mode 100644 index 000000000000..f0b7225b0cbc --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/logs/some_invalid_log.json @@ -0,0 +1,26 @@ +{ + "messageType":"DATA_MESSAGE", + "owner":"1234567890", + "logGroup":"test", + "logStream":"test-log-stream", + "subscriptionFilters":[ + "test logs filter" + ], + "logEvents":[ + { + "id":"111", + "timestamp":1646096348000, + "message":"REJECT", + "extractedFields":{ + "action":"REJECT" + } + }, + { + "id":"111", + "message":"it should fail without timestamp", + "extractedFields":{ + "action":"REJECT" + } + } + ] +} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/two_logs.json b/extension/encoding/firehoseencodingextension/testdata/logs/two_logs.json new file mode 100644 index 000000000000..34b33b06d1d3 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/logs/two_logs.json @@ -0,0 +1,24 @@ +{ + "messageType":"DATA_MESSAGE", + "owner":"1234567890", + "logGroup":"vpc-flow-logs", + "logStream":"test-log-stream-2", + "subscriptionFilters":[ + "vpc flow logs filter" + ], + "logEvents":[ + { + "id":"111", + "timestamp":1646096348000, + "message":"REJECT", + "extractedFields":{ + "action":"REJECT" + } + }, + { + "id":"38480917865042697267627490045603633139480491071049695232", + "timestamp":1725544035523, + "message":"Hello world!" + } + ] +} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/two_logs_expected.json b/extension/encoding/firehoseencodingextension/testdata/logs/two_logs_expected.json new file mode 100644 index 000000000000..fb361b2d8610 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/logs/two_logs_expected.json @@ -0,0 +1,53 @@ +{ + "resourceLogs":[ + { + "resource":{ + "attributes":[ + { + "key":"cloud.account.id", + "value":{ + "stringValue":"1234567890" + } + }, + { + "key":"aws.cloudwatch.log_group_name", + "value":{ + "stringValue":"vpc-flow-logs" + } + }, + { + "key":"aws.cloudwatch.log_stream_name", + "value":{ + "stringValue":"test-log-stream-2" + } + } + ] + }, + "scopeLogs":[ + { + "scope":{ + + }, + "logRecords":[ + { + "timeUnixNano":"1646096348000000000", + "body":{ + "stringValue":"REJECT" + }, + "traceId":"", + "spanId":"" + }, + { + "timeUnixNano":"1725544035523000000", + "body":{ + "stringValue":"Hello world!" + }, + "traceId":"", + "spanId":"" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/two_resources.json b/extension/encoding/firehoseencodingextension/testdata/logs/two_resources.json new file mode 100644 index 000000000000..0b162177607d --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/logs/two_resources.json @@ -0,0 +1,34 @@ +[ + { + "messageType":"DATA_MESSAGE", + "owner":"1234567890", + "logGroup":"test", + "logStream":"test-log-stream-1", + "subscriptionFilters":[ + "logs filter" + ], + "logEvents":[ + { + "id":"38480917865042697267627490045603633139480491071049695232", + "timestamp":1725544035523, + "message":"Hello world!" + } + ] + }, + { + "messageType":"DATA_MESSAGE", + "owner":"1234567890", + "logGroup":"test", + "logStream":"test-log-stream-1", + "subscriptionFilters":[ + "logs filter" + ], + "logEvents":[ + { + "id":"38480917865042697267627490045603633139480491071049695232", + "timestamp":1725544035523, + "message":"Hello world!" + } + ] + } +] \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/empty_record.json b/extension/encoding/firehoseencodingextension/testdata/metrics/empty_record.json new file mode 100644 index 000000000000..0637a088a01e --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/metrics/empty_record.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/invalid_metric.json b/extension/encoding/firehoseencodingextension/testdata/metrics/invalid_metric.json new file mode 100644 index 000000000000..dfad993fcbd2 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/metrics/invalid_metric.json @@ -0,0 +1,5 @@ +[ + { + "invalid": "record" + } +] \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources.json b/extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources.json new file mode 100644 index 000000000000..bd75a1906dc2 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources.json @@ -0,0 +1,44 @@ +[ + { + "metric_stream_name": "MyMetricStream-1", + "account_id": "1234567890", + "region": "us-east-1", + "namespace": "AWS/EC2", + "metric_name": "DiskWriteOps", + "dimensions": { + "InstanceId": "i-123456789012" + }, + "timestamp": 1611929698000, + "value": { + "count": 3.0, + "sum": 20.0, + "max": 18.0, + "min": 0.0, + "p99": 17.56, + "p99.9": 17.8764, + "TM(25%:75%)": 16.43 + }, + "unit": "Seconds" + }, + { + "metric_stream_name": "MyMetricStream-2", + "account_id": "1234567890", + "region": "us-east-1", + "namespace": "AWS/EC2", + "metric_name": "DiskWriteOps", + "dimensions": { + "InstanceId": "i-123456789012" + }, + "timestamp": 1611929698000, + "value": { + "count": 3.0, + "sum": 20.0, + "max": 18.0, + "min": 0.0, + "p99": 17.56, + "p99.9": 17.8764, + "TM(25%:75%)": 16.43 + }, + "unit": "Seconds" + } +] \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources_expected.json b/extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources_expected.json new file mode 100644 index 000000000000..e67e2c2f6d51 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources_expected.json @@ -0,0 +1,182 @@ +{ + "resourceMetrics":[ + { + "resource":{ + "attributes":[ + { + "key":"cloud.provider", + "value":{ + "stringValue":"aws" + } + }, + { + "key":"cloud.account.id", + "value":{ + "stringValue":"1234567890" + } + }, + { + "key":"cloud.region", + "value":{ + "stringValue":"us-east-1" + } + }, + { + "key":"service.namespace", + "value":{ + "stringValue":"AWS" + } + }, + { + "key":"service.name", + "value":{ + "stringValue":"EC2" + } + }, + { + "key":"aws.cloudwatch.metric_stream_name", + "value":{ + "stringValue":"MyMetricStream-1" + } + } + ] + }, + "scopeMetrics":[ + { + "scope":{ + + }, + "metrics":[ + { + "name":"DiskWriteOps", + "unit":"Seconds", + "summary":{ + "dataPoints":[ + { + "attributes":[ + { + "key":"service.instance.id", + "value":{ + "stringValue":"i-123456789012" + } + } + ], + "timeUnixNano":"1611929698000000000", + "count":"3", + "sum":20, + "quantileValues":[ + { + "quantile":99, + "value":17.56 + }, + { + "quantile":99.9, + "value":17.8764 + }, + { + + }, + { + "quantile":100, + "value":18 + } + ] + } + ] + } + } + ] + } + ] + }, + { + "resource":{ + "attributes":[ + { + "key":"cloud.provider", + "value":{ + "stringValue":"aws" + } + }, + { + "key":"cloud.account.id", + "value":{ + "stringValue":"1234567890" + } + }, + { + "key":"cloud.region", + "value":{ + "stringValue":"us-east-1" + } + }, + { + "key":"service.namespace", + "value":{ + "stringValue":"AWS" + } + }, + { + "key":"service.name", + "value":{ + "stringValue":"EC2" + } + }, + { + "key":"aws.cloudwatch.metric_stream_name", + "value":{ + "stringValue":"MyMetricStream-2" + } + } + ] + }, + "scopeMetrics":[ + { + "scope":{ + + }, + "metrics":[ + { + "name":"DiskWriteOps", + "unit":"Seconds", + "summary":{ + "dataPoints":[ + { + "attributes":[ + { + "key":"service.instance.id", + "value":{ + "stringValue":"i-123456789012" + } + } + ], + "timeUnixNano":"1611929698000000000", + "count":"3", + "sum":20, + "quantileValues":[ + { + "quantile":99, + "value":17.56 + }, + { + "quantile":99.9, + "value":17.8764 + }, + { + + }, + { + "quantile":100, + "value":18 + } + ] + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/one_metric.json b/extension/encoding/firehoseencodingextension/testdata/metrics/one_metric.json new file mode 100644 index 000000000000..3e40df35a073 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/metrics/one_metric.json @@ -0,0 +1,23 @@ +[ + { + "metric_stream_name":"MyMetricStream", + "account_id":"1234567890", + "region":"us-east-1", + "namespace":"AWS/EC2", + "metric_name":"DiskWriteOps", + "dimensions":{ + "InstanceId":"i-123456789012" + }, + "timestamp":1611929698000, + "value":{ + "count":3.0, + "sum":20.0, + "max":18.0, + "min":0.0, + "p99":17.56, + "p99.9":17.8764, + "TM(25%:75%)":16.43 + }, + "unit":"Seconds" + } +] \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/one_metric_expected.json b/extension/encoding/firehoseencodingextension/testdata/metrics/one_metric_expected.json new file mode 100644 index 000000000000..3e9fe178648a --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/metrics/one_metric_expected.json @@ -0,0 +1,93 @@ +{ + "resourceMetrics":[ + { + "resource":{ + "attributes":[ + { + "key":"cloud.provider", + "value":{ + "stringValue":"aws" + } + }, + { + "key":"cloud.account.id", + "value":{ + "stringValue":"1234567890" + } + }, + { + "key":"cloud.region", + "value":{ + "stringValue":"us-east-1" + } + }, + { + "key":"service.namespace", + "value":{ + "stringValue":"AWS" + } + }, + { + "key":"service.name", + "value":{ + "stringValue":"EC2" + } + }, + { + "key":"aws.cloudwatch.metric_stream_name", + "value":{ + "stringValue":"MyMetricStream" + } + } + ] + }, + "scopeMetrics":[ + { + "scope":{ + + }, + "metrics":[ + { + "name":"DiskWriteOps", + "unit":"Seconds", + "summary":{ + "dataPoints":[ + { + "attributes":[ + { + "key":"service.instance.id", + "value":{ + "stringValue":"i-123456789012" + } + } + ], + "timeUnixNano":"1611929698000000000", + "count":"3", + "sum":20, + "quantileValues":[ + { + "quantile":99, + "value":17.56 + }, + { + "quantile":99.9, + "value":17.8764 + }, + { + + }, + { + "quantile":100, + "value":18 + } + ] + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/some_invalid_metrics.json b/extension/encoding/firehoseencodingextension/testdata/metrics/some_invalid_metrics.json new file mode 100644 index 000000000000..636000e58ca1 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/metrics/some_invalid_metrics.json @@ -0,0 +1,26 @@ +[ + { + "metric_stream_name":"MyMetricStream", + "account_id":"1234567890", + "region":"us-east-1", + "namespace":"AWS/EC2", + "metric_name":"DiskWriteOps", + "dimensions":{ + "InstanceId":"i-123456789012" + }, + "timestamp":1611929698000, + "value":{ + "count":3.0, + "sum":20.0, + "max":18.0, + "min":0.0, + "p99":17.56, + "p99.9":17.8764, + "TM(25%:75%)":16.43 + }, + "unit":"Seconds" + }, + { + "invalid": "record" + } +] \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics.json b/extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics.json new file mode 100644 index 000000000000..1127193c5ef1 --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics.json @@ -0,0 +1,44 @@ +[ + { + "metric_stream_name":"MyMetricStream", + "account_id":"1234567890", + "region":"us-east-1", + "namespace":"AWS/EC2", + "metric_name":"DiskWriteOps", + "dimensions":{ + "InstanceId":"i-123456789012" + }, + "timestamp":1611929698000, + "value":{ + "count":3.0, + "sum":20.0, + "max":18.0, + "min":0.0, + "p99":17.56, + "p99.9":17.8764, + "TM(25%:75%)":16.43 + }, + "unit":"Seconds" + }, + { + "metric_stream_name":"MyMetricStream", + "account_id":"1234567890", + "region":"us-east-1", + "namespace":"AWS/EC2", + "metric_name":"DiskWriteOps", + "dimensions":{ + "InstanceId":"i-123456789012" + }, + "timestamp":1611929698100, + "value":{ + "count":5.0, + "sum":21.0, + "max":18.0, + "min":0.0, + "p99":17.56, + "p99.9":17.8764, + "TM(25%:75%)":16.43 + }, + "unit":"Seconds" + } +] \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics_expected.json b/extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics_expected.json new file mode 100644 index 000000000000..6dbc5c2422ee --- /dev/null +++ b/extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics_expected.json @@ -0,0 +1,123 @@ +{ + "resourceMetrics":[ + { + "resource":{ + "attributes":[ + { + "key":"cloud.provider", + "value":{ + "stringValue":"aws" + } + }, + { + "key":"cloud.account.id", + "value":{ + "stringValue":"1234567890" + } + }, + { + "key":"cloud.region", + "value":{ + "stringValue":"us-east-1" + } + }, + { + "key":"service.namespace", + "value":{ + "stringValue":"AWS" + } + }, + { + "key":"service.name", + "value":{ + "stringValue":"EC2" + } + }, + { + "key":"aws.cloudwatch.metric_stream_name", + "value":{ + "stringValue":"MyMetricStream" + } + } + ] + }, + "scopeMetrics":[ + { + "scope":{ + + }, + "metrics":[ + { + "name":"DiskWriteOps", + "unit":"Seconds", + "summary":{ + "dataPoints":[ + { + "attributes":[ + { + "key":"service.instance.id", + "value":{ + "stringValue":"i-123456789012" + } + } + ], + "timeUnixNano":"1611929698000000000", + "count":"3", + "sum":20, + "quantileValues":[ + { + "quantile":99, + "value":17.56 + }, + { + "quantile":99.9, + "value":17.8764 + }, + { + + }, + { + "quantile":100, + "value":18 + } + ] + }, + { + "attributes":[ + { + "key":"service.instance.id", + "value":{ + "stringValue":"i-123456789012" + } + } + ], + "timeUnixNano":"1611929698100000000", + "count":"5", + "sum":21, + "quantileValues":[ + { + "quantile":99, + "value":17.56 + }, + { + "quantile":99.9, + "value":17.8764 + }, + { + + }, + { + "quantile":100, + "value":18 + } + ] + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/cwlogs_to_log_record.go b/pkg/translator/cloudwatch/cwlogs_to_log_record.go new file mode 100644 index 000000000000..cdda408b751c --- /dev/null +++ b/pkg/translator/cloudwatch/cwlogs_to_log_record.go @@ -0,0 +1,60 @@ +package cloudwatch + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "go.opentelemetry.io/collector/pdata/pcommon" + "time" + + "go.opentelemetry.io/collector/pdata/plog" + "io" +) + +type cloudwatchLog struct { + Timestamp int64 `json:"timestamp"` + Message string `json:"message"` +} + +// isLogValid validates that the cloudwatch log has been unmarshalled correctly +func isLogValid(log cloudwatchLog) (bool, error) { + if log.Timestamp == 0 { + return false, errors.New("cloudwatch log is missing timestamp field") + } + if len(log.Message) == 0 { + return false, errors.New("cloudwatch log message field is empty") + } + return true, nil +} + +func addRecord(log cloudwatchLog, records plog.LogRecordSlice) { + logRecord := records.AppendEmpty() + logRecord.SetTimestamp(pcommon.NewTimestampFromTime(time.UnixMilli(log.Timestamp))) + logRecord.Body().SetStr(log.Message) +} + +func UnmarshalLogs(record []byte) (plog.LogRecordSlice, error) { + logs := plog.NewLogRecordSlice() + decoder := json.NewDecoder(bytes.NewReader(record)) + for datumIndex := 0; ; datumIndex++ { + var log cloudwatchLog + if err := decoder.Decode(&log); err != nil { + if errors.Is(err, io.EOF) { + break + } + return plog.LogRecordSlice{}, + fmt.Errorf("unable to unmarshal datum [%d] into cloudwatch log: %w", datumIndex, err) + } + if valid, err := isLogValid(log); !valid { + return plog.LogRecordSlice{}, + fmt.Errorf("cloudwatch log from datum [%d] is invalid: %w", datumIndex, err) + } + addRecord(log, logs) + } + + if logs.Len() == 0 { + return logs, errors.New("no log records could be obtained from the record") + } + return logs, nil +} diff --git a/pkg/translator/cloudwatch/cwlogs_to_log_record_test.go b/pkg/translator/cloudwatch/cwlogs_to_log_record_test.go new file mode 100644 index 000000000000..30e28dea5c66 --- /dev/null +++ b/pkg/translator/cloudwatch/cwlogs_to_log_record_test.go @@ -0,0 +1,79 @@ +package cloudwatch + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "go.opentelemetry.io/collector/pdata/plog" + "os" + "path/filepath" + "testing" +) + +func TestUnmarshalLogs(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + filename string + err error + }{ + "SingleRecord": { + filename: "single_record", + }, + "MultipleRecords": { + filename: "multiple_records", + }, + "InvalidRecord": { + filename: "invalid_record", + err: fmt.Errorf( + "cloudwatch log from datum [0] is invalid: %w", + errors.New("cloudwatch log is missing timestamp field"), + ), + }, + "SomeInvalidRecord": { + filename: "some_invalid_record", + err: fmt.Errorf( + "cloudwatch log from datum [1] is invalid: %w", + errors.New("cloudwatch log is missing timestamp field"), + ), + }, + "EmptyRecord": { + filename: "empty_record", + err: errors.New("no log records could be obtained from the record"), + }, + } + unmarshaller := &plog.JSONUnmarshaler{} + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + content, err := os.ReadFile(filepath.Join("testdata/log", testCase.filename+".json")) + require.NoError(t, err) + // since new line represents the end of the record, we + // need to remove all new lines from the json file, so + // the record will not get incorrectly split. We keep + // the new lines between different records. + var buf bytes.Buffer + gjson.ParseBytes(content).ForEach(func(_, value gjson.Result) bool { + err := json.NewEncoder(&buf).Encode(value.Value()) + require.NoError(t, err) + return true + }) + result, err := UnmarshalLogs(buf.Bytes()) + require.Equal(t, testCase.err, err) + if err != nil { + return + } + + content, err = os.ReadFile(filepath.Join("testdata/log", testCase.filename+"_expected.json")) + require.NoError(t, err) + expected, err := unmarshaller.UnmarshalLogs(content) + require.NoError(t, err) + + // get log records + expectedLogs := expected.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords() + require.Equal(t, expectedLogs, result) + }) + } +} diff --git a/pkg/translator/cloudwatch/cwlogs_to_logs.go b/pkg/translator/cloudwatch/cwlogs_to_logs.go deleted file mode 100644 index d2ba5668eedd..000000000000 --- a/pkg/translator/cloudwatch/cwlogs_to_logs.go +++ /dev/null @@ -1,97 +0,0 @@ -package cloudwatch - -import ( - "bytes" - "compress/gzip" - "encoding/json" - "errors" - "fmt" - "github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil" - - conventions "go.opentelemetry.io/collector/semconv/v1.6.1" - - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/plog" - "go.uber.org/zap" - "io" - "time" -) - -type cloudwatchLog struct { - MessageType string `json:"messageType"` - Owner string `json:"owner"` - LogGroup string `json:"logGroup"` - LogStream string `json:"logStream"` - SubscriptionFilters []string `json:"subscriptionFilters"` - LogEvents []struct { - ID string `json:"id"` - Timestamp int64 `json:"timestamp"` - Message string `json:"message"` - } `json:"logEvents"` -} - -const ( - attributeAWSCloudWatchLogGroupName = "aws.cloudwatch.log_group_name" - attributeAWSCloudWatchLogStreamName = "aws.cloudwatch.log_stream_name" -) - -var ( - errInvalidLogRecord = errors.New("no resource logs were obtained from the compressed record") -) - -// isLogValid validates that the cloudwatch log has been unmarshalled correctly -func isLogValid(log cloudwatchLog) bool { - return log.Owner != "" && log.LogGroup != "" && log.LogStream != "" -} - -func UnmarshalLogs(compressedRecord []byte, logger *zap.Logger) (plog.Logs, error) { - r, err := gzip.NewReader(bytes.NewReader(compressedRecord)) - if err != nil { - return plog.Logs{}, fmt.Errorf("failed to create gzip reader from compressed record: %w", err) - } - decoder := json.NewDecoder(r) - - logs := plog.NewLogs() - for datumIndex := 0; ; datumIndex++ { - var log cloudwatchLog - if err = decoder.Decode(&log); err != nil { - if errors.Is(err, io.EOF) { - break - } - logger.Error( - "Unable to unmarshal input", - zap.Int("datum_index", datumIndex), - zap.Error(err), - ) - continue - } - if !isLogValid(log) { - logger.Error( - "Invalid cloudwatch log", - zap.Int("datum_index", datumIndex), - ) - continue - } - - rl := logs.ResourceLogs().AppendEmpty() - resourceAttrs := rl.Resource().Attributes() - resourceAttrs.PutStr(conventions.AttributeCloudAccountID, log.Owner) - resourceAttrs.PutStr(attributeAWSCloudWatchLogGroupName, log.LogGroup) - resourceAttrs.PutStr(attributeAWSCloudWatchLogStreamName, log.LogStream) - - logRecords := rl.ScopeLogs().AppendEmpty().LogRecords() - for _, event := range log.LogEvents { - logRecord := logRecords.AppendEmpty() - // pcommon.Timestamp is a time specified as UNIX Epoch time in nanoseconds - // but timestamp in cloudwatch logs are in milliseconds. - logRecord.SetTimestamp(pcommon.Timestamp(event.Timestamp * int64(time.Millisecond))) - logRecord.Body().SetStr(event.Message) - } - } - - if logs.ResourceLogs().Len() == 0 { - return logs, errInvalidLogRecord - } - pdatautil.GroupByResourceLogs(logs.ResourceLogs()) - return logs, nil -} diff --git a/pkg/translator/cloudwatch/cwlogs_to_logs_test.go b/pkg/translator/cloudwatch/cwlogs_to_logs_test.go deleted file mode 100644 index 8b9653f54e36..000000000000 --- a/pkg/translator/cloudwatch/cwlogs_to_logs_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package cloudwatch - -import ( - "bytes" - "compress/gzip" - "github.com/stretchr/testify/require" - "go.uber.org/zap" - "os" - "path/filepath" - "testing" -) - -func TestUnmarshalLogs(t *testing.T) { - t.Parallel() - - tests := map[string]struct { - filename string - resourceCount int - logCount int - err error - }{ - "WithMultipleRecords": { - filename: "multiple_records", - resourceCount: 1, - logCount: 2, - }, - "WithSingleRecord": { - filename: "single_record", - resourceCount: 1, - logCount: 1, - }, - "WithInvalidRecords": { - filename: "invalid_records", - err: errInvalidLogRecord, - }, - "WithSomeInvalidRecords": { - filename: "some_invalid_records", - resourceCount: 1, - logCount: 2, - }, - "WithMultipleResources": { - filename: "multiple_resources", - resourceCount: 3, - logCount: 6, - }, - } - for _, test := range tests { - t.Run(test.filename, func(t *testing.T) { - record, err := os.ReadFile(filepath.Join("testdata/log", test.filename)) - require.NoError(t, err) - - var buf bytes.Buffer - g := gzip.NewWriter(&buf) - _, err = g.Write(record) - require.NoError(t, err) - err = g.Close() - require.NoError(t, err) - compressedRecord := buf.Bytes() - - logs, err := UnmarshalLogs(compressedRecord, zap.NewNop()) - require.Equal(t, test.err, err) - require.Equal(t, test.resourceCount, logs.ResourceLogs().Len()) - require.Equal(t, test.logCount, logs.LogRecordCount()) - }) - } -} diff --git a/pkg/translator/cloudwatch/cwmetrics_to_metrics.go b/pkg/translator/cloudwatch/cwmetrics_to_metrics.go index 43e4028127ec..42170b82d310 100644 --- a/pkg/translator/cloudwatch/cwmetrics_to_metrics.go +++ b/pkg/translator/cloudwatch/cwmetrics_to_metrics.go @@ -1,18 +1,17 @@ -package cloudwatch // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch" +package cloudwatch import ( "bytes" "encoding/json" "errors" + "fmt" + expmetrics "github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" - "go.uber.org/zap" + conventions "go.opentelemetry.io/collector/semconv/v1.27.0" "io" "strings" "time" - - expmetrics "github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics" - conventions "go.opentelemetry.io/collector/semconv/v1.27.0" ) // The cloudwatchMetric is the format for the CloudWatch metric stream records. @@ -37,6 +36,8 @@ type cloudwatchMetricValue struct { Min float64 `json:"min"` Sum float64 `json:"sum"` Count float64 `json:"count"` + Q1 float64 `json:"p99"` + Q2 float64 `json:"p99.9"` } const ( @@ -45,13 +46,21 @@ const ( namespaceDelimiter = "/" ) -var ( - errInvalidMetricRecord = errors.New("no resource metrics were obtained from the record") -) - // isMetricValid validates that the cloudwatch metric has been unmarshalled correctly -func isMetricValid(metric cloudwatchMetric) bool { - return metric.MetricName != "" && metric.Namespace != "" && metric.Unit != "" && metric.Value != nil +func isMetricValid(metric cloudwatchMetric) (bool, error) { + if metric.MetricName == "" { + return false, errors.New("cloudwatch metric is missing metric name field") + } + if metric.Namespace == "" { + return false, errors.New("cloudwatch metric is missing namespace field") + } + if metric.Unit == "" { + return false, errors.New("cloudwatch metric is missing unit field") + } + if metric.Value == nil { + return false, errors.New("cloudwatch metric is missing value") + } + return true, nil } // setResourceAttributes sets attributes on a pcommon.Resource from a cwMetric. @@ -72,14 +81,15 @@ func setResourceAttributes(m cloudwatchMetric, resource pcommon.Resource) { // if prepended by AWS/. Otherwise, it returns the CloudWatch namespace as the // service name with an empty service namespace func toServiceAttributes(namespace string) (serviceNamespace, serviceName string) { - index := strings.Index(namespace, namespaceDelimiter) - if index != -1 && strings.EqualFold(namespace[:index], conventions.AttributeCloudProviderAWS) { - return namespace[:index], namespace[index+1:] + if strings.HasPrefix(namespace, strings.ToUpper(conventions.AttributeCloudProviderAWS)+"/") { + parts := strings.SplitN(namespace, namespaceDelimiter, 2) + return parts[0], parts[1] } return "", namespace } -// setResourceAttributes sets attributes on a metric data point from a cloudwatchMetric. +// setResourceAttributes sets attributes on a metric data point from +// a cloudwatchMetric dimensions func setDataPointAttributes(m cloudwatchMetric, dp pmetric.SummaryDataPoint) { attrs := dp.Attributes() for k, v := range m.Dimensions { @@ -92,7 +102,42 @@ func setDataPointAttributes(m cloudwatchMetric, dp pmetric.SummaryDataPoint) { } } -func UnmarshalMetrics(record []byte, logger *zap.Logger) (pmetric.Metrics, error) { +// addMetric transforms the cloudwatch metric to pmetric.Metric +// and appends it to pmetric.Metrics +func addMetric(cwMetric cloudwatchMetric, metrics pmetric.Metrics) { + rm := metrics.ResourceMetrics().AppendEmpty() + setResourceAttributes(cwMetric, rm.Resource()) + + metric := rm.ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() + metric.SetName(cwMetric.MetricName) + metric.SetUnit(cwMetric.Unit) + + dp := metric.SetEmptySummary().DataPoints().AppendEmpty() + dp.SetCount(uint64(cwMetric.Value.Count)) + dp.SetSum(cwMetric.Value.Sum) + qv := dp.QuantileValues() + qvs := qv.AppendEmpty() + qvs.SetQuantile(99) + qvs.SetValue(cwMetric.Value.Q1) + qvs = qv.AppendEmpty() + qvs.SetQuantile(99.9) + qvs.SetValue(cwMetric.Value.Q2) + qvs = qv.AppendEmpty() + qvs.SetQuantile(0) + qvs.SetValue(cwMetric.Value.Min) + qvs = qv.AppendEmpty() + qvs.SetQuantile(100) + qvs.SetValue(cwMetric.Value.Max) + + // all dimensions are metric attributes + setDataPointAttributes(cwMetric, dp) + + // cloudwatch timestamp is in ms, but the metric + // expects it in ns + dp.SetTimestamp(pcommon.NewTimestampFromTime(time.UnixMilli(cwMetric.Timestamp))) +} + +func UnmarshalMetrics(record []byte) (pmetric.Metrics, error) { decoder := json.NewDecoder(bytes.NewReader(record)) metrics := pmetric.NewMetrics() for datumIndex := 0; ; datumIndex++ { @@ -101,43 +146,18 @@ func UnmarshalMetrics(record []byte, logger *zap.Logger) (pmetric.Metrics, error if errors.Is(err, io.EOF) { break } - logger.Error( - "Unable to unmarshal input", - zap.Int("datum_index", datumIndex), - zap.Error(err), - ) - continue + return pmetric.Metrics{}, + fmt.Errorf("unable to unmarshal datum [%d] into cloudwatch metric: %w", datumIndex, err) } - if !isMetricValid(cwMetric) { - logger.Error( - "Invalid cloudwatch cwMetric", - zap.Int("datum_index", datumIndex), - ) - continue + if valid, err := isMetricValid(cwMetric); !valid { + return pmetric.Metrics{}, + fmt.Errorf("cloudwatch metric from datum [%d] is invalid: %w", datumIndex, err) } - - rm := metrics.ResourceMetrics().AppendEmpty() - setResourceAttributes(cwMetric, rm.Resource()) - - metric := rm.ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() - metric.SetName(cwMetric.MetricName) - metric.SetUnit(cwMetric.Unit) - - dp := metric.SetEmptySummary().DataPoints().AppendEmpty() - dp.SetTimestamp(pcommon.NewTimestampFromTime(time.UnixMilli(cwMetric.Timestamp))) - setDataPointAttributes(cwMetric, dp) - dp.SetCount(uint64(cwMetric.Value.Count)) - dp.SetSum(cwMetric.Value.Sum) - minQ := dp.QuantileValues().AppendEmpty() - minQ.SetQuantile(0) - minQ.SetValue(cwMetric.Value.Min) - maxQ := dp.QuantileValues().AppendEmpty() - maxQ.SetQuantile(1) - maxQ.SetValue(cwMetric.Value.Max) + addMetric(cwMetric, metrics) } if metrics.MetricCount() == 0 { - return metrics, errInvalidMetricRecord + return metrics, errors.New("no resource metrics could be obtained from the record") } metrics = expmetrics.Merge(pmetric.NewMetrics(), metrics) diff --git a/pkg/translator/cloudwatch/cwmetrics_to_metrics_test.go b/pkg/translator/cloudwatch/cwmetrics_to_metrics_test.go index ee5f9ee3c82d..129099eb7a77 100644 --- a/pkg/translator/cloudwatch/cwmetrics_to_metrics_test.go +++ b/pkg/translator/cloudwatch/cwmetrics_to_metrics_test.go @@ -1,8 +1,13 @@ package cloudwatch import ( + "bytes" + "encoding/json" + "errors" + "fmt" "github.com/stretchr/testify/require" - "go.uber.org/zap" + "github.com/tidwall/gjson" + "go.opentelemetry.io/collector/pdata/pmetric" "os" "path/filepath" "testing" @@ -12,46 +17,64 @@ func TestUnmarshalMetrics(t *testing.T) { t.Parallel() testCases := map[string]struct { - filename string - resourceCount int - metricCount int - datapointCount int - err error + filename string + err error }{ - "WithMultipleRecords": { - filename: "multiple_records", - resourceCount: 6, - metricCount: 33, - datapointCount: 127, + "SingleRecord": { + filename: "single_record", }, - "WithSingleRecord": { - filename: "single_record", - resourceCount: 1, - metricCount: 1, - datapointCount: 1, + "MultipleRecords": { + filename: "multiple_records", }, - "WithInvalidRecords": { - filename: "invalid_records", - err: errInvalidMetricRecord, + "MultipleResources": { + filename: "multiple_resources", }, - "WithSomeInvalidRecords": { - filename: "some_invalid_records", - resourceCount: 5, - metricCount: 36, - datapointCount: 88, + "InvalidRecord": { + filename: "invalid_record", + err: fmt.Errorf( + "cloudwatch metric from datum [0] is invalid: %w", + errors.New("cloudwatch metric is missing metric name field"), + ), + }, + "SomeInvalidRecord": { + filename: "some_invalid_record", + err: fmt.Errorf( + "cloudwatch metric from datum [1] is invalid: %w", + errors.New("cloudwatch metric is missing metric name field"), + ), + }, + "EmptyRecord": { + filename: "empty_record", + err: errors.New("no resource metrics could be obtained from the record"), }, } + unmarshaller := &pmetric.JSONUnmarshaler{} for name, testCase := range testCases { t.Run(name, func(t *testing.T) { - record, err := os.ReadFile(filepath.Join("testdata/metric", testCase.filename)) + content, err := os.ReadFile(filepath.Join("testdata/metric", testCase.filename+".json")) require.NoError(t, err) + // since new line represents the end of the record, we + // need to remove all new lines from the json file, so + // the record will not get incorrectly split. We keep + // the new lines between different records. + var buf bytes.Buffer + gjson.ParseBytes(content).ForEach(func(_, value gjson.Result) bool { + err := json.NewEncoder(&buf).Encode(value.Value()) + require.NoError(t, err) + return true + }) + result, err := UnmarshalMetrics(buf.Bytes()) + require.Equal(t, testCase.err, err) + if err != nil { + return + } - metrics, err := UnmarshalMetrics(record, zap.NewNop()) + content, err = os.ReadFile(filepath.Join("testdata/metric", testCase.filename+"_expected.json")) + require.NoError(t, err) + expected, err := unmarshaller.UnmarshalMetrics(content) + require.NoError(t, err) - require.Equal(t, testCase.err, err) - require.Equal(t, testCase.resourceCount, metrics.ResourceMetrics().Len()) - require.Equal(t, testCase.metricCount, metrics.MetricCount()) - require.Equal(t, testCase.datapointCount, metrics.DataPointCount()) + require.Equal(t, expected, result) }) } } diff --git a/pkg/translator/cloudwatch/go.mod b/pkg/translator/cloudwatch/go.mod index 819e57722b55..2a519d9c7ca2 100644 --- a/pkg/translator/cloudwatch/go.mod +++ b/pkg/translator/cloudwatch/go.mod @@ -6,6 +6,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.117.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0 github.com/stretchr/testify v1.10.0 + github.com/tidwall/gjson v1.18.0 go.opentelemetry.io/collector/pdata v1.23.0 go.opentelemetry.io/collector/semconv v0.117.0 go.uber.org/goleak v1.3.0 @@ -21,6 +22,8 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.117.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect diff --git a/pkg/translator/cloudwatch/go.sum b/pkg/translator/cloudwatch/go.sum index 2688b869e30d..9504d4150cb1 100644 --- a/pkg/translator/cloudwatch/go.sum +++ b/pkg/translator/cloudwatch/go.sum @@ -43,6 +43,12 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/collector/pdata v1.23.0 h1:tEk0dkfB8RdSukoOMfEa8duB938gfZowdfRkrJxGDrw= diff --git a/pkg/translator/cloudwatch/testdata/log/empty_record.json b/pkg/translator/cloudwatch/testdata/log/empty_record.json new file mode 100644 index 000000000000..0637a088a01e --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/log/empty_record.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/log/invalid_record.json b/pkg/translator/cloudwatch/testdata/log/invalid_record.json new file mode 100644 index 000000000000..dfad993fcbd2 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/log/invalid_record.json @@ -0,0 +1,5 @@ +[ + { + "invalid": "record" + } +] \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/log/invalid_records b/pkg/translator/cloudwatch/testdata/log/invalid_records deleted file mode 100644 index d2672dbd26d3..000000000000 --- a/pkg/translator/cloudwatch/testdata/log/invalid_records +++ /dev/null @@ -1,4 +0,0 @@ -{"CHANGE":-0.09,"PRICE":4.96,"TICKER_SYMBOL":"KIN","SECTOR":"ENERGY"} -{"CHANGE":-1.47,"PRICE":134.74,"TICKER_SYMBOL":"DFG","SECTOR":"TECHNOLOGY"} -{"CHANGE":1.96,"PRICE":57.53,"TICKER_SYMBOL":"SAC","SECTOR":"ENERGY"} -{"CHANGE":0.04,"PRICE":32.84,"TICKER_SYMBOL":"PJN","SECTOR":"RETAIL"} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/log/multiple_records b/pkg/translator/cloudwatch/testdata/log/multiple_records deleted file mode 100644 index 18a1888d99a8..000000000000 --- a/pkg/translator/cloudwatch/testdata/log/multiple_records +++ /dev/null @@ -1,2 +0,0 @@ -{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695232","timestamp":1725544035523,"message":"Hello world, here is our first log message!"}]} -{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695233","timestamp":1725554035523,"message":"Hello world, here is our second log message!"}]} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/log/multiple_records.json b/pkg/translator/cloudwatch/testdata/log/multiple_records.json new file mode 100644 index 000000000000..8895d708628a --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/log/multiple_records.json @@ -0,0 +1,10 @@ +[ + { + "timestamp": 1725544035523, + "message": "Hello world!" + }, + { + "timestamp": 1725544035523, + "message": "Hello world, 2!" + } +] \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/log/multiple_records_expected.json b/pkg/translator/cloudwatch/testdata/log/multiple_records_expected.json new file mode 100644 index 000000000000..a5c1bd0b2717 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/log/multiple_records_expected.json @@ -0,0 +1,34 @@ +{ + "resourceLogs":[ + { + "resource":{ + + }, + "scopeLogs":[ + { + "scope":{ + + }, + "logRecords":[ + { + "timeUnixNano":"1725544035523000000", + "body":{ + "stringValue":"Hello world!" + }, + "traceId":"", + "spanId":"" + }, + { + "timeUnixNano":"1725544035523000000", + "body":{ + "stringValue":"Hello world, 2!" + }, + "traceId":"", + "spanId":"" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/log/multiple_resources b/pkg/translator/cloudwatch/testdata/log/multiple_resources deleted file mode 100644 index 0115614f1b9b..000000000000 --- a/pkg/translator/cloudwatch/testdata/log/multiple_resources +++ /dev/null @@ -1,6 +0,0 @@ -{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695232","timestamp":1725544035523,"message":"Hello world, here is our first log message!"}]} -{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695233","timestamp":1725554035523,"message":"Hello world, here is our second log message!"}]} -{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test2","logStream":"test1","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695234","timestamp":1725564035523,"message":"Hello world, here is our third log message!"}]} -{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test2","logStream":"test2","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695235","timestamp":1725574035523,"message":"Hello world, here is our fourth log message!"}]} -{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test2","logStream":"test1","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695236","timestamp":1725584035523,"message":"Hello world, here is our fifth log message!"}]} -{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test2","logStream":"test2","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695237","timestamp":1725594035523,"message":"Hello world, here is our sixth log message!"}]} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/log/single_record b/pkg/translator/cloudwatch/testdata/log/single_record deleted file mode 100644 index b35e166e47da..000000000000 --- a/pkg/translator/cloudwatch/testdata/log/single_record +++ /dev/null @@ -1 +0,0 @@ -{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695232","timestamp":1725544035523,"message":"Hello world, here is our first log message!"}]} diff --git a/pkg/translator/cloudwatch/testdata/log/single_record.json b/pkg/translator/cloudwatch/testdata/log/single_record.json new file mode 100644 index 000000000000..70ff88be0ff0 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/log/single_record.json @@ -0,0 +1,6 @@ +[ + { + "timestamp": 1725544035523, + "message": "Hello world!" + } +] \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/log/single_record_expected.json b/pkg/translator/cloudwatch/testdata/log/single_record_expected.json new file mode 100644 index 000000000000..ea32f4b56e71 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/log/single_record_expected.json @@ -0,0 +1,26 @@ +{ + "resourceLogs":[ + { + "resource":{ + + }, + "scopeLogs":[ + { + "scope":{ + + }, + "logRecords":[ + { + "timeUnixNano":"1725544035523000000", + "body":{ + "stringValue":"Hello world!" + }, + "traceId":"", + "spanId":"" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/log/some_invalid_record.json b/pkg/translator/cloudwatch/testdata/log/some_invalid_record.json new file mode 100644 index 000000000000..b1a16b3491c6 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/log/some_invalid_record.json @@ -0,0 +1,9 @@ +[ + { + "timestamp": 1725544035523, + "message": "Hello world!" + }, + { + "invalid": "record" + } +] \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/log/some_invalid_records b/pkg/translator/cloudwatch/testdata/log/some_invalid_records deleted file mode 100644 index 4026ad877a43..000000000000 --- a/pkg/translator/cloudwatch/testdata/log/some_invalid_records +++ /dev/null @@ -1,3 +0,0 @@ -{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695232","timestamp":1725544035523,"message":"Hello world, here is our first log message!"}]} -{"CHANGE":1.96,"PRICE":57.53,"TICKER_SYMBOL":"SAC","SECTOR":"ENERGY"} -{"messageType":"DATA_MESSAGE","owner":"123","logGroup":"test","logStream":"test","subscriptionFilters":["test"],"logEvents":[{"id":"38480917865042697267627490045603633139480491071049695233","timestamp":1725554035523,"message":"Hello world, here is our second log message!"}]} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/metric/empty_record.json b/pkg/translator/cloudwatch/testdata/metric/empty_record.json new file mode 100644 index 000000000000..0637a088a01e --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/metric/empty_record.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/metric/invalid_record.json b/pkg/translator/cloudwatch/testdata/metric/invalid_record.json new file mode 100644 index 000000000000..dfad993fcbd2 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/metric/invalid_record.json @@ -0,0 +1,5 @@ +[ + { + "invalid": "record" + } +] \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/metric/invalid_records b/pkg/translator/cloudwatch/testdata/metric/invalid_records deleted file mode 100644 index d2672dbd26d3..000000000000 --- a/pkg/translator/cloudwatch/testdata/metric/invalid_records +++ /dev/null @@ -1,4 +0,0 @@ -{"CHANGE":-0.09,"PRICE":4.96,"TICKER_SYMBOL":"KIN","SECTOR":"ENERGY"} -{"CHANGE":-1.47,"PRICE":134.74,"TICKER_SYMBOL":"DFG","SECTOR":"TECHNOLOGY"} -{"CHANGE":1.96,"PRICE":57.53,"TICKER_SYMBOL":"SAC","SECTOR":"ENERGY"} -{"CHANGE":0.04,"PRICE":32.84,"TICKER_SYMBOL":"PJN","SECTOR":"RETAIL"} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/metric/multiple_records b/pkg/translator/cloudwatch/testdata/metric/multiple_records deleted file mode 100644 index 20fa9b367fe7..000000000000 --- a/pkg/translator/cloudwatch/testdata/metric/multiple_records +++ /dev/null @@ -1,127 +0,0 @@ -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ConnectionEstablishedCount","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ErrorPortAllocation","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":477184.0,"count":92.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916720000,"value":{"max":40.0,"min":40.0,"sum":40.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":59.98,"min":59.98,"sum":59.98,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"MetricUpdate","dimensions":{},"timestamp":1643916720000,"value":{"max":47.0,"min":41.0,"sum":88.0,"count":2.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916720000,"value":{"max":96768.0,"min":96768.0,"sum":96768.0,"count":1.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"MetricUpdate","dimensions":{"MetricStreamName":"MyMetricStream"},"timestamp":1643916720000,"value":{"max":47.0,"min":41.0,"sum":88.0,"count":2.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUSurplusCreditsCharged","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":0.00133333333333333,"min":0.00133333333333333,"sum":0.00133333333333333,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916720000,"value":{"max":0.006538,"min":0.006538,"sum":0.006538,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916720000,"value":{"max":0.028416,"min":0.028416,"sum":0.028416,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.08,"count":92.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916720000,"value":{"max":223744.0,"min":223744.0,"sum":223744.0,"count":1.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUCreditUsage","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.018548,"min":0.018548,"sum":0.018548,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ActiveConnectionCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"OnDemand","Service":"Fargate","Type":"Resource"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ConnectionAttemptCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ErrorPortAllocation","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916660000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":60.145003,"min":60.145003,"sum":60.145003,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"PublishErrorRate","dimensions":{"MetricStreamName":"MyMetricStream"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"BurstBalance","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":100.0,"min":100.0,"sum":100.0,"count":1.0},"unit":"Percent"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":34.0,"min":34.0,"sum":34.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"Spot","Service":"Fargate","Type":"Resource"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Logs","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeMetricFilters","Service":"Logs","Type":"API"},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ErrorPortAllocation","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Logs","metric_name":"CallCount","dimensions":{},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeInstanceStatus","Service":"EC2","Type":"API"},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916720000,"value":{"max":92.0,"min":92.0,"sum":92.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeMetricFilters","Service":"Logs","Type":"API"},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":60.07399,"min":60.07399,"sum":60.07399,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ConnectionEstablishedCount","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"PublishErrorRate","dimensions":{},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ConnectionAttemptCount","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"ActiveConnectionCount","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"ListGroups","Service":"Resource Groups","Type":"API"},"timestamp":1643916780000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":7.0,"min":7.0,"sum":7.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"MetricUpdate","dimensions":{},"timestamp":1643916780000,"value":{"max":48.0,"min":36.0,"sum":84.0,"count":2.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":178688.0,"min":178688.0,"sum":178688.0,"count":1.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/CloudWatch/MetricStreams","metric_name":"MetricUpdate","dimensions":{"MetricStreamName":"MyMetricStream"},"timestamp":1643916780000,"value":{"max":48.0,"min":36.0,"sum":84.0,"count":2.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916780000,"value":{"max":0.018855,"min":0.018855,"sum":0.018855,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":0.004314,"min":0.004314,"sum":0.004314,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-096efe715dd283535"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916780000,"value":{"max":40448.0,"min":40448.0,"sum":40448.0,"count":1.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-0e3d6e73c17f81566"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"OnDemand","Service":"Fargate","Type":"Resource"},"timestamp":1643916780000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916720000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} diff --git a/pkg/translator/cloudwatch/testdata/metric/multiple_records.json b/pkg/translator/cloudwatch/testdata/metric/multiple_records.json new file mode 100644 index 000000000000..1127193c5ef1 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/metric/multiple_records.json @@ -0,0 +1,44 @@ +[ + { + "metric_stream_name":"MyMetricStream", + "account_id":"1234567890", + "region":"us-east-1", + "namespace":"AWS/EC2", + "metric_name":"DiskWriteOps", + "dimensions":{ + "InstanceId":"i-123456789012" + }, + "timestamp":1611929698000, + "value":{ + "count":3.0, + "sum":20.0, + "max":18.0, + "min":0.0, + "p99":17.56, + "p99.9":17.8764, + "TM(25%:75%)":16.43 + }, + "unit":"Seconds" + }, + { + "metric_stream_name":"MyMetricStream", + "account_id":"1234567890", + "region":"us-east-1", + "namespace":"AWS/EC2", + "metric_name":"DiskWriteOps", + "dimensions":{ + "InstanceId":"i-123456789012" + }, + "timestamp":1611929698100, + "value":{ + "count":5.0, + "sum":21.0, + "max":18.0, + "min":0.0, + "p99":17.56, + "p99.9":17.8764, + "TM(25%:75%)":16.43 + }, + "unit":"Seconds" + } +] \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/metric/multiple_records_expected.json b/pkg/translator/cloudwatch/testdata/metric/multiple_records_expected.json new file mode 100644 index 000000000000..6dbc5c2422ee --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/metric/multiple_records_expected.json @@ -0,0 +1,123 @@ +{ + "resourceMetrics":[ + { + "resource":{ + "attributes":[ + { + "key":"cloud.provider", + "value":{ + "stringValue":"aws" + } + }, + { + "key":"cloud.account.id", + "value":{ + "stringValue":"1234567890" + } + }, + { + "key":"cloud.region", + "value":{ + "stringValue":"us-east-1" + } + }, + { + "key":"service.namespace", + "value":{ + "stringValue":"AWS" + } + }, + { + "key":"service.name", + "value":{ + "stringValue":"EC2" + } + }, + { + "key":"aws.cloudwatch.metric_stream_name", + "value":{ + "stringValue":"MyMetricStream" + } + } + ] + }, + "scopeMetrics":[ + { + "scope":{ + + }, + "metrics":[ + { + "name":"DiskWriteOps", + "unit":"Seconds", + "summary":{ + "dataPoints":[ + { + "attributes":[ + { + "key":"service.instance.id", + "value":{ + "stringValue":"i-123456789012" + } + } + ], + "timeUnixNano":"1611929698000000000", + "count":"3", + "sum":20, + "quantileValues":[ + { + "quantile":99, + "value":17.56 + }, + { + "quantile":99.9, + "value":17.8764 + }, + { + + }, + { + "quantile":100, + "value":18 + } + ] + }, + { + "attributes":[ + { + "key":"service.instance.id", + "value":{ + "stringValue":"i-123456789012" + } + } + ], + "timeUnixNano":"1611929698100000000", + "count":"5", + "sum":21, + "quantileValues":[ + { + "quantile":99, + "value":17.56 + }, + { + "quantile":99.9, + "value":17.8764 + }, + { + + }, + { + "quantile":100, + "value":18 + } + ] + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/metric/multiple_resources.json b/pkg/translator/cloudwatch/testdata/metric/multiple_resources.json new file mode 100644 index 000000000000..bd75a1906dc2 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/metric/multiple_resources.json @@ -0,0 +1,44 @@ +[ + { + "metric_stream_name": "MyMetricStream-1", + "account_id": "1234567890", + "region": "us-east-1", + "namespace": "AWS/EC2", + "metric_name": "DiskWriteOps", + "dimensions": { + "InstanceId": "i-123456789012" + }, + "timestamp": 1611929698000, + "value": { + "count": 3.0, + "sum": 20.0, + "max": 18.0, + "min": 0.0, + "p99": 17.56, + "p99.9": 17.8764, + "TM(25%:75%)": 16.43 + }, + "unit": "Seconds" + }, + { + "metric_stream_name": "MyMetricStream-2", + "account_id": "1234567890", + "region": "us-east-1", + "namespace": "AWS/EC2", + "metric_name": "DiskWriteOps", + "dimensions": { + "InstanceId": "i-123456789012" + }, + "timestamp": 1611929698000, + "value": { + "count": 3.0, + "sum": 20.0, + "max": 18.0, + "min": 0.0, + "p99": 17.56, + "p99.9": 17.8764, + "TM(25%:75%)": 16.43 + }, + "unit": "Seconds" + } +] \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/metric/multiple_resources_expected.json b/pkg/translator/cloudwatch/testdata/metric/multiple_resources_expected.json new file mode 100644 index 000000000000..e67e2c2f6d51 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/metric/multiple_resources_expected.json @@ -0,0 +1,182 @@ +{ + "resourceMetrics":[ + { + "resource":{ + "attributes":[ + { + "key":"cloud.provider", + "value":{ + "stringValue":"aws" + } + }, + { + "key":"cloud.account.id", + "value":{ + "stringValue":"1234567890" + } + }, + { + "key":"cloud.region", + "value":{ + "stringValue":"us-east-1" + } + }, + { + "key":"service.namespace", + "value":{ + "stringValue":"AWS" + } + }, + { + "key":"service.name", + "value":{ + "stringValue":"EC2" + } + }, + { + "key":"aws.cloudwatch.metric_stream_name", + "value":{ + "stringValue":"MyMetricStream-1" + } + } + ] + }, + "scopeMetrics":[ + { + "scope":{ + + }, + "metrics":[ + { + "name":"DiskWriteOps", + "unit":"Seconds", + "summary":{ + "dataPoints":[ + { + "attributes":[ + { + "key":"service.instance.id", + "value":{ + "stringValue":"i-123456789012" + } + } + ], + "timeUnixNano":"1611929698000000000", + "count":"3", + "sum":20, + "quantileValues":[ + { + "quantile":99, + "value":17.56 + }, + { + "quantile":99.9, + "value":17.8764 + }, + { + + }, + { + "quantile":100, + "value":18 + } + ] + } + ] + } + } + ] + } + ] + }, + { + "resource":{ + "attributes":[ + { + "key":"cloud.provider", + "value":{ + "stringValue":"aws" + } + }, + { + "key":"cloud.account.id", + "value":{ + "stringValue":"1234567890" + } + }, + { + "key":"cloud.region", + "value":{ + "stringValue":"us-east-1" + } + }, + { + "key":"service.namespace", + "value":{ + "stringValue":"AWS" + } + }, + { + "key":"service.name", + "value":{ + "stringValue":"EC2" + } + }, + { + "key":"aws.cloudwatch.metric_stream_name", + "value":{ + "stringValue":"MyMetricStream-2" + } + } + ] + }, + "scopeMetrics":[ + { + "scope":{ + + }, + "metrics":[ + { + "name":"DiskWriteOps", + "unit":"Seconds", + "summary":{ + "dataPoints":[ + { + "attributes":[ + { + "key":"service.instance.id", + "value":{ + "stringValue":"i-123456789012" + } + } + ], + "timeUnixNano":"1611929698000000000", + "count":"3", + "sum":20, + "quantileValues":[ + { + "quantile":99, + "value":17.56 + }, + { + "quantile":99.9, + "value":17.8764 + }, + { + + }, + { + "quantile":100, + "value":18 + } + ] + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/metric/single_record b/pkg/translator/cloudwatch/testdata/metric/single_record deleted file mode 100644 index 6d2fd8e1b658..000000000000 --- a/pkg/translator/cloudwatch/testdata/metric/single_record +++ /dev/null @@ -1 +0,0 @@ -{"metric_stream_name": "MyMetricStream","account_id": "1234567890","region": "us-east-1","namespace": "AWS/EC2","metric_name": "DiskWriteOps","dimensions": {"InstanceId": "i-123456789012"},"timestamp": 1611929698000,"value": {"count": 3.0,"sum": 20.0,"max": 18.0,"min": 0.0},"unit": "Seconds"} diff --git a/pkg/translator/cloudwatch/testdata/metric/single_record.json b/pkg/translator/cloudwatch/testdata/metric/single_record.json new file mode 100644 index 000000000000..3e40df35a073 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/metric/single_record.json @@ -0,0 +1,23 @@ +[ + { + "metric_stream_name":"MyMetricStream", + "account_id":"1234567890", + "region":"us-east-1", + "namespace":"AWS/EC2", + "metric_name":"DiskWriteOps", + "dimensions":{ + "InstanceId":"i-123456789012" + }, + "timestamp":1611929698000, + "value":{ + "count":3.0, + "sum":20.0, + "max":18.0, + "min":0.0, + "p99":17.56, + "p99.9":17.8764, + "TM(25%:75%)":16.43 + }, + "unit":"Seconds" + } +] \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/metric/single_record_expected.json b/pkg/translator/cloudwatch/testdata/metric/single_record_expected.json new file mode 100644 index 000000000000..3e9fe178648a --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/metric/single_record_expected.json @@ -0,0 +1,93 @@ +{ + "resourceMetrics":[ + { + "resource":{ + "attributes":[ + { + "key":"cloud.provider", + "value":{ + "stringValue":"aws" + } + }, + { + "key":"cloud.account.id", + "value":{ + "stringValue":"1234567890" + } + }, + { + "key":"cloud.region", + "value":{ + "stringValue":"us-east-1" + } + }, + { + "key":"service.namespace", + "value":{ + "stringValue":"AWS" + } + }, + { + "key":"service.name", + "value":{ + "stringValue":"EC2" + } + }, + { + "key":"aws.cloudwatch.metric_stream_name", + "value":{ + "stringValue":"MyMetricStream" + } + } + ] + }, + "scopeMetrics":[ + { + "scope":{ + + }, + "metrics":[ + { + "name":"DiskWriteOps", + "unit":"Seconds", + "summary":{ + "dataPoints":[ + { + "attributes":[ + { + "key":"service.instance.id", + "value":{ + "stringValue":"i-123456789012" + } + } + ], + "timeUnixNano":"1611929698000000000", + "count":"3", + "sum":20, + "quantileValues":[ + { + "quantile":99, + "value":17.56 + }, + { + "quantile":99.9, + "value":17.8764 + }, + { + + }, + { + "quantile":100, + "value":18 + } + ] + } + ] + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/metric/some_invalid_record.json b/pkg/translator/cloudwatch/testdata/metric/some_invalid_record.json new file mode 100644 index 000000000000..636000e58ca1 --- /dev/null +++ b/pkg/translator/cloudwatch/testdata/metric/some_invalid_record.json @@ -0,0 +1,26 @@ +[ + { + "metric_stream_name":"MyMetricStream", + "account_id":"1234567890", + "region":"us-east-1", + "namespace":"AWS/EC2", + "metric_name":"DiskWriteOps", + "dimensions":{ + "InstanceId":"i-123456789012" + }, + "timestamp":1611929698000, + "value":{ + "count":3.0, + "sum":20.0, + "max":18.0, + "min":0.0, + "p99":17.56, + "p99.9":17.8764, + "TM(25%:75%)":16.43 + }, + "unit":"Seconds" + }, + { + "invalid": "record" + } +] \ No newline at end of file diff --git a/pkg/translator/cloudwatch/testdata/metric/some_invalid_records b/pkg/translator/cloudwatch/testdata/metric/some_invalid_records deleted file mode 100644 index d1598692b12c..000000000000 --- a/pkg/translator/cloudwatch/testdata/metric/some_invalid_records +++ /dev/null @@ -1,92 +0,0 @@ -{"CHANGE":-0.09,"PRICE":4.96,"TICKER_SYMBOL":"KIN","SECTOR":"ENERGY"} -{"CHANGE":-1.47,"PRICE":134.74,"TICKER_SYMBOL":"DFG","SECTOR":"TECHNOLOGY"} -{"CHANGE":1.96,"PRICE":57.53,"TICKER_SYMBOL":"SAC","SECTOR":"ENERGY"} -{"CHANGE":0.04,"PRICE":32.84,"TICKER_SYMBOL":"PJN","SECTOR":"RETAIL"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":60.058357,"min":60.058357,"sum":60.058357,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadOps","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsOut","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":189.0,"min":187.0,"sum":376.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUUtilization","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":2.225,"min":2.208333333333333,"sum":4.433333333333334,"count":2.0},"unit":"Percent"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSByteBalance%","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":99.0,"min":99.0,"sum":198.0,"count":2.0},"unit":"Percent"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadBytes","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUUtilization","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":2.225,"min":2.1,"sum":8.675,"count":4.0},"unit":"Percent"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteOps","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":37.0,"min":12.0,"sum":49.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"BurstBalance","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":100.0,"min":100.0,"sum":100.0,"count":1.0},"unit":"Percent"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":7.0,"min":7.0,"sum":7.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteBytes","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":246272.0,"min":49664.0,"sum":552448.0,"count":4.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteBytes","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":246272.0,"min":49664.0,"sum":295936.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"Spot","Service":"Fargate","Type":"Resource"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsOut","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":189.0,"min":167.0,"sum":729.0,"count":4.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkOut","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":33477.0,"min":28026.0,"sum":127491.0,"count":4.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadOps","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":4.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Logs","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeMetricFilters","Service":"Logs","Type":"API"},"timestamp":1643916600000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Logs","metric_name":"CallCount","dimensions":{},"timestamp":1643916600000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadOps","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkIn","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":96546.0,"min":75023.0,"sum":330899.0,"count":4.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":172.0,"min":172.0,"sum":172.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"CallCount","dimensions":{"Class":"None","Resource":"DescribeMetricFilters","Service":"Logs","Type":"API"},"timestamp":1643916600000,"value":{"max":1.0,"min":1.0,"sum":1.0,"count":1.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":4.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkOut","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":33477.0,"min":33411.0,"sum":66888.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":60.107249,"min":60.107249,"sum":60.107249,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSByteBalance%","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":100.0,"min":100.0,"sum":200.0,"count":2.0},"unit":"Percent"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSByteBalance%","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":100.0,"min":99.0,"sum":398.0,"count":4.0},"unit":"Percent"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"Standard/OnDemand","Resource":"vCPU","Service":"EC2","Type":"Resource"},"timestamp":1643916600000,"value":{"max":6.0,"min":6.0,"sum":6.0,"count":1.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadBytes","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteOps","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":50.0,"min":7.0,"sum":106.0,"count":4.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsIn","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":210.0,"min":206.0,"sum":416.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsIn","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":205.0,"min":182.0,"sum":387.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_System","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsOut","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":186.0,"min":167.0,"sum":353.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesOutToDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":10.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteOps","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":50.0,"min":7.0,"sum":57.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":892416.0,"count":172.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteOps","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":12.0,"min":12.0,"sum":12.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeIdleTime","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":59.98,"min":59.98,"sum":59.98,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalReadTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkIn","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":75359.0,"min":75023.0,"sum":150382.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":49664.0,"min":49664.0,"sum":49664.0,"count":1.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSReadBytes","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":5.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":5.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadOps","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkIn","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":96546.0,"min":72132.0,"sum":252649.0,"count":3.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"CPUUtilization","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":2.1416666666666666,"min":2.1,"sum":4.241666666666667,"count":2.0},"unit":"Percent"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeReadBytes","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsOutToSource","dimensions":{"NatGatewayId":"nat-01a4160dfb995b990"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"BytesInFromDestination","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSIOBalance%","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":100.0,"min":99.0,"sum":497.0,"count":5.0},"unit":"Percent"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkPacketsIn","dimensions":{"AutoScalingGroupName":"eks-ng-1cf4ef4b-7abedb2b-97f8-1909-ff97-1dd8bc506a16"},"timestamp":1643916600000,"value":{"max":210.0,"min":182.0,"sum":991.0,"count":5.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"NetworkOut","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":32577.0,"min":28026.0,"sum":60603.0,"count":2.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSWriteBytes","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":223744.0,"min":78336.0,"sum":480256.0,"count":3.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"StatusCheckFailed_Instance","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeQueueLength","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":0.00133333333333333,"min":0.00133333333333333,"sum":0.00133333333333333,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0c3eb53f736eb91fe"},"timestamp":1643916600000,"value":{"max":0.004637,"min":0.004637,"sum":0.004637,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"MetadataNoToken","dimensions":{"InstanceId":"i-016363f300c5b1dfa"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":0.00977,"min":0.00977,"sum":0.00977,"count":1.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeTotalWriteTime","dimensions":{"VolumeId":"vol-0679c44f13fe46f14"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.08,"count":172.0},"unit":"Seconds"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSIOBalance%","dimensions":{"InstanceId":"i-012e19a954ce9e13b"},"timestamp":1643916600000,"value":{"max":99.0,"min":99.0,"sum":297.0,"count":3.0},"unit":"Percent"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EC2","metric_name":"EBSIOBalance%","dimensions":{"InstanceId":"i-02863582c45c2ec85"},"timestamp":1643916600000,"value":{"max":100.0,"min":100.0,"sum":200.0,"count":2.0},"unit":"Percent"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/EBS","metric_name":"VolumeWriteBytes","dimensions":{"VolumeId":"vol-02b3a84b1abff7ff5"},"timestamp":1643916600000,"value":{"max":78336.0,"min":78336.0,"sum":78336.0,"count":1.0},"unit":"Bytes"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/Usage","metric_name":"ResourceCount","dimensions":{"Class":"None","Resource":"OnDemand","Service":"Fargate","Type":"Resource"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":1.0},"unit":"None"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsInFromSource","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} -{"metric_stream_name":"MyMetricStream","account_id":"1234567890","region":"us-east-1","namespace":"AWS/NATGateway","metric_name":"PacketsDropCount","dimensions":{"NatGatewayId":"nat-054778c04c28550b7"},"timestamp":1643916600000,"value":{"max":0.0,"min":0.0,"sum":0.0,"count":2.0},"unit":"Count"} From 00fcc7dd6a1aae3d75356939d3de44d0bf8892c8 Mon Sep 17 00:00:00 2001 From: Constanca Manteigas Date: Tue, 14 Jan 2025 17:22:49 +0100 Subject: [PATCH 3/9] - Replace firehose encoding with cloudwatch encoding --- .../Makefile | 0 .../encoding/cloudwatchencoding/config.go | 31 +++ .../config_test.go | 10 +- .../doc.go | 2 +- .../encoding/cloudwatchencoding/extension.go | 79 ++++++ .../cloudwatchencoding/extension_test.go | 18 ++ .../factory.go | 2 +- .../factory_test.go | 2 +- .../generated_component_test.go | 4 +- .../generated_package_test.go | 2 +- .../go.mod | 0 .../internal/metadata/generated_status.go | 2 +- .../metadata.yaml | 2 +- .../cloudwatchencoding/testdata/config.yaml | 4 + .../firehoseencodingextension/config.go | 53 ---- .../firehoseencodingextension/extension.go | 209 --------------- .../extension_test.go | 238 ------------------ .../testdata/config.yaml | 9 - .../testdata/logs/empty_log.json | 10 - .../testdata/logs/invalid_log.json | 18 -- .../logs/multiple_resources_expected.json | 94 ------- .../testdata/logs/one_log.json | 19 -- .../testdata/logs/one_log_expected.json | 45 ---- .../testdata/logs/some_invalid_log.json | 26 -- .../testdata/logs/two_logs.json | 24 -- .../testdata/logs/two_logs_expected.json | 53 ---- .../testdata/logs/two_resources.json | 34 --- .../testdata/metrics/empty_record.json | 1 - .../testdata/metrics/invalid_metric.json | 5 - .../testdata/metrics/multiple_resources.json | 44 ---- .../metrics/multiple_resources_expected.json | 182 -------------- .../testdata/metrics/one_metric.json | 23 -- .../testdata/metrics/one_metric_expected.json | 93 ------- .../metrics/some_invalid_metrics.json | 26 -- .../testdata/metrics/two_metrics.json | 44 ---- .../metrics/two_metrics_expected.json | 123 --------- ...ogs_to_log_record.go => cwlogs_to_logs.go} | 18 +- ..._record_test.go => cwlogs_to_logs_test.go} | 4 +- 38 files changed, 156 insertions(+), 1397 deletions(-) rename extension/encoding/{firehoseencodingextension => cloudwatchencoding}/Makefile (100%) create mode 100644 extension/encoding/cloudwatchencoding/config.go rename extension/encoding/{firehoseencodingextension => cloudwatchencoding}/config_test.go (77%) rename extension/encoding/{firehoseencodingextension => cloudwatchencoding}/doc.go (52%) create mode 100644 extension/encoding/cloudwatchencoding/extension.go create mode 100644 extension/encoding/cloudwatchencoding/extension_test.go rename extension/encoding/{firehoseencodingextension => cloudwatchencoding}/factory.go (91%) rename extension/encoding/{firehoseencodingextension => cloudwatchencoding}/factory_test.go (91%) rename extension/encoding/{firehoseencodingextension => cloudwatchencoding}/generated_component_test.go (93%) rename extension/encoding/{firehoseencodingextension => cloudwatchencoding}/generated_package_test.go (81%) rename extension/encoding/{firehoseencodingextension => cloudwatchencoding}/go.mod (100%) rename extension/encoding/{firehoseencodingextension => cloudwatchencoding}/internal/metadata/generated_status.go (84%) rename extension/encoding/{firehoseencodingextension => cloudwatchencoding}/metadata.yaml (81%) create mode 100644 extension/encoding/cloudwatchencoding/testdata/config.yaml delete mode 100644 extension/encoding/firehoseencodingextension/config.go delete mode 100644 extension/encoding/firehoseencodingextension/extension.go delete mode 100644 extension/encoding/firehoseencodingextension/extension_test.go delete mode 100644 extension/encoding/firehoseencodingextension/testdata/config.yaml delete mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/empty_log.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/invalid_log.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/multiple_resources_expected.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/one_log.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/one_log_expected.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/some_invalid_log.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/two_logs.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/two_logs_expected.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/logs/two_resources.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/empty_record.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/invalid_metric.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources_expected.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/one_metric.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/one_metric_expected.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/some_invalid_metrics.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics.json delete mode 100644 extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics_expected.json rename pkg/translator/cloudwatch/{cwlogs_to_log_record.go => cwlogs_to_logs.go} (74%) rename pkg/translator/cloudwatch/{cwlogs_to_log_record_test.go => cwlogs_to_logs_test.go} (94%) diff --git a/extension/encoding/firehoseencodingextension/Makefile b/extension/encoding/cloudwatchencoding/Makefile similarity index 100% rename from extension/encoding/firehoseencodingextension/Makefile rename to extension/encoding/cloudwatchencoding/Makefile diff --git a/extension/encoding/cloudwatchencoding/config.go b/extension/encoding/cloudwatchencoding/config.go new file mode 100644 index 000000000000..ac69b6f0bf4d --- /dev/null +++ b/extension/encoding/cloudwatchencoding/config.go @@ -0,0 +1,31 @@ +package cloudwatchencoding + +import ( + "fmt" + "go.opentelemetry.io/collector/component" +) + +type contentEncoding string + +const ( + NoEncoding = "" + GZipEncoded = "gzip" +) + +type Config struct { + Encoding contentEncoding `mapstructure:"content_encoding"` +} + +func createDefaultConfig() component.Config { + return &Config{} +} + +func (c *Config) Validate() error { + switch c.Encoding { + case NoEncoding: + case GZipEncoded: + default: + return fmt.Errorf("unknown content encoding %s", c.Encoding) + } + return nil +} diff --git a/extension/encoding/firehoseencodingextension/config_test.go b/extension/encoding/cloudwatchencoding/config_test.go similarity index 77% rename from extension/encoding/firehoseencodingextension/config_test.go rename to extension/encoding/cloudwatchencoding/config_test.go index e068de93568f..ecc9bf5ba0fd 100644 --- a/extension/encoding/firehoseencodingextension/config_test.go +++ b/extension/encoding/cloudwatchencoding/config_test.go @@ -1,4 +1,4 @@ -package firehoseencodingextension +package cloudwatchencoding import ( "errors" @@ -25,12 +25,8 @@ func TestLoadConfig(t *testing.T) { expected: defaultCfg, }, { - id: component.NewIDWithName(metadata.Type, "metrics_invalid_encoding"), - expectedErr: errors.New("unknown metrics encoding invalid"), - }, - { - id: component.NewIDWithName(metadata.Type, "logs_invalid_encoding"), - expectedErr: errors.New("unknown logs encoding invalid"), + id: component.NewIDWithName(metadata.Type, "invalid_encoding"), + expectedErr: errors.New("unknown content encoding invalid"), }, } diff --git a/extension/encoding/firehoseencodingextension/doc.go b/extension/encoding/cloudwatchencoding/doc.go similarity index 52% rename from extension/encoding/firehoseencodingextension/doc.go rename to extension/encoding/cloudwatchencoding/doc.go index c981b29efef9..4bb6fdeb0be5 100644 --- a/extension/encoding/firehoseencodingextension/doc.go +++ b/extension/encoding/cloudwatchencoding/doc.go @@ -1,2 +1,2 @@ //go:generate mdatagen metadata.yaml -package firehoseencodingextension +package cloudwatchencoding diff --git a/extension/encoding/cloudwatchencoding/extension.go b/extension/encoding/cloudwatchencoding/extension.go new file mode 100644 index 000000000000..09110e7c0624 --- /dev/null +++ b/extension/encoding/cloudwatchencoding/extension.go @@ -0,0 +1,79 @@ +package cloudwatchencoding + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/extension" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.uber.org/zap" + "io" +) + +var ( + _ encoding.LogsUnmarshalerExtension = (*cloudwatchExtension)(nil) + _ encoding.MetricsUnmarshalerExtension = (*cloudwatchExtension)(nil) +) + +type cloudwatchExtension struct { + config *Config + logger *zap.Logger +} + +func createExtension(_ context.Context, settings extension.Settings, config component.Config) (extension.Extension, error) { + return &cloudwatchExtension{ + config: config.(*Config), + logger: settings.Logger, + }, nil +} + +func (c *cloudwatchExtension) Start(_ context.Context, _ component.Host) error { + return nil +} + +func (c *cloudwatchExtension) Shutdown(_ context.Context) error { + return nil +} + +func decompress(buf []byte, encoding contentEncoding) ([]byte, error) { + switch encoding { + case NoEncoding: + return buf, nil + case GZipEncoded: + reader, err := gzip.NewReader(bytes.NewReader(buf)) + if err != nil { + return nil, fmt.Errorf("failed to create gzip reader: %w", err) + } + defer reader.Close() + + decompressed, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("failed to read from gzip reader: %w", err) + } + return decompressed, nil + default: + // not possible, prevented by config.Validate + return nil, nil + } +} + +func (c *cloudwatchExtension) UnmarshalLogs(buf []byte) (plog.Logs, error) { + data, err := decompress(buf, c.config.Encoding) + if err != nil { + return plog.Logs{}, err + } + return cloudwatch.UnmarshalLogs(data) +} + +func (c *cloudwatchExtension) UnmarshalMetrics(buf []byte) (pmetric.Metrics, error) { + data, err := decompress(buf, c.config.Encoding) + if err != nil { + return pmetric.Metrics{}, err + } + return cloudwatch.UnmarshalMetrics(data) +} diff --git a/extension/encoding/cloudwatchencoding/extension_test.go b/extension/encoding/cloudwatchencoding/extension_test.go new file mode 100644 index 000000000000..7ba659e10f32 --- /dev/null +++ b/extension/encoding/cloudwatchencoding/extension_test.go @@ -0,0 +1,18 @@ +package cloudwatchencoding + +import ( + "context" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" + "testing" +) + +func TestExtension_Start_Shutdown(t *testing.T) { + extension := &cloudwatchExtension{} + + err := extension.Start(context.Background(), componenttest.NewNopHost()) + require.NoError(t, err) + + err = extension.Shutdown(context.Background()) + require.NoError(t, err) +} diff --git a/extension/encoding/firehoseencodingextension/factory.go b/extension/encoding/cloudwatchencoding/factory.go similarity index 91% rename from extension/encoding/firehoseencodingextension/factory.go rename to extension/encoding/cloudwatchencoding/factory.go index c3fd23fed14e..6b71df564412 100644 --- a/extension/encoding/firehoseencodingextension/factory.go +++ b/extension/encoding/cloudwatchencoding/factory.go @@ -1,4 +1,4 @@ -package firehoseencodingextension +package cloudwatchencoding import ( "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension/internal/metadata" diff --git a/extension/encoding/firehoseencodingextension/factory_test.go b/extension/encoding/cloudwatchencoding/factory_test.go similarity index 91% rename from extension/encoding/firehoseencodingextension/factory_test.go rename to extension/encoding/cloudwatchencoding/factory_test.go index 2bb6ea1790d9..06c73ea3d965 100644 --- a/extension/encoding/firehoseencodingextension/factory_test.go +++ b/extension/encoding/cloudwatchencoding/factory_test.go @@ -1,4 +1,4 @@ -package firehoseencodingextension +package cloudwatchencoding import ( "context" diff --git a/extension/encoding/firehoseencodingextension/generated_component_test.go b/extension/encoding/cloudwatchencoding/generated_component_test.go similarity index 93% rename from extension/encoding/firehoseencodingextension/generated_component_test.go rename to extension/encoding/cloudwatchencoding/generated_component_test.go index 5d4451c68c5d..7433cbe3ea71 100644 --- a/extension/encoding/firehoseencodingextension/generated_component_test.go +++ b/extension/encoding/cloudwatchencoding/generated_component_test.go @@ -1,6 +1,6 @@ // Code generated by mdatagen. DO NOT EDIT. -package firehoseencodingextension +package cloudwatchencoding import ( "context" @@ -13,7 +13,7 @@ import ( ) func TestComponentFactoryType(t *testing.T) { - require.Equal(t, "firehose_encoding", NewFactory().Type().String()) + require.Equal(t, "cloudwatch_encoding", NewFactory().Type().String()) } func TestComponentConfigStruct(t *testing.T) { diff --git a/extension/encoding/firehoseencodingextension/generated_package_test.go b/extension/encoding/cloudwatchencoding/generated_package_test.go similarity index 81% rename from extension/encoding/firehoseencodingextension/generated_package_test.go rename to extension/encoding/cloudwatchencoding/generated_package_test.go index 575f3c8e6b14..c7c082c67a6a 100644 --- a/extension/encoding/firehoseencodingextension/generated_package_test.go +++ b/extension/encoding/cloudwatchencoding/generated_package_test.go @@ -1,6 +1,6 @@ // Code generated by mdatagen. DO NOT EDIT. -package firehoseencodingextension +package cloudwatchencoding import ( "go.uber.org/goleak" diff --git a/extension/encoding/firehoseencodingextension/go.mod b/extension/encoding/cloudwatchencoding/go.mod similarity index 100% rename from extension/encoding/firehoseencodingextension/go.mod rename to extension/encoding/cloudwatchencoding/go.mod diff --git a/extension/encoding/firehoseencodingextension/internal/metadata/generated_status.go b/extension/encoding/cloudwatchencoding/internal/metadata/generated_status.go similarity index 84% rename from extension/encoding/firehoseencodingextension/internal/metadata/generated_status.go rename to extension/encoding/cloudwatchencoding/internal/metadata/generated_status.go index dc1b03ab839f..d64ea26b7694 100644 --- a/extension/encoding/firehoseencodingextension/internal/metadata/generated_status.go +++ b/extension/encoding/cloudwatchencoding/internal/metadata/generated_status.go @@ -7,7 +7,7 @@ import ( ) var ( - Type = component.MustNewType("firehose_encoding") + Type = component.MustNewType("cloudwatch_encoding") ScopeName = "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" ) diff --git a/extension/encoding/firehoseencodingextension/metadata.yaml b/extension/encoding/cloudwatchencoding/metadata.yaml similarity index 81% rename from extension/encoding/firehoseencodingextension/metadata.yaml rename to extension/encoding/cloudwatchencoding/metadata.yaml index c6c4cc5dae73..1b902fc6ce83 100644 --- a/extension/encoding/firehoseencodingextension/metadata.yaml +++ b/extension/encoding/cloudwatchencoding/metadata.yaml @@ -1,4 +1,4 @@ -type: firehose_encoding +type: cloudwatch_encoding status: class: extension diff --git a/extension/encoding/cloudwatchencoding/testdata/config.yaml b/extension/encoding/cloudwatchencoding/testdata/config.yaml new file mode 100644 index 000000000000..e56c7e252f49 --- /dev/null +++ b/extension/encoding/cloudwatchencoding/testdata/config.yaml @@ -0,0 +1,4 @@ +cloudwatch_encoding: + +cloudwatch_encoding/invalid_encoding: + content_encoding: invalid \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/config.go b/extension/encoding/firehoseencodingextension/config.go deleted file mode 100644 index 0ab666fcc949..000000000000 --- a/extension/encoding/firehoseencodingextension/config.go +++ /dev/null @@ -1,53 +0,0 @@ -package firehoseencodingextension - -import ( - "fmt" - "go.opentelemetry.io/collector/component" -) - -type contentEncoding string - -const ( - NoEncoding = "" - GZipEncoded = "gzip" -) - -type Config struct { - Logs `mapstructure:"logs"` - Metrics `mapstructure:"metrics"` -} - -type Logs struct { - LogsEncoding contentEncoding `mapstructure:"content_encoding"` -} - -type Metrics struct { - MetricsEncoding contentEncoding `mapstructure:"content_encoding"` -} - -func createDefaultConfig() component.Config { - return &Config{ - Metrics: Metrics{ - MetricsEncoding: NoEncoding, - }, - Logs: Logs{ - LogsEncoding: GZipEncoded, - }, - } -} - -func (c *Config) Validate() error { - switch c.MetricsEncoding { - case NoEncoding: - case GZipEncoded: - default: - return fmt.Errorf("unknown metrics encoding %s", c.MetricsEncoding) - } - switch c.LogsEncoding { - case NoEncoding: - case GZipEncoded: - default: - return fmt.Errorf("unknown logs encoding %s", c.LogsEncoding) - } - return nil -} diff --git a/extension/encoding/firehoseencodingextension/extension.go b/extension/encoding/firehoseencodingextension/extension.go deleted file mode 100644 index 33619d5ef566..000000000000 --- a/extension/encoding/firehoseencodingextension/extension.go +++ /dev/null @@ -1,209 +0,0 @@ -package firehoseencodingextension - -import ( - "bytes" - "compress/gzip" - "context" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding" - expmetrics "github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics" - "github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil" - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch" - "github.com/tidwall/gjson" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/extension" - "go.opentelemetry.io/collector/pdata/plog" - "go.opentelemetry.io/collector/pdata/pmetric" - conventions "go.opentelemetry.io/collector/semconv/v1.6.1" - "go.uber.org/zap" - "io" -) - -var ( - _ encoding.LogsUnmarshalerExtension = (*cloudwatchExtension)(nil) - _ encoding.MetricsUnmarshalerExtension = (*cloudwatchExtension)(nil) -) - -type cloudwatchExtension struct { - config *Config - logger *zap.Logger -} - -type payload struct { - RequestID string `json:"requestId"` - Timestamp int64 `json:"timestamp"` - Records []record `json:"records"` -} - -type record struct { - Data string `json:"data"` -} - -type cloudwatchLog struct { - Owner string `json:"owner"` - LogGroup string `json:"logGroup"` - LogStream string `json:"logStream"` - - // LogEvents should have the logs data. - // It is left as raw, so the translator - // for cloudwatch log can unmarshal it. - LogEvents json.RawMessage `json:"logEvents"` -} - -const ( - attributeAWSCloudWatchLogGroupName = "aws.cloudwatch.log_group_name" - attributeAWSCloudWatchLogStreamName = "aws.cloudwatch.log_stream_name" -) - -func createExtension(_ context.Context, settings extension.Settings, config component.Config) (extension.Extension, error) { - return &cloudwatchExtension{ - config: config.(*Config), - logger: settings.Logger, - }, nil -} - -func (c *cloudwatchExtension) Start(_ context.Context, _ component.Host) error { - return nil -} - -func (c *cloudwatchExtension) Shutdown(_ context.Context) error { - return nil -} - -func decompress(decoded []byte, encoding contentEncoding) ([]byte, error) { - switch encoding { - case NoEncoding: - return decoded, nil - case GZipEncoded: - reader, err := gzip.NewReader(bytes.NewReader(decoded)) - if err != nil { - return nil, fmt.Errorf("failed to create gzip reader: %w", err) - } - defer reader.Close() - - decompressed, err := io.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("failed to read from gzip reader: %w", err) - } - return decompressed, nil - default: - // not possible, prevented by config.Validate - return nil, nil - } -} - -func getRecordsData(buf []byte, encoding contentEncoding) ([][]byte, error) { - var p payload - if err := json.Unmarshal(buf, &p); err != nil { - return nil, fmt.Errorf("failed to unmarshall firehose payload: %w", err) - } - - var records [][]byte - for i, rec := range p.Records { - decoded, err := base64.StdEncoding.DecodeString(rec.Data) - if err != nil { - return nil, fmt.Errorf("failed to base64 decode data from record [%d]: %w", i, err) - } - decompressed, err := decompress(decoded, encoding) - if err != nil { - return nil, fmt.Errorf("failed to decompress decoded data from record [%d]: %w", i, err) - } - records = append(records, decompressed) - } - - return records, nil -} - -func isCloudwatchLogValid(log cloudwatchLog) (bool, error) { - if log.Owner == "" { - return false, errors.New("cloudwatch log from firehose record data is missing owner field") - } - if log.LogGroup == "" { - return false, errors.New("cloudwatch log from firehose record data is missing log group field") - } - if log.LogStream == "" { - return false, errors.New("cloudwatch log from firehose record data is missing log stream field") - } - return true, nil -} - -func addLog(log cloudwatchLog, logs plog.Logs) error { - // Log events is a list of cloudwatch logs. - // It needs to be sent to the translator - // as a bytes array that has a new line - // splitting all logs. - result := gjson.ParseBytes(log.LogEvents) - var buf bytes.Buffer - result.ForEach(func(_, value gjson.Result) bool { - buf.WriteString(value.Raw) - buf.WriteByte('\n') - return true - }) - - logRecords, err := cloudwatch.UnmarshalLogs(buf.Bytes()) - if err != nil { - return err - } - - rl := logs.ResourceLogs().AppendEmpty() - lrs := rl.ScopeLogs().AppendEmpty().LogRecords() - logRecords.CopyTo(lrs) - resourceAttrs := rl.Resource().Attributes() - resourceAttrs.PutStr(conventions.AttributeCloudAccountID, log.Owner) - resourceAttrs.PutStr(attributeAWSCloudWatchLogGroupName, log.LogGroup) - resourceAttrs.PutStr(attributeAWSCloudWatchLogStreamName, log.LogStream) - return nil -} - -func (c *cloudwatchExtension) UnmarshalLogs(buf []byte) (plog.Logs, error) { - allData, e := getRecordsData(buf, c.config.LogsEncoding) - if e != nil { - return plog.Logs{}, e - } - - logs := plog.NewLogs() - for i, data := range allData { - var log cloudwatchLog - if err := json.Unmarshal(data, &log); err != nil { - return plog.Logs{}, fmt.Errorf("failed to unmarshall cloudwatch log from record data [%d]: %w", i, err) - } - if valid, err := isCloudwatchLogValid(log); !valid { - return plog.Logs{}, fmt.Errorf("cloudwatch log from record data [%d] is invalid: %w", i, err) - } - if err := addLog(log, logs); err != nil { - return plog.Logs{}, fmt.Errorf("failed to add cloudwatch log from record data [%d]: %w", i, err) - } - } - - if logs.ResourceLogs().Len() == 0 { - return logs, errors.New("no resource logs could be obtained from the firehose records") - } - pdatautil.GroupByResourceLogs(logs.ResourceLogs()) - return logs, nil -} - -func (c *cloudwatchExtension) UnmarshalMetrics(buf []byte) (pmetric.Metrics, error) { - allData, e := getRecordsData(buf, c.config.MetricsEncoding) - if e != nil { - return pmetric.Metrics{}, e - } - - metrics := pmetric.NewMetrics() - for i, data := range allData { - recordMetrics, err := cloudwatch.UnmarshalMetrics(data) - if err != nil { - return pmetric.Metrics{}, fmt.Errorf("failed to unmarshall metrics from record data [%d]: %w", i, err) - } - metrics = expmetrics.Merge(metrics, recordMetrics) - } - - if metrics.MetricCount() == 0 { - return metrics, errors.New("no resource metrics could be obtained from the record") - } - - metrics = expmetrics.Merge(pmetric.NewMetrics(), metrics) - return metrics, nil -} diff --git a/extension/encoding/firehoseencodingextension/extension_test.go b/extension/encoding/firehoseencodingextension/extension_test.go deleted file mode 100644 index 2181bcd6d124..000000000000 --- a/extension/encoding/firehoseencodingextension/extension_test.go +++ /dev/null @@ -1,238 +0,0 @@ -package firehoseencodingextension - -import ( - "bytes" - "compress/gzip" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "github.com/stretchr/testify/require" - "github.com/tidwall/gjson" - "go.opentelemetry.io/collector/pdata/plog" - "go.opentelemetry.io/collector/pdata/pmetric" - "go.uber.org/zap" - "os" - "path/filepath" - "strings" - "testing" -) - -func compressAndEncode(content []byte) (string, error) { - var compressed bytes.Buffer - gzipWriter := gzip.NewWriter(&compressed) - defer gzipWriter.Close() - _, err := gzipWriter.Write(content) - if err != nil { - return "", err - } - gzipWriter.Close() - encoded := base64.StdEncoding.EncodeToString(compressed.Bytes()) - return encoded, nil -} - -func TestUnmarshalLogs_Records(t *testing.T) { - t.Parallel() - - extension := cloudwatchExtension{ - config: &Config{ - Metrics: Metrics{ - MetricsEncoding: NoEncoding, - }, - Logs: Logs{ - LogsEncoding: GZipEncoded, - }, - }, - logger: zap.NewNop(), - } - - tests := map[string]struct { - filename string - err error - }{ - "OneLogRecord": { - filename: "one_log", - }, - "TwoLogRecords": { - filename: "two_logs", - }, - "InvalidRecord": { - filename: "invalid_log", - err: fmt.Errorf("failed to add cloudwatch log from record data [0]: %w", - fmt.Errorf("cloudwatch log from datum [0] is invalid: %w", - errors.New("cloudwatch log is missing timestamp field"), - ), - ), - }, - "SomeInvalidRecord": { - filename: "some_invalid_log", - err: fmt.Errorf("failed to add cloudwatch log from record data [0]: %w", - fmt.Errorf("cloudwatch log from datum [1] is invalid: %w", - errors.New("cloudwatch log is missing timestamp field"), - ), - ), - }, - "EmptyLogEvents": { - filename: "empty_log", - err: fmt.Errorf("failed to add cloudwatch log from record data [0]: %w", - errors.New("no log records could be obtained from the record"), - ), - }, - } - - unmarshaller := plog.JSONUnmarshaler{} - for name, test := range tests { - t.Run(name, func(t *testing.T) { - content, err := os.ReadFile(filepath.Join("testdata/logs", test.filename+".json")) - require.NoError(t, err) - - encoded, err := compressAndEncode(content) - require.NoError(t, err) - - // put the data in the format firehose expects - // it to be - p := payload{Records: []record{{Data: encoded}}} - allData, err := json.Marshal(p) - require.NoError(t, err) - - result, err := extension.UnmarshalLogs(allData) - require.Equal(t, test.err, err) - if err != nil { - return - } - - content, err = os.ReadFile(filepath.Join("testdata/logs", test.filename+"_expected.json")) - require.NoError(t, err) - expected, err := unmarshaller.UnmarshalLogs(content) - require.NoError(t, err) - - require.Equal(t, expected, result) - }) - } -} - -func TestUnmarshalLogs_MultipleResources(t *testing.T) { - t.Parallel() - - extension := cloudwatchExtension{ - config: &Config{ - Metrics: Metrics{ - MetricsEncoding: NoEncoding, - }, - Logs: Logs{ - LogsEncoding: GZipEncoded, - }, - }, - logger: zap.NewNop(), - } - - filename1 := "one_log" - content1, err := os.ReadFile(filepath.Join("testdata/logs", filename1+".json")) - require.NoError(t, err) - encoded1, err := compressAndEncode(content1) - require.NoError(t, err) - - filename2 := "two_logs" - content2, err := os.ReadFile(filepath.Join("testdata/logs", filename2+".json")) - require.NoError(t, err) - encoded2, err := compressAndEncode(content2) - require.NoError(t, err) - - // put the data in the format firehose expects it to be - p := payload{Records: []record{{Data: encoded1}, {Data: encoded2}}} - allData, err := json.Marshal(p) - require.NoError(t, err) - - result, err := extension.UnmarshalLogs(allData) - require.NoError(t, err) - - content, err := os.ReadFile("testdata/logs/multiple_resources_expected.json") - require.NoError(t, err) - unmarshaller := plog.JSONUnmarshaler{} - expected, err := unmarshaller.UnmarshalLogs(content) - require.NoError(t, err) - - require.Equal(t, expected, result) -} - -func TestUnmarshalMetrics(t *testing.T) { - t.Parallel() - - extension := cloudwatchExtension{ - config: &Config{ - Metrics: Metrics{ - MetricsEncoding: NoEncoding, - }, - Logs: Logs{ - LogsEncoding: GZipEncoded, - }, - }, - logger: zap.NewNop(), - } - - tests := map[string]struct { - filename string - errPrefix string - }{ - "OneMetric": { - filename: "one_metric", - }, - "TwoMetrics": { - filename: "two_metrics", - }, - "MultipleResources": { - filename: "multiple_resources", - }, - "InvalidRecord": { - filename: "invalid_metric", - errPrefix: "failed to unmarshall metrics from record data [0]:", - }, - "SomeInvalidRecord": { - filename: "some_invalid_metrics", - errPrefix: "failed to unmarshall metrics from record data [1]:", - }, - "EmptyRecord": { - filename: "empty_record", - errPrefix: "no resource metrics could be obtained from the record", - }, - } - - unmarshaller := pmetric.JSONUnmarshaler{} - for name, test := range tests { - t.Run(name, func(t *testing.T) { - content, err := os.ReadFile(filepath.Join("testdata/metrics", test.filename+".json")) - require.NoError(t, err) - - // for each metric, encode base64 - var records []record - var buf bytes.Buffer - gjson.ParseBytes(content).ForEach(func(_, value gjson.Result) bool { - data, e := json.Marshal(value.Value()) - require.NoError(t, e) - encoded := base64.StdEncoding.EncodeToString(data) - _, err = buf.WriteString(encoded + "\n") - require.NoError(t, err) - records = append(records, record{Data: encoded}) - return true - }) - - p := payload{Records: records} - allData, err := json.Marshal(p) - require.NoError(t, err) - - result, err := extension.UnmarshalMetrics(allData) - if err != nil { - require.NotEmpty(t, test.errPrefix) - require.True(t, strings.HasPrefix(err.Error(), test.errPrefix)) - return - } - - content, err = os.ReadFile(filepath.Join("testdata/metrics", test.filename+"_expected.json")) - require.NoError(t, err) - expected, err := unmarshaller.UnmarshalMetrics(content) - require.NoError(t, err) - - require.Equal(t, expected, result) - }) - } -} diff --git a/extension/encoding/firehoseencodingextension/testdata/config.yaml b/extension/encoding/firehoseencodingextension/testdata/config.yaml deleted file mode 100644 index be3dbb1578d5..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -firehose_encoding: - -firehose_encoding/logs_invalid_encoding: - logs: - content_encoding: invalid - -firehose_encoding/metrics_invalid_encoding: - metrics: - content_encoding: invalid \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/empty_log.json b/extension/encoding/firehoseencodingextension/testdata/logs/empty_log.json deleted file mode 100644 index 7c35990d2bf2..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/logs/empty_log.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "messageType":"DATA_MESSAGE", - "owner":"1234567890", - "logGroup":"test", - "logStream":"test-log-stream", - "subscriptionFilters":[ - "test logs filter" - ], - "logEvents":[] -} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/invalid_log.json b/extension/encoding/firehoseencodingextension/testdata/logs/invalid_log.json deleted file mode 100644 index b71ac98d123a..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/logs/invalid_log.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "messageType":"DATA_MESSAGE", - "owner":"1234567890", - "logGroup":"test", - "logStream":"test-log-stream", - "subscriptionFilters":[ - "test logs filter" - ], - "logEvents":[ - { - "id":"111", - "message":"it should fail without timestamp", - "extractedFields":{ - "action":"REJECT" - } - } - ] -} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/multiple_resources_expected.json b/extension/encoding/firehoseencodingextension/testdata/logs/multiple_resources_expected.json deleted file mode 100644 index 0ff798dc63ae..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/logs/multiple_resources_expected.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "resourceLogs":[ - { - "resource":{ - "attributes":[ - { - "key":"cloud.account.id", - "value":{ - "stringValue":"1234567890" - } - }, - { - "key":"aws.cloudwatch.log_group_name", - "value":{ - "stringValue":"vpc-flow-logs" - } - }, - { - "key":"aws.cloudwatch.log_stream_name", - "value":{ - "stringValue":"test-log-stream" - } - } - ] - }, - "scopeLogs":[ - { - "scope":{ - - }, - "logRecords":[ - { - "timeUnixNano":"1646096348000000000", - "body":{ - "stringValue":"REJECT" - }, - "traceId":"", - "spanId":"" - } - ] - } - ] - }, - { - "resource":{ - "attributes":[ - { - "key":"cloud.account.id", - "value":{ - "stringValue":"1234567890" - } - }, - { - "key":"aws.cloudwatch.log_group_name", - "value":{ - "stringValue":"vpc-flow-logs" - } - }, - { - "key":"aws.cloudwatch.log_stream_name", - "value":{ - "stringValue":"test-log-stream-2" - } - } - ] - }, - "scopeLogs":[ - { - "scope":{ - - }, - "logRecords":[ - { - "timeUnixNano":"1646096348000000000", - "body":{ - "stringValue":"REJECT" - }, - "traceId":"", - "spanId":"" - }, - { - "timeUnixNano":"1725544035523000000", - "body":{ - "stringValue":"Hello world!" - }, - "traceId":"", - "spanId":"" - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/one_log.json b/extension/encoding/firehoseencodingextension/testdata/logs/one_log.json deleted file mode 100644 index 6f958db9e0f9..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/logs/one_log.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "messageType":"DATA_MESSAGE", - "owner":"1234567890", - "logGroup":"vpc-flow-logs", - "logStream":"test-log-stream", - "subscriptionFilters":[ - "vpc flow logs filter" - ], - "logEvents":[ - { - "id":"111", - "timestamp":1646096348000, - "message":"REJECT", - "extractedFields":{ - "action":"REJECT" - } - } - ] -} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/one_log_expected.json b/extension/encoding/firehoseencodingextension/testdata/logs/one_log_expected.json deleted file mode 100644 index 05fd75da83e4..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/logs/one_log_expected.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "resourceLogs":[ - { - "resource":{ - "attributes":[ - { - "key":"cloud.account.id", - "value":{ - "stringValue":"1234567890" - } - }, - { - "key":"aws.cloudwatch.log_group_name", - "value":{ - "stringValue":"vpc-flow-logs" - } - }, - { - "key":"aws.cloudwatch.log_stream_name", - "value":{ - "stringValue":"test-log-stream" - } - } - ] - }, - "scopeLogs":[ - { - "scope":{ - - }, - "logRecords":[ - { - "timeUnixNano":"1646096348000000000", - "body":{ - "stringValue":"REJECT" - }, - "traceId":"", - "spanId":"" - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/some_invalid_log.json b/extension/encoding/firehoseencodingextension/testdata/logs/some_invalid_log.json deleted file mode 100644 index f0b7225b0cbc..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/logs/some_invalid_log.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "messageType":"DATA_MESSAGE", - "owner":"1234567890", - "logGroup":"test", - "logStream":"test-log-stream", - "subscriptionFilters":[ - "test logs filter" - ], - "logEvents":[ - { - "id":"111", - "timestamp":1646096348000, - "message":"REJECT", - "extractedFields":{ - "action":"REJECT" - } - }, - { - "id":"111", - "message":"it should fail without timestamp", - "extractedFields":{ - "action":"REJECT" - } - } - ] -} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/two_logs.json b/extension/encoding/firehoseencodingextension/testdata/logs/two_logs.json deleted file mode 100644 index 34b33b06d1d3..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/logs/two_logs.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "messageType":"DATA_MESSAGE", - "owner":"1234567890", - "logGroup":"vpc-flow-logs", - "logStream":"test-log-stream-2", - "subscriptionFilters":[ - "vpc flow logs filter" - ], - "logEvents":[ - { - "id":"111", - "timestamp":1646096348000, - "message":"REJECT", - "extractedFields":{ - "action":"REJECT" - } - }, - { - "id":"38480917865042697267627490045603633139480491071049695232", - "timestamp":1725544035523, - "message":"Hello world!" - } - ] -} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/two_logs_expected.json b/extension/encoding/firehoseencodingextension/testdata/logs/two_logs_expected.json deleted file mode 100644 index fb361b2d8610..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/logs/two_logs_expected.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "resourceLogs":[ - { - "resource":{ - "attributes":[ - { - "key":"cloud.account.id", - "value":{ - "stringValue":"1234567890" - } - }, - { - "key":"aws.cloudwatch.log_group_name", - "value":{ - "stringValue":"vpc-flow-logs" - } - }, - { - "key":"aws.cloudwatch.log_stream_name", - "value":{ - "stringValue":"test-log-stream-2" - } - } - ] - }, - "scopeLogs":[ - { - "scope":{ - - }, - "logRecords":[ - { - "timeUnixNano":"1646096348000000000", - "body":{ - "stringValue":"REJECT" - }, - "traceId":"", - "spanId":"" - }, - { - "timeUnixNano":"1725544035523000000", - "body":{ - "stringValue":"Hello world!" - }, - "traceId":"", - "spanId":"" - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/logs/two_resources.json b/extension/encoding/firehoseencodingextension/testdata/logs/two_resources.json deleted file mode 100644 index 0b162177607d..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/logs/two_resources.json +++ /dev/null @@ -1,34 +0,0 @@ -[ - { - "messageType":"DATA_MESSAGE", - "owner":"1234567890", - "logGroup":"test", - "logStream":"test-log-stream-1", - "subscriptionFilters":[ - "logs filter" - ], - "logEvents":[ - { - "id":"38480917865042697267627490045603633139480491071049695232", - "timestamp":1725544035523, - "message":"Hello world!" - } - ] - }, - { - "messageType":"DATA_MESSAGE", - "owner":"1234567890", - "logGroup":"test", - "logStream":"test-log-stream-1", - "subscriptionFilters":[ - "logs filter" - ], - "logEvents":[ - { - "id":"38480917865042697267627490045603633139480491071049695232", - "timestamp":1725544035523, - "message":"Hello world!" - } - ] - } -] \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/empty_record.json b/extension/encoding/firehoseencodingextension/testdata/metrics/empty_record.json deleted file mode 100644 index 0637a088a01e..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/metrics/empty_record.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/invalid_metric.json b/extension/encoding/firehoseencodingextension/testdata/metrics/invalid_metric.json deleted file mode 100644 index dfad993fcbd2..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/metrics/invalid_metric.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - { - "invalid": "record" - } -] \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources.json b/extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources.json deleted file mode 100644 index bd75a1906dc2..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - { - "metric_stream_name": "MyMetricStream-1", - "account_id": "1234567890", - "region": "us-east-1", - "namespace": "AWS/EC2", - "metric_name": "DiskWriteOps", - "dimensions": { - "InstanceId": "i-123456789012" - }, - "timestamp": 1611929698000, - "value": { - "count": 3.0, - "sum": 20.0, - "max": 18.0, - "min": 0.0, - "p99": 17.56, - "p99.9": 17.8764, - "TM(25%:75%)": 16.43 - }, - "unit": "Seconds" - }, - { - "metric_stream_name": "MyMetricStream-2", - "account_id": "1234567890", - "region": "us-east-1", - "namespace": "AWS/EC2", - "metric_name": "DiskWriteOps", - "dimensions": { - "InstanceId": "i-123456789012" - }, - "timestamp": 1611929698000, - "value": { - "count": 3.0, - "sum": 20.0, - "max": 18.0, - "min": 0.0, - "p99": 17.56, - "p99.9": 17.8764, - "TM(25%:75%)": 16.43 - }, - "unit": "Seconds" - } -] \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources_expected.json b/extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources_expected.json deleted file mode 100644 index e67e2c2f6d51..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/metrics/multiple_resources_expected.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "resourceMetrics":[ - { - "resource":{ - "attributes":[ - { - "key":"cloud.provider", - "value":{ - "stringValue":"aws" - } - }, - { - "key":"cloud.account.id", - "value":{ - "stringValue":"1234567890" - } - }, - { - "key":"cloud.region", - "value":{ - "stringValue":"us-east-1" - } - }, - { - "key":"service.namespace", - "value":{ - "stringValue":"AWS" - } - }, - { - "key":"service.name", - "value":{ - "stringValue":"EC2" - } - }, - { - "key":"aws.cloudwatch.metric_stream_name", - "value":{ - "stringValue":"MyMetricStream-1" - } - } - ] - }, - "scopeMetrics":[ - { - "scope":{ - - }, - "metrics":[ - { - "name":"DiskWriteOps", - "unit":"Seconds", - "summary":{ - "dataPoints":[ - { - "attributes":[ - { - "key":"service.instance.id", - "value":{ - "stringValue":"i-123456789012" - } - } - ], - "timeUnixNano":"1611929698000000000", - "count":"3", - "sum":20, - "quantileValues":[ - { - "quantile":99, - "value":17.56 - }, - { - "quantile":99.9, - "value":17.8764 - }, - { - - }, - { - "quantile":100, - "value":18 - } - ] - } - ] - } - } - ] - } - ] - }, - { - "resource":{ - "attributes":[ - { - "key":"cloud.provider", - "value":{ - "stringValue":"aws" - } - }, - { - "key":"cloud.account.id", - "value":{ - "stringValue":"1234567890" - } - }, - { - "key":"cloud.region", - "value":{ - "stringValue":"us-east-1" - } - }, - { - "key":"service.namespace", - "value":{ - "stringValue":"AWS" - } - }, - { - "key":"service.name", - "value":{ - "stringValue":"EC2" - } - }, - { - "key":"aws.cloudwatch.metric_stream_name", - "value":{ - "stringValue":"MyMetricStream-2" - } - } - ] - }, - "scopeMetrics":[ - { - "scope":{ - - }, - "metrics":[ - { - "name":"DiskWriteOps", - "unit":"Seconds", - "summary":{ - "dataPoints":[ - { - "attributes":[ - { - "key":"service.instance.id", - "value":{ - "stringValue":"i-123456789012" - } - } - ], - "timeUnixNano":"1611929698000000000", - "count":"3", - "sum":20, - "quantileValues":[ - { - "quantile":99, - "value":17.56 - }, - { - "quantile":99.9, - "value":17.8764 - }, - { - - }, - { - "quantile":100, - "value":18 - } - ] - } - ] - } - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/one_metric.json b/extension/encoding/firehoseencodingextension/testdata/metrics/one_metric.json deleted file mode 100644 index 3e40df35a073..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/metrics/one_metric.json +++ /dev/null @@ -1,23 +0,0 @@ -[ - { - "metric_stream_name":"MyMetricStream", - "account_id":"1234567890", - "region":"us-east-1", - "namespace":"AWS/EC2", - "metric_name":"DiskWriteOps", - "dimensions":{ - "InstanceId":"i-123456789012" - }, - "timestamp":1611929698000, - "value":{ - "count":3.0, - "sum":20.0, - "max":18.0, - "min":0.0, - "p99":17.56, - "p99.9":17.8764, - "TM(25%:75%)":16.43 - }, - "unit":"Seconds" - } -] \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/one_metric_expected.json b/extension/encoding/firehoseencodingextension/testdata/metrics/one_metric_expected.json deleted file mode 100644 index 3e9fe178648a..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/metrics/one_metric_expected.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "resourceMetrics":[ - { - "resource":{ - "attributes":[ - { - "key":"cloud.provider", - "value":{ - "stringValue":"aws" - } - }, - { - "key":"cloud.account.id", - "value":{ - "stringValue":"1234567890" - } - }, - { - "key":"cloud.region", - "value":{ - "stringValue":"us-east-1" - } - }, - { - "key":"service.namespace", - "value":{ - "stringValue":"AWS" - } - }, - { - "key":"service.name", - "value":{ - "stringValue":"EC2" - } - }, - { - "key":"aws.cloudwatch.metric_stream_name", - "value":{ - "stringValue":"MyMetricStream" - } - } - ] - }, - "scopeMetrics":[ - { - "scope":{ - - }, - "metrics":[ - { - "name":"DiskWriteOps", - "unit":"Seconds", - "summary":{ - "dataPoints":[ - { - "attributes":[ - { - "key":"service.instance.id", - "value":{ - "stringValue":"i-123456789012" - } - } - ], - "timeUnixNano":"1611929698000000000", - "count":"3", - "sum":20, - "quantileValues":[ - { - "quantile":99, - "value":17.56 - }, - { - "quantile":99.9, - "value":17.8764 - }, - { - - }, - { - "quantile":100, - "value":18 - } - ] - } - ] - } - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/some_invalid_metrics.json b/extension/encoding/firehoseencodingextension/testdata/metrics/some_invalid_metrics.json deleted file mode 100644 index 636000e58ca1..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/metrics/some_invalid_metrics.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "metric_stream_name":"MyMetricStream", - "account_id":"1234567890", - "region":"us-east-1", - "namespace":"AWS/EC2", - "metric_name":"DiskWriteOps", - "dimensions":{ - "InstanceId":"i-123456789012" - }, - "timestamp":1611929698000, - "value":{ - "count":3.0, - "sum":20.0, - "max":18.0, - "min":0.0, - "p99":17.56, - "p99.9":17.8764, - "TM(25%:75%)":16.43 - }, - "unit":"Seconds" - }, - { - "invalid": "record" - } -] \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics.json b/extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics.json deleted file mode 100644 index 1127193c5ef1..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - { - "metric_stream_name":"MyMetricStream", - "account_id":"1234567890", - "region":"us-east-1", - "namespace":"AWS/EC2", - "metric_name":"DiskWriteOps", - "dimensions":{ - "InstanceId":"i-123456789012" - }, - "timestamp":1611929698000, - "value":{ - "count":3.0, - "sum":20.0, - "max":18.0, - "min":0.0, - "p99":17.56, - "p99.9":17.8764, - "TM(25%:75%)":16.43 - }, - "unit":"Seconds" - }, - { - "metric_stream_name":"MyMetricStream", - "account_id":"1234567890", - "region":"us-east-1", - "namespace":"AWS/EC2", - "metric_name":"DiskWriteOps", - "dimensions":{ - "InstanceId":"i-123456789012" - }, - "timestamp":1611929698100, - "value":{ - "count":5.0, - "sum":21.0, - "max":18.0, - "min":0.0, - "p99":17.56, - "p99.9":17.8764, - "TM(25%:75%)":16.43 - }, - "unit":"Seconds" - } -] \ No newline at end of file diff --git a/extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics_expected.json b/extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics_expected.json deleted file mode 100644 index 6dbc5c2422ee..000000000000 --- a/extension/encoding/firehoseencodingextension/testdata/metrics/two_metrics_expected.json +++ /dev/null @@ -1,123 +0,0 @@ -{ - "resourceMetrics":[ - { - "resource":{ - "attributes":[ - { - "key":"cloud.provider", - "value":{ - "stringValue":"aws" - } - }, - { - "key":"cloud.account.id", - "value":{ - "stringValue":"1234567890" - } - }, - { - "key":"cloud.region", - "value":{ - "stringValue":"us-east-1" - } - }, - { - "key":"service.namespace", - "value":{ - "stringValue":"AWS" - } - }, - { - "key":"service.name", - "value":{ - "stringValue":"EC2" - } - }, - { - "key":"aws.cloudwatch.metric_stream_name", - "value":{ - "stringValue":"MyMetricStream" - } - } - ] - }, - "scopeMetrics":[ - { - "scope":{ - - }, - "metrics":[ - { - "name":"DiskWriteOps", - "unit":"Seconds", - "summary":{ - "dataPoints":[ - { - "attributes":[ - { - "key":"service.instance.id", - "value":{ - "stringValue":"i-123456789012" - } - } - ], - "timeUnixNano":"1611929698000000000", - "count":"3", - "sum":20, - "quantileValues":[ - { - "quantile":99, - "value":17.56 - }, - { - "quantile":99.9, - "value":17.8764 - }, - { - - }, - { - "quantile":100, - "value":18 - } - ] - }, - { - "attributes":[ - { - "key":"service.instance.id", - "value":{ - "stringValue":"i-123456789012" - } - } - ], - "timeUnixNano":"1611929698100000000", - "count":"5", - "sum":21, - "quantileValues":[ - { - "quantile":99, - "value":17.56 - }, - { - "quantile":99.9, - "value":17.8764 - }, - { - - }, - { - "quantile":100, - "value":18 - } - ] - } - ] - } - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/pkg/translator/cloudwatch/cwlogs_to_log_record.go b/pkg/translator/cloudwatch/cwlogs_to_logs.go similarity index 74% rename from pkg/translator/cloudwatch/cwlogs_to_log_record.go rename to pkg/translator/cloudwatch/cwlogs_to_logs.go index cdda408b751c..d035bcd8671d 100644 --- a/pkg/translator/cloudwatch/cwlogs_to_log_record.go +++ b/pkg/translator/cloudwatch/cwlogs_to_logs.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil" "go.opentelemetry.io/collector/pdata/pcommon" "time" @@ -28,14 +29,16 @@ func isLogValid(log cloudwatchLog) (bool, error) { return true, nil } -func addRecord(log cloudwatchLog, records plog.LogRecordSlice) { - logRecord := records.AppendEmpty() +func addRecord(log cloudwatchLog, logs plog.Logs) { + rl := logs.ResourceLogs().AppendEmpty() + sl := rl.ScopeLogs().AppendEmpty() + logRecord := sl.LogRecords().AppendEmpty() logRecord.SetTimestamp(pcommon.NewTimestampFromTime(time.UnixMilli(log.Timestamp))) logRecord.Body().SetStr(log.Message) } -func UnmarshalLogs(record []byte) (plog.LogRecordSlice, error) { - logs := plog.NewLogRecordSlice() +func UnmarshalLogs(record []byte) (plog.Logs, error) { + logs := plog.NewLogs() decoder := json.NewDecoder(bytes.NewReader(record)) for datumIndex := 0; ; datumIndex++ { var log cloudwatchLog @@ -43,18 +46,19 @@ func UnmarshalLogs(record []byte) (plog.LogRecordSlice, error) { if errors.Is(err, io.EOF) { break } - return plog.LogRecordSlice{}, + return plog.Logs{}, fmt.Errorf("unable to unmarshal datum [%d] into cloudwatch log: %w", datumIndex, err) } if valid, err := isLogValid(log); !valid { - return plog.LogRecordSlice{}, + return plog.Logs{}, fmt.Errorf("cloudwatch log from datum [%d] is invalid: %w", datumIndex, err) } addRecord(log, logs) } - if logs.Len() == 0 { + if logs.LogRecordCount() == 0 { return logs, errors.New("no log records could be obtained from the record") } + pdatautil.GroupByResourceLogs(logs.ResourceLogs()) return logs, nil } diff --git a/pkg/translator/cloudwatch/cwlogs_to_log_record_test.go b/pkg/translator/cloudwatch/cwlogs_to_logs_test.go similarity index 94% rename from pkg/translator/cloudwatch/cwlogs_to_log_record_test.go rename to pkg/translator/cloudwatch/cwlogs_to_logs_test.go index 30e28dea5c66..5358d5fa3f83 100644 --- a/pkg/translator/cloudwatch/cwlogs_to_log_record_test.go +++ b/pkg/translator/cloudwatch/cwlogs_to_logs_test.go @@ -72,8 +72,8 @@ func TestUnmarshalLogs(t *testing.T) { require.NoError(t, err) // get log records - expectedLogs := expected.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords() - require.Equal(t, expectedLogs, result) + //expectedLogs := expected.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords() + require.Equal(t, expected, result) }) } } From f7f90eae6a1dcb2765920d37c87802e6be897079 Mon Sep 17 00:00:00 2001 From: Constanca Manteigas Date: Wed, 15 Jan 2025 08:44:29 +0100 Subject: [PATCH 4/9] Apply suggestions, run make commands --- .github/ISSUE_TEMPLATE/bug_report.yaml | 2 + .github/ISSUE_TEMPLATE/feature_request.yaml | 2 + .github/ISSUE_TEMPLATE/other.yaml | 2 + .github/ISSUE_TEMPLATE/unmaintained.yaml | 2 + .../encoding/cloudwatchencoding/config.go | 8 ++- .../cloudwatchencoding/config_test.go | 17 ++++--- extension/encoding/cloudwatchencoding/doc.go | 5 +- .../encoding/cloudwatchencoding/extension.go | 20 ++++---- .../cloudwatchencoding/extension_test.go | 6 ++- .../encoding/cloudwatchencoding/factory.go | 8 ++- .../cloudwatchencoding/factory_test.go | 6 ++- .../generated_package_test.go | 3 +- extension/encoding/cloudwatchencoding/go.mod | 12 +++++ .../encoding/cloudwatchencoding/metadata.yaml | 2 +- pkg/translator/cloudwatch/cwlogs_to_logs.go | 13 +++-- .../cloudwatch/cwlogs_to_logs_test.go | 12 +++-- .../cloudwatch/cwmetrics_to_metrics.go | 51 ++++++++++++++++--- .../cloudwatch/cwmetrics_to_metrics_test.go | 10 ++-- pkg/translator/cloudwatch/go.mod | 6 +++ pkg/translator/cloudwatch/metadata.yaml | 2 +- pkg/translator/cloudwatch/package_test.go | 3 ++ 21 files changed, 148 insertions(+), 44 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index ea469c572bcb..0d73135965d2 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -92,6 +92,7 @@ body: - extension/cgroupruntime - extension/encoding - extension/encoding/avrologencoding + - extension/encoding/cloudwatchencoding - extension/encoding/jaegerencoding - extension/encoding/jsonlogencoding - extension/encoding/otlpencoding @@ -158,6 +159,7 @@ body: - pkg/status - pkg/translator/azure - pkg/translator/azurelogs + - pkg/translator/cloudwatch - pkg/translator/jaeger - pkg/translator/loki - pkg/translator/opencensus diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index 99f72f1c13b1..86dafe69977b 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -86,6 +86,7 @@ body: - extension/cgroupruntime - extension/encoding - extension/encoding/avrologencoding + - extension/encoding/cloudwatchencoding - extension/encoding/jaegerencoding - extension/encoding/jsonlogencoding - extension/encoding/otlpencoding @@ -152,6 +153,7 @@ body: - pkg/status - pkg/translator/azure - pkg/translator/azurelogs + - pkg/translator/cloudwatch - pkg/translator/jaeger - pkg/translator/loki - pkg/translator/opencensus diff --git a/.github/ISSUE_TEMPLATE/other.yaml b/.github/ISSUE_TEMPLATE/other.yaml index c65b1b3fa089..ed37196f0c72 100644 --- a/.github/ISSUE_TEMPLATE/other.yaml +++ b/.github/ISSUE_TEMPLATE/other.yaml @@ -86,6 +86,7 @@ body: - extension/cgroupruntime - extension/encoding - extension/encoding/avrologencoding + - extension/encoding/cloudwatchencoding - extension/encoding/jaegerencoding - extension/encoding/jsonlogencoding - extension/encoding/otlpencoding @@ -152,6 +153,7 @@ body: - pkg/status - pkg/translator/azure - pkg/translator/azurelogs + - pkg/translator/cloudwatch - pkg/translator/jaeger - pkg/translator/loki - pkg/translator/opencensus diff --git a/.github/ISSUE_TEMPLATE/unmaintained.yaml b/.github/ISSUE_TEMPLATE/unmaintained.yaml index a6402ed0e246..685cd8d24c69 100644 --- a/.github/ISSUE_TEMPLATE/unmaintained.yaml +++ b/.github/ISSUE_TEMPLATE/unmaintained.yaml @@ -91,6 +91,7 @@ body: - extension/cgroupruntime - extension/encoding - extension/encoding/avrologencoding + - extension/encoding/cloudwatchencoding - extension/encoding/jaegerencoding - extension/encoding/jsonlogencoding - extension/encoding/otlpencoding @@ -157,6 +158,7 @@ body: - pkg/status - pkg/translator/azure - pkg/translator/azurelogs + - pkg/translator/cloudwatch - pkg/translator/jaeger - pkg/translator/loki - pkg/translator/opencensus diff --git a/extension/encoding/cloudwatchencoding/config.go b/extension/encoding/cloudwatchencoding/config.go index ac69b6f0bf4d..b905f7368c33 100644 --- a/extension/encoding/cloudwatchencoding/config.go +++ b/extension/encoding/cloudwatchencoding/config.go @@ -1,7 +1,11 @@ -package cloudwatchencoding +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cloudwatchencoding // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" import ( "fmt" + "go.opentelemetry.io/collector/component" ) @@ -25,7 +29,7 @@ func (c *Config) Validate() error { case NoEncoding: case GZipEncoded: default: - return fmt.Errorf("unknown content encoding %s", c.Encoding) + return fmt.Errorf("unknown content encoding %q", c.Encoding) } return nil } diff --git a/extension/encoding/cloudwatchencoding/config_test.go b/extension/encoding/cloudwatchencoding/config_test.go index ecc9bf5ba0fd..dc7247b1e20a 100644 --- a/extension/encoding/cloudwatchencoding/config_test.go +++ b/extension/encoding/cloudwatchencoding/config_test.go @@ -1,13 +1,18 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package cloudwatchencoding import ( "errors" - "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension/internal/metadata" + "path/filepath" + "testing" + "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap/confmaptest" - "path/filepath" - "testing" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension/internal/metadata" ) func TestLoadConfig(t *testing.T) { @@ -26,7 +31,7 @@ func TestLoadConfig(t *testing.T) { }, { id: component.NewIDWithName(metadata.Type, "invalid_encoding"), - expectedErr: errors.New("unknown content encoding invalid"), + expectedErr: errors.New("unknown content encoding \"invalid\""), }, } @@ -42,7 +47,8 @@ func TestLoadConfig(t *testing.T) { require.NoError(t, sub.Unmarshal(cfg)) if tt.expectedErr != nil { - require.Equal(t, tt.expectedErr, component.ValidateConfig(cfg)) + err = component.ValidateConfig(cfg) + require.Equal(t, tt.expectedErr, err) return } @@ -50,5 +56,4 @@ func TestLoadConfig(t *testing.T) { require.Equal(t, tt.expected, cfg) }) } - } diff --git a/extension/encoding/cloudwatchencoding/doc.go b/extension/encoding/cloudwatchencoding/doc.go index 4bb6fdeb0be5..86b00ea0d448 100644 --- a/extension/encoding/cloudwatchencoding/doc.go +++ b/extension/encoding/cloudwatchencoding/doc.go @@ -1,2 +1,5 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + //go:generate mdatagen metadata.yaml -package cloudwatchencoding +package cloudwatchencoding // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" diff --git a/extension/encoding/cloudwatchencoding/extension.go b/extension/encoding/cloudwatchencoding/extension.go index 09110e7c0624..e050ad472e9f 100644 --- a/extension/encoding/cloudwatchencoding/extension.go +++ b/extension/encoding/cloudwatchencoding/extension.go @@ -1,18 +1,23 @@ -package cloudwatchencoding +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cloudwatchencoding // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" import ( "bytes" "compress/gzip" "context" "fmt" - "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding" - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch" + "io" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/extension" "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/pmetric" "go.uber.org/zap" - "io" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch" ) var ( @@ -50,12 +55,7 @@ func decompress(buf []byte, encoding contentEncoding) ([]byte, error) { return nil, fmt.Errorf("failed to create gzip reader: %w", err) } defer reader.Close() - - decompressed, err := io.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("failed to read from gzip reader: %w", err) - } - return decompressed, nil + return io.ReadAll(reader) default: // not possible, prevented by config.Validate return nil, nil diff --git a/extension/encoding/cloudwatchencoding/extension_test.go b/extension/encoding/cloudwatchencoding/extension_test.go index 7ba659e10f32..cc0bc80fb6b3 100644 --- a/extension/encoding/cloudwatchencoding/extension_test.go +++ b/extension/encoding/cloudwatchencoding/extension_test.go @@ -1,10 +1,14 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package cloudwatchencoding import ( "context" + "testing" + "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component/componenttest" - "testing" ) func TestExtension_Start_Shutdown(t *testing.T) { diff --git a/extension/encoding/cloudwatchencoding/factory.go b/extension/encoding/cloudwatchencoding/factory.go index 6b71df564412..d903b52fa4e4 100644 --- a/extension/encoding/cloudwatchencoding/factory.go +++ b/extension/encoding/cloudwatchencoding/factory.go @@ -1,8 +1,12 @@ -package cloudwatchencoding +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cloudwatchencoding // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" import ( - "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension/internal/metadata" "go.opentelemetry.io/collector/extension" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension/internal/metadata" ) func NewFactory() extension.Factory { diff --git a/extension/encoding/cloudwatchencoding/factory_test.go b/extension/encoding/cloudwatchencoding/factory_test.go index 06c73ea3d965..961c5d9a8b92 100644 --- a/extension/encoding/cloudwatchencoding/factory_test.go +++ b/extension/encoding/cloudwatchencoding/factory_test.go @@ -1,10 +1,14 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package cloudwatchencoding import ( "context" + "testing" + "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/extension/extensiontest" - "testing" ) func TestCreateExtension(t *testing.T) { diff --git a/extension/encoding/cloudwatchencoding/generated_package_test.go b/extension/encoding/cloudwatchencoding/generated_package_test.go index c7c082c67a6a..a0fdb5d7509e 100644 --- a/extension/encoding/cloudwatchencoding/generated_package_test.go +++ b/extension/encoding/cloudwatchencoding/generated_package_test.go @@ -3,8 +3,9 @@ package cloudwatchencoding import ( - "go.uber.org/goleak" "testing" + + "go.uber.org/goleak" ) func TestMain(m *testing.M) { diff --git a/extension/encoding/cloudwatchencoding/go.mod b/extension/encoding/cloudwatchencoding/go.mod index 0a18fc111493..cb29ad9a740b 100644 --- a/extension/encoding/cloudwatchencoding/go.mod +++ b/extension/encoding/cloudwatchencoding/go.mod @@ -58,3 +58,15 @@ require ( ) replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch => ../../../pkg/translator/cloudwatch + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest => ../../../pkg/pdatatest + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden => ../../../pkg/golden + +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil => ../../../internal/pdatautil + +replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding => ../ + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../../pkg/pdatautil + +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics => ../../../internal/exp/metrics diff --git a/extension/encoding/cloudwatchencoding/metadata.yaml b/extension/encoding/cloudwatchencoding/metadata.yaml index 1b902fc6ce83..b51f39963317 100644 --- a/extension/encoding/cloudwatchencoding/metadata.yaml +++ b/extension/encoding/cloudwatchencoding/metadata.yaml @@ -6,4 +6,4 @@ status: alpha: [extension] distributions: [] codeowners: - active: [] + active: [constanca-m, axw] diff --git a/pkg/translator/cloudwatch/cwlogs_to_logs.go b/pkg/translator/cloudwatch/cwlogs_to_logs.go index d035bcd8671d..80c1d89dc473 100644 --- a/pkg/translator/cloudwatch/cwlogs_to_logs.go +++ b/pkg/translator/cloudwatch/cwlogs_to_logs.go @@ -1,16 +1,21 @@ -package cloudwatch +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cloudwatch // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch" import ( "bytes" "encoding/json" "errors" "fmt" - "github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil" - "go.opentelemetry.io/collector/pdata/pcommon" + "io" "time" + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil" + "go.opentelemetry.io/collector/pdata/plog" - "io" ) type cloudwatchLog struct { diff --git a/pkg/translator/cloudwatch/cwlogs_to_logs_test.go b/pkg/translator/cloudwatch/cwlogs_to_logs_test.go index 5358d5fa3f83..232879e8c434 100644 --- a/pkg/translator/cloudwatch/cwlogs_to_logs_test.go +++ b/pkg/translator/cloudwatch/cwlogs_to_logs_test.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package cloudwatch import ( @@ -5,12 +8,13 @@ import ( "encoding/json" "errors" "fmt" - "github.com/stretchr/testify/require" - "github.com/tidwall/gjson" - "go.opentelemetry.io/collector/pdata/plog" "os" "path/filepath" "testing" + + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "go.opentelemetry.io/collector/pdata/plog" ) func TestUnmarshalLogs(t *testing.T) { @@ -72,7 +76,7 @@ func TestUnmarshalLogs(t *testing.T) { require.NoError(t, err) // get log records - //expectedLogs := expected.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords() + // expectedLogs := expected.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords() require.Equal(t, expected, result) }) } diff --git a/pkg/translator/cloudwatch/cwmetrics_to_metrics.go b/pkg/translator/cloudwatch/cwmetrics_to_metrics.go index 42170b82d310..38e34b611727 100644 --- a/pkg/translator/cloudwatch/cwmetrics_to_metrics.go +++ b/pkg/translator/cloudwatch/cwmetrics_to_metrics.go @@ -1,17 +1,22 @@ -package cloudwatch +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package cloudwatch // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch" import ( "bytes" "encoding/json" "errors" "fmt" - expmetrics "github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics" - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/pmetric" - conventions "go.opentelemetry.io/collector/semconv/v1.27.0" "io" "strings" "time" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + conventions "go.opentelemetry.io/collector/semconv/v1.27.0" + + expmetrics "github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics" ) // The cloudwatchMetric is the format for the CloudWatch metric stream records. @@ -46,6 +51,38 @@ const ( namespaceDelimiter = "/" ) +// find the valid unit values for cloudwatch metric in unit section of +// https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDatum.html#API_MetricDatum_Contents +var validUnitValues = map[string]bool{ + "Seconds": true, + "Microseconds": true, + "Milliseconds": true, + "Bytes": true, + "Kilobytes": true, + "Megabytes": true, + "Gigabytes": true, + "Terabytes": true, + "Bits": true, + "Kilobits": true, + "Megabits": true, + "Gigabits": true, + "Terabits": true, + "Percent": true, + "Count": true, + "Bytes/Second": true, + "Kilobytes/Second": true, + "Megabytes/Second": true, + "Gigabytes/Second": true, + "Terabytes/Second": true, + "Bits/Second": true, + "Kilobits/Second": true, + "Megabits/Second": true, + "Gigabits/Second": true, + "Terabits/Second": true, + "Count/Second": true, + "None": true, +} + // isMetricValid validates that the cloudwatch metric has been unmarshalled correctly func isMetricValid(metric cloudwatchMetric) (bool, error) { if metric.MetricName == "" { @@ -54,8 +91,8 @@ func isMetricValid(metric cloudwatchMetric) (bool, error) { if metric.Namespace == "" { return false, errors.New("cloudwatch metric is missing namespace field") } - if metric.Unit == "" { - return false, errors.New("cloudwatch metric is missing unit field") + if _, exists := validUnitValues[metric.Unit]; !exists { + return false, fmt.Errorf("cloudwatch metric unit '%s' value is not valid", metric.Unit) } if metric.Value == nil { return false, errors.New("cloudwatch metric is missing value") diff --git a/pkg/translator/cloudwatch/cwmetrics_to_metrics_test.go b/pkg/translator/cloudwatch/cwmetrics_to_metrics_test.go index 129099eb7a77..004007b0375d 100644 --- a/pkg/translator/cloudwatch/cwmetrics_to_metrics_test.go +++ b/pkg/translator/cloudwatch/cwmetrics_to_metrics_test.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package cloudwatch import ( @@ -5,12 +8,13 @@ import ( "encoding/json" "errors" "fmt" - "github.com/stretchr/testify/require" - "github.com/tidwall/gjson" - "go.opentelemetry.io/collector/pdata/pmetric" "os" "path/filepath" "testing" + + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "go.opentelemetry.io/collector/pdata/pmetric" ) func TestUnmarshalMetrics(t *testing.T) { diff --git a/pkg/translator/cloudwatch/go.mod b/pkg/translator/cloudwatch/go.mod index 2a519d9c7ca2..1dff39702c63 100644 --- a/pkg/translator/cloudwatch/go.mod +++ b/pkg/translator/cloudwatch/go.mod @@ -43,3 +43,9 @@ retract ( v0.76.1 v0.65.0 ) + +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil => ../../../internal/pdatautil + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest => ../../pdatatest + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden => ../../golden diff --git a/pkg/translator/cloudwatch/metadata.yaml b/pkg/translator/cloudwatch/metadata.yaml index 47c8adf2f35a..9b4fab7a9533 100644 --- a/pkg/translator/cloudwatch/metadata.yaml +++ b/pkg/translator/cloudwatch/metadata.yaml @@ -1,3 +1,3 @@ status: codeowners: - active: [] + active: [constanca-m, axw] diff --git a/pkg/translator/cloudwatch/package_test.go b/pkg/translator/cloudwatch/package_test.go index aef2b34f0690..3d9766560a81 100644 --- a/pkg/translator/cloudwatch/package_test.go +++ b/pkg/translator/cloudwatch/package_test.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package cloudwatch import ( From 4e8654255c2619080f7db550815fcac390176cfc Mon Sep 17 00:00:00 2001 From: Constanca Manteigas Date: Wed, 15 Jan 2025 08:50:58 +0100 Subject: [PATCH 5/9] add go.sum --- extension/encoding/cloudwatchencoding/go.sum | 150 +++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 extension/encoding/cloudwatchencoding/go.sum diff --git a/extension/encoding/cloudwatchencoding/go.sum b/extension/encoding/cloudwatchencoding/go.sum new file mode 100644 index 000000000000..753c697d6419 --- /dev/null +++ b/extension/encoding/cloudwatchencoding/go.sum @@ -0,0 +1,150 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ= +github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding v0.117.0 h1:te8DpwzFocnxrfenTK3dLr8MQ3E8r3dRa1ZXfd/scss= +github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding v0.117.0/go.mod h1:xCGKcgGqkH3TmLj3FPtHa7YgmPgYK6FdJaqVYfrUE3c= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.117.0 h1:6BF8ymRgg1duNHXDpqgDayDlR+YktfQGQN1vzdxGcxQ= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.117.0/go.mod h1:T5aTYEzGf74ZagtJiruJC5idVLGKr78q/zqDzN4gPQQ= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0 h1:SjZHtgph6f675yK/p+GTo8I7VlIAjJFod3SMNgfpEAg= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0/go.mod h1:e4cIzlw2iGmMCgHKyJHLf+PF99A1Y6GvNC0+E3x1dA4= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.117.0 h1:tOJFUIZaAU4zm5CilqZN1/AuKQa7diTrcEhgQIYly6k= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.117.0/go.mod h1:PJ2FGCS+Hw+tlHUNNWVHNo3IXtEsb9RKgl/ssSi3Z98= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.117.0 h1:/wMNk8w1UEHKpKoNk1jA2aifHgfGZE+WelGNrCf0CJ0= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.117.0/go.mod h1:ESyMNHmgZYh8Ouhr2veecTMK6sB8gQ8u2s3dsy9Og6k= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.117.0 h1:GqlhXd6J8zgxCYenbI3ew03SJnGec1vEEGzGHw9X/Y0= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.117.0/go.mod h1:OGylX+Bp+urSNNGoI1XG7U6vaRDZk1wN/w6fHP1F7IY= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector/component v0.117.0 h1:A3Im4PqLyfduAdVyUgbOZdUs7J/USegdpnkoIAOuN3Y= +go.opentelemetry.io/collector/component v0.117.0/go.mod h1:+SxJgeMwNV6y3aKNR2sP0PfovcUlRwC0+pEv4tTYdXA= +go.opentelemetry.io/collector/component/componenttest v0.117.0 h1:r3k0BsU/cJlqVQRtgFjxfduNEGaM2qCAU7JitIGkRds= +go.opentelemetry.io/collector/component/componenttest v0.117.0/go.mod h1:MoBWSGb3KwGc5FAIO+htez/QWK2uqJ4fnbEnfHB384c= +go.opentelemetry.io/collector/config/configtelemetry v0.117.0 h1:xsMfc89VByIF2fJzWuxs/2eqy44DWfNBAysReG4TAr8= +go.opentelemetry.io/collector/config/configtelemetry v0.117.0/go.mod h1:SlBEwQg0qly75rXZ6W1Ig8jN25KBVBkFIIAUI1GiAAE= +go.opentelemetry.io/collector/confmap v1.23.0 h1:EY+auc0kbyZ4HIfkLYeJyLDCZIFzMA1u8QRGW4bC1Ag= +go.opentelemetry.io/collector/confmap v1.23.0/go.mod h1:Rrhs+MWoaP6AswZp+ReQ2VO9dfOfcUjdjiSHBsG+nec= +go.opentelemetry.io/collector/extension v0.117.0 h1:B3cG7g+wbhmpMFugaDxOcyiPKeulaW8+EQdJbZxDfho= +go.opentelemetry.io/collector/extension v0.117.0/go.mod h1:WjyD5h9N5Y0SF8azB2rulvHJieJoWqroGO5hi3ax5+8= +go.opentelemetry.io/collector/extension/extensiontest v0.117.0 h1:XH+tkHdATylYZtASZKK3rCoN/xlaFi8MXLh07ZlQQWw= +go.opentelemetry.io/collector/extension/extensiontest v0.117.0/go.mod h1:ABqB9D41p4MCeGVmABOgJi7i7roWZlFbqeFJDy7lskQ= +go.opentelemetry.io/collector/pdata v1.23.0 h1:tEk0dkfB8RdSukoOMfEa8duB938gfZowdfRkrJxGDrw= +go.opentelemetry.io/collector/pdata v1.23.0/go.mod h1:I2jggpBMiO8A+7TXhzNpcJZkJtvi1cU0iVNIi+6bc+o= +go.opentelemetry.io/collector/pdata/pprofile v0.117.0 h1:AyOK+rkNGeawmLGUqF84wYks22BSGJtEV++3YSfvD1I= +go.opentelemetry.io/collector/pdata/pprofile v0.117.0/go.mod h1:eh7TLIkLrSI79/R3RL+sZsKpLS0k+83WntucPtXC5Ak= +go.opentelemetry.io/collector/semconv v0.117.0 h1:SavOvSbHPVD/QdAnXlI/cMca+yxCNyXStY1mQzerHs4= +go.opentelemetry.io/collector/semconv v0.117.0/go.mod h1:N6XE8Q0JKgBN2fAhkUQtqK9LT7rEGR6+Wu/Rtbal1iI= +go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= +go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= +go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= +go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= +go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= +go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= +go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= +go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= +go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= +go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 h1:XVhgTWWV3kGQlwJHR3upFWZeTsei6Oks1apkZSeonIE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= +google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU= +google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 1508956cf1d95f06e2b4709991c9abfdb81c75ca Mon Sep 17 00:00:00 2001 From: Constanca Manteigas Date: Wed, 15 Jan 2025 14:25:50 +0100 Subject: [PATCH 6/9] fix module name --- extension/encoding/cloudwatchencoding/doc.go | 5 ----- .../Makefile | 0 .../config.go | 2 +- .../config_test.go | 2 +- extension/encoding/cloudwatchencodingextension/doc.go | 5 +++++ .../extension.go | 2 +- .../extension_test.go | 2 +- .../factory.go | 2 +- .../factory_test.go | 2 +- .../generated_component_test.go | 2 +- .../generated_package_test.go | 5 ++--- .../go.mod | 0 .../go.sum | 0 .../internal/metadata/generated_status.go | 0 .../metadata.yaml | 0 .../testdata/config.yaml | 0 16 files changed, 14 insertions(+), 15 deletions(-) delete mode 100644 extension/encoding/cloudwatchencoding/doc.go rename extension/encoding/{cloudwatchencoding => cloudwatchencodingextension}/Makefile (100%) rename extension/encoding/{cloudwatchencoding => cloudwatchencodingextension}/config.go (78%) rename extension/encoding/{cloudwatchencoding => cloudwatchencodingextension}/config_test.go (97%) create mode 100644 extension/encoding/cloudwatchencodingextension/doc.go rename extension/encoding/{cloudwatchencoding => cloudwatchencodingextension}/extension.go (92%) rename extension/encoding/{cloudwatchencoding => cloudwatchencodingextension}/extension_test.go (92%) rename extension/encoding/{cloudwatchencoding => cloudwatchencodingextension}/factory.go (73%) rename extension/encoding/{cloudwatchencoding => cloudwatchencodingextension}/factory_test.go (92%) rename extension/encoding/{cloudwatchencoding => cloudwatchencodingextension}/generated_component_test.go (97%) rename extension/encoding/{cloudwatchencoding => cloudwatchencodingextension}/generated_package_test.go (80%) rename extension/encoding/{cloudwatchencoding => cloudwatchencodingextension}/go.mod (100%) rename extension/encoding/{cloudwatchencoding => cloudwatchencodingextension}/go.sum (100%) rename extension/encoding/{cloudwatchencoding => cloudwatchencodingextension}/internal/metadata/generated_status.go (100%) rename extension/encoding/{cloudwatchencoding => cloudwatchencodingextension}/metadata.yaml (100%) rename extension/encoding/{cloudwatchencoding => cloudwatchencodingextension}/testdata/config.yaml (100%) diff --git a/extension/encoding/cloudwatchencoding/doc.go b/extension/encoding/cloudwatchencoding/doc.go deleted file mode 100644 index 86b00ea0d448..000000000000 --- a/extension/encoding/cloudwatchencoding/doc.go +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -//go:generate mdatagen metadata.yaml -package cloudwatchencoding // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" diff --git a/extension/encoding/cloudwatchencoding/Makefile b/extension/encoding/cloudwatchencodingextension/Makefile similarity index 100% rename from extension/encoding/cloudwatchencoding/Makefile rename to extension/encoding/cloudwatchencodingextension/Makefile diff --git a/extension/encoding/cloudwatchencoding/config.go b/extension/encoding/cloudwatchencodingextension/config.go similarity index 78% rename from extension/encoding/cloudwatchencoding/config.go rename to extension/encoding/cloudwatchencodingextension/config.go index b905f7368c33..51b98376eaad 100644 --- a/extension/encoding/cloudwatchencoding/config.go +++ b/extension/encoding/cloudwatchencodingextension/config.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package cloudwatchencoding // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" +package cloudwatchencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" import ( "fmt" diff --git a/extension/encoding/cloudwatchencoding/config_test.go b/extension/encoding/cloudwatchencodingextension/config_test.go similarity index 97% rename from extension/encoding/cloudwatchencoding/config_test.go rename to extension/encoding/cloudwatchencodingextension/config_test.go index dc7247b1e20a..4c01fe473ed2 100644 --- a/extension/encoding/cloudwatchencoding/config_test.go +++ b/extension/encoding/cloudwatchencodingextension/config_test.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package cloudwatchencoding +package cloudwatchencodingextension import ( "errors" diff --git a/extension/encoding/cloudwatchencodingextension/doc.go b/extension/encoding/cloudwatchencodingextension/doc.go new file mode 100644 index 000000000000..26f7685e8355 --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/doc.go @@ -0,0 +1,5 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//go:generate mdatagen metadata.yaml +package cloudwatchencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" diff --git a/extension/encoding/cloudwatchencoding/extension.go b/extension/encoding/cloudwatchencodingextension/extension.go similarity index 92% rename from extension/encoding/cloudwatchencoding/extension.go rename to extension/encoding/cloudwatchencodingextension/extension.go index e050ad472e9f..53ad1708eafe 100644 --- a/extension/encoding/cloudwatchencoding/extension.go +++ b/extension/encoding/cloudwatchencodingextension/extension.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package cloudwatchencoding // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" +package cloudwatchencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" import ( "bytes" diff --git a/extension/encoding/cloudwatchencoding/extension_test.go b/extension/encoding/cloudwatchencodingextension/extension_test.go similarity index 92% rename from extension/encoding/cloudwatchencoding/extension_test.go rename to extension/encoding/cloudwatchencodingextension/extension_test.go index cc0bc80fb6b3..a3e43191c0ab 100644 --- a/extension/encoding/cloudwatchencoding/extension_test.go +++ b/extension/encoding/cloudwatchencodingextension/extension_test.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package cloudwatchencoding +package cloudwatchencodingextension import ( "context" diff --git a/extension/encoding/cloudwatchencoding/factory.go b/extension/encoding/cloudwatchencodingextension/factory.go similarity index 73% rename from extension/encoding/cloudwatchencoding/factory.go rename to extension/encoding/cloudwatchencodingextension/factory.go index d903b52fa4e4..7a2dc855dace 100644 --- a/extension/encoding/cloudwatchencoding/factory.go +++ b/extension/encoding/cloudwatchencodingextension/factory.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package cloudwatchencoding // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" +package cloudwatchencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension" import ( "go.opentelemetry.io/collector/extension" diff --git a/extension/encoding/cloudwatchencoding/factory_test.go b/extension/encoding/cloudwatchencodingextension/factory_test.go similarity index 92% rename from extension/encoding/cloudwatchencoding/factory_test.go rename to extension/encoding/cloudwatchencodingextension/factory_test.go index 961c5d9a8b92..e7c8a1af3d47 100644 --- a/extension/encoding/cloudwatchencoding/factory_test.go +++ b/extension/encoding/cloudwatchencodingextension/factory_test.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package cloudwatchencoding +package cloudwatchencodingextension import ( "context" diff --git a/extension/encoding/cloudwatchencoding/generated_component_test.go b/extension/encoding/cloudwatchencodingextension/generated_component_test.go similarity index 97% rename from extension/encoding/cloudwatchencoding/generated_component_test.go rename to extension/encoding/cloudwatchencodingextension/generated_component_test.go index 7433cbe3ea71..ca9003d4db78 100644 --- a/extension/encoding/cloudwatchencoding/generated_component_test.go +++ b/extension/encoding/cloudwatchencodingextension/generated_component_test.go @@ -1,6 +1,6 @@ // Code generated by mdatagen. DO NOT EDIT. -package cloudwatchencoding +package cloudwatchencodingextension import ( "context" diff --git a/extension/encoding/cloudwatchencoding/generated_package_test.go b/extension/encoding/cloudwatchencodingextension/generated_package_test.go similarity index 80% rename from extension/encoding/cloudwatchencoding/generated_package_test.go rename to extension/encoding/cloudwatchencodingextension/generated_package_test.go index a0fdb5d7509e..6a98f31ff6c3 100644 --- a/extension/encoding/cloudwatchencoding/generated_package_test.go +++ b/extension/encoding/cloudwatchencodingextension/generated_package_test.go @@ -1,11 +1,10 @@ // Code generated by mdatagen. DO NOT EDIT. -package cloudwatchencoding +package cloudwatchencodingextension import ( - "testing" - "go.uber.org/goleak" + "testing" ) func TestMain(m *testing.M) { diff --git a/extension/encoding/cloudwatchencoding/go.mod b/extension/encoding/cloudwatchencodingextension/go.mod similarity index 100% rename from extension/encoding/cloudwatchencoding/go.mod rename to extension/encoding/cloudwatchencodingextension/go.mod diff --git a/extension/encoding/cloudwatchencoding/go.sum b/extension/encoding/cloudwatchencodingextension/go.sum similarity index 100% rename from extension/encoding/cloudwatchencoding/go.sum rename to extension/encoding/cloudwatchencodingextension/go.sum diff --git a/extension/encoding/cloudwatchencoding/internal/metadata/generated_status.go b/extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go similarity index 100% rename from extension/encoding/cloudwatchencoding/internal/metadata/generated_status.go rename to extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go diff --git a/extension/encoding/cloudwatchencoding/metadata.yaml b/extension/encoding/cloudwatchencodingextension/metadata.yaml similarity index 100% rename from extension/encoding/cloudwatchencoding/metadata.yaml rename to extension/encoding/cloudwatchencodingextension/metadata.yaml diff --git a/extension/encoding/cloudwatchencoding/testdata/config.yaml b/extension/encoding/cloudwatchencodingextension/testdata/config.yaml similarity index 100% rename from extension/encoding/cloudwatchencoding/testdata/config.yaml rename to extension/encoding/cloudwatchencodingextension/testdata/config.yaml From b602a6fdcf9d5c69b6e040e821d47c0e550807c9 Mon Sep 17 00:00:00 2001 From: Constanca Manteigas Date: Wed, 15 Jan 2025 16:23:17 +0100 Subject: [PATCH 7/9] add README file --- .../cloudwatchencodingextension/README.md | 22 +++++++++++++++++++ .../internal/metadata/generated_status.go | 2 +- .../cloudwatchencodingextension/metadata.yaml | 2 +- 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 extension/encoding/cloudwatchencodingextension/README.md diff --git a/extension/encoding/cloudwatchencodingextension/README.md b/extension/encoding/cloudwatchencodingextension/README.md new file mode 100644 index 000000000000..4e1bae80a1f2 --- /dev/null +++ b/extension/encoding/cloudwatchencodingextension/README.md @@ -0,0 +1,22 @@ +# Cloudwatch encoding extension + + +| Status | | +| ------------- |-----------| +| Stability | [development] | +| Distributions | [] | +| Issues | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Aextension%2Fcloudwatchencoding%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Aextension%2Fcloudwatchencoding) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Aextension%2Fcloudwatchencoding%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Aextension%2Fcloudwatchencoding) | +| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@constanca-m](https://www.github.com/constanca-m), [@axw](https://www.github.com/axw) | + +[development]: https://github.com/open-telemetry/opentelemetry-collector#development + + +The `cloudwatch_encoding` extension is used to unmarshal: +- cloudwatch logs into `plogs.Log`, if used in a logs receiver +- cloudwatch metrics into `pmetrics.Metric`, if used in a metrics receiver + +Example: +```yaml +extensions: + cloudwatch_encoding: +``` \ No newline at end of file diff --git a/extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go b/extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go index d64ea26b7694..067c5f2b0ffb 100644 --- a/extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go +++ b/extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go @@ -12,5 +12,5 @@ var ( ) const ( - ExtensionStability = component.StabilityLevelAlpha + ExtensionStability = component.StabilityLevelDevelopment ) diff --git a/extension/encoding/cloudwatchencodingextension/metadata.yaml b/extension/encoding/cloudwatchencodingextension/metadata.yaml index b51f39963317..ae5b79ace360 100644 --- a/extension/encoding/cloudwatchencodingextension/metadata.yaml +++ b/extension/encoding/cloudwatchencodingextension/metadata.yaml @@ -3,7 +3,7 @@ type: cloudwatch_encoding status: class: extension stability: - alpha: [extension] + development: [extension] distributions: [] codeowners: active: [constanca-m, axw] From 1acd7a54451b1efcae113a417ddc9ab4aa58efc2 Mon Sep 17 00:00:00 2001 From: Constanca Manteigas Date: Thu, 16 Jan 2025 08:25:04 +0100 Subject: [PATCH 8/9] Add changelog, run make commands --- .chloggen/add-cloudwatch-encoding.yaml | 27 +++++++++++++++++++ cmd/otelcontribcol/builder-config.yaml | 3 +++ .../cloudwatchencodingextension/README.md | 7 ++--- .../generated_package_test.go | 3 ++- .../cloudwatchencodingextension/go.mod | 7 ++--- .../cloudwatchencodingextension/go.sum | 12 --------- .../internal/metadata/generated_status.go | 2 +- .../cloudwatchencodingextension/metadata.yaml | 4 +-- pkg/translator/cloudwatch/go.mod | 1 - pkg/translator/cloudwatch/go.sum | 8 ------ versions.yaml | 2 ++ 11 files changed, 43 insertions(+), 33 deletions(-) create mode 100644 .chloggen/add-cloudwatch-encoding.yaml diff --git a/.chloggen/add-cloudwatch-encoding.yaml b/.chloggen/add-cloudwatch-encoding.yaml new file mode 100644 index 000000000000..957a0026bb86 --- /dev/null +++ b/.chloggen/add-cloudwatch-encoding.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: new_component + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: cloudwatch_encoding + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add a cloudwatch encoding extension, that translates cloudwatch logs and metrics to plog.Logs and pmetric.Metrics, respectively. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [37222] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/cmd/otelcontribcol/builder-config.yaml b/cmd/otelcontribcol/builder-config.yaml index bff9f514f030..19145674429d 100644 --- a/cmd/otelcontribcol/builder-config.yaml +++ b/cmd/otelcontribcol/builder-config.yaml @@ -45,6 +45,7 @@ extensions: - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/otlpencodingextension v0.117.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/jaegerencodingextension v0.117.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/avrologencodingextension v0.117.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension v0.117.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/jsonlogencodingextension v0.117.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/textencodingextension v0.117.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/zipkinencodingextension v0.117.0 @@ -378,6 +379,7 @@ replaces: - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/zipkinexporter => ../../exporter/zipkinexporter - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver => ../../receiver/hostmetricsreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/signalfx => ../../pkg/translator/signalfx + - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch => ../../pkg/translator/cloudwatch - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/signalfxreceiver => ../../receiver/signalfxreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/windowsperfcountersreceiver => ../../receiver/windowsperfcountersreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/simpleprometheusreceiver => ../../receiver/simpleprometheusreceiver @@ -481,6 +483,7 @@ replaces: - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/otlpencodingextension => ../../extension/encoding/otlpencodingextension - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/zipkinencodingextension => ../../extension/encoding/zipkinencodingextension - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/avrologencodingextension => ../../extension/encoding/avrologencodingextension + - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension => ../../extension/encoding/cloudwatchencodingextension - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/jsonlogencodingextension => ../../extension/encoding/jsonlogencodingextension - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/textencodingextension => ../../extension/encoding/textencodingextension - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/jaegerencodingextension => ../../extension/encoding/jaegerencodingextension diff --git a/extension/encoding/cloudwatchencodingextension/README.md b/extension/encoding/cloudwatchencodingextension/README.md index 4e1bae80a1f2..2076e7b7197a 100644 --- a/extension/encoding/cloudwatchencodingextension/README.md +++ b/extension/encoding/cloudwatchencodingextension/README.md @@ -3,12 +3,13 @@ | Status | | | ------------- |-----------| -| Stability | [development] | -| Distributions | [] | +| Stability | [alpha] | +| Distributions | [contrib] | | Issues | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Aextension%2Fcloudwatchencoding%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Aextension%2Fcloudwatchencoding) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Aextension%2Fcloudwatchencoding%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Aextension%2Fcloudwatchencoding) | | [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@constanca-m](https://www.github.com/constanca-m), [@axw](https://www.github.com/axw) | -[development]: https://github.com/open-telemetry/opentelemetry-collector#development +[alpha]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#alpha +[contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib The `cloudwatch_encoding` extension is used to unmarshal: diff --git a/extension/encoding/cloudwatchencodingextension/generated_package_test.go b/extension/encoding/cloudwatchencodingextension/generated_package_test.go index 6a98f31ff6c3..ec2b41ec9e52 100644 --- a/extension/encoding/cloudwatchencodingextension/generated_package_test.go +++ b/extension/encoding/cloudwatchencodingextension/generated_package_test.go @@ -3,8 +3,9 @@ package cloudwatchencodingextension import ( - "go.uber.org/goleak" "testing" + + "go.uber.org/goleak" ) func TestMain(m *testing.M) { diff --git a/extension/encoding/cloudwatchencodingextension/go.mod b/extension/encoding/cloudwatchencodingextension/go.mod index cb29ad9a740b..43eaf28362d4 100644 --- a/extension/encoding/cloudwatchencodingextension/go.mod +++ b/extension/encoding/cloudwatchencodingextension/go.mod @@ -4,17 +4,14 @@ go 1.23 require ( github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding v0.117.0 - github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch v0.0.0-00010101000000-000000000000 github.com/stretchr/testify v1.10.0 - github.com/tidwall/gjson v1.18.0 go.opentelemetry.io/collector/component v0.117.0 go.opentelemetry.io/collector/component/componenttest v0.117.0 go.opentelemetry.io/collector/confmap v1.23.0 go.opentelemetry.io/collector/extension v0.117.0 go.opentelemetry.io/collector/extension/extensiontest v0.117.0 go.opentelemetry.io/collector/pdata v1.23.0 - go.opentelemetry.io/collector/semconv v0.117.0 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 ) @@ -36,12 +33,12 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.117.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.117.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.117.0 // indirect go.opentelemetry.io/collector/pdata/pprofile v0.117.0 // indirect + go.opentelemetry.io/collector/semconv v0.117.0 // indirect go.opentelemetry.io/otel v1.32.0 // indirect go.opentelemetry.io/otel/metric v1.32.0 // indirect go.opentelemetry.io/otel/sdk v1.32.0 // indirect diff --git a/extension/encoding/cloudwatchencodingextension/go.sum b/extension/encoding/cloudwatchencodingextension/go.sum index 753c697d6419..8bac2895609a 100644 --- a/extension/encoding/cloudwatchencodingextension/go.sum +++ b/extension/encoding/cloudwatchencodingextension/go.sum @@ -43,18 +43,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding v0.117.0 h1:te8DpwzFocnxrfenTK3dLr8MQ3E8r3dRa1ZXfd/scss= -github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding v0.117.0/go.mod h1:xCGKcgGqkH3TmLj3FPtHa7YgmPgYK6FdJaqVYfrUE3c= -github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.117.0 h1:6BF8ymRgg1duNHXDpqgDayDlR+YktfQGQN1vzdxGcxQ= -github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.117.0/go.mod h1:T5aTYEzGf74ZagtJiruJC5idVLGKr78q/zqDzN4gPQQ= -github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0 h1:SjZHtgph6f675yK/p+GTo8I7VlIAjJFod3SMNgfpEAg= -github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0/go.mod h1:e4cIzlw2iGmMCgHKyJHLf+PF99A1Y6GvNC0+E3x1dA4= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.117.0 h1:tOJFUIZaAU4zm5CilqZN1/AuKQa7diTrcEhgQIYly6k= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.117.0/go.mod h1:PJ2FGCS+Hw+tlHUNNWVHNo3IXtEsb9RKgl/ssSi3Z98= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.117.0 h1:/wMNk8w1UEHKpKoNk1jA2aifHgfGZE+WelGNrCf0CJ0= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.117.0/go.mod h1:ESyMNHmgZYh8Ouhr2veecTMK6sB8gQ8u2s3dsy9Og6k= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.117.0 h1:GqlhXd6J8zgxCYenbI3ew03SJnGec1vEEGzGHw9X/Y0= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.117.0/go.mod h1:OGylX+Bp+urSNNGoI1XG7U6vaRDZk1wN/w6fHP1F7IY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go b/extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go index 067c5f2b0ffb..d64ea26b7694 100644 --- a/extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go +++ b/extension/encoding/cloudwatchencodingextension/internal/metadata/generated_status.go @@ -12,5 +12,5 @@ var ( ) const ( - ExtensionStability = component.StabilityLevelDevelopment + ExtensionStability = component.StabilityLevelAlpha ) diff --git a/extension/encoding/cloudwatchencodingextension/metadata.yaml b/extension/encoding/cloudwatchencodingextension/metadata.yaml index ae5b79ace360..f3e46a6b6eea 100644 --- a/extension/encoding/cloudwatchencodingextension/metadata.yaml +++ b/extension/encoding/cloudwatchencodingextension/metadata.yaml @@ -3,7 +3,7 @@ type: cloudwatch_encoding status: class: extension stability: - development: [extension] - distributions: [] + alpha: [extension] + distributions: [contrib] codeowners: active: [constanca-m, axw] diff --git a/pkg/translator/cloudwatch/go.mod b/pkg/translator/cloudwatch/go.mod index 1dff39702c63..e5ce785ab377 100644 --- a/pkg/translator/cloudwatch/go.mod +++ b/pkg/translator/cloudwatch/go.mod @@ -10,7 +10,6 @@ require ( go.opentelemetry.io/collector/pdata v1.23.0 go.opentelemetry.io/collector/semconv v0.117.0 go.uber.org/goleak v1.3.0 - go.uber.org/zap v1.27.0 ) require ( diff --git a/pkg/translator/cloudwatch/go.sum b/pkg/translator/cloudwatch/go.sum index 9504d4150cb1..b2c2b3890714 100644 --- a/pkg/translator/cloudwatch/go.sum +++ b/pkg/translator/cloudwatch/go.sum @@ -29,12 +29,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0 h1:SjZHtgph6f675yK/p+GTo8I7VlIAjJFod3SMNgfpEAg= -github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.117.0/go.mod h1:e4cIzlw2iGmMCgHKyJHLf+PF99A1Y6GvNC0+E3x1dA4= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.117.0 h1:tOJFUIZaAU4zm5CilqZN1/AuKQa7diTrcEhgQIYly6k= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.117.0/go.mod h1:PJ2FGCS+Hw+tlHUNNWVHNo3IXtEsb9RKgl/ssSi3Z98= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.117.0 h1:/wMNk8w1UEHKpKoNk1jA2aifHgfGZE+WelGNrCf0CJ0= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.117.0/go.mod h1:ESyMNHmgZYh8Ouhr2veecTMK6sB8gQ8u2s3dsy9Og6k= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -69,8 +63,6 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/versions.yaml b/versions.yaml index 1a3501ca6367..d76c5e33d1ea 100644 --- a/versions.yaml +++ b/versions.yaml @@ -82,6 +82,7 @@ module-sets: - github.com/open-telemetry/opentelemetry-collector-contrib/extension/cgroupruntimeextension - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/avrologencodingextension + - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/cloudwatchencodingextension - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/jaegerencodingextension - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/jsonlogencodingextension - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/otlpencodingextension @@ -164,6 +165,7 @@ module-sets: - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/signalfx - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/skywalking - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/zipkin + - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/cloudwatch - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/winperfcounters - github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor - github.com/open-telemetry/opentelemetry-collector-contrib/processor/cumulativetodeltaprocessor From 8977a5424397169449a2ede37acc12d052c3d6be Mon Sep 17 00:00:00 2001 From: Constanca Manteigas Date: Thu, 16 Jan 2025 08:28:51 +0100 Subject: [PATCH 9/9] Remove extra comment --- pkg/translator/cloudwatch/cwlogs_to_logs_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/translator/cloudwatch/cwlogs_to_logs_test.go b/pkg/translator/cloudwatch/cwlogs_to_logs_test.go index 232879e8c434..6f979cd44a8a 100644 --- a/pkg/translator/cloudwatch/cwlogs_to_logs_test.go +++ b/pkg/translator/cloudwatch/cwlogs_to_logs_test.go @@ -74,9 +74,6 @@ func TestUnmarshalLogs(t *testing.T) { require.NoError(t, err) expected, err := unmarshaller.UnmarshalLogs(content) require.NoError(t, err) - - // get log records - // expectedLogs := expected.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords() require.Equal(t, expected, result) }) }