diff --git a/.chloggen/remove_time_from_container_attributes.yaml b/.chloggen/remove_time_from_container_attributes.yaml new file mode 100644 index 000000000000..f4b47d14361b --- /dev/null +++ b/.chloggen/remove_time_from_container_attributes.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: 'enhancement' + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: receiver/filelog + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add filelog.container.removeOriginalTimeField feature-flag for removing original time field + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33946] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/pkg/stanza/docs/operators/container.md b/pkg/stanza/docs/operators/container.md index 8655b516282a..9935c04b3ddc 100644 --- a/pkg/stanza/docs/operators/container.md +++ b/pkg/stanza/docs/operators/container.md @@ -253,6 +253,7 @@ Configuration: + #### Parse multiline logs and recombine into a single one If you are using the Docker format (or log tag indicators are not working), @@ -319,3 +320,10 @@ receivers: + +### Removing original time field + +In order to remove the original time field from the log records users can enable the +`filelog.container.removeOriginalTimeField` feature gate. +The feature gate `filelog.container.removeOriginalTimeField` will be deprecated and eventually removed +in the future, following the [feature lifecycle](https://github.com/open-telemetry/opentelemetry-collector/tree/main/featuregate#feature-lifecycle). diff --git a/pkg/stanza/operator/parser/container/config.go b/pkg/stanza/operator/parser/container/config.go index fcd6eeeba38b..f1cab89e3024 100644 --- a/pkg/stanza/operator/parser/container/config.go +++ b/pkg/stanza/operator/parser/container/config.go @@ -8,6 +8,8 @@ import ( "sync" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/featuregate" + "go.uber.org/zap" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/entry" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/errors" @@ -17,9 +19,17 @@ import ( ) const ( - operatorType = "container" - recombineSourceIdentifier = "log.file.path" - recombineIsLastEntry = "attributes.logtag == 'F'" + operatorType = "container" + recombineSourceIdentifier = "log.file.path" + recombineIsLastEntry = "attributes.logtag == 'F'" + removeOriginalTimeFieldFeatureFlag = "filelog.container.removeOriginalTimeField" +) + +var removeOriginalTimeField = featuregate.GlobalRegistry().MustRegister( + removeOriginalTimeFieldFeatureFlag, + featuregate.StageAlpha, + featuregate.WithRegisterDescription("When enabled, deletes the original `time` field from the Log Attributes. Time is parsed to Timestamp field, which should be used instead."), + featuregate.WithRegisterReferenceURL("https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/33389"), ) func init() { @@ -77,6 +87,14 @@ func (c Config) Build(set component.TelemetrySettings) (operator.Operator, error } } + if !removeOriginalTimeField.IsEnabled() { + // https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/33389 + set.Logger.Info("`time` log record attribute will be removed in a future release. Switch now using the feature gate.", + zap.String("attribute", "time"), + zap.String("feature gate", removeOriginalTimeFieldFeatureFlag), + ) + } + p := &Parser{ ParserOperator: parserOperator, recombineParser: recombineParser, diff --git a/pkg/stanza/operator/parser/container/parser.go b/pkg/stanza/operator/parser/container/parser.go index af640c4a5d34..8b5bf3c8f345 100644 --- a/pkg/stanza/operator/parser/container/parser.go +++ b/pkg/stanza/operator/parser/container/parser.go @@ -356,5 +356,10 @@ func parseTime(e *entry.Entry, layout string) error { } // timeutils.ParseGotime calls timeutils.SetTimestampYear before returning the timeValue e.Timestamp = timeValue + + if removeOriginalTimeField.IsEnabled() { + e.Delete(entry.NewAttributeField(parseFrom)) + } + return nil } diff --git a/pkg/stanza/operator/parser/container/parser_test.go b/pkg/stanza/operator/parser/container/parser_test.go index 924f83546f4f..3ca78c314a91 100644 --- a/pkg/stanza/operator/parser/container/parser_test.go +++ b/pkg/stanza/operator/parser/container/parser_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/featuregate" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/entry" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator" @@ -378,3 +379,101 @@ func TestRecombineProcess(t *testing.T) { }) } } + +func TestProcessWithTimeRemovalFlag(t *testing.T) { + + require.NoError(t, featuregate.GlobalRegistry().Set(removeOriginalTimeField.ID(), true)) + t.Cleanup(func() { + require.NoError(t, featuregate.GlobalRegistry().Set(removeOriginalTimeField.ID(), false)) + }) + + cases := []struct { + name string + op func() (operator.Operator, error) + input *entry.Entry + expect *entry.Entry + }{ + { + "docker", + func() (operator.Operator, error) { + cfg := NewConfigWithID("test_id") + cfg.AddMetadataFromFilePath = false + cfg.Format = "docker" + set := componenttest.NewNopTelemetrySettings() + return cfg.Build(set) + }, + &entry.Entry{ + Body: `{"log":"INFO: log line here","stream":"stdout","time":"2029-03-30T08:31:20.545192187Z"}`, + }, + &entry.Entry{ + Attributes: map[string]any{ + "log.iostream": "stdout", + }, + Body: "INFO: log line here", + Timestamp: time.Date(2029, time.March, 30, 8, 31, 20, 545192187, time.UTC), + }, + }, + { + "docker_with_auto_detection", + func() (operator.Operator, error) { + cfg := NewConfigWithID("test_id") + cfg.AddMetadataFromFilePath = false + set := componenttest.NewNopTelemetrySettings() + return cfg.Build(set) + }, + &entry.Entry{ + Body: `{"log":"INFO: log line here","stream":"stdout","time":"2029-03-30T08:31:20.545192187Z"}`, + }, + &entry.Entry{ + Attributes: map[string]any{ + "log.iostream": "stdout", + }, + Body: "INFO: log line here", + Timestamp: time.Date(2029, time.March, 30, 8, 31, 20, 545192187, time.UTC), + }, + }, + { + "docker_with_auto_detection_and_metadata_from_file_path", + func() (operator.Operator, error) { + cfg := NewConfigWithID("test_id") + cfg.AddMetadataFromFilePath = true + set := componenttest.NewNopTelemetrySettings() + return cfg.Build(set) + }, + &entry.Entry{ + Body: `{"log":"INFO: log line here","stream":"stdout","time":"2029-03-30T08:31:20.545192187Z"}`, + Attributes: map[string]any{ + "log.file.path": "/var/log/pods/some_kube-scheduler-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d3/kube-scheduler44/1.log", + }, + }, + &entry.Entry{ + Attributes: map[string]any{ + "log.iostream": "stdout", + "log.file.path": "/var/log/pods/some_kube-scheduler-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d3/kube-scheduler44/1.log", + }, + Body: "INFO: log line here", + Resource: map[string]any{ + "k8s.pod.name": "kube-scheduler-kind-control-plane", + "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d3", + "k8s.container.name": "kube-scheduler44", + "k8s.container.restart_count": "1", + "k8s.namespace.name": "some", + }, + Timestamp: time.Date(2029, time.March, 30, 8, 31, 20, 545192187, time.UTC), + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + op, err := tc.op() + require.NoError(t, err, "did not expect operator function to return an error, this is a bug with the test case") + + err = op.Process(context.Background(), tc.input) + require.NoError(t, err) + require.Equal(t, tc.expect, tc.input) + // Stop the operator + require.NoError(t, op.Stop()) + }) + } +}