From f4f3527cbe4b0d15444281051fe8df01557343e9 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 4 Aug 2020 11:38:36 +0200 Subject: [PATCH] dataset -> datastream from fleet and rules --- .../_meta/config/common.p2.yml.tmpl | 10 +- .../_meta/config/common.reference.p2.yml.tmpl | 10 +- .../config/elastic-agent.docker.yml.tmpl | 10 +- x-pack/elastic-agent/_meta/elastic-agent.yml | 10 +- .../docs/elastic-agent-configuration.asciidoc | 10 +- .../elastic-agent_configuration_example.yml | 162 +++++++++--------- x-pack/elastic-agent/elastic-agent.docker.yml | 10 +- .../elastic-agent/elastic-agent.reference.yml | 10 +- x-pack/elastic-agent/elastic-agent.yml | 10 +- .../application/filters/stream_checker.go | 26 +-- .../filters/stream_checker_test.go | 76 ++++---- .../agent/application/managed_mode_test.go | 38 ++-- .../pkg/agent/program/supported.go | 2 +- .../program/testdata/constraints_config.yml | 2 +- .../endpoint_basic-endpoint-security.yml | 2 +- .../agent/program/testdata/endpoint_basic.yml | 2 +- .../program/testdata/endpoint_no_fleet.yml | 2 +- .../testdata/endpoint_unknown_output.yml | 2 +- .../single_config-endpoint-security.yml | 2 +- .../agent/program/testdata/single_config.yml | 14 +- .../pkg/agent/transpiler/rules.go | 68 ++++---- .../pkg/agent/transpiler/rules_test.go | 56 +++--- x-pack/elastic-agent/spec/filebeat.yml | 6 +- x-pack/elastic-agent/spec/metricbeat.yml | 6 +- 24 files changed, 273 insertions(+), 273 deletions(-) diff --git a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl index c81236e8844..387192d60d9 100644 --- a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl @@ -17,7 +17,7 @@ inputs: # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # # Cannot start with -, _, + # Cannot be . or .. - dataset.namespace: default + datastream.namespace: default use_output: default streams: - metricset: cpu @@ -27,13 +27,13 @@ inputs: # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # # Cannot start with -, _, + # Cannot be . or .. - dataset.name: system.cpu + datastream.dataset: system.cpu - metricset: memory - dataset.name: system.memory + datastream.dataset: system.memory - metricset: network - dataset.name: system.network + datastream.dataset: system.network - metricset: filesystem - dataset.name: system.filesystem + datastream.dataset: system.filesystem # agent.monitoring: # # enabled turns on monitoring of running processes diff --git a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl index e0b1b26dfd9..65746e60cb5 100644 --- a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl @@ -17,7 +17,7 @@ inputs: # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # # Cannot start with -, _, + # Cannot be . or .. - dataset.namespace: default + datastream.namespace: default use_output: default streams: - metricset: cpu @@ -27,13 +27,13 @@ inputs: # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # # Cannot start with -, _, + # Cannot be . or .. - dataset.name: system.cpu + datastream.dataset: system.cpu - metricset: memory - dataset.name: system.memory + datastream.dataset: system.memory - metricset: network - dataset.name: system.network + datastream.dataset: system.network - metricset: filesystem - dataset.name: system.filesystem + datastream.dataset: system.filesystem # management: # # Mode of management, the Elastic Agent support two modes of operation: diff --git a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl index 90e83056932..07059dc1964 100644 --- a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl @@ -17,7 +17,7 @@ inputs: # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # # Cannot start with -, _, + # Cannot be . or .. - dataset.namespace: default + datastream.namespace: default use_output: default streams: - metricset: cpu @@ -27,13 +27,13 @@ inputs: # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # # Cannot start with -, _, + # Cannot be . or .. - dataset.name: system.cpu + datastream.dataset: system.cpu - metricset: memory - dataset.name: system.memory + datastream.dataset: system.memory - metricset: network - dataset.name: system.network + datastream.dataset: system.network - metricset: filesystem - dataset.name: system.filesystem + datastream.dataset: system.filesystem # management: # # Mode of management, the Elastic Agent support two modes of operation: diff --git a/x-pack/elastic-agent/_meta/elastic-agent.yml b/x-pack/elastic-agent/_meta/elastic-agent.yml index b084f0661a1..2a956352334 100644 --- a/x-pack/elastic-agent/_meta/elastic-agent.yml +++ b/x-pack/elastic-agent/_meta/elastic-agent.yml @@ -17,7 +17,7 @@ inputs: # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # # Cannot start with -, _, + # Cannot be . or .. - dataset.namespace: default + datastream.namespace: default use_output: default streams: - metricset: cpu @@ -27,13 +27,13 @@ inputs: # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # # Cannot start with -, _, + # Cannot be . or .. - dataset.name: system.cpu + datastream.dataset: system.cpu - metricset: memory - dataset.name: system.memory + datastream.dataset: system.memory - metricset: network - dataset.name: system.network + datastream.dataset: system.network - metricset: filesystem - dataset.name: system.filesystem + datastream.dataset: system.filesystem # management: # # Mode of management, the Elastic Agent support two modes of operation: diff --git a/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc b/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc index 860b109c779..b62efdadf0c 100644 --- a/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc +++ b/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc @@ -93,17 +93,17 @@ filesystem metrics, and sends them to the default output. For example: ------------------------------------------------------------------------------------- inputs: - type: system/metrics - dataset.namespace: default + datastream.namespace: default use_output: default streams: - metricset: cpu - dataset.name: system.cpu + datastream.dataset: system.cpu - metricset: memory - dataset.name: system.memory + datastream.dataset: system.memory - metricset: network - dataset.name: system.network + datastream.dataset: system.network - metricset: filesystem - dataset.name: system.filesystem + datastream.dataset: system.filesystem ------------------------------------------------------------------------------------- If `use_output` is not specified, the `default` output is used. diff --git a/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml b/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml index 5bffc62d44d..db218922091 100644 --- a/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml +++ b/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml @@ -109,7 +109,7 @@ inputs: - type: logfile name: epm/nginx version: 1.7.0 - dataset.namespace: prod + datastream.namespace: prod # constraints are still Experimental and should not be used in production. constraints?: # Contraints look are not final @@ -120,16 +120,16 @@ inputs: streams: - id?: {id} enabled?: true # default to true - dataset.name: nginx.acccess + datastream.dataset: nginx.acccess paths: /var/log/nginx/access.log - id?: {id} enabled?: true # default to true - dataset.name: nginx.error + datastream.dataset: nginx.error paths: /var/log/nginx/error.log - type: nginx/metricspackage?: name: epm/nginx version: 1.7.0 - dataset.namespace: prod + datastream.namespace: prod # constraints are still Experimental and should not be used in production. constraints?: # Contraints look are not final @@ -139,7 +139,7 @@ inputs: streams: - id?: {id} enabled?: true # default to true - dataset.name: nginx.stub_status + datastream.dataset: nginx.stub_status metricset: stub_status ################################################################################################# @@ -148,11 +148,11 @@ inputs: - type: kafka id: kafka-x1 title: "Consume data from kafka" - dataset.namespace: prod + datastream.namespace: prod use_output: long_term_storage host: localhost:6566 streams: - - dataset.name: foo.dataset + - datastream.dataset: foo.dataset topic: foo processors: - extract_bro_specifics @@ -171,54 +171,54 @@ inputs: - id?: {id} enabled?: false # default true metricset: cpu - dataset.name: system.cpu + datastream.dataset: system.cpu metrics: ["percentages", "normalized_percentages"] period: 10s - metricset: memory - dataset.name: system.memory + datastream.dataset: system.memory period: 10s - metricset: diskio - dataset.name: system.diskio + datastream.dataset: system.diskio period: 10s - metricset: load - dataset.name: system.load + datastream.dataset: system.load period: 10s - metricset: memory - dataset.name: system.memory + datastream.dataset: system.memory period: 10s - metricset: process - dataset.name: system.process + datastream.dataset: system.process processes: ["firefox*"] include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory period: 10s - metricset: process_summary - dataset.name: system.process_summary + datastream.dataset: system.process_summary period: 10s - metricset: uptime - dataset.name: system.uptime + datastream.dataset: system.uptime period: 15m - metricset: socket_summary - dataset.name: system.socket_summary + datastream.dataset: system.socket_summary period: 10s - metricset: filesystem - dataset.name: system.filesystem + datastream.dataset: system.filesystem period: 10s - metricset: raid - dataset.name: system.raid + datastream.dataset: system.raid period: 10s - metricset: socket - dataset.name: system.socket + datastream.dataset: system.socket period: 10s - metricset: service - dataset.name: system.service + datastream.dataset: system.service period: 10s - metricset: fsstat - dataset.name: system.fsstat + datastream.dataset: system.fsstat period: 10s - metricset: foo - dataset.name: system.foo + datastream.dataset: system.foo period: 10s @@ -234,23 +234,23 @@ inputs: streams: - id?: {id} enabled?: true # default to true - dataset.name: elasticsearch.audit + datastream.dataset: elasticsearch.audit paths: [/var/log/elasticsearch/*_access.log, /var/log/elasticsearch/*_audit.log] - id?: {id} enabled?: true - dataset.name: elasticsearch.deprecation + datastream.dataset: elasticsearch.deprecation paths: [/var/log/elasticsearch/*_deprecation.log] - id?: {id} enabled?: true - dataset.name: elasticsearch.gc + datastream.dataset: elasticsearch.gc paths: [/var/log/elasticsearch/*_gc.log, /var/log/elasticsearch/*_gc.log.[0-9]*] - id?: {id} enabled?: true - dataset.name: elasticsearch.server + datastream.dataset: elasticsearch.server paths: [/var/log/elasticsearch/*.log] - id?: {id} enabled?: true - dataset.name: elasticsearch.slowlog + datastream.dataset: elasticsearch.slowlog paths: [/var/log/elasticsearch/*_index_search_slowlog.log, /var/log/elasticsearch/*_index_indexing_slowlog.log] - type: elasticsearch/metrics id?: my-endpoint @@ -271,36 +271,36 @@ inputs: streams: - id?: {id} metricset: ccr - dataset.name: elasticseach.ccr + datastream.dataset: elasticseach.ccr period: 10s - id?: {id} metricset: cluster_stats - dataset.name: elasticseach.cluster_stats + datastream.dataset: elasticseach.cluster_stats period: 10s - id?: {id} metricset: enrich - dataset.name: elasticseach.enrich + datastream.dataset: elasticseach.enrich period: 10s - id?: {id} metricset: index - dataset.name: elasticseach.index + datastream.dataset: elasticseach.index period: 10s - id?: {id} metricset: index_recovery - dataset.name: elasticseach.index_recovery + datastream.dataset: elasticseach.index_recovery active_only: true period: 10s - id?: {id} metricset: ml_jobs - dataset.name: elasticseach.ml_jobs + datastream.dataset: elasticseach.ml_jobs period: 10s - id?: {id} metricset: node_stats - dataset.name: elasticseach.node_stats + datastream.dataset: elasticseach.node_stats period: 10s - id?: {id} metricset: shard - dataset.name: elasticseach.shard + datastream.dataset: elasticseach.shard period: 10s ################################################################################################# @@ -318,16 +318,16 @@ inputs: #shared_credential_file: /etc/filebeat/aws_credentials streams: - id?: {id} - dataset.name: aws.s3 + datastream.dataset: aws.s3 queue_url: https://sqs.myregion.amazonaws.com/123456/sqs-queue - id?: {id} - dataset.name: aws.s3access + datastream.dataset: aws.s3access queue_url: https://sqs.myregion.amazonaws.com/123456/sqs-queue - id?: {id} - dataset.name: aws.vpcflow + datastream.dataset: aws.vpcflow queue_url: https://sqs.myregion.amazonaws.com/123456/sqs-queue - id?: {id} - dataset.name: aws.cloudtrail + datastream.dataset: aws.cloudtrail queue_url: https://sqs.myregion.amazonaws.com/123456/sqs-queue - type: aws/metrics id?: my-aws @@ -343,11 +343,11 @@ inputs: streams: - id?: {id} metricset: usage - dataset.name: aws.usage + datastream.dataset: aws.usage period: 5m - id?: {id} metricset: cloudwatch - dataset.name: aws.cloudwatch + datastream.dataset: aws.cloudwatch period: 5m name: ["CPUUtilization", "DiskWriteOps"] resource_type: ec2:instance @@ -357,43 +357,43 @@ inputs: statistic: ["Average", "Maximum"] - id?: {id} metricset: ebs - dataset.name: aws.ebs + datastream.dataset: aws.ebs period: 5m - id?: {id} metricset: ec2 - dataset.name: aws.ec2 + datastream.dataset: aws.ec2 period: 5m - id?: {id} metricset: elb - dataset.name: aws.elb + datastream.dataset: aws.elb period: 5m - id?: {id} metricset: sns - dataset.name: aws.sns + datastream.dataset: aws.sns period: 5m - id?: {id} metricset: sqs - dataset.name: aws.sqs + datastream.dataset: aws.sqs period: 5m - id?: {id} metricset: rds - dataset.name: aws.rds + datastream.dataset: aws.rds period: 5m - id?: {id} metricset: billing - dataset.name: aws.billing + datastream.dataset: aws.billing period: 12h - id?: {id} metricset: billing - dataset.name: aws.billing + datastream.dataset: aws.billing period: 12h - id?: {id} metricset: s3_daily_storage - dataset.name: aws.s3_daily_storage + datastream.dataset: aws.s3_daily_storage period: 24h - id?: {id} metricset: s3_request - dataset.name: aws.s3_request + datastream.dataset: aws.s3_request period: 24h @@ -418,27 +418,27 @@ inputs: streams: - id?: {id} metricset: node - dataset.name: kubernetes.node + datastream.dataset: kubernetes.node period: 10s - id?: {id} metricset: system - dataset.name: kubernetes.system + datastream.dataset: kubernetes.system period: 10s - id?: {id} metricset: pod - dataset.name: kubernetes.pod + datastream.dataset: kubernetes.pod period: 10s - id?: {id} metricset: container - dataset.name: kubernetes.container + datastream.dataset: kubernetes.container period: 10s - id?: {id} metricset: container - dataset.name: kubernetes.container + datastream.dataset: kubernetes.container period: 10s - id?: {id} metricset: event - dataset.name: kubernetes.event + datastream.dataset: kubernetes.event period: 10s - type: kubernetes-state/metrics id?: my-kubernetes @@ -450,51 +450,51 @@ inputs: streams: - id?: {id} metricset: state_node - dataset.name: kubernetes.node + datastream.dataset: kubernetes.node period: 10s - id?: {id} metricset: state_deployment - dataset.name: kubernetes.deployment + datastream.dataset: kubernetes.deployment period: 10s - id?: {id} metricset: state_replicaset - dataset.name: kubernetes.replicaset + datastream.dataset: kubernetes.replicaset period: 10s - id?: {id} metricset: state_statefulset - dataset.name: kubernetes.statefulset + datastream.dataset: kubernetes.statefulset period: 10s - id?: {id} metricset: state_pod - dataset.name: kubernetes.pod + datastream.dataset: kubernetes.pod period: 10s - id?: {id} metricset: state_container - dataset.name: kubernetes.container + datastream.dataset: kubernetes.container period: 10s - id?: {id} metricset: state_container - dataset.name: kubernetes.container + datastream.dataset: kubernetes.container period: 10s - id?: {id} metricset: state_cronjob - dataset.name: kubernetes.cronjob + datastream.dataset: kubernetes.cronjob period: 10s - id?: {id} metricset: state_resourcequota - dataset.name: kubernetes.resourcequota + datastream.dataset: kubernetes.resourcequota period: 10s - id?: {id} metricset: state_service - dataset.name: kubernetes.service + datastream.dataset: kubernetes.service period: 10s - id?: {id} metricset: state_persistentvolume - dataset.name: kubernetes.persistentvolume + datastream.dataset: kubernetes.persistentvolume period: 10s - id?: {id} metricset: state_persistentvolumeclaim - dataset.name: kubernetes.persistentvolumeclaim + datastream.dataset: kubernetes.persistentvolumeclaim period: 10s ################################################################################################# @@ -517,35 +517,35 @@ inputs: streams: - id?: {id} metricset: container - dataset.name: docker.container + datastream.dataset: docker.container period: 10s - id?: {id} metricset: cpu - dataset.name: docker.cpu + datastream.dataset: docker.cpu period: 10s - id?: {id} metricset: diskio - dataset.name: docker.diskio + datastream.dataset: docker.diskio period: 10s - id?: {id} metricset: event - dataset.name: docker.event + datastream.dataset: docker.event period: 10s - id?: {id} metricset: healthcheck - dataset.name: docker.healthcheck + datastream.dataset: docker.healthcheck period: 10s - id?: {id} metricset: info - dataset.name: docker.info + datastream.dataset: docker.info period: 10s - id?: {id} metricset: memory - dataset.name: docker.memory + datastream.dataset: docker.memory period: 10s - id?: {id} metricset: network - dataset.name: docker.network + datastream.dataset: docker.network period: 10s ################################################################################################# @@ -555,14 +555,14 @@ inputs: - type: log id?: suricata-x1 title: Suricata's data - dataset.namespace: "abc" + datastream.namespace: "abc" package: name: suricata version: x.x.x streams: - id?: {id} type: "typeX" - dataset.name: suricata.logs + datastream.dataset: suricata.logs path: /var/log/surcata/eve.json ################################################################################################# @@ -571,7 +571,7 @@ inputs: - type: endpoint # Reserved key word id?: myendpoint-x1 title: Endpoint configuration - dataset.namespace: "canada" + datastream.namespace: "canada" package: name: endpoint version: xxx @@ -627,7 +627,7 @@ inputs: - type: endpoint # Reserved key word id?: myendpoint-1 title: Endpoint configuration - dataset.namespace: "canada" + datastream.namespace: "canada" package: name: epm/endpoint # This establish the link with the package and will allow to link it to endpoint app. version: xxx diff --git a/x-pack/elastic-agent/elastic-agent.docker.yml b/x-pack/elastic-agent/elastic-agent.docker.yml index 90e83056932..07059dc1964 100644 --- a/x-pack/elastic-agent/elastic-agent.docker.yml +++ b/x-pack/elastic-agent/elastic-agent.docker.yml @@ -17,7 +17,7 @@ inputs: # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # # Cannot start with -, _, + # Cannot be . or .. - dataset.namespace: default + datastream.namespace: default use_output: default streams: - metricset: cpu @@ -27,13 +27,13 @@ inputs: # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # # Cannot start with -, _, + # Cannot be . or .. - dataset.name: system.cpu + datastream.dataset: system.cpu - metricset: memory - dataset.name: system.memory + datastream.dataset: system.memory - metricset: network - dataset.name: system.network + datastream.dataset: system.network - metricset: filesystem - dataset.name: system.filesystem + datastream.dataset: system.filesystem # management: # # Mode of management, the Elastic Agent support two modes of operation: diff --git a/x-pack/elastic-agent/elastic-agent.reference.yml b/x-pack/elastic-agent/elastic-agent.reference.yml index 6e9589c3fa1..eabca049078 100644 --- a/x-pack/elastic-agent/elastic-agent.reference.yml +++ b/x-pack/elastic-agent/elastic-agent.reference.yml @@ -23,7 +23,7 @@ inputs: # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # # Cannot start with -, _, + # Cannot be . or .. - dataset.namespace: default + datastream.namespace: default use_output: default streams: - metricset: cpu @@ -33,13 +33,13 @@ inputs: # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # # Cannot start with -, _, + # Cannot be . or .. - dataset.name: system.cpu + datastream.dataset: system.cpu - metricset: memory - dataset.name: system.memory + datastream.dataset: system.memory - metricset: network - dataset.name: system.network + datastream.dataset: system.network - metricset: filesystem - dataset.name: system.filesystem + datastream.dataset: system.filesystem # management: # # Mode of management, the Elastic Agent support two modes of operation: diff --git a/x-pack/elastic-agent/elastic-agent.yml b/x-pack/elastic-agent/elastic-agent.yml index a78b0598142..a315a5631b7 100644 --- a/x-pack/elastic-agent/elastic-agent.yml +++ b/x-pack/elastic-agent/elastic-agent.yml @@ -23,7 +23,7 @@ inputs: # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # # Cannot start with -, _, + # Cannot be . or .. - dataset.namespace: default + datastream.namespace: default use_output: default streams: - metricset: cpu @@ -33,13 +33,13 @@ inputs: # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # # Cannot start with -, _, + # Cannot be . or .. - dataset.name: system.cpu + datastream.dataset: system.cpu - metricset: memory - dataset.name: system.memory + datastream.dataset: system.memory - metricset: network - dataset.name: system.network + datastream.dataset: system.network - metricset: filesystem - dataset.name: system.filesystem + datastream.dataset: system.filesystem # agent.monitoring: # # enabled turns on monitoring of running processes diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go index 470cb776ab1..62f1730b079 100644 --- a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go +++ b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go @@ -16,11 +16,11 @@ import ( // ErrInvalidNamespace is error returned when namespace value provided is invalid. var ErrInvalidNamespace = errors.New("provided namespace is invalid", errors.TypeConfig) -// ErrInvalidDataset is error returned when dataset name value provided is invalid. -var ErrInvalidDataset = errors.New("provided dataset name is invalid", errors.TypeConfig) +// ErrInvalidDataset is error returned when datastream name value provided is invalid. +var ErrInvalidDataset = errors.New("provided datastream dataset is invalid", errors.TypeConfig) -// ErrInvalidIndex occurs when concatenation of {dataset.type}-{dataset.name}-{dataset.namespace} does not meet index criteria. -var ErrInvalidIndex = errors.New("provided combination of type, dataset name and namespace is invalid", errors.TypeConfig) +// ErrInvalidIndex occurs when concatenation of {datastream.type}-{datastream.dataset}-{datastream.namespace} does not meet index criteria. +var ErrInvalidIndex = errors.New("provided combination of type, datastream dataset and namespace is invalid", errors.TypeConfig) // StreamChecker checks for invalid values in stream namespace and dataset. func StreamChecker(log *logger.Logger, ast *transpiler.AST) error { @@ -42,9 +42,9 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error { for _, inputNode := range inputsNodeListCollection { namespace := "default" datasetName := "generic" - // fail only if dataset.namespace or dataset[namespace] is found and invalid + // fail only if datastream.namespace or datastream[namespace] is found and invalid // not provided values are ok and will be fixed by rules - if nsNode, found := inputNode.Find("dataset.namespace"); found { + if nsNode, found := inputNode.Find("datastream.namespace"); found { nsKey, ok := nsNode.(*transpiler.Key) if ok { newNamespace := nsKey.Value().(transpiler.Node).String() @@ -54,9 +54,9 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error { namespace = newNamespace } } else { - dsNode, found := inputNode.Find("dataset") + dsNode, found := inputNode.Find("datastream") if found { - // got a dataset + // got a datastream datasetMap, ok := dsNode.Value().(*transpiler.Dict) if ok { nsNode, found := datasetMap.Find("namespace") @@ -76,14 +76,14 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error { // get the type, longest type for now is metrics datasetType := "metrics" - if nsNode, found := inputNode.Find("dataset.type"); found { + if nsNode, found := inputNode.Find("datastream.type"); found { nsKey, ok := nsNode.(*transpiler.Key) if ok { newDataset := nsKey.Value().(transpiler.Node).String() datasetType = newDataset } } else { - dsNode, found := inputNode.Find("dataset") + dsNode, found := inputNode.Find("datastream") if found { // got a dataset datasetMap, ok := dsNode.Value().(*transpiler.Dict) @@ -116,7 +116,7 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error { } // fix this only if in compact form - if dsNameNode, found := streamMap.Find("dataset.name"); found { + if dsNameNode, found := streamMap.Find("datastream.dataset"); found { dsKey, ok := dsNameNode.(*transpiler.Key) if ok { newDataset := dsKey.Value().(transpiler.Node).String() @@ -126,14 +126,14 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error { datasetName = newDataset } } else { - datasetNode, found := streamMap.Find("dataset") + datasetNode, found := streamMap.Find("datastream") if found { datasetMap, ok := datasetNode.Value().(*transpiler.Dict) if !ok { continue } - dsNameNode, found := datasetMap.Find("name") + dsNameNode, found := datasetMap.Find("dataset") if found { dsKey, ok := dsNameNode.(*transpiler.Key) if ok { diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go index b6d9bc22e0d..08303eaebb3 100644 --- a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go @@ -40,8 +40,8 @@ func TestStreamCheck(t *testing.T) { configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ { - "dataset.namespace": "somenamespace", - "streams": []map[string]interface{}{{"dataset.name": "somedatasetname"}}, + "datastream.namespace": "somenamespace", + "streams": []map[string]interface{}{{"datastream.dataset": "somedatasetname"}}, }, }, }, @@ -52,13 +52,13 @@ func TestStreamCheck(t *testing.T) { configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ { - "dataset": map[string]interface{}{ + "datastream": map[string]interface{}{ "namespace": "somenamespace", }, "streams": []map[string]interface{}{ { - "dataset": map[string]interface{}{ - "name": "somedatasetname", + "datastream": map[string]interface{}{ + "dataset": "somedatasetname", }, }, }, @@ -68,23 +68,23 @@ func TestStreamCheck(t *testing.T) { result: nil, }, { - name: "dataset.name invalid - compact", + name: "dataset invalid - compact", configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ - {"streams": []map[string]interface{}{{"dataset.name": ""}}}, + {"streams": []map[string]interface{}{{"datastream.dataset": ""}}}, }, }, result: ErrInvalidDataset, }, { - name: "dataset.name invalid - long", + name: "dataset invalid - long", configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ { "streams": []map[string]interface{}{ { - "dataset": map[string]interface{}{ - "name": "", + "datastream": map[string]interface{}{ + "dataset": "", }, }, }, @@ -95,55 +95,55 @@ func TestStreamCheck(t *testing.T) { }, { - name: "dataset.name invalid dot - compact", + name: "dataset invalid dot - compact", configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ - {"streams": []map[string]interface{}{{"dataset.name": "."}}}, + {"streams": []map[string]interface{}{{"datastream.dataset": "."}}}, }, }, result: ErrInvalidDataset, }, { - name: "dataset.name invalid dotdot- compact", + name: "dataset invalid dotdot- compact", configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ - {"streams": []map[string]interface{}{{"dataset.name": ".."}}}, + {"streams": []map[string]interface{}{{"datastream.dataset": ".."}}}, }, }, result: ErrInvalidDataset, }, { - name: "dataset.name invalid uppercase - compact", + name: "dataset invalid uppercase - compact", configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ - {"streams": []map[string]interface{}{{"dataset.name": "myNameIs"}}}, + {"streams": []map[string]interface{}{{"datastream.dataset": "myNameIs"}}}, }, }, result: ErrInvalidDataset, }, { - name: "dataset.name invalid space- compact", + name: "dataset invalid space- compact", configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ - {"streams": []map[string]interface{}{{"dataset.name": "outer space"}}}, + {"streams": []map[string]interface{}{{"datastream.dataset": "outer space"}}}, }, }, result: ErrInvalidDataset, }, { - name: "dataset.name invalid invalid char- compact", + name: "dataset invalid invalid char- compact", configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ - {"streams": []map[string]interface{}{{"dataset.name": "is\\thisvalid"}}}, + {"streams": []map[string]interface{}{{"datastream.dataset": "is\\thisvalid"}}}, }, }, result: ErrInvalidDataset, }, { - name: "dataset.name invalid invalid prefix- compact", + name: "dataset invalid invalid prefix- compact", configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ - {"streams": []map[string]interface{}{{"dataset.name": "_isthisvalid"}}}, + {"streams": []map[string]interface{}{{"datastream.dataset": "_isthisvalid"}}}, }, }, result: ErrInvalidDataset, @@ -152,7 +152,7 @@ func TestStreamCheck(t *testing.T) { { name: "namespace invalid - compact", configMap: map[string]interface{}{ - "inputs": []map[string]interface{}{{"dataset.namespace": ""}}, + "inputs": []map[string]interface{}{{"datastream.namespace": ""}}, }, result: ErrInvalidNamespace, }, @@ -160,7 +160,7 @@ func TestStreamCheck(t *testing.T) { name: "namespace invalid name 1 - compact", configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ - {"dataset.namespace": "."}, + {"datastream.namespace": "."}, }, }, result: ErrInvalidNamespace, @@ -168,35 +168,35 @@ func TestStreamCheck(t *testing.T) { { name: "namespace invalid name 2 - compact", configMap: map[string]interface{}{ - "inputs": []map[string]interface{}{{"dataset.namespace": ".."}}, + "inputs": []map[string]interface{}{{"datastream.namespace": ".."}}, }, result: ErrInvalidNamespace, }, { name: "namespace invalid name uppercase - compact", configMap: map[string]interface{}{ - "inputs": []map[string]interface{}{{"dataset.namespace": "someUpper"}}, + "inputs": []map[string]interface{}{{"datastream.namespace": "someUpper"}}, }, result: ErrInvalidNamespace, }, { name: "namespace invalid name space - compact", configMap: map[string]interface{}{ - "inputs": []map[string]interface{}{{"dataset.namespace": "some space"}}, + "inputs": []map[string]interface{}{{"datastream.namespace": "some space"}}, }, result: ErrInvalidNamespace, }, { name: "namespace invalid name invalid char - compact", configMap: map[string]interface{}{ - "inputs": []map[string]interface{}{{"dataset.namespace": "isitok?"}}, + "inputs": []map[string]interface{}{{"datastream.namespace": "isitok?"}}, }, result: ErrInvalidNamespace, }, { name: "namespace invalid name invalid prefix - compact", configMap: map[string]interface{}{ - "inputs": []map[string]interface{}{{"dataset.namespace": "+isitok"}}, + "inputs": []map[string]interface{}{{"datastream.namespace": "+isitok"}}, }, result: ErrInvalidNamespace, }, @@ -205,7 +205,7 @@ func TestStreamCheck(t *testing.T) { configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ { - "dataset": map[string]interface{}{ + "datastream": map[string]interface{}{ "namespace": "", }, }, @@ -217,7 +217,7 @@ func TestStreamCheck(t *testing.T) { name: "type invalid name 1 - compact", configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ - {"dataset.type": "-invalidstart"}, + {"datastream.type": "-invalidstart"}, }, }, result: ErrInvalidIndex, @@ -227,9 +227,9 @@ func TestStreamCheck(t *testing.T) { configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ { - "dataset.type": semiLongString, - "dataset.namespace": semiLongString, - "streams": []map[string]interface{}{{"dataset.name": semiLongString}}, + "datastream.type": semiLongString, + "datastream.namespace": semiLongString, + "streams": []map[string]interface{}{{"datastream.dataset": semiLongString}}, }, }, }, @@ -239,7 +239,7 @@ func TestStreamCheck(t *testing.T) { name: "type invalid type length 1 - compact", configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ - {"dataset.type": longString}, + {"datastream.type": longString}, }, }, result: ErrInvalidIndex, @@ -249,17 +249,17 @@ func TestStreamCheck(t *testing.T) { name: "type invalid namespace length 1 - compact", configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ - {"dataset.namespace": longString}, + {"datastream.namespace": longString}, }, }, result: ErrInvalidNamespace, }, { - name: "type invalid dataset.name length 1 - compact", + name: "type invalid dataset length 1 - compact", configMap: map[string]interface{}{ "inputs": []map[string]interface{}{ - {"streams": []map[string]interface{}{{"dataset.name": longString}}}, + {"streams": []map[string]interface{}{{"datastream.dataset": longString}}}, }, }, result: ErrInvalidDataset, diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go index b9e4e4b5f3c..e6b316ebdb3 100644 --- a/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go @@ -115,7 +115,7 @@ const fleetResponse = ` "streams": [{ "id": "system/metrics-system.core", "enabled": true, - "dataset.name": "system.core", + "datastream.dataset": "system.core", "period": "10s", "metrics": [ "percentages" @@ -124,7 +124,7 @@ const fleetResponse = ` { "id": "system/metrics-system.cpu", "enabled": true, - "dataset.name": "system.cpu", + "datastream.dataset": "system.cpu", "period": "10s", "metrics": [ "percentages", @@ -134,59 +134,59 @@ const fleetResponse = ` { "id": "system/metrics-system.diskio", "enabled": true, - "dataset.name": "system.diskio", + "datastream.dataset": "system.diskio", "period": "10s", "include_devices": [] }, { "id": "system/metrics-system.entropy", "enabled": true, - "dataset.name": "system.entropy", + "datastream.dataset": "system.entropy", "period": "10s", "include_devices": [] }, { "id": "system/metrics-system.filesystem", "enabled": true, - "dataset.name": "system.filesystem", + "datastream.dataset": "system.filesystem", "period": "1m", "ignore_types": [] }, { "id": "system/metrics-system.fsstat", "enabled": true, - "dataset.name": "system.fsstat", + "datastream.dataset": "system.fsstat", "period": "1m", "ignore_types": [] }, { "id": "system/metrics-system.load", "enabled": true, - "dataset.name": "system.load", + "datastream.dataset": "system.load", "period": "10s" }, { "id": "system/metrics-system.memory", "enabled": true, - "dataset.name": "system.memory", + "datastream.dataset": "system.memory", "period": "10s" }, { "id": "system/metrics-system.network", "enabled": true, - "dataset.name": "system.network", + "datastream.dataset": "system.network", "period": "10s" }, { "id": "system/metrics-system.network_summary", "enabled": true, - "dataset.name": "system.network_summary", + "datastream.dataset": "system.network_summary", "period": "10s" }, { "id": "system/metrics-system.process", "enabled": true, - "dataset.name": "system.process", + "datastream.dataset": "system.process", "period": "10s", "processes": [ ".*" @@ -202,39 +202,39 @@ const fleetResponse = ` { "id": "system/metrics-system.process_summary", "enabled": true, - "dataset.name": "system.process_summary", + "datastream.dataset": "system.process_summary", "period": "10s" }, { "id": "system/metrics-system.raid", "enabled": true, - "dataset.name": "system.raid", + "datastream.dataset": "system.raid", "period": "10s", "mount_point": "/" }, { "id": "system/metrics-system.service", "enabled": true, - "dataset.name": "system.service", + "datastream.dataset": "system.service", "period": "10s", "state_filter": [] }, { "id": "system/metrics-system.socket_summary", "enabled": true, - "dataset.name": "system.socket_summary", + "datastream.dataset": "system.socket_summary", "period": "10s" }, { "id": "system/metrics-system.uptime", "enabled": true, - "dataset.name": "system.uptime", + "datastream.dataset": "system.uptime", "period": "15m" }, { "id": "system/metrics-system.users", "enabled": true, - "dataset.name": "system.users", + "datastream.dataset": "system.users", "period": "10s" } ] @@ -245,7 +245,7 @@ const fleetResponse = ` "streams": [{ "id": "logs-system.auth", "enabled": true, - "dataset.name": "system.auth", + "datastream.dataset": "system.auth", "paths": [ "/var/log/auth.log*", "/var/log/secure*" @@ -254,7 +254,7 @@ const fleetResponse = ` { "id": "logs-system.syslog", "enabled": true, - "dataset.name": "system.syslog", + "datastream.dataset": "system.syslog", "paths": [ "/var/log/messages*", "/var/log/syslog*" diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go index c87ac1a427b..cf907fe016b 100644 --- a/x-pack/elastic-agent/pkg/agent/program/supported.go +++ b/x-pack/elastic-agent/pkg/agent/program/supported.go @@ -20,7 +20,7 @@ func init() { // spec/endpoint.yml // spec/filebeat.yml // spec/metricbeat.yml - unpacked := packer.MustUnpack("eJy8mF1zozjWx++fj9G3z9YuiHZ22aq5sMnyZoe0SSIJ3SHJBmwJs22MDVv73bcEGIOT9EzPVM1Fyg4W0jlH5/zPT/rPl2OxYX/b5Lw4ZHn511qKL//8QqVdktdDEqLZnjlmQfN18gbgjmO/4O5+GQF9/5wtBJXhmQJx4pbeEBToTAptsy5SlocFkfaOPx4ScpujJA4EVh4IlpMiAm8P3mNkPD8mywikIgLlNkazhjv2kT4elquXhdg4cIcBKajz9mBl88SzFucIh4fnbJ6N52U327J+XMokb56TQ+JZ82T1Ms+4hHWMyMzrn3FHlASZurLxqZkvmWM23FbzBVqELsfn5FB6DvxKULAlUhzJ62Gp3vPcRcqd5MGz/I/9f/G6cY5dE+Opt3teepY/zO2N7Fq96DpzeB2hUNw9rwkOKo79HcFP2WieT9adjD9tpDh/5Guwm5+tfFETaOpUihMzwpQ65wcr0xKCUxHppozRRVxjxxxbix8PiSfhibiLKkYzbYUDERmwjnE4xDPCfs6aPkbXmKPZO5/f2+Lr1IFNF29SbGyz4a4vIqQ9eG5pWv1z6oaCCRNE6KITfI3roiHoIiIjrNjukMRoduY4bPrfvhO8f/DccMact37vSEpdKG52auP8XLYxkOLIHVhj426sGwjqwB13zPo5WxQ0X+jcfer3uhSb1zbX00heBJn3vkr7yBEc5eFCYzkUrU/X+dqcC6sh3gAeCQo0avjNc7agRM2H16cIBTuCgwYD+xxDU/l29BxyJAhqK1kWkbRPEdSmOTr8bp/jdVtTZYTnd7W0kNSBgvc2sxweb/Gdl57rC4pMQLo1r8/bvxjA2XO2SCMQCGYE2wgvCgxKsVkP/tYE6RWXcNuO7X0cxywGIovQLJ3s8+59zCd71sVk+H+67/PSc0yduwv96lNrByYFA6KiyWHJQSro7pC8OHazRrPZ0uJdDCxWeFYkvX+lKdOERpDeLC34dwTEiblQY7puLl/m0s8WXoSD5wjpghnK/7eDD/o1LXb0LC6oYzfcETsGYMpkcPDr/fLLXzq53WZiQzfxO7lVMoN8EeH1VWLbUowkTPm86CQtW1Av020vOydeHgjuwvNKiiN9mQkq7Yw6cP8NqdQNRDvmfmweCooXxwiHYiXhKUL+kaC1SaR9ZOAtW1nzbPXWfVJknyLEBUXwxK1ZSUEovuGkZI69i2u9D5l39CyvDF/Up1+qrSQAlkTJxGh+7vo6eZmMPVLA8xjN8pW8CC7h8RsKRZTD3BPaMsK+FiOSRsb6wXNUTMJm1bYCmBFka78qG1mbFv9WpYSBOBEHfr2mH3fFWcWbOmbOzm1ZFFQWSka2zAhrguwSG4uadmldDenomCcMgopKcoxRoHUyoNpZuI0Q0Qjupb+TnAfPuVTEeGplhSL7fC+pd3JVc3SZSFMEzPMGmil1LlvumFvqiIY/3iTWsxYabQ7J1WZ2HpfXO1tPFJjncfkSnO4IXmhtTuWBxiRMKX5q9z5G6/ZzkLR2n/0zk2YrQ0qe1D7d2apR3TzGONCmpa5KSe3LKKb50+/14xZzCSU1/F5OVVts66jfK1JToD14Tl+652v7+cftmTH4vOzbn8YUttidDxh0EvDZvt3bG+NQ0Nf3fkzWPH8qw9OW4g75fWsd0j4xcEn5gEPziV1tXq/HsdNT5i5ucjo8v1Skx6r2+zjebV4QQfN11WNPWyfj9Txroer1xC2z4U5YKCllRriP0de7dSBodcAId0zZ5wTnT+bRiTt/8Fy4Z/OpLWrtFQirCJTKj4Q45i4GsL6b50gBq5iE+xgHWwYuFQeXiqicap89vfe/NpsNDtR7D54bzNQ71zj8lrbFcSAw+KDN/Mp7xLG1CAfid71jzU6jljPUHpOwpAYRbet9nehDh0BOmHLHbhRGtDXSoYDqLRrB/vYeU24I4r2rn77laxu8EH3+3iGZar+TWHY5cstjtdeD7aOcKD2n3+dkHJuwYKO5+j27+d48Dd+phBqRl4rfxh+4G57jPKhG61dPTaRHWOXFDSEiXOhMwi4fnVSPbuN31DF14pCKK1TM9zcMAWZFwEUwF2bMgDdEd8qUyDL9w+jqDP3zU3xt+2q3nztqLIb8IblfKS26m7fVWTLijDHyfoigtziMWOQWS47DMx+jGIAzpnySbxPMelHaiJ8OPi7paP3T1T6MtRF6zVKKYKM4g7wkOTWgpnLQr8+JD+AxwoEWo6AhyK4jkOQra573dZ6vWtv49wiR79FLi2KKM9SxsoktVljJL79cMUxuyu8Z+wDEXhHUmBS7Hrx2FCkI0QV3/SICPaB1550E1QPQNASHOrNmBXW009A4H3UZoUtzBz/XsarBnKlja+TXAE7qKZV2TpCuxPlEkbknr/rXFVZicCx7Jv4RwN3mx2HN0R3sOWZOFCjVs2PbAB71PUG+TmqfW9KuuSNk1DXgFsBYbZYEh3WMgh7IFhUzwskZviuGTtwmZ+jJuVKviNueOU7EakVZwcJpg/ThzKgSV8Wb4LU6dysQbZvjSq4rZoiGO7Bc5aKk1kw1gCuwLG9nmTbJv+Jrw389JBtDm4BfjGZ7gpNrcbYC95wtrj42XTGLUyzbM1sPm/qWuX4VAdgwYA4FRMFsGwHzROSl6ABWnBiANbfNlOThIHADiPb51sNBrXKHouHOQzJplu8BIaxuz4KrPb2desoe7+4rPoCeT0CjFRIM7CO1PwG6bu3bmqNm/N73WUWN+bgpiY0TCOauHzz7mA2wVLd1UfSgOeRqd3iZgGOG13e2GmGFwaVgxnp6Pr0C2WiPJnD5U34Me5gRRFqB+5Oh8V3DxgYvuJNumYQ5welwwPigSdcqz3D29fsK9DpmPO1/2Ij+3Ob1B4EYbjkQWmybNUFcbNz5bwNk11c1vlk+mutv3aHt/1fZsXgfo+6vXePxkPjjO6UOwFowm8JbfwCZjL01YaXf6sB3gwY9jQHcRtivo/t7lj5HBp0AUJvY1ebK1WZ10H/6CTgcvfczUHp31/Vnwe9P3EvtIxymKlYrOQKKn7ibuoP8n4IsOmGIH+r6pI+u5EcH1qFH/gywTfvz53dj3wlWPdSsVS1+Bm0Tf7rLoLYe/xC4dbDWQX7dwtqPwe2///e/AAAA//+pKFpS") + unpacked := packer.MustUnpack("eJy8mN9zo7oVx9/7Z+xrOy2IdVo6cx9sUn7ZIWuSSEJvSLIBW8J0jbGh0/+9I8AYnGTv3d6Z+7CTLBHSOUfnfM/n8J8vx2LD/rbJeXHI8vKvtRRf/vmFSrskr4ckRLM9c8yC5uvkDcAdx37B3f0yAvr+OVsIKsMzBeLELb0hKNCZFNpmXaQsDwsi7R1/PCTktkdJHAisPBAsJ0UE3h68x8h4fkyWEUhFBMptjGYNd+wjfTwsVy8LsXHgDgNSUOftwcrmiWctzhEOD8/ZPBvvy262Zf26lEnePCeHxLPmyeplnnEJ6xiRmdc/444oCTJ1ZeNTM18yx2y4rfYLtAhdjs/JofQc+JWgYEukOJLXw1K957mLlDvJg2f5H/v/4nXrHLsmxlNv97z0LH/Y2xvZtXrRdebwOkKhuHteExxUHPs7gp+y0T6fnDtZf9pIcf7I12A3P1v5oibQ1KkUJ2aEKXXOD1amJQSnItJNGaOLuMaOObYWPx4ST8ITcRdVjGbaCgciMmAd43CIZ4T9nDV9jK4xR7N3Pr+3xdepA5su3qTY2GbDXV9ESHvw3NK0+ufUDQUTJojQRSf4GtdFQ9BFREZYsd0hidHszHHY9H/7TvD+wXPDGXPe+rsjKXWhuNmpjfNz2cZAiiN3YI2Nu7VuIKgDd9wx6+dsUdB8oXP3qb/rUmxe21xPI3kRZN77Ku0jR3CUhwuN5VC0Pl33a3MurIZ4A3gkKNCo4TfP2YIStR9enyIU7AgOGgzscwxN5dvRc8iRIKitZFlE0j5FUJvm6PB3+xyv25oqIzy/q6WFpA4UvLeZ5fB4i++89FxfUGQC0p15fd7+iwGcPWeLNAKBYEawjfCiwKAUm/Xgb02QXnEJt+3a3sdxzGIgsgjN0sk9797HfHJnXUyG/0/vfV56jqlzd6FffWrtwKRgQFQ0OSw5SAXdHZIXx27WaDZbWryLgcUKz4qk9680ZZrQCNKbpQX/joA4MRdqTNfN5ctc+tnCi3DwHCFdMEP5/3bwQX+mxY6exQV17IY7YscATJkMDn69X375Sye320xs6CZ+J7dKZpAvIry+SmxbipGEKZ8XnaRlC+pluu1l58TLA8FdeF5JcaQvM0GlnVEH7r8hlbqBaNfcr81DQfHiGOFQrCQ8Rcg/ErQ2ibSPDLxlK2uerd66nxTZpwhxQRE8cWtWUhCKbzgpmWPv4lrvQ+YdPcsrwxf10y/VVRIAS6JkYrQ/d32dvEzWHingeYxm+UpeBJfw+A2FIsph7gltGWFfixFJI2P94DkqJmGzalsBzAiytV+VjaxNi3+rUsJAnIgDv17Tj7virOJNHTNn57YsCioLJSNbZoQ1QXaJjUVNu7SuhnR0zBMGQUUlOcYo0DoZUO0s3EaIaAT30t9JzoPnXCpiPLWyQpF9vpfUO7mqObpMpCkC5nkDzZQ6ly13zC11RMMfbxLrWQuNNofkajM7j8vrna0nCszzuHwJTncEL7Q2p/JAYxKmFD+1dx+jdftzkLT2nv0zk2YrQ0qe1D3d2apR3TzGONCmpa5KSd3LKKb50//rxy3mEkpq+L2cqrbY1lF/V6SmQHvwnL50z9f284/bM2Pwedm3P40pbLE7HzDoJOCze7u3N8ahoK/v/Zicef5UhqctxR3y+9Y6pH1i4JLyAYfmE7vavF6PY6enzF3c5HR4fqlIj1Xt7+N4t3lBBM3XVY89bZ2Mz/OsharXE7fMhjthoaSUGeE+Rl/vzoGg1QEj3DFlnxOcP9lHJ+78wXPhns2ntqizVyCsIlAqPxLimLsYwPpunyMFrGIS7mMcbBm4VBxcKqJyqn329N7/2mw2OFDvPXhuMFPvXOPwW9oWx4HA4IM28yvvEcfWItzn3U+0ycl7L7PTqPUMNcgkLKlBRNuCXyc60aGQE6ZctaVOJ0/9nqr1q5xUvUYj2N/eY8sNSbx39dQjgLbBC9Hn8x2iqXY8iW2XM7e8Vnc/+DDKkdJz+ntPxnEKCzbaq7/DWwyap+F3KqFG5KXit/UH7obnOA+q0fnVUxPpEVZ5ckOKCBc6k7DLTyfVo9v6HXVMnTik4god8/0NS4BZEXARzIUZM+AN2Z0yJbJMfzfKOkM//RRn2z7bod6OGoshl0juV0qb7vZtdZeMuGOMwB8i6S0OIza5xZLj8MzHaAbgjCmf5NsEu16UVuKng49LOjr/dLUPY22EYrOUItgo7iAvSU4NqKkc9Otz4gN4VHURo6AhyK4jkOQra573dZ+vWtv49wiR79FLi2aKO9SY2cQWK6zkl1+uWCY35feMfQBmrwhqTIpdD2I7ihSU6IK7fhGBHti6+SdB9QA4DcGhzqxZQR3tNDTSR11G6NLcwdB1rWo4Z+rYGvk1oJN6SqWdE6QrsT5RZO7Jq/51hZUoHMuekX8EdLf9cVhzdAd/jpkTBU717Ng2hEd9T5Cvk9rnlrRr7ggZdQ25BTJWmyXBYR2joAe0RcWMcDLTd8XQCd1kpp7MmXpF3HYGORGrFWkFD6cN0ocZUiWuijfBazWHKzBtm+VKritmiIY7sFzloqTWTDWEK8Asb7NNm+Rf8RUAXg/JxtAmIBij2Z7g5FqcrcA9Z4urj01XzOIUy3aG60VV3zLXryIAGwbMoYAomG0jYJ6IvBQd0IoTA7DmtpmSPBwEbgDTPt96WKhV7lA0fAORTJrle2AIq9uz4GpPb6eesse77xcfQNAn4NEKCQb2kdqfAF539u3MUXN+7/usosZ83JzExgkEc9cPnn3MBniq27ooevAccrUbZiYgmeH1na1GWGFwKZixns6rV0Ab3dEENn/Kj+EOM4JIK3B/MES+a9jY4AV30i2TMCc4HQaOD5p0rfIMZ1+/r0CvY8bT/oeN6I9tXr8TkOGWA6HFtlkTxMXGnf82YHZ9VeOb5aO5/tYNcX9eZcfifYy6f+0Zj4fEH39j6kCsBbQpyPUDyWTtrQkr/VYD4A0a9DQGcBthv47uv7v0OTLoBIDaxK42V642q8H/6ScgcfTez4Dt3bevPxCG9xEO01EMfss3q+6dvsZWcgQYP2HH3RDwU9BFJ0zxQ52f9NWV/GigHXrmzwDctF9//u3sO8Gqp5q1qs3PIG7iT/exqK3P3wVyHbx10F+38PZjkPvvn/4XAAD//xtQZOQ=") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config.yml index 1dce338f6b9..c24b0a8b87b 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config.yml @@ -24,7 +24,7 @@ inputs: - type: apache/metrics constraints: - "validate_version(%{[agent.version]}, '1.0.0 - 7.0.0')" - dataset.namespace: testing + datastream.namespace: testing streams: - enabled: true metricset: info diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic-endpoint-security.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic-endpoint-security.yml index f1ad8c1c646..1402aa8544d 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic-endpoint-security.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic-endpoint-security.yml @@ -26,7 +26,7 @@ inputs: package: name: endpoint version: 0.3.0 - dataset: + datastream: namespace: default artifact_manifest: schema_version: v22 diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml index 54e3b549236..91b8b5d7901 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml @@ -25,7 +25,7 @@ inputs: package: name: endpoint version: 0.3.0 - dataset: + datastream: namespace: default artifact_manifest: schema_version: v22 diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_no_fleet.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_no_fleet.yml index 6a7aea5099d..e7dc911abc5 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_no_fleet.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_no_fleet.yml @@ -16,7 +16,7 @@ inputs: package: name: endpoint version: 0.3.0 - dataset: + datastream: namespace: default artifact_manifest: schema_version: v22 diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_unknown_output.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_unknown_output.yml index 7c813b3732d..edf812b1c73 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_unknown_output.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_unknown_output.yml @@ -21,7 +21,7 @@ inputs: package: name: endpoint version: 0.3.0 - dataset: + datastream: namespace: default artifact_manifest: schema_version: v22 diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-endpoint-security.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-endpoint-security.yml index 0296e357bee..feaf5efeb5a 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-endpoint-security.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-endpoint-security.yml @@ -26,7 +26,7 @@ inputs: package: name: endpoint version: 0.3.0 - dataset: + datastream: namespace: default artifact_manifest: schema_version: v22 diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml index a4b8a8024a3..b4eff414856 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml @@ -29,11 +29,11 @@ inputs: use_output: default streams: - metricset: status - dataset: - name: docker.status + datastream: + dataset: docker.status - metricset: info - dataset: - name: "" + datastream: + dataset: "" hosts: ["http://127.0.0.1:8080"] - type: logfile use_output: default @@ -44,7 +44,7 @@ inputs: vars: var: value - type: logfile - dataset: + datastream: type: testtype use_output: default streams: @@ -55,7 +55,7 @@ inputs: var: value - id: apache-metrics-id type: apache/metrics - dataset: + datastream: namespace: testing use_output: default processors: @@ -73,7 +73,7 @@ inputs: package: name: endpoint version: 0.3.0 - dataset: + datastream: namespace: default artifact_manifest: schema_version: v22 diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go index 5cda92a6a99..1b57338fa03 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go @@ -409,7 +409,7 @@ func (r *FixStreamRule) Apply(ast *AST) error { for _, inputNode := range inputsNodeList.value { // fix this only if in compact form - if nsNode, found := inputNode.Find("dataset.namespace"); found { + if nsNode, found := inputNode.Find("datastream.namespace"); found { nsKey, ok := nsNode.(*Key) if ok { if newNamespace := nsKey.value.String(); newNamespace == "" { @@ -417,12 +417,12 @@ func (r *FixStreamRule) Apply(ast *AST) error { } } } else { - dsNode, found := inputNode.Find("dataset") + dsNode, found := inputNode.Find("datastream") if found { - // got a dataset - datasetMap, ok := dsNode.Value().(*Dict) + // got a datastream + datastreamMap, ok := dsNode.Value().(*Dict) if ok { - nsNode, found := datasetMap.Find("namespace") + nsNode, found := datastreamMap.Find("namespace") if found { nsKey, ok := nsNode.(*Key) if ok { @@ -434,7 +434,7 @@ func (r *FixStreamRule) Apply(ast *AST) error { inputMap, ok := inputNode.(*Dict) if ok { inputMap.value = append(inputMap.value, &Key{ - name: "dataset.namespace", + name: "datastream.namespace", value: &StrVal{value: defaultNamespace}, }) } @@ -444,7 +444,7 @@ func (r *FixStreamRule) Apply(ast *AST) error { inputMap, ok := inputNode.(*Dict) if ok { inputMap.value = append(inputMap.value, &Key{ - name: "dataset.namespace", + name: "datastream.namespace", value: &StrVal{value: defaultNamespace}, }) } @@ -468,7 +468,7 @@ func (r *FixStreamRule) Apply(ast *AST) error { } // fix this only if in compact form - if dsNameNode, found := streamMap.Find("dataset.name"); found { + if dsNameNode, found := streamMap.Find("datastream.dataset"); found { dsKey, ok := dsNameNode.(*Key) if ok { if newDataset := dsKey.value.String(); newDataset == "" { @@ -477,14 +477,14 @@ func (r *FixStreamRule) Apply(ast *AST) error { } } else { - datasetNode, found := streamMap.Find("dataset") + datastreamNode, found := streamMap.Find("datastream") if found { - datasetMap, ok := datasetNode.Value().(*Dict) + datastreamMap, ok := datastreamNode.Value().(*Dict) if !ok { continue } - dsNameNode, found := datasetMap.Find("name") + dsNameNode, found := datastreamMap.Find("dataset") if found { dsKey, ok := dsNameNode.(*Key) if ok { @@ -494,13 +494,13 @@ func (r *FixStreamRule) Apply(ast *AST) error { } } else { streamMap.value = append(streamMap.value, &Key{ - name: "dataset.name", + name: "datastream.dataset", value: &StrVal{value: defaultDataset}, }) } } else { streamMap.value = append(streamMap.value, &Key{ - name: "dataset.name", + name: "datastream.dataset", value: &StrVal{value: defaultDataset}, }) } @@ -517,10 +517,10 @@ func FixStream() *FixStreamRule { } // InjectIndexRule injects index to each input. -// Index is in form {type}-{namespace}-{dataset-type} +// Index is in form {type}-{namespace}-{dataset} // type: is provided to the rule. // namespace: is collected from streams[n].namespace. If not found used 'default'. -// dataset-type: is collected from streams[n].dataset.type. If not found used 'generic'. +// dataset: is collected from streams[n].datastream.dataset. If not found used 'generic'. type InjectIndexRule struct { Type string } @@ -538,8 +538,8 @@ func (r *InjectIndexRule) Apply(ast *AST) error { } for _, inputNode := range inputsList.value { - namespace := datasetNamespaceFromInputNode(inputNode) - datasetType := datasetTypeFromInputNode(inputNode, r.Type) + namespace := datastreamNamespaceFromInputNode(inputNode) + datastreamType := datastreamTypeFromInputNode(inputNode, r.Type) streamsNode, ok := inputNode.Find("streams") if !ok { @@ -560,7 +560,7 @@ func (r *InjectIndexRule) Apply(ast *AST) error { dataset := datasetNameFromStreamNode(streamNode) streamMap.value = append(streamMap.value, &Key{ name: "index", - value: &StrVal{value: fmt.Sprintf("%s-%s-%s", datasetType, dataset, namespace)}, + value: &StrVal{value: fmt.Sprintf("%s-%s-%s", datastreamType, dataset, namespace)}, }) } } @@ -595,8 +595,8 @@ func (r *InjectStreamProcessorRule) Apply(ast *AST) error { } for _, inputNode := range inputsList.value { - namespace := datasetNamespaceFromInputNode(inputNode) - datasetType := datasetTypeFromInputNode(inputNode, r.Type) + namespace := datastreamNamespaceFromInputNode(inputNode) + datastreamType := datastreamTypeFromInputNode(inputNode, r.Type) streamsNode, ok := inputNode.Find("streams") if !ok { @@ -636,7 +636,7 @@ func (r *InjectStreamProcessorRule) Apply(ast *AST) error { processorMap := &Dict{value: make([]Node, 0)} processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "datastream"}}) processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{ - &Key{name: "type", value: &StrVal{value: datasetType}}, + &Key{name: "type", value: &StrVal{value: datastreamType}}, &Key{name: "namespace", value: &StrVal{value: namespace}}, &Key{name: "dataset", value: &StrVal{value: dataset}}, }}}) @@ -1242,10 +1242,10 @@ func keys(m map[string]interface{}) []string { return keys } -func datasetNamespaceFromInputNode(inputNode Node) string { +func datastreamNamespaceFromInputNode(inputNode Node) string { const defaultNamespace = "default" - if namespaceNode, found := inputNode.Find("dataset.namespace"); found { + if namespaceNode, found := inputNode.Find("datastream.namespace"); found { nsKey, ok := namespaceNode.(*Key) if ok { if newNamespace := nsKey.value.String(); newNamespace != "" { @@ -1254,7 +1254,7 @@ func datasetNamespaceFromInputNode(inputNode Node) string { } } - dsNode, found := inputNode.Find("dataset") + dsNode, found := inputNode.Find("datastream") if found { dsMapNode, ok := dsNode.Value().(*Dict) if ok { @@ -1273,17 +1273,17 @@ func datasetNamespaceFromInputNode(inputNode Node) string { return defaultNamespace } -func datasetTypeFromInputNode(inputNode Node, defaultType string) string { - if dsTypeNode, found := inputNode.Find("dataset.type"); found { +func datastreamTypeFromInputNode(inputNode Node, defaultType string) string { + if dsTypeNode, found := inputNode.Find("datastream.type"); found { dsTypeKey, ok := dsTypeNode.(*Key) if ok { - if newDatasetType := dsTypeKey.value.String(); newDatasetType != "" { - return newDatasetType + if newDatastreamType := dsTypeKey.value.String(); newDatastreamType != "" { + return newDatastreamType } } } - dsNode, found := inputNode.Find("dataset") + dsNode, found := inputNode.Find("datastream") if found { dsMapNode, ok := dsNode.Value().(*Dict) if ok { @@ -1291,8 +1291,8 @@ func datasetTypeFromInputNode(inputNode Node, defaultType string) string { if found { typeKey, ok := typeNode.(*Key) if ok { - if newDatasetType := typeKey.value.String(); newDatasetType != "" { - return newDatasetType + if newDatastreamType := typeKey.value.String(); newDatastreamType != "" { + return newDatastreamType } } } @@ -1305,7 +1305,7 @@ func datasetTypeFromInputNode(inputNode Node, defaultType string) string { func datasetNameFromStreamNode(streamNode Node) string { const defaultDataset = "generic" - if dsNameNode, found := streamNode.Find("dataset.name"); found { + if dsNameNode, found := streamNode.Find("datastream.dataset"); found { dsNameKey, ok := dsNameNode.(*Key) if ok { if newDatasetName := dsNameKey.value.String(); newDatasetName != "" { @@ -1314,11 +1314,11 @@ func datasetNameFromStreamNode(streamNode Node) string { } } - dsNode, found := streamNode.Find("dataset") + dsNode, found := streamNode.Find("datastream") if found { dsMapNode, ok := dsNode.Value().(*Dict) if ok { - dsNameNode, found := dsMapNode.Find("name") + dsNameNode, found := dsMapNode.Find("dataset") if found { dsKey, ok := dsNameNode.(*Key) if ok { diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/rules_test.go b/x-pack/elastic-agent/pkg/agent/transpiler/rules_test.go index 5619d847de6..7b34ce4eb9b 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/rules_test.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/rules_test.go @@ -31,59 +31,59 @@ inputs: - paths: /var/log/mysql/error.log - name: Specified namespace type: file - dataset.namespace: nsns + datastream.namespace: nsns streams: - paths: /var/log/mysql/error.log - name: Specified dataset type: file streams: - paths: /var/log/mysql/error.log - dataset.name: dsds + datastream.dataset: dsds - name: All specified type: file - dataset.namespace: nsns + datastream.namespace: nsns streams: - paths: /var/log/mysql/error.log - dataset.name: dsds + datastream.dataset: dsds - name: All specified with empty strings type: file - dataset.namespace: "" + datastream.namespace: "" streams: - paths: /var/log/mysql/error.log - dataset.name: "" + datastream.dataset: "" `, expectedYAML: ` inputs: - name: All default type: file - dataset.namespace: default + datastream.namespace: default streams: - paths: /var/log/mysql/error.log - dataset.name: generic + datastream.dataset: generic - name: Specified namespace type: file - dataset.namespace: nsns + datastream.namespace: nsns streams: - paths: /var/log/mysql/error.log - dataset.name: generic + datastream.dataset: generic - name: Specified dataset type: file - dataset.namespace: default + datastream.namespace: default streams: - paths: /var/log/mysql/error.log - dataset.name: dsds + datastream.dataset: dsds - name: All specified type: file - dataset.namespace: nsns + datastream.namespace: nsns streams: - paths: /var/log/mysql/error.log - dataset.name: dsds + datastream.dataset: dsds - name: All specified with empty strings type: file - dataset.namespace: default + datastream.namespace: default streams: - paths: /var/log/mysql/error.log - dataset.name: generic + datastream.dataset: generic `, rule: &RuleList{ Rules: []Rule{ @@ -101,7 +101,7 @@ inputs: - paths: /var/log/mysql/error.log - name: Specified namespace type: file - dataset.namespace: nsns + datastream.namespace: nsns streams: - paths: /var/log/mysql/error.log @@ -109,19 +109,19 @@ inputs: type: file streams: - paths: /var/log/mysql/error.log - dataset.name: dsds + datastream.dataset: dsds - name: All specified type: file - dataset.namespace: nsns + datastream.namespace: nsns streams: - paths: /var/log/mysql/error.log - dataset.name: dsds + datastream.dataset: dsds - name: All specified with empty strings type: file - dataset.namespace: "" + datastream.namespace: "" streams: - paths: /var/log/mysql/error.log - dataset.name: "" + datastream.dataset: "" `, expectedYAML: ` inputs: @@ -132,7 +132,7 @@ inputs: index: mytype-generic-default - name: Specified namespace type: file - dataset.namespace: nsns + datastream.namespace: nsns streams: - paths: /var/log/mysql/error.log index: mytype-generic-nsns @@ -141,21 +141,21 @@ inputs: type: file streams: - paths: /var/log/mysql/error.log - dataset.name: dsds + datastream.dataset: dsds index: mytype-dsds-default - name: All specified type: file - dataset.namespace: nsns + datastream.namespace: nsns streams: - paths: /var/log/mysql/error.log - dataset.name: dsds + datastream.dataset: dsds index: mytype-dsds-nsns - name: All specified with empty strings type: file - dataset.namespace: "" + datastream.namespace: "" streams: - paths: /var/log/mysql/error.log - dataset.name: "" + datastream.dataset: "" index: mytype-generic-default `, rule: &RuleList{ diff --git a/x-pack/elastic-agent/spec/filebeat.yml b/x-pack/elastic-agent/spec/filebeat.yml index f9dc8bb4507..0eb1dd40083 100644 --- a/x-pack/elastic-agent/spec/filebeat.yml +++ b/x-pack/elastic-agent/spec/filebeat.yml @@ -49,11 +49,11 @@ rules: - remove_key: key: use_output - remove_key: - key: dataset + key: datastream - remove_key: - key: dataset.namespace + key: datastream.namespace - remove_key: - key: dataset.name + key: datastream.dataset - filter_values: selector: inputs diff --git a/x-pack/elastic-agent/spec/metricbeat.yml b/x-pack/elastic-agent/spec/metricbeat.yml index fcc8428517a..c2fb2df553b 100644 --- a/x-pack/elastic-agent/spec/metricbeat.yml +++ b/x-pack/elastic-agent/spec/metricbeat.yml @@ -65,11 +65,11 @@ rules: - remove_key: key: enabled - remove_key: - key: dataset + key: datastream - remove_key: - key: dataset.name + key: datastream.dataset - remove_key: - key: dataset.namespace + key: datastream.namespace - remove_key: key: use_output