From ab12d3ec74bf4e7b59c4ccfec4006ca88662e3c0 Mon Sep 17 00:00:00 2001
From: Michal Pristas <michal.pristas@gmail.com>
Date: Thu, 2 Jul 2020 10:34:40 +0200
Subject: [PATCH 01/10] added stream check

---
 .../filters/constraints_filter_test.go        |  16 +--
 .../application/filters/stream_checker.go     | 125 ++++++++++++++++++
 .../filters/stream_checker_test.go            | 122 +++++++++++++++++
 .../pkg/agent/application/local_mode.go       |  10 +-
 .../pkg/agent/application/managed_mode.go     |   2 +-
 5 files changed, 265 insertions(+), 10 deletions(-)
 create mode 100644 x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go
 create mode 100644 x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go

diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter_test.go b/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter_test.go
index d7a748d7fbdc..d7e3190dd923 100644
--- a/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter_test.go
+++ b/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter_test.go
@@ -22,14 +22,14 @@ func TestEvaluation(t *testing.T) {
 	}
 
 	testCases := []testCase{
-		testCase{"simple version", "validate_version(%{[agent.version]}, '" + release.Version() + "')", true},
-		testCase{"~ version release", "validate_version(%{[agent.version]}, '~" + release.Version() + "')", true},
-		testCase{"^ version release", "validate_version(%{[agent.version]}, '^" + release.Version() + "')", true},
-		testCase{"range to release", "validate_version(%{[agent.version]}, '1.0.0 - " + release.Version() + "')", true},
-		testCase{"range lower", "validate_version(%{[agent.version]}, '1.0.0 - 5.0.0')", false},
-		testCase{"range include", "validate_version(%{[agent.version]}, '1.0.0 - 100.0.0')", true},
-		testCase{"family should equal", "%{[os.family]} == '" + runtime.GOOS + "'", true},
-		testCase{"family should not equal", "%{[os.family]} != '" + runtime.GOOS + "'", false},
+		{"simple version", "validate_version(%{[agent.version]}, '" + release.Version() + "')", true},
+		{"~ version release", "validate_version(%{[agent.version]}, '~" + release.Version() + "')", true},
+		{"^ version release", "validate_version(%{[agent.version]}, '^" + release.Version() + "')", true},
+		{"range to release", "validate_version(%{[agent.version]}, '1.0.0 - " + release.Version() + "')", true},
+		{"range lower", "validate_version(%{[agent.version]}, '1.0.0 - 5.0.0')", false},
+		{"range include", "validate_version(%{[agent.version]}, '1.0.0 - 100.0.0')", true},
+		{"family should equal", "%{[os.family]} == '" + runtime.GOOS + "'", true},
+		{"family should not equal", "%{[os.family]} != '" + runtime.GOOS + "'", false},
 	}
 
 	for _, tc := range testCases {
diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go
new file mode 100644
index 000000000000..3d6cd95df4f2
--- /dev/null
+++ b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go
@@ -0,0 +1,125 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package filters
+
+import (
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
+)
+
+// ErrInvalidNamespace is error returned when namespace value provided is invalid.
+var ErrInvalidNamespace = errors.New("provided namespace is invalid", errors.TypeConfig)
+
+// ErrInvalidDataset is error returned when dataset name value provided is invalid.
+var ErrInvalidDataset = errors.New("provided dataset name is invalid", errors.TypeConfig)
+
+// StreamChecker checks for invalid values in stream namespace and dataset.
+func StreamChecker(log *logger.Logger, ast *transpiler.AST) error {
+	inputsNode, found := transpiler.Lookup(ast, "inputs")
+	if !found {
+		return nil
+	}
+
+	inputsNodeList, ok := inputsNode.Value().(*transpiler.List)
+	if !ok {
+		return nil
+	}
+
+	inputsNodeListCollection, ok := inputsNodeList.Value().([]transpiler.Node)
+	if !ok {
+		return errors.New("inputs is not a list", errors.TypeConfig)
+	}
+
+	for _, inputNode := range inputsNodeListCollection {
+		// fail only if dataset.namespace or dataset[namespace] is found and invalid
+		// not provided values are ok and will be fixed by rules
+		if nsNode, found := inputNode.Find("dataset.namespace"); found {
+			nsKey, ok := nsNode.(*transpiler.Key)
+			if ok {
+				if newNamespace := nsKey.Value().(transpiler.Node).String(); !isNamespaceValid(newNamespace) {
+					return ErrInvalidNamespace
+				}
+			}
+		} else {
+			dsNode, found := inputNode.Find("dataset")
+			if found {
+				// got a dataset
+				datasetMap, ok := dsNode.Value().(*transpiler.Dict)
+				if ok {
+					nsNode, found := datasetMap.Find("namespace")
+					if found {
+						nsKey, ok := nsNode.(*transpiler.Key)
+						if ok {
+							if newNamespace := nsKey.Value().(transpiler.Node).String(); !isNamespaceValid(newNamespace) {
+								return ErrInvalidNamespace
+							}
+						}
+					}
+				}
+			}
+		}
+
+		streamsNode, ok := inputNode.Find("streams")
+		if !ok {
+			continue
+		}
+
+		streamsList, ok := streamsNode.Value().(*transpiler.List)
+		if !ok {
+			continue
+		}
+
+		streamNodes, ok := streamsList.Value().([]transpiler.Node)
+		if !ok {
+			return errors.New("streams is not a list", errors.TypeConfig)
+		}
+
+		for _, streamNode := range streamNodes {
+			streamMap, ok := streamNode.(*transpiler.Dict)
+			if !ok {
+				continue
+			}
+
+			// fix this only if in compact form
+			if dsNameNode, found := streamMap.Find("dataset.name"); found {
+				dsKey, ok := dsNameNode.(*transpiler.Key)
+				if ok {
+					if newDataset := dsKey.Value().(transpiler.Node).String(); !isDatasetValid(newDataset) {
+						return ErrInvalidDataset
+					}
+				}
+			} else {
+				datasetNode, found := streamMap.Find("dataset")
+				if found {
+					datasetMap, ok := datasetNode.Value().(*transpiler.Dict)
+					if !ok {
+						continue
+					}
+
+					dsNameNode, found := datasetMap.Find("name")
+					if found {
+						dsKey, ok := dsNameNode.(*transpiler.Key)
+						if ok {
+							if newDataset := dsKey.Value().(transpiler.Node).String(); !isDatasetValid(newDataset) {
+								return ErrInvalidDataset
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func isNamespaceValid(namespace string) bool {
+	return len(namespace) > 0
+}
+
+func isDatasetValid(dataset string) bool {
+	return len(dataset) > 0
+}
diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go
new file mode 100644
index 000000000000..2f6a77fcb550
--- /dev/null
+++ b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go
@@ -0,0 +1,122 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package filters
+
+import (
+	"testing"
+
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStreamCheck(t *testing.T) {
+	type testCase struct {
+		name      string
+		configMap map[string]interface{}
+		result    error
+	}
+
+	testCases := []testCase{
+		{
+			name:      "all missing",
+			configMap: map[string]interface{}{},
+			result:    nil,
+		},
+		{
+			name: "all ok - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{
+						"dataset.namespace": "someNamespace",
+						"streams":           []map[string]interface{}{{"dataset.name": "someDatasetName"}},
+					},
+				},
+			},
+			result: nil,
+		},
+		{
+			name: "all ok - long",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{
+						"dataset": map[string]interface{}{
+							"namespace": "someNamespace",
+						},
+						"streams": []map[string]interface{}{
+							{
+								"dataset": map[string]interface{}{
+									"name": "someDatasetName",
+								},
+							},
+						},
+					},
+				},
+			},
+			result: nil,
+		},
+		{
+			name: "dataset.name invalid - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{"streams": []map[string]interface{}{{"dataset.name": ""}}},
+				},
+			},
+			result: ErrInvalidDataset,
+		},
+		{
+			name: "dataset.name invalid - long",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{
+						"streams": []map[string]interface{}{
+							{
+								"dataset": map[string]interface{}{
+									"name": "",
+								},
+							},
+						},
+					},
+				},
+			},
+			result: ErrInvalidDataset,
+		},
+		{
+			name: "namespace invalid - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{"dataset.namespace": ""},
+				},
+			},
+			result: ErrInvalidNamespace,
+		},
+		{
+			name: "namespace invalid - long",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{
+						"dataset": map[string]interface{}{
+							"namespace": "",
+						},
+					},
+				},
+			},
+			result: ErrInvalidNamespace,
+		},
+	}
+
+	log, err := logger.New("")
+	assert.NoError(t, err)
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ast, err := transpiler.NewAST(tc.configMap)
+			assert.NoError(t, err)
+
+			result := StreamChecker(log, ast)
+			assert.Equal(t, tc.result, result)
+		})
+	}
+}
diff --git a/x-pack/elastic-agent/pkg/agent/application/local_mode.go b/x-pack/elastic-agent/pkg/agent/application/local_mode.go
index b6c8a462018a..b53b75c20d81 100644
--- a/x-pack/elastic-agent/pkg/agent/application/local_mode.go
+++ b/x-pack/elastic-agent/pkg/agent/application/local_mode.go
@@ -99,7 +99,15 @@ func newLocal(
 	}
 
 	discover := discoverer(pathConfigFile, c.Management.Path)
-	emit := emitter(log, router, &configModifiers{Decorators: []decoratorFunc{injectMonitoring}, Filters: []filterFunc{filters.ConstraintFilter}}, monitor)
+	emit := emitter(
+		log,
+		router,
+		&configModifiers{
+			Decorators: []decoratorFunc{injectMonitoring},
+			Filters:    []filterFunc{filters.StreamChecker, filters.ConstraintFilter},
+		},
+		monitor,
+	)
 
 	var cfgSource source
 	if !c.Management.Reload.Enabled {
diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go
index 6ecd6321cb23..858eae225a53 100644
--- a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go
+++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go
@@ -146,7 +146,7 @@ func newManaged(
 		router,
 		&configModifiers{
 			Decorators: []decoratorFunc{injectMonitoring},
-			Filters:    []filterFunc{injectFleet(config), filters.ConstraintFilter},
+			Filters:    []filterFunc{filters.StreamChecker, injectFleet(config), filters.ConstraintFilter},
 		},
 		monitor,
 	)

From 5f2f36901393ce7d48d572b3e24d172626dd2288 Mon Sep 17 00:00:00 2001
From: Michal Pristas <michal.pristas@gmail.com>
Date: Thu, 2 Jul 2020 10:37:09 +0200
Subject: [PATCH 02/10] changelog

---
 x-pack/elastic-agent/CHANGELOG.asciidoc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc
index b926202727a9..86ed7e4fe298 100644
--- a/x-pack/elastic-agent/CHANGELOG.asciidoc
+++ b/x-pack/elastic-agent/CHANGELOG.asciidoc
@@ -75,3 +75,4 @@
 - Rename input.type logs to logfile {pull}19360[19360]
 - Agent now installs/uninstalls Elastic Endpoint {pull}19248[19248]
 - Agent now downloads Elastic Endpoint {pull}19503[19503]
+- Refuse invalid stream values in configuration {pull}19587[19587]

From 794c6fb46e76fe80c920dab3af350416626d4475 Mon Sep 17 00:00:00 2001
From: Michal Pristas <michal.pristas@gmail.com>
Date: Thu, 2 Jul 2020 10:53:10 +0200
Subject: [PATCH 03/10] fmt

---
 .../pkg/agent/application/filters/stream_checker_test.go       | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go
index 2f6a77fcb550..ed91b6462a08 100644
--- a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go
+++ b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go
@@ -7,9 +7,10 @@ package filters
 import (
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
-	"github.com/stretchr/testify/assert"
 )
 
 func TestStreamCheck(t *testing.T) {

From 4c8f931cbe064b3058e5d9b66b1cc38cd1c904c3 Mon Sep 17 00:00:00 2001
From: Michal Pristas <michal.pristas@gmail.com>
Date: Thu, 2 Jul 2020 11:22:49 +0200
Subject: [PATCH 04/10] invalid names

---
 .../application/filters/stream_checker.go     |  64 ++++++++++-
 .../filters/stream_checker_test.go            | 108 +++++++++++++++++-
 2 files changed, 165 insertions(+), 7 deletions(-)

diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go
index 3d6cd95df4f2..a7371e4b267e 100644
--- a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go
+++ b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go
@@ -5,6 +5,8 @@
 package filters
 
 import (
+	"strings"
+
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
@@ -116,10 +118,68 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error {
 	return nil
 }
 
+// The only two requirement are that it has only characters allowed in an Elasticsearch index name
+// and does NOT contain a `-`.
+// Index names must meet the following criteria:
+//     Lowercase only
+//     Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+//     Cannot start with -, _, +
+//     Cannot be . or ..
+//     Names starting with . are deprecated, except for hidden indices and internal indices managed by plugins
+
 func isNamespaceValid(namespace string) bool {
-	return len(namespace) > 0
+	// Cannot be . or ..
+	if namespace == "." || namespace == ".." {
+		return false
+	}
+
+	if len(namespace) <= 0 || len(namespace) > 255 {
+		return false
+	}
+
+	// Lowercase only
+	if strings.ToLower(namespace) != namespace {
+		return false
+	}
+
+	// Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+	if strings.ContainsAny(namespace, "\\/*?\"<>| ,#-") {
+		return false
+	}
+
+	// Cannot start with -, _, +
+	if strings.HasPrefix(namespace, "-") || strings.HasPrefix(namespace, "_") || strings.HasPrefix(namespace, "+") {
+		return false
+	}
+
+	return true
 }
 
+// The same requirements as for the namespace apply.
 func isDatasetValid(dataset string) bool {
-	return len(dataset) > 0
+	// Cannot be . or ..
+	if dataset == "." || dataset == ".." {
+		return false
+	}
+
+	if len(dataset) <= 0 || len(dataset) > 255 {
+		return false
+	}
+
+	// Lowercase only
+	if strings.ToLower(dataset) != dataset {
+		return false
+	}
+
+	// Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+	if strings.ContainsAny(dataset, "\\/*?\"<>| ,#-") {
+		return false
+	}
+
+	// Cannot start with -, _, +
+	if strings.HasPrefix(dataset, "-") || strings.HasPrefix(dataset, "_") || strings.HasPrefix(dataset, "+") {
+		return false
+	}
+
+	return true
 }
diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go
index ed91b6462a08..79452448568d 100644
--- a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go
+++ b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go
@@ -31,8 +31,8 @@ func TestStreamCheck(t *testing.T) {
 			configMap: map[string]interface{}{
 				"inputs": []map[string]interface{}{
 					{
-						"dataset.namespace": "someNamespace",
-						"streams":           []map[string]interface{}{{"dataset.name": "someDatasetName"}},
+						"dataset.namespace": "somenamespace",
+						"streams":           []map[string]interface{}{{"dataset.name": "somedatasetname"}},
 					},
 				},
 			},
@@ -44,12 +44,12 @@ func TestStreamCheck(t *testing.T) {
 				"inputs": []map[string]interface{}{
 					{
 						"dataset": map[string]interface{}{
-							"namespace": "someNamespace",
+							"namespace": "somenamespace",
 						},
 						"streams": []map[string]interface{}{
 							{
 								"dataset": map[string]interface{}{
-									"name": "someDatasetName",
+									"name": "somedatasetname",
 								},
 							},
 						},
@@ -84,15 +84,113 @@ func TestStreamCheck(t *testing.T) {
 			},
 			result: ErrInvalidDataset,
 		},
+
+		{
+			name: "dataset.name invalid dot - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{"streams": []map[string]interface{}{{"dataset.name": "."}}},
+				},
+			},
+			result: ErrInvalidDataset,
+		},
+		{
+			name: "dataset.name invalid dotdot- compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{"streams": []map[string]interface{}{{"dataset.name": ".."}}},
+				},
+			},
+			result: ErrInvalidDataset,
+		},
+		{
+			name: "dataset.name invalid uppercase - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{"streams": []map[string]interface{}{{"dataset.name": "myNameIs"}}},
+				},
+			},
+			result: ErrInvalidDataset,
+		},
+		{
+			name: "dataset.name invalid space- compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{"streams": []map[string]interface{}{{"dataset.name": "outer space"}}},
+				},
+			},
+			result: ErrInvalidDataset,
+		},
+		{
+			name: "dataset.name invalid invalid char- compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{"streams": []map[string]interface{}{{"dataset.name": "is\\thisvalid"}}},
+				},
+			},
+			result: ErrInvalidDataset,
+		},
+		{
+			name: "dataset.name invalid invalid prefix- compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{"streams": []map[string]interface{}{{"dataset.name": "_isthisvalid"}}},
+				},
+			},
+			result: ErrInvalidDataset,
+		},
+
 		{
 			name: "namespace invalid - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{{"dataset.namespace": ""}},
+			},
+			result: ErrInvalidNamespace,
+		},
+		{
+			name: "namespace invalid name 1 - compact",
 			configMap: map[string]interface{}{
 				"inputs": []map[string]interface{}{
-					{"dataset.namespace": ""},
+					{"dataset.namespace": "."},
 				},
 			},
 			result: ErrInvalidNamespace,
 		},
+		{
+			name: "namespace invalid name 2 - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{{"dataset.namespace": ".."}},
+			},
+			result: ErrInvalidNamespace,
+		},
+		{
+			name: "namespace invalid name uppercase - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{{"dataset.namespace": "someUpper"}},
+			},
+			result: ErrInvalidNamespace,
+		},
+		{
+			name: "namespace invalid name space - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{{"dataset.namespace": "some space"}},
+			},
+			result: ErrInvalidNamespace,
+		},
+		{
+			name: "namespace invalid name invalid char - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{{"dataset.namespace": "isitok?"}},
+			},
+			result: ErrInvalidNamespace,
+		},
+		{
+			name: "namespace invalid name invalid prefix - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{{"dataset.namespace": "+isitok"}},
+			},
+			result: ErrInvalidNamespace,
+		},
 		{
 			name: "namespace invalid - long",
 			configMap: map[string]interface{}{

From 4d386baaa4f89b5ab43d0a213582d8bdd74ec0c1 Mon Sep 17 00:00:00 2001
From: Michal Pristas <michal.pristas@gmail.com>
Date: Thu, 2 Jul 2020 11:41:07 +0200
Subject: [PATCH 05/10] updated config files

---
 x-pack/elastic-agent/elastic-agent.reference.yml | 12 +++++++++++-
 x-pack/elastic-agent/elastic-agent.yml           | 10 ++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/x-pack/elastic-agent/elastic-agent.reference.yml b/x-pack/elastic-agent/elastic-agent.reference.yml
index d5a5966dfb48..8c587be54399 100644
--- a/x-pack/elastic-agent/elastic-agent.reference.yml
+++ b/x-pack/elastic-agent/elastic-agent.reference.yml
@@ -16,14 +16,24 @@ outputs:
 
 inputs:
   - type: system/metrics
+
+    # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+    # and does NOT contain a `-`.
+    # Index names must meet the following criteria:
+    #   Lowercase only
+    #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+    #   Cannot start with -, _, +
+    #   Cannot be . or ..
+    #   Names starting with . are deprecated, except for hidden indices and internal indices managed by plugins
     dataset.namespace: default
     use_output: default
     streams:
       - metricset: cpu
+        # Same requirements as for the namespace apply.
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory
-      - metricset: network
+      - metricset: network≈
         dataset.name: system.network
       - metricset: filesystem
         dataset.name: system.filesystem
diff --git a/x-pack/elastic-agent/elastic-agent.yml b/x-pack/elastic-agent/elastic-agent.yml
index d1c7c6220be5..00eb5d332d35 100644
--- a/x-pack/elastic-agent/elastic-agent.yml
+++ b/x-pack/elastic-agent/elastic-agent.yml
@@ -16,10 +16,20 @@ outputs:
 
 inputs:
   - type: system/metrics
+
+  # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+  # and does NOT contain a `-`.
+  # Index names must meet the following criteria:
+  #   Lowercase only
+  #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+  #   Cannot start with -, _, +
+  #   Cannot be . or ..
+  #   Names starting with . are deprecated, except for hidden indices and internal indices managed by plugins
     dataset.namespace: default
     use_output: default
     streams:
       - metricset: cpu
+        # Same requirements as for the namespace apply.
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory

From 4314afb86416661bb3f86a8e3da94863e9c01221 Mon Sep 17 00:00:00 2001
From: Michal Pristas <michal.pristas@gmail.com>
Date: Thu, 2 Jul 2020 11:41:37 +0200
Subject: [PATCH 06/10] updated config files

---
 x-pack/elastic-agent/elastic-agent.reference.yml                 | 1 -
 .../pkg/agent/application/filters/stream_checker.go              | 1 -
 2 files changed, 2 deletions(-)

diff --git a/x-pack/elastic-agent/elastic-agent.reference.yml b/x-pack/elastic-agent/elastic-agent.reference.yml
index 8c587be54399..f0e15c5a8a5d 100644
--- a/x-pack/elastic-agent/elastic-agent.reference.yml
+++ b/x-pack/elastic-agent/elastic-agent.reference.yml
@@ -24,7 +24,6 @@ inputs:
     #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
     #   Cannot start with -, _, +
     #   Cannot be . or ..
-    #   Names starting with . are deprecated, except for hidden indices and internal indices managed by plugins
     dataset.namespace: default
     use_output: default
     streams:
diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go
index a7371e4b267e..08f1bd213cb8 100644
--- a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go
+++ b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go
@@ -125,7 +125,6 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error {
 //     Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
 //     Cannot start with -, _, +
 //     Cannot be . or ..
-//     Names starting with . are deprecated, except for hidden indices and internal indices managed by plugins
 
 func isNamespaceValid(namespace string) bool {
 	// Cannot be . or ..

From f703cbede2a9c2f4dc157efa716cf9ba8cf3e900 Mon Sep 17 00:00:00 2001
From: Michal Pristas <michal.pristas@gmail.com>
Date: Thu, 2 Jul 2020 12:01:54 +0200
Subject: [PATCH 07/10] changes in all config files

---
 x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl   |  4 ++++
 .../_meta/config/common.reference.p2.yml.tmpl          |  4 ++++
 .../_meta/config/elastic-agent.docker.yml.tmpl         |  4 ++++
 x-pack/elastic-agent/_meta/elastic-agent.yml           |  4 ++++
 x-pack/elastic-agent/elastic-agent.docker.yml          |  4 ++++
 x-pack/elastic-agent/elastic-agent.reference.yml       |  5 -----
 x-pack/elastic-agent/elastic-agent.yml                 | 10 ++--------
 7 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl
index 81dbde654195..7f9da1d13c7c 100644
--- a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl
+++ b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl
@@ -10,10 +10,14 @@ outputs:
 
 inputs:
   - type: system/metrics
+
+    # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+    # and does NOT contain a `-`.
     dataset.namespace: default
     use_output: default
     streams:
       - metricset: cpu
+        # Same requirements as for the namespace apply.
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory
diff --git a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl
index 59ea36c70bc4..c300fe16a63e 100644
--- a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl
+++ b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl
@@ -10,10 +10,14 @@ outputs:
 
 inputs:
   - type: system/metrics
+
+    # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+    # and does NOT contain a `-`.
     dataset.namespace: default
     use_output: default
     streams:
       - metricset: cpu
+        # Same requirements as for the namespace apply.
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory
diff --git a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl
index 212d6944b6ef..4eec8877bca9 100644
--- a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl
+++ b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl
@@ -10,10 +10,14 @@ outputs:
 
 inputs:
   - type: system/metrics
+
+    # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+    # and does NOT contain a `-`.
     dataset.namespace: default
     use_output: default
     streams:
       - metricset: cpu
+        # Same requirements as for the namespace apply.
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory
diff --git a/x-pack/elastic-agent/_meta/elastic-agent.yml b/x-pack/elastic-agent/_meta/elastic-agent.yml
index e8d47c9ea75e..5dff5db4ce83 100644
--- a/x-pack/elastic-agent/_meta/elastic-agent.yml
+++ b/x-pack/elastic-agent/_meta/elastic-agent.yml
@@ -10,10 +10,14 @@ outputs:
 
 inputs:
   - type: system/metrics
+
+    # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+    # and does NOT contain a `-`.
     dataset.namespace: default
     use_output: default
     streams:
       - metricset: cpu
+        # Same requirements as for the namespace apply.
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory
diff --git a/x-pack/elastic-agent/elastic-agent.docker.yml b/x-pack/elastic-agent/elastic-agent.docker.yml
index 8c51ecd61206..c3cfbc6e4b3d 100644
--- a/x-pack/elastic-agent/elastic-agent.docker.yml
+++ b/x-pack/elastic-agent/elastic-agent.docker.yml
@@ -10,10 +10,14 @@ outputs:
 
 inputs:
   - type: system/metrics
+
+    # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+    # and does NOT contain a `-`.
     dataset.namespace: default
     use_output: default
     streams:
       - metricset: cpu
+        # Same requirements as for the namespace apply.
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory
diff --git a/x-pack/elastic-agent/elastic-agent.reference.yml b/x-pack/elastic-agent/elastic-agent.reference.yml
index f0e15c5a8a5d..828433ea6e87 100644
--- a/x-pack/elastic-agent/elastic-agent.reference.yml
+++ b/x-pack/elastic-agent/elastic-agent.reference.yml
@@ -19,11 +19,6 @@ inputs:
 
     # The only two requirement are that it has only characters allowed in an Elasticsearch index name
     # and does NOT contain a `-`.
-    # Index names must meet the following criteria:
-    #   Lowercase only
-    #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
-    #   Cannot start with -, _, +
-    #   Cannot be . or ..
     dataset.namespace: default
     use_output: default
     streams:
diff --git a/x-pack/elastic-agent/elastic-agent.yml b/x-pack/elastic-agent/elastic-agent.yml
index 00eb5d332d35..36e53c834cf2 100644
--- a/x-pack/elastic-agent/elastic-agent.yml
+++ b/x-pack/elastic-agent/elastic-agent.yml
@@ -17,14 +17,8 @@ outputs:
 inputs:
   - type: system/metrics
 
-  # The only two requirement are that it has only characters allowed in an Elasticsearch index name
-  # and does NOT contain a `-`.
-  # Index names must meet the following criteria:
-  #   Lowercase only
-  #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
-  #   Cannot start with -, _, +
-  #   Cannot be . or ..
-  #   Names starting with . are deprecated, except for hidden indices and internal indices managed by plugins
+    # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+    # and does NOT contain a `-`.
     dataset.namespace: default
     use_output: default
     streams:

From 128529ecc5c79694af4ef76b107dee83e5eaf249 Mon Sep 17 00:00:00 2001
From: Michal Pristas <michal.pristas@gmail.com>
Date: Thu, 2 Jul 2020 13:24:48 +0200
Subject: [PATCH 08/10] config typo

---
 x-pack/elastic-agent/elastic-agent.reference.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/elastic-agent/elastic-agent.reference.yml b/x-pack/elastic-agent/elastic-agent.reference.yml
index 828433ea6e87..e23af983335c 100644
--- a/x-pack/elastic-agent/elastic-agent.reference.yml
+++ b/x-pack/elastic-agent/elastic-agent.reference.yml
@@ -27,7 +27,7 @@ inputs:
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory
-      - metricset: network≈
+      - metricset: network
         dataset.name: system.network
       - metricset: filesystem
         dataset.name: system.filesystem

From a6d685cd4a10f9f269f73d818b4f8d189997aba6 Mon Sep 17 00:00:00 2001
From: Michal Pristas <michal.pristas@gmail.com>
Date: Fri, 3 Jul 2020 12:48:27 +0200
Subject: [PATCH 09/10] small changes

---
 .../_meta/config/common.p2.yml.tmpl           |  13 +-
 .../_meta/config/common.reference.p2.yml.tmpl |  13 +-
 .../config/elastic-agent.docker.yml.tmpl      |  13 +-
 x-pack/elastic-agent/_meta/elastic-agent.yml  |  13 +-
 x-pack/elastic-agent/elastic-agent.docker.yml |  13 +-
 .../elastic-agent/elastic-agent.reference.yml |  13 +-
 x-pack/elastic-agent/elastic-agent.yml        |  13 +-
 .../application/filters/stream_checker.go     | 147 ++++++++++--------
 .../filters/stream_checker_test.go            |  70 +++++++++
 9 files changed, 228 insertions(+), 80 deletions(-)

diff --git a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl
index 7f9da1d13c7c..926d67f69c2d 100644
--- a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl
+++ b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl
@@ -12,12 +12,21 @@ inputs:
   - type: system/metrics
 
     # The only two requirement are that it has only characters allowed in an Elasticsearch index name
-    # and does NOT contain a `-`.
+    # Index names must meet the following criteria:
+    #   Lowercase only
+    #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+    #   Cannot start with -, _, +
+    #   Cannot be . or ..
     dataset.namespace: default
     use_output: default
     streams:
       - metricset: cpu
-        # Same requirements as for the namespace apply.
+        # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+        # Index names must meet the following criteria:
+        #   Lowercase only
+        #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+        #   Cannot start with -, _, +
+        #   Cannot be . or ..
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory
diff --git a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl
index c300fe16a63e..7f8b2743b828 100644
--- a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl
+++ b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl
@@ -12,12 +12,21 @@ inputs:
   - type: system/metrics
 
     # The only two requirement are that it has only characters allowed in an Elasticsearch index name
-    # and does NOT contain a `-`.
+    # Index names must meet the following criteria:
+    #   Lowercase only
+    #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+    #   Cannot start with -, _, +
+    #   Cannot be . or ..
     dataset.namespace: default
     use_output: default
     streams:
       - metricset: cpu
-        # Same requirements as for the namespace apply.
+        # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+        # Index names must meet the following criteria:
+        #   Lowercase only
+        #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+        #   Cannot start with -, _, +
+        #   Cannot be . or ..
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory
diff --git a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl
index 4eec8877bca9..144c2cd6962a 100644
--- a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl
+++ b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl
@@ -12,12 +12,21 @@ inputs:
   - type: system/metrics
 
     # The only two requirement are that it has only characters allowed in an Elasticsearch index name
-    # and does NOT contain a `-`.
+    # Index names must meet the following criteria:
+    #   Lowercase only
+    #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+    #   Cannot start with -, _, +
+    #   Cannot be . or ..
     dataset.namespace: default
     use_output: default
     streams:
       - metricset: cpu
-        # Same requirements as for the namespace apply.
+        # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+        # Index names must meet the following criteria:
+        #   Lowercase only
+        #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+        #   Cannot start with -, _, +
+        #   Cannot be . or ..
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory
diff --git a/x-pack/elastic-agent/_meta/elastic-agent.yml b/x-pack/elastic-agent/_meta/elastic-agent.yml
index 5dff5db4ce83..a5ad3cd0f942 100644
--- a/x-pack/elastic-agent/_meta/elastic-agent.yml
+++ b/x-pack/elastic-agent/_meta/elastic-agent.yml
@@ -12,12 +12,21 @@ inputs:
   - type: system/metrics
 
     # The only two requirement are that it has only characters allowed in an Elasticsearch index name
-    # and does NOT contain a `-`.
+    # Index names must meet the following criteria:
+    #   Lowercase only
+    #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+    #   Cannot start with -, _, +
+    #   Cannot be . or ..
     dataset.namespace: default
     use_output: default
     streams:
       - metricset: cpu
-        # Same requirements as for the namespace apply.
+        # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+        # Index names must meet the following criteria:
+        #   Lowercase only
+        #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+        #   Cannot start with -, _, +
+        #   Cannot be . or ..
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory
diff --git a/x-pack/elastic-agent/elastic-agent.docker.yml b/x-pack/elastic-agent/elastic-agent.docker.yml
index c3cfbc6e4b3d..0f9b2935d4a2 100644
--- a/x-pack/elastic-agent/elastic-agent.docker.yml
+++ b/x-pack/elastic-agent/elastic-agent.docker.yml
@@ -12,12 +12,21 @@ inputs:
   - type: system/metrics
 
     # The only two requirement are that it has only characters allowed in an Elasticsearch index name
-    # and does NOT contain a `-`.
+    # Index names must meet the following criteria:
+    #   Lowercase only
+    #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+    #   Cannot start with -, _, +
+    #   Cannot be . or ..
     dataset.namespace: default
     use_output: default
     streams:
       - metricset: cpu
-        # Same requirements as for the namespace apply.
+        # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+        # Index names must meet the following criteria:
+        #   Lowercase only
+        #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+        #   Cannot start with -, _, +
+        #   Cannot be . or ..
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory
diff --git a/x-pack/elastic-agent/elastic-agent.reference.yml b/x-pack/elastic-agent/elastic-agent.reference.yml
index e23af983335c..c3c156c09fef 100644
--- a/x-pack/elastic-agent/elastic-agent.reference.yml
+++ b/x-pack/elastic-agent/elastic-agent.reference.yml
@@ -18,12 +18,21 @@ inputs:
   - type: system/metrics
 
     # The only two requirement are that it has only characters allowed in an Elasticsearch index name
-    # and does NOT contain a `-`.
+    # Index names must meet the following criteria:
+    #   Lowercase only
+    #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+    #   Cannot start with -, _, +
+    #   Cannot be . or ..
     dataset.namespace: default
     use_output: default
     streams:
       - metricset: cpu
-        # Same requirements as for the namespace apply.
+        # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+        # Index names must meet the following criteria:
+        #   Lowercase only
+        #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+        #   Cannot start with -, _, +
+        #   Cannot be . or ..
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory
diff --git a/x-pack/elastic-agent/elastic-agent.yml b/x-pack/elastic-agent/elastic-agent.yml
index 36e53c834cf2..847abfdcf9b9 100644
--- a/x-pack/elastic-agent/elastic-agent.yml
+++ b/x-pack/elastic-agent/elastic-agent.yml
@@ -18,12 +18,21 @@ inputs:
   - type: system/metrics
 
     # The only two requirement are that it has only characters allowed in an Elasticsearch index name
-    # and does NOT contain a `-`.
+    # Index names must meet the following criteria:
+    #   Lowercase only
+    #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+    #   Cannot start with -, _, +
+    #   Cannot be . or ..
     dataset.namespace: default
     use_output: default
     streams:
       - metricset: cpu
-        # Same requirements as for the namespace apply.
+        # The only two requirement are that it has only characters allowed in an Elasticsearch index name
+        # Index names must meet the following criteria:
+        #   Lowercase only
+        #   Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
+        #   Cannot start with -, _, +
+        #   Cannot be . or ..
         dataset.name: system.cpu
       - metricset: memory
         dataset.name: system.memory
diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go
index 08f1bd213cb8..470cb776ab13 100644
--- a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go
+++ b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go
@@ -5,6 +5,7 @@
 package filters
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
@@ -18,6 +19,9 @@ var ErrInvalidNamespace = errors.New("provided namespace is invalid", errors.Typ
 // ErrInvalidDataset is error returned when dataset name value provided is invalid.
 var ErrInvalidDataset = errors.New("provided dataset name is invalid", errors.TypeConfig)
 
+// ErrInvalidIndex occurs when concatenation of {dataset.type}-{dataset.name}-{dataset.namespace} does not meet index criteria.
+var ErrInvalidIndex = errors.New("provided combination of type, dataset name and namespace is invalid", errors.TypeConfig)
+
 // StreamChecker checks for invalid values in stream namespace and dataset.
 func StreamChecker(log *logger.Logger, ast *transpiler.AST) error {
 	inputsNode, found := transpiler.Lookup(ast, "inputs")
@@ -36,14 +40,18 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error {
 	}
 
 	for _, inputNode := range inputsNodeListCollection {
+		namespace := "default"
+		datasetName := "generic"
 		// fail only if dataset.namespace or dataset[namespace] is found and invalid
 		// not provided values are ok and will be fixed by rules
 		if nsNode, found := inputNode.Find("dataset.namespace"); found {
 			nsKey, ok := nsNode.(*transpiler.Key)
 			if ok {
-				if newNamespace := nsKey.Value().(transpiler.Node).String(); !isNamespaceValid(newNamespace) {
+				newNamespace := nsKey.Value().(transpiler.Node).String()
+				if !isValid(newNamespace) {
 					return ErrInvalidNamespace
 				}
+				namespace = newNamespace
 			}
 		} else {
 			dsNode, found := inputNode.Find("dataset")
@@ -55,64 +63,96 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error {
 					if found {
 						nsKey, ok := nsNode.(*transpiler.Key)
 						if ok {
-							if newNamespace := nsKey.Value().(transpiler.Node).String(); !isNamespaceValid(newNamespace) {
+							newNamespace := nsKey.Value().(transpiler.Node).String()
+							if !isValid(newNamespace) {
 								return ErrInvalidNamespace
 							}
+							namespace = newNamespace
 						}
 					}
 				}
 			}
 		}
 
-		streamsNode, ok := inputNode.Find("streams")
-		if !ok {
-			continue
-		}
-
-		streamsList, ok := streamsNode.Value().(*transpiler.List)
-		if !ok {
-			continue
-		}
-
-		streamNodes, ok := streamsList.Value().([]transpiler.Node)
-		if !ok {
-			return errors.New("streams is not a list", errors.TypeConfig)
-		}
-
-		for _, streamNode := range streamNodes {
-			streamMap, ok := streamNode.(*transpiler.Dict)
-			if !ok {
-				continue
+		// get the type, longest type for now is metrics
+		datasetType := "metrics"
+		if nsNode, found := inputNode.Find("dataset.type"); found {
+			nsKey, ok := nsNode.(*transpiler.Key)
+			if ok {
+				newDataset := nsKey.Value().(transpiler.Node).String()
+				datasetType = newDataset
 			}
-
-			// fix this only if in compact form
-			if dsNameNode, found := streamMap.Find("dataset.name"); found {
-				dsKey, ok := dsNameNode.(*transpiler.Key)
+		} else {
+			dsNode, found := inputNode.Find("dataset")
+			if found {
+				// got a dataset
+				datasetMap, ok := dsNode.Value().(*transpiler.Dict)
 				if ok {
-					if newDataset := dsKey.Value().(transpiler.Node).String(); !isDatasetValid(newDataset) {
-						return ErrInvalidDataset
+					nsNode, found := datasetMap.Find("type")
+					if found {
+						nsKey, ok := nsNode.(*transpiler.Key)
+						if ok {
+							newDataset := nsKey.Value().(transpiler.Node).String()
+							datasetType = newDataset
+						}
 					}
 				}
-			} else {
-				datasetNode, found := streamMap.Find("dataset")
-				if found {
-					datasetMap, ok := datasetNode.Value().(*transpiler.Dict)
+			}
+		}
+
+		streamsNode, ok := inputNode.Find("streams")
+		if ok {
+			streamsList, ok := streamsNode.Value().(*transpiler.List)
+			if ok {
+				streamNodes, ok := streamsList.Value().([]transpiler.Node)
+				if !ok {
+					return errors.New("streams is not a list", errors.TypeConfig)
+				}
+
+				for _, streamNode := range streamNodes {
+					streamMap, ok := streamNode.(*transpiler.Dict)
 					if !ok {
 						continue
 					}
 
-					dsNameNode, found := datasetMap.Find("name")
-					if found {
+					// fix this only if in compact form
+					if dsNameNode, found := streamMap.Find("dataset.name"); found {
 						dsKey, ok := dsNameNode.(*transpiler.Key)
 						if ok {
-							if newDataset := dsKey.Value().(transpiler.Node).String(); !isDatasetValid(newDataset) {
+							newDataset := dsKey.Value().(transpiler.Node).String()
+							if !isValid(newDataset) {
 								return ErrInvalidDataset
 							}
+							datasetName = newDataset
+						}
+					} else {
+						datasetNode, found := streamMap.Find("dataset")
+						if found {
+							datasetMap, ok := datasetNode.Value().(*transpiler.Dict)
+							if !ok {
+								continue
+							}
+
+							dsNameNode, found := datasetMap.Find("name")
+							if found {
+								dsKey, ok := dsNameNode.(*transpiler.Key)
+								if ok {
+									newDataset := dsKey.Value().(transpiler.Node).String()
+									if !isValid(newDataset) {
+										return ErrInvalidDataset
+									}
+									datasetName = newDataset
+								}
+							}
 						}
 					}
 				}
 			}
 		}
+
+		if indexName := fmt.Sprintf("%s-%s-%s", datasetType, datasetName, namespace); !matchesIndexContraints(indexName) {
+			return ErrInvalidIndex
+		}
 	}
 
 	return nil
@@ -120,13 +160,17 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error {
 
 // The only two requirement are that it has only characters allowed in an Elasticsearch index name
 // and does NOT contain a `-`.
+func isValid(namespace string) bool {
+	return matchesIndexContraints(namespace) && !strings.Contains(namespace, "-")
+}
+
+// The only two requirement are that it has only characters allowed in an Elasticsearch index name
 // Index names must meet the following criteria:
 //     Lowercase only
 //     Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
 //     Cannot start with -, _, +
 //     Cannot be . or ..
-
-func isNamespaceValid(namespace string) bool {
+func matchesIndexContraints(namespace string) bool {
 	// Cannot be . or ..
 	if namespace == "." || namespace == ".." {
 		return false
@@ -142,7 +186,7 @@ func isNamespaceValid(namespace string) bool {
 	}
 
 	// Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
-	if strings.ContainsAny(namespace, "\\/*?\"<>| ,#-") {
+	if strings.ContainsAny(namespace, "\\/*?\"<>| ,#") {
 		return false
 	}
 
@@ -153,32 +197,3 @@ func isNamespaceValid(namespace string) bool {
 
 	return true
 }
-
-// The same requirements as for the namespace apply.
-func isDatasetValid(dataset string) bool {
-	// Cannot be . or ..
-	if dataset == "." || dataset == ".." {
-		return false
-	}
-
-	if len(dataset) <= 0 || len(dataset) > 255 {
-		return false
-	}
-
-	// Lowercase only
-	if strings.ToLower(dataset) != dataset {
-		return false
-	}
-
-	// Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
-	if strings.ContainsAny(dataset, "\\/*?\"<>| ,#-") {
-		return false
-	}
-
-	// Cannot start with -, _, +
-	if strings.HasPrefix(dataset, "-") || strings.HasPrefix(dataset, "_") || strings.HasPrefix(dataset, "+") {
-		return false
-	}
-
-	return true
-}
diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go
index 79452448568d..b6d9bc22e0d9 100644
--- a/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go
+++ b/x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go
@@ -5,6 +5,9 @@
 package filters
 
 import (
+	"crypto/sha512"
+	"encoding/hex"
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -13,6 +16,8 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 )
 
+const semiLongString = ""
+
 func TestStreamCheck(t *testing.T) {
 	type testCase struct {
 		name      string
@@ -20,6 +25,10 @@ func TestStreamCheck(t *testing.T) {
 		result    error
 	}
 
+	h := hex.EncodeToString(sha512.New().Sum(nil))
+	semiLongString := h[:86]
+	longString := fmt.Sprintf("%s%s", h, h)
+
 	testCases := []testCase{
 		{
 			name:      "all missing",
@@ -204,6 +213,67 @@ func TestStreamCheck(t *testing.T) {
 			},
 			result: ErrInvalidNamespace,
 		},
+		{
+			name: "type invalid name 1 - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{"dataset.type": "-invalidstart"},
+				},
+			},
+			result: ErrInvalidIndex,
+		},
+		{
+			name: "type invalid combined length 1 - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{
+						"dataset.type":      semiLongString,
+						"dataset.namespace": semiLongString,
+						"streams":           []map[string]interface{}{{"dataset.name": semiLongString}},
+					},
+				},
+			},
+			result: ErrInvalidIndex,
+		},
+		{
+			name: "type invalid type length 1 - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{"dataset.type": longString},
+				},
+			},
+			result: ErrInvalidIndex,
+		},
+
+		{
+			name: "type invalid namespace length 1 - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{"dataset.namespace": longString},
+				},
+			},
+			result: ErrInvalidNamespace,
+		},
+
+		{
+			name: "type invalid dataset.name length 1 - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{"streams": []map[string]interface{}{{"dataset.name": longString}}},
+				},
+			},
+			result: ErrInvalidDataset,
+		},
+
+		{
+			name: "type empty streams - compact",
+			configMap: map[string]interface{}{
+				"inputs": []map[string]interface{}{
+					{"streams": []map[string]interface{}{}},
+				},
+			},
+			result: nil,
+		},
 	}
 
 	log, err := logger.New("")

From 90594b79adb77f9070990c6198c38683827190d6 Mon Sep 17 00:00:00 2001
From: Michal Pristas <michal.pristas@gmail.com>
Date: Mon, 13 Jul 2020 12:43:51 +0200
Subject: [PATCH 10/10] Update go.sum

---
 go.sum | 1 -
 1 file changed, 1 deletion(-)

diff --git a/go.sum b/go.sum
index 73e445d1b72e..fd1e638b44b1 100644
--- a/go.sum
+++ b/go.sum
@@ -222,7 +222,6 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA=
 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
-github.com/elastic/beats v7.6.2+incompatible h1:jHdLv83KURaqWUC6f55iMyVP6LYZrgElfeqxKWcskVE=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
 github.com/elastic/ecs v1.5.0 h1:/VEIBsRU4ecq2+U3RPfKNc6bFyomP6qnthYEcQZu8GU=