Have filesets disabled unless explicitly configured

This changes fileset loading so that only filesets that are explicitly
defined in the configuration are enabled.

Until now, an enabled module will have all its filesets enabled unless
explicitly disabled, which makes for a bad user experience with modules
that contain a lot of filesets.

Closes elastic#17256

CHANGELOG.next.asciidoc

@@ -84,6 +84,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove all alias fields pointing to ECS fields from modules. This affects the Suricata and Traefik modules. {issue}10535[10535] {pull}26627[26627]
 - Add option for S3 input to work without SQS notification {issue}18205[18205] {pull}27332[27332]
 - Fix Crowdstrike ingest pipeline that was creating flattened `process` fields. {issue}27622[27622] {pull}27623[27623]
+- Only filesets that are explicitly configured will be enabled. {issue}17256[17256] {pull}27526[27526]
 
 *Heartbeat*
 - Remove long deprecated `watch_poll` functionality. {pull}27166[27166]

filebeat/fileset/modules.go

@@ -69,33 +69,13 @@ func newModuleRegistry(modulesPath string,
 			return nil, fmt.Errorf("error getting filesets for module %s: %v", mcfg.Module, err)
 		}
 
-		for _, filesetName := range moduleFilesets {
-			fcfg, exists := mcfg.Filesets[filesetName]
-			if !exists {
-				fcfg = &FilesetConfig{}
-			}
+		for filesetName, fcfg := range mcfg.Filesets {
 
 			fcfg, err = applyOverrides(fcfg, mcfg.Module, filesetName, overrides)
 			if err != nil {
 				return nil, fmt.Errorf("error applying overrides on fileset %s/%s: %v", mcfg.Module, filesetName, err)
 			}
 
-			if fcfg.Enabled != nil && !(*fcfg.Enabled) {
-				continue
-			}
-
-			fileset, err := New(modulesPath, filesetName, mcfg, fcfg)
-			if err != nil {
-				return nil, err
-			}
-			if err = fileset.Read(beatInfo); err != nil {
-				return nil, fmt.Errorf("error reading fileset %s/%s: %v", mcfg.Module, filesetName, err)
-			}
-			reg.registry[mcfg.Module][filesetName] = fileset
-		}
-
-		// check that no extra filesets are configured
-		for filesetName, fcfg := range mcfg.Filesets {
 			if fcfg.Enabled != nil && !(*fcfg.Enabled) {
 				continue
 			}
@@ -108,6 +88,15 @@ func newModuleRegistry(modulesPath string,
 			if !found {
 				return nil, fmt.Errorf("fileset %s/%s is configured but doesn't exist", mcfg.Module, filesetName)
 			}
+
+			fileset, err := New(modulesPath, filesetName, mcfg, fcfg)
+			if err != nil {
+				return nil, err
+			}
+			if err = fileset.Read(beatInfo); err != nil {
+				return nil, fmt.Errorf("error reading fileset %s/%s: %v", mcfg.Module, filesetName, err)
+			}
+			reg.registry[mcfg.Module][filesetName] = fileset
 		}
 	}
 
@@ -171,11 +160,18 @@ func mcfgFromConfig(cfg *common.Config) (*ModuleConfig, error) {
 	}
 
 	mcfg.Filesets = map[string]*FilesetConfig{}
-	for name, filesetConfig := range dict {
+
+	// This calls cfg.GetFields() instead of iterating over `dict` keys
+	// because cfg.Unpack above doesn't return keys that map to a nil value,
+	// but GetFields() returns all keys. We need to observe filesets that
+	// don't contain any configuration (all default values).
+	for _, name := range cfg.GetFields() {
 		if name == "module" || name == "enabled" || name == "path" {
 			continue
 		}
 
+		filesetConfig, _ := dict[name] // Nil config if name is not present.
+
 		tmpCfg, err := common.NewConfigFrom(filesetConfig)
 		if err != nil {
 			return nil, fmt.Errorf("error creating config from fileset %s/%s: %v", mcfg.Module, name, err)

filebeat/fileset/modules_test.go

@@ -46,10 +46,34 @@ func TestNewModuleRegistry(t *testing.T) {
 	require.NoError(t, err)
 
 	configs := []*ModuleConfig{
-		{Module: "nginx"},
-		{Module: "mysql"},
-		{Module: "system"},
-		{Module: "auditd"},
+		{
+			Module: "nginx",
+			Filesets: map[string]*FilesetConfig{
+				"access":             {},
+				"error":              {},
+				"ingress_controller": {},
+			},
+		},
+		{
+			Module: "mysql",
+			Filesets: map[string]*FilesetConfig{
+				"slowlog": {},
+				"error":   {},
+			},
+		},
+		{
+			Module: "system",
+			Filesets: map[string]*FilesetConfig{
+				"syslog": {},
+				"auth":   {},
+			},
+		},
+		{
+			Module: "auditd",
+			Filesets: map[string]*FilesetConfig{
+				"log": {},
+			},
+		},
 	}
 
 	reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"})
@@ -374,6 +398,19 @@ func TestMcfgFromConfig(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "empty fileset (nil)",
+			config: load(t, map[string]interface{}{
+				"module": "nginx",
+				"error":  nil,
+			}),
+			expected: ModuleConfig{
+				Module: "nginx",
+				Filesets: map[string]*FilesetConfig{
+					"error": {},
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {