diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index 008bb36860b8..b80adc5c8483 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -116,6 +116,29 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *common.Config) (b } if !moduleRegistry.Empty() { logp.Info("Enabled modules/filesets: %s", moduleRegistry.InfoString()) + + // Deprecation warning logic for v8.0 (https://github.com/elastic/beats/pull/27526) + for _, mod := range moduleRegistry.ModuleNames() { + if mod == "" { + continue + } + + loadedFilesets, err := moduleRegistry.ModuleFilesets(mod) + if err != nil { + logp.Err("Error retrieving module filesets: %+v", err) + return nil, err + } + + configuredFilesets := moduleRegistry.ModuleConfiguredFilesets(mod) + if len(configuredFilesets) != len(loadedFilesets) { + for _, loadedFileset := range loadedFilesets { + if _, ok := configuredFilesets[loadedFileset]; !ok { + logp.Warn("Fileset `%s` for module `%s` is loaded but was not explicitly defined in the config. "+ + "Starting from v8.0 this fileset won't be loaded unless explicitly defined.", loadedFileset, mod) + } + } + } + } } moduleInputs, err := moduleRegistry.GetInputConfigs() diff --git a/filebeat/fileset/modules.go b/filebeat/fileset/modules.go index 69b49b013b6e..f3c94deefca6 100644 --- a/filebeat/fileset/modules.go +++ b/filebeat/fileset/modules.go @@ -44,19 +44,22 @@ var availableMLModules = map[string]string{ const logName = "modules" type ModuleRegistry struct { - registry map[string]map[string]*Fileset // module -> fileset -> Fileset - log *logp.Logger + configuredFilesets map[string]map[string]struct{} // module -> fileset -> struct{} + registry map[string]map[string]*Fileset // module -> fileset -> Fileset + log *logp.Logger } // newModuleRegistry reads and loads the configured module into the registry. func newModuleRegistry(modulesPath string, moduleConfigs []*ModuleConfig, overrides *ModuleOverrides, + configuredFilesets map[string]map[string]struct{}, beatInfo beat.Info, ) (*ModuleRegistry, error) { reg := ModuleRegistry{ - registry: map[string]map[string]*Fileset{}, - log: logp.NewLogger(logName), + configuredFilesets: configuredFilesets, + registry: map[string]map[string]*Fileset{}, + log: logp.NewLogger(logName), } for _, mcfg := range moduleConfigs { @@ -77,15 +80,23 @@ func newModuleRegistry(modulesPath string, } for _, filesetName := range moduleFilesets { + var hasOverride bool fcfg, exists := mcfg.Filesets[filesetName] if !exists { fcfg = &FilesetConfig{} } - fcfg, err = applyOverrides(fcfg, mcfg.Module, filesetName, overrides) + fcfg, hasOverride, err = applyOverrides(fcfg, mcfg.Module, filesetName, overrides) if err != nil { return nil, fmt.Errorf("error applying overrides on fileset %s/%s: %v", mcfg.Module, filesetName, err) } + if hasOverride { + if _, ok := reg.configuredFilesets[mcfg.Module]; !ok { + reg.configuredFilesets[mcfg.Module] = map[string]struct{}{} + } + + reg.configuredFilesets[mcfg.Module][filesetName] = struct{}{} + } if fcfg.Enabled != nil && !(*fcfg.Enabled) { continue @@ -141,16 +152,19 @@ func NewModuleRegistry(moduleConfigs []*common.Config, beatInfo beat.Info, init } } var mcfgs []*ModuleConfig + configuredFilesets := map[string]map[string]struct{}{} for _, cfg := range moduleConfigs { cfg, err = mergePathDefaults(cfg) if err != nil { return nil, err } - moduleConfig, err := mcfgFromConfig(cfg) + moduleConfig, moduleConfiguredFilests, err := mcfgFromConfig(cfg) if err != nil { return nil, errors.Wrap(err, "error unpacking module config") } + + configuredFilesets[moduleConfig.Module] = moduleConfiguredFilests mcfgs = append(mcfgs, moduleConfig) } @@ -159,22 +173,22 @@ func NewModuleRegistry(moduleConfigs []*common.Config, beatInfo beat.Info, init return nil, err } - return newModuleRegistry(modulesPath, mcfgs, modulesOverrides, beatInfo) + return newModuleRegistry(modulesPath, mcfgs, modulesOverrides, configuredFilesets, beatInfo) } -func mcfgFromConfig(cfg *common.Config) (*ModuleConfig, error) { +func mcfgFromConfig(cfg *common.Config) (*ModuleConfig, map[string]struct{}, error) { var mcfg ModuleConfig err := cfg.Unpack(&mcfg) if err != nil { - return nil, err + return nil, nil, err } var dict map[string]interface{} err = cfg.Unpack(&dict) if err != nil { - return nil, fmt.Errorf("error unpacking module %s in a dict: %v", mcfg.Module, err) + return nil, nil, fmt.Errorf("error unpacking module %s in a dict: %v", mcfg.Module, err) } mcfg.Filesets = map[string]*FilesetConfig{} @@ -185,17 +199,31 @@ func mcfgFromConfig(cfg *common.Config) (*ModuleConfig, error) { tmpCfg, err := common.NewConfigFrom(filesetConfig) if err != nil { - return nil, fmt.Errorf("error creating config from fileset %s/%s: %v", mcfg.Module, name, err) + return nil, nil, fmt.Errorf("error creating config from fileset %s/%s: %v", mcfg.Module, name, err) } fcfg, err := NewFilesetConfig(tmpCfg) if err != nil { - return nil, fmt.Errorf("error creating config from fileset %s/%s: %v", mcfg.Module, name, err) + return nil, nil, fmt.Errorf("error creating config from fileset %s/%s: %v", mcfg.Module, name, err) } mcfg.Filesets[name] = fcfg } - return &mcfg, nil + // This calls cfg.GetFields() instead of iterating over `dict` keys + // because cfg.Unpack above doesn't return keys that map to a nil value, + // but GetFields() returns all keys. We need to observe filesets that + // don't contain any configuration (all default values). + configuredFilets := map[string]struct{}{} + + for _, name := range cfg.GetFields() { + if name == "module" || name == "enabled" || name == "path" { + continue + } + + configuredFilets[name] = struct{}{} + } + + return &mcfg, configuredFilets, nil } func getCurrentModuleName(modulePath, module string) (string, bool) { @@ -239,20 +267,20 @@ func getModuleFilesets(modulePath, module string) ([]string, error) { func applyOverrides(fcfg *FilesetConfig, module, fileset string, - overrides *ModuleOverrides) (*FilesetConfig, error) { + overrides *ModuleOverrides) (*FilesetConfig, bool, error) { if overrides == nil { - return fcfg, nil + return fcfg, false, nil } overridesConfigs := overrides.Get(module, fileset) if len(overridesConfigs) == 0 { - return fcfg, nil + return fcfg, false, nil } config, err := common.NewConfigFrom(fcfg) if err != nil { - return nil, fmt.Errorf("error creating vars config object: %v", err) + return nil, false, fmt.Errorf("error creating vars config object: %v", err) } toMerge := []*common.Config{config} @@ -260,15 +288,15 @@ func applyOverrides(fcfg *FilesetConfig, resultConfig, err := common.MergeConfigs(toMerge...) if err != nil { - return nil, fmt.Errorf("error merging configs: %v", err) + return nil, false, fmt.Errorf("error merging configs: %v", err) } res, err := NewFilesetConfig(resultConfig) if err != nil { - return nil, fmt.Errorf("error unpacking configs: %v", err) + return nil, false, fmt.Errorf("error unpacking configs: %v", err) } - return res, nil + return res, true, nil } // appendWithoutDuplicates appends basic module configuration for each module in the @@ -485,3 +513,13 @@ func (reg *ModuleRegistry) ModuleFilesets(module string) ([]string, error) { modulesPath := paths.Resolve(paths.Home, "module") return getModuleFilesets(modulesPath, module) } + +// ModuleConfiguredFilesets return the map of configured filesets for the given module +// it returns an empty map if the module doesn't exist +func (reg *ModuleRegistry) ModuleConfiguredFilesets(module string) map[string]struct{} { + if _, ok := reg.configuredFilesets[module]; ok { + return reg.configuredFilesets[module] + } + + return map[string]struct{}{} +} diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index 2c81665aad11..41c58e56929f 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -108,7 +108,7 @@ func TestSetupNginx(t *testing.T) { {Module: "nginx"}, } - reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("5.2.0")) + reg, err := newModuleRegistry(modulesPath, configs, nil, nil, makeTestInfo("5.2.0")) if err != nil { t.Fatal(err) } @@ -187,7 +187,7 @@ func TestLoadMultiplePipelines(t *testing.T) { &ModuleConfig{"foo", &enabled, filesetConfigs}, } - reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0")) + reg, err := newModuleRegistry(modulesPath, configs, nil, nil, makeTestInfo("6.6.0")) if err != nil { t.Fatal(err) } @@ -232,7 +232,7 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) { {"foo", &enabled, filesetConfigs}, } - reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0")) + reg, err := newModuleRegistry(modulesPath, configs, nil, nil, makeTestInfo("6.6.0")) if err != nil { t.Fatal(err) } diff --git a/filebeat/fileset/modules_test.go b/filebeat/fileset/modules_test.go index f69db27648c5..ab580e198919 100644 --- a/filebeat/fileset/modules_test.go +++ b/filebeat/fileset/modules_test.go @@ -52,7 +52,7 @@ func TestNewModuleRegistry(t *testing.T) { {Module: "auditd"}, } - reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}) + reg, err := newModuleRegistry(modulesPath, configs, nil, nil, beat.Info{Version: "5.2.0"}) require.NoError(t, err) assert.NotNil(t, reg) @@ -117,7 +117,7 @@ func TestNewModuleRegistryConfig(t *testing.T) { }, } - reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}) + reg, err := newModuleRegistry(modulesPath, configs, nil, nil, beat.Info{Version: "5.2.0"}) require.NoError(t, err) assert.NotNil(t, reg) @@ -142,7 +142,7 @@ func TestMovedModule(t *testing.T) { }, } - reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}) + reg, err := newModuleRegistry(modulesPath, configs, nil, nil, beat.Info{Version: "5.2.0"}) require.NoError(t, err) assert.NotNil(t, reg) } @@ -157,7 +157,28 @@ func TestApplyOverrides(t *testing.T) { module, fileset string overrides *ModuleOverrides expected FilesetConfig + hasOverride bool }{ + { + name: "no overrides", + fcfg: FilesetConfig{ + Var: map[string]interface{}{ + "a": "test", + "b.c": "test", + }, + Input: map[string]interface{}{}, + }, + module: "nginx", + fileset: "access", + expected: FilesetConfig{ + Var: map[string]interface{}{ + "a": "test", + "b.c": "test", + }, + Input: map[string]interface{}{}, + }, + hasOverride: false, + }, { name: "var overrides", fcfg: FilesetConfig{ @@ -183,6 +204,7 @@ func TestApplyOverrides(t *testing.T) { }, Input: map[string]interface{}{}, }, + hasOverride: true, }, { name: "enable and var overrides", @@ -209,6 +231,7 @@ func TestApplyOverrides(t *testing.T) { }, Input: map[string]interface{}{}, }, + hasOverride: true, }, { name: "input overrides", @@ -228,14 +251,16 @@ func TestApplyOverrides(t *testing.T) { }, Var: map[string]interface{}{}, }, + hasOverride: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - result, err := applyOverrides(&test.fcfg, test.module, test.fileset, test.overrides) + result, hasOverride, err := applyOverrides(&test.fcfg, test.module, test.fileset, test.overrides) require.NoError(t, err) assert.Equal(t, &test.expected, result) + assert.Equal(t, test.hasOverride, hasOverride) }) } } @@ -335,17 +360,28 @@ func TestAppendWithoutDuplicates(t *testing.T) { func TestMcfgFromConfig(t *testing.T) { falseVar := false tests := []struct { - name string - config *common.Config - expected ModuleConfig + name string + config *common.Config + expectedModuleConfig ModuleConfig + expectedConfiguredFilesets map[string]struct{} }{ + { + name: "not defined fileset", + config: load(t, map[string]interface{}{ + "module": "nginx", + }), + expectedModuleConfig: ModuleConfig{ + Module: "nginx", + }, + expectedConfiguredFilesets: map[string]struct{}{}, + }, { name: "disable fileset", config: load(t, map[string]interface{}{ "module": "nginx", "error.enabled": false, }), - expected: ModuleConfig{ + expectedModuleConfig: ModuleConfig{ Module: "nginx", Filesets: map[string]*FilesetConfig{ "error": { @@ -355,6 +391,9 @@ func TestMcfgFromConfig(t *testing.T) { }, }, }, + expectedConfiguredFilesets: map[string]struct{}{ + "error": struct{}{}, + }, }, { name: "set variable", @@ -362,7 +401,7 @@ func TestMcfgFromConfig(t *testing.T) { "module": "nginx", "access.var.test": false, }), - expected: ModuleConfig{ + expectedModuleConfig: ModuleConfig{ Module: "nginx", Filesets: map[string]*FilesetConfig{ "access": { @@ -373,16 +412,20 @@ func TestMcfgFromConfig(t *testing.T) { }, }, }, + expectedConfiguredFilesets: map[string]struct{}{ + "access": struct{}{}, + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - result, err := mcfgFromConfig(test.config) + result, configuredFilets, err := mcfgFromConfig(test.config) require.NoError(t, err) - assert.Equal(t, test.expected.Module, result.Module) - assert.Equal(t, len(test.expected.Filesets), len(result.Filesets)) - for name, fileset := range test.expected.Filesets { + assert.Equal(t, test.expectedConfiguredFilesets, configuredFilets) + assert.Equal(t, test.expectedModuleConfig.Module, result.Module) + assert.Equal(t, len(test.expectedModuleConfig.Filesets), len(result.Filesets)) + for name, fileset := range test.expectedModuleConfig.Filesets { assert.Equal(t, fileset, result.Filesets[name]) } })