-
Notifications
You must be signed in to change notification settings - Fork 4.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Allow template pattern to be overwritten #4769
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
With the current template loading is was not possible to overload the pattern used as name and pattern were mixed. For example in the case where someone wanted to define pattern name `a` that applies to the indice `a` this was not possible because the pattern generated was `a-*` so it didn't apply to the index `a`. This change now allows to set pattern and name separately. To make sure index pattern and templates are in sync, the beat will not start in case the index pattern was modified but the template patterns not. We should figure out later some automated mechanisms here. We should find a better solution for 6.x to require less config options and also include dashboard generation in this.
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,8 +2,11 @@ package template | |
|
||
import ( | ||
"fmt" | ||
"time" | ||
|
||
"github.com/elastic/beats/libbeat/beat" | ||
"github.com/elastic/beats/libbeat/common" | ||
"github.com/elastic/beats/libbeat/common/fmtstr" | ||
"github.com/elastic/go-ucfg/yaml" | ||
) | ||
|
||
|
@@ -17,19 +20,58 @@ var ( | |
) | ||
|
||
type Template struct { | ||
index string | ||
name string | ||
pattern string | ||
beatVersion common.Version | ||
esVersion common.Version | ||
settings TemplateSettings | ||
} | ||
|
||
// New creates a new template instance | ||
func New(beatVersion string, esVersion string, index string, settings TemplateSettings) (*Template, error) { | ||
func New(beatVersion string, beatName string, esVersion string, config TemplateConfig) (*Template, error) { | ||
bV, err := common.NewVersion(beatVersion) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
name := config.Name | ||
if name == "" { | ||
name = fmt.Sprintf("%s-%s", beatName, bV.String()) | ||
} | ||
|
||
pattern := config.Pattern | ||
if pattern == "" { | ||
pattern = name + "-*" | ||
} | ||
|
||
event := &beat.Event{ | ||
Fields: common.MapStr{ | ||
"beat": common.MapStr{ | ||
"name": beatName, | ||
"version": bV.String(), | ||
}, | ||
}, | ||
Timestamp: time.Now(), | ||
} | ||
|
||
nameFormatter, err := fmtstr.CompileEvent(name) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @urso I changed by approach here. Instead of building my own compiler and formatter, I just create an event object and use There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why? What happens if an user indeed uses There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It will work. I think this is a good thing or do you see an issue? This implementation allows us to add more fields later on and still use the same implementation. |
||
if err != nil { | ||
return nil, err | ||
} | ||
name, err = nameFormatter.Run(event) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
patternFormatter, err := fmtstr.CompileEvent(pattern) | ||
if err != nil { | ||
return nil, err | ||
} | ||
pattern, err = patternFormatter.Run(event) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
// In case no esVersion is set, it is assumed the same as beat version | ||
if esVersion == "" { | ||
esVersion = beatVersion | ||
|
@@ -41,10 +83,11 @@ func New(beatVersion string, esVersion string, index string, settings TemplateSe | |
} | ||
|
||
return &Template{ | ||
index: index, | ||
pattern: pattern, | ||
name: name, | ||
beatVersion: *bV, | ||
esVersion: *esV, | ||
settings: settings, | ||
settings: config.Settings, | ||
}, nil | ||
} | ||
|
||
|
@@ -62,9 +105,14 @@ func (t *Template) Load(file string) (common.MapStr, error) { | |
return output, nil | ||
} | ||
|
||
// GetName returns the name of the template which is {index}-{version} | ||
// GetName returns the name of the template | ||
func (t *Template) GetName() string { | ||
return fmt.Sprintf("%s-%s", t.index, t.beatVersion.String()) | ||
return t.name | ||
} | ||
|
||
// GetPattern returns the pattern of the template | ||
func (t *Template) GetPattern() string { | ||
return t.pattern | ||
} | ||
|
||
// generate generates the full template | ||
|
@@ -130,9 +178,9 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. | |
|
||
// ES 6 moved from template to index_patterns: https://github.com/elastic/elasticsearch/pull/21009 | ||
if t.esVersion.Major >= 6 { | ||
basicStructure.Put("index_patterns", []string{t.GetName() + "-*"}) | ||
basicStructure.Put("index_patterns", []string{t.GetPattern()}) | ||
} else { | ||
basicStructure.Put("template", t.GetName()+"-*") | ||
basicStructure.Put("template", t.GetPattern()) | ||
} | ||
|
||
if t.esVersion.IsMajor(2) { | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Should we remove here the
-
and make this just*
?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is just the default, right? I'd say let's keep the
-*
, so that it's less likely to apply accidentally to other indices.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes 👍