Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add basic wildcard/keyword fallback for upcoming ECS releases #22521

Merged
merged 5 commits into from
Nov 12, 2020
Merged

Add basic wildcard/keyword fallback for upcoming ECS releases #22521

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4e93baf
Add basic wildcard/keyword fallback for upcoming ECS releases
Nov 10, 2020
df8f31e
update changelog
Nov 10, 2020
3d0384e
Invert global boolean
Nov 10, 2020
165004c
Remove global
Nov 11, 2020
591d5a5
Rename XPack to ElasticLicensed
Nov 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add proxy metricset for istio module. {pull}21751[21751]
- Added Kafka version 2.2 to the list of supported versions. {pull}22328[22328]
- Add support for ephemeral containers in kubernetes autodiscover and `add_kubernetes_metadata`. {pull}22389[22389] {pull}22439[22439]
- Added support for wildcard fields and keyword fallback in beats setup commands. {pull}22521[22521]

*Auditbeat*

Expand Down
34 changes: 32 additions & 2 deletions libbeat/template/processor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@ import (

// Processor struct to process fields to template
type Processor struct {
EsVersion common.Version
Migration bool
EsVersion common.Version
Migration bool
ElasticLicensed bool
}

var (
Expand Down Expand Up @@ -73,6 +74,13 @@ func (p *Processor) Process(fields mapping.Fields, state *fieldState, output com
indexMapping = p.integer(&field)
case "text":
indexMapping = p.text(&field)
case "wildcard":
noWildcards := p.EsVersion.LessThan(common.MustNewVersion("7.9.0"))
if !p.ElasticLicensed || noWildcards {
indexMapping = p.keyword(&field)
} else {
indexMapping = p.wildcard(&field)
}
case "", "keyword":
indexMapping = p.keyword(&field)
case "object":
Expand Down Expand Up @@ -229,6 +237,28 @@ func (p *Processor) keyword(f *mapping.Field) common.MapStr {
return property
}

func (p *Processor) wildcard(f *mapping.Field) common.MapStr {
property := getDefaultProperties(f)

property["type"] = "wildcard"

switch f.IgnoreAbove {
case 0: // Use libbeat default
property["ignore_above"] = defaultIgnoreAbove
case -1: // Use ES default
default: // Use user value
property["ignore_above"] = f.IgnoreAbove
}

if len(f.MultiFields) > 0 {
fields := common.MapStr{}
p.Process(f.MultiFields, nil, fields)
property["fields"] = fields
}

return property
}

func (p *Processor) text(f *mapping.Field) common.MapStr {
properties := getDefaultProperties(f)

Expand Down
126 changes: 126 additions & 0 deletions libbeat/template/processor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -676,3 +676,129 @@ func TestProcessDefaultField(t *testing.T) {
"nested.bar",
)
}

func TestProcessWildcardOSS(t *testing.T) {
// Test common fields are combined even if they come from different objects
fields := mapping.Fields{
mapping.Field{
Name: "test",
Type: "group",
Fields: mapping.Fields{
mapping.Field{
Name: "one",
Type: "wildcard",
},
},
},
}

output := common.MapStr{}
version, err := common.NewVersion("8.0.0")
if err != nil {
t.Fatal(err)
}

p := Processor{EsVersion: *version}
err = p.Process(fields, nil, output)
if err != nil {
t.Fatal(err)
}

// Make sure fields without a name are skipped during template generation
expectedOutput := common.MapStr{
"test": common.MapStr{
"properties": common.MapStr{
"one": common.MapStr{
"ignore_above": 1024,
"type": "keyword",
},
},
},
}

assert.Equal(t, expectedOutput, output)
}

func TestProcessWildcardElastic(t *testing.T) {
// Test common fields are combined even if they come from different objects
fields := mapping.Fields{
mapping.Field{
Name: "test",
Type: "group",
Fields: mapping.Fields{
mapping.Field{
Name: "one",
Type: "wildcard",
},
},
},
}

output := common.MapStr{}
version, err := common.NewVersion("8.0.0")
if err != nil {
t.Fatal(err)
}

p := Processor{EsVersion: *version, ElasticLicensed: true}
err = p.Process(fields, nil, output)
if err != nil {
t.Fatal(err)
}

// Make sure fields without a name are skipped during template generation
expectedOutput := common.MapStr{
"test": common.MapStr{
"properties": common.MapStr{
"one": common.MapStr{
"ignore_above": 1024,
"type": "wildcard",
},
},
},
}

assert.Equal(t, expectedOutput, output)
}

func TestProcessWildcardPreSupport(t *testing.T) {
// Test common fields are combined even if they come from different objects
fields := mapping.Fields{
mapping.Field{
Name: "test",
Type: "group",
Fields: mapping.Fields{
mapping.Field{
Name: "one",
Type: "wildcard",
},
},
},
}

output := common.MapStr{}
version, err := common.NewVersion("7.8.0")
if err != nil {
t.Fatal(err)
}

p := Processor{EsVersion: *version, ElasticLicensed: true}
err = p.Process(fields, nil, output)
if err != nil {
t.Fatal(err)
}

// Make sure fields without a name are skipped during template generation
expectedOutput := common.MapStr{
"test": common.MapStr{
"properties": common.MapStr{
"one": common.MapStr{
"ignore_above": 1024,
"type": "keyword",
},
},
},
}

assert.Equal(t, expectedOutput, output)
}
3 changes: 2 additions & 1 deletion libbeat/template/template.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/elastic/beats/v7/libbeat/common"
"github.com/elastic/beats/v7/libbeat/common/fmtstr"
"github.com/elastic/beats/v7/libbeat/mapping"
"github.com/elastic/beats/v7/libbeat/version"
)

var (
Expand Down Expand Up @@ -157,7 +158,7 @@ func (t *Template) load(fields mapping.Fields) (common.MapStr, error) {

// Start processing at the root
properties := common.MapStr{}
processor := Processor{EsVersion: t.esVersion, Migration: t.migration}
processor := Processor{EsVersion: t.esVersion, ElasticLicensed: version.ElasticLicensed, Migration: t.migration}
if err := processor.Process(fields, nil, properties); err != nil {
return nil, err
}
Expand Down
3 changes: 3 additions & 0 deletions libbeat/version/helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ var (
buildTime = "unknown"
commit = "unknown"
qualifier = ""

// ElasticLicensed provides if this is an Elastic Licensed distro or OSS
ElasticLicensed = false
)

// BuildTime exposes the compile-time build time information.
Expand Down
5 changes: 5 additions & 0 deletions x-pack/libbeat/cmd/inject.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package cmd

import (
"github.com/elastic/beats/v7/libbeat/cmd"
"github.com/elastic/beats/v7/libbeat/version"

// register central management
"github.com/elastic/beats/v7/x-pack/libbeat/licenser"
Expand All @@ -21,6 +22,10 @@ import (
_ "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws/elb"
)

func init() {
version.ElasticLicensed = true
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is setting a global, that we are trying to avoid cc @urso

Perhaps we should come up with a way to passing this to the beat initialization?

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, +1 on not introducing globals. Beats query the ES version and Basic queries the license on startup.

We pass the agent name and version via beat.Info. Maybe this would be the right place to put this information.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not a fan of globals either, and made the change, but due to the way that the root cobra commands are initialized in each beat and the way the info propagates from the command to the processor, the changes are quite a bit more invasive. So, I'm going to keep this conversation unresolved for some context in case we want to revert back to the global.

}

// AddXPack extends the given root folder with XPack features
func AddXPack(root *cmd.BeatsRootCmd, name string) {
licenser.Enforce(name, licenser.BasicAndAboveOrTrial)
Expand Down