Add basic wildcard/keyword fallback for upcoming ECS releases

libbeat/template/processor.go

@@ -30,6 +30,7 @@ import (
 type Processor struct {
 	EsVersion common.Version
 	Migration bool
+	OSS       bool
 }
 
 var (
@@ -73,6 +74,13 @@ func (p *Processor) Process(fields mapping.Fields, state *fieldState, output com
 			indexMapping = p.integer(&field)
 		case "text":
 			indexMapping = p.text(&field)
+		case "wildcard":
+			noWildcards := p.EsVersion.LessThan(common.MustNewVersion("7.9.0"))
+			if p.OSS || noWildcards {
+				indexMapping = p.keyword(&field)
+			} else {
+				indexMapping = p.wildcard(&field)
+			}
 		case "", "keyword":
 			indexMapping = p.keyword(&field)
 		case "object":
@@ -229,6 +237,28 @@ func (p *Processor) keyword(f *mapping.Field) common.MapStr {
 	return property
 }
 
+func (p *Processor) wildcard(f *mapping.Field) common.MapStr {
+	property := getDefaultProperties(f)
+
+	property["type"] = "wildcard"
+
+	switch f.IgnoreAbove {
+	case 0: // Use libbeat default
+		property["ignore_above"] = defaultIgnoreAbove
+	case -1: // Use ES default
+	default: // Use user value
+		property["ignore_above"] = f.IgnoreAbove
+	}
+
+	if len(f.MultiFields) > 0 {
+		fields := common.MapStr{}
+		p.Process(f.MultiFields, nil, fields)
+		property["fields"] = fields
+	}
+
+	return property
+}
+
 func (p *Processor) text(f *mapping.Field) common.MapStr {
 	properties := getDefaultProperties(f)
 

libbeat/template/processor_test.go

@@ -676,3 +676,129 @@ func TestProcessDefaultField(t *testing.T) {
 		"nested.bar",
 	)
 }
+
+func TestProcessWildcardOSS(t *testing.T) {
+	// Test common fields are combined even if they come from different objects
+	fields := mapping.Fields{
+		mapping.Field{
+			Name: "test",
+			Type: "group",
+			Fields: mapping.Fields{
+				mapping.Field{
+					Name: "one",
+					Type: "wildcard",
+				},
+			},
+		},
+	}
+
+	output := common.MapStr{}
+	version, err := common.NewVersion("8.0.0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	p := Processor{EsVersion: *version, OSS: true}
+	err = p.Process(fields, nil, output)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Make sure fields without a name are skipped during template generation
+	expectedOutput := common.MapStr{
+		"test": common.MapStr{
+			"properties": common.MapStr{
+				"one": common.MapStr{
+					"ignore_above": 1024,
+					"type":         "keyword",
+				},
+			},
+		},
+	}
+
+	assert.Equal(t, expectedOutput, output)
+}
+
+func TestProcessWildcardElastic(t *testing.T) {
+	// Test common fields are combined even if they come from different objects
+	fields := mapping.Fields{
+		mapping.Field{
+			Name: "test",
+			Type: "group",
+			Fields: mapping.Fields{
+				mapping.Field{
+					Name: "one",
+					Type: "wildcard",
+				},
+			},
+		},
+	}
+
+	output := common.MapStr{}
+	version, err := common.NewVersion("8.0.0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	p := Processor{EsVersion: *version, OSS: false}
+	err = p.Process(fields, nil, output)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Make sure fields without a name are skipped during template generation
+	expectedOutput := common.MapStr{
+		"test": common.MapStr{
+			"properties": common.MapStr{
+				"one": common.MapStr{
+					"ignore_above": 1024,
+					"type":         "wildcard",
+				},
+			},
+		},
+	}
+
+	assert.Equal(t, expectedOutput, output)
+}
+
+func TestProcessWildcardPreSupport(t *testing.T) {
+	// Test common fields are combined even if they come from different objects
+	fields := mapping.Fields{
+		mapping.Field{
+			Name: "test",
+			Type: "group",
+			Fields: mapping.Fields{
+				mapping.Field{
+					Name: "one",
+					Type: "wildcard",
+				},
+			},
+		},
+	}
+
+	output := common.MapStr{}
+	version, err := common.NewVersion("7.8.0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	p := Processor{EsVersion: *version, OSS: false}
+	err = p.Process(fields, nil, output)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Make sure fields without a name are skipped during template generation
+	expectedOutput := common.MapStr{
+		"test": common.MapStr{
+			"properties": common.MapStr{
+				"one": common.MapStr{
+					"ignore_above": 1024,
+					"type":         "keyword",
+				},
+			},
+		},
+	}
+
+	assert.Equal(t, expectedOutput, output)
+}

libbeat/template/template.go

@@ -28,6 +28,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/fmtstr"
 	"github.com/elastic/beats/v7/libbeat/mapping"
+	"github.com/elastic/beats/v7/libbeat/version"
 )
 
 var (
@@ -157,7 +158,7 @@ func (t *Template) load(fields mapping.Fields) (common.MapStr, error) {
 
 	// Start processing at the root
 	properties := common.MapStr{}
-	processor := Processor{EsVersion: t.esVersion, Migration: t.migration}
+	processor := Processor{EsVersion: t.esVersion, OSS: version.OSS, Migration: t.migration}
 	if err := processor.Process(fields, nil, properties); err != nil {
 		return nil, err
 	}

libbeat/version/helper.go

@@ -32,6 +32,9 @@ var (
 	buildTime = "unknown"
 	commit    = "unknown"
 	qualifier = ""
+
+	// OSS provides if this is the OSS distro
+	OSS = true
 )
 
 // BuildTime exposes the compile-time build time information.

x-pack/libbeat/cmd/inject.go

@@ -6,6 +6,7 @@ package cmd
 
 import (
 	"github.com/elastic/beats/v7/libbeat/cmd"
+	"github.com/elastic/beats/v7/libbeat/version"
 
 	// register central management
 	"github.com/elastic/beats/v7/x-pack/libbeat/licenser"
@@ -21,6 +22,10 @@ import (
 	_ "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws/elb"
 )
 
+func init() {
+	version.OSS = false
+}
+
 // AddXPack extends the given root folder with XPack features
 func AddXPack(root *cmd.BeatsRootCmd, name string) {
 	licenser.Enforce(name, licenser.BasicAndAboveOrTrial)