From 4395eb34785a2483e7fae29e7f1e4372d327c50d Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Tue, 5 Mar 2024 17:23:27 +0100 Subject: [PATCH 01/17] Load JSON schema for ECS when package-spec >= 3.1.3 With the change elastic-package load JSON schema when: - package-spec > 2.3.0 and import_mappings: true - package-spec > 3.1.3 If package-spec is >= 2.3.0 and < 3.1.3 elastic-package continue to embed the legacy ecs@mappings. --- internal/builder/dynamic_mappings.go | 8 +++++++- internal/fields/dependency_manager.go | 2 +- internal/fields/validate.go | 6 +++++- internal/packages/buildmanifest/build_manifest.go | 2 +- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index beeb9a978..e23350c12 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -28,6 +28,7 @@ var staticEcsMappings string const prefixMapping = "_embedded_ecs" var semver2_3_0 = semver.MustParse("2.3.0") +var semver3_1_3 = semver.MustParse("3.1.3") type ecsTemplates struct { Mappings struct { @@ -75,7 +76,7 @@ func addDynamicMappings(packageRoot, destinationDir string) error { if err != nil { return err } - os.WriteFile(packageManifest, contents, 0664) + err = os.WriteFile(packageManifest, contents, 0664) if err != nil { return err } @@ -103,6 +104,11 @@ func shouldImportEcsMappings(specVersion, packageRoot string) (bool, error) { return false, nil } + if !v.LessThan(semver3_1_3) { + logger.Debugf("Required spec version < %s to import ECS mappings", semver3_1_3.String()) + return false, nil + } + if v.LessThan(semver2_3_0) { logger.Debugf("Required spec version >= %s to import ECS mappings", semver2_3_0.String()) return false, nil diff --git a/internal/fields/dependency_manager.go b/internal/fields/dependency_manager.go index f187fb6e2..d8ec2970a 100644 --- a/internal/fields/dependency_manager.go +++ b/internal/fields/dependency_manager.go @@ -302,7 +302,7 @@ func (dm *DependencyManager) importField(schemaName, fieldPath string) (FieldDef return *imported, nil } -// ImportAllFields method resolves all fields avaialble in the default ECS schema. +// ImportAllFields method resolves all fields available in the default ECS schema. func (dm *DependencyManager) ImportAllFields(schemaName string) ([]FieldDefinition, error) { if dm == nil { return []FieldDefinition{}, fmt.Errorf(`importing all external fields: external fields not allowed because dependencies file "_dev/build/build.yml" is missing`) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 13ede4cfa..d978e1983 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -33,6 +33,7 @@ var ( semver2_0_0 = semver.MustParse("2.0.0") semver2_3_0 = semver.MustParse("2.3.0") semver3_0_1 = semver.MustParse("3.0.1") + semver3_1_3 = semver.MustParse("3.1.3") defaultExternal = "ecs" ) @@ -216,7 +217,10 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im } var schema []FieldDefinition - if buildManifest.ImportMappings() && !specVersion.LessThan(semver2_3_0) && importECSSchema { + //if buildManifest.ImportMappings() && !specVersion.LessThan(semver2_3_0) && importECSSchema { + if (buildManifest.ImportMappings() && !specVersion.LessThan(semver2_3_0) || !specVersion.LessThan(semver3_1_3)) && importECSSchema { + // Import all fields from external schema (most likely ECS) to + // validate the package fields against it. ecsSchema, err := fdm.ImportAllFields(defaultExternal) if err != nil { return nil, nil, err diff --git a/internal/packages/buildmanifest/build_manifest.go b/internal/packages/buildmanifest/build_manifest.go index 09034046a..7f57a38cb 100644 --- a/internal/packages/buildmanifest/build_manifest.go +++ b/internal/packages/buildmanifest/build_manifest.go @@ -35,7 +35,7 @@ func (bm *BuildManifest) HasDependencies() bool { return bm.Dependencies.ECS.Reference != "" } -// HasDependencies function checks if there are any dependencies defined. +// ImportMappings function checks if there are any dependencies defined. func (bm *BuildManifest) ImportMappings() bool { return bm.Dependencies.ECS.ImportMappings } From 91c90855c742306eaa27911db46867450c5dee27 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Wed, 6 Mar 2024 13:01:12 +0100 Subject: [PATCH 02/17] Continue support import mappings (deprecated only) We should continue support import mappings for integrations that want to. We don't want to force integrations devs to migrate. --- internal/builder/dynamic_mappings.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index e23350c12..b2534c3a7 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -104,11 +104,6 @@ func shouldImportEcsMappings(specVersion, packageRoot string) (bool, error) { return false, nil } - if !v.LessThan(semver3_1_3) { - logger.Debugf("Required spec version < %s to import ECS mappings", semver3_1_3.String()) - return false, nil - } - if v.LessThan(semver2_3_0) { logger.Debugf("Required spec version >= %s to import ECS mappings", semver2_3_0.String()) return false, nil From 02c8e9c078b8bc7630dadc3769b999173dcccd49 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Wed, 6 Mar 2024 13:02:05 +0100 Subject: [PATCH 03/17] Cleanup --- internal/builder/dynamic_mappings.go | 1 - internal/fields/validate.go | 1 - 2 files changed, 2 deletions(-) diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index b2534c3a7..ce411c08a 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -28,7 +28,6 @@ var staticEcsMappings string const prefixMapping = "_embedded_ecs" var semver2_3_0 = semver.MustParse("2.3.0") -var semver3_1_3 = semver.MustParse("3.1.3") type ecsTemplates struct { Mappings struct { diff --git a/internal/fields/validate.go b/internal/fields/validate.go index d978e1983..281154e9e 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -217,7 +217,6 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im } var schema []FieldDefinition - //if buildManifest.ImportMappings() && !specVersion.LessThan(semver2_3_0) && importECSSchema { if (buildManifest.ImportMappings() && !specVersion.LessThan(semver2_3_0) || !specVersion.LessThan(semver3_1_3)) && importECSSchema { // Import all fields from external schema (most likely ECS) to // validate the package fields against it. From 96f3418f16252e6f0a6adae7c941d589a46c5df1 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Thu, 7 Mar 2024 11:12:02 +0100 Subject: [PATCH 04/17] Trying to infer if all stack versions support ECS This is an experiment to try inferring if all supported stack versions in the Kibana constraints support ECS mappings. If not, we don't include the ECS JSON schema mappings. In this case, users can opt for one of the following: - enable import mappings - add field mappings - bump the Kibana constraints >= 8.13.0 --- internal/fields/validate.go | 83 +++++++++++++++++++++++++++++++- internal/fields/validate_test.go | 48 ++++++++++++++++++ 2 files changed, 129 insertions(+), 2 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 281154e9e..aa630285b 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -33,7 +33,6 @@ var ( semver2_0_0 = semver.MustParse("2.0.0") semver2_3_0 = semver.MustParse("2.3.0") semver3_0_1 = semver.MustParse("3.0.1") - semver3_1_3 = semver.MustParse("3.1.3") defaultExternal = "ecs" ) @@ -216,8 +215,36 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im return nil, nil, fmt.Errorf("can't create field dependency manager: %w", err) } + // + // Check if the package embeds ECS mappings + // + packageEmbedsEcsMappings := buildManifest.ImportMappings() && !specVersion.LessThan(semver2_3_0) + if !packageEmbedsEcsMappings { + logger.Debugf("Package does not embed ECS mappings") + } + + // + // Check if the stack version includes ECS mappings + // + stackIncludesEcsMapping := false + + packageManifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot) + if err != nil { + return nil, nil, fmt.Errorf("can't read package manifest: %w", err) + } + + if len(packageManifest.Conditions.Kibana.Version) > 0 { + kibanaConstraints, err := semver.NewConstraint(packageManifest.Conditions.Kibana.Version) + if err != nil { + return nil, nil, fmt.Errorf("invalid constraint for Kibana: %w", err) + } + stackIncludesEcsMapping = onlySupportsStackVersionsWithEcsMappings(kibanaConstraints) + } + + // If the package embeds ECS mappings, or the stack version includes ECS mappings, then + // we should import the ECS schema to validate the package fields against it. var schema []FieldDefinition - if (buildManifest.ImportMappings() && !specVersion.LessThan(semver2_3_0) || !specVersion.LessThan(semver3_1_3)) && importECSSchema { + if (packageEmbedsEcsMappings || stackIncludesEcsMapping) && importECSSchema { // Import all fields from external schema (most likely ECS) to // validate the package fields against it. ecsSchema, err := fdm.ImportAllFields(defaultExternal) @@ -230,6 +257,58 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im return fdm, schema, nil } +func onlySupportsStackVersionsWithEcsMappings(kibanaConstraints *semver.Constraints) bool { + //versionsStr := []string{ + // "7.17.999", + // "8.0.999", + // "8.1.999", + // "8.2.999", + // "8.3.999", + // "8.4.999", + // "8.5.999", + // "8.6.999", + // "8.7.999", + // "8.8.999", + // "8.9.999", + // "8.10.999", + // "8.11.999", + // "8.12.999", + //} + //var minVersion *semver.Version + // + //for _, vStr := range versionsStr { + // v, err := semver.NewVersion(vStr) + // if err != nil { + // continue + // } + // + // if kibanaConstraints.Check(v) { + // if minVersion == nil || v.LessThan(minVersion) { + // minVersion = v + // } + // } + //} + // + //firstStackVersionWithEcsMappings := semver.MustParse("8.13.0") + //return minVersion == nil || minVersion.Compare(firstStackVersionWithEcsMappings) >= 0 // greater or equal + + // This check works under the assumption the constraints are not limited + // upwards. + // + // For example, if the constraint is `>= 8.12.0` and the stack version is + // `8.12.999`, the constraint will be satisfied. + // + // However, if the constraint is `= 8.0.0, < 8.10.0` the check will not + // return the right result. + // + // To support this, we would need to check the constraint against a larger + // set of versions, and check if the constraint is satisfied for all + // of them, like in the commented out example above. + // + lastStackVersionWithoutEcsMappings := semver.MustParse("8.12.999") + return !kibanaConstraints.Check(lastStackVersionWithoutEcsMappings) +} + //go:embed _static/allowed_geo_ips.txt var allowedGeoIPs string diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go index b44ab6bb4..2f8bc2e11 100644 --- a/internal/fields/validate_test.go +++ b/internal/fields/validate_test.go @@ -961,6 +961,54 @@ func TestValidateExternalMultiField(t *testing.T) { require.Empty(t, errs) } +func TestValidateStackVersionsWithEcsMappings(t *testing.T) { + // List of unique stack constraints extracted from the + // package manifest files in the elastic/integrations repository. + constraints := []struct { + Constraints string + SupportEcs bool + }{ + {"^7.14.0 || ^8.0.0", false}, + {"^7.14.1 || ^8.0.0", false}, + {"^7.14.1 || ^8.8.0", false}, + {"^7.16.0 || ^8.0.0", false}, + {"^7.17.0 || ^8.0.0", false}, + {"^8.0.0", false}, + {"^8.10.1", false}, + {"^8.10.2", false}, + {"^8.11.0", false}, + {"^8.11.2", false}, + {"^8.12.0", false}, + {"^8.12.1", false}, + {"^8.12.2", false}, + {"^8.13.0", true}, + {"^8.14.0", true}, + {"^8.2.0", false}, + {"^8.2.1", false}, + {"^8.3.0", false}, + {"^8.4.0", false}, + {"^8.5.0", false}, + {"^8.5.1", false}, + {"^8.6.0", false}, + {"^8.6.1", false}, + {"^8.7.0", false}, + {"^8.7.1", false}, + {"^8.8.0", false}, + {"^8.8.1", false}, + {"^8.8.2", false}, + {"^8.9.0", false}, + //{">= 8.0.0, < 8.10.0", false}, // FIXME: do we need to support this? + } + + for _, c := range constraints { + constraint, err := semver.NewConstraint(c.Constraints) + if err != nil { + require.NoError(t, err) + } + require.Equal(t, c.SupportEcs, onlySupportsStackVersionsWithEcsMappings(constraint), "constraint: %s", c.Constraints) + } +} + func readTestResults(t *testing.T, path string) (f results) { c, err := os.ReadFile(path) require.NoError(t, err) From edc0bf965238df67e194156a8b903c6d82f32bfb Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Thu, 7 Mar 2024 18:36:23 +0100 Subject: [PATCH 05/17] Use all ES releases < 8.13.0 to infer ECS support We compare all ES releases without ECS support against the kibana constraints to assess if the package can use the ECS JSON schema or not. --- internal/fields/validate.go | 205 +++++++++++++++++++++++++------ internal/fields/validate_test.go | 8 +- 2 files changed, 172 insertions(+), 41 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index aa630285b..11d294a24 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -34,6 +34,8 @@ var ( semver2_3_0 = semver.MustParse("2.3.0") semver3_0_1 = semver.MustParse("3.0.1") + firstStackVersionWithEcsMappings = semver.MustParse("8.13.0") + defaultExternal = "ecs" ) @@ -238,7 +240,7 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im if err != nil { return nil, nil, fmt.Errorf("invalid constraint for Kibana: %w", err) } - stackIncludesEcsMapping = onlySupportsStackVersionsWithEcsMappings(kibanaConstraints) + stackIncludesEcsMapping = allVersionsIncludeECS(kibanaConstraints) } // If the package embeds ECS mappings, or the stack version includes ECS mappings, then @@ -257,40 +259,167 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im return fdm, schema, nil } -func onlySupportsStackVersionsWithEcsMappings(kibanaConstraints *semver.Constraints) bool { - //versionsStr := []string{ - // "7.17.999", - // "8.0.999", - // "8.1.999", - // "8.2.999", - // "8.3.999", - // "8.4.999", - // "8.5.999", - // "8.6.999", - // "8.7.999", - // "8.8.999", - // "8.9.999", - // "8.10.999", - // "8.11.999", - // "8.12.999", - //} - //var minVersion *semver.Version - // - //for _, vStr := range versionsStr { - // v, err := semver.NewVersion(vStr) - // if err != nil { - // continue - // } - // - // if kibanaConstraints.Check(v) { - // if minVersion == nil || v.LessThan(minVersion) { - // minVersion = v - // } - // } - //} - // - //firstStackVersionWithEcsMappings := semver.MustParse("8.13.0") - //return minVersion == nil || minVersion.Compare(firstStackVersionWithEcsMappings) >= 0 // greater or equal +// allVersionsIncludeECS Check if all the stack versions in the constraints include ECS mappings. Only the stack +// versions 8.13.0 and above include ECS mappings. +// +// Returns true if all the stack versions in the constraints include ECS mappings, otherwise returns false. +func allVersionsIncludeECS(kibanaConstraints *semver.Constraints) bool { + // List of Elasticsearch releases that do not include ECS mappings (all versions before 8.13.0). + versionsStr := []string{ + "6.8.1", + "6.8.10", + "6.8.11", + "6.8.12", + "6.8.13", + "6.8.14", + "6.8.15", + "6.8.16", + "6.8.17", + "6.8.18", + "6.8.19", + "6.8.2", + "6.8.20", + "6.8.21", + "6.8.22", + "6.8.23", + "6.8.3", + "6.8.4", + "6.8.5", + "6.8.6", + "6.8.7", + "6.8.8", + "6.8.9", + "7.10.0", + "7.10.1", + "7.10.2", + "7.11.0", + "7.11.1", + "7.11.2", + "7.12.0", + "7.12.1", + "7.13.0", + "7.13.1", + "7.13.2", + "7.13.3", + "7.13.4", + "7.14.0", + "7.14.1", + "7.14.2", + "7.15.0", + "7.15.1", + "7.15.2", + "7.16.0", + "7.16.1", + "7.16.2", + "7.16.3", + "7.17.0", + "7.17.1", + "7.17.10", + "7.17.11", + "7.17.12", + "7.17.13", + "7.17.14", + "7.17.15", + "7.17.16", + "7.17.17", + "7.17.18", + "7.17.2", + "7.17.3", + "7.17.4", + "7.17.5", + "7.17.6", + "7.17.7", + "7.17.8", + "7.17.9", + "7.2.0", + "7.2.1", + "7.3.0", + "7.3.1", + "7.3.2", + "7.4.0", + "7.4.1", + "7.4.2", + "7.5.0", + "7.5.1", + "7.5.2", + "7.6.0", + "7.6.1", + "7.6.2", + "7.7.0", + "7.7.1", + "7.8.0", + "7.8.1", + "7.9.0", + "7.9.1", + "7.9.2", + "7.9.3", + "8.0.0", + "8.0.1", + "8.1.0", + "8.1.1", + "8.1.2", + "8.1.3", + "8.10.0", + "8.10.1", + "8.10.2", + "8.10.3", + "8.10.4", + "8.11.0", + "8.11.1", + "8.11.2", + "8.11.3", + "8.11.4", + "8.12.0", + "8.12.1", + "8.12.2", + "8.2.0", + "8.2.1", + "8.2.2", + "8.2.3", + "8.3.0", + "8.3.1", + "8.3.2", + "8.3.3", + "8.4.0", + "8.4.1", + "8.4.2", + "8.4.3", + "8.5.0", + "8.5.1", + "8.5.2", + "8.5.3", + "8.6.0", + "8.6.1", + "8.6.2", + "8.7.0", + "8.7.1", + "8.8.0", + "8.8.1", + "8.8.2", + "8.9.0", + "8.9.1", + "8.9.2", + } + + // Looking for a version that satisfies the package constraints. + for _, vStr := range versionsStr { + v, err := semver.NewVersion(vStr) + if err != nil { + logger.Errorf("invalid stack version %q: %v", vStr, err) + continue + } + + if kibanaConstraints.Check(v) { + // Found a version that satisfies the constraints, + // so at least this version does not include + // ECS mappings. + return false + } + } + + // If no version satisfies the constraints, then all versions + // include ECS mappings. + return true // This check works under the assumption the constraints are not limited // upwards. @@ -298,15 +427,15 @@ func onlySupportsStackVersionsWithEcsMappings(kibanaConstraints *semver.Constrai // For example, if the constraint is `>= 8.12.0` and the stack version is // `8.12.999`, the constraint will be satisfied. // - // However, if the constraint is `= 8.0.0, < 8.10.0` the check will not + // However, if the constraint is `>= 8.0.0, < 8.10.0` the check will not // return the right result. // // To support this, we would need to check the constraint against a larger // set of versions, and check if the constraint is satisfied for all // of them, like in the commented out example above. // - lastStackVersionWithoutEcsMappings := semver.MustParse("8.12.999") - return !kibanaConstraints.Check(lastStackVersionWithoutEcsMappings) + // lastStackVersionWithoutEcsMappings := semver.MustParse("8.12.999") + // return !kibanaConstraints.Check(lastStackVersionWithoutEcsMappings) } //go:embed _static/allowed_geo_ips.txt diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go index 2f8bc2e11..3fa21aa91 100644 --- a/internal/fields/validate_test.go +++ b/internal/fields/validate_test.go @@ -963,7 +963,8 @@ func TestValidateExternalMultiField(t *testing.T) { func TestValidateStackVersionsWithEcsMappings(t *testing.T) { // List of unique stack constraints extracted from the - // package manifest files in the elastic/integrations repository. + // package manifest files in the elastic/integrations + // repository. constraints := []struct { Constraints string SupportEcs bool @@ -997,7 +998,8 @@ func TestValidateStackVersionsWithEcsMappings(t *testing.T) { {"^8.8.1", false}, {"^8.8.2", false}, {"^8.9.0", false}, - //{">= 8.0.0, < 8.10.0", false}, // FIXME: do we need to support this? + {">= 8.0.0, < 8.10.0", false}, + {">= 8.0.0, < 8.0.1", false}, } for _, c := range constraints { @@ -1005,7 +1007,7 @@ func TestValidateStackVersionsWithEcsMappings(t *testing.T) { if err != nil { require.NoError(t, err) } - require.Equal(t, c.SupportEcs, onlySupportsStackVersionsWithEcsMappings(constraint), "constraint: %s", c.Constraints) + require.Equal(t, c.SupportEcs, allVersionsIncludeECS(constraint), "constraint: %s", c.Constraints) } } From 1eb3df1cb4f98ca197b15a88b803ef366d46d2ea Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Thu, 7 Mar 2024 18:48:48 +0100 Subject: [PATCH 06/17] Fix linter objections (sorry pal, you're right!) --- internal/fields/validate.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 11d294a24..8a706e34f 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -34,8 +34,6 @@ var ( semver2_3_0 = semver.MustParse("2.3.0") semver3_0_1 = semver.MustParse("3.0.1") - firstStackVersionWithEcsMappings = semver.MustParse("8.13.0") - defaultExternal = "ecs" ) From bb169858a9b4837c59527d40706df93622c600c7 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 8 Mar 2024 11:29:50 +0100 Subject: [PATCH 07/17] Update internal/fields/validate.go Co-authored-by: Jaime Soriano Pastor --- internal/fields/validate.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 8a706e34f..1393494f4 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -215,9 +215,7 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im return nil, nil, fmt.Errorf("can't create field dependency manager: %w", err) } - // // Check if the package embeds ECS mappings - // packageEmbedsEcsMappings := buildManifest.ImportMappings() && !specVersion.LessThan(semver2_3_0) if !packageEmbedsEcsMappings { logger.Debugf("Package does not embed ECS mappings") From bb05a753da99b6c77338e212ad6e3dd026c40377 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 8 Mar 2024 11:49:26 +0100 Subject: [PATCH 08/17] Remove stack versions that predate Fleet GA There are no package versions for older stacks. Sort the stack version numbers (`sort -V` is your friend here). --- internal/fields/validate.go | 102 ++++++++---------------------------- 1 file changed, 22 insertions(+), 80 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 1393494f4..33476d99f 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -262,43 +262,7 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im func allVersionsIncludeECS(kibanaConstraints *semver.Constraints) bool { // List of Elasticsearch releases that do not include ECS mappings (all versions before 8.13.0). versionsStr := []string{ - "6.8.1", - "6.8.10", - "6.8.11", - "6.8.12", - "6.8.13", - "6.8.14", - "6.8.15", - "6.8.16", - "6.8.17", - "6.8.18", - "6.8.19", - "6.8.2", - "6.8.20", - "6.8.21", - "6.8.22", - "6.8.23", - "6.8.3", - "6.8.4", - "6.8.5", - "6.8.6", - "6.8.7", - "6.8.8", - "6.8.9", - "7.10.0", - "7.10.1", - "7.10.2", - "7.11.0", - "7.11.1", - "7.11.2", - "7.12.0", - "7.12.1", - "7.13.0", - "7.13.1", - "7.13.2", - "7.13.3", - "7.13.4", - "7.14.0", + "7.14.0", // First version of Fleet in GA; there are no packages older than this version. "7.14.1", "7.14.2", "7.15.0", @@ -310,6 +274,14 @@ func allVersionsIncludeECS(kibanaConstraints *semver.Constraints) bool { "7.16.3", "7.17.0", "7.17.1", + "7.17.2", + "7.17.3", + "7.17.4", + "7.17.5", + "7.17.6", + "7.17.7", + "7.17.8", + "7.17.9", "7.17.10", "7.17.11", "7.17.12", @@ -319,55 +291,12 @@ func allVersionsIncludeECS(kibanaConstraints *semver.Constraints) bool { "7.17.16", "7.17.17", "7.17.18", - "7.17.2", - "7.17.3", - "7.17.4", - "7.17.5", - "7.17.6", - "7.17.7", - "7.17.8", - "7.17.9", - "7.2.0", - "7.2.1", - "7.3.0", - "7.3.1", - "7.3.2", - "7.4.0", - "7.4.1", - "7.4.2", - "7.5.0", - "7.5.1", - "7.5.2", - "7.6.0", - "7.6.1", - "7.6.2", - "7.7.0", - "7.7.1", - "7.8.0", - "7.8.1", - "7.9.0", - "7.9.1", - "7.9.2", - "7.9.3", "8.0.0", "8.0.1", "8.1.0", "8.1.1", "8.1.2", "8.1.3", - "8.10.0", - "8.10.1", - "8.10.2", - "8.10.3", - "8.10.4", - "8.11.0", - "8.11.1", - "8.11.2", - "8.11.3", - "8.11.4", - "8.12.0", - "8.12.1", - "8.12.2", "8.2.0", "8.2.1", "8.2.2", @@ -395,6 +324,19 @@ func allVersionsIncludeECS(kibanaConstraints *semver.Constraints) bool { "8.9.0", "8.9.1", "8.9.2", + "8.10.0", + "8.10.1", + "8.10.2", + "8.10.3", + "8.10.4", + "8.11.0", + "8.11.1", + "8.11.2", + "8.11.3", + "8.11.4", + "8.12.0", + "8.12.1", + "8.12.2", } // Looking for a version that satisfies the package constraints. From cf82a7d02183cdf8fc52b1022d29fd5fc1e3ab5d Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 8 Mar 2024 12:02:33 +0100 Subject: [PATCH 09/17] Pre-parse all stack versions and move them out fn --- internal/fields/validate.go | 167 +++++++++++++++++------------------- 1 file changed, 81 insertions(+), 86 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 33476d99f..e3918e423 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -34,6 +34,86 @@ var ( semver2_3_0 = semver.MustParse("2.3.0") semver3_0_1 = semver.MustParse("3.0.1") + // List of stack releases that do not + // include ECS mappings (all versions before 8.13.0). + stackVersions = []*semver.Version{ + semver.MustParse("7.14.0"), // First version of Fleet in GA; there are no packages older than this version. + semver.MustParse("7.14.1"), + semver.MustParse("7.14.2"), + semver.MustParse("7.15.0"), + semver.MustParse("7.15.1"), + semver.MustParse("7.15.2"), + semver.MustParse("7.16.0"), + semver.MustParse("7.16.1"), + semver.MustParse("7.16.2"), + semver.MustParse("7.16.3"), + semver.MustParse("7.17.0"), + semver.MustParse("7.17.1"), + semver.MustParse("7.17.2"), + semver.MustParse("7.17.3"), + semver.MustParse("7.17.4"), + semver.MustParse("7.17.5"), + semver.MustParse("7.17.6"), + semver.MustParse("7.17.7"), + semver.MustParse("7.17.8"), + semver.MustParse("7.17.9"), + semver.MustParse("7.17.10"), + semver.MustParse("7.17.11"), + semver.MustParse("7.17.12"), + semver.MustParse("7.17.13"), + semver.MustParse("7.17.14"), + semver.MustParse("7.17.15"), + semver.MustParse("7.17.16"), + semver.MustParse("7.17.17"), + semver.MustParse("7.17.18"), + semver.MustParse("8.0.0"), + semver.MustParse("8.0.1"), + semver.MustParse("8.1.0"), + semver.MustParse("8.1.1"), + semver.MustParse("8.1.2"), + semver.MustParse("8.1.3"), + semver.MustParse("8.2.0"), + semver.MustParse("8.2.1"), + semver.MustParse("8.2.2"), + semver.MustParse("8.2.3"), + semver.MustParse("8.3.0"), + semver.MustParse("8.3.1"), + semver.MustParse("8.3.2"), + semver.MustParse("8.3.3"), + semver.MustParse("8.4.0"), + semver.MustParse("8.4.1"), + semver.MustParse("8.4.2"), + semver.MustParse("8.4.3"), + semver.MustParse("8.5.0"), + semver.MustParse("8.5.1"), + semver.MustParse("8.5.2"), + semver.MustParse("8.5.3"), + semver.MustParse("8.6.0"), + semver.MustParse("8.6.1"), + semver.MustParse("8.6.2"), + semver.MustParse("8.7.0"), + semver.MustParse("8.7.1"), + semver.MustParse("8.8.0"), + semver.MustParse("8.8.1"), + semver.MustParse("8.8.2"), + semver.MustParse("8.9.0"), + semver.MustParse("8.9.1"), + semver.MustParse("8.9.2"), + semver.MustParse("8.10.0"), + semver.MustParse("8.10.1"), + semver.MustParse("8.10.2"), + semver.MustParse("8.10.3"), + semver.MustParse("8.10.4"), + semver.MustParse("8.11.0"), + semver.MustParse("8.11.1"), + semver.MustParse("8.11.2"), + semver.MustParse("8.11.3"), + semver.MustParse("8.11.4"), + semver.MustParse("8.12.0"), + semver.MustParse("8.12.1"), + semver.MustParse("8.12.2"), + } + defaultExternal = "ecs" ) @@ -260,93 +340,8 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im // // Returns true if all the stack versions in the constraints include ECS mappings, otherwise returns false. func allVersionsIncludeECS(kibanaConstraints *semver.Constraints) bool { - // List of Elasticsearch releases that do not include ECS mappings (all versions before 8.13.0). - versionsStr := []string{ - "7.14.0", // First version of Fleet in GA; there are no packages older than this version. - "7.14.1", - "7.14.2", - "7.15.0", - "7.15.1", - "7.15.2", - "7.16.0", - "7.16.1", - "7.16.2", - "7.16.3", - "7.17.0", - "7.17.1", - "7.17.2", - "7.17.3", - "7.17.4", - "7.17.5", - "7.17.6", - "7.17.7", - "7.17.8", - "7.17.9", - "7.17.10", - "7.17.11", - "7.17.12", - "7.17.13", - "7.17.14", - "7.17.15", - "7.17.16", - "7.17.17", - "7.17.18", - "8.0.0", - "8.0.1", - "8.1.0", - "8.1.1", - "8.1.2", - "8.1.3", - "8.2.0", - "8.2.1", - "8.2.2", - "8.2.3", - "8.3.0", - "8.3.1", - "8.3.2", - "8.3.3", - "8.4.0", - "8.4.1", - "8.4.2", - "8.4.3", - "8.5.0", - "8.5.1", - "8.5.2", - "8.5.3", - "8.6.0", - "8.6.1", - "8.6.2", - "8.7.0", - "8.7.1", - "8.8.0", - "8.8.1", - "8.8.2", - "8.9.0", - "8.9.1", - "8.9.2", - "8.10.0", - "8.10.1", - "8.10.2", - "8.10.3", - "8.10.4", - "8.11.0", - "8.11.1", - "8.11.2", - "8.11.3", - "8.11.4", - "8.12.0", - "8.12.1", - "8.12.2", - } - // Looking for a version that satisfies the package constraints. - for _, vStr := range versionsStr { - v, err := semver.NewVersion(vStr) - if err != nil { - logger.Errorf("invalid stack version %q: %v", vStr, err) - continue - } - + for _, v := range stackVersions { if kibanaConstraints.Check(v) { // Found a version that satisfies the constraints, // so at least this version does not include From e279d80f8569f1663946627d2dfdd828dd63c8f7 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 8 Mar 2024 12:17:45 +0100 Subject: [PATCH 10/17] Reverse stack version order to improve efficiency Since most of current integrations support 8.x: $ cat all_kibana_version.txt | awk -F',' '{print $3}' | sort | uniq -c 3 ^7.14.0 || ^8.0.0 5 ^7.14.1 || ^8.0.0 1 ^7.14.1 || ^8.8.0 10 ^7.16.0 || ^8.0.0 4 ^7.17.0 || ^8.0.0 6 ^8.0.0 7 ^8.10.1 6 ^8.10.2 3 ^8.11.0 2 ^8.11.2 75 ^8.12.0 1 ^8.12.1 1 ^8.12.2 44 ^8.13.0 1 ^8.2.0 2 ^8.2.1 6 ^8.3.0 2 ^8.4.0 4 ^8.5.0 1 ^8.5.1 2 ^8.6.0 1 ^8.6.1 1 ^8.7.0 24 ^8.7.1 27 ^8.8.0 1 ^8.8.1 1 ^8.8.2 11 ^8.9.0 It's more efficient to check for recent stack versions. --- internal/fields/validate.go | 148 ++++++++++++++++++------------------ 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index e3918e423..81658caed 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -37,81 +37,81 @@ var ( // List of stack releases that do not // include ECS mappings (all versions before 8.13.0). stackVersions = []*semver.Version{ - semver.MustParse("7.14.0"), // First version of Fleet in GA; there are no packages older than this version. - semver.MustParse("7.14.1"), - semver.MustParse("7.14.2"), - semver.MustParse("7.15.0"), - semver.MustParse("7.15.1"), - semver.MustParse("7.15.2"), - semver.MustParse("7.16.0"), - semver.MustParse("7.16.1"), - semver.MustParse("7.16.2"), - semver.MustParse("7.16.3"), - semver.MustParse("7.17.0"), - semver.MustParse("7.17.1"), - semver.MustParse("7.17.2"), - semver.MustParse("7.17.3"), - semver.MustParse("7.17.4"), - semver.MustParse("7.17.5"), - semver.MustParse("7.17.6"), - semver.MustParse("7.17.7"), - semver.MustParse("7.17.8"), - semver.MustParse("7.17.9"), - semver.MustParse("7.17.10"), - semver.MustParse("7.17.11"), - semver.MustParse("7.17.12"), - semver.MustParse("7.17.13"), - semver.MustParse("7.17.14"), - semver.MustParse("7.17.15"), - semver.MustParse("7.17.16"), - semver.MustParse("7.17.17"), - semver.MustParse("7.17.18"), - semver.MustParse("8.0.0"), - semver.MustParse("8.0.1"), - semver.MustParse("8.1.0"), - semver.MustParse("8.1.1"), - semver.MustParse("8.1.2"), - semver.MustParse("8.1.3"), - semver.MustParse("8.2.0"), - semver.MustParse("8.2.1"), - semver.MustParse("8.2.2"), - semver.MustParse("8.2.3"), - semver.MustParse("8.3.0"), - semver.MustParse("8.3.1"), - semver.MustParse("8.3.2"), - semver.MustParse("8.3.3"), - semver.MustParse("8.4.0"), - semver.MustParse("8.4.1"), - semver.MustParse("8.4.2"), - semver.MustParse("8.4.3"), - semver.MustParse("8.5.0"), - semver.MustParse("8.5.1"), - semver.MustParse("8.5.2"), - semver.MustParse("8.5.3"), - semver.MustParse("8.6.0"), - semver.MustParse("8.6.1"), - semver.MustParse("8.6.2"), - semver.MustParse("8.7.0"), - semver.MustParse("8.7.1"), - semver.MustParse("8.8.0"), - semver.MustParse("8.8.1"), - semver.MustParse("8.8.2"), - semver.MustParse("8.9.0"), - semver.MustParse("8.9.1"), - semver.MustParse("8.9.2"), - semver.MustParse("8.10.0"), - semver.MustParse("8.10.1"), - semver.MustParse("8.10.2"), - semver.MustParse("8.10.3"), - semver.MustParse("8.10.4"), - semver.MustParse("8.11.0"), - semver.MustParse("8.11.1"), - semver.MustParse("8.11.2"), - semver.MustParse("8.11.3"), - semver.MustParse("8.11.4"), - semver.MustParse("8.12.0"), - semver.MustParse("8.12.1"), semver.MustParse("8.12.2"), + semver.MustParse("8.12.1"), + semver.MustParse("8.12.0"), + semver.MustParse("8.11.4"), + semver.MustParse("8.11.3"), + semver.MustParse("8.11.2"), + semver.MustParse("8.11.1"), + semver.MustParse("8.11.0"), + semver.MustParse("8.10.4"), + semver.MustParse("8.10.3"), + semver.MustParse("8.10.2"), + semver.MustParse("8.10.1"), + semver.MustParse("8.10.0"), + semver.MustParse("8.9.2"), + semver.MustParse("8.9.1"), + semver.MustParse("8.9.0"), + semver.MustParse("8.8.2"), + semver.MustParse("8.8.1"), + semver.MustParse("8.8.0"), + semver.MustParse("8.7.1"), + semver.MustParse("8.7.0"), + semver.MustParse("8.6.2"), + semver.MustParse("8.6.1"), + semver.MustParse("8.6.0"), + semver.MustParse("8.5.3"), + semver.MustParse("8.5.2"), + semver.MustParse("8.5.1"), + semver.MustParse("8.5.0"), + semver.MustParse("8.4.3"), + semver.MustParse("8.4.2"), + semver.MustParse("8.4.1"), + semver.MustParse("8.4.0"), + semver.MustParse("8.3.3"), + semver.MustParse("8.3.2"), + semver.MustParse("8.3.1"), + semver.MustParse("8.3.0"), + semver.MustParse("8.2.3"), + semver.MustParse("8.2.2"), + semver.MustParse("8.2.1"), + semver.MustParse("8.2.0"), + semver.MustParse("8.1.3"), + semver.MustParse("8.1.2"), + semver.MustParse("8.1.1"), + semver.MustParse("8.1.0"), + semver.MustParse("8.0.1"), + semver.MustParse("8.0.0"), + semver.MustParse("7.17.18"), + semver.MustParse("7.17.17"), + semver.MustParse("7.17.16"), + semver.MustParse("7.17.15"), + semver.MustParse("7.17.14"), + semver.MustParse("7.17.13"), + semver.MustParse("7.17.12"), + semver.MustParse("7.17.11"), + semver.MustParse("7.17.10"), + semver.MustParse("7.17.9"), + semver.MustParse("7.17.8"), + semver.MustParse("7.17.7"), + semver.MustParse("7.17.6"), + semver.MustParse("7.17.5"), + semver.MustParse("7.17.4"), + semver.MustParse("7.17.3"), + semver.MustParse("7.17.2"), + semver.MustParse("7.17.1"), + semver.MustParse("7.17.0"), + semver.MustParse("7.16.3"), + semver.MustParse("7.16.2"), + semver.MustParse("7.16.1"), + semver.MustParse("7.16.0"), + semver.MustParse("7.15.2"), + semver.MustParse("7.15.1"), + semver.MustParse("7.15.0"), + semver.MustParse("7.14.2"), + semver.MustParse("7.14.1"), + semver.MustParse("7.14.0"), // First version of Fleet in GA; there are no packages older than this version. } defaultExternal = "ecs" From cb9f6890d4aad8d0e0e0784b61d215bdf0707e0a Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 8 Mar 2024 12:18:43 +0100 Subject: [PATCH 11/17] Update internal/fields/validate_test.go Co-authored-by: Jaime Soriano Pastor --- internal/fields/validate_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go index 3fa21aa91..630541dd6 100644 --- a/internal/fields/validate_test.go +++ b/internal/fields/validate_test.go @@ -969,6 +969,7 @@ func TestValidateStackVersionsWithEcsMappings(t *testing.T) { Constraints string SupportEcs bool }{ + {"^7.17.0", false}, {"^7.14.0 || ^8.0.0", false}, {"^7.14.1 || ^8.0.0", false}, {"^7.14.1 || ^8.8.0", false}, From e8f0bae8d8f6499887ff2425416f72d2538b85a4 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 8 Mar 2024 12:55:33 +0100 Subject: [PATCH 12/17] Move ECS support checking in dedicated function I'm also trying to improve naming and flow. --- internal/fields/validate.go | 42 +++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 81658caed..a30efb8f4 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -36,7 +36,7 @@ var ( // List of stack releases that do not // include ECS mappings (all versions before 8.13.0). - stackVersions = []*semver.Version{ + stackVersionsWithoutECS = []*semver.Version{ semver.MustParse("8.12.2"), semver.MustParse("8.12.1"), semver.MustParse("8.12.0"), @@ -302,27 +302,17 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im } // - // Check if the stack version includes ECS mappings + // Check if all stack versions support ECS mappings // - stackIncludesEcsMapping := false - - packageManifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot) + stackSupportsEcsMapping, err := supportsECSMappings(packageRoot) if err != nil { - return nil, nil, fmt.Errorf("can't read package manifest: %w", err) - } - - if len(packageManifest.Conditions.Kibana.Version) > 0 { - kibanaConstraints, err := semver.NewConstraint(packageManifest.Conditions.Kibana.Version) - if err != nil { - return nil, nil, fmt.Errorf("invalid constraint for Kibana: %w", err) - } - stackIncludesEcsMapping = allVersionsIncludeECS(kibanaConstraints) + return nil, nil, fmt.Errorf("can't check if stack version includes ECS mappings: %w", err) } // If the package embeds ECS mappings, or the stack version includes ECS mappings, then // we should import the ECS schema to validate the package fields against it. var schema []FieldDefinition - if (packageEmbedsEcsMappings || stackIncludesEcsMapping) && importECSSchema { + if (packageEmbedsEcsMappings || stackSupportsEcsMapping) && importECSSchema { // Import all fields from external schema (most likely ECS) to // validate the package fields against it. ecsSchema, err := fdm.ImportAllFields(defaultExternal) @@ -335,13 +325,33 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im return fdm, schema, nil } +// supportsECSMappings check if all the versions of the stack the package can run on support ECS mappings. +func supportsECSMappings(packageRoot string) (bool, error) { + packageManifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot) + if err != nil { + return false, fmt.Errorf("can't read package manifest: %w", err) + } + + if len(packageManifest.Conditions.Kibana.Version) == 0 { + logger.Debugf("No Kibana version constraint found in package manifest; assuming it does not support ECS mappings.") + return false, nil + } + + kibanaConstraints, err := semver.NewConstraint(packageManifest.Conditions.Kibana.Version) + if err != nil { + return false, fmt.Errorf("invalid constraint for Kibana: %w", err) + } + + return allVersionsIncludeECS(kibanaConstraints), nil +} + // allVersionsIncludeECS Check if all the stack versions in the constraints include ECS mappings. Only the stack // versions 8.13.0 and above include ECS mappings. // // Returns true if all the stack versions in the constraints include ECS mappings, otherwise returns false. func allVersionsIncludeECS(kibanaConstraints *semver.Constraints) bool { // Looking for a version that satisfies the package constraints. - for _, v := range stackVersions { + for _, v := range stackVersionsWithoutECS { if kibanaConstraints.Check(v) { // Found a version that satisfies the constraints, // so at least this version does not include From 151c5270ece5d8881c5adc1e12014b7a3db4f4a3 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 8 Mar 2024 18:44:26 +0100 Subject: [PATCH 13/17] Add upcoming 7.17.x patch releases Adding a few extra releases to support unlikely yet possible edge cases. --- internal/fields/validate.go | 6 ++++++ internal/fields/validate_test.go | 1 + 2 files changed, 7 insertions(+) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index a30efb8f4..b5bd4d80d 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -83,6 +83,12 @@ var ( semver.MustParse("8.1.0"), semver.MustParse("8.0.1"), semver.MustParse("8.0.0"), + semver.MustParse("7.17.24"), + semver.MustParse("7.17.23"), + semver.MustParse("7.17.22"), + semver.MustParse("7.17.21"), + semver.MustParse("7.17.20"), + semver.MustParse("7.17.19"), semver.MustParse("7.17.18"), semver.MustParse("7.17.17"), semver.MustParse("7.17.16"), diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go index 630541dd6..813c0477c 100644 --- a/internal/fields/validate_test.go +++ b/internal/fields/validate_test.go @@ -970,6 +970,7 @@ func TestValidateStackVersionsWithEcsMappings(t *testing.T) { SupportEcs bool }{ {"^7.17.0", false}, + {"7.17.19 || > 8.13", false}, {"^7.14.0 || ^8.0.0", false}, {"^7.14.1 || ^8.0.0", false}, {"^7.14.1 || ^8.8.0", false}, From f15fd184208d44732e054ebfda98a4d52ada5dd6 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 8 Mar 2024 18:54:25 +0100 Subject: [PATCH 14/17] Add package to test ecs@mappings support --- .../other/ecs_mappings/_dev/build/build.yml | 3 + .../ecs_mappings/_dev/build/docs/README.md | 14 +++++ .../packages/other/ecs_mappings/changelog.yml | 6 ++ .../first/agent/stream/stream.yml.hbs | 7 +++ .../elasticsearch/ingest_pipeline/default.yml | 10 ++++ .../data_stream/first/fields/base-fields.yml | 12 ++++ .../first/fields/histogram-fields.yml | 3 + .../data_stream/first/manifest.yml | 13 +++++ .../data_stream/first/sample_event.json | 29 ++++++++++ .../other/ecs_mappings/docs/README.md | 55 +++++++++++++++++++ test/packages/other/ecs_mappings/manifest.yml | 22 ++++++++ 11 files changed, 174 insertions(+) create mode 100644 test/packages/other/ecs_mappings/_dev/build/build.yml create mode 100644 test/packages/other/ecs_mappings/_dev/build/docs/README.md create mode 100644 test/packages/other/ecs_mappings/changelog.yml create mode 100644 test/packages/other/ecs_mappings/data_stream/first/agent/stream/stream.yml.hbs create mode 100644 test/packages/other/ecs_mappings/data_stream/first/elasticsearch/ingest_pipeline/default.yml create mode 100644 test/packages/other/ecs_mappings/data_stream/first/fields/base-fields.yml create mode 100644 test/packages/other/ecs_mappings/data_stream/first/fields/histogram-fields.yml create mode 100644 test/packages/other/ecs_mappings/data_stream/first/manifest.yml create mode 100644 test/packages/other/ecs_mappings/data_stream/first/sample_event.json create mode 100644 test/packages/other/ecs_mappings/docs/README.md create mode 100644 test/packages/other/ecs_mappings/manifest.yml diff --git a/test/packages/other/ecs_mappings/_dev/build/build.yml b/test/packages/other/ecs_mappings/_dev/build/build.yml new file mode 100644 index 000000000..e2b012548 --- /dev/null +++ b/test/packages/other/ecs_mappings/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@v8.11.0 diff --git a/test/packages/other/ecs_mappings/_dev/build/docs/README.md b/test/packages/other/ecs_mappings/_dev/build/docs/README.md new file mode 100644 index 000000000..644d5428e --- /dev/null +++ b/test/packages/other/ecs_mappings/_dev/build/docs/README.md @@ -0,0 +1,14 @@ +# ECS Mappings Test + +Test package to verify support for ECS mappings available to integrations running on stack version 8.13.0 and later. + +Please note that the package: + +- does not embed the legacy ECS mappings (no `import_mappings`). +- does not define fields in the `ecs.yml` file. + +Mappings for ECS fields (for example, `ecs.version`) come from the `ecs@mappings` component template in the integration index template, which has been available since 8.13.0. + +{{event "first"}} + +{{fields "first"}} diff --git a/test/packages/other/ecs_mappings/changelog.yml b/test/packages/other/ecs_mappings/changelog.yml new file mode 100644 index 000000000..bb0320a52 --- /dev/null +++ b/test/packages/other/ecs_mappings/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link diff --git a/test/packages/other/ecs_mappings/data_stream/first/agent/stream/stream.yml.hbs b/test/packages/other/ecs_mappings/data_stream/first/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..5845510de --- /dev/null +++ b/test/packages/other/ecs_mappings/data_stream/first/agent/stream/stream.yml.hbs @@ -0,0 +1,7 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +processors: + - add_locale: ~ diff --git a/test/packages/other/ecs_mappings/data_stream/first/elasticsearch/ingest_pipeline/default.yml b/test/packages/other/ecs_mappings/data_stream/first/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..81221adf3 --- /dev/null +++ b/test/packages/other/ecs_mappings/data_stream/first/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,10 @@ +--- +description: Pipeline for processing sample logs +processors: +- set: + field: sample_field + value: "1" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/test/packages/other/ecs_mappings/data_stream/first/fields/base-fields.yml b/test/packages/other/ecs_mappings/data_stream/first/fields/base-fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/other/ecs_mappings/data_stream/first/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/other/ecs_mappings/data_stream/first/fields/histogram-fields.yml b/test/packages/other/ecs_mappings/data_stream/first/fields/histogram-fields.yml new file mode 100644 index 000000000..128a0cb1c --- /dev/null +++ b/test/packages/other/ecs_mappings/data_stream/first/fields/histogram-fields.yml @@ -0,0 +1,3 @@ +- name: service.status.*.histogram + type: object + object_type: histogram diff --git a/test/packages/other/ecs_mappings/data_stream/first/manifest.yml b/test/packages/other/ecs_mappings/data_stream/first/manifest.yml new file mode 100644 index 000000000..979ef29d6 --- /dev/null +++ b/test/packages/other/ecs_mappings/data_stream/first/manifest.yml @@ -0,0 +1,13 @@ +title: "First" +type: logs +streams: + - input: logfile + title: Sample logs + description: Collect sample logs + vars: + - name: paths + type: text + title: Paths + multi: true + default: + - /var/log/*.log diff --git a/test/packages/other/ecs_mappings/data_stream/first/sample_event.json b/test/packages/other/ecs_mappings/data_stream/first/sample_event.json new file mode 100644 index 000000000..1f90d2db9 --- /dev/null +++ b/test/packages/other/ecs_mappings/data_stream/first/sample_event.json @@ -0,0 +1,29 @@ +{ + "source.geo.location": { + "lat": 1.0, + "lon": "2.0" + }, + "destination.geo.location.lat": 3.0, + "destination.geo.location.lon": 4.0, + "service.status.duration.histogram": { + "counts": [ + 8, + 17, + 8, + 7, + 6, + 2 + ], + "values": [ + 0.1, + 0.25, + 0.35, + 0.4, + 0.45, + 0.5 + ] + }, + "ecs": { + "version": "8.11.0" + } +} \ No newline at end of file diff --git a/test/packages/other/ecs_mappings/docs/README.md b/test/packages/other/ecs_mappings/docs/README.md new file mode 100644 index 000000000..24c234895 --- /dev/null +++ b/test/packages/other/ecs_mappings/docs/README.md @@ -0,0 +1,55 @@ +# ECS Mappings Test + +Test package to verify support for ECS mappings available to integrations running on stack version 8.13.0 and later. + +Please note that the package: + +- does not embed the legacy ECS mappings (no `import_mappings`). +- does not define fields in the `ecs.yml` file. + +Mappings for ECS fields (for example, `ecs.version`) come from the `ecs@mappings` component template in the integration index template, which has been available since 8.13.0. + +An example event for `first` looks as following: + +```json +{ + "source.geo.location": { + "lat": 1.0, + "lon": "2.0" + }, + "destination.geo.location.lat": 3.0, + "destination.geo.location.lon": 4.0, + "service.status.duration.histogram": { + "counts": [ + 8, + 17, + 8, + 7, + 6, + 2 + ], + "values": [ + 0.1, + 0.25, + 0.35, + 0.4, + 0.45, + 0.5 + ] + }, + "ecs": { + "version": "8.11.0" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| service.status.\*.histogram | | object | + diff --git a/test/packages/other/ecs_mappings/manifest.yml b/test/packages/other/ecs_mappings/manifest.yml new file mode 100644 index 000000000..409de2781 --- /dev/null +++ b/test/packages/other/ecs_mappings/manifest.yml @@ -0,0 +1,22 @@ +format_version: 3.1.2 +name: ecs_mappings +title: "ECS Mappings Tests" +version: 0.0.1 +description: "These tests fields validation using ECS mappings." +type: integration +categories: + - custom +conditions: + kibana: + version: "^8.13.0" +policy_templates: + - name: sample + title: Sample logs + description: Collect sample logs + inputs: + - type: logfile + title: Collect sample logs from instances + description: Collecting sample logs +owner: + github: elastic/integrations + type: elastic From f0a1ee1da9e83874c45eb65eb531a033183f3a48 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Wed, 13 Mar 2024 17:22:05 +0100 Subject: [PATCH 15/17] Apply suggestions from code review Co-authored-by: Mario Rodriguez Molins --- internal/fields/validate.go | 2 -- internal/fields/validate_test.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index b5bd4d80d..c7b602a60 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -307,9 +307,7 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im logger.Debugf("Package does not embed ECS mappings") } - // // Check if all stack versions support ECS mappings - // stackSupportsEcsMapping, err := supportsECSMappings(packageRoot) if err != nil { return nil, nil, fmt.Errorf("can't check if stack version includes ECS mappings: %w", err) diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go index 813c0477c..ecc38f02c 100644 --- a/internal/fields/validate_test.go +++ b/internal/fields/validate_test.go @@ -1009,7 +1009,7 @@ func TestValidateStackVersionsWithEcsMappings(t *testing.T) { if err != nil { require.NoError(t, err) } - require.Equal(t, c.SupportEcs, allVersionsIncludeECS(constraint), "constraint: %s", c.Constraints) + assert.Equal(t, c.SupportEcs, allVersionsIncludeECS(constraint), "constraint: %s", c.Constraints) } } From 15f2529ed88ff86485c344685c5cfc985c242cb3 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Wed, 13 Mar 2024 17:24:18 +0100 Subject: [PATCH 16/17] Log on importing JSON schema from external source --- internal/fields/validate.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index c7b602a60..78d3611db 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -319,6 +319,7 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im if (packageEmbedsEcsMappings || stackSupportsEcsMapping) && importECSSchema { // Import all fields from external schema (most likely ECS) to // validate the package fields against it. + logger.Debug("Importing ECS fields definition from external schema to validate the package fields against it") ecsSchema, err := fdm.ImportAllFields(defaultExternal) if err != nil { return nil, nil, err From 9e333acdbb608b63c73f39129df7d53eeffe756c Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Wed, 13 Mar 2024 17:33:31 +0100 Subject: [PATCH 17/17] Move log message to the right place --- internal/fields/validate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 78d3611db..db7f37828 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -319,11 +319,11 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im if (packageEmbedsEcsMappings || stackSupportsEcsMapping) && importECSSchema { // Import all fields from external schema (most likely ECS) to // validate the package fields against it. - logger.Debug("Importing ECS fields definition from external schema to validate the package fields against it") ecsSchema, err := fdm.ImportAllFields(defaultExternal) if err != nil { return nil, nil, err } + logger.Debug("Imported ECS fields definition from external schema for validation") schema = ecsSchema }