Add support for dimension fields

code/go/internal/validator/semantic/types.go

@@ -0,0 +1,97 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"gopkg.in/yaml.v3"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+
+	ve "github.com/elastic/package-spec/code/go/internal/errors"
+)
+
+type fields []field
+
+type field struct {
+	Name       string `yaml:"name"`
+	Type       string `yaml:"type"`
+	Unit       string `yaml:"unit"`
+	MetricType string `yaml:"metric_type"`
+	Dimension  bool   `yaml:"dimension"`
+
+	Fields fields `yaml:"fields"`
+}
+
+type validateFunc func(fieldsFile string, f field) ve.ValidationErrors
+
+func validateFields(pkgRoot string, validate validateFunc) ve.ValidationErrors {
+	fieldsFiles, err := listFieldsFiles(pkgRoot)
+	if err != nil {
+		return ve.ValidationErrors{errors.Wrap(err, "can't list fields files")}
+	}
+
+	var vErrs ve.ValidationErrors
+	for _, fieldsFile := range fieldsFiles {
+		unmarshaled, err := unmarshalFields(fieldsFile)
+		if err != nil {
+			vErrs = append(vErrs, errors.Wrapf(err, `file "%s" is invalid: can't unmarshal fields`, fieldsFile))
+		}
+
+		for _, u := range unmarshaled {
+			errs := validate(fieldsFile, u)
+			if len(errs) > 0 {
+				vErrs = append(vErrs, errs...)
+			}
+		}
+	}
+	return vErrs
+}
+
+func listFieldsFiles(pkgRoot string) ([]string, error) {
+	var fieldsFiles []string
+
+	dataStreamDir := filepath.Join(pkgRoot, "data_stream")
+	dataStreams, err := ioutil.ReadDir(dataStreamDir)
+	if errors.Is(err, os.ErrNotExist) {
+		return fieldsFiles, nil
+	}
+	if err != nil {
+		return nil, errors.Wrap(err, "can't list data streams directory")
+	}
+
+	for _, dataStream := range dataStreams {
+		fieldsDir := filepath.Join(dataStreamDir, dataStream.Name(), "fields")
+		fs, err := ioutil.ReadDir(fieldsDir)
+		if errors.Is(err, os.ErrNotExist) {
+			continue
+		}
+		if err != nil {
+			return nil, errors.Wrapf(err, "can't list fields directory (path: %s)", fieldsDir)
+		}
+
+		for _, f := range fs {
+			fieldsFiles = append(fieldsFiles, filepath.Join(fieldsDir, f.Name()))
+		}
+	}
+
+	return fieldsFiles, nil
+}
+
+func unmarshalFields(fieldsPath string) (fields, error) {
+	content, err := ioutil.ReadFile(fieldsPath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't read file (path: %s)", fieldsPath)
+	}
+
+	var f fields
+	err = yaml.Unmarshal(content, &f)
+	if err != nil {
+		return nil, errors.Wrapf(err, "yaml.Unmarshal failed (path: %s)", fieldsPath)
+	}
+	return f, nil
+}

code/go/internal/validator/semantic/validate_dimensions.go

@@ -0,0 +1,46 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/elastic/package-spec/code/go/internal/errors"
+)
+
+// ValidateDimensionFields verifies if dimension fields are of one of the expected types.
+func ValidateDimensionFields(pkgRoot string) errors.ValidationErrors {
+	return validateFields(pkgRoot, validateDimensionField)
+}
+
+func validateDimensionField(fieldsFile string, f field) errors.ValidationErrors {
+	if f.Dimension && !isAllowedDimensionType(f.Type) {
+		return errors.ValidationErrors{fmt.Errorf(`file "%s" is invalid: field "%s" of type %s can't be a dimension, allowed types for dimensions: %s`, fieldsFile, f.Name, f.Type, strings.Join(allowedDimensionTypes, ", "))}
+	}
+
+	return nil
+}
+
+var allowedDimensionTypes = []string{
+	// Keywords
+	"constant_keyword", "keyword",
+
+	// Numeric types
+	"long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float", "unsigned_long",
+
+	// IPs
+	"ip",
+}
+
+func isAllowedDimensionType(fieldType string) bool {
+	for _, allowedType := range allowedDimensionTypes {
+		if fieldType == allowedType {
+			return true
+		}
+	}
+
+	return false
+}

code/go/internal/validator/semantic/validate_dimensions_test.go

@@ -0,0 +1,84 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestValidateDimensionFields(t *testing.T) {
+	cases := []struct {
+		title string
+		field field
+		valid bool
+	}{
+		{
+			title: "usual keyword dimension",
+			field: field{
+				Name:      "host.id",
+				Type:      "keyword",
+				Dimension: true,
+			},
+			valid: true,
+		},
+		{
+			title: "not a dimension",
+			field: field{
+				Name: "host.id",
+				Type: "histogram",
+			},
+			valid: true,
+		},
+		{
+			title: "ip dimension",
+			field: field{
+				Name:      "source.ip",
+				Type:      "ip",
+				Dimension: true,
+			},
+			valid: true,
+		},
+		{
+			title: "numeric dimension",
+			field: field{
+				Name:      "http.body.size",
+				Type:      "long",
+				Dimension: true,
+			},
+			valid: true,
+		},
+		{
+			title: "histogram dimension is not supported",
+			field: field{
+				Name:      "http.response.time",
+				Type:      "histogram",
+				Dimension: true,
+			},
+			valid: false,
+		},
+		{
+			title: "nested field as dimension is not supported",
+			field: field{
+				Name:      "process.child",
+				Type:      "nested",
+				Dimension: true,
+			},
+			valid: false,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			errs := validateDimensionField("fields.yml", c.field)
+			if c.valid {
+				assert.Empty(t, errs)
+			} else {
+				assert.NotEmpty(t, errs)
+			}
+		})
+	}
+}

code/go/internal/validator/semantic/validate_field_groups.go

@@ -6,107 +6,27 @@ package semantic
 
 import (
 	"fmt"
-	"gopkg.in/yaml.v3"
-	"io/ioutil"
-	"os"
-	"path/filepath"
 
-	"github.com/pkg/errors"
-
-	ve "github.com/elastic/package-spec/code/go/internal/errors"
+	"github.com/elastic/package-spec/code/go/internal/errors"
 )
 
-type fields []field
-
-type field struct {
-	Name       string `yaml:"name"`
-	Type       string `yaml:"type"`
-	Unit       string `yaml:"unit"`
-	MetricType string `yaml:"metric_type"`
-
-	Fields fields `yaml:"fields"`
-}
-
 // ValidateFieldGroups verifies if field groups don't have units and metric types defined.
-func ValidateFieldGroups(pkgRoot string) ve.ValidationErrors {
-	fieldsFiles, err := listFieldsFiles(pkgRoot)
-	if err != nil {
-		return ve.ValidationErrors{errors.Wrapf(err, "can't list fields files")}
-	}
-
-	var vErrs ve.ValidationErrors
-	for _, fieldsFile := range fieldsFiles {
-		unmarshaled, err := unmarshalFields(fieldsFile)
-		if err != nil {
-			vErrs = append(vErrs, errors.Wrap(err, "can't unmarshal fields"))
-		}
-
-		for _, u := range unmarshaled {
-			errs := validateFieldUnit(u)
-			if len(errs) > 0 {
-				vErrs = append(vErrs, errs...)
-			}
-		}
-	}
-	return vErrs
-}
-
-func listFieldsFiles(pkgRoot string) ([]string, error) {
-	var fieldsFiles []string
-
-	dataStreamDir := filepath.Join(pkgRoot, "data_stream")
-	dataStreams, err := ioutil.ReadDir(dataStreamDir)
-	if errors.Is(err, os.ErrNotExist) {
-		return fieldsFiles, nil
-	}
-	if err != nil {
-		return nil, errors.Wrap(err, "can't list data streams directory")
-	}
-
-	for _, dataStream := range dataStreams {
-		fieldsDir := filepath.Join(dataStreamDir, dataStream.Name(), "fields")
-		fs, err := ioutil.ReadDir(fieldsDir)
-		if errors.Is(err, os.ErrNotExist) {
-			continue
-		}
-		if err != nil {
-			return nil, errors.Wrapf(err, "can't list fields directory (path: %s)", fieldsDir)
-		}
-
-		for _, f := range fs {
-			fieldsFiles = append(fieldsFiles, filepath.Join(fieldsDir, f.Name()))
-		}
-	}
-
-	return fieldsFiles, nil
-}
-
-func unmarshalFields(fieldsPath string) (fields, error) {
-	content, err := ioutil.ReadFile(fieldsPath)
-	if err != nil {
-		return nil, errors.Wrapf(err, "can't read file (path: %s)", fieldsPath)
-	}
-
-	var f fields
-	err = yaml.Unmarshal(content, &f)
-	if err != nil {
-		return nil, errors.Wrapf(err, "yaml.Unmarshal failed (path: %s)", fieldsPath)
-	}
-	return f, nil
+func ValidateFieldGroups(pkgRoot string) errors.ValidationErrors {
+	return validateFields(pkgRoot, validateFieldUnit)
 }
 
-func validateFieldUnit(f field) ve.ValidationErrors {
+func validateFieldUnit(fieldsFile string, f field) errors.ValidationErrors {
 	if f.Type == "group" && f.Unit != "" {
-		return ve.ValidationErrors{fmt.Errorf(`field "%s" can't have unit property'`, f.Name)}
+		return errors.ValidationErrors{fmt.Errorf(`file "%s" is invalid: field "%s" can't have unit property'`, fieldsFile, f.Name)}
 	}
 
 	if f.Type == "group" && f.MetricType != "" {
-		return ve.ValidationErrors{fmt.Errorf(`field "%s" can't have metric type property'`, f.Name)}
+		return errors.ValidationErrors{fmt.Errorf(`file "%s" is invalid: field "%s" can't have metric type property'`, fieldsFile, f.Name)}
 	}
 
-	var vErrs ve.ValidationErrors
+	var vErrs errors.ValidationErrors
 	for _, aField := range f.Fields {
-		errs := validateFieldUnit(aField)
+		errs := validateFieldUnit(fieldsFile, aField)
 		if len(errs) > 0 {
 			vErrs = append(vErrs, errs...)
 		}

code/go/internal/validator/semantic/validate_field_groups_test.go

@@ -5,8 +5,11 @@
 package semantic
 
 import (
+	"fmt"
+	"path/filepath"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -20,9 +23,16 @@ func TestValidateFieldGroups_Good(t *testing.T) {
 func TestValidateFieldGroups_Bad(t *testing.T) {
 	pkgRoot := "../../../../../test/packages/bad_group_unit"
 
+	fileError := func(name string, expected string) string {
+		return fmt.Sprintf(`file "%s" is invalid: %s`,
+			filepath.Join(pkgRoot, name),
+			expected)
+	}
+
 	errs := ValidateFieldGroups(pkgRoot)
-	require.Len(t, errs, 3)
-	require.Equal(t, `field "bbb" can't have unit property'`, errs[0].Error())
-	require.Equal(t, `field "eee" can't have unit property'`, errs[1].Error())
-	require.Equal(t, `field "fff" can't have metric type property'`, errs[2].Error())
-}
+	if assert.Len(t, errs, 3) {
+		assert.Equal(t, fileError("data_stream/bar/fields/hello-world.yml", `field "bbb" can't have unit property'`), errs[0].Error())
+		assert.Equal(t, fileError("data_stream/bar/fields/hello-world.yml", `field "eee" can't have unit property'`), errs[1].Error())
+		assert.Equal(t, fileError("data_stream/foo/fields/bad-file.yml", `field "fff" can't have metric type property'`), errs[2].Error())
+	}
+}

code/go/internal/validator/spec.go

@@ -68,6 +68,7 @@ func (s Spec) ValidatePackage(pkg Package) ve.ValidationErrors {
 		semantic.ValidateKibanaObjectIDs,
 		semantic.ValidateVersionIntegrity,
 		semantic.ValidateFieldGroups,
+		semantic.ValidateDimensionFields,
 	}
 	return rules.validate(pkg.RootPath)
 }