[pkg/ottl] Add lists to the OTTL grammar (#14709)

This makes it possible to use lists in the OTTL grammar. Existing instances of list parameters are converted to variadic parameters and maintain the same functionality.

Co-authored-by: Evan Bradley <[email protected]>

.chloggen/ottl-lists.yaml

@@ -0,0 +1,20 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: pkg/ottl
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add lists to the OTTL grammar and change the function signature of `Concat`.
+
+# One or more tracking issues related to the change
+issues: [13391]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  The following functions have changed:
+  `keep_keys` now has a function signature of `keep_keys(target, keys[])`.
+  `Concat` now has a function signature of `Concat(keys[], delimiter)`.
+

.chloggen/transformprocessor-lists.yaml

@@ -0,0 +1,20 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: processor/transform
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Convert the `keep_keys` and `Concat` functions to use list parameters and change the function signature of `Concat`.
+
+# One or more tracking issues related to the change
+issues: [13391]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  The updated functions now have the following signatures:
+  `keep_keys` now has a function signature of `keep_keys(target, keys[])`.
+  `Concat` now has a function signature of `Concat(keys[], delimiter)`.
+

pkg/ottl/README.md

@@ -25,7 +25,7 @@ Example Invocations
 
 #### Invocation parameters
 
-The OTTL will use reflection to determine parameter types when parsing an invocation within a statement.  When interpreting slice parameter types, the OTTL will attempt to build the slice from all remaining Values in the Invocation's arguments.  As a result, function implementations of Invocations may only contain one slice argument and it must be the last argument in the function definition.  See [function syntax guidelines](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/processing.md#function-syntax) for more details.
+The OTTL will use reflection to determine parameter types when parsing an invocation within a statement.
 
 The following types are supported for single parameter values:
 - `Setter`
@@ -41,13 +41,14 @@ For slice parameters, the following types are supported:
 - `string`
 - `float64`
 - `int64`
-- `uint8`. Slices of bytes will be interpreted as a byte array.
+- `uint8`. Byte slice literals are parsed as byte slices by the OTTL.
 - `Getter`
 
 ### Values
 
 Values are passed as input to an Invocation or are used in an Expression. Values can take the form of:
 - [Paths](#paths).
+- [Lists](#lists).
 - [Literals](#literals).
 - [Enums](#enums).
 - [Invocations](#invocations).
@@ -68,6 +69,16 @@ Example Paths
 - `resource.name`
 - `resource.attributes["key"]`
 
+#### Lists
+
+A List Value comprises a sequence of Expressions or supported Literals.
+
+Example List Values:
+- `[]`
+- `[1]`
+- `["1", "2", "3"]`
+- `["a", attributes["key"], Concat(["a", "b"], "-")]`
+
 #### Literals
 
 Literals are literal interpretations of the Value into a Go value.  Accepted literals are:
@@ -180,11 +191,11 @@ logs:
 
 ```
 traces:
-  keep_keys(attributes, "http.method", "http.status_code")
+  keep_keys(attributes, ["http.method", "http.status_code"])
 metrics:
-  keep_keys(attributes, "http.method", "http.status_code")
+  keep_keys(attributes, ["http.method", "http.status_code"])
 logs:
-  keep_keys(attributes, "http.method", "http.status_code")
+  keep_keys(attributes, ["http.method", "http.status_code"])
 ```
 
 ### Reduce cardinality of an attribute

pkg/ottl/functions.go

@@ -58,18 +58,17 @@ func (p *Parser[K]) buildArgs(inv invocation, fType reflect.Type) ([]reflect.Val
 	for i := 0; i < fType.NumIn(); i++ {
 		argType := fType.In(i)
 
-		switch argType.Kind() {
-		case reflect.Slice:
-			err := p.buildSliceArg(inv, argType, i, &args)
+		if argType.Kind() == reflect.Slice {
+			arg, err := p.buildSliceArg(inv, argType, i)
 			if err != nil {
 				return nil, err
 			}
-			// Slice arguments must be the final argument in an invocation.
-			return args, nil
-		default:
-			isInternalArg := p.buildInternalArg(argType, &args)
+			args = append(args, arg)
+		} else {
+			arg, isInternalArg := p.buildInternalArg(argType)
 
 			if isInternalArg {
+				args = append(args, arg)
 				continue
 			}
 
@@ -78,12 +77,16 @@ func (p *Parser[K]) buildArgs(inv invocation, fType reflect.Type) ([]reflect.Val
 			}
 
 			argDef := inv.Arguments[DSLArgumentIndex]
-			err := p.buildArg(argDef, argType, DSLArgumentIndex, &args)
-			DSLArgumentIndex++
+			val, err := p.buildArg(argDef, argType, DSLArgumentIndex)
+
 			if err != nil {
 				return nil, err
 			}
+
+			args = append(args, reflect.ValueOf(val))
 		}
+
+		DSLArgumentIndex++
 	}
 
 	if len(inv.Arguments) > DSLArgumentIndex {
@@ -93,116 +96,124 @@ func (p *Parser[K]) buildArgs(inv invocation, fType reflect.Type) ([]reflect.Val
 	return args, nil
 }
 
-func (p *Parser[K]) buildSliceArg(inv invocation, argType reflect.Type, startingIndex int, args *[]reflect.Value) error {
+func (p *Parser[K]) buildSliceArg(inv invocation, argType reflect.Type, index int) (reflect.Value, error) {
 	name := argType.Elem().Name()
 	switch {
+	case name == reflect.Uint8.String():
+		if inv.Arguments[index].Bytes == nil {
+			return reflect.ValueOf(nil), fmt.Errorf("invalid argument for slice parameter at position %v, must be a byte slice literal", index)
+		}
+		return reflect.ValueOf(([]byte)(*inv.Arguments[index].Bytes)), nil
 	case name == reflect.String.String():
-		var arg []string
-		for j := startingIndex; j < len(inv.Arguments); j++ {
-			if inv.Arguments[j].String == nil {
-				return fmt.Errorf("invalid argument for slice parameter at position %v, must be a string", j)
-			}
-			arg = append(arg, *inv.Arguments[j].String)
+		arg, err := buildSlice[string](inv, argType, index, p.buildArg, name)
+		if err != nil {
+			return reflect.ValueOf(nil), err
 		}
-		*args = append(*args, reflect.ValueOf(arg))
+		return arg, nil
 	case name == reflect.Float64.String():
-		var arg []float64
-		for j := startingIndex; j < len(inv.Arguments); j++ {
-			if inv.Arguments[j].Float == nil {
-				return fmt.Errorf("invalid argument for slice parameter at position %v, must be a float", j)
-			}
-			arg = append(arg, *inv.Arguments[j].Float)
+		arg, err := buildSlice[float64](inv, argType, index, p.buildArg, name)
+		if err != nil {
+			return reflect.ValueOf(nil), err
 		}
-		*args = append(*args, reflect.ValueOf(arg))
+		return arg, nil
 	case name == reflect.Int64.String():
-		var arg []int64
-		for j := startingIndex; j < len(inv.Arguments); j++ {
-			if inv.Arguments[j].Int == nil {
-				return fmt.Errorf("invalid argument for slice parameter at position %v, must be an int", j)
-			}
-			arg = append(arg, *inv.Arguments[j].Int)
-		}
-		*args = append(*args, reflect.ValueOf(arg))
-	case name == reflect.Uint8.String():
-		if inv.Arguments[startingIndex].Bytes == nil {
-			return fmt.Errorf("invalid argument for slice parameter at position %v, must be a byte slice literal", startingIndex)
+		arg, err := buildSlice[int64](inv, argType, index, p.buildArg, name)
+		if err != nil {
+			return reflect.ValueOf(nil), err
 		}
-		*args = append(*args, reflect.ValueOf(([]byte)(*inv.Arguments[startingIndex].Bytes)))
+		return arg, nil
 	case strings.HasPrefix(name, "Getter"):
-		var arg []Getter[K]
-		for j := startingIndex; j < len(inv.Arguments); j++ {
-			val, err := p.newGetter(inv.Arguments[j])
-			if err != nil {
-				return err
-			}
-			arg = append(arg, val)
+		arg, err := buildSlice[Getter[K]](inv, argType, index, p.buildArg, name)
+		if err != nil {
+			return reflect.ValueOf(nil), err
 		}
-		*args = append(*args, reflect.ValueOf(arg))
+		return arg, nil
 	default:
-		return fmt.Errorf("unsupported slice type '%s' for function '%v'", argType.Elem().Name(), inv.Function)
+		return reflect.ValueOf(nil), fmt.Errorf("unsupported slice type '%s' for function '%v'", argType.Elem().Name(), inv.Function)
 	}
-	return nil
 }
 
 // Handle interfaces that can be passed as arguments to OTTL function invocations.
-func (p *Parser[K]) buildArg(argDef value, argType reflect.Type, index int, args *[]reflect.Value) error {
+func (p *Parser[K]) buildArg(argDef value, argType reflect.Type, index int) (any, error) {
 	name := argType.Name()
 	switch {
 	case strings.HasPrefix(name, "Setter"):
 		fallthrough
 	case strings.HasPrefix(name, "GetSetter"):
 		arg, err := p.pathParser(argDef.Path)
 		if err != nil {
-			return fmt.Errorf("invalid argument at position %v %w", index, err)
+			return nil, fmt.Errorf("invalid argument at position %v %w", index, err)
 		}
-		*args = append(*args, reflect.ValueOf(arg))
+		return arg, nil
 	case strings.HasPrefix(name, "Getter"):
 		arg, err := p.newGetter(argDef)
 		if err != nil {
-			return fmt.Errorf("invalid argument at position %v %w", index, err)
+			return nil, fmt.Errorf("invalid argument at position %v %w", index, err)
 		}
-		*args = append(*args, reflect.ValueOf(arg))
+		return arg, nil
 	case name == "Enum":
 		arg, err := p.enumParser(argDef.Enum)
 		if err != nil {
-			return fmt.Errorf("invalid argument at position %v must be an Enum", index)
+			return nil, fmt.Errorf("invalid argument at position %v must be an Enum", index)
 		}
-		*args = append(*args, reflect.ValueOf(*arg))
-	case name == "string":
+		return *arg, nil
+	case name == reflect.String.String():
 		if argDef.String == nil {
-			return fmt.Errorf("invalid argument at position %v, must be an string", index)
+			return nil, fmt.Errorf("invalid argument at position %v, must be an string", index)
 		}
-		*args = append(*args, reflect.ValueOf(*argDef.String))
-	case name == "float64":
+		return *argDef.String, nil
+	case name == reflect.Float64.String():
 		if argDef.Float == nil {
-			return fmt.Errorf("invalid argument at position %v, must be an float", index)
+			return nil, fmt.Errorf("invalid argument at position %v, must be an float", index)
 		}
-		*args = append(*args, reflect.ValueOf(*argDef.Float))
-	case name == "int64":
+		return *argDef.Float, nil
+	case name == reflect.Int64.String():
 		if argDef.Int == nil {
-			return fmt.Errorf("invalid argument at position %v, must be an int", index)
+			return nil, fmt.Errorf("invalid argument at position %v, must be an int", index)
 		}
-		*args = append(*args, reflect.ValueOf(*argDef.Int))
-	case name == "bool":
+		return *argDef.Int, nil
+	case name == reflect.Bool.String():
 		if argDef.Bool == nil {
-			return fmt.Errorf("invalid argument at position %v, must be a bool", index)
+			return nil, fmt.Errorf("invalid argument at position %v, must be a bool", index)
 		}
-		*args = append(*args, reflect.ValueOf(bool(*argDef.Bool)))
+		return bool(*argDef.Bool), nil
 	default:
-		return errors.New("unsupported argument type")
+		return nil, errors.New("unsupported argument type")
 	}
-	return nil
 }
 
 // Handle interfaces that can be declared as parameters to a OTTL function, but will
 // never be called in an invocation. Returns whether the arg is an internal arg.
-func (p *Parser[K]) buildInternalArg(argType reflect.Type, args *[]reflect.Value) bool {
-	switch argType.Name() {
-	case "TelemetrySettings":
-		*args = append(*args, reflect.ValueOf(p.telemetrySettings))
-	default:
-		return false
+func (p *Parser[K]) buildInternalArg(argType reflect.Type) (reflect.Value, bool) {
+	if argType.Name() == "TelemetrySettings" {
+		return reflect.ValueOf(p.telemetrySettings), true
+	}
+	return reflect.ValueOf(nil), false
+}
+
+type buildArgFunc func(value, reflect.Type, int) (any, error)
+
+func buildSlice[T any](inv invocation, argType reflect.Type, index int, buildArg buildArgFunc, name string) (reflect.Value, error) {
+	if inv.Arguments[index].List == nil {
+		return reflect.ValueOf(nil), fmt.Errorf("invalid argument for parameter at position %v, must be a list of type %v", index, name)
+	}
+
+	vals := []T{}
+	values := inv.Arguments[index].List.Values
+	for j := 0; j < len(values); j++ {
+		untypedVal, err := buildArg(values[j], argType.Elem(), j)
+		if err != nil {
+			return reflect.ValueOf(nil), err
+		}
+
+		val, ok := untypedVal.(T)
+
+		if !ok {
+			return reflect.ValueOf(nil), fmt.Errorf("invalid element type at list index %v for argument at position %v, must be of type %v", j, index, name)
+		}
+
+		vals = append(vals, val)
 	}
 
-	return true
+	return reflect.ValueOf(vals), nil
 }