TEP-0075: Object variable replacement on Pipeline/PipelineRun level

Replace the following references with actual value
- the reference to the whole object param defined in PipelineSpec
- the reference to the individual keys of an object param

pkg/apis/pipeline/v1alpha1/param_types_test.go

@@ -173,7 +173,7 @@ func TestArrayOrString_ApplyReplacements(t *testing.T) {
 	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tt.args.input.ApplyReplacements(tt.args.stringReplacements, tt.args.arrayReplacements)
+			tt.args.input.ApplyReplacements(tt.args.stringReplacements, tt.args.arrayReplacements, nil)
 			if d := cmp.Diff(tt.expectedOutput, tt.args.input); d != "" {
 				t.Errorf("ApplyReplacements() output did not match expected value %s", diff.PrintWantGot(d))
 			}

pkg/apis/pipeline/v1alpha1/pipeline_resource_types.go

@@ -50,7 +50,6 @@ var AllResourceTypes = resource.AllResourceTypes
 
 // PipelineResource describes a resource that is an input to or output from a
 // Task.
-//
 type PipelineResource = resource.PipelineResource
 
 // PipelineResourceSpec defines  an individual resources used in the pipeline.

pkg/apis/pipeline/v1alpha1/taskrun_validation.go

@@ -132,9 +132,9 @@ func validateWorkspaceBindings(ctx context.Context, wb []WorkspaceBinding) *apis
 }
 
 // validatePipelineResources validates that
-//	1. resource is not declared more than once
-//	2. if both resource reference and resource spec is defined at the same time
-//	3. at least resource ref or resource spec is defined
+//  1. resource is not declared more than once
+//  2. if both resource reference and resource spec is defined at the same time
+//  3. at least resource ref or resource spec is defined
 func validatePipelineResources(ctx context.Context, resources []TaskResourceBinding, path string) *apis.FieldError {
 	encountered := sets.NewString()
 	for _, r := range resources {

pkg/apis/pipeline/v1beta1/param_types.go

@@ -195,15 +195,58 @@ func (arrayOrString ArrayOrString) MarshalJSON() ([]byte, error) {
 }
 
 // ApplyReplacements applyes replacements for ArrayOrString type
-func (arrayOrString *ArrayOrString) ApplyReplacements(stringReplacements map[string]string, arrayReplacements map[string][]string) {
-	if arrayOrString.Type == ParamTypeString {
-		arrayOrString.StringVal = substitution.ApplyReplacements(arrayOrString.StringVal, stringReplacements)
-	} else {
+func (arrayOrString *ArrayOrString) ApplyReplacements(stringReplacements map[string]string, arrayReplacements map[string][]string, objectReplacements map[string]map[string]string) {
+	switch arrayOrString.Type {
+	case ParamTypeArray:
 		var newArrayVal []string
 		for _, v := range arrayOrString.ArrayVal {
 			newArrayVal = append(newArrayVal, substitution.ApplyArrayReplacements(v, stringReplacements, arrayReplacements)...)
 		}
 		arrayOrString.ArrayVal = newArrayVal
+	case ParamTypeObject:
+		newObjectVal := map[string]string{}
+		for k, v := range arrayOrString.ObjectVal {
+			newObjectVal[k] = substitution.ApplyReplacements(v, stringReplacements)
+		}
+		arrayOrString.ObjectVal = newObjectVal
+	default:
+		arrayOrString.applyOrCorrect(stringReplacements, arrayReplacements, objectReplacements)
+	}
+}
+
+// applyOrCorrect deals with string param whose value can be string literal or a reference to a string/array/object param/result.
+// If the value of arrayOrString is a reference to array or object, the type will be corrected from string to array/object.
+func (arrayOrString *ArrayOrString) applyOrCorrect(stringReplacements map[string]string, arrayReplacements map[string][]string, objectReplacements map[string]map[string]string) {
+	stringVal := arrayOrString.StringVal
+
+	// if the stringVal is a string literal or a string that mixed with var references
+	// just do the normal string replacement
+	if !exactVariableSubstitutionRegex.MatchString(stringVal) {
+		arrayOrString.StringVal = substitution.ApplyReplacements(arrayOrString.StringVal, stringReplacements)
+		return
+	}
+
+	// trim the head "$(" and the tail ")" or "[*])"
+	// i.e. get "params.name" from "$(params.name)" or "$(params.name[*])"
+	trimedStringVal := strings.TrimSuffix(strings.TrimSuffix(strings.TrimPrefix(stringVal, "$("), ")"), "[*]")
+
+	// if the stringVal is a reference to a string param
+	if _, ok := stringReplacements[trimedStringVal]; ok {
+		arrayOrString.StringVal = substitution.ApplyReplacements(arrayOrString.StringVal, stringReplacements)
+	}
+
+	// if the stringVal is a reference to an array param, we need to change the type other than apply replacement
+	if _, ok := arrayReplacements[trimedStringVal]; ok {
+		arrayOrString.StringVal = ""
+		arrayOrString.ArrayVal = substitution.ApplyArrayReplacements(stringVal, stringReplacements, arrayReplacements)
+		arrayOrString.Type = ParamTypeArray
+	}
+
+	// if the stringVal is a reference an object param, we need to change the type other than apply replacement
+	if _, ok := objectReplacements[trimedStringVal]; ok {
+		arrayOrString.StringVal = ""
+		arrayOrString.ObjectVal = objectReplacements[trimedStringVal]
+		arrayOrString.Type = ParamTypeObject
 	}
 }
 

pkg/apis/pipeline/v1beta1/param_types_test.go

@@ -162,6 +162,7 @@ func TestArrayOrString_ApplyReplacements(t *testing.T) {
 		input              *v1beta1.ArrayOrString
 		stringReplacements map[string]string
 		arrayReplacements  map[string][]string
+		objectReplacements map[string]map[string]string
 	}
 	tests := []struct {
 		name           string
@@ -176,7 +177,15 @@ func TestArrayOrString_ApplyReplacements(t *testing.T) {
 		},
 		expectedOutput: v1beta1.NewArrayOrString("an", "array"),
 	}, {
-		name: "string replacements on string",
+		name: "single string replacement on string",
+		args: args{
+			input:              v1beta1.NewArrayOrString("$(params.myString1)"),
+			stringReplacements: map[string]string{"params.myString1": "value1", "params.myString2": "value2"},
+			arrayReplacements:  map[string][]string{"arraykey": {"array", "value"}, "sdfdf": {"asdf", "sdfsd"}},
+		},
+		expectedOutput: v1beta1.NewArrayOrString("value1"),
+	}, {
+		name: "multiple string replacements on string",
 		args: args{
 			input:              v1beta1.NewArrayOrString("astring$(some) asdf $(anotherkey)"),
 			stringReplacements: map[string]string{"some": "value", "anotherkey": "value"},
@@ -207,10 +216,70 @@ func TestArrayOrString_ApplyReplacements(t *testing.T) {
 			arrayReplacements:  map[string][]string{"arraykey": {}},
 		},
 		expectedOutput: v1beta1.NewArrayOrString("firstvalue", "lastvalue"),
+	}, {
+		name: "array replacement on string val",
+		args: args{
+			input:             v1beta1.NewArrayOrString("$(params.myarray)"),
+			arrayReplacements: map[string][]string{"params.myarray": {"a", "b", "c"}},
+		},
+		expectedOutput: v1beta1.NewArrayOrString("a", "b", "c"),
+	}, {
+		name: "array star replacement on string val",
+		args: args{
+			input:             v1beta1.NewArrayOrString("$(params.myarray[*])"),
+			arrayReplacements: map[string][]string{"params.myarray": {"a", "b", "c"}},
+		},
+		expectedOutput: v1beta1.NewArrayOrString("a", "b", "c"),
+	}, {
+		name: "object replacement on string val",
+		args: args{
+			input: v1beta1.NewArrayOrString("$(params.object)"),
+			objectReplacements: map[string]map[string]string{
+				"params.object": {
+					"url":    "abc.com",
+					"commit": "af234",
+				},
+			},
+		},
+		expectedOutput: v1beta1.NewObject(map[string]string{
+			"url":    "abc.com",
+			"commit": "af234",
+		}),
+	}, {
+		name: "object star replacement on string val",
+		args: args{
+			input: v1beta1.NewArrayOrString("$(params.object[*])"),
+			objectReplacements: map[string]map[string]string{
+				"params.object": {
+					"url":    "abc.com",
+					"commit": "af234",
+				},
+			},
+		},
+		expectedOutput: v1beta1.NewObject(map[string]string{
+			"url":    "abc.com",
+			"commit": "af234",
+		}),
+	}, {
+		name: "string replacement on object individual variables",
+		args: args{
+			input: v1beta1.NewObject(map[string]string{
+				"key1": "$(mystring)",
+				"key2": "$(anotherObject.key)",
+			}),
+			stringReplacements: map[string]string{
+				"mystring":          "foo",
+				"anotherObject.key": "bar",
+			},
+		},
+		expectedOutput: v1beta1.NewObject(map[string]string{
+			"key1": "foo",
+			"key2": "bar",
+		}),
 	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tt.args.input.ApplyReplacements(tt.args.stringReplacements, tt.args.arrayReplacements)
+			tt.args.input.ApplyReplacements(tt.args.stringReplacements, tt.args.arrayReplacements, tt.args.objectReplacements)
 			if d := cmp.Diff(tt.expectedOutput, tt.args.input); d != "" {
 				t.Errorf("ApplyReplacements() output did not match expected value %s", diff.PrintWantGot(d))
 			}

pkg/apis/pipeline/v1beta1/resource_types_validation.go

@@ -73,9 +73,9 @@ func (tr *TaskRunResources) Validate(ctx context.Context) *apis.FieldError {
 }
 
 // validateTaskRunResources validates that
-//	1. resource is not declared more than once
-//	2. if both resource reference and resource spec is defined at the same time
-//	3. at least resource ref or resource spec is defined
+//  1. resource is not declared more than once
+//  2. if both resource reference and resource spec is defined at the same time
+//  3. at least resource ref or resource spec is defined
 func validateTaskRunResources(ctx context.Context, resources []TaskResourceBinding, path string) *apis.FieldError {
 	encountered := sets.NewString()
 	for _, r := range resources {

pkg/apis/pipeline/v1beta1/resultref.go

@@ -43,14 +43,18 @@ const (
 	ResultResultPart = "results"
 	// TODO(#2462) use one regex across all substitutions
 	// variableSubstitutionFormat matches format like $result.resultname, $result.resultname[int] and $result.resultname[*]
-	variableSubstitutionFormat = `\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*(\[([0-9])*\*?\])?\)`
+	variableSubstitutionFormat = `\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*(\[([0-9]+|\*)\])?\)`
+	// exactVariableSubstitutionFormat matches strings that only contain a single reference to result or param variables, but nothing else
+	// i.e. `$(result.resultname)` is a match, but `foo $(result.resultname)` is not.
+	exactVariableSubstitutionFormat = `^\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*(\[([0-9]+|\*)\])?\)$`
 	// arrayIndexing will match all `[int]` and `[*]` for parseExpression
 	arrayIndexing = `\[([0-9])*\*?\]`
 	// ResultNameFormat Constant used to define the the regex Result.Name should follow
 	ResultNameFormat = `^([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$`
 )
 
 var variableSubstitutionRegex = regexp.MustCompile(variableSubstitutionFormat)
+var exactVariableSubstitutionRegex = regexp.MustCompile(exactVariableSubstitutionFormat)
 var resultNameFormatRegex = regexp.MustCompile(ResultNameFormat)
 var arrayIndexingRegex = regexp.MustCompile(arrayIndexing)
 

pkg/apis/pipeline/v1beta1/taskrun_defaults_test.go

@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+	http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,

pkg/controller/filter.go

@@ -32,10 +32,10 @@ import (
 // For example, a controller impl that wants to be notified of updates to Runs
 // which reference a Task with apiVersion "example.dev/v0" and kind "Example":
 //
-//     runinformer.Get(ctx).Informer().AddEventHandler(cache.FilteringResourceEventHandler{
-//       FilterFunc: FilterRunRef("example.dev/v0", "Example"),
-//       Handler:    controller.HandleAll(impl.Enqueue),
-//     })
+//	runinformer.Get(ctx).Informer().AddEventHandler(cache.FilteringResourceEventHandler{
+//	  FilterFunc: FilterRunRef("example.dev/v0", "Example"),
+//	  Handler:    controller.HandleAll(impl.Enqueue),
+//	})
 func FilterRunRef(apiVersion, kind string) func(interface{}) bool {
 	return func(obj interface{}) bool {
 		r, ok := obj.(*v1alpha1.Run)
@@ -65,10 +65,10 @@ func FilterRunRef(apiVersion, kind string) func(interface{}) bool {
 // For example, a controller impl that wants to be notified of updates to TaskRuns that are controlled by
 // a Run which references a custom task with apiVersion "example.dev/v0" and kind "Example":
 //
-//     taskruninformer.Get(ctx).Informer().AddEventHandler(cache.FilteringResourceEventHandler{
-//       FilterFunc: FilterOwnerRunRef("example.dev/v0", "Example"),
-//       Handler:    controller.HandleAll(impl.Enqueue),
-//     })
+//	taskruninformer.Get(ctx).Informer().AddEventHandler(cache.FilteringResourceEventHandler{
+//	  FilterFunc: FilterOwnerRunRef("example.dev/v0", "Example"),
+//	  Handler:    controller.HandleAll(impl.Enqueue),
+//	})
 func FilterOwnerRunRef(runLister listersalpha.RunLister, apiVersion, kind string) func(interface{}) bool {
 	return func(obj interface{}) bool {
 		object, ok := obj.(metav1.Object)

pkg/git/git_test.go

@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+	http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,

pkg/internal/affinityassistant/transformer_test.go

@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+	http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,

pkg/internal/limitrange/transformer_test.go

@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+	http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,

pkg/reconciler/doc.go

@@ -21,10 +21,13 @@ limitations under the License.
 //
 // Despite defining a Reconciler, each of the packages here are expected to
 // expose a controller constructor like:
-//    func NewController(...) *controller.Impl { ... }
+//
+//	func NewController(...) *controller.Impl { ... }
+//
 // These constructors will:
 // 1. Construct the Reconciler,
 // 2. Construct a controller.Impl with that Reconciler,
 // 3. Wire the assorted informers this Reconciler watches to call appropriate
-//   enqueue methods on the controller.
+//
+//	enqueue methods on the controller.
 package reconciler