Implement WhenExpressions

Adding `WhenExpressions` used to efficiently specify guarded execution of `Tasks`, without spinning up new pods. We use `WhenExpressions` to avoid adding an opinionated andcomplex expression language to the Tekton API to ensure Tekton can be supported by as many systems as possible. The components of `WhenExpressions` are `Input`, `Operator` and `Values`: - `Input` is the input for the `Guard` checking which can be static inputs or variables, such as `Parameters` or `Results`. - `Values` is an array of string values. The `Values` array must be non-empty. It can contain static values or variables (`Parameters` or `Results`). - `Operator` represents an `Input`'s relationship to a set of `Values`. `Operators` we will use in `WhenExpressions` are `In` and `NotIn`. The declared `WhenExpressions` are evaluated before the `Task` is run. If all the `WhenExpressions` evaluate to `True`, the `Task` is run. If any of the `WhenExpressions` evaluate to `False`, the `Task` is skipped. Further details in [Conditions Beta TEP](https://github.com/tektoncd/community/blob/master/teps/0007-conditions-beta.md).
tektoncd · Aug 19, 2020 · e7a95aa · e7a95aa
1 parent 7d7119a
commit e7a95aa
Show file tree

Hide file tree

Showing 19 changed files with 1,634 additions and 21 deletions.
diff --git a/docs/pipelines.md b/docs/pipelines.md
@@ -15,6 +15,7 @@ weight: 3
     - [Using the `from` parameter](#using-the-from-parameter)
     - [Using the `runAfter` parameter](#using-the-runafter-parameter)
     - [Using the `retries` parameter](#using-the-retries-parameter)
+    - [Guard `Task` execution using `When Expressions`](#guard-task-execution-using-whenexpressions)
     - [Guard `Task` execution using `Conditions`](#guard-task-execution-using-conditions)
     - [Configuring the failure timeout](#configuring-the-failure-timeout)
   - [Using `Results`](#using-results)
@@ -316,6 +317,47 @@ tasks:
       name: build-push
 ```
 
+### Guard `Task` execution using `WhenExpressions`
+
+To run a `Task` only when certain conditions are met, it is possible to _guard_ task execution using
+the `when` field. The `when` field allows you to list a series of references to `WhenExpressions`.
+
+The components of `WhenExpressions` are `Input`, `Operator` and `Values`:
+
+- `Input` is the input for the `Guard` checking which can be static inputs or variables (`Parameters` or `Results`).
+- `Values` is an array of string values. The `Values` array must be non-empty. It can contain static values
+or variables (`Parameters` or `Results`).
+- `Operator` represents an `Input`'s relationship to a set of `Values`. `Operators` we will use in `Guards`
+are `In` and `NotIn`.
+
+The declared `WhenExpressions` are evaluated before the `Task` is run. If all the `WhenExpressions`
+evaluate to `True`, the `Task` is run. If any of the `WhenExpressions` evaluate to `False`, the `Task` is
+skipped.
+
+When `WhenExpressions` are specified in a `Task`, `Conditions` should not be speficied in the same `Task`.
+
+```yaml
+tasks:
+  - name: first-create-file
+    when:
+      - input: "$(params.path)"
+        operator: in
+        values: ["README.md"]
+    taskRef:
+      name: create-readme-file
+---
+tasks:
+  - name: echo-file-exists
+    when:
+      - input: "$(tasks.check-file.results.status)"
+        operator: in
+        values: ["exists"]
+    taskRef:
+        name: echo-file-exists
+```
+
+For an end-to-end example, see [`PipelineRun` with `WhenExpressions`](../examples/v1beta1/pipelineruns/pipelinerun-with-when-expressions.yaml).
+
 ### Guard `Task` execution using `Conditions`
 
 To run a `Task` only when certain conditions are met, it is possible to _guard_ task execution using

diff --git a/examples/v1beta1/pipelineruns/pipelinerun-with-when-expressions.yaml b/examples/v1beta1/pipelineruns/pipelinerun-with-when-expressions.yaml
@@ -0,0 +1,124 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: create-readme-file
+spec:
+  resources:
+    outputs:
+      - name: workspace
+        type: git
+  steps:
+    - name: write-new-stuff
+      image: ubuntu
+      script: 'touch $(resources.outputs.workspace.path)/README.md'
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: check-file
+spec:
+  params:
+    - name: path
+  resources:
+    inputs:
+      - name: workspace
+        type: git
+  results:
+    - name: status
+      description: indicating whether the file exists
+  steps:
+    - name: check-file
+      image: alpine
+      script: |
+        if test -f $(resources.inputs.workspace.path)/$(params.path); then
+          printf exists | tee /tekton/results/status
+        else
+          printf missing | tee /tekton/results/status
+        fi
+---
+apiVersion: tekton.dev/v1alpha1
+kind: PipelineResource
+metadata:
+  name: pipeline-git
+spec:
+  type: git
+  params:
+    - name: revision
+      value: master
+    - name: url
+      value: https://github.com/tektoncd/pipeline
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: echo-file-status
+spec:
+  params:
+    - name: status
+  steps:
+    - name: echo
+      image: ubuntu
+      script: 'echo file $(params.status)'
+---
+apiVersion: tekton.dev/v1beta1
+kind: Pipeline
+metadata:
+  name: guarded-pipeline
+spec:
+  resources:
+    - name: source-repo
+      type: git
+  params:
+    - name: path
+      default: "README.md"
+  tasks:
+    - name: first-create-file
+      when:
+        - input: "$(params.path)"
+          operator: in
+          values: ["README.md"]
+      taskRef:
+        name: create-readme-file
+      resources:
+        outputs:
+          - name: workspace
+            resource: source-repo
+    - name: check-file
+      when:
+        - input: "foo"
+          operator: in
+          values: ["foo", "bar"] 
+      taskRef: 
+        name: check-file
+      params:
+        - name: path
+          value: "$(params.path)"
+      resources:
+        inputs:
+          - name: workspace
+            resource: source-repo
+            from: [first-create-file]
+    - name: echo-file-exists
+      params:
+        - name: status
+          value: "$(tasks.check-file.results.status)" 
+      when:
+        - input: "$(tasks.check-file.results.status)"
+          operator: in 
+          values: ["exists"]
+      taskRef:
+        name: echo-file-status
+---
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: guarded-pr
+spec:
+  pipelineRef:
+    name: guarded-pipeline
+  serviceAccountName: 'default'
+  resources:
+    - name: source-repo
+      resourceRef:
+        name: pipeline-git
+
diff --git a/internal/builder/v1beta1/pipeline.go b/internal/builder/v1beta1/pipeline.go
@@ -24,6 +24,7 @@ import (
 	resource "github.com/tektoncd/pipeline/pkg/apis/resource/v1alpha1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/selection"
 	"knative.dev/pkg/apis"
 )
 
@@ -54,6 +55,8 @@ type PipelineRunStatusOp func(*v1beta1.PipelineRunStatus)
 // PipelineTaskConditionOp is an operation which modifies a PipelineTaskCondition
 type PipelineTaskConditionOp func(condition *v1beta1.PipelineTaskCondition)
 
+type PipelineTaskWhenExpressionOp func(*v1beta1.WhenExpression)
+
 // Pipeline creates a Pipeline with default values.
 // Any number of Pipeline modifier can be passed to transform it.
 func Pipeline(name string, ops ...PipelineOp) *v1beta1.Pipeline {
@@ -319,6 +322,17 @@ func PipelineTaskConditionResource(name, resource string, from ...string) Pipeli
 	}
 }
 
+// PipelineTaskWhenExpression adds a WhenExpression with the specified input, operator and values 
+func PipelineTaskWhenExpression(input string, op selection.Operator, values []string) PipelineTaskOp {
+	return func (pt *v1beta1.PipelineTask)  {
+		pt.WhenExpressions = append(pt.WhenExpressions, v1beta1.WhenExpression{
+			Input: input,
+			Operator: op,
+			Values: values,
+		})
+	}
+}
+
 // PipelineTaskWorkspaceBinding adds a workspace with the specified name, workspace and subpath on a PipelineTask.
 func PipelineTaskWorkspaceBinding(name, workspace, subPath string) PipelineTaskOp {
 	return func(pt *v1beta1.PipelineTask) {

diff --git a/internal/builder/v1beta1/pipeline_test.go b/internal/builder/v1beta1/pipeline_test.go
@@ -54,6 +54,7 @@ func TestPipeline(t *testing.T) {
 			tb.PipelineTaskOutputResource("some-image", "my-only-image-resource"),
 		),
 		tb.PipelineTask("never-gonna", "give-you-up",
+			tb.PipelineTaskWhenExpression("foo", "in", []string{"foo", "bar"}),
 			tb.RunAfter("foo"),
 			tb.PipelineTaskTimeout(5*time.Second),
 		),
@@ -133,6 +134,7 @@ func TestPipeline(t *testing.T) {
 			}, {
 				Name:     "never-gonna",
 				TaskRef:  &v1beta1.TaskRef{Name: "give-you-up"},
+				WhenExpressions: []v1beta1.WhenExpression{{Input: "foo", Operator: "in", Values: []string{"foo", "bar"}}},
 				RunAfter: []string{"foo"},
 				Timeout:  &metav1.Duration{Duration: 5 * time.Second},
 			}, {

diff --git a/pkg/apis/pipeline/v1beta1/pipeline_types.go b/pkg/apis/pipeline/v1beta1/pipeline_types.go
@@ -19,6 +19,7 @@ package v1beta1
 import (
 	"github.com/tektoncd/pipeline/pkg/reconciler/pipeline/dag"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/selection"
 )
 
 // +genclient
@@ -110,6 +111,10 @@ type PipelineTask struct {
 	// +optional
 	Conditions []PipelineTaskCondition `json:"conditions,omitempty"`
 
+	// WhenExpressions is a list of when expressions that need to be true for the task to run
+	// +optional
+	WhenExpressions []WhenExpression `json:"when,omitempty"`
+
 	// Retries represents how many times this task should be retried in case of task failure: ConditionSucceeded set to False
 	// +optional
 	Retries int `json:"retries,omitempty"`
@@ -176,6 +181,16 @@ func (pt PipelineTask) Deps() []string {
 			}
 		}
 	}
+	// Add any dependents from when expressions
+	for _, whenExpression := range pt.WhenExpressions {
+		expressions, ok := GetVarSubstitutionExpressionsForWhenExpression(whenExpression)
+		if ok {
+			resultRefs := NewResultRefs(expressions)
+			for _, resultRef := range resultRefs {
+				deps = append(deps, resultRef.PipelineTask)
+			}
+		}
+	}
 	return deps
 }
 
@@ -209,6 +224,18 @@ type PipelineTaskCondition struct {
 	Resources []PipelineTaskInputResource `json:"resources,omitempty"`
 }
 
+// WhenExpression allows a PipelineTask to declare expressions to be evaluated before the Task is run
+// to determine whether the Task should be executed or skipped
+type WhenExpression struct {
+	// Input is the string for guard checking which can be a static input or an output from a parent Task
+	Input string `json:"input,omitempty"`
+	// Operator that represents an Input's relationship to the values
+	Operator selection.Operator `json:"operator,omitempty"`
+	// Values is an array of strings, which is compared against the input, for guard checking
+	// It must be non-empty
+	Values []string `json:"values,omitempty"`
+}
+
 // PipelineDeclaredResource is used by a Pipeline to declare the types of the
 // PipelineResources that it will required to run and names which can be used to
 // refer to these PipelineResources in PipelineTaskResourceBindings.
@@ -270,3 +297,13 @@ type PipelineList struct {
 	metav1.ListMeta `json:"metadata,omitempty"`
 	Items           []Pipeline `json:"items"`
 }
+
+// ApplyReplacements applies replacements for When Expressions
+func (whenExpression *WhenExpression) ApplyReplacements(stringReplacements map[string]string) {
+	whenExpression.Input = ApplyReplacements(whenExpression.Input, stringReplacements)
+	var newValues []string
+	for _, val := range whenExpression.Values {
+		newValues = append(newValues, ApplyReplacements(val, stringReplacements))
+	}
+	whenExpression.Values = newValues
+}
diff --git a/pkg/apis/pipeline/v1beta1/pipeline_validation.go b/pkg/apis/pipeline/v1beta1/pipeline_validation.go
@@ -26,6 +26,7 @@ import (
 	"github.com/tektoncd/pipeline/pkg/reconciler/pipeline/dag"
 	"github.com/tektoncd/pipeline/pkg/substitution"
 	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation"
 	"knative.dev/pkg/apis"
@@ -181,6 +182,10 @@ func (ps *PipelineSpec) Validate(ctx context.Context) *apis.FieldError {
 		return apis.ErrInvalidValue(err.Error(), "spec.tasks.params.value")
 	}
 
+	if err := validateWhenExpressionsReferencesToTaskResults(ps.Tasks); err != nil {
+		return apis.ErrInvalidValue(err.Error(), "spec.tasks.when")
+	}
+
 	// The parameter variables should be valid
 	if err := validatePipelineParameterVariables(ps.Tasks, ps.Params); err != nil {
 		return err
@@ -194,6 +199,10 @@ func (ps *PipelineSpec) Validate(ctx context.Context) *apis.FieldError {
 		return err
 	}
 
+	if err := validateWhenExpressions(ps.Tasks); err != nil {
+		return err
+	}
+
 	// Validate the pipeline's workspaces.
 	if err := validatePipelineWorkspaces(ps.Workspaces, ps.Tasks, ps.Finally); err != nil {
 		return err
@@ -446,6 +455,25 @@ func validateParamResults(tasks []PipelineTask) error {
 	return nil
 }
 
+// validateWhenExpressionsReferencesToTaskResults ensures that task result variables are properly configured
+func validateWhenExpressionsReferencesToTaskResults(tasks []PipelineTask) error {
+	for _, task := range tasks {
+		for _, we := range task.WhenExpressions {
+			expressions, ok := GetVarSubstitutionExpressionsForWhenExpression(we)
+			if ok {
+				if LooksLikeContainsResultRefs(expressions) {
+					expressions = filter(expressions, looksLikeResultRef)
+					resultRefs := NewResultRefs(expressions)
+					if len(expressions) != len(resultRefs) {
+						return fmt.Errorf("expected all of the expressions %v to be result expressions but only %v were", expressions, resultRefs)
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func filter(arr []string, cond func(string) bool) []string {
 	result := []string{}
 	for i := range arr {
@@ -488,6 +516,9 @@ func validateFinalTasks(finalTasks []PipelineTask) *apis.FieldError {
 		if len(f.Conditions) != 0 {
 			return apis.ErrInvalidValue(fmt.Sprintf("no conditions allowed under spec.finally, final task %s has conditions specified", f.Name), "spec.finally")
 		}
+		if len(f.WhenExpressions) != 0 {
+			return apis.ErrInvalidValue(fmt.Sprintf("no when expressions allowed under spec.finally, final task %s has when expressions specified", f.Name), "spec.finally")
+		}
 	}
 
 	if err := validateTaskResultReferenceNotUsed(finalTasks); err != nil {
@@ -531,3 +562,23 @@ func validateTasksInputFrom(tasks []PipelineTask) *apis.FieldError {
 	}
 	return nil
 }
+
+func validateWhenExpressions(tasks []PipelineTask) *apis.FieldError {
+	for i, t := range tasks {
+		// can't have both WheExpressions and Conditions at the same time
+		prefix := "spec.tasks"
+		if t.WhenExpressions != nil && t.Conditions != nil {
+			return apis.ErrMultipleOneOf(fmt.Sprintf(fmt.Sprintf(prefix+"[%d].when", i), fmt.Sprintf(prefix+"[%d].conditions", i)))
+		}
+
+		for _, we := range t.WhenExpressions {
+			if we.Operator != selection.In && we.Operator != selection.NotIn {
+				return apis.ErrInvalidValue(fmt.Sprintf("operator '%v' is not recognized", we.Operator), "spec.task.when")
+			}
+			if len(we.Values) == 0 {
+				return apis.ErrInvalidValue("expecting non-empty values field", "spec.task.when")
+			}
+		}
+	}
+	return nil
+}