diff --git a/docs/matrix.md b/docs/matrix.md index 699df515871..80fd00bc4a4 100644 --- a/docs/matrix.md +++ b/docs/matrix.md @@ -15,6 +15,8 @@ weight: 11 - [Results](#results) - [Specifying Results in a Matrix](#specifying-results-in-a-matrix) - [Results from fanned out PipelineTasks](#results-from-fanned-out-pipelinetasks) +- [Fan Out](#fan-out) + - [`PipelineTasks` with `Tasks`](#pipelinetasks-with-tasks) ## Overview @@ -134,3 +136,28 @@ Consuming `Results` from previous `TaskRuns` or `Runs` in a `Matrix`, which woul Consuming `Results` from fanned out `PipelineTasks` will not be in the supported in the initial iteration of `Matrix`. Supporting consuming `Results` from fanned out `PipelineTasks` will be revisited after array and object `Results` are supported. + +## Fan Out + +### `PipelineTasks` with `Tasks` + +When a `PipelineTask` has a `Task` and a `Matrix`, the `Task` will be executed in parallel `TaskRuns` with +substitutions from combinations of `Parameters`. + +In this [example](/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix.yaml), nine `TaskRuns` are created +with combinations of platforms ("linux", "mac", "windows") and browsers ("chrome", "safari", "firefox"). + +```shell +$ tkn taskruns list + +NAME STARTED DURATION STATUS +matrixed-pr-6lvzk-platforms-and-browsers-8 11 seconds ago 7 seconds Succeeded +matrixed-pr-6lvzk-platforms-and-browsers-6 12 seconds ago 7 seconds Succeeded +matrixed-pr-6lvzk-platforms-and-browsers-7 12 seconds ago 9 seconds Succeeded +matrixed-pr-6lvzk-platforms-and-browsers-4 12 seconds ago 7 seconds Succeeded +matrixed-pr-6lvzk-platforms-and-browsers-5 12 seconds ago 6 seconds Succeeded +matrixed-pr-6lvzk-platforms-and-browsers-3 13 seconds ago 7 seconds Succeeded +matrixed-pr-6lvzk-platforms-and-browsers-1 13 seconds ago 8 seconds Succeeded +matrixed-pr-6lvzk-platforms-and-browsers-2 13 seconds ago 8 seconds Succeeded +matrixed-pr-6lvzk-platforms-and-browsers-0 13 seconds ago 8 seconds Succeeded +``` \ No newline at end of file diff --git a/docs/pipelineruns.md b/docs/pipelineruns.md index a4aa01fc3e9..d39f3294072 100644 --- a/docs/pipelineruns.md +++ b/docs/pipelineruns.md @@ -1012,17 +1012,19 @@ Task Runs: The name of the `TaskRuns` and `Runs` owned by a `PipelineRun` are univocally associated to the owning resource. If a `PipelineRun` resource is deleted and created with the same name, the child `TaskRuns` will be created with the -same name as before. The base format of the name is `-`. The name may vary -according the logic of [`kmeta.ChildName`](https://pkg.go.dev/github.com/knative/pkg/kmeta#ChildName). +same name as before. The base format of the name is `-`. If the `PipelineTask` +has a `Matrix`, the name will have an int suffix with format `--`. +The name may vary according the logic of [`kmeta.ChildName`](https://pkg.go.dev/github.com/knative/pkg/kmeta#ChildName). Some examples: -| `PipelineRun` Name | `PipelineTask` Name | `TaskRun` Name | -|--------------------------|------------------------------|--------------------| -| pipeline-run | task1 | pipeline-run-task1 | -| pipeline-run | task2-0123456789-0123456789-0123456789-0123456789-0123456789 | pipeline-runee4a397d6eab67777d4e6f9991cd19e6-task2-0123456789-0 | -| pipeline-run-0123456789-0123456789-0123456789-0123456789 | task3 | pipeline-run-0123456789-0123456789-0123456789-0123456789-task3 | -| pipeline-run-0123456789-0123456789-0123456789-0123456789 | task2-0123456789-0123456789-0123456789-0123456789-0123456789 | pipeline-run-0123456789-012345607ad8c7aac5873cdfabe472a68996b5c | +| `PipelineRun` Name | `PipelineTask` Name | `TaskRun` Names | +|----------------------------------------------------------|--------------------------------------------------------------|----------------------------------------------------------------------------------------| +| pipeline-run | task1 | pipeline-run-task1 | +| pipeline-run | task2-0123456789-0123456789-0123456789-0123456789-0123456789 | pipeline-runee4a397d6eab67777d4e6f9991cd19e6-task2-0123456789-0 | +| pipeline-run-0123456789-0123456789-0123456789-0123456789 | task3 | pipeline-run-0123456789-0123456789-0123456789-0123456789-task3 | +| pipeline-run-0123456789-0123456789-0123456789-0123456789 | task2-0123456789-0123456789-0123456789-0123456789-0123456789 | pipeline-run-0123456789-012345607ad8c7aac5873cdfabe472a68996b5c | +| pipeline-run | task4 (with 2x2 `Matrix`) | pipeline-run-task1-0, pipeline-run-task1-2, pipeline-run-task1-3, pipeline-run-task1-4 | ## Cancelling a `PipelineRun` diff --git a/docs/pipelines.md b/docs/pipelines.md index 86d1f7aa5eb..93b84b75141 100644 --- a/docs/pipelines.md +++ b/docs/pipelines.md @@ -366,7 +366,7 @@ spec: - firefox ``` -For further information, read [`Matrix`](./matrix.md). +For further information, read [`Matrix`](./matrix.md) and see [end-to-end example](/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix.yaml). ### Specifying `Workspaces` in `PipelineTasks` @@ -1199,7 +1199,7 @@ spec: - "bar" ``` -For further information, read [`Matrix`](./matrix.md). +For further information, read [`Matrix`](./matrix.md) and see [end-to-end example](/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix.yaml). ### Consuming `Task` execution results in `finally` @@ -1640,7 +1640,7 @@ spec: - thud ``` -For further information, read [`Matrix`](./matrix.md). +For further information, read [`Matrix`](./matrix.md) and see [end-to-end example](/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix.yaml). ### Specifying workspaces diff --git a/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix.yaml b/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix.yaml new file mode 100644 index 00000000000..5f94162cd64 --- /dev/null +++ b/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix.yaml @@ -0,0 +1,42 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: platform-browsers + annotations: + description: | + A task that does something cool with platforms and browsers +spec: + params: + - name: platform + - name: browser + steps: + - name: echo + image: alpine + script: | + echo "$(params.platform) and $(params.browser)" +--- +# run platform-browsers task with: +# platforms: linux, mac, windows +# browsers: chrome, safari, firefox +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + generateName: matrixed-pr- +spec: + serviceAccountName: 'default' + pipelineSpec: + tasks: + - name: platforms-and-browsers + matrix: + - name: platform + value: + - linux + - mac + - windows + - name: browser + value: + - chrome + - safari + - firefox + taskRef: + name: platform-browsers diff --git a/pkg/apis/config/testdata/config-defaults-empty.yaml b/pkg/apis/config/testdata/config-defaults-empty.yaml index 661246818e6..53d8f729dbc 100644 --- a/pkg/apis/config/testdata/config-defaults-empty.yaml +++ b/pkg/apis/config/testdata/config-defaults-empty.yaml @@ -37,3 +37,7 @@ data: # default-timeout-minutes contains the default number of # minutes to use for TaskRun, if none is specified. default-timeout-minutes: "60" # 60 minutes + + # default-max-matrix-combinations-count contains the default maximum number + # of combinations from a Matrix, if none is specified. + default-max-matrix-combinations-count: "256" diff --git a/pkg/apis/pipeline/v1beta1/pipeline_types.go b/pkg/apis/pipeline/v1beta1/pipeline_types.go index f36ae980d85..caa2f6a0523 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_types.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_types.go @@ -310,7 +310,7 @@ func (pt *PipelineTask) validateMatrix(ctx context.Context) (errs *apis.FieldErr } func (pt *PipelineTask) validateMatrixCombinationsCount(ctx context.Context) (errs *apis.FieldError) { - matrixCombinationsCount := pt.getMatrixCombinationsCount() + matrixCombinationsCount := pt.GetMatrixCombinationsCount() maxMatrixCombinationsCount := config.FromContextOrDefaults(ctx).Defaults.DefaultMaxMatrixCombinationsCount if matrixCombinationsCount > maxMatrixCombinationsCount { errs = errs.Also(apis.ErrOutOfBoundsValue(matrixCombinationsCount, 0, maxMatrixCombinationsCount, "matrix")) @@ -318,7 +318,8 @@ func (pt *PipelineTask) validateMatrixCombinationsCount(ctx context.Context) (er return errs } -func (pt *PipelineTask) getMatrixCombinationsCount() int { +// GetMatrixCombinationsCount returns the count of combinations of Parameters generated from the Matrix in PipelineTask. +func (pt *PipelineTask) GetMatrixCombinationsCount() int { if len(pt.Matrix) == 0 { return 0 } diff --git a/pkg/apis/pipeline/v1beta1/pipeline_types_test.go b/pkg/apis/pipeline/v1beta1/pipeline_types_test.go index 9db6ded6de6..948ae9dd23a 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_types_test.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_types_test.go @@ -791,7 +791,7 @@ func TestPipelineTask_validateMatrix(t *testing.T) { } } -func TestPipelineTask_getMatrixCombinationsCount(t *testing.T) { +func TestPipelineTask_GetMatrixCombinationsCount(t *testing.T) { tests := []struct { name string pt *PipelineTask @@ -860,8 +860,8 @@ func TestPipelineTask_getMatrixCombinationsCount(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if d := cmp.Diff(tt.matrixCombinationsCount, tt.pt.getMatrixCombinationsCount()); d != "" { - t.Errorf("PipelineTask.getMatrixCombinationsCount() errors diff %s", diff.PrintWantGot(d)) + if d := cmp.Diff(tt.matrixCombinationsCount, tt.pt.GetMatrixCombinationsCount()); d != "" { + t.Errorf("PipelineTask.GetMatrixCombinationsCount() errors diff %s", diff.PrintWantGot(d)) } }) } diff --git a/pkg/reconciler/pipelinerun/pipelinerun.go b/pkg/reconciler/pipelinerun/pipelinerun.go index 4c7cafd8ac7..55524b74500 100644 --- a/pkg/reconciler/pipelinerun/pipelinerun.go +++ b/pkg/reconciler/pipelinerun/pipelinerun.go @@ -23,6 +23,7 @@ import ( "fmt" "path/filepath" "reflect" + "strconv" "strings" "time" @@ -38,6 +39,7 @@ import ( listersv1alpha1 "github.com/tektoncd/pipeline/pkg/client/listers/pipeline/v1alpha1" listers "github.com/tektoncd/pipeline/pkg/client/listers/pipeline/v1beta1" resourcelisters "github.com/tektoncd/pipeline/pkg/client/resource/listers/resource/v1alpha1" + "github.com/tektoncd/pipeline/pkg/matrix" "github.com/tektoncd/pipeline/pkg/pipelinerunmetrics" tknreconciler "github.com/tektoncd/pipeline/pkg/reconciler" "github.com/tektoncd/pipeline/pkg/reconciler/events" @@ -688,7 +690,8 @@ func (c *Reconciler) runNextSchedulableTask(ctx context.Context, pr *v1beta1.Pip if rprt == nil || rprt.Skip(pipelineRunFacts).IsSkipped || rprt.IsFinallySkipped(pipelineRunFacts).IsSkipped { continue } - if rprt.IsCustomTask() { + switch { + case rprt.IsCustomTask(): if rprt.IsFinalTask(pipelineRunFacts) { rprt.Run, err = c.createRun(ctx, rprt, pr, getFinallyTaskRunTimeout) } else { @@ -698,16 +701,27 @@ func (c *Reconciler) runNextSchedulableTask(ctx context.Context, pr *v1beta1.Pip recorder.Eventf(pr, corev1.EventTypeWarning, "RunCreationFailed", "Failed to create Run %q: %v", rprt.RunName, err) return fmt.Errorf("error creating Run called %s for PipelineTask %s from PipelineRun %s: %w", rprt.RunName, rprt.PipelineTask.Name, pr.Name, err) } - } else { + case rprt.IsMatrixed(): + if rprt.IsFinalTask(pipelineRunFacts) { + rprt.TaskRuns, err = c.createTaskRuns(ctx, rprt, pr, as.StorageBasePath(pr), getFinallyTaskRunTimeout) + } else { + rprt.TaskRuns, err = c.createTaskRuns(ctx, rprt, pr, as.StorageBasePath(pr), getTaskRunTimeout) + } + if err != nil { + recorder.Eventf(pr, corev1.EventTypeWarning, "TaskRunsCreationFailed", "Failed to create TaskRuns %q: %v", rprt.TaskRunNames, err) + return fmt.Errorf("error creating TaskRuns called %s for PipelineTask %s from PipelineRun %s: %w", rprt.TaskRunNames, rprt.PipelineTask.Name, pr.Name, err) + } + default: if rprt.IsFinalTask(pipelineRunFacts) { - rprt.TaskRun, err = c.createTaskRun(ctx, rprt, pr, as.StorageBasePath(pr), getFinallyTaskRunTimeout) + rprt.TaskRun, err = c.createTaskRun(ctx, rprt.TaskRunName, nil, rprt, pr, as.StorageBasePath(pr), getFinallyTaskRunTimeout) } else { - rprt.TaskRun, err = c.createTaskRun(ctx, rprt, pr, as.StorageBasePath(pr), getTaskRunTimeout) + rprt.TaskRun, err = c.createTaskRun(ctx, rprt.TaskRunName, nil, rprt, pr, as.StorageBasePath(pr), getTaskRunTimeout) } if err != nil { recorder.Eventf(pr, corev1.EventTypeWarning, "TaskRunCreationFailed", "Failed to create TaskRun %q: %v", rprt.TaskRunName, err) return fmt.Errorf("error creating TaskRun called %s for PipelineTask %s from PipelineRun %s: %w", rprt.TaskRunName, rprt.PipelineTask.Name, pr.Name, err) } + } } return nil @@ -750,10 +764,24 @@ func (c *Reconciler) updateRunsStatusDirectly(pr *v1beta1.PipelineRun) error { type getTimeoutFunc func(ctx context.Context, pr *v1beta1.PipelineRun, rprt *resources.ResolvedPipelineRunTask, c clock.PassiveClock) *metav1.Duration -func (c *Reconciler) createTaskRun(ctx context.Context, rprt *resources.ResolvedPipelineRunTask, pr *v1beta1.PipelineRun, storageBasePath string, getTimeoutFunc getTimeoutFunc) (*v1beta1.TaskRun, error) { +func (c *Reconciler) createTaskRuns(ctx context.Context, rprt *resources.ResolvedPipelineRunTask, pr *v1beta1.PipelineRun, storageBasePath string, getTimeoutFunc getTimeoutFunc) ([]*v1beta1.TaskRun, error) { + var taskRuns []*v1beta1.TaskRun + matrixCombinations := matrix.FanOut(rprt.PipelineTask.Matrix).ToMap() + for i, taskRunName := range rprt.TaskRunNames { + params := matrixCombinations[strconv.Itoa(i)] + taskRun, err := c.createTaskRun(ctx, taskRunName, params, rprt, pr, storageBasePath, getTimeoutFunc) + if err != nil { + return nil, err + } + taskRuns = append(taskRuns, taskRun) + } + return taskRuns, nil +} + +func (c *Reconciler) createTaskRun(ctx context.Context, taskRunName string, params []v1beta1.Param, rprt *resources.ResolvedPipelineRunTask, pr *v1beta1.PipelineRun, storageBasePath string, getTimeoutFunc getTimeoutFunc) (*v1beta1.TaskRun, error) { logger := logging.FromContext(ctx) - tr, _ := c.taskRunLister.TaskRuns(pr.Namespace).Get(rprt.TaskRunName) + tr, _ := c.taskRunLister.TaskRuns(pr.Namespace).Get(taskRunName) if tr != nil { // Don't modify the lister cache's copy. tr = tr.DeepCopy() @@ -767,16 +795,19 @@ func (c *Reconciler) createTaskRun(ctx context.Context, rprt *resources.Resolved rprt.PipelineTask = resources.ApplyPipelineTaskContexts(rprt.PipelineTask) taskRunSpec := pr.GetTaskRunSpec(rprt.PipelineTask.Name) + if len(params) == 0 { + params = rprt.PipelineTask.Params + } tr = &v1beta1.TaskRun{ ObjectMeta: metav1.ObjectMeta{ - Name: rprt.TaskRunName, + Name: taskRunName, Namespace: pr.Namespace, OwnerReferences: []metav1.OwnerReference{*kmeta.NewControllerRef(pr)}, Labels: combineTaskRunAndTaskSpecLabels(pr, rprt.PipelineTask), Annotations: combineTaskRunAndTaskSpecAnnotations(pr, rprt.PipelineTask), }, Spec: v1beta1.TaskRunSpec{ - Params: rprt.PipelineTask.Params, + Params: params, ServiceAccountName: taskRunSpec.TaskServiceAccountName, Timeout: getTimeoutFunc(ctx, pr, rprt, c.Clock), PodTemplate: taskRunSpec.TaskPodTemplate, @@ -803,7 +834,7 @@ func (c *Reconciler) createTaskRun(ctx context.Context, rprt *resources.Resolved } resources.WrapSteps(&tr.Spec, rprt.PipelineTask, rprt.ResolvedTaskResources.Inputs, rprt.ResolvedTaskResources.Outputs, storageBasePath) - logger.Infof("Creating a new TaskRun object %s for pipeline task %s", rprt.TaskRunName, rprt.PipelineTask.Name) + logger.Infof("Creating a new TaskRun object %s for pipeline task %s", taskRunName, rprt.PipelineTask.Name) return c.PipelineClientSet.TektonV1beta1().TaskRuns(pr.Namespace).Create(ctx, tr, metav1.CreateOptions{}) } diff --git a/pkg/reconciler/pipelinerun/pipelinerun_test.go b/pkg/reconciler/pipelinerun/pipelinerun_test.go index 313b9a7bd8f..437e5ecf5fd 100644 --- a/pkg/reconciler/pipelinerun/pipelinerun_test.go +++ b/pkg/reconciler/pipelinerun/pipelinerun_test.go @@ -24,6 +24,7 @@ import ( "fmt" "net/http/httptest" "net/url" + "strconv" "testing" "time" @@ -103,10 +104,11 @@ var ( ) const ( - apiFieldsFeatureFlag = "enable-api-fields" - customTasksFeatureFlag = "enable-custom-tasks" - ociBundlesFeatureFlag = "enable-tekton-oci-bundles" - embeddedStatusFeatureFlag = "embedded-status" + apiFieldsFeatureFlag = "enable-api-fields" + customTasksFeatureFlag = "enable-custom-tasks" + ociBundlesFeatureFlag = "enable-tekton-oci-bundles" + embeddedStatusFeatureFlag = "embedded-status" + maxMatrixCombinationsCountFlag = "default-max-matrix-combinations-count" ) type PipelineRunTest struct { @@ -1429,6 +1431,13 @@ func newFeatureFlagsConfigMap() *corev1.ConfigMap { } } +func newDefaultsConfigMap() *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: config.GetDefaultsConfigName(), Namespace: system.Namespace()}, + Data: make(map[string]string), + } +} + func withEnabledAlphaAPIFields(cm *corev1.ConfigMap) *corev1.ConfigMap { newCM := cm.DeepCopy() newCM.Data[apiFieldsFeatureFlag] = config.AlphaAPIFields @@ -1453,6 +1462,12 @@ func withEmbeddedStatus(cm *corev1.ConfigMap, flagVal string) *corev1.ConfigMap return newCM } +func withMaxMatrixCombinationsCount(cm *corev1.ConfigMap, count int) *corev1.ConfigMap { + newCM := cm.DeepCopy() + newCM.Data[maxMatrixCombinationsCountFlag] = strconv.Itoa(count) + return newCM +} + func TestReconcileOnCancelledPipelineRun(t *testing.T) { testCases := []struct { name string @@ -7433,3 +7448,304 @@ spec: t.Errorf("expected to see propagated metadata by the precedence from PipelineTaskRunSpec in TaskRun %v created. Diff %s", expectedTaskRun, diff.PrintWantGot(d)) } } + +func TestReconciler_PipelineTaskMatrix(t *testing.T) { + names.TestingSeed() + + task := parse.MustParseTask(t, ` +metadata: + name: mytask + namespace: foo +spec: + params: + - name: platform + - name: browser + steps: + - name: echo + image: alpine + script: | + echo "$(params.platform) and $(params.browser)" +`) + + expectedTaskRuns := []*v1beta1.TaskRun{ + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-platforms-and-browsers-0", "foo", + "pr", "p", "platforms-and-browsers", false), + ` +spec: + params: + - name: platform + value: linux + - name: browser + value: chrome + resources: {} + serviceAccountName: test-sa + taskRef: + name: mytask + timeout: 1h0m0s +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-platforms-and-browsers-1", "foo", + "pr", "p", "platforms-and-browsers", false), + ` +spec: + params: + - name: platform + value: mac + - name: browser + value: chrome + resources: {} + serviceAccountName: test-sa + taskRef: + name: mytask + timeout: 1h0m0s +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-platforms-and-browsers-2", "foo", + "pr", "p", "platforms-and-browsers", false), + ` +spec: + params: + - name: platform + value: windows + - name: browser + value: chrome + resources: {} + serviceAccountName: test-sa + taskRef: + name: mytask + timeout: 1h0m0s +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-platforms-and-browsers-3", "foo", + "pr", "p", "platforms-and-browsers", false), + ` +spec: + params: + - name: platform + value: linux + - name: browser + value: safari + resources: {} + serviceAccountName: test-sa + taskRef: + name: mytask + timeout: 1h0m0s +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-platforms-and-browsers-4", "foo", + "pr", "p", "platforms-and-browsers", false), + ` +spec: + params: + - name: platform + value: mac + - name: browser + value: safari + resources: {} + serviceAccountName: test-sa + taskRef: + name: mytask + timeout: 1h0m0s +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-platforms-and-browsers-5", "foo", + "pr", "p", "platforms-and-browsers", false), + ` +spec: + params: + - name: platform + value: windows + - name: browser + value: safari + resources: {} + serviceAccountName: test-sa + taskRef: + name: mytask + timeout: 1h0m0s +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-platforms-and-browsers-6", "foo", + "pr", "p", "platforms-and-browsers", false), + ` +spec: + params: + - name: platform + value: linux + - name: browser + value: firefox + resources: {} + serviceAccountName: test-sa + taskRef: + name: mytask + timeout: 1h0m0s +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-platforms-and-browsers-7", "foo", + "pr", "p", "platforms-and-browsers", false), + ` +spec: + params: + - name: platform + value: mac + - name: browser + value: firefox + resources: {} + serviceAccountName: test-sa + taskRef: + name: mytask + timeout: 1h0m0s +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-platforms-and-browsers-8", "foo", + "pr", "p", "platforms-and-browsers", false), + ` +spec: + params: + - name: platform + value: windows + - name: browser + value: firefox + resources: {} + serviceAccountName: test-sa + taskRef: + name: mytask + timeout: 1h0m0s +`), + } + + cms := []*corev1.ConfigMap{withEnabledAlphaAPIFields(newFeatureFlagsConfigMap())} + cms = append(cms, withMaxMatrixCombinationsCount(newDefaultsConfigMap(), 10)) + + tests := []struct { + name string + memberOf string + p *v1beta1.Pipeline + tr *v1beta1.TaskRun + }{{ + name: "p-dag", + memberOf: "tasks", + p: parse.MustParsePipeline(t, fmt.Sprintf(` +metadata: + name: %s + namespace: foo +spec: + tasks: + - name: platforms-and-browsers + taskRef: + name: mytask + matrix: + - name: platform + value: + - linux + - mac + - windows + - name: browser + value: + - chrome + - safari + - firefox +`, "p-dag")), + }, { + name: "p-finally", + memberOf: "finally", + p: parse.MustParsePipeline(t, fmt.Sprintf(` +metadata: + name: %s + namespace: foo +spec: + tasks: + - name: unmatrixed-pt + params: + - name: platform + value: linux + - name: browser + value: chrome + taskRef: + name: mytask + finally: + - name: platforms-and-browsers + taskRef: + name: mytask + matrix: + - name: platform + value: + - linux + - mac + - windows + - name: browser + value: + - chrome + - safari + - firefox +`, "p-finally")), + tr: mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-unmatrixed-pt", "foo", + "pr", "p-finally", "unmatrixed-pt", false), + ` +spec: + params: + - name: platform + value: linux + - name: browser + value: chrome + resources: {} + serviceAccountName: test-sa + taskRef: + name: mytask + timeout: 1h0m0s +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing +`), + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pr := parse.MustParsePipelineRun(t, fmt.Sprintf(` +metadata: + name: pr + namespace: foo +spec: + serviceAccountName: test-sa + pipelineRef: + name: %s +`, tt.name)) + d := test.Data{ + PipelineRuns: []*v1beta1.PipelineRun{pr}, + Pipelines: []*v1beta1.Pipeline{tt.p}, + Tasks: []*v1beta1.Task{task}, + ConfigMaps: cms, + } + if tt.tr != nil { + d.TaskRuns = []*v1beta1.TaskRun{tt.tr} + } + prt := newPipelineRunTest(d, t) + defer prt.Cancel() + + _, clients := prt.reconcileRun("foo", "pr", []string{}, false) + taskRuns, err := clients.Pipeline.TektonV1beta1().TaskRuns("foo").List(prt.TestAssets.Ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("tekton.dev/pipelineRun=pr,tekton.dev/pipeline=%s,tekton.dev/pipelineTask=platforms-and-browsers", tt.name), + Limit: 1, + }) + if err != nil { + t.Fatalf("Failure to list TaskRun's %s", err) + } + + if len(taskRuns.Items) != 9 { + t.Fatalf("Expected 9 TaskRuns got %d", len(taskRuns.Items)) + } + + for i := range taskRuns.Items { + expectedTaskRun := expectedTaskRuns[i] + expectedTaskRun.Labels["tekton.dev/pipeline"] = tt.name + expectedTaskRun.Labels["tekton.dev/memberOf"] = tt.memberOf + if d := cmp.Diff(expectedTaskRun, &taskRuns.Items[i], ignoreResourceVersion, ignoreTypeMeta); d != "" { + t.Errorf("expected to see TaskRun %v created. Diff %s", expectedTaskRuns[i].Name, diff.PrintWantGot(d)) + } + } + }) + } +} diff --git a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go index 3455741117f..17c87ad0dff 100644 --- a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go +++ b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go @@ -56,12 +56,12 @@ func (e *TaskNotFoundError) Error() string { return fmt.Sprintf("Couldn't retrieve Task %q: %s", e.Name, e.Msg) } -// ResolvedPipelineRunTask contains a Task and its associated TaskRun, if it -// exists. TaskRun can be nil to represent there being no TaskRun. +// ResolvedPipelineRunTask contains a PipelineTask and its associated TaskRun(s) or Runs, if they exist. type ResolvedPipelineRunTask struct { - TaskRunName string - TaskRun *v1beta1.TaskRun - TaskRuns []*v1beta1.TaskRun + TaskRunName string + TaskRun *v1beta1.TaskRun + TaskRunNames []string + TaskRuns []*v1beta1.TaskRun // If the PipelineTask is a Custom Task, RunName and Run will be set. CustomTask bool RunName string @@ -523,14 +523,22 @@ func ResolvePipelineRunTask( PipelineTask: &pipelineTask, } rprt.CustomTask = isCustomTask(ctx, rprt) - if rprt.IsCustomTask() { + switch { + case rprt.IsCustomTask(): rprt.RunName = getRunName(pipelineRun.Status.Runs, pipelineRun.Status.ChildReferences, pipelineTask.Name, pipelineRun.Name) run, err := getRun(rprt.RunName) if err != nil && !kerrors.IsNotFound(err) { return nil, fmt.Errorf("error retrieving Run %s: %w", rprt.RunName, err) } rprt.Run = run - } else { + case rprt.IsMatrixed(): + rprt.TaskRunNames = GetNamesOfTaskRuns(pipelineRun.Status.TaskRuns, pipelineRun.Status.ChildReferences, pipelineTask.Name, pipelineRun.Name, pipelineTask.GetMatrixCombinationsCount()) + for _, taskRunName := range rprt.TaskRunNames { + if err := rprt.resolvePipelineRunTaskWithTaskRun(ctx, taskRunName, getTask, getTaskRun, pipelineTask, providedResources); err != nil { + return nil, err + } + } + default: rprt.TaskRunName = GetTaskRunName(pipelineRun.Status.TaskRuns, pipelineRun.Status.ChildReferences, pipelineTask.Name, pipelineRun.Name) if err := rprt.resolvePipelineRunTaskWithTaskRun(ctx, rprt.TaskRunName, getTask, getTaskRun, pipelineTask, providedResources); err != nil { return nil, err @@ -554,10 +562,14 @@ func (t *ResolvedPipelineRunTask) resolvePipelineRunTaskWithTaskRun( } } if taskRun != nil { - t.TaskRun = taskRun + if t.IsMatrixed() { + t.TaskRuns = append(t.TaskRuns, taskRun) + } else { + t.TaskRun = taskRun + } } - if err := t.resolveTaskResources(ctx, getTask, pipelineTask, providedResources, t.TaskRun); err != nil { + if err := t.resolveTaskResources(ctx, getTask, pipelineTask, providedResources, taskRun); err != nil { return err } diff --git a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go index 99638717e1e..53cb8268081 100644 --- a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go +++ b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go @@ -1682,10 +1682,20 @@ func TestResolvePipelineRun_PipelineTaskHasNoResources(t *testing.T) { } func TestResolvePipelineRun_TaskDoesntExist(t *testing.T) { - pt := v1beta1.PipelineTask{ + pts := []v1beta1.PipelineTask{{ Name: "mytask1", TaskRef: &v1beta1.TaskRef{Name: "task"}, - } + }, { + Name: "mytask2", + TaskRef: &v1beta1.TaskRef{Name: "task"}, + Matrix: []v1beta1.Param{{ + Name: "foo", + Value: *v1beta1.NewArrayOrString("f", "o", "o"), + }, { + Name: "bar", + Value: *v1beta1.NewArrayOrString("b", "a", "r"), + }}, + }} providedResources := map[string]*resourcev1alpha1.PipelineResource{} // Return an error when the Task is retrieved, as if it didn't exist @@ -1700,14 +1710,16 @@ func TestResolvePipelineRun_TaskDoesntExist(t *testing.T) { Name: "pipelinerun", }, } - _, err := ResolvePipelineRunTask(context.Background(), pr, getTask, getTaskRun, nopGetRun, pt, providedResources) - switch err := err.(type) { - case nil: - t.Fatalf("Expected error getting non-existent Tasks for Pipeline %s but got none", p.Name) - case *TaskNotFoundError: - // expected error - default: - t.Fatalf("Expected specific error type returned by func for non-existent Task for Pipeline %s but got %s", p.Name, err) + for _, pt := range pts { + _, err := ResolvePipelineRunTask(context.Background(), pr, getTask, getTaskRun, nopGetRun, pt, providedResources) + switch err := err.(type) { + case nil: + t.Fatalf("Expected error getting non-existent Tasks for Pipeline %s but got none", p.Name) + case *TaskNotFoundError: + // expected error + default: + t.Fatalf("Expected specific error type returned by func for non-existent Task for Pipeline %s but got %s", p.Name, err) + } } } @@ -3001,6 +3013,111 @@ func TestIsMatrixed(t *testing.T) { } } +func TestResolvePipelineRunTask_WithMatrix(t *testing.T) { + pipelineRunName := "pipelinerun" + pipelineTaskName := "pipelinetask" + + pr := v1beta1.PipelineRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: pipelineRunName, + }, + } + + var taskRuns []*v1beta1.TaskRun + var taskRunsNames []string + taskRunsMap := map[string]*v1beta1.TaskRun{} + for i := 0; i < 9; i++ { + trName := fmt.Sprintf("%s-%s-%d", pipelineRunName, pipelineTaskName, i) + tr := &v1beta1.TaskRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: trName, + }, + } + taskRuns = append(taskRuns, tr) + taskRunsNames = append(taskRunsNames, trName) + taskRunsMap[trName] = tr + } + + pts := []v1beta1.PipelineTask{{ + Name: "pipelinetask", + TaskRef: &v1beta1.TaskRef{ + Name: "my-task", + }, + Matrix: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeArray, ArrayVal: []string{"linux", "mac", "windows"}}, + }}, + }, { + Name: "pipelinetask", + TaskRef: &v1beta1.TaskRef{ + Name: "my-task", + }, + Matrix: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeArray, ArrayVal: []string{"linux", "mac", "windows"}}, + }, { + Name: "browsers", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeArray, ArrayVal: []string{"chrome", "safari", "firefox"}}, + }}, + }} + + rtr := &resources.ResolvedTaskResources{ + TaskName: "task", + TaskSpec: &v1beta1.TaskSpec{Steps: []v1beta1.Step{{ + Name: "step1", + }}}, + Inputs: map[string]*v1alpha1.PipelineResource{}, + Outputs: map[string]*v1alpha1.PipelineResource{}, + } + + getTask := func(ctx context.Context, name string) (v1beta1.TaskObject, error) { return task, nil } + getTaskRun := func(name string) (*v1beta1.TaskRun, error) { return taskRunsMap[name], nil } + getRun := func(name string) (*v1alpha1.Run, error) { return &runs[0], nil } + + for _, tc := range []struct { + name string + pt v1beta1.PipelineTask + want *ResolvedPipelineRunTask + }{{ + name: "task with matrix - single parameter", + pt: pts[0], + want: &ResolvedPipelineRunTask{ + TaskRunNames: taskRunsNames[:3], + TaskRuns: taskRuns[:3], + PipelineTask: &pts[0], + ResolvedTaskResources: rtr, + }, + }, { + name: "task with matrix - multiple parameters", + pt: pts[1], + want: &ResolvedPipelineRunTask{ + TaskRunNames: taskRunsNames, + TaskRuns: taskRuns, + PipelineTask: &pts[1], + ResolvedTaskResources: rtr, + }, + }} { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + cfg := config.NewStore(logtesting.TestLogger(t)) + cfg.OnConfigChanged(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: config.GetFeatureFlagsConfigName()}, + Data: map[string]string{ + "enable-api-fields": "alpha", + }, + }) + ctx = cfg.ToContext(ctx) + rprt, err := ResolvePipelineRunTask(ctx, pr, getTask, getTaskRun, getRun, tc.pt, nil) + if err != nil { + t.Fatalf("Did not expect error when resolving PipelineRun: %v", err) + } + if d := cmp.Diff(tc.want, rprt); d != "" { + t.Errorf("Did not get expected ResolvePipelineRunTask with Matrix: %s", diff.PrintWantGot(d)) + } + }) + } +} + func TestIsSuccessful(t *testing.T) { for _, tc := range []struct { name string diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index b26240e7f62..dff26fa1fff 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -22,6 +22,7 @@ source $(git rev-parse --show-toplevel)/test/e2e-common.sh # Setting defaults PIPELINE_FEATURE_GATE=${PIPELINE_FEATURE_GATE:-stable} +MATRIX_MAX_CONCURRENCY=${MATRIX_MAX_CONCURRENCY:-"256"} SKIP_INITIALIZE=${SKIP_INITIALIZE:="false"} RUN_YAML_TESTS=${RUN_YAML_TESTS:="true"} failed=0 @@ -50,6 +51,14 @@ function set_feature_gate() { kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" } +function set_matrix_max_concurrency() { + local count="$1" + printf "Setting config defaults for matrix max concurrency to %s\n", ${count} + jsonpatch=$(printf "{\"data\": {\"default-max-matrix-combinations-count\": \"%s\"}}" $1) + echo "config-defaults ConfigMap patch: ${jsonpatch}" + kubectl patch configmap config-defaults -n tekton-pipelines -p "$jsonpatch" +} + function run_e2e() { # Run the integration tests header "Running Go e2e tests" @@ -64,6 +73,7 @@ function run_e2e() { } set_feature_gate "$PIPELINE_FEATURE_GATE" +set_matrix_max_concurrency "$MATRIX_MAX_CONCURRENCY" run_e2e (( failed )) && fail_test