From 8a1d2efdc18ef60b816af6dbdb4443debf4e62ef Mon Sep 17 00:00:00 2001 From: JeromeJu Date: Fri, 7 Jul 2023 18:33:46 +0000 Subject: [PATCH] Add upgrade test against previous server version to prevent regressions This commit adds an upgrade test cases to test against previous server version by adding a light-weight test checking the existing functionalities of simple resources so as to prevent regressions introduced in the current release. It creates a pipelineRun with a basic TaskRun and a basic Pipeline with a Simple PipelineTask and examines the runs created are successful. This helps prevent the regression where the client has introduced regressions towards the last release i.e. [Regression: PipelineTaskRunSpec.Metadata incompatible with older webhooks](https://github.com/tektoncd/pipeline/issues/4913) --- test/e2e-tests-upgrade.sh | 10 +- test/upgrade_test.go | 213 +++++++++++++++++++++++--------------- 2 files changed, 136 insertions(+), 87 deletions(-) diff --git a/test/e2e-tests-upgrade.sh b/test/e2e-tests-upgrade.sh index 981fcd9610f..8fa259c169d 100755 --- a/test/e2e-tests-upgrade.sh +++ b/test/e2e-tests-upgrade.sh @@ -18,8 +18,8 @@ # and deploy Tekton Pipelines to it for running upgrading tests. There are # two scenarios we need to cover in this script: -# Scenario 1: install the previous release, test on previous server version -# to prevent regression, and then upgrade to the current release, and +# Scenario 1: install the previous release, test compatibility with +# the current clients, and then upgrade to the current release, and # validate whether the Tekton pipeline works. # Scenario 2: install the previous release, create the pipelines and tasks, upgrade @@ -50,8 +50,10 @@ header "Install the previous release of Tekton pipeline $PREVIOUS_PIPELINE_VERSI install_pipeline_crd_version $PREVIOUS_PIPELINE_VERSION failed=0 -# The upgrade test for the old server version to prevent regression on existing changes. -go_test_e2e -timeout=20m ./test -run ^TestUpgradeAgainstPreviousVersion || failed=1 +# Run upgrade tests for the old server version to prevent regressions where a new client +# is incompatible with an old server (e.g. https://github.com/tektoncd/pipeline/issues/4913) +go_test_e2e -timeout=20m ./test -run ^TestSimpleTaskRun || failed=1 +go_test_e2e -timeout=20m ./test -run ^TestSimplePipelineRun || failed=1 # Upgrade to the current release. header "Upgrade to the current release of Tekton pipeline" diff --git a/test/upgrade_test.go b/test/upgrade_test.go index a7711070fda..eee39994f82 100644 --- a/test/upgrade_test.go +++ b/test/upgrade_test.go @@ -36,27 +36,12 @@ import ( ) var ( - filterOwnerReference = cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID", "Controller", "BlockOwnerDeletion") - filterV1SidecarState = cmpopts.IgnoreFields(v1.SidecarState{}, "ImageID") - filterVolumeMountsName = cmpopts.IgnoreFields(corev1.VolumeMount{}, "Name") - filterTaskRunStatusProvenance = cmpopts.IgnoreFields(v1.TaskRunStatusFields{}, "Provenance") - filterPipelineRunStatusProvenance = cmpopts.IgnoreFields(v1.PipelineRunStatusFields{}, "Provenance") -) - -// TestUpgradeAgainstPreviousVersion creates a pipelineRun with a basic -// TaskRun and a basic Pipeline and examines the events and runs created -// are successful. -func TestUpgradeAgainstPreviousVersion(t *testing.T) { - t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - c, namespace := setup(ctx, t) - - knativetest.CleanupOnInterrupt(func() { tearDown(context.Background(), t, c, namespace) }, t.Logf) - defer tearDown(context.Background(), t, c, namespace) + filterV1SidecarState = cmpopts.IgnoreFields(v1.SidecarState{}, "ImageID") + filterVolumeMountsName = cmpopts.IgnoreFields(corev1.VolumeMount{}, "Name") + filterTaskRunStatusFields = cmpopts.IgnoreFields(v1.TaskRunStatusFields{}, "StartTime", "CompletionTime", "Provenance", "PodName") + filterPipelineRunStatusFields = cmpopts.IgnoreFields(v1.PipelineRunStatusFields{}, "StartTime", "CompletionTime", "Provenance") - t.Logf("Creating Task in namespace %s", namespace) - task := parse.MustParseV1Task(t, fmt.Sprintf(` + simpleTaskYaml = ` metadata: name: %s namespace: %s @@ -78,12 +63,9 @@ spec: fi workspaces: - name: taskWorkspace -`, task1Name, namespace)) - if _, err := c.V1TaskClient.Create(ctx, task, metav1.CreateOptions{}); err != nil { - t.Fatalf("Failed to create Task `%s`: %s", task.Name, err) - } +` - pipeline := parse.MustParseV1Pipeline(t, fmt.Sprintf(` + simplePipelineYaml = ` metadata: name: %s namespace: %s @@ -97,9 +79,23 @@ spec: workspace: workspace workspaces: - name: workspace -`, helpers.ObjectNameForTest(t), namespace, task1Name, task.Name)) - pipelineName := pipeline.Name - pipelineRun := parse.MustParseV1PipelineRun(t, fmt.Sprintf(` +` + + simpleTaskRunYaml = ` +metadata: + name: %s + namespace: %s +spec: + taskRef: + name: %s + kind: Task + timeout: 30m + workspaces: + - name: taskWorkspace + emptyDir: {} +` + + simplePipelineRunYaml = ` metadata: name: %s namespace: %s @@ -111,58 +107,17 @@ spec: emptyDir: {} timeouts: pipeline: 180s -`, helpers.ObjectNameForTest(t), namespace, pipeline.Name)) - pipelineRunName := pipelineRun.Name - - if _, err := c.V1PipelineClient.Create(ctx, pipeline, metav1.CreateOptions{}); err != nil { - t.Fatalf("Failed to create Pipeline `%s`: %s", pipeline.Name, err) - } - if _, err := c.V1PipelineRunClient.Create(ctx, pipelineRun, metav1.CreateOptions{}); err != nil { - t.Fatalf("Failed to create PipelineRun `%s`: %s", pipelineRunName, err) - } +` - t.Logf("Waiting for PipelineRun %s in namespace %s to complete", pipelineRunName, namespace) - if err := WaitForPipelineRunState(ctx, c, pipelineRunName, timeout, PipelineRunSucceed(pipelineRunName), "PipelineRunSuccess", v1Version); err != nil { - t.Fatalf("Error waiting for PipelineRun %s to finish: %s", pipelineRunName, err) - } - - taskRunName := strings.Join([]string{pipelineRunName, task1Name}, "-") - r, err := c.V1TaskRunClient.Get(ctx, taskRunName, metav1.GetOptions{}) - if err != nil { - t.Fatalf("Couldn't get expected TaskRun %s: %s", taskRunName, err) - } - - expectedTaskRun := getExpectedTaskRunUpgrade(t, taskRunName, namespace, pipelineRunName, task1Name) - if d := cmp.Diff(expectedTaskRun, r, append([]cmp.Option{filterV1TaskRunSA, filterV1StepState, filterOwnerReference, filterV1TaskRunStatus, filterV1SidecarState, filterVolumeMountsName, filterTaskRunStatusProvenance}, filterV1TaskRunFields...)...); d != "" { - t.Errorf("Cannot get expected TaskRun, -want, +got: %v", d) - } - - pr, err := c.V1PipelineRunClient.Get(ctx, pipelineRunName, metav1.GetOptions{}) - if err != nil { - t.Fatalf("Couldn't get expected PipelineRun %s: %s", pipelineRunName, err) - } - - expectedPipelineRun := getExpectedPipelineRunUpgrade(t, pipelineRunName, namespace, pipelineName, taskRunName) - if d := cmp.Diff(expectedPipelineRun, pr, append([]cmp.Option{filterV1PipelineRunStatus, filterV1PipelineRunSA, filterPipelineRunStatusProvenance}, filterV1PipelineRunFields...)...); d != "" { - t.Errorf("Cannot get expected PipelineRun, -want, +got: %v", d) - } -} - -func getExpectedTaskRunUpgrade(t *testing.T, taskRunName, namespace, pipelineRunName, task1Name string) *v1.TaskRun { - t.Helper() - expectedTaskRun := parse.MustParseV1TaskRun(t, fmt.Sprintf(` + expectedSimpleTaskRunYaml = ` metadata: name: %s namespace: %s - ownerReferences: - - apiVersion: "tekton.dev/v1" - kind: "PipelineRun" - name: %s spec: taskRef: name: %s kind: Task - timeout: 1h + timeout: 30m workspaces: - name: taskWorkspace emptyDir: {} @@ -171,11 +126,9 @@ status: - type: Succeeded reason: Succeeded status: "True" - podName: %s-pod taskSpec: steps: - - computeResources: {} - image: alpine + - image: alpine name: echo-param script: | echo "response" @@ -200,13 +153,9 @@ status: name: check-workspace terminated: reason: Completed -`, taskRunName, namespace, pipelineRunName, task1Name, taskRunName)) - return expectedTaskRun -} +` -func getExpectedPipelineRunUpgrade(t *testing.T, pipelineRunName, namespace, pipelineName, taskRunName string) *v1.PipelineRun { - t.Helper() - expectedPipelineRun := parse.MustParseV1PipelineRun(t, fmt.Sprintf(` + expectedSimplePipelineRunYaml = ` metadata: name: %s namespace: %s @@ -241,6 +190,104 @@ status: pipelineTaskName: task1 apiVersion: "tekton.dev/v1" kind: TaskRun -`, pipelineRunName, namespace, pipelineName, taskRunName)) - return expectedPipelineRun +` +) + +// TestSimpleTaskRun creates a taskRun with a basic task and verifies the +// runs created are successful and as expected. +func TestSimpleTaskRun(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, namespace := setup(ctx, t) + + taskRunName := helpers.ObjectNameForTest(t) + + knativetest.CleanupOnInterrupt(func() { tearDown(context.Background(), t, c, namespace) }, t.Logf) + defer tearDown(context.Background(), t, c, namespace) + + t.Logf("Creating Task in namespace %s", namespace) + task := parse.MustParseV1Task(t, fmt.Sprintf(simpleTaskYaml, task1Name, namespace)) + if _, err := c.V1TaskClient.Create(ctx, task, metav1.CreateOptions{}); err != nil { + t.Fatalf("Failed to create Task `%s`: %s", task.Name, err) + } + + taskRun := parse.MustParseV1TaskRun(t, fmt.Sprintf(simpleTaskRunYaml, taskRunName, namespace, task1Name)) + if _, err := c.V1TaskRunClient.Create(ctx, taskRun, metav1.CreateOptions{}); err != nil { + t.Fatalf("Failed to create TaskRun `%s`: %s", taskRunName, err) + } + + t.Logf("Waiting for TaskRun %s in namespace %s to complete", taskRunName, namespace) + if err := WaitForTaskRunState(ctx, c, taskRunName, TaskRunSucceed(taskRunName), "TaskRunSuccess", v1Version); err != nil { + t.Fatalf("Error waiting for PipelineRun %s to finish: %s", taskRunName, err) + } + + r, err := c.V1TaskRunClient.Get(ctx, taskRunName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Couldn't get expected TaskRun %s: %s", taskRunName, err) + } + + expectedTaskRun := parse.MustParseV1TaskRun(t, fmt.Sprintf(expectedSimpleTaskRunYaml, taskRunName, namespace, task1Name)) + if d := cmp.Diff(expectedTaskRun, r, append([]cmp.Option{filterV1TaskRunSA, filterV1StepState, filterV1TaskRunStatus, filterV1SidecarState, filterVolumeMountsName, filterTaskRunStatusFields, cmpopts.EquateEmpty()}, filterV1TaskRunFields...)...); d != "" { + t.Errorf("Cannot get expected TaskRun, -want, +got: %v", d) + } +} + +// TestSimplePipelineRun creates a pipelineRun with a basic Pipeline +// and verifies the runs created are successful and as expected. +func TestSimplePipelineRun(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, namespace := setup(ctx, t) + + knativetest.CleanupOnInterrupt(func() { tearDown(context.Background(), t, c, namespace) }, t.Logf) + defer tearDown(context.Background(), t, c, namespace) + + t.Logf("Creating Task in namespace %s", namespace) + task := parse.MustParseV1Task(t, fmt.Sprintf(simpleTaskYaml, task1Name, namespace)) + if _, err := c.V1TaskClient.Create(ctx, task, metav1.CreateOptions{}); err != nil { + t.Fatalf("Failed to create Task `%s`: %s", task.Name, err) + } + + pipeline := parse.MustParseV1Pipeline(t, fmt.Sprintf(simplePipelineYaml, helpers.ObjectNameForTest(t), namespace, task1Name, task.Name)) + pipelineName := pipeline.Name + pipelineRun := parse.MustParseV1PipelineRun(t, fmt.Sprintf(` +metadata: + name: %s + namespace: %s +spec: + pipelineRef: + name: %s + workspaces: + - name: workspace + emptyDir: {} + timeouts: + pipeline: 180s +`, helpers.ObjectNameForTest(t), namespace, pipeline.Name)) + pipelineRunName := pipelineRun.Name + + if _, err := c.V1PipelineClient.Create(ctx, pipeline, metav1.CreateOptions{}); err != nil { + t.Fatalf("Failed to create Pipeline `%s`: %s", pipeline.Name, err) + } + if _, err := c.V1PipelineRunClient.Create(ctx, pipelineRun, metav1.CreateOptions{}); err != nil { + t.Fatalf("Failed to create PipelineRun `%s`: %s", pipelineRunName, err) + } + + t.Logf("Waiting for PipelineRun %s in namespace %s to complete", pipelineRunName, namespace) + if err := WaitForPipelineRunState(ctx, c, pipelineRunName, timeout, PipelineRunSucceed(pipelineRunName), "PipelineRunSuccess", v1Version); err != nil { + t.Fatalf("Error waiting for PipelineRun %s to finish: %s", pipelineRunName, err) + } + + taskRunName := strings.Join([]string{pipelineRunName, task1Name}, "-") + + pr, err := c.V1PipelineRunClient.Get(ctx, pipelineRunName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Couldn't get expected PipelineRun %s: %s", pipelineRunName, err) + } + + expectedPipelineRun := parse.MustParseV1PipelineRun(t, fmt.Sprintf(expectedSimplePipelineRunYaml, pipelineRunName, namespace, pipelineName, taskRunName)) + if d := cmp.Diff(expectedPipelineRun, pr, append([]cmp.Option{filterV1PipelineRunStatus, filterV1PipelineRunSA, filterPipelineRunStatusFields}, filterV1PipelineRunFields...)...); d != "" { + t.Errorf("Cannot get expected PipelineRun, -want, +got: %v", d) + } }