Run CI YAML Example tests using go test

- Accurately report specific example failures
- Test failures now use same output format as other tests
- Provides greater control over parallelism using args to `go test`
- Use the same test harness as other integration tests
  - De-dupe harness between bash/go

Future work:

  - allow e2e tests to be run entirely locally by expanding SubEnv
    - (avoid external deps, see tektoncd#1372)

Side effects:

  - new `yaml` build tag added so that existing integration tests aren't
    affected
  - Limited to 4 folders for testing
  - no-ci isn't explicitly excluded
  - no recursion in examples directory when finding examples to run

Fixes tektoncd#1251

test/README.md

@@ -206,9 +206,12 @@ go test -v -tags=e2e -count=1 ./test -run ^TestTaskRun
 To run the YAML e2e tests, run the following command:
 
 ```bash
-./test/e2e-tests-yaml.sh
+go test -v -count=1 -tags='e2e,yaml' -timeout=20m -test.run '^TestYaml' ./test/
 ```
 
+To limit parallelism of tests, use `-parallel=n` where `n` is the number of
+tests to run in parallel.
+
 ### Running upgrade tests
 
 There are two scenarios in upgrade tests. One is to install the previous release, upgrade to the current release, and

test/e2e-common.sh

@@ -18,115 +18,6 @@
 
 source $(git rev-parse --show-toplevel)/vendor/github.com/tektoncd/plumbing/scripts/e2e-tests.sh
 
-function teardown() {
-    subheader "Tearing down Tekton Pipelines"
-    ko delete --ignore-not-found=true -f config/
-    # teardown will be called when run against an existing cluster to cleanup before
-    # continuing, so we must wait for the cleanup to complete or the subsequent attempt
-    # to deploy to the same namespace will fail
-    wait_until_object_does_not_exist namespace tekton-pipelines
-}
-
-function output_yaml_test_results() {
-  # If formatting fails for any reason, use yaml as a fall back.
-  kubectl get $1.tekton.dev -o=custom-columns-file=${REPO_ROOT_DIR}/test/columns.txt || \
-    kubectl get $1.tekton.dev -oyaml
-}
-
-function output_pods_logs() {
-    echo ">>> $1"
-    kubectl get $1.tekton.dev -o yaml
-    local runs=$(kubectl get $1.tekton.dev --output=jsonpath="{.items[*].metadata.name}")
-    set +e
-    for run in ${runs}; do
-	echo ">>>> $1 ${run}"
-	case "$1" in
-	    "taskrun")
-		tkn taskrun logs --nocolour ${run}
-		;;
-	    "pipelinerun")
-		tkn pipelinerun logs --nocolour ${run}
-		;;
-	esac
-    done
-    set -e
-    echo ">>>> Pods"
-    kubectl get pods -o yaml
-}
-
-# Called by `fail_test` (provided by `e2e-tests.sh`) to dump info on test failure
-function dump_extra_cluster_state() {
-  echo ">>> Pipeline controller log:"
-  kubectl -n tekton-pipelines logs $(get_app_pod tekton-pipelines-controller tekton-pipelines)
-  echo ">>> Pipeline webhook log:"
-  kubectl -n tekton-pipelines logs $(get_app_pod tekton-pipelines-webhook tekton-pipelines)
-}
-
-function validate_run() {
-  local tests_finished=0
-  for i in {1..90}; do
-    local finished="$(kubectl get $1.tekton.dev --output=jsonpath='{.items[*].status.conditions[*].status}')"
-    if [[ ! "$finished" == *"Unknown"* ]]; then
-      tests_finished=1
-      break
-    fi
-    sleep 10
-  done
-
-  return ${tests_finished}
-}
-
-function check_results() {
-  local failed=0
-  results="$(kubectl get $1.tekton.dev --output=jsonpath='{range .items[*]}{.metadata.name}={.status.conditions[*].type}{.status.conditions[*].status}{" "}{end}')"
-  for result in ${results}; do
-    if [[ ! "${result,,}" == *"=succeededtrue" ]]; then
-      echo "ERROR: test ${result} but should be succeededtrue"
-      failed=1
-    fi
-  done
-
-  return ${failed}
-}
-
-function create_resources() {
-  local resource=$1
-  echo ">> Creating resources ${resource}"
-
-  # Applying the resources, either *taskruns or * *pipelineruns except those
-  # in the no-ci directory
-  for file in $(find ${REPO_ROOT_DIR}/examples/${resource}s/ -name '*.yaml' -not -path '*/no-ci/*' | sort); do
-    perl -p -e 's/gcr.io\/christiewilson-catfactory/$ENV{KO_DOCKER_REPO}/g' ${file} | ko create -f - || return 1
-  done
-}
-
-function run_tests() {
-  local resource=$1
-
-  # Wait for tests to finish.
-  echo ">> Waiting for tests to finish for ${resource}"
-  if validate_run $resource; then
-    echo "ERROR: tests timed out"
-  fi
-
-  # Check that tests passed.
-  echo ">> Checking test results for ${resource}"
-  if check_results $resource; then
-    echo ">> All YAML tests passed"
-    return 0
-  fi
-  return 1
-}
-
-function run_yaml_tests() {
-  echo ">> Starting tests for the resource ${1}/${2}"
-  create_resources ${1}/${2} || fail_test "Could not create ${2}/${1} from the examples"
-  if ! run_tests ${2}; then
-    return 1
-  fi
-  return 0
-}
-
 function install_pipeline_crd() {
   echo ">> Deploying Tekton Pipelines"
   ko resolve -f config/ \

test/e2e-tests-upgrade.sh

@@ -51,15 +51,7 @@ failed=0
 go_test_e2e -timeout=20m ./test || failed=1
 
 # Run the post-integration tests.
-for test in taskrun pipelinerun; do
-  header "Running YAML e2e tests for ${test}s"
-  if ! run_yaml_tests ${test}; then
-    echo "ERROR: one or more YAML tests failed"
-    output_yaml_test_results ${test}
-    output_pods_logs ${test}
-    failed=1
-  fi
-done
+go_test_e2e -tags='e2e,yaml' -timeout=20m -test.run '^TestYaml' ./test/ || failed=1
 
 # Remove all the pipeline CRDs, and clean up the environment for next Scenario.
 uninstall_pipeline_crd
@@ -87,15 +79,7 @@ go_test_e2e -timeout=20m ./test || failed=1
 # Run the post-integration tests. We do not need to install the resources again, since
 # they are installed before the upgrade. We verify if they still work, after going through
 # the upgrade.
-for test in taskrun pipelinerun; do
-  header "Running YAML e2e tests for ${test}s"
-  if ! run_tests ${test}; then
-    echo "ERROR: one or more YAML tests failed"
-    output_yaml_test_results ${test}
-    output_pods_logs ${test}
-    failed=1
-  fi
-done
+go_test_e2e -tags='e2e,yaml' -timeout=20m -test.run '^TestYaml' ./test/ || failed=1
 
 (( failed )) && fail_test