Split expect tests into separate circleci group

.circleci/config.yml

@@ -62,6 +62,23 @@ workflows:
                 - /rel\/.*/
                 - /hotfix\/.*/
           context: slack-secrets
+      - amd64_e2e_expect:
+          requires:
+            - amd64_build
+          filters:
+            branches:
+              ignore:
+                - /rel\/.*/
+                - /hotfix\/.*/
+      - amd64_e2e_expect_nightly:
+          requires:
+            - amd64_build
+          filters:
+            branches:
+              only:
+                - /rel\/.*/
+                - /hotfix\/.*/
+          context: slack-secrets
       - arm64_build
       - arm64_test:
           requires:
@@ -114,6 +131,23 @@ workflows:
                 - /rel\/.*/
                 - /hotfix\/.*/
           context: slack-secrets
+      - arm64_e2e_expect:
+          requires:
+            - arm64_build
+          filters:
+            branches:
+              ignore:
+                - /rel\/.*/
+                - /hotfix\/.*/
+      - arm64_e2e_expect_nightly:
+          requires:
+            - arm64_build
+          filters:
+            branches:
+              only:
+                - /rel\/.*/
+                - /hotfix\/.*/
+          context: slack-secrets
       - mac_amd64_build
       - mac_amd64_test:
           requires:
@@ -166,6 +200,23 @@ workflows:
                 - /rel\/.*/
                 - /hotfix\/.*/
           context: slack-secrets
+      - mac_amd64_e2e_expect:
+          requires:
+            - mac_amd64_build
+          filters:
+            branches:
+              ignore:
+                - /rel\/.*/
+                - /hotfix\/.*/
+      - mac_amd64_e2e_expect_nightly:
+          requires:
+            - mac_amd64_build
+          filters:
+            branches:
+              only:
+                - /rel\/.*/
+                - /hotfix\/.*/
+          context: slack-secrets
       #- windows_x64_build
 
 commands:
@@ -473,6 +524,33 @@ jobs:
           event: fail
           template: basic_fail_1
 
+  amd64_e2e_expect:
+    machine:
+      image: ubuntu-2004:202104-01
+    resource_class: large
+    environment:
+      E2E_TEST_FILTER: "EXPECT"
+    steps:
+      - prepare_go
+      - generic_integration:
+          result_subdir: amd64-e2e_subs
+          short_test_flag: "-short"
+
+  amd64_e2e_expect_nightly:
+    machine:
+      image: ubuntu-2004:202104-01
+    resource_class: large
+    environment:
+      E2E_TEST_FILTER: "EXPECT"
+    steps:
+      - prepare_go
+      - generic_integration:
+          result_subdir: amd64-e2e_subs_nightly
+          no_output_timeout: 45m
+      - slack/notify:
+          event: fail
+          template: basic_fail_1
+
   arm64_build:
     machine:
       image: ubuntu-2004:202101-01
@@ -571,6 +649,35 @@ jobs:
           event: fail
           template: basic_fail_1
 
+  arm64_e2e_expect:
+    machine:
+      image: ubuntu-2004:202101-01
+    resource_class: arm.large
+    environment:
+      E2E_TEST_FILTER: "EXPECT"
+    steps:
+      - checkout
+      - prepare_go
+      - generic_integration:
+          result_subdir: arm64-e2e_subs
+          short_test_flag: "-short"
+
+  arm64_e2e_expect_nightly:
+    machine:
+      image: ubuntu-2004:202101-01
+    resource_class: arm.large
+    environment:
+      E2E_TEST_FILTER: "EXPECT"
+    steps:
+      - checkout
+      - prepare_go
+      - generic_integration:
+          result_subdir: arm64-e2e_subs-nightly
+          no_output_timeout: 45m
+      - slack/notify:
+          event: fail
+          template: basic_fail_1
+
   mac_amd64_build:
     macos:
       xcode: 12.0.1
@@ -687,6 +794,39 @@ jobs:
           event: fail
           template: basic_fail_1
 
+  mac_amd64_e2e_expect:
+    macos:
+      xcode: 12.0.1
+    resource_class: large
+    environment:
+      E2E_TEST_FILTER: "EXPECT"
+      HOMEBREW_NO_AUTO_UPDATE: "true"
+    steps:
+      #- run: git -C /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core fetch --unshallow
+      - prepare_go
+      - generic_integration:
+          result_subdir: mac-amd64-e2e_subs
+          circleci_home: /Users/distiller
+          short_test_flag: "-short"
+
+  mac_amd64_e2e_expect_nightly:
+    macos:
+      xcode: 12.0.1
+    resource_class: large
+    environment:
+      E2E_TEST_FILTER: "EXPECT"
+      HOMEBREW_NO_AUTO_UPDATE: "true"
+    steps:
+      #- run: git -C /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core fetch --unshallow
+      - prepare_go
+      - generic_integration:
+          result_subdir: mac-amd64-e2e_subs-nightly
+          circleci_home: /Users/distiller
+          no_output_timeout: 45m
+      - slack/notify:
+          event: fail
+          template: basic_fail_1
+
   windows_x64_build:
     executor:
       name: win/default

scripts/travis/run_tests.sh

@@ -3,6 +3,11 @@
 set -e
 
 if [ "${BUILD_TYPE}" = "integration" ]; then
+    # Travis has trouble with the expect tests.
+    if [ "${TRAVIS}" = "true" ]; then
+      export RUN_EXPECT="FALSE"
+    fi
+
     # Run short tests when doing pull requests; leave the long testing for nightly runs.
     if [[ "${TRAVIS_BRANCH}" =~ ^rel/nightly ]] || [[ "${TRAVIS_BRANCH}" =~ ^hotfix/ ]]; then
         SHORTTEST=
@@ -11,7 +16,7 @@ if [ "${BUILD_TYPE}" = "integration" ]; then
     fi
     export SHORTTEST 
     make integration
-elif [ "${TRAVIS_EVENT_TYPE}" = "cron" ] || [[ "${TRAVIS_BRANCH}" =~ ^rel/ ]]  || [[ "${TRAVIS_BRANCH}" =~ ^hotfix/ ]]; then
+elif [ "${TRAVIS_EVENT_TYPE}" = "cron" ] || [[ "${TRAVIS_BRANCH}" =~ ^rel/ ]] || [[ "${TRAVIS_BRANCH}" =~ ^hotfix/ ]]; then
     make fulltest -j2
 else
     make shorttest -j2

test/e2e-go/cli/goal/expect/goal_expect_test.go

@@ -24,7 +24,6 @@ import (
 
 // TestGoalWithExpect Process all expect script files with suffix Test.exp within the test/e2e-go/cli/goal/expect directory
 func TestGoalWithExpect(t *testing.T) {
-	t.Skip("goal expect test are disabled due to flakiness")
 	et := fixtures.MakeExpectTest(t)
 	et.Run()
 }

test/framework/fixtures/expectFixture.go

@@ -98,6 +98,10 @@ func (ef *ExpectFixture) removeTestDir(workingDir string) (err error) {
 
 // MakeExpectTest creates an expect test fixture for the current directory
 func MakeExpectTest(t *testing.T) *ExpectFixture {
+	if skipExpectTests() {
+		t.Skip("Expect tests disabled by environment variables.")
+	}
+
 	ef := &ExpectFixture{}
 	ef.expectFiles = make(map[string]string) // map expect test to full file name.
 	err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
@@ -112,6 +116,33 @@ func MakeExpectTest(t *testing.T) *ExpectFixture {
 	return ef
 }
 
+func skipExpectTests() bool {
+	// Explicitly enabled.
+	if strings.ToUpper(os.Getenv("RUN_EXPECT")) == "TRUE" {
+		return false
+	}
+	if strings.ToUpper(os.Getenv("RUN_EXPECT")) == "FALSE" {
+		return true
+	}
+
+	// If any of the CI systems didn't set RUN_EXPECT, disable them.
+	if strings.ToUpper(os.Getenv("CI")) == "TRUE" {
+		return true
+	}
+	if strings.ToUpper(os.Getenv("CIRCLECI")) == "TRUE" {
+		return true
+	}
+	if strings.ToUpper(os.Getenv("TRAVIS")) == "TRUE" {
+		return true
+	}
+	if strings.ToUpper(os.Getenv("JENKINS_URL")) != "" {
+		return true
+	}
+
+	// Implicitly enable for devs running tests.
+	return false
+}
+
 // Run Process all expect script files with suffix Test.exp within the current directory
 func (ef *ExpectFixture) Run() {
 	disabledTest := map[string]string{

test/scripts/e2e.sh

@@ -45,6 +45,18 @@ TEST_RUN_ID=$(${SCRIPT_PATH}/testrunid.py)
 export TEMPDIR=${SRCROOT}/tmp/out/e2e/${TEST_RUN_ID}
 echo "Test output can be found in ${TEMPDIR}"
 
+function cleanup() {
+  echo "Cleaning up temp dir."
+
+  rm -rf "${TEMPDIR}"
+
+  if ! ${NO_BUILD} ; then
+      rm -rf ${PKG_ROOT}
+  fi
+}
+
+# Cleanup files created during tests.
+trap cleanup EXIT
 
 # ARM64 has an unoptimized scrypt() which can cause tests to timeout.
 # Run kmd with scrypt() configured to run less secure and fast to go through the motions for test.
@@ -101,7 +113,6 @@ export GOPATH=$(go env GOPATH)
 cd "${SCRIPT_PATH}"
 
 if [ -z "$E2E_TEST_FILTER" ] || [ "$E2E_TEST_FILTER" == "SCRIPTS" ]; then
-
     ./timeout 200 ./e2e_basic_start_stop.sh
     duration "e2e_basic_start_stop.sh"
 
@@ -122,28 +133,35 @@ if [ -z "$E2E_TEST_FILTER" ] || [ "$E2E_TEST_FILTER" == "SCRIPTS" ]; then
     for script in "$SRCROOT"/test/scripts/e2e_subs/serial/*; do
         "${TEMPDIR}/ve/bin/python3" e2e_client_runner.py ${RUN_KMD_WITH_UNSAFE_SCRYPT} $script
     done
-    duration "serial client runners"
 
     deactivate
+    duration "serial client runners"
 fi # if E2E_TEST_FILTER == "" or == "SCRIPTS"
 
 if [ -z "$E2E_TEST_FILTER" ] || [ "$E2E_TEST_FILTER" == "GO" ]; then
     # Export our root temp folder as 'TESTDIR' for tests to use as their root test folder
     # This allows us to clean up everything with our rm -rf trap.
-    export TESTDIR=${TEMPDIR}
+    mkdir "${TEMPDIR}/go"
+    export TESTDIR=${TEMPDIR}/go
     export TESTDATADIR=${SRCROOT}/test/testdata
     export SRCROOT=${SRCROOT}
 
     ./e2e_go_tests.sh ${GO_TEST_ARGS}
-    duration "e2e_go_tests.sh"
+    duration "go integration tests"
+fi # if E2E_TEST_FILTER == "" or == "GO"
 
-    rm -rf "${TEMPDIR}"
+if [ -z "$E2E_TEST_FILTER" ] || [ "$E2E_TEST_FILTER" == "EXPECT" ]; then
+    # Export our root temp folder as 'TESTDIR' for tests to use as their root test folder
+    # This allows us to clean up everything with our rm -rf trap.
+    mkdir "${TEMPDIR}/expect"
+    export TESTDIR=${TEMPDIR}/expect
+    export TESTDATADIR=${SRCROOT}/test/testdata
+    export SRCROOT=${SRCROOT}
 
-    if ! ${NO_BUILD} ; then
-        rm -rf ${PKG_ROOT}
-    fi
+    ./e2e_go_tests.sh -e ${GO_TEST_ARGS}
+    duration "expect tests"
+fi # if E2E_TEST_FILTER == "" or == "EXPECT"
 
-    echo "----------------------------------------------------------------------"
-    echo "  DONE: E2E"
-    echo "----------------------------------------------------------------------"
-fi # if E2E_TEST_FILTER == "" or == "GO"
+echo "----------------------------------------------------------------------"
+echo "  DONE: E2E"
+echo "----------------------------------------------------------------------"

test/scripts/e2e_go_tests.sh

@@ -13,8 +13,13 @@ GOTESTCOMMAND=${GOTESTCOMMAND:="go test"}
 
 TESTPATTERNS=()
 NORACEBUILD=""
+export RUN_EXPECT="FALSE"
 while [ "$1" != "" ]; do
     case "$1" in
+        -e)
+            # The test code checks this variable.
+            export RUN_EXPECT="TRUE"
+            ;;
         -t)
             shift
             TESTPATTERNS+=($1)
@@ -30,6 +35,11 @@ while [ "$1" != "" ]; do
     shift
 done
 
+if [[ -n $TESTPATTERNS && -n $RUN_EXPECT ]]; then
+    echo "-t and -e are mutually exclusive."
+    exit 1
+fi
+
 # Anchor our repo root reference location
 REPO_ROOT="$( cd "$(dirname "$0")" ; pwd -P )"/../..
 
@@ -89,18 +99,24 @@ if [[ "${ARCHTYPE}" = arm* ]]; then
     PARALLEL_FLAG="-p 1"
 fi
 
+PACKAGES="./..."
+if [ "$RUN_EXPECT" != "" ]; then
+  PACKAGES=$(go list ./...|grep expect)
+fi
+
 echo "PARALLEL_FLAG = ${PARALLEL_FLAG}"
+echo "PACKAGES = ${PACKAGES}"
 
 if [ "${#TESTPATTERNS[@]}" -eq 0 ]; then
-    ${GOTESTCOMMAND} ${RACE_OPTION} ${PARALLEL_FLAG} -timeout 1h -v ${SHORTTEST} ./...
+    ${GOTESTCOMMAND} ${RACE_OPTION} ${PARALLEL_FLAG} -timeout 1h -v ${SHORTTEST} ${PACKAGES}
 else
     for TEST in ${TESTPATTERNS[@]}; do
-        ${GOTESTCOMMAND} ${RACE_OPTION} ${PARALLEL_FLAG} -timeout 1h -v ${SHORTTEST} -run ${TEST} ./...
+        ${GOTESTCOMMAND} ${RACE_OPTION} ${PARALLEL_FLAG} -timeout 1h -v ${SHORTTEST} -run ${TEST} ${PACKAGES}
     done
 fi
 
 if [ ${CLEANUP_TEMPDIR} -ne 0 ]; then
-    rm -rf ${TEMPDIR}
+    rm -rf "${TEMPDIR}"
 fi
 
 echo "----------------------------------------------------------------------"