testing: move travis_retry to skip rebuilding (algorand#2324)

Remove the top-level retry command. This will allow build / lint failures to terminate the build with no retry, and may speed up test failure retries.
tsachiherman · Jun 24, 2021 · b55b53f · b55b53f
1 parent 90edab3
commit b55b53f
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 26 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -28,7 +28,7 @@ jobs:
     - stage: build_commit
       os: linux
       script:
-        - travis_retry scripts/travis/build_test.sh
+        - scripts/travis/build_test.sh
 
     - stage: build_pr
       os: linux
@@ -39,12 +39,12 @@ jobs:
       os: linux
       name: Ubuntu AMD64 Build
       script:
-        - travis_retry scripts/travis/build_test.sh
+        - scripts/travis/build_test.sh
     - # same stage, parallel job
       os: linux
       name: Ubuntu AMD64 Integration Test
       script:
-        - travis_retry ./scripts/travis/integration_test.sh
+        - ./scripts/travis/integration_test.sh
     - # same stage, parallel job
       name: External ARM64 Build
       os: linux
@@ -56,7 +56,7 @@ jobs:
           packages:
             - awscli
       script:
-        - travis_retry scripts/travis/external_build.sh ./scripts/travis/build_test.sh
+        - scripts/travis/external_build.sh ./scripts/travis/build_test.sh
     - # same stage, parallel job
       name: External ARM64 Integration Test
       os: linux
@@ -74,13 +74,13 @@ jobs:
       osx_image: xcode11
       name: MacOS AMD64 Build
       script:
-        - travis_retry scripts/travis/build_test.sh
+        - scripts/travis/build_test.sh
     - # same stage, parallel job
       os: osx
       osx_image: xcode11
       name: MacOS AMD64 Integration Test
       script:
-        - travis_retry ./scripts/travis/integration_test.sh
+        - ./scripts/travis/integration_test.sh
     - # same stage, parallel job
        os: windows
        name: Windows x64 Build
@@ -89,7 +89,7 @@ jobs:
            - $HOME/AppData/Local/Temp/chocolatey
            - /C/tools/msys64
        script:
-         - travis_retry $mingw64 scripts/travis/build_test.sh
+         - $mingw64 scripts/travis/build_test.sh
 
     - stage: build_release
       os: linux
@@ -100,12 +100,12 @@ jobs:
       os: linux
       name: Ubuntu AMD64 Build
       script:
-        - travis_retry ./scripts/travis/build_test.sh
+        - ./scripts/travis/build_test.sh
     - # same stage, parallel job
       os: linux
       name: Ubuntu AMD64 Integration Test
       script:
-        - travis_retry ./scripts/travis/integration_test.sh
+        - ./scripts/travis/integration_test.sh
     - # same stage, parallel job
       name: External ARM64 Build
       os: linux
@@ -117,7 +117,7 @@ jobs:
           packages:
             - awscli
       script:
-        - travis_retry scripts/travis/external_build.sh ./scripts/travis/build_test.sh
+        - scripts/travis/external_build.sh ./scripts/travis/build_test.sh
     - # same stage, parallel job
       name: External ARM64 Integration Test
       os: linux
@@ -135,13 +135,13 @@ jobs:
       osx_image: xcode11
       name: MacOS AMD64 Build
       script:
-        - travis_retry scripts/travis/build_test.sh
+        - scripts/travis/build_test.sh
     - # same stage, parallel job
       os: osx
       osx_image: xcode11
       name: MacOS AMD64 Integration Test
       script:
-        - travis_retry ./scripts/travis/integration_test.sh
+        - ./scripts/travis/integration_test.sh
     - # same stage, parallel job
        os: windows
        name: Windows x64 Build
@@ -150,7 +150,7 @@ jobs:
            - $HOME/AppData/Local/Temp/chocolatey
            - /C/tools/msys64
        script:
-         - travis_retry $mingw64 scripts/travis/build_test.sh
+         - $mingw64 scripts/travis/build_test.sh
 
     - stage: deploy
       name: Ubuntu Deploy

diff --git a/scripts/travis/build_test.sh b/scripts/travis/build_test.sh
@@ -13,12 +13,5 @@ ALGORAND_DEADLOCK=enable
 export ALGORAND_DEADLOCK
 SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
 
-if [ "${USER}" = "travis" ]; then
-    # we're running on a travis machine
-    "${SCRIPTPATH}/build.sh" --make_debug
-    "${SCRIPTPATH}/travis_wait.sh" 90 "${SCRIPTPATH}/test.sh"
-else
-    # we're running on an ephermal build machine
-    "${SCRIPTPATH}/build.sh" --make_debug
-    "${SCRIPTPATH}/test.sh"
-fi
+"${SCRIPTPATH}/build.sh" --make_debug
+"${SCRIPTPATH}/test.sh"
diff --git a/scripts/travis/run_tests.sh b/scripts/travis/run_tests.sh
@@ -2,6 +2,8 @@
 
 set -e
 
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+
 if [ "${BUILD_TYPE}" = "integration" ]; then
     # Run short tests when doing pull requests; leave the long testing for nightly runs.
     if [[ "${TRAVIS_BRANCH}" =~ ^rel/nightly ]]; then
@@ -10,9 +12,9 @@ if [ "${BUILD_TYPE}" = "integration" ]; then
         SHORTTEST=-short
     fi
     export SHORTTEST 
-    make integration
+    "${SCRIPTPATH}/travis_retry.sh" make integration
 elif [ "${TRAVIS_EVENT_TYPE}" = "cron" ] || [[ "${TRAVIS_BRANCH}" =~ ^rel/ ]]; then
-    make fulltest -j2
+    "${SCRIPTPATH}/travis_retry.sh" make fulltest -j2
 else
-    make shorttest -j2
+    "${SCRIPTPATH}/travis_retry.sh" make shorttest -j2
 fi
diff --git a/scripts/travis/travis_retry.sh b/scripts/travis/travis_retry.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_retry.bash
+
+ANSI_RED="\033[31;1m"
+ANSI_RESET="\033[0m"
+
+travis_retry() {
+  local result=0
+  local count=1
+  while [[ "${count}" -le 3 ]]; do
+    [[ "${result}" -ne 0 ]] && {
+      echo -e "\\n${ANSI_RED}The command \"${*}\" failed. Retrying, ${count} of 3.${ANSI_RESET}\\n" >&2
+    }
+    # run the command in a way that doesn't disable setting `errexit`
+    "${@}"
+    result="${?}"
+    if [[ $result -eq 0 ]]; then break; fi
+    count="$((count + 1))"
+    sleep 1
+  done
+
+  [[ "${count}" -gt 3 ]] && {
+    echo -e "\\n${ANSI_RED}The command \"${*}\" failed 3 times.${ANSI_RESET}\\n" >&2
+  }
+
+  return "${result}"
+}
+
+travis_retry "$@"
diff --git a/scripts/travis/travis_wait.sh b/scripts/travis/travis_wait.sh
@@ -69,4 +69,4 @@ travis_wait() {
   return $result
 }
 
-travis_wait $@
+travis_wait "$@"