fix(ci): Retry after docker log follow ssh failures (#4198)

* Retry after log ssh failures in full sync test * Retry after log failures in other docker tests
ZcashFoundation · Apr 26, 2022 · 4f92df4 · 4f92df4
1 parent d476c18
commit 4f92df4
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 18 deletions.
diff --git a/.github/workflows/test-full-sync.yml b/.github/workflows/test-full-sync.yml
@@ -236,12 +236,15 @@ jobs:
       - name: Full sync
         id: full-sync
         run: |
-          gcloud compute ssh \
-          full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
-          --zone ${{ env.ZONE }} \
-          --quiet \
-          --ssh-flag="-o ServerAliveInterval=5" \
-          --command="docker logs --follow ${{ env.CONTAINER_NAME }}"
+          for RETRY in 1 2 3 4; do
+              gcloud compute ssh \
+              full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
+              --zone ${{ env.ZONE }} \
+              --quiet \
+              --ssh-flag="-o ServerAliveInterval=15" \
+              --command="docker logs --follow ${{ env.CONTAINER_NAME }}" \
+              || echo "ssh disconnected $RETRY times"
+          done
 
           EXIT_CODE=$(\
           gcloud compute ssh \

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -347,12 +347,15 @@ jobs:
         id: sync-to-checkpoint
         if: ${{ steps.create-instance.outcome == 'success' }}
         run: |
-          gcloud compute ssh \
-          regenerate-disk-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
-          --zone ${{ env.ZONE }} \
-          --quiet \
-          --ssh-flag="-o ServerAliveInterval=5" \
-          --command="docker logs --follow ${{ env.CONTAINER_NAME }}"
+          for RETRY in 1 2 3 4; do
+              gcloud compute ssh \
+              regenerate-disk-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
+              --zone ${{ env.ZONE }} \
+              --quiet \
+              --ssh-flag="-o ServerAliveInterval=15" \
+              --command="docker logs --follow ${{ env.CONTAINER_NAME }}" \
+              || echo "ssh disconnected $RETRY times"
+          done
 
           EXIT_CODE=$(\
           gcloud compute ssh \
@@ -512,12 +515,15 @@ jobs:
       - name: Sync past mandatory checkpoint
         id: sync-past-checkpoint
         run: |
-          gcloud compute ssh \
-          sync-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
-          --zone ${{ env.ZONE }} \
-          --quiet \
-          --ssh-flag="-o ServerAliveInterval=5" \
-          --command="docker logs --follow ${{ env.CONTAINER_NAME }}"
+          for RETRY in 1 2 3 4; do
+              gcloud compute ssh \
+              sync-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
+              --zone ${{ env.ZONE }} \
+              --quiet \
+              --ssh-flag="-o ServerAliveInterval=15" \
+              --command="docker logs --follow ${{ env.CONTAINER_NAME }}" \
+              || echo "ssh disconnected $RETRY times"
+          done
 
           EXIT_CODE=$(\
           gcloud compute ssh \