diff --git a/.github/workflows/test-full-sync.yml b/.github/workflows/test-full-sync.yml index f819837114a..223b5d406ee 100644 --- a/.github/workflows/test-full-sync.yml +++ b/.github/workflows/test-full-sync.yml @@ -236,12 +236,15 @@ jobs: - name: Full sync id: full-sync run: | - gcloud compute ssh \ - full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ - --zone ${{ env.ZONE }} \ - --quiet \ - --ssh-flag="-o ServerAliveInterval=5" \ - --command="docker logs --follow ${{ env.CONTAINER_NAME }}" + for RETRY in 1 2 3 4; do + gcloud compute ssh \ + full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=15" \ + --command="docker logs --follow ${{ env.CONTAINER_NAME }}" \ + || echo "ssh disconnected $RETRY times" + done EXIT_CODE=$(\ gcloud compute ssh \ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1e623bf6ffb..9964803c00e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -347,12 +347,15 @@ jobs: id: sync-to-checkpoint if: ${{ steps.create-instance.outcome == 'success' }} run: | - gcloud compute ssh \ - regenerate-disk-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ - --zone ${{ env.ZONE }} \ - --quiet \ - --ssh-flag="-o ServerAliveInterval=5" \ - --command="docker logs --follow ${{ env.CONTAINER_NAME }}" + for RETRY in 1 2 3 4; do + gcloud compute ssh \ + regenerate-disk-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=15" \ + --command="docker logs --follow ${{ env.CONTAINER_NAME }}" \ + || echo "ssh disconnected $RETRY times" + done EXIT_CODE=$(\ gcloud compute ssh \ @@ -512,12 +515,15 @@ jobs: - name: Sync past mandatory checkpoint id: sync-past-checkpoint run: | - gcloud compute ssh \ - sync-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ - --zone ${{ env.ZONE }} \ - --quiet \ - --ssh-flag="-o ServerAliveInterval=5" \ - --command="docker logs --follow ${{ env.CONTAINER_NAME }}" + for RETRY in 1 2 3 4; do + gcloud compute ssh \ + sync-checkpoint-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=15" \ + --command="docker logs --follow ${{ env.CONTAINER_NAME }}" \ + || echo "ssh disconnected $RETRY times" + done EXIT_CODE=$(\ gcloud compute ssh \