upload log should not fail the CI (#2879)

alibaba · Jun 13, 2023 · 5a63faf · 5a63faf
1 parent 559b084
commit 5a63faf
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 128 deletions.
diff --git a/.github/workflows/gaia.yml b/.github/workflows/gaia.yml
@@ -101,6 +101,7 @@ jobs:
     - name: Upload GIE log
       if: failure()
       uses: actions/upload-artifact@v3
+      continue-on-error: true
       with:
         name: gie-log
         path: /var/log/graphscope
diff --git a/.github/workflows/gss.yml b/.github/workflows/gss.yml
@@ -184,8 +184,7 @@ jobs:
 
         minikube image load ${{ env.GSS_IMAGE }}:${SHORT_SHA}
 
-    - name: Start to export kubernetes logs
-      uses: dashanji/kubernetes-log-export-action@v4
+    - uses: dashanji/kubernetes-log-export-action@v4
       env:
         SHOW_TIMESTAMPS: 'true'
         OUTPUT_DIR: ${{ github.workspace }}/helm-installation-logs
@@ -203,19 +202,6 @@ jobs:
         helm install ci --set image.tag=${SHORT_SHA} ./graphscope-store
         helm test ci --timeout 5m0s
 
-    - name: Stop to export kubernetes logs
-      uses: dashanji/kubernetes-log-export-action@v4
-      env:
-        MODE: stop
-
-    - name: Start to export kubernetes logs
-      uses: dashanji/kubernetes-log-export-action@v4
-      env:
-        SHOW_TIMESTAMPS: 'true'
-        OUTPUT_DIR: ${{ github.workspace }}/demo-fresh-of-helm-installation-logs
-        NAMESPACES: "gs*,default"
-        MODE: start
-
     - name: Test the helm deployment
       run: |
         # 1. get gss service endpoint
@@ -248,19 +234,6 @@ jobs:
         python3 setup.py build_proto
         python3 -m pytest -s -vvv graphscope/tests/kubernetes/test_store_service.py -k test_demo_fresh
 
-    - name: Stop to export kubernetes logs
-      uses: dashanji/kubernetes-log-export-action@v4
-      env:
-        MODE: stop
-
-    - name: Start to export kubernetes logs
-      uses: dashanji/kubernetes-log-export-action@v4
-      env:
-        SHOW_TIMESTAMPS: 'true'
-        OUTPUT_DIR: ${{ github.workspace }}/demo-script-of-helm-installation-with-pv-logs
-        NAMESPACES: "gs*,default"
-        MODE: start
-
     - name: restart helm and run demo with the PersistentVolume
       run: |
         # restart helm and run demo with the PersistentVolume
@@ -269,19 +242,6 @@ jobs:
         cd ${GITHUB_WORKSPACE}/charts
         helm install ci --set image.tag=${SHORT_SHA} ./graphscope-store
 
-    - name: Stop to export kubernetes logs
-      uses: dashanji/kubernetes-log-export-action@v4
-      env:
-        MODE: stop
-
-    - name: Start to export kubernetes logs
-      uses: dashanji/kubernetes-log-export-action@v4
-      env:
-        SHOW_TIMESTAMPS: 'true'
-        OUTPUT_DIR: ${{ github.workspace }}/helm-test-of-helm-installation-with-pv-logs
-        NAMESPACES: "gs*,default"
-        MODE: start
-
     - name: Helm Test with Helm Deployment and PersistentVolume
       run: |
         # helm test and python test on the restarted store
@@ -290,19 +250,6 @@ jobs:
         export NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
         helm test ci --timeout 10m0s
 
-    - name: Stop to export kubernetes logs
-      uses: dashanji/kubernetes-log-export-action@v4
-      env:
-        MODE: stop
-
-    - name: Start to export kubernetes logs
-      uses: dashanji/kubernetes-log-export-action@v4
-      env:
-        SHOW_TIMESTAMPS: 'true'
-        OUTPUT_DIR: ${{ github.workspace }}/demo-after-restart-of-helm-installation-with-pv-logs
-        NAMESPACES: "gs*,default"
-        MODE: start
-
     - name: Python Test with Helm Deployment and PersistentVolume
       run: |
         export GRPC_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services ci-graphscope-store-frontend)
@@ -311,14 +258,15 @@ jobs:
         cd ${GITHUB_WORKSPACE}/python
         python3 -m pytest -s -vvv graphscope/tests/kubernetes/test_store_service.py -k test_demo_after_restart
 
+    - uses: dashanji/kubernetes-log-export-action@v4
+      env:
+        MODE: stop
+
     - name: upload the k8s logs to artifact
       if: ${{ always() }}
       uses: actions/upload-artifact@v3
+      continue-on-error: true
       with:
         name: k8s-test-logs
         path: ${{ github.workspace }}/k8s-ci-*-logs
 
-    - name: Stop to export kubernetes logs
-      uses: dashanji/kubernetes-log-export-action@v4
-      env:
-        MODE: stop
diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml
@@ -377,23 +377,6 @@ jobs:
           # Use a dummy builder image (tag=ci, which is actually a busybox) to reduce time and space to pull the builder
           make graphscope CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci
 
-      - name: Prepare the log directory
-        run: |
-          # create the helm installation log directory
-          mkdir -p ${{ github.workspace }}/helm-installation-logs
-
-          # create the store service of helm installation log directory
-          mkdir -p ${{ github.workspace }}/store-service-of-helm-installation-logs
-
-          # create the demo script of helm installation log directory
-          mkdir -p ${{ github.workspace }}/demo-script-of-helm-installation-logs
-
-          # create the store service of kubernetes test log directory
-          mkdir -p ${{ github.workspace }}/store-service-of-kubernetes-test-logs
-
-          # create the demo script of hdfs test log directory
-          mkdir -p ${{ github.workspace }}/demo-script-of-hdfs-test-logs
-
       - name: Install Python dependencies
         run: |
           cd ${GITHUB_WORKSPACE}/python
@@ -448,8 +431,7 @@ jobs:
           minikube image load graphscope/learning:${SHORT_SHA}
           echo "loaded learning"
 
-      - name: Start to export kubernetes logs
-        uses: dashanji/kubernetes-log-export-action@v4
+      - uses: dashanji/kubernetes-log-export-action@v4
         env:
           SHOW_TIMESTAMPS: 'true'
           OUTPUT_DIR: ${{ github.workspace }}/k8s-ci-helm-installation-logs
@@ -467,19 +449,6 @@ jobs:
                                   ./graphscope
           helm test graphscope --timeout 5m0s
 
-      - name: Stop to export kubernetes logs
-        uses: dashanji/kubernetes-log-export-action@v4
-        env:
-          MODE: stop
-
-      - name: Start to export kubernetes logs
-        uses: dashanji/kubernetes-log-export-action@v4
-        env:
-          SHOW_TIMESTAMPS: 'true'
-          OUTPUT_DIR: ${{ github.workspace }}/k8s-ci-demo-script-of-helm-installation-logs
-          NAMESPACES: "gs*,default"
-          MODE: start
-
       - name: Test demo script of helm installation
         run: |
           export NODE_IP=$(kubectl get pod -lgraphscope.coordinator.name=coordinator-graphscope -ojsonpath="{.items[0].status.hostIP}")
@@ -490,20 +459,17 @@ jobs:
 
           python3 -m pytest -s -vvv ./graphscope/tests/kubernetes/test_demo_script.py -k test_helm_installation
 
-      - name: Stop to export kubernetes logs
-        uses: dashanji/kubernetes-log-export-action@v4
+      - uses: dashanji/kubernetes-log-export-action@v4
         env:
           MODE: stop
 
       - name: Delete Helm Cluster
-        run: |
-          helm delete graphscope
+        run: helm delete graphscope
 
-      - name: Start to export kubernetes logs
-        uses: dashanji/kubernetes-log-export-action@v4
+      - uses: dashanji/kubernetes-log-export-action@v4
         env:
           SHOW_TIMESTAMPS: 'true'
-          OUTPUT_DIR: ${{ github.workspace }}/k8s-ci-store-service-of-kubernetes-test-logs
+          OUTPUT_DIR: ${{ github.workspace }}/k8s-ci-demo-script-test-logs
           NAMESPACES: "gs*,default"
           MODE: start
 
@@ -522,25 +488,12 @@ jobs:
                           --cov-report=term --exitfirst -s -vvv --log-cli-level=INFO \
                           ./graphscope/tests/kubernetes
 
-      - name: Stop to export kubernetes logs
-        uses: dashanji/kubernetes-log-export-action@v4
-        env:
-          MODE: stop
-
       - name: Upload Coverage
         uses: codecov/codecov-action@v3
         with:
           file: ./python/coverage.xml
           fail_ci_if_error: false
 
-      - name: Start to export kubernetes logs
-        uses: dashanji/kubernetes-log-export-action@v4
-        env:
-          SHOW_TIMESTAMPS: 'true'
-          OUTPUT_DIR: ${{ github.workspace }}/k8s-ci-demo-script-of-hdfs-test-logs
-          NAMESPACES: "gs*,default"
-          MODE: start
-
       - name: HDFS test
         env:
           JAVA_HOME: /usr/lib/jvm/default-java
@@ -561,28 +514,24 @@ jobs:
           hadoop fs -put ${GS_TEST_DIR}/ldbc_sample/person_0_0.csv /ldbc_sample/person_0_0.csv
           hadoop fs -put ${GS_TEST_DIR}/ldbc_sample/person_knows_person_0_0.csv /ldbc_sample/person_knows_person_0_0.csv
 
-          # validate hadoop
-          hadoop fs -ls /ldbc_sample
-
           # prepare CI environments
           export HDFS_TEST_DIR=hdfs:///ldbc_sample
           export HDFS_HOST=$(hostname -I | awk '{print $1}')
 
           # run test
           cd ${GITHUB_WORKSPACE}/python
-
           python3 -m pytest -s -vvv ./graphscope/tests/kubernetes/test_demo_script.py -k test_demo_on_hdfs
           # Check the result file have successfully written to the given location
           # hdfs dfs -test -e /ldbc_sample/res.csv_0 && hdfs dfs -test -e /ldbc_sample/res.csv_1
 
-      - name: Stop to export kubernetes logs
-        uses: dashanji/kubernetes-log-export-action@v4
+      - uses: dashanji/kubernetes-log-export-action@v4
         env:
           MODE: stop
 
       - name: Upload the k8s logs to artifact
         uses: actions/upload-artifact@v3
         if: ${{ always() }}
+        continue-on-error: true
         with:
           name: k8s-test-logs
           path: ${{ github.workspace }}/k8s-ci-*-logs
@@ -718,11 +667,6 @@ jobs:
               -t registry.cn-hongkong.aliyuncs.com/graphscope/interactive-experimental:${SHORT_SHA} \
               -f .github/workflows/docker/interactive-experimental-local-artifacts.Dockerfile .
 
-      - name: Prepare the log directory
-        run: |
-          # create the k8s failover test log directory
-          mkdir -p ${{ github.workspace }}/k8s-failover-test-logs
-
       - name: Prepare Cluster and Data
         env:
           GS_TEST_DIR: ${{ github.workspace }}/gstest
@@ -743,8 +687,7 @@ jobs:
           # install python gremlin client
           pip install gremlinpython
 
-      - name: Start to export kubernetes logs
-        uses: dashanji/kubernetes-log-export-action@v4
+      - uses: dashanji/kubernetes-log-export-action@v4
         env:
           SHOW_TIMESTAMPS: 'true'
           OUTPUT_DIR: ${{ github.workspace }}/k8s-failover-logs
@@ -769,14 +712,14 @@ jobs:
           # run failover test
           cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_k8s_failover_ci.sh default test-gie-standalone 2 1
 
+      - uses: dashanji/kubernetes-log-export-action@v4
+        env:
+          MODE: stop
+
       - name: upload the k8s logs to artifact
         uses: actions/upload-artifact@v3
+        continue-on-error: true
         if: ${{ always() }}
         with:
           name: k8s-failover-logs
           path: ${{ github.workspace }}/k8s-failover-logs
-
-      - name: Stop to export kubernetes logs
-        uses: dashanji/kubernetes-log-export-action@v4
-        env:
-          MODE: stop
diff --git a/.github/workflows/local-ci.yml b/.github/workflows/local-ci.yml
@@ -313,6 +313,7 @@ jobs:
       - name: Upload GIE log
         if: failure()
         uses: actions/upload-artifact@v3
+        continue-on-error: true
         with:
           name: gie-log
           path: ~/.local/log/graphscope