Fix container-api twister param bug + compress workflows with matrix (z…

…ephyrproject-rtos#7)
pkunieck · Dec 1, 2021 · 84cc496 · 84cc496
1 parent c387026
commit 84cc496
Show file tree

Hide file tree

Showing 3 changed files with 65 additions and 137 deletions.
diff --git a/.github/workflows/1rtos-container-image-build.yml b/.github/workflows/1rtos-container-image-build.yml
@@ -18,77 +18,46 @@ jobs:
           docker push 127.0.0.1:5000/sdk-docker-intel:main.stg
         shell: bash
 
-  twisterTest_1of3:
-    needs: [ buildAndStageImage ]
-    defaults:
-      run:
-        shell: bash  
+  runTwister:
     runs-on: uzdo128
-    container:
-      image: 192.168.0.200:5000/sdk-docker-intel:main.stg
-      options: --privileged --user root:root -v/opt/toolchains:/opt/toolchains
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          path: 'zephyrproject/zephyr'
-          repository: 'intel-innersource/os.rtos.zephyr.zephyr.git'
-          ref: '1rtos-ci-self-test'
-          token: ${{ secrets.CHRISVINTC_AT_INNERSOURCE }} 
-      - name: 1rtos-ci
-        run: /opt/1rtos/container-api.sh 1rtos-ci $GITHUB_WORKSPACE 1 3 "--integration"
-      - name: log artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: twisterLog_1of3
-          path: ${{ github.workspace }}/zephyrproject/zephyr/twister-out/twister.log
-          retention-days: 30
-
-  twisterTest_2of3:
     needs: [ buildAndStageImage ]
     defaults:
       run:
-        shell: bash  
-    runs-on: uzdo128
+        shell: bash
     container:
       image: 192.168.0.200:5000/sdk-docker-intel:main.stg
       options: --privileged --user root:root -v/opt/toolchains:/opt/toolchains
+    strategy:
+      matrix:
+        batch: [1, 2, 3]
+        batch_total: [ 3 ] 
     steps:
-      - uses: actions/checkout@v2
+      - name: checkout zephyr ci-self-test tag
+        uses: actions/checkout@v2
         with:
           path: 'zephyrproject/zephyr'
           repository: 'intel-innersource/os.rtos.zephyr.zephyr.git'
-          ref: '1rtos-ci-self-test'
-          token: ${{ secrets.CHRISVINTC_AT_INNERSOURCE }} 
+          ref: '1rtos-ci-self-test' 
+          token: ${{ secrets.PAT_SYS_TMBUILD_SDK_DOCKER_INTEL }}
       - name: 1rtos-ci
-        run: /opt/1rtos/container-api.sh 1rtos-ci $GITHUB_WORKSPACE 2 3 "--integration"
+        run: /opt/1rtos/container-api.sh 1rtos-ci $GITHUB_WORKSPACE ${{ matrix.batch }} ${{ matrix.batch_total }} "--integration"
       - name: log artifacts
+        if: always()
         uses: actions/upload-artifact@v2
         with:
-          name: twisterLog_2of3
-          path: ${{ github.workspace }}/zephyrproject/zephyr/twister-out/twister.log
-          retention-days: 30
-
-  twisterTest_3of3:
-    needs: [ buildAndStageImage ]
-    defaults:
-      run:
-        shell: bash  
-    runs-on: uzdo128
-    container:
-      image: 192.168.0.200:5000/sdk-docker-intel:main.stg
-      options: --privileged --user root:root -v/opt/toolchains:/opt/toolchains
+          name: twisterLog_${{ matrix.batch }}of${{ matrix.batch_total }}
+          path: |
+            zephyrproject/zephyr/twister-out/twister.log
+            zephyrproject/zephyr/twister-out/twister.xml
+          retention-days: 1               
+
+# "OneRTOS-CI-Passed" is our exit job that matches branch-protection status-check
+# settings for PR workflow. Make sure all required tests are prereqs to this job.
+# For this build workflow, no actions are taken by OneRTOS-CI-Passed, just status
+# echo.
+  OneRTOS-CI-Passed:
+    needs: [ buildAndStageImage, runTwister ]
+    runs-on: testhead
     steps:
-      - uses: actions/checkout@v2
-        with:
-          path: 'zephyrproject/zephyr'
-          repository: 'intel-innersource/os.rtos.zephyr.zephyr.git'
-          ref: '1rtos-ci-self-test'
-          token: ${{ secrets.CHRISVINTC_AT_INNERSOURCE }} 
-      - name: 1rtos-ci
-        run: /opt/1rtos/container-api.sh 1rtos-ci $GITHUB_WORKSPACE 3 3 "--integration"
-      - name: log artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: twisterLog_3of3
-          path: ${{ github.workspace }}/zephyrproject/zephyr/twister-out/twister.log
-          retention-days: 30
+      - name: Post run steps
+        run: echo "OneRTOS-CI-Passsed"
diff --git a/.github/workflows/1rtos-container-image-push.yml b/.github/workflows/1rtos-container-image-push.yml
@@ -17,83 +17,41 @@ jobs:
           docker push 127.0.0.1:5000/sdk-docker-intel:main.stg
         shell: bash
 
-  twisterTest_1of3:
+  runTwister:
     needs: [ buildAndStageImage ]
     defaults:
       run:
-        shell: bash  
-    runs-on: uzdo128
-    container:
-      image: 192.168.0.200:5000/sdk-docker-intel:main.stg
-      options: --privileged --user root:root -v/opt/toolchains:/opt/toolchains
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          path: 'zephyrproject/zephyr'
-          repository: 'intel-innersource/os.rtos.zephyr.zephyr.git'
-          ref: '1rtos-ci-self-test'
-          token: ${{ secrets.CHRISVINTC_AT_INNERSOURCE }} 
-      - name: 1rtos-ci
-        run: /opt/1rtos/container-api.sh 1rtos-ci $GITHUB_WORKSPACE 1 3 "--integration"
-      - name: log artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: twisterLog_1of3
-          path: ${{ github.workspace }}/zephyrproject/zephyr/twister-out/twister.log
-          retention-days: 30
-
-  twisterTest_2of3:
-    needs: [ buildAndStageImage ]
-    defaults:
-      run:
-        shell: bash  
-    runs-on: uzdo128
-    container:
-      image: 192.168.0.200:5000/sdk-docker-intel:main.stg
-      options: --privileged --user root:root -v/opt/toolchains:/opt/toolchains
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          path: 'zephyrproject/zephyr'
-          repository: 'intel-innersource/os.rtos.zephyr.zephyr.git'
-          ref: '1rtos-ci-self-test'
-          token: ${{ secrets.CHRISVINTC_AT_INNERSOURCE }} 
-      - name: 1rtos-ci
-        run: /opt/1rtos/container-api.sh 1rtos-ci $GITHUB_WORKSPACE 2 3 "--integration"
-      - name: log artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: twisterLog_2of3
-          path: ${{ github.workspace }}/zephyrproject/zephyr/twister-out/twister.log
-          retention-days: 30
-
-  twisterTest_3of3:
-    needs: [ buildAndStageImage ]
-    defaults:
-      run:
-        shell: bash  
+        shell: bash
     runs-on: uzdo128
     container:
       image: 192.168.0.200:5000/sdk-docker-intel:main.stg
       options: --privileged --user root:root -v/opt/toolchains:/opt/toolchains
+    strategy:
+      matrix:
+        batch: [1, 2, 3]
+        batch_total: [ 3 ] 
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v2
         with:
           path: 'zephyrproject/zephyr'
           repository: 'intel-innersource/os.rtos.zephyr.zephyr.git'
-          ref: '1rtos-ci-self-test'
-          token: ${{ secrets.CHRISVINTC_AT_INNERSOURCE }} 
+          ref: '1rtos-ci-self-test' 
+          token: ${{ secrets.PAT_SYS_TMBUILD_SDK_DOCKER_INTEL }}
       - name: 1rtos-ci
-        run: /opt/1rtos/container-api.sh 1rtos-ci $GITHUB_WORKSPACE 3 3 "--integration"
+        run: /opt/1rtos/container-api.sh 1rtos-ci $GITHUB_WORKSPACE ${{ matrix.batch }} ${{ matrix.batch_total }}  "--integration"
       - name: log artifacts
+        if: always()
         uses: actions/upload-artifact@v2
         with:
-          name: twisterLog_3of3
-          path: ${{ github.workspace }}/zephyrproject/zephyr/twister-out/twister.log
-          retention-days: 30
+          name: twisterLog_${{ matrix.batch }}of${{ matrix.batch_total }}
+          path: |
+            zephyrproject/zephyr/twister-out/twister.log
+            zephyrproject/zephyr/twister-out/twister.xml
+          retention-days: 30              
 
-  promoteImage2Prod: 
-    needs: [ twisterTest_1of3, twisterTest_2of3, twisterTest_3of3 ]
+  OneRTOS-CI-Passed:
+    needs: [ buildAndStageImage, runTwister ]
     runs-on: testhead
     steps:
       - run: |
@@ -102,4 +60,3 @@ jobs:
             docker push 127.0.0.1:5000/sdk-docker-intel:main
             docker rmi 127.0.0.1:5000/sdk-docker-intel:main.stg
         shell: bash
-
diff --git a/1rtos/container-api.sh b/1rtos/container-api.sh
@@ -1,32 +1,34 @@
 #!/bin/bash
 
 # 1rtos/container-api.sh
-#  -a library of shell functions to standardize our 1rtos-container interfaces
+#  - standardized CI/QA functions via our 1rtos-test container ( sdk-docker-intel )
 #
 # Usage:
-#   a.) Source this file ahead of ahead of any automation calls and call functions directly
-#   b.) Call this script with the function name + parameters, eg:
-#	container-api.sh 1rtos-ci <workspace> <batch> ....
+#   Call this script with the function name + parameters, eg:
+#	$ container-api.sh 1rtos-ci <workspace> <batch> ....
 
-# 1rtos-ci <workspace> <batch> <parallel> <option>
+# 1rtos-ci workspace batch# parallel# <option-str>
 # ------------------------------------------------
 # * standard method for ci/qemu twister with skipList support, both container & repo defined.
-#	<workspace> ($1) = path to workspace in container
-#	<batch>	    ($2) = which batch number am I
-#	<parallel>  ($3) = total number of parallel jobs
-#	<extra>     ($4) = optional twister parameter to pass, typically used for "--integration"
+#	<workspace> ($2) = path to workspace in container
+#	<batch#>    ($3) = which batch number am I
+#	<parallel#> ($4) = total number of parallel jobs
+#	<option-str>($5) = optional twister parameter to pass, typically used for "--integration"
 1rtos-ci() {
-	cd "$1/zephyrproject"
+	cd "$2/zephyrproject"
 	if [ ! -d .west ]; then
 		west init -l zephyr
 	fi
 	west update
 
-	export ZEPHYR_BASE="$1/zephyrproject/zephyr"
-	cd "$1/zephyrproject/zephyr"
+	export ZEPHYR_BASE="$2/zephyrproject/zephyr"
+	cd "$ZEPHYR_BASE"
+
+	# set twister cmdline, injecting $5/option-str
+	TWISTER_CMD="scripts/twister -M -x=USE_CCACHE=0 -N --inline-logs -v $5" 
 
 	# store all testcases
-	scripts/twister -M -x=USE_CCACHE=0 -N --inline-logs -v --save-tests testcases "$4"
+	$TWISTER_CMD --save-tests testcases
 	cp testcases testcases.0
 
 	# apply twisterSkipList if exists in container
@@ -46,12 +48,12 @@
 	fi
 
 	# run twister with our standard options + injected batch/parallel & extra params
-	scripts/twister -B "$2/$3" -M --load-tests testcases -x=USE_CCACHE=0 -N --inline-logs -v --retry-failed 3 --retry-interval 60 "$4"
+	$TWISTER_CMD -B $3/$4 --load-tests testcases --retry-failed 3 --retry-interval 60
 }
-export -f 1rtos-ci
 
-# process internal method calls
+echo "oneRTOS container API invoked as: $@"
 
+# process internal method calls
 if [ "$1" == "1rtos-ci" ]; then
-	1rtos-ci "$2" "$3" "$4" "$5"
+	1rtos-ci "${@}"
 fi