Add full Android job in package pipeline (#9009)

* Add full Android job in package pipeline * Address CR comments
microsoft · Sep 9, 2021 · 1533f57 · 1533f57
1 parent c20cb76
commit 1533f57
Show file tree

Hide file tree

Showing 5 changed files with 158 additions and 74 deletions.
diff --git a/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh b/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh
@@ -1,22 +1,31 @@
 #!/bin/bash
 
 # This script will run build_aar_package.py to build Android AAR and copy all the artifacts
-# to a given folder for publishing to Maven Central
+# to a given folder for publishing to Maven Central, or building nuget package
 # This script is intended to be used in CI build only
 
 set -e
 set -x
 export PATH=/opt/python/cp37-cp37m/bin:$PATH
 
-# build the AAR package, using the build settings specified in
-# <repro root>/tools/ci_build/github/android/default_mobile_aar_build_settings.json
-python3 /onnxruntime_src/tools/ci_build/github/android/build_aar_package.py \
-    --build_dir /build \
-    --config $BUILD_CONFIG \
-    --android_sdk_path /android_home \
-    --android_ndk_path /ndk_home \
-    --include_ops_by_config /onnxruntime_src/tools/ci_build/github/android/mobile_package.required_operators.config \
-    /onnxruntime_src/tools/ci_build/github/android/default_mobile_aar_build_settings.json
+# build the AAR package, using the build settings under /home/onnxruntimedev/.build_settings/
+# if there is also include_ops_and_types.config exists in the same folder, use it to build with included ops/types
+if [ -f "/home/onnxruntimedev/.build_settings/include_ops_and_types.config" ]; then
+    python3 /onnxruntime_src/tools/ci_build/github/android/build_aar_package.py \
+        --build_dir /build \
+        --config $BUILD_CONFIG \
+        --android_sdk_path /android_home \
+        --android_ndk_path /ndk_home \
+        --include_ops_by_config /home/onnxruntimedev/.build_settings/include_ops_and_types.config \
+        /home/onnxruntimedev/.build_settings/build_settings.json
+else
+    python3 /onnxruntime_src/tools/ci_build/github/android/build_aar_package.py \
+        --build_dir /build \
+        --config $BUILD_CONFIG \
+        --android_sdk_path /android_home \
+        --android_ndk_path /ndk_home \
+        /home/onnxruntimedev/.build_settings/build_settings.json
+fi
 
 # Copy the built artifacts to give folder for publishing
 PACKAGE_NAME=onnxruntime-mobile

diff --git a/tools/ci_build/github/android/default_full_aar_build_settings.json b/tools/ci_build/github/android/default_full_aar_build_settings.json
@@ -0,0 +1,19 @@
+{
+    "build_abis": [
+        "armeabi-v7a",
+        "arm64-v8a",
+        "x86",
+        "x86_64"
+    ],
+    "android_min_sdk_version": 21,
+    "android_target_sdk_version": 24,
+    "build_params": [
+        "--android",
+        "--parallel",
+        "--cmake_generator=Ninja",
+        "--build_java",
+        "--build_shared_lib",
+        "--use_nnapi",
+        "--skip_tests"
+    ]
+}
diff --git a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml
@@ -0,0 +1,71 @@
+parameters:
+- name: artifactName
+  displayName: Artifact Name
+  type: string
+  default: ''
+
+- name: job_name_suffix
+  displayName: job name
+  type: string
+  default: ''
+
+jobs:
+- job: Final_AAR_Testing_Android_${{ parameters.job_name_suffix }}
+  workspace:
+    clean: all
+  pool:
+    vmImage: 'macOS-10.15'
+  variables:
+  - name: runCodesignValidationInjection
+    value: false
+  timeoutInMinutes: 60
+  dependsOn:
+    - Android_Java_API_AAR_Packaging_${{ parameters.job_name_suffix }}
+  steps:
+  - template: set-version-number-variables-step.yml
+
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Final Android AAR'
+    inputs:
+      buildType: 'current'
+      artifactName: '${{parameters.artifactName}}'
+      targetPath: '$(Build.BinariesDirectory)/final-android-aar'
+
+  - script: /bin/bash tools/ci_build/github/android/setup_gradle_wrapper.sh $(Build.SourcesDirectory)
+    displayName: Setup gradle wrapper to use gradle 6.8.3
+
+  - script: |
+      python3 $(Build.SourcesDirectory)/tools/python/run_android_emulator.py \
+        --android-sdk-root ${ANDROID_SDK_ROOT} \
+        --create-avd --system-image "system-images;android-30;google_apis;x86_64" \
+        --start --emulator-extra-args="-partition-size 4096" \
+        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
+    displayName: Start Android emulator
+
+  - script: |
+      set -e -x
+      mkdir android_test
+      cd android_test
+      cp -av $(Build.SourcesDirectory)/java/src/test/android ./
+      cd ./android
+      mkdir -p app/libs
+      cp $(Build.BinariesDirectory)/final-android-aar/onnxruntime-mobile-$(OnnxRuntimeVersion).aar app/libs/onnxruntime-mobile.aar
+      $(Build.SourcesDirectory)/java/gradlew --no-daemon clean connectedDebugAndroidTest
+    displayName: Run E2E test for Android
+    workingDirectory: $(Build.BinariesDirectory)
+
+  - script: |
+      python3 $(Build.SourcesDirectory)/tools/python/run_android_emulator.py \
+        --android-sdk-root ${ANDROID_SDK_ROOT} \
+        --stop \
+        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
+    displayName: Stop Android emulator
+    condition: always()
+
+  - template: component-governance-component-detection-steps.yml
+    parameters :
+      condition : 'succeeded'
+
+  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    displayName: 'Clean Agent Directories'
+    condition: always()
diff --git a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml
@@ -4,8 +4,28 @@ parameters:
   type: string
   default: 'Release'
 
+- name: buildSettings
+  displayName: Path to the build setting file
+  type: string
+  default: ''
+
+- name: includedOpsConfig
+  displayName: Path to the included ops config file
+  type: string
+  default: ''
+
+- name: artifactName
+  displayName: Artifact Name
+  type: string
+  default: ''
+
+- name: job_name_suffix
+  displayName: job name
+  type: string
+  default: ''
+
 jobs:
-- job: Android_Java_API_AAR_Packaging_Pipeline
+- job: Android_Java_API_AAR_Packaging_${{ parameters.job_name_suffix }}
   timeoutInMinutes: 120
   workspace:
     clean: all
@@ -39,13 +59,19 @@ jobs:
     displayName: Build Android AAR Packages
     inputs:
       script: |
+        set -e -x
         NDK_HOME=$(realpath $ANDROID_NDK_HOME)
+        mkdir $(Build.BinariesDirectory)/.build_settings
+        cp ${{parameters.buildSettings}} $(Build.BinariesDirectory)/.build_settings/build_settings.json
+        [ -f "${{parameters.includedOpsConfig}}" ] && \
+          cp ${{parameters.includedOpsConfig}} $(Build.BinariesDirectory)/.build_settings/include_ops_and_types.config
         docker run --rm \
           --volume $(Build.SourcesDirectory):/onnxruntime_src \
           --volume $(Build.BinariesDirectory):/build \
           --volume $ANDROID_HOME:/android_home \
           --volume $NDK_HOME:/ndk_home \
           --volume $(artifacts_directory):/home/onnxruntimedev/.artifacts \
+          --volume $(Build.BinariesDirectory)/.build_settings:/home/onnxruntimedev/.build_settings \
           -e NIGHTLY_BUILD \
           -e BUILD_BUILDNUMBER \
           -e BUILD_CONFIG=${{parameters.buildConfig}} \
@@ -57,4 +83,4 @@ jobs:
   - task: PublishBuildArtifacts@1
     inputs:
       pathtoPublish: '$(artifacts_directory)'
-      artifactName: 'onnxruntime-android-aar'
+      artifactName: '${{parameters.artifactName}}'
diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml
@@ -171,6 +171,17 @@ jobs:
 - template: android-java-api-aar.yml
   parameters:
     buildConfig: 'Release'
+    buildSettings: '$(Build.SourcesDirectory)/tools/ci_build/github/android/default_mobile_aar_build_settings.json'
+    includedOpsConfig: '$(Build.SourcesDirectory)/tools/ci_build/github/android/mobile_package.required_operators.config'
+    artifactName: 'onnxruntime-android-mobile-aar'
+    job_name_suffix: 'Mobile'
+
+- template: android-java-api-aar.yml
+  parameters:
+    buildConfig: 'Release'
+    buildSettings: '$(Build.SourcesDirectory)/tools/ci_build/github/android/default_full_aar_build_settings.json'
+    artifactName: 'onnxruntime-android-full-aar'
+    job_name_suffix: 'Full'
 
 - template: win-ci.yml
   parameters:
@@ -405,7 +416,7 @@ jobs:
       configuration: RelWithDebInfo
       arguments: '--configuration RelWithDebInfo -t:CreatePackage -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
       workingDirectory: '$(Build.SourcesDirectory)\csharp'
-  
+
   - task: CopyFiles@2
     displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)'
     inputs:
@@ -535,23 +546,23 @@ jobs:
           echo changing directory to artifact download path
           cd $(Build.BinariesDirectory)/nuget-artifact/final-package
           echo processing nupkg
-          SETLOCAL EnableDelayedExpansion   
+          SETLOCAL EnableDelayedExpansion
           FOR /R %%i IN (*.nupkg) do (
            set filename=%%~ni
            IF NOT "!filename:~25,7!"=="Managed" (
             echo processing %%~ni.nupkg
             copy %%~ni.nupkg %%~ni.zip
             echo copied to zip
-            echo listing lib files in the zip 
-            REM use a single .csv file to put the data 
+            echo listing lib files in the zip
+            REM use a single .csv file to put the data
             echo os,arch,build_config,size > $(Build.BinariesDirectory)\binary_size_data.txt
             7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a  in ('more') do if not "%%a" == "" echo linux,x64,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt
             7z.exe l -slt %%~ni.zip runtimes\linux-aarch64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a  in ('more') do if not "%%a" == "" echo linux,aarch64,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt
             7z.exe l -slt %%~ni.zip runtimes\osx.10.14-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a  in ('more') do if not "%%a" == "" echo osx,x64,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt
             7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a  in ('more') do if not "%%a" == "" echo win,x64,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt
-            7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a  in ('more') do if not "%%a" == "" echo win,x86,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt            
+            7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a  in ('more') do if not "%%a" == "" echo win,x86,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt
              )
-           ) 
+           )
   - task: AzureCLI@2
     displayName: 'Azure CLI'
     inputs:
@@ -749,61 +760,9 @@ jobs:
     displayName: 'Clean Agent Directories'
     condition: always()
 
-- job: Final_AAR_Testing_Android
-  workspace:
-    clean: all
-  pool:
-    vmImage: 'macOS-10.15'
-  variables:
-  - name: runCodesignValidationInjection
-    value: false
-  timeoutInMinutes: 60
-  dependsOn:
-    - Android_Java_API_AAR_Packaging_Pipeline
-  steps:
-  - template: set-version-number-variables-step.yml
-
-  - task: DownloadPipelineArtifact@2
-    displayName: 'Download Final Android AAR'
-    inputs:
-      buildType: 'current'
-      artifactName: 'onnxruntime-android-aar'
-      targetPath: '$(Build.BinariesDirectory)/final-android-aar'
-
-  - script: /bin/bash tools/ci_build/github/android/setup_gradle_wrapper.sh $(Build.SourcesDirectory)
-    displayName: Setup gradle wrapper to use gradle 6.8.3
-
-  - script: |
-      python3 $(Build.SourcesDirectory)/tools/python/run_android_emulator.py \
-        --android-sdk-root ${ANDROID_SDK_ROOT} \
-        --create-avd --system-image "system-images;android-30;google_apis;x86_64" \
-        --start --emulator-extra-args="-partition-size 4096" \
-        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
-    displayName: Start Android emulator
+- template: android-java-api-aar-test.yml
+  parameters:
+    artifactName: 'onnxruntime-android-mobile-aar'
+    job_name_suffix: 'Mobile'
 
-  - script: |
-      set -e -x
-      mkdir android_test
-      cd android_test
-      cp -av $(Build.SourcesDirectory)/java/src/test/android ./
-      cd ./android
-      mkdir -p app/libs
-      cp $(Build.BinariesDirectory)/final-android-aar/onnxruntime-mobile-$(OnnxRuntimeVersion).aar app/libs/onnxruntime-mobile.aar
-      $(Build.SourcesDirectory)/java/gradlew --no-daemon clean connectedDebugAndroidTest
-    displayName: Run E2E test for Android
-    workingDirectory: $(Build.BinariesDirectory)
 
-  - script: |
-      python3 $(Build.SourcesDirectory)/tools/python/run_android_emulator.py \
-        --android-sdk-root ${ANDROID_SDK_ROOT} \
-        --stop \
-        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
-    displayName: Stop Android emulator
-    condition: always()
-
-  - template: component-governance-component-detection-steps.yml
-    parameters :
-      condition : 'succeeded'
-  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
-    displayName: 'Clean Agent Directories'
-    condition: always()