_linux_ut.yml

name: Linux UT Test

on:
  workflow_call:
    inputs:
      pytorch:
        required: false
        type: string
        default: 'main'
        description: Pytorch branch/commit
      keep_torch_xpu_ops:
        required: false
        type: string
        default: 'false'
        description: Keep torch-xpu-ops pin. `true` means use pined commit
      triton:
        required: false
        type: string
        default: ''
        description: Triton commit. Use pytorch pined commit by default
      ut:
        required: true
        type: string
        default: ''
        description: UT scope. `op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu` Delimiter is comma
      abi:
        required: false
        type: string
        default: 1
        description: ABI version. Default abi as 1.
      python:
        required: false
        type: string
        default: '3.10'
        description: Python version
      runner:
        required: true
        type: string
        default: 'linux.idc.xpu'
        description: Runner label
      driver:
        required: false
        type: string
        default: 'lts'
        description: Driver lts/rolling

permissions: read-all

jobs:
  ut_test:
    runs-on: ${{ inputs.runner }} 
    if: ${{ inputs.ut != 'xpu_distributed' }}
    timeout-minutes: 900
    env:
      NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }}
      DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }}
    steps:
      - name: Checkout torch-xpu-ops
        uses: actions/checkout@v4
      - name: Prepare Stock Pytorch
        run: |
          pwd
          which conda && conda clean -ay
          conda remove --all -y -n xpu_op_${ZE_AFFINITY_MASK} || \
                rm -rf $(dirname ${CONDA_EXE})/../envs/xpu_op_${ZE_AFFINITY_MASK}
          conda create -n xpu_op_${ZE_AFFINITY_MASK} python=${{ inputs.python }} cmake ninja -y
          source activate xpu_op_${ZE_AFFINITY_MASK}
          cd ../ && rm -rf pytorch
          pip install requests
          git clone https://github.com/pytorch/pytorch pytorch
          if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then
            cd pytorch && git checkout $(echo ${{ inputs.pytorch }})
            # apply PRs for stock pytorch
            python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py
            git status && git show -s
            git submodule sync && git submodule update --init --recursive
            if [[ ${{ inputs.keep_torch_xpu_ops }} == 'true' ]]; then
              echo "Don't replace torch-xpu-ops!"
            else
              rm -rf third_party/torch-xpu-ops && cp -r ../torch-xpu-ops third_party/
              # Workaround for torch-xpu-ops ci test
              sed -i "s/checkout --quiet \${TORCH_XPU_OPS_COMMIT}/log -n 1/g" caffe2/CMakeLists.txt
            fi
          fi
      - name: Triton Installation
        run: |
          source activate xpu_op_${ZE_AFFINITY_MASK}
          cd ../pytorch
          TRITON_REPO="https://github.com/intel/intel-xpu-backend-for-triton"
          if [ -z ${{ inputs.triton }} ]; then
            TRITON_COMMIT_ID="$(<.ci/docker/ci_commit_pins/triton-xpu.txt)"
          else
            TRITON_COMMIT_ID="${{ inputs.triton }}"
          fi
          echo ${TRITON_REPO}@${TRITON_COMMIT_ID}
          if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then
            pip install --force-reinstall "git+${TRITON_REPO}@${TRITON_COMMIT_ID}#subdirectory=python"
          fi
      - name: Download Pytorch wheel
        if: ${{ inputs.pytorch != 'nightly_wheel' }}
        uses: actions/download-artifact@v4
        with:
          name: Torch-XPU-Wheel-${{ github.event.pull_request.number || github.sha }}-${{ inputs.abi }}
          path: ${{ github.workspace }}
      - name: Install Pytorch XPU
        run: |
          source activate xpu_op_${ZE_AFFINITY_MASK}
          source .github/scripts/env.sh ${{ inputs.pytorch }}
          pip install mkl-static==2025.0.1 mkl-include==2025.0.1
          if [[ ${{ inputs.abi }} == '0' ]]; then
            export _GLIBCXX_USE_CXX11_ABI=0
          else
            export _GLIBCXX_USE_CXX11_ABI=1
          fi
          if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then
            cd ../pytorch
            export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
            pip install -r requirements.txt
            pip install --force-reinstall ${{ github.workspace }}/torch*.whl
            git clone https://github.com/pytorch/vision && cd vision && python setup.py install && cd ..
          else
            pip install torch torchvision torchaudio --pre --index-url https://download.pytorch.org/whl/nightly/xpu
            TORCH_COMMIT_ID=$(python -c 'import torch; print(torch.version.git_version)')
            cd ../pytorch
            git reset --hard && git checkout ${TORCH_COMMIT_ID}
            TORCH_XPU_OPS_COMMIT=$(<third_party/xpu.txt)
            rm -rf third_party/torch-xpu-ops
            git clone https://github.com/intel/torch-xpu-ops.git third_party/torch-xpu-ops
            cd third_party/torch-xpu-ops
            git checkout ${TORCH_XPU_OPS_COMMIT}
            cd ../..
            python third_party/torch-xpu-ops/.github/scripts/apply_torch_pr.py
          fi
          pip install -r .ci/docker/requirements-ci.txt
      - name: Torch Config
        run: |
          source activate xpu_op_${ZE_AFFINITY_MASK}
          source .github/scripts/env.sh ${{ inputs.pytorch }}
          python -c "import torch; print(torch.__config__.show())"
          python -c "import torch; print(torch.__config__.parallel_info())"
          python -c "import torch; print(torch.__config__.torch.xpu.device_count())"
          python -c "import triton; print(triton.__version__)"

          cd ..
          python pytorch/torch/utils/collect_env.py
          rm -rf /tmp/torchinductor_*
          rm -rf ~/.triton/cache
      - name: Run XPU OP Regression
        if: contains(inputs.ut, 'op_regression') || github.event_name == 'schedule'
        run: |
          cd ${{ github.workspace }}
          mkdir -p ut_log/op_regression
          xpu-smi discovery
          source .github/scripts/env.sh ${{ inputs.pytorch }}
          source activate xpu_op_${ZE_AFFINITY_MASK}
          cd ${{ github.workspace }}
          cd test/regressions
          pip install pytest
          timeout 8000 pytest -v --junit-xml=${{ github.workspace }}/ut_log/op_regression.xml 2>${{ github.workspace }}/ut_log/op_regression/op_regression_test_error.log | tee ${{ github.workspace }}/ut_log/op_regression/op_regression_test.log
      - name: Run XPU OP Regressions test on device 1
        if: contains(inputs.ut, 'op_regression_dev1') || github.event_name == 'schedule'
        run: |
          cd ${{ github.workspace }}
          mkdir -p ut_log/op_regression_dev1
          xpu-smi discovery
          source .github/scripts/env.sh ${{ inputs.pytorch }}
          source activate xpu_op_${ZE_AFFINITY_MASK}
          export ZE_AFFINITY_MASK_OLD=${ZE_AFFINITY_MASK}
          unset ZE_AFFINITY_MASK
          cd ${{ github.workspace }}
          cd test/regressions
          pip install pytest
          timeout 8000 pytest -v --junit-xml=${{ github.workspace }}/ut_log/op_regression_dev1.xml test_operation_on_device_1.py 2>${{ github.workspace }}/ut_log/op_regression_dev1/op_regression_dev1_test_error.log | tee ${{ github.workspace }}/ut_log/op_regression_dev1/op_regression_dev1_test.log
          export ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK_OLD}
      - name: Run XPU OP Extended UT
        if: contains(inputs.ut, 'op_extended') || github.event_name == 'schedule'
        run: |
          source .github/scripts/env.sh ${{ inputs.pytorch }}
          source activate xpu_op_${ZE_AFFINITY_MASK}
          export PYTORCH_TEST_WITH_SLOW=1
          cd ${{ github.workspace }}
          mkdir -p ut_log/op_extended
          cd ../pytorch/third_party/torch-xpu-ops/test/xpu/extended/
          timeout 10000 python run_test_with_skip.py 2>${{ github.workspace }}/ut_log/op_extended/op_extended_test_error.log | tee ${{ github.workspace }}/ut_log/op_extended/op_extended_test.log
          cp op_extended.xml ${{ github.workspace }}/ut_log
      - name: Run XPU OP UT
        if: contains(inputs.ut, 'op_ut') || github.event_name == 'schedule'
        run: |
          source .github/scripts/env.sh ${{ inputs.pytorch }}
          source activate xpu_op_${ZE_AFFINITY_MASK}
          export PYTORCH_ENABLE_XPU_FALLBACK=1
          export PYTORCH_TEST_WITH_SLOW=1
          cd ${{ github.workspace }}
          mkdir -p ut_log/op_ut
          cd ../pytorch/third_party/torch-xpu-ops/test/xpu
          timeout 10000 python run_test_with_skip.py 2>${{ github.workspace }}/ut_log/op_ut/op_ut_with_skip_test_error.log | tee ${{ github.workspace }}/ut_log/op_ut/op_ut_with_skip_test.log
          cp *.xml ${{ github.workspace }}/ut_log
          # Cases run with a on-demand white list, since some suites are too
          # slow to go through all operators on CPU. So add cases on-demand
          # when XPU implementatoin is done.
          # test_foreach, test_decomp
          timeout 10000 python run_test_with_only.py 2>${{ github.workspace }}/ut_log/op_ut/op_ut_with_only_test_error.log | tee ${{ github.workspace }}/ut_log/op_ut/op_ut_with_only_test.log
          cp op_ut_with_only.xml ${{ github.workspace }}/ut_log
      - name: Run Torch XPU UT
        if: contains(inputs.ut, 'torch_xpu') || github.event_name == 'schedule'
        run: |
          source .github/scripts/env.sh ${{ inputs.pytorch }}
          source activate xpu_op_${ZE_AFFINITY_MASK}
          cd ${{ github.workspace }}
          mkdir -p ut_log/torch_xpu
          cd ../pytorch
          TEST_REPORTS_DIR=$(pwd)/test/test-reports
          rm -rf "$TEST_REPORTS_DIR" && mkdir -p "$TEST_REPORTS_DIR"
          # Run Pytorch XPU binary UT
          for xpu_case in build/bin/*{xpu,sycl}*; do
            if [[ "$xpu_case" != *"*"* && "$xpu_case" != *.so && "$xpu_case" != *.a ]]; then
              case_name=$(basename "$xpu_case")
              echo "Testing ${case_name} ..."
              "$xpu_case" --gtest_output=xml:"$TEST_REPORTS_DIR"/"$case_name".xml 2>${{ github.workspace }}/ut_log/torch_xpu/binary_ut_torch_xpu_${case_name}_test_error.log | tee ${{ github.workspace }}/ut_log/torch_xpu/binary_ut_torch_xpu_${case_name}_test.log
            fi
          done
          # Run Pytorch XPU python UT
          export PYTORCH_TEST_WITH_SLOW=1
          export PYTORCH_TESTING_DEVICE_ONLY_FOR="xpu"

          test_cmd="python test/run_test.py --include "
          # All Inductor UT under test/inductor
          for test in $(ls test/inductor | grep test);
          do 
              test_cmd="${test_cmd} inductor/$test";
          done
          # All xpu ut under test/xpu
          for test in $(ls test/xpu | grep test);
          do 
              test_cmd="${test_cmd} xpu/$test";
          done
          if [ -f "test/test_xpu.py" ]; then
            test_cmd="${test_cmd} test_xpu.py"
          fi
          eval $test_cmd 2>${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_test_error.log | tee ${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_test.log
      - name: Run Torch XPU Profile UT
        if: ${{ inputs.driver == 'rolling' && contains(inputs.ut, 'xpu_profiling') }} 
        run: |
          source .github/scripts/env.sh ${{ inputs.pytorch }}
          source activate xpu_op_${ZE_AFFINITY_MASK}
          mkdir -p ${{ github.workspace }}/ut_log/profile_test
          # RN50 Test
          PROFILE=1 python -u test/profiling/rn50.py -a resnet50 --dummy ./ --num-iterations 20 --xpu 0
          cp profiling.fp32.train.pt ${{ github.workspace }}/ut_log/profile_test

          # All Issue Reproduce UT
          mkdir -p ${{ github.workspace }}/ut_log/profile_test/issue_reproduce
          python -u test/profiling/correlation_id_mixed.py | tee ${{ github.workspace }}/ut_log/profile_test/issue_reproduce/correlation_id_mixed.log
          python -u test/profiling/reproducer.missing.gpu.kernel.time.py | tee ${{ github.workspace }}/ut_log/profile_test/issue_reproduce/reproducer.missing.gpu.kernel.time.log
          python -u test/profiling/time_precision_in_profile.py | tee ${{ github.workspace }}/ut_log/profile_test/issue_reproduce/time_precision_in_profile.log
          python -u test/profiling/profile_partial_runtime_ops.py | tee ${{ github.workspace }}/ut_log/profile_test/issue_reproduce/profile_partial_runtime_ops.log
          python -u test/profiling/triton_xpu_ops_time.py | tee ${{ github.workspace }}/ut_log/profile_test/issue_reproduce/triton_xpu_ops_time.log

          # All xpu ut under test/profiler
          cd ../pytorch/test/profiler
          python -m pytest -vs test_cpp_thread.py | tee ${{ github.workspace }}/ut_log/profile_test/test_cpp_thread.log
          python -m pytest -vs test_execution_trace.py | tee ${{ github.workspace }}/ut_log/profile_test/test_execution_trace.log
          python -m pytest -vs test_memory_profiler.py | tee ${{ github.workspace }}/ut_log/profile_test/test_memory_profiler.log
          python -m pytest -vs test_profiler_tree.py | tee ${{ github.workspace }}/ut_log/profile_test/test_profiler_tree.log
      - name: UT Test Results Summary
        run: |
          source activate xpu_op_${ZE_AFFINITY_MASK}
          pip install junitparser
          python .github/scripts/check-ut.py ${{ github.workspace }}/ut_log/*.xml >> $GITHUB_STEP_SUMMARY || true
      - name: UT Test Results Check
        shell: bash
        run: |
          function contains() {
              contains_status="echo 'Start $2 ...'"
              {
                [[ $1 =~ (^|,)$2($|,) ]]
              } || {
                echo "[Warning] $2 is not suppotted type! Skipped!"
                contains_status="continue"
              }
          }
          set -xe
          echo "UT_NAME=$(echo ${{ inputs.ut }} |sed 's/,/-/g')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
          for ut_suite in $(echo ${{ inputs.ut }} |sed 's/,/ /g')
          do
            contains "op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu" $ut_suite
            $contains_status
            cd ${{ github.workspace }}/ut_log/${ut_suite}
            cp ${{ github.workspace }}/.github/scripts/ut_result_check.sh ./
            bash ut_result_check.sh ${ut_suite}
          done
      - name: Upload Inductor XPU UT Log
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ inputs.abi }}-${{ env.UT_NAME }}
          path: ${{ github.workspace }}/ut_log
  
  distributed_ut_test:
    runs-on: pvc_e2e
    if: contains(inputs.ut, 'xpu_distributed')
    timeout-minutes: 900
    env:
      NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }}
      DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }}
    steps:
      - name: Checkout torch-xpu-ops
        uses: actions/checkout@v4
      - name: Prepare Stock Pytorch
        run: |
          pwd
          which conda && conda clean -ay
          conda remove --all -y -n xpu_op_${ZE_AFFINITY_MASK} || \
                rm -rf $(dirname ${CONDA_EXE})/../envs/xpu_op_${ZE_AFFINITY_MASK}
          conda create -n xpu_op_${ZE_AFFINITY_MASK} python=${{ inputs.python }} cmake ninja -y
          source activate xpu_op_${ZE_AFFINITY_MASK}
          cd ../ && rm -rf pytorch
          pip install requests
          git clone https://github.com/pytorch/pytorch pytorch
          if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then
            cd pytorch && git checkout $(echo ${{ inputs.pytorch }})
            # apply PRs for stock pytorch
            python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py
            git status && git show -s
            git submodule sync && git submodule update --init --recursive
            if [[ ${{ inputs.keep_torch_xpu_ops }} == 'true' ]]; then
              echo "Don't replace torch-xpu-ops!"
            else
              rm -rf third_party/torch-xpu-ops && cp -r ../torch-xpu-ops third_party/
              # Workaround for torch-xpu-ops ci test
              sed -i "s/checkout --quiet \${TORCH_XPU_OPS_COMMIT}/log -n 1/g" caffe2/CMakeLists.txt
            fi
          fi
      - name: Triton Installation
        run: |
          source activate xpu_op_${ZE_AFFINITY_MASK}
          cd ../pytorch
          TRITON_REPO="https://github.com/intel/intel-xpu-backend-for-triton"
          if [ -z ${{ inputs.triton }} ]; then
            TRITON_COMMIT_ID="$(<.ci/docker/ci_commit_pins/triton-xpu.txt)"
          else
            TRITON_COMMIT_ID="${{ inputs.triton }}"
          fi
          echo ${TRITON_REPO}@${TRITON_COMMIT_ID}
          if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then
            pip install --force-reinstall "git+${TRITON_REPO}@${TRITON_COMMIT_ID}#subdirectory=python"
          fi
      - name: Download Pytorch wheel
        if: ${{ inputs.pytorch != 'nightly_wheel' }}
        uses: actions/download-artifact@v4
        with:
          name: Torch-XPU-Wheel-${{ github.event.pull_request.number || github.sha }}-${{ inputs.abi }}
          path: ${{ github.workspace }}
      - name: Install Pytorch XPU
        run: |
          source activate xpu_op_${ZE_AFFINITY_MASK}
          source .github/scripts/env.sh ${{ inputs.pytorch }}
          pip install mkl-static==2025.0.1 mkl-include==2025.0.1
          if [[ ${{ inputs.abi }} == '0' ]]; then
            export _GLIBCXX_USE_CXX11_ABI=0
          else
            export _GLIBCXX_USE_CXX11_ABI=1
          fi
          if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then
            cd ../pytorch
            export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
            pip install -r requirements.txt
            pip install --force-reinstall ${{ github.workspace }}/torch*.whl
            git clone https://github.com/pytorch/vision && cd vision && python setup.py install && cd ..
          else
            pip install torch torchvision torchaudio --pre --index-url https://download.pytorch.org/whl/nightly/xpu
            TORCH_COMMIT_ID=$(python -c 'import torch; print(torch.version.git_version)')
            cd ../pytorch
            git reset --hard && git checkout ${TORCH_COMMIT_ID}
            TORCH_XPU_OPS_COMMIT=$(<third_party/xpu.txt)
            rm -rf third_party/torch-xpu-ops
            git clone https://github.com/intel/torch-xpu-ops.git third_party/torch-xpu-ops
            cd third_party/torch-xpu-ops
            git checkout ${TORCH_XPU_OPS_COMMIT}
            cd ../..
            python third_party/torch-xpu-ops/.github/scripts/apply_torch_pr.py
          fi
          pip install -r .ci/docker/requirements-ci.txt
      - name: Torch Config
        run: |
          source activate xpu_op_${ZE_AFFINITY_MASK}
          source .github/scripts/env.sh ${{ inputs.pytorch }}
          python -c "import torch; print(torch.__config__.show())"
          python -c "import torch; print(torch.__config__.parallel_info())"
          python -c "import torch; print(torch.__config__.torch.xpu.device_count())"
          python -c "import triton; print(triton.__version__)"

          cd ..
          python pytorch/torch/utils/collect_env.py
          rm -rf /tmp/torchinductor_*
          rm -rf ~/.triton/cache
      - name: Run Torch XPU Distributed UT
        run: |
          source .github/scripts/env.sh ${{ inputs.pytorch }}
          source activate xpu_op_${ZE_AFFINITY_MASK}
          pip install pytest
          cd ${{ github.workspace }}
          sudo cp /proc/sys/kernel/yama/ptrace_scope ptrace_scope.bk
          sudo echo "0"|sudo tee /proc/sys/kernel/yama/ptrace_scope
          mkdir -p ut_log/xpu_distributed
          cd ../pytorch/third_party/torch-xpu-ops/test/xpu
          python -c "import torch;print(torch.distributed.is_xccl_available())"
          timeout 10000 python run_distributed.py 2>${{ github.workspace }}/ut_log/xpu_distributed/xpu_distributed_test_error.log | tee ${{ github.workspace }}/ut_log/xpu_distributed/xpu_distributed_test.log
          cd ${{ github.workspace }}
          sudo cp ptrace_scope.bk /proc/sys/kernel/yama/ptrace_scope
      - name: UT Test Results Check
        shell: bash
        run: |
          function contains() {
              contains_status="echo 'Start $2 ...'"
              {
                [[ $1 =~ (^|,)$2($|,) ]]
              } || {
                echo "[Warning] $2 is not suppotted type! Skipped!"
                contains_status="continue"
              }
          }
          set -xe
          echo "UT_NAME=$(echo ${{ inputs.ut }} |sed 's/,/-/g')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
          cd ${{ github.workspace }}/ut_log/xpu_distributed
          cp ${{ github.workspace }}/.github/scripts/ut_result_check.sh ./
          bash ut_result_check.sh 'xpu_distributed'
      - name: Upload Inductor XPU UT Log
        if: ${{ ! cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ inputs.abi }}-xpu_distributed
          path: ${{ github.workspace }}/ut_log