diff --git a/.github/actions/inductor-xpu-e2e-test/action.yml b/.github/actions/inductor-xpu-e2e-test/action.yml index 52ec6c3b0..88551837e 100644 --- a/.github/actions/inductor-xpu-e2e-test/action.yml +++ b/.github/actions/inductor-xpu-e2e-test/action.yml @@ -111,6 +111,9 @@ runs: set -xe for suite in $(echo ${{ inputs.suite }} |sed 's/,/ /g') do + if [ "${suite}" == "pt2e" ];then + continue + fi contains "huggingface,timm_models,torchbench" $suite $contains_status for dt in $(echo ${{ inputs.dt }} |sed 's/,/ /g') diff --git a/.github/actions/pt2e/action.yml b/.github/actions/pt2e/action.yml new file mode 100644 index 000000000..a01c7bc91 --- /dev/null +++ b/.github/actions/pt2e/action.yml @@ -0,0 +1,147 @@ +name: inductor-xpu-pt2e-test + +inputs: + suite: + required: true + type: string + default: 'huggingface' + description: Dynamo benchmarks test suite. huggingface,timm_models,torchbench. Delimiter is comma + env_prepare: + required: false + description: If set to any value, will prepare suite test env + dt: + required: true + type: string + default: 'float32' + description: Data precision of the test.float32,int8. Delimiter is comma + mode: + required: true + type: string + default: 'inference' + description: inference. Delimiter is comma + scenario: + required: true + type: string + default: 'accuracy' + description: accuracy,performance. Delimiter is comma + cards: + required: false + type: string + default: 'all' + description: which cards can be used in the test + hf_token: + required: false + description: HUGGING_FACE_HUB_TOKEN for torchbench test + pytorch: + required: false + type: string + default: 'main' + description: Pytorch branch/commit + driver: + required: false + type: string + default: 'lts' + description: Driver lts/rolling + +runs: + using: composite + steps: + - name: Prepare ENV + if: ${{ inputs.env_prepare }} + shell: bash + run: | + source activate e2e_ci + source .github/scripts/env.sh ${{ inputs.pytorch }} + # accuracy code + if [[ "${{ inputs.scenario }}" == *"accuracy"* ]];then + rm -rf pt2e-accuracy + git clone -b yifeng/accuracy https://github.com/chuanqi129/inductor-tools pt2e-accuracy + fi + # performance code + if [[ "${{ inputs.scenario }}" == *"performance"* ]];then + rm -rf pt2e-performance + git clone -b yifeng/pt2e_xpu https://github.com/zxd1997066/benchmark pt2e-performance + fi + # deps + if [[ ${{ inputs.scenario }} == *"performance"* ]]; then + pip install pyyaml botocore + if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then + rm -rf pt2e-audio + git clone --single-branch -b main https://github.com/pytorch/audio pt2e-audio + cd pt2e-audio && git checkout $TORCHAUDIO_COMMIT_ID + python setup.py bdist_wheel && pip uninstall torchaudio -y && pip install --no-deps dist/*.whl + cd ../ + rm -rf pt2e-vision + git clone --single-branch -b main https://github.com/pytorch/vision pt2e-vision + cd pt2e-vision && git checkout $TORCHVISION_COMMIT_ID + python setup.py bdist_wheel && pip uninstall torchvision -y && pip install --no-deps dist/*.whl + cd ../ + fi + # torchbench + python -c "import torch, torchvision, torchaudio" + rm -rf pt2e-benchmark + git clone https://github.com/pytorch/benchmark pt2e-benchmark + cd pt2e-benchmark && git checkout $TORCHBENCH_COMMIT_ID && pip install --no-deps -r requirements.txt + pip install -U transformers tokenizers safetensors + python install.py --continue_on_fail + cd ../ + # deps for torchrec_dlrm + pip install pyre_extensions + pip install fbgemm-gpu + pip install --no-deps torchmetrics==1.0.3 torchrec + # transformers + pip install --force-reinstall git+https://github.com/huggingface/transformers@${TRANSFORMERS_VERSION} + # timm + pip install --no-deps git+https://github.com/huggingface/pytorch-image-models@$TIMM_COMMIT_ID + pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/$TIMM_COMMIT_ID/requirements.txt | grep -vE torch) + fi + pip install numpy==1.26.4 + # dataset + if [ ! -d ${HOME}/datasets/imagenet ];then + rm -rf ${HOME}/datasets/imagenet + mkdir -p ${HOME}/datasets/imagenet + cd ${HOME}/datasets/imagenet + wget https://image-net.org/data/ILSVRC/2012/ILSVRC2012_img_val.tar + tar -xf ILSVRC2012_img_val.tar + wget -O valprep.sh https://raw.githubusercontent.com/soumith/imagenetloader.torch/master/valprep.sh + bash valprep.sh + fi + - name: PT2E Test (${{ inputs.suite }} ${{ inputs.dt }} ${{ inputs.mode }} ${{ inputs.scenario }}) + env: + HUGGING_FACE_HUB_TOKEN: ${{ inputs.hf_token }} + NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }} + DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }} + shell: bash + run: | + source activate e2e_ci + source .github/scripts/env.sh ${{ inputs.pytorch }} + pt2e_logs_dir="${{ github.workspace }}/../pytorch/inductor_log" + rm -rf "${pt2e_logs_dir}" && mkdir -p "${pt2e_logs_dir}" + if [[ "${{ inputs.scenario }}" == *"accuracy"* ]];then + if [[ "${{ inputs.dt }}" == *"float32"* ]];then + python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --is_fp32 --dataset_dir ${HOME}/datasets/imagenet |\ + tee "${pt2e_logs_dir}/accuracy-fp32.log" + fi + if [[ "${{ inputs.dt }}" == *"int8"* ]];then + python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --dataset_dir ${HOME}/datasets/imagenet |\ + tee "${pt2e_logs_dir}/accuracy-int8.log" + fi + fi + if [[ "${{ inputs.scenario }}" == *"performance"* ]];then + models="alexnet,demucs,dlrm,hf_Albert,hf_Bert,hf_Bert_large,hf_DistilBert,hf_Roberta_base,mnasnet1_0,mobilenet_v2," + models+="mobilenet_v3_large,nvidia_deeprecommender,pytorch_CycleGAN_and_pix2pix,resnet152,resnet18,resnet50,resnext50_32x4d," + models+="shufflenet_v2_x1_0,squeezenet1_1,Super_SloMo,timm_efficientnet,timm_nfnet,timm_regnet,timm_resnest," + models+="timm_vision_transformer,timm_vision_transformer_large,timm_vovnet,vgg16" + if [[ "${{ inputs.dt }}" == *"float32"* ]];then + rm -rf pt2e-performance/.userbenchmark + python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs --torchdynamo inductor -m $models 2>&1 |\ + tee "${pt2e_logs_dir}/performance-fp32.log" + mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-fp32 + fi + if [[ "${{ inputs.dt }}" == *"float32"* ]];then + rm -rf pt2e-performance/.userbenchmark + python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs --torchdynamo inductor --quantization pt2e -m $models 2>&1 |\ + tee "${pt2e_logs_dir}/performance-int8.log" + mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-int8 + fi + fi diff --git a/.github/workflows/nightly_ondemand.yml b/.github/workflows/nightly_ondemand.yml index 26da75ff3..ed1e20618 100644 --- a/.github/workflows/nightly_ondemand.yml +++ b/.github/workflows/nightly_ondemand.yml @@ -32,7 +32,7 @@ on: required: true type: string default: 'huggingface' - description: Dynamo benchmarks test suite. `huggingface,timm_models,torchbench`. Delimiter is comma + description: Dynamo benchmarks test suite. `huggingface,timm_models,torchbench,pt2e`. Delimiter is comma dt: required: true type: string @@ -233,6 +233,16 @@ jobs: scenario: accuracy env_prepare: true hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + - name: Nightly PT2E Accuracy Test + if: github.event_name == 'schedule' && github.event.schedule == '0 13 * * 0-4' + uses: ./.github/actions/pt2e + with: + suite: pt2e + dt: float32,int8 + scenario: accuracy,performance + env_prepare: true + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + # Weekly launch - name: Weekly Huggingface Full Test if: github.event_name == 'schedule' && github.event.schedule == '0 16 * * 5' @@ -264,9 +274,19 @@ jobs: mode: inference,training scenario: accuracy,performance hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + - name: Weekly PT2E Accuracy Test + if: github.event_name == 'schedule' && github.event.schedule == '0 16 * * 5' + uses: ./.github/actions/pt2e + with: + suite: pt2e + env_prepare: true + dt: float32,int8 + scenario: accuracy,performance + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + # On-demand launch - name: OnDemand Test (${{ inputs.suite }} ${{ inputs.dt }} ${{ inputs.mode }} ${{ inputs.scenario }}) - if: github.event_name != 'schedule' + if: ${{ github.event_name != 'schedule' && inputs.suite != 'pt2e' }} uses: ./.github/actions/inductor-xpu-e2e-test with: suite: ${{ inputs.suite }} @@ -275,6 +295,16 @@ jobs: mode: ${{ inputs.mode }} scenario: ${{ inputs.scenario }} hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + - name: OnDemand PT2E Test (${{ inputs.suite }} ${{ inputs.dt }} ${{ inputs.mode }} ${{ inputs.scenario }}) + if: ${{ github.event_name != 'schedule' && contains(inputs.suite, 'pt2e') }} + uses: ./.github/actions/pt2e + with: + suite: ${{ inputs.suite }} + env_prepare: true + dt: ${{ inputs.dt }} + mode: ${{ inputs.mode }} + scenario: ${{ inputs.scenario }} + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} - name: Summarize archieve files id: summary diff --git a/.github/workflows/nightly_ondemand_rolling.yml b/.github/workflows/nightly_ondemand_rolling.yml index 5cf66b3c8..22cf0cbee 100644 --- a/.github/workflows/nightly_ondemand_rolling.yml +++ b/.github/workflows/nightly_ondemand_rolling.yml @@ -32,7 +32,7 @@ on: required: true type: string default: 'huggingface' - description: Dynamo benchmarks test suite. `huggingface,timm_models,torchbench`. Delimiter is comma + description: Dynamo benchmarks test suite. `huggingface,timm_models,torchbench,pt2e`. Delimiter is comma dt: required: true type: string @@ -241,6 +241,17 @@ jobs: env_prepare: true hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} driver: rolling + - name: Nightly PT2E Accuracy Test + if: github.event_name == 'schedule' && github.event.schedule == '30 13 * * 0-4' + uses: ./.github/actions/pt2e + with: + suite: pt2e + dt: float32,int8 + scenario: accuracy,performance + env_prepare: true + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + driver: rolling + # Weekly launch - name: Weekly Huggingface Full Test if: github.event_name == 'schedule' && github.event.schedule == '30 16 * * 5' @@ -275,9 +286,20 @@ jobs: scenario: accuracy,performance hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} driver: rolling + - name: Weekly PT2E Accuracy Test + if: github.event_name == 'schedule' && github.event.schedule == '30 16 * * 5' + uses: ./.github/actions/pt2e + with: + suite: pt2e + env_prepare: true + dt: float32,int8 + scenario: accuracy,performance + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + driver: rolling + # On-demand launch - name: OnDemand Test (${{ inputs.suite }} ${{ inputs.dt }} ${{ inputs.mode }} ${{ inputs.scenario }}) - if: github.event_name != 'schedule' + if: ${{ github.event_name != 'schedule' && inputs.suite != 'pt2e' }} uses: ./.github/actions/inductor-xpu-e2e-test with: suite: ${{ inputs.suite }} @@ -287,6 +309,17 @@ jobs: scenario: ${{ inputs.scenario }} hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} driver: rolling + - name: OnDemand PT2E Test (${{ inputs.suite }} ${{ inputs.dt }} ${{ inputs.mode }} ${{ inputs.scenario }}) + if: ${{ github.event_name != 'schedule' && contains(inputs.suite, 'pt2e') }} + uses: ./.github/actions/pt2e + with: + suite: ${{ inputs.suite }} + env_prepare: true + dt: ${{ inputs.dt }} + mode: ${{ inputs.mode }} + scenario: ${{ inputs.scenario }} + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + driver: rolling - name: Summarize archieve files id: summary diff --git a/.github/workflows/nightly_ondemand_whl.yml b/.github/workflows/nightly_ondemand_whl.yml index 86cc3e764..583d36040 100644 --- a/.github/workflows/nightly_ondemand_whl.yml +++ b/.github/workflows/nightly_ondemand_whl.yml @@ -22,7 +22,7 @@ on: required: true type: string default: 'huggingface' - description: Dynamo benchmarks test suite. `huggingface,timm_models,torchbench`. Delimiter is comma + description: Dynamo benchmarks test suite. `huggingface,timm_models,torchbench,pt2e`. Delimiter is comma dt: required: true type: string @@ -187,6 +187,16 @@ jobs: pytorch: nightly_wheel env_prepare: true hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + - name: Nightly PT2E Accuracy Test + if: github.event_name == 'schedule' && github.event.schedule == '0 14 * * 0-4' + uses: ./.github/actions/pt2e + with: + suite: pt2e + dt: float32,int8 + scenario: accuracy,performance + pytorch: nightly_wheel + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + # Weekly launch - name: Weekly Huggingface Full Test if: github.event_name == 'schedule' && github.event.schedule == '0 17 * * 5' @@ -221,9 +231,19 @@ jobs: scenario: accuracy,performance pytorch: nightly_wheel hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + - name: Weekly PT2E Accuracy Test + if: github.event_name == 'schedule' && github.event.schedule == '0 17 * * 5' + uses: ./.github/actions/pt2e + with: + suite: pt2e + dt: float32,int8 + scenario: accuracy,performance + pytorch: nightly_wheel + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + # On-demand launch - name: OnDemand Test (${{ inputs.suite }} ${{ inputs.dt }} ${{ inputs.mode }} ${{ inputs.scenario }}) - if: github.event_name != 'schedule' + if: ${{ github.event_name != 'schedule' && inputs.suite != 'pt2e' }} uses: ./.github/actions/inductor-xpu-e2e-test with: suite: ${{ inputs.suite }} @@ -233,6 +253,17 @@ jobs: scenario: ${{ inputs.scenario }} pytorch: nightly_wheel hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + - name: OnDemand PT2E Test (${{ inputs.suite }} ${{ inputs.dt }} ${{ inputs.mode }} ${{ inputs.scenario }}) + if: ${{ github.event_name != 'schedule' && contains(inputs.suite, 'pt2e') }} + uses: ./.github/actions/pt2e + with: + suite: ${{ inputs.suite }} + env_prepare: true + dt: ${{ inputs.dt }} + mode: ${{ inputs.mode }} + scenario: ${{ inputs.scenario }} + pytorch: nightly_wheel + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} - name: Summarize archieve files id: summary