-
Notifications
You must be signed in to change notification settings - Fork 33
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
5 changed files
with
250 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| name: inductor-xpu-pt2e-test | ||
|
|
||
| inputs: | ||
| suite: | ||
| required: true | ||
| type: string | ||
| default: 'huggingface' | ||
| description: Dynamo benchmarks test suite. huggingface,timm_models,torchbench. Delimiter is comma | ||
| env_prepare: | ||
| required: false | ||
| description: If set to any value, will prepare suite test env | ||
| dt: | ||
| required: true | ||
| type: string | ||
| default: 'float32' | ||
| description: Data precision of the test.float32,int8. Delimiter is comma | ||
| mode: | ||
| required: true | ||
| type: string | ||
| default: 'inference' | ||
| description: inference. Delimiter is comma | ||
| scenario: | ||
| required: true | ||
| type: string | ||
| default: 'accuracy' | ||
| description: accuracy,performance. Delimiter is comma | ||
| cards: | ||
| required: false | ||
| type: string | ||
| default: 'all' | ||
| description: which cards can be used in the test | ||
| hf_token: | ||
| required: false | ||
| description: HUGGING_FACE_HUB_TOKEN for torchbench test | ||
| pytorch: | ||
| required: false | ||
| type: string | ||
| default: 'main' | ||
| description: Pytorch branch/commit | ||
| driver: | ||
| required: false | ||
| type: string | ||
| default: 'lts' | ||
| description: Driver lts/rolling | ||
|
|
||
| runs: | ||
| using: composite | ||
| steps: | ||
| - name: Prepare ENV | ||
| if: ${{ inputs.env_prepare }} | ||
| shell: bash | ||
| run: | | ||
| source activate e2e_ci | ||
| source .github/scripts/env.sh ${{ inputs.pytorch }} | ||
| # accuracy code | ||
| if [[ "${{ inputs.scenario }}" == *"accuracy"* ]];then | ||
| rm -rf pt2e-accuracy | ||
| git clone -b yifeng/accuracy https://github.com/chuanqi129/inductor-tools pt2e-accuracy | ||
| fi | ||
| # performance code | ||
| if [[ "${{ inputs.scenario }}" == *"performance"* ]];then | ||
| rm -rf pt2e-performance | ||
| git clone -b yifeng/pt2e_xpu https://github.com/zxd1997066/benchmark pt2e-performance | ||
| fi | ||
| # deps | ||
| if [[ ${{ inputs.scenario }} == *"performance"* ]]; then | ||
| pip install pyyaml botocore | ||
| if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then | ||
| rm -rf pt2e-audio | ||
| git clone --single-branch -b main https://github.com/pytorch/audio pt2e-audio | ||
| cd pt2e-audio && git checkout $TORCHAUDIO_COMMIT_ID | ||
| python setup.py bdist_wheel && pip uninstall torchaudio -y && pip install --no-deps dist/*.whl | ||
| cd ../ | ||
| rm -rf pt2e-vision | ||
| git clone --single-branch -b main https://github.com/pytorch/vision pt2e-vision | ||
| cd pt2e-vision && git checkout $TORCHVISION_COMMIT_ID | ||
| python setup.py bdist_wheel && pip uninstall torchvision -y && pip install --no-deps dist/*.whl | ||
| cd ../ | ||
| fi | ||
| # torchbench | ||
| python -c "import torch, torchvision, torchaudio" | ||
| rm -rf pt2e-benchmark | ||
| git clone https://github.com/pytorch/benchmark pt2e-benchmark | ||
| cd pt2e-benchmark && git checkout $TORCHBENCH_COMMIT_ID && pip install --no-deps -r requirements.txt | ||
| pip install -U transformers tokenizers safetensors | ||
| python install.py --continue_on_fail | ||
| cd ../ | ||
| # deps for torchrec_dlrm | ||
| pip install pyre_extensions | ||
| pip install fbgemm-gpu | ||
| pip install --no-deps torchmetrics==1.0.3 torchrec | ||
| # transformers | ||
| pip install --force-reinstall git+https://github.com/huggingface/transformers@${TRANSFORMERS_VERSION} | ||
| # timm | ||
| pip install --no-deps git+https://github.com/huggingface/pytorch-image-models@$TIMM_COMMIT_ID | ||
| pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/$TIMM_COMMIT_ID/requirements.txt | grep -vE torch) | ||
| fi | ||
| pip install numpy==1.26.4 | ||
| # dataset | ||
| if [ ! -d ${HOME}/datasets/imagenet ];then | ||
| rm -rf ${HOME}/datasets/imagenet | ||
| mkdir -p ${HOME}/datasets/imagenet | ||
| cd ${HOME}/datasets/imagenet | ||
| wget https://image-net.org/data/ILSVRC/2012/ILSVRC2012_img_val.tar | ||
| tar -xf ILSVRC2012_img_val.tar | ||
| wget -O valprep.sh https://raw.githubusercontent.com/soumith/imagenetloader.torch/master/valprep.sh | ||
| bash valprep.sh | ||
| fi | ||
| - name: PT2E Test (${{ inputs.suite }} ${{ inputs.dt }} ${{ inputs.mode }} ${{ inputs.scenario }}) | ||
| env: | ||
| HUGGING_FACE_HUB_TOKEN: ${{ inputs.hf_token }} | ||
| NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }} | ||
| DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }} | ||
| shell: bash | ||
| run: | | ||
| source activate e2e_ci | ||
| source .github/scripts/env.sh ${{ inputs.pytorch }} | ||
| pt2e_logs_dir="${{ github.workspace }}/../pytorch/inductor_log" | ||
| rm -rf "${pt2e_logs_dir}" && mkdir -p "${pt2e_logs_dir}" | ||
| if [[ "${{ inputs.scenario }}" == *"accuracy"* ]];then | ||
| if [[ "${{ inputs.dt }}" == *"float32"* ]];then | ||
| python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --is_fp32 --dataset_dir ${HOME}/datasets/imagenet |\ | ||
| tee "${pt2e_logs_dir}/accuracy-fp32.log" | ||
| fi | ||
| if [[ "${{ inputs.dt }}" == *"int8"* ]];then | ||
| python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --dataset_dir ${HOME}/datasets/imagenet |\ | ||
| tee "${pt2e_logs_dir}/accuracy-int8.log" | ||
| fi | ||
| fi | ||
| if [[ "${{ inputs.scenario }}" == *"performance"* ]];then | ||
| models="alexnet,demucs,dlrm,hf_Albert,hf_Bert,hf_Bert_large,hf_DistilBert,hf_Roberta_base,mnasnet1_0,mobilenet_v2," | ||
| models+="mobilenet_v3_large,nvidia_deeprecommender,pytorch_CycleGAN_and_pix2pix,resnet152,resnet18,resnet50,resnext50_32x4d," | ||
| models+="shufflenet_v2_x1_0,squeezenet1_1,Super_SloMo,timm_efficientnet,timm_nfnet,timm_regnet,timm_resnest," | ||
| models+="timm_vision_transformer,timm_vision_transformer_large,timm_vovnet,vgg16" | ||
| if [[ "${{ inputs.dt }}" == *"float32"* ]];then | ||
| rm -rf pt2e-performance/.userbenchmark | ||
| python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs --torchdynamo inductor -m $models 2>&1 |\ | ||
| tee "${pt2e_logs_dir}/performance-fp32.log" | ||
| mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-fp32 | ||
| fi | ||
| if [[ "${{ inputs.dt }}" == *"float32"* ]];then | ||
| rm -rf pt2e-performance/.userbenchmark | ||
| python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs --torchdynamo inductor --quantization pt2e -m $models 2>&1 |\ | ||
| tee "${pt2e_logs_dir}/performance-int8.log" | ||
| mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-int8 | ||
| fi | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters