diff --git a/.github/workflows/linux-cpu-x64-build.yml b/.github/workflows/linux-cpu-x64-build.yml
index 102b9a790..cddfaea8b 100644
--- a/.github/workflows/linux-cpu-x64-build.yml
+++ b/.github/workflows/linux-cpu-x64-build.yml
@@ -94,7 +94,7 @@ jobs:
run: |
export ORTGENAI_LOG_ORT_LIB=1
cd test/csharp
- dotnet test /p:Configuration=Release /p:NativeBuildOutputDir="../../build/cpu/" /p:OrtLibDir="../../ort/lib/"
+ dotnet test /p:Configuration=Release /p:NativeBuildOutputDir="../../build/cpu/" /p:OrtLibDir="../../ort/lib/" --verbosity normal
- name: Run tests
run: |
diff --git a/.github/workflows/mac-cpu-arm64-build.yml b/.github/workflows/mac-cpu-arm64-build.yml
index fe5a72c68..2a019a2ca 100644
--- a/.github/workflows/mac-cpu-arm64-build.yml
+++ b/.github/workflows/mac-cpu-arm64-build.yml
@@ -82,7 +82,7 @@ jobs:
run: |
export ORTGENAI_LOG_ORT_LIB=1
cd test/csharp
- dotnet test /p:Configuration=Release /p:NativeBuildOutputDir="../../build/cpu/osx-arm64"
+ dotnet test /p:Configuration=Release /p:NativeBuildOutputDir="../../build/cpu/osx-arm64" --verbosity normal
- name: Run tests
run: |
diff --git a/.github/workflows/win-cpu-x64-build.yml b/.github/workflows/win-cpu-x64-build.yml
index 22399ed89..d6aeaed85 100644
--- a/.github/workflows/win-cpu-x64-build.yml
+++ b/.github/workflows/win-cpu-x64-build.yml
@@ -90,7 +90,7 @@ jobs:
- name: Build the C# API and Run the C# Tests
run: |
cd test\csharp
- dotnet test /p:NativeBuildOutputDir="$env:GITHUB_WORKSPACE\$env:binaryDir\Release" /p:OrtLibDir="$env:GITHUB_WORKSPACE\ort\lib"
+ dotnet test /p:NativeBuildOutputDir="$env:GITHUB_WORKSPACE\$env:binaryDir\Release" /p:OrtLibDir="$env:GITHUB_WORKSPACE\ort\lib" --verbosity normal
- name: Verify Build Artifacts
if: always()
diff --git a/.pipelines/stages/jobs/nuget-validation-job.yml b/.pipelines/stages/jobs/nuget-validation-job.yml
index 2f0fef0d0..88a1c0c0b 100644
--- a/.pipelines/stages/jobs/nuget-validation-job.yml
+++ b/.pipelines/stages/jobs/nuget-validation-job.yml
@@ -88,6 +88,16 @@ jobs:
${{ else }}:
value: 'cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4'
+ - name: prebuild_phi3_5_vision_model_folder
+ ${{ if eq(parameters.ep, 'cpu') }}:
+ value: 'cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4'
+ ${{ elseif eq(parameters.ep, 'cuda') }}:
+ value: 'gpu/gpu-int4-rtn-block-32'
+ ${{ elseif eq(parameters.ep, 'directml')}}:
+ value: 'gpu/gpu-int4-rtn-block-32'
+ ${{ else }}:
+ value: 'cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4'
+
- name: cuda_docker_image
${{ if eq(parameters.cuda_version, '11.8') }}:
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240531.1
@@ -120,98 +130,46 @@ jobs:
inputs:
version: '8.x'
+ - template: steps/utils/flex-download-pipeline-artifact.yml
+ parameters:
+ StepName: 'Download NuGet Artifacts'
+ ArtifactName: $(artifactName)-nuget
+ TargetPath: '$(Build.BinariesDirectory)/nuget'
+ SpecificArtifact: ${{ parameters.specificArtifact }}
+ BuildId: ${{ parameters.BuildId }}
+
- template: steps/utils/download-huggingface-model.yml
parameters:
- StepName: 'Download Model from HuggingFace'
HuggingFaceRepo: 'microsoft/Phi-3-mini-4k-instruct-onnx'
+ LocalFolder: 'phi3-mini'
RepoFolder: $(prebuild_phi3_mini_model_folder)
- LocalFolder: 'models'
WorkingDirectory: '$(Build.Repository.LocalPath)/examples/csharp/HelloPhi'
HuggingFaceToken: $(HF_TOKEN)
os: ${{ parameters.os }}
- - template: steps/utils//flex-download-pipeline-artifact.yml
+ - template: steps/nuget-validation-step.yml
parameters:
- StepName: 'Download NuGet Artifacts'
- ArtifactName: $(artifactName)-nuget
- TargetPath: '$(Build.BinariesDirectory)/nuget'
- SpecificArtifact: ${{ parameters.specificArtifact }}
- BuildId: ${{ parameters.BuildId }}
+ CsprojFolder: "examples/csharp/HelloPhi"
+ CsprojName: "HelloPhi"
+ CsprojConfiguration: $(csproj_configuration)
+ LocalFolder: 'phi3-mini'
+ ModelFolder: $(prebuild_phi3_mini_model_folder)
- - task: Docker@2
- inputs:
- containerRegistry: onnxruntimebuildcache
- command: "login"
- addPipelineData: false
- displayName: "Log in to container registry"
-
- - ${{ if eq(parameters.os, 'win') }}:
- - ${{ if eq(parameters.ep, 'cuda') }}:
- - powershell: |
- $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
- azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
- displayName: 'Download CUDA $(cuda_version)'
- workingDirectory: '$(Build.Repository.LocalPath)'
- - powershell: |
- if ("$(ep)" -eq "cuda") {
- $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
- $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
- Write-Host $env:PATH
- }
- dotnet --info
- Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination examples/csharp/HelloPhi/
- cd examples/csharp/HelloPhi
- Move-Item models\$(prebuild_phi3_mini_model_folder) models\phi-3
- dotnet restore -r $(os)-$(arch) /property:Configuration=$(csproj_configuration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet run -r $(os)-$(arch) --configuration $(csproj_configuration) --no-restore --verbosity normal -- -m ./models/phi-3
- displayName: 'Run Example With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
- env:
- NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
- NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
- - ${{ elseif or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
- - bash: |
- dotnet --info
- cp $(Build.BinariesDirectory)/nuget/* examples/csharp/HelloPhi/
- cd examples/csharp/HelloPhi
- mv models/$(prebuild_phi3_mini_model_folder) models/phi-3
- dotnet restore -r $(os)-$(arch) /property:Configuration=$(csproj_configuration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet build ./HelloPhi.csproj -r $(os)-$(arch) /property:Configuration=$(csproj_configuration) --no-restore --self-contained
- ls -l ./bin/$(csproj_configuration)/net6.0/$(os)-$(arch)/
- displayName: 'Perform dotnet restore & build'
- workingDirectory: '$(Build.Repository.LocalPath)'
- env:
- NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
- NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
-
- - ${{ if eq(parameters.ep, 'cuda') }}:
- - bash: |
- set -e -x
- docker pull $(cuda_docker_image)
-
- docker run \
- --gpus all \
- --rm \
- --volume $(Build.Repository.LocalPath):/ort_genai_src \
- --volume $(Build.BinariesDirectory):/ort_genai_binary \
- -e HF_TOKEN=$HF_TOKEN \
- -w /ort_genai_src/ $(cuda_docker_image) \
- bash -c " \
- export ORTGENAI_LOG_ORT_LIB=1 && \
- cd /ort_genai_src/examples/csharp/HelloPhi && \
- chmod +x ./bin/Release_Cuda/net6.0/linux-x64/HelloPhi && \
- ./bin/Release_Cuda/net6.0/linux-x64/HelloPhi -m ./models/phi-3"
-
- displayName: 'Run Example With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
-
- - ${{ elseif eq(parameters.ep, 'cpu') }}:
- - bash: |
- export ORTGENAI_LOG_ORT_LIB=1
- cd examples/csharp/HelloPhi
- dotnet run -r $(os)-$(arch) --configuration $(csproj_configuration) --no-build --verbosity normal -- -m ./models/phi-3
- displayName: 'Run Example With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
+ - template: steps/utils/download-huggingface-model.yml
+ parameters:
+ HuggingFaceRepo: 'microsoft/Phi-3.5-vision-instruct-onnx'
+ LocalFolder: 'phi3.5-vision'
+ RepoFolder: $(prebuild_phi3_5_vision_model_folder)
+ WorkingDirectory: '$(Build.Repository.LocalPath)/examples/csharp/HelloPhi3V'
+ HuggingFaceToken: $(HF_TOKEN)
+ os: ${{ parameters.os }}
- - template: steps/compliant-and-cleanup-step.yml
+ - template: steps/nuget-validation-step.yml
+ parameters:
+ CsprojFolder: "examples/csharp/HelloPhi3V"
+ CsprojName: "HelloPhi3V"
+ CsprojConfiguration: $(csproj_configuration)
+ LocalFolder: 'phi3.5-vision'
+ ModelFolder: $(prebuild_phi3_5_vision_model_folder)
+ - template: steps/compliant-and-cleanup-step.yml
diff --git a/.pipelines/stages/jobs/py-validation-job.yml b/.pipelines/stages/jobs/py-validation-job.yml
index 8543796d4..2930b8885 100644
--- a/.pipelines/stages/jobs/py-validation-job.yml
+++ b/.pipelines/stages/jobs/py-validation-job.yml
@@ -97,6 +97,16 @@ jobs:
${{ else }}:
value: 'cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4'
+ - name: prebuild_phi3_5_vision_model_folder
+ ${{ if eq(parameters.ep, 'cpu') }}:
+ value: 'cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4'
+ ${{ elseif eq(parameters.ep, 'cuda') }}:
+ value: 'gpu/gpu-int4-rtn-block-32'
+ ${{ elseif eq(parameters.ep, 'directml')}}:
+ value: 'gpu/gpu-int4-rtn-block-32'
+ ${{ else }}:
+ value: 'cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4'
+
- name: cuda_docker_image
${{ if eq(parameters.cuda_version, '11.8') }}:
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240531.1
@@ -147,99 +157,34 @@ jobs:
- template: steps/utils/download-huggingface-model.yml
parameters:
- StepName: 'Download Model from HuggingFace'
HuggingFaceRepo: 'microsoft/Phi-3-mini-4k-instruct-onnx'
+ LocalFolder: 'phi3-mini'
RepoFolder: $(prebuild_phi3_mini_model_folder)
- LocalFolder: 'models'
WorkingDirectory: '$(Build.Repository.LocalPath)/examples/python'
HuggingFaceToken: $(HF_TOKEN)
os: ${{ parameters.os }}
- - task: Docker@2
- inputs:
- containerRegistry: onnxruntimebuildcache
- command: "login"
- addPipelineData: false
- displayName: "Log in to container registry"
-
- - ${{ if or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
- - ${{ if eq(parameters.ep, 'cuda') }}:
- - bash: |
- set -e -x
- docker pull $(cuda_docker_image)
- python_exe=/opt/python/cp310-cp310/bin/python3.10
-
- docker run \
- --gpus all \
- --rm \
- --volume $(Build.Repository.LocalPath):/ort_genai_src \
- --volume $(Build.BinariesDirectory):/ort_genai_binary \
- -e HF_TOKEN=$HF_TOKEN \
- -w /ort_genai_src/ $(cuda_docker_image) \
- bash -c " \
- export ORTGENAI_LOG_ORT_LIB=1 && \
- $python_exe -m pip install -r /ort_genai_src/test/python/requirements.txt && \
- $python_exe -m pip install -r /ort_genai_src/test/python/cuda/torch/requirements.txt && \
- $python_exe -m pip install -r /ort_genai_src/test/python/cuda/ort/requirements.txt && \
- cd /ort_genai_src/examples/python && \
- $python_exe -m pip install --no-index --find-links=/ort_genai_binary/wheel $(pip_package_name) && \
- $python_exe model-generate.py -m ./models/$(prebuild_phi3_mini_model_folder) --min_length 25 --max_length 50 --verbose"
-
- displayName: 'Run Example With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
- - ${{ elseif eq(parameters.ep, 'cpu') }}:
- - bash: |
- export ORTGENAI_LOG_ORT_LIB=1
- python -m pip install -r test/python/requirements.txt
- if [[ "$(os)" == "linux" ]]; then
- python -m pip install -r test/python/cpu/torch/requirements.txt
- python -m pip install -r test/python/cpu/ort/requirements.txt
- fi
- if [[ "$(os)" == "osx" ]]; then
- python -m pip install -r test/python/macos/torch/requirements.txt
- python -m pip install -r test/python/macos/ort/requirements.txt
- fi
- cd examples/python
- python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
- python model-generate.py -m ./models/$(prebuild_phi3_mini_model_folder) --min_length 25 --max_length 50 --verbose
- displayName: 'Run Example With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
-
- - ${{ if eq(parameters.os, 'win') }}:
- - ${{ if eq(parameters.ep, 'cuda') }}:
- - powershell: |
- $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
- azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
- displayName: 'Download CUDA $(cuda_version)'
- workingDirectory: '$(Build.Repository.LocalPath)'
- - powershell: |
- if ("$(arch)" -ne "arm64") {
- python -m pip install -r test/python/requirements.txt
- }
- if ("$(ep)" -eq "cuda") {
- $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
- $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
- Write-Host $env:PATH
- python -m pip install -r test/python/cuda/torch/requirements.txt
- python -m pip install -r test/python/cuda/ort/requirements.txt
- }
- elseif ("$(ep)" -eq "directml") {
- python -m pip install -r test/python/directml/torch/requirements.txt
- python -m pip install -r test/python/directml/ort/requirements.txt
- }
- elseif ("$(arch)" -eq "arm64") {
- python -m pip install numpy<2
- python -m pip install onnxruntime-qnn==1.20.0
- }
- else {
- python -m pip install -r test/python/cpu/torch/requirements.txt
- python -m pip install -r test/python/cpu/ort/requirements.txt
- }
- cd examples\python
- python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
-
- python model-generate.py -m .\models\$(prebuild_phi3_mini_model_folder) --min_length 25 --max_length 50 --batch_size_for_cuda_graph 3 --verbose
- displayName: 'Run Example With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
+ - template: steps/python-validation-step.yml
+ parameters:
+ PythonScriptFolder: "examples/python"
+ PythonScriptName: "model-generate.py"
+ LocalFolder: 'phi3-mini'
+ ModelFolder: $(prebuild_phi3_mini_model_folder)
+
+ - template: steps/utils/download-huggingface-model.yml
+ parameters:
+ HuggingFaceRepo: 'microsoft/Phi-3.5-vision-instruct-onnx'
+ LocalFolder: 'phi3.5-vision'
+ RepoFolder: $(prebuild_phi3_5_vision_model_folder)
+ WorkingDirectory: '$(Build.Repository.LocalPath)/examples/python'
+ HuggingFaceToken: $(HF_TOKEN)
+ os: ${{ parameters.os }}
+
+ - template: steps/python-validation-step.yml
+ parameters:
+ PythonScriptFolder: "examples/python"
+ PythonScriptName: "phi3v.py"
+ LocalFolder: 'phi3.5-vision'
+ ModelFolder: $(prebuild_phi3_5_vision_model_folder)
- template: steps/compliant-and-cleanup-step.yml
diff --git a/.pipelines/stages/jobs/steps/nuget-validation-step.yml b/.pipelines/stages/jobs/steps/nuget-validation-step.yml
new file mode 100644
index 000000000..788a36732
--- /dev/null
+++ b/.pipelines/stages/jobs/steps/nuget-validation-step.yml
@@ -0,0 +1,88 @@
+parameters:
+- name: CsprojFolder
+ type: string
+- name: CsprojName
+ type: string
+- name: CsprojConfiguration
+ type: string
+- name: LocalFolder
+ type: string
+- name: ModelFolder
+ type: string
+
+steps:
+ - task: Docker@2
+ inputs:
+ containerRegistry: onnxruntimebuildcache
+ command: "login"
+ addPipelineData: false
+ displayName: "Log in to container registry"
+
+ - powershell: |
+ $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
+ azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
+ displayName: 'Download CUDA $(cuda_version)'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: and(eq(variables['os'], 'win'), eq(variables['ep'], 'cuda'))
+ - powershell: |
+ if ("$(ep)" -eq "cuda") {
+ $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
+ $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
+ Write-Host $env:PATH
+ }
+ dotnet --info
+ Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination ${{ parameters.CsprojFolder }}
+ cd ${{ parameters.CsprojFolder }}
+ dotnet restore -r $(os)-$(arch) /property:Configuration=${{ parameters.CsprojConfiguration }} --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
+ dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-restore --verbosity normal -- -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --non-interactive
+ displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Windows'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: eq(variables['os'], 'win')
+ env:
+ NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
+ NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
+
+ - bash: |
+ set -e -x
+ dotnet --info
+ cp $(Build.BinariesDirectory)/nuget/* ${{ parameters.CsprojFolder }}
+ cd ${{ parameters.CsprojFolder }}
+ dotnet restore -r $(os)-$(arch) /property:Configuration=${{ parameters.CsprojConfiguration }} --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
+ dotnet build ./${{ parameters.CsprojName }}.csproj -r $(os)-$(arch) /property:Configuration=${{ parameters.CsprojConfiguration }} --no-restore --self-contained --verbosity normal
+ ls -l ./bin/${{ parameters.CsprojConfiguration }}/net6.0/$(os)-$(arch)/
+ displayName: 'Perform dotnet restore & build'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx'))
+ env:
+ NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
+ NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
+ - bash: |
+ set -e -x
+ az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
+ az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
+ docker pull $(cuda_docker_image)
+
+ docker run \
+ --gpus all \
+ --rm \
+ --volume $(Build.Repository.LocalPath):/ort_genai_src \
+ --volume $(Build.BinariesDirectory):/ort_genai_binary \
+ -e HF_TOKEN=$HF_TOKEN \
+ -w /ort_genai_src/ $(cuda_docker_image) \
+ bash -c " \
+ export ORTGENAI_LOG_ORT_LIB=1 && \
+ cd /ort_genai_src/${{ parameters.CsprojFolder }} && \
+ chmod +x ./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} && \
+ ./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --non-interactive"
+
+ displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Linux CUDA'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: and(eq(variables['os'], 'linux'), eq(variables['ep'], 'cuda'))
+
+ - bash: |
+ export ORTGENAI_LOG_ORT_LIB=1
+ cd ${{ parameters.CsprojFolder }}
+ dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-build --verbosity normal -- -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --non-interactive
+ displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Linux/macOS CPU'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
diff --git a/.pipelines/stages/jobs/steps/python-validation-step.yml b/.pipelines/stages/jobs/steps/python-validation-step.yml
new file mode 100644
index 000000000..dc964b718
--- /dev/null
+++ b/.pipelines/stages/jobs/steps/python-validation-step.yml
@@ -0,0 +1,97 @@
+parameters:
+- name: PythonScriptFolder
+ type: string
+- name: PythonScriptName
+ type: string
+- name: LocalFolder
+ type: string
+- name: ModelFolder
+ type: string
+
+steps:
+ - task: Docker@2
+ inputs:
+ containerRegistry: onnxruntimebuildcache
+ command: "login"
+ addPipelineData: false
+ displayName: "Log in to container registry"
+
+ - powershell: |
+ $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
+ azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
+ displayName: 'Download CUDA $(cuda_version)'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: and(eq(variables['os'], 'win'), eq(variables['ep'], 'cuda'))
+ - powershell: |
+ python -m pip install -r test/python/requirements.txt
+ if ("$(ep)" -eq "cuda") {
+ $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
+ $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
+ Write-Host $env:PATH
+ python -m pip install -r test/python/cuda/torch/requirements.txt
+ python -m pip install -r test/python/cuda/ort/requirements.txt
+ }
+ elseif ("$(ep)" -eq "directml") {
+ python -m pip install -r test/python/directml/torch/requirements.txt
+ python -m pip install -r test/python/directml/ort/requirements.txt
+ }
+ else {
+ python -m pip install -r test/python/cpu/torch/requirements.txt
+ python -m pip install -r test/python/cpu/ort/requirements.txt
+ }
+ cd ${{ parameters.PythonScriptFolder }}
+ python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
+
+ if ("$(ep)" -eq "directml") {
+ python ${{ parameters.PythonScriptName }} -m .\${{ parameters.LocalFolder }}\${{ parameters.ModelFolder }} --provider dml --non-interactive
+ } else {
+ python ${{ parameters.PythonScriptName }} -m .\${{ parameters.LocalFolder }}\${{ parameters.ModelFolder }} --provider $(ep) --non-interactive
+ }
+ displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Windows'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: eq(variables['os'], 'win')
+
+ - bash: |
+ set -e -x
+ az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
+ az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
+ docker pull $(cuda_docker_image)
+ python_exe=/opt/python/cp310-cp310/bin/python3.10
+
+ docker run \
+ --gpus all \
+ --rm \
+ --volume $(Build.Repository.LocalPath):/ort_genai_src \
+ --volume $(Build.BinariesDirectory):/ort_genai_binary \
+ -e HF_TOKEN=$HF_TOKEN \
+ -w /ort_genai_src/ $(cuda_docker_image) \
+ bash -c " \
+ export ORTGENAI_LOG_ORT_LIB=1 && \
+ $python_exe -m pip install -r /ort_genai_src/test/python/requirements.txt && \
+ $python_exe -m pip install -r /ort_genai_src/test/python/cuda/torch/requirements.txt && \
+ $python_exe -m pip install -r /ort_genai_src/test/python/cuda/ort/requirements.txt && \
+ cd /ort_genai_src/${{ parameters.PythonScriptFolder }} && \
+ $python_exe -m pip install --no-index --find-links=/ort_genai_binary/wheel $(pip_package_name) && \
+ $python_exe ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --provider $(ep) --non-interactive"
+
+ displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Linux CUDA'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: and(eq(variables['os'], 'linux'), eq(variables['ep'], 'cuda'))
+
+ - bash: |
+ export ORTGENAI_LOG_ORT_LIB=1
+ python -m pip install -r test/python/requirements.txt
+ if [[ "$(os)" == "linux" ]]; then
+ python -m pip install -r test/python/cpu/torch/requirements.txt
+ python -m pip install -r test/python/cpu/ort/requirements.txt
+ fi
+ if [[ "$(os)" == "osx" ]]; then
+ python -m pip install -r test/python/macos/torch/requirements.txt
+ python -m pip install -r test/python/macos/ort/requirements.txt
+ fi
+ cd ${{ parameters.PythonScriptFolder }}
+ python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
+ python ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --provider $(ep) --non-interactive
+ displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Linux/macOS CPU'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
\ No newline at end of file
diff --git a/.pipelines/stages/jobs/steps/utils/download-huggingface-model.yml b/.pipelines/stages/jobs/steps/utils/download-huggingface-model.yml
index 3bb0caa22..c537cd86e 100644
--- a/.pipelines/stages/jobs/steps/utils/download-huggingface-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/download-huggingface-model.yml
@@ -1,6 +1,4 @@
parameters:
- - name: StepName
- type: string
- name: WorkingDirectory
type: string
- name: HuggingFaceRepo
@@ -20,17 +18,20 @@ steps:
python -m pip install "huggingface_hub[cli]"
huggingface-cli login --token $HF_TOKEN
huggingface-cli download ${{ parameters.HuggingFaceRepo }} --include ${{ parameters.RepoFolder }}/* --local-dir ${{ parameters.LocalFolder }} --local-dir-use-symlinks False
- displayName: ${{ parameters.StepName }}
+ displayName: Download ${{ parameters.HuggingFaceRepo }} from HuggingFace
workingDirectory: ${{ parameters.WorkingDirectory }}
env:
HF_TOKEN: ${{ parameters.HuggingFaceToken }}
+ condition: succeededOrFailed() # Run this even if previous tasks failed.
+
- ${{ if eq(parameters.os, 'win') }}:
- powershell: |
python -m pip install "huggingface_hub[cli]"
huggingface-cli login --token $env:HF_TOKEN
# Use maximum path length for Windows... otherwises hits the path character limit
huggingface-cli download ${{ parameters.HuggingFaceRepo }} --include ${{ parameters.RepoFolder }}/* --local-dir "\\?\${{ parameters.WorkingDirectory }}\\${{ parameters.LocalFolder }}" --local-dir-use-symlinks False
- displayName: ${{ parameters.StepName }}
+ displayName: Download ${{ parameters.HuggingFaceRepo }} from HuggingFace
workingDirectory: ${{ parameters.WorkingDirectory }}
env:
HF_TOKEN: ${{ parameters.HuggingFaceToken }}
+ condition: succeededOrFailed() # Run this even if previous tasks failed.
diff --git a/.pipelines/stages/jobs/steps/utils/flex-download-pipeline-artifact.yml b/.pipelines/stages/jobs/steps/utils/flex-download-pipeline-artifact.yml
index a83451a1b..232e8549e 100644
--- a/.pipelines/stages/jobs/steps/utils/flex-download-pipeline-artifact.yml
+++ b/.pipelines/stages/jobs/steps/utils/flex-download-pipeline-artifact.yml
@@ -30,3 +30,4 @@ steps:
pipeline: $(Build.DefinitionName)
runVersion: 'specific'
buildId: ${{ parameters.BuildId }}
+ condition: succeededOrFailed() # Run this even if previous tasks failed.
diff --git a/VERSION_INFO b/VERSION_INFO
index 5d4294b91..2411653a5 100644
--- a/VERSION_INFO
+++ b/VERSION_INFO
@@ -1 +1 @@
-0.5.1
\ No newline at end of file
+0.5.2
\ No newline at end of file
diff --git a/examples/csharp/HelloPhi/HelloPhi.csproj b/examples/csharp/HelloPhi/HelloPhi.csproj
index 1e9a86a30..8e357f3ae 100644
--- a/examples/csharp/HelloPhi/HelloPhi.csproj
+++ b/examples/csharp/HelloPhi/HelloPhi.csproj
@@ -10,9 +10,9 @@
-
-
-
+
+
+
diff --git a/examples/csharp/HelloPhi/Program.cs b/examples/csharp/HelloPhi/Program.cs
index 26e20a353..02190f21b 100644
--- a/examples/csharp/HelloPhi/Program.cs
+++ b/examples/csharp/HelloPhi/Program.cs
@@ -1,11 +1,15 @@
-// See https://aka.ms/new-console-template for more information
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
using Microsoft.ML.OnnxRuntimeGenAI;
void PrintUsage()
{
Console.WriteLine("Usage:");
Console.WriteLine(" -m model_path");
- Console.WriteLine(" -i (optional): Interactive mode");
+ Console.WriteLine("\t\t\t\tPath to the model");
+ Console.WriteLine(" --non-interactive (optional)");
+ Console.WriteLine("\t\t\t\tInteractive mode");
}
using OgaHandle ogaHandle = new OgaHandle();
@@ -16,16 +20,16 @@ void PrintUsage()
Environment.Exit(-1);
}
-bool interactive = false;
+bool interactive = true;
string modelPath = string.Empty;
uint i = 0;
while (i < args.Length)
{
var arg = args[i];
- if (arg == "-i")
+ if (arg == "--non-interactive")
{
- interactive = true;
+ interactive = false;
}
else if (arg == "-m")
{
diff --git a/examples/csharp/HelloPhi3V/HelloPhi3V.csproj b/examples/csharp/HelloPhi3V/HelloPhi3V.csproj
index ab7fc6c44..5ea0e25f8 100644
--- a/examples/csharp/HelloPhi3V/HelloPhi3V.csproj
+++ b/examples/csharp/HelloPhi3V/HelloPhi3V.csproj
@@ -9,9 +9,9 @@
-
-
-
+
+
+
diff --git a/examples/csharp/HelloPhi3V/Program.cs b/examples/csharp/HelloPhi3V/Program.cs
index 1f03a45d1..61f98b2d9 100644
--- a/examples/csharp/HelloPhi3V/Program.cs
+++ b/examples/csharp/HelloPhi3V/Program.cs
@@ -3,80 +3,155 @@
using Microsoft.ML.OnnxRuntimeGenAI;
using System.Linq;
+using System.Runtime.CompilerServices;
-class Program
+static string GetDirectoryInTreeThatContains(string currentDirectory, string targetDirectoryName)
{
- static void Run(string modelPath)
+ bool found = false;
+ foreach (string d in Directory.GetDirectories(currentDirectory, searchPattern: targetDirectoryName))
{
- using Model model = new Model(modelPath);
- using MultiModalProcessor processor = new MultiModalProcessor(model);
- using var tokenizerStream = processor.CreateStream();
+ found = true;
+ return Path.Combine(currentDirectory, targetDirectoryName);
+ }
+ if (!found)
+ {
+ DirectoryInfo dirInfo = new DirectoryInfo(currentDirectory);
+ if (dirInfo.Parent != null)
+ {
+ return GetDirectoryInTreeThatContains(Path.GetFullPath(Path.Combine(currentDirectory, "..")), targetDirectoryName);
+ }
+ else
+ {
+ return null;
+ }
+ }
+ return null;
+}
+
+void PrintUsage()
+{
+ Console.WriteLine("Usage:");
+ Console.WriteLine(" -m model_path");
+ Console.WriteLine("\t\t\t\tPath to the model");
+ Console.WriteLine(" --image_paths");
+ Console.WriteLine("\t\t\t\tPath to the images");
+ Console.WriteLine(" --non-interactive (optional), mainly for CI usage");
+ Console.WriteLine("\t\t\t\tInteractive mode");
+}
+
+using OgaHandle ogaHandle = new OgaHandle();
+
+if (args.Length < 1)
+{
+ PrintUsage();
+ Environment.Exit(-1);
+}
+
+bool interactive = true;
+string modelPath = string.Empty;
+List imagePaths = new List();
- while (true)
+uint i_arg = 0;
+while (i_arg < args.Length)
+{
+ var arg = args[i_arg];
+ if (arg == "--non-interactive")
+ {
+ interactive = false;
+ }
+ else if (arg == "-m")
+ {
+ if (i_arg + 1 < args.Length)
{
- Console.WriteLine("Image Path (comma separated; leave empty if no image):");
- string[] imagePaths = Console.ReadLine().Split(',').ToList().Select(i => i.ToString().Trim()).ToArray();
-
- Images images = null;
- if (imagePaths.Length == 0)
- {
- Console.WriteLine("No image provided");
- }
- else
- {
- for (int i = 0; i < imagePaths.Length; i++)
- {
- string imagePath = imagePaths[i].Trim();
- if (!File.Exists(imagePath))
- {
- throw new Exception("Image file not found: " + imagePath);
- }
- }
- images = Images.Load(imagePaths);
- }
-
- Console.WriteLine("Prompt:");
- string text = Console.ReadLine();
- string prompt = "<|user|>\n";
- if (images != null)
- {
- for (int i = 0; i < imagePaths.Length; i++)
- {
- prompt += "<|image_" + (i + 1) + "|>\n";
- }
- }
- prompt += text + "<|end|>\n<|assistant|>\n";
-
- Console.WriteLine("Processing image and prompt...");
- var inputTensors = processor.ProcessImages(prompt, images);
-
- Console.WriteLine("Generating response...");
- using GeneratorParams generatorParams = new GeneratorParams(model);
- generatorParams.SetSearchOption("max_length", 7680);
- generatorParams.SetInputs(inputTensors);
-
- using var generator = new Generator(model, generatorParams);
- while (!generator.IsDone())
- {
- generator.ComputeLogits();
- generator.GenerateNextToken();
- Console.Write(tokenizerStream.Decode(generator.GetSequence(0)[^1]));
- }
+ modelPath = Path.Combine(args[i_arg+1]);
}
+ }
+ else if (arg == "--image_paths")
+ {
+ if (i_arg + 1 < args.Length)
+ {
+ imagePaths = args[i_arg + 1].Split(',').ToList().Select(i => i.ToString().Trim()).ToList();
+ }
+ }
+ i_arg++;
+}
+Console.WriteLine("--------------------");
+Console.WriteLine("Hello, Phi-3-Vision!");
+Console.WriteLine("--------------------");
+
+Console.WriteLine("Model path: " + modelPath);
+Console.WriteLine("Interactive: " + interactive);
+
+using Model model = new Model(modelPath);
+using MultiModalProcessor processor = new MultiModalProcessor(model);
+using var tokenizerStream = processor.CreateStream();
+
+do
+{
+ if (interactive)
+ {
+ Console.WriteLine("Image Path (comma separated; leave empty if no image):");
+ imagePaths = Console.ReadLine().Split(',').ToList().Select(i => i.ToString().Trim()).ToList();
}
- static void Main(string[] args)
+ if (imagePaths.Count == 0)
{
- Console.WriteLine("--------------------");
- Console.WriteLine("Hello, Phi-3-Vision!");
- Console.WriteLine("--------------------");
+ Console.WriteLine("No image provided. Using default image.");
+ imagePaths.Add(Path.Combine(
+ GetDirectoryInTreeThatContains(Directory.GetCurrentDirectory(), "test"), "test_models", "images", "australia.jpg"));
+ }
+ for (int i = 0; i < imagePaths.Count; i++)
+ {
+ string imagePath = Path.GetFullPath(imagePaths[i].Trim());
+ if (!File.Exists(imagePath))
+ {
+ throw new Exception("Image file not found: " + imagePath);
+ }
+ Console.WriteLine("Using image: " + imagePath);
+ }
+
+ Images images = imagePaths.Count > 0 ? Images.Load(imagePaths.ToArray()) : null;
- if (args.Length != 1)
+ string text = "What is shown in this image?";
+ if (interactive) {
+ Console.WriteLine("Prompt:");
+ text = Console.ReadLine();
+ }
+
+ string prompt = "<|user|>\n";
+ if (images != null)
+ {
+ for (int i = 0; i < imagePaths.Count; i++)
{
- throw new Exception("Usage: .\\HelloPhi3V ");
+ prompt += "<|image_" + (i + 1) + "|>\n";
}
+ }
+ prompt += text + "<|end|>\n<|assistant|>\n";
+
+ Console.WriteLine("Processing image and prompt...");
+ using var inputTensors = processor.ProcessImages(prompt, images);
- Run(args[0]);
+ Console.WriteLine("Generating response...");
+ using GeneratorParams generatorParams = new GeneratorParams(model);
+ generatorParams.SetSearchOption("max_length", 7680);
+ generatorParams.SetInputs(inputTensors);
+
+ using var generator = new Generator(model, generatorParams);
+ var watch = System.Diagnostics.Stopwatch.StartNew();
+ while (!generator.IsDone())
+ {
+ generator.ComputeLogits();
+ generator.GenerateNextToken();
+ Console.Write(tokenizerStream.Decode(generator.GetSequence(0)[^1]));
+ }
+ watch.Stop();
+ var runTimeInSeconds = watch.Elapsed.TotalSeconds;
+ Console.WriteLine();
+ Console.WriteLine($"Total Time: {runTimeInSeconds:0.00}");
+
+ if (images != null)
+ {
+ images.Dispose();
}
-}
\ No newline at end of file
+} while (interactive);
\ No newline at end of file
diff --git a/examples/python/model-generate.py b/examples/python/model-generate.py
index 0a97f25b4..930eba5de 100644
--- a/examples/python/model-generate.py
+++ b/examples/python/model-generate.py
@@ -4,7 +4,17 @@
def main(args):
if args.verbose: print("Loading model...")
- model = og.Model(f'{args.model}')
+ if hasattr(og, 'Config'):
+ config = og.Config(args.model_path)
+ config.clear_providers()
+ if args.provider != "cpu":
+ if args.verbose:
+ print(f"Setting model to {args.provider}...")
+ config.append_provider(args.provider)
+ model = og.Model(config)
+ else:
+ model = og.Model(args.model_path)
+
if args.verbose: print("Model loaded")
tokenizer = og.Tokenizer(model)
if args.verbose: print("Tokenizer created")
@@ -12,22 +22,26 @@ def main(args):
if hasattr(args, 'prompts'):
prompts = args.prompts
else:
- prompts = ["I like walking my cute dog",
+ if args.non_interactive:
+ prompts = ["I like walking my cute dog",
"What is the best restaurant in town?",
"Hello, how are you today?"]
-
+ else:
+ text = input("Input: ")
+ prompts = [text]
+
if args.chat_template:
if args.chat_template.count('{') != 1 or args.chat_template.count('}') != 1:
print("Error, chat template must have exactly one pair of curly braces, e.g. '<|user|>\n{input} <|end|>\n<|assistant|>'")
exit(1)
prompts[:] = [f'{args.chat_template.format(input=text)}' for text in prompts]
-
+
input_tokens = tokenizer.encode_batch(prompts)
if args.verbose: print(f'Prompt(s) encoded: {prompts}')
params = og.GeneratorParams(model)
- search_options = {name:getattr(args, name) for name in ['do_sample', 'max_length', 'min_length', 'top_p', 'top_k', 'temperature', 'repetition_penalty'] if name in args}
+ search_options = {name:getattr(args, name) for name in ['do_sample', 'max_length', 'min_length', 'top_p', 'top_k', 'temperature', 'repetition_penalty'] if name in args}
if (args.verbose): print(f'Args: {args}')
if (args.verbose): print(f'Search options: {search_options}')
@@ -58,18 +72,20 @@ def main(args):
if __name__ == "__main__":
parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS, description="End-to-end token generation loop example for gen-ai")
- parser.add_argument('-m', '--model', type=str, required=True, help='Onnx model folder path (must contain config.json and model.onnx)')
+ parser.add_argument('-m', '--model_path', type=str, required=True, help='Onnx model folder path (must contain config.json and model.onnx)')
+ parser.add_argument("-p", "--provider", type=str, required=True, help="Provider to run model")
parser.add_argument('-pr', '--prompts', nargs='*', required=False, help='Input prompts to generate tokens from. Provide this parameter multiple times to batch multiple prompts')
- parser.add_argument('-i', '--min_length', type=int, help='Min number of tokens to generate including the prompt')
- parser.add_argument('-l', '--max_length', type=int, help='Max number of tokens to generate including the prompt')
+ parser.add_argument('-i', '--min_length', type=int, default=25, help='Min number of tokens to generate including the prompt')
+ parser.add_argument('-l', '--max_length', type=int, default=50, help='Max number of tokens to generate including the prompt')
parser.add_argument('-ds', '--do_random_sampling', action='store_true', help='Do random sampling. When false, greedy or beam search are used to generate the output. Defaults to false')
- parser.add_argument('-p', '--top_p', type=float, help='Top p probability to sample with')
+ parser.add_argument('--top_p', type=float, help='Top p probability to sample with')
parser.add_argument('-k', '--top_k', type=int, help='Top k tokens to sample from')
parser.add_argument('-t', '--temperature', type=float, help='Temperature to sample with')
parser.add_argument('-r', '--repetition_penalty', type=float, help='Repetition penalty to sample with')
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print verbose output and timing information. Defaults to false')
parser.add_argument('-b', '--batch_size_for_cuda_graph', type=int, default=1, help='Max batch size for CUDA graph')
parser.add_argument('-c', '--chat_template', type=str, default='', help='Chat template to use for the prompt. User input will be injected into {input}. If not set, the prompt is used as is.')
+ parser.add_argument('--non-interactive', action=argparse.BooleanOptionalAction, required=False, help='Non-interactive mode, mainly for CI usage')
args = parser.parse_args()
main(args)
\ No newline at end of file
diff --git a/examples/python/phi3v.py b/examples/python/phi3v.py
index d77fd4eda..67b8beba7 100644
--- a/examples/python/phi3v.py
+++ b/examples/python/phi3v.py
@@ -3,53 +3,90 @@
import argparse
import os
-import readline
import glob
+import time
+from pathlib import Path
import onnxruntime_genai as og
+
+def _find_dir_contains_sub_dir(current_dir: Path, target_dir_name):
+ curr_path = Path(current_dir).absolute()
+ target_dir = glob.glob(target_dir_name, root_dir=curr_path)
+ if target_dir:
+ return Path(curr_path / target_dir[0]).absolute()
+ else:
+ if curr_path.parent == curr_path:
+ # Root dir
+ return None
+ return _find_dir_contains_sub_dir(curr_path / '..', target_dir_name)
+
+
def _complete(text, state):
return (glob.glob(text + "*") + [None])[state]
def run(args: argparse.Namespace):
print("Loading model...")
- config = og.Config(args.model_path)
- config.clear_providers()
- if args.provider != "cpu":
- print(f"Setting model to {args.provider}...")
- config.append_provider(args.provider)
- model = og.Model(config)
+ if hasattr(og, 'Config'):
+ config = og.Config(args.model_path)
+ config.clear_providers()
+ if args.provider != "cpu":
+ print(f"Setting model to {args.provider}...")
+ config.append_provider(args.provider)
+ model = og.Model(config)
+ else:
+ model = og.Model(args.model_path)
+ print("Model loaded")
processor = model.create_multimodal_processor()
tokenizer_stream = processor.create_stream()
+ interactive = not args.non_interactive
+
while True:
- readline.set_completer_delims(" \t\n;")
- readline.parse_and_bind("tab: complete")
- readline.set_completer(_complete)
- image_paths = [
- image_path.strip()
- for image_path in input(
- "Image Path (comma separated; leave empty if no image): "
- ).split(",")
- ]
- image_paths = [image_path for image_path in image_paths if len(image_path)]
- print(image_paths)
+ if interactive:
+ try:
+ import readline
+ readline.set_completer_delims(" \t\n;")
+ readline.parse_and_bind("tab: complete")
+ readline.set_completer(_complete)
+ except ImportError:
+ # Not available on some platforms. Ignore it.
+ pass
+ image_paths = [
+ image_path.strip()
+ for image_path in input(
+ "Image Path (comma separated; leave empty if no image): "
+ ).split(",")
+ ]
+ else:
+ if args.image_paths:
+ image_paths = args.image_paths
+ else:
+ image_paths = [str(_find_dir_contains_sub_dir(Path(__file__).parent, "test") / "test_models" / "images" / "australia.jpg")]
+
+ image_paths = [image_path for image_path in image_paths]
images = None
prompt = "<|user|>\n"
if len(image_paths) == 0:
print("No image provided")
else:
- print("Loading images...")
for i, image_path in enumerate(image_paths):
if not os.path.exists(image_path):
raise FileNotFoundError(f"Image file not found: {image_path}")
+ print(f"Using image: {image_path}")
prompt += f"<|image_{i+1}|>\n"
images = og.Images.open(*image_paths)
- text = input("Prompt: ")
+ if interactive:
+ text = input("Prompt: ")
+ else:
+ if args.prompt:
+ text = args.prompt
+ else:
+ text = "What is shown in this image?"
prompt += f"{text}<|end|>\n<|assistant|>\n"
print("Processing images and prompt...")
inputs = processor(prompt, images=images)
@@ -60,6 +97,7 @@ def run(args: argparse.Namespace):
params.set_search_options(max_length=7680)
generator = og.Generator(model, params)
+ start_time = time.time()
while not generator.is_done():
generator.compute_logits()
@@ -68,12 +106,19 @@ def run(args: argparse.Namespace):
new_token = generator.get_next_tokens()[0]
print(tokenizer_stream.decode(new_token), end="", flush=True)
+ print()
+ total_run_time = time.time() - start_time
+ print(f"Total Time : {total_run_time:.2f}")
+
for _ in range(3):
print()
# Delete the generator to free the captured graph before creating another one
del generator
+ if not interactive:
+ break
+
if __name__ == "__main__":
parser = argparse.ArgumentParser()
@@ -83,5 +128,14 @@ def run(args: argparse.Namespace):
parser.add_argument(
"-p", "--provider", type=str, required=True, help="Provider to run model"
)
+ parser.add_argument(
+ "--image_paths", nargs='*', type=str, required=False, help="Path to the images, mainly for CI usage"
+ )
+ parser.add_argument(
+ '-pr', '--prompt', required=False, help='Input prompts to generate tokens from, mainly for CI usage'
+ )
+ parser.add_argument(
+ '--non-interactive', action=argparse.BooleanOptionalAction, required=False, help='Non-interactive mode, mainly for CI usage'
+ )
args = parser.parse_args()
run(args)
diff --git a/nuget/PACKAGE.md b/nuget/PACKAGE.md
index 7daeed797..565db4226 100644
--- a/nuget/PACKAGE.md
+++ b/nuget/PACKAGE.md
@@ -22,7 +22,7 @@ You can call a high level `generate()` method to generate all of the output at o
// See https://aka.ms/new-console-template for more information
using Microsoft.ML.OnnxRuntimeGenAI;
-OgaHandle ogaHandle = new OgaHandle();
+using OgaHandle ogaHandle = new OgaHandle();
// Specify the location of your downloaded model.
// Many models are published on HuggingFace e.g.
diff --git a/test/csharp/TestOnnxRuntimeGenAIAPI.cs b/test/csharp/TestOnnxRuntimeGenAIAPI.cs
index 3633a8c17..3ea5fd663 100644
--- a/test/csharp/TestOnnxRuntimeGenAIAPI.cs
+++ b/test/csharp/TestOnnxRuntimeGenAIAPI.cs
@@ -5,28 +5,41 @@
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
+using System.Runtime.CompilerServices;
using Xunit;
using Xunit.Abstractions;
namespace Microsoft.ML.OnnxRuntimeGenAI.Tests
{
- public class TestsFixture : IDisposable
+ public class OnnxRuntimeGenAITests
{
- private OgaHandle _handle;
- public TestsFixture()
- {
- _handle = new OgaHandle();
- }
+ private readonly ITestOutputHelper output;
- public void Dispose()
+ private static string GetDirectoryInTreeThatContains(string currentDirectory, string targetDirectoryName)
{
- _handle.Dispose();
+ bool found = false;
+ foreach (string d in Directory.GetDirectories(currentDirectory, searchPattern: targetDirectoryName))
+ {
+ found = true;
+ return Path.Combine(currentDirectory, targetDirectoryName);
+ }
+ if (!found)
+ {
+ DirectoryInfo dirInfo = new DirectoryInfo(currentDirectory);
+ if (dirInfo.Parent != null)
+ {
+ return GetDirectoryInTreeThatContains(Path.GetFullPath(Path.Combine(currentDirectory, "..")), targetDirectoryName);
+ }
+ else
+ {
+ return null;
+ }
+ }
+ return null;
}
- }
- public class OnnxRuntimeGenAITests : IClassFixture
- {
- private readonly ITestOutputHelper output;
+ private static readonly string _phi2Path = Path.Combine(
+ GetDirectoryInTreeThatContains(Directory.GetCurrentDirectory(), "test"), "test_models", "phi-2", "int4", "cpu");
public OnnxRuntimeGenAITests(ITestOutputHelper o)
{
@@ -37,7 +50,7 @@ private class IgnoreOnModelAbsenceFact : FactAttribute
{
public IgnoreOnModelAbsenceFact()
{
- string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_models", "cpu", "phi-2");
+ string modelPath = _phi2Path;
bool exists = System.IO.Directory.Exists(modelPath);
if (!System.IO.Directory.Exists(modelPath))
{
@@ -125,7 +138,7 @@ public void TestTopKSearch()
float temp = 0.6f;
ulong maxLength = 20;
- string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_models", "cpu", "phi-2");
+ string modelPath = _phi2Path;
using (var model = new Model(modelPath))
{
Assert.NotNull(model);
@@ -167,7 +180,7 @@ public void TestTopPSearch()
float temp = 0.6f;
ulong maxLength = 20;
- string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_models", "cpu", "phi-2");
+ string modelPath = _phi2Path;
using (var model = new Model(modelPath))
{
Assert.NotNull(model);
@@ -210,7 +223,7 @@ public void TestTopKTopPSearch()
float temp = 0.6f;
ulong maxLength = 20;
- string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_models", "cpu", "phi-2");
+ string modelPath = _phi2Path;
using (var model = new Model(modelPath))
{
Assert.NotNull(model);
@@ -249,7 +262,7 @@ public void TestTopKTopPSearch()
[IgnoreOnModelAbsenceFact(DisplayName = "TestTokenizerBatchEncodeDecode")]
public void TestTokenizerBatchEncodeDecode()
{
- string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_models", "cpu", "phi-2");
+ string modelPath = _phi2Path;
using (var model = new Model(modelPath))
{
Assert.NotNull(model);
@@ -278,7 +291,7 @@ public void TestTokenizerBatchEncodeDecode()
[IgnoreOnModelAbsenceFact(DisplayName = "TestTokenizerBatchEncodeSingleDecode")]
public void TestTokenizerBatchEncodeSingleDecode()
{
- string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_models", "cpu", "phi-2");
+ string modelPath = _phi2Path;
using (var model = new Model(modelPath))
{
Assert.NotNull(model);
@@ -309,7 +322,7 @@ public void TestTokenizerBatchEncodeSingleDecode()
[IgnoreOnModelAbsenceFact(DisplayName = "TestTokenizerBatchEncodeStreamDecode")]
public void TestTokenizerBatchEncodeStreamDecode()
{
- string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_models", "cpu", "phi-2");
+ string modelPath = _phi2Path;
using (var model = new Model(modelPath))
{
Assert.NotNull(model);
@@ -345,7 +358,7 @@ public void TestTokenizerBatchEncodeStreamDecode()
[IgnoreOnModelAbsenceFact(DisplayName = "TestTokenizerSingleEncodeDecode")]
public void TestTokenizerSingleEncodeDecode()
{
- string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_models", "cpu", "phi-2");
+ string modelPath = _phi2Path;
using (var model = new Model(modelPath))
{
Assert.NotNull(model);
@@ -369,7 +382,7 @@ public void TestTokenizerSingleEncodeDecode()
[IgnoreOnModelAbsenceFact(DisplayName = "TestPhi2")]
public void TestPhi2()
{
- string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_models", "cpu", "phi-2");
+ string modelPath = _phi2Path;
using (var model = new Model(modelPath))
{
Assert.NotNull(model);