From ee72da4222c6ceb5d47e8a4887f6e224cdb18965 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Tue, 2 Jul 2024 13:12:27 -0700 Subject: [PATCH 1/2] commit the file changes so that we can see them running --- eng/common/testproxy/install-test-proxy.ps1 | 30 +++ .../testproxy/test-proxy-standalone-tool.yml | 83 +++++++ eng/common/testproxy/test-proxy.ps1 | 209 ++++++++++++++++++ 3 files changed, 322 insertions(+) create mode 100644 eng/common/testproxy/install-test-proxy.ps1 create mode 100644 eng/common/testproxy/test-proxy-standalone-tool.yml create mode 100644 eng/common/testproxy/test-proxy.ps1 diff --git a/eng/common/testproxy/install-test-proxy.ps1 b/eng/common/testproxy/install-test-proxy.ps1 new file mode 100644 index 0000000000..2cafdf5e0a --- /dev/null +++ b/eng/common/testproxy/install-test-proxy.ps1 @@ -0,0 +1,30 @@ +Set-StrictMode -Version 4 +<# +.SYNOPSIS +Installs a standalone version of the test-proxy for use in tests. This function is intended to be used in CI/CD pipelines, and leaves behind +the pipeline variable PROXY_EXE which contains the path to the test-proxy executable. + +.PARAMETER Version +The version of the proxy to install. Requires a full version to be provided. EG "1.0.0-dev.20240617.1" + +.PARAMETER Directory +The directory within which the test-proxy exe will exist after this function invokes. Defaults to CWD. +#> +param( + [Parameter(Mandatory = $true)] + $Version, + [Parameter(Mandatory = $true)] + $InstallDirectory, +) + +. (Join-Path $PSScriptRoot test-proxy.ps1) + +Install-Standalone-TestProxy -Version $Version -InstallDirectory $InstallDirectory + +if $(IsWindows) { + $PROXY_EXE = Join-Path $InstallDirectory "Azure.Sdk.Tools.TestProxy.exe" +} else { + $PROXY_EXE = Join-Path $InstallDirectory "Azure.Sdk.Tools.TestProxy" +} +Write-Host "Downloaded test-proxy available at $PROXY_EXE." +Write-Host "##vso[task.setvariable variable=PROXY_EXE]$PROXY_EXE" \ No newline at end of file diff --git a/eng/common/testproxy/test-proxy-standalone-tool.yml b/eng/common/testproxy/test-proxy-standalone-tool.yml new file mode 100644 index 0000000000..d85691bb35 --- /dev/null +++ b/eng/common/testproxy/test-proxy-standalone-tool.yml @@ -0,0 +1,83 @@ +# This template sets variable PROXY_PID to be used for shutdown later. +parameters: + rootFolder: '$(Build.SourcesDirectory)' + runProxy: true + targetVersion: '' + templateRoot: '$(Build.SourcesDirectory)' + condition: true + +steps: + - pwsh: | + ${{ parameters.templateRoot }}/eng/common/scripts/trust-proxy-certificate.ps1 + displayName: 'Language Specific Certificate Trust' + condition: and(succeeded(), ${{ parameters.condition }}) + + - task: PowerShell@2 + displayName: 'Override proxy version if necessary' + condition: and(succeeded(), ${{ parameters.condition }}, ne('${{ parameters.targetVersion }}', '')) + inputs: + targetType: filePath + filePath: '${{ parameters.templateRoot }}/eng/common/testproxy/scripts/override-proxy-version.ps1' + arguments: '-TargetVersion "${{ parameters.targetVersion }}"' + pwsh: true + + - pwsh: | + $standardVersion = "${{ parameters.templateRoot }}/eng/common/testproxy/target_version.txt" + $overrideVersion = "${{ parameters.templateRoot }}/eng/target_proxy_version.txt" + + $version = $(Get-Content $standardVersion -Raw).Trim() + + if (Test-Path $overrideVersion) { + $version = $(Get-Content $overrideVersion -Raw).Trim() + } + + Write-Host "Installing test-proxy version $version" + ${{ parameters.templateRoot }}/eng/common/testproxy/scripts/install-test-proxy.ps1 -Version $version -InstallDirectory $(Build.BinariesDirectory)/test-proxy + displayName: "Install test-proxy" + condition: and(succeeded(), ${{ parameters.condition }}) + + - pwsh: | + Write-Host "##vso[task.prependpath]$(Build.BinariesDirectory)/test-proxy" + displayName: "Prepend path with test-proxy tool install location" + + - ${{ if eq(parameters.runProxy, 'true') }}: + - pwsh: | + Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Path]${{ parameters.templateRoot }}/eng/common/testproxy/dotnet-devcert.pfx" + Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Password]password" + Write-Host "##vso[task.setvariable variable=PROXY_MANUAL_START]true" + displayName: 'Configure Kestrel and PROXY_MANUAL_START Variables' + condition: and(succeeded(), ${{ parameters.condition }}) + + - pwsh: | + $Process = Start-Process $(PROXY_EXE) ` + -ArgumentList "start -u --storage-location ${{ parameters.rootFolder }}" ` + -NoNewWindow -PassThru -RedirectStandardOutput ${{ parameters.rootFolder }}/test-proxy.log + + Write-Host "##vso[task.setvariable variable=PROXY_PID]$($Process.Id)" + displayName: 'Run the testproxy - windows' + condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT'), ${{ parameters.condition }}) + + # nohup does NOT continue beyond the current session if you use it within powershell + - bash: | + nohup $(PROXY_EXE) &>${{ parameters.rootFolder }}/test-proxy.log & + + echo $! > $(Build.SourcesDirectory)/test-proxy.pid + echo "##vso[task.setvariable variable=PROXY_PID]$(cat $(Build.SourcesDirectory)/test-proxy.pid)" + displayName: "Run the testproxy - linux/mac" + condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT'), ${{ parameters.condition }}) + workingDirectory: "${{ parameters.rootFolder }}" + + - pwsh: | + for ($i = 0; $i -lt 10; $i++) { + try { + Invoke-WebRequest -Uri "http://localhost:5000/Admin/IsAlive" | Out-Null + exit 0 + } catch { + Write-Warning "Failed to successfully connect to test proxy. Retrying..." + Start-Sleep 6 + } + } + Write-Error "Could not connect to test proxy." + exit 1 + displayName: Test Proxy IsAlive + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/common/testproxy/test-proxy.ps1 b/eng/common/testproxy/test-proxy.ps1 new file mode 100644 index 0000000000..0514c934de --- /dev/null +++ b/eng/common/testproxy/test-proxy.ps1 @@ -0,0 +1,209 @@ +Set-StrictMode -Version 4 +$AVAILABLE_TEST_PROXY_BINARIES = @{ + "Windows" = @{ + "AMD64" = @{ + "system" = "Windows" + "machine" = "AMD64" + "file_name" = "test-proxy-standalone-win-x64.zip" + "executable" = "Azure.Sdk.Tools.TestProxy.exe" + } + } + "Linux" = @{ + "X86_64" = @{ + "system" = "Linux" + "machine" = "X86_64" + "file_name" = "test-proxy-standalone-linux-x64.tar.gz" + "executable" = "Azure.Sdk.Tools.TestProxy" + } + "ARM64" = @{ + "system" = "Linux" + "machine" = "ARM64" + "file_name" = "test-proxy-standalone-linux-arm64.tar.gz" + "executable" = "Azure.Sdk.Tools.TestProxy" + } + } + "Darwin" = @{ + "X86_64" = @{ + "system" = "Darwin" + "machine" = "X86_64" + "file_name" = "test-proxy-standalone-osx-x64.zip" + "executable" = "Azure.Sdk.Tools.TestProxy" + } + "ARM64" = @{ + "system" = "Darwin" + "machine" = "ARM64" + "file_name" = "test-proxy-standalone-osx-arm64.zip" + "executable" = "Azure.Sdk.Tools.TestProxy" + } + } +} + +function Get-SystemArchitecture { + $unameOutput = uname -m + switch ($unameOutput) { + "x86_64" { return "X86_64" } + "aarch64" { return "ARM64" } + "arm64" { return "ARM64" } + default { throw "Unable to determine system architecture. uname -m returned $unameOutput." } + } +} + +function Get-Proxy-Meta () { + $ErrorActionPreferenceDefault = $ErrorActionPreference + $ErrorActionPreference = "Stop" + + $os = "unknown" + $machine = Get-SystemArchitecture + + if ($IsWindows) { + $os = "Windows" + # we only support x64 on windows, if that doesn't work the platform is unsupported + $machine = "AMD64" + } elseif ($IsLinux) { + $os = "Linux" + } elseif ($IsMacOS) { + $os = "Darwin" + } + + $ErrorActionPreference = $ErrorActionPreferenceDefault + + return $AVAILABLE_TEST_PROXY_BINARIES[$os][$machine] +} + +function Get-Proxy-Url ( + [Parameter(mandatory=$true)]$Version +) { + $systemDetails = Get-Proxy-Meta + + $file = $systemDetails.file_name + $url = "https://github.com/Azure/azure-sdk-tools/releases/download/Azure.Sdk.Tools.TestProxy_$Version/$file" + + return $url +} + + +function Cleanup-Directory ($path) { + if (Test-Path -Path $path) { + Remove-Item -Path $path -Recurse -Force + } + New-Item -ItemType Directory -Path $path -Force +} + +function Is-Work-Necessary ( + [Parameter(mandatory=$true)] + $Version, + [Parameter(mandatory=$true)] + $Directory +) { + $savedVersionTxt = Join-Path $Directory "downloaded_version.txt" + if (Test-Path $savedVersionTxt) { + $result = (Get-Content -Raw $savedVersionTxt).Trim() + + if ($result -eq $Version) { + return $false + } + } + + return $true +} + +<# +.SYNOPSIS +Installs the test-proxy as a dotnet tool installation. +.PARAMETER Version +The version of the proxy to install. Defaults to latest release. +.PARAMETER Feed +The feed from which to install the test-proxy. Defaults to public azure sdk for net official feed. +.PARAMETER TargetDirectory +If provided, the tool will be installed to this directory, rather than globally. +#> +function Install-Tool-TestProxy ( + $Version = "1.0.0-dev*", + $Feed = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json", + $TargetDirectory = $null +) { + + $adjustableArgs = "--global" + + if ($TargetDirectory -ne $null) { + $adjustableArgs = "--tool-path `"$TargetDirectory`"" + } + + Write-Host "dotnet tool update azure.sdk.tools.testproxy $adjustableArgs --add-source $Feed --version $Version --ignore-failed-sources" + dotnet tool update azure.sdk.tools.testproxy $adjustableArgs --add-source $Feed --version $Version --ignore-failed-sources +} + +<# +.SYNOPSIS +Installs a local copy of the test-proxy. +.PARAMETER ProjectDirectory +Where does the solution or project directory exist? Defaults to ".". +.PARAMETER TargetDirectory +If provided, the tool will be installed to this directory, rather than globally. +#> +function Install-Local-TestProxy ( + $ProjectDirectory = ".", + $TargetDirectory = $null +) { + $tmpDir = Join-Path -Path ([System.IO.Path]::GetTempPath()) -ChildPath ([System.Guid]::NewGuid().ToString()) + New-Item -ItemType Directory -Path $tmpDir + + $adjustableArgs = "--global" + if ($TargetDirectory -ne $null) { + $adjustableArgs = "--tool-path `"$TargetDirectory`"" + } + + dotnet pack /p:ArtifactsPackagesDir=$tmpDir -c Release $ProjectDirectory + Write-Host "Built Test-Proxy located in $tmpDir" + dotnet tool update azure.sdk.tools.testproxy $adjustableArgs --prerelease --add-source $tmpDir +} + +<# +.SYNOPSIS +Installs a standalone version of the test-proxy. +.PARAMETER Version +The version of the proxy to install. Requires a full version to be provided. EG "1.0.0-dev.20240617.1" +.PARAMETER Directory +The directory within which the test-proxy exe will exist after this function invokes. Defaults to "." +#> +function Install-Standalone-TestProxy ( + [Parameter(mandatory=$true)] + $Version, + $Directory="." +) { + $ErrorActionPreference = "Stop" + $systemDetails = Get-Proxy-Meta + $downloadFolder = Resolve-Path $Directory + $downloadUrl = Get-Proxy-Url $Version + $downloadFile = $downloadUrl.Split('/')[-1] + $downloadLocation = Join-Path $downloadFolder $downloadFile + $savedVersionTxt = Join-Path $downloadFolder "downloaded_version.txt" + + if (Is-Work-Necessary $version $downloadFolder) { + Write-Host "Commencing installation of `"$Version`" to `"$downloadFolder`" from $downloadUrl." + Invoke-WebRequest -Uri $downloadUrl -OutFile $downloadLocation + + if ($downloadFile -like "*.zip") { + Expand-Archive -Path $downloadLocation -DestinationPath $downloadFolder -Force + } elseif ($downloadFile -like "*.tar.gz") { + tar -xzf $downloadLocation -C $downloadFolder + } else { + throw "Unsupported file format" + } + + # Remove the downloaded file after extraction + Remove-Item -Path $downloadLocation -Force + + # Record downloaded version + Set-Content -Path $savedVersionTxt -Value $Version + + # Set executable permissions if on macOS (Darwin) + $executable_path = Join-Path $downloadFolder $systemDetails.executable + if ($IsMacOS) { + chmod 755 $executable_path + } + } + else { + Write-Host "Target version `"$Version`" already present in target directory `"$downloadFolder.`"" + } +} \ No newline at end of file From a79f3725c52872890de15326ad61e0d8b20db109 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Tue, 2 Jul 2024 16:08:32 -0700 Subject: [PATCH 2/2] use standalone tool --- eng/common/testproxy/install-test-proxy.ps1 | 13 ++-- .../testproxy/test-proxy-standalone-tool.yml | 2 +- eng/common/testproxy/test-proxy.ps1 | 59 ++----------------- 3 files changed, 15 insertions(+), 59 deletions(-) diff --git a/eng/common/testproxy/install-test-proxy.ps1 b/eng/common/testproxy/install-test-proxy.ps1 index 2cafdf5e0a..402e5ddc8c 100644 --- a/eng/common/testproxy/install-test-proxy.ps1 +++ b/eng/common/testproxy/install-test-proxy.ps1 @@ -1,4 +1,3 @@ -Set-StrictMode -Version 4 <# .SYNOPSIS Installs a standalone version of the test-proxy for use in tests. This function is intended to be used in CI/CD pipelines, and leaves behind @@ -14,17 +13,21 @@ param( [Parameter(Mandatory = $true)] $Version, [Parameter(Mandatory = $true)] - $InstallDirectory, + $InstallDirectory ) . (Join-Path $PSScriptRoot test-proxy.ps1) -Install-Standalone-TestProxy -Version $Version -InstallDirectory $InstallDirectory +Write-Host "Attempting to download and install version `"$Version`" into `"$InstallDirectory`"" -if $(IsWindows) { +Install-Standalone-TestProxy -Version $Version -Directory $InstallDirectory + +$PROXY_EXE = "" + +if ($IsWindows) { $PROXY_EXE = Join-Path $InstallDirectory "Azure.Sdk.Tools.TestProxy.exe" } else { $PROXY_EXE = Join-Path $InstallDirectory "Azure.Sdk.Tools.TestProxy" } Write-Host "Downloaded test-proxy available at $PROXY_EXE." -Write-Host "##vso[task.setvariable variable=PROXY_EXE]$PROXY_EXE" \ No newline at end of file +Write-Host "##vso[task.setvariable variable=PROXY_EXE]$PROXY_EXE" diff --git a/eng/common/testproxy/test-proxy-standalone-tool.yml b/eng/common/testproxy/test-proxy-standalone-tool.yml index d85691bb35..a836427ad7 100644 --- a/eng/common/testproxy/test-proxy-standalone-tool.yml +++ b/eng/common/testproxy/test-proxy-standalone-tool.yml @@ -32,7 +32,7 @@ steps: } Write-Host "Installing test-proxy version $version" - ${{ parameters.templateRoot }}/eng/common/testproxy/scripts/install-test-proxy.ps1 -Version $version -InstallDirectory $(Build.BinariesDirectory)/test-proxy + ${{ parameters.templateRoot }}/eng/common/testproxy/install-test-proxy.ps1 -Version $version -InstallDirectory $(Build.BinariesDirectory)/test-proxy displayName: "Install test-proxy" condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/common/testproxy/test-proxy.ps1 b/eng/common/testproxy/test-proxy.ps1 index 0514c934de..f1bf1eca8f 100644 --- a/eng/common/testproxy/test-proxy.ps1 +++ b/eng/common/testproxy/test-proxy.ps1 @@ -81,7 +81,6 @@ function Get-Proxy-Url ( return $url } - function Cleanup-Directory ($path) { if (Test-Path -Path $path) { Remove-Item -Path $path -Recurse -Force @@ -107,57 +106,6 @@ function Is-Work-Necessary ( return $true } -<# -.SYNOPSIS -Installs the test-proxy as a dotnet tool installation. -.PARAMETER Version -The version of the proxy to install. Defaults to latest release. -.PARAMETER Feed -The feed from which to install the test-proxy. Defaults to public azure sdk for net official feed. -.PARAMETER TargetDirectory -If provided, the tool will be installed to this directory, rather than globally. -#> -function Install-Tool-TestProxy ( - $Version = "1.0.0-dev*", - $Feed = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json", - $TargetDirectory = $null -) { - - $adjustableArgs = "--global" - - if ($TargetDirectory -ne $null) { - $adjustableArgs = "--tool-path `"$TargetDirectory`"" - } - - Write-Host "dotnet tool update azure.sdk.tools.testproxy $adjustableArgs --add-source $Feed --version $Version --ignore-failed-sources" - dotnet tool update azure.sdk.tools.testproxy $adjustableArgs --add-source $Feed --version $Version --ignore-failed-sources -} - -<# -.SYNOPSIS -Installs a local copy of the test-proxy. -.PARAMETER ProjectDirectory -Where does the solution or project directory exist? Defaults to ".". -.PARAMETER TargetDirectory -If provided, the tool will be installed to this directory, rather than globally. -#> -function Install-Local-TestProxy ( - $ProjectDirectory = ".", - $TargetDirectory = $null -) { - $tmpDir = Join-Path -Path ([System.IO.Path]::GetTempPath()) -ChildPath ([System.Guid]::NewGuid().ToString()) - New-Item -ItemType Directory -Path $tmpDir - - $adjustableArgs = "--global" - if ($TargetDirectory -ne $null) { - $adjustableArgs = "--tool-path `"$TargetDirectory`"" - } - - dotnet pack /p:ArtifactsPackagesDir=$tmpDir -c Release $ProjectDirectory - Write-Host "Built Test-Proxy located in $tmpDir" - dotnet tool update azure.sdk.tools.testproxy $adjustableArgs --prerelease --add-source $tmpDir -} - <# .SYNOPSIS Installs a standalone version of the test-proxy. @@ -173,6 +121,11 @@ function Install-Standalone-TestProxy ( ) { $ErrorActionPreference = "Stop" $systemDetails = Get-Proxy-Meta + + if (!(Test-Path $Directory) -and $Directory -ne ".") { + New-Item -ItemType Directory -Path $Directory -Force + } + $downloadFolder = Resolve-Path $Directory $downloadUrl = Get-Proxy-Url $Version $downloadFile = $downloadUrl.Split('/')[-1] @@ -206,4 +159,4 @@ function Install-Standalone-TestProxy ( else { Write-Host "Target version `"$Version`" already present in target directory `"$downloadFolder.`"" } -} \ No newline at end of file +}