diff --git a/BuildModule.tests.ps1 b/BuildModule.tests.ps1 new file mode 100644 index 000000000..a9d6afb3a --- /dev/null +++ b/BuildModule.tests.ps1 @@ -0,0 +1,153 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# these are tests for the build module + +import-module -force "./build.psm1" +Describe "Build Module Tests" { + Context "Global.json" { + BeforeAll { + $globalJson = Get-Content (Join-Path $PSScriptRoot global.json) | ConvertFrom-Json + $expectedVersion = $globalJson.sdk.version + $result = Get-GlobalJsonSdkVersion + } + $propertyTestcases = @{ Name = "Major"; Type = "System.Int32" }, + @{ Name = "Minor"; Type = "System.Int32" }, + @{ Name = "Patch"; Type = "System.Int32" }, + @{ Name = "PrereleaseLabel"; Type = "System.String" } + It "Get-GlobalJsonSdkVersion returns a portable version object with property '' with type ''" -TestCases $propertyTestcases { + param ( $Name, $Type ) + $result.psobject.properties[$Name] | Should -BeOfType [System.Management.Automation.PSNoteProperty] + $result.psobject.properties[$Name].TypeNameOfValue | Should -Be $Type + } + It "Can retrieve the version from global.json" { + $result = Get-GlobalJsonSdkVersion + $resultString = "{0}.{1}.{2}" -f $result.Major,$result.Minor,$result.Patch + if ( $result.prereleasestring ) { $resultString += "-" + $result.prereleasestring } + $resultString | Should -Be $expectedVersion + } + } + Context "Test-SuiteableDotnet" { + It "Test-SuitableDotnet should return true when the expected version matches the installed version" { + Test-SuitableDotnet -availableVersions 2.1.2 -requiredVersion 2.1.2 | Should -Be $true + } + It "Test-SuitableDotnet should return true when the expected version matches the available versions" { + Test-SuitableDotnet -availableVersions "2.1.1","2.1.2","2.1.3" -requiredVersion 2.1.2 | Should -Be $true + } + It "Test-SuitableDotnet should return false when the expected version does not match an available" { + Test-SuitableDotnet -availableVersions "2.2.100","2.2.300" -requiredVersion 2.2.200 | Should -Be $false + } + It "Test-SuitableDotnet should return false when the expected version does not match an available" { + Test-SuitableDotnet -availableVersions "2.2.100","2.2.300" -requiredVersion 2.2.105 | Should -Be $false + } + It "Test-SuitableDotnet should return true when the expected version matches an available" { + Test-SuitableDotnet -availableVersions "2.2.150","2.2.300" -requiredVersion 2.2.105 | Should -Be $true + } + It "Test-SuitableDotnet should return false when the expected version does not match an available" { + Test-SuitableDotnet -availableVersions "2.2.400","2.2.401","2.2.405" -requiredVersion "2.2.410" | Should -Be $false + } + } + + Context "Test-DotnetInstallation" { + BeforeAll { + $availableVersions = ConvertTo-PortableVersion -strVersion "2.2.400","2.2.401","2.2.405" + $foundVersion = ConvertTo-PortableVersion -strVersion 2.2.402 + $missingVersion = ConvertTo-PortableVersion -strVersion 2.2.410 + } + + It "Test-DotnetInstallation finds a good version" { + Mock Get-InstalledCLIVersion { return $availableVersions } + Mock Get-GlobalJSonSdkVersion { return $foundVersion } + $result = Test-DotnetInstallation -requestedVersion (Get-GlobalJsonSdkVersion) -installedVersions (Get-InstalledCLIVersion) + Assert-MockCalled "Get-InstalledCLIVersion" -Times 1 + Assert-MockCalled "Get-GlobalJsonSdkVersion" -Times 1 + $result | Should -Be $true + } + + It "Test-DotnetInstallation cannot find a good version should return false" { + Mock Get-InstalledCLIVersion { return $availableVersions } + Mock Get-GlobalJSonSdkVersion { return $missingVersion } + $result = Test-DotnetInstallation -requestedVersion (Get-GlobalJsonSdkVersion) -installedVersions (Get-InstalledCLIVersion) + Assert-MockCalled "Get-InstalledCLIVersion" -Times 1 + Assert-MockCalled "Get-GlobalJsonSdkVersion" -Times 1 + $result | Should -Be $false + } + } + + Context "Receive-DotnetInstallScript" { + + Mock -ModuleName Build Receive-File { new-item -type file TestDrive:/dotnet-install.sh } + It "Downloads the proper non-Windows file" { + try { + push-location TestDrive: + Receive-DotnetInstallScript -platform NonWindows + "TestDrive:/dotnet-install.sh" | Should -Exist + } + finally { + Pop-Location + } + } + + Mock -ModuleName Build Receive-File { new-item -type file TestDrive:/dotnet-install.ps1 } + It "Downloads the proper file Windows file" { + try { + push-location TestDrive: + Receive-DotnetInstallScript -platform "Windows" + "TestDrive:/dotnet-install.ps1" | Should -Exist + } + finally { + Pop-Location + } + } + + } + + Context "Test result functions" { + BeforeAll { + $xmlFile = @' + + + + + + + + + + + + + + Expected 2, but got 1. + at <ScriptBlock>, /tmp/bad.tests.ps1: line 3 +3: It "a failing test" { 1 | Should -Be 2 } + + + + + + + + + +'@ + + $xmlFile | out-file TESTDRIVE:/results.xml + $results = Get-TestResults -logfile TESTDRIVE:/results.xml + $failures = Get-TestFailures -logfile TESTDRIVE:/results.xml + } + + It "Get-TestResults finds 2 results" { + $results.Count | Should -Be 2 + } + It "Get-TestResults finds 1 pass" { + @($results | ?{ $_.result -eq "Success" }).Count |Should -Be 1 + } + It "Get-TestResults finds 1 failure" { + @($results | ?{ $_.result -eq "Failure" }).Count |Should -Be 1 + } + It "Get-TestFailures finds 1 failure" { + $failures.Count | Should -Be 1 + } + } +} diff --git a/build.ps1 b/build.ps1 index 2473146e2..e0382f6cf 100644 --- a/build.ps1 +++ b/build.ps1 @@ -31,7 +31,10 @@ param( [switch] $Test, [Parameter(ParameterSetName='Test')] - [switch] $InProcess + [switch] $InProcess, + + [Parameter(ParameterSetName='Bootstrap')] + [switch] $Bootstrap ) END { @@ -58,6 +61,10 @@ END { } Start-ScriptAnalyzerBuild @buildArgs } + "Bootstrap" { + Install-DotNet + return + } "Test" { Test-ScriptAnalyzer -InProcess:$InProcess return diff --git a/build.psm1 b/build.psm1 index d283eef00..e9837479d 100644 --- a/build.psm1 +++ b/build.psm1 @@ -21,7 +21,7 @@ function Publish-File # attempt to get the users module directory function Get-UserModulePath { - if ( $IsCoreCLR -and ! $IsWindows ) + if ( $IsCoreCLR -and -not $IsWindows ) { $platformType = "System.Management.Automation.Platform" -as [Type] if ( $platformType ) { @@ -104,7 +104,7 @@ function Start-DocumentationBuild throw "Cannot find markdown documentation folder." } Import-Module platyPS - if ( ! (Test-Path $outputDocsPath)) { + if ( -not (Test-Path $outputDocsPath)) { $null = New-Item -Type Directory -Path $outputDocsPath -Force } $null = New-ExternalHelp -Path $markdownDocsPath -OutputPath $outputDocsPath -Force @@ -126,7 +126,26 @@ function Start-ScriptAnalyzerBuild [switch]$Documentation ) + BEGIN { + # don't allow the build to be started unless we have the proper Cli version + # this will not actually install dotnet if it's already present, but it will + # install the proper version + Install-Dotnet + if ( -not (Test-SuitableDotnet) ) { + $requiredVersion = Get-GlobalJsonSdkVersion + $foundVersion = Get-InstalledCLIVersion + Write-Warning "No suitable dotnet CLI found, requires version '$requiredVersion' found only '$foundVersion'" + } + } END { + + # Build docs either when -Documentation switch is being specified or the first time in a clean repo + $documentationFileExists = Test-Path (Join-Path $PSScriptRoot 'out\PSScriptAnalyzer\en-us\Microsoft.Windows.PowerShell.ScriptAnalyzer.dll-Help.xml') + if ( $Documentation -or -not $documentationFileExists ) + { + Start-DocumentationBuild + } + if ( $All ) { # Build all the versions of the analyzer @@ -136,13 +155,6 @@ function Start-ScriptAnalyzerBuild return } - $documentationFileExists = Test-Path (Join-Path $PSScriptRoot 'out\PSScriptAnalyzer\en-us\Microsoft.Windows.PowerShell.ScriptAnalyzer.dll-Help.xml') - # Build docs either when -Documentation switch is being specified or the first time in a clean repo - if ( $Documentation -or -not $documentationFileExists ) - { - Start-DocumentationBuild - } - if ($PSVersion -ge 6) { $framework = 'netstandard2.0' } @@ -193,7 +205,10 @@ function Start-ScriptAnalyzerBuild try { Push-Location $projectRoot/Rules Write-Progress "Building ScriptAnalyzer for PSVersion '$PSVersion' using framework '$framework' and configuration '$Configuration'" - $buildOutput = dotnet build --framework $framework --configuration "$config" + if ( -not $script:DotnetExe ) { + $script:DotnetExe = Get-DotnetExe + } + $buildOutput = & $script:DotnetExe build --framework $framework --configuration "$config" 2>&1 if ( $LASTEXITCODE -ne 0 ) { throw "$buildOutput" } } catch { @@ -271,3 +286,286 @@ function Get-TestFailures $results = [xml](Get-Content $logPath) $results.SelectNodes(".//test-case[@result='Failure']") } + +# BOOTSTRAPPING CODE FOR INSTALLING DOTNET +# install dotnet cli tools based on the version mentioned in global.json +function Install-Dotnet +{ + [CmdletBinding(SupportsShouldProcess=$true)] + param ( + [Parameter()][Switch]$Force, + [Parameter()]$version = $( Get-GlobalJsonSdkVersion -Raw ) + ) + + if ( Test-DotnetInstallation -requestedversion $version ) { + if ( $Force ) { + Write-Verbose -Verbose "Installing again" + } + else { + return + } + } + + try { + Push-Location $PSScriptRoot + $installScriptPath = Receive-DotnetInstallScript + $installScriptName = [System.IO.Path]::GetFileName($installScriptPath) + If ( $PSCmdlet.ShouldProcess("$installScriptName for $version")) { + & "${installScriptPath}" -c release -version $version + } + # this may be the first time that dotnet has been installed, + # set up the executable variable + if ( -not $script:DotnetExe ) { + $script:DotnetExe = Get-DotnetExe + } + } + catch { + throw $_ + } + finally { + if ( Test-Path $installScriptPath ) { + Remove-Item $installScriptPath + } + Pop-Location + } +} + +function Get-GlobalJsonSdkVersion { + param ( [switch]$Raw ) + $json = Get-Content -raw (Join-Path $PSScriptRoot global.json) | ConvertFrom-Json + $version = $json.sdk.Version + if ( $Raw ) { + return $version + } + else { + ConvertTo-PortableVersion $version + } +} + +# we don't have semantic version in earlier versions of PowerShell, so we need to +# create something that we can use +function ConvertTo-PortableVersion { + param ( [string[]]$strVersion ) + if ( -not $strVersion ) { + return (ConvertTo-PortableVersion "0.0.0-0") + } + foreach ( $v in $strVersion ) { + $ver, $pre = $v.split("-",2) + try { + [int]$major, [int]$minor, [int]$patch, $unused = $ver.Split(".",4) + if ( -not $pre ) { + $pre = $unused + } + } + catch { + Write-Warning "Cannot convert '$v' to portable version" + continue + } + $h = @{ + Major = $major + Minor = $minor + Patch = $patch + } + if ( $pre ) { + $h['PrereleaseLabel'] = $pre + } + else { + $h['PrereleaseLabel'] = [String]::Empty + } + $customObject = [pscustomobject]$h + # we do this so we can get an approximate sort, since this implements a pseudo-version + # type in script, we need a way to find the highest version of dotnet, it's not a great solution + # but it will work in most cases. + Add-Member -inputobject $customObject -Type ScriptMethod -Name ToString -Force -Value { + $str = "{0:0000}.{1:0000}.{2:0000}" -f $this.Major,$this.Minor,$this.Patch + if ( $this.PrereleaseLabel ) { + $str += "-{0}" -f $this.PrereleaseLabel + } + return $str + } + Add-Member -inputobject $customObject -Type ScriptMethod -Name IsContainedIn -Value { + param ( [object[]]$collection ) + foreach ( $object in $collection ) { + if ( + $this.Major -eq $object.Major -and + $this.Minor -eq $object.Minor -and + $this.Patch -eq $object.Patch -and + $this.PrereleaseLabel -eq $object.PrereleaseLabel + ) { + return $true + } + } + return $false + } + $customObject + } +} + +# see https://docs.microsoft.com/en-us/dotnet/core/tools/global-json for rules +# on how version checks are done +function Test-SuitableDotnet { + param ( + $availableVersions = $( Get-InstalledCliVersion), + $requiredVersion = $( Get-GlobalJsonSdkVersion ) + ) + + if ( $requiredVersion -is [String] -or $requiredVersion -is [Version] ) { + $requiredVersion = ConvertTo-PortableVersion "$requiredVersion" + } + + $availableVersionList = $availableVersions | ForEach-Object { if ( $_ -is [string] -or $_ -is [version] ) { ConvertTo-PortableVersion $_ } else { $_ } } + $availableVersions = $availableVersionList + # if we have what was requested, we can use it + if ( $RequiredVersion.IsContainedIn($availableVersions)) { + return $true + } + # if we had found a match, we would have returned $true above + # exact match required for 2.1.100 through 2.1.201 + if ( $RequiredVersion.Major -eq 2 -and $RequiredVersion.Minor -eq 1 -and $RequiredVersion.Patch -ge 100 -and $RequiredVersion.Patch -le 201 ) { + return $false + } + # we need to check each available version for something that's useable + foreach ( $version in $availableVersions ) { + # major/minor numbers don't match - keep looking + if ( $version.Major -ne $requiredVersion.Major -or $version.Minor -ne $requiredVersion.Minor ) { + continue + } + $requiredPatch = $requiredVersion.Patch + $possiblePatch = $version.Patch + + if ( $requiredPatch -gt $possiblePatch ) { + continue + } + if ( [math]::Abs(($requiredPatch - $possiblePatch)) -lt 100 ) { + return $true + } + } + return $false +} + +# these are mockable functions for testing +function Get-InstalledCLIVersion { + # dotnet might not have been installed _ever_, so just return 0.0.0.0 + if ( -not $script:DotnetExe ) { + Write-Warning "Dotnet executable not found" + return (ConvertTo-PortableVersion 0.0.0) + } + try { + # earlier versions of dotnet do not support --list-sdks, so we'll check the output + # and use dotnet --version as a fallback + $sdkList = & $script:DotnetExe --list-sdks 2>&1 + $sdkList = "Unknown option" + if ( $sdkList -match "Unknown option" ) { + $installedVersions = & $script:DotnetExe --version 2>$null + } + else { + $installedVersions = $sdkList | Foreach-Object { $_.Split()[0] } + } + } + catch { + Write-Verbose -Verbose "$_" + $installedVersions = & $script:DotnetExe --version 2>$null + } + return (ConvertTo-PortableVersion $installedVersions) +} + +function Test-DotnetInstallation +{ + param ( + $requestedVersion = $( Get-GlobalJsonSdkVersion ), + $installedVersions = $( Get-InstalledCLIVersion ) + ) + return (Test-SuitableDotnet -availableVersions $installedVersions -requiredVersion $requestedVersion ) +} + +function Receive-File { + param ( [Parameter(Mandatory,Position=0)]$uri ) + + # enable Tls12 for the request + # -SslProtocol parameter for Invoke-WebRequest wasn't in PSv3 + $securityProtocol = [System.Net.ServicePointManager]::SecurityProtocol + $tls12 = [System.Net.SecurityProtocolType]::Tls12 + try { + if ( ([System.Net.ServicePointManager]::SecurityProtocol -band $tls12) -eq 0 ) { + [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor $tls12 + } + $null = Invoke-WebRequest -Uri ${uri} -OutFile "${installScriptName}" + } + finally { + [System.Net.ServicePointManager]::SecurityProtocol = $securityProtocol + } + if ( (Test-Path Variable:IsWindows) -and -not $IsWindows ) { + chmod +x $installScriptName + } + $installScript = Get-Item $installScriptName -ErrorAction Stop + if ( -not $installScript ) { + throw "Download failure of ${uri}" + } + return $installScript +} + +function Receive-DotnetInstallScript +{ + # param '$platform' is a hook to enable forcing download of a specific + # install script, generally it should not be used except in testing. + param ( $platform = "" ) + + # if $platform has been set, it has priority + # if it's not set to Windows or NonWindows, it will be ignored + if ( $platform -eq "Windows" ) { + $installScriptName = "dotnet-install.ps1" + } + elseif ( $platform -eq "NonWindows" ) { + $installScriptName = "dotnet-install.sh" + } + elseif ( ((Test-Path Variable:IsWindows) -and -not $IsWindows) ) { + # if the variable IsWindows exists and it is set to false + $installScriptName = "dotnet-install.sh" + } + else { # the default case - we're running on a Windows system + $installScriptName = "dotnet-install.ps1" + } + $uri = "https://dot.net/v1/${installScriptName}" + + $installScript = Receive-File -Uri $uri + return $installScript.FullName +} + +function Get-DotnetExe +{ + $discoveredDotnet = Get-Command -CommandType Application dotnet -ErrorAction SilentlyContinu + if ( $discoveredDotnet ) { + # it's possible that there are multiples. Take the highest version we find + # the problem is that invoking dotnet on a version which is lower than the specified + # version in global.json will produce an error, so we can only take the dotnet which executes + $latestDotnet = $discoveredDotNet | + Where-Object { try { & $_ --version 2>$null } catch { } } | + Sort-Object { $pv = ConvertTo-PortableVersion (& $_ --version 2>$null); "$pv" } | + Select-Object -Last 1 + if ( $latestDotnet ) { + $script:DotnetExe = $latestDotnet + return $latestDotnet + } + } + # it's not in the path, try harder to find it by checking some usual places + if ( ! (test-path variable:IsWindows) -or $IsWindows ) { + $dotnetHuntPath = "$HOME\AppData\Local\Microsoft\dotnet\dotnet.exe" + Write-Verbose -Verbose "checking Windows $dotnetHuntPath" + if ( test-path $dotnetHuntPath ) { + $script:DotnetExe = $dotnetHuntPath + return $dotnetHuntPath + } + } + else { + $dotnetHuntPath = "$HOME/.dotnet/dotnet" + Write-Verbose -Verbose "checking non-Windows $dotnetHuntPath" + if ( test-path $dotnetHuntPath ) { + $script:DotnetExe = $dotnetHuntPath + return $dotnetHuntPath + } + } + + Write-Warning "Could not find dotnet executable" + return [String]::Empty +} +$script:DotnetExe = Get-DotnetExe diff --git a/tools/appveyor.psm1 b/tools/appveyor.psm1 index d9cbf3663..6367dda7d 100644 --- a/tools/appveyor.psm1 +++ b/tools/appveyor.psm1 @@ -5,7 +5,7 @@ $ErrorActionPreference = 'Stop' # Implements the AppVeyor 'install' step and installs the required versions of Pester, platyPS and the .Net Core SDK if needed. function Invoke-AppVeyorInstall { - $requiredPesterVersion = '4.4.1' + $requiredPesterVersion = '4.4.4' $pester = Get-Module Pester -ListAvailable | Where-Object { $_.Version -eq $requiredPesterVersion } if ($null -eq $pester) { if ($null -eq (Get-Module -ListAvailable PowershellGet)) { @@ -31,36 +31,11 @@ function Invoke-AppVeyorInstall { Install-Module -Name platyPS -Force -Scope CurrentUser -RequiredVersion $platyPSVersion } - # the legacy WMF4 image only has the old preview SDKs of dotnet - $globalDotJson = Get-Content (Join-Path $PSScriptRoot '..\global.json') -Raw | ConvertFrom-Json - $requiredDotNetCoreSDKVersion = $globalDotJson.sdk.version - if ($PSVersionTable.PSVersion.Major -gt 4) { - $requiredDotNetCoreSDKVersionPresent = (dotnet --list-sdks) -match $requiredDotNetCoreSDKVersion - } - else { - # WMF 4 image has old SDK that does not have --list-sdks parameter - $requiredDotNetCoreSDKVersionPresent = (dotnet --version).StartsWith($requiredDotNetCoreSDKVersion) - } - if (-not $requiredDotNetCoreSDKVersionPresent) { - Write-Verbose -Verbose "Installing required .Net CORE SDK $requiredDotNetCoreSDKVersion" - $originalSecurityProtocol = [Net.ServicePointManager]::SecurityProtocol - try { - [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 - if ($IsLinux -or $isMacOS) { - Invoke-WebRequest 'https://dot.net/v1/dotnet-install.sh' -OutFile dotnet-install.sh - bash dotnet-install.sh --version $requiredDotNetCoreSDKVersion - [System.Environment]::SetEnvironmentVariable('PATH', "/home/appveyor/.dotnet$([System.IO.Path]::PathSeparator)$PATH") - } - else { - Invoke-WebRequest 'https://dot.net/v1/dotnet-install.ps1' -OutFile dotnet-install.ps1 - .\dotnet-install.ps1 -Version $requiredDotNetCoreSDKVersion - } - } - finally { - [Net.ServicePointManager]::SecurityProtocol = $originalSecurityProtocol - Remove-Item .\dotnet-install.* - } - } + # the build script sorts out the problems of WMF4 and earlier versions of dotnet CLI + Write-Verbose -Verbose "Installing required .Net CORE SDK" + Write-Verbose "& $buildScriptDir/build.ps1 -bootstrap" + $buildScriptDir = (Resolve-Path "$PSScriptRoot/..").Path + & "$buildScriptDir/build.ps1" -bootstrap } # Implements AppVeyor 'test_script' step @@ -71,14 +46,19 @@ function Invoke-AppveyorTest { $CheckoutPath ) + # enforce the language to utf-8 to avoid issues + $env:LANG = "en_US.UTF-8" Write-Verbose -Verbose ("Running tests on PowerShell version " + $PSVersionTable.PSVersion) + Write-Verbose -Verbose "Language set to '${env:LANG}'" $modulePath = $env:PSModulePath.Split([System.IO.Path]::PathSeparator) | Where-Object { Test-Path $_} | Select-Object -First 1 Copy-Item "${CheckoutPath}\out\PSScriptAnalyzer" "$modulePath\" -Recurse -Force - $testResultsFile = ".\TestResults.xml" + $testResultsPath = Join-Path ${CheckoutPath} TestResults.xml $testScripts = "${CheckoutPath}\Tests\Engine","${CheckoutPath}\Tests\Rules","${CheckoutPath}\Tests\Documentation" - $testResults = Invoke-Pester -Script $testScripts -OutputFormat NUnitXml -OutputFile $testResultsFile -PassThru - (New-Object 'System.Net.WebClient').UploadFile("https://ci.appveyor.com/api/testresults/nunit/${env:APPVEYOR_JOB_ID}", (Resolve-Path $testResultsFile)) + $uploadUrl = "https://ci.appveyor.com/api/testresults/nunit/${env:APPVEYOR_JOB_ID}" + $testResults = Invoke-Pester -Script $testScripts -OutputFormat NUnitXml -OutputFile $testResultsPath -PassThru + Write-Verbose -Verbose "Uploading test results '$testResultsPath' to '${uploadUrl}'" + [byte[]]$response = (New-Object 'System.Net.WebClient').UploadFile("$uploadUrl" , $testResultsPath) if ($testResults.FailedCount -gt 0) { throw "$($testResults.FailedCount) tests failed." } @@ -91,6 +71,8 @@ function Invoke-AppveyorFinish { Add-Type -AssemblyName 'System.IO.Compression.FileSystem' [System.IO.Compression.ZipFile]::CreateFromDirectory((Join-Path $pwd 'out'), $zipFile) @( + # add test results as an artifact + (Get-ChildItem TestResults.xml) # You can add other artifacts here (Get-ChildItem $zipFile) ) | ForEach-Object { Push-AppveyorArtifact $_.FullName }