From d8a42da2fa1d9ba4635531a470deb5e018a4b02f Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 6 Dec 2024 15:37:36 -0800 Subject: [PATCH 1/8] always check for installation at least once, but only once --- eng/common/scripts/Helpers/Package-Helpers.ps1 | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1 index f1ea6ef91f2..69ea3472662 100644 --- a/eng/common/scripts/Helpers/Package-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Package-Helpers.ps1 @@ -46,7 +46,6 @@ function GetDocsTocDisplayName($pkg) { return $displayName } - <# .SYNOPSIS This function is a safe wrapper around `yq` and `ConvertFrom-Yaml` to convert YAML content to a PowerShell HashTable object @@ -68,7 +67,7 @@ Get-Content -Raw path/to/file.yml | CompatibleConvertFrom-Yaml #> function CompatibleConvertFrom-Yaml { param( - [Parameter(Mandatory=$true, ValueFromPipeline=$true)] + [Parameter(Mandatory = $true, ValueFromPipeline = $true)] [string]$Content ) @@ -76,20 +75,13 @@ function CompatibleConvertFrom-Yaml { throw "Content to parse is a required input." } - # Initialize any variables or checks that need to be done once - $yqPresent = Get-Command 'yq' -ErrorAction SilentlyContinue - if (-not $yqPresent) { + if (-not $script:IsYamlInstalled) { . (Join-Path $PSScriptRoot PSModule-Helpers.ps1) Install-ModuleIfNotInstalled -WhatIf:$false "powershell-yaml" "0.4.7" | Import-Module + $script:IsYamlInstalled = $true } - # Process the content (for example, you could convert from YAML here) - if ($yqPresent) { - return ($content | yq -o=json | ConvertFrom-Json -AsHashTable) - } - else { - return ConvertFrom-Yaml $content - } + return ConvertFrom-Yaml $content } <# From c8a665851270ebef488e4a911869eff6c6fb300c Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 6 Dec 2024 16:20:16 -0800 Subject: [PATCH 2/8] when in strict mode, the variable needs to be defined ahead of time --- eng/common/scripts/Helpers/Package-Helpers.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1 index 69ea3472662..5ac46209b44 100644 --- a/eng/common/scripts/Helpers/Package-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Package-Helpers.ps1 @@ -46,6 +46,7 @@ function GetDocsTocDisplayName($pkg) { return $displayName } +$script:IsYamlInstalled = $false <# .SYNOPSIS This function is a safe wrapper around `yq` and `ConvertFrom-Yaml` to convert YAML content to a PowerShell HashTable object From a2a22cf547b1bf4c6aa85b50439c8fa8bd58baab Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 11 Dec 2024 12:12:35 -0800 Subject: [PATCH 3/8] move the short-circuiting of install to PSModule-Helpers --- eng/common/scripts/Helpers/PSModule-Helpers.ps1 | 14 +++++++++++++- eng/common/scripts/Helpers/Package-Helpers.ps1 | 7 +------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 index 34dac96f8b1..ef75ffe9d0c 100644 --- a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 +++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 @@ -62,8 +62,18 @@ function Get-ModuleRepositories([string]$moduleName) { return $repoUrls } +if ($null -eq $script:InstalledModules) { + $script:InstalledModules = @{} +} + function moduleIsInstalled([string]$moduleName, [string]$version) { - $modules = (Get-Module -ListAvailable $moduleName) + if ($script:InstalledModules.ContainsKey("${moduleName}")) { + $modules = @($script:InstalledModules["${moduleName}"]) + } + else { + $modules = (Get-Module -ListAvailable $moduleName) + } + if ($version -as [Version]) { $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version }) if ($modules.Count -gt 0) @@ -102,6 +112,8 @@ function installModule([string]$moduleName, [string]$version, $repoUrl) { throw "Failed to install module $moduleName with version $version" } + $script:InstalledModules["${moduleName}"] = $modules + # Unregister repository as it can cause overlap issues with `dotnet tool install` # commands using the same devops feed Unregister-PSRepository -Name $repoUrl | Out-Null diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1 index 5ac46209b44..477e1bcdd19 100644 --- a/eng/common/scripts/Helpers/Package-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Package-Helpers.ps1 @@ -46,7 +46,6 @@ function GetDocsTocDisplayName($pkg) { return $displayName } -$script:IsYamlInstalled = $false <# .SYNOPSIS This function is a safe wrapper around `yq` and `ConvertFrom-Yaml` to convert YAML content to a PowerShell HashTable object @@ -76,11 +75,7 @@ function CompatibleConvertFrom-Yaml { throw "Content to parse is a required input." } - if (-not $script:IsYamlInstalled) { - . (Join-Path $PSScriptRoot PSModule-Helpers.ps1) - Install-ModuleIfNotInstalled -WhatIf:$false "powershell-yaml" "0.4.7" | Import-Module - $script:IsYamlInstalled = $true - } + Install-ModuleIfNotInstalled -WhatIf:$false "powershell-yaml" "0.4.7" | Import-Module return ConvertFrom-Yaml $content } From 49817103ab5771887d6dfa42dc7e4358b7fa972f Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 11 Dec 2024 13:03:27 -0800 Subject: [PATCH 4/8] now take advantage of a local script variable that remembers which modules have been installed --- .../scripts/Helpers/PSModule-Helpers.ps1 | 40 ++++++++++--------- .../scripts/Helpers/Package-Helpers.ps1 | 19 ++++----- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 index ef75ffe9d0c..d6725b35d1f 100644 --- a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 +++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 @@ -1,7 +1,6 @@ $global:CurrentUserModulePath = "" -function Update-PSModulePathForCI() -{ +function Update-PSModulePathForCI() { # Information on PSModulePath taken from docs # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_psmodulepath @@ -11,7 +10,8 @@ function Update-PSModulePathForCI() if ($IsWindows) { $hostedAgentModulePath = $env:SystemDrive + "\Modules" $moduleSeperator = ";" - } else { + } + else { $hostedAgentModulePath = "/usr/share" $moduleSeperator = ":" } @@ -55,29 +55,30 @@ function Get-ModuleRepositories([string]$moduleName) { $repoUrls = if ($packageFeedOverrides.Contains("${moduleName}")) { @($packageFeedOverrides["${moduleName}"], $DefaultPSRepositoryUrl) - } else { + } + else { @($DefaultPSRepositoryUrl) } return $repoUrls } -if ($null -eq $script:InstalledModules) { - $script:InstalledModules = @{} -} - function moduleIsInstalled([string]$moduleName, [string]$version) { + if ($null -eq $global:InstalledModules) { + $script:InstalledModules = @{} + } + if ($script:InstalledModules.ContainsKey("${moduleName}")) { - $modules = @($script:InstalledModules["${moduleName}"]) + $modules = $script:InstalledModules["${moduleName}"] } else { $modules = (Get-Module -ListAvailable $moduleName) + $script:InstalledModules["${moduleName}"] = $modules } if ($version -as [Version]) { $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version }) - if ($modules.Count -gt 0) - { + if ($modules.Count -gt 0) { Write-Host "Using module $($modules[0].Name) with version $($modules[0].Version)." return $modules[0] } @@ -87,8 +88,7 @@ function moduleIsInstalled([string]$moduleName, [string]$version) { function installModule([string]$moduleName, [string]$version, $repoUrl) { $repo = (Get-PSRepository).Where({ $_.SourceLocation -eq $repoUrl }) - if ($repo.Count -eq 0) - { + if ($repo.Count -eq 0) { Register-PSRepository -Name $repoUrl -SourceLocation $repoUrl -InstallationPolicy Trusted | Out-Null $repo = (Get-PSRepository).Where({ $_.SourceLocation -eq $repoUrl }) if ($repo.Count -eq 0) { @@ -123,8 +123,7 @@ function installModule([string]$moduleName, [string]$version, $repoUrl) { # Manual test at eng/common-tests/psmodule-helpers/Install-Module-Parallel.ps1 # If we want to use another default repository other then PSGallery we can update the default parameters -function Install-ModuleIfNotInstalled() -{ +function Install-ModuleIfNotInstalled() { [CmdletBinding(SupportsShouldProcess = $true)] param( [string]$moduleName, @@ -149,12 +148,14 @@ function Install-ModuleIfNotInstalled() foreach ($url in $repoUrls) { try { $module = installModule -moduleName $moduleName -version $version -repoUrl $url - } catch { + } + catch { if ($url -ne $repoUrls[-1]) { Write-Warning "Failed to install powershell module from '$url'. Retrying with fallback repository" Write-Warning $_ continue - } else { + } + else { Write-Warning "Failed to install powershell module from $url" throw } @@ -163,7 +164,8 @@ function Install-ModuleIfNotInstalled() } Write-Host "Using module '$($module.Name)' with version '$($module.Version)'." - } finally { + } + finally { $mutex.ReleaseMutex() } @@ -171,5 +173,5 @@ function Install-ModuleIfNotInstalled() } if ($null -ne $env:SYSTEM_TEAMPROJECTID) { - Update-PSModulePathForCI + Update-PSModulePathForCI } diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1 index 477e1bcdd19..81bf9038539 100644 --- a/eng/common/scripts/Helpers/Package-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Package-Helpers.ps1 @@ -75,6 +75,7 @@ function CompatibleConvertFrom-Yaml { throw "Content to parse is a required input." } + . (Join-Path $PSScriptRoot PSModule-Helpers.ps1) Install-ModuleIfNotInstalled -WhatIf:$false "powershell-yaml" "0.4.7" | Import-Module return ConvertFrom-Yaml $content @@ -102,7 +103,7 @@ LoadFrom-Yaml -YmlFile path/to/file.yml #> function LoadFrom-Yaml { param( - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] [string]$YmlFile ) if (Test-Path -Path $YmlFile) { @@ -147,19 +148,19 @@ GetValueSafelyFrom-Yaml -YamlContentAsHashtable $YmlFileContent -Keys @("extends #> function GetValueSafelyFrom-Yaml { param( - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] $YamlContentAsHashtable, - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] [string[]]$Keys ) $current = $YamlContentAsHashtable foreach ($key in $Keys) { - if ($current.ContainsKey($key) -or $current[$key]) { - $current = $current[$key] - } - else { - return $null - } + if ($current.ContainsKey($key) -or $current[$key]) { + $current = $current[$key] + } + else { + return $null + } } return [object]$current From 5ff01525b5ffe780fadeb48cc5751ec6e7bd4891 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 11 Dec 2024 13:09:42 -0800 Subject: [PATCH 5/8] $global -> $script --- eng/common/scripts/Helpers/PSModule-Helpers.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 index d6725b35d1f..11275a228d8 100644 --- a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 +++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 @@ -64,7 +64,7 @@ function Get-ModuleRepositories([string]$moduleName) { } function moduleIsInstalled([string]$moduleName, [string]$version) { - if ($null -eq $global:InstalledModules) { + if ($null -eq $script:InstalledModules) { $script:InstalledModules = @{} } From ef631d5b46e3f7bc42f7b718b18a558484dae13f Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 11 Dec 2024 13:57:01 -0800 Subject: [PATCH 6/8] just take ben's suggestion --- eng/common/scripts/Helpers/PSModule-Helpers.ps1 | 8 ++++++++ eng/common/scripts/Helpers/Package-Helpers.ps1 | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 index 11275a228d8..86505fa423e 100644 --- a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 +++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 @@ -121,6 +121,14 @@ function installModule([string]$moduleName, [string]$version, $repoUrl) { return $modules[0] } +function InstallAndImport-ModuleIfNotInstalled([string]$module, [string]$version) { + if ($null -eq (moduleIsInstalled $module $version)) { + Install-ModuleIfNotInstalled -WhatIf:$false $module $version | Import-Module + } elseif (!(Get-Module -Name $module)) { + Import-Module $module + } +} + # Manual test at eng/common-tests/psmodule-helpers/Install-Module-Parallel.ps1 # If we want to use another default repository other then PSGallery we can update the default parameters function Install-ModuleIfNotInstalled() { diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1 index 81bf9038539..e40fb41ff4a 100644 --- a/eng/common/scripts/Helpers/Package-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Package-Helpers.ps1 @@ -76,7 +76,7 @@ function CompatibleConvertFrom-Yaml { } . (Join-Path $PSScriptRoot PSModule-Helpers.ps1) - Install-ModuleIfNotInstalled -WhatIf:$false "powershell-yaml" "0.4.7" | Import-Module + InstallAndImport-ModuleIfNotInstalled -WhatIf:$false "powershell-yaml" "0.4.7" return ConvertFrom-Yaml $content } From 6945462823ba684e1da80ef0d7b6f3382682bbf3 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 11 Dec 2024 13:59:30 -0800 Subject: [PATCH 7/8] remove whatif --- eng/common/scripts/Helpers/Package-Helpers.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1 index e40fb41ff4a..b545872dbbb 100644 --- a/eng/common/scripts/Helpers/Package-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Package-Helpers.ps1 @@ -76,7 +76,7 @@ function CompatibleConvertFrom-Yaml { } . (Join-Path $PSScriptRoot PSModule-Helpers.ps1) - InstallAndImport-ModuleIfNotInstalled -WhatIf:$false "powershell-yaml" "0.4.7" + InstallAndImport-ModuleIfNotInstalled "powershell-yaml" "0.4.7" return ConvertFrom-Yaml $content } From b9a5a3b44f35b274698d8223d03d9338c16c4417 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 11 Dec 2024 16:50:36 -0800 Subject: [PATCH 8/8] avoid issue with higher strict mode usage of undeclared variables --- eng/common/scripts/Helpers/PSModule-Helpers.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 index 86505fa423e..da2e6de4cc6 100644 --- a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 +++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 @@ -64,7 +64,7 @@ function Get-ModuleRepositories([string]$moduleName) { } function moduleIsInstalled([string]$moduleName, [string]$version) { - if ($null -eq $script:InstalledModules) { + if (-not (Test-Path variable:script:InstalledModules)) { $script:InstalledModules = @{} }