From acdcea72efac53ac88ac196cd89ac1f075947a5d Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Fri, 11 Oct 2024 13:01:49 -0700 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools for PR 9102 (#31362) * Add CompatibleConvertFrom-Yaml to common Package-Helpers * Add CI artifact parsing to Save-Package-Properties, now each individual packageinfo.json file contains the relevant ci artifact lines for the package. This short circuits needing to re-parse this information in common checks. --------- Co-authored-by: Scott Beddall --- .vscode/cspell.json | 4 ++ .../scripts/Helpers/Package-Helpers.ps1 | 48 ++++++++++++++- eng/common/scripts/Package-Properties.ps1 | 60 ++++++++++++++++++- eng/scripts/Language-Settings.ps1 | 6 +- 4 files changed, 114 insertions(+), 4 deletions(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index fa6c6cc8f083..5da0892cef7b 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -46,9 +46,11 @@ ], "words": [ "AMQP", + "azuresdkdocs", "BRCPF", "Brcpf", "CONTOSO", + "docfx", "DTDL", "ECONNRESET", "ESDNI", @@ -66,6 +68,7 @@ "Kubernetes", "Localizable", "Lucene", + "mgmt", "MPNS", "MSRC", "Mibps", @@ -78,6 +81,7 @@ "Petabit", "Picometer", "Plregon", + "psobject", "Rasterize", "Resourceid", "Rollup", diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1 index 3cc8adbb7e3d..985ebcc4ee3b 100644 --- a/eng/common/scripts/Helpers/Package-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Package-Helpers.ps1 @@ -12,7 +12,7 @@ function GetPackageKey($pkg) { return $pkgKey } - + # Different language needs a different way to index the package. Build a map in convienice to lookup the package. # E.g. : is the package key in java. function GetPackageLookup($packageList) { @@ -44,4 +44,50 @@ function GetDocsTocDisplayName($pkg) { $displayName += " (deprecated)" } return $displayName +} + + +<# +.SYNOPSIS +This function is a safe wrapper around `yq` and `ConvertFrom-Yaml` to convert YAML content to a PowerShell HashTable object + +.DESCRIPTION +This function wraps `yq` and `ConvertFrom-Yaml` to convert YAML content to a PowerShell HashTable object. The reason this function exists is +because while on a local user's machine, installing a module from the powershell gallery is an easy task, in pipelines we often have failures +to install modules from the gallery. This function will attempt to use the `yq` command if it is available on the machine, and only will install +the yaml module if `yq` is not available. This means that for the majority of runs on CI machines, the yaml module will not be installed. + +.PARAMETER Content +The content to convert from YAML to a PowerShell HashTable object. Accepted as named argument or from the pipeline. + +.EXAMPLE +CompatibleConvertFrom-Yaml -Content (Get-Content -Raw path/to/file.yml) + +.EXAMPLE +Get-Content -Raw path/to/file.yml | CompatibleConvertFrom-Yaml +#> +function CompatibleConvertFrom-Yaml { + param( + [Parameter(Mandatory=$true, ValueFromPipeline=$true)] + [string]$Content + ) + + if (!($Content)) { + throw "Content to parse is a required input." + } + + # Initialize any variables or checks that need to be done once + $yqPresent = Get-Command 'yq' -ErrorAction SilentlyContinue + if (-not $yqPresent) { + . (Join-Path $PSScriptRoot PSModule-Helpers.ps1) + Install-ModuleIfNotInstalled -WhatIf:$false "powershell-yaml" "0.4.1" | Import-Module + } + + # Process the content (for example, you could convert from YAML here) + if ($yqPresent) { + return ($content | yq -o=json | ConvertFrom-Json -AsHashTable) + } + else { + return ConvertFrom-Yaml $content + } } \ No newline at end of file diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 486eeaca4b15..72810397c94f 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -1,6 +1,6 @@ -# Helper functions for retireving useful information from azure-sdk-for-* repo +# Helper functions for retrieving useful information from azure-sdk-for-* repo . "${PSScriptRoot}\logging.ps1" - +. "${PSScriptRoot}\Helpers\Package-Helpers.ps1" class PackageProps { [string]$Name @@ -19,6 +19,7 @@ class PackageProps [boolean]$IncludedForValidation # does this package include other packages that we should trigger validation for? [string[]]$AdditionalValidationPackages + [HashTable]$ArtifactDetails PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory) { @@ -66,6 +67,8 @@ class PackageProps { $this.ChangeLogPath = $null } + + $this.InitializeCIArtifacts() } hidden [void]Initialize( @@ -79,6 +82,59 @@ class PackageProps $this.Initialize($name, $version, $directoryPath, $serviceDirectory) $this.Group = $group } + + hidden [object]GetValueSafely($hashtable, [string[]]$keys) { + $current = $hashtable + foreach ($key in $keys) { + if ($current.ContainsKey($key) -or $current[$key]) { + $current = $current[$key] + } + else { + return $null + } + } + + return $current + } + + hidden [HashTable]ParseYmlForArtifact([string]$ymlPath) { + if (Test-Path -Path $ymlPath) { + try { + $content = Get-Content -Raw -Path $ymlPath | CompatibleConvertFrom-Yaml + if ($content) { + $artifacts = $this.GetValueSafely($content, @("extends", "parameters", "Artifacts")) + + $artifactForCurrentPackage = $artifacts | Where-Object { $_["name"] -eq $this.ArtifactName -or $_["name"] -eq $this.Name } + + if ($artifactForCurrentPackage) { + return [HashTable]$artifactForCurrentPackage + } + } + } + catch { + Write-Host "Exception while parsing yml file $($ymlPath): $_" + } + } + + return $null + } + + [void]InitializeCIArtifacts(){ + $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot ".." ".." "..") + + $ciFolderPath = Join-Path -Path $RepoRoot -ChildPath (Join-Path "sdk" $this.ServiceDirectory) + $ciFiles = Get-ChildItem -Path $ciFolderPath -Filter "ci*.yml" -File + + if (-not $this.ArtifactDetails) { + foreach($ciFile in $ciFiles) { + $ciArtifactResult = $this.ParseYmlForArtifact($ciFile.FullName) + if ($ciArtifactResult) { + $this.ArtifactDetails = [Hashtable]$ciArtifactResult + break + } + } + } + } } # Takes package name and service Name diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 35a21e6fbd3e..6358d7795f13 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -40,6 +40,10 @@ function Get-javascript-PackageInfoFromRepo ($pkgPath, $serviceDirectory) { } $pkgProp.IsNewSdk = ($pkgProp.SdkType -eq "client") -or ($pkgProp.SdkType -eq "mgmt") $pkgProp.ArtifactName = $jsStylePkgName + # the constructor for the package properties object attempts to initialize CI artifacts on instantiation + # of the class. however, due to the fact that we set the ArtifactName _after_ the constructor is called, + # we need to call it again here to ensure the CI artifacts are properly initialized + $pkgProp.InitializeCIArtifacts() return $pkgProp } return $null @@ -249,7 +253,7 @@ function SetPackageVersion ($PackageName, $Version, $ReleaseDate, $ReplaceLatest } # PackageName: Pass full package name e.g. @azure/abort-controller -# You can obtain full pacakge name using the 'Get-PkgProperties' function in 'eng\common\scripts\Package-Properties.Ps1' +# You can obtain full package name using the 'Get-PkgProperties' function in 'eng\common\scripts\Package-Properties.Ps1' function GetExistingPackageVersions ($PackageName, $GroupId = $null) { try { $existingVersion = Invoke-RestMethod -Method GET -Uri "http://registry.npmjs.com/${PackageName}"