Update dependencies from https://github.com/dotnet/arcade build 20210…

…108.3 (#693)

[master] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -3,9 +3,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.20630.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21058.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4c8515c18ebe0071c32fed467ee3890fbd488898</Sha>
+      <Sha>1571d6b095014ad63fdb48b10f5dea912f96872e</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/build.sh

@@ -81,7 +81,7 @@ runtime_source_feed_key=''
 
 properties=''
 while [[ $# > 0 ]]; do
-  opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+  opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
     -help|-h)
       usage

eng/common/cross/build-android-rootfs.sh

@@ -27,7 +27,7 @@ __AndroidToolchain=aarch64-linux-android
 
 for i in "$@"
     do
-        lowerI="$(echo $i | awk '{print tolower($0)}')"
+        lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
         case $lowerI in
         -?|-h|--help)
             usage

eng/common/cross/build-rootfs.sh

@@ -82,7 +82,7 @@ while :; do
         break
     fi
 
-    lowerI="$(echo $1 | awk '{print tolower($0)}')"
+    lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
     case $lowerI in
         -?|-h|--help)
             usage

eng/common/darc-init.sh

@@ -6,7 +6,7 @@ versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc
 verbosity='minimal'
 
 while [[ $# > 0 ]]; do
-  opt="$(echo "$1" | awk '{print tolower($0)}')"
+  opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
     --darcversion)
       darcVersion=$2

eng/common/dotnet-install.sh

@@ -19,7 +19,7 @@ runtime='dotnet'
 runtimeSourceFeed=''
 runtimeSourceFeedKey=''
 while [[ $# > 0 ]]; do
-  opt="$(echo "$1" | awk '{print tolower($0)}')"
+  opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
     -version|-v)
       shift

eng/common/init-tools-native.sh

@@ -16,7 +16,7 @@ declare -A native_assets
 . $scriptroot/native/common-library.sh
 
 while (($# > 0)); do
-  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
   case $lowerI in
     --baseuri)
       base_uri=$2
@@ -76,24 +76,89 @@ while (($# > 0)); do
 done
 
 function ReadGlobalJsonNativeTools {
-  # Get the native-tools section from the global.json.
-  local native_tools_section=$(cat $global_json_file | awk '/"native-tools"/,/}/')
-  # Only extract the contents of the object.
-  local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}')
-  native_tools_list=${native_tools_list//[\" ]/}
-  native_tools_list=$( echo "$native_tools_list" | sed 's/\s//g' | sed 's/,/\n/g' )
-
-  local old_IFS=$IFS
-  while read -r line; do
-    # Lines are of the form: 'tool:version'
-    IFS=:
-    while read -r key value; do
-     native_assets[$key]=$value
-    done <<< "$line"
-  done <<< "$native_tools_list"
-  IFS=$old_IFS
-
-  return 0;
+  # happy path: we have a proper JSON parsing tool `jq(1)` in PATH!
+  if command -v jq &> /dev/null; then
+
+    # jq: read each key/value pair under "native-tools" entry and emit:
+    #   KEY="<entry-key>" VALUE="<entry-value>"
+    # followed by a null byte.
+    #
+    # bash: read line with null byte delimeter and push to array (for later `eval`uation).
+
+    while IFS= read -rd '' line; do
+      native_assets+=("$line")
+    done < <(jq -r '. |
+        select(has("native-tools")) |
+        ."native-tools" |
+        keys[] as $k |
+        @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file")
+
+    return
+  fi
+
+  # Warning: falling back to manually parsing JSON, which is not recommended.
+
+  # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above.
+  # It has been tested with several weird strings with escaped characters in entries (key and value)
+  # and results were compared with the output of jq(1) in binary representation using xxd(1);
+  # just before the assignment to 'native_assets' array (above and below).
+
+  # try to capture the section under "native-tools".
+  if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then
+    return
+  fi
+
+  section="${BASH_REMATCH[1]}"
+
+  parseStarted=0
+  possibleEnd=0
+  escaping=0
+  escaped=0
+  isKey=1
+
+  for (( i=0; i<${#section}; i++ )); do
+    char="${section:$i:1}"
+    if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi
+
+    if ! ((escaping)) && [[ "$char" == "\\" ]]; then
+      escaping=1
+    elif ((escaping)) && ! ((escaped)); then
+      escaped=1
+    fi
+
+    if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then
+      parseStarted=1
+      possibleEnd=0
+    elif [[ "$char" == "'" ]]; then
+      token="$token'\\\''"
+      possibleEnd=0
+    elif ((escaping)) || [[ "$char" != "\"" ]]; then
+      token="$token$char"
+      possibleEnd=1
+    fi
+
+    if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then
+      # Use printf to unescape token to match jq(1)'s @sh formatting rules.
+      # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed.
+      printf -v token "'$token'"
+
+      if ((isKey)); then
+        KEY="$token"
+        isKey=0
+      else
+        line="KEY=$KEY VALUE=$token"
+        native_assets+=("$line")
+        isKey=1
+      fi
+
+      # reset for next token
+      parseStarted=0
+      token=
+    elif ((escaping)) && ((escaped)); then
+      escaping=0
+      escaped=0
+    fi
+  done
 }
 
 native_base_dir=$install_directory
@@ -111,14 +176,14 @@ if [[ ${#native_assets[@]} -eq 0 ]]; then
   exit 0;
 else
   native_installer_dir="$scriptroot/native"
-  for tool in "${!native_assets[@]}"
-  do
-    tool_version=${native_assets[$tool]}
-    installer_path="$native_installer_dir/install-$tool.sh"
+  for index in "${!native_assets[@]}"; do
+    eval "${native_assets["$index"]}"
+
+    installer_path="$native_installer_dir/install-$KEY.sh"
     installer_command="$installer_path"
     installer_command+=" --baseuri $base_uri"
     installer_command+=" --installpath $install_bin"
-    installer_command+=" --version $tool_version"
+    installer_command+=" --version $VALUE"
     echo $installer_command
 
     if [[ $force = true ]]; then

eng/common/internal-feed-operations.sh

@@ -103,7 +103,7 @@ authToken=''
 repoName=''
 
 while [[ $# > 0 ]]; do
-  opt="$(echo "$1" | awk '{print tolower($0)}')"
+  opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
     --operation)
       operation=$2

eng/common/msbuild.sh

@@ -19,7 +19,7 @@ prepare_machine=false
 extra_args=''
 
 while (($# > 0)); do
-  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
   case $lowerI in
     --verbosity)
       verbosity=$2

eng/common/native/install-cmake-test.sh

@@ -14,7 +14,7 @@ download_retries=5
 retry_wait_time_seconds=30
 
 while (($# > 0)); do
-  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
   case $lowerI in
     --baseuri)
       base_uri=$2
@@ -63,7 +63,7 @@ done
 
 tool_name="cmake-test"
 tool_os=$(GetCurrentOS)
-tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
 tool_arch="x86_64"
 tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
 tool_install_directory="$install_path/$tool_name/$version"
@@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then
   exit 1
 fi
 
-exit 0
+exit 0

eng/common/native/install-cmake.sh

@@ -14,7 +14,7 @@ download_retries=5
 retry_wait_time_seconds=30
 
 while (($# > 0)); do
-  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
   case $lowerI in
     --baseuri)
       base_uri=$2
@@ -63,7 +63,7 @@ done
 
 tool_name="cmake"
 tool_os=$(GetCurrentOS)
-tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
 tool_arch="x86_64"
 tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
 tool_install_directory="$install_path/$tool_name/$version"
@@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then
   exit 1
 fi
 
-exit 0
+exit 0

eng/common/performance/performance-setup.sh

@@ -29,7 +29,7 @@ using_wasm=false
 use_latest_dotnet=false
 
 while (($# > 0)); do
-  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
   case $lowerI in
     --sourcedirectory)
       source_directory=$2

eng/common/pipeline-logging-functions.sh

@@ -6,7 +6,7 @@ function Write-PipelineTelemetryError {
   local function_args=()
   local message=''
   while [[ $# -gt 0 ]]; do
-    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
     case "$opt" in
       -category|-c)
         telemetry_category=$2
@@ -48,7 +48,7 @@ function Write-PipelineTaskError {
   local force=false
 
   while [[ $# -gt 0 ]]; do
-    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
     case "$opt" in
       -type|-t)
         message_type=$2
@@ -122,7 +122,7 @@ function Write-PipelineSetVariable {
   local is_multi_job_variable=true
 
   while [[ $# -gt 0 ]]; do
-    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
     case "$opt" in
       -name|-n)
         name=$2
@@ -164,7 +164,7 @@ function Write-PipelinePrependPath {
   local prepend_path=''
 
   while [[ $# -gt 0 ]]; do
-    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
     case "$opt" in
       -path|-p)
         prepend_path=$2
@@ -186,7 +186,7 @@ function Write-PipelineSetResult {
   local message=''
 
   while [[ $# -gt 0 ]]; do
-    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
     case "$opt" in
       -result|-r)
         result=$2

eng/common/templates/job/publish-build-assets.yml

@@ -37,6 +37,7 @@ jobs:
     - name: _BuildConfig
       value: ${{ parameters.configuration }}
     - group: Publish-Build-Assets
+    - group: AzureDevOps-Artifact-Feeds-Pats
     # Skip component governance and codesign validation for SDL. These jobs
     # create no content.
     - name: skipComponentGovernanceDetection
@@ -57,6 +58,12 @@ jobs:
     - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
       - task: NuGetAuthenticate@0
 
+      - task: PowerShell@2 
+        displayName: Enable cross-org NuGet feed authentication 
+        inputs: 
+          filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 
+          arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw) 
+
     - task: PowerShell@2
       displayName: Publish Build Assets
       inputs:

eng/common/templates/job/source-index-stage1.yml

@@ -0,0 +1,46 @@
+parameters:
+  runAsPublic: false
+  sourceIndexPackageVersion: 1.0.0-beta5
+  sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
+  sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
+  binlogPath: artifacts/log/Debug/Build.binlog
+  pool:
+    vmImage: vs2017-win2016
+
+jobs:
+- job: SourceIndexStage1
+  variables:
+  - name: SourceIndexPackageVersion
+    value: ${{ parameters.sourceIndexPackageVersion }}
+  - name: SourceIndexPackageSource
+    value: ${{ parameters.sourceIndexPackageSource }}
+  - name: BinlogPath
+    value: ${{ parameters.binlogPath }}
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - group: source-dot-net stage1 variables
+
+  pool: ${{ parameters.pool }}
+  steps:
+  - task: UseDotNet@2
+    displayName: Use .NET Core sdk 3.1
+    inputs:
+      packageType: sdk
+      version: 3.1.x
+
+  - script: |
+      dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
+      dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
+      echo ##vso[task.prependpath]$(Build.SourcesDirectory)/.source-index/tools
+    displayName: Download Tools
+
+  - script: ${{ parameters.sourceIndexBuildCommand }}
+    displayName: Build Repository
+
+  - script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
+    displayName: Process Binlog into indexable sln
+
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
+      displayName: Upload stage1 artifacts to source index
+      env:
+        BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)