Sync with remote master

.github/CODEOWNERS

@@ -345,7 +345,7 @@ sdk/labservices/Microsoft.Azure.Management.LabServices/           @Tanmayeekamat
 /sdk/marketplaceordering/Microsoft.Azure.Management.MarketplaceOrdering/            @prbansa
 
 # ServiceLabel: %Media Services %Service Attention
-/sdk/mediaservices/            @akucer
+/sdk/mediaservices/            @akucer @naiteeks @bennage @giakas
 
 # ServiceLabel: %Migrate %Service Attention
 /sdk/resourcemover/Microsoft.Azure.Management.Migrate/            @shijojoy

CONTRIBUTING.md

@@ -11,8 +11,8 @@
 - Install the **.NET Core cross-platform development** workloads in VisualStudio
 - Install **.NET Core 5.0.100 SDK** for your specific platform. (or a higher version within the 5.0.*** band)  (https://dotnet.microsoft.com/download/dotnet-core/5.0)
 - Install the latest version of git (https://git-scm.com/downloads)
-- Install [PowerShell](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell), version 6 or higher, if you plan to make public API changes or are working with generated code snippets.
-- Install [NodeJS](https://nodejs.org/en/) (14.x.x) if you plan to use [C# code generation](https://github.com/Azure/autorest.csharp).
+- Install [PowerShell](https://docs.microsoft.com/powershell/scripting/install/installing-powershell), version 6 or higher, if you plan to make public API changes or are working with generated code snippets.
+- Install [NodeJS](https://nodejs.org/) (14.x.x) if you plan to use [C# code generation](https://github.com/Azure/autorest.csharp).
 
 ## GENERAL THINGS TO KNOW:
 

NuGet.Config

@@ -1,10 +1,17 @@
 <?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <packageSources>
-    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" protocolVersion="3" />
-    <add key="Local" value="tools/LocalNugetFeed" />
-    <add key="azure-sdk-tools" value="https://azuresdkartifacts.blob.core.windows.net/azure-sdk-tools/index.json" />
+    <clear />
+    <!-- Do not add any additional feeds if new packages are needed they need to come from nuget.org or our azure-sdk-for-net DevOps feed -->
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
     <add key="azure-sdk-for-net" value="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json" />
-    <add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
+
+    <!-- 
+      Used for azure-sdk-tools repo until issue https://github.com/Azure/azure-sdk-tools/issues/1329 is addressed
+    -->
+    <add key="azure-sdk-tools" value="https://azuresdkartifacts.blob.core.windows.net/azure-sdk-tools/index.json" />
   </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
 </configuration>

common/Perf/Azure.Test.Perf/PerfProgram.cs

@@ -24,6 +24,9 @@ public static class PerfProgram
         private static List<TimeSpan>[] _correctedLatencies;
         private static Channel<(TimeSpan, Stopwatch)> _pendingOperations;
 
+        private static int CompletedOperations => _completedOperations.Sum();
+        private static double OperationsPerSecond => _completedOperations.Zip(_lastCompletionTimes, (operations, time) => (operations / time.TotalSeconds)).Sum();
+
         public static async Task Main(Assembly assembly, string[] args)
         {
             var testTypes = assembly.ExportedTypes
@@ -215,13 +218,15 @@ private static async Task RunTestsAsync(IPerfTest[] tests, PerfOptions options,
             using var progressStatusCts = new CancellationTokenSource();
             var progressStatusThread = PerfStressUtilities.PrintStatus(
                 $"=== {title} ===" + Environment.NewLine +
-                "Current\t\tTotal",
+                "Current\t\tTotal\t\tAverage",
                 () =>
                 {
-                    var totalCompleted = _completedOperations.Sum();
+                    var totalCompleted = CompletedOperations;
                     var currentCompleted = totalCompleted - lastCompleted;
+                    var averageCompleted = OperationsPerSecond;
+
                     lastCompleted = totalCompleted;
-                    return currentCompleted + "\t\t" + totalCompleted;
+                    return $"{currentCompleted}\t\t{totalCompleted}\t\t{averageCompleted:F2}";
                 },
                 newLine: true,
                 progressStatusCts.Token,
@@ -273,12 +278,12 @@ private static async Task RunTestsAsync(IPerfTest[] tests, PerfOptions options,
 
             Console.WriteLine("=== Results ===");
 
-            var totalOperations = _completedOperations.Sum();
-            var operationsPerSecond = _completedOperations.Zip(_lastCompletionTimes, (operations, time) => (operations / time.TotalSeconds)).Sum();
+            var totalOperations = CompletedOperations;
+            var operationsPerSecond = OperationsPerSecond;
             var secondsPerOperation = 1 / operationsPerSecond;
             var weightedAverageSeconds = totalOperations / operationsPerSecond;
 
-            Console.WriteLine($"Completed {totalOperations} operations in a weighted-average of {weightedAverageSeconds:N2}s " +
+            Console.WriteLine($"Completed {totalOperations:N0} operations in a weighted-average of {weightedAverageSeconds:N2}s " +
                 $"({operationsPerSecond:N2} ops/s, {secondsPerOperation:N3} s/op)");
             Console.WriteLine();
 

common/SmokeTests/SmokeTest/KeyVaultTest.cs

@@ -14,10 +14,10 @@ class KeyVaultTest
     {
         private static Dictionary<string, Uri> authorityHostMap = new Dictionary<string, Uri>
         {
-            { "AzureCloud", KnownAuthorityHosts.AzureCloud },
-            { "AzureChinaCloud", KnownAuthorityHosts.AzureChinaCloud },
-            { "AzureGermanCloud", KnownAuthorityHosts.AzureGermanCloud },
-            { "AzureUSGovernment", KnownAuthorityHosts.AzureUSGovernment },
+            { "AzureCloud", AzureAuthorityHosts.AzurePublicCloud },
+            { "AzureChinaCloud", AzureAuthorityHosts.AzureChina },
+            { "AzureGermanCloud", AzureAuthorityHosts.AzureGermany },
+            { "AzureUSGovernment", AzureAuthorityHosts.AzureGovernment },
         };
 
         private static string SecretName = $"SmokeTestSecret-{Guid.NewGuid()}";
@@ -38,7 +38,7 @@ public static async Task RunTests()
             Console.WriteLine("3.- Delete that Secret (Clean up)\n");
 
             string keyVaultUri = Environment.GetEnvironmentVariable("KEY_VAULT_URI");
-            var authorityHost = GetAuthorityHost(Environment.GetEnvironmentVariable("AZURE_CLOUD"), KnownAuthorityHosts.AzureCloud);
+            var authorityHost = GetAuthorityHost(Environment.GetEnvironmentVariable("AZURE_CLOUD"), AzureAuthorityHosts.AzurePublicCloud);
 
             var defaultAzureCredentialOptions = new DefaultAzureCredentialOptions 
             {
@@ -95,4 +95,4 @@ private static Uri GetAuthorityHost(string cloudName, Uri defaultAuthorityHost)
             return defaultAuthorityHost;
         }
     }
-}
+}

common/SmokeTests/SmokeTest/SmokeTest.csproj

@@ -5,26 +5,24 @@
     <LangVersion>latest</LangVersion>
     <StartupObject>SmokeTest.Program</StartupObject>
   </PropertyGroup>
-
   <ItemGroup Condition="'$(TargetFramework)' == 'net461'">
     <Reference Include="System" />
     <Reference Include="Microsoft.CSharp" />
   </ItemGroup>
-
   <ItemGroup>
-    <PackageReference Include="Azure.Core" Version="1.4.0-dev.*" />
-    <PackageReference Include="Azure.Identity" Version="1.2.0-dev.*" />
-    <PackageReference Include="Azure.Messaging.EventHubs" Version="5.1.0-dev.*" />
-    <PackageReference Include="Azure.Messaging.EventHubs.Processor" Version="5.1.0-dev.*" />
-    <PackageReference Include="Azure.Security.Keyvault.Secrets" Version="4.1.0-dev.*" />
-    <PackageReference Include="Azure.Storage.Blobs" Version="12.5.0-dev.*" />
-    <PackageReference Include="Microsoft.Azure.Amqp" Version="2.4.3" />
+    <PackageReference Include="Azure.Core" Version="1.8.0" />
+    <PackageReference Include="Azure.Identity" Version="1.4.0-beta.1" />
+    <!-- The OverrideDailyVersion attribute prevents the Update-Dependencies script from overwriting it with a daily build version -->
+    <PackageReference Include="Azure.Messaging.EventHubs" Version="5.3.0-beta.4" OverrideDailyVersion="" />
+    <PackageReference Include="Azure.Messaging.EventHubs.Processor" Version="5.3.0-beta.4" />
+    <PackageReference Include="Azure.Security.Keyvault.Secrets" Version="4.2.0-beta.3" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.8.0-beta.1" />
+    <PackageReference Include="Microsoft.Azure.Amqp" Version="2.4.9" />
     <PackageReference Include="Microsoft.Azure.Devices" Version="1.20.1" />
     <PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.10.0" />
   </ItemGroup>
-
   <!-- Sample: IoT Hub Connection String Translation -->
   <ItemGroup>
     <Compile Include="$(MSBuildThisFileDirectory)..\..\..\samples\iothub-connect-to-eventhubs\**\*.cs" Link="Samples\IoTHubConnection\SharedSource\%(Filename)%(Extension)" />
   </ItemGroup>
-</Project>
+</Project>

common/SmokeTests/SmokeTest/Update-Dependencies.ps1

@@ -1,27 +1,21 @@
 param(
     [string]$ProjectFile = './SmokeTest.csproj',
+    [string]$ArtifactsPath,
     [switch]$SkipVersionValidation,
-    [switch]$CI
+    [switch]$CI,
+    [switch]$Daily
 )
 
-# To exclude a package version create an entry whose key is the package to
-# exclude whose value is a hash table of versions to exclude.
-# Example:
-# $PACKAGE_EXCLUSIONS = @{
-#     'Azure.Security.Keyvault.Secrets' = @{
-#         '4.1.0-dev.20191102.1' = $true;
-#         '4.1.0-dev.20191103.1' = $true;
-#     }
-# }
-$PACKAGE_EXCLUSIONS = @{ }
+. $PSScriptRoot/../../../eng/common/scripts/SemVer.ps1
 
 $PACKAGE_REFERENCE_XPATH = '//Project/ItemGroup/PackageReference'
 
 # Matches the dev.yyyymmdd portion of the version string
-$DEV_DATE_REGEX = 'dev\.(\d{8})'
+$ALPHA_DATE_REGEX = 'alpha\.(\d{8})'
 
-$NIGHTLY_FEED_NAME = 'NightlyFeed'
-$NIGHTLY_FEED_URL = 'https://azuresdkartifacts.blob.core.windows.net/azure-sdk-for-net/index.json'
+$baselineVersionDate = $null;
+
+$PACKAGE_FEED_URL = 'https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json'
 
 function Log-Warning($message) {
     if ($CI) {
@@ -31,63 +25,94 @@ function Log-Warning($message) {
     }
 }
 
-Register-PackageSource `
-    -Name $NIGHTLY_FEED_NAME `
-    -Location $NIGHTLY_FEED_URL `
-    -ProviderName Nuget `
-    -ErrorAction SilentlyContinue `
+function GetAllPackages {
+    $packages = Find-Package -Source $PACKAGE_FEED_URL -AllVersion -AllowPrereleaseVersions
+    if ($Daily) {
+        return $packages | Where-Object { $_.Version.Contains("alpha") }
+    }
+    return $packages | Where-Object { !$_.Version.Contains("alpha") -and !$_.Version.Contains("dev") }
+}
 
-# List all packages from the source specified by $FeedName. Packages are sorted
-# ascending by version according to semver rules (e.g. 4.0.0-preview.1 comes
-# before 4.0.0) not lexicographically.
-# Packages cannot be filtered at this stage because the sleet feed to which they
-# are published does not support filtering by name.
-$allPackages = Find-Package -Source $NIGHTLY_FEED_NAME -AllVersion -AllowPrereleaseVersions
+function GetLatestPackage([array]$packageList, [string]$packageName) {
+    $versions = ($packageList
+        | Where-Object { $_.Name -eq $packageName }
+        | Select-Object -ExpandProperty Version)
 
-$baselineVersionDate = $null;
+    if (!$versions) {
+        Write-Warning "Did not find any versions for $($packageName)"
+        return
+    }
 
-# For each PackageReferecne in the csproj, find the latest version of that
-# package from the dev feed which is not in the excluded list.
-$projectFilePath = Resolve-Path -Path $ProjectFile
-[xml]$csproj = Get-Content $ProjectFile
-$csproj |
-    Select-XML $PACKAGE_REFERENCE_XPATH |
-    Where-Object { $_.Node.HasAttribute('Version') } |
-    ForEach-Object {
-        # Resolve package version:
-        $packageName = $_.Node.Include
-
-        # This assumes that the versions coming back from Find-Package are
-        # sorted ascending. It excludes any version that is in the corresponding
-        # $PACKAGE_EXCLUSIONS entry
-        $targetVersion = ($allPackages |
-            Where-Object { $_.Name -eq $packageName } |
-            Where-Object { -not ( $PACKAGE_EXCLUSIONS.ContainsKey($packageName) -and $PACKAGE_EXCLUSIONS[$packageName].ContainsKey($_.Version)) } |
-            Select-Object -Last 1).Version
-
-        if ($targetVersion -eq $null) {
-            return
-        }
+    $sorted = [AzureEngSemanticVersion]::SortVersionStrings($versions)
+    return $sorted | Select-Object -First 1
+}
 
-        Write-Host "Setting $packageName to $targetVersion"
-        $_.Node.Version = "$targetVersion"
+function GetPackageVersion([array]$packageList, [string]$packageName) {
+    if ($Daily -or -not $ArtifactsPath) {
+        return GetLatestPackage $packageList $packageName
+    }
 
+    if (-not (Test-Path (Join-Path $ArtifactsPath $packageName))) {
+      Write-Host "No build artifact directory for smoke test dependency $packageName. Using latest upstream version."
+      return GetLatestPackage $packageList $packageName
+    }
 
-        # Validate package version date component matches
-        if ($SkipVersionValidation) {
-            return
-        }
+    $pkg = Get-ChildItem "$ArtifactsPath/$packageName/*.nupkg" | Select-Object -First 1
+    if ($pkg -match "$packageName\.(.*)\.nupkg") {
+        $version = $matches[1]
+        Write-Host "Found build artifact for $packageName with version $version. Using artifact version."
+        return $version
+    } else {
+      throw "No build artifact packages found for smoke test dependency '$packageName'."
+    }
+}
 
-        if ($_.Node.Version -match $DEV_DATE_REGEX) {
-            if ($baselineVersionDate -eq $null) {
-                Write-Host "Using baseline version date: $($matches[1])"
-                $baselineVersionDate = $matches[1]
+function SetLatestPackageVersions([xml]$csproj) {
+    # For each PackageReference in the csproj, find the latest version of that
+    # package from the dev feed which is not in the excluded list.
+    $allPackages = GetAllPackages
+    $csproj |
+        Select-XML $PACKAGE_REFERENCE_XPATH
+        | Where-Object { $_.Node.HasAttribute('Version') }
+        | Where-Object { -not ($Daily -and $_.Node.HasAttribute('OverrideDailyVersion')) }
+        | ForEach-Object {
+            # Resolve package version:
+            $packageName = $_.Node.Include
+
+            $targetVersion = GetPackageVersion $allPackages $packageName
+
+            if ($null -eq $targetVersion) {
+                return
             }
 
-            if ($baselineVersionDate -ne $matches[1]) {
-                Log-Warning "$($_.Node.Include) uses invalid version. Expected: $baselineVersionDate, Actual: $($matches[1])"
+            Write-Host "Setting $packageName to $targetVersion"
+            $_.Node.Version = "$targetVersion"
+
+
+            # Validate package version date component matches
+            if ($SkipVersionValidation) {
+                return
+            }
+
+            if ($_.Node.Version -match $ALPHA_DATE_REGEX) {
+                $capture = $matches[1]
+                if ($null -eq $baselineVersionDate) {
+                    Write-Host "Using baseline version date: $capture"
+                    $baselineVersionDate = $capture
+                }
+
+                if ($baselineVersionDate -ne $matches[1]) {
+                    Log-Warning "$($_.Node.Include) uses invalid version. Expected: $baselineVersionDate, Actual: $capture"
+                }
             }
         }
-    }
+}
+
+function UpdateCsprojVersions {
+    $projectFilePath = Resolve-Path -Path $ProjectFile
+    [xml]$csproj = Get-Content $projectFilePath
+    SetLatestPackageVersions $csproj
+    $csproj.Save($projectFilePath)
+}
 
-$csproj.Save($projectFilePath)
+UpdateCsprojVersions