Only provision and test Managed HSM when opted in

sdk/core/Azure.Core.TestFramework/src/RecordedTestBase.cs

@@ -8,6 +8,7 @@
 using System.Linq;
 using System.Reflection;
 using NUnit.Framework;
+using NUnit.Framework.Interfaces;
 
 namespace Azure.Core.TestFramework
 {
@@ -136,7 +137,7 @@ public virtual void StartTestRecording()
         [TearDown]
         public virtual void StopTestRecording()
         {
-            bool save = TestContext.CurrentContext.Result.FailCount == 0;
+            bool save = TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Passed;
 #if DEBUG
             save |= SaveDebugRecordingsOnFailure;
 #endif

sdk/keyvault/Azure.Security.KeyVault.Administration/tests/AdministrationTestBase.cs

@@ -34,7 +34,12 @@ protected AdministrationTestBase(bool isAsync, RecordedTestMode? mode)
         /// <summary>
         /// Gets the endpoint to connect. By default it is <see cref="KeyVaultTestEnvironment.ManagedHsmUrl"/>.
         /// </summary>
-        public virtual Uri Uri => new Uri(TestEnvironment.ManagedHsmUrl);
+        public Uri Uri =>
+            Uri.TryCreate(TestEnvironment.ManagedHsmUrl, UriKind.Absolute, out Uri uri)
+                ? uri
+                // If the AZURE_MANAGEDHSM_URL variable is not defined, we didn't provision one
+                // due to limitations: https://github.com/Azure/azure-sdk-for-net/issues/16531
+                : throw new IgnoreException($"Required variable 'AZURE_MANAGEDHSM_URL' is not defined");
 
         /// <summary>
         /// Gets a polling interval based on whether we're playing back recorded tests (0s) or not (2s).

sdk/keyvault/Azure.Security.KeyVault.Keys/tests/CryptographyClientLiveTests.cs

@@ -29,8 +29,6 @@ public void ClearChallengeCacheforRecord()
             // is always made.  This allows tests to be replayed independently and in any order
             if (Mode == RecordedTestMode.Record || Mode == RecordedTestMode.Playback)
             {
-                Client = GetClient();
-
                 ChallengeBasedAuthenticationPolicy.AuthenticationChallenge.ClearCache();
             }
         }

sdk/keyvault/Azure.Security.KeyVault.Keys/tests/KeyClientLiveTests.cs

@@ -35,8 +35,6 @@ public void ClearChallengeCacheforRecord()
             // is always made.  This allows tests to be replayed independently and in any order
             if (Mode == RecordedTestMode.Record || Mode == RecordedTestMode.Playback)
             {
-                Client = GetClient();
-
                 ChallengeBasedAuthenticationPolicy.AuthenticationChallenge.ClearCache();
             }
         }

sdk/keyvault/Azure.Security.KeyVault.Keys/tests/KeysTestBase.cs

@@ -22,7 +22,7 @@ public abstract class KeysTestBase : RecordedTestBase<KeyVaultTestEnvironment>
             ? TimeSpan.Zero
             : TimeSpan.FromSeconds(2);
 
-        public KeyClient Client { get; set; }
+        public KeyClient Client { get; private set; }
 
         public virtual Uri Uri => new Uri(TestEnvironment.KeyVaultUrl);
 
@@ -34,7 +34,7 @@ public abstract class KeysTestBase : RecordedTestBase<KeyVaultTestEnvironment>
         private KeyVaultTestEventListener _listener;
 
         protected KeysTestBase(bool isAsync, KeyClientOptions.ServiceVersion serviceVersion, RecordedTestMode? mode)
-            : base(isAsync, mode ?? RecordedTestUtilities.GetModeFromEnvironment())
+            : base(isAsync, mode ?? RecordedTestUtilities.GetModeFromEnvironment() /* RecordedTestMode.Record */)
         {
             _serviceVersion = serviceVersion;
         }

sdk/keyvault/Azure.Security.KeyVault.Keys/tests/ManagedHsmLiveTests.cs

@@ -3,6 +3,7 @@
 
 using System;
 using Azure.Core.TestFramework;
+using NUnit.Framework;
 
 namespace Azure.Security.KeyVault.Keys.Tests
 {
@@ -13,6 +14,11 @@ public ManagedHsmLiveTests(bool isAsync, KeyClientOptions.ServiceVersion service
         {
         }
 
-        public override Uri Uri => new Uri(TestEnvironment.ManagedHsmUrl);
+        public override Uri Uri =>
+            Uri.TryCreate(TestEnvironment.ManagedHsmUrl, UriKind.Absolute, out Uri uri)
+                ? uri
+                // If the AZURE_MANAGEDHSM_URL variable is not defined, we didn't provision one
+                // due to limitations: https://github.com/Azure/azure-sdk-for-net/issues/16531
+                : throw new IgnoreException($"Required variable 'AZURE_MANAGEDHSM_URL' is not defined");
     }
 }

sdk/keyvault/Azure.Security.KeyVault.Shared/tests/KeyVaultTestEnvironment.cs

@@ -29,7 +29,7 @@ public KeyVaultTestEnvironment() : base("keyvault")
         /// <summary>
         /// Gets the URI to Managed HSM.
         /// </summary>
-        public string ManagedHsmUrl => GetRecordedVariable("AZURE_MANAGEDHSM_URL");
+        public string ManagedHsmUrl => GetRecordedOptionalVariable("AZURE_MANAGEDHSM_URL");
 
         /// <summary>
         /// Gets an OID for the client within the tenant.

sdk/keyvault/CONTRIBUTING.md

@@ -57,12 +57,11 @@ Before running or recording live tests you need to create [live test resources][
 
 ```powershell
 eng\common\TestResources\New-TestResources.ps1 `
-  -BaseName 'myusername' `
   -ServiceDirectory 'keyvault' `
-  -TestApplicationId $sp.ApplicationId `
-  -TestApplicationSecret (ConvertFrom-SecureString $sp.Secret -AsPlainText) `
   -AdditionalParameters @{
-    enableSoftDelete = $false # Enable or disable soft delete. Default is $true (enabled).
+    # Enable Managed HSM provisioning and testing.
+    # Disabled by default due to limitations: https://github.com/Azure/azure-sdk-for-net/issues/16531
+    enableHsm = $true
   }
 ```
 

sdk/keyvault/test-resources-post.ps1

@@ -93,7 +93,7 @@ $wrappingFiles = foreach ($i in 0..2) {
 
 # TODO: Use Az module when available; for now, assumes Azure CLI is installed and in $Env:PATH.
 Log "Logging '$username' into the Azure CLI"
-az login --service-principal --tenant $tenant --username $username --password $password
+az login --service-principal --tenant "$tenant" --username "$username" --password="$password"
 
 Log "Downloading security domain from '$hsmUrl'"
 

sdk/keyvault/test-resources.json

@@ -43,6 +43,13 @@
                 "description": "The location of the Managed HSM. By default, this is 'southcentralus'."
             }
         },
+        "enableHsm": {
+            "type": "bool",
+            "defaultValue": false,
+            "metadata": {
+                "description": "Whether to enable deployment of Managed HSM. The default is false."
+            }
+        },
         "enableSoftDelete": {
             "type": "bool",
             "defaultValue": true,
@@ -165,6 +172,7 @@
             "type": "Microsoft.KeyVault/managedHSMs",
             "apiVersion": "[variables('hsmApiVersion')]",
             "name": "[variables('hsmName')]",
+            "condition": "[parameters('enableHsm')]",
             "location": "[parameters('hsmLocation')]",
             "sku": {
                 "family": "B",
@@ -236,6 +244,7 @@
         },
         "AZURE_MANAGEDHSM_URL": {
             "type": "string",
+            "condition": "[parameters('enableHsm')]",
             "value": "[reference(variables('hsmName')).hsmUri]"
         },
         "KEYVAULT_SKU": {