Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update provisioning tests to retry on throttling #2411

Merged
merged 4 commits into from
May 25, 2022
Merged

Update provisioning tests to retry on throttling #2411

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f6f1a09
refactor(e2e-tests): Update provisioning tests to retry on throttling
abhipsaMisra May 24, 2022
ae07328
fluent
abhipsaMisra May 24, 2022
4b36d42
only reprov
abhipsaMisra May 24, 2022
deeeafc
Revert "only reprov"
abhipsaMisra May 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions e2e/test/E2ETests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
<Compile Remove="iothub\service\DigitalTwinClientE2ETests.cs" />
<Compile Remove="helpers\digitaltwins\models\ThermostatTwin.cs" />
<Compile Remove="helpers\digitaltwins\models\TemperatureControllerTwin.cs" />
<Compile Remove="helpers\ProvisioningServiceRetryPolicy.cs" />
</ItemGroup>

<!-- All Platforms -->
Expand Down
53 changes: 53 additions & 0 deletions e2e/test/helpers/ProvisioningServiceRetryPolicy.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System;
using System.Collections.Generic;
using Microsoft.Azure.Devices.Client;
using Microsoft.Azure.Devices.Provisioning.Service;

namespace Microsoft.Azure.Devices.E2ETests.Helpers
{
public class ProvisioningServiceRetryPolicy : IRetryPolicy
{
private const string RetryAfterKey = "Retry-After";
private const int MaxRetryCount = 5;

private static readonly TimeSpan s_defaultRetryInterval = TimeSpan.FromSeconds(5);

private static readonly IRetryPolicy s_exponentialBackoffRetryStrategy = new ExponentialBackoff(
retryCount: MaxRetryCount,
minBackoff: s_defaultRetryInterval,
maxBackoff: TimeSpan.FromSeconds(10),
deltaBackoff: TimeSpan.FromMilliseconds(100));

public bool ShouldRetry(int currentRetryCount, Exception lastException, out TimeSpan retryInterval)
{
retryInterval = TimeSpan.Zero;

var provisioningException = lastException as ProvisioningServiceClientHttpException;

if (provisioningException == null || currentRetryCount > MaxRetryCount)
{
return false;
}
else if ((int)provisioningException.StatusCode == 429) // HttpStatusCode.TooManyRequests is not available in net472
{
IDictionary<string, string> httpHeaders = provisioningException.Fields;
bool retryAfterPresent = httpHeaders.TryGetValue(RetryAfterKey, out string retryAfter);

retryInterval = retryAfterPresent
? TimeSpan.FromSeconds(Convert.ToDouble(retryAfter))
: s_defaultRetryInterval;

return true;
}
else if ((int)provisioningException.StatusCode > 500 && (int)provisioningException.StatusCode < 600)
{
return s_exponentialBackoffRetryStrategy.ShouldRetry(currentRetryCount, lastException, out retryInterval);
}

return false;
}
}
}
9 changes: 6 additions & 3 deletions e2e/test/provisioning/ProvisioningE2ETests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1085,7 +1085,8 @@ private async Task<SecurityProvider> CreateSecurityProviderFromNameAsync(
allocationPolicy,
customAllocationDefinition,
iothubs,
capabilities).ConfigureAwait(false);
capabilities,
Logger).ConfigureAwait(false);

return new SecurityProviderTpmSimulator(tpmEnrollment.RegistrationId);

Expand Down Expand Up @@ -1113,7 +1114,8 @@ private async Task<SecurityProvider> CreateSecurityProviderFromNameAsync(
allocationPolicy,
customAllocationDefinition,
iothubs,
capabilities).ConfigureAwait(false);
capabilities,
Logger).ConfigureAwait(false);

x509IndividualEnrollment.Attestation.Should().BeAssignableTo<X509Attestation>();
}
Expand Down Expand Up @@ -1166,7 +1168,8 @@ private async Task<SecurityProvider> CreateSecurityProviderFromNameAsync(
allocationPolicy,
customAllocationDefinition,
iothubs,
capabilities).ConfigureAwait(false);
capabilities,
Logger).ConfigureAwait(false);

Assert.IsTrue(symmetricKeyEnrollment.Attestation is SymmetricKeyAttestation);
symmetricKeyAttestation = (SymmetricKeyAttestation)symmetricKeyEnrollment.Attestation;
Expand Down
121 changes: 94 additions & 27 deletions e2e/test/provisioning/ProvisioningServiceClientE2ETests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,12 @@
using System.Security.Cryptography.X509Certificates;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Azure.Devices.Client;
using Microsoft.Azure.Devices.E2ETests.Helpers;
using Microsoft.Azure.Devices.Provisioning.Security.Samples;
using Microsoft.Azure.Devices.Provisioning.Service;
using Microsoft.Azure.Devices.Shared;
using Microsoft.Extensions.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace Microsoft.Azure.Devices.E2ETests.Provisioning
Expand All @@ -22,6 +25,9 @@ public class ProvisioningServiceClientE2ETests : E2EMsTestBase
private static readonly string s_proxyServerAddress = TestConfiguration.IoTHub.ProxyServerAddress;
private static readonly string s_devicePrefix = $"E2E_{nameof(ProvisioningServiceClientE2ETests)}_";

private static readonly HashSet<Type> s_retryableExceptions = new HashSet<Type> { typeof(ProvisioningServiceClientHttpException) };
private static readonly IRetryPolicy s_provisioningServiceRetryPolicy = new ProvisioningServiceRetryPolicy();

#pragma warning disable CA1823
private readonly VerboseTestLogger _verboseLog = VerboseTestLogger.GetInstance();
#pragma warning restore CA1823
Expand Down Expand Up @@ -138,7 +144,8 @@ public async Task ProvisioningServiceClient_GetIndividualEnrollmentAttestation(A
AllocationPolicy.Static,
null,
null,
null)
null,
Logger)
.ConfigureAwait(false);

AttestationMechanism attestationMechanism = await provisioningServiceClient.GetIndividualEnrollmentAttestationAsync(individualEnrollment.RegistrationId);
Expand Down Expand Up @@ -215,12 +222,18 @@ private async Task ProvisioningServiceClient_IndividualEnrollments_Query_Ok(stri
}
}

public static async Task ProvisioningServiceClient_IndividualEnrollments_Create_Ok(string proxyServerAddress, AttestationMechanismType attestationType)
public async Task ProvisioningServiceClient_IndividualEnrollments_Create_Ok(string proxyServerAddress, AttestationMechanismType attestationType)
{
await ProvisioningServiceClient_IndividualEnrollments_Create_Ok(proxyServerAddress, attestationType, null, AllocationPolicy.Hashed, null, null).ConfigureAwait(false);
}

public static async Task ProvisioningServiceClient_IndividualEnrollments_Create_Ok(string proxyServerAddress, AttestationMechanismType attestationType, ReprovisionPolicy reprovisionPolicy, AllocationPolicy allocationPolicy, CustomAllocationDefinition customAllocationDefinition, ICollection<string> iotHubsToProvisionTo)
public async Task ProvisioningServiceClient_IndividualEnrollments_Create_Ok(
string proxyServerAddress,
AttestationMechanismType attestationType,
ReprovisionPolicy reprovisionPolicy,
AllocationPolicy allocationPolicy,
CustomAllocationDefinition customAllocationDefinition,
ICollection<string> iotHubsToProvisionTo)
{
using (ProvisioningServiceClient provisioningServiceClient = CreateProvisioningService(proxyServerAddress))
{
Expand All @@ -235,7 +248,8 @@ public static async Task ProvisioningServiceClient_IndividualEnrollments_Create_
allocationPolicy,
customAllocationDefinition,
iotHubsToProvisionTo,
null).ConfigureAwait(false);
null,
Logger).ConfigureAwait(false);
IndividualEnrollment individualEnrollmentResult = await provisioningServiceClient.GetIndividualEnrollmentAsync(individualEnrollment.RegistrationId).ConfigureAwait(false);
Assert.AreEqual(individualEnrollmentResult.ProvisioningStatus, ProvisioningStatus.Enabled);

Expand Down Expand Up @@ -321,17 +335,18 @@ public static async Task<IndividualEnrollment> CreateIndividualEnrollmentAsync(
AllocationPolicy allocationPolicy,
CustomAllocationDefinition customAllocationDefinition,
ICollection<string> iotHubsToProvisionTo,
DeviceCapabilities capabilities)
DeviceCapabilities capabilities,
MsTestLogger logger)
{
Attestation attestation;
IndividualEnrollment individualEnrollment;
IndividualEnrollment createdEnrollment = null;
switch (attestationType)
{
case AttestationMechanismType.Tpm:
using (var tpmSim = new SecurityProviderTpmSimulator(registrationId))
{
string base64Ek = Convert.ToBase64String(tpmSim.GetEndorsementKey());
using var provisioningService = ProvisioningServiceClient.CreateFromConnectionString(TestConfiguration.Provisioning.ConnectionString);
individualEnrollment = new IndividualEnrollment(registrationId, new TpmAttestation(base64Ek))
{
Capabilities = capabilities,
Expand All @@ -341,11 +356,45 @@ public static async Task<IndividualEnrollment> CreateIndividualEnrollmentAsync(
IotHubs = iotHubsToProvisionTo
};

IndividualEnrollment enrollment = await provisioningService.CreateOrUpdateIndividualEnrollmentAsync(individualEnrollment).ConfigureAwait(false);
IndividualEnrollment temporaryCreatedEnrollment = null;
await RetryOperationHelper
.RetryOperationsAsync(
async () =>
{
temporaryCreatedEnrollment = await provisioningServiceClient.CreateOrUpdateIndividualEnrollmentAsync(individualEnrollment).ConfigureAwait(false);
},
s_provisioningServiceRetryPolicy,
s_retryableExceptions,
logger)
.ConfigureAwait(false);

if (temporaryCreatedEnrollment == null)
{
throw new ArgumentException("The enrollment entry could not be created, exiting test.");
}

attestation = new TpmAttestation(base64Ek);
enrollment.Attestation = attestation;
return await provisioningService.CreateOrUpdateIndividualEnrollmentAsync(enrollment).ConfigureAwait(false);
temporaryCreatedEnrollment.Attestation = attestation;

await RetryOperationHelper
.RetryOperationsAsync(
async () =>
{
createdEnrollment = await provisioningServiceClient.CreateOrUpdateIndividualEnrollmentAsync(temporaryCreatedEnrollment).ConfigureAwait(false);
},
s_provisioningServiceRetryPolicy,
s_retryableExceptions,
logger)
.ConfigureAwait(false);

if (createdEnrollment == null)
{
throw new ArgumentException("The enrollment entry could not be created, exiting test.");
}

return createdEnrollment;
}

case AttestationMechanismType.SymmetricKey:
string primaryKey = CryptoKeyGenerator.GenerateKey(32);
string secondaryKey = CryptoKeyGenerator.GenerateKey(32);
Expand All @@ -354,31 +403,49 @@ public static async Task<IndividualEnrollment> CreateIndividualEnrollmentAsync(

case AttestationMechanismType.X509:
attestation = X509Attestation.CreateFromClientCertificates(authenticationCertificate);

individualEnrollment = new IndividualEnrollment(registrationId, attestation)
{
Capabilities = capabilities,
AllocationPolicy = allocationPolicy,
ReprovisionPolicy = reprovisionPolicy,
CustomAllocationDefinition = customAllocationDefinition,
IotHubs = iotHubsToProvisionTo,
};
return await provisioningServiceClient.CreateOrUpdateIndividualEnrollmentAsync(individualEnrollment).ConfigureAwait(false);
break;

default:
throw new NotSupportedException("Test code has not been written for testing this attestation type yet");
}

individualEnrollment = new IndividualEnrollment(registrationId, attestation);
individualEnrollment.Capabilities = capabilities;
individualEnrollment.CustomAllocationDefinition = customAllocationDefinition;
individualEnrollment.ReprovisionPolicy = reprovisionPolicy;
individualEnrollment.IotHubs = iotHubsToProvisionTo;
individualEnrollment.AllocationPolicy = allocationPolicy;
return await provisioningServiceClient.CreateOrUpdateIndividualEnrollmentAsync(individualEnrollment).ConfigureAwait(false);
individualEnrollment = new IndividualEnrollment(registrationId, attestation)
{
Capabilities = capabilities,
AllocationPolicy = allocationPolicy,
ReprovisionPolicy = reprovisionPolicy,
CustomAllocationDefinition = customAllocationDefinition,
IotHubs = iotHubsToProvisionTo,
};

await RetryOperationHelper
.RetryOperationsAsync(
async () =>
{
createdEnrollment = await provisioningServiceClient.CreateOrUpdateIndividualEnrollmentAsync(individualEnrollment).ConfigureAwait(false);
},
s_provisioningServiceRetryPolicy,
s_retryableExceptions,
logger)
.ConfigureAwait(false);

if (createdEnrollment == null)
{
throw new ArgumentException("The enrollment entry could not be created, exiting test.");
}

return createdEnrollment;
}

public static async Task<EnrollmentGroup> CreateEnrollmentGroup(ProvisioningServiceClient provisioningServiceClient, AttestationMechanismType attestationType, string groupId, ReprovisionPolicy reprovisionPolicy, AllocationPolicy allocationPolicy, CustomAllocationDefinition customAllocationDefinition, ICollection<string> iothubs, DeviceCapabilities capabilities)
public static async Task<EnrollmentGroup> CreateEnrollmentGroup(
ProvisioningServiceClient provisioningServiceClient,
AttestationMechanismType attestationType,
string groupId,
ReprovisionPolicy reprovisionPolicy,
AllocationPolicy allocationPolicy,
CustomAllocationDefinition customAllocationDefinition,
ICollection<string> iothubs,
DeviceCapabilities capabilities)
{
Attestation attestation;
switch (attestationType)
Expand Down
Loading