Introduce TelemetryResilienceStrategy (#1140)

eng/Library.targets

@@ -40,7 +40,7 @@
     </ItemGroup>
   </Target>
 
-  <Target Name="AddInternalsVisibleToTest" BeforeTargets="BeforeCompile">
+  <Target Name="AddInternalsVisibleToTest" BeforeTargets="BeforeCompile" Condition="'@(InternalsVisibleToTest)' != ''">
     <ItemGroup>
       <InternalsVisibleTo Include="%(InternalsVisibleToTest.Identity)$(PollyPublicKeySuffix)" />
     </ItemGroup>

src/Polly.Core.Tests/Polly.Core.Tests.csproj

@@ -13,9 +13,11 @@
 
   <ItemGroup>
     <ProjectReference Include="..\Polly.Core\Polly.Core.csproj" />
+    <ProjectReference Include="..\Polly.TestUtils\Polly.TestUtils.csproj" />
   </ItemGroup>
 
   <ItemGroup>
+    <Using Include="Polly.TestUtils" />
     <Using Include="Polly.Core.Tests.Helpers" />
   </ItemGroup>
 </Project>

src/Polly.Core.Tests/ResilienceContextTests.cs

@@ -1,3 +1,5 @@
+using Polly.Strategy;
+
 namespace Polly.Core.Tests;
 
 public class ResilienceContextTests
@@ -19,7 +21,7 @@ public void Get_EnsureDefaults()
     [Fact]
     public async Task Get_EnsurePooled()
     {
-        await TestUtils.AssertWithTimeoutAsync(() =>
+        await TestUtilities.AssertWithTimeoutAsync(() =>
         {
             var context = ResilienceContext.Get();
 
@@ -35,17 +37,29 @@ public void Return_Null_Throws()
         Assert.Throws<ArgumentNullException>(() => ResilienceContext.Return(null!));
     }
 
+    [Fact]
+    public void AddResilienceEvent_Ok()
+    {
+        var context = ResilienceContext.Get();
+
+        context.AddResilienceEvent(new ReportedResilienceEvent("Dummy"));
+
+        context.ResilienceEvents.Should().HaveCount(1);
+        context.ResilienceEvents.Should().Contain(new ReportedResilienceEvent("Dummy"));
+    }
+
     [Fact]
     public async Task Return_EnsureDefaults()
     {
-        await TestUtils.AssertWithTimeoutAsync(() =>
+        await TestUtilities.AssertWithTimeoutAsync(() =>
         {
             using var cts = new CancellationTokenSource();
             var context = ResilienceContext.Get();
             context.CancellationToken = cts.Token;
             context.Initialize<bool>(true);
             context.CancellationToken.Should().Be(cts.Token);
             context.Properties.Set(new ResiliencePropertyKey<int>("abc"), 10);
+            context.AddResilienceEvent(new ReportedResilienceEvent("dummy"));
             ResilienceContext.Return(context);
 
             AssertDefaults(context);
@@ -91,5 +105,6 @@ private static void AssertDefaults(ResilienceContext context)
         context.IsSynchronous.Should().BeFalse();
         context.CancellationToken.Should().Be(CancellationToken.None);
         context.Properties.Should().BeEmpty();
+        context.ResilienceEvents.Should().BeEmpty();
     }
 }

src/Polly.Core.Tests/ResilienceStrategyBuilderTests.cs

@@ -292,6 +292,29 @@ public void BuildStrategy_EnsureCorrectContext()
         verified2.Should().BeTrue();
     }
 
+    [Fact]
+    public void Build_OnCreatingStrategy_EnsureRespected()
+    {
+        // arrange
+        var strategy = new TestResilienceStrategy();
+        var builder = new ResilienceStrategyBuilder
+        {
+            OnCreatingStrategy = strategies =>
+            {
+                strategies.Should().ContainSingle(s => s == strategy);
+                strategies.Insert(0, new TestResilienceStrategy());
+            }
+        };
+
+        builder.AddStrategy(strategy);
+
+        // act
+        var finalStrategy = builder.Build();
+
+        // assert
+        finalStrategy.Should().BeOfType<ResilienceStrategyPipeline>();
+    }
+
     private class Strategy : ResilienceStrategy
     {
         public Action? Before { get; set; }

src/Polly.Core.Tests/Retry/RetryResilienceStrategyTests.cs

@@ -11,7 +11,7 @@ public class RetryResilienceStrategyTests
     private readonly ResilienceStrategyTelemetry _telemetry;
     private readonly Mock<DiagnosticSource> _diagnosticSource = new();
 
-    public RetryResilienceStrategyTests() => _telemetry = TestUtils.CreateResilienceTelemetry(_diagnosticSource.Object);
+    public RetryResilienceStrategyTests() => _telemetry = TestUtilities.CreateResilienceTelemetry(_diagnosticSource.Object);
 
     [Fact]
     public void ShouldRetryEmpty_Skipped()

src/Polly.Core.Tests/Strategy/ReportedResilienceEventTests.cs

@@ -0,0 +1,22 @@
+using Polly.Strategy;
+
+namespace Polly.Core.Tests.Strategy;
+
+public class ReportedResilienceEventTests
+{
+    [Fact]
+    public void Ctor_Ok()
+    {
+        var ev = new ReportedResilienceEvent("A");
+
+        ev.ToString().Should().Be("A");
+    }
+
+    [Fact]
+    public void Equality_Ok()
+    {
+        (new ReportedResilienceEvent("A") == new ReportedResilienceEvent("A")).Should().BeTrue();
+        (new ReportedResilienceEvent("A") != new ReportedResilienceEvent("A")).Should().BeFalse();
+        (new ReportedResilienceEvent("A") == new ReportedResilienceEvent("B")).Should().BeFalse();
+    }
+}

src/Polly.Core.Tests/Timeout/TimeoutResilienceStrategyTests.cs

@@ -1,5 +1,4 @@
 using Moq;
-using Polly.Core.Tests.Helpers;
 using Polly.Strategy;
 using Polly.Timeout;
 
@@ -16,7 +15,7 @@ public class TimeoutResilienceStrategyTests : IDisposable
 
     public TimeoutResilienceStrategyTests()
     {
-        _telemetry = TestUtils.CreateResilienceTelemetry(_diagnosticSource.Object);
+        _telemetry = TestUtilities.CreateResilienceTelemetry(_diagnosticSource.Object);
         _timeProvider = new FakeTimeProvider();
         _options = new TimeoutStrategyOptions();
         _cancellationSource = new CancellationTokenSource();

src/Polly.Core.Tests/Utils/SystemTimeProviderTests.cs

@@ -13,7 +13,7 @@ public void TimestampFrequency_Ok()
     [Fact]
     public async Task CancelAfter_Ok()
     {
-        await TestUtils.AssertWithTimeoutAsync(async () =>
+        await TestUtilities.AssertWithTimeoutAsync(async () =>
         {
             using var cts = new CancellationTokenSource();
             TimeProvider.System.CancelAfter(cts, TimeSpan.FromMilliseconds(10));
@@ -28,7 +28,7 @@ public async Task Delay_Ok()
     {
         using var cts = new CancellationTokenSource();
 
-        await TestUtils.AssertWithTimeoutAsync(() =>
+        await TestUtilities.AssertWithTimeoutAsync(() =>
         {
             TimeProvider.System.Delay(TimeSpan.FromMilliseconds(10)).IsCompleted.Should().BeFalse();
         });
@@ -48,16 +48,20 @@ public async Task GetElapsedTime_Ok()
         var delay = TimeSpan.FromMilliseconds(10);
         var delayWithTolerance = TimeSpan.FromMilliseconds(30);
 
-        await TestUtils.AssertWithTimeoutAsync(async () =>
+        await TestUtilities.AssertWithTimeoutAsync(async () =>
         {
             var stamp1 = TimeProvider.System.GetTimestamp();
             await Task.Delay(10);
             var stamp2 = TimeProvider.System.GetTimestamp();
 
             var elapsed = TimeProvider.System.GetElapsedTime(stamp1, stamp2);
+            var elapsed2 = TimeProvider.System.GetElapsedTime(stamp1);
 
             elapsed.Should().BeGreaterThanOrEqualTo(delay);
             elapsed.Should().BeLessThan(delayWithTolerance);
+
+            elapsed2.Should().BeGreaterThanOrEqualTo(delay);
+            elapsed2.Should().BeLessThan(delayWithTolerance);
         });
     }
 
@@ -81,7 +85,7 @@ public void GetElapsedTime_Mocked_Ok()
     [Fact]
     public async Task UtcNow_Ok()
     {
-        await TestUtils.AssertWithTimeoutAsync(() =>
+        await TestUtilities.AssertWithTimeoutAsync(() =>
         {
             var now = TimeProvider.System.UtcNow;
             (DateTimeOffset.UtcNow - now).Should().BeLessThanOrEqualTo(TimeSpan.FromMilliseconds(10));

src/Polly.Core.Tests/Utils/TimeProviderExtensionsTests.cs

@@ -25,7 +25,7 @@ public async Task DelayAsync_System_Ok(bool synchronous, bool mocked, bool hasCa
         context.CancellationToken = token;
         mock.SetupDelay(delay, token);
 
-        await TestUtils.AssertWithTimeoutAsync(async () =>
+        await TestUtilities.AssertWithTimeoutAsync(async () =>
         {
             var task = timeProvider.DelayAsync(delay, context);
             task.IsCompleted.Should().Be(synchronous || mocked);
@@ -45,7 +45,7 @@ public async Task DelayAsync_SystemSynchronous_Ok()
         var context = ResilienceContext.Get();
         context.Initialize<VoidResult>(isSynchronous: true);
 
-        await TestUtils.AssertWithTimeoutAsync(async () =>
+        await TestUtilities.AssertWithTimeoutAsync(async () =>
         {
             var watch = Stopwatch.StartNew();
             await TimeProvider.System.DelayAsync(delay, context);
@@ -63,7 +63,7 @@ public async Task DelayAsync_SystemSynchronousWhenCancelled_Ok()
         context.Initialize<VoidResult>(isSynchronous: true);
         context.CancellationToken = cts.Token;
 
-        await TestUtils.AssertWithTimeoutAsync(async () =>
+        await TestUtilities.AssertWithTimeoutAsync(async () =>
         {
             await TimeProvider.System
                 .Invoking(p => p.DelayAsync(delay, context))
@@ -102,7 +102,7 @@ public async Task DelayAsync_CancellationAfter_Throws(bool synchronous, bool moc
     {
         var delay = TimeSpan.FromMilliseconds(20);
 
-        await TestUtils.AssertWithTimeoutAsync(async () =>
+        await TestUtilities.AssertWithTimeoutAsync(async () =>
         {
             var mock = new FakeTimeProvider();
             using var tcs = new CancellationTokenSource();

src/Polly.Core/Polly.Core.csproj

@@ -18,6 +18,7 @@
     <Using Include="Polly.Utils" />
     <Using Remove="System.Net.Http" />
     <InternalsVisibleToTest Include="Polly.Core.Tests" />
+    <InternalsVisibleToTest Include="Polly.TestUtils" />
     <InternalsVisibleToTest Include="Polly.Extensions" />
   </ItemGroup>
 

src/Polly.Core/ResilienceContext.cs

@@ -1,4 +1,5 @@
 using System.Diagnostics.CodeAnalysis;
+using Polly.Strategy;
 
 namespace Polly;
 
@@ -16,6 +17,8 @@ public sealed class ResilienceContext
 
     private static readonly ObjectPool<ResilienceContext> Pool = new(static () => new ResilienceContext(), static c => c.Reset());
 
+    private readonly List<ReportedResilienceEvent> _resilienceEvents = new();
+
     private ResilienceContext()
     {
     }
@@ -55,6 +58,14 @@ private ResilienceContext()
     /// </summary>
     public ResilienceProperties Properties { get; } = new();
 
+    /// <summary>
+    /// Gets the collection of resilience events that occurred while executing the resilience strategy.
+    /// </summary>
+    /// <remarks>
+    /// If the number of resilience events is greater than zero it's an indication that the execution was unhealthy.
+    /// </remarks>
+    public IReadOnlyCollection<ReportedResilienceEvent> ResilienceEvents => _resilienceEvents;
+
     /// <summary>
     /// Gets a <see cref="ResilienceContext"/> instance from the pool.
     /// </summary>
@@ -92,14 +103,19 @@ internal ResilienceContext Initialize<TResult>(bool isSynchronous)
         return this;
     }
 
+    internal void AddResilienceEvent(ReportedResilienceEvent @event)
+    {
+        _resilienceEvents.Add(@event);
+    }
+
     private bool Reset()
     {
         IsSynchronous = false;
         ResultType = typeof(UnknownResult);
         ContinueOnCapturedContext = false;
         CancellationToken = default;
         ((IDictionary<string, object?>)Properties).Clear();
-
+        _resilienceEvents.Clear();
         return true;
     }
 

src/Polly.Core/ResilienceStrategyBuilder.cs

@@ -37,6 +37,11 @@ public class ResilienceStrategyBuilder
     [Required]
     internal TimeProvider TimeProvider { get; set; } = TimeProvider.System;
 
+    /// <summary>
+    /// Gets or sets the callback that is invoked just before the final resilience strategy is being created.
+    /// </summary>
+    internal Action<IList<ResilienceStrategy>>? OnCreatingStrategy { get; set; }
+
     /// <summary>
     /// Adds an already created strategy instance to the builder.
     /// </summary>
@@ -85,18 +90,19 @@ public ResilienceStrategy Build()
 
         _used = true;
 
-        if (_entries.Count == 0)
+        var strategies = _entries.Select(CreateResilienceStrategy).ToList();
+        OnCreatingStrategy?.Invoke(strategies);
+
+        if (strategies.Count == 0)
         {
             return NullResilienceStrategy.Instance;
         }
 
-        if (_entries.Count == 1)
+        if (strategies.Count == 1)
         {
-            return CreateResilienceStrategy(_entries[0]);
+            return strategies[0];
         }
 
-        var strategies = _entries.Select(CreateResilienceStrategy).ToList();
-
         return ResilienceStrategyPipeline.CreatePipeline(strategies);
     }
 

src/Polly.Core/Strategy/ReportedResilienceEvent.cs

@@ -0,0 +1,11 @@
+namespace Polly.Strategy;
+
+/// <summary>
+/// Represents a resilience event that has been reported.
+/// </summary>
+/// <param name="EventName">The event name.</param>
+public readonly record struct ReportedResilienceEvent(string EventName)
+{
+    /// <inheritdoc/>
+    public override string ToString() => EventName;
+}