From b1f02d5cb9b1829f40c6e0a754c9a92711d112ba Mon Sep 17 00:00:00 2001
From: Reuben Bond <203839+ReubenBond@users.noreply.github.com>
Date: Tue, 7 Mar 2023 05:30:17 -0800
Subject: [PATCH] Improve accuracy and stability of ActivationWorkingSet
 (#8321)

---
 src/Orleans.Runtime/Catalog/ActivationData.cs |  3 +-
 .../Catalog/ActivationWorkingSet.cs           | 68 ++++++++++---------
 2 files changed, 38 insertions(+), 33 deletions(-)

diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs
index c071f07dbb..5b8dbdf397 100644
--- a/src/Orleans.Runtime/Catalog/ActivationData.cs
+++ b/src/Orleans.Runtime/Catalog/ActivationData.cs
@@ -657,9 +657,10 @@ public TExtensionInterface GetExtension<TExtensionInterface>()
 
         bool IActivationWorkingSetMember.IsCandidateForRemoval(bool wouldRemove)
         {
+            const int IdlenessLowerBound = 10_000;
             lock (this)
             {
-                var inactive = IsInactive;
+                var inactive = IsInactive && _idleDuration.ElapsedMilliseconds > IdlenessLowerBound;
 
                 // This instance will remain in the working set if it is either not pending removal or if it is currently active.
                 _isInWorkingSet = !wouldRemove || !inactive;
diff --git a/src/Orleans.Runtime/Catalog/ActivationWorkingSet.cs b/src/Orleans.Runtime/Catalog/ActivationWorkingSet.cs
index 2ff7520e33..0f36b959e9 100644
--- a/src/Orleans.Runtime/Catalog/ActivationWorkingSet.cs
+++ b/src/Orleans.Runtime/Catalog/ActivationWorkingSet.cs
@@ -6,6 +6,7 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Orleans.Internal;
+using Orleans.Runtime.Internal;
 
 namespace Orleans.Runtime
 {
@@ -16,7 +17,7 @@ internal sealed class ActivationWorkingSet : IActivationWorkingSet, ILifecyclePa
     {
         private class MemberState
         {
-            public bool IsMarkedForRemoval { get; set; }
+            public bool IsIdle { get; set; }
         }
 
         private readonly ConcurrentDictionary<IActivationWorkingSetMember, MemberState> _members = new();
@@ -33,7 +34,7 @@ public ActivationWorkingSet(
             IEnumerable<IActivationWorkingSetObserver> observers)
         {
             _logger = logger;
-            _scanPeriodTimer = asyncTimerFactory.Create(TimeSpan.FromMilliseconds(100), nameof(ActivationWorkingSet) + "." + nameof(MonitorWorkingSet));
+            _scanPeriodTimer = asyncTimerFactory.Create(TimeSpan.FromMilliseconds(5_000), nameof(ActivationWorkingSet) + "." + nameof(MonitorWorkingSet));
             _observers = observers.ToList();
             CatalogInstruments.RegisterActivationWorkingSetObserve(() => Count);
         }
@@ -42,23 +43,25 @@ public ActivationWorkingSet(
 
         public void OnActivated(IActivationWorkingSetMember member)
         {
-            if (!_members.TryAdd(member, new MemberState()))
+            if (_members.TryAdd(member, new MemberState()))
             {
-                throw new InvalidOperationException($"Member {member} is already a member of the working set");
-            }
+                Interlocked.Increment(ref _activeCount);
+                foreach (var observer in _observers)
+                {
+                    observer.OnAdded(member);
+                }
 
-            Interlocked.Increment(ref _activeCount);
-            foreach (var observer in _observers)
-            {
-                observer.OnAdded(member);
+                return;
             }
+
+            throw new InvalidOperationException($"Member {member} is already a member of the working set");
         }
 
         public void OnActive(IActivationWorkingSetMember member)
         {
             if (_members.TryGetValue(member, out var state))
             {
-                state.IsMarkedForRemoval = false;
+                state.IsIdle = false;
             }
             else if (_members.TryAdd(member, new()))
             {
@@ -101,9 +104,27 @@ public void OnDeactivated(IActivationWorkingSetMember member)
             }
         }
 
+        private async Task MonitorWorkingSet()
+        {
+            while (await _scanPeriodTimer.NextTick())
+            {
+                foreach (var pair in _members)
+                {
+                    try
+                    {
+                        VisitMember(pair.Key, pair.Value);
+                    }
+                    catch (Exception exception)
+                    {
+                        _logger.LogError(exception, "Exception visiting working set member {Member}", pair.Key);
+                    }
+                }
+            }
+        }
+
         private void VisitMember(IActivationWorkingSetMember member, MemberState state)
         {
-            var wouldRemove = state.IsMarkedForRemoval;
+            var wouldRemove = state.IsIdle;
             if (member.IsCandidateForRemoval(wouldRemove))
             {
                 if (wouldRemove)
@@ -112,7 +133,7 @@ private void VisitMember(IActivationWorkingSetMember member, MemberState state)
                 }
                 else
                 {
-                    state.IsMarkedForRemoval = true;
+                    state.IsIdle = true;
                     foreach (var observer in _observers)
                     {
                         observer.OnIdle(member);
@@ -121,7 +142,7 @@ private void VisitMember(IActivationWorkingSetMember member, MemberState state)
             }
             else
             {
-                state.IsMarkedForRemoval = false;
+                state.IsIdle = false;
                 foreach (var observer in _observers)
                 {
                     observer.OnActive(member);
@@ -129,24 +150,6 @@ private void VisitMember(IActivationWorkingSetMember member, MemberState state)
             }
         }
 
-        private async Task MonitorWorkingSet()
-        {
-            while (await _scanPeriodTimer.NextTick())
-            {
-                foreach (var pair in _members)
-                {
-                    try
-                    {
-                        VisitMember(pair.Key, pair.Value);
-                    }
-                    catch (Exception exception)
-                    {
-                        _logger.LogError(exception, "Exception visiting working set member {Member}", pair.Key);
-                    }
-                }
-            }
-        }
-
         void ILifecycleParticipant<ISiloLifecycle>.Participate(ISiloLifecycle lifecycle)
         {
             lifecycle.Subscribe(
@@ -154,7 +157,8 @@ void ILifecycleParticipant<ISiloLifecycle>.Participate(ISiloLifecycle lifecycle)
                 ServiceLifecycleStage.BecomeActive,
                 ct =>
                 {
-                    _runTask = Task.Run(this.MonitorWorkingSet);
+                    using var _ = new ExecutionContextSuppressor();
+                    _runTask = Task.Run(MonitorWorkingSet);
                     return Task.CompletedTask;
                 },
                 async ct =>