From 7504aaebabecd564397ac47d987141947ffb6580 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Tue, 19 Nov 2024 14:13:02 -0800 Subject: [PATCH 1/5] add exponential backoff to file read --- src/Config/FileSystemRuntimeConfigLoader.cs | 28 +++++++++++++++++++-- src/Config/Utilities/FileUtilities.cs | 8 +++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/Config/FileSystemRuntimeConfigLoader.cs b/src/Config/FileSystemRuntimeConfigLoader.cs index 7528bf0e52..bf53103486 100644 --- a/src/Config/FileSystemRuntimeConfigLoader.cs +++ b/src/Config/FileSystemRuntimeConfigLoader.cs @@ -7,7 +7,9 @@ using System.Reflection; using System.Text.Json; using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Config.Utilities; using Azure.DataApiBuilder.Service.Exceptions; +using Humanizer; using Microsoft.Extensions.Logging; namespace Azure.DataApiBuilder.Config; @@ -199,8 +201,30 @@ public bool TryLoadConfig( // and ensures the file handle is released immediately after reading. // Previous usage of File.Open may cause file locking issues when // actively using hot-reload and modifying the config file in a text editor. - string json = _fileSystem.File.ReadAllText(path); - if (TryParseConfig(json, out RuntimeConfig, connectionString: _connectionString, replaceEnvVar: replaceEnvVar)) + // Includes an exponential back-off retry mechanism to accommodate + // circumstances where the file may be in use by another process. + int runCount = 1; + string json = string.Empty; + while (runCount <= FileUtilities.RunLimit) + { + try + { + json = _fileSystem.File.ReadAllText(path); + } + catch (IOException ex) + { + Console.WriteLine($"IO Exception, retrying due to {ex.Message}"); + if (runCount == FileUtilities.RunLimit) + { + throw; + } + + Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(FileUtilities.ExponentialRetryBase, runCount))); + runCount++; + } + } + + if (!string.IsNullOrEmpty(json) && TryParseConfig(json, out RuntimeConfig, connectionString: _connectionString, replaceEnvVar: replaceEnvVar)) { if (TrySetupConfigFileWatcher()) { diff --git a/src/Config/Utilities/FileUtilities.cs b/src/Config/Utilities/FileUtilities.cs index 024f3e8f33..579115b4ee 100644 --- a/src/Config/Utilities/FileUtilities.cs +++ b/src/Config/Utilities/FileUtilities.cs @@ -11,14 +11,14 @@ internal class FileUtilities /// /// Limit the number of retries when failures occur. /// - private static readonly int _runLimit = 3; + public static readonly int RunLimit = 3; /// /// Base of the exponential retry back-off. /// -> base ^ runCount /// e.g. 2^1 , 2^2, 2^3, etc. /// - private static readonly int _exponentialRetryBase = 2; + public static readonly int ExponentialRetryBase = 2; /// /// Computes the SHA256 hash for the input data (file contents) at the given path. @@ -42,7 +42,7 @@ public static byte[] ComputeHash(IFileSystem fileSystem, string filePath) int runCount = 1; // Maximum 2^_runLimit seconds of wait time due to retries. - while (runCount <= _runLimit) + while (runCount <= RunLimit) { try { @@ -73,7 +73,7 @@ public static byte[] ComputeHash(IFileSystem fileSystem, string filePath) throw; } - Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(_exponentialRetryBase, runCount))); + Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(ExponentialRetryBase, runCount))); runCount++; } } From 42b6cf97b93b228b723ad71ccbd4c9536afb3c95 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Tue, 19 Nov 2024 14:14:00 -0800 Subject: [PATCH 2/5] remove magic number --- src/Config/Utilities/FileUtilities.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config/Utilities/FileUtilities.cs b/src/Config/Utilities/FileUtilities.cs index 579115b4ee..9ea0dbbb0d 100644 --- a/src/Config/Utilities/FileUtilities.cs +++ b/src/Config/Utilities/FileUtilities.cs @@ -68,7 +68,7 @@ public static byte[] ComputeHash(IFileSystem fileSystem, string filePath) catch (IOException ex) { Console.WriteLine($"IO Exception, retrying due to {ex.Message}"); - if (runCount == 3) + if (runCount == RunLimit) { throw; } From f3c00f6054389bf3377b3ee559869aa9b1f8564d Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Tue, 19 Nov 2024 14:19:51 -0800 Subject: [PATCH 3/5] remove using directive --- src/Config/FileSystemRuntimeConfigLoader.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Config/FileSystemRuntimeConfigLoader.cs b/src/Config/FileSystemRuntimeConfigLoader.cs index bf53103486..b7b867b818 100644 --- a/src/Config/FileSystemRuntimeConfigLoader.cs +++ b/src/Config/FileSystemRuntimeConfigLoader.cs @@ -9,7 +9,6 @@ using Azure.DataApiBuilder.Config.ObjectModel; using Azure.DataApiBuilder.Config.Utilities; using Azure.DataApiBuilder.Service.Exceptions; -using Humanizer; using Microsoft.Extensions.Logging; namespace Azure.DataApiBuilder.Config; From 62cbd346e0ee409b8cd846608505a81e3e4dd65e Mon Sep 17 00:00:00 2001 From: aaronburtle <93220300+aaronburtle@users.noreply.github.com> Date: Wed, 20 Nov 2024 16:29:41 -0800 Subject: [PATCH 4/5] Update src/Config/Utilities/FileUtilities.cs Co-authored-by: RubenCerna2079 <32799214+RubenCerna2079@users.noreply.github.com> --- src/Config/Utilities/FileUtilities.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config/Utilities/FileUtilities.cs b/src/Config/Utilities/FileUtilities.cs index 9ea0dbbb0d..b63e35ff37 100644 --- a/src/Config/Utilities/FileUtilities.cs +++ b/src/Config/Utilities/FileUtilities.cs @@ -41,7 +41,7 @@ public static byte[] ComputeHash(IFileSystem fileSystem, string filePath) // Exponential back-off retry mechanism. int runCount = 1; - // Maximum 2^_runLimit seconds of wait time due to retries. + // Maximum 2^RunLimit seconds of wait time due to retries. while (runCount <= RunLimit) { try From cb9f9bb732becf1b463c932b108da833c1bba803 Mon Sep 17 00:00:00 2001 From: Aaron Burtle Date: Wed, 20 Nov 2024 19:08:42 -0800 Subject: [PATCH 5/5] forgot to push the break --- src/Config/FileSystemRuntimeConfigLoader.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Config/FileSystemRuntimeConfigLoader.cs b/src/Config/FileSystemRuntimeConfigLoader.cs index b7b867b818..4632029af0 100644 --- a/src/Config/FileSystemRuntimeConfigLoader.cs +++ b/src/Config/FileSystemRuntimeConfigLoader.cs @@ -209,6 +209,7 @@ public bool TryLoadConfig( try { json = _fileSystem.File.ReadAllText(path); + break; } catch (IOException ex) {