Handle polly timeout rejected exception for create engine wait

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+# v0.9.14-alpha
+* Fix failing tests with `engine already exists`.
+* Add `EngineProvisionTimeoutException` thrown from `CreateEngineWaitAsync` when requested engine timeout to provision.
+
 # v0.9.13-alpha
 * Add `IAccessTokenHandler` interface to implement custom access Auth0 token handlers.
 * `Client` class consturctor extended to accept custom access token handlers implementations.

RelationalAI.Test/TestConfig.cs

@@ -13,7 +13,6 @@ namespace RelationalAI.Test
 {
     public class EngineFixture : IDisposable
     {
-        private Engine _engine;
         private readonly string engineName = "sdk-csharp-" + Guid.NewGuid().ToString();
         // Semaphore is used to lock the CreateEngine function so that only one test creates the engine.
         private static readonly SemaphoreSlim semaphoreSlim = new SemaphoreSlim(1);
@@ -37,9 +36,27 @@ public async Task<Engine> CreateEngineWaitAsync(Client client)
             try
             {
                 await semaphoreSlim.WaitAsync();
-                if (_engine == null)
+                try
                 {
-                    _engine = await client.CreateEngineWaitAsync(engineName);
+                    Engine = await client.GetEngineAsync(engineName);
+                    if (Engine.State != "PROVISIONED")
+                    {
+                        throw new EngineProvisionFailedException(Engine);
+                    }
+
+                    return Engine;
+                }
+                catch (HttpError ex)
+                {
+                    if (ex.StatusCode != 404)
+                    {
+                        throw ex;
+                    }
+                }
+
+                if (Engine == null)
+                {
+                    Engine = await client.CreateEngineWaitAsync(engineName);
                 }
             }
             finally
@@ -48,16 +65,10 @@ public async Task<Engine> CreateEngineWaitAsync(Client client)
             }
 
 
-            return _engine;
+            return Engine;
         }
 
-        public Engine Engine
-        {
-            get
-            {
-                return _engine;
-            }
-        }
+        public Engine Engine { get; private set; }
     }
 
     [CollectionDefinition("RelationalAI.Test")]

RelationalAI/Client.cs

@@ -24,6 +24,7 @@
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Polly;
+using Polly.Timeout;
 using Relationalai.Protocol;
 
 namespace RelationalAI
@@ -157,17 +158,24 @@ public async Task<Engine> CreateEngineWaitAsync(string engine, string size = "XS
         {
             var startTime = DateTimeOffset.Now.ToUnixTimeMilliseconds();
             await CreateEngineAsync(engine, size);
-            var resp = await Policy
+            try
+            {
+                var resp = await Policy
                     .HandleResult<Engine>(e => !EngineStates.IsTerminalState(e.State, EngineStates.Provisioned))
                     .RetryWithTimeout(startTime, 0.1, 120, 10 * 60)
                     .ExecuteAsync(() => GetEngineAsync(engine));
 
-            if (resp.State != EngineStates.Provisioned)
+                if (resp.State != EngineStates.Provisioned)
+                {
+                    throw new EngineProvisionFailedException(engine);
+                }
+
+                return resp;
+            }
+            catch (TimeoutRejectedException)
             {
-                throw new EngineProvisionFailedException(resp);
+                throw new EngineProvisionTimeoutException(engine);
             }
-
-            return resp;
         }
 
         public async Task<Engine> GetEngineAsync(string engine)

RelationalAI/EngineProvisionFailedException.cs

@@ -29,6 +29,11 @@ public EngineProvisionFailedException(Engine engine)
             Engine = engine;
         }
 
+        public EngineProvisionFailedException(string engine)
+            : base($"Engine with name `{engine}` failed to provision")
+        {
+        }
+
         /// <summary>
         /// Gets the name of the engine that failed to provision.
         /// </summary>

RelationalAI/EngineProvisionTimeoutException.cs

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2022 RelationalAI, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace RelationalAI
+{
+    /// <summary>
+    /// Represents error thrown when engine requested to provision timeout.
+    /// </summary>
+    public class EngineProvisionTimeoutException : EngineProvisionFailedException
+    {
+        public EngineProvisionTimeoutException(string engine)
+            : base($"Engine with name `{engine}` provisioning timeout")
+        {
+        }
+    }
+}

RelationalAI/RelationalAI.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <Version>0.9.13-alpha</Version>
+    <Version>0.9.14-alpha</Version>
     <TargetFramework>netcoreapp3.1</TargetFramework>
     <PackageId>RAI</PackageId>
     <Authors>RelationalAI</Authors>