feat: retry mechanism (#1474)

apps/docs/.vitepress/config.ts

@@ -240,6 +240,10 @@ export default defineConfig({
                 text: 'Querying the Chain',
                 link: '/guide/providers/querying-the-chain',
               },
+              {
+                text: 'Retrying calls',
+                link: '/guide/providers/retrying-calls',
+              },
             ],
           },
           {

apps/docs/src/guide/providers/retrying-calls.md

@@ -0,0 +1,5 @@
+# Retrying calls
+
+The default behavior of calls done via the `Provider` towards a fuel node is that they'll fail if the connection breaks. Specifying retry options allows you to customize how many additional attempts you want to make when the connection to the node breaks before ultimately throwing an error. You can also specify the back-off algorithm as well as the base duration that algorithm will use to calculate the wait time for each request.
+
+<<< @/../../../packages/providers/test/retry.test.ts#provider-retry-options{ts:line-numbers}

packages/providers/src/call-retrier.ts

@@ -0,0 +1,70 @@
+import type { ProviderOptions } from './provider';
+import { sleep } from './utils';
+
+type Backoff = 'linear' | 'exponential' | 'fixed';
+
+export interface RetryOptions {
+  /**
+   * Amount of attempts to retry before failing the call.
+   */
+  maxRetries: number;
+  /**
+   * Backoff strategy to use when retrying. Default is exponential.
+   */
+  backoff?: Backoff;
+  /**
+   * Base duration for backoff strategy. Default is 150ms.
+   */
+  baseDuration?: number;
+}
+
+function getWaitDuration(options: RetryOptions, attempt: number) {
+  const duration = options.baseDuration ?? 150;
+
+  if (attempt === 0) {
+    return duration;
+  }
+
+  switch (options.backoff) {
+    case 'linear':
+      return duration * attempt;
+    case 'fixed':
+      return duration;
+    case 'exponential':
+    default:
+      return duration * (2 ^ (attempt - 1));
+  }
+}
+
+export function retrier(
+  fetchFn: NonNullable<ProviderOptions['fetch']>,
+  options: RetryOptions | undefined,
+  retryAttempt: number = 0
+): NonNullable<ProviderOptions['fetch']> {
+  if (options === undefined) {
+    return fetchFn;
+  }
+
+  return async (...args) => {
+    try {
+      return await fetchFn(...args);
+    } catch (e: unknown) {
+      const error = e as Error & { cause?: { code?: string } };
+
+      if (error.cause?.code !== 'ECONNREFUSED') {
+        throw e;
+      }
+
+      if (retryAttempt === options.maxRetries) {
+        throw e;
+      }
+
+      // eslint-disable-next-line no-param-reassign
+      retryAttempt += 1;
+
+      await sleep(getWaitDuration(options, retryAttempt));
+
+      return retrier(fetchFn, options, retryAttempt)(...args);
+    }
+  };
+}

packages/providers/src/provider.ts

@@ -24,6 +24,8 @@ import type {
   GqlGetBlocksQueryVariables,
   GqlPeerInfo,
 } from './__generated__/operations';
+import type { RetryOptions } from './call-retrier';
+import { retrier } from './call-retrier';
 import type { Coin } from './coin';
 import type { CoinQuantity, CoinQuantityLike } from './coin-quantity';
 import { coinQuantityfy } from './coin-quantity';
@@ -218,6 +220,7 @@ export type ProviderOptions = {
   ) => Promise<Response>;
   timeout?: number;
   cacheUtxo?: number;
+  retryOptions?: RetryOptions;
 };
 
 /**
@@ -279,17 +282,23 @@ export default class Provider {
     timeout: undefined,
     cacheUtxo: undefined,
     fetch: undefined,
+    retryOptions: undefined,
   };
 
-  private static getFetchFn(options: ProviderOptions) {
-    return options.fetch !== undefined
-      ? options.fetch
-      : (url: string, request: FetchRequestOptions) =>
-          fetch(url, {
-            ...request,
-            signal:
-              options.timeout !== undefined ? AbortSignal.timeout(options.timeout) : undefined,
-          });
+  private static getFetchFn(options: ProviderOptions): NonNullable<ProviderOptions['fetch']> {
+    const { retryOptions, timeout } = options;
+
+    return retrier((...args) => {
+      if (options.fetch) {
+        return options.fetch(...args);
+      }
+
+      const url = args[0];
+      const request = args[1];
+      const signal = timeout ? AbortSignal.timeout(timeout) : undefined;
+
+      return fetch(url, { ...request, signal });
+    }, retryOptions);
   }
 
   /**

packages/providers/test/retry.test.ts

@@ -0,0 +1,150 @@
+import { safeExec } from '@fuel-ts/errors/test-utils';
+
+import type { RetryOptions } from '../src/call-retrier';
+import Provider from '../src/provider';
+
+// TODO: Figure out a way to import this constant from `@fuel-ts/wallet/configs`
+const FUEL_NETWORK_URL = 'http://127.0.0.1:4000/graphql';
+
+function mockFetch(maxAttempts: number, callTimes: number[]) {
+  const fetchSpy = vi.spyOn(global, 'fetch');
+
+  fetchSpy.mockImplementation((...args: unknown[]) => {
+    callTimes.push(Date.now());
+
+    if (fetchSpy.mock.calls.length <= maxAttempts) {
+      const error = new Error();
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore TS is not happy with this property, but it works. ts-expect-error doesn't work for some reason, so I chose ts-ignore
+      error.cause = {
+        code: 'ECONNREFUSED',
+      };
+
+      throw error;
+    }
+
+    fetchSpy.mockRestore();
+
+    return fetch(args[0] as URL, args[1] as RequestInit);
+  });
+}
+
+/**
+ * @group node
+ */
+describe('Retries correctly', () => {
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  const maxRetries = 4;
+  const baseDuration = 150;
+
+  function assertBackoff(callTime: number, index: number, arr: number[], expectedWaitTime: number) {
+    if (index === 0) {
+      return;
+    } // initial call doesn't count as it's not a retry
+
+    const waitTime = callTime - arr[index - 1];
+
+    // in one test run the waitTime was 1ms less than the expectedWaitTime
+    // meaning that the call happened before the wait duration expired
+    // this might be something related to the event loop and how it schedules setTimeouts
+    // expectedWaitTime minus 5ms seems like reasonable to allow
+    expect(waitTime).toBeGreaterThanOrEqual(expectedWaitTime - 5);
+    expect(waitTime).toBeLessThanOrEqual(expectedWaitTime + 15);
+  }
+
+  test('fixed backoff', async () => {
+    const retryOptions: RetryOptions = { maxRetries, baseDuration, backoff: 'fixed' };
+
+    const provider = await Provider.create(FUEL_NETWORK_URL, { retryOptions });
+
+    const callTimes: number[] = [];
+
+    mockFetch(maxRetries, callTimes);
+
+    const expectedChainInfo = await provider.operations.getChain();
+
+    const chainInfo = await provider.operations.getChain();
+
+    expect(chainInfo.chain.name).toEqual(expectedChainInfo.chain.name);
+    expect(callTimes.length - 1).toBe(maxRetries); // callTimes.length - 1 is for the initial call that's not a retry so we ignore it
+
+    callTimes.forEach((callTime, index) => assertBackoff(callTime, index, callTimes, baseDuration));
+  });
+
+  test('linear backoff', async () => {
+    const retryOptions = {
+      maxRetries,
+      backoff: 'linear' as const,
+    };
+
+    const provider = await Provider.create(FUEL_NETWORK_URL, { retryOptions });
+    const callTimes: number[] = [];
+
+    mockFetch(maxRetries, callTimes);
+
+    const expectedChainInfo = await provider.operations.getChain();
+
+    const chainInfo = await provider.operations.getChain();
+
+    expect(chainInfo.chain.name).toEqual(expectedChainInfo.chain.name);
+    expect(callTimes.length - 1).toBe(maxRetries); // callTimes.length - 1 is for the initial call that's not a retry so we ignore it
+
+    callTimes.forEach((callTime, index) =>
+      assertBackoff(callTime, index, callTimes, baseDuration * index)
+    );
+  });
+
+  test('exponential backoff', async () => {
+    // #region provider-retry-options
+    const retryOptions: RetryOptions = {
+      maxRetries,
+      baseDuration,
+      backoff: 'exponential',
+    };
+
+    const provider = await Provider.create(FUEL_NETWORK_URL, { retryOptions });
+    // #endregion provider-retry-options
+
+    const callTimes: number[] = [];
+
+    mockFetch(maxRetries, callTimes);
+
+    const expectedChainInfo = await provider.operations.getChain();
+
+    const chainInfo = await provider.operations.getChain();
+
+    expect(chainInfo.chain.name).toEqual(expectedChainInfo.chain.name);
+    expect(callTimes.length - 1).toBe(maxRetries); // callTimes.length - 1 is for the initial call that's not a retry so we ignore it
+
+    callTimes.forEach((callTime, index) =>
+      assertBackoff(callTime, index, callTimes, baseDuration * (2 ^ (index - 1)))
+    );
+  });
+
+  test('throws if last attempt fails', async () => {
+    const retryOptions = {
+      maxRetries,
+      backoff: 'fixed' as const,
+    };
+
+    const provider = await Provider.create(FUEL_NETWORK_URL, { retryOptions });
+
+    const fetchSpy = vi.spyOn(global, 'fetch').mockImplementation(() => {
+      const error = new Error() as Error & { cause: { code: string } };
+      error.cause = {
+        code: 'ECONNREFUSED',
+      };
+
+      throw error;
+    });
+
+    const { error } = await safeExec(() => provider.operations.getChain());
+
+    expect(error).toMatchObject({ cause: { code: 'ECONNREFUSED' } });
+    // the added one is for the initial call which isn't considered a retry attempt
+    expect(fetchSpy).toHaveBeenCalledTimes(maxRetries + 1);
+  });
+});