Resolve TODOs (#84)

* Remove TODOs from diagram

* Rename fetchInterval to signedDataFetchInterval

* Remove non-actionable TODOs

* Remove ESLint TODO

* Fix logic to determine beacon set updates, call real update feeds

* Minor renaming

* Initialize sponsor wallets in e2e test

* Log batch processing result

* Log warning when there are no active dAPIs

* Fix tests

* Apply review suggestions

* Make tests more reliable

.eslintrc.js

@@ -20,8 +20,6 @@ module.exports = {
   rules: {
     'unicorn/prefer-top-level-await': 'off',
     'unicorn/no-process-exit': 'off',
-    // TODO this can be removed after removing non-camelcased config values
-    camelcase: 'off',
 
     // Typescript
     '@typescript-eslint/no-var-requires': 'off',

README.md

@@ -181,7 +181,7 @@ The interval specifying how often to run the data feed update loop. In seconds.
 
 The batch size of active dAPIs that are to be fetched in a single RPC call.
 
-#### `fetchInterval`
+#### `signedDataFetchInterval`
 
 The fetch interval in seconds between retrievals of signed API data.
 

airseeker_v2_pipeline.drawio

@@ -1,13 +1,16 @@
-<mxfile host="app.diagrams.net" modified="2023-11-02T12:21:57.340Z" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36" etag="EZSKkGVehWDqeleKX_O1" version="22.0.8" type="device">
+<mxfile host="app.diagrams.net" modified="2023-11-09T08:39:31.290Z" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36" etag="yRIwGNZNJzACz53eiixr" version="22.1.0" type="device">
   <diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1">
-    <mxGraphModel dx="1221" dy="640" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="3300" pageHeight="4681" math="0" shadow="0">
+    <mxGraphModel dx="1259" dy="680" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="3300" pageHeight="4681" math="0" shadow="0">
       <root>
         <mxCell id="WIyWlLk6GJQsqaUBKTNV-0" />
         <mxCell id="WIyWlLk6GJQsqaUBKTNV-1" parent="WIyWlLk6GJQsqaUBKTNV-0" />
         <mxCell id="ci7EG28U3f9VGxeywyoC-37" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="ci7EG28U3f9VGxeywyoC-28" target="ci7EG28U3f9VGxeywyoC-34" edge="1">
           <mxGeometry relative="1" as="geometry" />
         </mxCell>
-        <mxCell id="ci7EG28U3f9VGxeywyoC-28" value="Repeat indefinitely, after every&lt;br&gt;&lt;i&gt;fetchInterval.&lt;br&gt;&lt;/i&gt;" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;align=center;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+        <mxCell id="HytMPlxkX1mnba_mCnJT-0" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="ci7EG28U3f9VGxeywyoC-28" target="ci7EG28U3f9VGxeywyoC-115">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="ci7EG28U3f9VGxeywyoC-28" value="Repeat indefinitely, after every&lt;br&gt;&lt;i&gt;signedDataFetchInterval.&lt;br&gt;&lt;/i&gt;" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;align=center;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
           <mxGeometry x="80" y="800" width="240" height="80" as="geometry" />
         </mxCell>
         <mxCell id="ci7EG28U3f9VGxeywyoC-29" value="&lt;font style=&quot;&quot;&gt;&lt;b&gt;&lt;font style=&quot;font-size: 15px;&quot;&gt;Signed data fetching loop&lt;/font&gt;&lt;/b&gt;&lt;br&gt;&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
@@ -82,7 +85,7 @@
         <mxCell id="ci7EG28U3f9VGxeywyoC-70" value="for all active dAPIs&lt;br&gt;in the batch" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="ci7EG28U3f9VGxeywyoC-57" target="ci7EG28U3f9VGxeywyoC-69" edge="1">
           <mxGeometry relative="1" as="geometry" />
         </mxCell>
-        <mxCell id="ci7EG28U3f9VGxeywyoC-57" value="Fetch the active dAPIs batch together with Airnode address and template ID(s).&lt;br style=&quot;border-color: var(--border-color);&quot;&gt;&lt;br style=&quot;border-color: var(--border-color);&quot;&gt;TODO: This is not yet finalized.&lt;br&gt;" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;align=center;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+        <mxCell id="ci7EG28U3f9VGxeywyoC-57" value="" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;align=center;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
           <mxGeometry x="1280" y="1479" width="240" height="80" as="geometry" />
         </mxCell>
         <mxCell id="ci7EG28U3f9VGxeywyoC-58" value="fetch&amp;nbsp;from" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="ci7EG28U3f9VGxeywyoC-57" target="ci7EG28U3f9VGxeywyoC-59" edge="1">
@@ -115,8 +118,8 @@
             <mxPoint x="1920" y="1520" as="sourcePoint" />
           </mxGeometry>
         </mxCell>
-        <mxCell id="ci7EG28U3f9VGxeywyoC-78" value="Fetch the active dAPIs batch together with Airnode address and template ID(s).&lt;br style=&quot;border-color: var(--border-color);&quot;&gt;&lt;br style=&quot;border-color: var(--border-color);&quot;&gt;TODO: This is not yet finalized.&lt;br&gt;" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;align=center;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
-          <mxGeometry x="1290" y="1489" width="240" height="80" as="geometry" />
+        <mxCell id="ci7EG28U3f9VGxeywyoC-78" value="" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;align=center;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+          <mxGeometry x="1290" y="1490" width="240" height="80" as="geometry" />
         </mxCell>
         <mxCell id="ci7EG28U3f9VGxeywyoC-79" value="Fetch the rest of the active dAPI batches together with Airnode address, template ID(s), signed API URLs and on chain values." style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;align=center;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
           <mxGeometry x="1300" y="1499" width="240" height="80" as="geometry" />
@@ -155,7 +158,7 @@
         <mxCell id="ci7EG28U3f9VGxeywyoC-109" value="Config" style="ellipse;whiteSpace=wrap;html=1;rounded=1;glass=0;strokeWidth=1;shadow=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
           <mxGeometry x="640" y="960" width="80" height="80" as="geometry" />
         </mxCell>
-        <mxCell id="ci7EG28U3f9VGxeywyoC-114" value="read&lt;i style=&quot;border-color: var(--border-color); font-size: 12px; background-color: rgb(251, 251, 251);&quot;&gt;&amp;nbsp;&lt;/i&gt;&lt;i style=&quot;border-color: var(--border-color); font-size: 12px; background-color: rgb(251, 251, 251);&quot;&gt;fetchInterval&lt;/i&gt;&lt;br&gt;from" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="WIyWlLk6GJQsqaUBKTNV-1" target="ci7EG28U3f9VGxeywyoC-115" edge="1">
+        <mxCell id="ci7EG28U3f9VGxeywyoC-114" value="read&lt;i style=&quot;border-color: var(--border-color); font-size: 12px; background-color: rgb(251, 251, 251);&quot;&gt;&amp;nbsp;signedDataFetchInterval&lt;br&gt;&lt;/i&gt;from" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="WIyWlLk6GJQsqaUBKTNV-1" target="ci7EG28U3f9VGxeywyoC-115" edge="1">
           <mxGeometry relative="1" as="geometry">
             <mxPoint x="200" y="880" as="sourcePoint" />
           </mxGeometry>

config/airseeker.example.json

@@ -23,5 +23,5 @@
     }
   },
   "deviationThresholdCoefficient": 1,
-  "fetchInterval": 10000
+  "signedDataFetchInterval": 10000
 }

src/condition-check/condition-check.ts

@@ -14,6 +14,7 @@ export const calculateUpdateInPercentage = (initialValue: ethers.BigNumber, upda
 };
 
 export const calculateMedian = (arr: ethers.BigNumber[]) => {
+  if (arr.length === 0) throw new Error('Cannot calculate median of empty array');
   const mid = Math.floor(arr.length / 2);
 
   const nums = [...arr].sort((a, b) => {
@@ -22,7 +23,7 @@ export const calculateMedian = (arr: ethers.BigNumber[]) => {
     else return 0;
   });
 
-  return arr.length % 2 === 0 ? nums[mid - 1]!.add(nums[mid]!).div(2) : nums[mid];
+  return arr.length % 2 === 0 ? nums[mid - 1]!.add(nums[mid]!).div(2) : nums[mid]!;
 };
 
 /**

src/config/schema.ts

@@ -22,7 +22,7 @@ export type Provider = z.infer<typeof providerSchema>;
 export const optionalContractsSchema = z
   .object({
     Api3ServerV1: evmAddressSchema.optional(),
-    DapiDataRegistry: evmAddressSchema, // TODO: Make optional and load from "airnode-protocol-v1" or some other location and document it accordingly.
+    DapiDataRegistry: evmAddressSchema,
   })
   .strict();
 
@@ -120,7 +120,7 @@ export const configSchema = z
   .object({
     sponsorWalletMnemonic: z.string().refine((mnemonic) => ethers.utils.isValidMnemonic(mnemonic), 'Invalid mnemonic'),
     chains: chainsSchema,
-    fetchInterval: z.number().positive(), // TODO: Rename to signedDataFetchInterval
+    signedDataFetchInterval: z.number().positive(),
     deviationThresholdCoefficient: z.number().positive().optional().default(1), // Explicitly agreed to make this optional. See: https://github.com/api3dao/airseeker-v2/pull/20#issuecomment-1750856113.
   })
   .strict();

src/constants.ts

@@ -1,6 +1,13 @@
+import { ethers } from 'ethers';
+
 export const HTTP_SIGNED_DATA_API_ATTEMPT_TIMEOUT = 10_000;
 export const HTTP_SIGNED_DATA_API_HEADROOM = 1000;
 
 export const HUNDRED_PERCENT = 1e8;
 
 export const AIRSEEKER_PROTOCOL_ID = '5'; // From: https://github.com/api3dao/airnode/blob/ef16c54f33d455a1794e7886242567fc47ee14ef/packages/airnode-protocol/src/index.ts#L46
+
+// Solidity type(int224).min
+export const INT224_MIN = ethers.BigNumber.from(2).pow(ethers.BigNumber.from(223)).mul(ethers.BigNumber.from(-1));
+// Solidity type(int224).max
+export const INT224_MAX = ethers.BigNumber.from(2).pow(ethers.BigNumber.from(223)).sub(ethers.BigNumber.from(1));

src/gas-price/index.ts

@@ -0,0 +1 @@
+export * from './gas-price';

src/signed-api-fetch/data-fetcher.ts

@@ -32,7 +32,6 @@ const callSignedDataApi = async (url: string): Promise<SignedData[]> => {
         url,
         headers: {
           Accept: 'application/json',
-          // TODO add API key?
         },
       }),
     {
@@ -53,12 +52,16 @@ const callSignedDataApi = async (url: string): Promise<SignedData[]> => {
 
 export const runDataFetcher = async () => {
   const state = getState();
-  const { config, signedApiUrlStore, dataFetcherInterval } = state;
+  const {
+    config: { signedDataFetchInterval },
+    signedApiUrlStore,
+    dataFetcherInterval,
+  } = state;
 
-  const fetchInterval = config.fetchInterval * 1000;
+  const signedDataFetchIntervalMs = signedDataFetchInterval * 1000;
 
   if (!dataFetcherInterval) {
-    const dataFetcherInterval = setInterval(runDataFetcher, fetchInterval);
+    const dataFetcherInterval = setInterval(runDataFetcher, signedDataFetchIntervalMs);
     updateState((draft) => {
       draft.dataFetcherInterval = dataFetcherInterval;
     });
@@ -82,8 +85,8 @@ export const runDataFetcher = async () => {
         },
         {
           retries: 0,
-          totalTimeoutMs: fetchInterval + HTTP_SIGNED_DATA_API_HEADROOM,
-          attemptTimeoutMs: fetchInterval + HTTP_SIGNED_DATA_API_HEADROOM - 100,
+          totalTimeoutMs: signedDataFetchIntervalMs + HTTP_SIGNED_DATA_API_HEADROOM,
+          attemptTimeoutMs: signedDataFetchIntervalMs + HTTP_SIGNED_DATA_API_HEADROOM - 100,
         }
       )
     )

src/state/state.ts

@@ -9,8 +9,8 @@ import type {
   ChainId,
   SignedData,
   DataFeedId,
-  Provider,
-  DApiName,
+  ProviderName,
+  SignedApiUrl,
 } from '../types';
 
 interface GasState {
@@ -41,8 +41,8 @@ export interface State {
   gasPriceStore: Record<string, Record<string, GasState>>;
   derivedSponsorWallets: Record<DapiName, PrivateKey>;
   signedApiStore: Record<DataFeedId, SignedData>;
-  signedApiUrlStore: Record<ChainId, Record<Provider, string[]>>;
-  dapis: Record<DApiName, DapiState>;
+  signedApiUrlStore: Record<ChainId, Record<ProviderName, SignedApiUrl[]>>;
+  dapis: Record<DapiName, DapiState>;
 }
 
 type StateUpdater = (draft: Draft<State>) => void;

src/types.ts

@@ -8,8 +8,8 @@ export type DapiName = string;
 export type PrivateKey = string;
 export type DataFeedId = EvmId;
 export type ChainId = string;
-export type DApiName = string;
-export type Provider = string;
+export type ProviderName = string;
+export type SignedApiUrl = string;
 
 // Taken from https://github.com/api3dao/signed-api/blob/main/packages/api/src/schema.ts
 export const signedDataSchema = z.object({
@@ -30,7 +30,7 @@ export const signedApiResponseSchema = z.object({
 export interface Beacon {
   airnodeAddress: AirnodeAddress;
   templateId: TemplateId;
-  dataFeedId: string;
+  beaconId: string;
 }
 
 export interface DecodedDataFeed {

src/update-feeds/api3-server-v1.ts

@@ -1,5 +1,5 @@
-import { Api3ServerV1__factory } from '@api3/airnode-protocol-v1';
+import { Api3ServerV1__factory as Api3ServerV1Factory } from '@api3/airnode-protocol-v1';
 import type { ethers } from 'ethers';
 
 export const getApi3ServerV1 = (address: string, provider: ethers.providers.StaticJsonRpcProvider) =>
-  Api3ServerV1__factory.connect(address, provider);
+  Api3ServerV1Factory.connect(address, provider);

src/update-feeds/dapi-data-registry.test.ts

@@ -5,19 +5,19 @@ import { decodeDataFeed } from './dapi-data-registry';
 describe('helper functions', () => {
   it('decodes dataFeed bytes into objects', () => {
     const single = encodeBeaconFeed({
-      dataFeedId: '0xf5c140bcb4814dfec311d38f6293e86c02d32ba1b7da027fe5b5202cae35dbc6',
+      beaconId: '0xf5c140bcb4814dfec311d38f6293e86c02d32ba1b7da027fe5b5202cae35dbc6',
       airnodeAddress: '0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4',
       templateId: '0x457a3b3da67e394a895ea49e534a4d91b2d009477bef15eab8cbed313925b010',
     });
 
     const multiple = encodeBeaconFeedSet([
       {
-        dataFeedId: '0xf5c140bcb4814dfec311d38f6293e86c02d32ba1b7da027fe5b5202cae35dbc6',
+        beaconId: '0xf5c140bcb4814dfec311d38f6293e86c02d32ba1b7da027fe5b5202cae35dbc6',
         airnodeAddress: '0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4',
         templateId: '0x457a3b3da67e394a895ea49e534a4d91b2d009477bef15eab8cbed313925b010',
       },
       {
-        dataFeedId: '0xf5c140bcb4814dfec311d38f6293e86c02d32ba1b7da027fe5b5202cae35dbc6',
+        beaconId: '0xf5c140bcb4814dfec311d38f6293e86c02d32ba1b7da027fe5b5202cae35dbc6',
         airnodeAddress: '0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4',
         templateId: '0x457a3b3da67e394a895ea49e534a4d91b2d009477bef15eab8cbed313925b010',
       },
@@ -31,7 +31,7 @@ describe('helper functions', () => {
       beacons: [
         {
           airnodeAddress: '0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4',
-          dataFeedId: '0xf5c140bcb4814dfec311d38f6293e86c02d32ba1b7da027fe5b5202cae35dbc6',
+          beaconId: '0xf5c140bcb4814dfec311d38f6293e86c02d32ba1b7da027fe5b5202cae35dbc6',
           templateId: '0x457a3b3da67e394a895ea49e534a4d91b2d009477bef15eab8cbed313925b010',
         },
       ],
@@ -41,12 +41,12 @@ describe('helper functions', () => {
       beacons: [
         {
           airnodeAddress: '0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4',
-          dataFeedId: '0xf5c140bcb4814dfec311d38f6293e86c02d32ba1b7da027fe5b5202cae35dbc6',
+          beaconId: '0xf5c140bcb4814dfec311d38f6293e86c02d32ba1b7da027fe5b5202cae35dbc6',
           templateId: '0x457a3b3da67e394a895ea49e534a4d91b2d009477bef15eab8cbed313925b010',
         },
         {
           airnodeAddress: '0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4',
-          dataFeedId: '0xf5c140bcb4814dfec311d38f6293e86c02d32ba1b7da027fe5b5202cae35dbc6',
+          beaconId: '0xf5c140bcb4814dfec311d38f6293e86c02d32ba1b7da027fe5b5202cae35dbc6',
           templateId: '0x457a3b3da67e394a895ea49e534a4d91b2d009477bef15eab8cbed313925b010',
         },
       ],

src/update-feeds/dapi-data-registry.ts

@@ -1,12 +1,12 @@
 import { ethers } from 'ethers';
 
 // NOTE: The contract is not yet published, so we generate the Typechain artifacts locally and import it from there.
-import { type DapiDataRegistry, DapiDataRegistry__factory } from '../../typechain-types';
+import { type DapiDataRegistry, DapiDataRegistry__factory as DapiDataRegistryFactory } from '../../typechain-types';
 import type { DecodedDataFeed } from '../types';
 import { deriveBeaconId, deriveBeaconSetId } from '../utils';
 
 export const getDapiDataRegistry = (address: string, provider: ethers.providers.StaticJsonRpcProvider) =>
-  DapiDataRegistry__factory.connect(address, provider);
+  DapiDataRegistryFactory.connect(address, provider);
 
 export const verifyMulticallResponse = (
   response: Awaited<ReturnType<DapiDataRegistry['callStatic']['tryMulticall']>>
@@ -32,21 +32,21 @@ export const decodeDataFeed = (dataFeed: string): DecodedDataFeed => {
 
     const dataFeedId = deriveBeaconId(airnodeAddress, templateId)!;
 
-    return { dataFeedId, beacons: [{ dataFeedId, airnodeAddress, templateId }] };
+    return { dataFeedId, beacons: [{ beaconId: dataFeedId, airnodeAddress, templateId }] };
   }
 
   const [airnodeAddresses, templateIds] = ethers.utils.defaultAbiCoder.decode(['address[]', 'bytes32[]'], dataFeed);
 
-  const dataFeeds = (airnodeAddresses as string[]).map((airnodeAddress: string, idx: number) => {
+  const beacons = (airnodeAddresses as string[]).map((airnodeAddress: string, idx: number) => {
     const templateId = templateIds[idx] as string;
-    const dataFeedId = deriveBeaconId(airnodeAddress, templateId)!;
+    const beaconId = deriveBeaconId(airnodeAddress, templateId)!;
 
-    return { dataFeedId, airnodeAddress, templateId };
+    return { beaconId, airnodeAddress, templateId };
   });
 
-  const dataFeedId = deriveBeaconSetId(dataFeeds.map((df) => df.dataFeedId))!;
+  const dataFeedId = deriveBeaconSetId(beacons.map((b) => b.beaconId))!;
 
-  return { dataFeedId, beacons: dataFeeds };
+  return { dataFeedId, beacons };
 };
 
 export const decodeReadDapiWithIndexResponse = (