feat: load interactions in pages

warp-contracts · Feb 28, 2024 · e5e33e2 · e5e33e2
1 parent e31aa05
commit e5e33e2
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 25 deletions.
diff --git a/.env.defaults b/.env.defaults
@@ -25,8 +25,8 @@ APPSYNC_STREAM=''
 NODE_JWK_KEY=''
 UPDATE_MODE='poll'
 # 3h (3 * 60 * 60s), 1h (60 * 60s), 1m (60s), 2s
-SYNC_WINDOW_SECONDS='[10800,3600,60,2]'
-POLL_RESPONSE_LENGTH_LIMIT=30000
+SYNC_WINDOW_SECONDS='[3600,60,2]'
+POLL_RESPONSE_LENGTH_LIMIT=500
 POLL_INTERACTIONS_URL='http://api-dre-sync.warp.cc/v1/ro/interactions'
 # Spawns a separate node process to evaluate a partition of interactions
 POLL_FORK_PROCESS=false

diff --git a/src/config.js b/src/config.js
@@ -108,7 +108,7 @@ const config = {
   },
   syncWindowSeconds: JSON.parse(process.env.SYNC_WINDOW_SECONDS),
   firstInteractionTimestamp: parseInt(process.env.FIRST_INTERACTION_TIMESTAMP),
-  pollResponseLengthLimit: process.env.POLL_RESPONSE_LENGTH_LIMIT ? parseInt(process.env.POLL_RESPONSE_LENGTH_LIMIT) : 15000,
+  pollResponseLengthLimit: process.env.POLL_RESPONSE_LENGTH_LIMIT ? parseInt(process.env.POLL_RESPONSE_LENGTH_LIMIT) : 500,
   pollLoadInteractionsUrl: readLoadInteractionsUrl(),
   pollForkProcess: process.env.POLL_FORK_PROCESS === 'true',
   whitelistMode: JSON.parse(process.env.EVALUATION_WHITELIST_SOURCES).length > 0,

diff --git a/src/loadInteractions.js b/src/loadInteractions.js
@@ -1,9 +1,10 @@
 const { config } = require('./config');
-module.exports = async (startTimestamp, endTimestamp, whiteListedSources, blacklistedContracts, limit) => {
+module.exports = async (startTimestamp, endTimestamp, whiteListedSources, blacklistedContracts, limit, offset) => {
   const response = await postData({
     start: startTimestamp,
     end: endTimestamp,
     limit,
+    offset,
     src_ids: whiteListedSources,
     blacklisted_contracts: blacklistedContracts
   });

diff --git a/src/workers/pollGateway.js b/src/workers/pollGateway.js
@@ -11,12 +11,7 @@ const { init } = require('./pollWorkerRunner');
 const logger = LoggerFactory.INST.create('syncer');
 LoggerFactory.INST.logLevel('info', 'syncer');
 
-function filterInvalidEntries(entries, responseSizeLimit) {
-  if (entries.length >= responseSizeLimit) {
-    logger.warn(`Max entries in response (${responseSizeLimit}), either reduce window size or increase interactions limit in response via the .env.POLL_RESPONSE_LENGTH_LIMIT`);
-    process.exit(0);
-  }
-
+function filterInvalidEntries(entries) {
   const validEntries = entries.filter((entry) => {
     if (!entry.contractTxId || !isTxIdValid(entry.contractTxId)) {
       logger.warn(`No valid 'contractTxId' in entry: ${JSON.stringify(entry)}`);
@@ -89,22 +84,36 @@ module.exports = async function (
         toDate: new Date(endTimestamp)
       });
 
-      let result;
+      let result = {
+        interactions: []
+      };
+      let partialResult;
+      let offset = 0;
       try {
-        result = await loadInteractions(
-          startTimestamp,
-          endTimestamp,
-          whitelistedSources,
-          blacklistedContracts,
-          config.pollResponseLengthLimit
-        );
-        if (!result) {
-          throw new Error('Result is null or undefined');
-        }
-        if (!result.interactions) {
-          throw new Error("Result does not contain 'interactions' field");
-        }
-        result.interactions = filterInvalidEntries(result.interactions, config.pollResponseLengthLimit);
+        do {
+          logger.info(`Loading interactions for`, {
+            startTimestamp,
+            endTimestamp,
+            limit: config.pollResponseLengthLimit,
+            offset
+          });
+          partialResult = await loadInteractions(
+              startTimestamp,
+              endTimestamp,
+              whitelistedSources,
+              blacklistedContracts,
+              config.pollResponseLengthLimit,
+              offset
+          );
+          if (!partialResult) {
+            throw new Error('Result is null or undefined');
+          }
+          if (!partialResult.interactions) {
+            throw new Error("Result does not contain 'interactions' field");
+          }
+          result.interactions.concat(filterInvalidEntries(partialResult.interactions));
+          offset += config.pollResponseLengthLimit;
+        } while (partialResult.interactions.length === config.pollResponseLengthLimit)
       } catch (e) {
         logger.error(
           'Error while loading interactions',