Merge pull request #384 from HubSpot/issue/381

Fix for #381 - downloadHubDbTable would download incorrect results for large tables
HubSpot · Nov 10, 2020 · b8764f3 · b8764f3
2 parents 1631d53 + 6866b28
commit b8764f3
Show file tree

Hide file tree

Showing 4 changed files with 68 additions and 32 deletions.
diff --git a/packages/cms-lib/__tests__/fixtures/hubdb/hubdbFetchRowsResponse.json b/packages/cms-lib/__tests__/fixtures/hubdb/hubdbFetchRowsResponse.json
@@ -10,5 +10,6 @@
       "name": "My Best Event",
       "values": { "first_col": "e", "second_col": "f" }
     }
-  ]
+  ],
+  "paging": null
 }
diff --git a/packages/cms-lib/__tests__/fixtures/hubdb/hubdbFetchRowsWithPaging.json b/packages/cms-lib/__tests__/fixtures/hubdb/hubdbFetchRowsWithPaging.json
@@ -0,0 +1,19 @@
+{
+  "total": 3,
+  "results": [
+    { "name": "Paging 1", "values": { "first_col": "a", "second_col": "b" } },
+    {
+      "name": "Paging 2",
+      "values": { "first_col": "c", "second_col": "d" }
+    },
+    {
+      "name": "Paging 3",
+      "values": { "first_col": "e", "second_col": "f" }
+    }
+  ],
+  "paging": {
+    "next": {
+      "after": "testing"
+    }
+  }
+}
diff --git a/packages/cms-lib/__tests__/hubdb.js b/packages/cms-lib/__tests__/hubdb.js
@@ -4,6 +4,7 @@ const hubdb = require('../api/hubdb');
 const { getCwd } = require('../path');
 const hubdbJson = require('./fixtures/hubdb/hubdbTableData');
 const hubdbFetchRowResponse = require('./fixtures/hubdb/hubdbFetchRowsResponse.json');
+const hubdbFetchRowsWithPagingResponse = require('./fixtures/hubdb/hubdbFetchRowsWithPaging.json');
 const hubdbFetchTableResponse = require('./fixtures/hubdb/hubdbFetchTableResponse.json');
 const hubdbCreateTableResponse = require('./fixtures/hubdb/hubdbCreateTableResponse.json');
 const hubdbCreateRowsResponse = require('./fixtures/hubdb/hubdbCreateRowsResponse.json');
@@ -43,6 +44,28 @@ describe('cms-lib/hubdb', () => {
     });
   });
 
+  describe('paging', () => {
+    it('fetches all results', async () => {
+      const accountId = 123;
+      const tableId = 456;
+      const destPath = 'tmp.json';
+      const projectCwd = '/home/tom/projects';
+
+      getCwd.mockReturnValue(projectCwd);
+
+      hubdb.fetchRows.mockReturnValue(hubdbFetchRowResponse);
+      hubdb.fetchTable.mockReturnValue(hubdbFetchTableResponse);
+
+      hubdb.fetchRows.mockReturnValueOnce(hubdbFetchRowsWithPagingResponse);
+
+      await downloadHubDbTable(accountId, tableId, destPath);
+      const fileOutput = JSON.parse(fs.outputFile.mock.results[1].value);
+      expect(fileOutput.rows.length).toEqual(6);
+      expect(fileOutput.rows[0].name).toMatchInlineSnapshot(`"Paging 1"`);
+      expect(fileOutput.rows[5].name).toMatchInlineSnapshot(`"My Best Event"`);
+    });
+  });
+
   it('uploads hubdb table', async () => {
     const accountId = 123;
     const srcPath = 'tmp.json';

diff --git a/packages/cms-lib/hubdb.js b/packages/cms-lib/hubdb.js
@@ -124,6 +124,23 @@ function convertToJSON(table, rows) {
   };
 }
 
+async function fetchAllRows(accountId, tableId) {
+  let rows = [];
+  let after = null;
+  do {
+    const { paging, results } = await fetchRows(
+      accountId,
+      tableId,
+      after ? { after } : null
+    );
+
+    rows = rows.concat(results);
+    after = paging && paging.next ? paging.next.after : null;
+  } while (after !== null);
+
+  return rows;
+}
+
 async function downloadHubDbTable(accountId, tableId, dest) {
   const table = await fetchTable(accountId, tableId);
 
@@ -135,21 +152,7 @@ async function downloadHubDbTable(accountId, tableId, dest) {
     validateJsonPath(dest);
   }
 
-  let totalRows = null;
-  let rows = [];
-  let count = 0;
-  let offset = 0;
-  while (totalRows === null || count < totalRows) {
-    const response = await fetchRows(accountId, tableId, { offset });
-    if (totalRows === null) {
-      totalRows = response.total;
-    }
-
-    count += response.results.length;
-    offset += response.results.length;
-    rows = rows.concat(response.results);
-  }
-
+  const rows = await fetchAllRows(accountId, tableId);
   const tableToWrite = JSON.stringify(convertToJSON(table, rows));
   const tableJson = prettier.format(tableToWrite, {
     parser: 'json',
@@ -161,22 +164,12 @@ async function downloadHubDbTable(accountId, tableId, dest) {
 }
 
 async function clearHubDbTableRows(accountId, tableId) {
-  let totalRows = null;
-  let rows = [];
-  let count = 0;
-  let offset = 0;
-  while (totalRows === null || count < totalRows) {
-    const response = await fetchRows(accountId, tableId, { offset });
-    if (totalRows === null) {
-      totalRows = response.total;
-    }
-
-    count += response.results.length;
-    offset += response.results.length;
-    const rowIds = response.results.map(row => row.id);
-    rows = rows.concat(rowIds);
-  }
-  await deleteRows(accountId, tableId, rows);
+  const rows = await fetchAllRows(accountId, tableId);
+  await deleteRows(
+    accountId,
+    tableId,
+    rows.map(row => row.id)
+  );
 
   return {
     deletedRowCount: rows.length,