fix(smartsheet): Fix Pagination and add more tests

apps/smartsheet/bundle/bots/load/smartsheet_load/bot.ts

@@ -25,13 +25,15 @@ export default function smartsheet_load(bot: LoadBotApi) {
 		batchNumber = 0,
 		batchSize,
 		collectionMetadata,
-		conditions,
+		conditions
 	} = bot.loadRequest
 	const queryParams = {} as Record<string, FieldValue>
-	if (typeof batchSize === "number" && batchSize > 0) {
+	const doPagination = typeof batchSize === "number" && batchSize > 0
+	if (doPagination) {
 		// SmartSheet pagination uses 1-based indexing
 		queryParams.page = batchNumber + 1
-		queryParams.pageSize = batchSize
+		// Always add one to to the batch size to determine if there are more records
+		queryParams.pageSize = batchSize + 1
 	} else {
 		queryParams.includeAll = true
 	}
@@ -59,7 +61,7 @@ export default function smartsheet_load(bot: LoadBotApi) {
 
 	const response = bot.http.request({
 		method: "GET",
-		url,
+		url
 	})
 	const body = response.body as SheetResponse
 	const fieldsMetadata = collectionMetadata.getAllFieldMetadata()
@@ -71,9 +73,19 @@ export default function smartsheet_load(bot: LoadBotApi) {
 		}
 	})
 
-	body.rows.forEach((row) => {
+	const maxRecordForPagination = doPagination
+		? (queryParams.pageSize as number) - 1
+		: -1
+	body.rows.forEach((row, i) => {
+		// If we are on the final row, do NOT add it to the bot's records,
+		// since we requested one more than we needed, but DO set has more records
+		// so that Uesio pagination controls will work
+		if (doPagination && i === maxRecordForPagination) {
+			bot.setHasMoreRecords()
+			return
+		}
 		const record: Record<string, FieldValue> = {
-			"uesio/core.id": row.id + "",
+			"uesio/core.id": row.id + ""
 		}
 
 		row.cells.forEach((cell) => {

apps/smartsheet/tests/smartsheet_load.spec.ts

@@ -1,35 +1,58 @@
-import { LoadBotApi } from "@uesio/bots"
+import { FieldValue, LoadBotApi, LoadRequestMetadata } from "@uesio/bots"
 import smartsheet_load from "../bundle/bots/load/smartsheet_load/bot"
 
+const sampleUesioCollectionName = "tasks"
+const sampleUesioNS = "luigi/foo"
+const mockBot = (
+	returnRecords: Record<string, FieldValue>[],
+	loadRequest?: Partial<LoadRequestMetadata>
+) => ({
+	loadRequest: {
+		collection: `${sampleUesioNS}.${sampleUesioCollectionName}`,
+		collectionMetadata: getSampleCollectionMetadata(),
+		...(loadRequest || {})
+	},
+	addRecord: jest.fn(),
+	setHasMoreRecords: jest.fn(),
+	http: {
+		request: jest.fn(() => ({
+			code: 200,
+			body: {
+				rows: returnRecords
+			}
+		}))
+	}
+})
+
 const row1 = {
 	id: 1,
 	cells: [
 		{
 			columnId: "taskname",
 			displayValue: "Test",
-			value: "Test",
+			value: "Test"
 		},
 		{
 			columnId: "taskstatus",
 			displayValue: "In Progress",
-			value: "in_progress",
-		},
-	],
+			value: "in_progress"
+		}
+	]
 }
 const row2 = {
 	id: 2,
 	cells: [
 		{
 			columnId: "taskname",
 			displayValue: "Another test",
-			value: "Another test",
+			value: "Another test"
 		},
 		{
 			columnId: "taskstatus",
 			displayValue: "Completed",
-			value: "completed",
-		},
-	],
+			value: "completed"
+		}
+	]
 }
 
 const getSampleCollectionMetadata = () => ({
@@ -38,120 +61,88 @@ const getSampleCollectionMetadata = () => ({
 		"uesio/smartsheet.name": {
 			externalName: "taskname",
 			name: "name",
-			namespace: "uesio/smartsheet",
+			namespace: "uesio/smartsheet"
 		},
 		"uesio/smartsheet.status": {
 			externalName: "taskstatus",
 			name: "status",
-			namespace: "uesio/smartsheet",
-		},
-	})),
+			namespace: "uesio/smartsheet"
+		}
+	}))
 })
 const uesioRow1 = {
 	"uesio/core.id": "1",
 	"uesio/smartsheet.name": "Test",
-	"uesio/smartsheet.status": "in_progress",
+	"uesio/smartsheet.status": "in_progress"
 }
 const uesioRow2 = {
 	"uesio/core.id": "2",
 	"uesio/smartsheet.name": "Another test",
-	"uesio/smartsheet.status": "completed",
+	"uesio/smartsheet.status": "completed"
 }
 
 describe("Smartsheet Load", () => {
 	it("should load data from Smartsheet", () => {
-		const addRecord = jest.fn()
-		const bot = {
-			loadRequest: {
-				collectionMetadata: getSampleCollectionMetadata(),
-			},
-			addRecord,
-			http: {
-				request: jest.fn(() => ({
-					code: 200,
-					body: {
-						rows: [row1, row2],
-					},
-				})),
-			},
-		}
-
+		const bot = mockBot([row1, row2])
 		smartsheet_load(bot as unknown as LoadBotApi)
-
 		expect(bot.addRecord).toHaveBeenCalledWith(uesioRow1)
 		expect(bot.addRecord).toHaveBeenCalledWith(uesioRow2)
 	})
 	it("should only return specific rows from a sheet if uesio/core.id condition exists (multi-value)", () => {
-		const addRecord = jest.fn()
-		const request = jest.fn(() => ({
-			code: 200,
-			body: {
-				rows: [row1, row2],
-			},
-		}))
-		const bot = {
-			loadRequest: {
-				batchSize: 1,
-				batchNumber: 0,
-				collectionMetadata: getSampleCollectionMetadata(),
-				conditions: [
-					{
-						field: "uesio/core.id",
-						operator: "IN",
-						values: ["1", "2"],
-					},
-				],
-			},
-			addRecord,
-			http: {
-				request,
-			},
-		}
-
+		const bot = mockBot([row1, row2], {
+			batchSize: 10,
+			batchNumber: 0,
+			conditions: [
+				{
+					field: "uesio/core.id",
+					operator: "IN",
+					values: ["1", "2"]
+				}
+			]
+		})
 		smartsheet_load(bot as unknown as LoadBotApi)
-
 		expect(bot.addRecord).toHaveBeenCalledWith(uesioRow1)
 		expect(bot.addRecord).toHaveBeenCalledWith(uesioRow2)
-
-		expect(request).toHaveBeenCalledWith({
+		// We asked for up to 10 records, but only got back 2, so there are no more records available
+		expect(bot.setHasMoreRecords).toHaveBeenCalledTimes(0)
+		expect(bot.http.request).toHaveBeenCalledWith({
 			method: "GET",
-			url: "https://api.smartsheet.com/2.0/sheets/somesheetid?page=1&pageSize=1&rowIds=1%2C2",
+			url: "https://api.smartsheet.com/2.0/sheets/somesheetid?page=1&pageSize=11&rowIds=1%2C2"
 		})
 	})
 	it("should only return one row from a sheet if uesio/core.id condition exists (single-value)", () => {
-		const addRecord = jest.fn()
-		const request = jest.fn(() => ({
-			code: 200,
-			body: {
-				rows: [row2],
-			},
-		}))
-		const bot = {
-			loadRequest: {
-				batchSize: 1,
-				batchNumber: 0,
-				collectionMetadata: getSampleCollectionMetadata(),
-				conditions: [
-					{
-						field: "uesio/core.id",
-						operator: "EQ",
-						value: "2",
-					},
-				],
-			},
-			addRecord,
-			http: {
-				request,
-			},
-		}
-
+		const bot = mockBot([row2], {
+			batchSize: 1,
+			batchNumber: 0,
+			conditions: [
+				{
+					field: "uesio/core.id",
+					operator: "EQ",
+					value: "2"
+				}
+			]
+		})
 		smartsheet_load(bot as unknown as LoadBotApi)
-
 		expect(bot.addRecord).toHaveBeenCalledWith(uesioRow2)
-
-		expect(request).toHaveBeenCalledWith({
+		// We asked for up to 1 records, but only got back 1, so there are no more records available
+		expect(bot.setHasMoreRecords).toHaveBeenCalledTimes(0)
+		expect(bot.http.request).toHaveBeenCalledWith({
+			method: "GET",
+			url: "https://api.smartsheet.com/2.0/sheets/somesheetid?page=1&pageSize=2&rowIds=2"
+		})
+	})
+	it("should indicate that there are more records available from server", () => {
+		const bot = mockBot([row1, row2], {
+			batchSize: 1,
+			batchNumber: 0
+		})
+		smartsheet_load(bot as unknown as LoadBotApi)
+		expect(bot.addRecord).toHaveBeenCalledWith(uesioRow1)
+		// We asked for up to 1 records, and got back 2, so there ARE records available
+		expect(bot.setHasMoreRecords).toHaveBeenCalled()
+		expect(bot.http.request).toHaveBeenCalledWith({
 			method: "GET",
-			url: "https://api.smartsheet.com/2.0/sheets/somesheetid?page=1&pageSize=1&rowIds=2",
+			url: "https://api.smartsheet.com/2.0/sheets/somesheetid?page=1&pageSize=2"
 		})
 	})
 })