From af65b8809cefce15c867b991b844a6a577a3d89e Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Fri, 12 Apr 2024 13:42:50 -0500 Subject: [PATCH] wip --- package-lock.json | 139 +++++++-------- plugins/webhook-egress/package.json | 3 +- .../src/fetch.and.process.sheets.ts | 32 ++++ plugins/webhook-egress/src/post.to.webhook.ts | 35 ++++ plugins/webhook-egress/src/types.ts | 13 ++ .../webhook-egress/src/webhook.egress.spec.ts | 18 +- plugins/webhook-egress/src/webhook.egress.ts | 165 +++++++++++------- utils/common/src/get.secret.ts | 15 ++ utils/common/src/index.ts | 2 + utils/common/src/logging.helper.ts | 5 + utils/common/src/valid.url.ts | 11 ++ utils/response-rejection/src/index.ts | 137 ++++----------- utils/response-rejection/src/prepare.sheet.ts | 27 +++ utils/response-rejection/src/update.sheet.ts | 49 ++++++ 14 files changed, 409 insertions(+), 242 deletions(-) create mode 100644 plugins/webhook-egress/src/fetch.and.process.sheets.ts create mode 100644 plugins/webhook-egress/src/post.to.webhook.ts create mode 100644 plugins/webhook-egress/src/types.ts create mode 100644 utils/common/src/get.secret.ts create mode 100644 utils/common/src/valid.url.ts create mode 100644 utils/response-rejection/src/prepare.sheet.ts create mode 100644 utils/response-rejection/src/update.sheet.ts diff --git a/package-lock.json b/package-lock.json index 0ceee8457..e8a8d4e6f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14890,14 +14890,14 @@ }, "plugins/autocast": { "name": "@flatfile/plugin-autocast", - "version": "0.8.1", + "version": "0.8.2", "license": "ISC", "dependencies": { "@flatfile/hooks": "^1.3.2", - "@flatfile/util-common": "^1.3.1" + "@flatfile/util-common": "^1.3.2" }, "devDependencies": { - "@flatfile/plugin-record-hook": "^1.5.2", + "@flatfile/plugin-record-hook": "^1.5.3", "@flatfile/rollup-config": "0.1.1" }, "engines": { @@ -14906,19 +14906,19 @@ "peerDependencies": { "@flatfile/api": "^1.8.9", "@flatfile/listener": "^1.0.1", - "@flatfile/plugin-record-hook": "^1.5.2" + "@flatfile/plugin-record-hook": "^1.5.3" } }, "plugins/automap": { "name": "@flatfile/plugin-automap", - "version": "0.3.1", + "version": "0.3.2", "license": "ISC", "dependencies": { "@flatfile/common-plugin-utils": "^1.0.2", "remeda": "^1.23.0" }, "devDependencies": { - "@flatfile/utils-testing": "^0.1.5" + "@flatfile/utils-testing": "^0.1.6" }, "engines": { "node": ">= 16" @@ -14930,10 +14930,10 @@ }, "plugins/constraints": { "name": "@flatfile/plugin-constraints", - "version": "1.2.1", + "version": "1.2.2", "license": "ISC", "devDependencies": { - "@flatfile/plugin-record-hook": "^1.5.2", + "@flatfile/plugin-record-hook": "^1.5.3", "@flatfile/rollup-config": "0.1.1" }, "engines": { @@ -14942,16 +14942,16 @@ "peerDependencies": { "@flatfile/api": "^1.8.9", "@flatfile/listener": "^1.0.1", - "@flatfile/plugin-record-hook": "^1.5.2" + "@flatfile/plugin-record-hook": "^1.5.3" } }, "plugins/dedupe": { "name": "@flatfile/plugin-dedupe", - "version": "1.1.1", + "version": "1.1.3", "license": "ISC", "dependencies": { - "@flatfile/plugin-job-handler": "^0.5.1", - "@flatfile/util-common": "^1.3.1" + "@flatfile/plugin-job-handler": "^0.5.2", + "@flatfile/util-common": "^1.3.2" }, "devDependencies": { "@faker-js/faker": "^7.6.0", @@ -14967,10 +14967,10 @@ }, "plugins/delimiter-extractor": { "name": "@flatfile/plugin-delimiter-extractor", - "version": "0.10.1", + "version": "0.10.2", "license": "ISC", "dependencies": { - "@flatfile/util-extractor": "^0.6.1", + "@flatfile/util-extractor": "^0.6.2", "papaparse": "^5.4.1", "remeda": "^1.14.0" }, @@ -14987,7 +14987,7 @@ }, "plugins/dxp-configure": { "name": "@flatfile/plugin-dxp-configure", - "version": "1.1.1", + "version": "1.1.2", "license": "ISC", "devDependencies": { "@flatfile/configure": "^1.0.1", @@ -15003,11 +15003,11 @@ }, "plugins/export-workbook": { "name": "@flatfile/plugin-export-workbook", - "version": "0.2.1", + "version": "0.3.0", "license": "ISC", "devDependencies": { - "@flatfile/plugin-job-handler": "^0.5.1", - "@flatfile/util-common": "^1.3.1" + "@flatfile/plugin-job-handler": "^0.5.2", + "@flatfile/util-common": "^1.3.2" }, "engines": { "node": ">= 16" @@ -15015,15 +15015,15 @@ "peerDependencies": { "@flatfile/api": "^1.8.9", "@flatfile/listener": "^1.0.1", - "@flatfile/plugin-job-handler": "^0.5.1", - "@flatfile/util-common": "^1.3.1", + "@flatfile/plugin-job-handler": "^0.5.2", + "@flatfile/util-common": "^1.3.2", "remeda": "^1.14.0", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" } }, "plugins/foreign-db-extractor": { "name": "@flatfile/plugin-foreign-db-extractor", - "version": "0.1.1", + "version": "0.1.2", "license": "ISC", "dependencies": { "cross-fetch": "^4.0.0", @@ -15059,14 +15059,14 @@ }, "plugins/job-handler": { "name": "@flatfile/plugin-job-handler", - "version": "0.5.1", + "version": "0.5.2", "license": "ISC", "dependencies": { - "@flatfile/util-common": "^1.3.1" + "@flatfile/util-common": "^1.3.2" }, "devDependencies": { "@flatfile/rollup-config": "0.1.1", - "@flatfile/utils-testing": "^0.1.5" + "@flatfile/utils-testing": "^0.1.6" }, "engines": { "node": ">= 16" @@ -15092,10 +15092,10 @@ }, "plugins/json-schema": { "name": "@flatfile/plugin-convert-json-schema", - "version": "0.3.2", + "version": "0.3.3", "license": "ISC", "dependencies": { - "@flatfile/plugin-space-configure": "^0.5.1", + "@flatfile/plugin-space-configure": "^0.5.2", "cross-fetch": "^4.0.0" }, "engines": { @@ -15108,11 +15108,11 @@ }, "plugins/merge-connection": { "name": "@flatfile/plugin-connect-via-merge", - "version": "0.3.1", + "version": "0.3.2", "license": "ISC", "dependencies": { - "@flatfile/plugin-convert-openapi-schema": "^0.2.2", - "@flatfile/plugin-job-handler": "^0.5.1", + "@flatfile/plugin-convert-openapi-schema": "^0.2.3", + "@flatfile/plugin-job-handler": "^0.5.2", "@mergeapi/merge-node-client": "^1.0.4" }, "engines": { @@ -15125,10 +15125,10 @@ }, "plugins/openapi-schema": { "name": "@flatfile/plugin-convert-openapi-schema", - "version": "0.2.2", + "version": "0.2.3", "license": "ISC", "dependencies": { - "@flatfile/plugin-space-configure": "^0.5.1", + "@flatfile/plugin-space-configure": "^0.5.2", "cross-fetch": "^4.0.0" }, "devDependencies": { @@ -15144,11 +15144,11 @@ }, "plugins/pdf-extractor": { "name": "@flatfile/plugin-pdf-extractor", - "version": "0.2.1", + "version": "0.2.2", "license": "ISC", "dependencies": { - "@flatfile/util-common": "^1.3.1", - "@flatfile/util-file-buffer": "^0.3.1", + "@flatfile/util-common": "^1.3.2", + "@flatfile/util-file-buffer": "^0.3.2", "cross-fetch": "^4.0.0", "form-data": "^4.0.0", "fs-extra": "^11.1.1", @@ -15196,7 +15196,7 @@ }, "plugins/psv-extractor": { "name": "@flatfile/plugin-psv-extractor", - "version": "1.8.1", + "version": "1.8.2", "license": "ISC", "engines": { "node": ">= 16" @@ -15208,10 +15208,10 @@ }, "plugins/record-hook": { "name": "@flatfile/plugin-record-hook", - "version": "1.5.2", + "version": "1.5.3", "license": "ISC", "dependencies": { - "@flatfile/util-common": "^1.3.1" + "@flatfile/util-common": "^1.3.2" }, "devDependencies": { "@flatfile/rollup-config": "0.1.1" @@ -15227,15 +15227,15 @@ }, "plugins/space-configure": { "name": "@flatfile/plugin-space-configure", - "version": "0.5.1", + "version": "0.5.2", "license": "ISC", "dependencies": { - "@flatfile/plugin-job-handler": "^0.5.1" + "@flatfile/plugin-job-handler": "^0.5.2" }, "devDependencies": { "@flatfile/api": "^1.8.9", "@flatfile/rollup-config": "0.1.1", - "@flatfile/utils-testing": "^0.1.5" + "@flatfile/utils-testing": "^0.1.6" }, "engines": { "node": ">= 16" @@ -15247,11 +15247,11 @@ }, "plugins/sql-ddl-converter": { "name": "@flatfile/plugin-convert-sql-ddl", - "version": "0.1.2", + "version": "0.1.3", "license": "ISC", "dependencies": { - "@flatfile/plugin-convert-json-schema": "^0.3.2", - "@flatfile/plugin-space-configure": "^0.5.1", + "@flatfile/plugin-convert-json-schema": "^0.3.3", + "@flatfile/plugin-space-configure": "^0.5.2", "sql-ddl-to-json-schema": "^4.1.0" }, "engines": { @@ -15264,7 +15264,7 @@ }, "plugins/tsv-extractor": { "name": "@flatfile/plugin-tsv-extractor", - "version": "1.7.1", + "version": "1.7.2", "license": "ISC", "engines": { "node": ">= 16" @@ -15276,17 +15276,18 @@ }, "plugins/webhook-egress": { "name": "@flatfile/plugin-webhook-egress", - "version": "1.3.1", + "version": "1.3.2", "license": "ISC", "dependencies": { - "@flatfile/plugin-job-handler": "^0.5.1", - "@flatfile/util-common": "^1.3.1", - "@flatfile/util-response-rejection": "^1.3.1", - "cross-fetch": "^4.0.0" + "@flatfile/plugin-job-handler": "^0.5.2", + "@flatfile/util-common": "^1.3.2", + "@flatfile/util-response-rejection": "^1.3.3", + "cross-fetch": "^4.0.0", + "modern-async": "^2.0.0" }, "devDependencies": { "@flatfile/rollup-config": "0.1.1", - "@flatfile/utils-testing": "^0.1.5", + "@flatfile/utils-testing": "^0.1.6", "jest-fetch-mock": "^3.0.3" }, "engines": { @@ -15299,7 +15300,7 @@ }, "plugins/webhook-event-forwarder": { "name": "@flatfile/plugin-webhook-event-forwarder", - "version": "0.3.1", + "version": "0.3.2", "license": "ISC", "dependencies": { "cross-fetch": "^4.0.0" @@ -15350,16 +15351,16 @@ }, "plugins/yaml-schema": { "name": "@flatfile/plugin-convert-yaml-schema", - "version": "0.2.2", + "version": "0.2.3", "license": "ISC", "dependencies": { - "@flatfile/plugin-convert-json-schema": "^0.3.2", - "@flatfile/plugin-space-configure": "^0.5.1", + "@flatfile/plugin-convert-json-schema": "^0.3.3", + "@flatfile/plugin-space-configure": "^0.5.2", "cross-fetch": "^4.0.0", "js-yaml": "^4.1.0" }, "devDependencies": { - "@flatfile/utils-testing": "^0.1.5", + "@flatfile/utils-testing": "^0.1.6", "express": "^4.18.2", "jest-fetch-mock": "^3.0.3" }, @@ -15387,17 +15388,17 @@ }, "plugins/zip-extractor": { "name": "@flatfile/plugin-zip-extractor", - "version": "0.5.2", + "version": "0.5.3", "license": "ISC", "dependencies": { - "@flatfile/plugin-job-handler": "^0.5.1", - "@flatfile/util-common": "^1.3.0", - "@flatfile/util-file-buffer": "^0.3.0", + "@flatfile/plugin-job-handler": "^0.5.2", + "@flatfile/util-common": "^1.3.2", + "@flatfile/util-file-buffer": "^0.3.2", "adm-zip": "^0.5.10", "modern-async": "^2.0.0" }, "devDependencies": { - "@flatfile/utils-testing": "^0.1.5", + "@flatfile/utils-testing": "^0.1.6", "@types/adm-zip": "^0.4.3" }, "engines": { @@ -15418,7 +15419,7 @@ }, "utils/common": { "name": "@flatfile/util-common", - "version": "1.3.1", + "version": "1.3.2", "license": "ISC", "dependencies": { "@flatfile/cross-env-config": "^0.0.5", @@ -15441,11 +15442,11 @@ }, "utils/extractor": { "name": "@flatfile/util-extractor", - "version": "0.6.1", + "version": "0.6.2", "license": "ISC", "dependencies": { - "@flatfile/util-common": "^1.3.1", - "@flatfile/util-file-buffer": "^0.3.1" + "@flatfile/util-common": "^1.3.2", + "@flatfile/util-file-buffer": "^0.3.2" }, "engines": { "node": ">= 16" @@ -15457,7 +15458,7 @@ }, "utils/fetch-schema": { "name": "@flatfile/util-fetch-schema", - "version": "0.2.1", + "version": "0.2.2", "license": "ISC", "dependencies": { "cross-fetch": "^4.0.0" @@ -15471,7 +15472,7 @@ }, "utils/file-buffer": { "name": "@flatfile/util-file-buffer", - "version": "0.3.1", + "version": "0.3.2", "license": "ISC", "engines": { "node": ">= 16" @@ -15483,10 +15484,10 @@ }, "utils/response-rejection": { "name": "@flatfile/util-response-rejection", - "version": "1.3.2", + "version": "1.3.3", "license": "ISC", "dependencies": { - "@flatfile/util-common": "^1.3.1" + "@flatfile/util-common": "^1.3.2" }, "devDependencies": { "@flatfile/rollup-config": "0.1.1" @@ -15500,7 +15501,7 @@ }, "utils/testing": { "name": "@flatfile/utils-testing", - "version": "0.1.5", + "version": "0.1.6", "license": "ISC", "dependencies": { "@flatfile/api": "^1.8.9", diff --git a/plugins/webhook-egress/package.json b/plugins/webhook-egress/package.json index bfc358c44..ec3b82b74 100644 --- a/plugins/webhook-egress/package.json +++ b/plugins/webhook-egress/package.json @@ -54,7 +54,8 @@ "@flatfile/plugin-job-handler": "^0.5.2", "@flatfile/util-common": "^1.3.2", "@flatfile/util-response-rejection": "^1.3.3", - "cross-fetch": "^4.0.0" + "cross-fetch": "^4.0.0", + "modern-async": "^2.0.0" }, "peerDependencies": { "@flatfile/api": "^1.8.9", diff --git a/plugins/webhook-egress/src/fetch.and.process.sheets.ts b/plugins/webhook-egress/src/fetch.and.process.sheets.ts new file mode 100644 index 000000000..585a79184 --- /dev/null +++ b/plugins/webhook-egress/src/fetch.and.process.sheets.ts @@ -0,0 +1,32 @@ +import type { Flatfile } from '@flatfile/api' +import api from '@flatfile/api' +import { asyncMap } from 'modern-async' + +interface Part { + sheetId: string + pageNumber: number + pageSize: number +} + +export async function prepareParts( + workbookId: string, + pageSize: number, + filter: Flatfile.Filter +): Promise> { + const { data: sheets } = await api.sheets.list({ workbookId }) + + const partsArrays = await asyncMap(sheets, async (sheet) => { + const { + data: { + counts: { total }, + }, + } = await api.sheets.getRecordCounts(sheet.id, { filter }) + return Array.from({ length: Math.ceil(total / pageSize) }, (_, index) => ({ + sheetId: sheet.id, + pageNumber: index + 1, + pageSize, + })) + }) + + return partsArrays.flat() +} diff --git a/plugins/webhook-egress/src/post.to.webhook.ts b/plugins/webhook-egress/src/post.to.webhook.ts new file mode 100644 index 000000000..49344b617 --- /dev/null +++ b/plugins/webhook-egress/src/post.to.webhook.ts @@ -0,0 +1,35 @@ +import { getSecret, isValidUrl } from '@flatfile/util-common' +import { SheetExport } from './types' + +export async function postToWebhook( + sheetExport: SheetExport, + url: string | URL, + urlParams: Array<{ key: string; value: unknown }>, + secretName: string, + environmentId: string, + spaceId: string +) { + const baseUrl = isValidUrl(url) + ? url + : await getSecret(url as string, environmentId) + const queryParams = new URLSearchParams() + urlParams.forEach(({ key, value }) => { + Array.isArray(value) + ? value.forEach((v) => queryParams.append(key, String(v))) + : queryParams.set(key, String(value)) + }) + const secret = secretName + ? await getSecret(secretName, environmentId, spaceId) + : '' + const response = await fetch(`${baseUrl}?${queryParams}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(secret ? { Authorization: `Bearer ${secret}` } : {}), + }, + body: JSON.stringify({ sheet: sheetExport }), + }) + if (!response.ok) + throw new Error(`HTTP ${response.status} ${response.statusText}`) + return response.json() +} diff --git a/plugins/webhook-egress/src/types.ts b/plugins/webhook-egress/src/types.ts new file mode 100644 index 000000000..7a76d7738 --- /dev/null +++ b/plugins/webhook-egress/src/types.ts @@ -0,0 +1,13 @@ +import type { Flatfile } from '@flatfile/api' + +export interface SheetExport extends Flatfile.Sheet { + records: Flatfile.Record_[] +} + +export interface WebhookEgressOptions { + secretName?: string + urlParams?: Array<{ key: string; value: unknown }> + pageSize?: number + filter?: Flatfile.Filter + debug?: boolean +} diff --git a/plugins/webhook-egress/src/webhook.egress.spec.ts b/plugins/webhook-egress/src/webhook.egress.spec.ts index c01e3e17f..0cba2be09 100644 --- a/plugins/webhook-egress/src/webhook.egress.spec.ts +++ b/plugins/webhook-egress/src/webhook.egress.spec.ts @@ -17,9 +17,9 @@ fetchMock.dontMock() describe('webhookEgress() e2e', () => { const listener = setupListener() - let spaceId - let workbookId - let sheetId + let spaceId: string + let workbookId: string + let sheetId: string beforeAll(async () => { const space = await setupSpace() @@ -30,7 +30,7 @@ describe('webhookEgress() e2e', () => { 'notes', ]) workbookId = workbook.id - sheetId = workbook.sheets[0].id + sheetId = workbook.sheets![0].id await createRecords(sheetId, [ { name: 'John Doe', @@ -76,7 +76,7 @@ describe('webhookEgress() e2e', () => { await listener.waitFor('job:ready', 1, 'workbook:egressTestSuccess') const response = await api.jobs.get(successfulJobId) - expect(response.data.outcome.message).toEqual( + expect(response.data.outcome?.message).toEqual( `Data was successfully submitted to the provided webhook. Go check it out at example.com.` ) }) @@ -157,10 +157,10 @@ describe('webhookEgress() e2e', () => { await listener.waitFor('job:ready', 1, 'workbook:egressTestSuccess') const response = await api.jobs.get(successfulJobId) - expect(response.data.outcome.message).toEqual( + expect(response.data.outcome?.message).toEqual( 'The data has been successfully submitted without any rejections. This task is now complete.' ) - expect(response.data.outcome.heading).toEqual('Success!') + expect(response.data.outcome?.heading).toEqual('Success!') }) it('returns rejections', async () => { @@ -208,10 +208,10 @@ describe('webhookEgress() e2e', () => { await listener.waitFor('job:ready', 1, 'workbook:egressTestSuccess') const response = await api.jobs.get(successfulJobId) - expect(response.data.outcome.message).toEqual( + expect(response.data.outcome?.message).toEqual( 'During the data submission process, 1 records were rejected. Please review and correct these records before resubmitting.' ) - expect(response.data.outcome.heading).toEqual('Rejected Records') + expect(response.data.outcome?.heading).toEqual('Rejected Records') }) }) }) diff --git a/plugins/webhook-egress/src/webhook.egress.ts b/plugins/webhook-egress/src/webhook.egress.ts index 7b4b6e17d..5daf217c1 100644 --- a/plugins/webhook-egress/src/webhook.egress.ts +++ b/plugins/webhook-egress/src/webhook.egress.ts @@ -1,83 +1,126 @@ -import { FlatfileClient } from '@flatfile/api' -import { FlatfileListener } from '@flatfile/listener' -import { jobHandler } from '@flatfile/plugin-job-handler' -import { logError } from '@flatfile/util-common' -import { - RejectionResponse, - responseRejectionHandler, -} from '@flatfile/util-response-rejection' +import { Flatfile, FlatfileClient } from '@flatfile/api' +import type FlatfileListener from '@flatfile/listener' +import { FlatfileEvent } from '@flatfile/listener' +import { deleteRecords, getRecordsRaw, logError } from '@flatfile/util-common' +import { responseRejectionHandler } from '@flatfile/util-response-rejection' +import { prepareParts } from './fetch.and.process.sheets' +import { postToWebhook } from './post.to.webhook' +import type { SheetExport, WebhookEgressOptions } from './types' const api = new FlatfileClient() -export function webhookEgress(job: string, webhookUrl?: string) { - return function (listener: FlatfileListener) { - listener.use( - jobHandler(job, async (event, tick) => { - const { workbookId } = event.context - const { data: workbook } = await api.workbooks.get(workbookId) - const { data: workbookSheets } = await api.sheets.list({ workbookId }) +export function webhookEgress( + job: string, + url: string | URL, + options: WebhookEgressOptions = {} +) { + const { + secretName = 'WEBHOOK_TOKEN', + urlParams = [], + pageSize = 10_000, + filter = Flatfile.Filter.Valid, + debug = false, + } = options + let deleteSubmitted = false - await tick(30, 'Getting workbook data') + return (listener: FlatfileListener) => { + listener.on( + 'job:ready', + { job, isPart: false }, + async (event: FlatfileEvent) => { + const { jobId, workbookId } = event.context - const sheets = [] - for (const [_, element] of workbookSheets.entries()) { - const { data: records } = await api.records.get(element.id) - sheets.push({ - ...element, - ...records, + await api.jobs.ack(jobId, { info: 'Splitting Job', progress: 10 }) + + const parts = await prepareParts(workbookId, pageSize, filter) + if (parts.length > 0) { + await api.jobs.split(jobId, { parts }) + await api.jobs.ack(jobId, { + info: `Job Split into ${parts.length} parts.`, + progress: 20, + }) + } else { + await api.jobs.complete(jobId, { + outcome: { message: 'nothing to do' }, }) } + } + ) - await tick(60, 'Posting data to webhook') - + listener.on( + 'job:ready', + { job, isPart: true }, + async (event: FlatfileEvent) => { + const { jobId, environmentId, spaceId } = event.context try { - const webhookReceiver = webhookUrl || process.env.WEBHOOK_SITE_URL - const response = await fetch(webhookReceiver, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - workbook: { - ...workbook, - sheets, - }, - }), + const job = await api.jobs.get(jobId) + const { sheetId, pageNumber } = job.data.partData! + + const records = await getRecordsRaw(sheetId, { + pageNumber, + pageSize, + filter, }) - if (response.status === 200) { - const responseData = await response.json() - const rejections: RejectionResponse = responseData.rejections + if (records instanceof Error) { + throw new Error(`Error fetching records: ${records.message}`) + } - if (rejections) { - return await responseRejectionHandler(rejections) - } + const { data: sheet } = await api.sheets.get(sheetId) + const sheetExport: SheetExport = { ...sheet, records } + const responseData = await postToWebhook( + sheetExport, + url, + urlParams, + secretName, + environmentId, + spaceId + ) - return { - outcome: { - message: `Data was successfully submitted to the provided webhook. Go check it out at ${webhookReceiver}.`, - }, - } + const { rejections } = responseData + if (rejections) { + const response = await responseRejectionHandler(rejections) + deleteSubmitted = rejections.deleteSubmitted + + await api.jobs.complete(jobId, response.jobCompleteDetails) } else { - logError( - '@flatfile/plugin-webhook-egress', - `Failed to submit data to ${webhookReceiver}. Status: ${response.status} ${response.statusText}` - ) - return { + await api.jobs.complete(jobId, { outcome: { - message: `Data was not successfully submitted to the provided webhook. Status: ${response.status} ${response.statusText}`, + message: `Data was successfully submitted to the provided webhook. Check it out at ${url}.`, }, - } + }) } } catch (error) { - logError( - '@flatfile/plugin-webhook-egress', - JSON.stringify(error, null, 2) - ) - // Throw error to fail job - throw new Error(`Error posting data to webhook`) + if (debug) { + logError('@flatfile/plugin-webhook-egress', error.message) + } + await api.jobs.fail(jobId, { outcome: { message: error.message } }) } - }) + } + ) + + listener.on( + 'job:parts-completed', + { job, isPart: false }, + async (event: FlatfileEvent) => { + const { jobId, workbookId } = event.context + if (deleteSubmitted) { + const { data: sheets } = await api.sheets.list({ workbookId }) + for (const sheet of sheets) { + const { + data: { + counts: { valid }, + }, + } = await api.sheets.getRecordCounts(sheet.id) + if (valid > 0) await deleteRecords(sheet.id, { filter: 'valid' }) + } + } + await api.jobs.complete(jobId, { + outcome: { message: 'This job is now complete.' }, + }) + } ) } } + +export * from './types' diff --git a/utils/common/src/get.secret.ts b/utils/common/src/get.secret.ts new file mode 100644 index 000000000..d0aa3d9a0 --- /dev/null +++ b/utils/common/src/get.secret.ts @@ -0,0 +1,15 @@ +import api from '@flatfile/api' +import { handleError } from './logging.helper' + +export async function getSecret( + name: string, + environmentId: string, + spaceId?: string +): Promise { + try { + const secrets = await api.secrets.list({ spaceId, environmentId }) + return secrets.data.find((secret) => secret.name === name)?.value + } catch (e) { + handleError(e, `Error fetching secret ${name}`) + } +} diff --git a/utils/common/src/index.ts b/utils/common/src/index.ts index b3f666fb6..cae1ba01c 100644 --- a/utils/common/src/index.ts +++ b/utils/common/src/index.ts @@ -1,5 +1,7 @@ export * from './all.records' export * from './async.batch' export * from './delete.records' +export * from './get.secret' export * from './logging.helper' export * from './slugify' +export * from './valid.url' diff --git a/utils/common/src/logging.helper.ts b/utils/common/src/logging.helper.ts index e4bb6d852..657a2f157 100644 --- a/utils/common/src/logging.helper.ts +++ b/utils/common/src/logging.helper.ts @@ -24,3 +24,8 @@ export const logWarn = (packageName: string, msg: string): void => { export const logError = (packageName: string, msg: string): void => { log(packageName, msg, 'error') } + +export function handleError(error: any, message: string) { + console.error(error) + throw new Error(`${message}: ${error.message}`) +} diff --git a/utils/common/src/valid.url.ts b/utils/common/src/valid.url.ts new file mode 100644 index 000000000..0a6583ae9 --- /dev/null +++ b/utils/common/src/valid.url.ts @@ -0,0 +1,11 @@ +export function isValidUrl(url: string | URL) { + if (url instanceof URL) { + return true + } + try { + new URL(url) + return true + } catch (error) { + return false + } +} diff --git a/utils/response-rejection/src/index.ts b/utils/response-rejection/src/index.ts index 8f28bf2dc..ae1a512a5 100644 --- a/utils/response-rejection/src/index.ts +++ b/utils/response-rejection/src/index.ts @@ -1,5 +1,8 @@ -import { Flatfile, FlatfileClient } from '@flatfile/api' -import { deleteRecords, processRecords } from '@flatfile/util-common' +import type { Flatfile } from '@flatfile/api' + +import { FlatfileClient } from '@flatfile/api' +import { addSubmissionStatusField } from './prepare.sheet' +import { updateSheet } from './update.sheet' const api = new FlatfileClient() @@ -7,47 +10,55 @@ export interface RejectionResponse { id: string message?: string deleteSubmitted?: boolean - sheets: SheetRejections[] -} - -export interface SheetRejections { - sheetId: string - rejectedRecords: RecordRejections[] + rejectedRecords: RejectedRecord[] } -export interface RecordRejections { +export interface RejectedRecord { id: string values: { field: string; message: string }[] } +export interface RejectionHandlerResponse { + rejectedRecordsCount: number + jobCompleteDetails: Flatfile.JobCompleteDetails +} + export async function responseRejectionHandler( responseRejection: RejectionResponse -): Promise { - let totalRejectedRecords = 0 +): Promise { + const { + id: sheetId, + deleteSubmitted, + message, + rejectedRecords, + } = responseRejection - for (const sheet of responseRejection.sheets || []) { - const count = await updateSheet(sheet, responseRejection.deleteSubmitted) - totalRejectedRecords += count + if (!deleteSubmitted) { + await addSubmissionStatusField(sheetId) } - const message = responseRejection.message ?? getMessage(totalRejectedRecords) + await updateSheet(sheetId, rejectedRecords, deleteSubmitted ?? false) + let next - if (!responseRejection.deleteSubmitted && totalRejectedRecords > 0) { - next = getNext(totalRejectedRecords, responseRejection.sheets[0].sheetId) + if (!deleteSubmitted && rejectedRecords.length > 0) { + next = getNext(rejectedRecords.length, sheetId) } return { - outcome: { - buttonText: 'Close', - heading: totalRejectedRecords > 0 ? 'Rejected Records' : 'Success!', - acknowledge: true, - ...(next && !responseRejection.deleteSubmitted && { next }), - message, + rejectedRecordsCount: rejectedRecords.length, + jobCompleteDetails: { + outcome: { + buttonText: 'Close', + heading: rejectedRecords.length > 0 ? 'Rejected Records' : 'Success!', + acknowledge: true, + ...(next && !deleteSubmitted && { next }), + message: message ?? getMessage(rejectedRecords.length), + }, }, } } -function getMessage(totalRejectedRecords) { +function getMessage(totalRejectedRecords: number) { return totalRejectedRecords > 0 ? `During the data submission process, ${totalRejectedRecords} records were rejected. Please review and correct these records before resubmitting.` : 'The data has been successfully submitted without any rejections. This task is now complete.' @@ -66,81 +77,3 @@ function getNext( } : undefined } - -async function updateSheet( - sheetRejections: SheetRejections, - deleteSubmitted: boolean -): Promise { - const sheetId = sheetRejections.sheetId - if (!deleteSubmitted) { - await addSubmissionStatusField(sheetId) - } - - await processRecords( - sheetId, - async (records: Flatfile.RecordsWithLinks, _pageNumber?: number) => { - if (!records.length) { - return - } - records.forEach((record) => { - const rejectedRecord = sheetRejections.rejectedRecords.find( - (item) => item.id === record.id - ) - - rejectedRecord?.values.forEach((value) => { - if (record.values[value.field]) { - record.values[value.field].messages = [ - { type: 'error', message: value.message }, - ] - } - }) - - if (!deleteSubmitted) { - record.values['submissionStatus'].value = rejectedRecord - ? 'rejected' - : 'submitted' - } - }) - - try { - await api.records.update(sheetId, records) - } catch (error) { - console.error('Error updating records:', error) - throw new Error('Error updating records') - } - } - ) - - deleteSubmitted && - (await deleteRecords(sheetId, { - filter: 'valid', - })) - - return sheetRejections.rejectedRecords.length -} - -async function addSubmissionStatusField(sheetId: string): Promise { - try { - const { data: sheet } = await api.sheets.get(sheetId) - if ( - !sheet.config.fields.some((field) => field.key === 'submissionStatus') - ) { - await api.sheets.addField(sheet.id, { - key: 'submissionStatus', - label: 'Submission Status', - type: 'enum', - readonly: true, - config: { - allowCustom: false, - options: [ - { label: 'Rejected', value: 'rejected' }, - { label: 'Submitted', value: 'submitted' }, - ], - }, - }) - } - } catch (error) { - console.error('Error adding rejection status field:', error) - throw 'Error adding rejection status field' - } -} diff --git a/utils/response-rejection/src/prepare.sheet.ts b/utils/response-rejection/src/prepare.sheet.ts new file mode 100644 index 000000000..2044832c7 --- /dev/null +++ b/utils/response-rejection/src/prepare.sheet.ts @@ -0,0 +1,27 @@ +import api from '@flatfile/api' + +export async function addSubmissionStatusField(sheetId: string): Promise { + try { + const { data: sheet } = await api.sheets.get(sheetId) + if ( + !sheet.config.fields.some((field) => field.key === 'submissionStatus') + ) { + await api.sheets.addField(sheet.id, { + key: 'submissionStatus', + label: 'Submission Status', + type: 'enum', + readonly: true, + config: { + allowCustom: false, + options: [ + { label: 'Rejected', value: 'rejected' }, + { label: 'Submitted', value: 'submitted' }, + ], + }, + }) + } + } catch (error) { + console.error('Error adding rejection status field:', error) + throw new Error('Error adding rejection status field') + } +} diff --git a/utils/response-rejection/src/update.sheet.ts b/utils/response-rejection/src/update.sheet.ts new file mode 100644 index 000000000..063bbe9ec --- /dev/null +++ b/utils/response-rejection/src/update.sheet.ts @@ -0,0 +1,49 @@ +import type { Flatfile } from '@flatfile/api' +import api from '@flatfile/api' +import { chunkify, getRecordsRaw } from '@flatfile/util-common' +import { RejectedRecord } from '.' + +export async function updateSheet( + sheetId: string, + rejectedRecords: RejectedRecord[], + deleteSubmitted: boolean +) { + const recordIds = rejectedRecords.map((record) => record.id) + const chunkedRecordIds = chunkify(recordIds, 100) // records get API call limits the id filter to 100 ids + for (const ids of chunkedRecordIds) { + const records = await getRecordsRaw(sheetId, { + ids, + }) + if (Array.isArray(records)) { + records.forEach((record: Flatfile.Record_) => { + const rejectedRecord = rejectedRecords.find( + (item) => item.id === record.id + ) + + // Necessary record clean up to avoid Date/string type mismatch bug + Object.values(record.values).forEach((value) => delete value.updatedAt) + + rejectedRecord?.values.forEach((value) => { + if (record.values[value.field]) { + record.values[value.field].messages = [ + { type: 'error', message: value.message }, + ] + } + }) + + if (!deleteSubmitted) { + record.values['submissionStatus'].value = rejectedRecord + ? 'rejected' + : 'submitted' + } + }) + + try { + await api.records.update(sheetId, records) + } catch (error) { + console.error('Error updating records:', error) + throw new Error('Error updating records') + } + } + } +}