diff --git a/packages/records/lib/helpers/format.ts b/packages/records/lib/helpers/format.ts index 1f862f3fe1c..461d31fb795 100644 --- a/packages/records/lib/helpers/format.ts +++ b/packages/records/lib/helpers/format.ts @@ -46,7 +46,8 @@ export const formatRecords = ({ const formattedRecord: FormattedRecord = { id: stableId(datum), json: datum, - external_id: String(datum['id']), + // postgresql does not support null bytes in strings + external_id: String(datum['id']).replaceAll('\x00', ''), data_hash, model, connection_id: connectionId, diff --git a/packages/records/lib/models/records.integration.test.ts b/packages/records/lib/models/records.integration.test.ts index 5ef3bbdc4e5..00cf6d76109 100644 --- a/packages/records/lib/models/records.integration.test.ts +++ b/packages/records/lib/models/records.integration.test.ts @@ -616,6 +616,23 @@ describe('Records service', () => { expect(records).toContainEqual(expect.objectContaining({ id: '5' })); }); + it('should filter out 0x00 in ids', async () => { + const connectionId = rnd.number(); + const environmentId = rnd.number(); + const model = rnd.string(); + const syncId = uuid.v4(); + const toInsert = [{ id: '1', name: 'John Doe' }]; + await upsertRecords({ records: toInsert, connectionId, environmentId, model, syncId }); + + const response = await Records.getRecords({ connectionId, model, externalIds: ['\x001'] }); + + expect(response.isOk()).toBe(true); + const { records } = response.unwrap(); + + expect(records.length).toBe(1); + expect(records).toContainEqual(expect.objectContaining({ id: '1', name: 'John Doe' })); + }); + it('Should be able to retrieve 20K records in under 5s with a cursor', async () => { const numOfRecords = 20000; const limit = 1000; diff --git a/packages/records/lib/models/records.ts b/packages/records/lib/models/records.ts index 55ffd440bd8..6a4032356dc 100644 --- a/packages/records/lib/models/records.ts +++ b/packages/records/lib/models/records.ts @@ -101,7 +101,9 @@ export async function getRecords({ } if (externalIds) { - query = query.whereIn('external_id', externalIds); + // postgresql does not support null bytes in strings + const cleanIds = externalIds.map((id) => id.replaceAll('\x00', '')); + query = query.whereIn('external_id', cleanIds); } if (limit) {