fix(persist): Filter external ids to remove 0x00

packages/records/lib/helpers/format.ts

@@ -46,7 +46,8 @@ export const formatRecords = ({
         const formattedRecord: FormattedRecord = {
             id: stableId(datum),
             json: datum,
-            external_id: String(datum['id']),
+            // postgresql does not support null bytes in strings
+            external_id: String(datum['id']).replaceAll('\x00', ''),
             data_hash,
             model,
             connection_id: connectionId,

packages/records/lib/models/records.integration.test.ts

@@ -616,6 +616,23 @@ describe('Records service', () => {
             expect(records).toContainEqual(expect.objectContaining({ id: '5' }));
         });
 
+        it('should filter out 0x00 in ids', async () => {
+            const connectionId = rnd.number();
+            const environmentId = rnd.number();
+            const model = rnd.string();
+            const syncId = uuid.v4();
+            const toInsert = [{ id: '1', name: 'John Doe' }];
+            await upsertRecords({ records: toInsert, connectionId, environmentId, model, syncId });
+
+            const response = await Records.getRecords({ connectionId, model, externalIds: ['\x001'] });
+
+            expect(response.isOk()).toBe(true);
+            const { records } = response.unwrap();
+
+            expect(records.length).toBe(1);
+            expect(records).toContainEqual(expect.objectContaining({ id: '1', name: 'John Doe' }));
+        });
+
         it('Should be able to retrieve 20K records in under 5s with a cursor', async () => {
             const numOfRecords = 20000;
             const limit = 1000;

packages/records/lib/models/records.ts

@@ -101,7 +101,9 @@ export async function getRecords({
         }
 
         if (externalIds) {
-            query = query.whereIn('external_id', externalIds);
+            // postgresql does not support null bytes in strings
+            const cleanIds = externalIds.map((id) => id.replaceAll('\x00', ''));
+            query = query.whereIn('external_id', cleanIds);
         }
 
         if (limit) {