Skip to content

Commit

Permalink
Ignore _nango_metadata field in batchSave, batchUpdate, batchDelete
Browse files Browse the repository at this point in the history
  • Loading branch information
@nalanj
nalanj committed Feb 11, 2025
1 parent 12cb367 commit 59c97fa
Show file tree
Hide file tree
Showing 5 changed files with 98 additions and 75 deletions.
54 changes: 27 additions & 27 deletions packages/cli/lib/services/__snapshots__/model.service.unit.test.ts.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ interface JwtCredentials {
type: AuthModes['Jwt'];
privateKeyId?: string;
issuerId?: string;
privateKey:{
privateKey: {
id: string;
secret: string;
} | string;
Expand Down Expand Up @@ -275,15 +275,15 @@ export interface NangoProps {
track_deletes?: boolean;
attributes?: object | undefined;
logMessages?:
| {
counts: {
updated: number;
added: number;
deleted: number;
};
messages: unknown[];
}
| undefined;
| {
counts: {
updated: number;
added: number;
deleted: number;
};
messages: unknown[];
}
| undefined;
rawSaveOutput?: Map<string, unknown[]> | undefined;
rawDeleteOutput?: Map<string, unknown[]> | undefined;
stubbedMetadata?: Metadata | undefined;
Expand Down Expand Up @@ -378,12 +378,12 @@ export declare class NangoAction {
message: any,
options?:
| {
level?: LogLevel;
}
level?: LogLevel;
}
| {
[key: string]: any;
level?: never;
}
[key: string]: any;
level?: never;
}
): Promise<void>;
log(
message: string,
Expand All @@ -407,13 +407,13 @@ export declare class NangoSync extends NangoAction {
track_deletes: boolean;
logMessages?:
| {
counts: {
updated: number;
added: number;
deleted: number;
};
messages: unknown[];
}
counts: {
updated: number;
added: number;
deleted: number;
};
messages: unknown[];
}
| undefined;
rawSaveOutput?: Map<string, unknown[]>;
rawDeleteOutput?: Map<string, unknown[]>;
Expand All @@ -423,10 +423,10 @@ export declare class NangoSync extends NangoAction {
/**
* @deprecated please use batchSave
*/
batchSend<T = any>(results: T[], model: string): Promise<boolean | null>;
batchSave<T = any>(results: T[], model: string): Promise<boolean | null>;
batchDelete<T = any>(results: T[], model: string): Promise<boolean | null>;
batchUpdate<T = any>(results: T[], model: string): Promise<boolean | null>;
batchSend<T extends object>(results: T[], model: string): Promise<boolean | null>;
batchSave<T extends object>(results: T[], model: string): Promise<boolean | null>;
batchDelete<T extends object>(results: T[], model: string): Promise<boolean | null>;
batchUpdate<T extends object>(results: T[], model: string): Promise<boolean | null>;
getMetadata<T = Metadata>(): Promise<T>;
setMergingStrategy(merging: { strategy: 'ignore_if_modified_after' | 'override' }, model: string): Promise<void>;
}
Expand All @@ -437,7 +437,7 @@ export declare class NangoSync extends NangoAction {
* It has been split from the actual code to avoid making the code too dirty and to easily enable/disable tracing if there is an issue with it
*/
export declare function instrumentSDK(rawNango: NangoAction | NangoSync): NangoAction | NangoSync;
export {};
export { };
Expand Down
20 changes: 13 additions & 7 deletions packages/cli/lib/services/sdk.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,14 +130,16 @@ export class NangoSyncCLI extends NangoSyncBase {
log = NangoActionCLI['prototype']['log'];
triggerSync = NangoActionCLI['prototype']['triggerSync'];

public batchSave<T = any>(results: T[], model: string) {
public batchSave<T extends object>(results: T[], model: string) {
if (!results || results.length === 0) {
console.info('batchSave received an empty array. No records to save.');
return true;
}

const resultsWithoutMetadata = this.removeMetadata(results);

// Validate records
const hasErrors = this.validateRecords(model, results);
const hasErrors = this.validateRecords(model, resultsWithoutMetadata);

if (hasErrors.length > 0) {
this.log('Invalid record payload. Use `--validation` option to see the details', { level: 'warn' });
Expand All @@ -147,7 +149,7 @@ export class NangoSyncCLI extends NangoSyncBase {
}

this.logMessages?.messages.push(`A batch save call would save the following data to the ${model} model:`);
for (const msg of results) {
for (const msg of resultsWithoutMetadata) {
this.logMessages?.messages.push(msg);
}
if (this.logMessages && this.logMessages.counts) {
Expand All @@ -162,14 +164,16 @@ export class NangoSyncCLI extends NangoSyncBase {
return true;
}

public batchDelete<T = any>(results: T[], model: string) {
public batchDelete<T extends object>(results: T[], model: string) {
if (!results || results.length === 0) {
console.info('batchDelete received an empty array. No records to delete.');
return true;
}

const resultsWithoutMetadata = this.removeMetadata(results);

this.logMessages?.messages.push(`A batch delete call would delete the following data:`);
for (const msg of results) {
for (const msg of resultsWithoutMetadata) {
this.logMessages?.messages.push(msg);
}
if (this.logMessages && this.logMessages.counts) {
Expand All @@ -184,14 +188,16 @@ export class NangoSyncCLI extends NangoSyncBase {
return true;
}

public batchUpdate<T = any>(results: T[], model: string) {
public batchUpdate<T extends object>(results: T[], model: string) {
if (!results || results.length === 0) {
console.info('batchUpdate received an empty array. No records to update.');
return true;
}

const resultsWithoutMetadata = this.removeMetadata(results);

this.logMessages?.messages.push(`A batch update call would update the following data to the ${model} model:`);
for (const msg of results) {
for (const msg of resultsWithoutMetadata) {
this.logMessages?.messages.push(msg);
}
if (this.logMessages && this.logMessages.counts) {
Expand Down
19 changes: 15 additions & 4 deletions packages/runner-sdk/lib/sync.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,15 @@ export abstract class NangoSyncBase extends NangoActionBase {
/**
* @deprecated please use batchSave
*/
public async batchSend<T = any>(results: T[], model: string): Promise<boolean | null> {
public async batchSend<T extends object>(results: T[], model: string): Promise<boolean | null> {
return this.batchSave(results, model);
}

public abstract batchSave<T = any>(results: T[], model: string): MaybePromise<boolean>;
public abstract batchSave<T extends object>(results: T[], model: string): MaybePromise<boolean>;

public abstract batchDelete<T = any>(results: T[], model: string): MaybePromise<boolean>;
public abstract batchDelete<T extends object>(results: T[], model: string): MaybePromise<boolean>;

public abstract batchUpdate<T = any>(results: T[], model: string): MaybePromise<boolean>;
public abstract batchUpdate<T extends object>(results: T[], model: string): MaybePromise<boolean>;

protected validateRecords(model: string, records: unknown[]): { data: any; validation: ValidateDataError[] }[] {
// Validate records
Expand All @@ -55,4 +55,15 @@ export abstract class NangoSyncBase extends NangoActionBase {

return hasErrors;
}

protected removeMetadata<T extends object>(results: T[]) {
return results.map((result) => {
if ('_nango_metadata' in result) {
const { _nango_metadata, ...rest } = result;
return rest;
}

return result;
});
}
}
54 changes: 27 additions & 27 deletions packages/runner-sdk/models.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ interface JwtCredentials {
type: AuthModes['Jwt'];
privateKeyId?: string;
issuerId?: string;
privateKey:{
privateKey: {
id: string;
secret: string;
} | string;
Expand Down Expand Up @@ -232,15 +232,15 @@ export interface NangoProps {
track_deletes?: boolean;
attributes?: object | undefined;
logMessages?:
| {
counts: {
updated: number;
added: number;
deleted: number;
};
messages: unknown[];
}
| undefined;
| {
counts: {
updated: number;
added: number;
deleted: number;
};
messages: unknown[];
}
| undefined;
rawSaveOutput?: Map<string, unknown[]> | undefined;
rawDeleteOutput?: Map<string, unknown[]> | undefined;
stubbedMetadata?: Metadata | undefined;
Expand Down Expand Up @@ -335,12 +335,12 @@ export declare class NangoAction {
message: any,
options?:
| {
level?: LogLevel;
}
level?: LogLevel;
}
| {
[key: string]: any;
level?: never;
}
[key: string]: any;
level?: never;
}
): Promise<void>;
log(
message: string,
Expand All @@ -364,13 +364,13 @@ export declare class NangoSync extends NangoAction {
track_deletes: boolean;
logMessages?:
| {
counts: {
updated: number;
added: number;
deleted: number;
};
messages: unknown[];
}
counts: {
updated: number;
added: number;
deleted: number;
};
messages: unknown[];
}
| undefined;
rawSaveOutput?: Map<string, unknown[]>;
rawDeleteOutput?: Map<string, unknown[]>;
Expand All @@ -380,10 +380,10 @@ export declare class NangoSync extends NangoAction {
/**
* @deprecated please use batchSave
*/
batchSend<T = any>(results: T[], model: string): Promise<boolean | null>;
batchSave<T = any>(results: T[], model: string): Promise<boolean | null>;
batchDelete<T = any>(results: T[], model: string): Promise<boolean | null>;
batchUpdate<T = any>(results: T[], model: string): Promise<boolean | null>;
batchSend<T extends object>(results: T[], model: string): Promise<boolean | null>;
batchSave<T extends object>(results: T[], model: string): Promise<boolean | null>;
batchDelete<T extends object>(results: T[], model: string): Promise<boolean | null>;
batchUpdate<T extends object>(results: T[], model: string): Promise<boolean | null>;
getMetadata<T = Metadata>(): Promise<T>;
setMergingStrategy(merging: { strategy: 'ignore_if_modified_after' | 'override' }, model: string): Promise<void>;
}
Expand All @@ -394,4 +394,4 @@ export declare class NangoSync extends NangoAction {
* It has been split from the actual code to avoid making the code too dirty and to easily enable/disable tracing if there is an issue with it
*/
export declare function instrumentSDK(rawNango: NangoAction | NangoSync): NangoAction | NangoSync;
export {};
export { };
26 changes: 16 additions & 10 deletions packages/runner/lib/sdk/sdk.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,14 +270,16 @@ export class NangoSyncRunner extends NangoSyncBase {
this.mergingByModel.set(model, merging);
}

public async batchSave<T = any>(results: T[], model: string) {
public async batchSave<T extends object>(results: T[], model: string) {
this.throwIfAborted();
if (!results || results.length === 0) {
return true;
}

const resultsWithoutMetadata = this.removeMetadata(results);

// Validate records
const hasErrors = this.validateRecords(model, results);
const hasErrors = this.validateRecords(model, resultsWithoutMetadata);

if (hasErrors.length > 0) {
metrics.increment(metrics.Types.RUNNER_INVALID_SYNCS_RECORDS, hasErrors.length);
Expand All @@ -297,8 +299,8 @@ export class NangoSyncRunner extends NangoSyncBase {
);
}

for (let i = 0; i < results.length; i += this.batchSize) {
const batch = results.slice(i, i + this.batchSize);
for (let i = 0; i < resultsWithoutMetadata.length; i += this.batchSize) {
const batch = resultsWithoutMetadata.slice(i, i + this.batchSize);
const res = await this.persistClient.saveRecords({
model,
records: batch,
Expand All @@ -319,14 +321,16 @@ export class NangoSyncRunner extends NangoSyncBase {
return true;
}

public async batchDelete<T = any>(results: T[], model: string) {
public async batchDelete<T extends object>(results: T[], model: string) {
this.throwIfAborted();
if (!results || results.length === 0) {
return true;
}

for (let i = 0; i < results.length; i += this.batchSize) {
const batch = results.slice(i, i + this.batchSize);
const resultsWithoutMetadata = this.removeMetadata(results);

for (let i = 0; i < resultsWithoutMetadata.length; i += this.batchSize) {
const batch = resultsWithoutMetadata.slice(i, i + this.batchSize);
const res = await this.persistClient.deleteRecords({
model,
records: batch,
Expand All @@ -348,14 +352,16 @@ export class NangoSyncRunner extends NangoSyncBase {
return true;
}

public async batchUpdate<T = any>(results: T[], model: string) {
public async batchUpdate<T extends object>(results: T[], model: string) {
this.throwIfAborted();
if (!results || results.length === 0) {
return true;
}

for (let i = 0; i < results.length; i += this.batchSize) {
const batch = results.slice(i, i + this.batchSize);
const resultsWithoutMetadata = this.removeMetadata(results);

for (let i = 0; i < resultsWithoutMetadata.length; i += this.batchSize) {
const batch = resultsWithoutMetadata.slice(i, i + this.batchSize);
const res = await this.persistClient.updateRecords({
model,
records: batch,
Expand Down

0 comments on commit 59c97fa

Please sign in to comment.