From 97eb493fef648da1410a4e89b72d47fedcd789fd Mon Sep 17 00:00:00 2001 From: m417z Date: Fri, 20 Mar 2020 22:59:05 +0200 Subject: [PATCH] Fixed batchExecutor to not be useless --- src/lib/clear.ts | 10 +++++----- src/lib/export.ts | 16 ++++++++-------- src/lib/firestore-helpers.ts | 8 ++++---- src/lib/import.ts | 8 ++++---- tests/firestore-helpers.spec.ts | 6 +++--- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/lib/clear.ts b/src/lib/clear.ts index 7228811..8468f89 100644 --- a/src/lib/clear.ts +++ b/src/lib/clear.ts @@ -23,10 +23,10 @@ const clearData = async (startingRef: admin.firestore.Firestore | }; const clearCollections = async (startingRef: admin.firestore.Firestore | FirebaseFirestore.DocumentReference, logs = false) => { - const collectionPromises: Array> = []; + const collectionPromises: Array<() => Promise> = []; const collectionsSnapshot = await safelyGetCollectionsSnapshot(startingRef, logs); collectionsSnapshot.map((collectionRef: FirebaseFirestore.CollectionReference) => { - collectionPromises.push(clearDocuments(collectionRef, logs)); + collectionPromises.push(() => clearDocuments(collectionRef, logs)); }); return batchExecutor(collectionPromises); }; @@ -34,10 +34,10 @@ const clearCollections = async (startingRef: admin.firestore.Firestore | Firebas const clearDocuments = async (collectionRef: FirebaseFirestore.CollectionReference, logs = false) => { logs && console.log(`Retrieving documents from ${collectionRef.path}`); const allDocuments = await safelyGetDocumentReferences(collectionRef, logs); - const documentPromises: Array> = []; + const documentPromises: Array<() => Promise> = []; allDocuments.forEach((docRef: DocumentReference) => { - documentPromises.push(clearCollections(docRef, logs)); - documentPromises.push(docRef.delete()); + documentPromises.push(() => clearCollections(docRef, logs)); + documentPromises.push(() => docRef.delete()); }); return batchExecutor(documentPromises); }; diff --git a/src/lib/export.ts b/src/lib/export.ts index 6a96e97..8662578 100644 --- a/src/lib/export.ts +++ b/src/lib/export.ts @@ -12,12 +12,12 @@ const exportData = async (startingRef: admin.firestore.Firestore | FirebaseFirestore.DocumentReference | FirebaseFirestore.CollectionReference, logs = false) => { if (isLikeDocument(startingRef)) { - const collectionsPromise = getCollections(startingRef, logs); - let dataPromise: Promise; + const collectionsPromise = () => getCollections(startingRef, logs); + let dataPromise: () => Promise; if (isRootOfDatabase(startingRef)) { - dataPromise = Promise.resolve({}); + dataPromise = () => Promise.resolve({}); } else { - dataPromise = (startingRef).get() + dataPromise = () => (startingRef).get() .then(snapshot => snapshot.data()) .then(data => serializeSpecialTypes(data)); } @@ -31,11 +31,11 @@ const exportData = async (startingRef: admin.firestore.Firestore | const getCollections = async (startingRef: admin.firestore.Firestore | FirebaseFirestore.DocumentReference, logs = false) => { const collectionNames: Array = []; - const collectionPromises: Array> = []; + const collectionPromises: Array<() => Promise> = []; const collectionsSnapshot = await safelyGetCollectionsSnapshot(startingRef, logs); collectionsSnapshot.map((collectionRef: FirebaseFirestore.CollectionReference) => { collectionNames.push(collectionRef.id); - collectionPromises.push(getDocuments(collectionRef, logs)); + collectionPromises.push(() => getDocuments(collectionRef, logs)); }); const results = await batchExecutor(collectionPromises); const zipped: any = {}; @@ -48,11 +48,11 @@ const getCollections = async (startingRef: admin.firestore.Firestore | FirebaseF const getDocuments = async (collectionRef: FirebaseFirestore.CollectionReference, logs = false) => { logs && console.log(`Retrieving documents from ${collectionRef.path}`); const results: any = {}; - const documentPromises: Array> = []; + const documentPromises: Array<() => Promise> = []; const allDocuments = await safelyGetDocumentReferences(collectionRef, logs); allDocuments.forEach((doc) => { - documentPromises.push(new Promise(async (resolve) => { + documentPromises.push(() => new Promise(async (resolve) => { const docSnapshot = await doc.get(); const docDetails: any = {}; if (docSnapshot.exists) { diff --git a/src/lib/firestore-helpers.ts b/src/lib/firestore-helpers.ts index bae4627..8fb1520 100644 --- a/src/lib/firestore-helpers.ts +++ b/src/lib/firestore-helpers.ts @@ -45,11 +45,11 @@ const isRootOfDatabase = (ref: admin.firestore.Firestore | const sleep = (timeInMS: number): Promise => new Promise(resolve => setTimeout(resolve, timeInMS)); -const batchExecutor = async function (promises: Promise[], batchSize: number = 50) { +const batchExecutor = async function (promiseGenerators: (() => Promise)[], batchSize: number = 50) { const res: T[] = []; - while (promises.length > 0) { - const temp = await Promise.all(promises.splice(0, batchSize)); - res.push(...temp); + while (promiseGenerators.length > 0) { + const promises = promiseGenerators.splice(0, batchSize).map(generator => generator()); + res.push(...await Promise.all(promises)); } return res; }; diff --git a/src/lib/import.ts b/src/lib/import.ts index 3aa471b..24113b3 100644 --- a/src/lib/import.ts +++ b/src/lib/import.ts @@ -14,10 +14,10 @@ const importData = (data: any, throw new Error('Root or document reference doesn\'t contain a __collections__ property.'); } const collections = dataToImport['__collections__']; - const collectionPromises: Array> = []; + const collectionPromises: Array<() => Promise> = []; for (const collection in collections) { if (collections.hasOwnProperty(collection)) { - collectionPromises.push(setDocuments(collections[collection], startingRef.collection(collection), mergeWithExisting, logs)); + collectionPromises.push(() => setDocuments(collections[collection], startingRef.collection(collection), mergeWithExisting, logs)); } } if (isRootOfDatabase(startingRef)) { @@ -57,12 +57,12 @@ const setDocuments = (data: ICollection, startingRef: FirebaseFirestore.Collecti const documentData: any = unserializeSpecialTypes(documents); batch.set(startingRef.doc(documentKey), documentData, {merge: mergeWithExisting}); }); - return batch.commit(); + return () => batch.commit(); }); return batchExecutor(chunkPromises) .then(() => { return collections.map((col) => { - return setDocuments(col.collection, col.path, mergeWithExisting, logs); + return () => setDocuments(col.collection, col.path, mergeWithExisting, logs); }); }) .then(subCollectionPromises => batchExecutor(subCollectionPromises)) diff --git a/tests/firestore-helpers.spec.ts b/tests/firestore-helpers.spec.ts index ca87551..92e5e31 100644 --- a/tests/firestore-helpers.spec.ts +++ b/tests/firestore-helpers.spec.ts @@ -98,19 +98,19 @@ describe('Firestore Helpers', () => { }; it('should resolve lists smaller then the batchsize', async () => { - const input = [toPromise(1), toPromise(2)]; + const input = [() => toPromise(1), () => toPromise(2)]; let actual = await batchExecutor(input, 3); expect(actual).to.eql([1, 2]); }); it('should resolve lists equal to the batchsize', async () => { - const input = [toPromise(1), toPromise(2)]; + const input = [() => toPromise(1), () => toPromise(2)]; let actual = await batchExecutor(input, 1); expect(actual).to.eql([1, 2]); }); it('should resolve lists larger then the batchsize', async () => { - const input = [toPromise(1), toPromise(2)]; + const input = [() => toPromise(1), () => toPromise(2)]; let actual = await batchExecutor(input, 1); expect(actual).to.eql([1, 2]); });