Merge pull request #172 from brafdlog/baruchiro/issue170

Bug: ENOTDIR, not a directory

src/backend/consts.ts

@@ -0,0 +1,5 @@
+import { app, remote } from 'electron';
+
+const App = app || remote.app;
+
+export const userDataPath = App.getPath('userData');

src/backend/eventEmitters/EventEmitter.ts

@@ -1,10 +1,11 @@
 // eslint-disable-next-line max-classes-per-file
-import Emittery from 'emittery';
 import { EnrichedTransaction, OutputVendorName } from '@/backend/commonTypes';
+import Emittery from 'emittery';
 import { CompanyTypes } from 'israeli-bank-scrapers-core';
 
 export enum EventNames {
   IMPORT_PROCESS_START = 'IMPORT_PROCESS_START',
+  DOWNLOAD_CHROME = 'DOWNLOAD_CHROME',
   IMPORTER_START = 'IMPORTER_START',
   IMPORTER_PROGRESS = 'IMPORTER_PROGRESS',
   IMPORTER_ERROR = 'IMPORTER_ERROR',
@@ -88,8 +89,15 @@ export class ExporterEvent extends BudgetTrackingEvent {
   }
 }
 
+export class DownalodChromeEvent extends BudgetTrackingEvent {
+  constructor(percent: number) {
+    super({ message: `Download Chrome: ${percent}%` });
+  }
+}
+
 export type EventDataMap = {
   [EventNames.IMPORT_PROCESS_START]: BudgetTrackingEvent
+  [EventNames.DOWNLOAD_CHROME]: DownalodChromeEvent
   [EventNames.IMPORTER_START]: ImporterEvent
   [EventNames.IMPORTER_PROGRESS]: ImporterEvent
   [EventNames.IMPORTER_ERROR]: ImporterEvent

src/backend/import/bankScraper.ts

@@ -1,6 +1,5 @@
 import { CompanyTypes, createScraper, SCRAPERS } from 'israeli-bank-scrapers-core';
 import { AccountToScrapeConfig } from '../configManager/configManager';
-import getChrome from './downloadChromium';
 
 export { ScaperScrapingResult } from 'israeli-bank-scrapers-core/lib/scrapers/base-scraper';
 export { Transaction } from 'israeli-bank-scrapers-core/lib/transactions';
@@ -24,13 +23,11 @@ type EmitProgressEventFunction = (eventCompanyId: string, message: string) => Pr
 
 export async function scrape({
   companyId, credentials, startDate, showBrowser = false
-}: ScrapeParameters, emitProgressEvent: EmitProgressEventFunction) {
+}: ScrapeParameters, emitProgressEvent: EmitProgressEventFunction, chromePath: string) {
   if (!credentials || (!credentials.username && !credentials.num && !credentials.id) || !credentials.password) {
     throw new Error(`Missing credentials for scraper. CompanyId: ${companyId}`);
   }
 
-  const chromePath = await getChrome(undefined);
-
   const options = {
     companyId, // mandatory; one of 'hapoalim', 'discount', 'otsarHahayal', 'leumiCard', 'isracard', 'amex'
     startDate, // the date to fetch transactions from (can't be before the minimum allowed time difference for the scraper)

src/backend/import/downloadChromium.ts

@@ -1,23 +1,31 @@
-import download from 'download-chromium';
+import download, { OnProgress } from 'download-chromium';
 import { getPuppeteerConfig } from 'israeli-bank-scrapers-core';
 
-type OnProgress = ({ percent, transferred, total }) => void
-type DownloadOptions = {
-  platform: string,
-  revision: string,
-  log: boolean,
-  onProgress: OnProgress
-  installPath: string
-}
-type Download = (options: Partial<DownloadOptions>) => string
+const getIntegerPercent = (callback: OnProgress): OnProgress => {
+  let prevPercent = -1;
+
+  return ({ percent, ...rest }) => {
+    const p = Math.floor(percent * 10);
+    if (p > prevPercent) {
+      prevPercent = p;
+      callback({ percent: p, ...rest });
+    }
+  };
+};
 
 const revision = getPuppeteerConfig().chromiumRevision;
 
+let downloadProm: ReturnType<typeof download>;
+
 export default async function downloadChromium(installPath?: string, onProgress?: OnProgress) {
-  const func: Download = download;
-  return func({
+  if (downloadProm) return downloadProm;
+
+  downloadProm = download({
     revision,
     installPath,
-    onProgress,
+    onProgress: onProgress && getIntegerPercent(onProgress),
+    log: true
   });
+
+  return downloadProm;
 }

src/backend/import/importTransactions.ts

@@ -1,14 +1,16 @@
 import { EnrichedTransaction } from '@/backend/commonTypes';
 import * as configManager from '@/backend/configManager/configManager';
+import { userDataPath } from '@/backend/consts';
 import * as bankScraper from '@/backend/import/bankScraper';
 import { ScaperScrapingResult, Transaction } from '@/backend/import/bankScraper';
 import * as categoryCalculation from '@/backend/import/categoryCalculationScript';
 import _ from 'lodash';
 import moment from 'moment';
 import {
-  AccountStatus, BudgetTrackingEventEmitter, EventNames, EventPublisher, ImporterEvent
+  AccountStatus, BudgetTrackingEventEmitter, DownalodChromeEvent, EventNames, EventPublisher, ImporterEvent
 } from '../eventEmitters/EventEmitter';
 import { calculateTransactionHash } from '../transactions/transactions';
+import getChrome from './downloadChromium';
 
 type AccountToScrapeConfig = configManager.AccountToScrapeConfig;
 type Config = configManager.Config;
@@ -18,12 +20,15 @@ const TRANSACTION_STATUS_COMPLETED = 'completed';
 
 export async function scrapeFinancialAccountsAndFetchTransactions(scrapingConfig: ScrapingConfig, startDate: Date, eventPublisher: EventPublisher) {
   const companyIdToTransactions: Record<string, EnrichedTransaction[]> = {};
+
+  const dowloadedChrome = await getChrome(userDataPath, ({ percent }) => emitChromeDownload(eventPublisher, percent));
+
   const accountsToScrape = scrapingConfig.accountsToScrape.filter((accountToScrape) => accountToScrape.active !== false);
   const scrapingPromises = accountsToScrape.map(async (accountToScrape) => {
     const companyId = accountToScrape.key;
     try {
       await eventPublisher.emit(EventNames.IMPORTER_START, buildImporterEvent(accountToScrape, { message: 'Importer start' }));
-      const scrapeResult = await fetchTransactions(accountToScrape, startDate, scrapingConfig, eventPublisher);
+      const scrapeResult = await fetchTransactions(accountToScrape, startDate, scrapingConfig, eventPublisher, dowloadedChrome);
       const transactions = await postProcessTransactions(accountToScrape, scrapeResult);
       companyIdToTransactions[companyId] = transactions;
       await eventPublisher.emit(EventNames.IMPORTER_END, buildImporterEvent(accountToScrape, { message: 'Importer end', status: AccountStatus.DONE }));
@@ -47,6 +52,10 @@ function buildImporterEvent(accountConfig: AccountToScrapeConfig, additionalPara
   });
 }
 
+function emitChromeDownload(eventPublisher: EventPublisher, percent: number) {
+  eventPublisher.emit(EventNames.DOWNLOAD_CHROME, new DownalodChromeEvent(percent));
+}
+
 export async function getFinancialAccountNumbers() {
   const eventEmitter = new BudgetTrackingEventEmitter();
   const config = await configManager.getConfig();
@@ -69,7 +78,8 @@ export async function getFinancialAccountNumbers() {
 async function fetchTransactions(
   accountToScrapeConfig: AccountToScrapeConfig,
   startDate: Date, scrapingConfig: Config['scraping'],
-  eventPublisher: EventPublisher
+  eventPublisher: EventPublisher,
+  chromePath: string
 ) {
   const emitImporterProgressEvent = async (eventCompanyId: string, message: string) => {
     await eventPublisher.emit(EventNames.IMPORTER_PROGRESS, buildImporterEvent(accountToScrapeConfig, { message }));
@@ -80,7 +90,7 @@ async function fetchTransactions(
     credentials: accountToScrapeConfig.loginFields,
     startDate,
     showBrowser: scrapingConfig.showBrowser,
-  }, emitImporterProgressEvent);
+  }, emitImporterProgressEvent, chromePath);
   if (!scrapeResult.success) {
     throw new Error(scrapeResult.errorMessage || scrapeResult.errorType);
   }

src/types/download-chromium.d.ts

@@ -0,0 +1,17 @@
+declare module 'download-chromium' {
+    type Progress = {
+        percent: number,
+        transferred: number,
+        total: number
+    }
+    export type OnProgress = (progress: Progress) => void;
+    type DownloadOptions = {
+        platform: string,
+        revision: string,
+        log: boolean,
+        onProgress: OnProgress
+        installPath: string
+    }
+    function download(options: Partial<DownloadOptions>): Promise<string>
+    export default download
+}

test/unit/mock/electron.ts

@@ -1,4 +1,10 @@
+import { tmpdir } from 'os';
+
 export const ipcRenderer = {
   on: jest.fn(),
   send: jest.fn(),
 };
+
+export const app = {
+  getPath: jest.fn().mockReturnValue(tmpdir())
+};