feat: Introducing 2FA scrapers infrastructure + OneZero experimental scraper

README.md

@@ -29,6 +29,7 @@ Currently only the following banks are supported:
 - Massad
 - Yahav (Thanks to [@gczobel](https://github.com/gczobel))
 - Beyhad Bishvilha - [ביחד בשבילך](https://www.hist.org.il/) (thanks [@esakal](https://github.com/esakal))
+- OneZero (Experimental) (thanks [@orzarchi](https://github.com/orzarchi))
 
 # Prerequisites
 To use this you will need to have [Node.js](https://nodejs.org) >= 10.x installed.
@@ -127,6 +128,47 @@ The return value is a list of scraper metadata:
 }
 ```
 
+## Two-Factor Authentication Scrapers
+Some companies require two-factor authentication, and as such the scraper cannot be fully automated. When using the relevant scrapers, you have two options:
+1. Provide an async callback that knows how to retrieve real time secrets like OTP codes.
+2. When supported by the scraper - provide a "long term token". These are usually available if the financial provider only requires Two-Factor authentication periodically, and not on every login. You can retrieve your long term token from the relevant credit/banking app using reverse engineering and a MITM proxy, or use helper functions that are provided by some Two-Factor Auth scrapers (e.g. OneZero).
+
+
+```node
+import { CompanyTypes, createScraper } from 'israeli-bank-scrapers';
+import { prompt } from 'enquirer';
+
+// Option 1 - Provide a callback
+
+const result = await scraper.login({
+ email: relevantAccount.credentials.email,
+ password: relevantAccount.credentials.password,
+ phoneNumber,
+ otpCodeRetriever: async () => {
+  let otpCode;
+  while (!otpCode) {
+   otpCode = await questions('OTP Code?');
+  }
+
+  return otpCode[0];
+ }
+});
+
+// Option 2 - Retrieve a long term otp token (OneZero)
+await scraper.triggerTwoFactorAuth(phoneNumber);
+
+// OTP is sent, retrieve it somehow
+const otpCode='...';
+
+const result = scraper.getLongTermTwoFactorToken(otpCode);
+/*
+result = {
+  success: true;
+  longTermTwoFactorAuthToken: 'eyJraWQiOiJiNzU3OGM5Yy0wM2YyLTRkMzktYjBm...';
+}
+ */
+```
+
 # Getting deployed version of latest changes in master
 This library is deployed automatically to NPM with any change merged into the master branch. 
 

src/definitions.ts

@@ -18,7 +18,8 @@ export enum CompanyTypes {
   leumi = 'leumi',
   massad = 'massad',
   yahav = 'yahav',
-  beyahadBishvilha = 'beyahadBishvilha'
+  beyahadBishvilha = 'beyahadBishvilha',
+  oneZero = 'oneZero'
 }
 
 export const SCRAPERS = {
@@ -86,4 +87,8 @@ export const SCRAPERS = {
     name: 'Beyahad Bishvilha',
     loginFields: ['id', PASSWORD_FIELD],
   },
+  [CompanyTypes.oneZero]: {
+    name: 'One Zero',
+    loginFields: ['email', PASSWORD_FIELD, 'otpCodeRetriever', 'phoneNumber', 'otpLongTermToken'],
+  },
 };

src/helpers/fetch.ts

@@ -30,20 +30,26 @@ export async function fetchGet<TResult>(url: string,
 }
 
 export async function fetchPost(url: string, data: Record<string, any>,
-  extraHeaders: Record<string, any>) {
-  let headers = getJsonHeaders();
-  if (extraHeaders) {
-    headers = Object.assign(headers, extraHeaders);
-  }
+  extraHeaders: Record<string, any> = {}) {
   const request = {
     method: 'POST',
-    headers,
+    headers: { ...getJsonHeaders(), ...extraHeaders },
     body: JSON.stringify(data),
   };
   const result = await nodeFetch(url, request);
   return result.json();
 }
 
+export async function fetchGraphql<TResult>(url: string, query: string,
+  variables: Record<string, unknown> = {},
+  extraHeaders: Record<string, any> = {}): Promise<TResult> {
+  const result = await fetchPost(url, { operationName: null, query, variables }, extraHeaders);
+  if (result.errors?.length) {
+    throw new Error(result.errors[0].message);
+  }
+  return result.data as Promise<TResult>;
+}
+
 export function fetchGetWithinPage<TResult>(page: Page, url: string): Promise<TResult | null> {
   return page.evaluate((url) => {
     return new Promise<TResult | null>((resolve, reject) => {

src/index.ts

@@ -2,7 +2,19 @@ import puppeteerConfig from './puppeteer-config.json';
 
 export { default as createScraper } from './scrapers/factory';
 export { SCRAPERS, CompanyTypes } from './definitions';
-export { ScraperOptions } from './scrapers/base-scraper';
+
+// Note: the typo ScaperScrapingResult & ScraperLoginResult (sic) are exported here for backward compatibility
+export {
+  ScraperOptions,
+  ScraperScrapingResult as ScaperScrapingResult,
+  ScraperScrapingResult,
+  ScraperLoginResult as ScaperLoginResult,
+  ScraperLoginResult,
+  Scraper,
+  ScraperCredentials,
+} from './scrapers/interface';
+
+export { default as OneZeroScraper } from './scrapers/one-zero';
 
 export function getPuppeteerConfig() {
   return { ...puppeteerConfig };

src/scrapers/amex.test.ts

@@ -28,7 +28,7 @@ describe('AMEX legacy scraper', () => {
 
     const scraper = new AMEXScraper(options);
 
-    const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d' });
+    const result = await scraper.scrape({ id: 'e10s12', card6Digits: '123456', password: '3f3ss3d' });
 
     expect(result).toBeDefined();
     expect(result.success).toBeFalsy();

src/scrapers/amex.ts

@@ -1,5 +1,5 @@
 import IsracardAmexBaseScraper from './base-isracard-amex';
-import { ScraperOptions } from './base-scraper';
+import { ScraperOptions } from './interface';
 
 const BASE_URL = 'https://he.americanexpress.co.il';
 const COMPANY_CODE = '77';

src/scrapers/base-beinleumi-group.ts

@@ -13,7 +13,6 @@ import { SHEKEL_CURRENCY } from '../constants';
 import {
   TransactionsAccount, Transaction, TransactionStatuses, TransactionTypes,
 } from '../transactions';
-import { ScraperCredentials } from './base-scraper';
 
 const DATE_FORMAT = 'DD/MM/YYYY';
 const NO_TRANSACTION_IN_DATE_RANGE_TEXT = 'לא נמצאו נתונים בנושא המבוקש';
@@ -55,7 +54,7 @@ export function getPossibleLoginResults(): PossibleLoginResults {
   return urls;
 }
 
-export function createLoginFields(credentials: ScraperCredentials) {
+export function createLoginFields(credentials: ScraperSpecificCredentials) {
   return [
     { selector: '#username', value: credentials.username },
     { selector: '#password', value: credentials.password },
@@ -290,14 +289,16 @@ async function fetchAccounts(page: Page, startDate: Moment) {
   return accounts;
 }
 
-class BeinleumiGroupBaseScraper extends BaseScraperWithBrowser {
+type ScraperSpecificCredentials = {username: string, password: string};
+
+class BeinleumiGroupBaseScraper extends BaseScraperWithBrowser<ScraperSpecificCredentials> {
   BASE_URL = '';
 
   LOGIN_URL = '';
 
   TRANSACTIONS_URL = '';
 
-  getLoginOptions(credentials: ScraperCredentials) {
+  getLoginOptions(credentials: ScraperSpecificCredentials) {
     return {
       loginUrl: `${this.LOGIN_URL}`,
       fields: createLoginFields(credentials),

src/scrapers/base-isracard-amex.ts

@@ -17,12 +17,13 @@ import {
   TransactionStatuses, TransactionTypes,
 } from '../transactions';
 import {
-  ScraperErrorTypes,
-  ScraperOptions, ScaperScrapingResult, ScaperProgressTypes,
-  ScraperCredentials,
+  ScraperProgressTypes,
+
 } from './base-scraper';
 import { getDebug } from '../helpers/debug';
 import { runSerial } from '../helpers/waiting';
+import { ScraperErrorTypes } from './errors';
+import { ScraperScrapingResult, ScraperOptions } from './interface';
 
 const COUNTRY_CODE = '212';
 const ID_TYPE = '1';
@@ -320,8 +321,8 @@ async function fetchAllTransactions(page: Page, options: ExtendedScraperOptions,
   };
 }
 
-
-class IsracardAmexBaseScraper extends BaseScraperWithBrowser {
+type ScraperSpecificCredentials = {id: string, password: string, card6Digits: string};
+class IsracardAmexBaseScraper extends BaseScraperWithBrowser<ScraperSpecificCredentials> {
   private baseUrl: string;
 
   private companyCode: string;
@@ -336,7 +337,7 @@ class IsracardAmexBaseScraper extends BaseScraperWithBrowser {
     this.servicesUrl = `${baseUrl}/services/ProxyRequestHandler.ashx`;
   }
 
-  async login(credentials: ScraperCredentials): Promise<ScaperScrapingResult> {
+  async login(credentials: ScraperSpecificCredentials): Promise<ScraperScrapingResult> {
     await this.page.setRequestInterception(true);
     this.page.on('request', (request) => {
       if (request.url().includes('detector-dom.min.js')) {
@@ -350,7 +351,7 @@ class IsracardAmexBaseScraper extends BaseScraperWithBrowser {
     debug('navigate to login page');
     await this.navigateTo(`${this.baseUrl}/personalarea/Login`);
 
-    this.emitProgress(ScaperProgressTypes.LoggingIn);
+    this.emitProgress(ScraperProgressTypes.LoggingIn);
 
     const validateUrl = `${this.servicesUrl}?reqName=ValidateIdData`;
     const validateRequest = {
@@ -384,34 +385,34 @@ class IsracardAmexBaseScraper extends BaseScraperWithBrowser {
       debug(`user login with status '${loginResult?.status}'`);
 
       if (loginResult && loginResult.status === '1') {
-        this.emitProgress(ScaperProgressTypes.LoginSuccess);
+        this.emitProgress(ScraperProgressTypes.LoginSuccess);
         return { success: true };
       }
 
       if (loginResult && loginResult.status === '3') {
-        this.emitProgress(ScaperProgressTypes.ChangePassword);
+        this.emitProgress(ScraperProgressTypes.ChangePassword);
         return {
           success: false,
           errorType: ScraperErrorTypes.ChangePassword,
         };
       }
 
-      this.emitProgress(ScaperProgressTypes.LoginFailed);
+      this.emitProgress(ScraperProgressTypes.LoginFailed);
       return {
         success: false,
         errorType: ScraperErrorTypes.InvalidPassword,
       };
     }
 
     if (validateReturnCode === '4') {
-      this.emitProgress(ScaperProgressTypes.ChangePassword);
+      this.emitProgress(ScraperProgressTypes.ChangePassword);
       return {
         success: false,
         errorType: ScraperErrorTypes.ChangePassword,
       };
     }
 
-    this.emitProgress(ScaperProgressTypes.LoginFailed);
+    this.emitProgress(ScraperProgressTypes.LoginFailed);
     return {
       success: false,
       errorType: ScraperErrorTypes.InvalidPassword,

src/scrapers/base-scraper-with-browser.ts

@@ -1,13 +1,14 @@
 import puppeteer, { Browser, Frame, Page } from 'puppeteer';
 
 import {
-  ScraperErrorTypes,
-  BaseScraper, ScaperScrapingResult, ScaperProgressTypes,
-  ScraperCredentials,
+  BaseScraper, ScraperProgressTypes,
+
 } from './base-scraper';
 import { getCurrentUrl, waitForNavigation } from '../helpers/navigation';
 import { clickButton, fillInput, waitUntilElementFound } from '../helpers/elements-interactions';
 import { getDebug } from '../helpers/debug';
+import { ScraperErrorTypes } from './errors';
+import { ScraperScrapingResult, ScraperCredentials } from './interface';
 
 const VIEWPORT_WIDTH = 1024;
 const VIEWPORT_HEIGHT = 768;
@@ -75,22 +76,22 @@ async function getKeyByValue(object: PossibleLoginResults, value: string, page:
   return Promise.resolve(LoginResults.UnknownError);
 }
 
-function handleLoginResult(scraper: BaseScraperWithBrowser, loginResult: LoginResults) {
+function handleLoginResult(scraper: BaseScraperWithBrowser<ScraperCredentials>, loginResult: LoginResults) {
   switch (loginResult) {
     case LoginResults.Success:
-      scraper.emitProgress(ScaperProgressTypes.LoginSuccess);
+      scraper.emitProgress(ScraperProgressTypes.LoginSuccess);
       return { success: true };
     case LoginResults.InvalidPassword:
     case LoginResults.UnknownError:
-      scraper.emitProgress(ScaperProgressTypes.LoginFailed);
+      scraper.emitProgress(ScraperProgressTypes.LoginFailed);
       return {
         success: false,
         errorType: loginResult === LoginResults.InvalidPassword ? ScraperErrorTypes.InvalidPassword :
           ScraperErrorTypes.General,
         errorMessage: `Login failed with ${loginResult} error`,
       };
     case LoginResults.ChangePassword:
-      scraper.emitProgress(ScaperProgressTypes.ChangePassword);
+      scraper.emitProgress(ScraperProgressTypes.ChangePassword);
       return {
         success: false,
         errorType: ScraperErrorTypes.ChangePassword,
@@ -100,14 +101,14 @@ function handleLoginResult(scraper: BaseScraperWithBrowser, loginResult: LoginRe
   }
 }
 
-function createGeneralError(): ScaperScrapingResult {
+function createGeneralError(): ScraperScrapingResult {
   return {
     success: false,
     errorType: ScraperErrorTypes.General,
   };
 }
 
-class BaseScraperWithBrowser extends BaseScraper {
+class BaseScraperWithBrowser<TCredentials extends ScraperCredentials> extends BaseScraper<TCredentials> {
   // NOTICE - it is discourage to use bang (!) in general. It is used here because
   // all the classes that inherit from this base assume is it mandatory.
   protected browser!: Browser;
@@ -126,6 +127,7 @@ class BaseScraperWithBrowser extends BaseScraper {
   async initialize() {
     await super.initialize();
     debug('initialize scraper');
+    this.emitProgress(ScraperProgressTypes.Initializing);
 
     let env: Record<string, any> | undefined;
     if (this.options.verbose) {
@@ -225,7 +227,7 @@ class BaseScraperWithBrowser extends BaseScraper {
     }
   }
 
-  async login(credentials: Record<string, string>): Promise<ScaperScrapingResult> {
+  async login(credentials: ScraperCredentials): Promise<ScraperScrapingResult> {
     if (!credentials || !this.page) {
       return createGeneralError();
     }
@@ -262,7 +264,7 @@ class BaseScraperWithBrowser extends BaseScraper {
     } else {
       await loginOptions.submitButtonSelector();
     }
-    this.emitProgress(ScaperProgressTypes.LoggingIn);
+    this.emitProgress(ScraperProgressTypes.LoggingIn);
 
     if (loginOptions.postAction) {
       debug('execute \'postAction\' interceptor provided in login options');
@@ -281,7 +283,7 @@ class BaseScraperWithBrowser extends BaseScraper {
 
   async terminate(_success: boolean) {
     debug(`terminating browser with success = ${_success}`);
-    this.emitProgress(ScaperProgressTypes.Terminating);
+    this.emitProgress(ScraperProgressTypes.Terminating);
 
     if (!_success && !!this.options.storeFailureScreenShotPath) {
       debug(`create a snapshot before terminated in ${this.options.storeFailureScreenShotPath}`);