diff --git a/samples/degiro-export.csv b/samples/degiro-export.csv index b228e37c..82a15e8b 100644 --- a/samples/degiro-export.csv +++ b/samples/degiro-export.csv @@ -71,3 +71,43 @@ Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id 09-06-2020,12:05,08-06-2020,EUR CASH FUND FUNDSHARE,NL0010661914,"Conversion Fonds Monétaires finalisée: Vente 1 046,3825 @ 0,9854 EUR",,EUR,-4.81,EUR,84.89, 01-01-2025,06:46,31-12-2024,,,FX Credit,,EUR,12.28,EUR,217.01, 01-01-2025,06:46,31-12-2024,,,FX Debit,1.0379,USD,-12.75,USD,0.00, +13-09-2021,08:26,10-09-2021,MOODY'S CORP.,US6153691059,Dividend,,USD,0.62,USD,0.53, +13-09-2021,08:26,10-09-2021,MOODY'S CORP.,US6153691059,Dividendbelasting,,USD,-0.09,USD,-0.09, +10-09-2021,20:22,10-09-2021,MOODY'S CORP.,US6153691059,Valuta Debitering,1.1824,USD,-384.50,USD,0.00,80ff8574-14dd-46e1-bdf7-f11b8bbfa5c7 +10-09-2021,20:22,10-09-2021,MOODY'S CORP.,US6153691059,Valuta Creditering,,EUR,325.19,EUR,,80ff8574-14dd-46e1-bdf7-f11b8bbfa5c7 +10-09-2021,20:22,10-09-2021,MOODY'S CORP.,US6153691059,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-0.50,EUR,,80ff8574-14dd-46e1-bdf7-f11b8bbfa5c7 +10-09-2021,20:22,10-09-2021,MOODY'S CORP.,US6153691059,"Verkoop 1 @ 384,5 USD",,USD,384.50,USD,,80ff8574-14dd-46e1-bdf7-f11b8bbfa5c7 +16-12-2021,18:02,22-10-2021,MOODY'S CORP.,US6153691059,Dividendbelasting,,USD,-0.01,USD,0.00, +16-12-2021,18:02,22-10-2021,MORGAN STANLEY,US6174464486,Dividendbelasting,,USD,0.01,USD,0.01, +16-12-2021,18:02,22-10-2021,MOODY'S CORP.,US6153691059,Dividendbelasting,,USD,-0.01,USD,0.00, +16-12-2021,18:02,22-10-2021,MORGAN STANLEY,US6174464486,Dividendbelasting,,USD,0.01,USD,0.01, +23-12-2020,10:03,23-12-2020,ISHARES GLOBAL CLEAN ENERGY UCITS ETF,IE00B1XNHC34,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-0.79,EUR,,1d2e7a85-cf67-4325-82c7-d5994a0afb98 +23-12-2020,10:03,23-12-2020,ISHARES GLOBAL CLEAN ENERGY UCITS ETF,IE00B1XNHC34,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-4.00,EUR,,1d2e7a85-cf67-4325-82c7-d5994a0afb98 +23-12-2020,10:03,23-12-2020,ISHARES GLOBAL CLEAN ENERGY UCITS ETF,IE00B1XNHC34,Verkoop 120 @ 1.202 GBX,,GBP,1442.40,GBP,,1d2e7a85-cf67-4325-82c7-d5994a0afb98 +25-08-2022,10:21,25-08-2022,TESLA,US88160R1014,"STOCK SPLIT: Koop 3 @ 297,0967 USD",,USD,-891.29,USD,-0.01, +25-08-2022,10:21,25-08-2022,TESLA,US88160R1014,"STOCK SPLIT: Verkoop 1 @ 891,29 USD",,USD,891.29,USD,1.29, +11-07-2024,16:46,11-07-2024,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,"Verkoop 16 @ 124,28 EUR",,EUR,1988.48,EUR,,56e1f16b-4142-4373-851d-45d0505fe12c +11-07-2024,16:43,11-07-2024,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-1.00,EUR,,56e1f16b-4142-4373-851d-45d0505fe12c +11-07-2024,16:43,11-07-2024,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,"Verkoop 8 @ 124,28 EUR",,EUR,994.24,EUR,,56e1f16b-4142-4373-851d-45d0505fe12c +13-04-2016,09:00,13-04-2016,ABN AMRO BANK NV,NL0011540547,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-2.00,EUR,,df134e52-2753-4694-947b-418f08d4a352 +13-04-2016,09:00,13-04-2016,ABN AMRO BANK NV,NL0011540547,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-0.04,EUR,,df134e52-2753-4694-947b-418f08d4a352 +13-04-2016,09:00,13-04-2016,ABN AMRO BANK NV,NL0011540547,"Koop 10 @ 18,3 EUR",,EUR,-183.00,EUR,,df134e52-2753-4694- +,,,,,,,,,,,947b-418f08d4a352 +28-11-2023,15:30,28-11-2023,TESLA,US88160R1014,Valuta Debitering,1.1005,USD,-710.37,USD,0.00,81ac086b-8fab-4465- +,,,,,,,,,,,a458-118e34009c9e +28-11-2023,15:30,28-11-2023,TESLA,US88160R1014,Valuta Creditering,,EUR,645.47,EUR,672.29,81ac086b-8fab-4465-a458-118e34009c9e +28-11-2023,15:30,28-11-2023,TESLA,US88160R1014,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-2.00,EUR,26.82,81ac086b-8fab-4465-a458-118e34009c9e +28-11-2023,15:30,28-11-2023,TESLA,US88160R1014,"Verkoop 3 @ 236,79 USD",,USD,710.37,USD,710.37,81ac086b-8fab-4465-a458-118e34009c9e +24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-0.26,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 +24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-2.00,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 +24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-0.02,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 +24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,"Verkoop 14 @ 61,7 EUR",,EUR,863.80,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 +24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,"Verkoop 1 @ 61,7 EUR",,EUR,61.70,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 +30-12-2024,07:44,27-12-2024,ISHARES CHINA LARGE CAP UCITS ETF USD,IE00B02KXK85,Dividend,,USD,1.10,USD,15.30, +01-02-2022,09:56,01-02-2022,ISHARES CHINA LARGE CAP UCITS ETF USD,IE00B02KXK85,"Koop 2 @ 96,22 EUR",,EUR,-192.44,EUR,,b2a1b368-491d-4fad-81ba-460c7f6839cf +03-01-2022,08:21,31-12-2021,ISHARES CHINA LARGE CAP UCITS ETF USD,IE00B02KXK85,Dividend,,USD,1.22,USD,1.22, +01-06-2021,11:03,01-06-2021,ISHARES CHINA LARGE CAP UCITS ETF USD,IE00B02KXK85,"Koop 1 @ 111,11 EUR",,EUR,-111.11,EUR,,1ee86f48-4a0a-4fd2-9d32-98aee813217f +06-10-2022,10:13,06-10-2022,ETC ISSUANCE GMBH,DE000A3G01J0,WIJZIGING ISIN: Koop 12 @ 0 EUR,,EUR,0.00,EUR, +06-10-2022,08:16,14-09-2022,ETC ISSUANCE ETHETC - PHYSICAL,DE000A3G01J0,SPIN-OFF: Koop 12 @ 0 EUR,,EUR,0.00,EUR, +05-10-2022,15:43,14-09-2022,ETC ISSUANCE ETHETC - PHYSICAL,DE000A3G01J0,CLAIMEMISSIE: Verkoop 12 @ 0 EUR,,EUR,0.00,EUR, +05-10-2022,11:09,14-09-2022,ETC ISSUANCE ETHETC - PHYSICAL,DE000A3G01J0,CLAIMEMISSIE: Koop 12 @ 0 EUR,,EUR,0.00,EUR diff --git a/src/converters/degiroConverterV3.test.ts b/src/converters/degiroConverterV3.test.ts index 7c6f6fbe..c774e7ac 100644 --- a/src/converters/degiroConverterV3.test.ts +++ b/src/converters/degiroConverterV3.test.ts @@ -35,7 +35,7 @@ describe("degiroConverterV3", () => { // Assert expect(actualExport).toBeTruthy(); expect(actualExport.activities.length).toBeGreaterThan(0); - expect(actualExport.activities.length).toBe(26); + expect(actualExport.activities.length).toBe(43); done(); }, () => { done.fail("Should not have an error!"); }); diff --git a/src/converters/degiroConverterV3.ts b/src/converters/degiroConverterV3.ts index ff020d20..7a4493a3 100644 --- a/src/converters/degiroConverterV3.ts +++ b/src/converters/degiroConverterV3.ts @@ -1,4 +1,5 @@ import dayjs from "dayjs"; +import * as crypto from "crypto"; import { parse } from "csv-parse"; import { DeGiroRecord } from "../models/degiroRecord"; import { AbstractConverter } from "./abstractconverter"; @@ -21,6 +22,10 @@ export class DeGiroConverterV3 extends AbstractConverter { * @inheritdoc */ public processFileContents(input: string, successCallback: any, errorCallback: any): void { + // my exported file contained many broken records like this. It's easy to recover them. So, why not?! + // 13-04-2016,09:00,13-04-2016,ABN AMRO BANK NV,NL0011540547,"Koop 10 @ 18,3 EUR",,EUR,-183.00,EUR,,df134e52-2753-4694- + // ,,,,,,,,,,,947b-418f08d4a352 + input = input.replace(/(,[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-)\n,{11}([a-f0-9]{4}-[a-f0-9]{12})$/mg, '$1$2'); // Parse the CSV and convert to Ghostfolio import format. parse(input, { @@ -37,10 +42,10 @@ export class DeGiroConverterV3 extends AbstractConverter { return columnValue; } - }, async (err, records: DeGiroRecord[]) => { + }, async (err, plainRecords: DeGiroRecord[]) => { // Check if parsing failed.. - if (err || records === undefined || records.length === 0) { + if (err || plainRecords === undefined || plainRecords.length === 0) { let errorMsg = "An error ocurred while parsing!"; if (err) { @@ -66,9 +71,22 @@ export class DeGiroConverterV3 extends AbstractConverter { activities: [] }; + // Inverse records so they come in the chronological order. This is very important since + // selecion of security should happe based on buy/sell records, and not dividend/fee records. + // By default DeGiro's CSV file has the most recent records at the top. So, by inversing + // the order we process records in choronological order but keep internal releationships + // between records which we can break if, for instance, sort by date/time. + plainRecords.reverse(); + + // Map plain objects to DeGiroRecord instances + const records = plainRecords.map(record => DeGiroRecord.fromPlainObject(record)); + // Populate the progress bar. const bar1 = this.progress.create(records.length, 0); + // HashSet to skip processed records + const processedRecords = new Set(); + for (let idx = 0; idx < records.length; idx++) { const record = records[idx]; @@ -78,66 +96,28 @@ export class DeGiroConverterV3 extends AbstractConverter { continue; } - // Look if the current record was already processed previously by checking the orderId. - // Not all exports provide an order ID, so check for a buy/sell marking in those cases. - // Dividend records never have an order ID, so check for a marking there. - // If a match was found, skip the record and move next. - if (result.activities.findIndex(a => - a.comment !== "" && - a.comment === record.orderId || - a.comment.startsWith(`Buy ${record.isin} @ ${record.date}T`) || - a.comment.startsWith(`Sell ${record.isin} @ ${record.date}T`) || - a.comment.startsWith(`Dividend ${record.isin} @ ${record.date}T`)) > -1) { - - bar1.increment(); - continue; + // Check if the current record was already processed. + const recordHash = this.hashRecord(record); + if (processedRecords.has(recordHash)) { + bar1.increment(); + continue } + processedRecords.add(recordHash); + // TODO: Is is possible to add currency? So VWRL.AS is retrieved for IE00B3RBWM25 instead of VWRL.L. // Maybe add yahoo-finance2 library that Ghostfolio uses, so I dont need to call Ghostfolio for this. // Platform fees do not have a security, add those immediately. if (this.isPlatformFees(record)) { - - const feeAmount = Math.abs(parseFloat(record.amount.replace(",", "."))); - const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); - - result.activities.push({ - accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, - comment: "", - fee: feeAmount, - quantity: 1, - type: GhostfolioOrderType.fee, - unitPrice: 0, - currency: record.currency, - dataSource: "MANUAL", - date: date.format("YYYY-MM-DDTHH:mm:ssZ"), - symbol: record.description - }); - + result.activities.push(this.mapPlatformFeeRecord(record)); bar1.increment(1); continue; } // Interest does not have a security, add it immediately. if (this.isInterest(record)) { - - const interestAmount = Math.abs(parseFloat(record.amount.replace(",", "."))); - const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); - - result.activities.push({ - accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, - comment: "", - fee: 0, - quantity: 1, - type: GhostfolioOrderType.interest, - unitPrice: interestAmount, - currency: record.currency, - dataSource: "MANUAL", - date: date.format("YYYY-MM-DDTHH:mm:ssZ"), - symbol: record.description - }); - + result.activities.push(this.mapInterestRecord(record)); bar1.increment(1); continue; } @@ -164,35 +144,72 @@ export class DeGiroConverterV3 extends AbstractConverter { continue; } - // Look ahead in the remaining records if there is one with the samen orderId. - let matchingRecord = this.findMatchByOrderId(record, records.slice(idx + 1)); + // Look ahead in the remaining records if there are some with the same orderId. + let matchingRecords = []; + if (record.orderId) { + matchingRecords = this.findMatchesByOrderId(record, records.slice(idx + 1)); + } + + // If there was no match by orderId, and there was no orderId present on the current record, look ahead one record to find a match by ISIN + Product + Date. + if (matchingRecords.length == 0 && !record.orderId) { + matchingRecords = this.findMatchByIsinSameDateTime(record, records.slice(idx + 1, idx + 2)); + } + + // Filter out ignored records + matchingRecords = matchingRecords.filter(r => !this.isIgnoredRecord(r)); - // If there was no match by orderId, and there was no orderId present on the current record, look ahead in the remaining records to find a match by ISIN + Product. - if (!matchingRecord && !record.orderId) { - matchingRecord = this.findMatchByIsin(record, records.slice(idx + 1)); + // Register records as processed so they are skipped on next iteration(s) + matchingRecords.forEach(r => processedRecords.add(this.hashRecord(r))); + + if (matchingRecords.length == 0) { + // If it's a standalone record, add it immediately. + const mappedRecord = this.mapStandaloneRecord(record, security); + if (mappedRecord) result.activities.push(mappedRecord); + + bar1.increment(1); + continue; } - // If it's a standalone record, add it immediately. - if (!matchingRecord) { + // Now, need to check that all records uses the same currency. + // If not, we have to report them separately even though they are part of one operation in DeGiro + const sameCurrency = matchingRecords.every(r => r.currency == record.currency); + if (!sameCurrency) { + matchingRecords.unshift(record); // combine records to work with them as one whole + matchingRecords.forEach(r => { + const mr = this.mapStandaloneRecord(r, security); + if (mr) result.activities.push(mr); + }) - if (this.isBuyOrSellRecord(record)) { - result.activities.push(this.mapRecordToActivity(record, security)); - } - else { - result.activities.push(this.mapDividendRecord(record, null, security)); - } + bar1.increment(1); + continue; } - else { - // This is a pair of records. Check which type of record it is and then combine the records into a Ghostfolio activity. + // Okay, we're dealing with a set of records of the same currency. + // We can combined them and report as one (or multiple records). + matchingRecords.unshift(record); // combine records to work with them as one whole + + // Get main and fees records + const mainRecords = matchingRecords.filter(r => this.isBuyOrSellRecord(r) || this.isDividendRecord(r)); + let transactionFeeRecords = matchingRecords.filter(r => this.isTransactionFeeRecord(r)); - // Check wether it is a buy/sell record set. - if (this.isBuyOrSellRecordSet(record, matchingRecord)) { - result.activities.push(this.combineRecords(record, matchingRecord, security)); + if (mainRecords.length == 0 || matchingRecords.length != mainRecords.length + transactionFeeRecords.length) { + this.progress.log(`[i] Unexpected set of ${matchingRecords.length} records (see below)! Please add them manually..\n`); + matchingRecords.forEach(r => this.progress.log(`Record ${r.isin || r.product} from ${r.date} with ${r.amount}${r.currency}`)); + + bar1.increment(); + continue; + } + + // if there is one main record, report it combined with transaction fee(s). All other goes without TX fee. + mainRecords.forEach(r => { + if (this.isBuyOrSellRecord(r)) { + result.activities.push(this.mapBuySellRecord(r, transactionFeeRecords, security)); } else { - result.activities.push(this.mapDividendRecord(record, matchingRecord, security)); + result.activities.push(this.mapDividendRecord(r, transactionFeeRecords, security)); } - } + + transactionFeeRecords = []; // nullify tx after first iteration + }); bar1.increment(); } @@ -219,8 +236,8 @@ export class DeGiroConverterV3 extends AbstractConverter { "fx", "currency", "amount", - "col1", // Not relevant column. - "col2", // Not relevant column. + "balance_currency", + "balance", "orderId"]; return csvHeaders; @@ -240,12 +257,26 @@ export class DeGiroConverterV3 extends AbstractConverter { return true; } + // Sometimes there are records like the following. They happen when an ETF changes ISIN. + // They have zero value and zero price. Let's ignore them + // 06-10-2022,10:13,06-10-2022,ETC ISSUANCE GMBH,DE000A3G01J0,WIJZIGING ISIN: Koop 12 @ 0 EUR,,EUR,0.00,EUR, + // 06-10-2022,08:16,14-09-2022,ETC ISSUANCE ETHETC - PHYSICAL,DE000A3G01J0,SPIN-OFF: Koop 12 @ 0 EUR,,EUR,0.00,EUR, + // 05-10-2022,15:43,14-09-2022,ETC ISSUANCE ETHETC - PHYSICAL,DE000A3G01J0,CLAIMEMISSIE: Verkoop 12 @ 0 EUR,,EUR,0.00,EUR, + // 05-10-2022,11:09,14-09-2022,ETC ISSUANCE ETHETC - PHYSICAL,DE000A3G01J0,CLAIMEMISSIE: Koop 12 @ 0 EUR,,EUR,0.00,EUR, + if (record.getAbsoluteAmount() == 0 && !record.fx) { + return true; + } + const ignoredRecordTypes = [ "ideal", "flatex", "cash sweep", "withdrawal", "productwijziging", + "compensatie", + "terugstorting", + "geldmarktfonds", + "overboeking", "währungswechsel", "trasferisci", "deposito", @@ -271,123 +302,162 @@ export class DeGiroConverterV3 extends AbstractConverter { return ignoredRecordTypes.some((t) => record.description.toLocaleLowerCase().indexOf(t) > -1); } - private findMatchByOrderId(currentRecord: DeGiroRecord, records: DeGiroRecord[]): DeGiroRecord | undefined { - return records.find(r => r.orderId === currentRecord.orderId); + private findMatchesByOrderId(currentRecord: DeGiroRecord, records: DeGiroRecord[]): DeGiroRecord[] | undefined { + return records.filter(r => r.orderId === currentRecord.orderId); } - private findMatchByIsin(currentRecord: DeGiroRecord, records: DeGiroRecord[]): DeGiroRecord | undefined { - return records.find(r => r.isin === currentRecord.isin && r.product === currentRecord.product); + private findMatchByIsinSameDateTime(currentRecord: DeGiroRecord, records: DeGiroRecord[]): DeGiroRecord[] | undefined { + return records.filter(r => r.isin === currentRecord.isin && r.product === currentRecord.product && r.date == currentRecord.date); } - private mapRecordToActivity(record: DeGiroRecord, security?: YahooFinanceRecord, isTransactionFeeRecord: boolean = false): GhostfolioActivity { + private mapTransactionFeeRecord(record: DeGiroRecord, security?: YahooFinanceRecord): GhostfolioActivity { + const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); + const feeAmount = record.getAbsoluteAmount(); - let numberShares, unitPrice, feeAmount = 0; - let orderType; + const currency = this.getCurrencyIfConvertable(record.currency, security.currency) + const convertedFeeAmount = this.convertCurrencyIfConvertable(feeAmount, record.currency, security.currency); - // If it is not a transaction fee record, get data from the record. - if (!isTransactionFeeRecord) { + return { + accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, + comment: record.orderId, + fee: this.formatFloat(convertedFeeAmount), + quantity: 1, + type: GhostfolioOrderType.fee, + unitPrice: 0, + currency: currency, + dataSource: "MANUAL", + date: date.format("YYYY-MM-DDTHH:mm:ssZ"), + // ghostfolio doesn't like two MANUAL records with same name, hence adding date & time. + symbol: `Transaction fee ${security.symbol} @ ${record.date}T${record.time}` + }; + } - // Get the amount of shares from the description. - const numberSharesFromDescription = record.description.match(/([\d*\.?\,?\d*]+)/)[0]; - numberShares = parseFloat(numberSharesFromDescription); + private mapBuySellRecord(record: DeGiroRecord, transactionFeeRecords: DeGiroRecord[], security?: YahooFinanceRecord): GhostfolioActivity { + // !IMPORTANT It's assumed that all records (record + transactionFeeRecords) have same currency - // For buy/sale records, only the total amount is recorded. So the unit price needs to be calculated. - const totalAmount = parseFloat(record.amount.replace(",", ".")); - unitPrice = parseFloat((Math.abs(totalAmount) / numberShares).toFixed(3)); + const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); - // If amount is negative (so money has been removed) or it's stock dividend (so free shares), thus it's a buy record. - if (totalAmount < 0 || record.description.toLocaleLowerCase().indexOf("stock dividend") > -1) { - orderType = GhostfolioOrderType.buy; - } else { - orderType = GhostfolioOrderType.sell; - } - } - else { + /* Get the amount and unit price of shares from the description. + * For buy/sale records, only the total amount is recorded. So the unit price needs to be calculated. + * However, in the modern days degiro csv both quantity and price-per-unit are reported. + * Ex: Verkoop 1 @ 16,78 USD + * Ex: Koop 475 @ 583,5 GBX + * + * At the same time, we can't use the unit price because it may come with a different "currency". + * "Currency" is in quotes because it's the case between GBP and GBp/GBX. 1 GBP = 100 * GBp/GBX. + * So, it's not really a difference currency, but rather a different denomination. + * Yet, it makes it necessary to do the dance with dividing total amount by num of shares. + * Ex: + * 03-03-2020,11:14,03-03-2020,ISHARES GLOBAL CLEAN ENERGY UCITS ETF,IE00B1XNHC34,"Koop 475 @ 583,5 GBX",,GBP,-2771.63,GBP,,3b000105-xxxx-xxxx-xxxx-xxxxxxxxxxxx + * + * Another note: the old code contained regexp: + * record.description.match(/([\d*\.?\,?\d*]+)/)[0] + * + * This regexp seems broken to me. Confirmed by ChatGPT. + * I have no idea how it managed to work. + */ + + const num = record.description.match(/([0-9]+[.,]?[0-9]*)/)[0]; + const quantity = parseFloat(num); + const unitPrice = record.getAbsoluteAmount() / quantity; + + // If amount is negative (so money has been removed) + const orderType = record.getAmount() < 0 ? GhostfolioOrderType.buy : GhostfolioOrderType.sell; - // Otherwise, get the transaction fee info. - feeAmount = parseFloat(Math.abs(parseFloat(record.amount.replace(",", "."))).toFixed(3)); - } + // Extract the fee from the transaction fee record and put it in the action record. + const feeAmount = transactionFeeRecords.reduce((sum, r) => sum + r.getAbsoluteAmount(), 0); - const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); + const currency = this.getCurrencyIfConvertable(record.currency, security.currency) + const convertedFeeAmount = this.convertCurrencyIfConvertable(feeAmount, record.currency, security.currency); + const convertedUnitPrice = this.convertCurrencyIfConvertable(unitPrice, record.currency, security.currency); - // Create the record. return { accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, comment: record.orderId ?? `${orderType === GhostfolioOrderType.buy ? "Buy" : "Sell"} ${record.isin} @ ${record.date}T${record.time}`, - fee: feeAmount, - quantity: numberShares, + fee: this.formatFloat(convertedFeeAmount), + quantity: this.formatFloat(quantity), type: orderType, - unitPrice: unitPrice, - currency: record.currency ?? "", + unitPrice: this.formatFloat(convertedUnitPrice), + currency: currency, dataSource: "YAHOO", date: date.format("YYYY-MM-DDTHH:mm:ssZ"), symbol: security.symbol ?? "", }; } - private combineRecords(currentRecord: DeGiroRecord, nextRecord: DeGiroRecord, security: YahooFinanceRecord): GhostfolioActivity { + private mapDividendRecord(record: DeGiroRecord, transactionFeeRecords: DeGiroRecord[], security: YahooFinanceRecord): GhostfolioActivity { + // !IMPORTANT It's assumed that all records (record + transactionFeeRecords) have same currency - // Set the default values for the records. - let actionRecord = currentRecord; - let txFeeRecord: DeGiroRecord | null = nextRecord; - - // Determine which of the two records is the action record (e.g. buy/sell) and which contains the transaction fees. - // Firstly, check if the current record is the TxFee record. - if (this.isTransactionFeeRecord(currentRecord, true)) { - actionRecord = nextRecord; - txFeeRecord = currentRecord; - } - - // Map both records. - const mappedActionRecord = this.mapRecordToActivity(actionRecord, security); - const mappedTxFeeRecord = this.mapRecordToActivity(txFeeRecord, security, true); + const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); + const unitPrice = record.getAbsoluteAmount(); + const feeAmount = transactionFeeRecords.reduce((sum, r) => sum + r.getAbsoluteAmount(), 0); - // Extract the fee from the transaction fee record and put it in the action record. - mappedActionRecord.fee = mappedTxFeeRecord.fee; + const currency = this.getCurrencyIfConvertable(record.currency, security.currency) + const convertedFeeAmount = this.convertCurrencyIfConvertable(feeAmount, record.currency, security.currency); + const convertedUnitPrice = this.convertCurrencyIfConvertable(unitPrice, record.currency, security.currency); - return mappedActionRecord; + // Create the record. + return { + accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, + comment: `Dividend ${record.isin} @ ${record.date}T${record.time}`, + fee: this.formatFloat(convertedFeeAmount), + quantity: 1, + type: GhostfolioOrderType.dividend, + unitPrice: this.formatFloat(convertedUnitPrice), + currency: currency, + dataSource: "YAHOO", + date: date.format("YYYY-MM-DDTHH:mm:ssZ"), + symbol: security.symbol, + }; } - private mapDividendRecord(currentRecord: DeGiroRecord, nextRecord: DeGiroRecord | null = null, security: YahooFinanceRecord): GhostfolioActivity { - - // It's a dividend set. - // Set the default values for the records. - let dividendRecord = currentRecord; - let txFeeRecord: DeGiroRecord = nextRecord; + private mapStandaloneRecord(record: DeGiroRecord, security?: YahooFinanceRecord): GhostfolioActivity { + if (this.isBuyOrSellRecord(record)) + return this.mapBuySellRecord(record, [], security); - // Determine which of the two records is the dividend record and which contains the transaction fees. - // Firstly, check if the current record is the TxFee record. - if (nextRecord && this.isTransactionFeeRecord(currentRecord, false)) { - dividendRecord = nextRecord; - txFeeRecord = currentRecord; - } + if (this.isDividendRecord(record)) + return this.mapDividendRecord(record, [], security); - let unitPrice = Math.abs(parseFloat(dividendRecord.amount.replace(",", "."))); - let fees = 0; - if (txFeeRecord) { - fees = Math.abs(parseFloat(txFeeRecord.amount.replace(",", "."))); - } + if (this.isTransactionFeeRecord(record)) + // sometimes there are standalone transaction fee records + return this.mapTransactionFeeRecord(record, security); - const date = dayjs(`${dividendRecord.date} ${dividendRecord.time}:00`, "DD-MM-YYYY HH:mm"); + this.progress.log(`[i] Unknown standalone record ${record.isin || record.product} from ${record.date} with ${record.amount}${record.currency}! Please add this manually..\n`); + return undefined; + } - // Create the record. + private mapPlatformFeeRecord(record: DeGiroRecord): GhostfolioActivity { + const feeAmount = record.getAbsoluteAmount(); + const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); return { accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, - comment: `Dividend ${dividendRecord.isin} @ ${currentRecord.date}T${currentRecord.time}`, - fee: fees, + comment: "", + fee: this.formatFloat(feeAmount), quantity: 1, - type: GhostfolioOrderType.dividend, - unitPrice: unitPrice, - currency: dividendRecord.currency, - dataSource: "YAHOO", + type: GhostfolioOrderType.fee, + unitPrice: 0, + currency: record.currency, + dataSource: "MANUAL", date: date.format("YYYY-MM-DDTHH:mm:ssZ"), - symbol: security.symbol, + symbol: record.description }; } - private isBuyOrSellRecordSet(currentRecord: DeGiroRecord, nextRecord: DeGiroRecord): boolean { - return (this.isBuyOrSellRecord(currentRecord) && this.isTransactionFeeRecord(nextRecord, true)) || - (this.isTransactionFeeRecord(currentRecord, true) && this.isBuyOrSellRecord(nextRecord)) + private mapInterestRecord(record: DeGiroRecord): GhostfolioActivity { + const interestAmount = record.getAbsoluteAmount(); + const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); + return { + accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, + comment: "", + fee: 0, + quantity: 1, + type: GhostfolioOrderType.interest, + unitPrice: this.formatFloat(interestAmount), + currency: record.currency, + dataSource: "MANUAL", + date: date.format("YYYY-MM-DDTHH:mm:ssZ"), + symbol: record.description + }; } private isBuyOrSellRecord(record: DeGiroRecord): boolean { @@ -401,14 +471,24 @@ export class DeGiroConverterV3 extends AbstractConverter { return buySellRecordType.some((t) => record.description.toLocaleLowerCase().indexOf(t) > -1); } - private isTransactionFeeRecord(record: DeGiroRecord, isBuyOrSellTransactionFeeRecord: boolean): boolean { + private isDividendRecord(record: DeGiroRecord): boolean { if (!record) { return false; } - // When a dividend transaction must be found, there should not be an orderid. - if (!isBuyOrSellTransactionFeeRecord && record.orderId) { + if (this.isTransactionFeeRecord(record)) { + // dividend tax records often has 'Impôts sur dividende' or 'dividendbelasting' + // which make them match the condition below. + return false; + } + + return record.description.toLocaleLowerCase().indexOf("dividend") > -1 || record.description.toLocaleLowerCase().indexOf("capital return") > -1; + } + + private isTransactionFeeRecord(record: DeGiroRecord): boolean { + + if (!record) { return false; } @@ -430,4 +510,45 @@ export class DeGiroConverterV3 extends AbstractConverter { return platformFeeRecordType.some((t) => record.description.toLocaleLowerCase().indexOf(t) > -1); } + + private hashRecord(record: DeGiroRecord): string { + const md5 = crypto.createHash('md5'); + md5.update(record.date); + md5.update(record.time); + md5.update(record.currencyDate.toString()); + md5.update(record.product); + md5.update(record.isin); + md5.update(record.description); + md5.update(record.fx); + md5.update(record.currency); + md5.update(record.amount); + md5.update(record.balance_currency); + md5.update(record.balance); + md5.update(record.orderId); + return md5.digest('hex'); + } + + private formatFloat(val: number): number { + return parseFloat(val.toFixed(3)); + } + + private getCurrencyIfConvertable(from_currency: string, to_currency: string): string { + if (from_currency == "GBP" && (to_currency == "GBp" || to_currency == "GBX")) + return to_currency + + if ((from_currency == "GBp" || from_currency == "GBX") && to_currency == "GBP") + return to_currency; + + return from_currency; + } + + private convertCurrencyIfConvertable(val: number, from_currency: string, to_currency: string): number { + if (from_currency == "GBP" && (to_currency == "GBp" || to_currency == "GBX")) + return val * 100.0; + + if (from_currency == "GBp" && to_currency == "GBP") + return val / 100.0; + + return val; + } } diff --git a/src/models/degiroRecord.ts b/src/models/degiroRecord.ts index 9dc275b3..3c113f39 100644 --- a/src/models/degiroRecord.ts +++ b/src/models/degiroRecord.ts @@ -8,5 +8,14 @@ export class DeGiroRecord { fx: string; currency: string; amount: string; + balance_currency: string; // not used, but improtant for hashing + balance: string; //// not used, but improtant for hashing orderId: string; + + public getAmount(): number { return parseFloat(this.amount.replace(",", ".")); } + public getAbsoluteAmount(): number { return Math.abs(this.getAmount()); } + + static fromPlainObject(obj: any): DeGiroRecord { + return Object.assign(new DeGiroRecord(), obj); + } }