From 26d394dbd4dab8084f3d8e7d7f869a55222d1592 Mon Sep 17 00:00:00 2001 From: Ivan Kruglov Date: Sat, 4 Jan 2025 00:54:43 +0100 Subject: [PATCH 01/14] degiroConverterV3: introduce mapInterestRecord() && mapPlatformFeeRecord() --- src/converters/degiroConverterV3.ts | 70 +++++++++++++++-------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/src/converters/degiroConverterV3.ts b/src/converters/degiroConverterV3.ts index ff020d20..79cf894f 100644 --- a/src/converters/degiroConverterV3.ts +++ b/src/converters/degiroConverterV3.ts @@ -98,46 +98,14 @@ export class DeGiroConverterV3 extends AbstractConverter { // Platform fees do not have a security, add those immediately. if (this.isPlatformFees(record)) { - - const feeAmount = Math.abs(parseFloat(record.amount.replace(",", "."))); - const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); - - result.activities.push({ - accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, - comment: "", - fee: feeAmount, - quantity: 1, - type: GhostfolioOrderType.fee, - unitPrice: 0, - currency: record.currency, - dataSource: "MANUAL", - date: date.format("YYYY-MM-DDTHH:mm:ssZ"), - symbol: record.description - }); - + result.activities.push(this.mapPlatformFeeRecord(record)); bar1.increment(1); continue; } // Interest does not have a security, add it immediately. if (this.isInterest(record)) { - - const interestAmount = Math.abs(parseFloat(record.amount.replace(",", "."))); - const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); - - result.activities.push({ - accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, - comment: "", - fee: 0, - quantity: 1, - type: GhostfolioOrderType.interest, - unitPrice: interestAmount, - currency: record.currency, - dataSource: "MANUAL", - date: date.format("YYYY-MM-DDTHH:mm:ssZ"), - symbol: record.description - }); - + result.activities.push(this.mapInterestRecord(record)); bar1.increment(1); continue; } @@ -390,6 +358,40 @@ export class DeGiroConverterV3 extends AbstractConverter { (this.isTransactionFeeRecord(currentRecord, true) && this.isBuyOrSellRecord(nextRecord)) } + private mapPlatformFeeRecord(record: DeGiroRecord): GhostfolioActivity { + const feeAmount = Math.abs(parseFloat(record.amount.replace(",", "."))); + const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); + return { + accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, + comment: "", + fee: feeAmount, + quantity: 1, + type: GhostfolioOrderType.fee, + unitPrice: 0, + currency: record.currency, + dataSource: "MANUAL", + date: date.format("YYYY-MM-DDTHH:mm:ssZ"), + symbol: record.description + }; + } + + private mapInterestRecord(record: DeGiroRecord): GhostfolioActivity { + const interestAmount = Math.abs(parseFloat(record.amount.replace(",", "."))); + const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); + return { + accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, + comment: "", + fee: 0, + quantity: 1, + type: GhostfolioOrderType.interest, + unitPrice: interestAmount, + currency: record.currency, + dataSource: "MANUAL", + date: date.format("YYYY-MM-DDTHH:mm:ssZ"), + symbol: record.description + }; + } + private isBuyOrSellRecord(record: DeGiroRecord): boolean { if (!record) { From bdeccb815893e5e30e11c2a1406da03b3f5fa61f Mon Sep 17 00:00:00 2001 From: Ivan Kruglov Date: Sat, 4 Jan 2025 01:02:22 +0100 Subject: [PATCH 02/14] degiroConverterV3: dedup records base on md5 hash --- src/converters/degiroConverterV3.ts | 45 +++++++++++++++++++---------- src/models/degiroRecord.ts | 2 ++ 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/converters/degiroConverterV3.ts b/src/converters/degiroConverterV3.ts index 79cf894f..9bdc681f 100644 --- a/src/converters/degiroConverterV3.ts +++ b/src/converters/degiroConverterV3.ts @@ -1,4 +1,5 @@ import dayjs from "dayjs"; +import * as crypto from "crypto"; import { parse } from "csv-parse"; import { DeGiroRecord } from "../models/degiroRecord"; import { AbstractConverter } from "./abstractconverter"; @@ -69,6 +70,9 @@ export class DeGiroConverterV3 extends AbstractConverter { // Populate the progress bar. const bar1 = this.progress.create(records.length, 0); + // HashSet to skip processed records + const processedRecords = new Set(); + for (let idx = 0; idx < records.length; idx++) { const record = records[idx]; @@ -78,21 +82,15 @@ export class DeGiroConverterV3 extends AbstractConverter { continue; } - // Look if the current record was already processed previously by checking the orderId. - // Not all exports provide an order ID, so check for a buy/sell marking in those cases. - // Dividend records never have an order ID, so check for a marking there. - // If a match was found, skip the record and move next. - if (result.activities.findIndex(a => - a.comment !== "" && - a.comment === record.orderId || - a.comment.startsWith(`Buy ${record.isin} @ ${record.date}T`) || - a.comment.startsWith(`Sell ${record.isin} @ ${record.date}T`) || - a.comment.startsWith(`Dividend ${record.isin} @ ${record.date}T`)) > -1) { - - bar1.increment(); - continue; + // Check if the current record was already processed. + const recordHash = this.hashRecord(record); + if (processedRecords.has(recordHash)) { + bar1.increment(); + continue } + processedRecords.add(recordHash); + // TODO: Is is possible to add currency? So VWRL.AS is retrieved for IE00B3RBWM25 instead of VWRL.L. // Maybe add yahoo-finance2 library that Ghostfolio uses, so I dont need to call Ghostfolio for this. @@ -187,8 +185,8 @@ export class DeGiroConverterV3 extends AbstractConverter { "fx", "currency", "amount", - "col1", // Not relevant column. - "col2", // Not relevant column. + "balance_currency", + "balance", "orderId"]; return csvHeaders; @@ -432,4 +430,21 @@ export class DeGiroConverterV3 extends AbstractConverter { return platformFeeRecordType.some((t) => record.description.toLocaleLowerCase().indexOf(t) > -1); } + + private hashRecord(record: DeGiroRecord): string { + const md5 = crypto.createHash('md5'); + md5.update(record.date); + md5.update(record.time); + md5.update(record.currencyDate.toString()); + md5.update(record.product); + md5.update(record.isin); + md5.update(record.description); + md5.update(record.fx); + md5.update(record.currency); + md5.update(record.amount); + md5.update(record.balance_currency); + md5.update(record.balance); + md5.update(record.orderId); + return md5.digest('hex'); + } } diff --git a/src/models/degiroRecord.ts b/src/models/degiroRecord.ts index 9dc275b3..dfe5d21b 100644 --- a/src/models/degiroRecord.ts +++ b/src/models/degiroRecord.ts @@ -8,5 +8,7 @@ export class DeGiroRecord { fx: string; currency: string; amount: string; + balance_currency: string; // not used, but improtant for hashing + balance: string; //// not used, but improtant for hashing orderId: string; } From 1f59298c40a8bfd663912f9d044ac655d46a9011 Mon Sep 17 00:00:00 2001 From: Ivan Kruglov Date: Sat, 4 Jan 2025 18:43:58 +0100 Subject: [PATCH 03/14] degiroConverterV3: extend ignored records --- src/converters/degiroConverterV3.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/converters/degiroConverterV3.ts b/src/converters/degiroConverterV3.ts index 9bdc681f..e7be66ad 100644 --- a/src/converters/degiroConverterV3.ts +++ b/src/converters/degiroConverterV3.ts @@ -212,6 +212,10 @@ export class DeGiroConverterV3 extends AbstractConverter { "cash sweep", "withdrawal", "productwijziging", + "compensatie", + "terugstorting", + "geldmarktfonds", + "overboeking", "währungswechsel", "trasferisci", "deposito", From d257b692b7440c0bb726ede44f93295b512baa5c Mon Sep 17 00:00:00 2001 From: Ivan Kruglov Date: Sat, 4 Jan 2025 18:49:24 +0100 Subject: [PATCH 04/14] degiroConverterV3: improve handling This commit re-works degiroConverterV3 logic. The main change is that the code now can consider a set of records as one, instread action+tx record. The following set of records are one sell with two transaction fees records. They can be reported as one in ghostfolio. The previous code didn't handle this. 23-12-2020,10:03,23-12-2020,ISHARES GLOBAL CLEAN ENERGY UCITS ETF,IE00B1XNHC34,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-0.79,EUR,,1d2e7a85-cf67-4325-82c7-d5994a0afb98 23-12-2020,10:03,23-12-2020,ISHARES GLOBAL CLEAN ENERGY UCITS ETF,IE00B1XNHC34,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-4.00,EUR,,1d2e7a85-cf67-4325-82c7-d5994a0afb98 23-12-2020,10:03,23-12-2020,ISHARES GLOBAL CLEAN ENERGY UCITS ETF,IE00B1XNHC34,Verkoop 120 @ 1.202 GBX,,GBP,1442.40,GBP,,1d2e7a85-cf67-4325-82c7-d5994a0afb98 Another example is below. It has two sells and one tx record. 11-07-2024,16:46,11-07-2024,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,"Verkoop 16 @ 124,28 EUR",,EUR,1988.48,EUR,,56e1f16b-4142-4373-851d-45d0505fe12c 11-07-2024,16:43,11-07-2024,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-1.00,EUR,,56e1f16b-4142-4373-851d-45d0505fe12c 11-07-2024,16:43,11-07-2024,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,"Verkoop 8 @ 124,28 EUR",,EUR,994.24,EUR,,56e1f16b-4142-4373-851d-45d0505fe12c Or this 24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-0.26,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-2.00,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-0.02,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,"Verkoop 14 @ 61,7 EUR",,EUR,863.80,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,"Verkoop 1 @ 61,7 EUR",,EUR,61.70,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 --- src/converters/degiroConverterV3.ts | 152 +++++++++++++++------------- 1 file changed, 80 insertions(+), 72 deletions(-) diff --git a/src/converters/degiroConverterV3.ts b/src/converters/degiroConverterV3.ts index e7be66ad..d756a10d 100644 --- a/src/converters/degiroConverterV3.ts +++ b/src/converters/degiroConverterV3.ts @@ -130,33 +130,61 @@ export class DeGiroConverterV3 extends AbstractConverter { continue; } - // Look ahead in the remaining records if there is one with the samen orderId. - let matchingRecord = this.findMatchByOrderId(record, records.slice(idx + 1)); + // Look ahead in the remaining records if there are some with the same orderId. + let matchingRecords = []; + if (record.orderId) { + matchingRecords = this.findMatchesByOrderId(record, records.slice(idx + 1)); + } - // If there was no match by orderId, and there was no orderId present on the current record, look ahead in the remaining records to find a match by ISIN + Product. - if (!matchingRecord && !record.orderId) { - matchingRecord = this.findMatchByIsin(record, records.slice(idx + 1)); + // If there was no match by orderId, and there was no orderId present on the current record, look ahead one record to find a match by ISIN + Product + Date. + if (matchingRecords.length == 0 && !record.orderId) { + matchingRecords = this.findMatchByIsinSameDateTime(record, records.slice(idx + 1, idx + 2)); } - // If it's a standalone record, add it immediately. - if (!matchingRecord) { + if (matchingRecords.length > 0) { + // Filter out ignored records + matchingRecords = matchingRecords.filter(r => !this.isIgnoredRecord(r)); - if (this.isBuyOrSellRecord(record)) { - result.activities.push(this.mapRecordToActivity(record, security)); - } - else { - result.activities.push(this.mapDividendRecord(record, null, security)); - } + // Register records as processed so they are skipped on next iteration(s) + matchingRecords.forEach(r => processedRecords.add(this.hashRecord(r))); } - else { - // This is a pair of records. Check which type of record it is and then combine the records into a Ghostfolio activity. + if (matchingRecords.length > 0) { + // This is a set of records. Check which type of record it is and then combine the records into a Ghostfolio activity. + + // Get main and fees records + matchingRecords.unshift(record); + const mainRecords = matchingRecords.filter(r => this.isBuyOrSellRecord(r) || this.isDividendRecord(r)); + let transactionFeeRecords = matchingRecords.filter(r => this.isTransactionFeeRecord(r)); + + if (mainRecords.length == 0 || matchingRecords.length != mainRecords.length + transactionFeeRecords.length) { + this.progress.log(`[i] Unexpected set of ${matchingRecords.length} records (see below)! Please add them manually..\n`); + matchingRecords.forEach(r => this.progress.log(`Record ${r.isin || r.product} from ${r.date} with ${r.amount}${r.currency}`)); + bar1.increment(); + continue; + } - // Check wether it is a buy/sell record set. - if (this.isBuyOrSellRecordSet(record, matchingRecord)) { - result.activities.push(this.combineRecords(record, matchingRecord, security)); + // if there is one main record, report it combined with transaction fee(s). All other goes without TX fee. + mainRecords.forEach(r => { + if (this.isBuyOrSellRecord(r)) { + result.activities.push(this.combineRecords(r, transactionFeeRecords, security)); + } else { + result.activities.push(this.mapDividendRecord(r, transactionFeeRecords, security)); + } + + transactionFeeRecords = []; // nullify tx after first iteration + }); + } else { + // If it's a standalone record, add it immediately. + if (this.isBuyOrSellRecord(record)) { + result.activities.push(this.mapRecordToActivity(record, security)); + } else if (this.isDividendRecord(record)) { + result.activities.push(this.mapDividendRecord(record, [], security)); + } else if (this.isTransactionFeeRecord(record)) { + // sometimes there are stantalone transaction records, report them as platform fees + result.activities.push(this.mapPlatformFeeRecord(record)); } else { - result.activities.push(this.mapDividendRecord(record, matchingRecord, security)); + this.progress.log(`[i] Unknown standalone record ${record.isin || record.product} from ${record.date} with ${record.amount}${record.currency}! Please add this manually..\n`); } } @@ -241,12 +269,12 @@ export class DeGiroConverterV3 extends AbstractConverter { return ignoredRecordTypes.some((t) => record.description.toLocaleLowerCase().indexOf(t) > -1); } - private findMatchByOrderId(currentRecord: DeGiroRecord, records: DeGiroRecord[]): DeGiroRecord | undefined { - return records.find(r => r.orderId === currentRecord.orderId); + private findMatchesByOrderId(currentRecord: DeGiroRecord, records: DeGiroRecord[]): DeGiroRecord[] | undefined { + return records.filter(r => r.orderId === currentRecord.orderId); } - private findMatchByIsin(currentRecord: DeGiroRecord, records: DeGiroRecord[]): DeGiroRecord | undefined { - return records.find(r => r.isin === currentRecord.isin && r.product === currentRecord.product); + private findMatchByIsinSameDateTime(currentRecord: DeGiroRecord, records: DeGiroRecord[]): DeGiroRecord[] | undefined { + return records.filter(r => r.isin === currentRecord.isin && r.product === currentRecord.product && r.date == currentRecord.date); } private mapRecordToActivity(record: DeGiroRecord, security?: YahooFinanceRecord, isTransactionFeeRecord: boolean = false): GhostfolioActivity { @@ -271,11 +299,14 @@ export class DeGiroConverterV3 extends AbstractConverter { } else { orderType = GhostfolioOrderType.sell; } - } - else { + } else { // Otherwise, get the transaction fee info. - feeAmount = parseFloat(Math.abs(parseFloat(record.amount.replace(",", "."))).toFixed(3)); + const amount = parseFloat(record.amount.replace(",", ".")); + feeAmount = parseFloat(Math.abs(amount).toFixed(3)); + orderType = amount < 0 ? GhostfolioOrderType.sell : GhostfolioOrderType.buy; + numberShares = 1; + unitPrice = 0; } const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); @@ -295,56 +326,28 @@ export class DeGiroConverterV3 extends AbstractConverter { }; } - private combineRecords(currentRecord: DeGiroRecord, nextRecord: DeGiroRecord, security: YahooFinanceRecord): GhostfolioActivity { - - // Set the default values for the records. - let actionRecord = currentRecord; - let txFeeRecord: DeGiroRecord | null = nextRecord; - - // Determine which of the two records is the action record (e.g. buy/sell) and which contains the transaction fees. - // Firstly, check if the current record is the TxFee record. - if (this.isTransactionFeeRecord(currentRecord, true)) { - actionRecord = nextRecord; - txFeeRecord = currentRecord; - } - + private combineRecords(mainRecord: DeGiroRecord, transactionFeeRecords: DeGiroRecord[], security: YahooFinanceRecord): GhostfolioActivity { // Map both records. - const mappedActionRecord = this.mapRecordToActivity(actionRecord, security); - const mappedTxFeeRecord = this.mapRecordToActivity(txFeeRecord, security, true); + const mappedActionRecord = this.mapRecordToActivity(mainRecord, security); + const mappedTxFeeRecords = transactionFeeRecords.map(r => this.mapRecordToActivity(r, security, true)); // Extract the fee from the transaction fee record and put it in the action record. - mappedActionRecord.fee = mappedTxFeeRecord.fee; + const fee = mappedTxFeeRecords.reduce((sum, r) => sum + r.fee, 0); + mappedActionRecord.fee = parseFloat(fee.toFixed(3)); return mappedActionRecord; } - private mapDividendRecord(currentRecord: DeGiroRecord, nextRecord: DeGiroRecord | null = null, security: YahooFinanceRecord): GhostfolioActivity { - - // It's a dividend set. - // Set the default values for the records. - let dividendRecord = currentRecord; - let txFeeRecord: DeGiroRecord = nextRecord; - - // Determine which of the two records is the dividend record and which contains the transaction fees. - // Firstly, check if the current record is the TxFee record. - if (nextRecord && this.isTransactionFeeRecord(currentRecord, false)) { - dividendRecord = nextRecord; - txFeeRecord = currentRecord; - } - - let unitPrice = Math.abs(parseFloat(dividendRecord.amount.replace(",", "."))); - let fees = 0; - if (txFeeRecord) { - fees = Math.abs(parseFloat(txFeeRecord.amount.replace(",", "."))); - } - + private mapDividendRecord(dividendRecord: DeGiroRecord, transactionFeeRecords: DeGiroRecord[], security: YahooFinanceRecord): GhostfolioActivity { + const unitPrice = Math.abs(parseFloat(dividendRecord.amount.replace(",", "."))); + const feeAmount = transactionFeeRecords.reduce((sum, r) => sum + Math.abs(parseFloat(r.amount.replace(",", "."))), 0); const date = dayjs(`${dividendRecord.date} ${dividendRecord.time}:00`, "DD-MM-YYYY HH:mm"); // Create the record. return { accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, - comment: `Dividend ${dividendRecord.isin} @ ${currentRecord.date}T${currentRecord.time}`, - fee: fees, + comment: `Dividend ${dividendRecord.isin} @ ${dividendRecord.date}T${dividendRecord.time}`, + fee: feeAmount, quantity: 1, type: GhostfolioOrderType.dividend, unitPrice: unitPrice, @@ -355,11 +358,6 @@ export class DeGiroConverterV3 extends AbstractConverter { }; } - private isBuyOrSellRecordSet(currentRecord: DeGiroRecord, nextRecord: DeGiroRecord): boolean { - return (this.isBuyOrSellRecord(currentRecord) && this.isTransactionFeeRecord(nextRecord, true)) || - (this.isTransactionFeeRecord(currentRecord, true) && this.isBuyOrSellRecord(nextRecord)) - } - private mapPlatformFeeRecord(record: DeGiroRecord): GhostfolioActivity { const feeAmount = Math.abs(parseFloat(record.amount.replace(",", "."))); const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); @@ -405,14 +403,24 @@ export class DeGiroConverterV3 extends AbstractConverter { return buySellRecordType.some((t) => record.description.toLocaleLowerCase().indexOf(t) > -1); } - private isTransactionFeeRecord(record: DeGiroRecord, isBuyOrSellTransactionFeeRecord: boolean): boolean { + private isDividendRecord(record: DeGiroRecord): boolean { if (!record) { return false; } - // When a dividend transaction must be found, there should not be an orderid. - if (!isBuyOrSellTransactionFeeRecord && record.orderId) { + if (this.isTransactionFeeRecord(record)) { + // dividend tax records often has 'Impôts sur dividende' or 'dividendbelasting' + // which make them match the condition below. + return false; + } + + return record.description.toLocaleLowerCase().indexOf("dividend") > -1 || record.description.toLocaleLowerCase().indexOf("capital return") > -1; + } + + private isTransactionFeeRecord(record: DeGiroRecord): boolean { + + if (!record) { return false; } From 7f06480ed9ae39f11a8185ba4c121a539a5ba86c Mon Sep 17 00:00:00 2001 From: Ivan Kruglov Date: Sat, 4 Jan 2025 19:26:55 +0100 Subject: [PATCH 05/14] degiroConverterV3: cast records to DeGiroRecord objects --- src/converters/degiroConverterV3.ts | 7 +++++-- src/models/degiroRecord.ts | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/converters/degiroConverterV3.ts b/src/converters/degiroConverterV3.ts index d756a10d..6a70404d 100644 --- a/src/converters/degiroConverterV3.ts +++ b/src/converters/degiroConverterV3.ts @@ -38,10 +38,10 @@ export class DeGiroConverterV3 extends AbstractConverter { return columnValue; } - }, async (err, records: DeGiroRecord[]) => { + }, async (err, plainRecords: DeGiroRecord[]) => { // Check if parsing failed.. - if (err || records === undefined || records.length === 0) { + if (err || plainRecords === undefined || plainRecords.length === 0) { let errorMsg = "An error ocurred while parsing!"; if (err) { @@ -67,6 +67,9 @@ export class DeGiroConverterV3 extends AbstractConverter { activities: [] }; + // Map plain objects to DeGiroRecord instances + const records = plainRecords.map(record => DeGiroRecord.fromPlainObject(record)); + // Populate the progress bar. const bar1 = this.progress.create(records.length, 0); diff --git a/src/models/degiroRecord.ts b/src/models/degiroRecord.ts index dfe5d21b..8f301327 100644 --- a/src/models/degiroRecord.ts +++ b/src/models/degiroRecord.ts @@ -11,4 +11,8 @@ export class DeGiroRecord { balance_currency: string; // not used, but improtant for hashing balance: string; //// not used, but improtant for hashing orderId: string; + + static fromPlainObject(obj: any): DeGiroRecord { + return Object.assign(new DeGiroRecord(), obj); + } } From 1942b6e74d042b700c44dce4c49462356ce448f4 Mon Sep 17 00:00:00 2001 From: Ivan Kruglov Date: Sat, 4 Jan 2025 19:32:27 +0100 Subject: [PATCH 06/14] DeGiroRecord: getAmount() & getAbsoluteAmount() --- src/converters/degiroConverterV3.ts | 12 ++++++------ src/models/degiroRecord.ts | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/converters/degiroConverterV3.ts b/src/converters/degiroConverterV3.ts index 6a70404d..e654d2c5 100644 --- a/src/converters/degiroConverterV3.ts +++ b/src/converters/degiroConverterV3.ts @@ -293,7 +293,7 @@ export class DeGiroConverterV3 extends AbstractConverter { numberShares = parseFloat(numberSharesFromDescription); // For buy/sale records, only the total amount is recorded. So the unit price needs to be calculated. - const totalAmount = parseFloat(record.amount.replace(",", ".")); + const totalAmount = record.getAmount(); unitPrice = parseFloat((Math.abs(totalAmount) / numberShares).toFixed(3)); // If amount is negative (so money has been removed) or it's stock dividend (so free shares), thus it's a buy record. @@ -305,7 +305,7 @@ export class DeGiroConverterV3 extends AbstractConverter { } else { // Otherwise, get the transaction fee info. - const amount = parseFloat(record.amount.replace(",", ".")); + const amount = record.getAmount(); feeAmount = parseFloat(Math.abs(amount).toFixed(3)); orderType = amount < 0 ? GhostfolioOrderType.sell : GhostfolioOrderType.buy; numberShares = 1; @@ -342,8 +342,8 @@ export class DeGiroConverterV3 extends AbstractConverter { } private mapDividendRecord(dividendRecord: DeGiroRecord, transactionFeeRecords: DeGiroRecord[], security: YahooFinanceRecord): GhostfolioActivity { - const unitPrice = Math.abs(parseFloat(dividendRecord.amount.replace(",", "."))); - const feeAmount = transactionFeeRecords.reduce((sum, r) => sum + Math.abs(parseFloat(r.amount.replace(",", "."))), 0); + const unitPrice = dividendRecord.getAbsoluteAmount(); + const feeAmount = transactionFeeRecords.reduce((sum, r) => sum + r.getAbsoluteAmount(), 0); const date = dayjs(`${dividendRecord.date} ${dividendRecord.time}:00`, "DD-MM-YYYY HH:mm"); // Create the record. @@ -362,7 +362,7 @@ export class DeGiroConverterV3 extends AbstractConverter { } private mapPlatformFeeRecord(record: DeGiroRecord): GhostfolioActivity { - const feeAmount = Math.abs(parseFloat(record.amount.replace(",", "."))); + const feeAmount = record.getAbsoluteAmount(); const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); return { accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, @@ -379,7 +379,7 @@ export class DeGiroConverterV3 extends AbstractConverter { } private mapInterestRecord(record: DeGiroRecord): GhostfolioActivity { - const interestAmount = Math.abs(parseFloat(record.amount.replace(",", "."))); + const interestAmount = record.getAbsoluteAmount(); const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); return { accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, diff --git a/src/models/degiroRecord.ts b/src/models/degiroRecord.ts index 8f301327..3c113f39 100644 --- a/src/models/degiroRecord.ts +++ b/src/models/degiroRecord.ts @@ -12,6 +12,9 @@ export class DeGiroRecord { balance: string; //// not used, but improtant for hashing orderId: string; + public getAmount(): number { return parseFloat(this.amount.replace(",", ".")); } + public getAbsoluteAmount(): number { return Math.abs(this.getAmount()); } + static fromPlainObject(obj: any): DeGiroRecord { return Object.assign(new DeGiroRecord(), obj); } From ea9bbe6cf9ac02f7700b406c6e2179357987ac0e Mon Sep 17 00:00:00 2001 From: Ivan Kruglov Date: Sat, 4 Jan 2025 19:40:56 +0100 Subject: [PATCH 07/14] degiroConverterV3: formatFloat() --- src/converters/degiroConverterV3.ts | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/converters/degiroConverterV3.ts b/src/converters/degiroConverterV3.ts index e654d2c5..20ec3865 100644 --- a/src/converters/degiroConverterV3.ts +++ b/src/converters/degiroConverterV3.ts @@ -294,7 +294,7 @@ export class DeGiroConverterV3 extends AbstractConverter { // For buy/sale records, only the total amount is recorded. So the unit price needs to be calculated. const totalAmount = record.getAmount(); - unitPrice = parseFloat((Math.abs(totalAmount) / numberShares).toFixed(3)); + unitPrice = Math.abs(totalAmount) / numberShares; // If amount is negative (so money has been removed) or it's stock dividend (so free shares), thus it's a buy record. if (totalAmount < 0 || record.description.toLocaleLowerCase().indexOf("stock dividend") > -1) { @@ -305,9 +305,8 @@ export class DeGiroConverterV3 extends AbstractConverter { } else { // Otherwise, get the transaction fee info. - const amount = record.getAmount(); - feeAmount = parseFloat(Math.abs(amount).toFixed(3)); - orderType = amount < 0 ? GhostfolioOrderType.sell : GhostfolioOrderType.buy; + feeAmount = record.getAbsoluteAmount(); + orderType = record.getAmount() < 0 ? GhostfolioOrderType.sell : GhostfolioOrderType.buy; numberShares = 1; unitPrice = 0; } @@ -318,10 +317,10 @@ export class DeGiroConverterV3 extends AbstractConverter { return { accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, comment: record.orderId ?? `${orderType === GhostfolioOrderType.buy ? "Buy" : "Sell"} ${record.isin} @ ${record.date}T${record.time}`, - fee: feeAmount, + fee: this.formatFloat(feeAmount), quantity: numberShares, type: orderType, - unitPrice: unitPrice, + unitPrice: this.formatFloat(unitPrice), currency: record.currency ?? "", dataSource: "YAHOO", date: date.format("YYYY-MM-DDTHH:mm:ssZ"), @@ -335,8 +334,7 @@ export class DeGiroConverterV3 extends AbstractConverter { const mappedTxFeeRecords = transactionFeeRecords.map(r => this.mapRecordToActivity(r, security, true)); // Extract the fee from the transaction fee record and put it in the action record. - const fee = mappedTxFeeRecords.reduce((sum, r) => sum + r.fee, 0); - mappedActionRecord.fee = parseFloat(fee.toFixed(3)); + mappedActionRecord.fee = mappedTxFeeRecords.reduce((sum, r) => sum + r.fee, 0); return mappedActionRecord; } @@ -350,10 +348,10 @@ export class DeGiroConverterV3 extends AbstractConverter { return { accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, comment: `Dividend ${dividendRecord.isin} @ ${dividendRecord.date}T${dividendRecord.time}`, - fee: feeAmount, + fee: this.formatFloat(feeAmount), quantity: 1, type: GhostfolioOrderType.dividend, - unitPrice: unitPrice, + unitPrice: this.formatFloat(unitPrice), currency: dividendRecord.currency, dataSource: "YAHOO", date: date.format("YYYY-MM-DDTHH:mm:ssZ"), @@ -367,7 +365,7 @@ export class DeGiroConverterV3 extends AbstractConverter { return { accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, comment: "", - fee: feeAmount, + fee: this.formatFloat(feeAmount), quantity: 1, type: GhostfolioOrderType.fee, unitPrice: 0, @@ -387,7 +385,7 @@ export class DeGiroConverterV3 extends AbstractConverter { fee: 0, quantity: 1, type: GhostfolioOrderType.interest, - unitPrice: interestAmount, + unitPrice: this.formatFloat(interestAmount), currency: record.currency, dataSource: "MANUAL", date: date.format("YYYY-MM-DDTHH:mm:ssZ"), @@ -462,4 +460,8 @@ export class DeGiroConverterV3 extends AbstractConverter { md5.update(record.orderId); return md5.digest('hex'); } + + private formatFloat(val: number): number { + return parseFloat(val.toFixed(3)); + } } From 4441cccbb6b0df2013abbf330bf7bba1ae391852 Mon Sep 17 00:00:00 2001 From: Ivan Kruglov Date: Sat, 4 Jan 2025 20:17:09 +0100 Subject: [PATCH 08/14] degiroConverterV3: recover simple broken CSV records --- src/converters/degiroConverterV3.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/converters/degiroConverterV3.ts b/src/converters/degiroConverterV3.ts index 20ec3865..8ca23110 100644 --- a/src/converters/degiroConverterV3.ts +++ b/src/converters/degiroConverterV3.ts @@ -22,6 +22,10 @@ export class DeGiroConverterV3 extends AbstractConverter { * @inheritdoc */ public processFileContents(input: string, successCallback: any, errorCallback: any): void { + // my exported file contained many broken records like this. It's easy to recover them. So, why not?! + // 13-04-2016,09:00,13-04-2016,ABN AMRO BANK NV,NL0011540547,"Koop 10 @ 18,3 EUR",,EUR,-183.00,EUR,,df134e52-2753-4694- + // ,,,,,,,,,,,947b-418f08d4a352 + input = input.replace(/(,[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-)\n,{11}([a-f0-9]{4}-[a-f0-9]{12})$/mg, '$1$2'); // Parse the CSV and convert to Ghostfolio import format. parse(input, { From 1e66d27086f97e9811d7066ad052fc515705f53d Mon Sep 17 00:00:00 2001 From: Ivan Kruglov Date: Thu, 9 Jan 2025 23:56:19 +0100 Subject: [PATCH 09/14] degiroConverterV3: don't combine records with different currencies --- src/converters/degiroConverterV3.ts | 198 ++++++++++++++++------------ 1 file changed, 116 insertions(+), 82 deletions(-) diff --git a/src/converters/degiroConverterV3.ts b/src/converters/degiroConverterV3.ts index 8ca23110..d177a3fd 100644 --- a/src/converters/degiroConverterV3.ts +++ b/src/converters/degiroConverterV3.ts @@ -148,52 +148,61 @@ export class DeGiroConverterV3 extends AbstractConverter { matchingRecords = this.findMatchByIsinSameDateTime(record, records.slice(idx + 1, idx + 2)); } - if (matchingRecords.length > 0) { - // Filter out ignored records - matchingRecords = matchingRecords.filter(r => !this.isIgnoredRecord(r)); + // Filter out ignored records + matchingRecords = matchingRecords.filter(r => !this.isIgnoredRecord(r)); - // Register records as processed so they are skipped on next iteration(s) - matchingRecords.forEach(r => processedRecords.add(this.hashRecord(r))); + // Register records as processed so they are skipped on next iteration(s) + matchingRecords.forEach(r => processedRecords.add(this.hashRecord(r))); + + if (matchingRecords.length == 0) { + // If it's a standalone record, add it immediately. + const mappedRecord = this.mapStandaloneRecord(record, security); + if (mappedRecord) result.activities.push(mappedRecord); + + bar1.increment(1); + continue; + } + + // Now, need to check that all records uses the same currency. + // If not, we have to report them separately even though they are part of one operation in DeGiro + const sameCurrency = matchingRecords.every(r => r.currency == record.currency); + if (!sameCurrency) { + matchingRecords.unshift(record); // combine records to work with them as one whole + matchingRecords.forEach(r => { + const mr = this.mapStandaloneRecord(r, security); + if (mr) result.activities.push(mr); + }) + + bar1.increment(1); + continue; } - if (matchingRecords.length > 0) { - // This is a set of records. Check which type of record it is and then combine the records into a Ghostfolio activity. + // Okay, we're dealing with a set of records of the same currency. + // We can combined them and report as one (or multiple records). + matchingRecords.unshift(record); // combine records to work with them as one whole - // Get main and fees records - matchingRecords.unshift(record); - const mainRecords = matchingRecords.filter(r => this.isBuyOrSellRecord(r) || this.isDividendRecord(r)); - let transactionFeeRecords = matchingRecords.filter(r => this.isTransactionFeeRecord(r)); + // Get main and fees records + const mainRecords = matchingRecords.filter(r => this.isBuyOrSellRecord(r) || this.isDividendRecord(r)); + let transactionFeeRecords = matchingRecords.filter(r => this.isTransactionFeeRecord(r)); - if (mainRecords.length == 0 || matchingRecords.length != mainRecords.length + transactionFeeRecords.length) { - this.progress.log(`[i] Unexpected set of ${matchingRecords.length} records (see below)! Please add them manually..\n`); - matchingRecords.forEach(r => this.progress.log(`Record ${r.isin || r.product} from ${r.date} with ${r.amount}${r.currency}`)); - bar1.increment(); - continue; - } + if (mainRecords.length == 0 || matchingRecords.length != mainRecords.length + transactionFeeRecords.length) { + this.progress.log(`[i] Unexpected set of ${matchingRecords.length} records (see below)! Please add them manually..\n`); + matchingRecords.forEach(r => this.progress.log(`Record ${r.isin || r.product} from ${r.date} with ${r.amount}${r.currency}`)); - // if there is one main record, report it combined with transaction fee(s). All other goes without TX fee. - mainRecords.forEach(r => { - if (this.isBuyOrSellRecord(r)) { - result.activities.push(this.combineRecords(r, transactionFeeRecords, security)); - } else { - result.activities.push(this.mapDividendRecord(r, transactionFeeRecords, security)); - } - - transactionFeeRecords = []; // nullify tx after first iteration - }); - } else { - // If it's a standalone record, add it immediately. - if (this.isBuyOrSellRecord(record)) { - result.activities.push(this.mapRecordToActivity(record, security)); - } else if (this.isDividendRecord(record)) { - result.activities.push(this.mapDividendRecord(record, [], security)); - } else if (this.isTransactionFeeRecord(record)) { - // sometimes there are stantalone transaction records, report them as platform fees - result.activities.push(this.mapPlatformFeeRecord(record)); + bar1.increment(); + continue; + } + + // if there is one main record, report it combined with transaction fee(s). All other goes without TX fee. + mainRecords.forEach(r => { + if (this.isBuyOrSellRecord(r)) { + result.activities.push(this.mapBuySellRecord(r, transactionFeeRecords, security)); } else { - this.progress.log(`[i] Unknown standalone record ${record.isin || record.product} from ${record.date} with ${record.amount}${record.currency}! Please add this manually..\n`); + result.activities.push(this.mapDividendRecord(r, transactionFeeRecords, security)); } - } + + transactionFeeRecords = []; // nullify tx after first iteration + }); bar1.increment(); } @@ -284,45 +293,64 @@ export class DeGiroConverterV3 extends AbstractConverter { return records.filter(r => r.isin === currentRecord.isin && r.product === currentRecord.product && r.date == currentRecord.date); } - private mapRecordToActivity(record: DeGiroRecord, security?: YahooFinanceRecord, isTransactionFeeRecord: boolean = false): GhostfolioActivity { - - let numberShares, unitPrice, feeAmount = 0; - let orderType; - - // If it is not a transaction fee record, get data from the record. - if (!isTransactionFeeRecord) { + private mapTransactionFeeRecord(record: DeGiroRecord, security?: YahooFinanceRecord): GhostfolioActivity { + const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); + const feeAmount = record.getAbsoluteAmount(); - // Get the amount of shares from the description. - const numberSharesFromDescription = record.description.match(/([\d*\.?\,?\d*]+)/)[0]; - numberShares = parseFloat(numberSharesFromDescription); + return { + accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, + comment: record.orderId ?? `Transaction fee ${record.isin} @ ${record.date}T${record.time}`, + fee: this.formatFloat(feeAmount), + quantity: 1, + type: GhostfolioOrderType.fee, + unitPrice: 0, + currency: record.currency ?? "", + dataSource: "YAHOO", + date: date.format("YYYY-MM-DDTHH:mm:ssZ"), + symbol: security.symbol, + }; + } - // For buy/sale records, only the total amount is recorded. So the unit price needs to be calculated. - const totalAmount = record.getAmount(); - unitPrice = Math.abs(totalAmount) / numberShares; + private mapBuySellRecord(record: DeGiroRecord, transactionFeeRecords: DeGiroRecord[], security?: YahooFinanceRecord): GhostfolioActivity { + // !IMPORTANT It's assumed that all records (record + transactionFeeRecords) have same currency - // If amount is negative (so money has been removed) or it's stock dividend (so free shares), thus it's a buy record. - if (totalAmount < 0 || record.description.toLocaleLowerCase().indexOf("stock dividend") > -1) { - orderType = GhostfolioOrderType.buy; - } else { - orderType = GhostfolioOrderType.sell; - } - } else { + const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); - // Otherwise, get the transaction fee info. - feeAmount = record.getAbsoluteAmount(); - orderType = record.getAmount() < 0 ? GhostfolioOrderType.sell : GhostfolioOrderType.buy; - numberShares = 1; - unitPrice = 0; - } + /* Get the amount and unit price of shares from the description. + * For buy/sale records, only the total amount is recorded. So the unit price needs to be calculated. + * However, in the modern days degiro csv both quantity and price-per-unit are reported. + * Ex: Verkoop 1 @ 16,78 USD + * Ex: Koop 475 @ 583,5 GBX + * + * At the same time, we can't use the unit price because it may come with a different "currency". + * "Currency" is in quotes because it's the case between GBP and GBp/GBX. 1 GBP = 100 * GBp/GBX. + * So, it's not really a difference currency, but rather a different denomination. + * Yet, it makes it necessary to do the dance with dividing total amount by num of shares. + * Ex: + * 03-03-2020,11:14,03-03-2020,ISHARES GLOBAL CLEAN ENERGY UCITS ETF,IE00B1XNHC34,"Koop 475 @ 583,5 GBX",,GBP,-2771.63,GBP,,3b000105-xxxx-xxxx-xxxx-xxxxxxxxxxxx + * + * Another note: the old code contained regexp: + * record.description.match(/([\d*\.?\,?\d*]+)/)[0] + * + * This regexp seems broken to me. Confirmed by ChatGPT. + * I have no idea how it managed to work. + */ + + const num = record.description.match(/([0-9]+[.,]?[0-9]*)/)[0]; + const quantity = parseFloat(num); + const unitPrice = record.getAbsoluteAmount() / quantity; + + // If amount is negative (so money has been removed) + const orderType = record.getAmount() < 0 ? GhostfolioOrderType.buy : GhostfolioOrderType.sell; - const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); + // Extract the fee from the transaction fee record and put it in the action record. + const feeAmount = transactionFeeRecords.reduce((sum, r) => sum + r.getAbsoluteAmount(), 0); - // Create the record. return { accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, comment: record.orderId ?? `${orderType === GhostfolioOrderType.buy ? "Buy" : "Sell"} ${record.isin} @ ${record.date}T${record.time}`, fee: this.formatFloat(feeAmount), - quantity: numberShares, + quantity: this.formatFloat(quantity), type: orderType, unitPrice: this.formatFloat(unitPrice), currency: record.currency ?? "", @@ -332,37 +360,43 @@ export class DeGiroConverterV3 extends AbstractConverter { }; } - private combineRecords(mainRecord: DeGiroRecord, transactionFeeRecords: DeGiroRecord[], security: YahooFinanceRecord): GhostfolioActivity { - // Map both records. - const mappedActionRecord = this.mapRecordToActivity(mainRecord, security); - const mappedTxFeeRecords = transactionFeeRecords.map(r => this.mapRecordToActivity(r, security, true)); - - // Extract the fee from the transaction fee record and put it in the action record. - mappedActionRecord.fee = mappedTxFeeRecords.reduce((sum, r) => sum + r.fee, 0); + private mapDividendRecord(record: DeGiroRecord, transactionFeeRecords: DeGiroRecord[], security: YahooFinanceRecord): GhostfolioActivity { + // !IMPORTANT It's assumed that all records (record + transactionFeeRecords) have same currency - return mappedActionRecord; - } - - private mapDividendRecord(dividendRecord: DeGiroRecord, transactionFeeRecords: DeGiroRecord[], security: YahooFinanceRecord): GhostfolioActivity { - const unitPrice = dividendRecord.getAbsoluteAmount(); + const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); + const unitPrice = record.getAbsoluteAmount(); const feeAmount = transactionFeeRecords.reduce((sum, r) => sum + r.getAbsoluteAmount(), 0); - const date = dayjs(`${dividendRecord.date} ${dividendRecord.time}:00`, "DD-MM-YYYY HH:mm"); // Create the record. return { accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, - comment: `Dividend ${dividendRecord.isin} @ ${dividendRecord.date}T${dividendRecord.time}`, + comment: `Dividend ${record.isin} @ ${record.date}T${record.time}`, fee: this.formatFloat(feeAmount), quantity: 1, type: GhostfolioOrderType.dividend, unitPrice: this.formatFloat(unitPrice), - currency: dividendRecord.currency, + currency: record.currency ?? "", dataSource: "YAHOO", date: date.format("YYYY-MM-DDTHH:mm:ssZ"), symbol: security.symbol, }; } + private mapStandaloneRecord(record: DeGiroRecord, security?: YahooFinanceRecord): GhostfolioActivity { + if (this.isBuyOrSellRecord(record)) + return this.mapBuySellRecord(record, [], security); + + if (this.isDividendRecord(record)) + return this.mapDividendRecord(record, [], security); + + if (this.isTransactionFeeRecord(record)) + // sometimes there are standalone transaction fee records + return this.mapTransactionFeeRecord(record, security); + + this.progress.log(`[i] Unknown standalone record ${record.isin || record.product} from ${record.date} with ${record.amount}${record.currency}! Please add this manually..\n`); + return undefined; + } + private mapPlatformFeeRecord(record: DeGiroRecord): GhostfolioActivity { const feeAmount = record.getAbsoluteAmount(); const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); From 7621b02e4486955ba2f6044cd26c2c14cb3326ee Mon Sep 17 00:00:00 2001 From: Ivan Kruglov Date: Sat, 11 Jan 2025 16:55:05 +0100 Subject: [PATCH 10/14] degiroConverterV3: inverse processing order --- src/converters/degiroConverterV3.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/converters/degiroConverterV3.ts b/src/converters/degiroConverterV3.ts index d177a3fd..7fa9832b 100644 --- a/src/converters/degiroConverterV3.ts +++ b/src/converters/degiroConverterV3.ts @@ -71,6 +71,13 @@ export class DeGiroConverterV3 extends AbstractConverter { activities: [] }; + // Inverse records so they come in the chronological order. This is very important since + // selecion of security should happe based on buy/sell records, and not dividend/fee records. + // By default DeGiro's CSV file has the most recent records at the top. So, by inversing + // the order we process records in choronological order but keep internal releationships + // between records which we can break if, for instance, sort by date/time. + plainRecords.reverse(); + // Map plain objects to DeGiroRecord instances const records = plainRecords.map(record => DeGiroRecord.fromPlainObject(record)); From ce1eb9aaf73a365e393e752daf385670f547b2fa Mon Sep 17 00:00:00 2001 From: Ivan Kruglov Date: Sun, 19 Jan 2025 17:07:37 +0100 Subject: [PATCH 11/14] degiroConverterV3: generate unique transaction records --- src/converters/degiroConverterV3.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/converters/degiroConverterV3.ts b/src/converters/degiroConverterV3.ts index 7fa9832b..0c6ee1e1 100644 --- a/src/converters/degiroConverterV3.ts +++ b/src/converters/degiroConverterV3.ts @@ -306,15 +306,16 @@ export class DeGiroConverterV3 extends AbstractConverter { return { accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, - comment: record.orderId ?? `Transaction fee ${record.isin} @ ${record.date}T${record.time}`, + comment: record.orderId, fee: this.formatFloat(feeAmount), quantity: 1, type: GhostfolioOrderType.fee, unitPrice: 0, currency: record.currency ?? "", - dataSource: "YAHOO", + dataSource: "MANUAL", date: date.format("YYYY-MM-DDTHH:mm:ssZ"), - symbol: security.symbol, + // ghostfolio doesn't like two MANUAL records with same name, hence adding date & time. + symbol: `Transaction fee ${security.symbol} @ ${record.date}T${record.time}` }; } From 71e35b09cc793ade61b2d82a9271ddbbbd4da3c9 Mon Sep 17 00:00:00 2001 From: Ivan Kruglov Date: Sun, 19 Jan 2025 17:25:40 +0100 Subject: [PATCH 12/14] degiroConverterV3: GBP <-> GBp/GBX converstion --- src/converters/degiroConverterV3.ts | 47 ++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/src/converters/degiroConverterV3.ts b/src/converters/degiroConverterV3.ts index 0c6ee1e1..5d6b8084 100644 --- a/src/converters/degiroConverterV3.ts +++ b/src/converters/degiroConverterV3.ts @@ -304,14 +304,17 @@ export class DeGiroConverterV3 extends AbstractConverter { const date = dayjs(`${record.date} ${record.time}:00`, "DD-MM-YYYY HH:mm"); const feeAmount = record.getAbsoluteAmount(); + const currency = this.getCurrencyIfConvertable(record.currency, security.currency) + const convertedFeeAmount = this.convertCurrencyIfConvertable(feeAmount, record.currency, security.currency); + return { accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, comment: record.orderId, - fee: this.formatFloat(feeAmount), + fee: this.formatFloat(convertedFeeAmount), quantity: 1, type: GhostfolioOrderType.fee, unitPrice: 0, - currency: record.currency ?? "", + currency: currency, dataSource: "MANUAL", date: date.format("YYYY-MM-DDTHH:mm:ssZ"), // ghostfolio doesn't like two MANUAL records with same name, hence adding date & time. @@ -354,14 +357,18 @@ export class DeGiroConverterV3 extends AbstractConverter { // Extract the fee from the transaction fee record and put it in the action record. const feeAmount = transactionFeeRecords.reduce((sum, r) => sum + r.getAbsoluteAmount(), 0); + const currency = this.getCurrencyIfConvertable(record.currency, security.currency) + const convertedFeeAmount = this.convertCurrencyIfConvertable(feeAmount, record.currency, security.currency); + const convertedUnitPrice = this.convertCurrencyIfConvertable(unitPrice, record.currency, security.currency); + return { accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, comment: record.orderId ?? `${orderType === GhostfolioOrderType.buy ? "Buy" : "Sell"} ${record.isin} @ ${record.date}T${record.time}`, - fee: this.formatFloat(feeAmount), + fee: this.formatFloat(convertedFeeAmount), quantity: this.formatFloat(quantity), type: orderType, - unitPrice: this.formatFloat(unitPrice), - currency: record.currency ?? "", + unitPrice: this.formatFloat(convertedUnitPrice), + currency: currency, dataSource: "YAHOO", date: date.format("YYYY-MM-DDTHH:mm:ssZ"), symbol: security.symbol ?? "", @@ -375,15 +382,19 @@ export class DeGiroConverterV3 extends AbstractConverter { const unitPrice = record.getAbsoluteAmount(); const feeAmount = transactionFeeRecords.reduce((sum, r) => sum + r.getAbsoluteAmount(), 0); + const currency = this.getCurrencyIfConvertable(record.currency, security.currency) + const convertedFeeAmount = this.convertCurrencyIfConvertable(feeAmount, record.currency, security.currency); + const convertedUnitPrice = this.convertCurrencyIfConvertable(unitPrice, record.currency, security.currency); + // Create the record. return { accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, comment: `Dividend ${record.isin} @ ${record.date}T${record.time}`, - fee: this.formatFloat(feeAmount), + fee: this.formatFloat(convertedFeeAmount), quantity: 1, type: GhostfolioOrderType.dividend, - unitPrice: this.formatFloat(unitPrice), - currency: record.currency ?? "", + unitPrice: this.formatFloat(convertedUnitPrice), + currency: currency, dataSource: "YAHOO", date: date.format("YYYY-MM-DDTHH:mm:ssZ"), symbol: security.symbol, @@ -510,4 +521,24 @@ export class DeGiroConverterV3 extends AbstractConverter { private formatFloat(val: number): number { return parseFloat(val.toFixed(3)); } + + private getCurrencyIfConvertable(from_currency: string, to_currency: string): string { + if (from_currency == "GBP" && (to_currency == "GBp" || to_currency == "GBX")) + return to_currency + + if ((from_currency == "GBp" || from_currency == "GBX") && to_currency == "GBP") + return to_currency; + + return from_currency; + } + + private convertCurrencyIfConvertable(val: number, from_currency: string, to_currency: string): number { + if (from_currency == "GBP" && (to_currency == "GBp" || to_currency == "GBX")) + return val * 100.0; + + if (from_currency == "GBp" && to_currency == "GBP") + return val / 100.0; + + return val; + } } From 8c3ca17b2a434c4d08e8906c6aaa329eaef8afc2 Mon Sep 17 00:00:00 2001 From: Ivan Kruglov Date: Sun, 19 Jan 2025 20:09:41 +0100 Subject: [PATCH 13/14] degiroConverterV3: ignore record with zero change and zero fee --- src/converters/degiroConverterV3.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/converters/degiroConverterV3.ts b/src/converters/degiroConverterV3.ts index 5d6b8084..7a4493a3 100644 --- a/src/converters/degiroConverterV3.ts +++ b/src/converters/degiroConverterV3.ts @@ -257,6 +257,16 @@ export class DeGiroConverterV3 extends AbstractConverter { return true; } + // Sometimes there are records like the following. They happen when an ETF changes ISIN. + // They have zero value and zero price. Let's ignore them + // 06-10-2022,10:13,06-10-2022,ETC ISSUANCE GMBH,DE000A3G01J0,WIJZIGING ISIN: Koop 12 @ 0 EUR,,EUR,0.00,EUR, + // 06-10-2022,08:16,14-09-2022,ETC ISSUANCE ETHETC - PHYSICAL,DE000A3G01J0,SPIN-OFF: Koop 12 @ 0 EUR,,EUR,0.00,EUR, + // 05-10-2022,15:43,14-09-2022,ETC ISSUANCE ETHETC - PHYSICAL,DE000A3G01J0,CLAIMEMISSIE: Verkoop 12 @ 0 EUR,,EUR,0.00,EUR, + // 05-10-2022,11:09,14-09-2022,ETC ISSUANCE ETHETC - PHYSICAL,DE000A3G01J0,CLAIMEMISSIE: Koop 12 @ 0 EUR,,EUR,0.00,EUR, + if (record.getAbsoluteAmount() == 0 && !record.fx) { + return true; + } + const ignoredRecordTypes = [ "ideal", "flatex", From 9a92844a17c341c05a8265625cfb324f4c489423 Mon Sep 17 00:00:00 2001 From: Ivan Kruglov Date: Sun, 5 Jan 2025 14:21:03 +0100 Subject: [PATCH 14/14] degiroConverterV3.test: add tests --- samples/degiro-export.csv | 40 ++++++++++++++++++++++++ src/converters/degiroConverterV3.test.ts | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/samples/degiro-export.csv b/samples/degiro-export.csv index b228e37c..82a15e8b 100644 --- a/samples/degiro-export.csv +++ b/samples/degiro-export.csv @@ -71,3 +71,43 @@ Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id 09-06-2020,12:05,08-06-2020,EUR CASH FUND FUNDSHARE,NL0010661914,"Conversion Fonds Monétaires finalisée: Vente 1 046,3825 @ 0,9854 EUR",,EUR,-4.81,EUR,84.89, 01-01-2025,06:46,31-12-2024,,,FX Credit,,EUR,12.28,EUR,217.01, 01-01-2025,06:46,31-12-2024,,,FX Debit,1.0379,USD,-12.75,USD,0.00, +13-09-2021,08:26,10-09-2021,MOODY'S CORP.,US6153691059,Dividend,,USD,0.62,USD,0.53, +13-09-2021,08:26,10-09-2021,MOODY'S CORP.,US6153691059,Dividendbelasting,,USD,-0.09,USD,-0.09, +10-09-2021,20:22,10-09-2021,MOODY'S CORP.,US6153691059,Valuta Debitering,1.1824,USD,-384.50,USD,0.00,80ff8574-14dd-46e1-bdf7-f11b8bbfa5c7 +10-09-2021,20:22,10-09-2021,MOODY'S CORP.,US6153691059,Valuta Creditering,,EUR,325.19,EUR,,80ff8574-14dd-46e1-bdf7-f11b8bbfa5c7 +10-09-2021,20:22,10-09-2021,MOODY'S CORP.,US6153691059,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-0.50,EUR,,80ff8574-14dd-46e1-bdf7-f11b8bbfa5c7 +10-09-2021,20:22,10-09-2021,MOODY'S CORP.,US6153691059,"Verkoop 1 @ 384,5 USD",,USD,384.50,USD,,80ff8574-14dd-46e1-bdf7-f11b8bbfa5c7 +16-12-2021,18:02,22-10-2021,MOODY'S CORP.,US6153691059,Dividendbelasting,,USD,-0.01,USD,0.00, +16-12-2021,18:02,22-10-2021,MORGAN STANLEY,US6174464486,Dividendbelasting,,USD,0.01,USD,0.01, +16-12-2021,18:02,22-10-2021,MOODY'S CORP.,US6153691059,Dividendbelasting,,USD,-0.01,USD,0.00, +16-12-2021,18:02,22-10-2021,MORGAN STANLEY,US6174464486,Dividendbelasting,,USD,0.01,USD,0.01, +23-12-2020,10:03,23-12-2020,ISHARES GLOBAL CLEAN ENERGY UCITS ETF,IE00B1XNHC34,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-0.79,EUR,,1d2e7a85-cf67-4325-82c7-d5994a0afb98 +23-12-2020,10:03,23-12-2020,ISHARES GLOBAL CLEAN ENERGY UCITS ETF,IE00B1XNHC34,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-4.00,EUR,,1d2e7a85-cf67-4325-82c7-d5994a0afb98 +23-12-2020,10:03,23-12-2020,ISHARES GLOBAL CLEAN ENERGY UCITS ETF,IE00B1XNHC34,Verkoop 120 @ 1.202 GBX,,GBP,1442.40,GBP,,1d2e7a85-cf67-4325-82c7-d5994a0afb98 +25-08-2022,10:21,25-08-2022,TESLA,US88160R1014,"STOCK SPLIT: Koop 3 @ 297,0967 USD",,USD,-891.29,USD,-0.01, +25-08-2022,10:21,25-08-2022,TESLA,US88160R1014,"STOCK SPLIT: Verkoop 1 @ 891,29 USD",,USD,891.29,USD,1.29, +11-07-2024,16:46,11-07-2024,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,"Verkoop 16 @ 124,28 EUR",,EUR,1988.48,EUR,,56e1f16b-4142-4373-851d-45d0505fe12c +11-07-2024,16:43,11-07-2024,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-1.00,EUR,,56e1f16b-4142-4373-851d-45d0505fe12c +11-07-2024,16:43,11-07-2024,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,"Verkoop 8 @ 124,28 EUR",,EUR,994.24,EUR,,56e1f16b-4142-4373-851d-45d0505fe12c +13-04-2016,09:00,13-04-2016,ABN AMRO BANK NV,NL0011540547,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-2.00,EUR,,df134e52-2753-4694-947b-418f08d4a352 +13-04-2016,09:00,13-04-2016,ABN AMRO BANK NV,NL0011540547,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-0.04,EUR,,df134e52-2753-4694-947b-418f08d4a352 +13-04-2016,09:00,13-04-2016,ABN AMRO BANK NV,NL0011540547,"Koop 10 @ 18,3 EUR",,EUR,-183.00,EUR,,df134e52-2753-4694- +,,,,,,,,,,,947b-418f08d4a352 +28-11-2023,15:30,28-11-2023,TESLA,US88160R1014,Valuta Debitering,1.1005,USD,-710.37,USD,0.00,81ac086b-8fab-4465- +,,,,,,,,,,,a458-118e34009c9e +28-11-2023,15:30,28-11-2023,TESLA,US88160R1014,Valuta Creditering,,EUR,645.47,EUR,672.29,81ac086b-8fab-4465-a458-118e34009c9e +28-11-2023,15:30,28-11-2023,TESLA,US88160R1014,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-2.00,EUR,26.82,81ac086b-8fab-4465-a458-118e34009c9e +28-11-2023,15:30,28-11-2023,TESLA,US88160R1014,"Verkoop 3 @ 236,79 USD",,USD,710.37,USD,710.37,81ac086b-8fab-4465-a458-118e34009c9e +24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-0.26,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 +24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-2.00,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 +24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-0.02,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 +24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,"Verkoop 14 @ 61,7 EUR",,EUR,863.80,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 +24-03-2020,13:10,24-03-2020,VANGUARD FTSE ALL-WORLD UCITS ETF,IE00B3RBWM25,"Verkoop 1 @ 61,7 EUR",,EUR,61.70,EUR,,810d6e1d-af8e-43a4-830e-0b85bf90c818 +30-12-2024,07:44,27-12-2024,ISHARES CHINA LARGE CAP UCITS ETF USD,IE00B02KXK85,Dividend,,USD,1.10,USD,15.30, +01-02-2022,09:56,01-02-2022,ISHARES CHINA LARGE CAP UCITS ETF USD,IE00B02KXK85,"Koop 2 @ 96,22 EUR",,EUR,-192.44,EUR,,b2a1b368-491d-4fad-81ba-460c7f6839cf +03-01-2022,08:21,31-12-2021,ISHARES CHINA LARGE CAP UCITS ETF USD,IE00B02KXK85,Dividend,,USD,1.22,USD,1.22, +01-06-2021,11:03,01-06-2021,ISHARES CHINA LARGE CAP UCITS ETF USD,IE00B02KXK85,"Koop 1 @ 111,11 EUR",,EUR,-111.11,EUR,,1ee86f48-4a0a-4fd2-9d32-98aee813217f +06-10-2022,10:13,06-10-2022,ETC ISSUANCE GMBH,DE000A3G01J0,WIJZIGING ISIN: Koop 12 @ 0 EUR,,EUR,0.00,EUR, +06-10-2022,08:16,14-09-2022,ETC ISSUANCE ETHETC - PHYSICAL,DE000A3G01J0,SPIN-OFF: Koop 12 @ 0 EUR,,EUR,0.00,EUR, +05-10-2022,15:43,14-09-2022,ETC ISSUANCE ETHETC - PHYSICAL,DE000A3G01J0,CLAIMEMISSIE: Verkoop 12 @ 0 EUR,,EUR,0.00,EUR, +05-10-2022,11:09,14-09-2022,ETC ISSUANCE ETHETC - PHYSICAL,DE000A3G01J0,CLAIMEMISSIE: Koop 12 @ 0 EUR,,EUR,0.00,EUR diff --git a/src/converters/degiroConverterV3.test.ts b/src/converters/degiroConverterV3.test.ts index 7c6f6fbe..c774e7ac 100644 --- a/src/converters/degiroConverterV3.test.ts +++ b/src/converters/degiroConverterV3.test.ts @@ -35,7 +35,7 @@ describe("degiroConverterV3", () => { // Assert expect(actualExport).toBeTruthy(); expect(actualExport.activities.length).toBeGreaterThan(0); - expect(actualExport.activities.length).toBe(26); + expect(actualExport.activities.length).toBe(43); done(); }, () => { done.fail("Should not have an error!"); });