Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🐛 Binary data handling fixes #2629

Merged
merged 25 commits into from
Jan 3, 2022
Merged

🐛 Binary data handling fixes #2629

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
f4c92c3
Update node airtable
ahsanv Jan 3, 2022
e394929
Update nodenextcloud
ahsanv Jan 3, 2022
34fa669
Update node spreadsheet
ahsanv Jan 3, 2022
1335f00
Update node cortex, dropbox, editImage nodes
ahsanv Jan 3, 2022
a222d74
Update node emailSend
ahsanv Jan 3, 2022
def8193
Update node ftp
ahsanv Jan 3, 2022
d9b1c8c
Update node googleDrive
ahsanv Jan 3, 2022
7122612
Update node googleDrive fix
ahsanv Jan 3, 2022
643b974
Update node youtube
ahsanv Jan 3, 2022
fcb3901
Update node htmlExtract
ahsanv Jan 3, 2022
cdcb18e
Update node linkedIn
ahsanv Jan 3, 2022
9d6cc0a
Update node mailgun
ahsanv Jan 3, 2022
d9c79bc
Update node matrix
ahsanv Jan 3, 2022
0fd2512
Update node pipedrive
ahsanv Jan 3, 2022
b5ae362
Update node readPdf
ahsanv Jan 3, 2022
f767210
Update node facebookGraphApi
ahsanv Jan 3, 2022
7c69476
Update node httpRequest
ahsanv Jan 3, 2022
403b6b0
Update node nocoDB
ahsanv Jan 3, 2022
a4b1e35
Update node httpRequest, respondToWebhook
ahsanv Jan 3, 2022
4469ce1
Update node signi4
ahsanv Jan 3, 2022
5641ef4
Update node slack
ahsanv Jan 3, 2022
4cccf0c
Update node zulip
ahsanv Jan 3, 2022
ecf3aff
cleanup
ahsanv Jan 3, 2022
c26c054
fix generic funcs
ahsanv Jan 3, 2022
910de23
:bug: Fix EditImage Node
janober Jan 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions packages/nodes-base/nodes/Airtable/GenericFunctions.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,11 +126,7 @@ export async function downloadRecordAttachments(this: IExecuteFunctions | IPollF
if (record.fields[fieldName] !== undefined) {
for (const [index, attachment] of (record.fields[fieldName] as IAttachment[]).entries()) {
const file = await apiRequest.call(this, 'GET', '', {}, {}, attachment.url, { json: false, encoding: null });
element.binary![`${fieldName}_${index}`] = {
data: Buffer.from(file).toString('base64'),
fileName: attachment.filename,
mimeType: attachment.type,
};
element.binary![`${fieldName}_${index}`] = await this.helpers.prepareBinaryData(Buffer.from(file), attachment.filename, attachment.type);
}
}
}
Expand Down
5 changes: 2 additions & 3 deletions packages/nodes-base/nodes/Cortex/Cortex.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';

Expand Down Expand Up @@ -242,7 +241,7 @@ export class Cortex implements INodeType {
throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`);
}

const fileBufferData = Buffer.from(item.binary[binaryPropertyName].data, BINARY_ENCODING);
const fileBufferData = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName);

const options = {
formData: {
Expand Down Expand Up @@ -425,7 +424,7 @@ export class Cortex implements INodeType {
throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`);
}

const fileBufferData = Buffer.from(item.binary[binaryPropertyName].data, BINARY_ENCODING);
const fileBufferData = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName);
const sha256 = createHash('sha256').update(fileBufferData).digest('hex');

(body.data as IDataObject).attachment = {
Expand Down
7 changes: 2 additions & 5 deletions packages/nodes-base/nodes/Dropbox/Dropbox.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';

import {
IDataObject,
Expand Down Expand Up @@ -859,7 +856,7 @@ export class Dropbox implements INodeType {
throw new NodeOperationError(this.getNode(), `No binary data property "${propertyNameUpload}" does not exists on item!`);
}

body = Buffer.from(item.binary[propertyNameUpload].data, BINARY_ENCODING);
body = await this.helpers.getBinaryDataBuffer(i, propertyNameUpload);
} else {
// Is text file
body = Buffer.from(this.getNodeParameter('fileContent', i) as string, 'utf8');
Expand Down
10 changes: 6 additions & 4 deletions packages/nodes-base/nodes/EditImage/EditImage.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1179,7 +1179,8 @@ export class EditImage implements INodeType {
throw new NodeOperationError(this.getNode(), `Item does not contain any binary data with the name "${dataPropertyName}".`);
}

gmInstance = gm(Buffer.from(item.binary![dataPropertyName as string].data, BINARY_ENCODING));
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, dataPropertyName);
gmInstance = gm(binaryDataBuffer);
gmInstance = gmInstance.background('transparent');
}

Expand Down Expand Up @@ -1218,7 +1219,8 @@ export class EditImage implements INodeType {

const { fd, path, cleanup } = await file();
cleanupFunctions.push(cleanup);
await fsWriteFileAsync(fd, Buffer.from(item.binary![operationData.dataPropertyNameComposite as string].data, BINARY_ENCODING));
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, operationData.dataPropertyNameComposite as string);
await fsWriteFileAsync(fd, binaryDataBuffer);

if (operations[0].operation === 'create') {
// It seems like if the image gets created newly we have to create a new gm instance
Expand Down Expand Up @@ -1349,14 +1351,14 @@ export class EditImage implements INodeType {

returnData.push(await (new Promise<INodeExecutionData>((resolve, reject) => {
gmInstance
.toBuffer((error: Error | null, buffer: Buffer) => {
.toBuffer(async (error: Error | null, buffer: Buffer) => {
cleanupFunctions.forEach(async cleanup => await cleanup());

if (error) {
return reject(error);
}

newItem.binary![dataPropertyName as string].data = buffer.toString(BINARY_ENCODING);
newItem.binary![dataPropertyName as string].data = (await this.helpers.prepareBinaryData(Buffer.from(buffer))).data;

return resolve(newItem);
});
Expand Down
7 changes: 2 additions & 5 deletions packages/nodes-base/nodes/EmailSend/EmailSend.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IDataObject,
INodeExecutionData,
Expand Down Expand Up @@ -198,7 +195,7 @@ export class EmailSend implements INodeType {
}
attachments.push({
filename: item.binary[propertyName].fileName || 'unknown',
content: Buffer.from(item.binary[propertyName].data, BINARY_ENCODING),
content: await this.helpers.getBinaryDataBuffer(itemIndex, propertyName),
});
}

Expand Down
8 changes: 3 additions & 5 deletions packages/nodes-base/nodes/Facebook/FacebookGraphApi.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IBinaryData,
IDataObject,
Expand Down Expand Up @@ -391,9 +388,10 @@ export class FacebookGraphApi implements INodeType {

const binaryProperty = item.binary[binaryPropertyName] as IBinaryData;

const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, binaryPropertyName);
requestOptions.formData = {
[propertyName]: {
value: Buffer.from(binaryProperty.data, BINARY_ENCODING),
value: binaryDataBuffer,
options: {
filename: binaryProperty.fileName,
contentType: binaryProperty.mimeType,
Expand Down
9 changes: 3 additions & 6 deletions packages/nodes-base/nodes/Ftp/Ftp.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
ICredentialDataDecryptedObject,
IDataObject,
Expand Down Expand Up @@ -503,7 +500,7 @@ export class Ftp implements INodeType {
throw new NodeOperationError(this.getNode(), `No binary data property "${propertyNameUpload}" does not exists on item!`);
}

const buffer = Buffer.from(item.binary[propertyNameUpload].data, BINARY_ENCODING) as Buffer;
const buffer = await this.helpers.getBinaryDataBuffer(i, propertyNameUpload);
await sftp!.put(buffer, remotePath);
} else {
// Is text file
Expand Down Expand Up @@ -597,7 +594,7 @@ export class Ftp implements INodeType {
throw new NodeOperationError(this.getNode(), `No binary data property "${propertyNameUpload}" does not exists on item!`);
}

const buffer = Buffer.from(item.binary[propertyNameUpload].data, BINARY_ENCODING) as Buffer;
const buffer = await this.helpers.getBinaryDataBuffer(i, propertyNameUpload);

try {
await ftp!.put(buffer, remotePath);
Expand Down
7 changes: 2 additions & 5 deletions packages/nodes-base/nodes/Google/Drive/GoogleDrive.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';

import {
IDataObject,
Expand Down Expand Up @@ -2423,7 +2420,7 @@ export class GoogleDrive implements INodeType {
originalFilename = item.binary[propertyNameUpload].fileName;
}

body = Buffer.from(item.binary[propertyNameUpload].data, BINARY_ENCODING);
body = await this.helpers.getBinaryDataBuffer(i, propertyNameUpload);
} else {
// Is text file
body = Buffer.from(this.getNodeParameter('fileContent', i) as string, 'utf8');
Expand Down
9 changes: 3 additions & 6 deletions packages/nodes-base/nodes/Google/YouTube/YouTube.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';

import {
IDataObject,
Expand Down Expand Up @@ -426,7 +423,7 @@ export class YouTube implements INodeType {
mimeType = item.binary[binaryProperty].mimeType;
}

const body = Buffer.from(item.binary[binaryProperty].data, BINARY_ENCODING);
const body = await this.helpers.getBinaryDataBuffer(i, binaryProperty);

const requestOptions = {
headers: {
Expand Down Expand Up @@ -913,7 +910,7 @@ export class YouTube implements INodeType {
mimeType = item.binary[binaryProperty].mimeType;
}

const body = Buffer.from(item.binary[binaryProperty].data, BINARY_ENCODING);
const body = await this.helpers.getBinaryDataBuffer(i, binaryProperty);

const requestOptions = {
headers: {
Expand Down
4 changes: 3 additions & 1 deletion packages/nodes-base/nodes/HtmlExtract/HtmlExtract.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,9 @@ export class HtmlExtract implements INodeType {
if (item.binary[dataPropertyName] === undefined) {
throw new NodeOperationError(this.getNode(), `No property named "${dataPropertyName}" exists!`);
}
htmlArray = Buffer.from(item.binary[dataPropertyName].data, 'base64').toString('utf8');

const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, dataPropertyName);
htmlArray = binaryDataBuffer.toString('utf-8');
}

// Convert it always to array that it works with a string or an array of strings
Expand Down
9 changes: 5 additions & 4 deletions packages/nodes-base/nodes/HttpRequest/HttpRequest.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import {
Expand Down Expand Up @@ -771,8 +770,9 @@ export class HttpRequest implements INodeType {
if (item.binary[binaryPropertyName] === undefined) {
throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`);
}
const binaryProperty = item.binary[binaryPropertyName] as IBinaryData;
requestOptions.body = Buffer.from(binaryProperty.data, BINARY_ENCODING);

const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, binaryPropertyName);
requestOptions.body = binaryDataBuffer;
} else if (options.bodyContentType === 'multipart-form-data') {
requestOptions.body = {};
const binaryPropertyNameFull = this.getNodeParameter('binaryPropertyName', itemIndex) as string;
Expand All @@ -794,9 +794,10 @@ export class HttpRequest implements INodeType {
}

const binaryProperty = item.binary[binaryPropertyName] as IBinaryData;
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, binaryPropertyName);

requestOptions.body[propertyName] = {
value: Buffer.from(binaryProperty.data, BINARY_ENCODING),
value: binaryDataBuffer,
options: {
filename: binaryProperty.fileName,
contentType: binaryProperty.mimeType,
Expand Down
7 changes: 2 additions & 5 deletions packages/nodes-base/nodes/LinkedIn/LinkedIn.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IDataObject,
ILoadOptionsFunctions,
Expand Down Expand Up @@ -149,7 +146,7 @@ export class LinkedIn implements INodeType {
}

// Buffer binary data
const buffer = Buffer.from(item.binary[propertyNameUpload].data, BINARY_ENCODING) as Buffer;
const buffer = await this.helpers.getBinaryDataBuffer(i, propertyNameUpload);
// Upload image
await linkedInApiRequest.call(this, 'POST', uploadUrl, buffer, true);

Expand Down
8 changes: 3 additions & 5 deletions packages/nodes-base/nodes/Mailgun/Mailgun.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IDataObject,
INodeExecutionData,
Expand Down Expand Up @@ -158,8 +155,9 @@ export class Mailgun implements INodeType {
if (!item.binary.hasOwnProperty(propertyName)) {
continue;
}
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, propertyName);
attachments.push({
value: Buffer.from(item.binary[propertyName].data, BINARY_ENCODING),
value: binaryDataBuffer,
options: {
filename: item.binary[propertyName].fileName || 'unknown',

Expand Down
8 changes: 3 additions & 5 deletions packages/nodes-base/nodes/Matrix/GenericFunctions.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import {
import { IDataObject, NodeApiError, NodeOperationError, } from 'n8n-workflow';

import {
BINARY_ENCODING,
IExecuteFunctions,
IExecuteSingleFunctions,
ILoadOptionsFunctions,
Expand Down Expand Up @@ -69,7 +68,7 @@ export async function matrixApiRequest(this: IExecuteFunctions | IExecuteSingleF
}
}

export async function handleMatrixCall(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, item: IDataObject, index: number, resource: string, operation: string): Promise<any> { // tslint:disable-line:no-any
export async function handleMatrixCall(this: IExecuteFunctions, item: IDataObject, index: number, resource: string, operation: string): Promise<any> { // tslint:disable-line:no-any

if (resource === 'account') {
if (operation === 'me') {
Expand Down Expand Up @@ -193,13 +192,12 @@ export async function handleMatrixCall(this: IExecuteFunctions | IExecuteSingleF
throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`);
}

//@ts-ignore
// @ts-ignore
qs.filename = item.binary[binaryPropertyName].fileName;
//@ts-ignore
filename = item.binary[binaryPropertyName].fileName;

//@ts-ignore
body = Buffer.from(item.binary[binaryPropertyName].data, BINARY_ENCODING);
body = await this.helpers.getBinaryDataBuffer(index, binaryPropertyName);
//@ts-ignore
headers['Content-Type'] = item.binary[binaryPropertyName].mimeType;
headers['accept'] = 'application/json,text/*;q=0.99';
Expand Down
7 changes: 2 additions & 5 deletions packages/nodes-base/nodes/NextCloud/NextCloud.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';

import {
IDataObject,
Expand Down Expand Up @@ -1066,7 +1063,7 @@ export class NextCloud implements INodeType {
throw new NodeOperationError(this.getNode(), `No binary data property "${propertyNameUpload}" does not exists on item!`);
}

body = Buffer.from(item.binary[propertyNameUpload].data, BINARY_ENCODING);
body = await this.helpers.getBinaryDataBuffer(i, propertyNameUpload);
} else {
// Is text file
body = this.getNodeParameter('fileContent', i) as string;
Expand Down
6 changes: 1 addition & 5 deletions packages/nodes-base/nodes/NocoDB/GenericFunctions.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,11 +118,7 @@ export async function downloadRecordAttachments(this: IExecuteFunctions | IPollF
if (record[fieldName]) {
for (const [index, attachment] of (JSON.parse(record[fieldName] as string) as IAttachment[]).entries()) {
const file = await apiRequest.call(this, 'GET', '', {}, {}, attachment.url, { json: false, encoding: null });
element.binary![`${fieldName}_${index}`] = {
data: Buffer.from(file).toString('base64'),
fileName: attachment.title,
mimeType: attachment.mimetype,
};
element.binary![`${fieldName}_${index}`] = await this.helpers.prepareBinaryData(Buffer.from(file), attachment.title, attachment.mimetype);
}
}
}
Expand Down
3 changes: 1 addition & 2 deletions packages/nodes-base/nodes/Pipedrive/Pipedrive.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
ILoadOptionsFunctions,
} from 'n8n-core';
Expand Down Expand Up @@ -4500,7 +4499,7 @@ export class Pipedrive implements INodeType {
throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`);
}

const fileBufferData = Buffer.from(item.binary[binaryPropertyName].data, BINARY_ENCODING);
const fileBufferData = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName);

formData.file = {
value: fileBufferData,
Expand Down
7 changes: 2 additions & 5 deletions packages/nodes-base/nodes/ReadPdf/ReadPdf.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';

import {
INodeExecutionData,
Expand Down Expand Up @@ -55,7 +52,7 @@ export class ReadPdf implements INodeType {
item.binary = {};
}

const binaryData = Buffer.from(item.binary[binaryPropertyName].data, BINARY_ENCODING);
const binaryData = await this.helpers.getBinaryDataBuffer(itemIndex, binaryPropertyName);
returnData.push({
binary: item.binary,
json: await pdf(binaryData),
Expand Down
Loading