fix: sanitize filenames and upload based on filename (#138)

__tests__/tools/sanitze-filename.js

@@ -0,0 +1,13 @@
+const sanitizeFilename = require('../../src/utils/sanitize-filename');
+
+describe('sanitizeFilename method', () => {
+  it('should remove emojis from filename', () => {
+    expect(sanitizeFilename('🇯🇵japan.😢jpg')).toBe('japan.jpg');
+  });
+
+  it('should remove invalid character from filename', () => {
+    expect(sanitizeFilename('~/.\u0000ssh/authorized_keys')).toBe(
+      '~.sshauthorized_keys'
+    );
+  });
+});

example/create-board.js

@@ -28,17 +28,15 @@ function readFile(path) {
       name: 'Japan 🇯🇵',
     });
 
-    const links = await wtClient.board.addLinks(board, [
+    await wtClient.board.addLinks(board, [
       {
         url: 'https://en.wikipedia.org/wiki/Japan',
         title: 'Japan - Wikipedia',
       },
     ]);
 
-    console.log(links);
-
     const filePaths = [
-      path.join(__dirname, 'files/Japan-01.jpg'),
+      path.join(__dirname, 'files/Japan-01🇯🇵.jpg'),
       path.join(__dirname, 'files/Japan-02.jpg'),
       path.join(__dirname, 'files/Japan-03.jpg'),
       path.join(__dirname, 'files/Japan-04.jpg'),

example/create-transfer.js

@@ -29,7 +29,7 @@ function readFile(path) {
     });
 
     const filePaths = [
-      path.join(__dirname, 'files/Japan-01.jpg'),
+      path.join(__dirname, 'files/Japan-01🇯🇵.jpg'),
       path.join(__dirname, 'files/Japan-02.jpg'),
       path.join(__dirname, 'files/Japan-03.jpg'),
       path.join(__dirname, 'files/Japan-04.jpg'),

package.json

@@ -37,7 +37,9 @@
   },
   "dependencies": {
     "axios": "0.18.0",
+    "emoji-strip": "1.0.1",
     "lodash": "4.17.11",
+    "sanitize-filename": "1.6.1",
     "winston": "3.1.0"
   },
   "devDependencies": {

src/boards/actions/add-files.js

@@ -2,6 +2,7 @@ const WTError = require('../../error');
 const logger = require('../../config/logger');
 const futureFile = require('../models/future-file');
 const RemoteFile = require('../models/remote-file');
+const contentForFiles = require('../../utils/content-for-files');
 
 module.exports = function({ request, routes, uploadFileToBoard }) {
   /**
@@ -35,9 +36,10 @@ module.exports = function({ request, routes, uploadFileToBoard }) {
       board.addFiles(...boardFiles);
 
       if (shouldUploadFiles(files)) {
+        const filesContent = contentForFiles(files);
         await Promise.all(
-          boardFiles.map((file, index) => {
-            return uploadFileToBoard(board, file, files[index].content);
+          boardFiles.map((file) => {
+            return uploadFileToBoard(board, file, filesContent[file.name]);
           })
         );
       }

src/boards/models/future-file.js

@@ -1,11 +1,13 @@
+const sanitizeFileName = require('../../utils/sanitize-filename');
+
 /**
  * Normalizes a file object. Removes non-expected properties.
  * @param   {Object} file A file object
  * @returns {Object}      Normalized file object
  */
 function normalizeFile(file) {
   return {
-    name: file.name,
+    name: sanitizeFileName(file.name),
     size: file.size,
   };
 }

src/transfers/actions/create.js

@@ -1,3 +1,5 @@
+const contentForFiles = require('../../utils/content-for-files');
+
 const WTError = require('../../error');
 const logger = require('../../config/logger');
 const { futureTransfer, RemoteTransfer } = require('../models');
@@ -29,6 +31,7 @@ module.exports = function({
   return async function createTransfer(transfer) {
     try {
       logger.info('Creating a new transfer.');
+
       const response = await request.send(
         routes.transfers.create,
         futureTransfer(transfer)
@@ -40,12 +43,13 @@ module.exports = function({
       // If the files array contains the content of the file
       // lets uplopad directly, without asking the user to do it.
       if (shouldUploadFiles(transfer.files)) {
+        const filesContent = contentForFiles(transfer.files);
         await Promise.all(
-          remoteTransfer.files.map((file, index) => {
+          remoteTransfer.files.map((file) => {
             return uploadFileToTransfer(
               remoteTransfer,
               file,
-              transfer.files[index].content
+              filesContent[file.name]
             );
           })
         );

src/transfers/models/future-file.js

@@ -1,11 +1,13 @@
+const sanitizeFileName = require('../../utils/sanitize-filename');
+
 /**
  * Normalizes a file object. Removes non-expected properties.
  * @param   {Object} file A file object
  * @returns {Object}      Normalized file object
  */
 function normalizeFile(file) {
   return {
-    name: file.name,
+    name: sanitizeFileName(file.name),
     size: file.size,
   };
 }

src/utils/content-for-files.js

@@ -0,0 +1,18 @@
+const sanitizeFileName = require('./sanitize-filename');
+
+/**
+ * Create a hash where the key is the (sanitized) name of the file,
+ * and the value is the content of the file. This will be useful
+ * when mapping the remoteTransfer/Board files with the local files.
+ *
+ * @param   {Array}  files List of files with names and content
+ * @returns {Object}       Hash with file names => file content
+ */
+function contentForFiles(files) {
+  return files.reduce((filesContent, file) => {
+    filesContent[sanitizeFileName(file.name)] = file.content;
+    return filesContent;
+  }, {});
+}
+
+module.exports = contentForFiles;

src/utils/sanitize-filename.js

@@ -0,0 +1,13 @@
+const emojiStrip = require('emoji-strip');
+const sanitizeFileName = require('sanitize-filename');
+
+/**
+ * WeTransfer doesn't support emojis as a part of filenames 😢
+ * Let's also sanitize the filename, just in case.
+ *
+ * @param   {String} fileName Original filename
+ * @returns {String}          The sanitized filename
+ */
+module.exports = function(fileName = '') {
+  return sanitizeFileName(emojiStrip(fileName));
+};

yarn.lock

@@ -1337,7 +1337,7 @@ debug@^4.1.0:
   dependencies:
     ms "^2.1.1"
 
-debuglog@*, debuglog@^1.0.1:
+debuglog@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
 
@@ -1543,6 +1543,16 @@ editor@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/editor/-/editor-1.0.0.tgz#60c7f87bd62bcc6a894fa8ccd6afb7823a24f742"
 
+emoji-regex@^6.1.3:
+  version "6.5.1"
+  resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-6.5.1.tgz#9baea929b155565c11ea41c6626eaa65cef992c2"
+
+[email protected]:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/emoji-strip/-/emoji-strip-1.0.1.tgz#cf9390535044121888420ae4db436d03a6cc459d"
+  dependencies:
+    emoji-regex "^6.1.3"
+
 [email protected]:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/enabled/-/enabled-1.0.2.tgz#965f6513d2c2d1c5f4652b64a2e3396467fc2f93"
@@ -2559,7 +2569,7 @@ import-local@^1.0.0:
     pkg-dir "^2.0.0"
     resolve-cwd "^2.0.0"
 
-imurmurhash@*, imurmurhash@^0.1.4:
+imurmurhash@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea"
 
@@ -3596,39 +3606,17 @@ lockfile@^1.0.4:
   dependencies:
     signal-exit "^3.0.2"
 
-lodash._baseindexof@*:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/lodash._baseindexof/-/lodash._baseindexof-3.1.0.tgz#fe52b53a1c6761e42618d654e4a25789ed61822c"
-
 lodash._baseuniq@~4.6.0:
   version "4.6.0"
   resolved "https://registry.yarnpkg.com/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz#0ebb44e456814af7905c6212fa2c9b2d51b841e8"
   dependencies:
     lodash._createset "~4.0.0"
     lodash._root "~3.0.0"
 
-lodash._bindcallback@*:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/lodash._bindcallback/-/lodash._bindcallback-3.0.1.tgz#e531c27644cf8b57a99e17ed95b35c748789392e"
-
-lodash._cacheindexof@*:
-  version "3.0.2"
-  resolved "https://registry.yarnpkg.com/lodash._cacheindexof/-/lodash._cacheindexof-3.0.2.tgz#3dc69ac82498d2ee5e3ce56091bafd2adc7bde92"
-
-lodash._createcache@*:
-  version "3.1.2"
-  resolved "https://registry.yarnpkg.com/lodash._createcache/-/lodash._createcache-3.1.2.tgz#56d6a064017625e79ebca6b8018e17440bdcf093"
-  dependencies:
-    lodash._getnative "^3.0.0"
-
 lodash._createset@~4.0.0:
   version "4.0.3"
   resolved "https://registry.yarnpkg.com/lodash._createset/-/lodash._createset-4.0.3.tgz#0f4659fbb09d75194fa9e2b88a6644d363c9fe26"
 
-lodash._getnative@*, lodash._getnative@^3.0.0:
-  version "3.9.1"
-  resolved "https://registry.yarnpkg.com/lodash._getnative/-/lodash._getnative-3.9.1.tgz#570bc7dede46d61cdcde687d65d3eecbaa3aaff5"
-
 lodash._root@~3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/lodash._root/-/lodash._root-3.0.1.tgz#fba1c4524c19ee9a5f8136b4609f017cf4ded692"
@@ -3657,10 +3645,6 @@ lodash.isstring@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451"
 
-lodash.restparam@*:
-  version "3.6.1"
-  resolved "https://registry.yarnpkg.com/lodash.restparam/-/lodash.restparam-3.6.1.tgz#936a4e309ef330a7645ed4145986c85ae5b20805"
-
 lodash.sortby@^4.7.0:
   version "4.7.0"
   resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438"
@@ -4299,7 +4283,6 @@ npm@^6.3.0:
     cmd-shim "~2.0.2"
     columnify "~1.5.4"
     config-chain "~1.1.11"
-    debuglog "*"
     detect-indent "~5.0.0"
     detect-newline "^2.1.0"
     dezalgo "~1.0.3"
@@ -4314,7 +4297,6 @@ npm@^6.3.0:
     has-unicode "~2.0.1"
     hosted-git-info "^2.6.0"
     iferr "^1.0.0"
-    imurmurhash "*"
     inflight "~1.0.6"
     inherits "~2.0.3"
     ini "^1.3.5"
@@ -4327,14 +4309,8 @@ npm@^6.3.0:
     libnpx "^10.2.0"
     lock-verify "^2.0.2"
     lockfile "^1.0.4"
-    lodash._baseindexof "*"
     lodash._baseuniq "~4.6.0"
-    lodash._bindcallback "*"
-    lodash._cacheindexof "*"
-    lodash._createcache "*"
-    lodash._getnative "*"
     lodash.clonedeep "~4.5.0"
-    lodash.restparam "*"
     lodash.union "~4.6.0"
     lodash.uniq "~4.5.0"
     lodash.without "~4.4.0"
@@ -4373,7 +4349,6 @@ npm@^6.3.0:
     read-package-json "^2.0.13"
     read-package-tree "^5.2.1"
     readable-stream "^2.3.6"
-    readdir-scoped-modules "*"
     request "^2.81.0"
     retry "^0.12.0"
     rimraf "~2.6.2"
@@ -5067,7 +5042,7 @@ readable-stream@~1.1.10:
     isarray "0.0.1"
     string_decoder "~0.10.x"
 
-readdir-scoped-modules@*, readdir-scoped-modules@^1.0.0:
+readdir-scoped-modules@^1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/readdir-scoped-modules/-/readdir-scoped-modules-1.0.2.tgz#9fafa37d286be5d92cbaebdee030dc9b5f406747"
   dependencies:
@@ -5366,6 +5341,12 @@ sane@^2.0.0:
   optionalDependencies:
     fsevents "^1.1.1"
 
+[email protected]:
+  version "1.6.1"
+  resolved "https://registry.yarnpkg.com/sanitize-filename/-/sanitize-filename-1.6.1.tgz#612da1c96473fa02dccda92dcd5b4ab164a6772a"
+  dependencies:
+    truncate-utf8-bytes "^1.0.0"
+
 sax@^1.2.4:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
@@ -6019,6 +6000,12 @@ triple-beam@^1.2.0, triple-beam@^1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9"
 
+truncate-utf8-bytes@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz#405923909592d56f78a5818434b0b78489ca5f2b"
+  dependencies:
+    utf8-byte-length "^1.0.1"
+
 tslib@^1.9.0:
   version "1.9.3"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.9.3.tgz#d7e4dd79245d85428c4d7e4822a79917954ca286"
@@ -6159,6 +6146,10 @@ use@^3.1.0:
   dependencies:
     kind-of "^6.0.2"
 
+utf8-byte-length@^1.0.1:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz#f45f150c4c66eee968186505ab93fcbb8ad6bf61"
+
 util-deprecate@~1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"