From 88db807f6f4a7e12281cd1de57e47ddf4e95f070 Mon Sep 17 00:00:00 2001 From: Hannah Wolfe Date: Wed, 10 Dec 2014 21:50:00 +0000 Subject: [PATCH] Initial pass at importer refactor --- core/server/api/db.js | 111 ++----------- core/server/data/importer/handlers/json.js | 36 +++++ core/server/data/importer/importers/data.js | 15 ++ core/server/data/importer/index.js | 163 ++++++++++++++++++++ 4 files changed, 230 insertions(+), 95 deletions(-) create mode 100644 core/server/data/importer/handlers/json.js create mode 100644 core/server/data/importer/importers/data.js create mode 100644 core/server/data/importer/index.js diff --git a/core/server/api/db.js b/core/server/api/db.js index 21f4638711f4..2c7e6e23e2d2 100644 --- a/core/server/api/db.js +++ b/core/server/api/db.js @@ -1,57 +1,19 @@ // # DB API // API for DB operations -var dataExport = require('../data/export'), - dataImport = require('../data/import'), - dataProvider = require('../models'), - fs = require('fs-extra'), +var _ = require('lodash'), Promise = require('bluebird'), - _ = require('lodash'), - path = require('path'), - os = require('os'), - glob = require('glob'), - uuid = require('node-uuid'), - extract = require('extract-zip'), - errors = require('../../server/errors'), + dataExport = require('../data/export'), + importer = require('../data/importer'), + models = require('../models'), + errors = require('../errors'), canThis = require('../permissions').canThis, utils = require('./utils'), + api = {}, - db, - types = ['application/octet-stream', 'application/json', 'application/zip', 'application/x-zip-compressed'], - extensions = ['.json', '.zip']; + db; api.settings = require('./settings'); -// TODO refactor this out of here -function isJSON(ext) { - return ext === '.json'; -} - -function isZip(ext) { - return ext === '.zip'; -} - -function getJSONFileContents(filepath, ext) { - if (isJSON(ext)) { - // if it's just a JSON file, read it - return Promise.promisify(fs.readFile)(filepath); - } else if (isZip(ext)) { - var tmpdir = path.join(os.tmpdir(), uuid.v4()); - - return Promise.promisify(extract)(filepath, {dir: tmpdir}).then(function () { - return Promise.promisify(glob)('**/*.json', {cwd: tmpdir}).then(function (files) { - if (files[0]) { - // @TODO: handle multiple JSON files - return Promise.promisify(fs.readFile)(path.join(tmpdir, files[0])); - } else { - return Promise.reject(new errors.UnsupportedMediaTypeError( - 'Zip did not include any content to import.' - )); - } - }); - }); - } -} - /** * ## DB API Methods * @@ -90,9 +52,6 @@ db = { */ importContent: function (options) { options = options || {}; - var databaseVersion, - ext, - filepath; // Check if a file was provided if (!utils.checkFileExists(options, 'importfile')) { @@ -100,58 +59,20 @@ db = { } // Check if the file is valid - if (!utils.checkFileIsValid(options.importfile, types, extensions)) { + if (!utils.checkFileIsValid(options.importfile, importer.getTypes(), importer.getExtensions())) { return Promise.reject(new errors.UnsupportedMediaTypeError( - 'Please select either a .json or .zip file to import.' + 'Unsupported file. Please try any of the following formats: ' + + _.reduce(importer.getExtensions(), function (memo, ext) { + return memo ? memo + ', ' + ext : ext; + }) )); } - // TODO refactor this out of here - filepath = options.importfile.path; - ext = path.extname(options.importfile.name).toLowerCase(); - // Permissions check return canThis(options.context).importContent.db().then(function () { - return api.settings.read( - {key: 'databaseVersion', context: {internal: true}} - ).then(function (response) { - var setting = response.settings[0]; - - return setting.value; - }).then(function (version) { - databaseVersion = version; - // Read the file contents - return getJSONFileContents(filepath, ext); - }).then(function (fileContents) { - var importData; - - // Parse the json data - try { - importData = JSON.parse(fileContents); - - // if importData follows JSON-API format `{ db: [exportedData] }` - if (_.keys(importData).length === 1 && Array.isArray(importData.db)) { - importData = importData.db[0]; - } - } catch (e) { - errors.logError(e, 'API DB import content', 'check that the import file is valid JSON.'); - return Promise.reject(new errors.BadRequestError('Failed to parse the import JSON file.')); - } - - if (!importData.meta || !importData.meta.version) { - return Promise.reject( - new errors.ValidationError('Import data does not specify version', 'meta.version') - ); - } - - // Import for the current version - return dataImport(databaseVersion, importData); - }).then(api.settings.updateSettingsCache) - .return({db: []}) - .finally(function () { - // Unlink the file after import - return Promise.promisify(fs.unlink)(filepath); - }); + return importer.importFromFile(options.importfile) + .then(api.settings.updateSettingsCache) + .return({db: []}); }, function () { return Promise.reject(new errors.NoPermissionError('You do not have permission to import data (no rights).')); }); @@ -168,7 +89,7 @@ db = { options = options || {}; return canThis(options.context).deleteAllContent.db().then(function () { - return Promise.resolve(dataProvider.deleteAllContent()) + return Promise.resolve(models.deleteAllContent()) .return({db: []}) .catch(function (error) { return Promise.reject(new errors.InternalServerError(error.message || error)); diff --git a/core/server/data/importer/handlers/json.js b/core/server/data/importer/handlers/json.js new file mode 100644 index 000000000000..ec1b48dd950d --- /dev/null +++ b/core/server/data/importer/handlers/json.js @@ -0,0 +1,36 @@ +var _ = require('lodash'), + Promise = require('bluebird'), + fs = require('fs-extra'), + errors = require('../../../errors'), + JSONHandler; + +JSONHandler = { + type: 'data', + extensions: ['.json'], + types: ['application/octet-stream', 'application/json'], + + loadFile: function (files, startDir) { + /*jshint unused:false */ + // @TODO: Handle multiple JSON files + var filePath = files[0].path; + + return Promise.promisify(fs.readFile)(filePath).then(function (fileData) { + var importData; + try { + importData = JSON.parse(fileData); + + // if importData follows JSON-API format `{ db: [exportedData] }` + if (_.keys(importData).length === 1 && Array.isArray(importData.db)) { + importData = importData.db[0]; + } + + return importData; + } catch (e) { + errors.logError(e, 'API DB import content', 'check that the import file is valid JSON.'); + return Promise.reject(new errors.BadRequestError('Failed to parse the import JSON file.')); + } + }); + } +}; + +module.exports = JSONHandler; diff --git a/core/server/data/importer/importers/data.js b/core/server/data/importer/importers/data.js new file mode 100644 index 000000000000..455164e9fffb --- /dev/null +++ b/core/server/data/importer/importers/data.js @@ -0,0 +1,15 @@ +var importer = require('../../import'), + DataImporter; + +DataImporter = { + type: 'data', + preProcess: function (importData) { + importData.preProcessedByData = true; + return importData; + }, + doImport: function (importData) { + return importer('003', importData); + } +}; + +module.exports = DataImporter; diff --git a/core/server/data/importer/index.js b/core/server/data/importer/index.js new file mode 100644 index 000000000000..d1b2c9bad3ff --- /dev/null +++ b/core/server/data/importer/index.js @@ -0,0 +1,163 @@ +var _ = require('lodash'), + Promise = require('bluebird'), + sequence = require('../../utils/sequence'), + pipeline = require('../../utils/pipeline'), + fs = require('fs-extra'), + path = require('path'), + os = require('os'), + glob = require('glob'), + uuid = require('node-uuid'), + extract = require('extract-zip'), + errors = require('../../errors'), + JSONHandler = require('./handlers/json'), + DataImporter = require('./importers/data'), + + defaults; + +defaults = { + extensions: ['.zip'], + types: ['application/zip', 'application/x-zip-compressed'] +}; + +function ImportManager() { + this.importers = [DataImporter]; + this.handlers = [JSONHandler]; +} + +_.extend(ImportManager.prototype, { + getExtensions: function () { + return _.flatten(_.union(_.pluck(this.handlers, 'extensions'), defaults.extensions)); + }, + getTypes: function () { + return _.flatten(_.union(_.pluck(this.handlers, 'types'), defaults.types)); + }, + getGlobPattern: function (handler) { + return '**/*+(' + _.reduce(handler.extensions, function (memo, ext) { + return memo !== '' ? memo + '|' + ext : ext; + }, '') + ')'; + }, + isZip: function (ext) { + return _.contains(defaults.extensions, ext); + }, + extractZip: function (filePath) { + var tmpDir = path.join(os.tmpdir(), uuid.v4()); + return Promise.promisify(extract)(filePath, {dir: tmpDir}).then(function () { + return tmpDir; + }); + }, + processZip: function (file) { + var self = this; + return this.extractZip(file.path).then(function (directory) { + var ops = [], + importData = {}, + startDir = glob.sync(file.name.replace('.zip', ''), {cwd: directory}); + + startDir = startDir[0] || false; + + _.each(self.handlers, function (handler) { + if (importData.hasOwnProperty(handler.type)) { + // This limitation is here to reduce the complexity of the importer for now + return Promise.reject(new errors.UnsupportedMediaTypeError( + 'Zip file contains too many types of import data. Please split it up and import separately.' + )); + } + + var globPattern = self.getGlobPattern(handler), + files = _.map(glob.sync(globPattern, {cwd: directory}), function (file) { + return {name: file, path: path.join(directory, file)}; + }); + + if (files.length > 0) { + ops.push(function () { + return handler.loadFile(files, startDir).then(function (data) { + importData[handler.type] = data; + }); + }); + } + }); + + if (ops.length === 0) { + return Promise.reject(new errors.UnsupportedMediaTypeError( + 'Zip did not include any content to import.' + )); + } + + return sequence(ops).then(function () { + return importData; + }); + }); + }, + processFile: function (file, ext) { + var fileHandler = _.find(this.handlers, function (handler) { + return _.contains(handler.extensions, ext); + }); + + return fileHandler.loadFile([_.pick(file, 'name', 'path')]).then(function (loadedData) { + // normalize the returned data + var importData = {}; + importData[fileHandler.type] = loadedData; + return importData; + }); + }, + loadFile: function (file) { + var self = this, + ext = path.extname(file.name).toLowerCase(); + + return Promise.resolve(this.isZip(ext)).then(function (isZip) { + if (isZip) { + // If it's a zip, process the zip file + return self.processZip(file); + } else { + // Else process the file + return self.processFile(file, ext); + } + }).finally(function () { + return Promise.promisify(fs.unlink)(file.path); + }); + }, + preProcess: function (importData) { + var ops = []; + _.each(this.importers, function (importer) { + ops.push(function () { + return importer.preProcess(importData); + }); + }); + + return pipeline(ops); + }, + doImport: function (importData) { + var ops = []; + _.each(this.importers, function (importer) { + if (importData.hasOwnProperty(importer.type)) { + ops.push(function () { + return importer.doImport(importData[importer.type]); + }); + } + }); + + return sequence(ops).then(function (importResult) { + return importResult; + }); + }, + generateReport: function (importData) { + return importData; + }, + importFromFile: function (file) { + var self = this; + + // Step 1: Handle converting the file to usable data + return this.loadFile(file).then(function (importData) { + // Step 2: Let the importers pre-process the data + return self.preProcess(importData); + }).then(function (importData) { + // Step 3: Actually do the import + // @TODO: It would be cool to have some sort of dry run flag here + return self.doImport(importData); + }).then(function (importData) { + // Step 4: Finally, report on the import + return self.generateReport(importData); + }); + } +}); + +module.exports = new ImportManager();