Data importer improvements and fixes

refs TryGhost#4605, TryGhost#4479 - Removes versioning from the importer - Fixes an issue with SQLITE errors not being thrown properly for posts - Ensures that posts have a created_at date - Makes sure that the API wrapper is properly handled
ErisDS · Dec 18, 2014 · 3c9273e · 3c9273e
1 parent 68df133
commit 3c9273e
Show file tree

Hide file tree

Showing 15 changed files with 806 additions and 837 deletions.
diff --git a/core/server/data/import/001.js b/core/server/data/import/001.js
diff --git a/core/server/data/import/002.js b/core/server/data/import/002.js
diff --git a/core/server/data/import/003.js b/core/server/data/import/003.js
diff --git a/core/server/data/import/000.js → core/server/data/import/data-importer.js b/core/server/data/import/000.js → core/server/data/import/data-importer.js
@@ -3,37 +3,15 @@ var Promise = require('bluebird'),
     models  = require('../../models'),
     utils   = require('./utils'),
 
-    Importer000;
+    DataImporter;
 
-Importer000 = function () {
-    _.bindAll(this, 'doImport');
+DataImporter = function () {};
 
-    this.version = '000';
-
-    this.importFrom = {
-        '000': this.doImport,
-        '001': this.doImport,
-        '002': this.doImport,
-        '003': this.doImport
-    };
+DataImporter.prototype.importData = function (data) {
+    return this.doImport(data);
 };
 
-Importer000.prototype.importData = function (data) {
-    return this.canImport(data)
-        .then(function (importerFunc) {
-            return importerFunc(data);
-        });
-};
-
-Importer000.prototype.canImport = function (data) {
-    if (data.meta && data.meta.version && this.importFrom[data.meta.version]) {
-        return Promise.resolve(this.importFrom[data.meta.version]);
-    }
-
-    return Promise.reject('Unsupported version of data: ' + data.meta.version);
-};
-
-Importer000.prototype.loadUsers = function () {
+DataImporter.prototype.loadUsers = function () {
     var users = {all: {}};
 
     return models.User.findAll({include: ['roles']}).then(function (_users) {
@@ -52,11 +30,7 @@ Importer000.prototype.loadUsers = function () {
     });
 };
 
-// Importer000.prototype.importerFunction = function (t) {
-//
-// };
-
-Importer000.prototype.doUserImport = function (t, tableData, users, errors) {
+DataImporter.prototype.doUserImport = function (t, tableData, users, errors) {
     var userOps = [],
         imported = [];
 
@@ -89,7 +63,7 @@ Importer000.prototype.doUserImport = function (t, tableData, users, errors) {
     return Promise.resolve({});
 };
 
-Importer000.prototype.doImport = function (data) {
+DataImporter.prototype.doImport = function (data) {
     var self = this,
         tableData = data.data,
         imported = {},
@@ -171,8 +145,8 @@ Importer000.prototype.doImport = function (data) {
 };
 
 module.exports = {
-    Importer000: Importer000,
+    DataImporter: DataImporter,
     importData: function (data) {
-        return new Importer000().importData(data);
+        return new DataImporter().importData(data);
     }
 };
diff --git a/core/server/data/import/index.js b/core/server/data/import/index.js
@@ -3,7 +3,7 @@ var Promise         = require('bluebird'),
     validation      = require('../validation'),
     errors          = require('../../errors'),
     uuid            = require('node-uuid'),
-    validator       = require('validator'),
+    importer        = require('./data-importer'),
     tables          = require('../schema').tables,
     validate,
     handleErrors,
@@ -89,14 +89,14 @@ sanitize = function sanitize(data) {
         });
 
     _.each(tableNames, function (tableName) {
-        // Sanitize the table data for duplicates and valid uuid values
+        // Sanitize the table data for duplicates and valid uuid and created_at values
         var sanitizedTableData = _.transform(data.data[tableName], function (memo, importValues) {
             var uuidMissing = (!importValues.uuid && tables[tableName].uuid) ? true : false,
-                uuidMalformed = (importValues.uuid && !validator.isUUID(importValues.uuid)) ? true : false,
+                uuidMalformed = (importValues.uuid && !validation.validator.isUUID(importValues.uuid)) ? true : false,
                 isDuplicate,
                 problemTag;
 
-            // Check for correct UUID and fix if neccessary
+            // Check for correct UUID and fix if necessary
             if (uuidMissing || uuidMalformed) {
                 importValues.uuid = uuid.v4();
             }
@@ -184,25 +184,12 @@ validate = function validate(data) {
     });
 };
 
-module.exports = function (version, data) {
-    var importer,
-        sanitizeResults;
-
-    sanitizeResults = sanitize(data);
+module.exports = function (data) {
+    var sanitizeResults = sanitize(data);
 
     data = sanitizeResults.data;
 
     return validate(data).then(function () {
-        try {
-            importer = require('./' + version);
-        } catch (ignore) {
-            // Zero effs given
-        }
-
-        if (!importer) {
-            return Promise.reject('No importer found');
-        }
-
         return importer.importData(data);
     }).then(function () {
         return sanitizeResults;

diff --git a/core/server/data/import/utils.js b/core/server/data/import/utils.js
@@ -185,34 +185,25 @@ utils = {
         var ops = [];
 
         tableData = stripProperties(['id'], tableData);
-
         _.each(tableData, function (post) {
              // Validate minimum post fields
             if (areEmpty(post, 'title', 'slug', 'markdown')) {
                 return;
             }
 
-            ops.push(function () {
-                return models.Post.add(post, _.extend(internal, {transacting: transaction, importing: true}))
+             // The post importer has auto-timestamping disabled
+            if (!post.created_at) {
+                post.created_at = Date.now();
+            }
+
+            ops.push(models.Post.add(post, _.extend(internal, {transacting: transaction, importing: true}))
                     .catch(function (error) {
                         return Promise.reject({raw: error, model: 'post', data: post});
-                    });
-            });
+                    })
+            );
         });
 
-        return Promise.reduce(ops, function (results, op) {
-            return op().then(function (result) {
-                results.push(result);
-
-                return results;
-            }).catch(function (error) {
-                if (error) {
-                    results.push(error);
-                }
-
-                return results;
-            });
-        }, []).settle();
+        return Promise.settle(ops);
     },
 
     importUsers: function importUsers(tableData, existingUsers, transaction) {

diff --git a/core/server/data/importer/handlers/json.js b/core/server/data/importer/handlers/json.js
@@ -20,7 +20,11 @@ JSONHandler = {
                 importData = JSON.parse(fileData);
 
                 // if importData follows JSON-API format `{ db: [exportedData] }`
-                if (_.keys(importData).length === 1 && Array.isArray(importData.db)) {
+                if (_.keys(importData).length === 1) {
+                    if (!importData.db || !Array.isArray(importData.db)) {
+                        throw new Error('Invalid JSON format, expected `{ db: [exportedData] }`');
+                    }
+
                     importData = importData.db[0];
                 }
 

diff --git a/core/server/data/importer/importers/data.js b/core/server/data/importer/importers/data.js
@@ -8,7 +8,7 @@ DataImporter = {
         return importData;
     },
     doImport: function (importData) {
-        return importer('003', importData);
+        return importer(importData);
     }
 };
 

diff --git a/core/server/data/validation/index.js b/core/server/data/validation/index.js
@@ -27,9 +27,8 @@ validator.extend('isEmptyOrURL', function (str) {
     return (_.isEmpty(str) || validator.isURL(str, {require_protocol: false}));
 });
 
-// Validation validation against schema attributes
-// values are checked against the validation objects
-// form schema.js
+// Validation against schema attributes
+// values are checked against the validation objects from schema.js
 validateSchema = function (tableName, model) {
     var columns = _.keys(schema[tableName]),
         validationErrors = [];
@@ -163,6 +162,7 @@ validate = function (value, key, validations) {
 };
 
 module.exports = {
+    validator: validator,
     validateSchema: validateSchema,
     validateSettings: validateSettings,
     validateActiveTheme: validateActiveTheme