Skip to content

Commit

Permalink
Use workers to parse files and extract dependencies properly.
Browse files Browse the repository at this point in the history
  • Loading branch information
@cpojer
cpojer committed Apr 18, 2016
1 parent 8fc2753 commit 275af0c
Show file tree
Hide file tree
Showing 18 changed files with 232 additions and 215 deletions.
131 changes: 76 additions & 55 deletions packages/jest-haste-map/src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,23 +10,20 @@

const crypto = require('crypto');
const denodeify = require('denodeify');
const docblock = require('./lib/docblock');
const execSync = require('child_process').execSync;
const fs = require('graceful-fs');
const getPlatformExtension = require('./lib/getPlatformExtension');
const nodeCrawl = require('./crawlers/node');
const os = require('os');
const path = require('./fastpath');
const watchmanCrawl = require('./crawlers/watchman');
const worker = require('./worker');
const workerFarm = require('worker-farm');

const GENERIC_PLATFORM = 'generic';
const GENERIC_PLATFORM = 'g';
const NODE_MODULES = path.sep + 'node_modules' + path.sep;
const PACKAGE_JSON = path.sep + 'package.json';
const VERSION = require('../package.json').version;

const readFile = denodeify(fs.readFile);
const writeFile = denodeify(fs.writeFile);

const canUseWatchman = (() => {
try {
execSync('watchman version', {stdio: 'ignore'});
Expand All @@ -42,6 +39,9 @@ class HasteMap {
cacheDirectory: options.cacheDirectory || os.tmpDir(),
extensions: options.extensions,
ignorePattern: options.ignorePattern,
maxWorkers: options.maxWorkers,
mocksPattern:
options.mocksPattern ? new RegExp(options.mocksPattern) : null,
platforms: options.platforms,
resetCache: options.resetCache,
roots: options.roots,
Expand All @@ -54,14 +54,17 @@ class HasteMap {
? new RegExp('(.+?)' + NODE_MODULES + '(' + list.join('|') + ')(?!' + NODE_MODULES + ')')
: null;

this._buildPromise = null;
this._cachePath = HasteMap.getCacheFilePath(
this._options.cacheDirectory,
VERSION,
this._options.roots.join(':'),
this._options.extensions.join(':'),
this._options.platforms.join(':')
this._options.platforms.join(':'),
options.mocksPattern
);
this._buildPromise = null;
this._workerPromise = null;
this._workerFarm = null;
}

static getCacheFilePath(tmpdir) {
Expand All @@ -73,15 +76,12 @@ class HasteMap {
build() {
if (!this._buildPromise) {
this._buildPromise = this._buildFileMap()
.then(data => this._buildHasteMap(data));
.then(data => this._buildHasteMap(data))
.then(data => this._persist(data));
}
return this._buildPromise;
}

persist(data) {
return writeFile(this._cachePath, JSON.stringify(data)).then(() => data);
}

read() {
return this._parse(fs.readFileSync(this._cachePath, 'utf-8'));
}
Expand All @@ -101,6 +101,11 @@ class HasteMap {
});
}

_persist(data) {
fs.writeFileSync(this._cachePath, JSON.stringify(data), 'utf-8');
return data;
}

_buildFileMap() {
const read = this._options.resetCache ? this._createEmptyMap : this.read;

Expand All @@ -112,6 +117,9 @@ class HasteMap {

_buildHasteMap(data) {
const map = Object.create(null);
const mocks = Object.create(null);
const mocksPattern = this._options.mocksPattern;
const promises = [];
const setModule = module => {
if (!map[module.id]) {
map[module.id] = Object.create(null);
Expand All @@ -129,13 +137,14 @@ class HasteMap {
);
}

const fileData = data.files[module.path];
fileData.id = module.id;
moduleMap[platform] = module;
};

const promises = [];
for (const filePath in data.files) {
if (mocksPattern && mocksPattern.test(filePath)) {
mocks[path.basename(filePath, path.extname(filePath))] = filePath;
}

if (!this._isNodeModulesDir(filePath)) {
const fileData = data.files[filePath];
const moduleData = data.map[fileData.id];
Expand All @@ -148,19 +157,60 @@ class HasteMap {
}
}

fileData.visited = true;
if (filePath.endsWith(PACKAGE_JSON)) {
promises.push(this._processHastePackage(filePath, setModule));
} else {
promises.push(this._processHasteModule(filePath, setModule));
promises.push(
this._getWorker()({filePath}).then(data => {
fileData.visited = true;
if (data.module) {
fileData.id = data.module.id;
setModule(data.module);
}
if (data.dependencies) {
fileData.dependencies = data.dependencies;
}
})
);
}
}

return Promise.all(promises)
.then(() => {
if (this._workerFarm) {
workerFarm.end(this._workerFarm);
}
this._workerFarm = null;
this._workerPromise = null;
})
.then(() => {
data.map = map;
data.mocks = mocks;
return data;
});
}

_getWorker() {
if (!this._workerPromise) {
if (this._options.maxWorkers === 1) {
this._workerPromise = data => new Promise(
(resolve, reject) => worker(data, (error, data) => {
if (error) {
reject(error);
} else {
resolve(data);
}
})
);
} else {
this._workerFarm = workerFarm(
{
maxConcurrentWorkers: this._options.maxWorkers,
},
require.resolve('./worker')
);
this._workerPromise = denodeify(this._workerFarm);
}
}

return Promise.all(promises).then(() => {
data.map = map;
return data;
});
return this._workerPromise;
}

_parse(data) {
Expand Down Expand Up @@ -201,39 +251,10 @@ class HasteMap {
clocks: Object.create(null),
files: Object.create(null),
map: Object.create(null),
mocks: Object.create(null),
};
}

_processHastePackage(filePath, setModule) {
return readFile(filePath, 'utf-8')
.then(data => {
data = JSON.parse(data);
if (data.name) {
setModule({
id: data.name,
path: filePath,
type: 'package',
});
}
});
}

_processHasteModule(filePath, setModule) {
return readFile(filePath, 'utf-8')
.then(data => {
const doc = docblock.parseAsObject(data);
const id = doc.providesModule || doc.provides;

if (id) {
setModule({
id,
path: filePath,
type: 'module',
});
}
});
}

}

HasteMap.GENERIC_PLATFORM = GENERIC_PLATFORM;
Expand Down
51 changes: 10 additions & 41 deletions packages/jest-haste-map/src/lib/docblock.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,32 +9,20 @@

'use strict';

const commentEndRe = /\*\/$/;
const commentStartRe = /^\/\*\*/;
const docblockRe = /^\s*(\/\*\*(.|\r?\n)*?\*\/)/;

const ltrimRe = /^\s*/;
/**
* @param {String} contents
* @return {String}
*/
const multilineRe = /(?:^|\r?\n) *(@[^\r\n]*?) *\r?\n *([^@\r\n\s][^@\r\n]+?) *\r?\n/g;
const propertyRe = /(?:^|\r?\n) *@(\S+) *([^\r\n]*)/g;
const stringStartRe = /(\r?\n|^) *\*/g;
const wsRe = /[\t ]+/g;

function extract(contents) {
const match = contents.match(docblockRe);
if (match) {
return match[0].replace(ltrimRe, '') || '';
}
return '';
return match ? match[0].replace(ltrimRe, '') || '' : '';
}

const commentStartRe = /^\/\*\*/;
const commentEndRe = /\*\/$/;
const wsRe = /[\t ]+/g;
const stringStartRe = /(\r?\n|^) *\*/g;
const multilineRe = /(?:^|\r?\n) *(@[^\r\n]*?) *\r?\n *([^@\r\n\s][^@\r\n]+?) *\r?\n/g;
const propertyRe = /(?:^|\r?\n) *@(\S+) *([^\r\n]*)/g;

/**
* @param {String} contents
* @return {Array}
*/
function parse(docblock) {
docblock = docblock
.replace(commentStartRe, '')
Expand All @@ -50,32 +38,13 @@ function parse(docblock) {
}
docblock = docblock.trim();

const result = [];
const result = Object.create(null);
let match;
while ((match = propertyRe.exec(docblock))) {
result.push([match[1], match[2]]);
}

return result;
}

/**
* Same as parse but returns an object of prop: value instead of array of paris
* If a property appers more than once the last one will be returned
*
* @param {String} contents
* @return {Object}
*/
function parseAsObject(docblock) {
const pairs = parse(docblock);
const result = {};
for (let i = 0; i < pairs.length; i++) {
result[pairs[i][0]] = pairs[i][1];
result[match[1]] = match[2];
}
return result;
}


exports.extract = extract;
exports.parse = parse;
exports.parseAsObject = parseAsObject;
52 changes: 17 additions & 35 deletions packages/jest-haste-map/src/lib/extractRequires.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,52 +8,34 @@
*/
'use strict';

const blockCommentRe = /\/\*[^]*?\*\//g;
const lineCommentRe = /\/\/.*/g;

/* eslint-disable max-len */
const replacePatterns = {
IMPORT_RE: /(\bimport\s+(?:[^'"]+\s+from\s+)??)(['"])([^'"]+)(\2)/g,
EXPORT_RE: /(\bexport\s+(?:[^'"]+\s+from\s+)??)(['"])([^'"]+)(\2)/g,
IMPORT_RE: /(\bimport\s+(?:[^'"]+\s+from\s+)??)(['"])([^'"]+)(\2)/g,
REQUIRE_EXTENSIONS_PATTERN: /(\b(?:require\s*?\.\s*?(?:requireActual|requireMock)|jest\s*?\.\s*?genMockFromModule)\s*?\(\s*?)(['"])([^'"]+)(\2\s*?\))/g,
REQUIRE_RE: /(\brequire\s*?\(\s*?)(['"])([^'"]+)(\2\s*?\))/g,
};
/* eslint-enable max-len */

/**
* Extract all required modules from a `code` string.
*/
const blockCommentRe = /\/\*[^]*?\*\//g;
const lineCommentRe = /\/\/.*/g;
function extractRequires(code) {
const cache = Object.create(null);
const deps = {
sync: [],
};

const addDependency = dep => {
if (!cache[dep]) {
cache[dep] = true;
deps.sync.push(dep);
}
const dependencies = new Set();
const addDependency = (match, pre, quot, dep, post) => {
dependencies.add(dep);
return match;
};

code = code
code
.replace(blockCommentRe, '')
.replace(lineCommentRe, '')
// Parse the sync dependencies this module has. When the module is
// required, all it's sync dependencies will be loaded into memory.
// Sync dependencies can be defined either using `require` or the ES6
// `import` or `export` syntaxes:
// var dep1 = require('dep1');
.replace(replacePatterns.IMPORT_RE, (match, pre, quot, dep, post) => {
addDependency(dep);
return match;
})
.replace(replacePatterns.EXPORT_RE, (match, pre, quot, dep, post) => {
addDependency(dep);
return match;
})
.replace(replacePatterns.REQUIRE_RE, (match, pre, quot, dep, post) => {
addDependency(dep);
return match;
});
.replace(replacePatterns.EXPORT_RE, addDependency)
.replace(replacePatterns.IMPORT_RE, addDependency)
.replace(replacePatterns.REQUIRE_EXTENSIONS_PATTERN, addDependency)
.replace(replacePatterns.REQUIRE_RE, addDependency);

return {code, deps};
return Array.from(dependencies);
}

module.exports = extractRequires;
Loading

0 comments on commit 275af0c

Please sign in to comment.