diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 9354dee014c..d66bcc8e09a 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -1,18 +1,139 @@ // @flow import {Bundler} from '@parcel/plugin'; +const ISOLATED_ENVS = new Set(['web-worker', 'service-worker']); +const OPTIONS = { + minBundles: 1, + minBundleSize: 30000, + maxParallelRequests: 5 +}; + export default new Bundler({ - async bundle(graph) { - let assets = Array.from(graph.nodes.values()) - .filter(node => node.type === 'asset') - .map(node => node.value); - - return [ - { - type: 'js', - filePath: 'bundle.js', - assets + bundle(assetGraph, bundleGraph) { + // RULES: + // 1. If dep.isAsync or dep.isEntry, start a new bundle group. + // 2. If an asset is a different type than the current bundle, make a parallel bundle in the same bundle group. + // 3. If an asset is already in a parent bundle in the same entry point, exclude from child bundles. + // 4. If an asset is only in separate isolated entry points (e.g. workers, different HTML pages), duplicate it. + // 5. If the sub-graph from an asset is >= 30kb, and the number of parallel requests in the bundle group is < 5, create a new bundle containing the sub-graph. + // 6. If two assets are always seen together, put them in the same extracted bundle. + + // Step 1: create bundles for each of the explicit code split points. + assetGraph.traverse((node, context) => { + if (node.type === 'dependency') { + let dep = node.value; + + // Start a new bundle if this is an async dependency, or entry point. + if (dep.isAsync || dep.isEntry) { + let isIsolated = + !context || dep.isEntry || ISOLATED_ENVS.has(dep.env.context); + let bundleGroup = {dependency: dep}; + bundleGraph.addBundleGroup( + isIsolated ? null : context.bundle, + bundleGroup + ); + + return {bundleGroup}; + } + } else if (node.type === 'asset') { + if (!context.bundle || node.value.type !== context.bundle.type) { + let bundle = assetGraph.createBundle(node.value); + + // If there is a current bundle, but this asset is of a different type, + // separate it out into a parallel bundle in the same bundle group. + if (context.bundle) { + let bundles = bundleGraph.getBundles(context.bundleGroup); + let existingBundle = bundles.find(b => b.type === node.value.type); + + // If there is an existing bundle of the asset's type, combine with that. + // Otherwise, a new bundle will be created. + if (existingBundle) { + existingBundle.assetGraph.merge(bundle.assetGraph); + return {bundleGroup: context.bundleGroup, bundle: existingBundle}; + } + } + + bundleGraph.addBundle(context.bundleGroup, bundle); + return {bundleGroup: context.bundleGroup, bundle}; + } + } + }); + + // Step 2: remove assets that are duplicated in a parent bundle + bundleGraph.traverseBundles(bundle => { + let assetGraph = bundle.assetGraph; + assetGraph.traverseAssets(asset => { + if (bundleGraph.isAssetInAncestorBundle(bundle, asset)) { + assetGraph.removeAsset(asset); + } + }); + }); + + // Step 3: Find duplicated assets in different bundle groups, and separate them into their own parallel bundles. + // If multiple assets are always seen together in the same bundles, combine them together. + + let candidateBundles = new Map(); + + assetGraph.traverseAssets((asset, context, traversal) => { + // If this asset is duplicated in the minimum number of bundles, it is a candidate to be separated into its own bundle. + let bundles = bundleGraph.findBundlesWithAsset(asset); + if (bundles.length > OPTIONS.minBundles) { + let bundle = assetGraph.createBundle(asset); + let size = bundle.assetGraph.getTotalSize(); + + let id = bundles.map(b => b.id).join(':'); + let candidate = candidateBundles.get(id); + if (!candidate) { + candidateBundles.set(id, {bundles, bundle, size}); + } else { + candidate.size += size; + candidate.bundle.assetGraph.merge(bundle.assetGraph); + } + + // Skip children from consideration since we added a parent already. + traversal.skipChildren(); + } + }); + + // Sort candidates by size (consider larger bundles first), and ensure they meet the size threshold + let sortedCandidates = Array.from(candidateBundles.values()) + .filter(bundle => bundle.size >= OPTIONS.minBundleSize) + .sort((a, b) => b.size - a.size); + + for (let {bundle, bundles} of sortedCandidates) { + // Find all bundle groups connected to the original bundles + let bundleGroups = bundles.reduce( + (arr, bundle) => arr.concat(bundleGraph.getBundleGroups(bundle)), + [] + ); + + // Check that all the bundle groups are inside the parallel request limit. + if ( + !bundleGroups.every( + group => + bundleGraph.getBundles(group).length < OPTIONS.maxParallelRequests + ) + ) { + continue; } - ]; + + // Remove all of the root assets from each of the original bundles + for (let asset of bundle.assetGraph.getEntryAssets()) { + for (let bundle of bundles) { + bundle.assetGraph.removeAsset(asset); + } + } + + // Create new bundle node and connect it to all of the original bundle groups + for (let bundleGroup of bundleGroups) { + bundleGraph.addBundle(bundleGroup, bundle); + } + } + + bundleGraph.dumpGraphViz(); + + bundleGraph.traverseBundles(bundle => { + bundle.assetGraph.dumpGraphViz(); + }); } }); diff --git a/packages/core/core/src/Asset.js b/packages/core/core/src/Asset.js index a6bc897935f..14e3788a8b8 100644 --- a/packages/core/core/src/Asset.js +++ b/packages/core/core/src/Asset.js @@ -13,6 +13,7 @@ import type { Config, PackageJSON } from '@parcel/types'; +import type Cache from '@parcel/cache'; import md5 from '@parcel/utils/md5'; import {loadConfig} from '@parcel/utils/config'; import createDependency from './createDependency'; @@ -28,7 +29,8 @@ type AssetOptions = { connectedFiles?: Array, output?: AssetOutput, env: Environment, - meta?: JSONObject + meta?: JSONObject, + cache?: Cache }; export default class Asset implements IAsset { @@ -41,8 +43,10 @@ export default class Asset implements IAsset { dependencies: Array; connectedFiles: Array; output: AssetOutput; + outputSize: number; env: Environment; meta: JSONObject; + #cache; // no type annotation because prettier dies... constructor(options: AssetOptions) { this.id = @@ -60,8 +64,10 @@ export default class Asset implements IAsset { ? options.connectedFiles.slice() : []; this.output = options.output || {code: this.code}; + this.outputSize = this.output.code.length; this.env = options.env; this.meta = options.meta || {}; + this.#cache = options.cache; } toJSON(): AssetOptions { @@ -74,6 +80,7 @@ export default class Asset implements IAsset { dependencies: this.dependencies, connectedFiles: this.connectedFiles, output: this.output, + outputSize: this.outputSize, env: this.env, meta: this.meta }; @@ -101,7 +108,7 @@ export default class Asset implements IAsset { } createChildAsset(result: TransformerResult) { - let code = result.code || (result.output && result.output.code) || ''; + let code = (result.output && result.output.code) || result.code || ''; let opts: AssetOptions = { hash: this.hash || md5(code), filePath: this.filePath, @@ -111,6 +118,7 @@ export default class Asset implements IAsset { env: mergeEnvironment(this.env, result.env), dependencies: this.dependencies, connectedFiles: this.connectedFiles, + output: result.output, meta: Object.assign({}, this.meta, result.meta) }; @@ -132,6 +140,11 @@ export default class Asset implements IAsset { } async getOutput() { + if (this.#cache) { + await this.#cache.readBlobs(this); + this.#cache = null; + } + return this.output; } diff --git a/packages/core/core/src/AssetGraph.js b/packages/core/core/src/AssetGraph.js index 3d2c3716391..a7b0c7c8d49 100644 --- a/packages/core/core/src/AssetGraph.js +++ b/packages/core/core/src/AssetGraph.js @@ -9,12 +9,17 @@ import type { FilePath, TransformerRequest, Target, - Environment + Environment, + Bundle, + GraphTraversalCallback, + DependencyResolution } from '@parcel/types'; import path from 'path'; import md5 from '@parcel/utils/md5'; import createDependency from './createDependency'; +let BUNDLECOUNT = 0; + export const nodeFromRootDir = (rootDir: string) => ({ id: rootDir, type: 'root', @@ -90,15 +95,15 @@ export default class AssetGraph extends Graph { incompleteNodes: Map; invalidNodes: Map; - constructor() { - super(); + constructor(opts) { + super(opts); this.incompleteNodes = new Map(); this.invalidNodes = new Map(); } initializeGraph({entries, targets, rootDir}: AssetGraphOpts) { let rootNode = nodeFromRootDir(rootDir); - this.addNode(rootNode); + this.setRootNode(rootNode); let depNodes = []; for (let entry of entries) { @@ -107,7 +112,8 @@ export default class AssetGraph extends Graph { createDependency( { moduleSpecifier: entry, - env: target.env + env: target.env, + isEntry: true }, path.resolve(rootDir, 'index') ) @@ -220,6 +226,96 @@ export default class AssetGraph extends Graph { } } + getDependencies(asset: Asset): Array { + let node = this.getNode(asset.id); + if (!node) { + return []; + } + + return this.getNodesConnectedFrom(node).map(node => node.value); + } + + getDependencyResolution(dep: Dependency): DependencyResolution { + let depNode = this.getNode(dep.id); + if (!depNode) { + return {}; + } + + let node = this.getNodesConnectedFrom(depNode)[0]; + if (node.type === 'transformer_request') { + let assetNode = this.getNodesConnectedFrom(node).find( + node => node.type === 'asset' || node.type === 'asset_reference' + ); + if (assetNode) { + return {asset: assetNode.value}; + } + } else if (node.type === 'bundle_group') { + let bundles = this.getNodesConnectedFrom(node).map(node => node.value); + return {bundles}; + } + + return {}; + } + + traverseAssets(visit: GraphTraversalCallback, startNode: ?Node) { + return this.traverse((node, ...args) => { + if (node.type === 'asset') { + return visit(node.value, ...args); + } + }, startNode); + } + + createBundle(asset: Asset): Bundle { + let assetNode = this.getNode(asset.id); + if (!assetNode) { + throw new Error('Cannot get bundle for non-existant asset'); + } + + let graph = this.getSubGraph(assetNode); + graph.setRootNode({ + type: 'root', + id: 'root', + value: null + }); + + graph.addEdge({from: 'root', to: assetNode.id}); + return { + id: 'bundle:' + asset.id, + type: asset.type, + assetGraph: graph, + filePath: 'bundle.' + BUNDLECOUNT++ + '.js' + }; + } + + getTotalSize(asset?: Asset): number { + let size = 0; + let assetNode = asset ? this.getNode(asset.id) : null; + this.traverseAssets(asset => { + size += asset.outputSize; + }, assetNode); + + return size; + } + + getEntryAssets(): Array { + return this.getNodesConnectedFrom(this.getRootNode()).map( + node => node.value + ); + } + + removeAsset(asset: Asset) { + let assetNode = this.getNode(asset.id); + if (!assetNode) { + return; + } + + this.replaceNode(assetNode, { + type: 'asset_reference', + id: 'asset_reference:' + assetNode.id, + value: asset + }); + } + async dumpGraphViz() { let graphviz = require('graphviz'); let tempy = require('tempy'); @@ -237,14 +333,6 @@ export default class AssetGraph extends Graph { }; let nodes = Array.from(this.nodes.values()); - let root; - for (let node of nodes) { - if (node.type === 'root') { - root = node; - break; - } - } - let rootPath = root ? root.value : '/'; for (let node of nodes) { let n = g.addNode(node.id); @@ -260,11 +348,10 @@ export default class AssetGraph extends Graph { let parts = []; if (node.value.isEntry) parts.push('entry'); if (node.value.isAsync) parts.push('async'); - if (node.value.isIncluded) parts.push('included'); if (node.value.isOptional) parts.push('optional'); - if (parts.length) label += '(' + parts.join(', ') + ')'; + if (parts.length) label += ' (' + parts.join(', ') + ')'; if (node.value.env) label += ` (${getEnvDescription(node.value.env)})`; - } else if (node.type === 'asset') { + } else if (node.type === 'asset' || node.type === 'asset_reference') { label += path.basename(node.value.filePath) + '#' + node.value.type; } else if (node.type === 'file') { label += path.basename(node.value.filePath); @@ -272,6 +359,21 @@ export default class AssetGraph extends Graph { label += path.basename(node.value.filePath) + ` (${getEnvDescription(node.value.env)})`; + } else if (node.type === 'bundle') { + let rootAssets = node.value.assetGraph.getNodesConnectedFrom( + node.value.assetGraph.getRootNode() + ); + label += rootAssets + .map(asset => { + let parts = asset.value.filePath.split(path.sep); + let index = parts.lastIndexOf('node_modules'); + if (index >= 0) { + return parts[index + 1]; + } + + return path.basename(asset.value.filePath); + }) + .join(', '); } else { // label += node.id; label = node.type; diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js new file mode 100644 index 00000000000..0c797ab562a --- /dev/null +++ b/packages/core/core/src/BundleGraph.js @@ -0,0 +1,140 @@ +// @flow +import type { + Asset, + Bundle, + BundleGroup, + GraphTraversalCallback +} from '@parcel/types'; +import AssetGraph from './AssetGraph'; + +const getBundleGroupId = (bundleGroup: BundleGroup) => + 'bundle_group:' + bundleGroup.dependency.id; + +export default class BundleGraph extends AssetGraph { + constructor() { + super(); + this.setRootNode({ + type: 'root', + id: 'root', + value: null + }); + } + + addBundleGroup(parentBundle: ?Bundle, bundleGroup: BundleGroup) { + let node = { + id: getBundleGroupId(bundleGroup), + type: 'bundle_group', + value: bundleGroup + }; + + // Add a connection from the dependency to the new bundle group in all bundles + this.traverse(bundle => { + if (bundle.type === 'bundle') { + let depNode = bundle.value.assetGraph.getNode( + bundleGroup.dependency.id + ); + if (depNode) { + bundle.value.assetGraph.replaceNodesConnectedTo(depNode, [node]); + } + } + }); + + this.addNode(node); + this.addEdge({ + from: !parentBundle ? 'root' : parentBundle.id, + to: node.id + }); + } + + addBundle(bundleGroup: BundleGroup, bundle: Bundle) { + let bundleGroupId = getBundleGroupId(bundleGroup); + let bundleNode = { + id: bundle.id, + type: 'bundle', + value: bundle + }; + + this.addNode(bundleNode); + this.addEdge({ + from: bundleGroupId, + to: bundleNode.id + }); + + // Add a connection from the bundle group to the bundle in all bundles + this.traverse(node => { + if ( + node.type === 'bundle' && + node.value.assetGraph.hasNode(bundleGroupId) + ) { + node.value.assetGraph.addNode(bundleNode); + node.value.assetGraph.addEdge({ + from: bundleGroupId, + to: bundleNode.id + }); + } + }); + } + + getBundles(bundleGroup: BundleGroup): Array { + let bundleGroupId = getBundleGroupId(bundleGroup); + let node = this.getNode(bundleGroupId); + if (!node) { + return []; + } + + return this.getNodesConnectedFrom(node).map(node => node.value); + } + + getBundleGroups(bundle: Bundle): Array { + let node = this.getNode(bundle.id); + if (!node) { + return []; + } + + return this.getNodesConnectedTo(node).map(node => node.value); + } + + isAssetInAncestorBundle(bundle: Bundle, asset: Asset): boolean { + let bundleNode = this.getNode(bundle.id); + if (!bundleNode) { + return false; + } + + let ret = null; + this.traverseAncestors(bundleNode, (node, context, traversal) => { + // Skip starting node + if (node === bundleNode) { + return; + } + + // If this is the first bundle we've seen, initialize result to true + if (node.type === 'bundle' && ret === null) { + ret = true; + } + + if (node.type === 'bundle' && !node.value.assetGraph.hasNode(asset.id)) { + ret = false; + traversal.stop(); + } + }); + + return !!ret; + } + + findBundlesWithAsset(asset: Asset): Array { + return Array.from(this.nodes.values()) + .filter( + node => + node.type === 'bundle' && node.value.assetGraph.hasNode(asset.id) + ) + .map(node => node.value); + } + + traverseBundles(visit: GraphTraversalCallback): any { + return this.traverse((node, ...args) => { + if (node.type === 'bundle') { + return visit(node.value, ...args); + } + }); + } +} diff --git a/packages/core/core/src/BundlerRunner.js b/packages/core/core/src/BundlerRunner.js index 2a1b2d9748e..885e7ac0e98 100644 --- a/packages/core/core/src/BundlerRunner.js +++ b/packages/core/core/src/BundlerRunner.js @@ -1,7 +1,11 @@ +// @flow import path from 'path'; -import Config from './Config'; +import type Config from './Config'; +import BundleGraph from './BundleGraph'; export default class BundlerRunner { + config: Config; + constructor(opts) { this.config = opts.config; } @@ -9,6 +13,8 @@ export default class BundlerRunner { async bundle(graph /* , opts */) { let bundler = await this.config.getBundler(); - return bundler.bundle(graph); + let bundleGraph = new BundleGraph(); + bundler.bundle(graph, bundleGraph); + return bundleGraph; } } diff --git a/packages/core/core/src/Graph.js b/packages/core/core/src/Graph.js index adbc51a2e9c..32766f5c983 100644 --- a/packages/core/core/src/Graph.js +++ b/packages/core/core/src/Graph.js @@ -1,5 +1,6 @@ // @flow 'use strict'; +import type {TraversalContext} from '@parcel/types'; export type NodeId = string; @@ -22,10 +23,20 @@ type GraphUpdates = { export default class Graph { nodes: Map; edges: Set; + rootNodeId: ?NodeId; - constructor() { - this.nodes = new Map(); - this.edges = new Set(); + constructor(opts = {}) { + this.nodes = new Map(opts.nodes); + this.edges = new Set(opts.edges); + this.rootNodeId = opts.rootNodeId || null; + } + + toJSON() { + return { + nodes: [...this.nodes], + edges: [...this.edges], + rootNodeId: this.rootNodeId + }; } addNode(node: Node) { @@ -41,6 +52,15 @@ export default class Graph { return this.nodes.get(id); } + setRootNode(node: Node) { + this.addNode(node); + this.rootNodeId = node.id; + } + + getRootNode(): ?Node { + return this.rootNodeId ? this.getNode(this.rootNodeId) : null; + } + addEdge(edge: Edge) { this.edges.add(edge); return edge; @@ -77,12 +97,24 @@ export default class Graph { } // Removes node and any edges coming from that node - removeNode(node: Node): Graph { - let removed = new Graph(); + removeNode(node: Node): this { + let removed = new this.constructor(); this.nodes.delete(node.id); removed.addNode(node); + for (let edge of this.edges) { + if (edge.from === node.id || edge.to === node.id) { + removed.merge(this.removeEdge(edge)); + } + } + + return removed; + } + + removeEdges(node: Node): this { + let removed = new this.constructor(); + for (let edge of this.edges) { if (edge.from === node.id) { removed.merge(this.removeEdge(edge)); @@ -93,8 +125,8 @@ export default class Graph { } // Removes edge and node the edge is to if the node is orphaned - removeEdge(edge: Edge): Graph { - let removed = new Graph(); + removeEdge(edge: Edge): this { + let removed = new this.constructor(); this.edges.delete(edge); removed.addEdge(edge); @@ -119,11 +151,23 @@ export default class Graph { return true; } + replaceNode(fromNode: Node, toNode: Node) { + this.addNode(toNode); + + for (let edge of this.edges) { + if (edge.to === fromNode.id) { + edge.to = toNode.id; + } + } + + this.removeNode(fromNode); + } + // Update a node's downstream nodes making sure to prune any orphaned branches // Also keeps track of all added and removed edges and nodes replaceNodesConnectedTo(fromNode: Node, toNodes: Array): GraphUpdates { - let removed = new Graph(); - let added = new Graph(); + let removed = new this.constructor(); + let added = new this.constructor(); let edgesBefore = Array.from(this.edges).filter( edge => edge.from === fromNode.id @@ -154,4 +198,128 @@ export default class Graph { return {removed, added}; } + + traverse( + visit: (node: Node, context?: any, traversal: TraversalContext) => any, + startNode: ?Node + ) { + return this.dfs({ + visit, + startNode, + getChildren: this.getNodesConnectedFrom.bind(this) + }); + } + + traverseAncestors( + startNode: Node, + visit: (node: Node, context?: any, traversal: TraversalContext) => any + ) { + return this.dfs({ + visit, + startNode, + getChildren: this.getNodesConnectedTo.bind(this) + }); + } + + dfs({ + visit, + startNode, + getChildren + }: { + visit(node: Node, context?: any, traversal: TraversalContext): any, + getChildren(node: Node): Array, + startNode?: ?Node + }): ?Node { + let root = startNode || this.getRootNode(); + if (!root) { + return null; + } + + let visited = new Set(); + let stopped = false; + let skipped = false; + let ctx: TraversalContext = { + skipChildren() { + skipped = true; + }, + stop() { + stopped = true; + } + }; + + let walk = (node, context) => { + visited.add(node); + + skipped = false; + let newContext = visit(node, context, ctx); + if (typeof newContext !== 'undefined') { + context = newContext; + } + + if (skipped) { + return; + } + + if (stopped) { + return context; + } + + for (let child of getChildren(node)) { + if (visited.has(child)) { + continue; + } + + visited.add(child); + let result = walk(child, context); + if (stopped) { + return result; + } + } + }; + + return walk(root); + } + + bfs(visit: (node: Node) => ?boolean): ?Node { + let root = this.getRootNode(); + if (!root) { + return null; + } + + let queue: Array = [root]; + let visited = new Set([root]); + + while (queue.length > 0) { + let node = queue.shift(); + let stop = visit(node); + if (stop === true) { + return node; + } + + for (let child of this.getNodesConnectedFrom(node)) { + if (!visited.has(child)) { + visited.add(child); + queue.push(child); + } + } + } + + return null; + } + + getSubGraph(node: Node): this { + let graph = new this.constructor(); + graph.setRootNode(node); + + this.traverse(node => { + graph.addNode(node); + + let edges = Array.from(this.edges).filter(edge => edge.from === node.id); + for (let edge of edges) { + graph.addEdge(edge); + } + }, node); + + return graph; + } } diff --git a/packages/core/core/src/PackagerRunner.js b/packages/core/core/src/PackagerRunner.js index a444e353b0e..ca65a1881b5 100644 --- a/packages/core/core/src/PackagerRunner.js +++ b/packages/core/core/src/PackagerRunner.js @@ -5,6 +5,8 @@ import {mkdirp, writeFile} from '@parcel/fs'; import path from 'path'; import type {Bundle, CLIOptions, Blob, FilePath} from '@parcel/types'; import clone from 'clone'; +import AssetGraph from './AssetGraph'; +import Asset from './Asset'; type Opts = { config: Config, @@ -27,6 +29,14 @@ export default class PackagerRunner { } async writeBundle(bundle: Bundle) { + // deserialize asset graph from JSON + bundle.assetGraph = new AssetGraph(bundle.assetGraph); + bundle.assetGraph.traverse(node => { + if (node.type === 'asset') { + node.value = new Asset({...node.value, cache: this.cache}); + } + }); + let contents = await this.package(bundle); contents = await this.optimize(bundle, contents); @@ -45,16 +55,6 @@ export default class PackagerRunner { async package(bundle: Bundle): Promise { let packager = await this.config.getPackager(bundle.filePath); - - // Read the contents of each asset in the bundle from the cache. - // This mutates the assets, so clone the bundle first. - bundle = clone(bundle); - await Promise.all( - bundle.assets.map(async asset => { - await this.cache.readBlobs(asset); - }) - ); - return await packager.package(bundle, this.cliOpts); } diff --git a/packages/core/core/src/Parcel.js b/packages/core/core/src/Parcel.js index 36365981156..8e4bf589b80 100644 --- a/packages/core/core/src/Parcel.js +++ b/packages/core/core/src/Parcel.js @@ -7,6 +7,7 @@ import AssetGraph from './AssetGraph'; import {Node} from './Graph'; import type { Bundle, + BundleGraph, CLIOptions, Dependency, File, @@ -128,8 +129,8 @@ export default class Parcel { await this.updateGraph({signal}); await this.completeGraph({signal}); await this.graph.dumpGraphViz(); - let bundles = await this.bundle(); - await this.package(bundles); + let bundleGraph = await this.bundle(); + await this.package(bundleGraph); if (!this.watcher) { await this.farm.end(); @@ -218,8 +219,12 @@ export default class Parcel { return this.bundlerRunner.bundle(this.graph); } - // TODO: implement bundle types - package(bundles: any[]) { - return Promise.all(bundles.map(bundle => this.runPackage(bundle))); + package(bundleGraph: BundleGraph) { + let promises = []; + bundleGraph.traverseBundles(bundle => { + promises.push(this.runPackage(bundle)); + }); + + return Promise.all(promises); } } diff --git a/packages/core/types/index.js b/packages/core/types/index.js index d17bec0cbce..3b54a41cd1b 100644 --- a/packages/core/types/index.js +++ b/packages/core/types/index.js @@ -97,15 +97,12 @@ export type DependencyOptions = { isAsync?: boolean, isEntry?: boolean, isOptional?: boolean, - isIncluded?: boolean, - isConfig?: boolean, loc?: SourceLocation, env?: Environment, meta?: JSONObject }; -export type Dependency = { - ...DependencyOptions, +export type Dependency = DependencyOptions & { id: string, env: Environment, @@ -144,6 +141,7 @@ export interface Asset { getPackage(): Promise; addDependency(dep: DependencyOptions): string; createChildAsset(result: TransformerResult): Asset; + getOutput(): AssetOutput; } export type AssetOutput = { @@ -204,17 +202,60 @@ export type CacheEntry = { initialAssets: ?Array // Initial assets, pre-post processing }; +export interface TraversalContext { + skipChildren(): void; + stop(): void; +} + +export type GraphTraversalCallback = ( + asset: T, + context?: any, + traversal: TraversalContext +) => any; + +interface Graph { + merge(graph: Graph): void; +} + +export type DependencyResolution = { + asset?: Asset, + bundles?: Array +}; + // TODO: what do we want to expose here? -interface AssetGraph {} +interface AssetGraph extends Graph { + traverseAssets(visit: GraphTraversalCallback): any; + createBundle(asset: Asset): Bundle; + getTotalSize(asset?: Asset): number; + getEntryAssets(): Array; + removeAsset(asset: Asset): void; + getDependencies(asset: Asset): Array; + getDependencyResolution(dependency: Dependency): DependencyResolution; +} + +export type BundleGroup = { + dependency: Dependency +}; export type Bundle = { + id: string, type: string, - assets: Array, + assetGraph: AssetGraph, filePath?: FilePath }; +export interface BundleGraph { + addBundleGroup(parentBundle: ?Bundle, bundleGroup: BundleGroup): void; + addBundle(bundleGroup: BundleGroup, bundle: Bundle): void; + isAssetInAncestorBundle(bundle: Bundle, asset: Asset): boolean; + findBundlesWithAsset(asset: Asset): Array; + getBundles(bundleGroup: BundleGroup): Array; + getBundleGroups(bundle: Bundle): Array; + traverseBundles(visit: GraphTraversalCallback): any; +} + export type Bundler = { - bundle(graph: AssetGraph, opts: CLIOptions): Array + bundle(graph: AssetGraph, bundleGraph: BundleGraph, opts: CLIOptions): void }; export type Namer = { diff --git a/packages/dev/babel-preset/common.js b/packages/dev/babel-preset/common.js index 481fb3b3736..dacc55ef5af 100644 --- a/packages/dev/babel-preset/common.js +++ b/packages/dev/babel-preset/common.js @@ -1,5 +1,6 @@ const flow = require('@babel/preset-flow'); module.exports = { - presets: [flow] + presets: [flow], + plugins: [require('@babel/plugin-proposal-class-properties')] }; diff --git a/packages/dev/babel-preset/legacy.js b/packages/dev/babel-preset/legacy.js index 4ddb7148bac..1ff78e1903f 100644 --- a/packages/dev/babel-preset/legacy.js +++ b/packages/dev/babel-preset/legacy.js @@ -11,5 +11,6 @@ module.exports = () => ({ } ], ...common.presets - ] + ], + plugins: common.plugins }); diff --git a/packages/dev/babel-preset/modern.js b/packages/dev/babel-preset/modern.js index 794664620e7..c1894fa1444 100644 --- a/packages/dev/babel-preset/modern.js +++ b/packages/dev/babel-preset/modern.js @@ -11,5 +11,6 @@ module.exports = () => ({ } ], ...common.presets - ] + ], + plugins: common.plugins }); diff --git a/packages/dev/babel-preset/package.json b/packages/dev/babel-preset/package.json index d79acb2f302..b5040147e06 100644 --- a/packages/dev/babel-preset/package.json +++ b/packages/dev/babel-preset/package.json @@ -3,6 +3,7 @@ "version": "2.0.0", "license": "MIT", "dependencies": { + "@babel/plugin-proposal-class-properties": "^7.1.0", "@babel/preset-env": "^7.1.0", "@babel/preset-flow": "^7.0.0" }, diff --git a/packages/examples/simple/package.json b/packages/examples/simple/package.json index 82cb597a670..773bc9b4447 100644 --- a/packages/examples/simple/package.json +++ b/packages/examples/simple/package.json @@ -10,16 +10,8 @@ "@parcel/babel-register": "^2.0.0", "@parcel/core": "^2.0.0" }, - "browser": "dist/browser.js", "browserModern": "dist/modern.js", "targets": { - "browser": { - "engines": { - "browsers": [ - "> 1%" - ] - } - }, "browserModern": { "engines": { "browsers": [ @@ -27,5 +19,10 @@ ] } } + }, + "dependencies": { + "lodash": "^4.17.11", + "react": "^16.6.3", + "react-dom": "^16.6.3" } } diff --git a/packages/examples/simple/src/async.js b/packages/examples/simple/src/async.js new file mode 100644 index 00000000000..ecc11eea642 --- /dev/null +++ b/packages/examples/simple/src/async.js @@ -0,0 +1,2 @@ +console.log(require('react')); +require('lodash'); diff --git a/packages/examples/simple/src/async2.js b/packages/examples/simple/src/async2.js new file mode 100644 index 00000000000..ecc11eea642 --- /dev/null +++ b/packages/examples/simple/src/async2.js @@ -0,0 +1,2 @@ +console.log(require('react')); +require('lodash'); diff --git a/packages/examples/simple/src/index.js b/packages/examples/simple/src/index.js index 224d00f6d0c..bb9772db59a 100644 --- a/packages/examples/simple/src/index.js +++ b/packages/examples/simple/src/index.js @@ -1,7 +1,12 @@ -const message = require('./message'); -const fs = require('fs'); +import('./async'); +import('./async2'); -console.log(message); // eslint-disable-line no-console -console.log(fs.readFileSync(__dirname + '/test.txt', 'utf8')); +new Worker('./worker.js'); + +// const message = require('./message'); +// const fs = require('fs'); + +// console.log(message); // eslint-disable-line no-console +// console.log(fs.readFileSync(__dirname + '/test.txt', 'utf8')); class Test {} diff --git a/packages/examples/simple/src/worker.js b/packages/examples/simple/src/worker.js new file mode 100644 index 00000000000..a8baa369c92 --- /dev/null +++ b/packages/examples/simple/src/worker.js @@ -0,0 +1 @@ +module.exports = 'worker'; diff --git a/packages/packagers/js/src/JSPackager.js b/packages/packagers/js/src/JSPackager.js index 5e25d358b89..6d38028bc3d 100644 --- a/packages/packagers/js/src/JSPackager.js +++ b/packages/packagers/js/src/JSPackager.js @@ -11,38 +11,49 @@ const PRELUDE = fs export default new Packager({ async package(bundle) { - let assets = bundle.assets - .map((asset, i) => { - let deps = {}; - - for (let dep of asset.dependencies) { - let resolvedAsset = bundle.assets.find( - a => a.filePath === dep.resolvedPath - ); - if (resolvedAsset) { - deps[dep.moduleSpecifier] = resolvedAsset.id; - } + let promises = []; + bundle.assetGraph.traverseAssets(asset => { + promises.push(asset.getOutput()); + }); + let outputs = await Promise.all(promises); + + let assets = ''; + let i = 0; + bundle.assetGraph.traverseAssets(asset => { + let deps = {}; + + let dependencies = bundle.assetGraph.getDependencies(asset); + for (let dep of dependencies) { + let resolved = bundle.assetGraph.getDependencyResolution(dep); + if (resolved.bundles) { + deps[dep.moduleSpecifier] = resolved.bundles.map(b => b.id); + } else if (resolved.asset) { + deps[dep.moduleSpecifier] = resolved.asset.id; } + } - let wrapped = i === 0 ? '' : ','; - wrapped += - JSON.stringify(asset.id) + - ':[function(require,module,exports) {\n' + - (asset.output.code || '') + - '\n},'; - wrapped += JSON.stringify(deps); - wrapped += ']'; + let output = outputs[i]; + let wrapped = i === 0 ? '' : ','; + wrapped += + JSON.stringify(asset.id) + + ':[function(require,module,exports) {\n' + + (output.code || '') + + '\n},'; + wrapped += JSON.stringify(deps); + wrapped += ']'; - return wrapped; - }) - .join(''); + i++; + assets += wrapped; + }); return ( PRELUDE + '({' + assets + '},{},' + - JSON.stringify([bundle.assets[0].id]) + + JSON.stringify( + bundle.assetGraph.getEntryAssets().map(asset => asset.id) + ) + ', ' + 'null' + ')' diff --git a/packages/transformers/js/src/JSTransformer.js b/packages/transformers/js/src/JSTransformer.js index db0e5414e33..2c065ae497d 100644 --- a/packages/transformers/js/src/JSTransformer.js +++ b/packages/transformers/js/src/JSTransformer.js @@ -63,17 +63,17 @@ export default new Transformer({ return [asset]; } + // Inline environment variables + if (asset.env.context === 'browser' && ENV_RE.test(asset.code)) { + walk.simple(asset.ast.program, envVisitor, asset); + } + // Collect dependencies if (canHaveDependencies(asset.code)) { walk.ancestor(asset.ast.program, collectDependencies, asset); } if (asset.env.context === 'browser') { - // Inline environment variables - if (ENV_RE.test(asset.code)) { - walk.simple(asset.ast.program, envVisitor, asset); - } - // Inline fs calls let fsDep = asset.dependencies.find(dep => dep.moduleSpecifier === 'fs'); if (fsDep && FS_RE.test(asset.code)) { @@ -89,9 +89,9 @@ export default new Transformer({ } // Insert node globals - if (GLOBAL_RE.test(asset.code)) { - walk.ancestor(asset.ast.program, insertGlobals, asset); - } + // if (GLOBAL_RE.test(asset.code)) { + // walk.ancestor(asset.ast.program, insertGlobals, asset); + // } } // Do some transforms diff --git a/packages/transformers/js/src/visitors/dependencies.js b/packages/transformers/js/src/visitors/dependencies.js index 4e9b798ae27..ccef40093b2 100644 --- a/packages/transformers/js/src/visitors/dependencies.js +++ b/packages/transformers/js/src/visitors/dependencies.js @@ -53,12 +53,12 @@ export default { types.isStringLiteral(args[0]); if (isDynamicImport) { - asset.addDependency({moduleSpecifier: '_bundle_loader'}); + // asset.addDependency({moduleSpecifier: '_bundle_loader'}); addDependency(asset, args[0], {isAsync: true}); - node.callee = requireTemplate().expression; - node.arguments[0] = argTemplate({MODULE: args[0]}).expression; - asset.ast.isDirty = true; + // node.callee = requireTemplate().expression; + // node.arguments[0] = argTemplate({MODULE: args[0]}).expression; + // asset.ast.isDirty = true; return; } @@ -71,7 +71,7 @@ export default { // https://developers.google.com/web/fundamentals/primers/service-workers/lifecycle#avoid_changing_the_url_of_your_service_worker_script addURLDependency(asset, args[0], { isEntry: true, - context: 'service-worker' + env: {context: 'service-worker'} }); return; } @@ -87,7 +87,7 @@ export default { types.isStringLiteral(args[0]); if (isWebWorker) { - addURLDependency(asset, args[0], {context: 'web-worker'}); + addURLDependency(asset, args[0], {env: {context: 'web-worker'}}); return; } } @@ -190,6 +190,10 @@ function addDependency(asset, node, opts = {}) { function addURLDependency(asset, node, opts = {}) { opts.loc = node.loc && node.loc.start; - node.value = asset.addDependency({moduleSpecifier: node.value, ...opts}); + node.value = asset.addDependency({ + moduleSpecifier: node.value, + isAsync: true, + ...opts + }); asset.ast.isDirty = true; } diff --git a/packages/transformers/js/src/visitors/env.js b/packages/transformers/js/src/visitors/env.js index 4c71ad3f89c..048399aa99f 100644 --- a/packages/transformers/js/src/visitors/env.js +++ b/packages/transformers/js/src/visitors/env.js @@ -11,7 +11,7 @@ export default { let value = types.valueToNode(prop); morph(node, value); asset.ast.isDirty = true; - asset.meta.env[key.value] = process.env[key.value]; + // asset.meta.env[key.value] = process.env[key.value]; } } } diff --git a/yarn.lock b/yarn.lock index 3552e9f97ca..faa040fbe9d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -297,6 +297,18 @@ "@babel/helper-remap-async-to-generator" "^7.1.0" "@babel/plugin-syntax-async-generators" "^7.0.0" +"@babel/plugin-proposal-class-properties@^7.1.0": + version "7.1.0" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.1.0.tgz#9af01856b1241db60ec8838d84691aa0bd1e8df4" + integrity sha512-/PCJWN+CKt5v1xcGn4vnuu13QDoV+P7NcICP44BoonAJoPSGwVkgrXihFIQGiEjjPlUDBIw1cM7wYFLARS2/hw== + dependencies: + "@babel/helper-function-name" "^7.1.0" + "@babel/helper-member-expression-to-functions" "^7.0.0" + "@babel/helper-optimise-call-expression" "^7.0.0" + "@babel/helper-plugin-utils" "^7.0.0" + "@babel/helper-replace-supers" "^7.1.0" + "@babel/plugin-syntax-class-properties" "^7.0.0" + "@babel/plugin-proposal-json-strings@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.0.0.tgz#3b4d7b5cf51e1f2e70f52351d28d44fc2970d01e" @@ -337,6 +349,13 @@ dependencies: "@babel/helper-plugin-utils" "^7.0.0" +"@babel/plugin-syntax-class-properties@^7.0.0": + version "7.0.0" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.0.0.tgz#e051af5d300cbfbcec4a7476e37a803489881634" + integrity sha512-cR12g0Qzn4sgkjrbrzWy2GE7m9vMl/sFkqZ3gIpAQdrvPDnLM8180i+ANDFIXfjHo9aqp0ccJlQ0QNZcFUbf9w== + dependencies: + "@babel/helper-plugin-utils" "^7.0.0" + "@babel/plugin-syntax-export-default-from@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-export-default-from/-/plugin-syntax-export-default-from-7.0.0.tgz#084b639bce3d42f3c5bf3f68ccb42220bb2d729d"