From e517b40901cf41181b73086d613c5d0438dd6f8b Mon Sep 17 00:00:00 2001 From: patak Date: Tue, 28 Feb 2023 00:02:52 +0100 Subject: [PATCH 1/3] feat: cancellable scan during optimization --- packages/vite/src/node/optimizer/index.ts | 44 +++++---- packages/vite/src/node/optimizer/optimizer.ts | 11 ++- packages/vite/src/node/optimizer/scan.ts | 89 +++++++++++-------- 3 files changed, 90 insertions(+), 54 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index d04939906326e8..1a545d2514e81c 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -239,7 +239,7 @@ export async function optimizeDeps( return cachedMetadata } - const deps = await discoverProjectDependencies(config) + const deps = await (await discoverProjectDependencies(config)).result const depsString = depsLogString(Object.keys(deps)) log(colors.green(`Optimizing dependencies:\n ${depsString}`)) @@ -382,24 +382,32 @@ export function loadCachedDepOptimizationMetadata( */ export async function discoverProjectDependencies( config: ResolvedConfig, -): Promise> { - const { deps, missing } = await scanImports(config) - - const missingIds = Object.keys(missing) - if (missingIds.length) { - throw new Error( - `The following dependencies are imported but could not be resolved:\n\n ${missingIds - .map( - (id) => - `${colors.cyan(id)} ${colors.white( - colors.dim(`(imported by ${missing[id]})`), - )}`, +): Promise<{ + cancel: () => Promise + result: Promise> +}> { + const { cancel, result } = await scanImports(config) + + return { + cancel, + result: result.then(({ deps, missing }) => { + const missingIds = Object.keys(missing) + if (missingIds.length) { + throw new Error( + `The following dependencies are imported but could not be resolved:\n\n ${missingIds + .map( + (id) => + `${colors.cyan(id)} ${colors.white( + colors.dim(`(imported by ${missing[id]})`), + )}`, + ) + .join(`\n `)}\n\nAre they installed?`, ) - .join(`\n `)}\n\nAre they installed?`, - ) - } + } - return deps + return deps + }), + } } export function toDiscoveredDependencies( @@ -679,7 +687,7 @@ export async function findKnownImports( config: ResolvedConfig, ssr: boolean, ): Promise { - const deps = (await scanImports(config)).deps + const deps = (await (await scanImports(config)).result).deps await addManuallyIncludedOptimizeDeps(deps, config, ssr) return Object.keys(deps) } diff --git a/packages/vite/src/node/optimizer/optimizer.ts b/packages/vite/src/node/optimizer/optimizer.ts index 12721b140787c6..b125ed434233c7 100644 --- a/packages/vite/src/node/optimizer/optimizer.ts +++ b/packages/vite/src/node/optimizer/optimizer.ts @@ -161,11 +161,18 @@ async function createDepsOptimizer( let firstRunCalled = !!cachedMetadata let postScanOptimizationResult: Promise | undefined + let discover: + | { + cancel: () => Promise + result: Promise> + } + | undefined let optimizingNewDeps: Promise | undefined async function close() { closed = true await Promise.allSettled([ + discover?.cancel(), depsOptimizer.scanProcessing, postScanOptimizationResult, optimizingNewDeps, @@ -204,7 +211,9 @@ async function createDepsOptimizer( try { debug(colors.green(`scanning for dependencies...`)) - const deps = await discoverProjectDependencies(config) + discover = await discoverProjectDependencies(config) + const deps = await discover.result + discover = undefined debug( colors.green( diff --git a/packages/vite/src/node/optimizer/scan.ts b/packages/vite/src/node/optimizer/scan.ts index a7de005573fd92..c933807e957e46 100644 --- a/packages/vite/src/node/optimizer/scan.ts +++ b/packages/vite/src/node/optimizer/scan.ts @@ -3,7 +3,7 @@ import path from 'node:path' import { performance } from 'node:perf_hooks' import glob from 'fast-glob' import type { Loader, OnLoadResult, Plugin } from 'esbuild' -import { build, formatMessages, transform } from 'esbuild' +import esbuild, { formatMessages, transform } from 'esbuild' import colors from 'picocolors' import type { ResolvedConfig } from '..' import { @@ -48,8 +48,11 @@ export const importsRE = /(? - missing: Record + cancel: () => Promise + result: Promise<{ + deps: Record + missing: Record + }> }> { // Only used to scan non-ssr code @@ -93,7 +96,10 @@ export async function scanImports(config: ResolvedConfig): Promise<{ ), ) } - return { deps: {}, missing: {} } + return { + cancel: () => Promise.resolve(), + result: Promise.resolve({ deps: {}, missing: {} }), + } } else { debug(`Crawling dependencies using entries:\n ${entries.join('\n ')}`) } @@ -106,44 +112,57 @@ export async function scanImports(config: ResolvedConfig): Promise<{ const { plugins = [], ...esbuildOptions } = config.optimizeDeps?.esbuildOptions ?? {} - try { - await build({ - absWorkingDir: process.cwd(), - write: false, - stdin: { - contents: entries.map((e) => `import ${JSON.stringify(e)}`).join('\n'), - loader: 'js', - }, - bundle: true, - format: 'esm', - logLevel: 'silent', - plugins: [...plugins, plugin], - ...esbuildOptions, + const esbuildContext = await esbuild.context({ + absWorkingDir: process.cwd(), + write: false, + stdin: { + contents: entries.map((e) => `import ${JSON.stringify(e)}`).join('\n'), + loader: 'js', + }, + bundle: true, + format: 'esm', + logLevel: 'silent', + plugins: [...plugins, plugin], + ...esbuildOptions, + }) + + const result = esbuildContext + .rebuild() + .then(() => { + return { + // Ensure a fixed order so hashes are stable and improve logs + deps: orderedDependencies(deps), + missing, + } }) - } catch (e) { - const prependMessage = colors.red(`\ + .catch(async (e) => { + const prependMessage = colors.red(`\ Failed to scan for dependencies from entries: ${entries.join('\n')} `) - if (e.errors) { - const msgs = await formatMessages(e.errors, { - kind: 'error', - color: true, - }) - e.message = prependMessage + msgs.join('\n') - } else { - e.message = prependMessage + e.message - } - throw e - } - - debug(`Scan completed in ${(performance.now() - start).toFixed(2)}ms:`, deps) + if (e.errors) { + const msgs = await formatMessages(e.errors, { + kind: 'error', + color: true, + }) + e.message = prependMessage + msgs.join('\n') + } else { + e.message = prependMessage + e.message + } + throw e + }) + .finally(() => { + esbuildContext.dispose() + debug( + `Scan completed in ${(performance.now() - start).toFixed(2)}ms:`, + deps, + ) + }) return { - // Ensure a fixed order so hashes are stable and improve logs - deps: orderedDependencies(deps), - missing, + cancel: () => esbuildContext.cancel(), + result, } } From 5ff8d8b5942860e1d897ebd825cd085acc6e58f6 Mon Sep 17 00:00:00 2001 From: patak Date: Tue, 28 Feb 2023 11:27:34 +0100 Subject: [PATCH 2/3] refactor: simplify scanImports return type --- packages/vite/src/node/optimizer/index.ts | 12 +- packages/vite/src/node/optimizer/optimizer.ts | 2 +- packages/vite/src/node/optimizer/scan.ts | 149 ++++++++++-------- 3 files changed, 93 insertions(+), 70 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 1a545d2514e81c..155d5af3cb3d19 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -239,7 +239,7 @@ export async function optimizeDeps( return cachedMetadata } - const deps = await (await discoverProjectDependencies(config)).result + const deps = await discoverProjectDependencies(config).result const depsString = depsLogString(Object.keys(deps)) log(colors.green(`Optimizing dependencies:\n ${depsString}`)) @@ -380,13 +380,11 @@ export function loadCachedDepOptimizationMetadata( * Initial optimizeDeps at server start. Perform a fast scan using esbuild to * find deps to pre-bundle and include user hard-coded dependencies */ -export async function discoverProjectDependencies( - config: ResolvedConfig, -): Promise<{ +export function discoverProjectDependencies(config: ResolvedConfig): { cancel: () => Promise result: Promise> -}> { - const { cancel, result } = await scanImports(config) +} { + const { cancel, result } = scanImports(config) return { cancel, @@ -687,7 +685,7 @@ export async function findKnownImports( config: ResolvedConfig, ssr: boolean, ): Promise { - const deps = (await (await scanImports(config)).result).deps + const { deps } = await scanImports(config).result await addManuallyIncludedOptimizeDeps(deps, config, ssr) return Object.keys(deps) } diff --git a/packages/vite/src/node/optimizer/optimizer.ts b/packages/vite/src/node/optimizer/optimizer.ts index b125ed434233c7..757e120e6a2f30 100644 --- a/packages/vite/src/node/optimizer/optimizer.ts +++ b/packages/vite/src/node/optimizer/optimizer.ts @@ -211,7 +211,7 @@ async function createDepsOptimizer( try { debug(colors.green(`scanning for dependencies...`)) - discover = await discoverProjectDependencies(config) + discover = discoverProjectDependencies(config) const deps = await discover.result discover = undefined diff --git a/packages/vite/src/node/optimizer/scan.ts b/packages/vite/src/node/optimizer/scan.ts index c933807e957e46..3147656dc7b6ba 100644 --- a/packages/vite/src/node/optimizer/scan.ts +++ b/packages/vite/src/node/optimizer/scan.ts @@ -2,7 +2,7 @@ import fs from 'node:fs' import path from 'node:path' import { performance } from 'node:perf_hooks' import glob from 'fast-glob' -import type { Loader, OnLoadResult, Plugin } from 'esbuild' +import type { BuildContext, Loader, OnLoadResult, Plugin } from 'esbuild' import esbuild, { formatMessages, transform } from 'esbuild' import colors from 'picocolors' import type { ResolvedConfig } from '..' @@ -47,17 +47,92 @@ const htmlTypesRE = /\.(html|vue|svelte|astro|imba)$/ export const importsRE = /(? Promise result: Promise<{ deps: Record missing: Record }> -}> { +} { // Only used to scan non-ssr code const start = performance.now() + const deps: Record = {} + const missing: Record = {} + let entries: string[] + + const esbuildContext: Promise = computeEntries( + config, + ).then((computedEntries) => { + entries = computedEntries + + if (!entries.length) { + if (!config.optimizeDeps.entries && !config.optimizeDeps.include) { + config.logger.warn( + colors.yellow( + '(!) Could not auto-determine entry point from rollupOptions or html files ' + + 'and there are no explicit optimizeDeps.include patterns. ' + + 'Skipping dependency pre-bundling.', + ), + ) + } + return + } else { + debug(`Crawling dependencies using entries:\n ${entries.join('\n ')}`) + } + + return prepareEsbuildScanner(config, entries, deps, missing) + }) + + const result = esbuildContext + .then((context) => { + if (!context) { + return { deps: {}, missing: {} } + } + return context + .rebuild() + .then(() => { + return { + // Ensure a fixed order so hashes are stable and improve logs + deps: orderedDependencies(deps), + missing, + } + }) + .finally(() => { + context.dispose() + }) + }) + .catch(async (e) => { + const prependMessage = colors.red(`\ + Failed to scan for dependencies from entries: + ${entries.join('\n')} + + `) + if (e.errors) { + const msgs = await formatMessages(e.errors, { + kind: 'error', + color: true, + }) + e.message = prependMessage + msgs.join('\n') + } else { + e.message = prependMessage + e.message + } + throw e + }) + .finally(() => { + debug( + `Scan completed in ${(performance.now() - start).toFixed(2)}ms:`, + deps, + ) + }) + + return { + cancel: () => esbuildContext.then((context) => context?.cancel()), + result, + } +} +async function computeEntries(config: ResolvedConfig) { let entries: string[] = [] const explicitEntryPatterns = config.optimizeDeps.entries @@ -86,33 +161,22 @@ export async function scanImports(config: ResolvedConfig): Promise<{ (entry) => isScannable(entry) && fs.existsSync(entry), ) - if (!entries.length) { - if (!explicitEntryPatterns && !config.optimizeDeps.include) { - config.logger.warn( - colors.yellow( - '(!) Could not auto-determine entry point from rollupOptions or html files ' + - 'and there are no explicit optimizeDeps.include patterns. ' + - 'Skipping dependency pre-bundling.', - ), - ) - } - return { - cancel: () => Promise.resolve(), - result: Promise.resolve({ deps: {}, missing: {} }), - } - } else { - debug(`Crawling dependencies using entries:\n ${entries.join('\n ')}`) - } + return entries +} - const deps: Record = {} - const missing: Record = {} +async function prepareEsbuildScanner( + config: ResolvedConfig, + entries: string[], + deps: Record, + missing: Record, +) { const container = await createPluginContainer(config) const plugin = esbuildScanPlugin(config, container, deps, missing, entries) const { plugins = [], ...esbuildOptions } = config.optimizeDeps?.esbuildOptions ?? {} - const esbuildContext = await esbuild.context({ + return await esbuild.context({ absWorkingDir: process.cwd(), write: false, stdin: { @@ -125,45 +189,6 @@ export async function scanImports(config: ResolvedConfig): Promise<{ plugins: [...plugins, plugin], ...esbuildOptions, }) - - const result = esbuildContext - .rebuild() - .then(() => { - return { - // Ensure a fixed order so hashes are stable and improve logs - deps: orderedDependencies(deps), - missing, - } - }) - .catch(async (e) => { - const prependMessage = colors.red(`\ -Failed to scan for dependencies from entries: -${entries.join('\n')} - -`) - if (e.errors) { - const msgs = await formatMessages(e.errors, { - kind: 'error', - color: true, - }) - e.message = prependMessage + msgs.join('\n') - } else { - e.message = prependMessage + e.message - } - throw e - }) - .finally(() => { - esbuildContext.dispose() - debug( - `Scan completed in ${(performance.now() - start).toFixed(2)}ms:`, - deps, - ) - }) - - return { - cancel: () => esbuildContext.cancel(), - result, - } } function orderedDependencies(deps: Record) { From 55b24a882df88f64c0d088bcd334b0e5eda1f6a2 Mon Sep 17 00:00:00 2001 From: patak Date: Tue, 28 Feb 2023 18:55:03 +0100 Subject: [PATCH 3/3] chore: simplify else Co-authored-by: Ben McCann <322311+benmccann@users.noreply.github.com> --- packages/vite/src/node/optimizer/scan.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/vite/src/node/optimizer/scan.ts b/packages/vite/src/node/optimizer/scan.ts index 3147656dc7b6ba..f2fccb4a3c3232 100644 --- a/packages/vite/src/node/optimizer/scan.ts +++ b/packages/vite/src/node/optimizer/scan.ts @@ -77,10 +77,9 @@ export function scanImports(config: ResolvedConfig): { ) } return - } else { - debug(`Crawling dependencies using entries:\n ${entries.join('\n ')}`) } + debug(`Crawling dependencies using entries:\n ${entries.join('\n ')}`) return prepareEsbuildScanner(config, entries, deps, missing) })