From b44f16838258040b3013ed68b8aa9af9dbc9a0e5 Mon Sep 17 00:00:00 2001 From: Craigory Coppola Date: Wed, 15 Jan 2025 12:54:53 -0500 Subject: [PATCH] feat(core): propagate errors and retry on cancellation --- .../devkit/readCachedProjectGraph.md | 4 +- ...project-graph-incremental-recomputation.ts | 12 +++--- packages/nx/src/native/index.d.ts | 1 + packages/nx/src/native/utils/file_lock.rs | 15 +++++++ packages/nx/src/project-graph/error-types.ts | 19 ++++---- .../nx/src/project-graph/nx-deps-cache.ts | 14 ++++-- .../nx/src/project-graph/project-graph.ts | 43 ++++++++++++++----- packages/nx/src/utils/project-graph-utils.ts | 2 +- 8 files changed, 78 insertions(+), 32 deletions(-) diff --git a/docs/generated/devkit/readCachedProjectGraph.md b/docs/generated/devkit/readCachedProjectGraph.md index be49a12990d771..32b36db5f25057 100644 --- a/docs/generated/devkit/readCachedProjectGraph.md +++ b/docs/generated/devkit/readCachedProjectGraph.md @@ -1,12 +1,12 @@ # Function: readCachedProjectGraph -▸ **readCachedProjectGraph**(): [`ProjectGraph`](../../devkit/documents/ProjectGraph) +▸ **readCachedProjectGraph**(): [`ProjectGraph`](../../devkit/documents/ProjectGraph) & \{ `computedAt`: `number` ; `errors`: `ProjectGraphErrorTypes`[] } Synchronously reads the latest cached copy of the workspace's ProjectGraph. #### Returns -[`ProjectGraph`](../../devkit/documents/ProjectGraph) +[`ProjectGraph`](../../devkit/documents/ProjectGraph) & \{ `computedAt`: `number` ; `errors`: `ProjectGraphErrorTypes`[] } **`Throws`** diff --git a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts index 23d744cc05e538..4856da1d523605 100644 --- a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts +++ b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts @@ -299,7 +299,12 @@ async function processFilesAndCreateAndSerializeProjectGraph( }; } } - + writeCache( + g.projectFileMapCache, + g.projectGraph, + projectConfigurationsResult.sourceMaps, + errors + ); if (errors.length > 0) { return { error: new DaemonProjectGraphError( @@ -316,11 +321,6 @@ async function processFilesAndCreateAndSerializeProjectGraph( serializedSourceMaps: null, }; } else { - writeCache( - g.projectFileMapCache, - g.projectGraph, - projectConfigurationsResult.sourceMaps - ); return g; } } catch (err) { diff --git a/packages/nx/src/native/index.d.ts b/packages/nx/src/native/index.d.ts index ce8309cc27d4c2..59321395f80e77 100644 --- a/packages/nx/src/native/index.d.ts +++ b/packages/nx/src/native/index.d.ts @@ -17,6 +17,7 @@ export declare class FileLock { locked: boolean constructor(lockFilePath: string) unlock(): void + check(): boolean wait(): Promise lock(): void } diff --git a/packages/nx/src/native/utils/file_lock.rs b/packages/nx/src/native/utils/file_lock.rs index 0edfa6f11c2859..ee46af1f646098 100644 --- a/packages/nx/src/native/utils/file_lock.rs +++ b/packages/nx/src/native/utils/file_lock.rs @@ -62,6 +62,20 @@ impl FileLock { self.locked = false; } + #[napi] + pub fn check(&mut self) -> Result { + // Check if the file is locked + let file_lock: std::result::Result<(), std::io::Error> = self.file.try_lock_exclusive(); + + if file_lock.is_ok() { + // Checking if the file is locked, locks it, so unlock it. + self.file.unlock()?; + } + + self.locked = file_lock.is_err(); + Ok(self.locked) + } + #[napi(ts_return_type = "Promise")] pub fn wait(&mut self, env: Env) -> napi::Result { if self.locked { @@ -74,6 +88,7 @@ impl FileLock { .create(true) .open(&lock_file_path)?; file.lock_shared()?; + file.unlock()?; Ok(()) }) } else { diff --git a/packages/nx/src/project-graph/error-types.ts b/packages/nx/src/project-graph/error-types.ts index 093bcff466a7fc..cd75770a2b4f6b 100644 --- a/packages/nx/src/project-graph/error-types.ts +++ b/packages/nx/src/project-graph/error-types.ts @@ -6,20 +6,21 @@ import { ProjectConfiguration } from '../config/workspace-json-project-json'; import { ProjectGraph } from '../config/project-graph'; import { CreateNodesFunctionV2 } from './plugins/public-api'; +export type ProjectGraphErrorTypes = + | AggregateCreateNodesError + | MergeNodesError + | CreateMetadataError + | ProjectsWithNoNameError + | MultipleProjectsWithSameNameError + | ProcessDependenciesError + | WorkspaceValidityError; + export class ProjectGraphError extends Error { readonly #partialProjectGraph: ProjectGraph; readonly #partialSourceMaps: ConfigurationSourceMaps; constructor( - private readonly errors: Array< - | AggregateCreateNodesError - | MergeNodesError - | ProjectsWithNoNameError - | MultipleProjectsWithSameNameError - | ProcessDependenciesError - | CreateMetadataError - | WorkspaceValidityError - >, + private readonly errors: Array, partialProjectGraph: ProjectGraph, partialSourceMaps: ConfigurationSourceMaps ) { diff --git a/packages/nx/src/project-graph/nx-deps-cache.ts b/packages/nx/src/project-graph/nx-deps-cache.ts index f0c60f0e7f3674..34e9379591e358 100644 --- a/packages/nx/src/project-graph/nx-deps-cache.ts +++ b/packages/nx/src/project-graph/nx-deps-cache.ts @@ -19,6 +19,7 @@ import { import { PackageJson } from '../utils/package-json'; import { nxVersion } from '../utils/versions'; import { ConfigurationSourceMaps } from './utils/project-configuration-utils'; +import { ProjectGraphError, ProjectGraphErrorTypes } from './error-types'; export interface FileMapCache { version: string; @@ -80,7 +81,9 @@ export function readFileMapCache(): null | FileMapCache { return data ?? null; } -export function readProjectGraphCache(): null | ProjectGraph { +export function readProjectGraphCache(): + | null + | (ProjectGraph & { errors: ProjectGraphErrorTypes[]; computedAt: number }) { performance.mark('read project-graph:start'); ensureCacheDirectory(); @@ -152,7 +155,8 @@ export function createProjectFileMapCache( export function writeCache( cache: FileMapCache, projectGraph: ProjectGraph, - sourceMaps: ConfigurationSourceMaps + sourceMaps: ConfigurationSourceMaps, + errors: ProjectGraphErrorTypes[] ): void { performance.mark('write cache:start'); let retry = 1; @@ -169,7 +173,11 @@ export function writeCache( const tmpSourceMapPath = `${nxSourceMaps}~${unique}`; try { - writeJsonFile(tmpProjectGraphPath, projectGraph); + writeJsonFile(tmpProjectGraphPath, { + ...projectGraph, + errors, + computedAt: Date.now(), + }); renameSync(tmpProjectGraphPath, nxProjectGraph); writeJsonFile(tmpFileMapPath, cache); diff --git a/packages/nx/src/project-graph/project-graph.ts b/packages/nx/src/project-graph/project-graph.ts index e6f5996d6640fa..febfe96ca9afa2 100644 --- a/packages/nx/src/project-graph/project-graph.ts +++ b/packages/nx/src/project-graph/project-graph.ts @@ -21,6 +21,7 @@ import { isAggregateProjectGraphError, ProjectConfigurationsError, ProjectGraphError, + ProjectGraphErrorTypes, } from './error-types'; import { readFileMapCache, @@ -44,8 +45,11 @@ import { DelayedSpinner } from '../utils/delayed-spinner'; * Synchronously reads the latest cached copy of the workspace's ProjectGraph. * @throws {Error} if there is no cached ProjectGraph to read from */ -export function readCachedProjectGraph(): ProjectGraph { - const projectGraphCache: ProjectGraph = readProjectGraphCache(); +export function readCachedProjectGraph(): ProjectGraph & { + errors: ProjectGraphErrorTypes[]; + computedAt: number; +} { + const projectGraphCache = readProjectGraphCache(); if (!projectGraphCache) { const angularSpecificError = fileExists(`${workspaceRoot}/angular.json`) ? stripIndents` @@ -168,12 +172,13 @@ export async function buildProjectGraphAndSourceMapsWithoutDaemon() { ...(projectGraphError?.errors ?? []), ]; + if (cacheEnabled) { + writeCache(projectFileMapCache, projectGraph, sourceMaps, errors); + } + if (errors.length > 0) { throw new ProjectGraphError(errors, projectGraph, sourceMaps); } else { - if (cacheEnabled) { - writeCache(projectFileMapCache, projectGraph, sourceMaps); - } return { projectGraph, sourceMaps }; } } @@ -252,7 +257,11 @@ export async function createProjectGraphAsync( if (process.env.NX_FORCE_REUSE_CACHED_GRAPH === 'true') { try { // If no cached graph is found, we will fall through to the normal flow - return readCachedGraphAndHydrateFileMap(); + const graph = await readCachedGraphAndHydrateFileMap(); + if (graph.errors.length > 0) { + throw new ProjectGraphError(graph.errors, graph, readSourceMapsCache()); + } + return graph; } catch (e) { logger.verbose('Unable to use cached project graph', e); } @@ -276,8 +285,10 @@ export async function createProjectGraphAndSourceMapsAsync( const lock = new FileLock( join(workspaceDataDirectory, 'project-graph.lock') ); + const initiallyLocked = lock.locked; + let locked = lock.locked; - if (lock.locked) { + while (lock.locked) { logger.verbose( 'Waiting for graph construction in another process to complete' ); @@ -300,12 +311,22 @@ export async function createProjectGraphAndSourceMapsAsync( 'The project graph was computed in another process, but the source maps are missing.' ); } - return { - projectGraph: await readCachedGraphAndHydrateFileMap(), - sourceMaps, - }; + const graph = await readCachedGraphAndHydrateFileMap(); + if (graph.errors.length > 0) { + throw new ProjectGraphError(graph.errors, graph, sourceMaps); + } + if (Date.now() - graph.computedAt < 10_000) { + // If the graph was computed in the last 10 seconds, we can assume it's fresh + return { + projectGraph: graph, + sourceMaps, + }; + } + locked = lock.check(); } + // if (!initiallyLocked) { lock.lock(); + // } try { const res = await buildProjectGraphAndSourceMapsWithoutDaemon(); performance.measure( diff --git a/packages/nx/src/utils/project-graph-utils.ts b/packages/nx/src/utils/project-graph-utils.ts index e5f51e71b843db..01bbf7a73bc7fd 100644 --- a/packages/nx/src/utils/project-graph-utils.ts +++ b/packages/nx/src/utils/project-graph-utils.ts @@ -58,7 +58,7 @@ export function getSourceDirOfDependentProjects( */ export function findAllProjectNodeDependencies( parentNodeName: string, - projectGraph = readCachedProjectGraph(), + projectGraph: ProjectGraph = readCachedProjectGraph(), includeExternalDependencies = false ): string[] { const dependencyNodeNames = new Set();