diff --git a/packages/metro-file-map/src/__tests__/index-test.js b/packages/metro-file-map/src/__tests__/index-test.js index f7c6805840..aae9842f8a 100644 --- a/packages/metro-file-map/src/__tests__/index-test.js +++ b/packages/metro-file-map/src/__tests__/index-test.js @@ -74,8 +74,7 @@ jest.mock('../crawlers/watchman', () => ]); } } else { - const fileData = previousState.files.get(relativeFilePath); - if (fileData) { + if (previousState.fileSystem.exists(relativeFilePath)) { removedFiles.add(relativeFilePath); } } diff --git a/packages/metro-file-map/src/crawlers/__tests__/integration-test.js b/packages/metro-file-map/src/crawlers/__tests__/integration-test.js index e1c0715120..4c207d627e 100644 --- a/packages/metro-file-map/src/crawlers/__tests__/integration-test.js +++ b/packages/metro-file-map/src/crawlers/__tests__/integration-test.js @@ -9,6 +9,7 @@ * @oncall react_native */ +import TreeFS from '../../lib/TreeFS'; import nodeCrawl from '../node'; import watchmanCrawl from '../watchman'; import {execSync} from 'child_process'; @@ -109,7 +110,10 @@ describe.each(Object.keys(CRAWLERS))( invariant(crawl, 'crawl should not be null within maybeTest'); const result = await crawl({ previousState: { - files: new Map([['removed.js', ['', 123, 234, 0, '', null, 0]]]), + fileSystem: new TreeFS({ + rootDir: FIXTURES_DIR, + files: new Map([['removed.js', ['', 123, 234, 0, '', null, 0]]]), + }), clocks: new Map(), }, includeSymlinks, diff --git a/packages/metro-file-map/src/crawlers/__tests__/node-test.js b/packages/metro-file-map/src/crawlers/__tests__/node-test.js index a9bc1da346..76bb3fa5c2 100644 --- a/packages/metro-file-map/src/crawlers/__tests__/node-test.js +++ b/packages/metro-file-map/src/crawlers/__tests__/node-test.js @@ -9,6 +9,7 @@ */ import {AbortController} from 'node-abort-controller'; +import TreeFS from '../../lib/TreeFS'; jest.useRealTimers(); @@ -124,6 +125,8 @@ const createMap = obj => new Map(Object.keys(obj).map(key => [normalize(key), obj[key]])); const rootDir = '/project'; +const emptyFS = new TreeFS({rootDir, files: new Map()}); +const getFS = (files: FileData) => new TreeFS({rootDir, files}); let mockResponse; let mockSpawnExit; let nodeCrawl; @@ -154,9 +157,7 @@ describe('node crawler', () => { ].join('\n'); const {changedFiles, removedFiles} = await nodeCrawl({ - previousState: { - files: new Map(), - }, + previousState: {fileSystem: emptyFS}, extensions: ['js', 'json'], ignore: pearMatcher, rootDir, @@ -205,7 +206,7 @@ describe('node crawler', () => { }); const {changedFiles, removedFiles} = await nodeCrawl({ - previousState: {files}, + previousState: {fileSystem: getFS(files)}, extensions: ['js'], ignore: pearMatcher, rootDir, @@ -234,7 +235,7 @@ describe('node crawler', () => { }); const {changedFiles, removedFiles} = await nodeCrawl({ - previousState: {files}, + previousState: {fileSystem: getFS(files)}, extensions: ['js'], ignore: pearMatcher, rootDir, @@ -258,9 +259,7 @@ describe('node crawler', () => { nodeCrawl = require('../node'); const {changedFiles, removedFiles} = await nodeCrawl({ - previousState: { - files: new Map(), - }, + previousState: {fileSystem: emptyFS}, extensions: ['js'], ignore: pearMatcher, rootDir, @@ -289,9 +288,7 @@ describe('node crawler', () => { nodeCrawl = require('../node'); const {changedFiles, removedFiles} = await nodeCrawl({ - previousState: { - files: new Map(), - }, + previousState: {fileSystem: emptyFS}, extensions: ['js'], ignore: pearMatcher, rootDir, @@ -311,9 +308,8 @@ describe('node crawler', () => { childProcess = require('child_process'); nodeCrawl = require('../node'); - const files = new Map(); const {changedFiles, removedFiles} = await nodeCrawl({ - previousState: {files}, + previousState: {fileSystem: emptyFS}, extensions: ['js'], forceNodeFilesystemAPI: true, ignore: pearMatcher, @@ -334,9 +330,8 @@ describe('node crawler', () => { it('completes with empty roots', async () => { nodeCrawl = require('../node'); - const files = new Map(); const {changedFiles, removedFiles} = await nodeCrawl({ - previousState: {files}, + previousState: {fileSystem: emptyFS}, extensions: ['js'], forceNodeFilesystemAPI: true, ignore: pearMatcher, @@ -351,9 +346,8 @@ describe('node crawler', () => { it('completes with fs.readdir throwing an error', async () => { nodeCrawl = require('../node'); - const files = new Map(); const {changedFiles, removedFiles} = await nodeCrawl({ - previousState: {files}, + previousState: {fileSystem: emptyFS}, extensions: ['js'], forceNodeFilesystemAPI: true, ignore: pearMatcher, @@ -369,9 +363,8 @@ describe('node crawler', () => { nodeCrawl = require('../node'); const fs = require('graceful-fs'); - const files = new Map(); const {changedFiles, removedFiles} = await nodeCrawl({ - previousState: {files}, + previousState: {fileSystem: emptyFS}, extensions: ['js'], forceNodeFilesystemAPI: true, ignore: pearMatcher, @@ -398,9 +391,7 @@ describe('node crawler', () => { await expect( nodeCrawl({ abortSignal: abortSignalWithReason(err), - previousState: { - files: new Map(), - }, + previousState: {fileSystem: emptyFS}, extensions: ['js', 'json'], ignore: pearMatcher, rootDir, @@ -431,9 +422,7 @@ describe('node crawler', () => { nodeCrawl({ perfLogger: fakePerfLogger, abortSignal: abortController.signal, - previousState: { - files: new Map(), - }, + previousState: {fileSystem: emptyFS}, extensions: ['js', 'json'], ignore: pearMatcher, rootDir, diff --git a/packages/metro-file-map/src/crawlers/__tests__/watchman-test.js b/packages/metro-file-map/src/crawlers/__tests__/watchman-test.js index e87c9165b1..833743d157 100644 --- a/packages/metro-file-map/src/crawlers/__tests__/watchman-test.js +++ b/packages/metro-file-map/src/crawlers/__tests__/watchman-test.js @@ -9,6 +9,7 @@ */ import {AbortController} from 'node-abort-controller'; +import TreeFS from '../../lib/TreeFS'; const path = require('path'); @@ -45,6 +46,7 @@ let watchman; let watchmanCrawl; let mockResponse; let mockFiles; +const getFS = files => new TreeFS({files, rootDir: ROOT_MOCK}); const ROOT_MOCK = path.sep === '/' ? '/root-mock' : 'M:\\root-mock'; const FRUITS_RELATIVE = 'fruits'; @@ -129,7 +131,7 @@ describe('watchman watch', () => { const {changedFiles, clocks, removedFiles} = await watchmanCrawl({ previousState: { clocks: new Map(), - files: new Map(), + fileSystem: getFS(new Map()), }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -204,7 +206,7 @@ describe('watchman watch', () => { clocks: createMap({ '': 'c:fake-clock:1', }), - files: mockFiles, + fileSystem: getFS(mockFiles), }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -272,7 +274,7 @@ describe('watchman watch', () => { clocks: createMap({ '': 'c:fake-clock:1', }), - files: mockFiles, + fileSystem: getFS(mockFiles), }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -352,7 +354,7 @@ describe('watchman watch', () => { [FRUITS_RELATIVE]: 'c:fake-clock:1', [VEGETABLES_RELATIVE]: 'c:fake-clock:2', }), - files: mockFiles, + fileSystem: getFS(mockFiles), }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -407,7 +409,7 @@ describe('watchman watch', () => { const {changedFiles, clocks, removedFiles} = await watchmanCrawl({ previousState: { clocks: new Map(), - files: new Map(), + fileSystem: getFS(new Map()), }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -470,7 +472,7 @@ describe('watchman watch', () => { computeSha1: true, previousState: { clocks: new Map(), - files: new Map(), + fileSystem: getFS(new Map()), }, extensions: ['js', 'json'], rootDir: ROOT_MOCK, diff --git a/packages/metro-file-map/src/crawlers/node/index.js b/packages/metro-file-map/src/crawlers/node/index.js index 6411ca6ff0..f0dc36f869 100644 --- a/packages/metro-file-map/src/crawlers/node/index.js +++ b/packages/metro-file-map/src/crawlers/node/index.js @@ -9,7 +9,6 @@ * @oncall react_native */ -import type {Path, FileMetaData} from '../../flow-types'; import type { CanonicalPath, CrawlerOptions, @@ -18,7 +17,6 @@ import type { } from '../../flow-types'; import hasNativeFindSupport from './hasNativeFindSupport'; -import H from '../../constants'; import * as fastPath from '../../lib/fast_path'; import {spawn} from 'child_process'; import * as fs from 'graceful-fs'; @@ -194,19 +192,7 @@ module.exports = async function nodeCrawl(options: CrawlerOptions): Promise<{ return new Promise((resolve, reject) => { const callback = (fileData: FileData) => { - const changedFiles = new Map(); - const removedFiles = new Set(previousState.files.keys()); - for (const [normalPath, fileMetaData] of fileData) { - const existingFile = previousState.files.get(normalPath); - removedFiles.delete(normalPath); - if ( - existingFile == null || - existingFile[H.MTIME] !== fileMetaData[H.MTIME] - ) { - // See ../constants.js; SHA-1 will always be null and fulfilled later. - changedFiles.set(normalPath, fileMetaData); - } - } + const difference = previousState.fileSystem.getDifference(fileData); perfLogger?.point('nodeCrawl_end'); @@ -216,10 +202,7 @@ module.exports = async function nodeCrawl(options: CrawlerOptions): Promise<{ } catch (e) { reject(e); } - resolve({ - changedFiles, - removedFiles, - }); + resolve(difference); }; if (useNativeFind) { diff --git a/packages/metro-file-map/src/crawlers/watchman/index.js b/packages/metro-file-map/src/crawlers/watchman/index.js index 56f9f03823..e55ed93fe4 100644 --- a/packages/metro-file-map/src/crawlers/watchman/index.js +++ b/packages/metro-file-map/src/crawlers/watchman/index.js @@ -20,7 +20,6 @@ import type { } from '../../flow-types'; import type {WatchmanQueryResponse, WatchmanWatchResponse} from 'fb-watchman'; -import H from '../../constants'; import * as fastPath from '../../lib/fast_path'; import normalizePathSeparatorsToSystem from '../../lib/normalizePathSeparatorsToSystem'; import {planQuery} from './planQuery'; @@ -244,22 +243,15 @@ module.exports = async function watchmanCrawl({ } let removedFiles: Set = new Set(); - const changedFiles: FileData = new Map(); + let changedFiles: FileData = new Map(); let results: Map; let isFresh = false; let queryError: ?Error; try { const watchmanRoots = await getWatchmanRoots(roots); const watchmanFileResults = await queryWatchmanForDirs(watchmanRoots); - - // Reset the file map if watchman was restarted and sends us a list of - // files. - if (watchmanFileResults.isFresh) { - removedFiles = new Set(previousState.files.keys()); - isFresh = true; - } - results = watchmanFileResults.results; + isFresh = watchmanFileResults.isFresh; } catch (e) { queryError = e; } @@ -291,6 +283,8 @@ module.exports = async function watchmanCrawl({ perfLogger?.point('watchmanCrawl/processResults_start'); + const freshFileData: FileData = new Map(); + for (const [watchRoot, response] of results) { const fsRoot = normalizePathSeparatorsToSystem(watchRoot); const relativeFsRoot = fastPath.relative(rootDir, fsRoot); @@ -306,23 +300,13 @@ module.exports = async function watchmanCrawl({ const filePath = fsRoot + path.sep + normalizePathSeparatorsToSystem(fileData.name); const relativeFilePath = fastPath.relative(rootDir, filePath); - const existingFileData = previousState.files.get(relativeFilePath); - - // If watchman is fresh, the removed files map starts with all files - // and we remove them as we verify they still exist. - if (isFresh && existingFileData && fileData.exists) { - removedFiles.delete(relativeFilePath); - } if (!fileData.exists) { - // No need to act on files that do not exist and were not tracked. - if (existingFileData) { - // If watchman is not fresh, we will know what specific files were - // deleted since we last ran and can track only those files. - if (!isFresh) { - removedFiles.add(relativeFilePath); - } + if (!isFresh) { + removedFiles.add(relativeFilePath); } + // Whether watchman can return exists: false in a fresh instance + // response is unknown, but there's nothing we need to do in that case. } else if (!ignore(filePath)) { const {mtime_ms, size} = fileData; invariant( @@ -332,10 +316,6 @@ module.exports = async function watchmanCrawl({ const mtime = typeof mtime_ms === 'number' ? mtime_ms : mtime_ms.toNumber(); - if (existingFileData && existingFileData[H.MTIME] === mtime) { - continue; - } - let sha1hex = fileData['content.sha1hex']; if (typeof sha1hex !== 'string' || sha1hex.length !== 40) { sha1hex = undefined; @@ -346,7 +326,7 @@ module.exports = async function watchmanCrawl({ symlinkInfo = fileData['symlink_target'] ?? 1; } - let nextData: FileMetaData = [ + const nextData: FileMetaData = [ '', mtime, size, @@ -356,31 +336,22 @@ module.exports = async function watchmanCrawl({ symlinkInfo, ]; - if ( - existingFileData && - sha1hex != null && - existingFileData[H.SHA1] === sha1hex && - // File is still of the same type - (existingFileData[H.SYMLINK] !== 0) === (fileData.type === 'l') - ) { - // Special case - file touched but not modified, so we can reuse the - // metadata and just update mtime. - nextData = [ - existingFileData[0], - mtime, - existingFileData[2], - existingFileData[3], - existingFileData[4], - existingFileData[5], - typeof symlinkInfo === 'string' ? symlinkInfo : existingFileData[6], - ]; + // If watchman is fresh, the removed files map starts with all files + // and we remove them as we verify they still exist. + if (isFresh) { + freshFileData.set(relativeFilePath, nextData); + } else { + changedFiles.set(relativeFilePath, nextData); } - - changedFiles.set(relativeFilePath, nextData); } } } + if (isFresh) { + ({changedFiles, removedFiles} = + previousState.fileSystem.getDifference(freshFileData)); + } + perfLogger?.point('watchmanCrawl/processResults_end'); perfLogger?.point('watchmanCrawl_end'); abortSignal?.throwIfAborted(); diff --git a/packages/metro-file-map/src/flow-types.js b/packages/metro-file-map/src/flow-types.js index e87f8f9000..00d6b22e49 100644 --- a/packages/metro-file-map/src/flow-types.js +++ b/packages/metro-file-map/src/flow-types.js @@ -96,7 +96,7 @@ export type CrawlerOptions = { perfLogger?: ?PerfLogger, previousState: $ReadOnly<{ clocks: $ReadOnlyMap, - files: $ReadOnlyMap, + fileSystem: FileSystem, }>, rootDir: string, roots: $ReadOnlyArray, @@ -171,6 +171,10 @@ export interface FileSystem { exists(file: Path): boolean; getAllFiles(): Array; getDependencies(file: Path): ?Array; + getDifference(files: FileData): { + changedFiles: FileData, + removedFiles: Set, + }; getModuleName(file: Path): ?string; getRealPath(file: Path): ?string; getSerializableSnapshot(): FileData; diff --git a/packages/metro-file-map/src/index.js b/packages/metro-file-map/src/index.js index e310b1f6e4..5964260520 100644 --- a/packages/metro-file-map/src/index.js +++ b/packages/metro-file-map/src/index.js @@ -373,7 +373,7 @@ export default class HasteMap extends EventEmitter { }; const fileDelta = await this._buildFileDelta({ - files: initialData.files, + fileSystem, clocks: initialData.clocks, }); diff --git a/packages/metro-file-map/src/lib/TreeFS.js b/packages/metro-file-map/src/lib/TreeFS.js index 55f0c42a72..354bbacf9b 100644 --- a/packages/metro-file-map/src/lib/TreeFS.js +++ b/packages/metro-file-map/src/lib/TreeFS.js @@ -75,6 +75,47 @@ export default class TreeFS implements MutableFileSystem { } } + getDifference(files: FileData): { + changedFiles: FileData, + removedFiles: Set, + } { + const changedFiles: FileData = new Map(files); + const removedFiles: Set = new Set(); + for (const [normalPath, metadata] of this.#files) { + const newMetadata = files.get(normalPath); + if (newMetadata) { + if ((newMetadata[H.SYMLINK] === 0) !== (metadata[H.SYMLINK] === 0)) { + // Types differ, file has changed + continue; + } + if ( + newMetadata[H.MTIME] != null && + // TODO: Remove when mtime is null if not populated + newMetadata[H.MTIME] != 0 && + newMetadata[H.MTIME] === metadata[H.MTIME] + ) { + // Types and modified time match - not changed. + changedFiles.delete(normalPath); + } else if ( + newMetadata[H.SHA1] != null && + newMetadata[H.SHA1] === metadata[H.SHA1] && + metadata[H.VISITED] === 1 + ) { + // Content matches - update modified time but don't revisit + const updatedMetadata = [...metadata]; + updatedMetadata[H.MTIME] = newMetadata[H.MTIME]; + changedFiles.set(normalPath, updatedMetadata); + } + } else { + removedFiles.add(normalPath); + } + } + return { + changedFiles, + removedFiles, + }; + } + getSha1(mixedPath: Path): ?string { const fileMetadata = this._getFileData(mixedPath); return (fileMetadata && fileMetadata[H.SHA1]) ?? null; diff --git a/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js b/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js index 16a24c2527..fcb9a7b826 100644 --- a/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js +++ b/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js @@ -10,6 +10,7 @@ */ import type TreeFS from '../TreeFS'; +import type {FileData} from '../../flow-types'; let mockPathModule; jest.mock('path', () => mockPathModule); @@ -37,10 +38,10 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { [p('bar.js'), ['', 234, 0, 0, '', '', 0]], [p('link-to-foo'), ['', 456, 0, 0, '', '', p('./foo')]], [p('root'), ['', 0, 0, 0, '', '', '..']], - [p('link-to-nowhere'), ['', 0, 0, 0, '', '', p('./nowhere')]], - [p('link-to-self'), ['', 0, 0, 0, '', '', p('./link-to-self')]], - [p('link-cycle-1'), ['', 0, 0, 0, '', '', p('./link-cycle-2')]], - [p('link-cycle-2'), ['', 0, 0, 0, '', '', p('./link-cycle-1')]], + [p('link-to-nowhere'), ['', 123, 0, 0, '', '', p('./nowhere')]], + [p('link-to-self'), ['', 123, 0, 0, '', '', p('./link-to-self')]], + [p('link-cycle-1'), ['', 123, 0, 0, '', '', p('./link-cycle-2')]], + [p('link-cycle-2'), ['', 123, 0, 0, '', '', p('./link-cycle-1')]], ]), }); }); @@ -108,6 +109,36 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { ); }); + describe('getDifference', () => { + test('returns changed (inc. new) and removed files in given FileData', () => { + const newFiles: FileData = new Map([ + [p('new-file'), ['', 789, 0, 0, '', '', 0]], + [p('link-to-foo'), ['', 456, 0, 0, '', '', p('./foo')]], + // Different modified time, expect new mtime in changedFiles + [p('foo/another.js'), ['', 124, 0, 0, '', '', 0]], + [p('link-cycle-1'), ['', 123, 0, 0, '', '', p('./link-cycle-2')]], + [p('link-cycle-2'), ['', 123, 0, 0, '', '', p('./link-cycle-1')]], + // Was a symlink, now a regular file + [p('link-to-self'), ['', 123, 0, 0, '', '', 0]], + [p('link-to-nowhere'), ['', 123, 0, 0, '', '', p('./nowhere')]], + ]); + expect(tfs.getDifference(newFiles)).toEqual({ + changedFiles: new Map([ + [p('new-file'), ['', 789, 0, 0, '', '', 0]], + [p('foo/another.js'), ['', 124, 0, 0, '', '', 0]], + [p('link-to-self'), ['', 123, 0, 0, '', '', 0]], + ]), + removedFiles: new Set([ + p('foo/link-to-bar.js'), + p('foo/link-to-another.js'), + p('../outside/external.js'), + p('bar.js'), + p('root'), + ]), + }); + }); + }); + describe('matchFilesWithContext', () => { test('non-recursive, skipping deep paths', () => { expect(