Merge branch 'main' into markjm/split

.github/workflows/test.yml

@@ -3,10 +3,10 @@ name: Test
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
-      - master
+      - main
 
 jobs:
   lint:
@@ -36,7 +36,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        node-version: [10.x, 12.x, 14.x, 16.x]
+        node-version: [10.x, 12.x, 14.x, 16.x, 17.x]
         polling: ["false", "200"]
         exclude:
           - os: macos-latest

README.md

@@ -49,6 +49,8 @@ var wp = new Watchpack({
 	// ignored: "string" - a glob pattern for files or folders that should not be watched
 	// ignored: ["string", "string"] - multiple glob patterns that should be ignored
 	// ignored: /regexp/ - a regular expression for files or folders that should not be watched
+	// ignored: (entry) => boolean - an arbitrary function which must return truthy to ignore an entry
+	// For all cases expect the arbitrary function the path will have path separator normalized to '/'.
 	// All subdirectories are ignored too
 });
 
@@ -133,13 +135,13 @@ var fileTimes = wp.getTimes();
 // key: absolute path, value: timestamp as number
 ```
 
-[build-status]: https://travis-ci.org/webpack/watchpack.svg?branch=master
+[build-status]: https://travis-ci.org/webpack/watchpack.svg?branch=main
 [build-status-url]: https://travis-ci.org/webpack/watchpack
-[build-status-veyor]: https://ci.appveyor.com/api/projects/status/e5u2qvmugtv0r647/branch/master?svg=true
-[build-status-veyor-url]: https://ci.appveyor.com/project/sokra/watchpack/branch/master
+[build-status-veyor]: https://ci.appveyor.com/api/projects/status/e5u2qvmugtv0r647/branch/main?svg=true
+[build-status-veyor-url]: https://ci.appveyor.com/project/sokra/watchpack/branch/main
 [coveralls-url]: https://coveralls.io/r/webpack/watchpack/
 [coveralls-image]: https://img.shields.io/coveralls/webpack/watchpack.svg
-[codecov]: https://codecov.io/gh/webpack/watchpack/branch/master/graph/badge.svg
+[codecov]: https://codecov.io/gh/webpack/watchpack/branch/main/graph/badge.svg
 [codecov-url]: https://codecov.io/gh/webpack/watchpack
 [downloads]: https://img.shields.io/npm/dm/watchpack.svg
 [downloads-url]: https://www.npmjs.com/package/watchpack

lib/DirectoryWatcher.js

@@ -70,7 +70,7 @@ class DirectoryWatcher extends EventEmitter {
 		this.directories = new Map();
 		this.lastWatchEvent = 0;
 		this.initialScan = true;
-		this.ignored = options.ignored;
+		this.ignored = options.ignored || (() => false);
 		this.nestedWatching = false;
 		this.polledWatching =
 			typeof options.poll === "number"
@@ -95,12 +95,6 @@ class DirectoryWatcher extends EventEmitter {
 		this.doScan(true);
 	}
 
-	checkIgnore(path) {
-		if (!this.ignored) return false;
-		path = path.replace(/\\/g, "/");
-		return this.ignored.test(path);
-	}
-
 	createWatcher() {
 		try {
 			if (this.polledWatching) {
@@ -175,7 +169,7 @@ class DirectoryWatcher extends EventEmitter {
 	setFileTime(filePath, mtime, initial, ignoreWhenEqual, type) {
 		const now = Date.now();
 
-		if (this.checkIgnore(filePath)) return;
+		if (this.ignored(filePath)) return;
 
 		const old = this.files.get(filePath);
 
@@ -233,7 +227,7 @@ class DirectoryWatcher extends EventEmitter {
 	}
 
 	setDirectory(directoryPath, birthtime, initial, type) {
-		if (this.checkIgnore(directoryPath)) return;
+		if (this.ignored(directoryPath)) return;
 		if (directoryPath === this.path) {
 			if (!initial) {
 				this.forEachWatcher(this.path, w =>
@@ -391,7 +385,7 @@ class DirectoryWatcher extends EventEmitter {
 		}
 
 		const filePath = path.join(this.path, filename);
-		if (this.checkIgnore(filePath)) return;
+		if (this.ignored(filePath)) return;
 
 		if (this._activeEvents.get(filename) === undefined) {
 			this._activeEvents.set(filename, false);
@@ -637,6 +631,7 @@ class DirectoryWatcher extends EventEmitter {
 							if (
 								err2.code === "ENOENT" ||
 								err2.code === "EPERM" ||
+								err2.code === "EACCES" ||
 								err2.code === "EBUSY"
 							) {
 								this.setMissing(itemPath, initial, "scan (" + err2.code + ")");

lib/watchpack.js

@@ -14,7 +14,8 @@ const EMPTY_ARRAY = [];
 const EMPTY_OPTIONS = {};
 
 function addWatchersToSet(watchers, set) {
-	for (const w of watchers) {
+	for (const ww of watchers) {
+		const w = ww.watcher;
 		if (!set.has(w.directoryWatcher)) {
 			set.add(w.directoryWatcher);
 		}
@@ -28,24 +29,28 @@ const stringToRegexp = ignored => {
 	return matchingStart;
 };
 
-const ignoredToRegexp = ignored => {
+const ignoredToFunction = ignored => {
 	if (Array.isArray(ignored)) {
-		return new RegExp(ignored.map(i => stringToRegexp(i)).join("|"));
+		const regexp = new RegExp(ignored.map(i => stringToRegexp(i)).join("|"));
+		return x => regexp.test(x.replace(/\\/g, "/"));
 	} else if (typeof ignored === "string") {
-		return new RegExp(stringToRegexp(ignored));
+		const regexp = new RegExp(stringToRegexp(ignored));
+		return x => regexp.test(x.replace(/\\/g, "/"));
 	} else if (ignored instanceof RegExp) {
+		return x => ignored.test(x.replace(/\\/g, "/"));
+	} else if (ignored instanceof Function) {
 		return ignored;
 	} else if (ignored) {
 		throw new Error(`Invalid option for 'ignored': ${ignored}`);
 	} else {
-		return undefined;
+		return () => false;
 	}
 };
 
 const normalizeOptions = options => {
 	return {
 		followSymlinks: !!options.followSymlinks,
-		ignored: ignoredToRegexp(options.ignored),
+		ignored: ignoredToFunction(options.ignored),
 		poll: options.poll
 	};
 };
@@ -59,6 +64,85 @@ const cachedNormalizeOptions = options => {
 	return normalized;
 };
 
+class WatchpackFileWatcher {
+	constructor(watchpack, watcher, files) {
+		if (!watcher) throw new Error();
+		this.files = Array.isArray(files) ? files : [files];
+		this.watcher = watcher;
+		watcher.on("initial-missing", type => {
+			for (const file of this.files) {
+				if (!watchpack._missing.has(file))
+					watchpack._onRemove(file, file, type);
+			}
+		});
+		watcher.on("change", (mtime, type) => {
+			for (const file of this.files) {
+				watchpack._onChange(file, mtime, file, type);
+			}
+		});
+		watcher.on("remove", type => {
+			for (const file of this.files) {
+				watchpack._onRemove(file, file, type);
+			}
+		});
+	}
+
+	update(files) {
+		if (!Array.isArray(files)) {
+			if (this.files.length !== 1) {
+				this.files = [files];
+			} else if (this.files[0] !== files) {
+				this.files[0] = files;
+			}
+		} else {
+			this.files = files;
+		}
+	}
+
+	close() {
+		this.watcher.close();
+	}
+}
+
+class WatchpackDirectoryWatcher {
+	constructor(watchpack, watcher, directories) {
+		if (!watcher) throw new Error();
+		this.directories = Array.isArray(directories) ? directories : [directories];
+		this.watcher = watcher;
+		watcher.on("initial-missing", type => {
+			for (const item of this.directories) {
+				watchpack._onRemove(item, item, type);
+			}
+		});
+		watcher.on("change", (file, mtime, type) => {
+			for (const item of this.directories) {
+				watchpack._onChange(item, mtime, file, type);
+			}
+		});
+		watcher.on("remove", type => {
+			for (const item of this.directories) {
+				watchpack._onRemove(item, item, type);
+			}
+		});
+	}
+
+	update(directories) {
+		if (!Array.isArray(directories)) {
+			if (this.directories.length !== 1) {
+				this.directories = [directories];
+			} else if (this.directories[0] !== directories) {
+				this.directories[0] = directories;
+			}
+		} else {
+			this.directories = directories;
+		}
+	}
+
+	close() {
+		this.watcher.close();
+	}
+}
+
 class Watchpack extends EventEmitter {
 	constructor(options) {
 		super();
@@ -72,6 +156,7 @@ class Watchpack extends EventEmitter {
 		this.watcherManager = getWatcherManager(this.watcherOptions);
 		this.fileWatchers = new Map();
 		this.directoryWatchers = new Map();
+		this._missing = new Set();
 		this.startTime = undefined;
 		this.paused = false;
 		this.aggregatedChanges = new Set();
@@ -96,18 +181,18 @@ class Watchpack extends EventEmitter {
 			startTime = arg3;
 		}
 		this.paused = false;
-		const oldFileWatchers = this.fileWatchers;
-		const oldDirectoryWatchers = this.directoryWatchers;
+		const fileWatchers = this.fileWatchers;
+		const directoryWatchers = this.directoryWatchers;
 		const ignored = this.watcherOptions.ignored;
-		const filter = ignored
-			? path => !ignored.test(path.replace(/\\/g, "/"))
-			: () => true;
+		const filter = path => !ignored(path);
 		const addToMap = (map, key, item) => {
 			const list = map.get(key);
 			if (list === undefined) {
-				map.set(key, [item]);
-			} else {
+				map.set(key, item);
+			} else if (Array.isArray(list)) {
 				list.push(item);
+			} else {
+				map.set(key, [list, item]);
 			}
 		};
 		const fileWatchersNeeded = new Map();
@@ -167,82 +252,47 @@ class Watchpack extends EventEmitter {
 				}
 			}
 		}
-		const newFileWatchers = new Map();
-		const newDirectoryWatchers = new Map();
-		const setupFileWatcher = (watcher, key, files) => {
-			watcher.on("initial-missing", type => {
-				for (const file of files) {
-					if (!missingFiles.has(file)) this._onRemove(file, file, type);
-				}
-			});
-			watcher.on("change", (mtime, type) => {
-				for (const file of files) {
-					this._onChange(file, mtime, file, type);
-				}
-			});
-			watcher.on("remove", type => {
-				for (const file of files) {
-					this._onRemove(file, file, type);
-				}
-			});
-			newFileWatchers.set(key, watcher);
-		};
-		const setupDirectoryWatcher = (watcher, key, directories) => {
-			watcher.on("initial-missing", type => {
-				for (const item of directories) {
-					this._onRemove(item, item, type);
-				}
-			});
-			watcher.on("change", (file, mtime, type) => {
-				for (const item of directories) {
-					this._onChange(item, mtime, file, type);
-				}
-			});
-			watcher.on("remove", type => {
-				for (const item of directories) {
-					this._onRemove(item, item, type);
-				}
-			});
-			newDirectoryWatchers.set(key, watcher);
-		};
 		// Close unneeded old watchers
-		const fileWatchersToClose = [];
-		const directoryWatchersToClose = [];
-		for (const [key, w] of oldFileWatchers) {
-			if (!fileWatchersNeeded.has(key)) {
+		// and update existing watchers
+		for (const [key, w] of fileWatchers) {
+			const needed = fileWatchersNeeded.get(key);
+			if (needed === undefined) {
 				w.close();
+				fileWatchers.delete(key);
 			} else {
-				fileWatchersToClose.push(w);
+				w.update(needed);
+				fileWatchersNeeded.delete(key);
 			}
 		}
-		for (const [key, w] of oldDirectoryWatchers) {
-			if (!directoryWatchersNeeded.has(key)) {
+		for (const [key, w] of directoryWatchers) {
+			const needed = directoryWatchersNeeded.get(key);
+			if (needed === undefined) {
 				w.close();
+				directoryWatchers.delete(key);
 			} else {
-				directoryWatchersToClose.push(w);
+				w.update(needed);
+				directoryWatchersNeeded.delete(key);
 			}
 		}
 		// Create new watchers and install handlers on these watchers
 		watchEventSource.batch(() => {
 			for (const [key, files] of fileWatchersNeeded) {
 				const watcher = this.watcherManager.watchFile(key, startTime);
 				if (watcher) {
-					setupFileWatcher(watcher, key, files);
+					fileWatchers.set(key, new WatchpackFileWatcher(this, watcher, files));
 				}
 			}
 			for (const [key, directories] of directoryWatchersNeeded) {
 				const watcher = this.watcherManager.watchDirectory(key, startTime);
 				if (watcher) {
-					setupDirectoryWatcher(watcher, key, directories);
+					directoryWatchers.set(
+						key,
+						new WatchpackDirectoryWatcher(this, watcher, directories)
+					);
 				}
 			}
 		});
-		// Close old watchers
-		for (const w of fileWatchersToClose) w.close();
-		for (const w of directoryWatchersToClose) w.close();
-		// Store watchers
-		this.fileWatchers = newFileWatchers;
-		this.directoryWatchers = newDirectoryWatchers;
+		this._missing = missingFiles;
 		this.startTime = startTime;
 	}
 

test/ManyWatchers.js

@@ -66,6 +66,12 @@ describe("ManyWatchers", function() {
 			}
 			w.watch({ files });
 			console.timeEnd("creating/closing watchers");
+			console.time("calling watch with the same files");
+			for (let i = 0; i < 2000; i++) {
+				w.watch({ files });
+			}
+			console.timeEnd("calling watch with the same files");
+
 			testHelper.tick(10000, () => {
 				console.time("detecting change event");
 				testHelper.file("4096/900/file");