[node-modules] Improve deduplication efficiency by tracking ancestor …

…dependencies (#955)

* Improve deduplication efficiency by tracking ancestor dependencies

* Sets versions

* Commit current changes

* Add support for NM_DEBUG_LEVEL env variable to dump the tree and reasons the packages were not hoisted

* Fix unit tests

* Add support of special hoisting case - dropping nodes

* Count only direct ancestors for popularity measurements

* Add check for hoisting peer dependent packages with same ident

* Speedup ancestor map building

* Remove unneeded node dropping concept

* Remove hoisting blacklist, it doesn't help now

* Improve ancestor map buiding time orders of magnitude

* Implement fast hoisting (direct to current root versus buble up as before)

* Remove regular deps from packagePeers (temporary fix)

.yarn/versions/e79b7c69.yml

@@ -0,0 +1,24 @@
+releases:
+  "@yarnpkg/cli": prerelease
+  "@yarnpkg/plugin-node-modules": prerelease
+  "@yarnpkg/pnpify": prerelease
+
+declined:
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-essentials"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - vscode-zipfs
+  - "@yarnpkg/builder"
+  - "@yarnpkg/core"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/pnp"

packages/yarnpkg-pnpify/sources/buildNodeModulesTree.ts

@@ -61,7 +61,7 @@ export const getArchivePath = (packagePath: PortablePath): PortablePath | null =
 export const buildNodeModulesTree = (pnp: PnpApi, options: NodeModulesTreeOptions): NodeModulesTree => {
   const packageTree = buildPackageTree(pnp);
 
-  const hoistedTree = hoist(packageTree, {check: false});
+  const hoistedTree = hoist(packageTree, {finalCheck: false});
 
   return populateNodeModulesTree(pnp, hoistedTree, options);
 };
@@ -121,7 +121,7 @@ const buildPackageTree = (pnp: PnpApi): HoisterTree => {
 
   const nodes = new Map<LocatorKey, HoisterTree>();
 
-  const addPackageToTree = (pkg: PackageInformation<NativePath>, locator: PackageLocator, parent: HoisterTree) => {
+  const addPackageToTree = (pkg: PackageInformation<NativePath>, locator: PackageLocator, parent: HoisterTree, parentPkg: PackageInformation<NativePath>) => {
     const locatorKey = stringifyLocator(locator);
     let node = nodes.get(locatorKey);
     const isSeen = !!node;
@@ -130,11 +130,17 @@ const buildPackageTree = (pnp: PnpApi): HoisterTree => {
     if (!node) {
       const {name, reference} = locator;
 
+      // TODO: remove this code when `packagePeers` will not contain regular dependencies
+      const peerNames = new Set<string>();
+      for (const peerName of pkg.packagePeers)
+        if (pkg.packageDependencies.get(peerName) === parentPkg.packageDependencies.get(peerName))
+          peerNames.add(peerName);
+
       node = {
         name: name!,
         reference: reference!,
         dependencies: new Set(),
-        peerNames: pkg.packagePeers,
+        peerNames,
       };
       nodes.set(locatorKey, node);
     }
@@ -148,14 +154,14 @@ const buildPackageTree = (pnp: PnpApi): HoisterTree => {
           const depPkg = pnp.getPackageInformation(pkgLocator)!;
           // Skip package self-references
           if (stringifyLocator(depLocator) !== locatorKey) {
-            addPackageToTree(depPkg, depLocator, node);
+            addPackageToTree(depPkg, depLocator, node, pkg);
           }
         }
       }
     }
   };
 
-  addPackageToTree(topPkg, topLocator, packageTree);
+  addPackageToTree(topPkg, topLocator, packageTree, topPkg);
 
   return packageTree;
 };