feat: rapid bar

packages/cli/lib/download_dependency.js

@@ -43,23 +43,19 @@ async function download(options) {
   const blobManager = new NpmBlobManager();
   const entryListener = entryListenerFactory(blobManager);
 
-  console.time('[rapid] downloader new');
   const downloader = new Downloader({
     entryListener,
     productionMode: options.productionMode,
   });
-  console.timeEnd('[rapid] downloader new');
-  console.time('[rapid] downloader init');
   await downloader.init();
-  console.timeEnd('[rapid] downloader init');
+
   options.downloader = downloader;
   const depsTree = options.depsTree;
 
-  console.time('[rapid] parallel download time');
   await downloader.download(depsTree);
-  console.log('[rapid] download finished');
   const { tocMap, indices } = downloader.dumpdata;
   await downloader.shutdown();
+  console.log('[rapid] download finished');
 
   for (const [ blobId, tocIndex ] of Object.entries(tocMap)) {
     blobManager.addBlob(blobId, tocIndex);
@@ -98,20 +94,23 @@ async function download(options) {
       }
     }
   }
-  console.timeEnd('[rapid] parallel download time');
-  console.time('[rapid] generate fs meta');
+
+  console.log('[rapid] generate fs meta');
+
+
   const npmFs = new NpmFs(blobManager, options);
   const allPkgs = await util.getAllPkgPaths(options.cwd, options.pkg);
+
   await Promise.all(allPkgs.map(async pkgPath => {
     const { tarIndex } = await util.getWorkdir(options.cwd, pkgPath);
     const fsMeta = await npmFs.getFsMeta(depsTree, pkgPath);
     await fs.mkdir(path.dirname(tarIndex), { recursive: true });
     await fs.writeFile(tarIndex, JSON.stringify(fsMeta), 'utf8');
   }));
+
   // FIXME atomic write
   await fs.writeFile(npmCacheConfigPath, JSON.stringify(tocMap), 'utf8');
   await fs.writeFile(npmIndexConfigPath, JSON.stringify(indices), 'utf8');
-  console.timeEnd('[rapid] generate fs meta');
   return {
     depsTree,
   };

packages/cli/lib/downloader.js

@@ -3,6 +3,7 @@
 const Util = require('./util');
 const { tarBucketsDir, npmCacheConfigPath, npmIndexConfigPath } = require('./constants');
 const os = require('node:os');
+const { Bar } = require('./logger');
 
 const platform = os.platform();
 const arch = os.arch();
@@ -38,6 +39,7 @@ class Downloader {
   async shutdown() {
     if (!this.rapidDownloader) return;
     await this.rapidDownloader.shutdown();
+    this.bar.stop();
   }
 
   createRapidDownloader() {
@@ -47,7 +49,10 @@ class Downloader {
       httpConcurrentCount: this.httpConcurrentCount,
       downloadDir: tarBucketsDir,
       entryWhitelist: [ '*/package.json', '*/binding.gyp' ],
-      entryListener: this.entryListener,
+      entryListener: entry => {
+        this.bar.update(entry?.pkgName);
+        this.entryListener(entry);
+      },
       downloadTimeout: this.downloadTimeout,
       tocPath: {
         map: npmCacheConfigPath,
@@ -59,6 +64,11 @@ class Downloader {
   async download(pkgLockJson) {
     const tasks = this.createDownloadTask(pkgLockJson);
     console.log('[rapid] downloads %d packages', tasks.length);
+    this.bar = new Bar({
+      type: 'download',
+      total: tasks.length,
+      autoFinish: false,
+    });
     await this.rapidDownloader.batchDownloads(tasks);
   }
 

packages/cli/lib/index.js

@@ -75,7 +75,8 @@ exports.clean = async function clean({ nydusMode = NYDUS_TYPE.FUSE, cwd, force,
     return;
   }
   if (!pkg) {
-    pkg = await util.readPkgJSON(cwd);
+    const pkgRes = await util.readPkgJSON(cwd);
+    pkg = pkgRes.pkg;
   }
   await nydusd.endNydusFs(nydusMode, cwd, pkg, force);
 };

packages/cli/lib/logger.js

@@ -0,0 +1,73 @@
+const cliProgress = require('cli-progress');
+
+const MAX_TITLE_LENGTH = 11;
+
+function padCenter(str, length, char = ' ') {
+  const padLength = length - str.length;
+  const padLeft = Math.floor(padLength / 2);
+  const padRight = padLength - padLeft;
+  return char.repeat(padLeft) + str + char.repeat(padRight);
+}
+
+class Bar {
+  constructor({ type, total }) {
+    const title = padCenter(type, MAX_TITLE_LENGTH);
+    this.type = type;
+    this.total = total;
+    this.multiBar = new cliProgress.MultiBar(
+      {
+        clearOnComplete: false,
+        hideCursor: true,
+        format: `[rapid] [{bar}] {percentage}% |${title}| {status} | {message}`,
+      },
+      cliProgress.Presets.shades_grey
+    );
+
+    this.startTime = Date.now();
+    this.isTTY = process.stdout.isTTY;
+
+    // init
+    if (this.isTTY) {
+      this.bar = this.multiBar.create(total, 1, {
+        status: 'Running',
+        warning: '',
+        message: '',
+      });
+    }
+
+  }
+
+  update(current = '') {
+
+    if (!this.isTTY) {
+      return;
+    }
+
+    const { value, total } = this.bar;
+    if (value < total) {
+      this.isTTY && this.bar.update(value + 1, { status: 'Running', message: current });
+    }
+
+    if (value >= total - 1) {
+      this.isTTY && this.bar.update(total - 1, { status: 'Processing', message: 'Processing...' });
+    }
+  }
+
+  stop() {
+    if (!this.isTTY) {
+      console.log('[rapid] %s complete, %dms', this.type, Date.now() - this.startTime);
+      return;
+    }
+
+    const { total } = this.bar;
+    this.bar.update(total, {
+      status: 'Complete',
+      message: Date.now() - this.startTime + 'ms',
+    });
+
+    this.multiBar.stop();
+
+  }
+}
+
+exports.Bar = Bar;

packages/cli/lib/npm_fs/index.js

@@ -4,6 +4,7 @@ const assert = require('node:assert');
 const NpmFsBuilder = require('./npm_fs_builder');
 const TnpmFsBuilder = require('./tnpm_fs_builder');
 const { NpmFsMode } = require('../constants');
+const { Bar } = require('../logger');
 
 class NpmFs {
   /**
@@ -15,6 +16,7 @@ class NpmFs {
    */
   constructor(blobManager, options) {
     this.blobManager = blobManager;
+
     this.options = Object.assign({
       uid: process.getuid(),
       gid: process.getgid(),
@@ -27,9 +29,17 @@ class NpmFs {
   }
 
   async getFsMeta(pkgLockJson, pkgPath = '') {
+    this.bar = new Bar({
+      type: 'fs meta',
+      total: Object.keys(pkgLockJson.packages).length,
+    });
     const builderClazz = this._getFsMetaBuilder();
     const builder = new builderClazz(this.blobManager, this.options);
-    return await builder.generateFsMeta(pkgLockJson, pkgPath);
+    const res = await builder.generateFsMeta(pkgLockJson, pkgPath, entryName => {
+      this.bar.update(entryName);
+    });
+    this.bar.stop();
+    return res;
   }
 
   _getFsMetaBuilder() {

packages/cli/lib/npm_fs/npm_fs_builder.js

@@ -19,16 +19,18 @@ class NpmFsMetaBuilder {
     this.uid = options.uid;
     this.gid = options.gid;
     this.cwd = options.cwd;
+    this.entryListener = options.entryListener;
     this.productionMode = options.productionMode;
     // 项目直接依赖的 bin
     this.pkgBinSet = new Set();
   }
 
-  async generateFsMeta(packageLockJson, currentPkgPath) {
+  async generateFsMeta(packageLockJson, currentPkgPath, entryListener) {
     const packageLock = new PackageLock({ cwd: this.cwd, packageLockJson });
     await packageLock.load();
     const packages = packageLock.packages;
     for (const [ pkgPath, pkgItem ] of Object.entries(packages)) {
+      entryListener?.(pkgPath);
       if (!pkgPath || !Util.validDep(pkgItem, this.productionMode)) continue;
       // npm alias or normal npm package
       const name = Util.getAliasPackageNameFromPackagePath(pkgPath, packages);

packages/cli/lib/npm_fs/tnpm_fs_builder.js

@@ -15,7 +15,7 @@ class TnpmFsBuilder {
    */
   constructor(blobManager, options) {
     this.blobManager = blobManager;
-    this.fsMeta = new FsMeta();
+    this.fsMeta = new FsMeta(options.entryListener);
     this.uid = options.uid;
     this.gid = options.gid;
     this.productionMode = options.productionMode;
@@ -25,7 +25,7 @@ class TnpmFsBuilder {
     this.projectVersions = new Map();
   }
 
-  generateFsMeta(packageLockJson) {
+  generateFsMeta(packageLockJson, _, entryListener) {
     this.getProjectVersions(packageLockJson);
     this.getLatestVersions(packageLockJson);
     this.createRealPkgs(packageLockJson);
@@ -34,6 +34,7 @@ class TnpmFsBuilder {
     const blobId = this.fsMeta.blobIds[0];
     const packages = packageLockJson.packages;
     for (const [ pkgPath, pkgItem ] of Object.entries(packages)) {
+      entryListener?.(pkgPath);
       if (this.shouldSkipGenerate(pkgPath, pkgItem)) continue;
       const name = Util.getPackageNameFromPackagePath(pkgPath, packages);
       const version = pkgItem.version;

packages/cli/lib/nydusd/fuse_mode.js

@@ -17,45 +17,56 @@ const {
   listMountInfo,
 } = require('../util');
 const nydusdApi = require('./nydusd_api');
+const { Bar } = require('../logger');
 
 async function startNydusFs(cwd, pkg) {
-  await Promise.all([
-    nydusdApi.initDaemon(),
-    generateBootstrapFile(cwd, pkg),
-  ]);
+  await nydusdApi.initDaemon();
 
+  console.log('[rapid] generate bootstrap');
+  await generateBootstrapFile(cwd, pkg);
+
+  console.log('[rapid] mount nydusd');
   await mountNydus(cwd, pkg);
 
-  console.time('[rapid] mount overlay');
+  console.log('[rapid] mount overlay, it may take a few seconds');
   await mountOverlay(cwd, pkg);
-  console.timeEnd('[rapid] mount overlay');
 }
 
 async function generateBootstrapFile(cwd, pkg) {
-  console.time('[rapid] generate bootstrap');
   const allPkgs = await getAllPkgPaths(cwd, pkg);
+  const bar = new Bar({ type: 'bootstrap', total: allPkgs.length });
   await Promise.all(allPkgs.map(async pkgPath => {
-    const { bootstrap, tarIndex } = await getWorkdir(cwd, pkgPath);
+    const { bootstrap, tarIndex, nodeModulesDir } = await getWorkdir(cwd, pkgPath);
     await fs.mkdir(path.dirname(bootstrap), { recursive: true });
     await execa.command(`${BOOTSTRAP_BIN} --stargz-config-path=${tarIndex} --stargz-dir=${tarBucketsDir} --bootstrap=${bootstrap}`);
+    bar.update(nodeModulesDir);
   }));
-  console.timeEnd('[rapid] generate bootstrap');
+  bar.stop();
 }
 
 async function mountNydus(cwd, pkg) {
   const allPkgs = await getAllPkgPaths(cwd, pkg);
 
+  const bar = new Bar({
+    type: 'mount',
+    total: allPkgs.length,
+  });
+
   // 需要串行 mount，并发创建时 nydusd 会出现问题
   for (const pkgPath of allPkgs) {
     const { dirname, bootstrap } = await getWorkdir(cwd, pkgPath);
-    console.time(`[rapid] mount '/${dirname}' to nydusd daemon using socket api`);
     await nydusdApi.mount(`/${dirname}`, cwd, bootstrap);
-    console.timeEnd(`[rapid] mount '/${dirname}' to nydusd daemon using socket api`);
+    bar.update(dirname);
   }
+  bar.stop();
 }
 
 async function mountOverlay(cwd, pkg) {
   const allPkgs = await getAllPkgPaths(cwd, pkg);
+  const bar = new Bar({
+    type: 'overlay',
+    total: allPkgs.length,
+  });
   await Promise.all(allPkgs.map(async pkgPath => {
     const {
       upper,
@@ -102,14 +113,14 @@ ${nodeModulesDir}`);
 ${upper}=RW:${mnt}=RO \
 ${nodeModulesDir}`;
     }
-    console.info('[rapid] mountOverlay: `%s`', shScript);
-    console.time(`[rapid] overlay ${overlay} mounted.`);
+    // console.info('[rapid] mountOverlay: `%s`', shScript);
     await execa.command(shScript);
-    console.timeEnd(`[rapid] overlay ${overlay} mounted.`);
+    bar.update(nodeModulesDir);
   }));
+  bar.stop();
 }
 
-async function endNydusFs(cwd, pkg, force = false) {
+async function endNydusFs(cwd, pkg, force = true) {
   const allPkgs = await getAllPkgPaths(cwd, pkg);
   const umountCmd = force ? 'umount -f' : 'umount';
   await Promise.all(allPkgs.map(async pkgPath => {
@@ -153,7 +164,7 @@ async function endNydusFs(cwd, pkg, force = false) {
         },
         fallback: force
           ? async () => {
-            await execa.command(`hdiutil detach -force ${overlay}`);
+            await execa.command(`umount -f ${overlay}`);
           }
           : undefined,
       });

packages/cli/package.json

@@ -17,6 +17,7 @@
     "await-event": "^2.1.0",
     "binary-mirror-config": "^2.5.0",
     "chalk": "^4.0.0",
+    "cli-progress": "^3.12.0",
     "execa": "^5.1.1",
     "inquirer": "^8.2.6",
     "ms": "^0.7.1",