Merge pull request #150 from quantcdn/feature/unfolding-meta

Update client to return all pages for meta calls.
quantcdn · May 24, 2022 · 27d9dbe · 27d9dbe
2 parents 3a35f7d + a6def65
commit 27d9dbe
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 5 deletions.
diff --git a/src/commands/scan.js b/src/commands/scan.js
@@ -16,7 +16,23 @@ const md5File = require('md5-file');
 const command = {};
 
 command.command = 'scan';
-command.describe = 'Validate file checksums';
+command.describe = 'Validate local file checksums';
+command.builder = (yargs) => {
+  yargs.options('diff-only', {
+    describe: 'Show only source files different from Quant',
+    type: 'boolean',
+    default: false,
+  });
+  yargs.options('unpublish-only', {
+    describe: 'Show only the unpublished results',
+    type: 'boolean',
+    default: false,
+  });
+  yargs.options('skip-unpublish-regex', {
+    describe: 'Skip the unpublish process for specific regex',
+    type: 'string',
+  });
+};
 
 command.handler = async function(argv) {
   config.fromArgs(argv);
@@ -42,9 +58,16 @@ command.handler = async function(argv) {
     yargs.exit(1);
   }
 
+  const relativeFiles = [];
+
   files.map(async (file) => {
     const filepath = path.relative(p, file);
     let revision = false;
+    relativeFiles.push(`/${filepath.toLowerCase()}`);
+
+    if (argv['unpublish-only']) {
+      return;
+    }
 
     try {
       revision = await quant.revisions(filepath);
@@ -58,9 +81,31 @@ command.handler = async function(argv) {
     const localmd5 = md5File.sync(file);
 
     if (revision.md5 == localmd5) {
-      console.log(chalk.green(`[info]: ${filepath} is up-to-date`));
+      if (!argv['diff-only']) {
+        console.log(chalk.green(`[info]: ${filepath} is up-to-date`));
+      }
     } else {
-      console.log(chalk.yellow(`[info]: ${filepath} is different.`));
+      if (argv['diff-only']) {
+        console.log(chalk.yellow(`[info]: ${filepath} is different.`));
+      }
+    }
+  });
+
+  data.records.map(async (item) => {
+    const f = item.url.replace('/index.html', '.html');
+
+    if (relativeFiles.includes(item.url) || relativeFiles.includes(f)) {
+      return;
+    }
+    // Skip unpublish process if skip unpublish regex matches.
+    if (argv['skip-unpublish-regex']) {
+      const match = item.url.match(argv['skip-unpublish-regex']);
+      if (match) {
+        return;
+      }
+    }
+    if (!argv['diff-only']) {
+      console.log(chalk.magenta(`[info]: ${item.url} is to be unpublished.`));
     }
   });
 };

diff --git a/src/quant-client.js b/src/quant-client.js
@@ -107,7 +107,6 @@ const client = function(config) {
         published: true,
       }, extend);
       const url = `${config.get('endpoint')}/global-meta?${querystring.stringify(query)}`;
-
       const doUnfold = async function(i) {
         const res = await get({
           url: `${url}&page=${i}`,
@@ -142,7 +141,7 @@ const client = function(config) {
 
       if (unfold) {
         page++;
-        while (res.body.global_meta.total_pages > page) {
+        while (res.body.global_meta.total_pages >= page) {
           await doUnfold(page);
           page++;
         }