feat(batch): add batch capability

- batch analyse multiple repos (repo urls)
- reduce default check subtask console output
- improve check subtask completion progress output
- add global "show-check-subtasks" flag
- naming cleanup

lib/batch/batch.ts

@@ -7,6 +7,7 @@ import { initWorkspaceTask } from './tasks/init-workspace.task';
 import { initJobTask } from './tasks/init-job.task';
 import { runJobsTask } from './tasks/run-jobs.task';
 import { retrieveSettingsTask } from '../tasks/retrieve-settings.task';
+import { retrieveChecksTask } from '../tasks/retrieve-checks.task';
 
 const logger = createLogger('BATCH');
 
@@ -19,20 +20,43 @@ export const describe =
 
 export const builder = (yargs: Argv) =>
   yargs
+
     .option('job-path', {
       type: 'string',
       default: './omniboard-job.json',
       description: 'Location of Omniboard batch job file',
     })
+    .option('preserve-queue', {
+      type: 'boolean',
+      default: false,
+      description:
+        'Preserves Omniboard batch job queue (good for multiple runs)',
+    })
     .option('workspace-path', {
       type: 'string',
       default: './omniboard-workspace',
       description: 'Location where the Omniboard batch workspace is stored',
+    })
+    .option('json', {
+      type: 'boolean',
+      default: false,
+      description: 'Store results data in local json file',
+    })
+    .option('check-pattern', {
+      alias: 'cp',
+      type: 'string',
+      description: 'Only run checks matching provided pattern',
     });
 
 export const handler = async (argv: any) =>
   runner(
-    [retrieveSettingsTask, initJobTask, initWorkspaceTask, runJobsTask],
+    [
+      initWorkspaceTask,
+      initJobTask,
+      retrieveSettingsTask,
+      retrieveChecksTask,
+      runJobsTask,
+    ],
     argv,
     logger
   );

lib/batch/tasks/batch-save-project-json.task.ts

@@ -0,0 +1,18 @@
+import { ListrTask } from 'listr2';
+
+import * as fs from '../../services/fs.service';
+import { Context } from '../../interface';
+import { getRepoNameFromUrl } from '../../services/git.service';
+
+export function batchSaveProjectJsonTaskFactory(job: string): ListrTask {
+  return {
+    title: `Save project (local json file)`,
+    skip: (ctx: Context) => ctx.control.skipEverySubsequentTask,
+    enabled: (ctx: Context) => ctx.options.json,
+    task: async (ctx: Context, task) => {
+      const path = `../_dist/${getRepoNameFromUrl(job)}.json`;
+      fs.writeJson(path, ctx.processedResults);
+      task.title = `${task.title}, saved to: ${path}`;
+    },
+  };
+}

lib/batch/tasks/finalize-job.task.ts

@@ -0,0 +1,33 @@
+import { ListrTask } from 'listr2';
+
+import { writeJson } from '../../services/fs.service';
+import { Context, ParentTask } from '../../interface';
+
+export function finalizeJobTaskFactory(
+  job: string,
+  parentTask: ParentTask
+): ListrTask {
+  return {
+    title: 'Finalize job',
+    task: async (ctx: Context, task) => {
+      // reset job CTX state
+      ctx.control = { skipEverySubsequentTask: false };
+      ctx.results = { checks: {} };
+      ctx.handledCheckFailures = [];
+      ctx.processedResults = undefined;
+
+      // reset cwd
+      process.chdir('../../');
+
+      // update batch state
+      if (!ctx.options.preserveQueue) {
+        ctx.batch.completed.push(job);
+        ctx.batch.queue = ctx.batch.queue.filter((j) => j !== job);
+        writeJson(ctx.options.jobPath, ctx.batch);
+      }
+
+      task.title = `${task.title} successful`;
+      parentTask.title = `${parentTask.title} successful`;
+    },
+  };
+}

lib/batch/tasks/init-job-repo.task.ts

@@ -1,17 +1,20 @@
 import { ListrTask } from 'listr2';
 
-import { Context } from '../../interface';
 import {
   getRepoNameFromUrl,
   cloneRepo,
   pullLatest,
 } from '../../services/git.service';
+import { Context } from '../../interface';
 import { directoryExists, pathJoin } from '../../services/fs.service';
 
 export function initJobRepo(job: string): ListrTask {
   return {
     title: 'Init job repo',
     task: async (ctx: Context, task) => {
+      if (job.includes('b2c')) {
+        throw new Error('B2C not supported');
+      }
       const { workspacePath, verbose } = ctx.options;
       const repoName = getRepoNameFromUrl(job);
       const repoPath = pathJoin(workspacePath, repoName);
@@ -22,6 +25,7 @@ export function initJobRepo(job: string): ListrTask {
         await pullLatest(repoPath);
         task.title = `${task.title}, updated`;
       }
+      process.chdir(repoPath);
     },
   };
 }

lib/batch/tasks/init-job.task.ts

@@ -1,10 +1,9 @@
 import { ListrTask } from 'listr2';
 
-import { BatchJob, Context } from '../../interface';
+import { Batch, Context } from '../../interface';
 import { readJson, writeJson } from '../../services/fs.service';
 
-const DEFAULT_JOB: BatchJob = {
-  running: '',
+const DEFAULT_JOB: Batch = {
   queue: [],
   completed: [],
   failed: [],
@@ -18,10 +17,13 @@ export const initJobTask: ListrTask = {
     if (!job) {
       job = DEFAULT_JOB;
       writeJson(jobPath, job);
-      task.title = `${task.title} - created new job file at ${jobPath}`;
+      task.title = `${task.title} successful, created new job file at ${jobPath}`;
     } else {
-      task.title = `${task.title} successful, found ${job.queue?.length} jobs`;
+      task.title = `${task.title} successful, found ${jobPath} with ${job.queue?.length} jobs`;
+    }
+    ctx.batch = job;
+    if (!ctx.batch.queue?.length) {
+      ctx.control.skipEverySubsequentTask = true;
     }
-    ctx.batchJob = job;
   },
 };

lib/batch/tasks/init-workspace.task.ts

@@ -1,13 +1,18 @@
 import { ListrTask } from 'listr2';
 
 import { Context } from '../../interface';
-import { ensureDirectoryExists } from '../../services/fs.service';
+import {
+  ensureDirectoryExists,
+  removeDirectoryRecursive,
+  pathJoin,
+} from '../../services/fs.service';
 
 export const initWorkspaceTask: ListrTask = {
   title: 'Init workspace',
   task: async (ctx: Context, task) => {
     const { workspacePath } = ctx.options;
     ensureDirectoryExists(workspacePath);
+    removeDirectoryRecursive(pathJoin(workspacePath, '_dist'));
     task.title = `${task.title} successful, workspace initialized ${workspacePath}`;
   },
 };

lib/batch/tasks/run-job.task.ts

@@ -1,14 +1,47 @@
 import { ListrTask } from 'listr2';
 
 import { Context } from '../../interface';
+import { writeJson } from '../../services/fs.service';
+import { getRepoNameFromUrl } from '../../services/git.service';
+
+import { projectInfoTask } from '../../tasks/project-info.task';
+import { saveProjectApiTask } from '../../tasks/save-project-api.task';
+import { runChecksWrapperTask } from '../../tasks/run-checks-wrapper.task';
 
 import { initJobRepo } from './init-job-repo.task';
+import { finalizeJobTaskFactory } from './finalize-job.task';
+import { batchSaveProjectJsonTaskFactory } from './batch-save-project-json.task';
 
 export function runJobTaskFactory(job: string): ListrTask {
   return {
-    title: `${job}`,
+    title: `${getRepoNameFromUrl(job)}`,
+    rollback: (ctx: Context, task) => {
+      // update batch state
+      if (!ctx.options.preserveQueue) {
+        ctx.batch.failed.push(job);
+        ctx.batch.queue = ctx.batch.queue.filter((j) => j !== job);
+        writeJson(ctx.options.jobPath, ctx.batch);
+        task.title = `${task.title} failed, added to failed jobs`;
+      } else {
+        task.title = `${task.title} failed`;
+      }
+    },
     task: async (ctx: Context, task) => {
-      return task.newListr([initJobRepo(job)]);
+      return task.newListr(
+        [
+          initJobRepo(job),
+          projectInfoTask,
+          runChecksWrapperTask,
+          batchSaveProjectJsonTaskFactory(job),
+          saveProjectApiTask,
+          finalizeJobTaskFactory(job, task),
+        ],
+        {
+          rendererOptions: {
+            collapse: true,
+          },
+        }
+      );
     },
   };
 }

lib/batch/tasks/run-jobs.task.ts

@@ -6,8 +6,13 @@ import { runJobTaskFactory } from './run-job.task';
 
 export const runJobsTask: ListrTask = {
   title: 'Run jobs',
+  skip: (ctx: Context) => ctx.control.skipEverySubsequentTask,
   task: async (ctx: Context, task) =>
     task.newListr(
-      ctx.batchJob.queue.map((queuedJob) => runJobTaskFactory(queuedJob))
+      ctx.batch.queue.map((queuedJob) => runJobTaskFactory(queuedJob)),
+      {
+        exitOnError: false,
+        exitAfterRollback: false,
+      }
     ),
 };

lib/checks/check.service.ts

@@ -2,7 +2,7 @@ import {
   DEFAULT_EXCLUDE_FILES_PATTERN_FLAGS,
   DEFAULT_INCLUDE_FILES_FLAG,
 } from '../consts';
-import { BaseCheckDefinition } from '../interface';
+import { BaseCheckDefinition, ParentTask } from '../interface';
 import { resolveActiveFlags } from '../utils/regexp';
 import * as fs from '../services/fs.service';
 
@@ -25,6 +25,12 @@ export function resolveCheckTaskFulfilledTitle(
   } ${title}${matches.length > 0 ? `, found matches: ${matches.length}` : ''}`;
 }
 
+export function resolveCheckParentTaskProgress(parentTask: ParentTask) {
+  const regexp = /(?<current>\d*)\//;
+  const current = regexp.exec(parentTask.title)?.groups?.current ?? 0;
+  parentTask.title = parentTask.title.replace(regexp, `${+current + 1}/`);
+}
+
 export function getCheckFiles(
   definition: BaseCheckDefinition,
   defaultExcludeFilesPattern: string

lib/checks/content.check.ts

@@ -3,6 +3,7 @@ import { ListrDefaultRenderer, ListrTaskWrapper } from 'listr2';
 import {
   ContentCheckDefinition,
   Context,
+  ParentTask,
   ProjectCheckMatch,
   ProjectCheckMatchDetails,
 } from '../interface';
@@ -16,12 +17,16 @@ import * as fs from '../services/fs.service';
 import {
   CheckResultSymbol,
   getCheckFiles,
+  resolveCheckParentTaskProgress,
   resolveCheckTaskFulfilledTitle,
 } from './check.service';
 
 const DEFAULT_CONTENT_PATTERN_FLAGS = 'ig';
 
-export function contentCheckTaskFactory(definition: ContentCheckDefinition) {
+export function contentCheckTaskFactory(
+  definition: ContentCheckDefinition,
+  parentTask: ParentTask
+) {
   async function contentCheckTask(
     ctx: Context,
     task: ListrTaskWrapper<Context, ListrDefaultRenderer>
@@ -41,6 +46,7 @@ export function contentCheckTaskFactory(definition: ContentCheckDefinition) {
         value: false,
       };
       task.title = `${CheckResultSymbol.UNFULFILLED} ${task.title}`;
+      resolveCheckParentTaskProgress(parentTask);
       return;
     }
 
@@ -97,6 +103,7 @@ export function contentCheckTaskFactory(definition: ContentCheckDefinition) {
             matches,
           };
           task.title = resolveCheckTaskFulfilledTitle(task, matches);
+          resolveCheckParentTaskProgress(parentTask);
           resolve();
         }
       }

lib/checks/file.check.ts

@@ -1,11 +1,18 @@
 import { ListrDefaultRenderer, ListrTaskWrapper } from 'listr2';
 
-import { ContentCheckDefinition, Context } from '../interface';
+import { ContentCheckDefinition, Context, ParentTask } from '../interface';
 import { DEFAULT_EXCLUDE_FILES_PATTERN_CONTENT } from '../consts';
 
-import { CheckResultSymbol, getCheckFiles } from './check.service';
+import {
+  CheckResultSymbol,
+  getCheckFiles,
+  resolveCheckParentTaskProgress,
+} from './check.service';
 
-export function fileCheckTaskFactory(definition: ContentCheckDefinition) {
+export function fileCheckTaskFactory(
+  definition: ContentCheckDefinition,
+  parentTask: ParentTask
+) {
   async function fileCheckTask(
     ctx: Context,
     task: ListrTaskWrapper<Context, ListrDefaultRenderer>
@@ -25,6 +32,7 @@ export function fileCheckTaskFactory(definition: ContentCheckDefinition) {
         value: false,
       };
       task.title = `${CheckResultSymbol.UNFULFILLED} ${task.title}`;
+      resolveCheckParentTaskProgress(parentTask);
       return;
     } else {
       ctx.results.checks![name] = {
@@ -37,6 +45,7 @@ export function fileCheckTaskFactory(definition: ContentCheckDefinition) {
         })),
       };
       task.title = `${CheckResultSymbol.FULFILLED} ${task.title}`;
+      resolveCheckParentTaskProgress(parentTask);
     }
   }
   return fileCheckTask;