feat(batch): batch feature skeleton

lib/batch/batch.ts

@@ -0,0 +1,38 @@
+import { Argv } from 'yargs';
+
+import { runner } from '../utils/process';
+import { createLogger } from '../services/logger.service';
+
+import { initWorkspaceTask } from './tasks/init-workspace.task';
+import { initJobTask } from './tasks/init-job.task';
+import { runJobsTask } from './tasks/run-jobs.task';
+import { retrieveSettingsTask } from '../tasks/retrieve-settings.task';
+
+const logger = createLogger('BATCH');
+
+export const command = 'batch';
+
+export const aliases = ['b'];
+
+export const describe =
+  'Check out and analyze multiple project repositories and upload results to Omniboard.dev';
+
+export const builder = (yargs: Argv) =>
+  yargs
+    .option('job-path', {
+      type: 'string',
+      default: './omniboard-job.json',
+      description: 'Location of Omniboard batch job file',
+    })
+    .option('workspace-path', {
+      type: 'string',
+      default: './omniboard-workspace',
+      description: 'Location where the Omniboard batch workspace is stored',
+    });
+
+export const handler = async (argv: any) =>
+  runner(
+    [retrieveSettingsTask, initJobTask, initWorkspaceTask, runJobsTask],
+    argv,
+    logger
+  );

lib/batch/tasks/init-job-repo.task.ts

@@ -0,0 +1,27 @@
+import { ListrTask } from 'listr2';
+
+import { Context } from '../../interface';
+import {
+  getRepoNameFromUrl,
+  cloneRepo,
+  pullLatest,
+} from '../../services/git.service';
+import { directoryExists, pathJoin } from '../../services/fs.service';
+
+export function initJobRepo(job: string): ListrTask {
+  return {
+    title: 'Init job repo',
+    task: async (ctx: Context, task) => {
+      const { workspacePath, verbose } = ctx.options;
+      const repoName = getRepoNameFromUrl(job);
+      const repoPath = pathJoin(workspacePath, repoName);
+      if (!directoryExists(repoPath)) {
+        await cloneRepo(job, workspacePath);
+        task.title = `${task.title}, cloned`;
+      } else {
+        await pullLatest(repoPath);
+        task.title = `${task.title}, updated`;
+      }
+    },
+  };
+}

lib/batch/tasks/init-job.task.ts

@@ -0,0 +1,27 @@
+import { ListrTask } from 'listr2';
+
+import { BatchJob, Context } from '../../interface';
+import { readJson, writeJson } from '../../services/fs.service';
+
+const DEFAULT_JOB: BatchJob = {
+  running: '',
+  queue: [],
+  completed: [],
+  failed: [],
+};
+
+export const initJobTask: ListrTask = {
+  title: 'Init job',
+  task: async (ctx: Context, task) => {
+    const { jobPath } = ctx.options;
+    let job = readJson(jobPath);
+    if (!job) {
+      job = DEFAULT_JOB;
+      writeJson(jobPath, job);
+      task.title = `${task.title} - created new job file at ${jobPath}`;
+    } else {
+      task.title = `${task.title} successful, found ${job.queue?.length} jobs`;
+    }
+    ctx.batchJob = job;
+  },
+};

lib/batch/tasks/init-workspace.task.ts

@@ -0,0 +1,13 @@
+import { ListrTask } from 'listr2';
+
+import { Context } from '../../interface';
+import { ensureDirectoryExists } from '../../services/fs.service';
+
+export const initWorkspaceTask: ListrTask = {
+  title: 'Init workspace',
+  task: async (ctx: Context, task) => {
+    const { workspacePath } = ctx.options;
+    ensureDirectoryExists(workspacePath);
+    task.title = `${task.title} successful, workspace initialized ${workspacePath}`;
+  },
+};

lib/batch/tasks/run-job.task.ts

@@ -0,0 +1,14 @@
+import { ListrTask } from 'listr2';
+
+import { Context } from '../../interface';
+
+import { initJobRepo } from './init-job-repo.task';
+
+export function runJobTaskFactory(job: string): ListrTask {
+  return {
+    title: `${job}`,
+    task: async (ctx: Context, task) => {
+      return task.newListr([initJobRepo(job)]);
+    },
+  };
+}

lib/batch/tasks/run-jobs.task.ts

@@ -0,0 +1,13 @@
+import { ListrTask } from 'listr2';
+
+import { Context } from '../../interface';
+
+import { runJobTaskFactory } from './run-job.task';
+
+export const runJobsTask: ListrTask = {
+  title: 'Run jobs',
+  task: async (ctx: Context, task) =>
+    task.newListr(
+      ctx.batchJob.queue.map((queuedJob) => runJobTaskFactory(queuedJob))
+    ),
+};

lib/index.ts

@@ -18,6 +18,7 @@ import yargs, { Argv } from 'yargs';
 import * as analyzeCommand from './commands/analyze';
 import * as testCheckCommand from './commands/test-check';
 import * as testConnectionCommand from './commands/test-connection';
+import * as batchCommand from './batch/batch';
 import { apiMiddleware } from './middlewares/api.middleware';
 import { loggerMiddleware } from './middlewares/logger.middleware';
 
@@ -29,6 +30,7 @@ import { loggerMiddleware } from './middlewares/logger.middleware';
   .command(analyzeCommand)
   .command(testCheckCommand)
   .command(testConnectionCommand)
+  .command(batchCommand)
   .option('verbose', {
     type: 'boolean',
     default: false,

lib/interface.ts

@@ -9,6 +9,9 @@ export interface Options {
   apiKey?: string;
   silent: boolean;
   sanitizeRepoUrl: boolean;
+  // batch
+  jobPath: string;
+  workspacePath: string;
 }
 
 export interface Context {
@@ -36,6 +39,14 @@ export interface Context {
   };
   handledCheckFailures: Error[];
   processedResults?: any;
+  batchJob: BatchJob;
+}
+
+export interface BatchJob {
+  running: string;
+  queue: string[];
+  completed: string[];
+  failed: string[];
 }
 
 export interface CustomProjectResolver {

lib/services/fs.service.ts

@@ -88,9 +88,9 @@ export function readXmlAsDom(
 
 export function writeJson(destinationPath: string, data: any) {
   const { base, dir } = path.parse(destinationPath);
-  const stringifiedData = JSON.stringify(data, null, 2);
+  const dataAsString = JSON.stringify(data, null, 2);
   fs.mkdirSync(dir, { recursive: true });
-  fs.writeFileSync(destinationPath, stringifiedData);
+  fs.writeFileSync(destinationPath, dataAsString);
 }
 
 export function readFile(path: string) {
@@ -105,3 +105,17 @@ export function getFileSize(path: string) {
 export function getHumanReadableFileSize(size: number) {
   return filesize(size);
 }
+
+export function ensureDirectoryExists(path: string) {
+  if (!fs.existsSync(path)) {
+    fs.mkdirSync(path, { recursive: true });
+  }
+}
+
+export function directoryExists(path: string) {
+  return fs.existsSync(path);
+}
+
+export function pathJoin(...parts: string[]) {
+  return path.join(...parts);
+}

lib/services/git.service.ts

@@ -0,0 +1,14 @@
+import { run } from './shell.service';
+
+export function getRepoNameFromUrl(url: string): string {
+  const parts = url.split('/');
+  return parts[parts.length - 1].replace(/\.git$/, '');
+}
+
+export async function cloneRepo(url: string, targetDir: string) {
+  return await run(`git clone --depth 1 ${url}`, targetDir);
+}
+
+export async function pullLatest(targetDir: string) {
+  return await run(`git checkout --force && git pull`, targetDir);
+}

lib/services/shell.service.ts

@@ -0,0 +1,20 @@
+import cp from 'child_process';
+import { promisify } from 'util';
+
+const execAsync = promisify(cp.exec);
+const MAX_BUFFER_SIZE = 1 * 1024 * 1024;
+
+export async function run(
+  command: string,
+  targetDir: string
+): Promise<CommandResult> {
+  return await execAsync(command, {
+    cwd: targetDir,
+    maxBuffer: MAX_BUFFER_SIZE,
+  });
+}
+
+export interface CommandResult {
+  stdout: string;
+  stderr: string;
+}

lib/utils/process.ts

@@ -19,6 +19,7 @@ export const runner = async (
     results: { checks: {} },
     definitions: {},
     handledCheckFailures: [],
+    batchJob: { running: '', queue: [], completed: [], failed: [] },
   };
   await new Listr(tasks, {
     rendererFallback: () => options?.verbose,