Merge pull request #576 from steveukx/feat/progress

feat: Progress Handler

readme.md

@@ -77,6 +77,36 @@ const git: SimpleGit = simpleGit('/some/path', { config: ['http.proxy=someproxy'
 await git.pull();
 ```
 
+## Progress Events
+
+To receive progress updates, pass a `progress` configuration option to the `simpleGit` instance:
+
+```typescript
+import simpleGit, { SimpleGit, SimpleGitProgressEvent } from 'simple-git';
+
+const progress = ({method, stage, progress}: SimpleGitProgressEvent) => {
+   console.log(`git.${method} ${stage} stage ${progress}% complete`);
+}
+const git: SimpleGit = simpleGit({baseDir: '/some/path', progress});
+
+// pull automatically triggers progress events when the progress plugin is configured
+await git.pull();
+
+// supply the `--progress` option to any other command that supports it to receive
+// progress events into your handler
+await git.raw('pull', '--progress');
+```
+
+The `checkout`, `clone`, `pull`, `push` methods will automatically enable progress events when
+a progress handler has been set. For any other method that _can_ support progress events, set
+`--progress` in the task's `TaskOptions` for example to receive progress events when running
+submodule tasks:
+
+```typescript
+await git.submoduleUpdate('submodule-name', { '--progress': null });
+await git.submoduleUpdate('submodule-name', ['--progress']);
+```
+
 ## Using task callbacks
 
 Each of the methods in the API listing below can be called in a chain and will be called in series after each other.

src/git-factory.js

@@ -1,7 +1,9 @@
 const Git = require('./git');
+
 const {GitConstructError} = require('./lib/api');
 const {PluginStore} = require("./lib/plugins/plugin-store");
 const {commandConfigPrefixingPlugin} = require('./lib/plugins/command-config-prefixing-plugin');
+const {progressMonitorPlugin} = require('./lib/plugins/progress-monitor-plugin');
 const {createInstanceConfig, folderExists} = require('./lib/utils');
 
 const api = Object.create(null);
@@ -45,9 +47,11 @@ module.exports.gitInstanceFactory = function gitInstanceFactory (baseDir, option
       throw new GitConstructError(config, `Cannot use simple-git on a directory that does not exist`);
    }
 
-   if (config.config) {
+   if (Array.isArray(config.config)) {
       plugins.add(commandConfigPrefixingPlugin(config.config));
    }
 
+   config.progress && plugins.add(progressMonitorPlugin(config.progress));
+
    return new Git(config, plugins);
 };

src/lib/plugins/plugin-store.ts

@@ -1,13 +1,16 @@
 import { SimpleGitPlugin, SimpleGitPluginType, SimpleGitPluginTypes } from './simple-git-plugin';
+import { asArray } from '../utils';
 
 export class PluginStore {
 
    private plugins: Set<SimpleGitPlugin<SimpleGitPluginType>> = new Set();
 
-   public add<T extends SimpleGitPluginType>(plugin: SimpleGitPlugin<T>) {
-      this.plugins.add(plugin);
+   public add<T extends SimpleGitPluginType>(plugin: SimpleGitPlugin<T> | SimpleGitPlugin<T>[]) {
+      const plugins = asArray(plugin);
+      plugins.forEach(plugin => this.plugins.add(plugin));
+
       return () => {
-         this.plugins.delete(plugin);
+         plugins.forEach(plugin => this.plugins.delete(plugin));
       };
    }
 

src/lib/plugins/progress-monitor-plugin.ts

@@ -0,0 +1,50 @@
+import { SimpleGitOptions } from '../types';
+import { asNumber, including } from '../utils';
+
+import { SimpleGitPlugin } from './simple-git-plugin';
+
+export function progressMonitorPlugin(progress: Exclude<SimpleGitOptions['progress'], void>) {
+   const progressCommand = '--progress';
+   const progressMethods = ['checkout', 'clone', 'pull', 'push'];
+
+   const onProgress: SimpleGitPlugin<'spawn.after'> = {
+      type: 'spawn.after',
+      action(_data, context) {
+         if (!context.commands.includes(progressCommand)) {
+            return;
+         }
+
+         context.spawned.stderr?.on('data', (chunk: Buffer) => {
+            const message = /^([a-zA-Z ]+):\s*(\d+)% \((\d+)\/(\d+)\)/.exec(chunk.toString('utf8'));
+            if (!message) {
+               return;
+            }
+
+            progress({
+               method: context.method,
+               stage: progressEventStage(message[1]),
+               progress: asNumber(message[2]),
+               processed: asNumber(message[3]),
+               total: asNumber(message[4]),
+            });
+         });
+      }
+   };
+
+   const onArgs: SimpleGitPlugin<'spawn.args'> = {
+      type: 'spawn.args',
+      action(args, context) {
+         if (!progressMethods.includes(context.method)) {
+            return args;
+         }
+
+         return including(args, progressCommand);
+      }
+   }
+
+   return [onArgs, onProgress];
+}
+
+function progressEventStage (input: string) {
+   return String(input.toLowerCase().split(' ', 1)) || 'unknown';
+}

src/lib/plugins/simple-git-plugin.ts

@@ -1,8 +1,21 @@
+import { ChildProcess } from 'child_process';
+
+type SimpleGitTaskPluginContext = {
+   readonly method: string;
+   readonly commands: string[];
+}
+
 export interface SimpleGitPluginTypes {
    'spawn.args': {
       data: string[];
-      context: {};
+      context: SimpleGitTaskPluginContext & {};
    };
+   'spawn.after': {
+      data: void;
+      context: SimpleGitTaskPluginContext & {
+         spawned: ChildProcess,
+      };
+   }
 }
 
 export type SimpleGitPluginType = keyof SimpleGitPluginTypes;

src/lib/runners/git-executor-chain.ts

@@ -1,9 +1,8 @@
 import { spawn, SpawnOptions } from 'child_process';
 import { GitError } from '../api';
 import { OutputLogger } from '../git-logger';
+import { PluginStore } from '../plugins';
 import { EmptyTask, isBufferTask, isEmptyTask, } from '../tasks/task';
-import { Scheduler } from './scheduler';
-import { TasksPendingQueue } from './tasks-pending-queue';
 import {
    GitExecutorResult,
    Maybe,
@@ -13,8 +12,9 @@ import {
    SimpleGitTask,
    TaskResponseFormat
 } from '../types';
-import { callTaskParser, GitOutputStreams, objectToString } from '../utils';
-import { PluginStore } from '../plugins/plugin-store';
+import { callTaskParser, first, GitOutputStreams, objectToString } from '../utils';
+import { Scheduler } from './scheduler';
+import { TasksPendingQueue } from './tasks-pending-queue';
 
 export class GitExecutorChain implements SimpleGitExecutor {
 
@@ -82,9 +82,10 @@ export class GitExecutorChain implements SimpleGitExecutor {
    }
 
    private async attemptRemoteTask<R>(task: RunnableTask<R>, logger: OutputLogger) {
-      const args = this._plugins.exec('spawn.args', task.commands, {});
+      const args = this._plugins.exec('spawn.args', [...task.commands], pluginContext(task, task.commands));
 
       const raw = await this.gitResponse(
+         task,
          this.binary, args, this.outputHandler, logger.step('SPAWN'),
       );
       const outputStreams = await this.handleTaskData(task, raw, logger.step('HANDLE'));
@@ -148,7 +149,7 @@ export class GitExecutorChain implements SimpleGitExecutor {
       });
    }
 
-   private async gitResponse(command: string, args: string[], outputHandler: Maybe<outputHandler>, logger: OutputLogger): Promise<GitExecutorResult> {
+   private async gitResponse<R>(task: SimpleGitTask<R>, command: string, args: string[], outputHandler: Maybe<outputHandler>, logger: OutputLogger): Promise<GitExecutorResult> {
       const outputLogger = logger.sibling('output');
       const spawnOptions: SpawnOptions = {
          cwd: this.cwd,
@@ -202,11 +203,23 @@ export class GitExecutorChain implements SimpleGitExecutor {
             outputHandler(command, spawned.stdout!, spawned.stderr!, [...args]);
          }
 
+         this._plugins.exec('spawn.after', undefined, {
+            ...pluginContext(task, args),
+            spawned,
+         });
+
       });
    }
 
 }
 
+function pluginContext<R>(task: SimpleGitTask<R>, commands: string[]) {
+   return {
+      method: first(task.commands) || '',
+      commands,
+   }
+}
+
 function onErrorReceived(target: Buffer[], logger: OutputLogger) {
    return (err: Error) => {
       logger(`[ERROR] child process exception %o`, err);

src/lib/types/handlers.ts

@@ -9,3 +9,19 @@ export interface SimpleGitTaskCallback<T = string, E extends GitError = GitError
 
    (err: E): void;
 }
+
+/**
+ * The event data emitted to the progress handler whenever progress detail is received.
+ */
+export interface SimpleGitProgressEvent {
+   /** The underlying method called - push, pull etc */
+   method: string;
+   /** The type of progress being reported, note that any one task may emit many stages - for example `git clone` emits both `receiving` and `resolving` */
+   stage: 'compressing' | 'counting' | 'receiving' | 'resolving' | 'unknown' | 'writing' | string;
+   /** The percent progressed as a number 0 - 100 */
+   progress: number;
+   /** The number of items processed so far */
+   processed: number;
+   /** The total number of items to be processed */
+   total: number;
+}

src/lib/types/index.ts

@@ -1,4 +1,5 @@
 import { SimpleGitTask } from './tasks';
+import { SimpleGitProgressEvent } from './handlers';
 
 export * from './handlers';
 export * from './tasks';
@@ -37,9 +38,6 @@ export type outputHandler = (
 export type GitExecutorEnv = NodeJS.ProcessEnv | undefined;
 
 
-
-
-
 /**
  * Public interface of the Executor
  */
@@ -50,6 +48,7 @@ export interface SimpleGitExecutor {
    cwd: string;
 
    chain(): SimpleGitExecutor;
+
    push<R>(task: SimpleGitTask<R>): Promise<R>;
 }
 
@@ -71,6 +70,8 @@ export interface SimpleGitOptions {
    binary: string;
    maxConcurrentProcesses: number;
    config: string[];
+
+   progress?(data: SimpleGitProgressEvent): void;
 }
 
 export type Maybe<T> = T | undefined;

src/lib/utils/util.ts

@@ -75,7 +75,7 @@ export function folderExists(path: string): boolean {
 }
 
 /**
- * Adds `item` into the `target` `Array` or `Set` when it is not already present.
+ * Adds `item` into the `target` `Array` or `Set` when it is not already present and returns the `item`.
  */
 export function append<T>(target: T[] | Set<T>, item: T): typeof item {
    if (Array.isArray(target)) {
@@ -88,6 +88,17 @@ export function append<T>(target: T[] | Set<T>, item: T): typeof item {
    return item;
 }
 
+/**
+ * Adds `item` into the `target` `Array` when it is not already present and returns the `target`.
+ */
+export function including<T>(target: T[], item: T): typeof target {
+   if (Array.isArray(target) && !target.includes(item)) {
+      target.push(item);
+   }
+
+   return target;
+}
+
 export function remove<T>(target: Set<T> | T[], item: T): T {
    if (Array.isArray(target)) {
       const index = target.indexOf(item);

test/integration/progress-plugin.spec.ts

@@ -0,0 +1,43 @@
+import { createTestContext, newSimpleGit, SimpleGitTestContext } from '../__fixtures__';
+import { SimpleGitOptions } from '../../src/lib/types';
+
+describe('progress-monitor', () => {
+
+   const upstream = 'https://github.com/steveukx/git-js.git';
+
+   let context: SimpleGitTestContext;
+
+   beforeEach(async () => context = await createTestContext());
+
+   it('emits progress events', async () => {
+      const progress = jest.fn();
+      const opt: Partial<SimpleGitOptions> = {
+         baseDir: context.root,
+         progress,
+      };
+
+      await newSimpleGit(opt).clone(upstream);
+
+      const receivingUpdates = progressEventsAtStage(progress, 'receiving');
+
+      expect(receivingUpdates.length).toBeGreaterThan(0);
+
+      receivingUpdates.reduce((previous, update) => {
+         expect(update).toEqual({
+            method: 'clone',
+            stage: 'receiving',
+            progress: expect.any(Number),
+            processed: expect.any(Number),
+            total: expect.any(Number),
+         });
+
+         expect(update.progress).toBeGreaterThanOrEqual(previous);
+         return update.progress;
+      }, 0);
+   });
+
+});
+
+function progressEventsAtStage (mock: jest.Mock, stage: string) {
+   return mock.mock.calls.filter(c => c[0].stage === stage).map(c => c[0]);
+}

test/unit/__fixtures__/child-processes.ts

@@ -4,6 +4,21 @@ import { wait } from '../../__fixtures__';
 const EXIT_CODE_SUCCESS = 0;
 const EXIT_CODE_ERROR = 1;
 
+export async function writeToStdErr (data = '') {
+   await wait();
+   const proc = mockChildProcessModule.$mostRecent();
+
+   if (!proc) {
+      throw new Error(`writeToStdErr unable to find matching child process`);
+   }
+
+   if (proc.$emitted('exit')) {
+      throw new Error('writeToStdErr: attempting to write to an already closed process');
+   }
+
+   proc.stderr.$emit('data', Buffer.from(data));
+}
+
 export async function closeWithError (stack = 'CLOSING WITH ERROR', code = EXIT_CODE_ERROR) {
    await wait();
    const match = mockChildProcessModule.$mostRecent();

test/unit/__fixtures__/expectations.ts

@@ -27,6 +27,10 @@ export function assertExecutedCommandsContains(command: string) {
    expect(mockChildProcessModule.$mostRecent().$args.indexOf(command)).not.toBe(-1);
 }
 
+export function assertExecutedCommandsContainsOnce(command: string) {
+   expect(mockChildProcessModule.$mostRecent().$args.filter(c => c === command)).toHaveLength(1);
+}
+
 export function assertChildProcessEnvironmentVariables(env: any) {
    expect(mockChildProcessModule.$mostRecent()).toHaveProperty('$env', env);
 }