feat: Support ignoring paths

This patch adds the ability for upstream templates to main a
`.demsignore` file, which is a list of glob patterns separated by
newlines. This is helpful for two reasons;

- Skipping specific files that may contain template variables you want
  to skip; i.e. CI configuration, or UI libraries using handlebars.
- Reducing the number of files that will get walked to have their
  content replaced with mustache variables

package.json

@@ -41,5 +41,9 @@
     "tar-fs": "^2.0.0",
     "ts-jest": "^24.0.2",
     "typescript": "^3.5.3"
+  },
+  "dependencies": {
+    "@types/minimatch": "^3.0.3",
+    "minimatch": "^3.0.4"
   }
 }

src/cli.ts

@@ -1,3 +1,4 @@
+import { join } from 'path';
 import { walkAndRender } from './walk';
 import { help } from './help';
 import { getValidConfig } from './validate';
@@ -19,8 +20,8 @@ export async function cli(
   }
 
   try {
-    const { resolvedDest, archiveUrl } = await getValidConfig(...args);
-    const templateVariables = await downloadRepo(archiveUrl, resolvedDest);
+    const config = await getValidConfig(...args);
+    const templateVariables = await downloadRepo(config);
 
     if (templateVariables.size > 0) {
       const variables: { [key: string]: string } = {};
@@ -35,12 +36,15 @@ export async function cli(
 
       debug(`Variables: ${JSON.stringify(variables, null, 2)}`);
 
-      await walkAndRender(resolvedDest, variables);
+      const resolvedGlobs = config.ignoreGlobs.map((glob) =>
+        join(config.resolvedDest, glob),
+      );
+      await walkAndRender(config.resolvedDest, variables, resolvedGlobs);
     }
 
     return {
       code: 0,
-      message: `Successfully downloaded to ${resolvedDest}`,
+      message: `Successfully downloaded to ${config.resolvedDest}`,
     };
   } catch ({ code, message }) {
     return { code, message };

src/download-repo.ts

@@ -3,6 +3,8 @@ import { createGunzip } from 'zlib';
 import { extract, Headers } from 'tar-fs';
 import { ReadStream } from 'fs';
 import { parse } from 'mustache';
+import minimatch from 'minimatch';
+import { Config } from './types';
 
 function map(header: Headers) {
   // eslint-disable-next-line no-param-reassign
@@ -13,37 +15,41 @@ function map(header: Headers) {
   return header;
 }
 
-function getMapStream(templateVariables: Set<string>) {
-  return (fileStream: ReadStream, { type }: Headers) => {
-    if (type === 'file') {
-      const templateChunks: string[] = [];
-
-      fileStream.on('data', (chunk) => templateChunks.push(chunk.toString()));
-      fileStream.on('end', () => {
-        const template: string[] = parse(templateChunks.join(''));
-        template
-          .filter((entry) => entry[0] === 'name')
-          .map((entry) => entry[1])
-          .forEach((entry) => templateVariables.add(entry));
-      });
-
+function getMapStream(templateVariables: Set<string>, globsToIgnore: string[]) {
+  return (fileStream: ReadStream, { type, name }: Headers) => {
+    if (
+      type !== 'file' ||
+      globsToIgnore.some((glob) => minimatch(name, glob))
+    ) {
       return fileStream;
     }
 
+    const templateChunks: string[] = [];
+
+    fileStream.on('data', (chunk) => templateChunks.push(chunk.toString()));
+    fileStream.on('end', () => {
+      const template: string[] = parse(templateChunks.join(''));
+      template
+        .filter((entry) => entry[0] === 'name')
+        .map((entry) => entry[1])
+        .forEach((entry) => templateVariables.add(entry));
+    });
+
     return fileStream;
   };
 }
 
-export async function downloadRepo(
-  archiveUrl: string,
-  dest: string,
-): Promise<Set<string>> {
+export async function downloadRepo({
+  archiveUrl,
+  resolvedDest,
+  ignoreGlobs,
+}: Config): Promise<Set<string>> {
   return new Promise((resolve, reject) => {
     // TODO: Clean up this side-effect nightmare
     const templateVariables: Set<string> = new Set();
 
     // eslint-disable-next-line consistent-return
-    get(archiveUrl, (response) => {
+    get(archiveUrl, async (response) => {
       if (!response.statusCode || response.statusCode >= 400) {
         return reject(new Error(response.statusMessage));
       }
@@ -59,13 +65,18 @@ export async function downloadRepo(
           );
         }
 
-        downloadRepo(redirectUrl, dest).then(resolve, reject);
+        downloadRepo({
+          archiveUrl: redirectUrl,
+          resolvedDest,
+          ignoreGlobs,
+        }).then(resolve, reject);
       } else {
-        const mapStream = getMapStream(templateVariables);
+        const mapStream = getMapStream(templateVariables, ignoreGlobs);
+
         response
           .pipe(createGunzip())
           .pipe(
-            extract(dest, {
+            extract(resolvedDest, {
               map,
               mapStream,
             }),

src/get-globs-to-ignore.ts

@@ -0,0 +1,35 @@
+import { get } from 'https';
+import { debug } from './log';
+
+export async function getGlobsToIgnore(
+  ignoreFileUrl: string,
+): Promise<string[]> {
+  let globs: string[] = [];
+
+  return new Promise((resolve) => {
+    try {
+      get(ignoreFileUrl, (response) => {
+        if (response.statusCode !== 200) {
+          debug(`No .demsignore file found at ${ignoreFileUrl}`);
+          resolve([]);
+        }
+
+        const globChunks: string[] = [];
+
+        response.on('data', (chunk) => {
+          globChunks.push(chunk);
+        });
+
+        response.on('end', () => {
+          globs = globChunks
+            .join('')
+            .split('\n')
+            .filter(Boolean);
+          resolve(globs);
+        });
+      });
+    } catch {
+      debug(`Error retrieving .demsignore file at ${ignoreFileUrl}`);
+    }
+  });
+}

src/prompt.ts

@@ -3,8 +3,6 @@ import { createInterface } from 'readline';
 /**
  * Prompts for user input, displaying the given `question` string, and returns
  * the users input a a string
- *
- * @param question
  */
 export async function prompt(question: string): Promise<string> {
   const rl = createInterface({

src/types.ts

@@ -3,6 +3,7 @@ export type SCMType = 'github' | 'gitlab' | 'bitbucket';
 export interface Config {
   resolvedDest: string;
   archiveUrl: string;
+  ignoreGlobs: string[];
 }
 
 export interface CommitSHAMap {

src/validate.ts

@@ -1,10 +1,20 @@
 import { resolve } from 'path';
 import { existsSync } from 'fs';
-import { MissingSourceArgError, DestExistsError } from './errors';
-import { Config, CommitSHAMap } from './types';
+import {
+  MissingSourceArgError,
+  DestExistsError,
+  InvalidSourceError,
+} from './errors';
+import { Config, CommitSHAMap, SCMType } from './types';
 import { pExec } from './exec-promise';
 import { debug } from './log';
-import { getScmInfo } from './get-scm-info';
+import { getGlobsToIgnore } from './get-globs-to-ignore';
+
+const repoUrlMap = {
+  github: (repo: string) => `https://github.com/${repo}`,
+  gitlab: (repo: string) => `https://gitlab.com/${repo}`,
+  bitbucket: (repo: string) => `https://bitbucket.org/${repo}`,
+};
 
 const scmArchiveUrls = {
   gitlab: (url: string, sha: string) =>
@@ -13,6 +23,15 @@ const scmArchiveUrls = {
   github: (url: string, sha: string) => `${url}/archive/${sha}.tar.gz`,
 };
 
+const ignoreFileUrls = {
+  gitlab: (repo: string, sha: string) =>
+    `https://gitlab.com/${repo}/raw/${sha}/.demsignore`,
+  bitbucket: (repo: string, sha: string) =>
+    `https://bitbucket.org/${repo}/raw/${sha}/.demsignore`,
+  github: (repo: string, sha: string) =>
+    `https://raw.githubusercontent.com/${repo}/${sha}/.demsignore`,
+};
+
 /**
  * Run several checks to see if the user input appears valid. If valid,
  * transform it into a stricter structure for the rest of the program.
@@ -34,7 +53,17 @@ export async function getValidConfig(
   }
 
   debug('Checking source is valid');
-  const { url, type } = getScmInfo(descriptor);
+  const pattern = /(?:https:\/\/|git@)?(github|bitbucket|gitlab)?(?::)?(?:\.org|\.com)?(?:\/|:)?([\w-]+\/[\w-]+)(?:\.git)?/;
+  const match = descriptor.match(pattern);
+
+  if (!match) {
+    throw new InvalidSourceError();
+  }
+
+  const type = (match[1] || 'github') as SCMType;
+  const repo = match[2];
+  const url = repoUrlMap[type](repo);
+
   debug(`Valid source: ${url}`);
 
   const { stdout } = await pExec(`git ls-remote ${url}`);
@@ -57,6 +86,12 @@ export async function getValidConfig(
   const archiveUrl = scmArchiveUrls[type](url, sha);
   debug(`Archive URL: ${archiveUrl}`);
 
+  const ignoreFileUrl = ignoreFileUrls[type](repo, sha);
+  debug(`Ignore file URL: ${ignoreFileUrl}`);
+
+  const ignoreGlobs = await getGlobsToIgnore(ignoreFileUrl);
+  debug(`Globs to ignore: ${ignoreGlobs}`);
+
   debug('Checking dest is valid');
   // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
   const resolvedDest = resolve(dest || url.split('/').pop()!);
@@ -65,5 +100,5 @@ export async function getValidConfig(
   }
   debug(`Valid dest: ${resolvedDest}`);
 
-  return { resolvedDest, archiveUrl };
+  return { resolvedDest, archiveUrl, ignoreGlobs };
 }

src/walk.ts

@@ -1,11 +1,13 @@
 import { render } from 'mustache';
 import { promises as fsp } from 'fs';
-import { join } from 'path';
+import { join, resolve } from 'path';
+import minimatch from 'minimatch';
 import { debug } from './log';
 
 export async function walkAndRender(
   dir: string,
   templateVariables: { [key: string]: string },
+  ignoreGlobs: string[],
 ) {
   const files = await fsp.readdir(dir, { withFileTypes: true });
 
@@ -14,10 +16,18 @@ export async function walkAndRender(
       const filePath = join(dir, file.name);
 
       if (file.isDirectory()) {
-        return walkAndRender(filePath, templateVariables);
+        return walkAndRender(filePath, templateVariables, ignoreGlobs);
       }
 
       if (file.isFile()) {
+        if (file.name === '.demsignore') {
+          return fsp.unlink(filePath);
+        }
+
+        if (ignoreGlobs.some((glob) => minimatch(filePath, resolve(glob)))) {
+          return Promise.resolve();
+        }
+
         const fileToRender = await fsp.readFile(filePath, { encoding: 'utf8' });
         debug(`Writing file ${filePath}`);
         return fsp.writeFile(filePath, render(fileToRender, templateVariables));

test/cleanup.ts

@@ -1,22 +1,34 @@
 import { existsSync, rmdirSync, readdirSync, unlinkSync } from 'fs';
 import { resolve } from 'path';
 
+function deleteDirectory(path: string) {
+  if (existsSync(path)) {
+    const dirents = readdirSync(path, { withFileTypes: true });
+    dirents.forEach((ent) => {
+      if (ent.isFile()) {
+        unlinkSync(resolve(path, ent.name));
+      }
+
+      if (ent.isDirectory()) {
+        deleteDirectory(resolve(path, ent.name));
+      }
+    });
+
+    rmdirSync(path);
+  }
+}
+
 /**
  * Not an ideal solution, but relatively safe if the tests are not executed in
  * parallel, and provides an easier solution than mocking write streams. Note
  * that due to performance issues with Jest and parallelism in CircleCI,
  * --runInBand is recommended anyway.
  */
 export function removeTestDir() {
-  const dir = resolve('dems-example');
-  if (existsSync(dir)) {
-    const dirents = readdirSync(dir, { withFileTypes: true });
-    dirents.forEach((ent) => {
-      if (ent.isFile()) {
-        unlinkSync(resolve(dir, ent.name));
-      }
-    });
+  const fixturePaths = ['dems-example', 'dems-fixture-with-ignores'];
 
-    rmdirSync(dir);
-  }
+  fixturePaths.forEach((fixturePath) => {
+    const dir = resolve(fixturePath);
+    deleteDirectory(dir);
+  });
 }