diff --git a/package.json b/package.json index ae61759..67a5fbf 100644 --- a/package.json +++ b/package.json @@ -41,5 +41,9 @@ "tar-fs": "^2.0.0", "ts-jest": "^24.0.2", "typescript": "^3.5.3" + }, + "dependencies": { + "@types/minimatch": "^3.0.3", + "minimatch": "^3.0.4" } } diff --git a/src/cli.ts b/src/cli.ts index d69bd34..73d638f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,3 +1,4 @@ +import { join } from 'path'; import { walkAndRender } from './walk'; import { help } from './help'; import { getValidConfig } from './validate'; @@ -19,8 +20,8 @@ export async function cli( } try { - const { resolvedDest, archiveUrl } = await getValidConfig(...args); - const templateVariables = await downloadRepo(archiveUrl, resolvedDest); + const config = await getValidConfig(...args); + const templateVariables = await downloadRepo(config); if (templateVariables.size > 0) { const variables: { [key: string]: string } = {}; @@ -35,12 +36,15 @@ export async function cli( debug(`Variables: ${JSON.stringify(variables, null, 2)}`); - await walkAndRender(resolvedDest, variables); + const resolvedGlobs = config.ignoreGlobs.map((glob) => + join(config.resolvedDest, glob), + ); + await walkAndRender(config.resolvedDest, variables, resolvedGlobs); } return { code: 0, - message: `Successfully downloaded to ${resolvedDest}`, + message: `Successfully downloaded to ${config.resolvedDest}`, }; } catch ({ code, message }) { return { code, message }; diff --git a/src/download-repo.ts b/src/download-repo.ts index d285c13..bac358d 100644 --- a/src/download-repo.ts +++ b/src/download-repo.ts @@ -3,6 +3,8 @@ import { createGunzip } from 'zlib'; import { extract, Headers } from 'tar-fs'; import { ReadStream } from 'fs'; import { parse } from 'mustache'; +import minimatch from 'minimatch'; +import { Config } from './types'; function map(header: Headers) { // eslint-disable-next-line no-param-reassign @@ -13,37 +15,41 @@ function map(header: Headers) { return header; } -function getMapStream(templateVariables: Set) { - return (fileStream: ReadStream, { type }: Headers) => { - if (type === 'file') { - const templateChunks: string[] = []; - - fileStream.on('data', (chunk) => templateChunks.push(chunk.toString())); - fileStream.on('end', () => { - const template: string[] = parse(templateChunks.join('')); - template - .filter((entry) => entry[0] === 'name') - .map((entry) => entry[1]) - .forEach((entry) => templateVariables.add(entry)); - }); - +function getMapStream(templateVariables: Set, globsToIgnore: string[]) { + return (fileStream: ReadStream, { type, name }: Headers) => { + if ( + type !== 'file' || + globsToIgnore.some((glob) => minimatch(name, glob)) + ) { return fileStream; } + const templateChunks: string[] = []; + + fileStream.on('data', (chunk) => templateChunks.push(chunk.toString())); + fileStream.on('end', () => { + const template: string[] = parse(templateChunks.join('')); + template + .filter((entry) => entry[0] === 'name') + .map((entry) => entry[1]) + .forEach((entry) => templateVariables.add(entry)); + }); + return fileStream; }; } -export async function downloadRepo( - archiveUrl: string, - dest: string, -): Promise> { +export async function downloadRepo({ + archiveUrl, + resolvedDest, + ignoreGlobs, +}: Config): Promise> { return new Promise((resolve, reject) => { // TODO: Clean up this side-effect nightmare const templateVariables: Set = new Set(); // eslint-disable-next-line consistent-return - get(archiveUrl, (response) => { + get(archiveUrl, async (response) => { if (!response.statusCode || response.statusCode >= 400) { return reject(new Error(response.statusMessage)); } @@ -59,13 +65,18 @@ export async function downloadRepo( ); } - downloadRepo(redirectUrl, dest).then(resolve, reject); + downloadRepo({ + archiveUrl: redirectUrl, + resolvedDest, + ignoreGlobs, + }).then(resolve, reject); } else { - const mapStream = getMapStream(templateVariables); + const mapStream = getMapStream(templateVariables, ignoreGlobs); + response .pipe(createGunzip()) .pipe( - extract(dest, { + extract(resolvedDest, { map, mapStream, }), diff --git a/src/get-globs-to-ignore.ts b/src/get-globs-to-ignore.ts new file mode 100644 index 0000000..1df30d8 --- /dev/null +++ b/src/get-globs-to-ignore.ts @@ -0,0 +1,35 @@ +import { get } from 'https'; +import { debug } from './log'; + +export async function getGlobsToIgnore( + ignoreFileUrl: string, +): Promise { + let globs: string[] = []; + + return new Promise((resolve) => { + try { + get(ignoreFileUrl, (response) => { + if (response.statusCode !== 200) { + debug(`No .demsignore file found at ${ignoreFileUrl}`); + resolve([]); + } + + const globChunks: string[] = []; + + response.on('data', (chunk) => { + globChunks.push(chunk); + }); + + response.on('end', () => { + globs = globChunks + .join('') + .split('\n') + .filter(Boolean); + resolve(globs); + }); + }); + } catch { + debug(`Error retrieving .demsignore file at ${ignoreFileUrl}`); + } + }); +} diff --git a/src/get-scm-info.ts b/src/get-scm-info.ts deleted file mode 100644 index 644cda8..0000000 --- a/src/get-scm-info.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { InvalidSourceError } from './errors'; -import { SCMType } from './types'; - -const repoUrlMap = { - github: (repo: string) => `https://github.com/${repo}`, - gitlab: (repo: string) => `https://gitlab.com/${repo}`, - bitbucket: (repo: string) => `https://bitbucket.org/${repo}`, -}; - -export function getScmInfo(descriptor: string): { type: SCMType; url: string } { - const pattern = /(?:https:\/\/|git@)?(github|bitbucket|gitlab)?(?::)?(?:\.org|\.com)?(?:\/|:)?([\w-]+\/[\w-]+)(?:\.git)?/; - const match = descriptor.match(pattern); - - if (!match) { - throw new InvalidSourceError(); - } - - const type = (match[1] || 'github') as SCMType; - const url = repoUrlMap[type](match[2]); - - return { type, url }; -} diff --git a/src/prompt.ts b/src/prompt.ts index 946aa2a..5c299a8 100644 --- a/src/prompt.ts +++ b/src/prompt.ts @@ -3,8 +3,6 @@ import { createInterface } from 'readline'; /** * Prompts for user input, displaying the given `question` string, and returns * the users input a a string - * - * @param question */ export async function prompt(question: string): Promise { const rl = createInterface({ diff --git a/src/types.ts b/src/types.ts index dec6c89..7ef6b44 100644 --- a/src/types.ts +++ b/src/types.ts @@ -3,6 +3,7 @@ export type SCMType = 'github' | 'gitlab' | 'bitbucket'; export interface Config { resolvedDest: string; archiveUrl: string; + ignoreGlobs: string[]; } export interface CommitSHAMap { diff --git a/src/validate.ts b/src/validate.ts index ef5d929..ecbf7bf 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -1,10 +1,20 @@ import { resolve } from 'path'; import { existsSync } from 'fs'; -import { MissingSourceArgError, DestExistsError } from './errors'; -import { Config, CommitSHAMap } from './types'; +import { + MissingSourceArgError, + DestExistsError, + InvalidSourceError, +} from './errors'; +import { Config, CommitSHAMap, SCMType } from './types'; import { pExec } from './exec-promise'; import { debug } from './log'; -import { getScmInfo } from './get-scm-info'; +import { getGlobsToIgnore } from './get-globs-to-ignore'; + +const repoUrlMap = { + github: (repo: string) => `https://github.com/${repo}`, + gitlab: (repo: string) => `https://gitlab.com/${repo}`, + bitbucket: (repo: string) => `https://bitbucket.org/${repo}`, +}; const scmArchiveUrls = { gitlab: (url: string, sha: string) => @@ -13,6 +23,15 @@ const scmArchiveUrls = { github: (url: string, sha: string) => `${url}/archive/${sha}.tar.gz`, }; +const ignoreFileUrls = { + gitlab: (repo: string, sha: string) => + `https://gitlab.com/${repo}/raw/${sha}/.demsignore`, + bitbucket: (repo: string, sha: string) => + `https://bitbucket.org/${repo}/raw/${sha}/.demsignore`, + github: (repo: string, sha: string) => + `https://raw.githubusercontent.com/${repo}/${sha}/.demsignore`, +}; + /** * Run several checks to see if the user input appears valid. If valid, * transform it into a stricter structure for the rest of the program. @@ -34,7 +53,17 @@ export async function getValidConfig( } debug('Checking source is valid'); - const { url, type } = getScmInfo(descriptor); + const pattern = /(?:https:\/\/|git@)?(github|bitbucket|gitlab)?(?::)?(?:\.org|\.com)?(?:\/|:)?([\w-]+\/[\w-]+)(?:\.git)?/; + const match = descriptor.match(pattern); + + if (!match) { + throw new InvalidSourceError(); + } + + const type = (match[1] || 'github') as SCMType; + const repo = match[2]; + const url = repoUrlMap[type](repo); + debug(`Valid source: ${url}`); const { stdout } = await pExec(`git ls-remote ${url}`); @@ -57,6 +86,12 @@ export async function getValidConfig( const archiveUrl = scmArchiveUrls[type](url, sha); debug(`Archive URL: ${archiveUrl}`); + const ignoreFileUrl = ignoreFileUrls[type](repo, sha); + debug(`Ignore file URL: ${ignoreFileUrl}`); + + const ignoreGlobs = await getGlobsToIgnore(ignoreFileUrl); + debug(`Globs to ignore: ${ignoreGlobs}`); + debug('Checking dest is valid'); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const resolvedDest = resolve(dest || url.split('/').pop()!); @@ -65,5 +100,5 @@ export async function getValidConfig( } debug(`Valid dest: ${resolvedDest}`); - return { resolvedDest, archiveUrl }; + return { resolvedDest, archiveUrl, ignoreGlobs }; } diff --git a/src/walk.ts b/src/walk.ts index f22a1ea..54f713c 100644 --- a/src/walk.ts +++ b/src/walk.ts @@ -1,11 +1,13 @@ import { render } from 'mustache'; import { promises as fsp } from 'fs'; -import { join } from 'path'; +import { join, resolve } from 'path'; +import minimatch from 'minimatch'; import { debug } from './log'; export async function walkAndRender( dir: string, templateVariables: { [key: string]: string }, + ignoreGlobs: string[], ) { const files = await fsp.readdir(dir, { withFileTypes: true }); @@ -14,10 +16,18 @@ export async function walkAndRender( const filePath = join(dir, file.name); if (file.isDirectory()) { - return walkAndRender(filePath, templateVariables); + return walkAndRender(filePath, templateVariables, ignoreGlobs); } if (file.isFile()) { + if (file.name === '.demsignore') { + return fsp.unlink(filePath); + } + + if (ignoreGlobs.some((glob) => minimatch(filePath, resolve(glob)))) { + return Promise.resolve(); + } + const fileToRender = await fsp.readFile(filePath, { encoding: 'utf8' }); debug(`Writing file ${filePath}`); return fsp.writeFile(filePath, render(fileToRender, templateVariables)); diff --git a/test/cleanup.ts b/test/cleanup.ts index 09dc65d..c7556cc 100644 --- a/test/cleanup.ts +++ b/test/cleanup.ts @@ -1,6 +1,23 @@ import { existsSync, rmdirSync, readdirSync, unlinkSync } from 'fs'; import { resolve } from 'path'; +function deleteDirectory(path: string) { + if (existsSync(path)) { + const dirents = readdirSync(path, { withFileTypes: true }); + dirents.forEach((ent) => { + if (ent.isFile()) { + unlinkSync(resolve(path, ent.name)); + } + + if (ent.isDirectory()) { + deleteDirectory(resolve(path, ent.name)); + } + }); + + rmdirSync(path); + } +} + /** * Not an ideal solution, but relatively safe if the tests are not executed in * parallel, and provides an easier solution than mocking write streams. Note @@ -8,15 +25,10 @@ import { resolve } from 'path'; * --runInBand is recommended anyway. */ export function removeTestDir() { - const dir = resolve('dems-example'); - if (existsSync(dir)) { - const dirents = readdirSync(dir, { withFileTypes: true }); - dirents.forEach((ent) => { - if (ent.isFile()) { - unlinkSync(resolve(dir, ent.name)); - } - }); + const fixturePaths = ['dems-example', 'dems-fixture-with-ignores']; - rmdirSync(dir); - } + fixturePaths.forEach((fixturePath) => { + const dir = resolve(fixturePath); + deleteDirectory(dir); + }); } diff --git a/test/cli.test.ts b/test/cli.test.ts index 0e63535..43ac28b 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -1,5 +1,5 @@ import { mocked } from 'ts-jest/utils'; -import { promises as fsp } from 'fs'; +import { promises as fsp, existsSync } from 'fs'; import { resolve } from 'path'; import { cli } from '../src/cli'; import { prompt } from '../src/prompt'; @@ -73,7 +73,7 @@ describe('cli', () => { promptMock.mockResolvedValueOnce('foo'); promptMock.mockResolvedValueOnce('bar'); - const args = ['github:robcresswell/dems-example']; + const args = [url]; const { code } = await cli(args); expect(writeMock).toHaveBeenCalledTimes(4); @@ -88,5 +88,32 @@ describe('cli', () => { expect(code).toBe(0); }); }); + + it('handles .demsignore files', async () => { + promptMock.mockResolvedValueOnce('foo'); + promptMock.mockResolvedValueOnce('bar'); + + const args = ['robcresswell/dems-fixture-with-ignores']; + const { code } = await cli(args); + + expect(writeMock).toHaveBeenCalledTimes(4); + expect(writeMock).toHaveBeenCalledWith( + resolve('dems-fixture-with-ignores', 'LICENSE.md'), + expect.any(String), + ); + expect(writeMock).toHaveBeenCalledWith( + resolve('dems-fixture-with-ignores', 'README.md'), + expect.any(String), + ); + + // Assert that this file exists, but was not written to + const fixtureTestFile = resolve( + 'dems-fixture-with-ignores', + 'test/test-file.js', + ); + expect(existsSync(fixtureTestFile)).toBe(true); + expect(writeMock).not.toHaveBeenCalledWith(fixtureTestFile); + expect(code).toBe(0); + }); }); }); diff --git a/test/get-scm-info.test.ts b/test/get-scm-info.test.ts deleted file mode 100644 index 22b4c4d..0000000 --- a/test/get-scm-info.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { getScmInfo } from '../src/get-scm-info'; -import { SCMType } from '../src/types'; - -jest.mock('../src/exec-promise'); - -const validSourceFixtures: { - [key: string]: { - type: SCMType; - url: string; - }; -} = { - 'robcresswell/dems-example-1': { - type: 'github', - url: 'https://github.com/robcresswell/dems-example-1', - }, - 'github:robcresswell/dems-example-2': { - type: 'github', - url: 'https://github.com/robcresswell/dems-example-2', - }, - 'https://github.com/robcresswell/dems-example-3': { - type: 'github', - url: 'https://github.com/robcresswell/dems-example-3', - }, - 'git@github.com:robcresswell/dems-example-4.git': { - type: 'github', - url: 'https://github.com/robcresswell/dems-example-4', - }, - 'gitlab:robcresswell/dems-example-5': { - type: 'gitlab', - url: 'https://gitlab.com/robcresswell/dems-example-5', - }, - 'https://gitlab.com/robcresswell/dems-example-6': { - type: 'gitlab', - url: 'https://gitlab.com/robcresswell/dems-example-6', - }, - 'git@gitlab.com:robcresswell/dems-example-7.git': { - type: 'gitlab', - url: 'https://gitlab.com/robcresswell/dems-example-7', - }, - 'bitbucket:robcresswell/dems-example-8': { - type: 'bitbucket', - url: 'https://bitbucket.org/robcresswell/dems-example-8', - }, - 'https://bitbucket.org/robcresswell/dems-example-9': { - type: 'bitbucket', - url: 'https://bitbucket.org/robcresswell/dems-example-9', - }, -}; - -describe('get-scm-info', () => { - describe('parses SCM info correctly', () => { - Object.entries(validSourceFixtures).forEach(([sourceUrl, sourceInfo]) => { - it(sourceUrl, () => { - const { type, url } = getScmInfo(sourceUrl); - - expect(type).toEqual(sourceInfo.type); - expect(url).toEqual(sourceInfo.url); - }); - }); - }); - - describe('throws an error if the source is invalid', () => { - it('throws an error if an empty source is provided', () => { - expect(() => getScmInfo('')).toThrow(); - }); - - it('throws an error if there is no "/"', () => { - expect(() => getScmInfo('nonsense')).toThrow(); - }); - - it('throws an error if the project name is missing', () => { - expect(() => getScmInfo('partial/')).toThrow(); - }); - - it('throws an error if the user or org name is missing', () => { - expect(() => getScmInfo('/partial')).toThrow(); - }); - }); -}); diff --git a/test/walk.test.ts b/test/walk.test.ts index 1f0ab22..08d8e18 100644 --- a/test/walk.test.ts +++ b/test/walk.test.ts @@ -19,7 +19,7 @@ describe('walk', () => { name: 'foo', title: 'bar', }; - await walkAndRender('./test/fixtures/test-dir', templateVariables); + await walkAndRender('./test/fixtures/test-dir', templateVariables, []); expect(writeMock).toHaveBeenCalledTimes(2); expect(writeMock).toHaveBeenNthCalledWith( @@ -39,7 +39,7 @@ describe('walk', () => { name: 'foo', title: 'bar', }; - await walkAndRender('./test/fixtures/test-dir', templateVariables); + await walkAndRender('./test/fixtures/test-dir', templateVariables, []); expect(writeMock).toHaveBeenCalledTimes(2); diff --git a/yarn.lock b/yarn.lock index 03fcf25..bf19e10 100644 --- a/yarn.lock +++ b/yarn.lock @@ -360,6 +360,11 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.3.tgz#bdfd69d61e464dcc81b25159c270d75a73c1a636" integrity sha512-Il2DtDVRGDcqjDtE+rF8iqg1CArehSK84HZJCT7AMITlyXRBpuPhqGLDQMowraqqu1coEaimg4ZOqggt6L6L+A== +"@types/minimatch@^3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" + integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA== + "@types/mustache@^0.8.32": version "0.8.32" resolved "https://registry.yarnpkg.com/@types/mustache/-/mustache-0.8.32.tgz#7db3b81f2bf450bd38805f596d20eca97c4ed595"