Cache output files (#95)

* cache output files

* update help snapshots

* feed task outputs into downstream inputs

* add tests for makeGlobsAbsolute

* cleanup tests

* flesh out makeGlobsAbsolute tests

* add output file caching tests

* make timeout a little more forgiving

* add tests for passing cached outputs downstream

package.json

@@ -12,7 +12,7 @@
   "type": "module",
   "main": "index.js",
   "scripts": {
-    "test": "rimraf .test && jest",
+    "test": "rm -rf .test && jest",
     "update-snapshots": "jest -u",
     "clean": "rm -rf node_modules && pnpm install",
     "format": "prettier --write .",
@@ -94,7 +94,6 @@
     "node-gyp": "^9.3.1",
     "prettier": "^2.8.7",
     "prettier-plugin-organize-imports": "^3.2.2",
-    "rimraf": "^5.0.0",
     "semver": "^7.4.0",
     "typescript": "^5.0.4",
     "wrangler": "2.17.0"

src/commands/clean.js

@@ -1,7 +1,7 @@
 import glob from 'fast-glob'
 import { logger } from '../logger/logger.js'
 import { Project } from '../project/Project.js'
-import { rimraf } from '../rimraf.js'
+import { rimraf } from '../utils/rimraf.js'
 
 export function clean() {
   const project = Project.fromCwd(process.cwd())

src/commands/inherit.js

@@ -14,7 +14,7 @@ export async function inherit(options) {
       'No npm_lifecycle_event found. Did you run `lazy inherit` directly instead of via "scripts"?',
     )
   }
-  const config = await Config.fromCwd(process.cwd())
+  const config = await Config.fromCwd(process.cwd(), options.verbose)
   const workspace =
     process.cwd() === config.project.root.dir
       ? config.project.root

src/commands/run.js

@@ -1,20 +1,20 @@
 import pc from 'picocolors'
 import { dedent } from 'ts-dedent'
-import { TaskGraph } from '../TaskGraph.js'
 import { Config } from '../config/config.js'
-import { createTimer } from '../createTimer.js'
 import { InteractiveLogger } from '../logger/InteractiveLogger.js'
 import { getColorForString, pipe } from '../logger/formatting.js'
 import { logger } from '../logger/logger.js'
-import { rainbow } from '../rainbow.js'
+import { rainbow } from '../logger/rainbow.js'
+import { TaskGraph } from '../tasks/TaskGraph.js'
+import { createTimer } from '../utils/createTimer.js'
 
 /**
  * @param {{scriptName: string, options: import('../types.js').CLIOption}} args
  * @param {import('../config/config.js').Config} [_config]
  */
 export async function run({ scriptName, options }, _config) {
   const timer = createTimer()
-  const config = _config ?? (await Config.fromCwd(process.cwd()))
+  const config = _config ?? (await Config.fromCwd(process.cwd(), options.verbose))
 
   const filterPaths = options.filter
     ? Array.isArray(options.filter)
@@ -51,7 +51,7 @@ export async function run({ scriptName, options }, _config) {
 
   const failedTasks = tasks.allFailedTaskKeys()
   if (failedTasks.length > 0) {
-    logger.logErr(
+    logger.log(
       pc.bold(pc.red('\nFailed tasks:')),
       failedTasks.map((t) => getColorForString(t).fg(t)).join(', '),
     )

src/config/config.js

@@ -55,7 +55,11 @@ export class TaskConfig {
   get urlSafeName() {
     // Script names can contain any character, so let's slugify them and add a hex-encoded full name to avoid slug collisions
     // h/t wireit https://github.com/google/wireit/blob/27712b767dbebd90460049a3daf87329b9fb3279/src/util/script-data-dir.ts#L25
-    return slugify(this.name) + '-' + Buffer.from(this.name).toString('hex')
+    const slug = slugify(this.name)
+    if (slug !== this.name) {
+      return slug + '-' + Buffer.from(this.name).toString('hex')
+    }
+    return this.name
   }
 
   /** @private */
@@ -67,6 +71,10 @@ export class TaskConfig {
     return join(this.dataDir, 'manifest.tsv')
   }
 
+  getOutputManifestPath() {
+    return join(this.dataDir, 'output-manifest.tsv')
+  }
+
   getNextManifestPath() {
     return join(this.dataDir, 'manifest.next.tsv')
   }
@@ -187,8 +195,8 @@ export class TaskConfig {
 
       return {
         envInputs: this.scriptConfig.cache?.envInputs ?? [],
-        inputs: extractGlobPattern(this.scriptConfig.cache?.inputs),
-        outputs: extractGlobPattern(this.scriptConfig.cache?.outputs),
+        inputs: extractGlobPattern(this.scriptConfig.cache?.inputs, ['**/*']),
+        outputs: extractGlobPattern(this.scriptConfig.cache?.outputs, []),
         inheritsInputFromDependencies,
         usesOutputFromDependencies,
       }
@@ -243,14 +251,14 @@ export function extractInheritMatch(command) {
 }
 
 /**
- *
  * @param {import('./config-types.js').GlobConfig | null | undefined} glob
- * @returns {{include: string[], exclude: string[]}}
+ * @param {string[]} defaultInclude
+ * @returns {{include: string[];exclude: string[];}}
  */
-function extractGlobPattern(glob) {
+function extractGlobPattern(glob, defaultInclude) {
   if (!glob) {
     return {
-      include: ['**/*'],
+      include: defaultInclude,
       exclude: [],
     }
   }
@@ -261,7 +269,7 @@ function extractGlobPattern(glob) {
     }
   }
 
-  return { include: glob.include ?? ['**/*'], exclude: glob.exclude ?? [] }
+  return { include: glob.include ?? defaultInclude, exclude: glob.exclude ?? [] }
 }
 
 export class Config {
@@ -273,17 +281,19 @@ export class Config {
    *
    * @property {Project} project
    * @property {import('./resolveConfig.js').ResolvedConfig} rootConfig
+   * @property {boolean} isVerbose
    */
   /** @param {ConfigWrapperOptions} options */
-  constructor({ project, rootConfig }) {
+  constructor({ project, rootConfig, isVerbose }) {
     this.project = project
     this.rootConfig = rootConfig
+    this.isVerbose = !!isVerbose
   }
 
   /**
    * @param {string} cwd
    */
-  static async fromCwd(cwd) {
+  static async fromCwd(cwd, isVerbose = false) {
     let project = Project.fromCwd(cwd)
     const rootConfig = await resolveConfig(project.root.dir)
     project = project.withoutIgnoredWorkspaces(rootConfig.config.ignoreWorkspaces ?? [])
@@ -297,6 +307,7 @@ export class Config {
     return new Config({
       project,
       rootConfig,
+      isVerbose,
     })
   }
   /**

src/config/resolveConfig.js

@@ -1,8 +1,8 @@
 import glob from 'fast-glob'
 import { join } from 'path'
 import { existsSync, mkdirSync, writeFileSync } from '../fs.js'
-import { isTest } from '../isTest.js'
 import { logger } from '../logger/logger.js'
+import { isTest } from '../utils/isTest.js'
 import { validateConfig } from './validateConfig.js'
 
 /**

src/execCli.js

@@ -5,10 +5,10 @@ import { inherit } from './commands/inherit.js'
 import { init } from './commands/init.js'
 import { run } from './commands/run.js'
 import { readFileSync } from './fs.js'
-import { isTest } from './isTest.js'
 import { LazyError } from './logger/LazyError.js'
 import { logger } from './logger/logger.js'
-import { rainbow } from './rainbow.js'
+import { rainbow } from './logger/rainbow.js'
+import { isTest } from './utils/isTest.js'
 
 const cli = cac('lazy')
 
@@ -21,6 +21,9 @@ cli
   .option('--force', '[boolean] ignore the cache', {
     default: false,
   })
+  .option('--verbose', '[boolean] verbose log output', {
+    default: false,
+  })
   .action(async (scriptName, options) => {
     // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
     return await run({ scriptName, options })
@@ -35,6 +38,9 @@ cli
   .option('--force', '[boolean] ignore the cache', {
     default: false,
   })
+  .option('--verbose', '[boolean] verbose log output', {
+    default: false,
+  })
   .action(async (scriptName, options) => {
     // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
     return await run({ scriptName, options })
@@ -54,6 +60,9 @@ cli
     '(use in package.json "scripts" only) Runs the command specified in the lazy config file for the script name.',
   )
   .option('--force', '[boolean] ignore the cache', { default: false })
+  .option('--verbose', '[boolean] verbose log output', {
+    default: false,
+  })
   .action(async (options) => {
     // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
     return await inherit(options)
@@ -95,10 +104,10 @@ export async function execCli(argv) {
       console.log(pc.red(msg) + '\n')
       cli.outputHelp()
     } else if (e instanceof LazyError) {
-      logger.logErr(e.format())
+      logger.log(e.format())
     } else {
       // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-member-access
-      logger.logErr(e.stack ?? e.message ?? e)
+      logger.log(e.stack ?? e.message ?? e)
     }
     return 1
   } finally {

src/logger/InteractiveLogger.js

@@ -2,7 +2,7 @@
 
 import pc from 'picocolors'
 import _sliceAnsi from 'slice-ansi'
-import { createTimer } from '../createTimer.js'
+import { createTimer } from '../utils/createTimer.js'
 import { LazyError } from './LazyError.js'
 import {
   formatDiffMessage,
@@ -111,12 +111,6 @@ export class InteractiveLogger {
       this.tty.write(args.join(' ') + '\n')
     })
   }
-  /**
-   * @param {string[]} args
-   */
-  logErr(...args) {
-    this.log(...args)
-  }
 
   /**
    * @param {string} headline
@@ -159,11 +153,16 @@ export class InteractiveLogger {
     this.log(formatSuccessMessage(message))
   }
 
+  get isVerbose() {
+    return false
+  }
+
   /**
    * @param {string} taskName
+   * @param {boolean} isVerbose
    * @returns {import('../types.js').TaskLogger}
    */
-  task(taskName) {
+  task(taskName, isVerbose) {
     const timer = createTimer()
     const color = getColorForString(taskName)
     const prefix = color.fg(`${taskName} `)
@@ -236,6 +235,7 @@ export class InteractiveLogger {
     }
 
     return {
+      isVerbose,
       restartTimer: () => {
         assertNotDone()
         timer.reset()
@@ -244,9 +244,6 @@ export class InteractiveLogger {
       log: (...args) => {
         log('running', ...args)
       },
-      logErr: (...args) => {
-        log('running', ...args)
-      },
       fail: (headline, more) => {
         complete('failed', formatFailMessage(headline, more))
       },