[kbn/optimizer/node] use shorter cache keys to avoid overflowing limit (

#93316) (#93444) Co-authored-by: spalger <[email protected]> Co-authored-by: Kibana Machine <[email protected]> # Conflicts: # packages/kbn-optimizer/src/node/node_auto_tranpilation.ts
elastic · Mar 3, 2021 · 92f82a0 · 92f82a0
1 parent 874b59f
commit 92f82a0
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 15 deletions.
diff --git a/packages/kbn-optimizer/src/node/cache.ts b/packages/kbn-optimizer/src/node/cache.ts
@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import Path from 'path';
 import { Writable } from 'stream';
 
 import chalk from 'chalk';
@@ -25,11 +26,17 @@ export class Cache {
   private readonly atimes: LmdbStore.Database<string, string>;
   private readonly mtimes: LmdbStore.Database<string, string>;
   private readonly sourceMaps: LmdbStore.Database<string, string>;
+  private readonly pathRoot: string;
   private readonly prefix: string;
   private readonly log?: Writable;
   private readonly timer: NodeJS.Timer;
 
-  constructor(config: { dir: string; prefix: string; log?: Writable }) {
+  constructor(config: { pathRoot: string; dir: string; prefix: string; log?: Writable }) {
+    if (!Path.isAbsolute(config.pathRoot)) {
+      throw new Error('cache requires an absolute path to resolve paths relative to');
+    }
+
+    this.pathRoot = config.pathRoot;
     this.prefix = config.prefix;
     this.log = config.log;
 
@@ -110,7 +117,12 @@ export class Cache {
   }
 
   private getKey(path: string) {
-    return `${this.prefix}${path}`;
+    const normalizedPath =
+      Path.sep !== '/'
+        ? Path.relative(this.pathRoot, path).split(Path.sep).join('/')
+        : Path.relative(this.pathRoot, path);
+
+    return `${this.prefix}${normalizedPath}`;
   }
 
   private safeGet<V>(db: LmdbStore.Database<V, string>, key: string) {

diff --git a/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts b/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts
@@ -53,8 +53,9 @@ it('returns undefined until values are set', async () => {
   const log = makeTestLog();
   const cache = makeCache({
     dir: DIR,
-    prefix: 'foo',
+    prefix: 'prefix:',
     log,
+    pathRoot: '/foo/',
   });
 
   expect(cache.getMtime(path)).toBe(undefined);
@@ -71,16 +72,16 @@ it('returns undefined until values are set', async () => {
   expect(cache.getCode(path)).toBe('var x = 1');
   expect(cache.getSourceMap(path)).toEqual({ foo: 'bar' });
   expect(log.output).toMatchInlineSnapshot(`
-    "MISS   [mtimes]   foo/foo/bar.js
-    MISS   [codes]   foo/foo/bar.js
-    MISS   [sourceMaps]   foo/foo/bar.js
-    PUT   [atimes]   foo/foo/bar.js
-    PUT   [mtimes]   foo/foo/bar.js
-    PUT   [codes]   foo/foo/bar.js
-    PUT   [sourceMaps]   foo/foo/bar.js
-    HIT   [mtimes]   foo/foo/bar.js
-    HIT   [codes]   foo/foo/bar.js
-    HIT   [sourceMaps]   foo/foo/bar.js
+    "MISS   [mtimes]   prefix:bar.js
+    MISS   [codes]   prefix:bar.js
+    MISS   [sourceMaps]   prefix:bar.js
+    PUT   [atimes]   prefix:bar.js
+    PUT   [mtimes]   prefix:bar.js
+    PUT   [codes]   prefix:bar.js
+    PUT   [sourceMaps]   prefix:bar.js
+    HIT   [mtimes]   prefix:bar.js
+    HIT   [codes]   prefix:bar.js
+    HIT   [sourceMaps]   prefix:bar.js
     "
   `);
 });
diff --git a/packages/kbn-optimizer/src/node/node_auto_tranpilation.ts b/packages/kbn-optimizer/src/node/node_auto_tranpilation.ts
@@ -91,7 +91,7 @@ function determineCachePrefix() {
     tsx: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.tsx')),
   });
 
-  const checksum = Crypto.createHash('sha256').update(json).digest('hex');
+  const checksum = Crypto.createHash('sha256').update(json).digest('hex').slice(0, 8);
   return `${checksum}:`;
 }
 
@@ -134,7 +134,8 @@ export function registerNodeAutoTranspilation() {
   installed = true;
 
   const cache = new Cache({
-    dir: Path.resolve(REPO_ROOT, 'data/node_auto_transpilation_cache', UPSTREAM_BRANCH),
+    pathRoot: REPO_ROOT,
+    dir: Path.resolve(REPO_ROOT, 'data/node_auto_transpilation_cache_v3', UPSTREAM_BRANCH),
     prefix: determineCachePrefix(),
     log: process.env.DEBUG_NODE_TRANSPILER_CACHE
       ? Fs.createWriteStream(Path.resolve(REPO_ROOT, 'node_auto_transpilation_cache.log'), {