chore: simplified bb.js bundling (#11622)

Attempt to be clever about wasm compression. In production, it's highly likely servers will provide the correct `Content-Encoding` and `Content-type` headers so that the browser can decompress on the fly. However, that's a big assumption in development, and configuring webpack-server/vite/web-test-runner or any other of the 1000s of options that can be used creates friction with consumers. Also, initial attempt at providing an external URL for wasm resolution, so a version with debug symbols (for example) can be hot-swapped without recompilation.
AztecProtocol · Jan 30, 2025 · 8408ede · 8408ede
1 parent a7f8d96
commit 8408ede
Show file tree

Hide file tree

Showing 16 changed files with 70 additions and 64 deletions.
diff --git a/barretenberg/acir_tests/browser-test-app/webpack.config.js b/barretenberg/acir_tests/browser-test-app/webpack.config.js
@@ -12,9 +12,6 @@ export default {
     index: "./src/index.ts",
   },
   module: {
-    parser: {
-      javascript: { importMeta: false },
-    },
     rules: [
       {
         test: /\.gz$/,

diff --git a/barretenberg/acir_tests/browser-test-app/yarn.lock b/barretenberg/acir_tests/browser-test-app/yarn.lock
@@ -7,7 +7,7 @@ __metadata:
 
 "@aztec/bb.js@file:../../ts::locator=browser-test-app%40workspace%3A.":
   version: 0.72.1
-  resolution: "@aztec/bb.js@file:../../ts#../../ts::hash=2e6237&locator=browser-test-app%40workspace%3A."
+  resolution: "@aztec/bb.js@file:../../ts#../../ts::hash=7ab3ef&locator=browser-test-app%40workspace%3A."
   dependencies:
     comlink: "npm:^4.4.1"
     commander: "npm:^12.1.0"
@@ -17,7 +17,7 @@ __metadata:
     tslib: "npm:^2.4.0"
   bin:
     bb.js: ./dest/node/main.js
-  checksum: 10c0/174074ea937d9e4888e1aa600717ace0f0c5745d15bdd23730c136d927c2d58eec87ba83cb21fbfb72f7aa00ec128a564d7fa6d822f709e0327e01aa6475e255
+  checksum: 10c0/463d29bb01102874431d5a83af71a18a92b6b7fd817311474744bfe4af2b82d71efa1796d0a00093bcc87b163922ff748cb6a8480d7c295c726a855d1bdf321d
   languageName: node
   linkType: hard
 

diff --git a/barretenberg/ts/src/barretenberg/index.ts b/barretenberg/ts/src/barretenberg/index.ts
@@ -22,6 +22,9 @@ export type BackendOptions = {
 
   /** @description Path to download CRS files */
   crsPath?: string;
+
+  /** @description Path to download WASM files */
+  wasmPath?: string;
 };
 
 export type CircuitOptions = {
@@ -50,7 +53,7 @@ export class Barretenberg extends BarretenbergApi {
   static async new(options: BackendOptions = {}) {
     const worker = createMainWorker();
     const wasm = getRemoteBarretenbergWasm<BarretenbergWasmMainWorker>(worker);
-    const { module, threads } = await fetchModuleAndThreads(options.threads);
+    const { module, threads } = await fetchModuleAndThreads(options.threads, options.wasmPath);
     await wasm.init(module, threads, proxy(debug), options.memory?.initial, options.memory?.maximum);
     return new Barretenberg(worker, wasm, options);
   }
@@ -97,16 +100,16 @@ export class BarretenbergSync extends BarretenbergApiSync {
     super(wasm);
   }
 
-  private static async new() {
+  private static async new(wasmPath?: string) {
     const wasm = new BarretenbergWasmMain();
-    const { module, threads } = await fetchModuleAndThreads(1);
+    const { module, threads } = await fetchModuleAndThreads(1, wasmPath);
     await wasm.init(module, threads);
     return new BarretenbergSync(wasm);
   }
 
-  static async initSingleton() {
+  static async initSingleton(wasmPath?: string) {
     if (!barrentenbergSyncSingletonPromise) {
-      barrentenbergSyncSingletonPromise = BarretenbergSync.new();
+      barrentenbergSyncSingletonPromise = BarretenbergSync.new(wasmPath);
     }
 
     barretenbergSyncSingleton = await barrentenbergSyncSingletonPromise;

diff --git a/barretenberg/ts/src/barretenberg_wasm/fetch_code/browser/index.ts b/barretenberg/ts/src/barretenberg_wasm/fetch_code/browser/index.ts
@@ -1,9 +1,25 @@
 import barretenbergModule from '../../barretenberg.wasm.gz';
 import barretenbergThreadsModule from '../../barretenberg-threads.wasm.gz';
+import pako from 'pako';
 
 // Annoyingly the wasm declares if it's memory is shared or not. So now we need two wasms if we want to be
 // able to fallback on "non shared memory" situations.
-export async function fetchCode(multithreaded: boolean) {
-  const res = await fetch(multithreaded ? barretenbergThreadsModule : barretenbergModule);
-  return res.arrayBuffer();
+export async function fetchCode(multithreaded: boolean, wasmPath?: string) {
+  let url = multithreaded ? barretenbergThreadsModule : barretenbergModule;
+  url = wasmPath ? `${wasmPath}/${/[^/]+(?=\/$|$)/.exec(url)?.[0]}` : url;
+  const res = await fetch(url);
+  const maybeCompressedData = await res.arrayBuffer();
+  const buffer = new Uint8Array(maybeCompressedData);
+  const isGzip =
+    // Check magic number
+    buffer[0] === 0x1f &&
+    buffer[1] === 0x8b &&
+    // Check compression method:
+    buffer[2] === 0x08;
+  if (isGzip) {
+    const decompressedData = pako.ungzip(buffer);
+    return decompressedData.buffer;
+  } else {
+    return buffer;
+  }
 }
diff --git a/barretenberg/ts/src/barretenberg_wasm/fetch_code/node/index.ts b/barretenberg/ts/src/barretenberg_wasm/fetch_code/node/index.ts
@@ -14,7 +14,7 @@ function getCurrentDir() {
 }
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-export async function fetchCode(multithreaded: boolean) {
+export async function fetchCode(multithreaded: boolean, wasmPath?: string) {
   const path = getCurrentDir() + '/../../barretenberg-threads.wasm.gz';
   const compressedData = await readFile(path);
   const decompressedData = pako.ungzip(new Uint8Array(compressedData));

diff --git a/barretenberg/ts/src/barretenberg_wasm/index.ts b/barretenberg/ts/src/barretenberg_wasm/index.ts
@@ -7,14 +7,14 @@ import { fetchCode } from './fetch_code/index.js';
 
 const debug = createDebug('bb.js:wasm');
 
-export async function fetchModuleAndThreads(desiredThreads = 32) {
+export async function fetchModuleAndThreads(desiredThreads = 32, wasmPath?: string) {
   const shared = getSharedMemoryAvailable();
 
   const availableThreads = shared ? await getAvailableThreads() : 1;
   // We limit the number of threads to 32 as we do not benefit from greater numbers.
   const limitedThreads = Math.min(desiredThreads, availableThreads, 32);
 
-  const code = await fetchCode(shared);
+  const code = await fetchCode(shared, wasmPath);
   const module = await WebAssembly.compile(code);
   return { module, threads: limitedThreads };
 }
@@ -38,10 +38,10 @@ export class BarretenbergWasm extends BarretenbergWasmMain {
    * Construct and initialize BarretenbergWasm within a Worker. Return both the worker and the wasm proxy.
    * Used when running in the browser, because we can't block the main thread.
    */
-  public static async new(desiredThreads?: number) {
+  public static async new(desiredThreads?: number, wasmPath?: string) {
     const worker = createMainWorker();
     const wasm = getRemoteBarretenbergWasm<BarretenbergWasmMainWorker>(worker);
-    const { module, threads } = await fetchModuleAndThreads(desiredThreads);
+    const { module, threads } = await fetchModuleAndThreads(desiredThreads, wasmPath);
     await wasm.init(module, threads, proxy(debug));
     return { worker, wasm };
   }

diff --git a/barretenberg/ts/webpack.config.js b/barretenberg/ts/webpack.config.js
@@ -17,7 +17,19 @@ export default {
     rules: [
       {
         test: /\.wasm\.gz$/,
-        type: 'asset/resource'
+        type: 'asset/resource',
+        generator: {
+          // The wasm filenames are actually the same, but we symlink them to the correct one
+          // (threads or not) on the .ts folder. Unfortunately webpack uses the original name,
+          // so we have to manually correct it here.
+          filename: (path) => {
+            if(path.filename.includes('wasm-threads')) {
+              return 'barretenberg-threads.wasm.gz';
+            }
+            return '[base]';
+          },
+          publicPath: '/'
+        }
       },
       {
         test: /\.worker\.ts$/,
@@ -62,6 +74,11 @@ export default {
     fallback: {
       os: false,
     },
+    alias: {
+      // All node specific code, wherever it's located, should be imported as below.
+      // Provides a clean and simple way to always strip out the node code for the web build.
+      './node/index.js': false,
+    }
   },
   devServer: {
     hot: false,

diff --git a/boxes/boxes/react/webpack.config.js b/boxes/boxes/react/webpack.config.js
@@ -12,9 +12,6 @@ export default (_, argv) => ({
     main: './src/index.tsx',
   },
   module: {
-    parser: {
-      javascript: { importMeta: false },
-    },
     rules: [
       {
         test: /\.gz$/,
@@ -73,11 +70,5 @@ export default (_, argv) => ({
     port: 5173,
     open: true,
     historyApiFallback: true,
-    headers: (req, res) => {
-      if (req.originalUrl.endsWith('.gz')) {
-        res.setHeader('Content-Encoding', 'gzip');
-        res.setHeader('Content-Type', 'application/wasm');
-      }
-    },
   },
 });
diff --git a/boxes/boxes/vanilla/webpack.config.js b/boxes/boxes/vanilla/webpack.config.js
@@ -12,9 +12,6 @@ export default (_, argv) => ({
     main: './src/index.ts',
   },
   module: {
-    parser: {
-      javascript: { importMeta: false },
-    },
     rules: [
       {
         test: /\.gz$/,
@@ -69,11 +66,5 @@ export default (_, argv) => ({
     port: 5173,
     open: true,
     historyApiFallback: true,
-    headers: (req, res) => {
-      if (req.originalUrl.endsWith('.gz')) {
-        res.setHeader('Content-Encoding', 'gzip');
-        res.setHeader('Content-Type', 'application/wasm');
-      }
-    },
   },
 });
diff --git a/boxes/boxes/vite/vite.config.ts b/boxes/boxes/vite/vite.config.ts
@@ -3,6 +3,8 @@ import react from "@vitejs/plugin-react-swc";
 import { PolyfillOptions, nodePolyfills } from "vite-plugin-node-polyfills";
 import { viteStaticCopy } from "vite-plugin-static-copy";
 
+const nodeModulesPath = `${searchForWorkspaceRoot(process.cwd())}/node_modules`;
+
 // Unfortunate, but needed due to https://github.com/davidmyersdev/vite-plugin-node-polyfills/issues/81
 // Suspected to be because of the yarn workspace setup, but not sure
 const nodePolyfillsFix = (options?: PolyfillOptions | undefined): Plugin => {
@@ -15,15 +17,15 @@ const nodePolyfillsFix = (options?: PolyfillOptions | undefined): Plugin => {
           source,
         );
       if (m) {
-        return `../../node_modules/vite-plugin-node-polyfills/shims/${m[1]}/dist/index.cjs`;
+        return `${nodeModulesPath}/vite-plugin-node-polyfills/shims/${m[1]}/dist/index.cjs`;
       }
     },
   };
 };
 
 // https://vite.dev/config/
 export default defineConfig({
-  logLevel: 'error',
+  logLevel: "error",
   server: {
     // Headers needed for bb WASM to work in multithreaded mode
     headers: {
@@ -32,13 +34,13 @@ export default defineConfig({
     },
     // Allow vite to serve files from these directories, since they are symlinked
     // These are the protocol circuit artifacts and noir/bb WASMs.
+    // ONLY REQUIRED TO RUN FROM THE MONOREPO
     fs: {
       allow: [
         searchForWorkspaceRoot(process.cwd()),
         "../../../yarn-project/noir-protocol-circuits-types/artifacts",
         "../../../noir/packages/noirc_abi/web",
         "../../../noir/packages/acvm_js/web",
-        "../../../barretenberg/ts/dest/browser",
       ],
     },
   },
@@ -48,8 +50,8 @@ export default defineConfig({
     viteStaticCopy({
       targets: [
         {
-          src: "../../../barretenberg/ts/dest/browser/*.gz",
-          dest: "assets/",
+          src: `${nodeModulesPath}/@aztec/aztec.js/dest/*.wasm.gz`,
+          dest: "./",
         },
       ],
     }),

diff --git a/gaztec/vite.config.ts b/gaztec/vite.config.ts
@@ -37,7 +37,6 @@ export default defineConfig({
         "../yarn-project/noir-protocol-circuits-types/artifacts",
         "../noir/packages/noirc_abi/web",
         "../noir/packages/acvm_js/web",
-        "../barretenberg/ts/dest/browser",
       ],
     },
   },
@@ -47,8 +46,8 @@ export default defineConfig({
     viteStaticCopy({
       targets: [
         {
-          src: "../barretenberg/ts/dest/browser/*.gz",
-          dest: "assets/",
+          src: "../barretenberg/ts/dest/browser/*.wasm.gz",
+          dest: "./",
         },
       ],
     }),

diff --git a/noir/noir-repo/compiler/integration-tests/web-test-runner.config.mjs b/noir/noir-repo/compiler/integration-tests/web-test-runner.config.mjs
@@ -26,10 +26,9 @@ export default {
     // playwrightLauncher({ product: "webkit" }),
     // playwrightLauncher({ product: "firefox" }),
   ],
-  middleware: [async function setGzHeader(ctx, next) {
-    if (ctx.url.endsWith('.gz')) {
-      ctx.set('Content-Encoding', 'gzip');
-      ctx.res.removeHeader('Content-Length');
+  middleware: [async (ctx, next) => {
+    if (ctx.url.endsWith('.wasm.gz')) {
+      ctx.url = ctx.url.replace('/', "/node_modules/@aztec/bb.js/dest/browser/");
     }
     await next();
   }],

diff --git a/yarn-project/aztec.js/webpack.config.js b/yarn-project/aztec.js/webpack.config.js
@@ -15,13 +15,14 @@ export default {
     main: './src/index.ts',
   },
   module: {
-    parser: {
-      javascript: { importMeta: false },
-    },
     rules: [
       {
-        test: /\.gz$/,
+        test: /\.wasm\.gz$/,
         type: 'asset/resource',
+        generator: {
+          filename: '[base]',
+          publicPath: '/',
+        },
       },
       {
         test: /\.tsx?$/,

diff --git a/yarn-project/end-to-end/src/composed/e2e_aztec_js_browser.test.ts b/yarn-project/end-to-end/src/composed/e2e_aztec_js_browser.test.ts
@@ -49,13 +49,6 @@ const setupApp = async () => {
   }
 
   const app = new Koa();
-  app.use(async (ctx, next) => {
-    if (ctx.url.endsWith('.gz')) {
-      ctx.set('Content-Encoding', 'gzip');
-      ctx.res.removeHeader('Content-Length');
-    }
-    await next();
-  });
   app.use(serve(path.resolve(__dirname, '../web')));
   const server = app.listen(PORT, () => {
     logger.info(`Web Server started at http://localhost:${PORT}`);

diff --git a/yarn-project/end-to-end/webpack.config.js b/yarn-project/end-to-end/webpack.config.js
@@ -15,9 +15,6 @@ export default {
     main: './src/web/main.ts',
   },
   module: {
-    parser: {
-      javascript: { importMeta: false },
-    },
     rules: [
       {
         test: /\.gz$/,

diff --git a/yarn-project/package.json b/yarn-project/package.json
@@ -12,7 +12,7 @@
     "format": "yarn prettier --cache -w .",
     "test": "RAYON_NUM_THREADS=4 FORCE_COLOR=true yarn workspaces foreach -A --exclude @aztec/aztec3-packages --exclude @aztec/prover-client -p -v run test --config=../jest.root.config.js",
     "build": "FORCE_COLOR=true yarn workspaces foreach -A --parallel --topological-dev --verbose --exclude @aztec/aztec3-packages --exclude @aztec/docs run build",
-    "build:fast": "cd foundation && yarn build && cd ../l1-artifacts && yarn build && cd ../circuits.js && yarn build && cd .. && yarn generate && tsc -b",
+    "build:fast": "cd foundation && yarn build && cd ../l1-artifacts && yarn build && cd ../circuits.js && yarn build && cd ../aztec.js && yarn build && cd .. && yarn generate && tsc -b",
     "build:dev": "./watch.sh",
     "generate": "FORCE_COLOR=true yarn workspaces foreach -A --parallel --topological-dev --verbose run generate",
     "clean": "yarn workspaces foreach -A -p -v run clean"