[WasmFS] Async proxied JS backend

src/library.js

@@ -3648,7 +3648,6 @@ LibraryManager.library = {
     return x.indexOf('dynCall_') == 0 || unmangledSymbols.includes(x) ? x : '_' + x;
   },
 
-
   $asyncLoad__docs: '/** @param {boolean=} noRunDep */',
   $asyncLoad: function(url, onload, onerror, noRunDep) {
     var dep = !noRunDep ? getUniqueRunDependency('al ' + url) : '';

src/library_wasmfs.js

@@ -1,13 +1,9 @@
-var WasmfsLibrary = {
-  $wasmFS$JSMemoryFiles : [],
-  $wasmFS$JSMemoryFreeList: [],
+var WasmFSLibrary = {
   $wasmFS$preloadedFiles: [],
   $wasmFS$preloadedDirs: [],
   $FS__deps: [
     '$wasmFS$preloadedFiles',
     '$wasmFS$preloadedDirs',
-    '$wasmFS$JSMemoryFiles',
-    '$wasmFS$JSMemoryFreeList',
     '$asyncLoad',
 #if !MINIMAL_RUNTIME
     // TODO: when preload-plugins are not used, we do not need this.
@@ -145,64 +141,121 @@ var WasmfsLibrary = {
     var len = lengthBytesUTF8(s) + 1;
     stringToUTF8(s, fileNameBuffer, len);
   },
-  _wasmfs_write_js_file: function(index, buffer, length, offset) {
-    try {
-      if (!wasmFS$JSMemoryFiles[index]) {
-        // Initialize typed array on first write operation.
-        wasmFS$JSMemoryFiles[index] = new Uint8Array(offset + length);
-      }
-
-      if (offset + length > wasmFS$JSMemoryFiles[index].length) {
-        // Resize the typed array if the length of the write buffer exceeds its capacity.
-        var oldContents = wasmFS$JSMemoryFiles[index];
-        var newContents = new Uint8Array(offset + length);
-        newContents.set(oldContents);
-        wasmFS$JSMemoryFiles[index] = newContents;
-      }
-
-      wasmFS$JSMemoryFiles[index].set(HEAPU8.subarray(buffer, buffer + length), offset);
-      return 0;
-    } catch (err) {
-      return {{{ cDefine('EIO') }}};
-    }
-  },
-  _wasmfs_read_js_file: function(index, buffer, length, offset) {
-    try {
-      HEAPU8.set(wasmFS$JSMemoryFiles[index].subarray(offset, offset + length), buffer);
-      return 0;
-    } catch (err) {
-      return {{{ cDefine('EIO') }}};
-    }
-  },
-  _wasmfs_get_js_file_size: function(index) {
-    return wasmFS$JSMemoryFiles[index] ? wasmFS$JSMemoryFiles[index].length : 0;
-  },
-  _wasmfs_create_js_file: function() {
-    // Find a free entry in the $wasmFS$JSMemoryFreeList or append a new entry to
-    // wasmFS$JSMemoryFiles.
-    if (wasmFS$JSMemoryFreeList.length) {
-      // Pop off the top of the free list.
-      var index = wasmFS$JSMemoryFreeList.pop();
-      return index;
-    }
-    wasmFS$JSMemoryFiles.push(null);
-    return wasmFS$JSMemoryFiles.length - 1;
-  },
-  _wasmfs_remove_js_file: function(index) {
-    wasmFS$JSMemoryFiles[index] = null;
-    // Add the index to the free list.
-    wasmFS$JSMemoryFreeList.push(index);
-  },
   _wasmfs_get_preloaded_file_size: function(index) {
     return wasmFS$preloadedFiles[index].fileData.length;
   },
   _wasmfs_copy_preloaded_file_data: function(index, buffer) {
     HEAPU8.set(wasmFS$preloadedFiles[index].fileData, buffer);
   },
-}
 
-mergeInto(LibraryManager.library, WasmfsLibrary);
+  // Backend support. wasmFS$backends will contain a mapping of backend IDs to
+  // the JS code that implements them. This is the JS side of the JSImpl* class
+  // in C++, together with the js_impl calls defined right after it.
+  $wasmFS$backends: {},
+
+  // JSImpl
+
+  _wasmfs_jsimpl_alloc_file: function(backend, file) {
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    return wasmFS$backends[backend].allocFile(file);
+  },
+
+  _wasmfs_jsimpl_free_file: function(backend, file) {
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    return wasmFS$backends[backend].freeFile(file);
+  },
+
+  _wasmfs_jsimpl_write: function(backend, file, buffer, length, {{{ defineI64Param('offset') }}}) {
+    {{{ receiveI64ParamAsDouble('offset') }}}
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    return wasmFS$backends[backend].write(file, buffer, length, offset);
+  },
+
+  _wasmfs_jsimpl_read: function(backend, file, buffer, length, {{{ defineI64Param('offset') }}}) {
+    {{{ receiveI64ParamAsDouble('offset') }}}
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    return wasmFS$backends[backend].read(file, buffer, length, offset);
+  },
 
-if (WASMFS) {
-  DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.push('$FS');
+  _wasmfs_jsimpl_get_size: function(backend, file) {
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    return wasmFS$backends[backend].getSize(file);
+  },
+
+  // ProxiedAsyncJSImpl. Each function receives a function pointer and a
+  // parameter. We convert those into a convenient Promise API for the
+  // implementors of backends: the hooks we call should return Promises, which
+  // we then connect to the calling C++.
+
+  _wasmfs_jsimpl_async_alloc_file: async function(backend, file, fptr, arg) {
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    {{{ runtimeKeepalivePush() }}}
+    await wasmFS$backends[backend].allocFile(file);
+    {{{ runtimeKeepalivePop() }}}
+    {{{ makeDynCall('vi', 'fptr') }}}(arg);
+  },
+
+  _wasmfs_jsimpl_async_free_file: async function(backend, file, fptr, arg) {
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    {{{ runtimeKeepalivePush() }}}
+    await wasmFS$backends[backend].freeFile(file);
+    {{{ runtimeKeepalivePop() }}}
+    {{{ makeDynCall('vi', 'fptr') }}}(arg);
+  },
+
+  _wasmfs_jsimpl_async_write: async function(backend, file, buffer, length, {{{ defineI64Param('offset') }}}, fptr, arg) {
+    {{{ receiveI64ParamAsDouble('offset') }}}
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    {{{ runtimeKeepalivePush() }}}
+    var size = await wasmFS$backends[backend].write(file, buffer, length, offset);
+    {{{ runtimeKeepalivePop() }}}
+    {{{ makeSetValue('arg', C_STRUCTS.CallbackState.result, '0', 'i32') }}};
+    {{{ makeSetValue('arg', C_STRUCTS.CallbackState.offset, 'size', 'i64') }}};
+    {{{ makeDynCall('vi', 'fptr') }}}(arg);
+  },
+
+  _wasmfs_jsimpl_async_read: async function(backend, file, buffer, length, {{{ defineI64Param('offset') }}}, fptr, arg) {
+    {{{ receiveI64ParamAsDouble('offset') }}}
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    {{{ runtimeKeepalivePush() }}}
+    var size = await wasmFS$backends[backend].read(file, buffer, length, offset);
+    {{{ runtimeKeepalivePop() }}}
+    {{{ makeSetValue('arg', C_STRUCTS.CallbackState.result, '0', 'i32') }}};
+    {{{ makeSetValue('arg', C_STRUCTS.CallbackState.offset, 'size', 'i64') }}};
+    {{{ makeDynCall('vi', 'fptr') }}}(arg);
+  },
+
+  _wasmfs_jsimpl_async_get_size: async function(backend, file, fptr, arg) {
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    {{{ runtimeKeepalivePush() }}}
+    var size = await wasmFS$backends[backend].getSize(file);
+    {{{ runtimeKeepalivePop() }}}
+    {{{ makeSetValue('arg', C_STRUCTS.CallbackState.result, '0', 'i32') }}};
+    {{{ makeSetValue('arg', C_STRUCTS.CallbackState.offset, 'size', 'i64') }}};
+    {{{ makeDynCall('vi', 'fptr') }}}(arg);
+  },
 }
+
+mergeInto(LibraryManager.library, WasmFSLibrary);
+
+DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.push('$FS');

src/library_wasmfs_fetch.js

@@ -0,0 +1,63 @@
+mergeInto(LibraryManager.library, {
+  // Fetch backend: On first access of the file (either a read or a getSize), it
+  // will fetch() the data from the network asynchronously. Otherwise, after
+  // that fetch it behaves just like JSFile (and it reuses the code from there).
+
+  _wasmfs_create_fetch_backend_js__deps: [
+    '$wasmFS$backends',
+    '$wasmFS$JSMemoryFiles',
+    '_wasmfs_create_js_file_backend_js',
+  ],
+  _wasmfs_create_fetch_backend_js: async function(backend) {
+    // Get a promise that fetches the data and stores it in JS memory (if it has
+    // not already been fetched).
+    async function getFile(file) {
+      if (wasmFS$JSMemoryFiles[file]) {
+        // The data is already here, so nothing to do before we continue on to
+        // the actual read below.
+        return Promise.resolve();
+      }
+
+      // This is the first time we want the file's data.
+      // TODO: real URL!
+      var url = 'data.dat';
+      var response = await fetch(url);
+      var buffer = await response['arrayBuffer']();
+      wasmFS$JSMemoryFiles[file] = new Uint8Array(buffer);
+    }
+
+    // Start with the normal JSFile operations. This sets
+    //   wasmFS$backends[backend]
+    // which we will then augment.
+    __wasmfs_create_js_file_backend_js(backend);
+
+    // Add the async operations on top.
+    var jsFileOps = wasmFS$backends[backend];
+    wasmFS$backends[backend] = {
+      // alloc/free operations are not actually async. Just forward to the
+      // parent class, but we must return a Promise as the caller expects.
+      allocFile: async function(file) {
+        jsFileOps.allocFile(file);
+        return Promise.resolve();
+      },
+      freeFile: async function(file) {
+        jsFileOps.freeFile(file);
+        return Promise.resolve();
+      },
+
+      write: async function(file, buffer, length, offset) {
+        abort("TODO: file writing in fetch backend? read-only for now");
+      },
+
+      // read/getSize fetch the data, then forward to the parent class.
+      read: async function(file, buffer, length, offset) {
+        await getFile(file);
+        return jsFileOps.read(file, buffer, length, offset);
+      },
+      getSize: async function(file) {
+        await getFile(file);
+        return jsFileOps.getSize(file);
+      },
+    };
+  },
+});

src/library_wasmfs_js_file.js

@@ -0,0 +1,53 @@
+mergeInto(LibraryManager.library, {
+  // JSFile backend: Store a file's data in JS. We map File objects in C++ to
+  // entries here that contain typed arrays.
+  $wasmFS$JSMemoryFiles: {},
+
+  _wasmfs_create_js_file_backend_js__deps: [
+    '$wasmFS$backends',
+    '$wasmFS$JSMemoryFiles',
+  ],
+  _wasmfs_create_js_file_backend_js: function(backend) {
+    wasmFS$backends[backend] = {
+      allocFile: function(file) {
+        // Do nothing: we allocate the typed array lazily, see write()
+      },
+      freeFile: function(file) {
+        // Release the memory, as it now has no references to it any more.
+        wasmFS$JSMemoryFiles[file] = undefined;
+      },
+      write: function(file, buffer, length, offset) {
+        try {
+          if (!wasmFS$JSMemoryFiles[file]) {
+            // Initialize typed array on first write operation.
+            wasmFS$JSMemoryFiles[file] = new Uint8Array(offset + length);
+          }
+
+          if (offset + length > wasmFS$JSMemoryFiles[file].length) {
+            // Resize the typed array if the length of the write buffer exceeds its capacity.
+            var oldContents = wasmFS$JSMemoryFiles[file];
+            var newContents = new Uint8Array(offset + length);
+            newContents.set(oldContents);
+            wasmFS$JSMemoryFiles[file] = newContents;
+          }
+
+          wasmFS$JSMemoryFiles[file].set(HEAPU8.subarray(buffer, buffer + length), offset);
+          return 0;
+        } catch (err) {
+          return {{{ cDefine('EIO') }}};
+        }
+      },
+      read: function(file, buffer, length, offset) {
+        try {
+          HEAPU8.set(wasmFS$JSMemoryFiles[file].subarray(offset, offset + length), buffer);
+          return 0;
+        } catch (err) {
+          return {{{ cDefine('EIO') }}};
+        }
+      },
+      getSize: function(file) {
+        return wasmFS$JSMemoryFiles[file] ? wasmFS$JSMemoryFiles[file].length : 0;
+      },
+    };
+  },
+});

src/modules.js

@@ -95,6 +95,8 @@ global.LibraryManager = {
       }
     } else if (WASMFS) {
       libraries.push('library_wasmfs.js');
+      libraries.push('library_wasmfs_js_file.js');
+      libraries.push('library_wasmfs_fetch.js');
     }
 
     // Additional JS libraries (without AUTO_JS_LIBRARIES, link to these explicitly via -lxxx.js)

src/parseTools.js

@@ -1073,6 +1073,19 @@ function receiveI64ParamAsI32s(name) {
   return '';
 }
 
+// TODO: use this in library_wasi.js and other places. but we need to add an
+//       error-handling hook here.
+function receiveI64ParamAsDouble(name) {
+  if (WASM_BIGINT) {
+    // Just convert the bigint into a double.
+    return `${name} = Number(${name});`;
+  }
+
+  // Combine the i32 params. Use an unsigned operator on low and shift high by
+  // 32 bits.
+  return `${name} = ${name}_high * 0x100000000 + (${name}_low >>> 0);`;
+}
+
 function sendI64Argument(low, high) {
   if (WASM_BIGINT) {
     return 'BigInt(low) | (BigInt(high) << BigInt(32))';

src/struct_info_internal.json

@@ -47,5 +47,14 @@
               "name"
             ]
         }
+    },
+    {
+        "file": "async_callback.h",
+        "structs": {
+            "CallbackState": [
+                "result",
+                "offset"
+            ]
+        }
     }
 ]

system/include/emscripten/wasmfs.h

@@ -40,6 +40,8 @@ typedef backend_t (*backend_constructor_t)(void*);
 backend_t wasmfs_create_proxied_backend(backend_constructor_t create_backend,
                                         void* arg);
 
+backend_t wasmfs_create_fetch_backend(char* base_url);
+
 #ifdef __cplusplus
 }
 #endif

system/lib/wasmfs/async_callback.h

@@ -0,0 +1,27 @@
+// Copyright 2022 The Emscripten Authors.  All rights reserved.
+// Emscripten is available under two separate licenses, the MIT license and the
+// University of Illinois/NCSA Open Source License.  Both these licenses can be
+// found in the LICENSE file.
+
+// This file defines the JS file backend and JS file of the new file system.
+// Current Status: Work in Progress.
+// See https://github.com/emscripten-core/emscripten/issues/15041.
+
+#pragma once
+
+#include "sys/types.h"
+#include "wasi/api.h"
+
+// Callbacks for the async API between C and JS. This is declared in a small
+// separate header for convenience of gen_struct_info.
+
+// Callbacks take a pointer to a CallbackState structure, which contains both
+// the function to call to resume execution, and storage for any out params.
+// Basically this stores the state during an async call.
+struct CallbackState {
+  // The result of the operation, either success or an error code.
+  __wasi_errno_t result;
+
+  // Some syscalls return an offset.
+  off_t offset;
+};