Add filesystem interface for the File System Access API

ChangeLog.md

@@ -20,6 +20,8 @@ See docs/process.md for more on how version tagging works.
 
 3.1.10
 ------
+- A new file system interface using the browser File System Access API is
+  available: `-lfsfs.js`
 
 3.1.9 - 04/21/2022
 ------------------

site/source/docs/api_reference/Filesystem-API.rst

@@ -57,7 +57,7 @@ However, due to JavaScript's event-driven nature, most *persistent* storage opti
 File systems
 ============
 
-.. note:: Only the :ref:`MEMFS <filesystem-api-memfs>` filesystem is included by default. All others must be enabled explicitly, using ``-lnodefs.js`` (:ref:`NODEFS <filesystem-api-nodefs>`), ``-lidbfs.js`` (:ref:`IDBFS <filesystem-api-idbfs>`), ``-lworkerfs.js`` (:ref:`WORKERFS <filesystem-api-workerfs>`), or ``-lproxyfs.js`` (:ref:`PROXYFS <filesystem-api-proxyfs>`).
+.. note:: Only the :ref:`MEMFS <filesystem-api-memfs>` filesystem is included by default. All others must be enabled explicitly, using ``-lnodefs.js`` (:ref:`NODEFS <filesystem-api-nodefs>`), ``-lidbfs.js`` (:ref:`IDBFS <filesystem-api-idbfs>`), ``-lfsfs.js`` (:ref:`FSFS <filesystem-api-fsfs>`), ``-lworkerfs.js`` (:ref:`WORKERFS <filesystem-api-workerfs>`), or ``-lproxyfs.js`` (:ref:`PROXYFS <filesystem-api-proxyfs>`).
 
 .. _filesystem-api-memfs:
 
@@ -99,6 +99,20 @@ The *IDBFS* file system implements the :js:func:`FS.syncfs` interface, which whe
 
 This is provided to overcome the limitation that browsers do not offer synchronous APIs for persistent storage, and so (by default) all writes exist only temporarily in-memory.
 
+.. _filesystem-api-fsfs:
+
+FSFS
+-----
+
+.. note:: This file system is only for use when running code inside a browser.
+
+The *FSFS* file system implements the :js:func:`FS.syncfs` interface, which when called will persist any operations to the attached ``FileSystemDirectoryHandle``.
+
+This uses the `File System Access API <https://web.dev/file-system-access/>`_. To use, pass a ``FileSystemDirectoryHandle`` as ``opts.dirHandle``, which can be created via:
+
+-  `navigator.storage.getDirectory()` – this is the Origin Private File System, currently only supported by Safari and Chromium browsers
+-  `self.showDirectoryPicker()` – currently only supported by Chromium browsers
+
 .. _filesystem-api-workerfs:
 
 WORKERFS
@@ -233,7 +247,7 @@ File system API
   Responsible for iterating and synchronizing all mounted file systems in an
   asynchronous fashion.
 
-  .. note:: Currently, only the :ref:`filesystem-api-idbfs` file system implements the
+  .. note:: Currently, only :ref:`filesystem-api-idbfs` and :ref:`filesystem-api-fsfs` file systems implement the
     interfaces needed for synchronization. All other file systems are completely
     synchronous and don't require synchronization.
 

src/library_fs.js

@@ -9,6 +9,9 @@ mergeInto(LibraryManager.library, {
 #if LibraryManager.has('library_idbfs.js')
     '$IDBFS',
 #endif
+#if LibraryManager.has('library_fsfs.js')
+    '$FSFS',
+#endif
 #if LibraryManager.has('library_nodefs.js')
     '$NODEFS',
 #endif
@@ -1464,6 +1467,9 @@ FS.staticInit();` +
 #if LibraryManager.has('library_idbfs.js')
         'IDBFS': IDBFS,
 #endif
+#if LibraryManager.has('library_fsfs.js')
+        'FSFS': FSFS,
+#endif
 #if LibraryManager.has('library_nodefs.js')
         'NODEFS': NODEFS,
 #endif

src/library_fsfs.js

@@ -0,0 +1,225 @@
+/**
+ * @license
+ * Copyright 2022 The Emscripten Authors
+ * SPDX-License-Identifier: MIT
+ */
+
+mergeInto(LibraryManager.library, {
+  $FSFS__deps: ['$FS', '$MEMFS', '$PATH'],
+  $FSFS__postset: function() {
+    return '';
+  },
+  $FSFS: {
+    DIR_MODE: Number("{{{ cDefine('S_IFDIR') }}}") | 511 /* 0777 */,
+    FILE_MODE: Number("{{{ cDefine('S_IFREG') }}}") | 511 /* 0777 */,
+    mount: function(mount) {
+      if (!mount.opts.dirHandle) {
+        throw new Error('opts.dirHandle is required');
+      }
+
+      // reuse all of the core MEMFS functionality
+      return MEMFS.mount.apply(null, arguments);
+    },
+    syncfs: async (mount, populate, callback) => {
+      try {
+        const local = await FSFS.getLocalSet(mount);
+        const remote = await FSFS.getRemoteSet(mount);
+        const src = populate ? remote : local;
+        const dst = populate ? local : remote;
+        await FSFS.reconcile(mount, src, dst);
+        callback(null);
+      } catch (e) {
+        callback(e);
+      }
+    },
+    // Returns file set of emscripten's filesystem at the mountpoint.
+    getLocalSet: (mount) => {
+      var entries = Object.create(null);
+
+      function isRealDir(p) {
+        return p !== '.' && p !== '..';
+      };
+      function toAbsolute(root) {
+        return (p) => {
+          return PATH.join2(root, p);
+        }
+      };
+
+      var check = FS.readdir(mount.mountpoint).filter(isRealDir).map(toAbsolute(mount.mountpoint));
+
+      while (check.length) {
+        var path = check.pop();
+        var stat = FS.stat(path);
+
+        if (FS.isDir(stat.mode)) {
+          check.push.apply(check, FS.readdir(path).filter(isRealDir).map(toAbsolute(path)));
+        }
+
+        entries[path] = { timestamp: stat.mtime, mode: stat.mode };
+      }
+
+      return { type: 'local', entries: entries };
+    },
+    // Returns file set of the real, on-disk filesystem at the mountpoint.
+    getRemoteSet: async (mount) => {
+      const entries = Object.create(null);
+
+      const handles = await FSFS.getFsHandles(mount.opts.dirHandle, true);
+      for (const [path, handle] of handles) {
+        if (path === '.') continue;
+
+        entries[PATH.join2(mount.mountpoint, path)] = {
+          timestamp: handle.kind === 'file' ? (await handle.getFile()).lastModifiedDate : new Date(),
+          mode: handle.kind === 'file' ? FSFS.FILE_MODE : FSFS.DIR_MODE,
+        };
+      }
+
+      return { type: 'remote', entries, handles };
+    },
+    loadLocalEntry: (path) => {
+      const lookup = FS.lookupPath(path);
+      const node = lookup.node;
+      const stat = FS.stat(path);      
+
+      if (FS.isDir(stat.mode)) {
+        return { 'timestamp': stat.mtime, 'mode': stat.mode };
+      } else if (FS.isFile(stat.mode)) {
+        node.contents = MEMFS.getFileDataAsTypedArray(node);
+        return { timestamp: stat.mtime, mode: stat.mode, contents: node.contents };
+      } else {
+        throw new Error('node type not supported');
+      }
+    },
+    storeLocalEntry: (path, entry) => {
+      if (FS.isDir(entry['mode'])) {
+        FS.mkdirTree(path, entry['mode']);
+      } else if (FS.isFile(entry['mode'])) {
+        FS.writeFile(path, entry['contents'], { canOwn: true });
+      } else {
+        throw new Error('node type not supported');
+      }
+
+      FS.chmod(path, entry['mode']);
+      FS.utime(path, entry['timestamp'], entry['timestamp']);
+    },
+    removeLocalEntry: (path) => {
+      var stat = FS.stat(path);
+
+      if (FS.isDir(stat.mode)) {
+        FS.rmdir(path);
+      } else if (FS.isFile(stat.mode)) {
+        FS.unlink(path);
+      }
+    },
+    loadRemoteEntry: async (handle) => {
+      if (handle.kind === 'file') {
+        const file = await handle.getFile();
+        return {
+          contents: new Uint8Array(await file.arrayBuffer()),
+          mode: FSFS.FILE_MODE,
+          timestamp: file.lastModifiedDate,
+        };
+      } else if (handle.kind === 'directory') {
+        return {
+          mode: FSFS.DIR_MODE,
+          timestamp: new Date(),
+        };
+      } else {
+        throw new Error('unknown kind: ' + handle.kind);
+      }
+    },
+    storeRemoteEntry: async (handles, path, entry) => {
+      const parentDirHandle = handles.get(PATH.dirname(path));
+      const handle = FS.isFile(entry.mode) ?
+        await parentDirHandle.getFileHandle(PATH.basename(path), {create: true}) :
+        await parentDirHandle.getDirectoryHandle(PATH.basename(path), {create: true});
+      if (handle.kind === 'file') {
+        const writable = await handle.createWritable();
+        await writable.write(entry.contents);
+        await writable.close();
+      }
+      handles.set(path, handle);
+    },
+    removeRemoteEntry: async (handles, path) => {
+      const parentDirHandle = handles.get(PATH.dirname(path));
+      await parentDirHandle.removeEntry(PATH.basename(path));
+      handles.delete(path);
+    },
+    reconcile: async (mount, src, dst) => {
+      let total = 0;
+
+      const create = [];
+      Object.keys(src.entries).forEach(function (key) {
+        const e = src.entries[key];
+        const e2 = dst.entries[key];
+        if (!e2 || (FS.isFile(e.mode) && e['timestamp'].getTime() > e2['timestamp'].getTime())) {
+          create.push(key);
+          total++;
+        }
+      });
+      // sort paths in ascending order so directory entries are created
+      // before the files inside them
+      create.sort();
+
+      const remove = [];
+      Object.keys(dst.entries).forEach(function (key) {
+        if (!src.entries[key]) {
+          remove.push(key);
+          total++;
+        }
+      });
+      // sort paths in descending order so files are deleted before their
+      // parent directories
+      remove.sort().reverse();
+
+      if (!total) {
+        return;
+      }
+
+      const handles = src.type === 'remote' ? src.handles : dst.handles;
+
+      for (const path of create) {
+        const relPath = PATH.normalize(path.replace(mount.mountpoint, '/')).substring(1);;
+        if (dst.type === 'local') {
+          const handle = handles.get(relPath);
+          const entry = await FSFS.loadRemoteEntry(handle);
+          FSFS.storeLocalEntry(path, entry);
+        } else {
+          const entry = FSFS.loadLocalEntry(path);
+          await FSFS.storeRemoteEntry(handles, relPath, entry);
+        }
+      }
+
+      for (const path of remove) {
+        if (dst.type === 'local') {
+          FSFS.removeLocalEntry(path);
+        } else {
+          const relPath = PATH.normalize(path.replace(mount.mountpoint, '/')).substring(1);
+          await FSFS.removeRemoteEntry(handles, relPath);
+        }
+      }
+    },
+    getFsHandles: async (dirHandle) => {
+      const handles = [];
+
+      async function collect(curDirHandle) {
+        for await (const entry of curDirHandle.values()) {
+          handles.push(entry);
+          if (entry.kind === 'directory') {
+            await collect(entry);
+          }
+        }
+      }
+
+      await collect(dirHandle);
+
+      const result = new Map();
+      result.set('.', dirHandle);
+      for (const handle of handles) {
+        const relativePath = (await dirHandle.resolve(handle)).join('/');
+        result.set(relativePath, handle);
+      }
+      return result;
+    },
+  }
+});

src/shell.js

@@ -467,6 +467,7 @@ assert(typeof Module['TOTAL_MEMORY'] == 'undefined', 'Module.TOTAL_MEMORY has be
 {{{ makeRemovedModuleAPIAssert('readBinary') }}}
 {{{ makeRemovedModuleAPIAssert('setWindowTitle') }}}
 {{{ makeRemovedFSAssert('IDBFS') }}}
+{{{ makeRemovedFSAssert('FSFS') }}}
 {{{ makeRemovedFSAssert('PROXYFS') }}}
 {{{ makeRemovedFSAssert('WORKERFS') }}}
 #if !NODERAWFS