Implement support for WebAssembly threads

... and add a parallel raytracing demo!

This commit adds enough support to `wasm-bindgen` to produce a workable
wasm binary *today* with the experimental WebAssembly threads support
implemented in Firefox Nightly. I've tried to comment what's going on in
the commits and such, but at a high level the changes made here are:

* A new transformation, living in a new `wasm-bindgen-threads-xform`
  crate, prepares a wasm module for parallel execution. This performs a
  number of mundane tasks which I hope to detail in a blog post later on.
* The `--no-modules` output is enhanced with more support for when
  shared memory is enabled, allowing passing in the module/memory to
  initialize the wasm instance on multiple threads (sharing both module
  and memory).
* The `wasm-bindgen` crate now offers the ability, in `--no-modules`
  mode, to get a handle on the `WebAssembly.Module` instance.
* The example itself requires Xargo to recompile the standard library
  with atomics and an experimental feature enabled. Afterwards it
  experimentally also enables threading support in wasm-bindgen.

I've also added hopefully enough CI support to compile this example in a
builder so we can upload it and poke around live online. I hope to
detail more about the technical details here in a blog post soon as
well!

.travis.yml

@@ -80,14 +80,14 @@ matrix:
 
     # All examples work
     - rust: nightly
-      env: JOB=examples-build
+      name: "examples - almost all examples"
       install:
         - *INSTALL_NODE_VIA_NVM
         - *INSTALL_AWS
         - npm install
       script:
         - |
-          for dir in `ls examples | grep -v README | grep -v asm.js | grep -v no_modules`; do
+          for dir in `ls examples | grep -v README | grep -v asm.js | grep -v raytrace | grep -v no_modules`; do
             (cd examples/$dir &&
              sed -i "s|: \"webpack-dev-server\"|: \"webpack --output-path $HOME/$TRAVIS_BUILD_NUMBER/exbuild/$dir\"|" package.json &&
              sed -i 's/npm install//' build.sh &&
@@ -96,6 +96,21 @@ matrix:
           done
         - if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then aws s3 sync --quiet ~/$TRAVIS_BUILD_NUMBER s3://wasm-bindgen-ci/$TRAVIS_BUILD_NUMBER; fi
       if: branch = master
+    - rust: nightly
+      name: "examples - raytracer"
+      install:
+        - *INSTALL_AWS
+        - rustup component add rust-src
+        - curl -L https://github.com/japaric/xargo/releases/download/v0.3.11/xargo-v0.3.11-x86_64-unknown-linux-musl.tar.gz | tar xzf -
+        - export PATH=$PATH:`pwd`
+      script:
+        - sed -i 's/python/#python/' examples/raytrace-parallel/build.sh
+        - (cd examples/raytrace-parallel && ./build.sh)
+        - dst=$HOME/$TRAVIS_BUILD_NUMBER/exbuild/raytrace-parallel
+        - mkdir -p $dst
+        - cp examples/raytrace-parallel/*.{js,html,wasm} $dst
+        - if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then aws s3 sync ~/$TRAVIS_BUILD_NUMBER s3://wasm-bindgen-ci/$TRAVIS_BUILD_NUMBER; fi
+      if: branch = master
 
     # The `web-sys` crate's tests pass
     - rust: beta

Cargo.toml

@@ -72,6 +72,7 @@ members = [
   "examples/no_modules",
   "examples/paint",
   "examples/performance",
+  "examples/raytrace-parallel",
   "examples/wasm-in-wasm",
   "examples/wasm2js",
   "examples/webaudio",

crates/cli-support/Cargo.toml

@@ -15,6 +15,7 @@ base64 = "0.9"
 failure = "0.1.2"
 parity-wasm = "0.35"
 tempfile = "3.0"
+wasm-bindgen-gc = { path = '../gc', version = '=0.2.25' }
 wasm-bindgen-shared = { path = "../shared", version = '=0.2.25' }
+wasm-bindgen-threads-xform = { path = '../threads-xform', version = '=0.2.25' }
 wasm-bindgen-wasm-interpreter = { path = "../wasm-interpreter", version = '=0.2.25' }
-wasm-bindgen-gc = { path = '../gc', version = '=0.2.25' }

crates/cli-support/src/js/mod.rs

@@ -6,7 +6,7 @@ use decode;
 use failure::{Error, ResultExt};
 use parity_wasm::elements::*;
 use shared;
-use wasm_bindgen_gc;
+use gc;
 
 use super::Bindgen;
 use descriptor::{Descriptor, VectorKind};
@@ -388,12 +388,26 @@ impl<'a> Context<'a> {
             ))
         })?;
 
+        self.bind("__wbindgen_module", &|me| {
+            if !me.config.no_modules {
+                bail!("`wasm_bindgen::module` is currently only supported with \
+                       --no-modules");
+            }
+            me.expose_add_heap_object();
+            Ok(format!(
+                "
+                function() {{
+                    return addHeapObject(init.__wbindgen_wasm_module);
+                }}
+                ",
+            ))
+        })?;
+
         self.bind("__wbindgen_rethrow", &|me| {
             me.expose_take_object();
             Ok(String::from("function(idx) { throw takeObject(idx); }"))
         })?;
 
-        self.create_memory_export();
         self.unexport_unused_internal_exports();
         closures::rewrite(self)?;
         self.gc();
@@ -415,7 +429,76 @@ impl<'a> Context<'a> {
 
         self.rewrite_imports(module_name);
 
-        let mut js = if self.config.no_modules {
+        let mut js = if self.config.threads.is_some() {
+            // TODO: It's not clear right now how to best use threads with
+            // bundlers like webpack. We need a way to get the existing
+            // module/memory into web workers for now and we don't quite know
+            // idiomatically how to do that! In the meantime, always require
+            // `--no-modules`
+            if !self.config.no_modules {
+                bail!("most use `--no-modules` with threads for now")
+            }
+            self.memory(); // set `memory_limit` if it's not already set
+            let limits = match &self.memory_init {
+                Some(l) if l.shared() => l.clone(),
+                _ => bail!("must impot a shared memory with threads"),
+            };
+
+            let mut memory = String::from("new WebAssembly.Memory({");
+            memory.push_str(&format!("initial:{}", limits.initial()));
+            if let Some(max) = limits.maximum() {
+                memory.push_str(&format!(",maximum:{}", max));
+            }
+            if limits.shared() {
+                memory.push_str(",shared:true");
+            }
+            memory.push_str("})");
+
+            format!(
+                    "\
+(function() {{
+    var wasm;
+    var memory;
+    const __exports = {{}};
+    {globals}
+    function init(module_or_path, maybe_memory) {{
+        let result;
+        const imports = {{ './{module}': __exports }};
+        if (module_or_path instanceof WebAssembly.Module) {{
+            memory = __exports.memory = maybe_memory;
+            result = WebAssembly.instantiate(module_or_path, imports)
+                .then(instance => {{
+                    return {{ instance, module: module_or_path }}
+                }});
+        }} else {{
+            memory = __exports.memory = {init_memory};
+            const response = fetch(module_or_path);
+            if (typeof WebAssembly.instantiateStreaming === 'function') {{
+                result = WebAssembly.instantiateStreaming(response, imports);
+            }} else {{
+                result = response
+                    .then(r => r.arrayBuffer())
+                    .then(bytes => WebAssembly.instantiate(bytes, imports));
+            }}
+        }}
+        return result.then(({{instance, module}}) => {{
+            wasm = init.wasm = instance.exports;
+            init.__wbindgen_wasm_instance = instance;
+            init.__wbindgen_wasm_module = module;
+            init.__wbindgen_wasm_memory = __exports.memory;
+        }});
+    }};
+    self.{global_name} = Object.assign(init, __exports);
+}})();",
+                    globals = self.globals,
+                    module = module_name,
+                    global_name = self.config.no_modules_global
+                        .as_ref()
+                        .map(|s| &**s)
+                        .unwrap_or("wasm_bindgen"),
+                    init_memory = memory,
+            )
+        } else if self.config.no_modules {
             format!(
                     "\
 (function() {{
@@ -637,20 +720,6 @@ impl<'a> Context<'a> {
         }
     }
 
-    fn create_memory_export(&mut self) {
-        let limits = match self.memory_init.clone() {
-            Some(limits) => limits,
-            None => return,
-        };
-        let mut initializer = String::from("new WebAssembly.Memory({");
-        initializer.push_str(&format!("initial:{}", limits.initial()));
-        if let Some(max) = limits.maximum() {
-            initializer.push_str(&format!(",maximum:{}", max));
-        }
-        initializer.push_str("})");
-        self.export("memory", &initializer, None);
-    }
-
     fn rewrite_imports(&mut self, module_name: &str) {
         for (name, contents) in self._rewrite_imports(module_name) {
             self.export(&name, &contents, None);
@@ -1621,7 +1690,7 @@ impl<'a> Context<'a> {
 
     fn gc(&mut self) {
         self.parse_wasm_names();
-        wasm_bindgen_gc::Config::new()
+        gc::Config::new()
             .demangle(self.config.demangle)
             .keep_debug(self.config.keep_debug || self.config.debug)
             .run(&mut self.module);
@@ -1671,7 +1740,6 @@ impl<'a> Context<'a> {
                 _ => None,
             }).next()
             .expect("must import memory");
-        assert_eq!(entry.module(), "env");
         assert_eq!(entry.field(), "memory");
         self.memory_init = Some(mem.limits().clone());
         "memory"

crates/cli-support/src/lib.rs

@@ -3,10 +3,11 @@
 extern crate parity_wasm;
 #[macro_use]
 extern crate wasm_bindgen_shared as shared;
-extern crate wasm_bindgen_gc;
+extern crate wasm_bindgen_gc as gc;
 #[macro_use]
 extern crate failure;
 extern crate wasm_bindgen_wasm_interpreter as wasm_interpreter;
+extern crate wasm_bindgen_threads_xform as threads_xform;
 
 use std::collections::BTreeSet;
 use std::env;
@@ -37,6 +38,9 @@ pub struct Bindgen {
     // Experimental support for `WeakRefGroup`, an upcoming ECMAScript feature.
     // Currently only enable-able through an env var.
     weak_refs: bool,
+    // Experimental support for the wasm threads proposal, transforms the wasm
+    // module to be "ready to be instantiated on any thread"
+    threads: Option<threads_xform::Config>,
 }
 
 enum Input {
@@ -59,6 +63,7 @@ impl Bindgen {
             demangle: true,
             keep_debug: false,
             weak_refs: env::var("WASM_BINDGEN_WEAKREF").is_ok(),
+            threads: threads_config(),
         }
     }
 
@@ -143,6 +148,11 @@ impl Bindgen {
         let programs = extract_programs(&mut module, &mut program_storage)
             .with_context(|_| "failed to extract wasm-bindgen custom sections")?;
 
+        if let Some(cfg) = &self.threads {
+            cfg.run(&mut module)
+                .with_context(|_| "failed to prepare module for threading")?;
+        }
+
         // Here we're actually instantiating the module we've parsed above for
         // execution. Why, you might be asking, are we executing wasm code? A
         // good question!
@@ -447,3 +457,20 @@ fn reset_indentation(s: &str) -> String {
     }
     return dst;
 }
+
+// Eventually these will all be CLI options, but while they're unstable features
+// they're left as environment variables. We don't guarantee anything about
+// backwards-compatibility with these options.
+fn threads_config() -> Option<threads_xform::Config> {
+    if env::var("WASM_BINDGEN_THREADS").is_err() {
+        return None
+    }
+    let mut cfg = threads_xform::Config::new();
+    if let Ok(s) = env::var("WASM_BINDGEN_THREADS_MAX_MEMORY") {
+        cfg.maximum_memory(s.parse().unwrap());
+    }
+    if let Ok(s) = env::var("WASM_BINDGEN_THREADS_STACK_SIZE") {
+        cfg.thread_stack_size(s.parse().unwrap());
+    }
+    Some(cfg)
+}

crates/threads-xform/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "wasm-bindgen-threads-xform"
+version = "0.2.25"
+authors = ["The wasm-bindgen Developers"]
+license = "MIT/Apache-2.0"
+repository = "https://github.com/rustwasm/wasm-bindgen/tree/master/crates/threads-xform"
+homepage = "https://rustwasm.github.io/wasm-bindgen/"
+documentation = "https://docs.rs/wasm-bindgen-threads-xform"
+description = """
+Support for threading-related transformations in wasm-bindgen
+"""
+
+[dependencies]
+parity-wasm = "0.35"
+failure = "0.1"