test: prototype faster test runner

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         os: [macOS-latest, windows-2019, ubuntu-16.04]
-        kind: ['test', 'test_debug', 'test_std', 'bench', 'lint']
+        kind: ['test', 'test_debug', 'bench', 'lint']
         exclude:
           - os: windows-2019
             kind: 'bench'
@@ -118,18 +118,9 @@ jobs:
         run: cargo clippy --all-targets --release --locked -- -D clippy::all
 
       - name: Build
-        if: matrix.kind == 'test' || matrix.kind == 'bench' || matrix.kind == 'test_std'
+        if: matrix.kind == 'test' || matrix.kind == 'bench'
         run: cargo build --release --locked --all-targets
 
-      # TODO(ry) Remove this step, and move the following test to
-      # cli/tests/std_tests.rs
-      # TODO(ry) Remove the "cd std".
-      - name: std test
-        if: matrix.kind == 'test_std'
-        run: |
-          cd std
-          ../target/release/deno test -A
-
       - name: Test
         if: matrix.kind == 'test'
         run: cargo test --release --locked --all-targets

cli/tests/std_tests.rs

@@ -1,25 +1,32 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-// TODO(ry) Current std tests are run in .github/workflows/build.yml but ideally
-// they would be called as part of "cargo test". "deno test" is too slow to do
-// this desierable thing: https://github.com/denoland/deno/issues/3088
-/*
-#[macro_use]
-extern crate lazy_static;
-extern crate tempfile;
-mod util;
-use util::*;
 
-#[test]
-fn std_tests() {
-  let mut deno = deno_cmd()
-    .current_dir(root_path())
-    .arg("test")
-    .arg("-A")
-    .arg("std")
-    .spawn()
-    .expect("failed to spawn script");
-  let status = deno.wait().expect("failed to wait for the child process");
-  assert_eq!(Some(0), status.code());
-  assert!(status.success());
+// TODO: fix tests in debug mode
+// Runs only on release build
+#[cfg(not(debug_assertions))]
+mod tests {
+  extern crate lazy_static;
+  extern crate tempfile;
+  use deno_cli::test_util::*;
+  use std::process::Command;
+  use tempfile::TempDir;
+
+  #[test]
+  fn std_tests() {
+    let dir = TempDir::new().expect("tempdir fail");
+    let mut deno_cmd = Command::new(deno_exe_path());
+    deno_cmd.env("DENO_DIR", dir.path());
+
+    let mut cwd = root_path();
+    cwd.push("std");
+    let mut deno = deno_cmd
+      .current_dir(cwd) // note: std tests expect to run from "std" dir
+      .arg("-A")
+      // .arg("-Ldebug")
+      .arg("./testing/runner.ts")
+      .arg("--exclude=testing/testdata")
+      .spawn()
+      .expect("failed to spawn script");
+    let status = deno.wait().expect("failed to wait for the child process");
+    assert!(status.success());
+  }
 }
-*/

std/testing/runner.ts

@@ -116,6 +116,28 @@ export interface RunTestModulesOptions extends RunTestsOptions {
   allowNone?: boolean;
 }
 
+/**
+ * Renders test file that will be run.
+ *
+ * It's done to optimize compilation of test files, because
+ * dynamically importing them one by one takes very long time.
+ * @TODO(bartlomieju): try to optimize compilation by reusing same compiler host
+ * multiple times
+ * @param testModules
+ */
+function renderTestFile(testModules: string[]): string {
+  let testFile = "";
+
+  for (const testModule of testModules) {
+    // NOTE: this is intentional that template string is not used
+    // because of TS compiler quirkness of trying to import it
+    // rather than treating it like a variable
+    testFile += 'import "' + testModule + '"\n';
+  }
+
+  return testFile;
+}
+
 /**
  * Import the specified test modules and run their tests as a suite.
  *
@@ -153,8 +175,9 @@ export async function runTestModules({
   disableLog = false
 }: RunTestModulesOptions = {}): Promise<void> {
   let moduleCount = 0;
+  const testModules = [];
   for await (const testModule of findTestModules(include, exclude)) {
-    await import(testModule);
+    testModules.push(testModule);
     moduleCount++;
   }
 
@@ -168,6 +191,44 @@ export async function runTestModules({
     return;
   }
 
+  // Create temporary test file which contains
+  // all matched modules as import statements.
+  const testFile = renderTestFile(testModules);
+  // Select where temporary test file will be stored.
+  // If `DENO_DIR` is set it means that user intentionally wants to store
+  // modules there - so it's a sane default to store there.
+  // Fallback is current directory which again seems like a sane default,
+  // user is probably working on project in this directory or even
+  // cd'ed into current directory to quickly run test from this directory.
+  const root = Deno.env("DENO_DIR") || Deno.cwd();
+  const testFilePath = join(root, ".deno.test.ts");
+  const data = new TextEncoder().encode(testFile);
+  await Deno.writeFile(testFilePath, data);
+
+  // Import temporary test file and delete it immediately after importing so it's not cluttering disk.
+  //
+  // You may think that this will cause recompilation on each run, but this actually
+  // tricks Deno to not recompile files if there's no need.
+  // Eg.
+  //   1. On first run of $DENO_DIR/.deno.test.ts Deno will compile and cache temporary test file and all of its imports
+  //   2. Temporary test file is removed by test runner
+  //   3. On next test run file is created again. If no new modules were added then temporary file contents are identical.
+  //      Deno will not compile temporary test file again, but load it directly into V8.
+  //   4. Deno starts loading imports one by one.
+  //   5. If imported file is outdated, Deno will recompile this single file.
+  let err;
+  try {
+    await import(`file://${testFilePath}`);
+  } catch (e) {
+    err = e;
+  } finally {
+    await Deno.remove(testFilePath);
+  }
+
+  if (err) {
+    throw err;
+  }
+
   if (!disableLog) {
     console.log(`Found ${moduleCount} matching test modules.`);
   }