test(node): enable native node tests (#695)

README.md

@@ -83,6 +83,9 @@ Examples of bad title:
 
 Ensure there is a related issue and it is referenced in the PR text.
 
+For contributions to the Node compatibility library please check the
+[`std/node` contributing guide](./node/README.md)
+
 _For maintainers_:
 
 To release a new version a tag in the form of `x.y.z` should be added.

node/README.md

@@ -85,6 +85,21 @@ const leftPad = require("left-pad");
 
 ## Contributing
 
+### Setting up the test runner
+
+This library contains automated tests pulled directly from the Node repo in
+order ensure compatibility.
+
+Setting up the test runner is as simple as running the `node/_tools/setup.ts`
+file, this will pull the configured tests in and then add them to the test
+workflow.
+
+To enable new tests, simply add a new entry inside `node/_tools/config.json`
+under the `tests` property. The structure this entries must have has to resemble
+a path inside `https://github.com/nodejs/node/tree/master/test`.
+
+### Best practices
+
 When converting from promise-based to callback-based APIs, the most obvious way
 is like this:
 

node/_tools/.gitignore

@@ -0,0 +1 @@
+versions

node/_tools/common.ts

@@ -0,0 +1,45 @@
+import { join } from "../../path/mod.ts";
+
+/**
+ * The test suite matches the folders inside the `test` folder inside the
+ * node repo
+ * 
+ * Each test suite contains a list of files (which can be paths
+ * or a regex to match) that will be pulled from the node repo
+ */
+type TestSuites = Record<string, string[]>;
+
+interface Config {
+  nodeVersion: string;
+  /** Ignored files won't regenerated by the update script */
+  ignore: TestSuites;
+  /**
+   * The files that will be run by the test suite
+   * 
+   * The files to be generated with the update script must be listed here as well,
+   * but they won't be regenerated if they are listed in the `ignore` configuration
+   * */
+  tests: TestSuites;
+  suitesFolder: string;
+  versionsFolder: string;
+}
+
+export const config: Config = JSON.parse(
+  await Deno.readTextFile(new URL("./config.json", import.meta.url)),
+);
+
+export const ignoreList = Object.entries(config.ignore).reduce(
+  (total: RegExp[], [suite, paths]) => {
+    paths.forEach((path) => total.push(new RegExp(join(suite, path))));
+    return total;
+  },
+  [],
+);
+
+export const testList = Object.entries(config.tests).reduce(
+  (total: RegExp[], [suite, paths]) => {
+    paths.forEach((path) => total.push(new RegExp(join(suite, path))));
+    return total;
+  },
+  [],
+);

node/_tools/config.json

@@ -0,0 +1,15 @@
+{
+  "nodeVersion": "15.5.1",
+  "ignore": {
+    "common": [
+      "index.js"
+    ]
+  },
+  "tests": {
+    "parallel": [
+      "test-event-emitter-listener-count.js"
+    ]
+  },
+  "suitesFolder": "suites",
+  "versionsFolder": "versions"
+}

node/_tools/require.ts

@@ -0,0 +1,23 @@
+import { createRequire } from "../module.ts";
+import { isAbsolute } from "../../path/mod.ts";
+
+/**
+ * This module is used as an entry point for each test file
+ * 
+ * The idea is to emulate a CommonJS environment without having to modify
+ * the test files in any way
+ * 
+ * Running with all permissions and unstable is recommended
+ * 
+ * Usage: `deno run -A --unstable require.ts my_commonjs_file.js`
+ */
+
+const file = Deno.args[0];
+if (!file) {
+  throw new Error("No file provided");
+} else if (!isAbsolute(file)) {
+  throw new Error("Path for file must be absolute");
+}
+
+const require = createRequire(file);
+require(file);

node/_tools/setup.ts

@@ -0,0 +1,179 @@
+import { gunzip } from "https://deno.land/x/[email protected]/gzip/gzip.ts";
+import { Untar } from "../../archive/tar.ts";
+import { walk } from "../../fs/walk.ts";
+import { basename, fromFileUrl, join, resolve } from "../../path/mod.ts";
+import { ensureFile } from "../../fs/ensure_file.ts";
+import { config, ignoreList } from "./common.ts";
+
+/**
+ * This script will download and extract the test files specified in the
+ * configuration file
+ * 
+ * It will delete any previous tests unless they are specified on the `ignore`
+ * section of the configuration file
+ * 
+ * Usage: `deno run --allow-read --allow-net --allow-write setup.ts`
+ */
+
+const NODE_URL = "https://nodejs.org/dist/vNODE_VERSION";
+const NODE_FILE = "node-vNODE_VERSION.tar.gz";
+
+/** URL for the download */
+const url = `${NODE_URL}/${NODE_FILE}`.replaceAll(
+  "NODE_VERSION",
+  config.nodeVersion,
+);
+/** Local url location */
+const path = join(
+  config.versionsFolder,
+  NODE_FILE.replaceAll("NODE_VERSION", config.nodeVersion),
+);
+
+/**
+ * This will overwrite the file if found
+ * */
+async function downloadFile(url: string, path: string) {
+  console.log(`Downloading: ${url}...`);
+  const fileContent = await fetch(url)
+    .then((response) => {
+      if (response.ok) {
+        if (!response.body) {
+          throw new Error(
+            `The requested download url ${url} doesn't contain an archive to download`,
+          );
+        }
+        return response.body.getIterator();
+      } else if (response.status === 404) {
+        throw new Error(
+          `The requested version ${config.nodeVersion} could not be found for download`,
+        );
+      }
+      throw new Error(`Request failed with status ${response.status}`);
+    });
+
+  const filePath = fromFileUrl(new URL(path, import.meta.url));
+
+  await ensureFile(filePath);
+  const file = await Deno.open(filePath, {
+    truncate: true,
+    write: true,
+  });
+  for await (const chunk of fileContent) {
+    await Deno.write(file.rid, chunk);
+  }
+  file.close();
+  console.log(`Downloaded: ${url} into ${path}`);
+}
+
+async function clearTests() {
+  console.log("Cleaning up previous tests");
+
+  const files = walk(
+    (fromFileUrl(new URL(config.suitesFolder, import.meta.url))),
+    {
+      includeDirs: false,
+      skip: ignoreList,
+    },
+  );
+
+  for await (const file of files) {
+    await Deno.remove(file.path);
+  }
+}
+
+/**
+ * This will iterate over the ignore and test lists defined in the
+ * configuration file
+ * 
+ * If it were to be found in the ignore list or not found in the test list, the
+ * function will return undefined, meaning the file won't be regenerated
+ */
+function getRequestedFileSuite(file: string): string | undefined {
+  for (const regex of ignoreList) {
+    if (regex.test(file)) {
+      return;
+    }
+  }
+
+  for (const suite in config.tests) {
+    for (const regex of config.tests[suite]) {
+      if (new RegExp(regex).test(file)) {
+        return suite;
+      }
+    }
+  }
+}
+
+async function decompressTests(filePath: string) {
+  console.log(`Processing ${basename(filePath)}...`);
+  const compressedFile = await Deno.open(
+    new URL(filePath, import.meta.url),
+    { read: true },
+  );
+
+  const buffer = new Deno.Buffer(gunzip(await Deno.readAll(compressedFile)));
+  Deno.close(compressedFile.rid);
+
+  const tar = new Untar(buffer);
+
+  for await (const entry of tar) {
+    if (entry.type !== "file") continue;
+    const suite = getRequestedFileSuite(entry.fileName);
+    if (!suite) continue;
+    const path = resolve(
+      fromFileUrl(new URL(config.suitesFolder, import.meta.url)),
+      suite,
+      basename(entry.fileName),
+    );
+    await ensureFile(path);
+    const file = await Deno.open(path, {
+      create: true,
+      truncate: true,
+      write: true,
+    });
+    // This will allow CI to pass without checking linting and formatting
+    // on the test suite files, removing the need to mantain that as well
+    await Deno.writeAll(
+      file,
+      new TextEncoder().encode(
+        "// deno-fmt-ignore-file\n// deno-lint-ignore-file\n" +
+          "\n// Copyright Joyent and Node contributors. All rights reserved. MIT license.\n" +
+          `// Taken from Node ${config.nodeVersion}\n` +
+          '// This file is automatically generated by "node/_tools/setup.ts". Do not modify this file manually\n\n',
+      ),
+    );
+    await Deno.copy(entry, file);
+    Deno.close(file.rid);
+  }
+}
+
+let shouldDownload = false;
+try {
+  Deno.lstatSync(new URL(path, import.meta.url)).isFile;
+  while (true) {
+    const r = (prompt(`File "${path}" found, use file? Y/N:`) ?? "").trim()
+      .toUpperCase();
+    if (r === "Y") {
+      break;
+    } else if (r === "N") {
+      shouldDownload = true;
+      break;
+    } else {
+      console.log(`Unexpected: "${r}"`);
+    }
+  }
+} catch (e) {
+  if (!(e instanceof Deno.errors.NotFound)) {
+    throw e;
+  }
+  shouldDownload = true;
+}
+
+if (shouldDownload) {
+  console.log(`Downloading ${url} in path "${path}" ...`);
+  await downloadFile(url, path);
+}
+
+await clearTests();
+
+await decompressTests(path);