chore(tests/specs): ability to have sub tests in file

tests/Cargo.toml

@@ -43,7 +43,7 @@ deno_lockfile.workspace = true
 deno_terminal.workspace = true
 deno_tls.workspace = true
 fastwebsockets = { workspace = true, features = ["upgrade", "unstable-split"] }
-file_test_runner = "0.4.1"
+file_test_runner = "0.5.0"
 flaky_test = "=0.1.0"
 http.workspace = true
 http-body-util.workspace = true

tests/specs/README.md

@@ -28,7 +28,7 @@ cargo test test_name
 
 ## `__test__.json` file
 
-This file describes the test to execute and the steps to execute. A basic
+This file describes the test(s) to execute and the steps to execute. A basic
 example looks like:
 
 ```json
@@ -57,6 +57,23 @@ Or another example that runs multiple steps:
 }
 ```
 
+Or if you want to run several tests at the same time:
+
+```json
+{
+  "tests": {
+    "ignore_dir": {
+      "args": "run script.ts",
+      "output": "script.out"
+    },
+    "some_other_test": {
+      "args": "run other.ts",
+      "output": "other.out"
+    }
+  }
+}
+```
+
 ### Top level properties
 
 - `base` - The base config to use for the test. Options:
@@ -71,13 +88,12 @@ Or another example that runs multiple steps:
 ### Step properties
 
 When writing a single step, these may be at the top level rather than nested in
-a "steps" array.
+a "steps" array or "tests" object.
 
 - `args` - A string (that will be spilt on whitespace into an args array) or an
   array of arguments.
-- `output` - Path to use to assert the output.
-- `cleanDenoDir` (boolean) - Whether to empty the deno_dir before running the
-  step.
+- `output` - Path to use to assert the output or text (must end with an .out
+  extension) _or_ text to pattern match against the output.
 - `flaky` - Step should be repeated until success a maximum of 3 times.
 - `if` (`"windows"`, `"linux"`, `"mac"`, `"unix"`) - Whether to run this step.
 - `exitCode` (number) - Expected exit code.

tests/specs/bench/collect/__test__.jsonc

@@ -1,19 +1,21 @@
 {
-  "steps": [{
-    "args": "bench --ignore=collect/ignore collect",
-    "output": "collect.out"
-  }, {
-    "cleanDenoDir": true,
-    "args": "bench --config collect/deno.jsonc collect",
-    "output": "collect.out"
-  }, {
-    "cleanDenoDir": true,
-    "args": "bench --config collect/deno2.jsonc collect",
-    "output": "collect2.out"
-  }, {
-    "cleanDenoDir": true,
-    "args": "bench --config collect/deno.malformed.jsonc",
-    "exitCode": 1,
-    "output": "collect_with_malformed_config.out"
-  }]
+  "tests": {
+    "ignore": {
+      "args": "bench --ignore=collect/ignore collect",
+      "output": "collect.out"
+    },
+    "config_sub_dir": {
+      "args": "bench --config collect/deno.jsonc collect",
+      "output": "collect.out"
+    },
+    "config_sub_dir_with_exclude": {
+      "args": "bench --config collect/deno2.jsonc collect",
+      "output": "collect2.out"
+    },
+    "config_malformed": {
+      "args": "bench --config collect/deno.malformed.jsonc",
+      "exitCode": 1,
+      "output": "collect_with_malformed_config.out"
+    }
+  }
 }

tests/specs/import_map/import_map_config/__test__.jsonc

@@ -1,28 +1,34 @@
 {
-  "steps": [{
-    "args": "run --quiet --reload --import-map=import_map.json test.ts",
-    "output": "run.out"
-  }, {
-    "args": "run --quiet --reload --import-map=import_map_invalid.json --config=config.json test.ts",
-    "output": "flag_has_precedence.out",
-    "exitCode": 1
-  }, {
-    "args": "run --reload --config=config.json test.ts",
-    "output": "config.out"
-  }, {
-    "cleanDenoDir": true,
-    "args": "cache --quiet --reload --import-map=import_map.json test.ts",
-    "output": "cache.out"
-  }, {
-    "cleanDenoDir": true,
-    "args": "info --quiet --import-map=import_map.json test.ts",
-    "output": "info.out"
-  }, {
-    "args": "run --quiet --reload --import-map=import_map.json unmapped_bare_specifier.ts",
-    "output": "unmapped_bare_specifier.out",
-    "exitCode": 1
-  }, {
-    "args": "run --quiet --reload --import-map import_map.json import_data_url.ts",
-    "output": "import_data_url.out"
-  }]
+  "tests": {
+    "run_import_map": {
+      "args": "run --quiet --reload --import-map=import_map.json test.ts",
+      "output": "run.out"
+    },
+    "un_invalid_import_map": {
+      "args": "run --quiet --reload --import-map=import_map_invalid.json --config=config.json test.ts",
+      "output": "flag_has_precedence.out",
+      "exitCode": 1
+    },
+    "run_config": {
+      "args": "run --reload --config=config.json test.ts",
+      "output": "config.out"
+    },
+    "cache": {
+      "args": "cache --quiet --reload --import-map=import_map.json test.ts",
+      "output": "cache.out"
+    },
+    "info": {
+      "args": "info --quiet --import-map=import_map.json test.ts",
+      "output": "info.out"
+    },
+    "unmapped_bare_specifier": {
+      "args": "run --quiet --reload --import-map=import_map.json unmapped_bare_specifier.ts",
+      "output": "unmapped_bare_specifier.out",
+      "exitCode": 1
+    },
+    "data_url": {
+      "args": "run --quiet --reload --import-map import_map.json import_data_url.ts",
+      "output": "import_data_url.out"
+    }
+  }
 }

tests/specs/jsr/deps/__test__.jsonc

@@ -1,10 +1,12 @@
 {
-  "steps": [{
-    "args": "run main.ts",
-    "output": "main.out"
-  }, {
-    "cleanDenoDir": true,
-    "args": "info main.ts",
-    "output": "main_info.out"
-  }]
+  "tests": {
+    "run": {
+      "args": "run main.ts",
+      "output": "main.out"
+    },
+    "info": {
+      "args": "info main.ts",
+      "output": "main_info.out"
+    }
+  }
 }

tests/specs/jsr/import_https_url/__test__.jsonc

@@ -1,12 +1,14 @@
 {
-  "steps": [{
-    "args": "run -A analyzable.ts",
-    "output": "analyzable.out",
-    "exitCode": 1
-  }, {
-    "cleanDenoDir": true,
-    "args": "run -A nonanalyzable.ts",
-    "output": "nonanalyzable.out",
-    "exitCode": 1
-  }]
+  "tests": {
+    "analyzable": {
+      "args": "run -A analyzable.ts",
+      "output": "analyzable.out",
+      "exitCode": 1
+    },
+    "non_analyzable": {
+      "args": "run -A nonanalyzable.ts",
+      "output": "nonanalyzable.out",
+      "exitCode": 1
+    }
+  }
 }

tests/specs/jsr/module_graph/__test__.jsonc

@@ -1,10 +1,12 @@
 {
-  "steps": [{
-    "args": "run --log-level=debug main.ts",
-    "output": "main.out"
-  }, {
-    "cleanDenoDir": true,
-    "args": "info main.ts",
-    "output": "main_info.out"
-  }]
+  "tests": {
+    "run": {
+      "args": "run --log-level=debug main.ts",
+      "output": "main.out"
+    },
+    "info": {
+      "args": "info main.ts",
+      "output": "main_info.out"
+    }
+  }
 }

tests/specs/jsr/module_graph2/__test__.jsonc

@@ -1,10 +1,12 @@
 {
-  "steps": [{
-    "args": "run --log-level=debug main.ts",
-    "output": "main.out"
-  }, {
-    "cleanDenoDir": true,
-    "args": "info main.ts",
-    "output": "main_info.out"
-  }]
+  "tests": {
+    "run": {
+      "args": "run --log-level=debug main.ts",
+      "output": "main.out"
+    },
+    "info": {
+      "args": "info main.ts",
+      "output": "main_info.out"
+    }
+  }
 }

tests/specs/jsr/no_module_graph/__test__.jsonc

@@ -1,13 +1,16 @@
 {
-  "steps": [{
-    "args": "run main.ts",
-    "output": "main.out"
-  }, {
-    "cleanDenoDir": true,
-    "args": "info main.ts",
-    "output": "main_info.out"
-  }, {
-    "args": "run --quiet multiple.ts",
-    "output": "multiple.out"
-  }]
+  "tests": {
+    "single": {
+      "args": "run main.ts",
+      "output": "main.out"
+    },
+    "single_info": {
+      "args": "info main.ts",
+      "output": "main_info.out"
+    },
+    "multiple": {
+      "args": "run --quiet multiple.ts",
+      "output": "multiple.out"
+    }
+  }
 }