diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 24955a75589e9..6ff9695365d7b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -39,9 +39,6 @@ jobs:
         if: matrix.deno == 'canary'
         run: deno upgrade --canary
 
-      - name: Generate Node tests
-        run: deno run --allow-all node/testdata/setup.ts
-
       - name: Test
         run: deno test --unstable --allow-all
 
diff --git a/node/testdata/test.ts b/node/testdata/test.ts
index 16e7dcfcde183..000b43ddaf341 100644
--- a/node/testdata/test.ts
+++ b/node/testdata/test.ts
@@ -1,6 +1,6 @@
 import { walk } from "../../fs/walk.ts";
 import { basename, dirname, fromFileUrl } from "../../path/mod.ts";
-import { assertEquals } from "../../testing/asserts.ts";
+import { assertEquals, fail } from "../../testing/asserts.ts";
 import { config, testList } from "./common.ts";
 
 /**
@@ -10,6 +10,23 @@ import { config, testList } from "./common.ts";
  * code for the test is reported, the test suite will fail inmediately
  */
 
+const process = Deno.run({
+  cwd: dirname(fromFileUrl(import.meta.url)),
+  cmd: [
+    Deno.execPath(),
+    "run",
+    "--allow-read",
+    "--allow-net",
+    "--allow-write",
+    "./setup.ts",
+  ],
+});
+
+const { code } = await process.status();
+if (code !== 0) {
+  fail(`Node tests generation process failed with code: "${code}"`);
+}
+
 const dir = walk(fromFileUrl(new URL(config.suitesFolder, import.meta.url)), {
   includeDirs: false,
   match: testList,