diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 083e5b6..11fd187 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -135,7 +135,6 @@ jobs:
       matrix:
         ember-cli: [
           "~3.4.0",
-          "~3.8.0",
           "~3.12.0",
           "~3.16.0",
           "~3.20.0",
@@ -176,9 +175,11 @@ jobs:
         steps.cache-dependencies.outputs.cache-hit != 'true'
 
     - name: Install Ember CLI
-      run: yarn global add ember-cli@${EMBER_CLI_VERSION}
+      run: |
+        yarn global add ember-cli@${EMBER_CLI_VERSION}
+        echo "$(yarn global bin)" >> $GITHUB_PATH
       env:
         EMBER_CLI_VERSION: ${{ matrix.ember-cli }}
 
     - name: Test
-      run: yarn test:node
+      run: ./node_modules/.bin/mocha node-tests/**/*-test.js
diff --git a/node-tests/e2e/test-support-test.js b/node-tests/e2e/test-support-test.js
index affaebc..b031f8d 100644
--- a/node-tests/e2e/test-support-test.js
+++ b/node-tests/e2e/test-support-test.js
@@ -21,10 +21,17 @@ async function adjustForCompatibility(testProject) {
   // internally unless configured to not do so. This violates the default CSP and causes an
   // `EvalError` to be thrown. This uncatched error will cause the tests to fail - regardless
   // of our custom test support.
-  // To avoid this issue we uninstall Ember Auto Import for these tests. This can be removed
-  // as soon as Ember CLI Content Security Policy works out of the box with Ember Auto Import.
+  // We need to adjust Ember Auto Import's configuration prevent this. It can be removed as
+  // soon as Ember CLI Content Security Policy works out of the box with Ember Auto Import.
   try {
-    await testProject.runCommand('yarn', 'remove', 'ember-auto-import');
+    const emberCliBuildJs = await testProject.readFile('ember-cli-build.js');
+    await testProject.writeFile(
+      'ember-cli-build.js',
+      emberCliBuildJs.replace(
+        '// Add options here',
+        'autoImport: { forbidEval: true }'
+      )
+    );
   } catch (error) {
     // Trying to remove ember-auto-import dependency may fail cause that dependency is not
     // present for older Ember CLI versions.
diff --git a/node-tests/e2e/test-support-warns-on-outdated-qunit-test.js b/node-tests/e2e/test-support-warns-on-outdated-qunit-test.js
index 2cb2dbe..eba5990 100644
--- a/node-tests/e2e/test-support-warns-on-outdated-qunit-test.js
+++ b/node-tests/e2e/test-support-warns-on-outdated-qunit-test.js
@@ -1,6 +1,7 @@
 const expect = require('chai').expect;
 const TestProject = require('ember-addon-tests').default;
 const path = require('path');
+const { readPackageJson } = require('../utils');
 
 describe('e2e: test support warns if dependencies are not supported', function () {
   this.timeout(300000);
@@ -11,9 +12,24 @@ describe('e2e: test support warns if dependencies are not supported', function (
     });
 
     await testProject.createEmberApp();
+
+    // ember-qunit was upgraded to 5.x and qunit added as a direct dependency
+    // in Ember CLI 3.24.0. This test can not be performed for such new Ember
+    // CLI versions.
+    const packageJson = await readPackageJson(testProject);
+    const hasQunitAsDirectDependency = Object.keys(
+      packageJson.devDependencies
+    ).includes('qunit');
+    if (hasQunitAsDirectDependency) {
+      return;
+    }
+
     await testProject.addOwnPackageAsDevDependency(
       'ember-cli-content-security-policy'
     );
+
+    // Ensure that projects uses an old ember-qunit version, which depends on
+    // incompatible qunit version.
     await testProject.addDevDependency('ember-qunit', '4.0.0');
 
     try {