updated npm perf test script

microsoft · May 13, 2022 · ccbcf72 · ccbcf72
1 parent 66538b6
commit ccbcf72
Show file tree

Hide file tree

Showing 7 changed files with 27 additions and 106 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -152,6 +152,7 @@
                 "CI_PYTHON_PATH": "<Python Path>",
                 "VSC_JUPYTER_PERF_TEST": "1",
                 "TEST_FILES_SUFFIX": "notebookCellExecution.perf.test",
+                "VSC_JUPYTER_TEST_TIMEOUT": "60000",
             },
             "sourceMaps": true,
             "outFiles": ["${workspaceFolder}/out/**/*.js", "!${workspaceFolder}/**/node_modules**/*"],

diff --git a/package.json b/package.json
@@ -2114,7 +2114,6 @@
         "testNativeNotebooksAndWebviews": "cross-env CODE_TESTS_WORKSPACE=src/test/datascience VSC_JUPYTER_CI_TEST_VSC_CHANNEL=insiders TEST_FILES_SUFFIX=vscode.test* VSC_JUPYTER_FORCE_LOGGING=1 VSC_JUPYTER_CI_TEST_GREP=webview-test VSC_JUPYTER_LOAD_EXPERIMENTS_FROM_FILE=true node ./out/test/testBootstrap.node.js ./out/test/standardTest.node.js",
         "testWebExtension": "node ./build/launchWebTest.js",
         "testPerformance": "node ./out/test/testBootstrap.node.js ./out/test/perfTest.node.js",
-        "testPerformance2": "cross-env CODE_TESTS_WORKSPACE=src/test/datascience VSC_JUPYTER_CI_TEST_VSC_CHANNEL=insiders TEST_FILES_SUFFIX=notebookCellExecution.perf.test VSC_JUPYTER_FORCE_LOGGING=1 VSC_JUPYTER_LOAD_EXPERIMENTS_FROM_FILE=true node ./out/test/testBootstrap.node.js ./out/test/standardTest.node.js",
         "testSmoke": "node ./out/test/testBootstrap.node.js ./out/test/smokeTest.node.js",
         "testSmokeLogged": "cross-env VSC_JUPYTER_FORCE_LOGGING=true VSC_JUPYTER_LOG_FILE=smoke-test.log node --no-force-async-hooks-checks ./out/test/testBootstrap.node.js ./out/test/smokeTest.node.js",
         "lint": "eslint -c .eslintrc.js --ext .ts src",

diff --git a/src/test/index.node.ts b/src/test/index.node.ts
@@ -74,13 +74,14 @@ function configure(): SetupOptions {
     // So the solution is to run them separately and first on CI.
     const grep = IS_CI_SERVER_TEST_DEBUGGER ? 'Debug' : defaultGrep;
     const testFilesSuffix = process.env.TEST_FILES_SUFFIX || '.test*';
+    const testTimeout = process.env.VSC_JUPYTER_TEST_TIMEOUT || TEST_TIMEOUT;
 
     const options: SetupOptions & { retries: number; invert: boolean } = {
         ui: 'tdd',
         color: true,
         rootHooks: rootHooks,
         invert,
-        timeout: TEST_TIMEOUT,
+        timeout: testTimeout,
         retries: IS_CI_SERVER ? TEST_RETRYCOUNT : 0,
         grep,
         testFilesSuffix,
@@ -185,11 +186,10 @@ export async function run(): Promise<void> {
         );
     });
 
-    console.log(`found ${testFiles.length} test files with suffix ${options.testFilesSuffix} and ignoreing ${ignoreGlob.join(', ')}`);
-
     // Setup test files that need to be run.
     testFiles.forEach((file) => mocha.addFile(path.join(testsRoot, file)));
 
+    // for performance tests, extension activation is part of the test run
     if (!IS_PERF_TEST) {
         /* eslint-disable no-console */
         console.time('Time taken to activate the extension');

diff --git a/src/test/perfTest.node.ts b/src/test/perfTest.node.ts
@@ -7,22 +7,38 @@
 process.env.VSC_JUPYTER_SMOKE_TEST = '1';
 
 import { spawn } from 'child_process';
+import glob from 'glob';
 import * as path from '../platform/vscode-path/path';
 import { EXTENSION_ROOT_DIR_FOR_TESTS } from './constants.node';
 
 class TestRunner {
     public async start() {
         console.log('Start Test Runner');
-        await this.launchTest();
+        const testFiles = await new Promise<string[]>((resolve, reject) => {
+            glob(`**/*perf.test.js`, (error, files) => {
+                if (error) {
+                    return reject(error);
+                }
+                resolve(files);
+            });
+        });
+
+        // warm up with a basic notebook operation before running the tests
+        await this.launchTest('notebookCellExecution.perf.test', true);
+
+        testFiles.forEach((file) => this.launchTest(file));
     }
 
-    private async launchTest() {
+    private async launchTest(testFile: string, warmupRun?: boolean) {
         console.log('Launch tests in test runner');
         await new Promise<void>((resolve, reject) => {
             const env: Record<string, string> = {
-                TEST_FILES_SUFFIX: 'notebookCellExecution.perf.test',
+                TEST_FILES_SUFFIX: testFile,
+                VSC_JUPYTER_CI_TEST_VSC_CHANNEL: 'insiders',
                 VSC_JUPYTER_PERF_TEST: '1',
                 CODE_TESTS_WORKSPACE: path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'src', 'test', 'datascience'),
+                VSC_JUPYTER_WARMUP_RUN: warmupRun ? '1' : '0',
+                VSC_JUPYTER_TEST_TIMEOUT: '60000',
                 ...process.env
             };
             const proc = spawn('node', [path.join(__dirname, 'standardTest.node.js')], {

diff --git a/src/test/performance/load.perf.vscode.test.ts b/src/test/performance/load.perf.vscode.test.ts
diff --git a/src/test/performance/notebookCellExecution.perf.test.ts b/src/test/performance/notebookCellExecution.perf.test.ts
@@ -5,9 +5,9 @@ import { IDisposable } from '../../platform/common/types';
 import { IExtensionTestApi } from '../common';
 import { createEmptyPythonNotebook, insertCodeCell, runCell, waitForTextOutput } from '../datascience/notebook/helper';
 import { activateExtension, initializePython } from '../initialize.node';
-import { PerformanceTracker } from './performanceTracker';
+import { PerformanceTracker } from './performanceTracker.node';
 
-suite('Initial Notebook Cell Execution Perf Test', () => {
+suite('Initial Notebook Cell Execution Perf Test', function () {
     let tracker: PerformanceTracker;
     setup(function () {
         sinon.restore();
@@ -21,7 +21,6 @@ suite('Initial Notebook Cell Execution Perf Test', () => {
     test('Initial Notebook Cell Execution Perf Test', async function () {
         const disposables: IDisposable[] = [];
         sinon.restore();
-
         await initializePython();
         tracker.markTime('pythonExtensionActivation');
 

diff --git a/src/test/performance/performanceTracker.ts → ...st/performance/performanceTracker.node.ts b/src/test/performance/performanceTracker.ts → ...st/performance/performanceTracker.node.ts
@@ -4,15 +4,14 @@ import { extensions } from 'vscode';
 import { JVSC_EXTENSION_ID, AppinsightsKey, Telemetry } from '../../platform/common/constants';
 import { StopWatch } from '../../platform/common/utils/stopWatch';
 import { IS_CI_SERVER } from '../ciConstants.node';
-import { JVSC_EXTENSION_ID_FOR_TESTS } from '../constants';
 import { sleep } from '../core';
 
 export class PerformanceTracker implements IDisposable {
     stopWatch = new StopWatch();
     durations: Record<string, number> = {};
     checkpointCount = 0;
     telemetryReporter: TelemetryReporter | undefined;
-    telemetryEnabled = true || IS_CI_SERVER;
+    telemetryEnabled = true || IS_CI_SERVER && !process.env.VSC_JUPYTER_WARMUP;
 
     constructor(private testName: string) {
         if (this.telemetryEnabled) {
@@ -30,7 +29,7 @@ export class PerformanceTracker implements IDisposable {
 
     public finishAndReport(result: string) {
         this.durations.totalTime = this.stopWatch.elapsedTime;
-        console.log(`test ${result} with times ${this.durations}`); // not showing in debug console?
+        console.log(`test ${result} with times: ${JSON.stringify(this.durations)}`);
         this.telemetryReporter?.sendDangerousTelemetryEvent(
             Telemetry.RunTest,
             {