From ccbcf7208e18eac4b1cfabc1581b9699e38e379f Mon Sep 17 00:00:00 2001 From: aamunger Date: Fri, 13 May 2022 10:57:59 -0700 Subject: [PATCH] updated npm perf test script --- .vscode/launch.json | 1 + package.json | 1 - src/test/index.node.ts | 6 +- src/test/perfTest.node.ts | 22 ++++- src/test/performance/load.perf.vscode.test.ts | 93 ------------------- .../notebookCellExecution.perf.test.ts | 5 +- ...eTracker.ts => performanceTracker.node.ts} | 5 +- 7 files changed, 27 insertions(+), 106 deletions(-) delete mode 100644 src/test/performance/load.perf.vscode.test.ts rename src/test/performance/{performanceTracker.ts => performanceTracker.node.ts} (90%) diff --git a/.vscode/launch.json b/.vscode/launch.json index e0716d3613a..0d90f0c8b6a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -152,6 +152,7 @@ "CI_PYTHON_PATH": "", "VSC_JUPYTER_PERF_TEST": "1", "TEST_FILES_SUFFIX": "notebookCellExecution.perf.test", + "VSC_JUPYTER_TEST_TIMEOUT": "60000", }, "sourceMaps": true, "outFiles": ["${workspaceFolder}/out/**/*.js", "!${workspaceFolder}/**/node_modules**/*"], diff --git a/package.json b/package.json index 999b66609a0..29dbe09f807 100644 --- a/package.json +++ b/package.json @@ -2114,7 +2114,6 @@ "testNativeNotebooksAndWebviews": "cross-env CODE_TESTS_WORKSPACE=src/test/datascience VSC_JUPYTER_CI_TEST_VSC_CHANNEL=insiders TEST_FILES_SUFFIX=vscode.test* VSC_JUPYTER_FORCE_LOGGING=1 VSC_JUPYTER_CI_TEST_GREP=webview-test VSC_JUPYTER_LOAD_EXPERIMENTS_FROM_FILE=true node ./out/test/testBootstrap.node.js ./out/test/standardTest.node.js", "testWebExtension": "node ./build/launchWebTest.js", "testPerformance": "node ./out/test/testBootstrap.node.js ./out/test/perfTest.node.js", - "testPerformance2": "cross-env CODE_TESTS_WORKSPACE=src/test/datascience VSC_JUPYTER_CI_TEST_VSC_CHANNEL=insiders TEST_FILES_SUFFIX=notebookCellExecution.perf.test VSC_JUPYTER_FORCE_LOGGING=1 VSC_JUPYTER_LOAD_EXPERIMENTS_FROM_FILE=true node ./out/test/testBootstrap.node.js ./out/test/standardTest.node.js", "testSmoke": "node ./out/test/testBootstrap.node.js ./out/test/smokeTest.node.js", "testSmokeLogged": "cross-env VSC_JUPYTER_FORCE_LOGGING=true VSC_JUPYTER_LOG_FILE=smoke-test.log node --no-force-async-hooks-checks ./out/test/testBootstrap.node.js ./out/test/smokeTest.node.js", "lint": "eslint -c .eslintrc.js --ext .ts src", diff --git a/src/test/index.node.ts b/src/test/index.node.ts index 1fc193616ec..4a4e1c4554a 100644 --- a/src/test/index.node.ts +++ b/src/test/index.node.ts @@ -74,13 +74,14 @@ function configure(): SetupOptions { // So the solution is to run them separately and first on CI. const grep = IS_CI_SERVER_TEST_DEBUGGER ? 'Debug' : defaultGrep; const testFilesSuffix = process.env.TEST_FILES_SUFFIX || '.test*'; + const testTimeout = process.env.VSC_JUPYTER_TEST_TIMEOUT || TEST_TIMEOUT; const options: SetupOptions & { retries: number; invert: boolean } = { ui: 'tdd', color: true, rootHooks: rootHooks, invert, - timeout: TEST_TIMEOUT, + timeout: testTimeout, retries: IS_CI_SERVER ? TEST_RETRYCOUNT : 0, grep, testFilesSuffix, @@ -185,11 +186,10 @@ export async function run(): Promise { ); }); - console.log(`found ${testFiles.length} test files with suffix ${options.testFilesSuffix} and ignoreing ${ignoreGlob.join(', ')}`); - // Setup test files that need to be run. testFiles.forEach((file) => mocha.addFile(path.join(testsRoot, file))); + // for performance tests, extension activation is part of the test run if (!IS_PERF_TEST) { /* eslint-disable no-console */ console.time('Time taken to activate the extension'); diff --git a/src/test/perfTest.node.ts b/src/test/perfTest.node.ts index 648a9b172c7..ffc228d0c78 100644 --- a/src/test/perfTest.node.ts +++ b/src/test/perfTest.node.ts @@ -7,22 +7,38 @@ process.env.VSC_JUPYTER_SMOKE_TEST = '1'; import { spawn } from 'child_process'; +import glob from 'glob'; import * as path from '../platform/vscode-path/path'; import { EXTENSION_ROOT_DIR_FOR_TESTS } from './constants.node'; class TestRunner { public async start() { console.log('Start Test Runner'); - await this.launchTest(); + const testFiles = await new Promise((resolve, reject) => { + glob(`**/*perf.test.js`, (error, files) => { + if (error) { + return reject(error); + } + resolve(files); + }); + }); + + // warm up with a basic notebook operation before running the tests + await this.launchTest('notebookCellExecution.perf.test', true); + + testFiles.forEach((file) => this.launchTest(file)); } - private async launchTest() { + private async launchTest(testFile: string, warmupRun?: boolean) { console.log('Launch tests in test runner'); await new Promise((resolve, reject) => { const env: Record = { - TEST_FILES_SUFFIX: 'notebookCellExecution.perf.test', + TEST_FILES_SUFFIX: testFile, + VSC_JUPYTER_CI_TEST_VSC_CHANNEL: 'insiders', VSC_JUPYTER_PERF_TEST: '1', CODE_TESTS_WORKSPACE: path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'src', 'test', 'datascience'), + VSC_JUPYTER_WARMUP_RUN: warmupRun ? '1' : '0', + VSC_JUPYTER_TEST_TIMEOUT: '60000', ...process.env }; const proc = spawn('node', [path.join(__dirname, 'standardTest.node.js')], { diff --git a/src/test/performance/load.perf.vscode.test.ts b/src/test/performance/load.perf.vscode.test.ts deleted file mode 100644 index 0c2e15c2baf..00000000000 --- a/src/test/performance/load.perf.vscode.test.ts +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -'use strict'; - -/* eslint-disable no-invalid-this, no-console */ - -import { expect } from 'chai'; -import * as fs from 'fs-extra'; -import { EOL } from 'os'; -import * as path from '../../platform/vscode-path/path'; -import { commands, extensions } from 'vscode'; -import { JVSC_EXTENSION_ID } from '../../platform/common/constants'; -import { StopWatch } from '../../platform/common/utils/stopWatch'; -import { - ACTIVATION_TIMES_DEV_LANGUAGE_SERVER_LOG_FILE_PATHS, - ACTIVATION_TIMES_DEV_LOG_FILE_PATHS, - ACTIVATION_TIMES_EXT_VERSION, - ACTIVATION_TIMES_LOG_FILE_PATH, - ACTIVATION_TIMES_RELEASE_LOG_FILE_PATHS -} from './constants.node'; - -const AllowedIncreaseInActivationDelayInMS = 500; - -suite('Activation Times', () => { - if (ACTIVATION_TIMES_LOG_FILE_PATH) { - const logFile = ACTIVATION_TIMES_LOG_FILE_PATH; - const sampleCounter = fs.existsSync(logFile) - ? fs.readFileSync(logFile, { encoding: 'utf8' }).toString().split(/\r?\n/g).length - : 1; - if (sampleCounter > 5) { - return; - } - test(`Capture Extension Activation Times (Version: ${ACTIVATION_TIMES_EXT_VERSION}, sample: ${sampleCounter})`, async () => { - const pythonExtension = extensions.getExtension(JVSC_EXTENSION_ID); - if (!pythonExtension) { - throw new Error('Python Extension not found'); - } - const stopWatch = new StopWatch(); - await pythonExtension!.activate(); - const elapsedTime = stopWatch.elapsedTime; - if (elapsedTime > 10) { - await fs.ensureDir(path.dirname(logFile)); - await fs.appendFile(logFile, `${elapsedTime}${EOL}`, { encoding: 'utf8' }); - console.log(`Loaded in ${elapsedTime}ms`); - } - await commands.executeCommand('workbench.action.reloadWindow'); - }); - } - - if ( - ACTIVATION_TIMES_DEV_LOG_FILE_PATHS && - ACTIVATION_TIMES_RELEASE_LOG_FILE_PATHS && - ACTIVATION_TIMES_DEV_LANGUAGE_SERVER_LOG_FILE_PATHS - ) { - test('Test activation times of Dev vs Release Extension', async () => { - function getActivationTimes(files: string[]) { - const activationTimes: number[] = []; - for (const file of files) { - fs.readFileSync(file, { encoding: 'utf8' }) - .toString() - .split(/\r?\n/g) - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => parseInt(line, 10)) - .forEach((item) => activationTimes.push(item)); - } - return activationTimes; - } - const devActivationTimes = getActivationTimes(JSON.parse(ACTIVATION_TIMES_DEV_LOG_FILE_PATHS!)); - const releaseActivationTimes = getActivationTimes(JSON.parse(ACTIVATION_TIMES_RELEASE_LOG_FILE_PATHS!)); - const languageServerActivationTimes = getActivationTimes( - JSON.parse(ACTIVATION_TIMES_DEV_LANGUAGE_SERVER_LOG_FILE_PATHS!) - ); - const devActivationAvgTime = - devActivationTimes.reduce((sum, item) => sum + item, 0) / devActivationTimes.length; - const releaseActivationAvgTime = - releaseActivationTimes.reduce((sum, item) => sum + item, 0) / releaseActivationTimes.length; - const languageServerActivationAvgTime = - languageServerActivationTimes.reduce((sum, item) => sum + item, 0) / - languageServerActivationTimes.length; - - console.log(`Dev version loaded in ${devActivationAvgTime}ms`); - console.log(`Release version loaded in ${releaseActivationAvgTime}ms`); - console.log(`Language server loaded in ${languageServerActivationAvgTime}ms`); - - expect(devActivationAvgTime - releaseActivationAvgTime).to.be.lessThan( - AllowedIncreaseInActivationDelayInMS, - 'Activation times have increased above allowed threshold.' - ); - }); - } -}); diff --git a/src/test/performance/notebookCellExecution.perf.test.ts b/src/test/performance/notebookCellExecution.perf.test.ts index a345a3308a7..3bb9065a2e2 100644 --- a/src/test/performance/notebookCellExecution.perf.test.ts +++ b/src/test/performance/notebookCellExecution.perf.test.ts @@ -5,9 +5,9 @@ import { IDisposable } from '../../platform/common/types'; import { IExtensionTestApi } from '../common'; import { createEmptyPythonNotebook, insertCodeCell, runCell, waitForTextOutput } from '../datascience/notebook/helper'; import { activateExtension, initializePython } from '../initialize.node'; -import { PerformanceTracker } from './performanceTracker'; +import { PerformanceTracker } from './performanceTracker.node'; -suite('Initial Notebook Cell Execution Perf Test', () => { +suite('Initial Notebook Cell Execution Perf Test', function () { let tracker: PerformanceTracker; setup(function () { sinon.restore(); @@ -21,7 +21,6 @@ suite('Initial Notebook Cell Execution Perf Test', () => { test('Initial Notebook Cell Execution Perf Test', async function () { const disposables: IDisposable[] = []; sinon.restore(); - await initializePython(); tracker.markTime('pythonExtensionActivation'); diff --git a/src/test/performance/performanceTracker.ts b/src/test/performance/performanceTracker.node.ts similarity index 90% rename from src/test/performance/performanceTracker.ts rename to src/test/performance/performanceTracker.node.ts index b7e4af80835..aadb6e0b6cb 100644 --- a/src/test/performance/performanceTracker.ts +++ b/src/test/performance/performanceTracker.node.ts @@ -4,7 +4,6 @@ import { extensions } from 'vscode'; import { JVSC_EXTENSION_ID, AppinsightsKey, Telemetry } from '../../platform/common/constants'; import { StopWatch } from '../../platform/common/utils/stopWatch'; import { IS_CI_SERVER } from '../ciConstants.node'; -import { JVSC_EXTENSION_ID_FOR_TESTS } from '../constants'; import { sleep } from '../core'; export class PerformanceTracker implements IDisposable { @@ -12,7 +11,7 @@ export class PerformanceTracker implements IDisposable { durations: Record = {}; checkpointCount = 0; telemetryReporter: TelemetryReporter | undefined; - telemetryEnabled = true || IS_CI_SERVER; + telemetryEnabled = true || IS_CI_SERVER && !process.env.VSC_JUPYTER_WARMUP; constructor(private testName: string) { if (this.telemetryEnabled) { @@ -30,7 +29,7 @@ export class PerformanceTracker implements IDisposable { public finishAndReport(result: string) { this.durations.totalTime = this.stopWatch.elapsedTime; - console.log(`test ${result} with times ${this.durations}`); // not showing in debug console? + console.log(`test ${result} with times: ${JSON.stringify(this.durations)}`); this.telemetryReporter?.sendDangerousTelemetryEvent( Telemetry.RunTest, {