diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b947f8dfede4b..b06cd9e04c219 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -465,6 +465,8 @@ packages/kbn-safer-lodash-set @elastic/kibana-security packages/kbn-saved-objects-settings @elastic/appex-sharedux packages/kbn-saved-search-component @elastic/obs-ux-logs-team packages/kbn-scout @elastic/appex-qa +packages/kbn-scout-info @elastic/appex-qa +packages/kbn-scout-reporting @elastic/appex-qa packages/kbn-screenshotting-server @elastic/appex-sharedux packages/kbn-search-api-keys-components @elastic/search-kibana packages/kbn-search-api-keys-server @elastic/search-kibana diff --git a/package.json b/package.json index 7a93af2e2ac7a..d628499e5bf22 100644 --- a/package.json +++ b/package.json @@ -1493,6 +1493,8 @@ "@kbn/repo-source-classifier": "link:packages/kbn-repo-source-classifier", "@kbn/repo-source-classifier-cli": "link:packages/kbn-repo-source-classifier-cli", "@kbn/scout": "link:packages/kbn-scout", + "@kbn/scout-info": "link:packages/kbn-scout-info", + "@kbn/scout-reporting": "link:packages/kbn-scout-reporting", "@kbn/security-api-integration-helpers": "link:x-pack/test/security_api_integration/packages/helpers", "@kbn/serverless-storybook-config": "link:packages/serverless/storybook/config", "@kbn/some-dev-log": "link:packages/kbn-some-dev-log", diff --git a/packages/kbn-scout-info/README.md b/packages/kbn-scout-info/README.md new file mode 100644 index 0000000000000..31db067b02443 --- /dev/null +++ b/packages/kbn-scout-info/README.md @@ -0,0 +1,6 @@ +# @kbn/scout-info + +This package stores information that's commonly used by packages in the `@kbn/scout*` namespace, and any other + package that wishes to extend Scout functionality. + +Check out the `@kbn/scout` package if you want to learn more about Scout. diff --git a/scripts/scout_test.js b/packages/kbn-scout-info/index.ts similarity index 86% rename from scripts/scout_test.js rename to packages/kbn-scout-info/index.ts index 8b14ebd33da19..ff7948350e1ad 100644 --- a/scripts/scout_test.js +++ b/packages/kbn-scout-info/index.ts @@ -7,5 +7,5 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -require('../src/setup_node_env'); -require('@kbn/scout').runTestsCli(); +export * from './src/paths'; +export * from './src/reporting'; diff --git a/packages/kbn-scout-info/jest.config.js b/packages/kbn-scout-info/jest.config.js new file mode 100644 index 0000000000000..6e70c1cc5996b --- /dev/null +++ b/packages/kbn-scout-info/jest.config.js @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-scout-info'], +}; diff --git a/packages/kbn-scout-info/kibana.jsonc b/packages/kbn-scout-info/kibana.jsonc new file mode 100644 index 0000000000000..a2a9f00b951c0 --- /dev/null +++ b/packages/kbn-scout-info/kibana.jsonc @@ -0,0 +1,6 @@ +{ + "type": "shared-common", + "id": "@kbn/scout-info", + "owner": "@elastic/appex-qa", + "devOnly": true +} diff --git a/packages/kbn-scout-info/package.json b/packages/kbn-scout-info/package.json new file mode 100644 index 0000000000000..c6e1076d04833 --- /dev/null +++ b/packages/kbn-scout-info/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/scout-info", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0" +} \ No newline at end of file diff --git a/packages/kbn-scout-info/src/paths.ts b/packages/kbn-scout-info/src/paths.ts new file mode 100644 index 0000000000000..f471cdcfa0201 --- /dev/null +++ b/packages/kbn-scout-info/src/paths.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import path from 'node:path'; +import { REPO_ROOT } from '@kbn/repo-info'; + +export const SCOUT_OUTPUT_ROOT = path.resolve(REPO_ROOT, '.scout'); + +// Servers +export const SCOUT_SERVERS_ROOT = path.resolve(SCOUT_OUTPUT_ROOT, 'servers'); + +// Reporting +export const SCOUT_REPORT_OUTPUT_ROOT = path.resolve(SCOUT_OUTPUT_ROOT, 'reports'); diff --git a/packages/kbn-scout-info/src/reporting.ts b/packages/kbn-scout-info/src/reporting.ts new file mode 100644 index 0000000000000..8c9660dde2daa --- /dev/null +++ b/packages/kbn-scout-info/src/reporting.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export const SCOUT_TEST_EVENTS_TEMPLATE_NAME = 'scout-test-events'; +export const SCOUT_TEST_EVENTS_INDEX_PATTERN = `${SCOUT_TEST_EVENTS_TEMPLATE_NAME}-*`; +export const SCOUT_TEST_EVENTS_DATA_STREAM_NAME = `${SCOUT_TEST_EVENTS_TEMPLATE_NAME}-kibana`; diff --git a/packages/kbn-scout-info/tsconfig.json b/packages/kbn-scout-info/tsconfig.json new file mode 100644 index 0000000000000..b7a5e0164b099 --- /dev/null +++ b/packages/kbn-scout-info/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/repo-info" + ] +} diff --git a/packages/kbn-scout-reporting/README.md b/packages/kbn-scout-reporting/README.md new file mode 100644 index 0000000000000..152e2421d97b1 --- /dev/null +++ b/packages/kbn-scout-reporting/README.md @@ -0,0 +1,5 @@ +# @kbn/scout-reporting + +This package contains reporting functionality for Scout. + +Check out the `@kbn/scout` package if you want to learn more about Scout. diff --git a/packages/kbn-scout-reporting/index.ts b/packages/kbn-scout-reporting/index.ts new file mode 100644 index 0000000000000..474c557f60687 --- /dev/null +++ b/packages/kbn-scout-reporting/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export * as cli from './src/cli'; +export * as datasources from './src/datasources'; +export * from './src/reporting'; diff --git a/packages/kbn-scout-reporting/jest.config.js b/packages/kbn-scout-reporting/jest.config.js new file mode 100644 index 0000000000000..bf77cdb10a86b --- /dev/null +++ b/packages/kbn-scout-reporting/jest.config.js @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-scout-reporting'], +}; diff --git a/packages/kbn-scout-reporting/kibana.jsonc b/packages/kbn-scout-reporting/kibana.jsonc new file mode 100644 index 0000000000000..3f73871b2e174 --- /dev/null +++ b/packages/kbn-scout-reporting/kibana.jsonc @@ -0,0 +1,6 @@ +{ + "type": "shared-common", + "id": "@kbn/scout-reporting", + "owner": "@elastic/appex-qa", + "devOnly": true +} diff --git a/packages/kbn-scout-reporting/package.json b/packages/kbn-scout-reporting/package.json new file mode 100644 index 0000000000000..6c87cb11358bc --- /dev/null +++ b/packages/kbn-scout-reporting/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/scout-reporting", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0" +} \ No newline at end of file diff --git a/packages/kbn-scout-reporting/src/cli/common.ts b/packages/kbn-scout-reporting/src/cli/common.ts new file mode 100644 index 0000000000000..e7d88f5b6576a --- /dev/null +++ b/packages/kbn-scout-reporting/src/cli/common.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { Client as ESClient, ClientOptions as ESClientOptions } from '@elastic/elasticsearch'; +import { ToolingLog } from '@kbn/tooling-log'; +import { createFailError } from '@kbn/dev-cli-errors'; + +/** + * Get an Elasticsearch client for which connectivity has been validated + * + * @param options Elasticsearch client options + * @param log Logger instance + * @throws FailError if cluster information cannot be read from the target Elasticsearch instance + */ +export async function getValidatedESClient( + options: ESClientOptions, + log: ToolingLog +): Promise { + const es = new ESClient(options); + + await es.info().then( + (esInfo) => { + log.info(`Connected to Elasticsearch node '${esInfo.name}'`); + }, + (err) => { + throw createFailError(`Failed to connect to Elasticsearch\n${err}`); + } + ); + + return es; +} diff --git a/packages/kbn-scout-reporting/src/cli/index.ts b/packages/kbn-scout-reporting/src/cli/index.ts new file mode 100644 index 0000000000000..774275d5b9453 --- /dev/null +++ b/packages/kbn-scout-reporting/src/cli/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export { initializeReportDatastream } from './initialize_report_datastream'; +export { uploadEvents } from './upload_events'; diff --git a/packages/kbn-scout-reporting/src/cli/initialize_report_datastream.ts b/packages/kbn-scout-reporting/src/cli/initialize_report_datastream.ts new file mode 100644 index 0000000000000..314794a9181b7 --- /dev/null +++ b/packages/kbn-scout-reporting/src/cli/initialize_report_datastream.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { Command } from '@kbn/dev-cli-runner'; +import { ScoutReportDataStream } from '../reporting/report'; +import { getValidatedESClient } from './common'; + +export const initializeReportDatastream: Command = { + name: 'initialize-report-datastream', + description: 'Initialize a Scout report datastream in Elasticsearch', + flags: { + string: ['esURL', 'esAPIKey'], + boolean: ['verifyTLSCerts'], + default: { + esURL: process.env.ES_URL, + esAPIKey: process.env.ES_API_KEY, + }, + help: ` + --esURL (required) Elasticsearch URL [env: ES_URL] + --esAPIKey (required) Elasticsearch API Key [env: ES_API_KEY] + --verifyTLSCerts (optional) Verify TLS certificates + `, + }, + run: async ({ flagsReader, log }) => { + const esURL = flagsReader.requiredString('esURL'); + const esAPIKey = flagsReader.requiredString('esAPIKey'); + + // ES connection + log.info(`Connecting to Elasticsearch at ${esURL}`); + const es = await getValidatedESClient( + { + node: esURL, + auth: { apiKey: esAPIKey }, + tls: { + rejectUnauthorized: flagsReader.boolean('verifyTLSCerts'), + }, + }, + log + ); + + // Initialize the report datastream + const reportDataStream = new ScoutReportDataStream(es, log); + await reportDataStream.initialize(); + + log.success('Scout report data stream initialized'); + }, +}; diff --git a/packages/kbn-scout-reporting/src/cli/upload_events.ts b/packages/kbn-scout-reporting/src/cli/upload_events.ts new file mode 100644 index 0000000000000..8c2ef1bd67347 --- /dev/null +++ b/packages/kbn-scout-reporting/src/cli/upload_events.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import fs from 'node:fs'; +import { Command } from '@kbn/dev-cli-runner'; +import { createFlagError } from '@kbn/dev-cli-errors'; +import { ScoutReportDataStream } from '../reporting/report'; +import { getValidatedESClient } from './common'; + +export const uploadEvents: Command = { + name: 'upload-events', + description: 'Upload events recorded by the Scout reporter to Elasticsearch', + flags: { + string: ['eventLogPath', 'esURL', 'esAPIKey'], + boolean: ['verifyTLSCerts'], + default: { + esURL: process.env.ES_URL, + esAPIKey: process.env.ES_API_KEY, + }, + help: ` + --eventLogPath (required) Path to the event log to upload + --esURL (required) Elasticsearch URL [env: ES_URL] + --esAPIKey (required) Elasticsearch API Key [env: ES_API_KEY] + --verifyTLSCerts (optional) Verify TLS certificates + `, + }, + run: async ({ flagsReader, log }) => { + // Read & validate CLI options + const eventLogPath = flagsReader.requiredString('eventLogPath'); + + if (!fs.existsSync(eventLogPath)) { + throw createFlagError(`Event log path '${eventLogPath}' does not exist.`); + } + + const esURL = flagsReader.requiredString('esURL'); + const esAPIKey = flagsReader.requiredString('esAPIKey'); + + // ES connection + log.info(`Connecting to Elasticsearch at ${esURL}`); + const es = await getValidatedESClient( + { + node: esURL, + auth: { apiKey: esAPIKey }, + tls: { + rejectUnauthorized: flagsReader.boolean('verifyTLSCerts'), + }, + }, + log + ); + + // Event log upload + const reportDataStream = new ScoutReportDataStream(es, log); + await reportDataStream.addEventsFromFile(eventLogPath); + }, +}; diff --git a/packages/kbn-scout-reporting/src/datasources/buildkite.ts b/packages/kbn-scout-reporting/src/datasources/buildkite.ts new file mode 100644 index 0000000000000..e58889510f4d7 --- /dev/null +++ b/packages/kbn-scout-reporting/src/datasources/buildkite.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +/** + * Buildkite info + */ +export interface BuildkiteMetadata { + branch?: string; + commit?: string; + job_id?: string; + message?: string; + build: { + id?: string; + number?: string; + url?: string; + }; + pipeline: { + id?: string; + name?: string; + slug?: string; + }; + agent: { + name?: string; + }; + group: { + id?: string; + key?: string; + label?: string; + }; + step: { + id?: string; + key?: string; + label?: string; + }; + command?: string; +} + +/** + * Buildkite information extracted from environment variables + * + * This object is empty if the process is not running in a Buildkite pipeline. + */ +export const buildkite: BuildkiteMetadata = + process.env.BUILDKITE === 'true' + ? { + branch: process.env.BUILDKITE_BRANCH, + commit: process.env.BUILDKITE_COMMIT, + job_id: process.env.BUILDKITE_JOB_ID, + message: process.env.BUILDKITE_MESSAGE, + build: { + id: process.env.BUILDKITE_BUILD_ID, + number: process.env.BUILDKITE_BUILD_NUMBER, + url: process.env.BUILDKITE_BUILD_URL, + }, + pipeline: { + id: process.env.BUILDKITE_PIPELINE_ID, + name: process.env.BUILDKITE_PIPELINE_NAME, + slug: process.env.BUILDKITE_PIPELINE_SLUG, + }, + agent: { + name: process.env.BUILDKITE_AGENT_NAME, + }, + group: { + id: process.env.BUILDKITE_GROUP_ID, + key: process.env.BUILDKITE_GROUP_KEY, + label: process.env.BUILDKITE_GROUP_LABEL, + }, + step: { + id: process.env.BUILDKITE_STEP_ID, + key: process.env.BUILDKITE_STEP_KEY, + label: process.env.BUILDKITE_LABEL, + }, + command: process.env.BUILDKITE_COMMAND, + } + : { + build: {}, + pipeline: {}, + agent: {}, + group: {}, + step: {}, + }; diff --git a/packages/kbn-scout-reporting/src/datasources/host.ts b/packages/kbn-scout-reporting/src/datasources/host.ts new file mode 100644 index 0000000000000..c7d528947a1c0 --- /dev/null +++ b/packages/kbn-scout-reporting/src/datasources/host.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import os from 'node:os'; + +/** + * Host info + */ +export interface HostMetadata { + architecture: string; + hostname: string; + os: OSMetadata; +} + +/** + * Operating system info + */ +export interface OSMetadata { + platform: string; + version: string; + family: string; +} + +/** + * Information about the host this process is running on + */ +export const host: HostMetadata = { + architecture: os.arch(), + hostname: os.hostname(), + os: { + platform: os.platform(), + version: os.release(), + family: os.type(), + }, +}; diff --git a/packages/kbn-scout-reporting/src/datasources/index.ts b/packages/kbn-scout-reporting/src/datasources/index.ts new file mode 100644 index 0000000000000..fc467d6f05380 --- /dev/null +++ b/packages/kbn-scout-reporting/src/datasources/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { buildkite } from './buildkite'; +import { host } from './host'; + +export * from './buildkite'; +export * from './host'; + +export const environmentMetadata = { + buildkite, + host, +}; diff --git a/packages/kbn-scout-reporting/src/reporting/index.ts b/packages/kbn-scout-reporting/src/reporting/index.ts new file mode 100644 index 0000000000000..58d4002320047 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/index.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { createHash, randomBytes } from 'node:crypto'; +import type { ReporterDescription } from 'playwright/test'; +import type { ScoutPlaywrightReporterOptions } from './playwright'; + +export * from './report'; + +// ID helpers +export function generateTestRunId() { + return randomBytes(8).toString('hex'); +} + +export function getTestIDForTitle(title: string) { + return createHash('sha256').update(title).digest('hex').slice(0, 31); +} + +// Playwright reporting +export const scoutPlaywrightReporter = ( + options?: ScoutPlaywrightReporterOptions +): ReporterDescription => { + return ['@kbn/scout-reporting/src/reporting/playwright.ts', options]; +}; diff --git a/packages/kbn-scout-reporting/src/reporting/playwright.ts b/packages/kbn-scout-reporting/src/reporting/playwright.ts new file mode 100644 index 0000000000000..fdea17cb844c0 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/playwright.ts @@ -0,0 +1,300 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { + FullConfig, + FullResult, + Reporter, + Suite, + TestCase, + TestError, + TestResult, + TestStep, +} from '@playwright/test/reporter'; + +import path from 'node:path'; +import { ToolingLog } from '@kbn/tooling-log'; +import { SCOUT_REPORT_OUTPUT_ROOT } from '@kbn/scout-info'; +import stripANSI from 'strip-ansi'; +import { REPO_ROOT } from '@kbn/repo-info'; +import { PathWithOwners, getPathsWithOwnersReversed, getCodeOwnersForFile } from '@kbn/code-owners'; +import { generateTestRunId, getTestIDForTitle, ScoutReport, ScoutReportEventAction } from '.'; +import { environmentMetadata } from '../datasources'; + +/** + * Configuration options for the Scout Playwright reporter + */ +export interface ScoutPlaywrightReporterOptions { + name?: string; + outputPath?: string; +} + +/** + * Scout Playwright reporter + */ +export class ScoutPlaywrightReporter implements Reporter { + readonly log: ToolingLog; + readonly name: string; + readonly runId: string; + private report: ScoutReport; + private readonly pathsWithOwners: PathWithOwners[]; + + constructor(private reporterOptions: ScoutPlaywrightReporterOptions = {}) { + this.log = new ToolingLog({ + level: 'info', + writeTo: process.stdout, + }); + + this.name = this.reporterOptions.name || 'unknown'; + this.runId = generateTestRunId(); + this.log.info(`Scout test run ID: ${this.runId}`); + + this.report = new ScoutReport(this.log); + this.pathsWithOwners = getPathsWithOwnersReversed(); + } + + private getFileOwners(filePath: string): string[] { + const concatenatedOwners = getCodeOwnersForFile(filePath, this.pathsWithOwners); + + if (concatenatedOwners === undefined) { + return []; + } + + return concatenatedOwners + .replace(/#.+$/, '') + .split(',') + .filter((value) => value.length > 0); + } + + /** + * Root path of this reporter's output + */ + public get reportRootPath(): string { + const outputPath = this.reporterOptions.outputPath || SCOUT_REPORT_OUTPUT_ROOT; + return path.join(outputPath, `scout-playwright-${this.runId}`); + } + + printsToStdio(): boolean { + // Don't take over console output + return false; + } + + onBegin(config: FullConfig, suite: Suite) { + this.report.logEvent({ + ...environmentMetadata, + reporter: { + name: this.name, + type: 'playwright', + }, + test_run: { + id: this.runId, + }, + event: { + action: ScoutReportEventAction.RUN_BEGIN, + }, + }); + } + + onTestBegin(test: TestCase, result: TestResult) { + this.report.logEvent({ + '@timestamp': result.startTime, + ...environmentMetadata, + reporter: { + name: this.name, + type: 'playwright', + }, + test_run: { + id: this.runId, + }, + suite: { + title: test.parent.titlePath().join(' '), + type: test.parent.type, + }, + test: { + id: getTestIDForTitle(test.titlePath().join(' ')), + title: test.title, + tags: test.tags, + annotations: test.annotations, + expected_status: test.expectedStatus, + }, + event: { + action: ScoutReportEventAction.TEST_BEGIN, + }, + file: { + path: path.relative(REPO_ROOT, test.location.file), + owner: this.getFileOwners(path.relative(REPO_ROOT, test.location.file)), + }, + }); + } + + onStepBegin(test: TestCase, result: TestResult, step: TestStep) { + this.report.logEvent({ + '@timestamp': step.startTime, + ...environmentMetadata, + reporter: { + name: this.name, + type: 'playwright', + }, + test_run: { + id: this.runId, + }, + suite: { + title: test.parent.titlePath().join(' '), + type: test.parent.type, + }, + test: { + id: getTestIDForTitle(test.titlePath().join(' ')), + title: test.title, + tags: test.tags, + annotations: test.annotations, + expected_status: test.expectedStatus, + step: { + title: step.titlePath().join(' '), + category: step.category, + }, + }, + event: { + action: ScoutReportEventAction.TEST_STEP_BEGIN, + }, + file: { + path: path.relative(REPO_ROOT, test.location.file), + owner: this.getFileOwners(path.relative(REPO_ROOT, test.location.file)), + }, + }); + } + + onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + this.report.logEvent({ + ...environmentMetadata, + reporter: { + name: this.name, + type: 'playwright', + }, + test_run: { + id: this.runId, + }, + suite: { + title: test.parent.titlePath().join(' '), + type: test.parent.type, + }, + test: { + id: getTestIDForTitle(test.titlePath().join(' ')), + title: test.title, + tags: test.tags, + annotations: test.annotations, + expected_status: test.expectedStatus, + step: { + title: step.titlePath().join(' '), + category: step.category, + duration: step.duration, + }, + }, + event: { + action: ScoutReportEventAction.TEST_STEP_END, + error: { + message: step.error?.message ? stripANSI(step.error.message) : undefined, + stack_trace: step.error?.stack ? stripANSI(step.error.stack) : undefined, + }, + }, + file: { + path: path.relative(REPO_ROOT, test.location.file), + owner: this.getFileOwners(path.relative(REPO_ROOT, test.location.file)), + }, + }); + } + + onTestEnd(test: TestCase, result: TestResult) { + this.report.logEvent({ + ...environmentMetadata, + reporter: { + name: this.name, + type: 'playwright', + }, + test_run: { + id: this.runId, + }, + suite: { + title: test.parent.titlePath().join(' '), + type: test.parent.type, + }, + test: { + id: getTestIDForTitle(test.titlePath().join(' ')), + title: test.title, + tags: test.tags, + annotations: test.annotations, + expected_status: test.expectedStatus, + status: result.status, + duration: result.duration, + }, + event: { + action: ScoutReportEventAction.TEST_END, + error: { + message: result.error?.message ? stripANSI(result.error.message) : undefined, + stack_trace: result.error?.stack ? stripANSI(result.error.stack) : undefined, + }, + }, + file: { + path: path.relative(REPO_ROOT, test.location.file), + owner: this.getFileOwners(path.relative(REPO_ROOT, test.location.file)), + }, + }); + } + + onEnd(result: FullResult) { + this.report.logEvent({ + ...environmentMetadata, + reporter: { + name: this.name, + type: 'playwright', + }, + test_run: { + id: this.runId, + status: result.status, + duration: result.duration, + }, + event: { + action: ScoutReportEventAction.RUN_END, + }, + }); + + // Save & conclude the report + try { + this.report.save(this.reportRootPath); + } finally { + this.report.conclude(); + } + } + + async onExit() { + // noop + } + + onError(error: TestError) { + this.report.logEvent({ + ...environmentMetadata, + reporter: { + name: this.name, + type: 'playwright', + }, + test_run: { + id: this.runId, + }, + event: { + action: ScoutReportEventAction.ERROR, + error: { + message: error.message ? stripANSI(error.message) : undefined, + stack_trace: error.stack ? stripANSI(error.stack) : undefined, + }, + }, + }); + } +} + +// eslint-disable-next-line import/no-default-export +export default ScoutPlaywrightReporter; diff --git a/packages/kbn-scout-reporting/src/reporting/report/event.ts b/packages/kbn-scout-reporting/src/reporting/report/event.ts new file mode 100644 index 0000000000000..1f6f8251f3b60 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/report/event.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { BuildkiteMetadata, HostMetadata } from '../../datasources'; + +/** + * Scout reporter event type + */ +export enum ScoutReportEventAction { + RUN_BEGIN = 'run-begin', + RUN_END = 'run-end', + TEST_BEGIN = 'test-begin', + TEST_END = 'test-end', + TEST_STEP_BEGIN = 'test-step-begin', + TEST_STEP_END = 'test-step-end', + ERROR = 'error', +} + +/** + * Scout report event info + */ +export interface ScoutReportEventInfo { + action: ScoutReportEventAction; + outcome?: 'failure' | 'success' | 'unknown'; + error?: { + message?: string; + id?: string; + code?: string; + stack_trace?: string; + type?: string; + }; +} + +/** + * Scout reporter info + */ +export interface ScoutReporterInfo { + name: string; + type: 'jest' | 'ftr' | 'playwright'; +} + +/** + * Scout test run info + */ +export interface ScoutTestRunInfo { + id: string; + status?: string; + duration?: number; +} + +/** + * Scout suite info + */ +export interface ScoutSuiteInfo { + title: string; + type: string; +} + +/** + * Scout test info + */ +export interface ScoutTestInfo { + id: string; + title: string; + tags: string[]; + annotations?: Array<{ + type: string; + description?: string; + }>; + expected_status?: string; + duration?: number; + status?: string; + step?: { + title: string; + category?: string; + duration?: number; + }; +} + +/** + * Scout file info + */ +export interface ScoutFileInfo { + path: string; + owner: string | string[]; +} + +/** + * Document that records an event to be logged by the Scout reporter + */ +export interface ScoutReportEvent { + '@timestamp'?: Date; + buildkite?: BuildkiteMetadata; + host?: HostMetadata; + event: ScoutReportEventInfo; + file?: ScoutFileInfo; + labels?: { [id: string]: string }; + reporter: ScoutReporterInfo; + test_run: ScoutTestRunInfo; + suite?: ScoutSuiteInfo; + test?: ScoutTestInfo; +} diff --git a/packages/kbn-scout-reporting/src/reporting/report/index.ts b/packages/kbn-scout-reporting/src/reporting/report/index.ts new file mode 100644 index 0000000000000..b678463c185f9 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/report/index.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +// eslint-disable-next-line max-classes-per-file +import os from 'node:os'; +import path from 'node:path'; +import fs from 'node:fs'; +import { ToolingLog } from '@kbn/tooling-log'; +import { ScoutReportEvent } from './event'; + +/** + * Generic error raised by a Scout report + */ +export class ScoutReportError extends Error {} + +/** + * + */ +export class ScoutReport { + log: ToolingLog; + workDir: string; + concluded = false; + + constructor(log?: ToolingLog) { + this.log = log || new ToolingLog(); + this.workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scout-report-')); + } + + public get eventLogPath(): string { + return path.join(this.workDir, `event-log.ndjson`); + } + + private raiseIfConcluded(additionalInfo?: string) { + if (this.concluded) { + let message = `Report at ${this.workDir} was concluded`; + + if (additionalInfo) { + message += `: ${additionalInfo}`; + } + + throw new ScoutReportError(message); + } + } + + /** + * Logs an event to be processed by this reporter + * + * @param event {ScoutReportEvent} - Event to record + */ + logEvent(event: ScoutReportEvent) { + this.raiseIfConcluded('logging new events is no longer allowed'); + + if (event['@timestamp'] === undefined) { + event['@timestamp'] = new Date(); + } + + fs.appendFileSync(this.eventLogPath, JSON.stringify(event) + '\n'); + } + + /** + * Save the report to a non-ephemeral location + * + * @param destination - Full path to the save location. Must not exist. + */ + save(destination: string) { + this.raiseIfConcluded('nothing to save because workdir has been cleared'); + + if (fs.existsSync(destination)) { + throw new ScoutReportError(`Save destination path '${destination}' already exists`); + } + + // Create the destination directory + this.log.info(`Saving Scout report to ${destination}`); + fs.mkdirSync(destination, { recursive: true }); + + // Copy the workdir data to the destination + fs.cpSync(this.workDir, destination, { recursive: true }); + } + + /** + * Call this when you're done adding information to this report. + * + * ⚠️**This will delete all the contents of the report's working directory** + */ + conclude() { + // Remove the working directory + this.log.info(`Removing Scout report working directory ${this.workDir}`); + fs.rmSync(this.workDir, { recursive: true, force: true }); + + // Mark this report as concluded + this.concluded = true; + this.log.success('Scout report has concluded.'); + } +} + +export * from './event'; +export * from './persistence'; diff --git a/packages/kbn-scout-reporting/src/reporting/report/persistence/component_templates.ts b/packages/kbn-scout-reporting/src/reporting/report/persistence/component_templates.ts new file mode 100644 index 0000000000000..bfc578825b98c --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/report/persistence/component_templates.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { + buildkiteProperties, + reporterProperties, + testRunProperties, + suiteProperties, + testProperties, +} from './mappings'; + +export const buildkiteMappings: ClusterPutComponentTemplateRequest = { + name: 'scout-test-event.mappings.buildkite', + version: 1, + template: { + mappings: { + properties: { + buildkite: { + type: 'object', + properties: buildkiteProperties, + }, + }, + }, + }, +}; + +export const reporterMappings: ClusterPutComponentTemplateRequest = { + name: 'scout-test-event.mappings.reporter', + version: 1, + template: { + mappings: { + properties: { + reporter: { + type: 'object', + properties: reporterProperties, + }, + }, + }, + }, +}; + +export const testRunMappings: ClusterPutComponentTemplateRequest = { + name: 'scout-test-event.mappings.test-run', + version: 1, + template: { + mappings: { + properties: { + test_run: { + type: 'object', + properties: testRunProperties, + }, + }, + }, + }, +}; + +export const suiteMappings: ClusterPutComponentTemplateRequest = { + name: 'scout-test-event.mappings.suite', + version: 1, + template: { + mappings: { + properties: { + suite: { + type: 'object', + properties: suiteProperties, + }, + }, + }, + }, +}; + +export const testMappings: ClusterPutComponentTemplateRequest = { + name: 'scout-test-event.mappings.test', + version: 1, + template: { + mappings: { + properties: { + test: { + type: 'object', + properties: testProperties, + }, + }, + }, + }, +}; diff --git a/packages/kbn-scout-reporting/src/reporting/report/persistence/index.ts b/packages/kbn-scout-reporting/src/reporting/report/persistence/index.ts new file mode 100644 index 0000000000000..cc95e3c4d94d4 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/report/persistence/index.ts @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import readline from 'node:readline'; +import { ToolingLog } from '@kbn/tooling-log'; +import { Client as ESClient } from '@elastic/elasticsearch'; +import { SCOUT_TEST_EVENTS_DATA_STREAM_NAME } from '@kbn/scout-info'; +import { ScoutReportEvent } from '../event'; +import * as componentTemplates from './component_templates'; +import * as indexTemplates from './index_templates'; + +export class ScoutReportDataStream { + private log: ToolingLog; + + constructor(private es: ESClient, log?: ToolingLog) { + this.log = log || new ToolingLog(); + } + + async exists() { + return await this.es.indices.exists({ index: SCOUT_TEST_EVENTS_DATA_STREAM_NAME }); + } + + async initialize() { + await this.setupComponentTemplates(); + await this.setupIndexTemplate(); + + if (await this.exists()) { + return; + } + + this.log.info(`Creating data stream '${SCOUT_TEST_EVENTS_DATA_STREAM_NAME}'`); + await this.es.indices.createDataStream({ + name: SCOUT_TEST_EVENTS_DATA_STREAM_NAME, + }); + } + + async setupComponentTemplates() { + for (const template of [ + componentTemplates.buildkiteMappings, + componentTemplates.reporterMappings, + componentTemplates.testRunMappings, + componentTemplates.suiteMappings, + componentTemplates.testMappings, + ]) { + const templateExists = await this.es.cluster.existsComponentTemplate({ name: template.name }); + if (!templateExists) { + this.log.info(`Creating component template '${template.name}'`); + await this.es.cluster.putComponentTemplate(template); + continue; + } + + // Template exists but might need to be updated + const newTemplateVersion = template.version || 0; + const existingTemplateVersion = + (await this.es.cluster.getComponentTemplate({ name: template.name })).component_templates[0] + .component_template.version || 0; + + if (existingTemplateVersion >= newTemplateVersion) { + this.log.info(`Component template '${template.name} exists and is up to date.`); + continue; + } + + this.log.info( + `Updating component template '${template.name}' (version ${existingTemplateVersion} -> ${newTemplateVersion})` + ); + await this.es.cluster.putComponentTemplate(template); + } + } + + async setupIndexTemplate() { + const template = indexTemplates.testEvents; + const templateExists: boolean = await this.es.indices.existsIndexTemplate({ + name: template.name, + }); + + if (!templateExists) { + this.log.info(`Creating index template '${template.name}'`); + await this.es.indices.putIndexTemplate(template); + return; + } + + // Template exists but might need to be updated + const newTemplateVersion = template.version || 0; + const existingTemplateVersion = + (await this.es.indices.getIndexTemplate({ name: template.name })).index_templates[0] + .index_template.version || 0; + + if (existingTemplateVersion >= newTemplateVersion) { + this.log.info(`Index template '${template.name} exists and is up to date.`); + return; + } + + this.log.info( + `Updating index template '${template.name}' (version ${existingTemplateVersion} -> ${newTemplateVersion})` + ); + await this.es.indices.putIndexTemplate(template); + } + + async addEvent(event: ScoutReportEvent) { + await this.es.index({ index: SCOUT_TEST_EVENTS_DATA_STREAM_NAME, document: event }); + } + + async addEventsFromFile(eventLogPath: string) { + // Make the given event log path absolute + eventLogPath = path.resolve(eventLogPath); + + const events = async function* () { + const lineReader = readline.createInterface({ + input: fs.createReadStream(eventLogPath), + crlfDelay: Infinity, + }); + + for await (const line of lineReader) { + yield line; + } + }; + + this.log.info( + `Uploading events from file ${eventLogPath} to data stream '${SCOUT_TEST_EVENTS_DATA_STREAM_NAME}'` + ); + + const stats = await this.es.helpers.bulk({ + datasource: events(), + onDocument: () => { + return { create: { _index: SCOUT_TEST_EVENTS_DATA_STREAM_NAME } }; + }, + }); + + this.log.info(`Uploaded ${stats.total} events in ${stats.time / 1000}s.`); + + if (stats.failed > 0) { + this.log.warning(`Failed to upload ${stats.failed} events`); + } + } +} diff --git a/packages/kbn-scout-reporting/src/reporting/report/persistence/index_templates.ts b/packages/kbn-scout-reporting/src/reporting/report/persistence/index_templates.ts new file mode 100644 index 0000000000000..3ad0a5809009b --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/report/persistence/index_templates.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { SCOUT_TEST_EVENTS_TEMPLATE_NAME, SCOUT_TEST_EVENTS_INDEX_PATTERN } from '@kbn/scout-info'; +import * as componentTemplates from './component_templates'; + +export const testEvents: IndicesPutIndexTemplateRequest = { + name: SCOUT_TEST_EVENTS_TEMPLATE_NAME, + version: 1, + data_stream: {}, + index_patterns: SCOUT_TEST_EVENTS_INDEX_PATTERN, + composed_of: [ + 'ecs@mappings', + componentTemplates.buildkiteMappings.name, + componentTemplates.reporterMappings.name, + componentTemplates.testRunMappings.name, + componentTemplates.suiteMappings.name, + componentTemplates.testMappings.name, + ], +}; diff --git a/packages/kbn-scout-reporting/src/reporting/report/persistence/mappings.ts b/packages/kbn-scout-reporting/src/reporting/report/persistence/mappings.ts new file mode 100644 index 0000000000000..083b3a87dac3b --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/report/persistence/mappings.ts @@ -0,0 +1,172 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +import { PropertyName, MappingProperty } from '@elastic/elasticsearch/lib/api/types'; + +export const buildkiteProperties: Record = { + branch: { + type: 'keyword', + }, + commit: { + type: 'wildcard', + }, + job_id: { + type: 'wildcard', + }, + message: { + type: 'text', + }, + build: { + type: 'object', + properties: { + id: { + type: 'wildcard', + }, + number: { + type: 'integer', + }, + url: { + type: 'wildcard', + }, + }, + }, + pipeline: { + type: 'object', + properties: { + id: { + type: 'wildcard', + }, + name: { + type: 'text', + }, + slug: { + type: 'wildcard', + }, + }, + }, + agent: { + type: 'object', + properties: { + name: { + type: 'wildcard', + }, + }, + }, + group: { + type: 'object', + properties: { + id: { + type: 'wildcard', + }, + key: { + type: 'wildcard', + }, + label: { + type: 'keyword', + }, + }, + }, + step: { + type: 'object', + properties: { + id: { + type: 'wildcard', + }, + key: { + type: 'wildcard', + }, + label: { + type: 'keyword', + }, + }, + }, + command: { + type: 'wildcard', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, +}; + +export const reporterProperties: Record = { + name: { + type: 'text', + }, + type: { + type: 'keyword', + }, +}; + +export const testRunProperties: Record = { + id: { + type: 'wildcard', + }, + status: { + type: 'keyword', + }, + duration: { + type: 'long', + }, +}; + +export const suiteProperties: Record = { + title: { + type: 'text', + }, + type: { + type: 'keyword', + }, +}; + +export const testProperties: Record = { + id: { + type: 'wildcard', + }, + title: { + type: 'text', + }, + tags: { + type: 'keyword', + }, + annotations: { + type: 'object', + properties: { + type: { + type: 'keyword', + }, + description: { + type: 'text', + }, + }, + }, + expected_status: { + type: 'keyword', + }, + duration: { + type: 'long', + }, + status: { + type: 'keyword', + }, + step: { + type: 'object', + properties: { + title: { + type: 'text', + }, + category: { + type: 'keyword', + }, + duration: { + type: 'long', + }, + }, + }, +}; diff --git a/packages/kbn-scout-reporting/tsconfig.json b/packages/kbn-scout-reporting/tsconfig.json new file mode 100644 index 0000000000000..30b5e3fca4e6c --- /dev/null +++ b/packages/kbn-scout-reporting/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/tooling-log", + "@kbn/dev-cli-runner", + "@kbn/dev-cli-errors", + "@kbn/scout-info", + "@kbn/repo-info", + "@kbn/code-owners", + ] +} diff --git a/packages/kbn-scout/index.ts b/packages/kbn-scout/index.ts index 34a906fcf755d..5cb95662f4228 100644 --- a/packages/kbn-scout/index.ts +++ b/packages/kbn-scout/index.ts @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export { startServersCli, runTestsCli } from './src/cli'; +export * as cli from './src/cli'; export { expect, test, createPlaywrightConfig, createLazyPageObject } from './src/playwright'; export type { ScoutPage, diff --git a/packages/kbn-scout/src/cli/index.ts b/packages/kbn-scout/src/cli/index.ts index f30b384f351d9..c26a255bd0340 100644 --- a/packages/kbn-scout/src/cli/index.ts +++ b/packages/kbn-scout/src/cli/index.ts @@ -6,6 +6,16 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ +import { RunWithCommands } from '@kbn/dev-cli-runner'; +import { cli as reportingCLI } from '@kbn/scout-reporting'; +import { startServer } from './start_server'; +import { runTests } from './run_tests'; -export { runTestsCli } from './run_tests_cli'; -export { startServersCli } from './start_servers_cli'; +export async function run() { + await new RunWithCommands( + { + description: 'Scout CLI', + }, + [startServer, runTests, reportingCLI.initializeReportDatastream, reportingCLI.uploadEvents] + ).execute(); +} diff --git a/packages/kbn-scout/src/cli/run_tests.ts b/packages/kbn-scout/src/cli/run_tests.ts new file mode 100644 index 0000000000000..80a235cc1b721 --- /dev/null +++ b/packages/kbn-scout/src/cli/run_tests.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { Command } from '@kbn/dev-cli-runner'; +import { initLogsDir } from '@kbn/test'; +import { TEST_FLAG_OPTIONS } from '../playwright/runner'; +import { parseTestFlags, runTests as runTestsFn } from '../playwright/runner'; + +/** + * Start servers and run the tests + */ +export const runTests: Command = { + name: 'run-tests', + description: ` + Run a Scout Playwright config. + + Note: + This also handles server starts. Make sure a Scout test server is not already running before invoking this command. + + Common usage: + node scripts/scout run-tests --stateful --config + node scripts/scout run-tests --serverless=es --headed --config + `, + flags: TEST_FLAG_OPTIONS, + run: async ({ flagsReader, log }) => { + const options = await parseTestFlags(flagsReader); + + if (options.logsDir) { + await initLogsDir(log, options.logsDir); + } + + await runTestsFn(log, options); + }, +}; diff --git a/packages/kbn-scout/src/cli/run_tests_cli.ts b/packages/kbn-scout/src/cli/run_tests_cli.ts deleted file mode 100644 index 913f09a310a63..0000000000000 --- a/packages/kbn-scout/src/cli/run_tests_cli.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { run } from '@kbn/dev-cli-runner'; -import { initLogsDir } from '@kbn/test'; -import { TEST_FLAG_OPTIONS, parseTestFlags, runTests } from '../playwright/runner'; - -/** - * Start servers and run the tests - */ -export function runTestsCli() { - run( - async ({ flagsReader, log }) => { - const options = await parseTestFlags(flagsReader); - - if (options.logsDir) { - initLogsDir(log, options.logsDir); - } - - await runTests(log, options); - }, - { - description: `Run Scout UI Tests`, - usage: ` - Usage: - node scripts/scout_test --help - node scripts/scout_test --stateful --config - node scripts/scout_test --serverless=es --headed --config - `, - flags: TEST_FLAG_OPTIONS, - } - ); -} diff --git a/packages/kbn-scout/src/cli/start_servers_cli.ts b/packages/kbn-scout/src/cli/start_server.ts similarity index 59% rename from packages/kbn-scout/src/cli/start_servers_cli.ts rename to packages/kbn-scout/src/cli/start_server.ts index 3006f87f5ba57..ff6f8f164626c 100644 --- a/packages/kbn-scout/src/cli/start_servers_cli.ts +++ b/packages/kbn-scout/src/cli/start_server.ts @@ -7,8 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { run } from '@kbn/dev-cli-runner'; - +import { Command } from '@kbn/dev-cli-runner'; import { initLogsDir } from '@kbn/test'; import { startServers, parseServerFlags, SERVER_FLAG_OPTIONS } from '../servers'; @@ -16,19 +15,16 @@ import { startServers, parseServerFlags, SERVER_FLAG_OPTIONS } from '../servers' /** * Start servers */ -export function startServersCli() { - run( - async ({ flagsReader: flags, log }) => { - const options = parseServerFlags(flags); - - if (options.logsDir) { - initLogsDir(log, options.logsDir); - } +export const startServer: Command = { + name: 'start-server', + description: 'Start Elasticsearch & Kibana for testing purposes', + flags: SERVER_FLAG_OPTIONS, + run: async ({ flagsReader, log }) => { + const options = parseServerFlags(flagsReader); - await startServers(log, options); - }, - { - flags: SERVER_FLAG_OPTIONS, + if (options.logsDir) { + await initLogsDir(log, options.logsDir); } - ); -} + await startServers(log, options); + }, +}; diff --git a/packages/kbn-scout/src/config/utils.ts b/packages/kbn-scout/src/config/utils.ts index 61bdc1b7b81ac..d15e0e094b2db 100644 --- a/packages/kbn-scout/src/config/utils.ts +++ b/packages/kbn-scout/src/config/utils.ts @@ -12,7 +12,7 @@ import getopts from 'getopts'; import path from 'path'; import { ToolingLog } from '@kbn/tooling-log'; import { ServerlessProjectType } from '@kbn/es'; -import { REPO_ROOT } from '@kbn/repo-info'; +import { SCOUT_SERVERS_ROOT } from '@kbn/scout-info'; import { CliSupportedServerModes, ScoutServerConfig } from '../types'; import { getConfigFilePath } from './get_config_file'; import { loadConfig } from './loader/config_load'; @@ -30,15 +30,14 @@ export const formatCurrentDate = () => { }; const saveTestServersConfigOnDisk = (testServersConfig: ScoutServerConfig, log: ToolingLog) => { - const configDirPath = path.resolve(REPO_ROOT, '.scout', 'servers'); - const configFilePath = path.join(configDirPath, `local.json`); + const configFilePath = path.join(SCOUT_SERVERS_ROOT, `local.json`); try { const jsonData = JSON.stringify(testServersConfig, null, 2); - if (!Fs.existsSync(configDirPath)) { - log.debug(`scout: creating configuration directory: ${configDirPath}`); - Fs.mkdirSync(configDirPath, { recursive: true }); + if (!Fs.existsSync(SCOUT_SERVERS_ROOT)) { + log.debug(`scout: creating configuration directory: ${SCOUT_SERVERS_ROOT}`); + Fs.mkdirSync(SCOUT_SERVERS_ROOT, { recursive: true }); } Fs.writeFileSync(configFilePath, jsonData, 'utf-8'); diff --git a/packages/kbn-scout/src/playwright/config/index.ts b/packages/kbn-scout/src/playwright/config/index.ts index 62f5261c08e25..cb1e371cb43e7 100644 --- a/packages/kbn-scout/src/playwright/config/index.ts +++ b/packages/kbn-scout/src/playwright/config/index.ts @@ -8,8 +8,8 @@ */ import { defineConfig, PlaywrightTestConfig, devices } from '@playwright/test'; -import * as Path from 'path'; -import { REPO_ROOT } from '@kbn/repo-info'; +import { scoutPlaywrightReporter } from '@kbn/scout-reporting'; +import { SCOUT_SERVERS_ROOT } from '@kbn/scout-info'; import { ScoutPlaywrightOptions, ScoutTestOptions, VALID_CONFIG_MARKER } from '../types'; export function createPlaywrightConfig(options: ScoutPlaywrightOptions): PlaywrightTestConfig { @@ -27,10 +27,11 @@ export function createPlaywrightConfig(options: ScoutPlaywrightOptions): Playwri reporter: [ ['html', { outputFolder: './output/reports', open: 'never' }], // HTML report configuration ['json', { outputFile: './output/reports/test-results.json' }], // JSON report + scoutPlaywrightReporter({ name: 'scout-playwright' }), // Scout report ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { - serversConfigDir: Path.resolve(REPO_ROOT, '.scout', 'servers'), + serversConfigDir: SCOUT_SERVERS_ROOT, [VALID_CONFIG_MARKER]: true, /* Base URL to use in actions like `await page.goto('/')`. */ // baseURL: 'http://127.0.0.1:3000', diff --git a/packages/kbn-scout/src/types/config.d.ts b/packages/kbn-scout/src/types/config.d.ts index 14cd27b47fde2..2f0c3a764d65f 100644 --- a/packages/kbn-scout/src/types/config.d.ts +++ b/packages/kbn-scout/src/types/config.d.ts @@ -29,6 +29,6 @@ export interface ScoutLoaderConfig { buildArgs?: string[]; sourceArgs?: string[]; serverArgs: string[]; - useDedicatedTastRunner?: boolean; + useDedicatedTestRunner?: boolean; }; } diff --git a/packages/kbn-scout/tsconfig.json b/packages/kbn-scout/tsconfig.json index 35d74c6437618..4be38ce4c80fd 100644 --- a/packages/kbn-scout/tsconfig.json +++ b/packages/kbn-scout/tsconfig.json @@ -27,5 +27,7 @@ "@kbn/mock-idp-utils", "@kbn/test-suites-xpack", "@kbn/test-subj-selector", + "@kbn/scout-info", + "@kbn/scout-reporting" ] } diff --git a/packages/kbn-test/src/functional_test_runner/fake_mocha_types.ts b/packages/kbn-test/src/functional_test_runner/fake_mocha_types.ts index c87839dfcf481..ac7db945e9492 100644 --- a/packages/kbn-test/src/functional_test_runner/fake_mocha_types.ts +++ b/packages/kbn-test/src/functional_test_runner/fake_mocha_types.ts @@ -25,6 +25,7 @@ export interface Suite extends Runnable { suites: Suite[]; tests: Test[]; title: string; + fullTitle(): string; file: string; parent?: Suite; eachTest: (cb: (test: Test) => void) => void; @@ -39,6 +40,7 @@ export interface Test extends Runnable { parent?: Suite; isPassed: () => boolean; pending?: boolean; + err?: Error; } export interface Runnable { @@ -51,10 +53,22 @@ export interface Runnable { parent?: Suite; } +interface Stats { + suites: number; + tests: number; + passes: number; + pending: number; + failures: number; + start?: Date; + end?: Date; + duration?: number; +} + export interface Runner extends EventEmitter { abort(): void; failures: any[]; uncaught: (error: Error) => void; + stats?: Stats; } export interface Mocha { diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index bb240a0416479..1039b53064cfb 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -178,6 +178,12 @@ export const schema = Joi.object() }) .default(), + scoutReporter: Joi.object() + .keys({ + enabled: Joi.boolean().default(process.env.ENABLE_SCOUT_REPORTER || false), + }) + .default(), + users: Joi.object().pattern( ID_PATTERN, Joi.object() diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js index 42fbc6a4f7386..0ea1792266274 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js @@ -20,6 +20,7 @@ import * as symbols from './symbols'; import { ms } from './ms'; import { writeEpilogue } from './write_epilogue'; import { setupCiStatsFtrTestGroupReporter } from './ci_stats_ftr_reporter'; +import { ScoutFTRReporter } from './scout_ftr_reporter'; export function MochaReporterProvider({ getService }) { const log = getService('log'); @@ -65,6 +66,10 @@ export function MochaReporterProvider({ getService }) { }); } } + + if (config.get('scoutReporter.enabled')) { + new ScoutFTRReporter(runner); + } } onStart = () => { diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/scout_ftr_reporter.ts b/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/scout_ftr_reporter.ts new file mode 100644 index 0000000000000..4ffef48ec6443 --- /dev/null +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/scout_ftr_reporter.ts @@ -0,0 +1,203 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import path from 'node:path'; +import { ToolingLog } from '@kbn/tooling-log'; +import { SCOUT_REPORT_OUTPUT_ROOT } from '@kbn/scout-info'; +import { REPO_ROOT } from '@kbn/repo-info'; +import { + generateTestRunId, + getTestIDForTitle, + ScoutReport, + ScoutReportEventAction, + datasources, +} from '@kbn/scout-reporting'; +import { getCodeOwnersForFile, getPathsWithOwnersReversed, PathWithOwners } from '@kbn/code-owners'; +import { Runner, Test } from '../../../fake_mocha_types'; + +/** + * Configuration options for the Scout Mocha reporter + */ +export interface ScoutFTRReporterOptions { + name?: string; + outputPath?: string; +} + +/** + * Scout Mocha reporter + */ +export class ScoutFTRReporter { + readonly log: ToolingLog; + readonly name: string; + readonly runId: string; + private report: ScoutReport; + private readonly pathsWithOwners: PathWithOwners[]; + + constructor(private runner: Runner, private reporterOptions: ScoutFTRReporterOptions = {}) { + this.log = new ToolingLog({ + level: 'info', + writeTo: process.stdout, + }); + + this.name = this.reporterOptions.name || 'ftr'; + this.runId = generateTestRunId(); + this.log.info(`Scout test run ID: ${this.runId}`); + + this.report = new ScoutReport(this.log); + this.pathsWithOwners = getPathsWithOwnersReversed(); + + // Register event listeners + for (const [eventName, listener] of Object.entries({ + start: this.onRunStart, + end: this.onRunEnd, + test: this.onTestStart, + 'test end': this.onTestEnd, + })) { + runner.on(eventName, listener); + } + } + + private getFileOwners(filePath: string): string[] { + const concatenatedOwners = getCodeOwnersForFile(filePath, this.pathsWithOwners); + + if (concatenatedOwners === undefined) { + return []; + } + + return concatenatedOwners + .replace(/#.+$/, '') + .split(',') + .filter((value) => value.length > 0); + } + + /** + * Root path of this reporter's output + */ + public get reportRootPath(): string { + const outputPath = this.reporterOptions.outputPath || SCOUT_REPORT_OUTPUT_ROOT; + return path.join(outputPath, `scout-ftr-${this.runId}`); + } + + onRunStart = () => { + /** + * Root suite execution began (all files have been parsed and hooks/tests are ready for execution) + */ + this.report.logEvent({ + ...datasources.environmentMetadata, + reporter: { + name: this.name, + type: 'ftr', + }, + test_run: { + id: this.runId, + }, + event: { + action: ScoutReportEventAction.RUN_BEGIN, + }, + }); + }; + + onTestStart = (test: Test) => { + /** + * Test execution started + */ + this.report.logEvent({ + ...datasources.environmentMetadata, + reporter: { + name: this.name, + type: 'ftr', + }, + test_run: { + id: this.runId, + }, + suite: { + title: test.parent?.fullTitle() || 'unknown', + type: test.parent?.root ? 'root' : 'suite', + }, + test: { + id: getTestIDForTitle(test.fullTitle()), + title: test.title, + tags: [], + }, + event: { + action: ScoutReportEventAction.TEST_BEGIN, + }, + file: { + path: test.file ? path.relative(REPO_ROOT, test.file) : 'unknown', + owner: test.file ? this.getFileOwners(path.relative(REPO_ROOT, test.file)) : 'unknown', + }, + }); + }; + + onTestEnd = (test: Test) => { + /** + * Test execution ended + */ + this.report.logEvent({ + ...datasources.environmentMetadata, + reporter: { + name: this.name, + type: 'ftr', + }, + test_run: { + id: this.runId, + }, + suite: { + title: test.parent?.fullTitle() || 'unknown', + type: test.parent?.root ? 'root' : 'suite', + }, + test: { + id: getTestIDForTitle(test.fullTitle()), + title: test.title, + tags: [], + status: test.isPending() ? 'skipped' : test.isPassed() ? 'passed' : 'failed', + duration: test.duration, + }, + event: { + action: ScoutReportEventAction.TEST_END, + error: { + message: test.err?.message, + stack_trace: test.err?.stack, + }, + }, + file: { + path: test.file ? path.relative(REPO_ROOT, test.file) : 'unknown', + owner: test.file ? this.getFileOwners(path.relative(REPO_ROOT, test.file)) : 'unknown', + }, + }); + }; + + onRunEnd = () => { + /** + * Root suite execution has ended + */ + this.report.logEvent({ + ...datasources.environmentMetadata, + reporter: { + name: this.name, + type: 'ftr', + }, + test_run: { + id: this.runId, + status: this.runner.stats?.failures === 0 ? 'passed' : 'failed', + duration: this.runner.stats?.duration || 0, + }, + event: { + action: ScoutReportEventAction.RUN_END, + }, + }); + + // Save & conclude the report + try { + this.report.save(this.reportRootPath); + } finally { + this.report.conclude(); + } + }; +} diff --git a/packages/kbn-test/tsconfig.json b/packages/kbn-test/tsconfig.json index af4473869eb75..b4cd88e05db5a 100644 --- a/packages/kbn-test/tsconfig.json +++ b/packages/kbn-test/tsconfig.json @@ -37,6 +37,8 @@ "@kbn/core-saved-objects-api-server", "@kbn/mock-idp-utils", "@kbn/code-owners", + "@kbn/scout-reporting", + "@kbn/scout-info", "@kbn/react-mute-legacy-root-warning", ] } diff --git a/scripts/scout_start_servers.js b/scripts/scout.js old mode 100644 new mode 100755 similarity index 89% rename from scripts/scout_start_servers.js rename to scripts/scout.js index b93ec0e456454..dce9e52c870f2 --- a/scripts/scout_start_servers.js +++ b/scripts/scout.js @@ -1,3 +1,5 @@ +#!/usr/bin/env node + /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the "Elastic License @@ -8,4 +10,4 @@ */ require('../src/setup_node_env'); -require('@kbn/scout').startServersCli(); +void require('@kbn/scout').cli.run(); diff --git a/tsconfig.base.json b/tsconfig.base.json index 1c05a23678871..7d1f6ea696819 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1548,6 +1548,10 @@ "@kbn/saved-search-plugin/*": ["src/plugins/saved_search/*"], "@kbn/scout": ["packages/kbn-scout"], "@kbn/scout/*": ["packages/kbn-scout/*"], + "@kbn/scout-info": ["packages/kbn-scout-info"], + "@kbn/scout-info/*": ["packages/kbn-scout-info/*"], + "@kbn/scout-reporting": ["packages/kbn-scout-reporting"], + "@kbn/scout-reporting/*": ["packages/kbn-scout-reporting/*"], "@kbn/screenshot-mode-example-plugin": ["examples/screenshot_mode_example"], "@kbn/screenshot-mode-example-plugin/*": ["examples/screenshot_mode_example/*"], "@kbn/screenshot-mode-plugin": ["src/plugins/screenshot_mode"], diff --git a/yarn.lock b/yarn.lock index 703518b9f0afe..df4c2f65299f2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6908,6 +6908,14 @@ version "0.0.0" uid "" +"@kbn/scout-info@link:packages/kbn-scout-info": + version "0.0.0" + uid "" + +"@kbn/scout-reporting@link:packages/kbn-scout-reporting": + version "0.0.0" + uid "" + "@kbn/scout@link:packages/kbn-scout": version "0.0.0" uid ""