Custom event-oriented test reporter & persistence

packages/kbn-scout/index.ts

@@ -7,7 +7,8 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-export { startServersCli, runTestsCli } from './src/cli';
+export * as cli from './src/cli';
+export * from './src/reporting';
 export { expect, test, createPlaywrightConfig, createLazyPageObject } from './src/playwright';
 export type {
   ScoutPage,

packages/kbn-scout/src/cli/common.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Client as ESClient, ClientOptions as ESClientOptions } from '@elastic/elasticsearch';
+import { ToolingLog } from '@kbn/tooling-log';
+import { createFailError } from '@kbn/dev-cli-errors';
+
+/**
+ * Get an Elasticsearch client for which connectivity has been validated
+ *
+ * @param options Elasticsearch client options
+ * @param log Logger instance
+ * @throws FailError if cluster information cannot be read from the target Elasticsearch instance
+ */
+export async function getValidatedESClient(
+  options: ESClientOptions,
+  log: ToolingLog
+): Promise<ESClient> {
+  const es = new ESClient(options);
+
+  await es.info().then(
+    (esInfo) => {
+      log.info(`Connected to Elasticsearch node '${esInfo.name}'`);
+    },
+    (err) => {
+      throw createFailError(`Failed to connect to Elasticsearch\n${err}`);
+    }
+  );
+
+  return es;
+}

packages/kbn-scout/src/cli/index.ts

@@ -6,6 +6,17 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
+import { RunWithCommands } from '@kbn/dev-cli-runner';
+import { initializeReportDatastream } from './initialize-report-datastream';
+import { startServer } from './start_server';
+import { runTests } from './run_tests';
+import { uploadEvents } from './upload_events';
 
-export { runTestsCli } from './run_tests_cli';
-export { startServersCli } from './start_servers_cli';
+export async function run() {
+  await new RunWithCommands(
+    {
+      description: 'Scout CLI',
+    },
+    [startServer, runTests, initializeReportDatastream, uploadEvents]
+  ).execute();
+}

packages/kbn-scout/src/cli/initialize-report-datastream.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Command } from '@kbn/dev-cli-runner';
+import { ScoutReportDataStream } from '../reporting/report';
+import { getValidatedESClient } from './common';
+
+export const initializeReportDatastream: Command<void> = {
+  name: 'initialize-report-datastream',
+  description: 'Initialize a Scout report datastream in Elasticsearch',
+  flags: {
+    string: ['esURL', 'esAPIKey'],
+    boolean: ['verifyTLSCerts'],
+    default: {
+      esURL: process.env.ES_URL,
+      esAPIKey: process.env.ES_API_KEY,
+    },
+    help: `
+    --esURL           (required)  Elasticsearch URL [env: ES_URL]
+    --esAPIKey        (required)  Elasticsearch API Key [env: ES_API_KEY]
+    --verifyTLSCerts  (optional)  Verify TLS certificates
+    `,
+  },
+  run: async ({ flagsReader, log }) => {
+    const esURL = flagsReader.requiredString('esURL');
+    const esAPIKey = flagsReader.requiredString('esAPIKey');
+
+    // ES connection
+    log.info(`Connecting to Elasticsearch at ${esURL}`);
+    const es = await getValidatedESClient(
+      {
+        node: esURL,
+        auth: { apiKey: esAPIKey },
+        tls: {
+          rejectUnauthorized: flagsReader.boolean('verifyTLSCerts'),
+        },
+      },
+      log
+    );
+
+    // Initialize the report datastream
+    const reportDataStream = new ScoutReportDataStream(es, log);
+    await reportDataStream.initialize();
+
+    log.success('Scout report data stream initialized');
+  },
+};

packages/kbn-scout/src/cli/run_tests.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Command } from '@kbn/dev-cli-runner';
+import { initLogsDir } from '@kbn/test';
+import { TEST_FLAG_OPTIONS } from '../playwright/runner';
+import { parseTestFlags, runTests as runTestsFn } from '../playwright/runner';
+
+/**
+ * Start servers and run the tests
+ */
+export const runTests: Command<void> = {
+  name: 'run-tests',
+  description: `
+  Run Scout UI Tests
+
+  Common usage:
+  ./scripts/scout run-tests --stateful --config <playwright_config_path>
+  ./scripts/scout run-tests --serverless=es --headed --config <playwright_config_path>
+  `,
+  flags: TEST_FLAG_OPTIONS,
+  run: async ({ flagsReader, log }) => {
+    const options = await parseTestFlags(flagsReader);
+
+    if (options.logsDir) {
+      await initLogsDir(log, options.logsDir);
+    }
+
+    await runTestsFn(log, options);
+  },
+};

packages/kbn-scout/src/cli/start_servers_cli.ts → packages/kbn-scout/src/cli/start_server.ts

@@ -7,28 +7,24 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { run } from '@kbn/dev-cli-runner';
-
+import { Command } from '@kbn/dev-cli-runner';
 import { initLogsDir } from '@kbn/test';
 
 import { startServers, parseServerFlags, SERVER_FLAG_OPTIONS } from '../servers';
 
 /**
  * Start servers
  */
-export function startServersCli() {
-  run(
-    async ({ flagsReader: flags, log }) => {
-      const options = parseServerFlags(flags);
-
-      if (options.logsDir) {
-        initLogsDir(log, options.logsDir);
-      }
+export const startServer: Command<void> = {
+  name: 'start-server',
+  description: 'Start a Kibana instance for testing purposes',
+  flags: SERVER_FLAG_OPTIONS,
+  run: async ({ flagsReader, log }) => {
+    const options = parseServerFlags(flagsReader);
 
-      await startServers(log, options);
-    },
-    {
-      flags: SERVER_FLAG_OPTIONS,
+    if (options.logsDir) {
+      await initLogsDir(log, options.logsDir);
     }
-  );
-}
+    await startServers(log, options);
+  },
+};

packages/kbn-scout/src/cli/upload_events.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import fs from 'node:fs';
+import { Command } from '@kbn/dev-cli-runner';
+import { createFlagError } from '@kbn/dev-cli-errors';
+import { ScoutReportDataStream } from '../reporting/report';
+import { getValidatedESClient } from './common';
+
+export const uploadEvents: Command<void> = {
+  name: 'upload-events',
+  description: 'Upload events recorded by the Scout reporter to Elasticsearch',
+  flags: {
+    string: ['eventLogPath', 'esURL', 'esAPIKey'],
+    boolean: ['verifyTLSCerts'],
+    default: {
+      esURL: process.env.ES_URL,
+      esAPIKey: process.env.ES_API_KEY,
+    },
+    help: `
+    --eventLogPath    (required)  Path to the event log to upload
+    --esURL           (required)  Elasticsearch URL [env: ES_URL]
+    --esAPIKey        (required)  Elasticsearch API Key [env: ES_API_KEY]
+    --verifyTLSCerts  (optional)  Verify TLS certificates
+    `,
+  },
+  run: async ({ flagsReader, log }) => {
+    // Read & validate CLI options
+    const eventLogPath = flagsReader.requiredString('eventLogPath');
+
+    if (!fs.existsSync(eventLogPath)) {
+      throw createFlagError(`Event log path '${eventLogPath}' does not exist.`);
+    }
+
+    const esURL = flagsReader.requiredString('esURL');
+    const esAPIKey = flagsReader.requiredString('esAPIKey');
+
+    // ES connection
+    log.info(`Connecting to Elasticsearch at ${esURL}`);
+    const es = await getValidatedESClient(
+      {
+        node: esURL,
+        auth: { apiKey: esAPIKey },
+        tls: {
+          rejectUnauthorized: flagsReader.boolean('verifyTLSCerts'),
+        },
+      },
+      log
+    );
+
+    // Event log upload
+    const reportDataStream = new ScoutReportDataStream(es, log);
+    await reportDataStream.addEventsFromFile(eventLogPath);
+  },
+};

packages/kbn-scout/src/datasources/buildkite.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+/**
+ * Buildkite info
+ */
+export interface BuildkiteMetadata {
+  branch?: string;
+  commit?: string;
+  job_id?: string;
+  message?: string;
+  build?: {
+    id?: string;
+    number?: string;
+    url?: string;
+  };
+  pipeline?: {
+    id?: string;
+    name?: string;
+    slug?: string;
+  };
+  agent?: {
+    name?: string;
+  };
+  group?: {
+    id?: string;
+    key?: string;
+    label?: string;
+  };
+  step?: {
+    id?: string;
+    key?: string;
+    label?: string;
+  };
+  command?: string;
+}
+
+/**
+ * Buildkite information extracted from environment variables
+ *
+ * This object is empty if the process is not running in a Buildkite pipeline.
+ */
+export const buildkite: BuildkiteMetadata =
+  process.env.BUILDKITE === 'true'
+    ? {
+        branch: process.env.BUILDKITE_BRANCH,
+        commit: process.env.BUILDKITE_COMMIT,
+        job_id: process.env.BUILDKITE_JOB_ID,
+        message: process.env.BUILDKITE_MESSAGE,
+        build: {
+          id: process.env.BUILDKITE_BUILD_ID,
+          number: process.env.BUILDKITE_BUILD_NUMBER,
+          url: process.env.BUILDKITE_BUILD_URL,
+        },
+        pipeline: {
+          id: process.env.BUILDKITE_PIPELINE_ID,
+          name: process.env.BUILDKITE_PIPELINE_NAME,
+          slug: process.env.BUILDKITE_PIPELINE_SLUG,
+        },
+        agent: {
+          name: process.env.BUILDKITE_AGENT_NAME,
+        },
+        group: {
+          id: process.env.BUILDKITE_GROUP_ID,
+          key: process.env.BUILDKITE_GROUP_KEY,
+          label: process.env.BUILDKITE_GROUP_LABEL,
+        },
+        step: {
+          id: process.env.BUILDKITE_STEP_ID,
+          key: process.env.BUILDKITE_STEP_KEY,
+          label: process.env.BUILDKITE_LABEL,
+        },
+        command: process.env.BUILDKITE_COMMAND,
+      }
+    : {};