From 9b80e4513de67ac3de6c799768113129d7493870 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Mon, 13 Nov 2023 16:32:11 +0100 Subject: [PATCH 01/21] adding the script --- x-pack/test/security_solution_cypress/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/security_solution_cypress/package.json b/x-pack/test/security_solution_cypress/package.json index 0e34d7867d37a..e43f32a447575 100644 --- a/x-pack/test/security_solution_cypress/package.json +++ b/x-pack/test/security_solution_cypress/package.json @@ -28,6 +28,7 @@ "cypress:changed-specs-only:serverless": "yarn cypress:serverless --changed-specs-only --env burn=5", "cypress:burn:serverless": "yarn cypress:serverless --env burn=2", "cypress:qa:serverless": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider node ../../plugins/security_solution/scripts/start_cypress_parallel_serverless --config-file ../../test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts", + "cypress:open:qa:serverless": "yarn cypress:qa:serverless open", "cypress:run:qa:serverless": "yarn cypress:qa:serverless --spec './cypress/e2e/!(investigations|explore)/**/*.cy.ts'", "cypress:run:qa:serverless:investigations": "yarn cypress:qa:serverless --spec './cypress/e2e/investigations/**/*.cy.ts'", "cypress:run:qa:serverless:explore": "yarn cypress:qa:serverless --spec './cypress/e2e/explore/**/*.cy.ts'" From 9d12316e56e4d40c69d55c1142f3214bc5026064 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Mon, 13 Nov 2023 16:32:51 +0100 Subject: [PATCH 02/21] trying to delete the project when cypress is not executed anymore --- .../scripts/run_cypress/parallel_serverless.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index c00ec17116ca9..5cbe9daee11b8 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -125,6 +125,7 @@ async function deleteSecurityProject( apiKey: string ): Promise { try { + log.info(`Deleting the project`); await axios.delete(`${BASE_ENV_URL}/api/v1/serverless/projects/security/${projectId}`, { headers: { Authorization: `ApiKey ${apiKey}`, @@ -327,7 +328,7 @@ function waitForKibanaLogin(kbUrl: string, credentials: Credentials): Promise { run( - async () => { + async (context) => { log = new ToolingLog({ level: 'info', writeTo: process.stdout, @@ -474,6 +475,12 @@ ${JSON.stringify(cypressConfigFile, null, 2)} return process.exit(1); } + context.addCleanupTask(async () => { + await deleteSecurityProject(project.id, PROJECT_NAME, API_KEY).then(() => { + log.info('Clean up done'); + }); + }); + // Reset credentials for elastic user const credentials = await resetCredentials(project.id, id, API_KEY); From 0bdc4e72ae2b561d4d6b05959bbc989b24900691 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Mon, 13 Nov 2023 20:26:27 +0100 Subject: [PATCH 03/21] clean up task working solution --- .../scripts/run_cypress/parallel_serverless.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index 5cbe9daee11b8..bcd600aa9b632 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -24,6 +24,7 @@ import pRetry from 'p-retry'; import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; import { INITIAL_REST_VERSION } from '@kbn/data-views-plugin/server/constants'; +import { exec } from 'child_process'; import { renderSummaryTable } from './print_run'; import type { SecuritySolutionDescribeBlockFtrConfig } from './utils'; import { parseTestFileConfig, retrieveIntegrations } from './utils'; @@ -475,10 +476,9 @@ ${JSON.stringify(cypressConfigFile, null, 2)} return process.exit(1); } - context.addCleanupTask(async () => { - await deleteSecurityProject(project.id, PROJECT_NAME, API_KEY).then(() => { - log.info('Clean up done'); - }); + context.addCleanupTask(() => { + const command = `curl -X DELETE ${BASE_ENV_URL}/api/v1/serverless/projects/security/${project.id} -H "Authorization: ApiKey ${API_KEY}"`; + exec(command); }); // Reset credentials for elastic user From 9925a082f5eb2731cf9b84e186302404f796976e Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Mon, 13 Nov 2023 20:34:12 +0100 Subject: [PATCH 04/21] removes severlessQA tag from test --- .../cypress/e2e/explore/overview/overview.cy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts index 1205d2420b7ab..f857805462877 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts @@ -16,7 +16,7 @@ import { OVERVIEW_URL } from '../../../urls/navigation'; import { createTimeline, favoriteTimeline } from '../../../tasks/api_calls/timelines'; import { getTimeline } from '../../../objects/timeline'; -describe('Overview Page', { tags: ['@ess', '@serverless', '@serverlessQA'] }, () => { +describe('Overview Page', { tags: ['@ess', '@serverless'] }, () => { before(() => { cy.task('esArchiverLoad', { archiveName: 'overview' }); }); From dabe4820cf180f290635401f26ab1b927c485cf0 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Mon, 13 Nov 2023 20:34:31 +0100 Subject: [PATCH 05/21] updates tags section in readme --- x-pack/test/security_solution_cypress/cypress/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/README.md b/x-pack/test/security_solution_cypress/cypress/README.md index c85ba1bea305f..75a1177ad1e7d 100644 --- a/x-pack/test/security_solution_cypress/cypress/README.md +++ b/x-pack/test/security_solution_cypress/cypress/README.md @@ -42,11 +42,10 @@ Please, before opening a PR with the new test, please make sure that the test fa Note that we use tags in order to select which tests we want to execute: -- `@serverless` includes a test in the Serverless test suite for PRs (the so-called first quality gate). You need to explicitly add this tag to any test you want to run in CI for open PRs. These tests will run against a local, "simulated" serverless environment. -- `@serverlessQA` includes a test in the Serverless test suite for QA (the so-called second quality gate). You need to explicitly add this tag to any test you want to run in the CD pipeline against real serverless projects deployed in the Serverless QA environment. - - **NOTE:** We are adding this tag temporarily until we check the behavior of our tests in the second quality gate. +- `@serverless` includes a test in the Serverless test suite for PRs (the so-called first quality gate) and QA environemnt (the so-called second quality gate). You need to explicitly add this tag to any test you want to run in CI for serverless. - `@ess` includes a test in the normal, non-Serverless test suite. You need to explicitly add this tag to any test you want to run against a non-Serverless environment. - `@brokenInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Indicates that a test should run in Serverless, but currently is broken. +- `@brokenInServerlessQA` excludes a test form the Serverless QA enviornment (second quality gate). Indicates that a test should run on it, but currently is broken. - `@skipInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Could indicate many things, e.g. "the test is flaky in Serverless", "the test is Flaky in any type of environemnt", "the test has been temporarily excluded, see the comment above why". Please, before opening a PR with a new test, make sure that the test fails. If you never see your test fail you don’t know if your test is actually testing the right thing, or testing anything at all. From d45456a3d253afe03d60159d21e73b5972adc167 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Tue, 14 Nov 2023 07:42:43 +0100 Subject: [PATCH 06/21] adding the new scripts to readme --- x-pack/test/security_solution_cypress/cypress/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/test/security_solution_cypress/cypress/README.md b/x-pack/test/security_solution_cypress/cypress/README.md index 75a1177ad1e7d..f15c3fbe56707 100644 --- a/x-pack/test/security_solution_cypress/cypress/README.md +++ b/x-pack/test/security_solution_cypress/cypress/README.md @@ -71,6 +71,10 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio | cypress:explore:run:ess | Runs all tests tagged as ESS in the `e2e/explore` directory in headless mode | | cypress:investigations:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode | | cypress:explore:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode | +| cypress:open:qa:serverless | Opens the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment. The kibana instance will reload when you make code changes. This is the recommended way to debug tests in QA. Follow the readme in order to learn about the known limitations. | +| cypress:run:qa:serverless | Runs all tests tagged as SERVERLESS placed in the `e2e` directory excluding `investigations` and `explore` directories in headless mode using the QA environment and real MKI projects.| +| cypress:run:qa:serverless:explore | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode using the QA environment and real MKI prorjects. | +| cypress:run:qa:serverless:investigations | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode using the QA environment and reak MKI projects. | | junit:merge | Merges individual test reports into a single report and moves the report to the `junit` directory | Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging. From eb0de0f037470c95b274194a7cea8ab83c2aefe6 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Tue, 14 Nov 2023 07:44:51 +0100 Subject: [PATCH 07/21] removing log.info added for testing purposes --- .../security_solution/scripts/run_cypress/parallel_serverless.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index bcd600aa9b632..842564d3c43ea 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -126,7 +126,6 @@ async function deleteSecurityProject( apiKey: string ): Promise { try { - log.info(`Deleting the project`); await axios.delete(`${BASE_ENV_URL}/api/v1/serverless/projects/security/${projectId}`, { headers: { Authorization: `ApiKey ${apiKey}`, From 3ed045818732f61a417d82c12c1f780b60aaa7c4 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Tue, 14 Nov 2023 08:02:10 +0100 Subject: [PATCH 08/21] extending readme --- .../cypress/README.md | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_cypress/cypress/README.md b/x-pack/test/security_solution_cypress/cypress/README.md index f15c3fbe56707..ed599f5a289bc 100644 --- a/x-pack/test/security_solution_cypress/cypress/README.md +++ b/x-pack/test/security_solution_cypress/cypress/README.md @@ -193,7 +193,7 @@ Task [cypress/support/es_archiver.ts](https://github.com/elastic/kibana/blob/mai Note that we use tags in order to select which tests we want to execute, if you want a test to be executed on serverless you need to add @serverless tag to it. -### Running the serverless tests locally +### Running serverless tests locally pointing to FTR serverless (First Quality Gate) Run the tests with the following yarn scripts from `x-pack/test/security_solution_cypress`: @@ -206,6 +206,33 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging. + +### Running serverless tests locally pointing to a MKI project created in QA environment (Second Quality Gate) +#### Setup required + +Setup a valid Elastic Cloud API key for QA environment: + +1. Navigate to QA environment. +2. Click on the `User menu button` located on the top right of the header. +3. Click on `Organization`. +4. Click on the `API keys` tab. +5. Click on `Create API key` button. +6. Add a name, set an expiration date, assign an organization owner role. +7. Click on `Create API key` +8. Save the value of the key + +Store the saved key on `~/.elastic/cloud.json` using the following format: + +```json +{ + "api_key": { + "qa": "" + } +} +``` + +#### Known limitations + ### PLIs When running serverless Cypress tests, the following PLIs are set by default: From a7366d37f09d92de948ad9b819bb9e3ce34f4ccc Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Tue, 14 Nov 2023 08:29:13 +0100 Subject: [PATCH 09/21] adding more details to readme --- .../cypress/README.md | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_cypress/cypress/README.md b/x-pack/test/security_solution_cypress/cypress/README.md index ed599f5a289bc..6b50dd2672996 100644 --- a/x-pack/test/security_solution_cypress/cypress/README.md +++ b/x-pack/test/security_solution_cypress/cypress/README.md @@ -208,6 +208,19 @@ Please note that all the headless mode commands do not open the Cypress UI and a ### Running serverless tests locally pointing to a MKI project created in QA environment (Second Quality Gate) + +Run the tests with the following yarn scripts from `x-pack/test/security_solution_cypress`: + +| Script Name | Description | +| ----------- | ----------- | +| cypress:open:qa:serverless | Opens the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment. The kibana instance will reload when you make code changes. This is the recommended way to debug tests in QA. Follow the readme in order to learn about the known limitations. | +| cypress:run:qa:serverless | Runs all tests tagged as SERVERLESS placed in the `e2e` directory excluding `investigations` and `explore` directories in headless mode using the QA environment and real MKI projects.| +| cypress:run:qa:serverless:explore | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode using the QA environment and real MKI prorjects. | +| cypress:run:qa:serverless:investigations | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode using the QA environment and reak MKI projects. | + +Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging. + + #### Setup required Setup a valid Elastic Cloud API key for QA environment: @@ -231,7 +244,13 @@ Store the saved key on `~/.elastic/cloud.json` using the following format: } ``` -#### Known limitations +#### Known limitations on headless mode +- Currently RBAC cannot be tested. + +#### Known limitations on visual mode +- Currently RBAC cannot be tested. +- The project is created as `complete` by default, if you want to execute a test on a `essentials` project do the following: + ### PLIs When running serverless Cypress tests, the following PLIs are set by default: From eb188b5f0b3cccd729e2960f376fd78b3e24445b Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Tue, 14 Nov 2023 14:56:11 +0100 Subject: [PATCH 10/21] adding node option to avoid es_archiver to fail when using cypress in visual mode --- .../cypress/cypress_ci_serverless_qa.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts b/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts index 4688cbe343e1c..342c3da34bef6 100644 --- a/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts +++ b/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts @@ -41,6 +41,7 @@ export default defineCypressConfig({ specPattern: './cypress/e2e/**/*.cy.ts', setupNodeEvents(on, config) { esArchiver(on, config); + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; // eslint-disable-next-line @typescript-eslint/no-var-requires require('@cypress/grep/src/plugin')(config); return config; From 8ec88f7d261fe700036cb7b4c72b65b8940a65de Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Tue, 14 Nov 2023 18:10:46 +0100 Subject: [PATCH 11/21] parametrizes the script to force a specific tier or product line --- config/serverless.security.yml | 1 + .../run_cypress/parallel_serverless.ts | 45 ++++++++++++++----- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/config/serverless.security.yml b/config/serverless.security.yml index 7f749afc7626d..4a5be14efd0f1 100644 --- a/config/serverless.security.yml +++ b/config/serverless.security.yml @@ -18,6 +18,7 @@ xpack.securitySolutionServerless.productTypes: [ { product_line: 'security', product_tier: 'complete' }, { product_line: 'endpoint', product_tier: 'complete' }, + { product_line: 'cloud', product_tier: 'complete'}, ] xpack.securitySolution.offeringSettings: { diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index 842564d3c43ea..265e9bc1390a3 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -26,7 +26,6 @@ import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; import { INITIAL_REST_VERSION } from '@kbn/data-views-plugin/server/constants'; import { exec } from 'child_process'; import { renderSummaryTable } from './print_run'; -import type { SecuritySolutionDescribeBlockFtrConfig } from './utils'; import { parseTestFileConfig, retrieveIntegrations } from './utils'; interface ProductType { @@ -83,19 +82,14 @@ const getApiKeyFromElasticCloudJsonFile = (): string | undefined => { async function createSecurityProject( projectName: string, apiKey: string, - ftrConfig: SecuritySolutionDescribeBlockFtrConfig + productTypes: ProductType[] ): Promise { const body: CreateProjectRequestBody = { name: projectName, region_id: DEFAULT_REGION, + product_types: productTypes, }; - const productTypes: ProductType[] = []; - ftrConfig?.productTypes?.forEach((t) => { - productTypes.push(t as ProductType); - }); - if (productTypes.length > 0) body.product_types = productTypes; - try { const response = await axios.post(`${BASE_ENV_URL}/api/v1/serverless/projects/security`, body, { headers: { @@ -326,6 +320,28 @@ function waitForKibanaLogin(kbUrl: string, credentials: Credentials): Promise { + let productTypes = parseTestFileConfig(filePath).productTypes as ProductType[]; + if (tier) { + productTypes = productTypes.map((product) => ({ + ...product, + product_tier: tier, + })); + } + if (ignoreEndpointLine) { + productTypes = productTypes.filter((product) => product.product_line !== 'endpoint'); + } + if (ignoreCloudLine) { + productTypes = productTypes.filter((product) => product.product_line !== 'cloud'); + } + return productTypes; +}; + export const cli = () => { run( async (context) => { @@ -389,6 +405,10 @@ ${JSON.stringify(argv, null, 2)} const cypressConfigFilePath = require.resolve(`../../${argv.configFile}`) as string; const cypressConfigFile = await import(cypressConfigFilePath); + const projectTier = argv.tier as string; + const ignoreEndpoint = argv.ignoreEndpoint as boolean; + const ignoreCloud = argv.ignoreCloud as boolean; + log.info(` ---------------------------------------------- Cypress config for file: ${cypressConfigFilePath}: @@ -457,7 +477,12 @@ ${JSON.stringify(cypressConfigFile, null, 2)} await withProcRunner(log, async (procs) => { const id = crypto.randomBytes(8).toString('hex'); const PROJECT_NAME = `${PROJECT_NAME_PREFIX}-${id}`; - const specFileFTRConfig = parseTestFileConfig(filePath); + const productTypes = getProductTypes( + filePath, + projectTier, + ignoreEndpoint, + ignoreCloud + ); if (!API_KEY) { log.info('API KEY to create project could not be retrieved.'); @@ -467,7 +492,7 @@ ${JSON.stringify(cypressConfigFile, null, 2)} log.info(`${id}: Creating project ${PROJECT_NAME}...`); // Creating project for the test to run - const project = await createSecurityProject(PROJECT_NAME, API_KEY, specFileFTRConfig); + const project = await createSecurityProject(PROJECT_NAME, API_KEY, productTypes); if (!project) { log.info('Failed to create project.'); From 57d1183da07043c2b15d201592b288d21c4be701 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Tue, 14 Nov 2023 18:34:31 +0100 Subject: [PATCH 12/21] completing readme --- .../cypress/README.md | 65 ++++++++++++++++--- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/README.md b/x-pack/test/security_solution_cypress/cypress/README.md index 6b50dd2672996..842b9c75b6d70 100644 --- a/x-pack/test/security_solution_cypress/cypress/README.md +++ b/x-pack/test/security_solution_cypress/cypress/README.md @@ -206,6 +206,37 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging. +#### PLIs +When running serverless Cypress tests, the following PLIs are set by default: + +``` + { product_line: 'security', product_tier: 'complete' }, + { product_line: 'endpoint', product_tier: 'complete' }, + { product_line: 'cloud', product_tier: 'complete' }, +``` + +With the above configuration we'll be able to cover most of the scenarios, but there are some cases were we might want to use a different configuration. In that case, we just need to pass to the header of the test, which is the configuration we want for it. + +```typescript +describe( + 'Entity Analytics Dashboard in Serverless', + { + tags: '@serverless', + env: { + ftrConfig: { + productTypes: [ + { product_line: 'security', product_tier: 'essentials' }, + { product_line: 'endpoint', product_tier: 'essentials' }, + ], + }, + }, + }, +``` + +Per the way we set the environment during the execution process on CI, the above configuration is going to be valid when the test is executed on headless mode. + +For test developing or test debugging purposes, you need to modify the configuration but without committing and pushing the changes in `x-pack/test/security_solution_cypress/serverless_config.ts`. + ### Running serverless tests locally pointing to a MKI project created in QA environment (Second Quality Gate) @@ -244,17 +275,12 @@ Store the saved key on `~/.elastic/cloud.json` using the following format: } ``` -#### Known limitations on headless mode -- Currently RBAC cannot be tested. - -#### Known limitations on visual mode +#### Known limitations - Currently RBAC cannot be tested. -- The project is created as `complete` by default, if you want to execute a test on a `essentials` project do the following: - -### PLIs -When running serverless Cypress tests, the following PLIs are set by default: +#### PLIs +When running serverless Cypress tests on QA environment, the following PLIs are set by default: ``` { product_line: 'security', product_tier: 'complete' }, { product_line: 'endpoint', product_tier: 'complete' }, @@ -279,9 +305,28 @@ describe( }, ``` -Per the way we set the environment during the execution process on CI, the above configuration is going to be valid when the test is executed on headless mode. +For test developing or test debugging purposes on QA, you have avaialable the following options: + +``` +yarn cypress:open:qa:serverless --tier +``` + +The above command will open the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment with the passed tier essentials or complete. If no flag is passed, the project will be cretaed as complete. + +``` +yarn cypress:open:qa:serverless --ignoreEndpoint +``` + +The above command will open the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment without the endpoint add-on. + +``` +yarn cypress:open:qa:serverless --ignoreCloud +``` + +The above command will open the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment without the cloud add-on. + +Note that all the above flags can be combined. -For test developing or test debugging purposes, you need to modify the configuration but without committing and pushing the changes in `x-pack/test/security_solution_cypress/serverless_config.ts`. ## Development Best Practices From 7e2f4e0ffed898b0043ac5825ca68d2a6937f949 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Tue, 14 Nov 2023 19:16:32 +0100 Subject: [PATCH 13/21] little refactor --- .../run_cypress/parallel_serverless.ts | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index 265e9bc1390a3..ab49b40c71690 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -48,6 +48,12 @@ interface Project { product: string; } +interface ProjectConfigurationOverride { + tier: string; + ignoreEndpoint: boolean; + ignoreCloud: boolean; +} + interface Credentials { username: string; password: string; @@ -322,21 +328,19 @@ function waitForKibanaLogin(kbUrl: string, credentials: Credentials): Promise { let productTypes = parseTestFileConfig(filePath).productTypes as ProductType[]; - if (tier) { + if (projectConfigurationOverride.tier) { productTypes = productTypes.map((product) => ({ ...product, - product_tier: tier, + product_tier: projectConfigurationOverride.tier, })); } - if (ignoreEndpointLine) { + if (projectConfigurationOverride.ignoreEndpoint) { productTypes = productTypes.filter((product) => product.product_line !== 'endpoint'); } - if (ignoreCloudLine) { + if (projectConfigurationOverride.ignoreCloud) { productTypes = productTypes.filter((product) => product.product_line !== 'cloud'); } return productTypes; @@ -405,9 +409,11 @@ ${JSON.stringify(argv, null, 2)} const cypressConfigFilePath = require.resolve(`../../${argv.configFile}`) as string; const cypressConfigFile = await import(cypressConfigFilePath); - const projectTier = argv.tier as string; - const ignoreEndpoint = argv.ignoreEndpoint as boolean; - const ignoreCloud = argv.ignoreCloud as boolean; + const projectConfigurationOverride: ProjectConfigurationOverride = { + tier: argv.tier as string, + ignoreEndpoint: argv.ignoreEndpoint as boolean, + ignoreCloud: argv.ignoreClous as boolean, + }; log.info(` ---------------------------------------------- @@ -477,12 +483,7 @@ ${JSON.stringify(cypressConfigFile, null, 2)} await withProcRunner(log, async (procs) => { const id = crypto.randomBytes(8).toString('hex'); const PROJECT_NAME = `${PROJECT_NAME_PREFIX}-${id}`; - const productTypes = getProductTypes( - filePath, - projectTier, - ignoreEndpoint, - ignoreCloud - ); + const productTypes = getProductTypes(filePath, projectConfigurationOverride); if (!API_KEY) { log.info('API KEY to create project could not be retrieved.'); From 5a6e17df442cd05fb910f2cda3fd021cd0083ca2 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Wed, 15 Nov 2023 10:53:13 +0100 Subject: [PATCH 14/21] last adjusments --- config/serverless.security.yml | 1 - .../run_cypress/parallel_serverless.ts | 33 ++++++++++++------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/config/serverless.security.yml b/config/serverless.security.yml index 4a5be14efd0f1..7f749afc7626d 100644 --- a/config/serverless.security.yml +++ b/config/serverless.security.yml @@ -18,7 +18,6 @@ xpack.securitySolutionServerless.productTypes: [ { product_line: 'security', product_tier: 'complete' }, { product_line: 'endpoint', product_tier: 'complete' }, - { product_line: 'cloud', product_tier: 'complete'}, ] xpack.securitySolution.offeringSettings: { diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index ab49b40c71690..7b71400ec5408 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -48,7 +48,7 @@ interface Project { product: string; } -interface ProjectConfigurationOverride { +interface ProjectConfigurationParameters { tier: string; ignoreEndpoint: boolean; ignoreCloud: boolean; @@ -59,6 +59,12 @@ interface Credentials { password: string; } +const DEFAULT_CONFIGURATION: ProductType[] = [ + { product_line: 'security', product_tier: 'complete' }, + { product_line: 'cloud', product_tier: 'complete' }, + { product_line: 'endpoint', product_tier: 'complete' }, +] as const; + const DEFAULT_REGION = 'aws-eu-west-1'; const PROJECT_NAME_PREFIX = 'kibana-cypress-security-solution-ephemeral'; const BASE_ENV_URL = 'https://global.qa.cld.elstc.co'; @@ -326,21 +332,21 @@ function waitForKibanaLogin(kbUrl: string, credentials: Credentials): Promise { - let productTypes = parseTestFileConfig(filePath).productTypes as ProductType[]; - if (projectConfigurationOverride.tier) { + let productTypes = DEFAULT_CONFIGURATION; + + if (projectConfigurationParameters.tier) { productTypes = productTypes.map((product) => ({ ...product, - product_tier: projectConfigurationOverride.tier, + product_tier: projectConfigurationParameters.tier, })); } - if (projectConfigurationOverride.ignoreEndpoint) { + if (projectConfigurationParameters.ignoreEndpoint) { productTypes = productTypes.filter((product) => product.product_line !== 'endpoint'); } - if (projectConfigurationOverride.ignoreCloud) { + if (projectConfigurationParameters.ignoreCloud) { productTypes = productTypes.filter((product) => product.product_line !== 'cloud'); } return productTypes; @@ -409,10 +415,10 @@ ${JSON.stringify(argv, null, 2)} const cypressConfigFilePath = require.resolve(`../../${argv.configFile}`) as string; const cypressConfigFile = await import(cypressConfigFilePath); - const projectConfigurationOverride: ProjectConfigurationOverride = { + const projectConfigurationParameters: ProjectConfigurationParameters = { tier: argv.tier as string, ignoreEndpoint: argv.ignoreEndpoint as boolean, - ignoreCloud: argv.ignoreClous as boolean, + ignoreCloud: argv.ignoreCloud as boolean, }; log.info(` @@ -483,7 +489,10 @@ ${JSON.stringify(cypressConfigFile, null, 2)} await withProcRunner(log, async (procs) => { const id = crypto.randomBytes(8).toString('hex'); const PROJECT_NAME = `${PROJECT_NAME_PREFIX}-${id}`; - const productTypes = getProductTypes(filePath, projectConfigurationOverride); + + const productTypes = isOpen + ? getOverridedProductTypes(projectConfigurationParameters) + : (parseTestFileConfig(filePath).productTypes as ProductType[]); if (!API_KEY) { log.info('API KEY to create project could not be retrieved.'); From 2facb09235ef64bfc995adc8490899c2938c9b1a Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Wed, 15 Nov 2023 12:12:29 +0100 Subject: [PATCH 15/21] fixes --- .../scripts/run_cypress/parallel_serverless.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index 7b71400ec5408..bd75910594dcd 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -59,7 +59,7 @@ interface Credentials { password: string; } -const DEFAULT_CONFIGURATION: ProductType[] = [ +const DEFAULT_CONFIGURATION: Readonly = [ { product_line: 'security', product_tier: 'complete' }, { product_line: 'cloud', product_tier: 'complete' }, { product_line: 'endpoint', product_tier: 'complete' }, @@ -335,7 +335,7 @@ function waitForKibanaLogin(kbUrl: string, credentials: Credentials): Promise { - let productTypes = DEFAULT_CONFIGURATION; + let productTypes: ProductType[] = [...DEFAULT_CONFIGURATION]; if (projectConfigurationParameters.tier) { productTypes = productTypes.map((product) => ({ From b48532da24d2fd1e14f9537306cb6a8b9f4865ea Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Wed, 15 Nov 2023 14:14:34 +0100 Subject: [PATCH 16/21] refactor --- .../run_cypress/parallel_serverless.ts | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index bd75910594dcd..ecd05175f4696 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -50,8 +50,8 @@ interface Project { interface ProjectConfigurationParameters { tier: string; - ignoreEndpoint: boolean; - ignoreCloud: boolean; + endpointAddon: boolean; + cloudAddon: boolean; } interface Credentials { @@ -343,12 +343,13 @@ const getOverridedProductTypes = ( product_tier: projectConfigurationParameters.tier, })); } - if (projectConfigurationParameters.ignoreEndpoint) { - productTypes = productTypes.filter((product) => product.product_line !== 'endpoint'); - } - if (projectConfigurationParameters.ignoreCloud) { + if (projectConfigurationParameters.endpointAddon) { productTypes = productTypes.filter((product) => product.product_line !== 'cloud'); } + if (projectConfigurationParameters.cloudAddon) { + productTypes = productTypes.filter((product) => product.product_line !== 'endpoint'); + } + return productTypes; }; @@ -398,7 +399,19 @@ export const cli = () => { } return acc; }, {} as Record) - ); + ) + .option('tier', { + type: 'string', + default: 'complete', + }) + .option('endpointAddon', { + type: 'boolean', + default: true, + }) + .option('cloudAddon', { + type: 'boolean', + default: true, + }); log.info(` ---------------------------------------------- @@ -417,8 +430,8 @@ ${JSON.stringify(argv, null, 2)} const projectConfigurationParameters: ProjectConfigurationParameters = { tier: argv.tier as string, - ignoreEndpoint: argv.ignoreEndpoint as boolean, - ignoreCloud: argv.ignoreCloud as boolean, + endpointAddon: argv.endpointAddon as boolean, + cloudAddon: argv.cloudAddon as boolean, }; log.info(` From 57923b255d2497048385c60bf556c60d999e98d4 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Wed, 15 Nov 2023 14:35:27 +0100 Subject: [PATCH 17/21] last adjustments --- .../scripts/run_cypress/parallel_serverless.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index ecd05175f4696..f1c4d1575f985 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -343,10 +343,10 @@ const getOverridedProductTypes = ( product_tier: projectConfigurationParameters.tier, })); } - if (projectConfigurationParameters.endpointAddon) { + if (projectConfigurationParameters.endpointAddon && !projectConfigurationParameters.cloudAddon) { productTypes = productTypes.filter((product) => product.product_line !== 'cloud'); } - if (projectConfigurationParameters.cloudAddon) { + if (projectConfigurationParameters.cloudAddon && !projectConfigurationParameters.endpointAddon) { productTypes = productTypes.filter((product) => product.product_line !== 'endpoint'); } @@ -401,14 +401,17 @@ export const cli = () => { }, {} as Record) ) .option('tier', { + alias: 't', type: 'string', default: 'complete', }) .option('endpointAddon', { + alias: 'ea', type: 'boolean', default: true, }) .option('cloudAddon', { + alias: 'ca', type: 'boolean', default: true, }); From 073d7ece499e17504762e2be398119fc76efdd50 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Wed, 15 Nov 2023 14:45:48 +0100 Subject: [PATCH 18/21] updates readme --- x-pack/test/security_solution_cypress/cypress/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/README.md b/x-pack/test/security_solution_cypress/cypress/README.md index 842b9c75b6d70..8940d6c86e73e 100644 --- a/x-pack/test/security_solution_cypress/cypress/README.md +++ b/x-pack/test/security_solution_cypress/cypress/README.md @@ -314,13 +314,13 @@ yarn cypress:open:qa:serverless --tier The above command will open the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment with the passed tier essentials or complete. If no flag is passed, the project will be cretaed as complete. ``` -yarn cypress:open:qa:serverless --ignoreEndpoint +yarn cypress:open:qa:serverless --no-endpoint-addon ``` The above command will open the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment without the endpoint add-on. ``` -yarn cypress:open:qa:serverless --ignoreCloud +yarn cypress:open:qa:serverless --no-cloud-addon ``` The above command will open the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment without the cloud add-on. From b48924caa19a1efda987b2df5e376f1d9cad7025 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Wed, 15 Nov 2023 15:49:55 +0100 Subject: [PATCH 19/21] Update x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts Co-authored-by: Sergi Massaneda --- .../scripts/run_cypress/parallel_serverless.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index f1c4d1575f985..981b5107c7c6a 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -343,10 +343,10 @@ const getOverridedProductTypes = ( product_tier: projectConfigurationParameters.tier, })); } - if (projectConfigurationParameters.endpointAddon && !projectConfigurationParameters.cloudAddon) { + if (!projectConfigurationParameters.cloudAddon) { productTypes = productTypes.filter((product) => product.product_line !== 'cloud'); } - if (projectConfigurationParameters.cloudAddon && !projectConfigurationParameters.endpointAddon) { + if (!projectConfigurationParameters.endpointAddon) { productTypes = productTypes.filter((product) => product.product_line !== 'endpoint'); } From 6fbfbaeee492df6858c43d2494118d8a091d9e28 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Wed, 15 Nov 2023 16:12:19 +0100 Subject: [PATCH 20/21] addressing comments --- .../run_cypress/parallel_serverless.ts | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index 981b5107c7c6a..c22ebbfc1edd1 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -48,12 +48,6 @@ interface Project { product: string; } -interface ProjectConfigurationParameters { - tier: string; - endpointAddon: boolean; - cloudAddon: boolean; -} - interface Credentials { username: string; password: string; @@ -332,21 +326,23 @@ function waitForKibanaLogin(kbUrl: string, credentials: Credentials): Promise { let productTypes: ProductType[] = [...DEFAULT_CONFIGURATION]; - if (projectConfigurationParameters.tier) { + if (tier) { productTypes = productTypes.map((product) => ({ ...product, - product_tier: projectConfigurationParameters.tier, + product_tier: tier, })); } - if (!projectConfigurationParameters.cloudAddon) { + if (!cloudAddon) { productTypes = productTypes.filter((product) => product.product_line !== 'cloud'); } - if (!projectConfigurationParameters.endpointAddon) { + if (!endpointAddon) { productTypes = productTypes.filter((product) => product.product_line !== 'endpoint'); } @@ -431,11 +427,9 @@ ${JSON.stringify(argv, null, 2)} const cypressConfigFilePath = require.resolve(`../../${argv.configFile}`) as string; const cypressConfigFile = await import(cypressConfigFilePath); - const projectConfigurationParameters: ProjectConfigurationParameters = { - tier: argv.tier as string, - endpointAddon: argv.endpointAddon as boolean, - cloudAddon: argv.cloudAddon as boolean, - }; + const tier: string = argv.tier; + const endpointAddon: boolean = argv.endpointAddon; + const cloudAddon: boolean = argv.cloudAddon; log.info(` ---------------------------------------------- @@ -507,7 +501,7 @@ ${JSON.stringify(cypressConfigFile, null, 2)} const PROJECT_NAME = `${PROJECT_NAME_PREFIX}-${id}`; const productTypes = isOpen - ? getOverridedProductTypes(projectConfigurationParameters) + ? getProductTypes(tier, endpointAddon, cloudAddon) : (parseTestFileConfig(filePath).productTypes as ProductType[]); if (!API_KEY) { From 715660490ca0013c3ccb8b810053ce3f7faee1d6 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Wed, 15 Nov 2023 16:38:01 +0100 Subject: [PATCH 21/21] refactor --- .../scripts/run_cypress/parallel_serverless.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index c22ebbfc1edd1..a64f4027f456e 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -604,15 +604,13 @@ ${JSON.stringify(cypressConfigFile, null, 2)} env: cyCustomEnv, }, }); + // Delete serverless project + log.info(`${id} : Deleting project ${PROJECT_NAME}...`); + await deleteSecurityProject(project.id, PROJECT_NAME, API_KEY); } catch (error) { result = error; } } - - // Delete serverless project - log.info(`${id} : Deleting project ${PROJECT_NAME}...`); - await deleteSecurityProject(project.id, PROJECT_NAME, API_KEY); - return result; }); return result;