[QA] Code coverage: unskip tests, collect tests results, exclude bund…

…les from report (#64477)

* custom nyc configs, run all tests

* copy tests results back to original folder

src/dev/code_coverage/nyc_config/nyc.functional.config.js

@@ -0,0 +1,31 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const defaultExclude = require('@istanbuljs/schema/default-exclude');
+const extraExclude = ['data/optimize/**', 'src/core/server/**', '**/test/**'];
+const path = require('path');
+
+module.exports = {
+  'temp-dir': process.env.COVERAGE_TEMP_DIR
+    ? path.resolve(process.env.COVERAGE_TEMP_DIR, 'functional')
+    : 'target/kibana-coverage/functional',
+  'report-dir': 'target/kibana-coverage/functional-combined',
+  reporter: ['html', 'json-summary'],
+  exclude: extraExclude.concat(defaultExclude),
+};

src/dev/code_coverage/nyc_config/nyc.jest.config.js

@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const path = require('path');
+
+module.exports = {
+  'temp-dir': process.env.COVERAGE_TEMP_DIR
+    ? path.resolve(process.env.COVERAGE_TEMP_DIR, 'jest')
+    : 'target/kibana-coverage/jest',
+  'report-dir': 'target/kibana-coverage/jest-combined',
+  reporter: ['html', 'json-summary'],
+};

src/dev/code_coverage/shell_scripts/merge_jest_and_functional.sh

@@ -1,10 +1,9 @@
 #!/bin/bash
 
-EXTRACT_START_DIR=tmp/extracted_coverage
-EXTRACT_END_DIR=target/kibana-coverage
-COMBINED_EXTRACT_DIR=/${EXTRACT_START_DIR}/${EXTRACT_END_DIR}
+COVERAGE_TEMP_DIR=/tmp/extracted_coverage/target/kibana-coverage/
+export COVERAGE_TEMP_DIR
 
 echo "### Merge coverage reports"
 for x in jest functional; do
-  yarn nyc report --temp-dir $COMBINED_EXTRACT_DIR/${x} --report-dir $EXTRACT_END_DIR/${x}-combined --reporter=html --reporter=json-summary
+  yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.${x}.config.js
 done

test/functional/apps/bundles/index.js

@@ -25,7 +25,7 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
 
   describe('bundle compression', function () {
-    this.tags('ciGroup12');
+    this.tags(['ciGroup12', 'skipCoverage']);
 
     let buildNum;
     before(async () => {

test/scripts/jenkins_ci_group.sh

@@ -31,4 +31,12 @@ else
     mkdir -p ../kibana/target/kibana-coverage/functional
     mv target/kibana-coverage/functional/* ../kibana/target/kibana-coverage/functional/
   fi
+
+  echo " -> moving junit output, silently fail in case of no report"
+  mkdir -p ../kibana/target/junit
+  mv target/junit/* ../kibana/target/junit/ || echo "copying junit failed"
+
+  echo " -> copying screenshots and html for failures"
+  cp -r test/functional/screenshots/* ../kibana/test/functional/screenshots/ || echo "copying screenshots failed"
+  cp -r test/functional/failure_debug ../kibana/test/functional/ || echo "copying html failed"
 fi

test/scripts/jenkins_xpack_ci_group.sh

@@ -32,4 +32,12 @@ else
     mkdir -p ../../kibana/target/kibana-coverage/functional
     mv ../target/kibana-coverage/functional/* ../../kibana/target/kibana-coverage/functional/
   fi
+
+  echo " -> moving junit output, silently fail in case of no report"
+  mkdir -p ../../kibana/target/junit
+  mv ../target/junit/* ../../kibana/target/junit/ || echo "copying junit failed"
+
+  echo " -> copying screenshots and html for failures"
+  cp -r test/functional/screenshots/* ../../kibana/x-pack/test/functional/screenshots/ || echo "copying screenshots failed"
+  cp -r test/functional/failure_debug ../../kibana/x-pack/test/functional/ || echo "copying html failed"
 fi

x-pack/dev-tools/jest/create_jest_config.js

@@ -43,6 +43,7 @@ export function createJestConfig({ kibanaDirectory, rootDir, xPackKibanaDirector
       '!**/scripts/**',
       '!**/mocks/**',
       '!**/plugins/apm/e2e/**',
+      '!**/plugins/siem/cypress/**',
     ],
     coveragePathIgnorePatterns: ['.*\\.d\\.ts'],
     coverageDirectory: `${kibanaDirectory}/target/kibana-coverage/jest`,

x-pack/test/functional/apps/advanced_settings/feature_controls/advanced_settings_security.ts

@@ -140,7 +140,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
 
     // FLAKY: https://github.com/elastic/kibana/issues/57377
     describe.skip('no advanced_settings privileges', function () {
-      this.tags(['skipCoverage']);
       before(async () => {
         await security.role.create('no_advanced_settings_privileges_role', {
           elasticsearch: {

x-pack/test/functional/apps/advanced_settings/feature_controls/advanced_settings_spaces.ts

@@ -60,7 +60,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
     });
 
     describe('space with Advanced Settings disabled', function () {
-      this.tags('skipCoverage');
       before(async () => {
         // we need to load the following in every situation as deleting
         // a space deletes all of the associated saved objects

x-pack/test/functional/apps/dev_tools/feature_controls/index.ts

@@ -7,7 +7,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('feature controls', function () {
-    this.tags(['skipFirefox', 'skipCoverage']);
+    this.tags(['skipFirefox']);
     loadTestFile(require.resolve('./dev_tools_security'));
     loadTestFile(require.resolve('./dev_tools_spaces'));
   });

x-pack/test/functional/apps/visualize/index.ts

@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function visualize({ loadTestFile }: FtrProviderContext) {
   describe('Visualize', function visualizeTestSuite() {
-    this.tags(['ciGroup4', 'skipFirefox', 'skipCoverage']);
+    this.tags(['ciGroup4', 'skipFirefox']);
 
     loadTestFile(require.resolve('./feature_controls/visualize_security'));
     loadTestFile(require.resolve('./feature_controls/visualize_spaces'));