Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Security Solution] Fix flaky test: x-pack/test/detection_engine_api_integration/security_and_spaces/update_prebuilt_rules_package/update_prebuilt_rules_package·ts - update_prebuilt_rules_package should allow user to install prebuilt rules from scratch, then install new rules and upgrade existing rules from the new package #163241

Merged
merged 8 commits into from
Aug 11, 2023
Merged

[Security Solution] Fix flaky test: x-pack/test/detection_engine_api_integration/security_and_spaces/update_prebuilt_rules_package/update_prebuilt_rules_package·ts - update_prebuilt_rules_package should allow user to install prebuilt rules from scratch, then install new rules and upgrade existing rules from the new package #163241

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a25be3f
RUn flaky tets runner
jpdjere Aug 7, 2023
d7472f8
Add indices refresh after installing and upgrading rules
jpdjere Aug 8, 2023
e8395b3
[CI] Auto-commit changed files from 'node scripts/precommit_hook.js -…
kibanamachine Aug 8, 2023
ce1cb78
Merge branch 'main' into update-prebuilt-rules-package-flay-test-runner
jpdjere Aug 8, 2023
7ee84f3
Refactor
jpdjere Aug 10, 2023
2d0a116
Merge branch 'main' into update-prebuilt-rules-package-flay-test-runner
jpdjere Aug 10, 2023
4b11128
Fix
jpdjere Aug 10, 2023
4857be9
Merge branch 'main' into update-prebuilt-rules-package-flay-test-runner
jpdjere Aug 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion x-pack/test/detection_engine_api_integration/basic/tests/coverage_overview.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,7 @@ export default ({ getService }: FtrProviderContext): void => {
threat: generateThreatArray(1),
}),
]);
await installPrebuiltRulesAndTimelines(supertest);
await installPrebuiltRulesAndTimelines(es, supertest);

const expectedRule = await createRule(supertest, log, {
...getSimpleRule('rule-1'),
Expand Down
18 changes: 9 additions & 9 deletions ...curity_and_spaces/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { deleteAllPrebuiltRuleAssets, deleteAllRules } from '../../utils';
import { getPrebuiltRulesStatus } from '../../utils/prebuilt_rules/get_prebuilt_rules_status';
import { installPrebuiltRulesPackageByVersion } from '../../utils/prebuilt_rules/install_fleet_package_by_url';

// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext): void => {
Expand Down Expand Up @@ -55,18 +56,17 @@ export default ({ getService }: FtrProviderContext): void => {
expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_install).toBe(0);
expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0);

const EPM_URL = `/api/fleet/epm/packages/security_detection_engine/99.0.0`;

const bundledInstallResponse = await supertest
.post(EPM_URL)
.set('kbn-xsrf', 'xxxx')
.type('application/json')
.send({ force: true })
.expect(200);
const bundledInstallResponse = await installPrebuiltRulesPackageByVersion(
es,
supertest,
'99.0.0'
);

// As opposed to "registry"
expect(bundledInstallResponse.body._meta.install_source).toBe('bundled');
expect(bundledInstallResponse._meta.install_source).toBe('bundled');

// Refresh ES indices to avoid race conditions between write and reading of indeces
// See implementation utility function at x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/install_prebuilt_rules_fleet_package.ts
await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES });

// Verify that status is updated after package installation
Expand Down
12 changes: 3 additions & 9 deletions ...api_integration/security_and_spaces/bundled_prebuilt_rules_package/prerelease_packages.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,10 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server';
import { DETECTION_ENGINE_RULES_URL_FIND } from '@kbn/security-solution-plugin/common/constants';
import expect from 'expect';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { deleteAllPrebuiltRuleAssets, deleteAllRules } from '../../utils';
import { getInstalledRules } from '../../utils/prebuilt_rules/get_installed_rules';
import { getPrebuiltRulesStatus } from '../../utils/prebuilt_rules/get_prebuilt_rules_status';
import { installPrebuiltRules } from '../../utils/prebuilt_rules/install_prebuilt_rules';

Expand Down Expand Up @@ -38,8 +37,7 @@ export default ({ getService }: FtrProviderContext): void => {
expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_install).toBe(0);
expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0);

await installPrebuiltRules(supertest);
await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES });
await installPrebuiltRules(es, supertest);

// Verify that status is updated after package installation
const statusAfterPackageInstallation = await getPrebuiltRulesStatus(supertest);
Expand All @@ -48,11 +46,7 @@ export default ({ getService }: FtrProviderContext): void => {
expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0);

// Get installed rules
const { body: rulesResponse } = await supertest
.get(DETECTION_ENGINE_RULES_URL_FIND)
.set('kbn-xsrf', 'true')
.send()
.expect(200);
const rulesResponse = await getInstalledRules(supertest);

// Assert that installed rules are from package 99.0.0 and not from prerelease (beta) package
expect(rulesResponse.data.length).toBe(1);
Expand Down
4 changes: 1 addition & 3 deletions .../security_and_spaces/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
* 2.0.
*/
import expect from 'expect';
import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { deleteAllRules, getPrebuiltRulesAndTimelinesStatus } from '../../utils';
import { deleteAllPrebuiltRuleAssets } from '../../utils/prebuilt_rules/delete_all_prebuilt_rule_assets';
Expand Down Expand Up @@ -36,8 +35,7 @@ export default ({ getService }: FtrProviderContext): void => {
expect(statusBeforePackageInstallation.rules_not_updated).toBe(0);

// Install the package with 15000 prebuilt historical version of rules rules and 750 unique rules
await installPrebuiltRulesAndTimelines(supertest);
await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES });
await installPrebuiltRulesAndTimelines(es, supertest);

// Verify that status is updated after package installation
const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest);
Expand Down
27 changes: 2 additions & 25 deletions .../detection_engine_api_integration/security_and_spaces/prebuilt_rules/fleet_integration.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
* 2.0.
*/
import expect from 'expect';
import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import {
deleteAllRules,
Expand Down Expand Up @@ -43,44 +42,22 @@ export default ({ getService }: FtrProviderContext): void => {
expect(statusBeforePackageInstallation.rules_not_updated).toBe(0);

await installPrebuiltRulesFleetPackage({
es,
supertest,
overrideExistingPackage: true,
});

// Before we proceed, we need to refresh saved object indices. This comment will explain why.
// At the previous step we installed the Fleet package with prebuilt detection rules.
// Prebuilt rules are assets that Fleet indexes as saved objects of a certain type.
// Fleet does this via a savedObjectsClient.import() call with explicit `refresh: false`.
// So, despite of the fact that the endpoint waits until the prebuilt rule assets will be
// successfully indexed, it doesn't wait until they become "visible" for subsequent read
// operations. Which is what we do next: we read these SOs in getPrebuiltRulesAndTimelinesStatus().
// Now, the time left until the next refresh can be anything from 0 to the default value, and
// it depends on the time when savedObjectsClient.import() call happens relative to the time of
// the next refresh. Also, probably the refresh time can be delayed when ES is under load?
// Anyway, here we have a race condition between a write and subsequent read operation, and to
// fix it deterministically we have to refresh saved object indices and wait until it's done.
await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES });

// Verify that status is updated after package installation
const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest);
expect(statusAfterPackageInstallation.rules_installed).toBe(0);
expect(statusAfterPackageInstallation.rules_not_installed).toBeGreaterThan(0);
expect(statusAfterPackageInstallation.rules_not_updated).toBe(0);

// Verify that all previously not installed rules were installed
const response = await installPrebuiltRulesAndTimelines(supertest);
const response = await installPrebuiltRulesAndTimelines(es, supertest);
expect(response.rules_installed).toBe(statusAfterPackageInstallation.rules_not_installed);
expect(response.rules_updated).toBe(0);

// Similar to the previous refresh, we need to do it again between the two operations:
// - previous write operation: install prebuilt rules and timelines
// - subsequent read operation: get prebuilt rules and timelines status
// You may ask why? I'm not sure, probably because the write operation can install the Fleet
// package under certain circumstances, and it all works with `refresh: false` again.
// Anyway, there were flaky runs failing specifically at one of the next assertions,
// which means some kind of the same race condition we have here too.
await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES });

// Verify that status is updated after rules installation
const statusAfterRuleInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest);
expect(statusAfterRuleInstallation.rules_installed).toBe(response.rules_installed);
Expand Down
40 changes: 20 additions & 20 deletions ...on_engine_api_integration/security_and_spaces/prebuilt_rules/get_prebuilt_rules_status.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ export default ({ getService }: FtrProviderContext): void => {

it('should return the number of installed prebuilt rules after installing them', async () => {
await createPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects());
await installPrebuiltRules(supertest);
await installPrebuiltRules(es, supertest);

const { stats } = await getPrebuiltRulesStatus(supertest);
expect(stats).toMatchObject({
Expand All @@ -95,7 +95,7 @@ export default ({ getService }: FtrProviderContext): void => {

it('should notify the user again that a rule is available for install after it is deleted', async () => {
await createPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects());
await installPrebuiltRules(supertest);
await installPrebuiltRules(es, supertest);
await deleteRule(supertest, 'rule-1');

const { stats } = await getPrebuiltRulesStatus(supertest);
Expand All @@ -110,7 +110,7 @@ export default ({ getService }: FtrProviderContext): void => {
it('should return available rule updates', async () => {
const ruleAssetSavedObjects = getRuleAssetSavedObjects();
await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects);
await installPrebuiltRules(supertest);
await installPrebuiltRules(es, supertest);

// Clear previous rule assets
await deleteAllPrebuiltRuleAssets(es);
Expand All @@ -130,15 +130,15 @@ export default ({ getService }: FtrProviderContext): void => {
it('should not return any available update if rule has been successfully upgraded', async () => {
const ruleAssetSavedObjects = getRuleAssetSavedObjects();
await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects);
await installPrebuiltRules(supertest);
await installPrebuiltRules(es, supertest);

// Clear previous rule assets
await deleteAllPrebuiltRuleAssets(es);
// Increment the version of one of the installed rules and create the new rule assets
ruleAssetSavedObjects[0]['security-rule'].version += 1;
await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects);
// Upgrade all rules
await upgradePrebuiltRules(supertest);
await upgradePrebuiltRules(es, supertest);

const { stats } = await getPrebuiltRulesStatus(supertest);
expect(stats).toMatchObject({
Expand All @@ -152,7 +152,7 @@ export default ({ getService }: FtrProviderContext): void => {
it('should not return any updates if none are available', async () => {
const ruleAssetSavedObjects = getRuleAssetSavedObjects();
await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects);
await installPrebuiltRules(supertest);
await installPrebuiltRules(es, supertest);

// Clear previous rule assets
await deleteAllPrebuiltRuleAssets(es);
Expand Down Expand Up @@ -193,7 +193,7 @@ export default ({ getService }: FtrProviderContext): void => {

it('should return the number of installed prebuilt rules after installing them', async () => {
await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects());
await installPrebuiltRules(supertest);
await installPrebuiltRules(es, supertest);

const { stats } = await getPrebuiltRulesStatus(supertest);
expect(stats).toMatchObject({
Expand All @@ -206,7 +206,7 @@ export default ({ getService }: FtrProviderContext): void => {

it('should notify the user again that a rule is available for install after it is deleted', async () => {
await createPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects());
await installPrebuiltRules(supertest);
await installPrebuiltRules(es, supertest);
await deleteRule(supertest, 'rule-1');

const { stats } = await getPrebuiltRulesStatus(supertest);
Expand All @@ -220,7 +220,7 @@ export default ({ getService }: FtrProviderContext): void => {

it('should return available rule updates when previous historical versions available', async () => {
await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects());
await installPrebuiltRules(supertest);
await installPrebuiltRules(es, supertest);

// Add a new version of one of the installed rules
await createHistoricalPrebuiltRuleAssetSavedObjects(es, [
Expand All @@ -238,7 +238,7 @@ export default ({ getService }: FtrProviderContext): void => {

it('should return available rule updates when previous historical versions unavailable', async () => {
await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects());
await installPrebuiltRules(supertest);
await installPrebuiltRules(es, supertest);

// Delete the previous versions of rule assets
await deleteAllPrebuiltRuleAssets(es);
Expand All @@ -261,7 +261,7 @@ export default ({ getService }: FtrProviderContext): void => {

it('should not return available rule updates after rule has been upgraded', async () => {
await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects());
await installPrebuiltRules(supertest);
await installPrebuiltRules(es, supertest);

// Delete the previous versions of rule assets
await deleteAllPrebuiltRuleAssets(es);
Expand All @@ -272,7 +272,7 @@ export default ({ getService }: FtrProviderContext): void => {
]);

// Upgrade the rule
await upgradePrebuiltRules(supertest);
await upgradePrebuiltRules(es, supertest);

const { stats } = await getPrebuiltRulesStatus(supertest);
expect(stats).toMatchObject({
Expand Down Expand Up @@ -339,7 +339,7 @@ export default ({ getService }: FtrProviderContext): void => {

it('should return the number of installed prebuilt rules after installing them', async () => {
await createPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects());
await installPrebuiltRulesAndTimelines(supertest);
await installPrebuiltRulesAndTimelines(es, supertest);

const body = await getPrebuiltRulesAndTimelinesStatus(supertest);
expect(body).toMatchObject({
Expand All @@ -352,7 +352,7 @@ export default ({ getService }: FtrProviderContext): void => {

it('should notify the user again that a rule is available for install after it is deleted', async () => {
await createPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects());
await installPrebuiltRulesAndTimelines(supertest);
await installPrebuiltRulesAndTimelines(es, supertest);
await deleteRule(supertest, 'rule-1');

const body = await getPrebuiltRulesAndTimelinesStatus(supertest);
Expand All @@ -367,7 +367,7 @@ export default ({ getService }: FtrProviderContext): void => {
it('should return available rule updates', async () => {
const ruleAssetSavedObjects = getRuleAssetSavedObjects();
await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects);
await installPrebuiltRulesAndTimelines(supertest);
await installPrebuiltRulesAndTimelines(es, supertest);

// Clear previous rule assets
await deleteAllPrebuiltRuleAssets(es);
Expand All @@ -387,7 +387,7 @@ export default ({ getService }: FtrProviderContext): void => {
it('should not return any updates if none are available', async () => {
const ruleAssetSavedObjects = getRuleAssetSavedObjects();
await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects);
await installPrebuiltRulesAndTimelines(supertest);
await installPrebuiltRulesAndTimelines(es, supertest);

// Clear previous rule assets
await deleteAllPrebuiltRuleAssets(es);
Expand Down Expand Up @@ -428,7 +428,7 @@ export default ({ getService }: FtrProviderContext): void => {

it('should return the number of installed prebuilt rules after installing them', async () => {
await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects());
await installPrebuiltRulesAndTimelines(supertest);
await installPrebuiltRulesAndTimelines(es, supertest);

const body = await getPrebuiltRulesAndTimelinesStatus(supertest);
expect(body).toMatchObject({
Expand All @@ -441,7 +441,7 @@ export default ({ getService }: FtrProviderContext): void => {

it('should notify the user again that a rule is available for install after it is deleted', async () => {
await createPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects());
await installPrebuiltRulesAndTimelines(supertest);
await installPrebuiltRulesAndTimelines(es, supertest);
await deleteRule(supertest, 'rule-1');

const body = await getPrebuiltRulesAndTimelinesStatus(supertest);
Expand All @@ -455,7 +455,7 @@ export default ({ getService }: FtrProviderContext): void => {

it('should return available rule updates when previous historical versions available', async () => {
await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects());
await installPrebuiltRulesAndTimelines(supertest);
await installPrebuiltRulesAndTimelines(es, supertest);

// Add a new version of one of the installed rules
await createHistoricalPrebuiltRuleAssetSavedObjects(es, [
Expand All @@ -473,7 +473,7 @@ export default ({ getService }: FtrProviderContext): void => {

it('should return available rule updates when previous historical versions unavailable', async () => {
await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects());
await installPrebuiltRulesAndTimelines(supertest);
await installPrebuiltRulesAndTimelines(es, supertest);

// Delete the previous versions of rule assets
await deleteAllPrebuiltRuleAssets(es);
Expand Down
2 changes: 1 addition & 1 deletion ...ngine_api_integration/security_and_spaces/prebuilt_rules/get_prebuilt_timelines_status.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ export default ({ getService }: FtrProviderContext): void => {
});

it('should return the number of installed timeline templates after installing them', async () => {
await installPrebuiltRulesAndTimelines(supertest);
await installPrebuiltRulesAndTimelines(es, supertest);

const body = await getPrebuiltRulesAndTimelinesStatus(supertest);
expect(body).toMatchObject({
Expand Down
Loading