From 4af2cc5de907e53a6606a9c4fc643c89161e443c Mon Sep 17 00:00:00 2001 From: Davis Plumlee Date: Thu, 8 Aug 2024 11:55:16 -0400 Subject: [PATCH 1/5] adds kql query algorithm --- .../diff/calculation/algorithms/index.ts | 1 + .../kql_query_diff_algorithm.test.ts | 551 ++++++++++++++++++ .../algorithms/kql_query_diff_algorithm.ts | 167 ++++++ 3 files changed, 719 insertions(+) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/index.ts index fc895543e66b2..d9ad131fc51ab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/index.ts @@ -11,3 +11,4 @@ export { scalarArrayDiffAlgorithm } from './scalar_array_diff_algorithm'; export { simpleDiffAlgorithm } from './simple_diff_algorithm'; export { multiLineStringDiffAlgorithm } from './multi_line_string_diff_algorithm'; export { dataSourceDiffAlgorithm } from './data_source_diff_algorithm'; +export { kqlQueryDiffAlgorithm } from './kql_query_diff_algorithm'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts new file mode 100644 index 0000000000000..054a4f925f74e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts @@ -0,0 +1,551 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + RuleKqlQuery, + ThreeVersionsOf, +} from '../../../../../../../../common/api/detection_engine'; +import { + ThreeWayDiffOutcome, + ThreeWayMergeOutcome, + MissingVersion, + ThreeWayDiffConflict, + KqlQueryType, + KqlQueryLanguageEnum, +} from '../../../../../../../../common/api/detection_engine'; +import { kqlQueryDiffAlgorithm } from './kql_query_diff_algorithm'; + +describe('kqlQueryDiffAlgorithm', () => { + describe('returns current_version as merged output if there is no update - scenario AAA', () => { + it('if all versions are inline query type', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + current_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + target_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.StockValueNoUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + it('if all versions are saved query type', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id', + }, + current_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id', + }, + target_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id', + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.StockValueNoUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + }); + + describe('returns current_version as merged output if current_version is different and there is no update - scenario ABA', () => { + it('if current version is different query type than base and target', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id', + }, + current_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + target_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id', + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueNoUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + it('if all versions are same data type', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + current_version: { + type: KqlQueryType.inline_query, + query: 'query string = false', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + target_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueNoUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + }); + + describe('returns target_version as merged output if current_version is the same and there is an update - scenario AAB', () => { + it('if target version is different query type than base and current', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + current_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + target_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id', + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.target_version, + diff_outcome: ThreeWayDiffOutcome.StockValueCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Target, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + it('if all versions are same data type', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + current_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + target_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [{ field: 'some filter' }], + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.target_version, + diff_outcome: ThreeWayDiffOutcome.StockValueCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Target, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + }); + + describe('returns current_version as merged output if current version is different but it matches the update - scenario ABB', () => { + it('if base version is different query type from current and target versions', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + current_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id', + }, + target_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id', + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueSameUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + it('if all versions are query types', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + type: KqlQueryType.inline_query, + query: 'query string = false', + language: KqlQueryLanguageEnum.lucene, + filters: [], + }, + current_version: { + type: KqlQueryType.inline_query, + query: 'query string = false', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + target_version: { + type: KqlQueryType.inline_query, + query: 'query string = false', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueSameUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + }); + + describe('if all three versions are different - scenario ABC', () => { + describe('if all versions are saved query type', () => { + it('returns the current_version with a conflict', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id-1', + }, + current_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id-2', + }, + target_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id-3', + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NON_SOLVABLE, + }) + ); + }); + }); + + describe('if all versions are inline query type', () => { + it('returns a computated merged version without a conflict if 3 way merge is possible', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + type: KqlQueryType.inline_query, + query: `My description.\f\nThis is a second\u2001 line.\f\nThis is a third line.`, + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + current_version: { + type: KqlQueryType.inline_query, + query: `My GREAT description.\f\nThis is a second\u2001 line.\f\nThis is a third line.`, + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + target_version: { + type: KqlQueryType.inline_query, + query: `My description.\f\nThis is a second\u2001 line.\f\nThis is a GREAT line.`, + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + }; + + const expectedMergedVersion: RuleKqlQuery = { + type: KqlQueryType.inline_query, + query: `My GREAT description.\f\nThis is a second\u2001 line.\f\nThis is a GREAT line.`, + language: KqlQueryLanguageEnum.kuery, + filters: [], + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: expectedMergedVersion, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Merged, + conflict: ThreeWayDiffConflict.SOLVABLE, + }) + ); + }); + + it('returns the current_version with a conflict if 3 way merge is not possible', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + type: KqlQueryType.inline_query, + query: 'My description.\nThis is a second line.', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + current_version: { + type: KqlQueryType.inline_query, + query: 'My GREAT description.\nThis is a third line.', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + target_version: { + type: KqlQueryType.inline_query, + query: 'My EXCELLENT description.\nThis is a fourth.', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NON_SOLVABLE, + }) + ); + }); + }); + + describe('if versions are different types', () => { + it('returns the current_version with a conflict', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + type: KqlQueryType.inline_query, + query: 'My description.\nThis is a second line.', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + current_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id-2', + }, + target_version: { + type: KqlQueryType.inline_query, + query: 'My EXCELLENT description.\nThis is a fourth.', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NON_SOLVABLE, + }) + ); + }); + }); + }); + + describe('if base_version is missing', () => { + describe('if current_version and target_version are the same - scenario -AA', () => { + it('returns current_version as merged output if if all versions are inline query types', () => { + const mockVersions: ThreeVersionsOf = { + base_version: MissingVersion, + current_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + target_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + has_base_version: false, + base_version: undefined, + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.MissingBaseNoUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + it('returns current_version as merged output if if all versions are saved query types', () => { + const mockVersions: ThreeVersionsOf = { + base_version: MissingVersion, + current_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id', + }, + target_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id', + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + has_base_version: false, + base_version: undefined, + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.MissingBaseNoUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + }); + + describe('if current_version and target_version are different - scenario -AB', () => { + it('returns target_version as merged output if all versions are the same', () => { + const mockVersions: ThreeVersionsOf = { + base_version: MissingVersion, + current_version: { + type: KqlQueryType.inline_query, + query: 'query string = true', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + target_version: { + type: KqlQueryType.inline_query, + query: 'query string = false', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + has_base_version: false, + base_version: undefined, + merged_version: mockVersions.target_version, + diff_outcome: ThreeWayDiffOutcome.MissingBaseCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Target, + conflict: ThreeWayDiffConflict.SOLVABLE, + }) + ); + }); + + it('returns target_version as merged output if all versions are different', () => { + const mockVersions: ThreeVersionsOf = { + base_version: MissingVersion, + current_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id-2', + }, + target_version: { + type: KqlQueryType.saved_query, + saved_query_id: 'saved-query-id-3', + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + has_base_version: false, + base_version: undefined, + merged_version: mockVersions.target_version, + diff_outcome: ThreeWayDiffOutcome.MissingBaseCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Target, + conflict: ThreeWayDiffConflict.SOLVABLE, + }) + ); + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.ts new file mode 100644 index 0000000000000..be9776fb5992d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.ts @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { merge } from 'node-diff3'; +import { isEqual } from 'lodash'; +import { assertUnreachable } from '../../../../../../../../common/utility_types'; +import type { + RuleKqlQuery, + ThreeVersionsOf, + ThreeWayDiff, +} from '../../../../../../../../common/api/detection_engine/prebuilt_rules'; +import { + determineIfValueCanUpdate, + ThreeWayDiffOutcome, + ThreeWayMergeOutcome, + MissingVersion, + ThreeWayDiffConflict, + determineDiffOutcome, + KqlQueryType, +} from '../../../../../../../../common/api/detection_engine/prebuilt_rules'; + +export const kqlQueryDiffAlgorithm = ( + versions: ThreeVersionsOf +): ThreeWayDiff => { + const { + base_version: baseVersion, + current_version: currentVersion, + target_version: targetVersion, + } = versions; + + const diffOutcome = determineDiffOutcome(baseVersion, currentVersion, targetVersion); + + const valueCanUpdate = determineIfValueCanUpdate(diffOutcome); + + const hasBaseVersion = baseVersion !== MissingVersion; + + const { mergeOutcome, conflict, mergedVersion } = mergeVersions({ + baseVersion: hasBaseVersion ? baseVersion : undefined, + currentVersion, + targetVersion, + diffOutcome, + }); + + return { + has_base_version: hasBaseVersion, + base_version: hasBaseVersion ? baseVersion : undefined, + current_version: currentVersion, + target_version: targetVersion, + merged_version: mergedVersion, + merge_outcome: mergeOutcome, + + diff_outcome: diffOutcome, + conflict, + has_update: valueCanUpdate, + }; +}; + +interface MergeResult { + mergeOutcome: ThreeWayMergeOutcome; + mergedVersion: RuleKqlQuery; + conflict: ThreeWayDiffConflict; +} + +interface MergeArgs { + baseVersion: RuleKqlQuery | undefined; + currentVersion: RuleKqlQuery; + targetVersion: RuleKqlQuery; + diffOutcome: ThreeWayDiffOutcome; +} + +const mergeVersions = ({ + baseVersion, + currentVersion, + targetVersion, + diffOutcome, +}: MergeArgs): MergeResult => { + switch (diffOutcome) { + // Scenario -AA is treated as scenario AAA: + // https://github.com/elastic/kibana/pull/184889#discussion_r1636421293 + case ThreeWayDiffOutcome.MissingBaseNoUpdate: + case ThreeWayDiffOutcome.StockValueNoUpdate: + case ThreeWayDiffOutcome.CustomizedValueNoUpdate: + case ThreeWayDiffOutcome.CustomizedValueSameUpdate: + return { + conflict: ThreeWayDiffConflict.NONE, + mergeOutcome: ThreeWayMergeOutcome.Current, + mergedVersion: currentVersion, + }; + + case ThreeWayDiffOutcome.StockValueCanUpdate: + return { + conflict: ThreeWayDiffConflict.NONE, + mergeOutcome: ThreeWayMergeOutcome.Target, + mergedVersion: targetVersion, + }; + + case ThreeWayDiffOutcome.CustomizedValueCanUpdate: { + if ( + baseVersion && + baseVersion.type === KqlQueryType.inline_query && + currentVersion.type === KqlQueryType.inline_query && + targetVersion.type === KqlQueryType.inline_query + ) { + // TS does not realize that in ABC scenario, baseVersion cannot be missing + // Missing baseVersion scenarios were handled as -AA and -AB. + const mergedVersion = merge(currentVersion.query, baseVersion.query, targetVersion.query, { + stringSeparator: /(\S+|\s+)/g, // Retains all whitespace, which we keep to preserve formatting + }); + + const baseNonMergeableFields = { + filters: baseVersion.filters, + language: baseVersion.language, + }; + + const currentNonMergeableFields = { + filters: currentVersion.filters, + language: currentVersion.language, + }; + + const targetNonMergeableFields = { + filters: targetVersion.filters, + language: targetVersion.language, + }; + + const baseEqlCurrent = isEqual(baseNonMergeableFields, currentNonMergeableFields); + const baseEqlTarget = isEqual(baseNonMergeableFields, targetNonMergeableFields); + const currentEqlTarget = isEqual(currentNonMergeableFields, targetNonMergeableFields); + + if ( + baseEqlCurrent && + baseEqlTarget && + currentEqlTarget && + mergedVersion.conflict === false + ) { + return { + conflict: ThreeWayDiffConflict.SOLVABLE, + mergedVersion: { ...currentVersion, query: mergedVersion.result.join('') }, + mergeOutcome: ThreeWayMergeOutcome.Merged, + }; + } + } + + return { + conflict: ThreeWayDiffConflict.NON_SOLVABLE, + mergeOutcome: ThreeWayMergeOutcome.Current, + mergedVersion: currentVersion, + }; + } + + // Scenario -AB is treated as scenario ABC, but marked as + // SOLVABLE, and returns the target version as the merged version + // https://github.com/elastic/kibana/pull/184889#discussion_r1636421293 + case ThreeWayDiffOutcome.MissingBaseCanUpdate: { + return { + mergedVersion: targetVersion, + mergeOutcome: ThreeWayMergeOutcome.Target, + conflict: ThreeWayDiffConflict.SOLVABLE, + }; + } + default: + return assertUnreachable(diffOutcome); + } +}; From 1f6e217fe8c557824b4b865243539f06ecfa5463 Mon Sep 17 00:00:00 2001 From: Davis Plumlee Date: Thu, 8 Aug 2024 14:49:52 -0400 Subject: [PATCH 2/5] adds eql query diff algorithm --- .../eql_query_diff_algorithm.test.ts | 305 ++++++++++++++++++ .../algorithms/eql_query_diff_algorithm.ts | 148 +++++++++ .../diff/calculation/algorithms/helpers.ts | 14 +- .../diff/calculation/algorithms/index.ts | 1 + .../kql_query_diff_algorithm.test.ts | 40 ++- .../algorithms/kql_query_diff_algorithm.ts | 43 +-- 6 files changed, 526 insertions(+), 25 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.test.ts new file mode 100644 index 0000000000000..203c39ad62df1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.test.ts @@ -0,0 +1,305 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + RuleEqlQuery, + ThreeVersionsOf, +} from '../../../../../../../../common/api/detection_engine'; +import { + ThreeWayDiffOutcome, + ThreeWayMergeOutcome, + MissingVersion, + ThreeWayDiffConflict, +} from '../../../../../../../../common/api/detection_engine'; +import { eqlQueryDiffAlgorithm } from './eql_query_diff_algorithm'; + +describe('eqlQueryDiffAlgorithm', () => { + it('returns current_version as merged output if there is no update - scenario AAA', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + query: 'query where true', + language: 'eql', + filters: [], + }, + current_version: { + query: 'query where true', + language: 'eql', + filters: [], + }, + target_version: { + query: 'query where true', + language: 'eql', + filters: [], + }, + }; + + const result = eqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.StockValueNoUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + it('returns current_version as merged output if current_version is different and there is no update - scenario ABA', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + query: 'query where true', + language: 'eql', + filters: [], + }, + current_version: { + query: 'query where false', + language: 'eql', + filters: [], + }, + target_version: { + query: 'query where true', + language: 'eql', + filters: [], + }, + }; + + const result = eqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueNoUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + it('returns target_version as merged output if current_version is the same and there is an update - scenario AAB', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + query: 'query where true', + language: 'eql', + filters: [], + }, + current_version: { + query: 'query where true', + language: 'eql', + filters: [], + }, + target_version: { + query: 'query where true', + language: 'eql', + filters: [{ query: 'some_field' }], + }, + }; + + const result = eqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.target_version, + diff_outcome: ThreeWayDiffOutcome.StockValueCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Target, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + it('returns current_version as merged output if current version is different but it matches the update - scenario ABB', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + query: 'query where true', + language: 'eql', + filters: [], + }, + current_version: { + query: 'query where false', + language: 'eql', + filters: [], + }, + target_version: { + query: 'query where false', + language: 'eql', + filters: [], + }, + }; + + const result = eqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueSameUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + describe('if all three versions are different - scenario ABC', () => { + it('returns a computated merged version with a solvable conflict if 3-way query field merge is possible', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + query: 'My description.\f\nThis is a second\u2001 line.\f\nThis is a third line.', + language: 'eql', + filters: [], + }, + current_version: { + query: 'My GREAT description.\f\nThis is a second\u2001 line.\f\nThis is a third line.', + language: 'eql', + filters: [], + }, + target_version: { + query: 'My description.\f\nThis is a second\u2001 line.\f\nThis is a GREAT line.', + language: 'eql', + filters: [], + }, + }; + + const expectedMergedVersion: RuleEqlQuery = { + query: `My GREAT description.\f\nThis is a second\u2001 line.\f\nThis is a GREAT line.`, + language: 'eql', + filters: [], + }; + + const result = eqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: expectedMergedVersion, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Merged, + conflict: ThreeWayDiffConflict.SOLVABLE, + }) + ); + }); + + it('returns the current_version with a non-solvable conflict if 3-way query field merge is not possible', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + query: 'My description.\nThis is a second line.', + language: 'eql', + filters: [], + }, + current_version: { + query: 'My GREAT description.\nThis is a third line.', + language: 'eql', + filters: [], + }, + target_version: { + query: 'My EXCELLENT description.\nThis is a fourth.', + language: 'eql', + filters: [], + }, + }; + + const result = eqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NON_SOLVABLE, + }) + ); + }); + + it('returns the current_version with a non-solvable conflict if non-mergeable fields are different', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + query: 'query where false', + language: 'eql', + filters: [], + }, + current_version: { + query: 'query where false', + language: 'eql', + filters: [{ field: 'some query' }], + }, + target_version: { + query: 'query where false', + language: 'eql', + filters: [{ field: 'a different query' }], + }, + }; + + const result = eqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NON_SOLVABLE, + }) + ); + }); + }); + + describe('if base_version is missing', () => { + it('returns current_version as merged output if current_version and target_version are the same - scenario -AA', () => { + const mockVersions: ThreeVersionsOf = { + base_version: MissingVersion, + current_version: { + query: 'query where true', + language: 'eql', + filters: [], + }, + target_version: { + query: 'query where true', + language: 'eql', + filters: [], + }, + }; + + const result = eqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + has_base_version: false, + base_version: undefined, + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.MissingBaseNoUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + it('returns target_version as merged output if current_version and target_version are different - scenario -AB', () => { + const mockVersions: ThreeVersionsOf = { + base_version: MissingVersion, + current_version: { + query: 'query where true', + language: 'eql', + filters: [], + }, + target_version: { + query: 'query where false', + language: 'eql', + filters: [], + }, + }; + + const result = eqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + has_base_version: false, + base_version: undefined, + merged_version: mockVersions.target_version, + diff_outcome: ThreeWayDiffOutcome.MissingBaseCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Target, + conflict: ThreeWayDiffConflict.SOLVABLE, + }) + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.ts new file mode 100644 index 0000000000000..948e5f33cf69c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { merge } from 'node-diff3'; +import { assertUnreachable } from '../../../../../../../../common/utility_types'; +import type { + RuleEqlQuery, + ThreeVersionsOf, + ThreeWayDiff, +} from '../../../../../../../../common/api/detection_engine/prebuilt_rules'; +import { + determineIfValueCanUpdate, + ThreeWayDiffOutcome, + ThreeWayMergeOutcome, + MissingVersion, + ThreeWayDiffConflict, + determineDiffOutcome, +} from '../../../../../../../../common/api/detection_engine/prebuilt_rules'; +import { determineIfAllVersionsAreEqual } from './helpers'; + +/** + * Diff algorithm for eql query types + */ +export const eqlQueryDiffAlgorithm = ( + versions: ThreeVersionsOf +): ThreeWayDiff => { + const { + base_version: baseVersion, + current_version: currentVersion, + target_version: targetVersion, + } = versions; + + const diffOutcome = determineDiffOutcome(baseVersion, currentVersion, targetVersion); + + const valueCanUpdate = determineIfValueCanUpdate(diffOutcome); + + const hasBaseVersion = baseVersion !== MissingVersion; + + const { mergeOutcome, conflict, mergedVersion } = mergeVersions({ + baseVersion: hasBaseVersion ? baseVersion : undefined, + currentVersion, + targetVersion, + diffOutcome, + }); + + return { + has_base_version: hasBaseVersion, + base_version: hasBaseVersion ? baseVersion : undefined, + current_version: currentVersion, + target_version: targetVersion, + merged_version: mergedVersion, + merge_outcome: mergeOutcome, + + diff_outcome: diffOutcome, + conflict, + has_update: valueCanUpdate, + }; +}; + +interface MergeResult { + mergeOutcome: ThreeWayMergeOutcome; + mergedVersion: RuleEqlQuery; + conflict: ThreeWayDiffConflict; +} + +interface MergeArgs { + baseVersion: RuleEqlQuery | undefined; + currentVersion: RuleEqlQuery; + targetVersion: RuleEqlQuery; + diffOutcome: ThreeWayDiffOutcome; +} + +const mergeVersions = ({ + baseVersion, + currentVersion, + targetVersion, + diffOutcome, +}: MergeArgs): MergeResult => { + switch (diffOutcome) { + // Scenario -AA is treated as scenario AAA: + // https://github.com/elastic/kibana/pull/184889#discussion_r1636421293 + case ThreeWayDiffOutcome.MissingBaseNoUpdate: + case ThreeWayDiffOutcome.StockValueNoUpdate: + case ThreeWayDiffOutcome.CustomizedValueNoUpdate: + case ThreeWayDiffOutcome.CustomizedValueSameUpdate: + return { + conflict: ThreeWayDiffConflict.NONE, + mergeOutcome: ThreeWayMergeOutcome.Current, + mergedVersion: currentVersion, + }; + + case ThreeWayDiffOutcome.StockValueCanUpdate: + return { + conflict: ThreeWayDiffConflict.NONE, + mergeOutcome: ThreeWayMergeOutcome.Target, + mergedVersion: targetVersion, + }; + + case ThreeWayDiffOutcome.CustomizedValueCanUpdate: { + if (baseVersion) { + // TS does not realize that in ABC scenario, baseVersion cannot be missing + // Missing baseVersion scenarios were handled as -AA and -AB. + const mergedVersion = merge(currentVersion.query, baseVersion.query, targetVersion.query, { + stringSeparator: /(\S+|\s+)/g, // Retains all whitespace, which we keep to preserve formatting + }); + + // Determines if all non-mergeable fields are equal to one another + // filters are the only other variable field in the `RuleEqlQuery` type + const nonMergeableFieldsEqual = determineIfAllVersionsAreEqual( + baseVersion.filters, + currentVersion.filters, + targetVersion.filters + ); + + if (nonMergeableFieldsEqual && mergedVersion.conflict === false) { + return { + conflict: ThreeWayDiffConflict.SOLVABLE, + mergedVersion: { ...currentVersion, query: mergedVersion.result.join('') }, + mergeOutcome: ThreeWayMergeOutcome.Merged, + }; + } + } + + return { + conflict: ThreeWayDiffConflict.NON_SOLVABLE, + mergeOutcome: ThreeWayMergeOutcome.Current, + mergedVersion: currentVersion, + }; + } + + // Scenario -AB is treated as scenario ABC, but marked as + // SOLVABLE, and returns the target version as the merged version + // https://github.com/elastic/kibana/pull/184889#discussion_r1636421293 + case ThreeWayDiffOutcome.MissingBaseCanUpdate: { + return { + mergedVersion: targetVersion, + mergeOutcome: ThreeWayMergeOutcome.Target, + conflict: ThreeWayDiffConflict.SOLVABLE, + }; + } + default: + return assertUnreachable(diffOutcome); + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/helpers.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/helpers.ts index 498b857eb0428..fa6e75cfcc88e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/helpers.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { difference, union, uniq } from 'lodash'; +import { difference, isEqual, union, uniq } from 'lodash'; import type { RuleDataSource } from '../../../../../../../../common/api/detection_engine'; import { DataSourceType } from '../../../../../../../../common/api/detection_engine'; @@ -35,3 +35,15 @@ export const getDedupedDataSourceVersion = (version: RuleDataSource): RuleDataSo } return version; }; + +export const determineIfAllVersionsAreEqual = ( + baseVersion: T, + currentVersion: T, + targetVersion: T +): boolean => { + const baseEqlCurrent = isEqual(baseVersion, currentVersion); + const baseEqlTarget = isEqual(baseVersion, targetVersion); + const currentEqlTarget = isEqual(currentVersion, targetVersion); + + return baseEqlCurrent && baseEqlTarget && currentEqlTarget; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/index.ts index d9ad131fc51ab..ccc0250d3bb92 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/index.ts @@ -12,3 +12,4 @@ export { simpleDiffAlgorithm } from './simple_diff_algorithm'; export { multiLineStringDiffAlgorithm } from './multi_line_string_diff_algorithm'; export { dataSourceDiffAlgorithm } from './data_source_diff_algorithm'; export { kqlQueryDiffAlgorithm } from './kql_query_diff_algorithm'; +export { eqlQueryDiffAlgorithm } from './eql_query_diff_algorithm'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts index 054a4f925f74e..d752b1aecee1f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts @@ -316,7 +316,7 @@ describe('kqlQueryDiffAlgorithm', () => { }); describe('if all versions are inline query type', () => { - it('returns a computated merged version without a conflict if 3 way merge is possible', () => { + it('returns a computated merged version with a solvable conflict if 3-way query field merge is possible', () => { const mockVersions: ThreeVersionsOf = { base_version: { type: KqlQueryType.inline_query, @@ -357,7 +357,7 @@ describe('kqlQueryDiffAlgorithm', () => { ); }); - it('returns the current_version with a conflict if 3 way merge is not possible', () => { + it('returns the current_version with a non-solvable conflict if 3-way query field merge is not possible', () => { const mockVersions: ThreeVersionsOf = { base_version: { type: KqlQueryType.inline_query, @@ -390,10 +390,44 @@ describe('kqlQueryDiffAlgorithm', () => { }) ); }); + + it('returns the current_version with a non-solvable conflict if non-mergeable fields are not equal', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + type: KqlQueryType.inline_query, + query: 'query string = false', + language: KqlQueryLanguageEnum.lucene, + filters: [], + }, + current_version: { + type: KqlQueryType.inline_query, + query: 'query string = false', + language: KqlQueryLanguageEnum.kuery, + filters: [], + }, + target_version: { + type: KqlQueryType.inline_query, + query: 'query string = false', + language: KqlQueryLanguageEnum.kuery, + filters: [{ field: 'some query' }], + }, + }; + + const result = kqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NON_SOLVABLE, + }) + ); + }); }); describe('if versions are different types', () => { - it('returns the current_version with a conflict', () => { + it('returns the current_version with a non-solvable conflict', () => { const mockVersions: ThreeVersionsOf = { base_version: { type: KqlQueryType.inline_query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.ts index be9776fb5992d..70bb709e4cec8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.ts @@ -6,7 +6,6 @@ */ import { merge } from 'node-diff3'; -import { isEqual } from 'lodash'; import { assertUnreachable } from '../../../../../../../../common/utility_types'; import type { RuleKqlQuery, @@ -22,10 +21,14 @@ import { determineDiffOutcome, KqlQueryType, } from '../../../../../../../../common/api/detection_engine/prebuilt_rules'; +import { determineIfAllVersionsAreEqual } from './helpers'; -export const kqlQueryDiffAlgorithm = ( - versions: ThreeVersionsOf -): ThreeWayDiff => { +/** + * Diff algorithm for all kql query types (`inline_query` and `saved_query`) + */ +export const kqlQueryDiffAlgorithm = ( + versions: ThreeVersionsOf +): ThreeWayDiff => { const { base_version: baseVersion, current_version: currentVersion, @@ -59,25 +62,25 @@ export const kqlQueryDiffAlgorithm = ( }; }; -interface MergeResult { +interface MergeResult { mergeOutcome: ThreeWayMergeOutcome; - mergedVersion: RuleKqlQuery; + mergedVersion: T; conflict: ThreeWayDiffConflict; } -interface MergeArgs { - baseVersion: RuleKqlQuery | undefined; - currentVersion: RuleKqlQuery; - targetVersion: RuleKqlQuery; +interface MergeArgs { + baseVersion: T | undefined; + currentVersion: T; + targetVersion: T; diffOutcome: ThreeWayDiffOutcome; } -const mergeVersions = ({ +const mergeVersions = ({ baseVersion, currentVersion, targetVersion, diffOutcome, -}: MergeArgs): MergeResult => { +}: MergeArgs): MergeResult => { switch (diffOutcome) { // Scenario -AA is treated as scenario AAA: // https://github.com/elastic/kibana/pull/184889#discussion_r1636421293 @@ -126,16 +129,14 @@ const mergeVersions = ({ language: targetVersion.language, }; - const baseEqlCurrent = isEqual(baseNonMergeableFields, currentNonMergeableFields); - const baseEqlTarget = isEqual(baseNonMergeableFields, targetNonMergeableFields); - const currentEqlTarget = isEqual(currentNonMergeableFields, targetNonMergeableFields); + // Determines if all non-mergeable fields are equal to one another + const nonMergeableFieldsEqual = determineIfAllVersionsAreEqual( + baseNonMergeableFields, + currentNonMergeableFields, + targetNonMergeableFields + ); - if ( - baseEqlCurrent && - baseEqlTarget && - currentEqlTarget && - mergedVersion.conflict === false - ) { + if (nonMergeableFieldsEqual && mergedVersion.conflict === false) { return { conflict: ThreeWayDiffConflict.SOLVABLE, mergedVersion: { ...currentVersion, query: mergedVersion.result.join('') }, From bf6e35672b1588a0437c697b03d28b5be505ab80 Mon Sep 17 00:00:00 2001 From: Davis Plumlee Date: Thu, 8 Aug 2024 14:59:17 -0400 Subject: [PATCH 3/5] adds esql query diff algorithm --- .../esql_query_diff_algorithm.test.ts | 251 ++++++++++++++++++ .../algorithms/esql_query_diff_algorithm.ts | 139 ++++++++++ .../diff/calculation/algorithms/index.ts | 1 + 3 files changed, 391 insertions(+) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.test.ts new file mode 100644 index 0000000000000..0c04c8a15ea50 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.test.ts @@ -0,0 +1,251 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + RuleEsqlQuery, + ThreeVersionsOf, +} from '../../../../../../../../common/api/detection_engine'; +import { + ThreeWayDiffOutcome, + ThreeWayMergeOutcome, + MissingVersion, + ThreeWayDiffConflict, +} from '../../../../../../../../common/api/detection_engine'; +import { esqlQueryDiffAlgorithm } from './esql_query_diff_algorithm'; + +describe('esqlQueryDiffAlgorithm', () => { + it('returns current_version as merged output if there is no update - scenario AAA', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + query: 'query where true', + language: 'esql', + }, + current_version: { + query: 'query where true', + language: 'esql', + }, + target_version: { + query: 'query where true', + language: 'esql', + }, + }; + + const result = esqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.StockValueNoUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + it('returns current_version as merged output if current_version is different and there is no update - scenario ABA', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + query: 'query where true', + language: 'esql', + }, + current_version: { + query: 'query where false', + language: 'esql', + }, + target_version: { + query: 'query where true', + language: 'esql', + }, + }; + + const result = esqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueNoUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + it('returns target_version as merged output if current_version is the same and there is an update - scenario AAB', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + query: 'query where true', + language: 'esql', + }, + current_version: { + query: 'query where true', + language: 'esql', + }, + target_version: { + query: 'query where false', + language: 'esql', + }, + }; + + const result = esqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.target_version, + diff_outcome: ThreeWayDiffOutcome.StockValueCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Target, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + it('returns current_version as merged output if current version is different but it matches the update - scenario ABB', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + query: 'query where true', + language: 'esql', + }, + current_version: { + query: 'query where false', + language: 'esql', + }, + target_version: { + query: 'query where false', + language: 'esql', + }, + }; + + const result = esqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueSameUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + describe('if all three versions are different - scenario ABC', () => { + it('returns a computated merged version with a solvable conflict if 3-way query field merge is possible', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + query: 'My description.\f\nThis is a second\u2001 line.\f\nThis is a third line.', + language: 'esql', + }, + current_version: { + query: 'My GREAT description.\f\nThis is a second\u2001 line.\f\nThis is a third line.', + language: 'esql', + }, + target_version: { + query: 'My description.\f\nThis is a second\u2001 line.\f\nThis is a GREAT line.', + language: 'esql', + }, + }; + + const expectedMergedVersion: RuleEsqlQuery = { + query: `My GREAT description.\f\nThis is a second\u2001 line.\f\nThis is a GREAT line.`, + language: 'esql', + }; + + const result = esqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: expectedMergedVersion, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Merged, + conflict: ThreeWayDiffConflict.SOLVABLE, + }) + ); + }); + + it('returns the current_version with a non-solvable conflict if 3-way query field merge is not possible', () => { + const mockVersions: ThreeVersionsOf = { + base_version: { + query: 'My description.\nThis is a second line.', + language: 'esql', + }, + current_version: { + query: 'My GREAT description.\nThis is a third line.', + language: 'esql', + }, + target_version: { + query: 'My EXCELLENT description.\nThis is a fourth.', + language: 'esql', + }, + }; + + const result = esqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.CustomizedValueCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NON_SOLVABLE, + }) + ); + }); + }); + + describe('if base_version is missing', () => { + it('returns current_version as merged output if current_version and target_version are the same - scenario -AA', () => { + const mockVersions: ThreeVersionsOf = { + base_version: MissingVersion, + current_version: { + query: 'query where true', + language: 'esql', + }, + target_version: { + query: 'query where true', + language: 'esql', + }, + }; + + const result = esqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + has_base_version: false, + base_version: undefined, + merged_version: mockVersions.current_version, + diff_outcome: ThreeWayDiffOutcome.MissingBaseNoUpdate, + merge_outcome: ThreeWayMergeOutcome.Current, + conflict: ThreeWayDiffConflict.NONE, + }) + ); + }); + + it('returns target_version as merged output if current_version and target_version are different - scenario -AB', () => { + const mockVersions: ThreeVersionsOf = { + base_version: MissingVersion, + current_version: { + query: 'query where true', + language: 'esql', + }, + target_version: { + query: 'query where false', + language: 'esql', + }, + }; + + const result = esqlQueryDiffAlgorithm(mockVersions); + + expect(result).toEqual( + expect.objectContaining({ + has_base_version: false, + base_version: undefined, + merged_version: mockVersions.target_version, + diff_outcome: ThreeWayDiffOutcome.MissingBaseCanUpdate, + merge_outcome: ThreeWayMergeOutcome.Target, + conflict: ThreeWayDiffConflict.SOLVABLE, + }) + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.ts new file mode 100644 index 0000000000000..1de4e85432a83 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { merge } from 'node-diff3'; +import { assertUnreachable } from '../../../../../../../../common/utility_types'; +import type { + RuleEsqlQuery, + ThreeVersionsOf, + ThreeWayDiff, +} from '../../../../../../../../common/api/detection_engine/prebuilt_rules'; +import { + determineIfValueCanUpdate, + ThreeWayDiffOutcome, + ThreeWayMergeOutcome, + MissingVersion, + ThreeWayDiffConflict, + determineDiffOutcome, +} from '../../../../../../../../common/api/detection_engine/prebuilt_rules'; + +/** + * Diff algorithm for esql query types + */ +export const esqlQueryDiffAlgorithm = ( + versions: ThreeVersionsOf +): ThreeWayDiff => { + const { + base_version: baseVersion, + current_version: currentVersion, + target_version: targetVersion, + } = versions; + + const diffOutcome = determineDiffOutcome(baseVersion, currentVersion, targetVersion); + + const valueCanUpdate = determineIfValueCanUpdate(diffOutcome); + + const hasBaseVersion = baseVersion !== MissingVersion; + + const { mergeOutcome, conflict, mergedVersion } = mergeVersions({ + baseVersion: hasBaseVersion ? baseVersion : undefined, + currentVersion, + targetVersion, + diffOutcome, + }); + + return { + has_base_version: hasBaseVersion, + base_version: hasBaseVersion ? baseVersion : undefined, + current_version: currentVersion, + target_version: targetVersion, + merged_version: mergedVersion, + merge_outcome: mergeOutcome, + + diff_outcome: diffOutcome, + conflict, + has_update: valueCanUpdate, + }; +}; + +interface MergeResult { + mergeOutcome: ThreeWayMergeOutcome; + mergedVersion: RuleEsqlQuery; + conflict: ThreeWayDiffConflict; +} + +interface MergeArgs { + baseVersion: RuleEsqlQuery | undefined; + currentVersion: RuleEsqlQuery; + targetVersion: RuleEsqlQuery; + diffOutcome: ThreeWayDiffOutcome; +} + +const mergeVersions = ({ + baseVersion, + currentVersion, + targetVersion, + diffOutcome, +}: MergeArgs): MergeResult => { + switch (diffOutcome) { + // Scenario -AA is treated as scenario AAA: + // https://github.com/elastic/kibana/pull/184889#discussion_r1636421293 + case ThreeWayDiffOutcome.MissingBaseNoUpdate: + case ThreeWayDiffOutcome.StockValueNoUpdate: + case ThreeWayDiffOutcome.CustomizedValueNoUpdate: + case ThreeWayDiffOutcome.CustomizedValueSameUpdate: + return { + conflict: ThreeWayDiffConflict.NONE, + mergeOutcome: ThreeWayMergeOutcome.Current, + mergedVersion: currentVersion, + }; + + case ThreeWayDiffOutcome.StockValueCanUpdate: + return { + conflict: ThreeWayDiffConflict.NONE, + mergeOutcome: ThreeWayMergeOutcome.Target, + mergedVersion: targetVersion, + }; + + case ThreeWayDiffOutcome.CustomizedValueCanUpdate: { + if (baseVersion) { + // TS does not realize that in ABC scenario, baseVersion cannot be missing + // Missing baseVersion scenarios were handled as -AA and -AB. + const mergedVersion = merge(currentVersion.query, baseVersion.query, targetVersion.query, { + stringSeparator: /(\S+|\s+)/g, // Retains all whitespace, which we keep to preserve formatting + }); + + if (mergedVersion.conflict === false) { + return { + conflict: ThreeWayDiffConflict.SOLVABLE, + mergedVersion: { ...currentVersion, query: mergedVersion.result.join('') }, + mergeOutcome: ThreeWayMergeOutcome.Merged, + }; + } + } + + return { + conflict: ThreeWayDiffConflict.NON_SOLVABLE, + mergeOutcome: ThreeWayMergeOutcome.Current, + mergedVersion: currentVersion, + }; + } + + // Scenario -AB is treated as scenario ABC, but marked as + // SOLVABLE, and returns the target version as the merged version + // https://github.com/elastic/kibana/pull/184889#discussion_r1636421293 + case ThreeWayDiffOutcome.MissingBaseCanUpdate: { + return { + mergedVersion: targetVersion, + mergeOutcome: ThreeWayMergeOutcome.Target, + conflict: ThreeWayDiffConflict.SOLVABLE, + }; + } + default: + return assertUnreachable(diffOutcome); + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/index.ts index ccc0250d3bb92..629f329c72b9b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/index.ts @@ -13,3 +13,4 @@ export { multiLineStringDiffAlgorithm } from './multi_line_string_diff_algorithm export { dataSourceDiffAlgorithm } from './data_source_diff_algorithm'; export { kqlQueryDiffAlgorithm } from './kql_query_diff_algorithm'; export { eqlQueryDiffAlgorithm } from './eql_query_diff_algorithm'; +export { esqlQueryDiffAlgorithm } from './esql_query_diff_algorithm'; From f34a6437c2f58630bcb403260276f00dec905dc6 Mon Sep 17 00:00:00 2001 From: Davis Plumlee Date: Tue, 13 Aug 2024 13:38:58 -0400 Subject: [PATCH 4/5] removes the mergeable logic in algorithms --- .../eql_query_diff_algorithm.test.ts | 51 +----- .../algorithms/eql_query_diff_algorithm.ts | 136 +-------------- .../esql_query_diff_algorithm.test.ts | 41 +---- .../algorithms/esql_query_diff_algorithm.ts | 127 +------------- .../kql_query_diff_algorithm.test.ts | 53 +----- .../algorithms/kql_query_diff_algorithm.ts | 157 +----------------- 6 files changed, 27 insertions(+), 538 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.test.ts index 203c39ad62df1..acbb63016aeda 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.test.ts @@ -143,57 +143,20 @@ describe('eqlQueryDiffAlgorithm', () => { }); describe('if all three versions are different - scenario ABC', () => { - it('returns a computated merged version with a solvable conflict if 3-way query field merge is possible', () => { + it('returns the current_version with a non-solvable conflict', () => { const mockVersions: ThreeVersionsOf = { base_version: { - query: 'My description.\f\nThis is a second\u2001 line.\f\nThis is a third line.', - language: 'eql', - filters: [], - }, - current_version: { - query: 'My GREAT description.\f\nThis is a second\u2001 line.\f\nThis is a third line.', - language: 'eql', - filters: [], - }, - target_version: { - query: 'My description.\f\nThis is a second\u2001 line.\f\nThis is a GREAT line.', - language: 'eql', - filters: [], - }, - }; - - const expectedMergedVersion: RuleEqlQuery = { - query: `My GREAT description.\f\nThis is a second\u2001 line.\f\nThis is a GREAT line.`, - language: 'eql', - filters: [], - }; - - const result = eqlQueryDiffAlgorithm(mockVersions); - - expect(result).toEqual( - expect.objectContaining({ - merged_version: expectedMergedVersion, - diff_outcome: ThreeWayDiffOutcome.CustomizedValueCanUpdate, - merge_outcome: ThreeWayMergeOutcome.Merged, - conflict: ThreeWayDiffConflict.SOLVABLE, - }) - ); - }); - - it('returns the current_version with a non-solvable conflict if 3-way query field merge is not possible', () => { - const mockVersions: ThreeVersionsOf = { - base_version: { - query: 'My description.\nThis is a second line.', + query: 'query where false', language: 'eql', filters: [], }, current_version: { - query: 'My GREAT description.\nThis is a third line.', + query: 'query where true', language: 'eql', filters: [], }, target_version: { - query: 'My EXCELLENT description.\nThis is a fourth.', + query: 'query two where false', language: 'eql', filters: [], }, @@ -211,7 +174,7 @@ describe('eqlQueryDiffAlgorithm', () => { ); }); - it('returns the current_version with a non-solvable conflict if non-mergeable fields are different', () => { + it('returns the current_version with a non-solvable conflict if one subfield has an ABA scenario and another has an AAB', () => { const mockVersions: ThreeVersionsOf = { base_version: { query: 'query where false', @@ -224,9 +187,9 @@ describe('eqlQueryDiffAlgorithm', () => { filters: [{ field: 'some query' }], }, target_version: { - query: 'query where false', + query: 'query where true', language: 'eql', - filters: [{ field: 'a different query' }], + filters: [], }, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.ts index 948e5f33cf69c..fa3e87397a2a7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/eql_query_diff_algorithm.ts @@ -5,144 +5,14 @@ * 2.0. */ -import { merge } from 'node-diff3'; -import { assertUnreachable } from '../../../../../../../../common/utility_types'; import type { RuleEqlQuery, ThreeVersionsOf, - ThreeWayDiff, } from '../../../../../../../../common/api/detection_engine/prebuilt_rules'; -import { - determineIfValueCanUpdate, - ThreeWayDiffOutcome, - ThreeWayMergeOutcome, - MissingVersion, - ThreeWayDiffConflict, - determineDiffOutcome, -} from '../../../../../../../../common/api/detection_engine/prebuilt_rules'; -import { determineIfAllVersionsAreEqual } from './helpers'; +import { simpleDiffAlgorithm } from './simple_diff_algorithm'; /** * Diff algorithm for eql query types */ -export const eqlQueryDiffAlgorithm = ( - versions: ThreeVersionsOf -): ThreeWayDiff => { - const { - base_version: baseVersion, - current_version: currentVersion, - target_version: targetVersion, - } = versions; - - const diffOutcome = determineDiffOutcome(baseVersion, currentVersion, targetVersion); - - const valueCanUpdate = determineIfValueCanUpdate(diffOutcome); - - const hasBaseVersion = baseVersion !== MissingVersion; - - const { mergeOutcome, conflict, mergedVersion } = mergeVersions({ - baseVersion: hasBaseVersion ? baseVersion : undefined, - currentVersion, - targetVersion, - diffOutcome, - }); - - return { - has_base_version: hasBaseVersion, - base_version: hasBaseVersion ? baseVersion : undefined, - current_version: currentVersion, - target_version: targetVersion, - merged_version: mergedVersion, - merge_outcome: mergeOutcome, - - diff_outcome: diffOutcome, - conflict, - has_update: valueCanUpdate, - }; -}; - -interface MergeResult { - mergeOutcome: ThreeWayMergeOutcome; - mergedVersion: RuleEqlQuery; - conflict: ThreeWayDiffConflict; -} - -interface MergeArgs { - baseVersion: RuleEqlQuery | undefined; - currentVersion: RuleEqlQuery; - targetVersion: RuleEqlQuery; - diffOutcome: ThreeWayDiffOutcome; -} - -const mergeVersions = ({ - baseVersion, - currentVersion, - targetVersion, - diffOutcome, -}: MergeArgs): MergeResult => { - switch (diffOutcome) { - // Scenario -AA is treated as scenario AAA: - // https://github.com/elastic/kibana/pull/184889#discussion_r1636421293 - case ThreeWayDiffOutcome.MissingBaseNoUpdate: - case ThreeWayDiffOutcome.StockValueNoUpdate: - case ThreeWayDiffOutcome.CustomizedValueNoUpdate: - case ThreeWayDiffOutcome.CustomizedValueSameUpdate: - return { - conflict: ThreeWayDiffConflict.NONE, - mergeOutcome: ThreeWayMergeOutcome.Current, - mergedVersion: currentVersion, - }; - - case ThreeWayDiffOutcome.StockValueCanUpdate: - return { - conflict: ThreeWayDiffConflict.NONE, - mergeOutcome: ThreeWayMergeOutcome.Target, - mergedVersion: targetVersion, - }; - - case ThreeWayDiffOutcome.CustomizedValueCanUpdate: { - if (baseVersion) { - // TS does not realize that in ABC scenario, baseVersion cannot be missing - // Missing baseVersion scenarios were handled as -AA and -AB. - const mergedVersion = merge(currentVersion.query, baseVersion.query, targetVersion.query, { - stringSeparator: /(\S+|\s+)/g, // Retains all whitespace, which we keep to preserve formatting - }); - - // Determines if all non-mergeable fields are equal to one another - // filters are the only other variable field in the `RuleEqlQuery` type - const nonMergeableFieldsEqual = determineIfAllVersionsAreEqual( - baseVersion.filters, - currentVersion.filters, - targetVersion.filters - ); - - if (nonMergeableFieldsEqual && mergedVersion.conflict === false) { - return { - conflict: ThreeWayDiffConflict.SOLVABLE, - mergedVersion: { ...currentVersion, query: mergedVersion.result.join('') }, - mergeOutcome: ThreeWayMergeOutcome.Merged, - }; - } - } - - return { - conflict: ThreeWayDiffConflict.NON_SOLVABLE, - mergeOutcome: ThreeWayMergeOutcome.Current, - mergedVersion: currentVersion, - }; - } - - // Scenario -AB is treated as scenario ABC, but marked as - // SOLVABLE, and returns the target version as the merged version - // https://github.com/elastic/kibana/pull/184889#discussion_r1636421293 - case ThreeWayDiffOutcome.MissingBaseCanUpdate: { - return { - mergedVersion: targetVersion, - mergeOutcome: ThreeWayMergeOutcome.Target, - conflict: ThreeWayDiffConflict.SOLVABLE, - }; - } - default: - return assertUnreachable(diffOutcome); - } -}; +export const eqlQueryDiffAlgorithm = (versions: ThreeVersionsOf) => + simpleDiffAlgorithm(versions); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.test.ts index 0c04c8a15ea50..ecaaa4142b5e0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.test.ts @@ -131,51 +131,18 @@ describe('esqlQueryDiffAlgorithm', () => { }); describe('if all three versions are different - scenario ABC', () => { - it('returns a computated merged version with a solvable conflict if 3-way query field merge is possible', () => { + it('returns the current_version with a non-solvable conflict', () => { const mockVersions: ThreeVersionsOf = { base_version: { - query: 'My description.\f\nThis is a second\u2001 line.\f\nThis is a third line.', - language: 'esql', - }, - current_version: { - query: 'My GREAT description.\f\nThis is a second\u2001 line.\f\nThis is a third line.', - language: 'esql', - }, - target_version: { - query: 'My description.\f\nThis is a second\u2001 line.\f\nThis is a GREAT line.', - language: 'esql', - }, - }; - - const expectedMergedVersion: RuleEsqlQuery = { - query: `My GREAT description.\f\nThis is a second\u2001 line.\f\nThis is a GREAT line.`, - language: 'esql', - }; - - const result = esqlQueryDiffAlgorithm(mockVersions); - - expect(result).toEqual( - expect.objectContaining({ - merged_version: expectedMergedVersion, - diff_outcome: ThreeWayDiffOutcome.CustomizedValueCanUpdate, - merge_outcome: ThreeWayMergeOutcome.Merged, - conflict: ThreeWayDiffConflict.SOLVABLE, - }) - ); - }); - - it('returns the current_version with a non-solvable conflict if 3-way query field merge is not possible', () => { - const mockVersions: ThreeVersionsOf = { - base_version: { - query: 'My description.\nThis is a second line.', + query: 'query where true', language: 'esql', }, current_version: { - query: 'My GREAT description.\nThis is a third line.', + query: 'query where false', language: 'esql', }, target_version: { - query: 'My EXCELLENT description.\nThis is a fourth.', + query: 'different query where true', language: 'esql', }, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.ts index 1de4e85432a83..8360800ad4676 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/esql_query_diff_algorithm.ts @@ -5,135 +5,14 @@ * 2.0. */ -import { merge } from 'node-diff3'; -import { assertUnreachable } from '../../../../../../../../common/utility_types'; import type { RuleEsqlQuery, ThreeVersionsOf, - ThreeWayDiff, -} from '../../../../../../../../common/api/detection_engine/prebuilt_rules'; -import { - determineIfValueCanUpdate, - ThreeWayDiffOutcome, - ThreeWayMergeOutcome, - MissingVersion, - ThreeWayDiffConflict, - determineDiffOutcome, } from '../../../../../../../../common/api/detection_engine/prebuilt_rules'; +import { simpleDiffAlgorithm } from './simple_diff_algorithm'; /** * Diff algorithm for esql query types */ -export const esqlQueryDiffAlgorithm = ( - versions: ThreeVersionsOf -): ThreeWayDiff => { - const { - base_version: baseVersion, - current_version: currentVersion, - target_version: targetVersion, - } = versions; - - const diffOutcome = determineDiffOutcome(baseVersion, currentVersion, targetVersion); - - const valueCanUpdate = determineIfValueCanUpdate(diffOutcome); - - const hasBaseVersion = baseVersion !== MissingVersion; - - const { mergeOutcome, conflict, mergedVersion } = mergeVersions({ - baseVersion: hasBaseVersion ? baseVersion : undefined, - currentVersion, - targetVersion, - diffOutcome, - }); - - return { - has_base_version: hasBaseVersion, - base_version: hasBaseVersion ? baseVersion : undefined, - current_version: currentVersion, - target_version: targetVersion, - merged_version: mergedVersion, - merge_outcome: mergeOutcome, - - diff_outcome: diffOutcome, - conflict, - has_update: valueCanUpdate, - }; -}; - -interface MergeResult { - mergeOutcome: ThreeWayMergeOutcome; - mergedVersion: RuleEsqlQuery; - conflict: ThreeWayDiffConflict; -} - -interface MergeArgs { - baseVersion: RuleEsqlQuery | undefined; - currentVersion: RuleEsqlQuery; - targetVersion: RuleEsqlQuery; - diffOutcome: ThreeWayDiffOutcome; -} - -const mergeVersions = ({ - baseVersion, - currentVersion, - targetVersion, - diffOutcome, -}: MergeArgs): MergeResult => { - switch (diffOutcome) { - // Scenario -AA is treated as scenario AAA: - // https://github.com/elastic/kibana/pull/184889#discussion_r1636421293 - case ThreeWayDiffOutcome.MissingBaseNoUpdate: - case ThreeWayDiffOutcome.StockValueNoUpdate: - case ThreeWayDiffOutcome.CustomizedValueNoUpdate: - case ThreeWayDiffOutcome.CustomizedValueSameUpdate: - return { - conflict: ThreeWayDiffConflict.NONE, - mergeOutcome: ThreeWayMergeOutcome.Current, - mergedVersion: currentVersion, - }; - - case ThreeWayDiffOutcome.StockValueCanUpdate: - return { - conflict: ThreeWayDiffConflict.NONE, - mergeOutcome: ThreeWayMergeOutcome.Target, - mergedVersion: targetVersion, - }; - - case ThreeWayDiffOutcome.CustomizedValueCanUpdate: { - if (baseVersion) { - // TS does not realize that in ABC scenario, baseVersion cannot be missing - // Missing baseVersion scenarios were handled as -AA and -AB. - const mergedVersion = merge(currentVersion.query, baseVersion.query, targetVersion.query, { - stringSeparator: /(\S+|\s+)/g, // Retains all whitespace, which we keep to preserve formatting - }); - - if (mergedVersion.conflict === false) { - return { - conflict: ThreeWayDiffConflict.SOLVABLE, - mergedVersion: { ...currentVersion, query: mergedVersion.result.join('') }, - mergeOutcome: ThreeWayMergeOutcome.Merged, - }; - } - } - - return { - conflict: ThreeWayDiffConflict.NON_SOLVABLE, - mergeOutcome: ThreeWayMergeOutcome.Current, - mergedVersion: currentVersion, - }; - } - - // Scenario -AB is treated as scenario ABC, but marked as - // SOLVABLE, and returns the target version as the merged version - // https://github.com/elastic/kibana/pull/184889#discussion_r1636421293 - case ThreeWayDiffOutcome.MissingBaseCanUpdate: { - return { - mergedVersion: targetVersion, - mergeOutcome: ThreeWayMergeOutcome.Target, - conflict: ThreeWayDiffConflict.SOLVABLE, - }; - } - default: - return assertUnreachable(diffOutcome); - } -}; +export const esqlQueryDiffAlgorithm = (versions: ThreeVersionsOf) => + simpleDiffAlgorithm(versions); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts index d752b1aecee1f..60641d52e6381 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts @@ -316,64 +316,23 @@ describe('kqlQueryDiffAlgorithm', () => { }); describe('if all versions are inline query type', () => { - it('returns a computated merged version with a solvable conflict if 3-way query field merge is possible', () => { - const mockVersions: ThreeVersionsOf = { - base_version: { - type: KqlQueryType.inline_query, - query: `My description.\f\nThis is a second\u2001 line.\f\nThis is a third line.`, - language: KqlQueryLanguageEnum.kuery, - filters: [], - }, - current_version: { - type: KqlQueryType.inline_query, - query: `My GREAT description.\f\nThis is a second\u2001 line.\f\nThis is a third line.`, - language: KqlQueryLanguageEnum.kuery, - filters: [], - }, - target_version: { - type: KqlQueryType.inline_query, - query: `My description.\f\nThis is a second\u2001 line.\f\nThis is a GREAT line.`, - language: KqlQueryLanguageEnum.kuery, - filters: [], - }, - }; - - const expectedMergedVersion: RuleKqlQuery = { - type: KqlQueryType.inline_query, - query: `My GREAT description.\f\nThis is a second\u2001 line.\f\nThis is a GREAT line.`, - language: KqlQueryLanguageEnum.kuery, - filters: [], - }; - - const result = kqlQueryDiffAlgorithm(mockVersions); - - expect(result).toEqual( - expect.objectContaining({ - merged_version: expectedMergedVersion, - diff_outcome: ThreeWayDiffOutcome.CustomizedValueCanUpdate, - merge_outcome: ThreeWayMergeOutcome.Merged, - conflict: ThreeWayDiffConflict.SOLVABLE, - }) - ); - }); - - it('returns the current_version with a non-solvable conflict if 3-way query field merge is not possible', () => { + it('returns the current_version with a non-solvable conflict', () => { const mockVersions: ThreeVersionsOf = { base_version: { type: KqlQueryType.inline_query, - query: 'My description.\nThis is a second line.', + query: 'query string = true', language: KqlQueryLanguageEnum.kuery, filters: [], }, current_version: { type: KqlQueryType.inline_query, - query: 'My GREAT description.\nThis is a third line.', + query: 'query string = false', language: KqlQueryLanguageEnum.kuery, filters: [], }, target_version: { type: KqlQueryType.inline_query, - query: 'My EXCELLENT description.\nThis is a fourth.', + query: 'query string two = true', language: KqlQueryLanguageEnum.kuery, filters: [], }, @@ -391,7 +350,7 @@ describe('kqlQueryDiffAlgorithm', () => { ); }); - it('returns the current_version with a non-solvable conflict if non-mergeable fields are not equal', () => { + it('returns the current_version with a non-solvable conflict if one subfield has an ABA scenario and another has an AAB', () => { const mockVersions: ThreeVersionsOf = { base_version: { type: KqlQueryType.inline_query, @@ -401,7 +360,7 @@ describe('kqlQueryDiffAlgorithm', () => { }, current_version: { type: KqlQueryType.inline_query, - query: 'query string = false', + query: 'query string = true', language: KqlQueryLanguageEnum.kuery, filters: [], }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.ts index 70bb709e4cec8..500c3211cc9c2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.ts @@ -5,164 +5,15 @@ * 2.0. */ -import { merge } from 'node-diff3'; -import { assertUnreachable } from '../../../../../../../../common/utility_types'; import type { RuleKqlQuery, ThreeVersionsOf, - ThreeWayDiff, } from '../../../../../../../../common/api/detection_engine/prebuilt_rules'; -import { - determineIfValueCanUpdate, - ThreeWayDiffOutcome, - ThreeWayMergeOutcome, - MissingVersion, - ThreeWayDiffConflict, - determineDiffOutcome, - KqlQueryType, -} from '../../../../../../../../common/api/detection_engine/prebuilt_rules'; -import { determineIfAllVersionsAreEqual } from './helpers'; +import { simpleDiffAlgorithm } from './simple_diff_algorithm'; /** * Diff algorithm for all kql query types (`inline_query` and `saved_query`) */ -export const kqlQueryDiffAlgorithm = ( - versions: ThreeVersionsOf -): ThreeWayDiff => { - const { - base_version: baseVersion, - current_version: currentVersion, - target_version: targetVersion, - } = versions; - - const diffOutcome = determineDiffOutcome(baseVersion, currentVersion, targetVersion); - - const valueCanUpdate = determineIfValueCanUpdate(diffOutcome); - - const hasBaseVersion = baseVersion !== MissingVersion; - - const { mergeOutcome, conflict, mergedVersion } = mergeVersions({ - baseVersion: hasBaseVersion ? baseVersion : undefined, - currentVersion, - targetVersion, - diffOutcome, - }); - - return { - has_base_version: hasBaseVersion, - base_version: hasBaseVersion ? baseVersion : undefined, - current_version: currentVersion, - target_version: targetVersion, - merged_version: mergedVersion, - merge_outcome: mergeOutcome, - - diff_outcome: diffOutcome, - conflict, - has_update: valueCanUpdate, - }; -}; - -interface MergeResult { - mergeOutcome: ThreeWayMergeOutcome; - mergedVersion: T; - conflict: ThreeWayDiffConflict; -} - -interface MergeArgs { - baseVersion: T | undefined; - currentVersion: T; - targetVersion: T; - diffOutcome: ThreeWayDiffOutcome; -} - -const mergeVersions = ({ - baseVersion, - currentVersion, - targetVersion, - diffOutcome, -}: MergeArgs): MergeResult => { - switch (diffOutcome) { - // Scenario -AA is treated as scenario AAA: - // https://github.com/elastic/kibana/pull/184889#discussion_r1636421293 - case ThreeWayDiffOutcome.MissingBaseNoUpdate: - case ThreeWayDiffOutcome.StockValueNoUpdate: - case ThreeWayDiffOutcome.CustomizedValueNoUpdate: - case ThreeWayDiffOutcome.CustomizedValueSameUpdate: - return { - conflict: ThreeWayDiffConflict.NONE, - mergeOutcome: ThreeWayMergeOutcome.Current, - mergedVersion: currentVersion, - }; - - case ThreeWayDiffOutcome.StockValueCanUpdate: - return { - conflict: ThreeWayDiffConflict.NONE, - mergeOutcome: ThreeWayMergeOutcome.Target, - mergedVersion: targetVersion, - }; - - case ThreeWayDiffOutcome.CustomizedValueCanUpdate: { - if ( - baseVersion && - baseVersion.type === KqlQueryType.inline_query && - currentVersion.type === KqlQueryType.inline_query && - targetVersion.type === KqlQueryType.inline_query - ) { - // TS does not realize that in ABC scenario, baseVersion cannot be missing - // Missing baseVersion scenarios were handled as -AA and -AB. - const mergedVersion = merge(currentVersion.query, baseVersion.query, targetVersion.query, { - stringSeparator: /(\S+|\s+)/g, // Retains all whitespace, which we keep to preserve formatting - }); - - const baseNonMergeableFields = { - filters: baseVersion.filters, - language: baseVersion.language, - }; - - const currentNonMergeableFields = { - filters: currentVersion.filters, - language: currentVersion.language, - }; - - const targetNonMergeableFields = { - filters: targetVersion.filters, - language: targetVersion.language, - }; - - // Determines if all non-mergeable fields are equal to one another - const nonMergeableFieldsEqual = determineIfAllVersionsAreEqual( - baseNonMergeableFields, - currentNonMergeableFields, - targetNonMergeableFields - ); - - if (nonMergeableFieldsEqual && mergedVersion.conflict === false) { - return { - conflict: ThreeWayDiffConflict.SOLVABLE, - mergedVersion: { ...currentVersion, query: mergedVersion.result.join('') }, - mergeOutcome: ThreeWayMergeOutcome.Merged, - }; - } - } - - return { - conflict: ThreeWayDiffConflict.NON_SOLVABLE, - mergeOutcome: ThreeWayMergeOutcome.Current, - mergedVersion: currentVersion, - }; - } - - // Scenario -AB is treated as scenario ABC, but marked as - // SOLVABLE, and returns the target version as the merged version - // https://github.com/elastic/kibana/pull/184889#discussion_r1636421293 - case ThreeWayDiffOutcome.MissingBaseCanUpdate: { - return { - mergedVersion: targetVersion, - mergeOutcome: ThreeWayMergeOutcome.Target, - conflict: ThreeWayDiffConflict.SOLVABLE, - }; - } - default: - return assertUnreachable(diffOutcome); - } -}; +export const kqlQueryDiffAlgorithm = ( + versions: ThreeVersionsOf +) => simpleDiffAlgorithm(versions); From 8263b1a78bd68da22138400aab9f83becfd52540 Mon Sep 17 00:00:00 2001 From: Davis Plumlee Date: Mon, 9 Sep 2024 15:50:49 -0400 Subject: [PATCH 5/5] addresses comments --- .../logic/diff/calculation/algorithms/helpers.ts | 14 +------------- .../algorithms/kql_query_diff_algorithm.test.ts | 14 ++++++++------ 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/helpers.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/helpers.ts index fa6e75cfcc88e..498b857eb0428 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/helpers.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { difference, isEqual, union, uniq } from 'lodash'; +import { difference, union, uniq } from 'lodash'; import type { RuleDataSource } from '../../../../../../../../common/api/detection_engine'; import { DataSourceType } from '../../../../../../../../common/api/detection_engine'; @@ -35,15 +35,3 @@ export const getDedupedDataSourceVersion = (version: RuleDataSource): RuleDataSo } return version; }; - -export const determineIfAllVersionsAreEqual = ( - baseVersion: T, - currentVersion: T, - targetVersion: T -): boolean => { - const baseEqlCurrent = isEqual(baseVersion, currentVersion); - const baseEqlTarget = isEqual(baseVersion, targetVersion); - const currentEqlTarget = isEqual(currentVersion, targetVersion); - - return baseEqlCurrent && baseEqlTarget && currentEqlTarget; -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts index 60641d52e6381..fe97a222448df 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/kql_query_diff_algorithm.test.ts @@ -422,7 +422,7 @@ describe('kqlQueryDiffAlgorithm', () => { describe('if base_version is missing', () => { describe('if current_version and target_version are the same - scenario -AA', () => { - it('returns current_version as merged output if if all versions are inline query types', () => { + it('returns current_version as merged output if all versions are inline query types', () => { const mockVersions: ThreeVersionsOf = { base_version: MissingVersion, current_version: { @@ -453,7 +453,7 @@ describe('kqlQueryDiffAlgorithm', () => { ); }); - it('returns current_version as merged output if if all versions are saved query types', () => { + it('returns current_version as merged output if all versions are saved query types', () => { const mockVersions: ThreeVersionsOf = { base_version: MissingVersion, current_version: { @@ -482,7 +482,7 @@ describe('kqlQueryDiffAlgorithm', () => { }); describe('if current_version and target_version are different - scenario -AB', () => { - it('returns target_version as merged output if all versions are the same', () => { + it('returns target_version as merged output if current and target versions have the same types', () => { const mockVersions: ThreeVersionsOf = { base_version: MissingVersion, current_version: { @@ -513,7 +513,7 @@ describe('kqlQueryDiffAlgorithm', () => { ); }); - it('returns target_version as merged output if all versions are different', () => { + it('returns target_version as merged output if current and target versions have different types', () => { const mockVersions: ThreeVersionsOf = { base_version: MissingVersion, current_version: { @@ -521,8 +521,10 @@ describe('kqlQueryDiffAlgorithm', () => { saved_query_id: 'saved-query-id-2', }, target_version: { - type: KqlQueryType.saved_query, - saved_query_id: 'saved-query-id-3', + type: KqlQueryType.inline_query, + query: 'query string = false', + language: KqlQueryLanguageEnum.kuery, + filters: [], }, };