[Security Solution] Handle specific fields in /upgrade/_review endpoint and refactor diff logic to use Zod

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts

@@ -304,8 +304,29 @@ export type TimestampOverrideFallbackDisabled = z.infer<typeof TimestampOverride
 export const TimestampOverrideFallbackDisabled = z.boolean();
 
 /**
- * Describes an Elasticsearch field that is needed for the rule to function
- */
+  * Describes an Elasticsearch field that is needed for the rule to function.
+
+Almost all types of Security rules check source event documents for a match to some kind of
+query or filter. If a document has certain field with certain values, then it's a match and
+the rule will generate an alert.
+
+Required field is an event field that must be present in the source indices of a given rule.
+
+@example
+const standardEcsField: RequiredField = {
+  name: 'event.action',
+  type: 'keyword',
+  ecs: true,
+};
+
+@example
+const nonEcsField: RequiredField = {
+  name: 'winlog.event_data.AttributeLDAPDisplayName',
+  type: 'keyword',
+  ecs: false,
+};
+
+  */
 export type RequiredField = z.infer<typeof RequiredField>;
 export const RequiredField = z.object({
   /**
@@ -368,6 +389,39 @@ export const SavedObjectResolveAliasPurpose = z.enum([
 export type SavedObjectResolveAliasPurposeEnum = typeof SavedObjectResolveAliasPurpose.enum;
 export const SavedObjectResolveAliasPurposeEnum = SavedObjectResolveAliasPurpose.enum;
 
+/**
+  * Related integration is a potential dependency of a rule. It's assumed that if the user installs
+one of the related integrations of a rule, the rule might start to work properly because it will
+have source events (generated by this integration) potentially matching the rule's query.
+
+NOTE: Proper work is not guaranteed, because a related integration, if installed, can be
+configured differently or generate data that is not necessarily relevant for this rule.
+
+Related integration is a combination of a Fleet package and (optionally) one of the
+package's "integrations" that this package contains. It is represented by 3 properties:
+
+- `package`: name of the package (required, unique id)
+- `version`: version of the package (required, semver-compatible)
+- `integration`: name of the integration of this package (optional, id within the package)
+
+There are Fleet packages like `windows` that contain only one integration; in this case,
+`integration` should be unspecified. There are also packages like `aws` and `azure` that contain
+several integrations; in this case, `integration` should be specified.
+
+@example
+const x: RelatedIntegration = {
+  package: 'windows',
+  version: '1.5.x',
+};
+
+@example
+const x: RelatedIntegration = {
+  package: 'azure',
+  version: '~1.1.6',
+  integration: 'activitylogs',
+};
+
+  */
 export type RelatedIntegration = z.infer<typeof RelatedIntegration>;
 export const RelatedIntegration = z.object({
   package: NonEmptyString,
@@ -378,6 +432,22 @@ export const RelatedIntegration = z.object({
 export type RelatedIntegrationArray = z.infer<typeof RelatedIntegrationArray>;
 export const RelatedIntegrationArray = z.array(RelatedIntegration);
 
+/**
+  * Schema for fields relating to investigation fields. These are user defined fields we use to highlight
+in various features in the UI such as alert details flyout and exceptions auto-population from alert.
+Added in PR #163235
+Right now we only have a single field but anticipate adding more related fields to store various
+configuration states such as `override` - where a user might say if they want only these fields to
+display, or if they want these fields + the fields we select. When expanding this field, it may look
+something like:
+```typescript
+const investigationFields = z.object({
+  field_names: NonEmptyArray(NonEmptyString),
+  override: z.boolean().optional(),
+});
+```
+
+  */
 export type InvestigationFields = z.infer<typeof InvestigationFields>;
 export const InvestigationFields = z.object({
   field_names: z.array(NonEmptyString).min(1),

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml

@@ -315,7 +315,28 @@ components:
 
     RequiredField:
       type: object
-      description: Describes an Elasticsearch field that is needed for the rule to function
+      description: |
+        Describes an Elasticsearch field that is needed for the rule to function.
+
+        Almost all types of Security rules check source event documents for a match to some kind of
+        query or filter. If a document has certain field with certain values, then it's a match and
+        the rule will generate an alert.
+
+        Required field is an event field that must be present in the source indices of a given rule.
+
+        @example
+        const standardEcsField: RequiredField = {
+          name: 'event.action',
+          type: 'keyword',
+          ecs: true,
+        };
+
+        @example
+        const nonEcsField: RequiredField = {
+          name: 'winlog.event_data.AttributeLDAPDisplayName',
+          type: 'keyword',
+          ecs: false,
+        };
       properties:
         name:
           $ref: '../../../model/primitives.schema.yaml#/components/schemas/NonEmptyString'
@@ -376,6 +397,37 @@ components:
 
     RelatedIntegration:
       type: object
+      description: |
+        Related integration is a potential dependency of a rule. It's assumed that if the user installs
+        one of the related integrations of a rule, the rule might start to work properly because it will
+        have source events (generated by this integration) potentially matching the rule's query.
+
+        NOTE: Proper work is not guaranteed, because a related integration, if installed, can be
+        configured differently or generate data that is not necessarily relevant for this rule.
+
+        Related integration is a combination of a Fleet package and (optionally) one of the
+        package's "integrations" that this package contains. It is represented by 3 properties:
+
+        - `package`: name of the package (required, unique id)
+        - `version`: version of the package (required, semver-compatible)
+        - `integration`: name of the integration of this package (optional, id within the package)
+
+        There are Fleet packages like `windows` that contain only one integration; in this case,
+        `integration` should be unspecified. There are also packages like `aws` and `azure` that contain
+        several integrations; in this case, `integration` should be specified.
+
+        @example
+        const x: RelatedIntegration = {
+          package: 'windows',
+          version: '1.5.x',
+        };
+
+        @example
+        const x: RelatedIntegration = {
+          package: 'azure',
+          version: '~1.1.6',
+          integration: 'activitylogs',
+        };
       properties:
         package:
           $ref: '../../../model/primitives.schema.yaml#/components/schemas/NonEmptyString'
@@ -392,10 +444,22 @@ components:
       items:
         $ref: '#/components/schemas/RelatedIntegration'
 
-    # Schema for fields relating to investigation fields, these are user defined fields we use to highlight in various features in the UI such as alert details flyout and exceptions auto-population from alert. Added in PR #163235
-    # Right now we only have a single field but anticipate adding more related fields to store various configuration states such as `override` - where a user might say if they want only these fields to display, or if they want these fields + the fields we select.
     InvestigationFields:
       type: object
+      description: |
+        Schema for fields relating to investigation fields. These are user defined fields we use to highlight
+        in various features in the UI such as alert details flyout and exceptions auto-population from alert.
+        Added in PR #163235
+        Right now we only have a single field but anticipate adding more related fields to store various
+        configuration states such as `override` - where a user might say if they want only these fields to
+        display, or if they want these fields + the fields we select. When expanding this field, it may look
+        something like:
+        ```typescript
+        const investigationFields = z.object({
+          field_names: NonEmptyArray(NonEmptyString),
+          override: z.boolean().optional(),
+        });
+        ```
       properties:
         field_names:
           type: array

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/time_duration.test.ts

@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { expectParseError, expectParseSuccess, stringifyZodError } from '@kbn/zod-helpers';
+import { TimeDuration } from './time_duration'; // Update with the actual path to your TimeDuration file
+
+describe('TimeDuration schema', () => {
+  test('it should validate a correctly formed TimeDuration with time unit of seconds', () => {
+    const payload = '1s';
+    const schema = TimeDuration({ allowedUnits: ['s'] });
+
+    const result = schema.safeParse(payload);
+    expectParseSuccess(result);
+    expect(result.data).toEqual(payload);
+  });
+
+  test('it should validate a correctly formed TimeDuration with time unit of minutes', () => {
+    const payload = '100m';
+    const schema = TimeDuration({ allowedUnits: ['s', 'm'] });
+
+    const result = schema.safeParse(payload);
+    expectParseSuccess(result);
+    expect(result.data).toEqual(payload);
+  });
+
+  test('it should validate a correctly formed TimeDuration with time unit of hours', () => {
+    const payload = '10000000h';
+    const schema = TimeDuration({ allowedUnits: ['s', 'm', 'h'] });
+
+    const result = schema.safeParse(payload);
+    expectParseSuccess(result);
+    expect(result.data).toEqual(payload);
+  });
+
+  test('it should validate a correctly formed TimeDuration with time unit of days', () => {
+    const payload = '7d';
+    const schema = TimeDuration({ allowedUnits: ['s', 'm', 'h', 'd'] });
+
+    const result = schema.safeParse(payload);
+    expectParseSuccess(result);
+    expect(result.data).toEqual(payload);
+  });
+
+  test('it should NOT validate a correctly formed TimeDuration with time unit of seconds if it is not an allowed unit', () => {
+    const payload = '30s';
+    const schema = TimeDuration({ allowedUnits: ['m', 'h', 'd'] });
+
+    const result = schema.safeParse(payload);
+    expectParseError(result);
+    expect(stringifyZodError(result.error)).toMatchInlineSnapshot(
+      `"Invalid time duration format. Must be a string that is not empty, and composed of a positive integer greater than 0 followed by a unit of time in the format {safe_integer}{timeUnit}, e.g. \\"30s\\", \\"1m\\", \\"2h\\", \\"7d\\""`
+    );
+  });
+
+  test('it should NOT validate a negative TimeDuration', () => {
+    const payload = '-10s';
+    const schema = TimeDuration({ allowedUnits: ['s', 'm', 'h', 'd'] });
+
+    const result = schema.safeParse(payload);
+    expectParseError(result);
+    expect(stringifyZodError(result.error)).toMatchInlineSnapshot(
+      `"Invalid time duration format. Must be a string that is not empty, and composed of a positive integer greater than 0 followed by a unit of time in the format {safe_integer}{timeUnit}, e.g. \\"30s\\", \\"1m\\", \\"2h\\", \\"7d\\""`
+    );
+  });
+
+  test('it should NOT validate a fractional number', () => {
+    const payload = '1.5s';
+    const schema = TimeDuration({ allowedUnits: ['s', 'm', 'h', 'd'] });
+
+    const result = schema.safeParse(payload);
+    expectParseError(result);
+    expect(stringifyZodError(result.error)).toMatchInlineSnapshot(
+      `"Invalid time duration format. Must be a string that is not empty, and composed of a positive integer greater than 0 followed by a unit of time in the format {safe_integer}{timeUnit}, e.g. \\"30s\\", \\"1m\\", \\"2h\\", \\"7d\\""`
+    );
+  });
+
+  test('it should NOT validate a TimeDuration with an invalid time unit', () => {
+    const payload = '10000000days';
+    const schema = TimeDuration({ allowedUnits: ['s', 'm', 'h', 'd'] });
+
+    const result = schema.safeParse(payload);
+    expectParseError(result);
+    expect(stringifyZodError(result.error)).toMatchInlineSnapshot(
+      `"Invalid time duration format. Must be a string that is not empty, and composed of a positive integer greater than 0 followed by a unit of time in the format {safe_integer}{timeUnit}, e.g. \\"30s\\", \\"1m\\", \\"2h\\", \\"7d\\""`
+    );
+  });
+
+  test('it should NOT validate a TimeDuration with a time interval with incorrect format', () => {
+    const payload = '100ff0000w';
+    const schema = TimeDuration({ allowedUnits: ['s', 'm', 'h'] });
+
+    const result = schema.safeParse(payload);
+    expectParseError(result);
+    expect(stringifyZodError(result.error)).toMatchInlineSnapshot(
+      `"Invalid time duration format. Must be a string that is not empty, and composed of a positive integer greater than 0 followed by a unit of time in the format {safe_integer}{timeUnit}, e.g. \\"30s\\", \\"1m\\", \\"2h\\", \\"7d\\""`
+    );
+  });
+
+  test('it should NOT validate an empty string', () => {
+    const payload = '';
+    const schema = TimeDuration({ allowedUnits: ['s', 'm', 'h'] });
+
+    const result = schema.safeParse(payload);
+    expectParseError(result);
+    expect(stringifyZodError(result.error)).toMatchInlineSnapshot(
+      `"Invalid time duration format. Must be a string that is not empty, and composed of a positive integer greater than 0 followed by a unit of time in the format {safe_integer}{timeUnit}, e.g. \\"30s\\", \\"1m\\", \\"2h\\", \\"7d\\""`
+    );
+  });
+
+  test('it should NOT validate a number', () => {
+    const payload = 100;
+    const schema = TimeDuration({ allowedUnits: ['s', 'm', 'h'] });
+
+    const result = schema.safeParse(payload);
+    expectParseError(result);
+    expect(stringifyZodError(result.error)).toMatchInlineSnapshot(
+      `"Expected string, received number"`
+    );
+  });
+
+  test('it should NOT validate a TimeDuration with a valid time unit but unsafe integer', () => {
+    const payload = `${Math.pow(2, 53)}h`;
+    const schema = TimeDuration({ allowedUnits: ['s', 'm', 'h'] });
+
+    const result = schema.safeParse(payload);
+    expectParseError(result);
+    expect(stringifyZodError(result.error)).toMatchInlineSnapshot(
+      `"Invalid time duration format. Must be a string that is not empty, and composed of a positive integer greater than 0 followed by a unit of time in the format {safe_integer}{timeUnit}, e.g. \\"30s\\", \\"1m\\", \\"2h\\", \\"7d\\""`
+    );
+  });
+});

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/time_duration.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+
+type TimeUnits = 's' | 'm' | 'h' | 'd' | 'w' | 'y';
+
+interface TimeDurationType {
+  allowedUnits: TimeUnits[];
+}
+
+const isTimeSafe = (time: number) => time >= 1 && Number.isSafeInteger(time);
+
+/**
+ * Types the TimeDuration as:
+ *   - A string that is not empty, and composed of a positive integer greater than 0 followed by a unit of time
+ *   - in the format {safe_integer}{timeUnit}, e.g. "30s", "1m", "2h", "7d"
+ *
+ * Example usage:
+ * ```
+ * const schedule: RuleSchedule = {
+ *   interval: TimeDuration({ allowedUnits: ['s', 'm', 'h'] }).parse('3h'),
+ * };
+ * ```
+ */
+export const TimeDuration = ({ allowedUnits }: TimeDurationType) => {
+  return z.string().refine(
+    (input) => {
+      if (input.trim() === '') return false;
+
+      try {
+        const inputLength = input.length;
+        const time = Number(input.trim().substring(0, inputLength - 1));
+        const unit = input.trim().at(-1) as TimeUnits;
+
+        return isTimeSafe(time) && allowedUnits.includes(unit);
+      } catch (error) {
+        return false;
+      }
+    },
+    {
+      message:
+        'Invalid time duration format. Must be a string that is not empty, and composed of a positive integer greater than 0 followed by a unit of time in the format {safe_integer}{timeUnit}, e.g. "30s", "1m", "2h", "7d"',
+    }
+  );
+};
+
+export type TimeDurationSchema = ReturnType<typeof TimeDuration>;
+export type TimeDuration = z.infer<TimeDurationSchema>;