From 70110b57b79de7b3c73ddc610991349a28c50e60 Mon Sep 17 00:00:00 2001 From: Mark G Date: Fri, 8 Mar 2024 12:10:05 -0500 Subject: [PATCH] Improve pipeline resource doc --- docs/resources/pipeline.md | 274 +++++++++++++++++++++++++++---------- 1 file changed, 203 insertions(+), 71 deletions(-) diff --git a/docs/resources/pipeline.md b/docs/resources/pipeline.md index 9940a0f..530bc78 100644 --- a/docs/resources/pipeline.md +++ b/docs/resources/pipeline.md @@ -8,63 +8,185 @@ description: |- # streamdal_pipeline (Resource) -Pipelines - - +This resource is used to create and manage pipelines in the Streamdal server. + +The pipeline resource can have multiple repeated `step` blocks for each step you wish to configure. + +## Example Usage + +```hcl +terraform { + required_providers { + streamdal = { + source = "streamdal/streamdal" + version = "0.1.1" + } + } +} + +provider "streamdal" { + token = "1234" + address = "localhost:8082" + connection_timeout = 10 +} + +resource "streamdal_pipeline" "mask_email" { + name = "Mask Email" + + step { + name = "Detect Email Field" + on_false { + abort = "abort_current" # No need to continue this pipeline if no email found + } + on_error { + abort = "abort_current" + } + dynamic = false + detective { + type = "pii_email" + args = [] # no args for this type + negate = false + path = "" # No path, we will scan the entire payload + } + } + + step { + name = "Mask Email Step" + dynamic = true + transform { + mask_value { + # No path needed since dynamic=true + # We will use the results from the first detective step + path = "" + + # Mask the email field(s) we find with asterisks + mask = "*" + } + } + } +} +``` ## Schema ### Required -- **name** (String) Name +- ``name`` - (String) Name +- ``step`` - (Repeated Blocks) Steps for this pipeline (see [below for nested schema](#nestedblock--step)) -### Optional + +### Nested Schema for `step` -- **step** (Block List) Steps for this pipeline (see [below for nested schema](#nestedblock--step)) +Required: -### Read-Only +- ``name`` - (String) Step Name -- **id** (String) Pipeline ID +At lease one of the following step type blocks is required: - -### Nested Schema for `step` +- ``detective`` - (Block List, Max: 1) Detective Step (see [below for nested schema](#nestedblock--step--detective)) +- ``dynamic`` - (Boolean) Should this step use the result from the previous step. This is valid **ONLY** for a transform step which immediately follows a detective step. Specifying `true` means +any result from the detective step will be used as the path(s) for the transform step. (Default: `false`) +- ``http_request`` - (Block List, Max: 1) HTTP Request Step (see [below for nested schema](#nestedblock--step--http_request)) -Optional: +- ``schema_validation`` - (Block List, Max: 1) Schema Validation Step (see [below for nested schema](#nestedblock--step--schema_validation)) +- ``transform`` - (Block List, Max: 1) Transform Step (see [below for nested schema](#nestedblock--step--transform)) + +Optional parameters: -- **detective** (Block List, Max: 1) Detective Step (see [below for nested schema](#nestedblock--step--detective)) -- **dynamic** (Boolean) Should this step use the result from the previous step -- **http_request** (Block List, Max: 1) HTTP Request Step (see [below for nested schema](#nestedblock--step--http_request)) -- **name** (String) Step Name -- **on_error** (Block List, Max: 1) Determines the next action if the result of the step is an error (see [below for nested schema](#nestedblock--step--on_error)) -- **on_false** (Block List, Max: 1) Determines the next action if the result of the step is false (see [below for nested schema](#nestedblock--step--on_false)) -- **on_true** (Block List, Max: 1) Determines the next action if the result of the step is true (see [below for nested schema](#nestedblock--step--on_true)) -- **schema_validation** (Block List, Max: 1) Schema Validation Step (see [below for nested schema](#nestedblock--step--schema_validation)) -- **transform** (Block List, Max: 1) Transform Step (see [below for nested schema](#nestedblock--step--transform)) +- ``on_error`` - (Optional Block) Determines the next action if the result of the step is an error (see [below for nested schema](#nestedblock--step--on_error)) +- ``on_false`` -(Optional Block) Determines the next action if the result of the step is false (see [below for nested schema](#nestedblock--step--on_false)) +- ``on_true`` - (Optional Block) Determines the next action if the result of the step is true (see [below for nested schema](#nestedblock--step--on_true)) ### Nested Schema for `step.detective` -Optional: +This step type is used to detect a specific field in a JSON payload based on it's content. -- **args** (List of String) Arguments -- **negate** (Boolean) Negate (Default: `false`) -- **path** (String) Path -- **type** (String) Detective Type +Optional: +- ``args`` - (List of String) Arguments. Used only for types: + - `string_contains_all` (any number of args) + - `string_contains_any` (any number of args) + - `string_length_min` (1 arg) + - `string_length_max` (1 arg) + - `string_length_range` (2 args) +- ``negate`` - (Boolean) Negate. Causes the step to return the opposite boolean value of what it normally would (Default: `false`) +- ``path`` - (String) JSON Path using dot notation to the field to be scanned. If a path is omitted, +- ``type`` - (String) Detective Type. One of the following + - `is_empty` + - `has_field` + - `is_type` + - `string_contains_any` + - `string_contains_all` + - `string_equal` + - `ipv4_address` + - `ipv6_address` + - `mac_address` + - `regex` + - `timestamp_rfc3339` + - `timestamp_unix_nano` + - `timestamp_unix` + - `boolean_true` + - `boolean_false` + - `uuid` + - `url` + - `hostname` + - `string_length_min` + - `string_length_max` + - `string_length_range` + - `semver` + - `pii_any` + - `pii_credit_card` + - `pii_ssn` + - `pii_email` + - `pii_phone` + - `pii_driver_license` + - `pii_passport_id` + - `pii_vin_number` + - `pii_serial_number` + - `pii_login` + - `pii_taxpayer_id` + - `pii_address` + - `pii_signature` + - `pii_geolocation` + - `pii_education` + - `pii_financial` + - `pii_health` + - `pii_aws_key_id` + - `pii_rsa_key` + - `pii_title` + - `pii_religion` + - `pii_slack_token` + - `pii_stripe_key` + - `pii_iban` + - `pii_swift_bic` + - `pii_bank_routing_number` + - `pii_crypto_address` + - `pii_github_pat` + - `numeric_equal_to` + - `numeric_greater_than` + - `numeric_greater_equal` + - `numeric_less_than` + - `numeric_less_equal` + - `numeric_range` + - `numeric_min` + - `numeric_max` ### Nested Schema for `step.http_request` +This step type performs a HTTP request + Required: -- **method** (String) HTTP Method -- **url** (String) URL +- ``method`` - (Enum) HTTP Method. Possible values: ``GET``, ``POST``, ``PUT``, ``PATCH``, ``DELETE`` +- ``url`` - (String) URL Optional: -- **body** (String) Body -- **headers** (Map of String) Headers +- ``headers`` - (Map of Strings) HTTP headers and values to pass in the request +- ``body`` - (String) Any payload you wish to send in the request @@ -72,109 +194,119 @@ Optional: Optional: -- **abort** (String) Abort (Default: `unset`) -- **metadata** (Map of String) Metadata (Default: `map[]`) -- **notification** (Block List, Max: 1) Notification Config (see [below for nested schema](#nestedblock--step--on_error--notification)) +- ``abort`` - (String) Abort (Default: `unset`) +- ``metadata`` - (Map of Strings) Any custom metadata you wish to return to the code calling the SDK `Process()` method (Default: `map[]`) +- ``notification`` - (Optional Block) Notification Config (see [below for nested schema](#nestedblock--step--on_error--notification)) ### Nested Schema for `step.on_error.notification` +Determines the next action if the result of the step is an error. + Optional: -- **notification_config_ids** (List of String) Notification Config IDs -- **paths** (List of String) Paths to Extract (If Payload Type is 'select_paths') -- **payload_type** (String) Payload Type (Default: `exclude`) +- ``notification_config_ids`` - (List of String) Notification Config IDs +- ``paths`` - (List of String) Paths to Extract (If Payload Type is 'select_paths') +- ``payload_type`` - (String) Payload Type (Default: `exclude`) ### Nested Schema for `step.on_false` +Determines the next action if the result of the step is false. + Optional: -- **abort** (String) Abort (Default: `unset`) -- **metadata** (Map of String) Metadata (Default: `map[]`) -- **notification** (Block List, Max: 1) Notification Config (see [below for nested schema](#nestedblock--step--on_false--notification)) +- ``abort`` - (String) Abort (Default: `unset`) +- ``metadata`` - (Map of Strings) Any custom metadata you wish to return to the code calling the SDK `Process()` method (Default: `map[]`) +- ``notification`` - (Optional Block) Notification Config (see [below for nested schema](#nestedblock--step--on_false--notification)) ### Nested Schema for `step.on_false.notification` Optional: -- **notification_config_ids** (List of String) Notification Config IDs -- **paths** (List of String) Paths to Extract (If Payload Type is 'select_paths') -- **payload_type** (String) Payload Type (Default: `exclude`) +- ``notification_config_ids`` - (List of String) Notification Config IDs +- ``paths`` - (List of String) Paths to Extract (If Payload Type is 'select_paths') +- ``payload_type`` - (String) Payload Type (Default: `exclude`) ### Nested Schema for `step.on_true` +Determines the next action if the result of the step is true. + Optional: -- **abort** (String) Abort (Default: `unset`) -- **metadata** (Map of String) Metadata (Default: `map[]`) -- **notification** (Block List, Max: 1) Notification Config (see [below for nested schema](#nestedblock--step--on_true--notification)) +- ``abort`` - (String) Abort (Default: `unset`) +- ``metadata`` - (Map of Strings) Any custom metadata you wish to return to the code calling the SDK `Process()` method (Default: `map[]`) +- ``notification`` - (Optional Block) Notification Config (see [below for nested schema](#nestedblock--step--on_true--notification)) ### Nested Schema for `step.on_true.notification` Optional: -- **notification_config_ids** (List of String) Notification Config IDs -- **paths** (List of String) Paths to Extract (If Payload Type is 'select_paths') -- **payload_type** (String) Payload Type (Default: `exclude`) +- ``notification_config_ids`` - (List of String) Notification Config IDs +- ``paths`` - (List of String) Paths to Extract (If Payload Type is 'select_paths') +- ``payload_type`` - (String) Payload Type (Default: `exclude`) ### Nested Schema for `step.schema_validation` -Required: - -- **condition** (String) Schema Validation Condition -- **type** (String) Schema Validation Type +Validates the JSON payload against a schema. Currently only JSONSchema is supported. -Optional: +Required: -- **json_schema** (Block List, Max: 1) JSON Schema (see [below for nested schema](#nestedblock--step--schema_validation--json_schema)) +- ``condition`` - (Enum) Schema Validation Condition. Possible values: ``match``, ``not_match`` +- ``type`` - (Enum) Schema Validation Type. Possible values: ``json_schema`` +- ``json_schema`` - (Block) JSON Schema (see [below for nested schema](#nestedblock--step--schema_validation--json_schema)) ### Nested Schema for `step.schema_validation.json_schema` Required: -- **draft** (String) JSON Schema Draft -- **json_schema** (String) Schema Definition +- ``draft`` - (String) JSON Schema Draft. Possible values: ``draft_04``, ``draft_06``, ``draft_07`` +- ``json_schema`` - (String) Schema Definition in JSON Schema format ### Nested Schema for `step.transform` -Optional: +At least one of the following transform type blocks must be specified. -- **delete_field** (Block List) Delete field (see [below for nested schema](#nestedblock--step--transform--delete_field)) -- **extract** (Block List) Extract value (see [below for nested schema](#nestedblock--step--transform--extract)) -- **mask_value** (Block List) Mask value (see [below for nested schema](#nestedblock--step--transform--mask_value)) -- **obfuscate** (Block List) Obfuscate value (see [below for nested schema](#nestedblock--step--transform--obfuscate)) -- **replace_value** (Block List) Replace value of a field (see [below for nested schema](#nestedblock--step--transform--replace_value)) -- **truncate** (Block List) Truncate value (see [below for nested schema](#nestedblock--step--transform--truncate)) +- ``delete_field`` - (Block) Delete field (see [below for nested schema](#nestedblock--step--transform--delete_field)) +- ``extract`` - (Block) Extract value (see [below for nested schema](#nestedblock--step--transform--extract)) +- ``mask_value`` - (Block) Mask value (see [below for nested schema](#nestedblock--step--transform--mask_value)) +- ``obfuscate`` - (Block) Obfuscate value (see [below for nested schema](#nestedblock--step--transform--obfuscate)) +- ``replace_value`` - (Block) Replace value of a field (see [below for nested schema](#nestedblock--step--transform--replace_value)) +- ``truncate`` - (Block) Truncate value (see [below for nested schema](#nestedblock--step--transform--truncate)) ### Nested Schema for `step.transform.delete_field` +This step type will delete the specified JSON paths(s) from a JSON payload. + Optional: -- **paths** (List of String) Paths +- ``paths`` - (List of Strings) JSON Paths of field(s) to delete. **If `dynamic=true`, this value is ignored** ### Nested Schema for `step.transform.extract` +Extract will pull the specified path(s) and create a new JSON payload with the extracted paths. +All other fields will be discarded + Optional: -- **paths** (List of String) Paths +- ``paths`` - (List of Strings) JSON Paths of field(s) whose value you wish to extract. **If `dynamic=true`, this value is ignored** @@ -182,8 +314,8 @@ Optional: Optional: -- **mask** (String) Mask (Default: `*`) -- **path** (String) Path +- ``mask`` - (String) Mask (Default: `*`) +- ``path`` - (String) Path @@ -191,7 +323,7 @@ Optional: Optional: -- **path** (String) Path +- ``path`` - (String) JSON Path of field to obfuscate. **If `dynamic=true`, this value is ignored** @@ -199,8 +331,8 @@ Optional: Optional: -- **path** (String) Path -- **value** (String) Value +- ``path`` - (String) JSON Path of field to replace. **If `dynamic=true`, this value is ignored** +- ``value`` - (String) Value @@ -208,10 +340,10 @@ Optional: Required: -- **type** (String) Truncate Type +- ``type`` - (String) Truncate Type Optional: -- **path** (String) Path +- ``path`` - (String) Path