VAULT-32206: verify audit log and systemd journal secret integrity (h…

…ashicorp#28932)

Verify vault secret integrity in unauthenticated I/O streams (audit log, STDOUT/STDERR via the systemd journal) by scanning the text with Vault Radar. We search for both known and unknown secrets by using an index of KVV2 values and also by radar's built-in heuristics for credentials, secrets, and keys.

The verification has been added to many scenarios where a slight time increase is allowed, as we now have to install Vault Radar and scan the text. In practice this adds less than 10 seconds to the overall duration of a scenario.

In the in-place upgrade scenario we explicitly exclude this verification when upgrading from a version that we know will fail the check. We also make the verification opt-in so as to not require a Vault Radar license to run Enos scenarios, though it will always be enabled in CI.

As part of this we also update our enos workflow to utilize secret values from our self-hosted Vault when executing in the vault-enterprise repo context.

Signed-off-by: Ryan Cragun <[email protected]>

.github/workflows/build.yml

@@ -238,6 +238,7 @@ jobs:
         include: ${{ fromJSON(needs.artifacts.outputs.testable-packages) }}
     with:
       build-artifact-name: ${{ matrix.artifact }}
+      runs-on: ${{ github.repository == 'hashicorp/vault' && '"ubuntu-latest"' || '["self-hosted","linux","small"]' }}
       sample-max: 1
       sample-name: ${{ matrix.sample }}
       ssh-key-name: ${{ github.event.repository.name }}-ci-ssh-key

enos/enos-descriptions.hcl

@@ -115,6 +115,10 @@ globals {
       Vault running in Agent mode uses templates to create log output.
     EOF
 
+    verify_log_secrets = <<-EOF
+      Verify that the vault audit log and systemd journal do not leak secret values.
+    EOF
+
     verify_raft_cluster_all_nodes_are_voters = <<-EOF
       When configured with a 'backend:raft' variant, verify that all nodes in the cluster are
       healthy and are voters.
@@ -198,7 +202,7 @@ globals {
     EOF
 
     verify_billing_start_date = <<-EOF
-      Verify that the billing start date has successfully rolled over to the latest billing year if needed.  
+      Verify that the billing start date has successfully rolled over to the latest billing year if needed.
     EOF
 
   }

enos/enos-modules.hcl

@@ -350,6 +350,12 @@ module "vault_wait_for_seal_rewrap" {
   vault_install_dir = var.vault_install_dir
 }
 
+module "verify_log_secrets" {
+  source = "./modules/verify_log_secrets"
+
+  radar_license_path = var.vault_radar_license_path != null ? abspath(var.vault_radar_license_path) : null
+}
+
 module "verify_seal_type" {
   source = "./modules/verify_seal_type"
 
@@ -363,4 +369,3 @@ module "vault_verify_billing_start_date" {
   vault_instance_count    = var.vault_instance_count
   vault_cluster_addr_port = global.ports["vault_cluster"]["port"]
 }
-

enos/enos-qualities.hcl

@@ -405,6 +405,10 @@ quality "vault_audit_log" {
   description = "The Vault audit sub-system is enabled with the log and writes to a log"
 }
 
+quality "vault_audit_log_secrets" {
+  description = "The Vault audit sub-system does not output secret values"
+}
+
 quality "vault_audit_socket" {
   description = "The Vault audit sub-system is enabled with the socket and writes to a socket"
 }
@@ -490,6 +494,10 @@ quality "vault_init" {
   description = "Vault initializes the cluster with the given seal parameters"
 }
 
+quality "vault_journal_secrets" {
+  description = "The Vault systemd journal does not output secret values"
+}
+
 quality "vault_license_required_ent" {
   description = "Vault Enterprise requires a license in order to start"
 }
@@ -532,6 +540,14 @@ quality "vault_proxy_cli_access" {
   EOF
 }
 
+quality "vault_radar_index_create" {
+  description = "Vault radar is able to create an index from KVv2 mounts"
+}
+
+quality "vault_radar_scan_file" {
+  description = "Vault radar is able to scan a file for secrets"
+}
+
 quality "vault_raft_voters" {
   description = global.description.verify_raft_cluster_all_nodes_are_voters
 }

enos/enos-scenario-agent.hcl

@@ -564,6 +564,34 @@ scenario "agent" {
     }
   }
 
+  step "verify_log_secrets" {
+    skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets
+
+    description = global.description.verify_log_secrets
+    module      = module.verify_log_secrets
+    depends_on = [
+      step.verify_secrets_engines_read,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_audit_log_secrets,
+      quality.vault_journal_secrets,
+      quality.vault_radar_index_create,
+      quality.vault_radar_scan_file,
+    ]
+
+    variables {
+      audit_log_file_path = step.create_vault_cluster.audit_device_file_path
+      leader_host         = step.get_vault_cluster_ips.leader_host
+      vault_addr          = step.create_vault_cluster.api_addr_localhost
+      vault_root_token    = step.create_vault_cluster.root_token
+    }
+  }
+
   step "verify_ui" {
     description = global.description.verify_ui
     module      = module.vault_verify_ui

enos/enos-scenario-autopilot.hcl

@@ -572,6 +572,34 @@ scenario "autopilot" {
     }
   }
 
+  step "verify_log_secrets" {
+    skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets
+
+    description = global.description.verify_log_secrets
+    module      = module.verify_log_secrets
+    depends_on = [
+      step.verify_secrets_engines_read,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_audit_log_secrets,
+      quality.vault_journal_secrets,
+      quality.vault_radar_index_create,
+      quality.vault_radar_scan_file,
+    ]
+
+    variables {
+      audit_log_file_path = step.create_vault_cluster.audit_device_file_path
+      leader_host         = step.get_updated_vault_cluster_ips.leader_host
+      vault_addr          = step.upgrade_vault_cluster_with_autopilot.api_addr_localhost
+      vault_root_token    = step.create_vault_cluster.root_token
+    }
+  }
+
   step "raft_remove_peers" {
     description = <<-EOF
       Remove the nodes that were running the prior version of Vault from the raft cluster

enos/enos-scenario-proxy.hcl

@@ -541,6 +541,34 @@ scenario "proxy" {
     }
   }
 
+  step "verify_log_secrets" {
+    skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets
+
+    description = global.description.verify_log_secrets
+    module      = module.verify_log_secrets
+    depends_on = [
+      step.verify_secrets_engines_read,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_audit_log_secrets,
+      quality.vault_journal_secrets,
+      quality.vault_radar_index_create,
+      quality.vault_radar_scan_file,
+    ]
+
+    variables {
+      audit_log_file_path = step.create_vault_cluster.audit_device_file_path
+      leader_host         = step.get_vault_cluster_ips.leader_host
+      vault_addr          = step.create_vault_cluster.api_addr_localhost
+      vault_root_token    = step.create_vault_cluster.root_token
+    }
+  }
+
   step "verify_ui" {
     description = global.description.verify_ui
     module      = module.vault_verify_ui

enos/enos-scenario-seal-ha.hcl

@@ -794,6 +794,34 @@ scenario "seal_ha" {
     }
   }
 
+  step "verify_log_secrets" {
+    skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets
+
+    description = global.description.verify_log_secrets
+    module      = module.verify_log_secrets
+    depends_on = [
+      step.verify_secrets_engines_read,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_audit_log_secrets,
+      quality.vault_journal_secrets,
+      quality.vault_radar_index_create,
+      quality.vault_radar_scan_file,
+    ]
+
+    variables {
+      audit_log_file_path = step.create_vault_cluster.audit_device_file_path
+      leader_host         = step.get_updated_cluster_ips.leader_host
+      vault_addr          = step.create_vault_cluster.api_addr_localhost
+      vault_root_token    = step.create_vault_cluster.root_token
+    }
+  }
+
   step "verify_ui" {
     description = global.description.verify_ui
     module      = module.vault_verify_ui

enos/enos-scenario-smoke.hcl

@@ -583,6 +583,34 @@ scenario "smoke" {
     }
   }
 
+  step "verify_log_secrets" {
+    skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets
+
+    description = global.description.verify_log_secrets
+    module      = module.verify_log_secrets
+    depends_on = [
+      step.verify_secrets_engines_read,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_audit_log_secrets,
+      quality.vault_journal_secrets,
+      quality.vault_radar_index_create,
+      quality.vault_radar_scan_file,
+    ]
+
+    variables {
+      audit_log_file_path = step.create_vault_cluster.audit_device_file_path
+      leader_host         = step.get_vault_cluster_ips.leader_host
+      vault_addr          = step.create_vault_cluster.api_addr_localhost
+      vault_root_token    = step.create_vault_cluster.root_token
+    }
+  }
+
   step "verify_ui" {
     description = global.description.verify_ui
     module      = module.vault_verify_ui

enos/enos-scenario-upgrade.hcl

@@ -649,6 +649,37 @@ scenario "upgrade" {
     }
   }
 
+  step "verify_log_secrets" {
+    // Only verify log secrets if the audit devices are turned on and we've enabled the check (as
+    // it requires a radar license). Some older versions have known issues so we'll skip this step
+    // in the event that we're upgrading from them, see VAULT-30557 for more information.
+    skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets || semverconstraint(var.vault_upgrade_initial_version, "=1.17.3 || =1.17.4 || =1.16.7 || =1.16.8")
+
+    description = global.description.verify_log_secrets
+    module      = module.verify_log_secrets
+    depends_on = [
+      step.verify_secrets_engines_read,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_audit_log_secrets,
+      quality.vault_journal_secrets,
+      quality.vault_radar_index_create,
+      quality.vault_radar_scan_file,
+    ]
+
+    variables {
+      audit_log_file_path = step.create_vault_cluster.audit_device_file_path
+      leader_host         = step.get_updated_vault_cluster_ips.leader_host
+      vault_addr          = step.create_vault_cluster.api_addr_localhost
+      vault_root_token    = step.create_vault_cluster.root_token
+    }
+  }
+
   step "verify_raft_auto_join_voter" {
     description = global.description.verify_raft_cluster_all_nodes_are_voters
     skip_step   = matrix.backend != "raft"

enos/enos-variables.hcl

@@ -188,6 +188,12 @@ variable "vault_product_version" {
   default     = null
 }
 
+variable "vault_radar_license_path" {
+  description = "The license for vault-radar which is used to verify the audit log"
+  type        = string
+  default     = null
+}
+
 variable "vault_revision" {
   description = "The git sha of Vault artifact we are testing"
   type        = string
@@ -199,3 +205,9 @@ variable "vault_upgrade_initial_version" {
   type        = string
   default     = "1.13.13"
 }
+
+variable "verify_log_secrets" {
+  description = "If true and var.vault_enable_audit_devices is true we'll verify that the audit log does not contain unencrypted secrets. Requires var.vault_radar_license_path to be set to a valid license file."
+  type        = bool
+  default     = false
+}

enos/modules/build_local/scripts/build.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/generate_failover_secondary_token/scripts/generate-failover-secondary-token.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 ## Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/install_packages/scripts/install-packages.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/softhsm_create_vault_keys/scripts/create-keys.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/softhsm_create_vault_keys/scripts/get-keys.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/softhsm_distribute_vault_keys/scripts/distribute-token.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/softhsm_init/scripts/init-softhsm.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/softhsm_install/scripts/find-shared-object.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 ## Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/vault_upgrade/scripts/maybe-remove-old-unit-file.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/vault_upgrade/scripts/restart-vault.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/vault_verify_autopilot/scripts/smoke-verify-autopilot.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/vault_verify_default_lcq/scripts/smoke-verify-default-lcq.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 

enos/modules/vault_verify_undo_logs/scripts/smoke-verify-undo-logs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1