[QT-554] Remove Terraform validations from Enos replication scenario

enos/modules/vault_verify_performance_replication/main.tf

@@ -44,18 +44,19 @@ variable "wrapping_token" {
 }
 
 locals {
-  primary_replication_status   = jsondecode(enos_remote_exec.verify_replication_on_primary.stdout)
-  secondary_replication_status = jsondecode(enos_remote_exec.verify_replication_on_secondary.stdout)
+  primary_replication_status   = jsondecode(enos_remote_exec.verify_replication_status_on_primary.stdout)
+  secondary_replication_status = jsondecode(enos_remote_exec.verify_replication_status_on_secondary.stdout)
 }
 
-resource "enos_remote_exec" "verify_replication_on_primary" {
+resource "enos_remote_exec" "verify_replication_status_on_primary" {
   environment = {
-    VAULT_ADDR        = "http://127.0.0.1:8200"
-    VAULT_INSTALL_DIR = var.vault_install_dir
-    REPLICATION_MODE  = "primary"
+    VAULT_ADDR               = "http://127.0.0.1:8200"
+    VAULT_INSTALL_DIR        = var.vault_install_dir
+    PRIMARY_LEADER_PRIV_IP   = var.primary_leader_private_ip
+    SECONDARY_LEADER_PRIV_IP = var.secondary_leader_private_ip
   }
 
-  scripts = ["${path.module}/scripts/verify-performance-replication.sh"]
+  scripts = ["${path.module}/scripts/verify-replication-status.sh"]
 
   transport = {
     ssh = {
@@ -64,23 +65,15 @@ resource "enos_remote_exec" "verify_replication_on_primary" {
   }
 }
 
-output "primary_replication_status" {
-  value = local.primary_replication_status
-
-  precondition {
-    condition     = local.primary_replication_status.data.mode == "primary" && local.primary_replication_status.data.state != "idle"
-    error_message = "Vault primary cluster mode must be \"primary\" and state must not be \"idle\"."
-  }
-}
-
-resource "enos_remote_exec" "verify_replication_on_secondary" {
+resource "enos_remote_exec" "verify_replication_status_on_secondary" {
   environment = {
-    VAULT_ADDR        = "http://127.0.0.1:8200"
-    VAULT_INSTALL_DIR = var.vault_install_dir
-    REPLICATION_MODE  = "secondary"
+    VAULT_ADDR               = "http://127.0.0.1:8200"
+    VAULT_INSTALL_DIR        = var.vault_install_dir
+    PRIMARY_LEADER_PRIV_IP   = var.primary_leader_private_ip
+    SECONDARY_LEADER_PRIV_IP = var.secondary_leader_private_ip
   }
 
-  scripts = ["${path.module}/scripts/verify-performance-replication.sh"]
+  scripts = ["${path.module}/scripts/verify-replication-status.sh"]
 
   transport = {
     ssh = {
@@ -89,52 +82,22 @@ resource "enos_remote_exec" "verify_replication_on_secondary" {
   }
 }
 
+output "primary_replication_status" {
+  value = local.primary_replication_status
+}
+
 output "known_primary_cluster_addrs" {
   value = local.secondary_replication_status.data.known_primary_cluster_addrs
-
-  precondition {
-    condition     = contains(local.secondary_replication_status.data.known_primary_cluster_addrs, "https://${var.primary_leader_private_ip}:8201")
-    error_message = "Vault secondary cluster known_primary_cluster_addrs must include ${var.primary_leader_private_ip}."
-  }
 }
 
 output "secondary_replication_status" {
   value = local.secondary_replication_status
-
-  precondition {
-    condition     = local.secondary_replication_status.data.mode == "secondary" && local.secondary_replication_status.data.state != "idle"
-    error_message = "Vault secondary cluster mode must be \"secondary\" and state must not be \"idle\"."
-  }
 }
 
 output "primary_replication_data_secondaries" {
   value = local.primary_replication_status.data.secondaries
-
-  # The secondaries connection_status should be "connected"
-  precondition {
-    condition     = local.primary_replication_status.data.secondaries[0].connection_status == "connected"
-    error_message = "connection status to primaries must be \"connected\"."
-  }
-
-  # The secondaries cluster address must have the secondary leader address
-  precondition {
-    condition     = local.primary_replication_status.data.secondaries[0].cluster_address == "https://${var.secondary_leader_private_ip}:8201"
-    error_message = "Vault secondaries cluster_address must be with ${var.secondary_leader_private_ip}."
-  }
 }
 
 output "secondary_replication_data_primaries" {
   value = local.secondary_replication_status.data.primaries
-
-  # The primaries connection_status should be "connected"
-  precondition {
-    condition     = local.secondary_replication_status.data.primaries[0].connection_status == "connected"
-    error_message = "connection status to primaries must be \"connected\"."
-  }
-
-  # The primaries cluster address must have the primary leader address
-  precondition {
-    condition     = local.secondary_replication_status.data.primaries[0].cluster_address == "https://${var.primary_leader_private_ip}:8201"
-    error_message = "Vault primaries cluster_address must be ${var.primary_leader_private_ip}."
-  }
 }

enos/modules/vault_verify_performance_replication/scripts/verify-replication-status.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+
+# This script waits for the replication status to be established
+# then verifies the performance replication between primary and
+# secondary clusters
+
+set -e
+
+binpath=${VAULT_INSTALL_DIR}/vault
+
+function fail() {
+	echo "$1" 1>&2
+	exit 1
+}
+
+retry() {
+  local retries=$1
+  shift
+  local count=0
+
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** count))
+    count=$((count + 1))
+    if [ "$count" -lt "$retries" ]; then
+      sleep "$wait"
+    else
+      return "$exit"
+    fi
+  done
+}
+
+test -x "$binpath" || exit 1
+
+check_pr_status() {
+  pr_status=$($binpath read -format=json sys/replication/performance/status)
+  cluster_state=$(echo $pr_status | jq -r '.data.state')
+  connection_mode=$(echo $pr_status | jq -r '.data.mode')
+
+  if [[ "$cluster_state" == 'idle' ]]; then
+    fail "replication cluster state is $cluster_state"
+  fi
+
+  if [[ "$connection_mode" == "primary" ]]; then
+    connection_status=$(echo $pr_status | jq -r '.data.secondaries[0].connection_status')
+    if [[ "$connection_status" == 'disconnected' ]]; then
+      fail "replication connection status of secondaries is $connection_status"
+    fi
+    secondary_cluster_addr=$(echo $pr_status | jq -r '.data.secondaries[0].cluster_address')
+    if [[ "$secondary_cluster_addr" != "https://"${SECONDARY_LEADER_PRIV_IP}":8201" ]]; then
+      fail "Expected secondary cluster address $SECONDARY_LEADER_PRIV_IP got  $secondary_cluster_addr "
+    fi
+  else
+    connection_status=$(echo $pr_status | jq -r '.data.primaries[0].connection_status')
+    if [[ "$connection_status" == 'disconnected' ]]; then
+      fail "replication connection status of secondaries is $connection_status"
+    fi
+    primary_cluster_addr=$(echo $pr_status | jq -r '.data.primaries[0].cluster_address')
+    if [[ "$primary_cluster_addr" != "https://"${PRIMARY_LEADER_PRIV_IP}":8201" ]]; then
+      fail "Expected primary cluster address $PRIMARY_LEADER_PRIV_IP got  $primary_cluster_addr"
+    fi
+    known_primary_cluster_addrs=$(echo $pr_status | jq -r '.data.known_primary_cluster_addrs')
+    # IFS="," read -a cluster_addr <<< ${known_primary_cluster_addrs}
+    if ! $(echo $known_primary_cluster_addrs |grep -q $PRIMARY_LEADER_PRIV_IP); then
+      fail "Primary leader address $PRIMARY_LEADER_PRIV_IP not found in Known primary cluster addresses $known_primary_cluster_addrs"
+    fi
+  fi
+  echo $pr_status
+}
+
+# Retry a few times because it can take some time for replication to sync
+retry 5 check_pr_status