refactor: network test scripts

.github/workflows/ci.yml

@@ -380,24 +380,17 @@ jobs:
             docker pull aztecprotocol/aztec:${{ env.GIT_COMMIT }}
             docker pull aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
             echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
-            cd yarn-project/end-to-end
 
-            export INSTALL_CHAOS_MESH=false
-            export INSTALL_METRICS=false
-            export NAMESPACE=smoke
-            export FRESH_INSTALL=true
-            export VALUES_FILE=ci-smoke.yaml
-
-            ./scripts/network_test.sh ./src/spartan/smoke.test.ts
+            INSTALL_METRICS=false ./spartan/scripts/test_kind.sh ./src/spartan/smoke.test.ts ci-smoke.yaml
       - name: Copy Network Logs
         if: always()
-        run: scripts/copy_from_tester yarn-project/end-to-end/scripts/network-test.log network-test.log || true
+        run: scripts/copy_from_tester spartan/scripts/logs/test_kind.log test_kind.log || true
       - name: Upload Network Logs
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: kind-network-smoke.log
-          path: network-test.log
+          path: test_kind.log
 
   kind-network-test:
     needs: [images-e2e, configure]
@@ -408,15 +401,15 @@ jobs:
       matrix:
         config:
           # - test: reorg.test.ts
-          #   values: ci
+          #   values: ci.yaml
           #   runner_type: 16core-tester-x86-high-memory
           #   timeout: 60
           - test: 4epochs.test.ts
-            values: ci
+            values: ci.yaml
             runner_type: 16core-tester-x86
             timeout: 40
           # - test: gating-passive.test.ts
-          #   values: ci
+          #   values: ci.yaml
           #   runner_type: 16core-tester-x86
           #   timeout: 40
     steps:
@@ -437,19 +430,19 @@ jobs:
             if ci3/test_should_run "$artifact"; then
               docker pull aztecprotocol/aztec:${{ env.GIT_COMMIT }}
               docker pull aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
-              ./ci.sh test-kind-network ${{ matrix.config.test }} ${{ matrix.config.values }}
+              INSTALL_METRICS=false ./spartan/scripts/test_kind.sh "./src/spartan/${{ matrix.config.test }}" "${{ matrix.config.values }}"
               ci3/cache_upload_flag "$artifact"
             fi
       - name: Copy Network Logs
         if: always()
-        run: scripts/copy_from_tester yarn-project/end-to-end/scripts/network-test.log network-test.log || true
+        run: scripts/copy_from_tester spartan/scripts/logs/test_kind.log test_kind.log || true
 
       - name: Upload Network Logs
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: kind-network-test-${{ matrix.config.values }}-${{ matrix.config.test }}.log
-          path: network-test.log
+          path: test_kind.log
 
   bb-bench:
     runs-on: ubuntu-latest

.github/workflows/network-deploy.yml

@@ -164,7 +164,7 @@ jobs:
 
           export MNEMONIC="${{ steps.get-mnemonic.outputs.mnemonic }}"
 
-          $REPO/yarn-project/end-to-end/scripts/bash/generate_devnet_config.sh
+          $REPO/spartan/scripts/generate_devnet_config.sh
 
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v2

.github/workflows/nightly-kind-test.yml

@@ -51,8 +51,8 @@ jobs:
     strategy:
       matrix:
         test:
-          - transfer
-          - reorg
+          - transfer.test.ts
+          - reorg.test.ts
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
@@ -66,7 +66,7 @@ jobs:
           run: |
             docker pull aztecprotocol/aztec:${{ env.GIT_COMMIT }}
             docker pull aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
-            FORCE_COLOR=1 CLEANUP_CLUSTER=true ./ci.sh test-kind-network ${{ matrix.test }}.test.ts 16-validators || true
+            FORCE_COLOR=1 INSTALL_METRICS=false ./spartan/scripts/test_kind.sh "./src/spartan/${{ matrix.config.test }}" 16-validators.yaml
 
   proving-test:
     needs: build
@@ -85,8 +85,8 @@ jobs:
           run: |
             docker pull aztecprotocol/aztec:${{ env.GIT_COMMIT }}
             docker pull aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
-            INSTALL_TIMEOUT=45m FORCE_COLOR=1 CLEANUP_CLUSTER=true \
-              ./ci.sh test-kind-network proving.test.ts 1-validator-with-proving || true
+            INSTALL_TIMEOUT=45m FORCE_COLOR=1 INSTALL_METRICS=false \
+              ./spartan/scripts/test_kind.sh ./src/spartan/proving.test.ts 1-validator-with-proving || true
 
   success-check:
     runs-on: ubuntu-20.04

bootstrap.sh

@@ -294,7 +294,7 @@ case "$cmd" in
     # Drop through. source_bootstrap on script entry has set flags.
   ;;
   *)
-    echo "usage: $0 <clean|full|fast|test|check|test-e2e|test-cache|test-boxes|test-kind-network|image-aztec|image-e2e|image-faucet>"
+    echo "usage: $0 <clean|full|fast|test|check|test-e2e|test-cache|test-boxes|image-aztec|image-e2e|image-faucet>"
     exit 1
   ;;
 esac

ci.sh

@@ -189,19 +189,6 @@ case "$cmd" in
     fi
     exit 0
     ;;
-  "test-kind-network")
-    test=${1:-transfer.test.ts}
-    values=${2:-3-validators}
-    ./bootstrap.sh image-e2e
-    cd yarn-project/end-to-end
-    NAMESPACE="kind-network-test" FRESH_INSTALL=true VALUES_FILE=$values.yaml ./scripts/network_test.sh ./src/spartan/$test
-    exit 0
-    ;;
-  "test-network")
-    shift 1
-    scripts/run_native_testnet.sh -i $@
-    exit 0
-    ;;
   "gha-url")
     workflow_id=$(gh workflow list --all --json name,id -q '.[] | select(.name == "CI").id')
     run_url=$(gh run list --workflow $workflow_id -b $BRANCH --limit 1 --json url -q '.[0].url')
@@ -213,7 +200,7 @@ case "$cmd" in
     exit 0
     ;;
   *)
-    echo "usage: $0 ec2|ec2-e2e|ec2-e2e-grind|local|run|wt|trigger|log|shell|attach|ssh-host|draft|ready|test-kind-network|test-network|gha-url"
+    echo "usage: $0 ec2|ec2-e2e|ec2-e2e-grind|local|run|wt|trigger|log|shell|attach|ssh-host|draft|ready|gha-url"
     exit 1
     ;;
 esac

ci3/find_ports

@@ -0,0 +1,6 @@
+#!/bin/bash
+source $(git rev-parse --show-toplevel)/ci3/source
+# Find 'num_ports' free ports between 9000 and 10000
+# Read first arg, default to 1 port
+num_ports="${1:-1}"
+comm -23 <(seq 9000 10000 | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n "$num_ports"

scripts/run_native_testnet.sh

@@ -40,7 +40,7 @@ L1_CONSENSUS_HOST_URL=
 OTEL_COLLECTOR_ENDPOINT=${OTEL_COLLECTOR_ENDPOINT:-"http://localhost:4318"}
 
 # Function to display help message
-display_help() {
+function display_help {
     echo "Usage: $0 [options]"
     echo
     echo "Options:"
@@ -161,16 +161,16 @@ cd $(git rev-parse --show-toplevel)
 
 # Base command
 BASE_CMD="INTERLEAVED=$INTERLEAVED ./yarn-project/end-to-end/scripts/native_network_test.sh \
-        $TEST_SCRIPT \
-        \"./deploy-l1-contracts.sh $NUM_VALIDATORS\" \
-        ./deploy-l2-contracts.sh \
-        ./boot-node.sh \
-        $ETHEREUM_SCRIPT \
-        \"./validators.sh $NUM_VALIDATORS\" \
-        $PROVER_SCRIPT \
-        ./pxe.sh \
-        ./transaction-bot.sh \
-        $BLOB_SINK_SCRIPT"
+  $TEST_SCRIPT \
+  \"./deploy-l1-contracts.sh $NUM_VALIDATORS\" \
+  ./deploy-l2-contracts.sh \
+  ./boot-node.sh \
+  $ETHEREUM_SCRIPT \
+  \"./validators.sh $NUM_VALIDATORS\" \
+  $PROVER_SCRIPT \
+  ./pxe.sh \
+  ./transaction-bot.sh \
+  $BLOB_SINK_SCRIPT"
 
 # Execute the command
 eval $BASE_CMD

spartan/README.md

@@ -100,10 +100,3 @@ Selecting individual pods provides options to open a console session as well as
 
 ![k8Lens Pod Access](./img/k8lens_3.png)
 
-## Auxiliary Services
-
-Outside of Aztec network deployments, the Spartan cluster has a few additional services installed to support cluster operations.
-
-### Prometheus and Grafana
-
-_(coming soon...)_

spartan/aztec-network/eth-devnet/create.sh

@@ -21,7 +21,7 @@ fi
 
 # Function to create execution genesis
 # Updates genesis timestamp to current time, helps with triggering Consensus layer
-create_execution_genesis() {
+function create_execution_genesis {
   local execution_genesis_path="$1"
   local execution_genesis_output="$2"
   echo "Creating execution genesis..."
@@ -62,7 +62,7 @@ create_execution_genesis() {
   echo "Execution genesis created at $execution_genesis_output"
 }
 
-prefund_accounts() {
+function prefund_accounts {
   local genesis_json="$1"
   local mnemonic="$2"
   local number_of_keys="$3"
@@ -91,7 +91,7 @@ prefund_accounts() {
 # Function to create beacon genesis
 # Uses the eth2-testnet-generator to generate beacon genesis state (genesis.ssz file)
 # The selected eth1 block
-create_beacon_genesis() {
+function create_beacon_genesis {
   local execution_genesis_path="$1"
   local beacon_mnemonics_path="./config/mnemonics.yaml"
   local beacon_config_path="./config/config.yaml"
@@ -151,13 +151,13 @@ create_beacon_genesis() {
   echo "Beacon genesis created at $beacon_genesis_path"
 }
 
-create_deposit_contract_block() {
+function create_deposit_contract_block {
   echo 0 > "$DIR_PATH/out/deposit_contract_block.txt"
   echo "Deposit contract block created at $DIR_PATH/out/deposit_contract_block.txt"
 }
 
 ## The ssz file must be written in base64 in order for a config map to accept it
-write_ssz_file_base64() {
+function write_ssz_file_base64 {
   local ssz_file="$DIR_PATH/out/genesis.ssz"
   local output_file="$DIR_PATH/out/genesis-ssz"
   base64 -w 0 "$ssz_file" > "$output_file"

spartan/bootstrap.sh

@@ -3,6 +3,96 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap
 
 cmd=${1:-}
 
-if [ "$cmd" = "hash" ]; then
-  hash_str $(cache_content_hash .rebuild_patterns) $(../yarn-project/bootstrap.sh hash)
-fi
+scripts/install_deps.sh
+
+function network_shaping {
+  namespace="$1"
+  chaos_values="$2"
+  if ! kubectl get service chaos-daemon -n chaos-mesh &>/dev/null; then
+    echo "Please set up chaos-mesh first. You can do this by running spartan/bootstrap.sh chaos-mesh"
+    exit 1
+  fi
+
+  echo "Deploying Aztec Chaos Scenarios..."
+  if ! helm upgrade --install aztec-chaos-scenarios aztec-chaos-scenarios \
+    --namespace chaos-mesh \
+    --values "aztec-chaos-scenarios/values/$chaos_values" \
+    --set global.targetNamespace="$namespace" \
+    --wait \
+    --timeout=5m; then
+    echo "Error: failed to deploy Aztec Chaos Scenarios!"
+    return 1
+  fi
+  echo "Aztec Chaos Scenarios applied successfully"
+  return 0
+}
+
+function gke {
+  # For GKE access
+  if ! command -v gcloud &> /dev/null; then
+    if [ -f /etc/os-release ] && grep -qi "Ubuntu" /etc/os-release; then
+      sudo apt update
+      sudo apt install -y apt-transport-https ca-certificates gnupg curl
+      sudo rm -f /usr/share/keyrings/cloud.google.gpg && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
+      echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+      sudo apt install -y google-cloud-cli
+      sudo apt install google-cloud-cli-gke-gcloud-auth-plugin
+      echo "Now you can run 'gcloud init'. Exiting with 1 as this is a necessary step."
+    else
+      echo "gcloud not found. This is needed for GKE kubernetes usage." >&2
+      echo "If needed, install glcoud and do 'gcloud components install gke-gcloud-auth-plugin', then 'gcloud init'" >&2
+    fi
+    exit 1
+  fi
+}
+
+case "$cmd" in
+  "")
+    # do nothing but the install_deps.sh above
+    ;;
+  "kind")
+    if kubectl config get-clusters | grep -q "^kind-kind$"; then
+      echo "Cluster 'kind' already exists. Skipping creation."
+    else
+      # Sometimes, kubectl does not have our kind context yet kind registers it as existing
+      # Ensure our context exists in kubectl
+      kind delete cluster || true
+      kind create cluster
+    fi
+    kubectl config use-context kind-kind || true
+    ;;
+  "chaos-mesh")
+    chaos-mesh/install.sh
+    ;;
+  "metrics-kind")
+    metrics/install-kind.sh
+    ;;
+  "metrics-prod")
+    metrics/install-prod.sh
+    ;;
+  "network-shaping")
+    shift
+    namespace="$1"
+    chaos_values="$2"
+    if network_shaping "$namespace" "$chaos_values"; then
+      exit
+    fi
+    # If we are unable to apply network shaping, as we cannot change existing chaos configurations, then delete existing configurations and try again
+    echo "Deleting existing network chaos experiments..."
+    kubectl delete networkchaos --all --all-namespaces
+    network_shaping "$namespace" "$chaos_values"
+    ;;
+  "hash")
+    hash_str $(cache_content_hash .rebuild_patterns) $(../yarn-project/bootstrap.sh hash)
+    ;;
+  "test-kind")
+    shift
+    scripts/test_kind.sh $@
+    ;;
+  "gke")
+    gke
+    ;;
+  *)
+    echo "Unknown command: $cmd"
+    exit 1
+esac

spartan/chaos-mesh/install.sh

@@ -2,6 +2,7 @@
 
 SCRIPT_DIR="$(dirname $(realpath -s "${BASH_SOURCE[0]}"))"
 cd "$SCRIPT_DIR"
+echo "Installing chaos mesh"
 
 # check if chaos-mesh is already installed
 if helm ls --namespace chaos-mesh | grep -q chaos; then

spartan/metrics/install-kind.sh

@@ -4,6 +4,8 @@ set -eu
 SCRIPT_DIR="$(dirname $(realpath -s "${BASH_SOURCE[0]}"))"
 cd "$SCRIPT_DIR"
 
+echo "Installing metrics (KIND)"
+
 # check if metrics is already installed
 if helm ls --namespace metrics | grep -q metrics; then
   echo "metrics is already installed"