AztecProtocol · PhilWindle · Jun 5, 2024 · Jun 4, 2024 · Jun 4, 2024 · Jun 4, 2024
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,7 +1,7 @@
 name: aztec-devnet
 services:
   pxe:
-    image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:afe0263560a82f2c84b9d76451ec4585a3ed389251fb90e916faaa85d9eb9a0d}
+    image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:edc36351a531c2d76c2a96e2a67dc1f4020d5dcb9c7132d7df8e75a8518ed1b3}
     # need to run bb for proofs and bb is only built for x86
     platform: linux/amd64
     environment:
@@ -28,7 +28,7 @@ services:
     expose:
       - 8080:8080
   cli:
-    image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:afe0263560a82f2c84b9d76451ec4585a3ed389251fb90e916faaa85d9eb9a0d}
+    image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:edc36351a531c2d76c2a96e2a67dc1f4020d5dcb9c7132d7df8e75a8518ed1b3}
     # run the same image as pxe
     platform: linux/amd64
     environment:

diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf
@@ -56,10 +56,10 @@ data "terraform_remote_state" "l1_contracts" {
 locals {
   publisher_private_keys = [var.SEQ_1_PUBLISHER_PRIVATE_KEY, var.SEQ_2_PUBLISHER_PRIVATE_KEY]
   node_p2p_private_keys  = [var.NODE_1_PRIVATE_KEY, var.NODE_2_PRIVATE_KEY]
-  node_count             = length(local.publisher_private_keys)
-  data_dir               = "/usr/src/yarn-project/aztec/data"
-  agents_per_sequencer   = var.AGENTS_PER_SEQUENCER
-  total_agents           = local.node_count * local.agents_per_sequencer
+  #node_count             = length(local.publisher_private_keys)
+  node_count           = 1
+  data_dir             = "/usr/src/yarn-project/aztec/data"
+  agents_per_sequencer = var.AGENTS_PER_SEQUENCER
 }
 
 resource "aws_cloudwatch_log_group" "aztec-node-log-group" {
@@ -115,18 +115,32 @@ resource "aws_efs_file_system" "node_data_store" {
   }
 }
 
-resource "aws_efs_mount_target" "private_az1" {
+# resource "aws_efs_mount_target" "private_az1" {
+#   count           = local.node_count
+#   file_system_id  = aws_efs_file_system.node_data_store[count.index].id
+#   subnet_id       = data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id
+#   security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id]
+# }
+
+# resource "aws_efs_mount_target" "private_az2" {
+#   count           = local.node_count
+#   file_system_id  = aws_efs_file_system.node_data_store[count.index].id
+#   subnet_id       = data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id
+#   security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id]
+# }
+
+resource "aws_efs_mount_target" "public_az1" {
   count           = local.node_count
   file_system_id  = aws_efs_file_system.node_data_store[count.index].id
-  subnet_id       = data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id
-  security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id]
+  subnet_id       = data.terraform_remote_state.setup_iac.outputs.subnet_az1_id
+  security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id]
 }
 
-resource "aws_efs_mount_target" "private_az2" {
+resource "aws_efs_mount_target" "public_az2" {
   count           = local.node_count
   file_system_id  = aws_efs_file_system.node_data_store[count.index].id
-  subnet_id       = data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id
-  security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id]
+  subnet_id       = data.terraform_remote_state.setup_iac.outputs.subnet_az2_id
+  security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id]
 }
 
 # Define task definitions for each node.
@@ -151,7 +165,7 @@ resource "aws_ecs_task_definition" "aztec-node" {
 [
   {
     "name": "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}",
-    "image": "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}",
+    "image": "${var.FULL_IMAGE}",
     "command": ["start", "--node", "--archiver", "--sequencer", "--prover"],
     "essential": true,
     "memoryReservation": 3776,
@@ -187,7 +201,7 @@ resource "aws_ecs_task_definition" "aztec-node" {
       },
       {
         "name": "DEBUG",
-        "value": "aztec:*,-json-rpc:json_proxy:*,-aztec:avm_simulator:*,discv5:*,libp2p:*"
+        "value": "aztec:*,-json-rpc:json_proxy:*,-aztec:avm_simulator:*,libp2p:*,discv5:*"
       },
       {
         "name": "ETHEREUM_HOST",
@@ -310,22 +324,6 @@ resource "aws_ecs_task_definition" "aztec-node" {
         "name": "P2P_PEER_CHECK_INTERVAL_MS",
         "value": "2000"
       },
-      {
-        "name": "ACVM_WORKING_DIRECTORY",
-        "value": "/usr/src/acvm"
-      },
-      {
-        "name": "BB_WORKING_DIRECTORY",
-        "value": "/usr/src/bb"
-      },
-      {
-        "name": "ACVM_BINARY_PATH",
-        "value": "/usr/src/noir/noir-repo/target/release/acvm"
-      },
-      {
-        "name": "BB_BINARY_PATH",
-        "value": "/usr/src/barretenberg/cpp/build/bin/bb"
-      },
       {
         "name": "PROVER_AGENTS",
         "value": "0"
@@ -380,11 +378,11 @@ resource "aws_ecs_service" "aztec-node" {
   }
 
 
-  load_balancer {
-    target_group_arn = aws_lb_target_group.aztec-node-tcp[count.index].arn
-    container_name   = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}"
-    container_port   = var.NODE_P2P_TCP_PORT + count.index
-  }
+  # load_balancer {
+  #   target_group_arn = aws_lb_target_group.aztec-node-tcp[count.index].arn
+  #   container_name   = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}"
+  #   container_port   = var.NODE_P2P_TCP_PORT + count.index
+  # }
 
   # load_balancer {
   #   target_group_arn = aws_lb_target_group.aztec-node-udp[count.index].arn
@@ -555,35 +553,30 @@ resource "aws_security_group_rule" "allow-node-udp-out" {
 // Configuration for proving agents
 
 resource "aws_cloudwatch_log_group" "aztec-proving-agent-log-group" {
-  count             = local.total_agents
-  name              = "/fargate/service/${var.DEPLOY_TAG}/aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}"
+  count             = local.node_count
+  name              = "/fargate/service/${var.DEPLOY_TAG}/aztec-proving-agent-group-${count.index + 1}"
   retention_in_days = 14
 }
 
 resource "aws_service_discovery_service" "aztec-proving-agent" {
-  count = local.total_agents
-  name  = "${var.DEPLOY_TAG}-aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}"
+  count = local.node_count
+  name  = "${var.DEPLOY_TAG}-aztec-proving-agent-group-${count.index + 1}"
 
   health_check_custom_config {
     failure_threshold = 1
   }
-
   dns_config {
     namespace_id = data.terraform_remote_state.setup_iac.outputs.local_service_discovery_id
-
     dns_records {
       ttl  = 60
       type = "A"
     }
-
     dns_records {
       ttl  = 60
       type = "SRV"
     }
-
     routing_policy = "MULTIVALUE"
   }
-
   # Terraform just fails if this resource changes and you have registered instances.
   provisioner "local-exec" {
     when    = destroy
@@ -593,23 +586,22 @@ resource "aws_service_discovery_service" "aztec-proving-agent" {
 
 # Define task definitions for each node.
 resource "aws_ecs_task_definition" "aztec-proving-agent" {
-  count                    = local.total_agents
-  family                   = "${var.DEPLOY_TAG}-aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}"
+  count                    = local.node_count
+  family                   = "${var.DEPLOY_TAG}-aztec-proving-agent-group-${count.index + 1}"
   requires_compatibilities = ["FARGATE"]
   network_mode             = "awsvpc"
   cpu                      = "16384"
-  memory                   = "65536"
+  memory                   = "98304"
   execution_role_arn       = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn
   task_role_arn            = data.terraform_remote_state.aztec2_iac.outputs.cloudwatch_logging_ecs_role_arn
-
-  container_definitions = <<DEFINITIONS
+  container_definitions    = <<DEFINITIONS
 [
   {
-    "name": "${var.DEPLOY_TAG}-aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}",
-    "image": "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}",
+    "name": "${var.DEPLOY_TAG}-aztec-proving-agent-group-${count.index + 1}",
+    "image": "${var.FULL_IMAGE}",
     "command": ["start", "--prover"],
     "essential": true,
-    "memoryReservation": 65536,
+    "memoryReservation": 98304,
     "portMappings": [
       {
         "containerPort": 80
@@ -629,8 +621,8 @@ resource "aws_ecs_task_definition" "aztec-proving-agent" {
         "value": "${var.DEPLOY_TAG}"
       },
       {
-        "name": "PROVER_URL",
-        "value": "http://${var.DEPLOY_TAG}-aztec-node-${floor(count.index / local.agents_per_sequencer) + 1}.local/${var.DEPLOY_TAG}/aztec-node-${floor(count.index / local.agents_per_sequencer) + 1}"
+        "name": "AZTEC_NODE_URL",
+        "value": "http://${var.DEPLOY_TAG}-aztec-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}"
       },
       {
         "name": "PROVER_AGENTS",
@@ -639,28 +631,12 @@ resource "aws_ecs_task_definition" "aztec-proving-agent" {
       {
         "name": "PROVER_REAL_PROOFS",
         "value": "${var.PROVING_ENABLED}"
-      },
-      {
-        "name": "ACVM_WORKING_DIRECTORY",
-        "value": "/usr/src/acvm"
-      },
-      {
-        "name": "BB_WORKING_DIRECTORY",
-        "value": "/usr/src/bb"
-      },
-      {
-        "name": "ACVM_BINARY_PATH",
-        "value": "/usr/src/noir/noir-repo/target/release/acvm"
-      },
-      {
-        "name": "BB_BINARY_PATH",
-        "value": "/usr/src/barretenberg/cpp/build/bin/bb"
       }
     ],
     "logConfiguration": {
       "logDriver": "awslogs",
       "options": {
-        "awslogs-group": "/fargate/service/${var.DEPLOY_TAG}/aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}",
+        "awslogs-group": "${aws_cloudwatch_log_group.aztec-proving-agent-log-group[count.index].name}",
         "awslogs-region": "eu-west-2",
         "awslogs-stream-prefix": "ecs"
       }
@@ -671,16 +647,14 @@ DEFINITIONS
 }
 
 resource "aws_ecs_service" "aztec-proving-agent" {
-  count                              = local.total_agents
-  name                               = "${var.DEPLOY_TAG}-aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}"
+  count                              = local.node_count
+  name                               = "${var.DEPLOY_TAG}-aztec-proving-agent-group-${count.index + 1}"
   cluster                            = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_id
   launch_type                        = "FARGATE"
   desired_count                      = 1
   deployment_maximum_percent         = 100
   deployment_minimum_healthy_percent = 0
   platform_version                   = "1.4.0"
-
-
   network_configuration {
     subnets = [
       data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id,
@@ -691,9 +665,101 @@ resource "aws_ecs_service" "aztec-proving-agent" {
 
   service_registries {
     registry_arn   = aws_service_discovery_service.aztec-proving-agent[count.index].arn
-    container_name = "${var.DEPLOY_TAG}-aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}"
+    container_name = "${var.DEPLOY_TAG}-aztec-proving-agent-group-${count.index + 1}"
     container_port = 80
   }
 
   task_definition = aws_ecs_task_definition.aztec-proving-agent[count.index].family
 }
+
+
+# Create CloudWatch metrics for the proving agents
+resource "aws_cloudwatch_metric_alarm" "cpu_high" {
+  count               = local.node_count
+  alarm_name          = "${var.DEPLOY_TAG}-proving-agent-cpu-high-${count.index + 1}"
+  comparison_operator = "GreaterThanThreshold"
+  evaluation_periods  = "1"
+  metric_name         = "CPUUtilization"
+  namespace           = "AWS/ECS"
+  period              = "60"
+  datapoints_to_alarm = 1
+  statistic           = "Maximum"
+  threshold           = "20"
+  alarm_description   = "Alert when CPU utilization is greater than 20%"
+  dimensions = {
+    ClusterName = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_name
+    ServiceName = "${aws_ecs_service.aztec-proving-agent[count.index].name}"
+  }
+  alarm_actions = [aws_appautoscaling_policy.scale_out[count.index].arn]
+}
+
+resource "aws_cloudwatch_metric_alarm" "cpu_low" {
+  count               = local.node_count
+  alarm_name          = "${var.DEPLOY_TAG}-proving-agent-cpu-low-${count.index + 1}"
+  comparison_operator = "LessThanThreshold"
+  evaluation_periods  = "3"
+  metric_name         = "CPUUtilization"
+  namespace           = "AWS/ECS"
+  period              = "60"
+  datapoints_to_alarm = 3
+  statistic           = "Maximum"
+  threshold           = "20"
+  alarm_description   = "Alarm when CPU utilization is less than 20%"
+  dimensions = {
+    ClusterName = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_name
+    ServiceName = "${aws_ecs_service.aztec-proving-agent[count.index].name}"
+  }
+  alarm_actions = [aws_appautoscaling_policy.scale_in[count.index].arn]
+}
+
+# Create Auto Scaling Target for ECS Service
+resource "aws_appautoscaling_target" "ecs_proving_agent" {
+  count              = local.node_count
+  max_capacity       = local.agents_per_sequencer
+  min_capacity       = 1
+  resource_id        = "service/${data.terraform_remote_state.setup_iac.outputs.ecs_cluster_id}/${aws_ecs_service.aztec-proving-agent[count.index].name}"
+  scalable_dimension = "ecs:service:DesiredCount"
+  service_namespace  = "ecs"
+}
+
+# Create Scaling Policy for Scaling Out
+resource "aws_appautoscaling_policy" "scale_out" {
+  count              = local.node_count
+  name               = "${var.DEPLOY_TAG}-scale-out-${count.index + 1}"
+  policy_type        = "StepScaling"
+  resource_id        = aws_appautoscaling_target.ecs_proving_agent[count.index].resource_id
+  scalable_dimension = aws_appautoscaling_target.ecs_proving_agent[count.index].scalable_dimension
+  service_namespace  = aws_appautoscaling_target.ecs_proving_agent[count.index].service_namespace
+
+  step_scaling_policy_configuration {
+    adjustment_type         = "ExactCapacity"
+    cooldown                = 60
+    metric_aggregation_type = "Maximum"
+
+    step_adjustment {
+      scaling_adjustment          = local.agents_per_sequencer
+      metric_interval_lower_bound = 0
+    }
+  }
+}
+
+# Create Scaling Policy for Scaling In
+resource "aws_appautoscaling_policy" "scale_in" {
+  count              = local.node_count
+  name               = "${var.DEPLOY_TAG}-scale-in-${count.index + 1}"
+  policy_type        = "StepScaling"
+  resource_id        = aws_appautoscaling_target.ecs_proving_agent[count.index].resource_id
+  scalable_dimension = aws_appautoscaling_target.ecs_proving_agent[count.index].scalable_dimension
+  service_namespace  = aws_appautoscaling_target.ecs_proving_agent[count.index].service_namespace
+
+  step_scaling_policy_configuration {
+    adjustment_type         = "ExactCapacity"
+    cooldown                = 60
+    metric_aggregation_type = "Maximum"
+
+    step_adjustment {
+      scaling_adjustment          = 1
+      metric_interval_upper_bound = 0
+    }
+  }
+}
diff --git a/yarn-project/aztec/terraform/node/variables.tf b/yarn-project/aztec/terraform/node/variables.tf
@@ -79,3 +79,11 @@ variable "PROVING_ENABLED" {
   type    = bool
   default = true
 }
+
+variable "IMAGE_TAG" {
+  type = string
+}
+
+variable "FULL_IMAGE" {
+  type = string
+}
diff --git a/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts b/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts
@@ -53,8 +53,8 @@ type ProvenSetup = {
  */
 
 export class FullProverTest {
-  static TOKEN_NAME = 'Aztec Token';
-  static TOKEN_SYMBOL = 'AZT';
+  static TOKEN_NAME = 'USDC';
+  static TOKEN_SYMBOL = 'USD';
   static TOKEN_DECIMALS = 18n;
   private snapshotManager: ISnapshotManager;
   logger: DebugLogger;

diff --git a/yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts b/yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts
@@ -24,8 +24,8 @@ import { TokenSimulator } from '../simulators/token_simulator.js';
 const { E2E_DATA_PATH: dataPath } = process.env;
 
 export class TokenContractTest {
-  static TOKEN_NAME = 'Aztec Token';
-  static TOKEN_SYMBOL = 'AZT';
+  static TOKEN_NAME = 'USDC';
+  static TOKEN_SYMBOL = 'USD';
   static TOKEN_DECIMALS = 18n;
   private snapshotManager: ISnapshotManager;
   logger: DebugLogger;