From 766db2d8cf4f05d85a44fd81a853373c46ae98e8 Mon Sep 17 00:00:00 2001
From: Kai Siren <kaisiren@navapbc.com>
Date: Thu, 11 Jan 2024 17:15:53 -0800
Subject: [PATCH 01/17] Backfill private subnets

---
 infra/networks/subnet-backfill.tf | 81 +++++++++++++++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100644 infra/networks/subnet-backfill.tf

diff --git a/infra/networks/subnet-backfill.tf b/infra/networks/subnet-backfill.tf
new file mode 100644
index 000000000..c43e9acf3
--- /dev/null
+++ b/infra/networks/subnet-backfill.tf
@@ -0,0 +1,81 @@
+# The purpose of this file is to backfill the default VPC with 3 private subnets.
+
+locals {
+  backfill_subnet_cidrs = {
+    # The CIDRs were chosen to be within `172.31.0.0/16` but not overlap with the nearest
+    # CIDRs already being used in the VPC.
+    "us-east-1a" = "172.31.144.0/20",
+    "us-east-1b" = "172.31.160.0/20",
+    "us-east-1c" = "172.31.176.0/20",
+  }
+}
+
+# ------- #
+# SUBNETS #
+# ------- #
+
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet
+resource "aws_subnet" "backfill_private" {
+  count                   = length(local.backfill_subnet_cidrs)
+  vpc_id                  = data.aws_vpc.default.id
+  availability_zone       = keys(local.backfill_subnet_cidrs)[count.index]
+  cidr_block              = values(local.backfill_subnet_cidrs)[count.index]
+  map_public_ip_on_launch = false
+  tags = {
+    Name        = "backfill-private-${count.index}"
+    subnet_type = "private"
+  }
+}
+
+# ----------- #
+# NAT GATEWAY #
+# ----------- #
+
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eip
+resource "aws_eip" "backfill_private" {
+  count  = length(local.backfill_subnet_cidrs)
+  domain = "vpc"
+  tags = {
+    Name = "backfill-private-${count.index}"
+  }
+}
+
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/nat_gateway
+resource "aws_nat_gateway" "backfill_private" {
+  count         = length(local.backfill_subnet_cidrs)
+  allocation_id = aws_eip.backfill_private[count.index].allocation_id
+  subnet_id     = aws_subnet.backfill_private[count.index].id
+  tags = {
+    Name = "backfill-private-${count.index}"
+  }
+}
+
+# ------------ #
+# ROUTE TABLES #
+# ------------ #
+
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table
+resource "aws_route_table" "backfill_private" {
+  count  = length(local.backfill_subnet_cidrs)
+  vpc_id = data.aws_vpc.default.id
+  tags = {
+    Name = "backfill-private-${count.index}"
+  }
+}
+
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table_association
+resource "aws_route_table_association" "backfill_private" {
+  count          = length(local.backfill_subnet_cidrs)
+  subnet_id      = aws_subnet.backfill_private[count.index].id
+  route_table_id = aws_route_table.backfill_private[count.index].id
+}
+
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route
+#
+# purpose: route external traffic to the NAT gateway
+resource "aws_route" "backfill_private" {
+  count                  = length(local.backfill_subnet_cidrs)
+  route_table_id         = aws_route_table.backfill_private[count.index].id
+  destination_cidr_block = "0.0.0.0/0"
+  nat_gateway_id         = aws_nat_gateway.backfill_private[count.index].id
+}

From 563b26d8c2474210e4a44d8dabbe6dfb0ef1d7f6 Mon Sep 17 00:00:00 2001
From: Kai Siren <kaisiren@navapbc.com>
Date: Fri, 12 Jan 2024 10:56:04 -0800
Subject: [PATCH 02/17] checkov

---
 infra/networks/subnet-backfill.tf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/infra/networks/subnet-backfill.tf b/infra/networks/subnet-backfill.tf
index c43e9acf3..d4c73067a 100644
--- a/infra/networks/subnet-backfill.tf
+++ b/infra/networks/subnet-backfill.tf
@@ -33,6 +33,7 @@ resource "aws_subnet" "backfill_private" {
 
 # docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eip
 resource "aws_eip" "backfill_private" {
+  # checkov:skip=CKV2_AWS_19: These EIPs are attached to NAT gateways
   count  = length(local.backfill_subnet_cidrs)
   domain = "vpc"
   tags = {

From 43babf25363b01792589ca2e0aa8db10952ac7c9 Mon Sep 17 00:00:00 2001
From: Kai Siren <kaisiren@navapbc.com>
Date: Fri, 12 Jan 2024 13:40:00 -0800
Subject: [PATCH 03/17] docs

---
 infra/networks/subnet-backfill.tf | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/infra/networks/subnet-backfill.tf b/infra/networks/subnet-backfill.tf
index d4c73067a..d2ec0dddc 100644
--- a/infra/networks/subnet-backfill.tf
+++ b/infra/networks/subnet-backfill.tf
@@ -4,9 +4,12 @@ locals {
   backfill_subnet_cidrs = {
     # The CIDRs were chosen to be within `172.31.0.0/16` but not overlap with the nearest
     # CIDRs already being used in the VPC.
-    "us-east-1a" = "172.31.144.0/20",
-    "us-east-1b" = "172.31.160.0/20",
-    "us-east-1c" = "172.31.176.0/20",
+    #
+    # You can can confirm the ranges with a tool like:
+    # https://www.ipaddressguide.com/cidr
+    "us-east-1a" = "172.31.144.0/20", # /20 = 4096 IPs, last address is 172.31.159.255
+    "us-east-1b" = "172.31.160.0/20", # /20 = 4096 IPs, last address is	172.31.175.255
+    "us-east-1c" = "172.31.176.0/20", # /20 = 4096 IPs, last address is 172.31.191.255
   }
 }
 
@@ -32,6 +35,9 @@ resource "aws_subnet" "backfill_private" {
 # ----------- #
 
 # docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eip
+#
+# purpose: All external traffic from the private subnets will be routed through this EIP
+#          via means of a NAT gateway.
 resource "aws_eip" "backfill_private" {
   # checkov:skip=CKV2_AWS_19: These EIPs are attached to NAT gateways
   count  = length(local.backfill_subnet_cidrs)
@@ -65,6 +71,8 @@ resource "aws_route_table" "backfill_private" {
 }
 
 # docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table_association
+#
+# purpose: Associate the private subnets with the private route table.
 resource "aws_route_table_association" "backfill_private" {
   count          = length(local.backfill_subnet_cidrs)
   subnet_id      = aws_subnet.backfill_private[count.index].id
@@ -73,7 +81,7 @@ resource "aws_route_table_association" "backfill_private" {
 
 # docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route
 #
-# purpose: route external traffic to the NAT gateway
+# purpose: Route external traffic through the NAT gateway.
 resource "aws_route" "backfill_private" {
   count                  = length(local.backfill_subnet_cidrs)
   route_table_id         = aws_route_table.backfill_private[count.index].id

From 43fde086179716a039c0545f788db86f9cdc5044 Mon Sep 17 00:00:00 2001
From: Kai Siren <kaisiren@navapbc.com>
Date: Fri, 12 Jan 2024 13:44:44 -0800
Subject: [PATCH 04/17] docs

---
 infra/networks/subnet-backfill.tf | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/infra/networks/subnet-backfill.tf b/infra/networks/subnet-backfill.tf
index d2ec0dddc..08e2eafd4 100644
--- a/infra/networks/subnet-backfill.tf
+++ b/infra/networks/subnet-backfill.tf
@@ -7,6 +7,8 @@ locals {
     #
     # You can can confirm the ranges with a tool like:
     # https://www.ipaddressguide.com/cidr
+    #
+    # The `/20` CIDR block was chosen because most of the existing subnets are `/20`.
     "us-east-1a" = "172.31.144.0/20", # /20 = 4096 IPs, last address is 172.31.159.255
     "us-east-1b" = "172.31.160.0/20", # /20 = 4096 IPs, last address is	172.31.175.255
     "us-east-1c" = "172.31.176.0/20", # /20 = 4096 IPs, last address is 172.31.191.255

From 175950c6664cd077bc4441782d20667fcd030536 Mon Sep 17 00:00:00 2001
From: "kai [they]" <coilysiren@gmail.com>
Date: Tue, 16 Jan 2024 09:25:35 -0800
Subject: [PATCH 05/17] Update infra/networks/subnet-backfill.tf

Co-authored-by: James Bursa <james@navapbc.com>
---
 infra/networks/subnet-backfill.tf | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/infra/networks/subnet-backfill.tf b/infra/networks/subnet-backfill.tf
index 08e2eafd4..428f8f931 100644
--- a/infra/networks/subnet-backfill.tf
+++ b/infra/networks/subnet-backfill.tf
@@ -21,13 +21,13 @@ locals {
 
 # docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet
 resource "aws_subnet" "backfill_private" {
-  count                   = length(local.backfill_subnet_cidrs)
+  for_each                = local.backfill_subnet_cidrs
   vpc_id                  = data.aws_vpc.default.id
-  availability_zone       = keys(local.backfill_subnet_cidrs)[count.index]
-  cidr_block              = values(local.backfill_subnet_cidrs)[count.index]
+  availability_zone       = each.key
+  cidr_block              = each.value
   map_public_ip_on_launch = false
   tags = {
-    Name        = "backfill-private-${count.index}"
+    Name        = "backfill-private-${each.key}"
     subnet_type = "private"
   }
 }

From d8dabe7140f49dff70c91bbe2edc317f9cc2cd5c Mon Sep 17 00:00:00 2001
From: "kai [they]" <coilysiren@gmail.com>
Date: Tue, 16 Jan 2024 09:26:42 -0800
Subject: [PATCH 06/17] Update infra/networks/subnet-backfill.tf

Co-authored-by: James Bursa <james@navapbc.com>
---
 infra/networks/subnet-backfill.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/infra/networks/subnet-backfill.tf b/infra/networks/subnet-backfill.tf
index 428f8f931..7fff8a586 100644
--- a/infra/networks/subnet-backfill.tf
+++ b/infra/networks/subnet-backfill.tf
@@ -42,10 +42,10 @@ resource "aws_subnet" "backfill_private" {
 #          via means of a NAT gateway.
 resource "aws_eip" "backfill_private" {
   # checkov:skip=CKV2_AWS_19: These EIPs are attached to NAT gateways
-  count  = length(local.backfill_subnet_cidrs)
+  for_each = local.backfill_subnet_cidrs
   domain = "vpc"
   tags = {
-    Name = "backfill-private-${count.index}"
+    Name = "backfill-private-${each.key}"
   }
 }
 

From c92157ac2cd1c46a4341afb75946cd49865805af Mon Sep 17 00:00:00 2001
From: "kai [they]" <coilysiren@gmail.com>
Date: Tue, 16 Jan 2024 09:26:49 -0800
Subject: [PATCH 07/17] Update infra/networks/subnet-backfill.tf

Co-authored-by: James Bursa <james@navapbc.com>
---
 infra/networks/subnet-backfill.tf | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/infra/networks/subnet-backfill.tf b/infra/networks/subnet-backfill.tf
index 7fff8a586..4e9bd62df 100644
--- a/infra/networks/subnet-backfill.tf
+++ b/infra/networks/subnet-backfill.tf
@@ -51,11 +51,11 @@ resource "aws_eip" "backfill_private" {
 
 # docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/nat_gateway
 resource "aws_nat_gateway" "backfill_private" {
-  count         = length(local.backfill_subnet_cidrs)
-  allocation_id = aws_eip.backfill_private[count.index].allocation_id
-  subnet_id     = aws_subnet.backfill_private[count.index].id
+  for_each      = local.backfill_subnet_cidrs
+  allocation_id = aws_eip.backfill_private[each.key].allocation_id
+  subnet_id     = aws_subnet.backfill_private[each.key].id
   tags = {
-    Name = "backfill-private-${count.index}"
+    Name = "backfill-private-${each.key}"
   }
 }
 

From 046898c865201ed4a9ba9a657db7d3d6de1e94d1 Mon Sep 17 00:00:00 2001
From: "kai [they]" <coilysiren@gmail.com>
Date: Tue, 16 Jan 2024 09:26:54 -0800
Subject: [PATCH 08/17] Update infra/networks/subnet-backfill.tf

Co-authored-by: James Bursa <james@navapbc.com>
---
 infra/networks/subnet-backfill.tf | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/infra/networks/subnet-backfill.tf b/infra/networks/subnet-backfill.tf
index 4e9bd62df..5345193b0 100644
--- a/infra/networks/subnet-backfill.tf
+++ b/infra/networks/subnet-backfill.tf
@@ -65,10 +65,10 @@ resource "aws_nat_gateway" "backfill_private" {
 
 # docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table
 resource "aws_route_table" "backfill_private" {
-  count  = length(local.backfill_subnet_cidrs)
+  for_each = local.backfill_subnet_cidrs
   vpc_id = data.aws_vpc.default.id
   tags = {
-    Name = "backfill-private-${count.index}"
+    Name = "backfill-private-${each.key}"
   }
 }
 
@@ -76,17 +76,17 @@ resource "aws_route_table" "backfill_private" {
 #
 # purpose: Associate the private subnets with the private route table.
 resource "aws_route_table_association" "backfill_private" {
-  count          = length(local.backfill_subnet_cidrs)
-  subnet_id      = aws_subnet.backfill_private[count.index].id
-  route_table_id = aws_route_table.backfill_private[count.index].id
+  for_each       = local.backfill_subnet_cidrs
+  subnet_id      = aws_subnet.backfill_private[each.key].id
+  route_table_id = aws_route_table.backfill_private[each.key].id
 }
 
 # docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route
 #
 # purpose: Route external traffic through the NAT gateway.
 resource "aws_route" "backfill_private" {
-  count                  = length(local.backfill_subnet_cidrs)
-  route_table_id         = aws_route_table.backfill_private[count.index].id
+  for_each               = local.backfill_subnet_cidrs
+  route_table_id         = aws_route_table.backfill_private[each.key].id
   destination_cidr_block = "0.0.0.0/0"
-  nat_gateway_id         = aws_nat_gateway.backfill_private[count.index].id
+  nat_gateway_id         = aws_nat_gateway.backfill_private[each.key].id
 }

From 3d9074dec2fcd055648cd1db0d2ab41ea6fc300c Mon Sep 17 00:00:00 2001
From: Kai Siren <kaisiren@navapbc.com>
Date: Tue, 16 Jan 2024 12:40:12 -0800
Subject: [PATCH 09/17] Stage 2 changes

---
 infra/frontend/app-config/main.tf | 14 ++++++++------
 infra/frontend/service/main.tf    |  6 ++++--
 infra/modules/service/main.tf     |  4 +---
 infra/networks/main.tf            | 21 +++++++++++++++++++--
 infra/networks/subnet-backfill.tf |  4 ++--
 5 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/infra/frontend/app-config/main.tf b/infra/frontend/app-config/main.tf
index 159a7d283..9f0414f80 100644
--- a/infra/frontend/app-config/main.tf
+++ b/infra/frontend/app-config/main.tf
@@ -1,6 +1,6 @@
 locals {
   app_name                        = "frontend"
-  environments                    = ["dev", "prod"]
+  environments                    = ["dev", "staging", "prod"]
   project_name                    = module.project_config.project_name
   image_repository_name           = "${local.project_name}-${local.app_name}"
   has_database                    = false
@@ -13,8 +13,9 @@ locals {
   }
 
   environment_configs = {
-    dev  = module.dev_config
-    prod = module.prod_config
+    dev     = module.dev_config
+    staging = module.staging_config
+    prod    = module.prod_config
   }
   # Map from environment name to the account name for the AWS account that
   # contains the resources for that environment. Resources that are shared
@@ -46,9 +47,10 @@ locals {
   #     prod    = "prod"
   #   }
   account_names_by_environment = {
-    shared = "simpler-grants-gov"
-    dev    = "simpler-grants-gov"
-    prod   = "simpler-grants-gov"
+    shared  = "simpler-grants-gov"
+    dev     = "simpler-grants-gov"
+    staging = "simpler-grants-gov"
+    prod    = "simpler-grants-gov"
   }
 }
 
diff --git a/infra/frontend/service/main.tf b/infra/frontend/service/main.tf
index 199eff25c..fe5279753 100644
--- a/infra/frontend/service/main.tf
+++ b/infra/frontend/service/main.tf
@@ -1,13 +1,15 @@
 # TODO(https://github.com/navapbc/template-infra/issues/152) use non-default VPC
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc
 data "aws_vpc" "default" {
   default = true
 }
 
 # TODO(https://github.com/navapbc/template-infra/issues/152) use private subnets
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet
 data "aws_subnets" "default" {
   filter {
-    name   = "default-for-az"
-    values = [true]
+    name   = "tag:subnet_type"
+    values = ["private"]
   }
 }
 
diff --git a/infra/modules/service/main.tf b/infra/modules/service/main.tf
index 07a6f887a..93a7c7030 100644
--- a/infra/modules/service/main.tf
+++ b/infra/modules/service/main.tf
@@ -49,9 +49,7 @@ resource "aws_ecs_service" "app" {
   }
 
   network_configuration {
-    # TODO(https://github.com/navapbc/template-infra/issues/152) set assign_public_ip = false after using private subnets
-    # checkov:skip=CKV_AWS_333:Switch to using private subnets
-    assign_public_ip = true
+    assign_public_ip = false
     subnets          = var.subnet_ids
     security_groups  = [aws_security_group.app.id]
   }
diff --git a/infra/networks/main.tf b/infra/networks/main.tf
index 140573d8c..37b933f34 100644
--- a/infra/networks/main.tf
+++ b/infra/networks/main.tf
@@ -17,9 +17,17 @@ locals {
   # see https://docs.aws.amazon.com/vpc/latest/privatelink/aws-services-privatelink-support.html
   #
   # The database module requires VPC access from private networks to SSM, KMS, and RDS
-  aws_service_integrations = toset(
-    module.app_config.has_database ? ["ssm", "kms"] : []
+  aws_service_integrations = setunion(
+    # AWS services used by ECS Fargate: ECR to fetch images, S3 for image layers, and CloudWatch for logs
+    ["ecr.api", "ecr.dkr", "s3", "logs"],
+
+    # AWS services used by the database's role manager
+    var.has_database ? ["ssm", "kms", "secretsmanager"] : [],
   )
+
+  # S3 and DynamoDB use Gateway VPC endpoints. All other services use Interface VPC endpoints
+  interface_vpc_endpoints = toset([for aws_service in local.aws_service_integrations : aws_service if !contains(["s3", "dynamodb"], aws_service)])
+  gateway_vpc_endpoints   = toset([for aws_service in local.aws_service_integrations : aws_service if contains(["s3", "dynamodb"], aws_service)])
 }
 
 terraform {
@@ -94,6 +102,15 @@ resource "aws_vpc_endpoint" "aws_service" {
   private_dns_enabled = true
 }
 
+resource "aws_vpc_endpoint" "gateway" {
+  for_each = local.gateway_vpc_endpoints
+
+  vpc_id            = data.aws_vpc.default.id
+  service_name      = "com.amazonaws.${data.aws_region.current.name}.${each.key}"
+  vpc_endpoint_type = "Gateway"
+  route_table_ids   = [for table in aws_route_table.backfill_private : table.id]
+}
+
 # VPC Configuration for DMS
 # ----------------------------------------
 
diff --git a/infra/networks/subnet-backfill.tf b/infra/networks/subnet-backfill.tf
index 5345193b0..7009f0457 100644
--- a/infra/networks/subnet-backfill.tf
+++ b/infra/networks/subnet-backfill.tf
@@ -43,7 +43,7 @@ resource "aws_subnet" "backfill_private" {
 resource "aws_eip" "backfill_private" {
   # checkov:skip=CKV2_AWS_19: These EIPs are attached to NAT gateways
   for_each = local.backfill_subnet_cidrs
-  domain = "vpc"
+  domain   = "vpc"
   tags = {
     Name = "backfill-private-${each.key}"
   }
@@ -66,7 +66,7 @@ resource "aws_nat_gateway" "backfill_private" {
 # docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table
 resource "aws_route_table" "backfill_private" {
   for_each = local.backfill_subnet_cidrs
-  vpc_id = data.aws_vpc.default.id
+  vpc_id   = data.aws_vpc.default.id
   tags = {
     Name = "backfill-private-${each.key}"
   }

From 62ac8c7f2649a96c6f8c3dcf316a9240f92d6306 Mon Sep 17 00:00:00 2001
From: Kai Siren <kaisiren@navapbc.com>
Date: Tue, 16 Jan 2024 13:04:06 -0800
Subject: [PATCH 10/17] move subnets

---
 infra/networks/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/infra/networks/main.tf b/infra/networks/main.tf
index 37b933f34..6ce43bdd7 100644
--- a/infra/networks/main.tf
+++ b/infra/networks/main.tf
@@ -98,7 +98,7 @@ resource "aws_vpc_endpoint" "aws_service" {
   service_name        = "com.amazonaws.${data.aws_region.current.name}.${each.key}"
   vpc_endpoint_type   = "Interface"
   security_group_ids  = [aws_security_group.aws_services[0].id]
-  subnet_ids          = data.aws_subnets.default.ids
+  subnet_ids          = [for subnet in aws_subnet.backfill_private : subnet.id]
   private_dns_enabled = true
 }
 

From d460cce00a28247a4a1f878c4bdf80aaa9ce764a Mon Sep 17 00:00:00 2001
From: Kai Siren <kaisiren@navapbc.com>
Date: Tue, 16 Jan 2024 13:18:32 -0800
Subject: [PATCH 11/17] security group ingress

---
 infra/networks/main.tf | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/infra/networks/main.tf b/infra/networks/main.tf
index 6ce43bdd7..ab1ccfb56 100644
--- a/infra/networks/main.tf
+++ b/infra/networks/main.tf
@@ -83,6 +83,7 @@ data "aws_subnets" "default" {
 # See https://repost.aws/knowledge-center/lambda-vpc-parameter-store
 # See https://docs.aws.amazon.com/vpc/latest/privatelink/create-interface-endpoint.html#create-interface-endpoint
 
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group
 resource "aws_security_group" "aws_services" {
   count = length(local.aws_service_integrations) > 0 ? 1 : 0
 
@@ -91,6 +92,19 @@ resource "aws_security_group" "aws_services" {
   vpc_id      = data.aws_vpc.default.id
 }
 
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule
+#
+# purpose: Allow all traffic from the VPC's CIDR block to the VPC endpoint security group
+resource "aws_vpc_security_group_ingress_rule" "aws_services" {
+  count = length(local.aws_service_integrations) > 0 ? 1 : 0
+
+  security_group_id = aws_security_group.aws_services[0].id
+  from_port         = 443
+  to_port           = 443
+  ip_protocol       = "tcp"
+  cidr_ipv4         = data.aws_vpc.default.cidr_block
+}
+
 resource "aws_vpc_endpoint" "aws_service" {
   for_each = local.aws_service_integrations
 

From 595df4f9d8f8026bbc85816cf2b109d7d799efbe Mon Sep 17 00:00:00 2001
From: Kai Siren <kaisiren@navapbc.com>
Date: Tue, 16 Jan 2024 13:18:39 -0800
Subject: [PATCH 12/17] staging deploy

---
 infra/frontend/app-config/staging.tf        | 12 ++++++++++++
 infra/frontend/service/staging.s3.tfbackend |  4 ++++
 infra/networks/variables.tf                 | 10 ++++++++++
 3 files changed, 26 insertions(+)
 create mode 100644 infra/frontend/app-config/staging.tf
 create mode 100644 infra/frontend/service/staging.s3.tfbackend
 create mode 100644 infra/networks/variables.tf

diff --git a/infra/frontend/app-config/staging.tf b/infra/frontend/app-config/staging.tf
new file mode 100644
index 000000000..1bf8cdf7e
--- /dev/null
+++ b/infra/frontend/app-config/staging.tf
@@ -0,0 +1,12 @@
+module "staging_config" {
+  source                          = "./env-config"
+  app_name                        = local.app_name
+  default_region                  = module.project_config.default_region
+  environment                     = "staging"
+  has_database                    = local.has_database
+  has_incident_management_service = local.has_incident_management_service
+  domain                          = "beta.grants.gov"
+  sendy_api_key                   = "/${local.app_name}/dev/sendy-api-key"
+  sendy_api_url                   = "/${local.app_name}/dev/sendy-api-url"
+  sendy_list_id                   = "/${local.app_name}/dev/sendy-list-id"
+}
diff --git a/infra/frontend/service/staging.s3.tfbackend b/infra/frontend/service/staging.s3.tfbackend
new file mode 100644
index 000000000..a19f6284a
--- /dev/null
+++ b/infra/frontend/service/staging.s3.tfbackend
@@ -0,0 +1,4 @@
+bucket         = "simpler-grants-gov-315341936575-us-east-1-tf"
+key            = "infra/frontend/service/staging.tfstate"
+dynamodb_table = "simpler-grants-gov-315341936575-us-east-1-tf-state-locks"
+region         = "us-east-1"
diff --git a/infra/networks/variables.tf b/infra/networks/variables.tf
new file mode 100644
index 000000000..84825e9ad
--- /dev/null
+++ b/infra/networks/variables.tf
@@ -0,0 +1,10 @@
+variable "aws_services_security_group_name_prefix" {
+  type    = string
+  default = "simpler"
+}
+
+variable "has_database" {
+  type        = bool
+  description = "whether the application has a database"
+  default     = true
+}

From e730721d70aaf45e6e785ce80d73d63bc622e018 Mon Sep 17 00:00:00 2001
From: Kai Siren <kaisiren@navapbc.com>
Date: Tue, 16 Jan 2024 13:26:29 -0800
Subject: [PATCH 13/17] docs

---
 infra/networks/main.tf | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/infra/networks/main.tf b/infra/networks/main.tf
index ab1ccfb56..f271aed9e 100644
--- a/infra/networks/main.tf
+++ b/infra/networks/main.tf
@@ -93,12 +93,11 @@ resource "aws_security_group" "aws_services" {
 }
 
 # docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule
-#
-# purpose: Allow all traffic from the VPC's CIDR block to the VPC endpoint security group
 resource "aws_vpc_security_group_ingress_rule" "aws_services" {
   count = length(local.aws_service_integrations) > 0 ? 1 : 0
 
   security_group_id = aws_security_group.aws_services[0].id
+  description       = "Allow all traffic from the VPC's CIDR block to the VPC endpoint security group"
   from_port         = 443
   to_port           = 443
   ip_protocol       = "tcp"

From 44342289614e7ebb583bf2404d26c6bbf9f54c9b Mon Sep 17 00:00:00 2001
From: Kai Siren <kaisiren@navapbc.com>
Date: Tue, 16 Jan 2024 14:20:35 -0800
Subject: [PATCH 14/17] pull in latest nava template changes

---
 infra/frontend/service/main.tf          | 13 ++++++++++---
 infra/modules/service/access-control.tf |  8 ++++++++
 infra/modules/service/load-balancer.tf  |  2 +-
 infra/modules/service/main.tf           |  4 ++--
 infra/modules/service/variables.tf      | 23 ++++++++++++++++++++---
 infra/networks/main.tf                  |  2 +-
 infra/networks/variables.tf             |  5 -----
 7 files changed, 42 insertions(+), 15 deletions(-)

diff --git a/infra/frontend/service/main.tf b/infra/frontend/service/main.tf
index fe5279753..2b12724bf 100644
--- a/infra/frontend/service/main.tf
+++ b/infra/frontend/service/main.tf
@@ -4,15 +4,21 @@ data "aws_vpc" "default" {
   default = true
 }
 
-# TODO(https://github.com/navapbc/template-infra/issues/152) use private subnets
 # docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet
-data "aws_subnets" "default" {
+data "aws_subnets" "private" {
   filter {
     name   = "tag:subnet_type"
     values = ["private"]
   }
 }
 
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet
+data "aws_subnets" "public" {
+  filter {
+    name   = "tag:subnet_type"
+    values = ["public"]
+  }
+}
 
 locals {
   # The prefix key/value pair is used for Terraform Workspaces, which is useful for projects with multiple infrastructure developers.
@@ -113,7 +119,8 @@ module "service" {
   image_repository_name = module.app_config.image_repository_name
   image_tag             = local.image_tag
   vpc_id                = data.aws_vpc.default.id
-  subnet_ids            = data.aws_subnets.default.ids
+  public_subnet_ids     = data.aws_subnets.public.ids
+  private_subnet_ids    = data.aws_subnets.private.ids
   enable_autoscaling    = module.app_config.enable_autoscaling
   cert_arn              = terraform.workspace == "default" ? data.aws_acm_certificate.cert[0].arn : null
   hostname              = module.app_config.hostname
diff --git a/infra/modules/service/access-control.tf b/infra/modules/service/access-control.tf
index 0b295088f..c7566c93f 100644
--- a/infra/modules/service/access-control.tf
+++ b/infra/modules/service/access-control.tf
@@ -69,3 +69,11 @@ resource "aws_iam_role_policy" "task_executor" {
   role   = aws_iam_role.task_executor.id
   policy = data.aws_iam_policy_document.task_executor.json
 }
+
+
+resource "aws_iam_role_policy_attachment" "extra_policies" {
+  for_each = var.extra_policies
+
+  role       = aws_iam_role.app_service.name
+  policy_arn = each.value
+}
diff --git a/infra/modules/service/load-balancer.tf b/infra/modules/service/load-balancer.tf
index 631c77db1..08876b068 100644
--- a/infra/modules/service/load-balancer.tf
+++ b/infra/modules/service/load-balancer.tf
@@ -9,7 +9,7 @@ resource "aws_lb" "alb" {
   idle_timeout    = "120"
   internal        = false
   security_groups = [aws_security_group.alb.id]
-  subnets         = var.subnet_ids
+  subnets         = var.public_subnet_ids
 
   # TODO(https://github.com/navapbc/template-infra/issues/163) Implement HTTPS
   # checkov:skip=CKV2_AWS_20:Redirect HTTP to HTTPS as part of implementing HTTPS support
diff --git a/infra/modules/service/main.tf b/infra/modules/service/main.tf
index 93a7c7030..4484e52db 100644
--- a/infra/modules/service/main.tf
+++ b/infra/modules/service/main.tf
@@ -28,7 +28,7 @@ locals {
     { name : "DB_NAME", value : var.db_vars.connection_info.db_name },
     { name : "DB_SCHEMA", value : var.db_vars.connection_info.schema_name },
   ]
-  environment_variables = concat(local.base_environment_variables, local.db_environment_variables)
+  environment_variables = concat(local.base_environment_variables, local.db_environment_variables, var.extra_environment_variables)
 }
 
 #-------------------
@@ -50,7 +50,7 @@ resource "aws_ecs_service" "app" {
 
   network_configuration {
     assign_public_ip = false
-    subnets          = var.subnet_ids
+    subnets          = var.private_subnet_ids
     security_groups  = [aws_security_group.app.id]
   }
 
diff --git a/infra/modules/service/variables.tf b/infra/modules/service/variables.tf
index c3f5b86dc..4b33931d3 100644
--- a/infra/modules/service/variables.tf
+++ b/infra/modules/service/variables.tf
@@ -70,9 +70,20 @@ variable "vpc_id" {
   description = "Uniquely identifies the VPC."
 }
 
-variable "subnet_ids" {
+variable "public_subnet_ids" {
   type        = list(any)
-  description = "Private subnet id from vpc module"
+  description = "Public subnet ids in VPC"
+}
+
+variable "private_subnet_ids" {
+  type        = list(any)
+  description = "Private subnet ids in VPC"
+}
+
+variable "extra_environment_variables" {
+  type        = list(object({ name = string, value = string }))
+  description = "Additional environment variables to pass to the service container"
+  default     = []
 }
 
 variable "db_vars" {
@@ -92,6 +103,12 @@ variable "db_vars" {
   default = null
 }
 
+variable "extra_policies" {
+  description = "Map of extra IAM policies to attach to the service's task role. The map's keys define the resource name in terraform."
+  type        = map(string)
+  default     = {}
+}
+
 variable "cert_arn" {
   description = "The ARN for the TLS certificate passed in from the app service layer"
   type        = string
@@ -120,4 +137,4 @@ variable "api_auth_token" {
   type        = string
   default     = null
   description = "Auth token for connecting to the API"
-}
\ No newline at end of file
+}
diff --git a/infra/networks/main.tf b/infra/networks/main.tf
index f271aed9e..46adccfb0 100644
--- a/infra/networks/main.tf
+++ b/infra/networks/main.tf
@@ -97,7 +97,7 @@ resource "aws_vpc_security_group_ingress_rule" "aws_services" {
   count = length(local.aws_service_integrations) > 0 ? 1 : 0
 
   security_group_id = aws_security_group.aws_services[0].id
-  description       = "Allow all traffic from the VPC's CIDR block to the VPC endpoint security group"
+  description       = "Allow all traffic from the VPCs CIDR block to the VPC endpoint security group"
   from_port         = 443
   to_port           = 443
   ip_protocol       = "tcp"
diff --git a/infra/networks/variables.tf b/infra/networks/variables.tf
index 84825e9ad..f45967381 100644
--- a/infra/networks/variables.tf
+++ b/infra/networks/variables.tf
@@ -1,8 +1,3 @@
-variable "aws_services_security_group_name_prefix" {
-  type    = string
-  default = "simpler"
-}
-
 variable "has_database" {
   type        = bool
   description = "whether the application has a database"

From 530f1bddaf7a91007b0d91a6da7f14d203ab7ed3 Mon Sep 17 00:00:00 2001
From: Kai Siren <kaisiren@navapbc.com>
Date: Tue, 16 Jan 2024 14:45:41 -0800
Subject: [PATCH 15/17] use staging sendy

---
 infra/frontend/app-config/staging.tf | 6 +++---
 infra/networks/main.tf               | 2 ++
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/infra/frontend/app-config/staging.tf b/infra/frontend/app-config/staging.tf
index 1bf8cdf7e..96265bf85 100644
--- a/infra/frontend/app-config/staging.tf
+++ b/infra/frontend/app-config/staging.tf
@@ -6,7 +6,7 @@ module "staging_config" {
   has_database                    = local.has_database
   has_incident_management_service = local.has_incident_management_service
   domain                          = "beta.grants.gov"
-  sendy_api_key                   = "/${local.app_name}/dev/sendy-api-key"
-  sendy_api_url                   = "/${local.app_name}/dev/sendy-api-url"
-  sendy_list_id                   = "/${local.app_name}/dev/sendy-list-id"
+  sendy_api_key                   = "/${local.app_name}/staging/sendy-api-key"
+  sendy_api_url                   = "/${local.app_name}/staging/sendy-api-url"
+  sendy_list_id                   = "/${local.app_name}/staging/sendy-list-id"
 }
diff --git a/infra/networks/main.tf b/infra/networks/main.tf
index 46adccfb0..9f5ac2057 100644
--- a/infra/networks/main.tf
+++ b/infra/networks/main.tf
@@ -104,6 +104,7 @@ resource "aws_vpc_security_group_ingress_rule" "aws_services" {
   cidr_ipv4         = data.aws_vpc.default.cidr_block
 }
 
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_endpoint
 resource "aws_vpc_endpoint" "aws_service" {
   for_each = local.aws_service_integrations
 
@@ -115,6 +116,7 @@ resource "aws_vpc_endpoint" "aws_service" {
   private_dns_enabled = true
 }
 
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_endpoint
 resource "aws_vpc_endpoint" "gateway" {
   for_each = local.gateway_vpc_endpoints
 

From f0ae7067ba648d0fa052adc60b89dee59b07dab7 Mon Sep 17 00:00:00 2001
From: Kai Siren <kaisiren@navapbc.com>
Date: Tue, 16 Jan 2024 16:41:10 -0800
Subject: [PATCH 16/17] add staging backend

---
 infra/api/app-config/main.tf            | 12 +++++++-----
 infra/api/app-config/staging.tf         |  8 ++++++++
 infra/api/database/staging.s3.tfbackend |  4 ++++
 infra/api/service/main.tf               | 19 ++++++++++++++-----
 infra/api/service/staging.s3.tfbackend  |  4 ++++
 5 files changed, 37 insertions(+), 10 deletions(-)
 create mode 100644 infra/api/app-config/staging.tf
 create mode 100644 infra/api/database/staging.s3.tfbackend
 create mode 100644 infra/api/service/staging.s3.tfbackend

diff --git a/infra/api/app-config/main.tf b/infra/api/app-config/main.tf
index 69bb12af5..d749d84b4 100644
--- a/infra/api/app-config/main.tf
+++ b/infra/api/app-config/main.tf
@@ -7,8 +7,9 @@ locals {
   has_incident_management_service = false
 
   environment_configs = {
-    dev  = module.dev_config
-    prod = module.prod_config
+    dev     = module.dev_config
+    staging = module.staging_config
+    prod    = module.prod_config
   }
 
   build_repository_config = {
@@ -44,9 +45,10 @@ locals {
   #     prod    = "prod"
   #   }
   account_names_by_environment = {
-    shared = "simpler-grants-gov"
-    dev    = "simpler-grants-gov"
-    prod   = "simpler-grants-gov"
+    shared  = "simpler-grants-gov"
+    dev     = "simpler-grants-gov"
+    staging = "simpler-grants-gov"
+    prod    = "simpler-grants-gov"
   }
 }
 
diff --git a/infra/api/app-config/staging.tf b/infra/api/app-config/staging.tf
new file mode 100644
index 000000000..4f3e94191
--- /dev/null
+++ b/infra/api/app-config/staging.tf
@@ -0,0 +1,8 @@
+module "staging_config" {
+  source                          = "./env-config"
+  app_name                        = local.app_name
+  default_region                  = module.project_config.default_region
+  environment                     = "staging"
+  has_database                    = local.has_database
+  has_incident_management_service = local.has_incident_management_service
+}
diff --git a/infra/api/database/staging.s3.tfbackend b/infra/api/database/staging.s3.tfbackend
new file mode 100644
index 000000000..c94da0d3f
--- /dev/null
+++ b/infra/api/database/staging.s3.tfbackend
@@ -0,0 +1,4 @@
+bucket         = "simpler-grants-gov-315341936575-us-east-1-tf"
+key            = "infra/api/database/staging.tfstate"
+dynamodb_table = "simpler-grants-gov-315341936575-us-east-1-tf-state-locks"
+region         = "us-east-1"
diff --git a/infra/api/service/main.tf b/infra/api/service/main.tf
index def913636..b12ffce8b 100644
--- a/infra/api/service/main.tf
+++ b/infra/api/service/main.tf
@@ -1,16 +1,24 @@
 # TODO(https://github.com/navapbc/template-infra/issues/152) use non-default VPC
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc
 data "aws_vpc" "default" {
   default = true
 }
 
-# TODO(https://github.com/navapbc/template-infra/issues/152) use private subnets
-data "aws_subnets" "default" {
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet
+data "aws_subnets" "private" {
   filter {
-    name   = "default-for-az"
-    values = [true]
+    name   = "tag:subnet_type"
+    values = ["private"]
   }
 }
 
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet
+data "aws_subnets" "public" {
+  filter {
+    name   = "tag:subnet_type"
+    values = ["public"]
+  }
+}
 
 locals {
   # The prefix key/value pair is used for Terraform Workspaces, which is useful for projects with multiple infrastructure developers.
@@ -95,7 +103,8 @@ module "service" {
   image_repository_name = module.app_config.image_repository_name
   image_tag             = local.image_tag
   vpc_id                = data.aws_vpc.default.id
-  subnet_ids            = data.aws_subnets.default.ids
+  public_subnet_ids     = data.aws_subnets.public.ids
+  private_subnet_ids    = data.aws_subnets.private.ids
   cpu                   = 1024
   memory                = 2048
 
diff --git a/infra/api/service/staging.s3.tfbackend b/infra/api/service/staging.s3.tfbackend
new file mode 100644
index 000000000..3bd263b17
--- /dev/null
+++ b/infra/api/service/staging.s3.tfbackend
@@ -0,0 +1,4 @@
+bucket         = "simpler-grants-gov-315341936575-us-east-1-tf"
+key            = "infra/api/service/staging.tfstate"
+dynamodb_table = "simpler-grants-gov-315341936575-us-east-1-tf-state-locks"
+region         = "us-east-1"

From e3998c2c57cbca38dcd6b49d4a72850bba01264a Mon Sep 17 00:00:00 2001
From: Kai Siren <kaisiren@navapbc.com>
Date: Wed, 17 Jan 2024 12:23:08 -0800
Subject: [PATCH 17/17] fix role manager lambda

---
 infra/modules/database/role_manager/role_manager.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/infra/modules/database/role_manager/role_manager.py b/infra/modules/database/role_manager/role_manager.py
index a435e65bd..a0544cf44 100644
--- a/infra/modules/database/role_manager/role_manager.py
+++ b/infra/modules/database/role_manager/role_manager.py
@@ -2,9 +2,14 @@
 import itertools
 from operator import itemgetter
 import os
+import json
 import logging
 from pg8000.native import Connection, identifier
 
+logging.basicConfig()
+logging.getLogger('botocore').setLevel(logging.DEBUG)
+logging.getLogger('boto3').setLevel(logging.DEBUG)
+
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)
 
@@ -115,9 +120,9 @@ def connect_using_iam(user: str) -> Connection:
     return Connection(user=user, host=host, port=port, database=database, password=token, ssl_context=True)
 
 def get_password() -> str:
-    ssm = boto3.client("ssm")
+    ssm = boto3.client("ssm",region_name=os.environ["AWS_REGION"])
     param_name = os.environ["DB_PASSWORD_PARAM_NAME"]
-    logger.info("Fetching password from parameter store")
+    logger.info("Fetching password from parameter store:\n%s"%param_name)
     result = json.loads(ssm.get_parameter(
         Name=param_name,
         WithDecryption=True,