[Issue 963] Moves ECS services into private subnets

infra/api/app-config/main.tf

@@ -7,8 +7,9 @@ locals {
   has_incident_management_service = false
 
   environment_configs = {
-    dev  = module.dev_config
-    prod = module.prod_config
+    dev     = module.dev_config
+    staging = module.staging_config
+    prod    = module.prod_config
   }
 
   build_repository_config = {
@@ -44,9 +45,10 @@ locals {
   #     prod    = "prod"
   #   }
   account_names_by_environment = {
-    shared = "simpler-grants-gov"
-    dev    = "simpler-grants-gov"
-    prod   = "simpler-grants-gov"
+    shared  = "simpler-grants-gov"
+    dev     = "simpler-grants-gov"
+    staging = "simpler-grants-gov"
+    prod    = "simpler-grants-gov"
   }
 }
 

infra/api/app-config/staging.tf

@@ -0,0 +1,8 @@
+module "staging_config" {
+  source                          = "./env-config"
+  app_name                        = local.app_name
+  default_region                  = module.project_config.default_region
+  environment                     = "staging"
+  has_database                    = local.has_database
+  has_incident_management_service = local.has_incident_management_service
+}

infra/api/database/staging.s3.tfbackend

@@ -0,0 +1,4 @@
+bucket         = "simpler-grants-gov-315341936575-us-east-1-tf"
+key            = "infra/api/database/staging.tfstate"
+dynamodb_table = "simpler-grants-gov-315341936575-us-east-1-tf-state-locks"
+region         = "us-east-1"

infra/api/service/main.tf

@@ -1,16 +1,24 @@
 # TODO(https://github.com/navapbc/template-infra/issues/152) use non-default VPC
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc
 data "aws_vpc" "default" {
   default = true
 }
 
-# TODO(https://github.com/navapbc/template-infra/issues/152) use private subnets
-data "aws_subnets" "default" {
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet
+data "aws_subnets" "private" {
   filter {
-    name   = "default-for-az"
-    values = [true]
+    name   = "tag:subnet_type"
+    values = ["private"]
   }
 }
 
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet
+data "aws_subnets" "public" {
+  filter {
+    name   = "tag:subnet_type"
+    values = ["public"]
+  }
+}
 
 locals {
   # The prefix key/value pair is used for Terraform Workspaces, which is useful for projects with multiple infrastructure developers.
@@ -95,7 +103,8 @@ module "service" {
   image_repository_name = module.app_config.image_repository_name
   image_tag             = local.image_tag
   vpc_id                = data.aws_vpc.default.id
-  subnet_ids            = data.aws_subnets.default.ids
+  public_subnet_ids     = data.aws_subnets.public.ids
+  private_subnet_ids    = data.aws_subnets.private.ids
   cpu                   = 1024
   memory                = 2048
 

infra/api/service/staging.s3.tfbackend

@@ -0,0 +1,4 @@
+bucket         = "simpler-grants-gov-315341936575-us-east-1-tf"
+key            = "infra/api/service/staging.tfstate"
+dynamodb_table = "simpler-grants-gov-315341936575-us-east-1-tf-state-locks"
+region         = "us-east-1"

infra/frontend/app-config/main.tf

@@ -1,6 +1,6 @@
 locals {
   app_name                        = "frontend"
-  environments                    = ["dev", "prod"]
+  environments                    = ["dev", "staging", "prod"]
   project_name                    = module.project_config.project_name
   image_repository_name           = "${local.project_name}-${local.app_name}"
   has_database                    = false
@@ -13,8 +13,9 @@ locals {
   }
 
   environment_configs = {
-    dev  = module.dev_config
-    prod = module.prod_config
+    dev     = module.dev_config
+    staging = module.staging_config
+    prod    = module.prod_config
   }
   # Map from environment name to the account name for the AWS account that
   # contains the resources for that environment. Resources that are shared
@@ -46,9 +47,10 @@ locals {
   #     prod    = "prod"
   #   }
   account_names_by_environment = {
-    shared = "simpler-grants-gov"
-    dev    = "simpler-grants-gov"
-    prod   = "simpler-grants-gov"
+    shared  = "simpler-grants-gov"
+    dev     = "simpler-grants-gov"
+    staging = "simpler-grants-gov"
+    prod    = "simpler-grants-gov"
   }
 }
 

infra/frontend/app-config/staging.tf

@@ -0,0 +1,12 @@
+module "staging_config" {
+  source                          = "./env-config"
+  app_name                        = local.app_name
+  default_region                  = module.project_config.default_region
+  environment                     = "staging"
+  has_database                    = local.has_database
+  has_incident_management_service = local.has_incident_management_service
+  domain                          = "beta.grants.gov"
+  sendy_api_key                   = "/${local.app_name}/staging/sendy-api-key"
+  sendy_api_url                   = "/${local.app_name}/staging/sendy-api-url"
+  sendy_list_id                   = "/${local.app_name}/staging/sendy-list-id"
+}

infra/frontend/service/main.tf

@@ -1,16 +1,24 @@
 # TODO(https://github.com/navapbc/template-infra/issues/152) use non-default VPC
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc
 data "aws_vpc" "default" {
   default = true
 }
 
-# TODO(https://github.com/navapbc/template-infra/issues/152) use private subnets
-data "aws_subnets" "default" {
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet
+data "aws_subnets" "private" {
   filter {
-    name   = "default-for-az"
-    values = [true]
+    name   = "tag:subnet_type"
+    values = ["private"]
   }
 }
 
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet
+data "aws_subnets" "public" {
+  filter {
+    name   = "tag:subnet_type"
+    values = ["public"]
+  }
+}
 
 locals {
   # The prefix key/value pair is used for Terraform Workspaces, which is useful for projects with multiple infrastructure developers.
@@ -111,7 +119,8 @@ module "service" {
   image_repository_name = module.app_config.image_repository_name
   image_tag             = local.image_tag
   vpc_id                = data.aws_vpc.default.id
-  subnet_ids            = data.aws_subnets.default.ids
+  public_subnet_ids     = data.aws_subnets.public.ids
+  private_subnet_ids    = data.aws_subnets.private.ids
   enable_autoscaling    = module.app_config.enable_autoscaling
   cert_arn              = terraform.workspace == "default" ? data.aws_acm_certificate.cert[0].arn : null
   hostname              = module.app_config.hostname

infra/frontend/service/staging.s3.tfbackend

@@ -0,0 +1,4 @@
+bucket         = "simpler-grants-gov-315341936575-us-east-1-tf"
+key            = "infra/frontend/service/staging.tfstate"
+dynamodb_table = "simpler-grants-gov-315341936575-us-east-1-tf-state-locks"
+region         = "us-east-1"

infra/modules/database/role_manager/role_manager.py

@@ -2,9 +2,14 @@
 import itertools
 from operator import itemgetter
 import os
+import json
 import logging
 from pg8000.native import Connection, identifier
 
+logging.basicConfig()
+logging.getLogger('botocore').setLevel(logging.DEBUG)
+logging.getLogger('boto3').setLevel(logging.DEBUG)
+
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)
 
@@ -115,9 +120,9 @@ def connect_using_iam(user: str) -> Connection:
     return Connection(user=user, host=host, port=port, database=database, password=token, ssl_context=True)
 
 def get_password() -> str:
-    ssm = boto3.client("ssm")
+    ssm = boto3.client("ssm",region_name=os.environ["AWS_REGION"])
     param_name = os.environ["DB_PASSWORD_PARAM_NAME"]
-    logger.info("Fetching password from parameter store")
+    logger.info("Fetching password from parameter store:\n%s"%param_name)
     result = json.loads(ssm.get_parameter(
         Name=param_name,
         WithDecryption=True,

infra/modules/service/access-control.tf

@@ -69,3 +69,11 @@ resource "aws_iam_role_policy" "task_executor" {
   role   = aws_iam_role.task_executor.id
   policy = data.aws_iam_policy_document.task_executor.json
 }
+
+
+resource "aws_iam_role_policy_attachment" "extra_policies" {
+  for_each = var.extra_policies
+
+  role       = aws_iam_role.app_service.name
+  policy_arn = each.value
+}

infra/modules/service/load-balancer.tf

@@ -9,7 +9,7 @@ resource "aws_lb" "alb" {
   idle_timeout    = "120"
   internal        = false
   security_groups = [aws_security_group.alb.id]
-  subnets         = var.subnet_ids
+  subnets         = var.public_subnet_ids
 
   # TODO(https://github.com/navapbc/template-infra/issues/163) Implement HTTPS
   # checkov:skip=CKV2_AWS_20:Redirect HTTP to HTTPS as part of implementing HTTPS support

infra/modules/service/main.tf

@@ -28,7 +28,7 @@ locals {
     { name : "DB_NAME", value : var.db_vars.connection_info.db_name },
     { name : "DB_SCHEMA", value : var.db_vars.connection_info.schema_name },
   ]
-  environment_variables = concat(local.base_environment_variables, local.db_environment_variables)
+  environment_variables = concat(local.base_environment_variables, local.db_environment_variables, var.extra_environment_variables)
 }
 
 #-------------------
@@ -49,10 +49,8 @@ resource "aws_ecs_service" "app" {
   }
 
   network_configuration {
-    # TODO(https://github.com/navapbc/template-infra/issues/152) set assign_public_ip = false after using private subnets
-    # checkov:skip=CKV_AWS_333:Switch to using private subnets
-    assign_public_ip = true
-    subnets          = var.subnet_ids
+    assign_public_ip = false
+    subnets          = var.private_subnet_ids
     security_groups  = [aws_security_group.app.id]
   }
 

infra/modules/service/variables.tf

@@ -70,9 +70,20 @@ variable "vpc_id" {
   description = "Uniquely identifies the VPC."
 }
 
-variable "subnet_ids" {
+variable "public_subnet_ids" {
   type        = list(any)
-  description = "Private subnet id from vpc module"
+  description = "Public subnet ids in VPC"
+}
+
+variable "private_subnet_ids" {
+  type        = list(any)
+  description = "Private subnet ids in VPC"
+}
+
+variable "extra_environment_variables" {
+  type        = list(object({ name = string, value = string }))
+  description = "Additional environment variables to pass to the service container"
+  default     = []
 }
 
 variable "db_vars" {
@@ -92,6 +103,12 @@ variable "db_vars" {
   default = null
 }
 
+variable "extra_policies" {
+  description = "Map of extra IAM policies to attach to the service's task role. The map's keys define the resource name in terraform."
+  type        = map(string)
+  default     = {}
+}
+
 variable "cert_arn" {
   description = "The ARN for the TLS certificate passed in from the app service layer"
   type        = string
@@ -120,4 +137,4 @@ variable "api_auth_token" {
   type        = string
   default     = null
   description = "Auth token for connecting to the API"
-}
+}

infra/networks/main.tf

@@ -17,9 +17,17 @@ locals {
   # see https://docs.aws.amazon.com/vpc/latest/privatelink/aws-services-privatelink-support.html
   #
   # The database module requires VPC access from private networks to SSM, KMS, and RDS
-  aws_service_integrations = toset(
-    module.app_config.has_database ? ["ssm", "kms"] : []
+  aws_service_integrations = setunion(
+    # AWS services used by ECS Fargate: ECR to fetch images, S3 for image layers, and CloudWatch for logs
+    ["ecr.api", "ecr.dkr", "s3", "logs"],
+
+    # AWS services used by the database's role manager
+    var.has_database ? ["ssm", "kms", "secretsmanager"] : [],
   )
+
+  # S3 and DynamoDB use Gateway VPC endpoints. All other services use Interface VPC endpoints
+  interface_vpc_endpoints = toset([for aws_service in local.aws_service_integrations : aws_service if !contains(["s3", "dynamodb"], aws_service)])
+  gateway_vpc_endpoints   = toset([for aws_service in local.aws_service_integrations : aws_service if contains(["s3", "dynamodb"], aws_service)])
 }
 
 terraform {
@@ -75,6 +83,7 @@ data "aws_subnets" "default" {
 # See https://repost.aws/knowledge-center/lambda-vpc-parameter-store
 # See https://docs.aws.amazon.com/vpc/latest/privatelink/create-interface-endpoint.html#create-interface-endpoint
 
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group
 resource "aws_security_group" "aws_services" {
   count = length(local.aws_service_integrations) > 0 ? 1 : 0
 
@@ -83,17 +92,40 @@ resource "aws_security_group" "aws_services" {
   vpc_id      = data.aws_vpc.default.id
 }
 
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule
+resource "aws_vpc_security_group_ingress_rule" "aws_services" {
+  count = length(local.aws_service_integrations) > 0 ? 1 : 0
+
+  security_group_id = aws_security_group.aws_services[0].id
+  description       = "Allow all traffic from the VPCs CIDR block to the VPC endpoint security group"
+  from_port         = 443
+  to_port           = 443
+  ip_protocol       = "tcp"
+  cidr_ipv4         = data.aws_vpc.default.cidr_block
+}
+
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_endpoint
 resource "aws_vpc_endpoint" "aws_service" {
   for_each = local.aws_service_integrations
 
   vpc_id              = data.aws_vpc.default.id
   service_name        = "com.amazonaws.${data.aws_region.current.name}.${each.key}"
   vpc_endpoint_type   = "Interface"
   security_group_ids  = [aws_security_group.aws_services[0].id]
-  subnet_ids          = data.aws_subnets.default.ids
+  subnet_ids          = [for subnet in aws_subnet.backfill_private : subnet.id]
   private_dns_enabled = true
 }
 
+# docs: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_endpoint
+resource "aws_vpc_endpoint" "gateway" {
+  for_each = local.gateway_vpc_endpoints
+
+  vpc_id            = data.aws_vpc.default.id
+  service_name      = "com.amazonaws.${data.aws_region.current.name}.${each.key}"
+  vpc_endpoint_type = "Gateway"
+  route_table_ids   = [for table in aws_route_table.backfill_private : table.id]
+}
+
 # VPC Configuration for DMS
 # ----------------------------------------