aws: Add an experimental IPv6 dev/test environment

data/data/aws/bootstrap/main.tf

@@ -184,6 +184,18 @@ resource "aws_security_group_rule" "ssh" {
   to_port     = 22
 }
 
+resource "aws_security_group_rule" "ssh_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
+  type              = "ingress"
+  security_group_id = aws_security_group.bootstrap.id
+
+  protocol         = "tcp"
+  ipv6_cidr_blocks = local.public_endpoints ? ["::/0"] : var.vpc_ipv6_cidrs
+  from_port        = 22
+  to_port          = 22
+}
+
 resource "aws_security_group_rule" "bootstrap_journald_gateway" {
   type              = "ingress"
   security_group_id = aws_security_group.bootstrap.id
@@ -194,3 +206,14 @@ resource "aws_security_group_rule" "bootstrap_journald_gateway" {
   to_port     = 19531
 }
 
+resource "aws_security_group_rule" "bootstrap_journald_gateway_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
+  type              = "ingress"
+  security_group_id = aws_security_group.bootstrap.id
+
+  protocol         = "tcp"
+  ipv6_cidr_blocks = local.public_endpoints ? ["::/0"] : var.vpc_ipv6_cidrs
+  from_port        = 19531
+  to_port          = 19531
+}

data/data/aws/bootstrap/variables.tf

@@ -68,6 +68,12 @@ variable "vpc_cidrs" {
   description = "VPC CIDR blocks."
 }
 
+variable "vpc_ipv6_cidrs" {
+  type        = list(string)
+  default     = []
+  description = "VPC IPv6 CIDR blocks."
+}
+
 variable "vpc_security_group_ids" {
   type        = list(string)
   default     = []
@@ -78,3 +84,8 @@ variable "publish_strategy" {
   type        = string
   description = "The publishing strategy for endpoints like load balancers"
 }
+
+variable "use_ipv6" {
+  description = "Use IPv6 instead of IPv4"
+  type        = bool
+}

data/data/aws/main.tf

@@ -23,8 +23,10 @@ module "bootstrap" {
   target_group_arns_length = module.vpc.aws_lb_target_group_arns_length
   vpc_id                   = module.vpc.vpc_id
   vpc_cidrs                = module.vpc.vpc_cidrs
+  vpc_ipv6_cidrs           = module.vpc.vpc_ipv6_cidrs
   vpc_security_group_ids   = [module.vpc.master_sg_id]
   publish_strategy         = var.aws_publish_strategy
+  use_ipv6                 = var.aws_use_ipv6
 
   tags = local.tags
 }
@@ -70,10 +72,12 @@ module "dns" {
   cluster_domain           = var.cluster_domain
   cluster_id               = var.cluster_id
   etcd_count               = var.master_count
-  etcd_ip_addresses        = flatten(module.masters.ip_addresses)
+  etcd_ip_addresses        = var.aws_use_ipv6 == true ? flatten(module.masters.ipv6_addresses) : flatten(module.masters.ip_addresses)
   tags                     = local.tags
   vpc_id                   = module.vpc.vpc_id
   publish_strategy         = var.aws_publish_strategy
+
+  use_ipv6 = var.aws_use_ipv6
 }
 
 module "vpc" {
@@ -95,6 +99,8 @@ module "vpc" {
   )
 
   tags = local.tags
+
+  use_ipv6 = var.aws_use_ipv6
 }
 
 resource "aws_ami_copy" "main" {

data/data/aws/master/main.tf

@@ -89,7 +89,16 @@ resource "aws_network_interface" "master" {
     var.tags,
   )
 }
-
+
+# NOTE(russellb) For some reason, I was not able to access get IPv6 addresses
+# on the resource, but was able to get them using the network interface data
+# source.
+data "aws_network_interface" "master" {
+  count = var.instance_count
+
+  id = aws_network_interface.master[count.index].id
+}
+
 resource "aws_instance" "master" {
   count = var.instance_count
   ami   = var.ec2_ami
@@ -137,4 +146,3 @@ resource "aws_lb_target_group_attachment" "master" {
   target_group_arn = var.target_group_arns[count.index % local.target_group_arns_length]
   target_id        = aws_instance.master[floor(count.index / local.target_group_arns_length)].private_ip
 }
-

data/data/aws/master/outputs.tf

@@ -2,3 +2,6 @@ output "ip_addresses" {
   value = aws_network_interface.master.*.private_ips
 }
 
+output "ipv6_addresses" {
+  value = data.aws_network_interface.master.*.ipv6_addresses
+}

data/data/aws/route53/base.tf

@@ -69,19 +69,20 @@ resource "aws_route53_record" "api_external_internal_zone" {
 }
 
 resource "aws_route53_record" "etcd_a_nodes" {
-  count   = var.etcd_count
+  count   = var.use_ipv6 == false ? var.etcd_count : 0
   type    = "A"
   ttl     = "60"
   zone_id = aws_route53_zone.int.zone_id
   name    = "etcd-${count.index}.${var.cluster_domain}"
-  # TF-UPGRADE-TODO: In Terraform v0.10 and earlier, it was sometimes necessary to
-  # force an interpolation expression to be interpreted as a list by wrapping it
-  # in an extra set of list brackets. That form was supported for compatibilty in
-  # v0.11, but is no longer supported in Terraform v0.12.
-  #
-  # If the expression in the following list itself returns a list, remove the
-  # brackets to avoid interpretation as a list of lists. If the expression
-  # returns a single list item then leave it as-is and remove this TODO comment.
+  records = [var.etcd_ip_addresses[count.index]]
+}
+
+resource "aws_route53_record" "etcd_aaaa_nodes" {
+  count   = var.use_ipv6 == true ? var.etcd_count : 0
+  type    = "AAAA"
+  ttl     = "60"
+  zone_id = aws_route53_zone.int.zone_id
+  name    = "etcd-${count.index}.${var.cluster_domain}"
   records = [var.etcd_ip_addresses[count.index]]
 }
 
@@ -90,6 +91,6 @@ resource "aws_route53_record" "etcd_cluster" {
   ttl     = "60"
   zone_id = aws_route53_zone.int.zone_id
   name    = "_etcd-server-ssl._tcp"
-  records = formatlist("0 10 2380 %s", aws_route53_record.etcd_a_nodes.*.fqdn)
+  records = formatlist("0 10 2380 %s", var.use_ipv6 == false ? aws_route53_record.etcd_a_nodes.*.fqdn : aws_route53_record.etcd_aaaa_nodes.*.fqdn)
 }
 

data/data/aws/route53/variables.tf

@@ -9,7 +9,7 @@ variable "etcd_count" {
 }
 
 variable "etcd_ip_addresses" {
-  description = "List of string IPs for machines running etcd members."
+  description = "List of string IPs (IPv4 or IPv6) for machines running etcd members."
   type        = list(string)
   default     = []
 }
@@ -64,3 +64,8 @@ based on if api_external_lb_dns_name for example, which will be null when there
 So publish_strategy serves an coordinated proxy for that decision.
 EOF
 }
+
+variable "use_ipv6" {
+  description = "Use IPv6 instead of IPv4"
+  type = bool
+}

data/data/aws/variables-aws.tf

@@ -91,3 +91,9 @@ variable "aws_publish_strategy" {
   type = string
   description = "The cluster publishing strategy, either Internal or External"
 }
+
+variable "aws_use_ipv6" {
+  type = bool
+  default = false
+  description = "Enable an experimental IPv6 environment"
+}

data/data/aws/vpc/master-elb.tf

@@ -153,4 +153,3 @@ resource "aws_lb_listener" "api_external_api" {
     type             = "forward"
   }
 }
-

data/data/aws/vpc/outputs.tf

@@ -6,6 +6,10 @@ output "vpc_cidrs" {
   value = [data.aws_vpc.cluster_vpc.cidr_block]
 }
 
+output "vpc_ipv6_cidrs" {
+  value = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+}
+
 output "az_to_private_subnet_id" {
   value = zipmap(data.aws_subnet.private.*.availability_zone, data.aws_subnet.private.*.id)
 }
@@ -63,4 +67,3 @@ output "aws_lb_api_internal_dns_name" {
 output "aws_lb_api_internal_zone_id" {
   value = aws_lb.api_internal.zone_id
 }
-

data/data/aws/vpc/sg-master.tf

@@ -23,6 +23,18 @@ resource "aws_security_group_rule" "master_mcs" {
   to_port     = 22623
 }
 
+resource "aws_security_group_rule" "master_mcs_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
+  type              = "ingress"
+  security_group_id = aws_security_group.master.id
+
+  protocol         = "tcp"
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+  from_port        = 22623
+  to_port          = 22623
+}
+
 resource "aws_security_group_rule" "master_egress" {
   type              = "egress"
   security_group_id = aws_security_group.master.id
@@ -33,6 +45,18 @@ resource "aws_security_group_rule" "master_egress" {
   cidr_blocks = ["0.0.0.0/0"]
 }
 
+resource "aws_security_group_rule" "master_egress_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
+  type              = "egress"
+  security_group_id = aws_security_group.master.id
+
+  from_port        = 0
+  to_port          = 0
+  protocol         = "-1"
+  ipv6_cidr_blocks = ["::/0"]
+}
+
 resource "aws_security_group_rule" "master_ingress_icmp" {
   type              = "ingress"
   security_group_id = aws_security_group.master.id
@@ -43,6 +67,18 @@ resource "aws_security_group_rule" "master_ingress_icmp" {
   to_port     = -1
 }
 
+resource "aws_security_group_rule" "master_ingress_icmp_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
+  type              = "ingress"
+  security_group_id = aws_security_group.master.id
+
+  protocol         = "icmp"
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+  from_port        = -1
+  to_port          = -1
+}
+
 resource "aws_security_group_rule" "master_ingress_ssh" {
   type              = "ingress"
   security_group_id = aws_security_group.master.id
@@ -53,6 +89,18 @@ resource "aws_security_group_rule" "master_ingress_ssh" {
   to_port     = 22
 }
 
+resource "aws_security_group_rule" "master_ingress_ssh_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
+  type              = "ingress"
+  security_group_id = aws_security_group.master.id
+
+  protocol         = "tcp"
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+  from_port        = 22
+  to_port          = 22
+}
+
 resource "aws_security_group_rule" "master_ingress_https" {
   type              = "ingress"
   security_group_id = aws_security_group.master.id
@@ -63,6 +111,18 @@ resource "aws_security_group_rule" "master_ingress_https" {
   to_port     = 6443
 }
 
+resource "aws_security_group_rule" "master_ingress_https_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
+  type              = "ingress"
+  security_group_id = aws_security_group.master.id
+
+  protocol         = "tcp"
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+  from_port        = 6443
+  to_port          = 6443
+}
+
 resource "aws_security_group_rule" "master_ingress_vxlan" {
   type              = "ingress"
   security_group_id = aws_security_group.master.id
@@ -253,3 +313,31 @@ resource "aws_security_group_rule" "master_ingress_services_udp" {
   self      = true
 }
 
+# For our AWS IPv6 environment, we run CoreDNS with host networking,
+# because it must use IPv4 to reach AWS DNS, so it can't be on our
+# IPv6 only SDN.
+resource "aws_security_group_rule" "master_dns_udp" {
+  count = var.use_ipv6 == true ? 1 : 0
+
+  type              = "ingress"
+  security_group_id = aws_security_group.master.id
+
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+
+  protocol  = "udp"
+  from_port = 5353
+  to_port   = 5353
+}
+
+resource "aws_security_group_rule" "master_dns_tcp" {
+  count = var.use_ipv6 == true ? 1 : 0
+
+  type              = "ingress"
+  security_group_id = aws_security_group.master.id
+
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+
+  protocol  = "tcp"
+  from_port = 5353
+  to_port   = 5353
+}