From 17778169d51e4d66ace696c9690fe3d39468a149 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Wed, 16 Oct 2019 11:28:40 -0400
Subject: [PATCH 01/34] validate: Don't force IPv4 CIDRs

This is a hack to not block IPv6 when doing CIDR validation.  We may
want to do something smarter later, like validate that IPv4 or IPv6 is
used consistently, or validate based on per-platform capabilities.
For now, removing this check unblocks IPv6 work.
---
 pkg/validate/validate.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/pkg/validate/validate.go b/pkg/validate/validate.go
index 6e2fcc52fc6..dd2c1fb9bd9 100644
--- a/pkg/validate/validate.go
+++ b/pkg/validate/validate.go
@@ -109,9 +109,6 @@ func ClusterName(v string) error {
 
 // SubnetCIDR checks if the given IP net is a valid CIDR.
 func SubnetCIDR(cidr *net.IPNet) error {
-	if cidr.IP.To4() == nil {
-		return errors.New("must use IPv4")
-	}
 	if cidr.IP.IsUnspecified() {
 		return errors.New("address must be specified")
 	}

From eb557b0bfb94a99c685ef96db6733488894bc24c Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Wed, 16 Oct 2019 20:54:40 -0400
Subject: [PATCH 02/34] aws: Start making terraform changes for IPv6

If you run "create cluster", the VPC will have an IPv6 subnet assigned
by AWS (a /56).  Then, each subnet created in the VPC will get a slice
of this (a /64).

I can see that the VMs created all had IPv6 addresses associated with
them, at least from the AWS perspective.

The networking section of my current install-config looks like this:

  networking:
    clusterNetwork:
    - cidr: fd01::/48
      hostPrefix: 64
    machineCIDR: 10.0.0.0/16
    networkType: OVNKubernetes
    serviceNetwork:
    - fd02::/112

I kept machineCIDR as IPv4, since on AWS we can't get IPv6 only.  We
get IPv4 + optionally IPv6.  We use machineCIDR to specify the IPv4
address range we carve up in the VPC.  For IPv6, AWS gives us the /56
that we carve up, so we don't need a subnet specified in the config.

For clusterNetwork and serviceNetwork, both of these are used
exclusively inside the cluster, so I changed them to be specified as
private IPv6 subnets.
---
 data/data/aws/vpc/vpc-private.tf | 4 ++++
 data/data/aws/vpc/vpc-public.tf  | 5 +++++
 data/data/aws/vpc/vpc.tf         | 3 +++
 3 files changed, 12 insertions(+)

diff --git a/data/data/aws/vpc/vpc-private.tf b/data/data/aws/vpc/vpc-private.tf
index 15f98bf7968..bb91475622e 100644
--- a/data/data/aws/vpc/vpc-private.tf
+++ b/data/data/aws/vpc/vpc-private.tf
@@ -33,6 +33,10 @@ resource "aws_subnet" "private_subnet" {
 
   availability_zone = var.availability_zones[count.index]
 
+  # TODO - Make this conditional if ipv6 support is made optional
+  ipv6_cidr_block                 = cidrsubnet(data.aws_vpc.cluster_vpc.ipv6_cidr_block, 8, count.index)
+  assign_ipv6_address_on_creation = true
+
   tags = merge(
     {
       "Name"                            = "${var.cluster_id}-private-${var.availability_zones[count.index]}"
diff --git a/data/data/aws/vpc/vpc-public.tf b/data/data/aws/vpc/vpc-public.tf
index e85ac8e92da..6bf8d952bfc 100644
--- a/data/data/aws/vpc/vpc-public.tf
+++ b/data/data/aws/vpc/vpc-public.tf
@@ -50,6 +50,11 @@ resource "aws_subnet" "public_subnet" {
   cidr_block        = cidrsubnet(local.new_public_cidr_range, 3, count.index)
   availability_zone = var.availability_zones[count.index]
 
+  # TODO - Make this conditional if ipv6 support is made optional
+  # We add length(var.availability_zones) here to skip over the subnets allocated to the private_subnet (vpc-private.tf)
+  ipv6_cidr_block                 = cidrsubnet(data.aws_vpc.cluster_vpc.ipv6_cidr_block, 8, count.index + length(var.availability_zones))
+  assign_ipv6_address_on_creation = true
+
   tags = merge(
     {
       "Name" = "${var.cluster_id}-public-${var.availability_zones[count.index]}"
diff --git a/data/data/aws/vpc/vpc.tf b/data/data/aws/vpc/vpc.tf
index a2534912f2a..40871bb17b2 100644
--- a/data/data/aws/vpc/vpc.tf
+++ b/data/data/aws/vpc/vpc.tf
@@ -10,6 +10,9 @@ resource "aws_vpc" "new_vpc" {
   enable_dns_hostnames = true
   enable_dns_support   = true
 
+  # TODO - This should probably be optional
+  assign_generated_ipv6_cidr_block = true
+
   tags = merge(
     {
       "Name" = "${var.cluster_id}-vpc"

From e770d2e71d6fdd46e1cc2a4520eb3f379fa65018 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Thu, 17 Oct 2019 13:14:23 -0400
Subject: [PATCH 03/34] aws: Add IPv6 route to the AWS internet gateway

This allows all of our VMs to speak IPv6 out to the internet.
---
 data/data/aws/vpc/vpc-public.tf | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/data/data/aws/vpc/vpc-public.tf b/data/data/aws/vpc/vpc-public.tf
index 6bf8d952bfc..e082c43214e 100644
--- a/data/data/aws/vpc/vpc-public.tf
+++ b/data/data/aws/vpc/vpc-public.tf
@@ -43,6 +43,18 @@ resource "aws_route" "igw_route" {
   }
 }
 
+resource "aws_route" "igw_route_v6" {
+  count = var.vpc == null ? 1 : 0
+
+  destination_ipv6_cidr_block = "::/0"
+  route_table_id              = aws_route_table.default[0].id
+  gateway_id                  = aws_internet_gateway.igw[0].id
+
+  timeouts {
+    create = "20m"
+  }
+}
+
 resource "aws_subnet" "public_subnet" {
   count = var.public_subnets == null ? length(var.availability_zones) : 0
 

From 0f719ef088678c05b3137ccf60f2eb205aa20905 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Thu, 17 Oct 2019 16:26:25 -0400
Subject: [PATCH 04/34] aws: Add IPv6 route from private subnet to internet gw

This adds a route on the private subnet to allow internet access
through the same internet gateway used by the public subnet.

In the IPv4 case, we use a NAT gateway from the private subnet.  For
IPv6, we don't use NAT since the addresses are publicly routable.
---
 data/data/aws/vpc/vpc-private.tf | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/data/data/aws/vpc/vpc-private.tf b/data/data/aws/vpc/vpc-private.tf
index bb91475622e..39998f4c698 100644
--- a/data/data/aws/vpc/vpc-private.tf
+++ b/data/data/aws/vpc/vpc-private.tf
@@ -24,6 +24,21 @@ resource "aws_route" "to_nat_gw" {
   }
 }
 
+# We can't target the NAT gw for our "private" IPv6 subnet.  Instead, we target the internet gateway,
+# since we want our private IPv6 addresses to be able to talk out to the internet, too.
+resource "aws_route" "private_igw_v6" {
+  count = var.private_subnets == null ? length(var.availability_zones) : 0
+
+  route_table_id              = aws_route_table.private_routes[count.index].id
+  destination_ipv6_cidr_block = "::/0"
+  gateway_id                  = aws_internet_gateway.igw[0].id
+  depends_on                  = [aws_route_table.private_routes]
+
+  timeouts {
+    create = "20m"
+  }
+}
+
 resource "aws_subnet" "private_subnet" {
   count = var.private_subnets == null ? length(var.availability_zones) : 0
 

From 5b44046ad363542252367b076623129ae96e43bd Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Thu, 17 Oct 2019 17:01:32 -0400
Subject: [PATCH 05/34] aws: Update security groups for IPv6

All security group rules that referenced an IPv6 CIDR have been
duplicated to reference the corresponding IPv6 CIDR.
---
 data/data/aws/bootstrap/main.tf | 19 ++++++++++++
 data/data/aws/vpc/sg-master.tf  | 51 +++++++++++++++++++++++++++++++++
 data/data/aws/vpc/sg-worker.tf  | 31 ++++++++++++++++++++
 3 files changed, 101 insertions(+)

diff --git a/data/data/aws/bootstrap/main.tf b/data/data/aws/bootstrap/main.tf
index 2b5cbfa869b..27ae453b552 100644
--- a/data/data/aws/bootstrap/main.tf
+++ b/data/data/aws/bootstrap/main.tf
@@ -184,6 +184,16 @@ resource "aws_security_group_rule" "ssh" {
   to_port     = 22
 }
 
+resource "aws_security_group_rule" "ssh_v6" {
+  type              = "ingress"
+  security_group_id = aws_security_group.bootstrap.id
+
+  protocol         = "tcp"
+  ipv6_cidr_blocks = ["::/0"]
+  from_port        = 22
+  to_port          = 22
+}
+
 resource "aws_security_group_rule" "bootstrap_journald_gateway" {
   type              = "ingress"
   security_group_id = aws_security_group.bootstrap.id
@@ -194,3 +204,12 @@ resource "aws_security_group_rule" "bootstrap_journald_gateway" {
   to_port     = 19531
 }
 
+resource "aws_security_group_rule" "bootstrap_journald_gateway_v6" {
+  type              = "ingress"
+  security_group_id = aws_security_group.bootstrap.id
+
+  protocol         = "tcp"
+  ipv6_cidr_blocks = ["::/0"]
+  from_port        = 19531
+  to_port          = 19531
+}
diff --git a/data/data/aws/vpc/sg-master.tf b/data/data/aws/vpc/sg-master.tf
index 4aa8736c623..91bd564473d 100644
--- a/data/data/aws/vpc/sg-master.tf
+++ b/data/data/aws/vpc/sg-master.tf
@@ -23,6 +23,16 @@ resource "aws_security_group_rule" "master_mcs" {
   to_port     = 22623
 }
 
+resource "aws_security_group_rule" "master_mcs_v6" {
+  type              = "ingress"
+  security_group_id = aws_security_group.master.id
+
+  protocol         = "tcp"
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+  from_port        = 22623
+  to_port          = 22623
+}
+
 resource "aws_security_group_rule" "master_egress" {
   type              = "egress"
   security_group_id = aws_security_group.master.id
@@ -33,6 +43,17 @@ resource "aws_security_group_rule" "master_egress" {
   cidr_blocks = ["0.0.0.0/0"]
 }
 
+# TODO make this conditional on ipv6 being enabled
+resource "aws_security_group_rule" "master_egress_v6" {
+  type              = "egress"
+  security_group_id = aws_security_group.master.id
+
+  from_port        = 0
+  to_port          = 0
+  protocol         = "-1"
+  ipv6_cidr_blocks = ["::/0"]
+}
+
 resource "aws_security_group_rule" "master_ingress_icmp" {
   type              = "ingress"
   security_group_id = aws_security_group.master.id
@@ -43,6 +64,16 @@ resource "aws_security_group_rule" "master_ingress_icmp" {
   to_port     = -1
 }
 
+resource "aws_security_group_rule" "master_ingress_icmp_v6" {
+  type              = "ingress"
+  security_group_id = aws_security_group.master.id
+
+  protocol         = "icmp"
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+  from_port        = -1
+  to_port          = -1
+}
+
 resource "aws_security_group_rule" "master_ingress_ssh" {
   type              = "ingress"
   security_group_id = aws_security_group.master.id
@@ -53,6 +84,16 @@ resource "aws_security_group_rule" "master_ingress_ssh" {
   to_port     = 22
 }
 
+resource "aws_security_group_rule" "master_ingress_ssh_v6" {
+  type              = "ingress"
+  security_group_id = aws_security_group.master.id
+
+  protocol         = "tcp"
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+  from_port        = 22
+  to_port          = 22
+}
+
 resource "aws_security_group_rule" "master_ingress_https" {
   type              = "ingress"
   security_group_id = aws_security_group.master.id
@@ -63,6 +104,16 @@ resource "aws_security_group_rule" "master_ingress_https" {
   to_port     = 6443
 }
 
+resource "aws_security_group_rule" "master_ingress_https_v6" {
+  type              = "ingress"
+  security_group_id = aws_security_group.master.id
+
+  protocol         = "tcp"
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+  from_port        = 6443
+  to_port          = 6443
+}
+
 resource "aws_security_group_rule" "master_ingress_vxlan" {
   type              = "ingress"
   security_group_id = aws_security_group.master.id
diff --git a/data/data/aws/vpc/sg-worker.tf b/data/data/aws/vpc/sg-worker.tf
index 1000c688dc5..717b9415903 100644
--- a/data/data/aws/vpc/sg-worker.tf
+++ b/data/data/aws/vpc/sg-worker.tf
@@ -23,6 +23,17 @@ resource "aws_security_group_rule" "worker_egress" {
   cidr_blocks = ["0.0.0.0/0"]
 }
 
+# TODO make this conditional on ipv6 being enabled
+resource "aws_security_group_rule" "worker_egress_v6" {
+  type              = "egress"
+  security_group_id = aws_security_group.worker.id
+
+  from_port        = 0
+  to_port          = 0
+  protocol         = "-1"
+  ipv6_cidr_blocks = ["::/0"]
+}
+
 resource "aws_security_group_rule" "worker_ingress_icmp" {
   type              = "ingress"
   security_group_id = aws_security_group.worker.id
@@ -33,6 +44,16 @@ resource "aws_security_group_rule" "worker_ingress_icmp" {
   to_port     = -1
 }
 
+resource "aws_security_group_rule" "worker_ingress_icmp_v6" {
+  type              = "ingress"
+  security_group_id = aws_security_group.worker.id
+
+  protocol         = "icmp"
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+  from_port        = -1
+  to_port          = -1
+}
+
 resource "aws_security_group_rule" "worker_ingress_ssh" {
   type              = "ingress"
   security_group_id = aws_security_group.worker.id
@@ -43,6 +64,16 @@ resource "aws_security_group_rule" "worker_ingress_ssh" {
   to_port     = 22
 }
 
+resource "aws_security_group_rule" "worker_ingress_ssh_v6" {
+  type              = "ingress"
+  security_group_id = aws_security_group.worker.id
+
+  protocol         = "tcp"
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+  from_port        = 22
+  to_port          = 22
+}
+
 resource "aws_security_group_rule" "worker_ingress_vxlan" {
   type              = "ingress"
   security_group_id = aws_security_group.worker.id

From 6d19648e85db937d38178c404d24468d094de206 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Thu, 17 Oct 2019 13:01:50 -0400
Subject: [PATCH 06/34] aws: Create IPv6 DNS records

All of the IPv4 records remain for now.
---
 data/data/aws/main.tf              |  1 +
 data/data/aws/master/main.tf       | 11 +++++-
 data/data/aws/master/outputs.tf    |  3 ++
 data/data/aws/route53/base.tf      | 56 ++++++++++++++++++++++++++++++
 data/data/aws/route53/variables.tf |  6 ++++
 5 files changed, 76 insertions(+), 1 deletion(-)

diff --git a/data/data/aws/main.tf b/data/data/aws/main.tf
index a7b6ec6d823..a5f0ac3913d 100644
--- a/data/data/aws/main.tf
+++ b/data/data/aws/main.tf
@@ -71,6 +71,7 @@ module "dns" {
   cluster_id               = var.cluster_id
   etcd_count               = var.master_count
   etcd_ip_addresses        = flatten(module.masters.ip_addresses)
+  etcd_ipv6_addresses      = flatten(module.masters.ipv6_addresses)
   tags                     = local.tags
   vpc_id                   = module.vpc.vpc_id
   publish_strategy         = var.aws_publish_strategy
diff --git a/data/data/aws/master/main.tf b/data/data/aws/master/main.tf
index 6a74f78e162..6866409461e 100644
--- a/data/data/aws/master/main.tf
+++ b/data/data/aws/master/main.tf
@@ -89,7 +89,16 @@ resource "aws_network_interface" "master" {
     var.tags,
   )
 }
-  
+
+# NOTE(russellb) For some reason, I was not able to access get IPv6 addresses
+# on the resource, but was able to get them using the network interface data
+# source.
+data "aws_network_interface" "master" {
+  count = var.instance_count
+
+  id = aws_network_interface.master[count.index].id
+}
+
 resource "aws_instance" "master" {
   count = var.instance_count
   ami   = var.ec2_ami
diff --git a/data/data/aws/master/outputs.tf b/data/data/aws/master/outputs.tf
index 06c06533e7f..eb3060db12f 100644
--- a/data/data/aws/master/outputs.tf
+++ b/data/data/aws/master/outputs.tf
@@ -2,3 +2,6 @@ output "ip_addresses" {
   value = aws_network_interface.master.*.private_ips
 }
 
+output "ipv6_addresses" {
+  value = data.aws_network_interface.master.*.ipv6_addresses
+}
diff --git a/data/data/aws/route53/base.tf b/data/data/aws/route53/base.tf
index 681b0e69c5c..19d96511a52 100644
--- a/data/data/aws/route53/base.tf
+++ b/data/data/aws/route53/base.tf
@@ -44,6 +44,18 @@ resource "aws_route53_record" "api_external" {
   }
 }
 
+resource "aws_route53_record" "api_external_v6" {
+  zone_id = data.aws_route53_zone.public[0].zone_id
+  name    = "api.${var.cluster_domain}"
+  type    = "AAAA"
+
+  alias {
+    name                   = var.api_external_lb_dns_name
+    zone_id                = var.api_external_lb_zone_id
+    evaluate_target_health = false
+  }
+}
+
 resource "aws_route53_record" "api_internal" {
   zone_id = aws_route53_zone.int.zone_id
   name    = "api-int.${var.cluster_domain}"
@@ -56,6 +68,19 @@ resource "aws_route53_record" "api_internal" {
   }
 }
 
+# TODO make this conditional if IPv6 is optional
+resource "aws_route53_record" "api_internal_v6" {
+  zone_id = aws_route53_zone.int.zone_id
+  name    = "api-int.${var.cluster_domain}"
+  type    = "AAAA"
+
+  alias {
+    name                   = var.api_internal_lb_dns_name
+    zone_id                = var.api_internal_lb_zone_id
+    evaluate_target_health = false
+  }
+}
+
 resource "aws_route53_record" "api_external_internal_zone" {
   zone_id = aws_route53_zone.int.zone_id
   name    = "api.${var.cluster_domain}"
@@ -68,6 +93,19 @@ resource "aws_route53_record" "api_external_internal_zone" {
   }
 }
 
+# TODO make this conditional if IPv6 is optional
+resource "aws_route53_record" "api_external_internal_zone_v6" {
+  zone_id = aws_route53_zone.int.zone_id
+  name    = "api.${var.cluster_domain}"
+  type    = "AAAA"
+
+  alias {
+    name                   = var.api_internal_lb_dns_name
+    zone_id                = var.api_internal_lb_zone_id
+    evaluate_target_health = false
+  }
+}
+
 resource "aws_route53_record" "etcd_a_nodes" {
   count   = var.etcd_count
   type    = "A"
@@ -85,6 +123,24 @@ resource "aws_route53_record" "etcd_a_nodes" {
   records = [var.etcd_ip_addresses[count.index]]
 }
 
+# TODO make this conditional if IPv6 is optional
+resource "aws_route53_record" "etcd_a_nodes_v6" {
+  count   = var.etcd_count
+  type    = "AAAA"
+  ttl     = "60"
+  zone_id = aws_route53_zone.int.zone_id
+  name    = "etcd-${count.index}.${var.cluster_domain}"
+  # TF-UPGRADE-TODO: In Terraform v0.10 and earlier, it was sometimes necessary to
+  # force an interpolation expression to be interpreted as a list by wrapping it
+  # in an extra set of list brackets. That form was supported for compatibilty in
+  # v0.11, but is no longer supported in Terraform v0.12.
+  #
+  # If the expression in the following list itself returns a list, remove the
+  # brackets to avoid interpretation as a list of lists. If the expression
+  # returns a single list item then leave it as-is and remove this TODO comment.
+  records = [var.etcd_ipv6_addresses[count.index]]
+}
+
 resource "aws_route53_record" "etcd_cluster" {
   type    = "SRV"
   ttl     = "60"
diff --git a/data/data/aws/route53/variables.tf b/data/data/aws/route53/variables.tf
index 06746923ac0..9274adbdc26 100644
--- a/data/data/aws/route53/variables.tf
+++ b/data/data/aws/route53/variables.tf
@@ -14,6 +14,12 @@ variable "etcd_ip_addresses" {
   default     = []
 }
 
+variable "etcd_ipv6_addresses" {
+  description = "List of string IPs (IPv6) for machines running etcd members."
+  type        = list(string)
+  default     = []
+}
+
 variable "base_domain" {
   description = "The base domain used for public records."
   type        = string

From e8ea7afc12aa7c298f5975bad200064d98456bde Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Fri, 18 Oct 2019 09:19:41 -0400
Subject: [PATCH 07/34] aws: Disable IPv4 DNS records

Introduce a "use_ipv6" terraform variable that is hard coded to
"true".  Use it to disable the IPv4 DNS records (A records), leaving
only the IPv6 (AAAA) records in place.
---
 data/data/aws/main.tf              |  5 +++++
 data/data/aws/route53/base.tf      | 20 +++++++++++++++-----
 data/data/aws/route53/variables.tf |  5 +++++
 3 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/data/data/aws/main.tf b/data/data/aws/main.tf
index a5f0ac3913d..d39c859bc94 100644
--- a/data/data/aws/main.tf
+++ b/data/data/aws/main.tf
@@ -5,6 +5,9 @@ locals {
     },
     var.aws_extra_tags,
   )
+
+  # Use IPv6 instead of IPv4
+  use_ipv6 = true
 }
 
 provider "aws" {
@@ -75,6 +78,8 @@ module "dns" {
   tags                     = local.tags
   vpc_id                   = module.vpc.vpc_id
   publish_strategy         = var.aws_publish_strategy
+
+  use_ipv6 = local.use_ipv6
 }
 
 module "vpc" {
diff --git a/data/data/aws/route53/base.tf b/data/data/aws/route53/base.tf
index 19d96511a52..466a88f0793 100644
--- a/data/data/aws/route53/base.tf
+++ b/data/data/aws/route53/base.tf
@@ -31,7 +31,7 @@ resource "aws_route53_zone" "int" {
 }
 
 resource "aws_route53_record" "api_external" {
-  count = local.public_endpoints ? 1 : 0
+  count = local.public_endpoints && var.use_ipv6 == false ? 1 : 0
 
   zone_id = data.aws_route53_zone.public[0].zone_id
   name    = "api.${var.cluster_domain}"
@@ -54,6 +54,8 @@ resource "aws_route53_record" "api_external_v6" {
     zone_id                = var.api_external_lb_zone_id
     evaluate_target_health = false
   }
+
+  count = local.public_endpoints && var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_route53_record" "api_internal" {
@@ -66,6 +68,8 @@ resource "aws_route53_record" "api_internal" {
     zone_id                = var.api_internal_lb_zone_id
     evaluate_target_health = false
   }
+
+  count = var.use_ipv6 == false ? 1 : 0
 }
 
 # TODO make this conditional if IPv6 is optional
@@ -79,6 +83,8 @@ resource "aws_route53_record" "api_internal_v6" {
     zone_id                = var.api_internal_lb_zone_id
     evaluate_target_health = false
   }
+
+  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_route53_record" "api_external_internal_zone" {
@@ -91,6 +97,8 @@ resource "aws_route53_record" "api_external_internal_zone" {
     zone_id                = var.api_internal_lb_zone_id
     evaluate_target_health = false
   }
+
+  count = var.use_ipv6 == false ? 1 : 0
 }
 
 # TODO make this conditional if IPv6 is optional
@@ -104,10 +112,12 @@ resource "aws_route53_record" "api_external_internal_zone_v6" {
     zone_id                = var.api_internal_lb_zone_id
     evaluate_target_health = false
   }
+
+  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_route53_record" "etcd_a_nodes" {
-  count   = var.etcd_count
+  count   = var.use_ipv6 == false ? var.etcd_count : 0
   type    = "A"
   ttl     = "60"
   zone_id = aws_route53_zone.int.zone_id
@@ -124,8 +134,8 @@ resource "aws_route53_record" "etcd_a_nodes" {
 }
 
 # TODO make this conditional if IPv6 is optional
-resource "aws_route53_record" "etcd_a_nodes_v6" {
-  count   = var.etcd_count
+resource "aws_route53_record" "etcd_aaaa_nodes" {
+  count   = var.use_ipv6 == true ? var.etcd_count : 0
   type    = "AAAA"
   ttl     = "60"
   zone_id = aws_route53_zone.int.zone_id
@@ -146,6 +156,6 @@ resource "aws_route53_record" "etcd_cluster" {
   ttl     = "60"
   zone_id = aws_route53_zone.int.zone_id
   name    = "_etcd-server-ssl._tcp"
-  records = formatlist("0 10 2380 %s", aws_route53_record.etcd_a_nodes.*.fqdn)
+  records = var.use_ipv6 == false ? formatlist("0 10 2380 %s", aws_route53_record.etcd_a_nodes.*.fqdn) : formatlist("0 10 2380 %s", aws_route53_record.etcd_aaaa_nodes.*.fqdn)
 }
 
diff --git a/data/data/aws/route53/variables.tf b/data/data/aws/route53/variables.tf
index 9274adbdc26..4477e87355e 100644
--- a/data/data/aws/route53/variables.tf
+++ b/data/data/aws/route53/variables.tf
@@ -70,3 +70,8 @@ based on if api_external_lb_dns_name for example, which will be null when there
 So publish_strategy serves an coordinated proxy for that decision.
 EOF
 }
+
+variable "use_ipv6" {
+  description = "Use IPv6 instead of IPv4"
+  type = bool
+}

From 9a7fa84614b0420c360dcac030fb55b5a9673348 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Mon, 21 Oct 2019 12:27:39 -0400
Subject: [PATCH 08/34] aws: Create IPv6 compatible load balancers

Back in commit 16dfbb354120a04925775092b4e43a0420dc2e05, the installer
switched from classic load balancers to network load balancers.
Unfortunately, NLBs don't support IPv6.  This commit introduces
conditional classic load balancers when use_ipv6 is turned on.
---
 data/data/aws/bootstrap/main.tf      |  14 ++-
 data/data/aws/bootstrap/variables.tf |  15 +++
 data/data/aws/main.tf                |  12 +++
 data/data/aws/master/main.tf         |  13 ++-
 data/data/aws/master/variables.tf    |  15 +++
 data/data/aws/route53/base.tf        |   9 +-
 data/data/aws/vpc/master-elb.tf      | 140 +++++++++++++++++++++++++--
 data/data/aws/vpc/outputs.tf         |  15 ++-
 data/data/aws/vpc/variables.tf       |   5 +
 9 files changed, 219 insertions(+), 19 deletions(-)

diff --git a/data/data/aws/bootstrap/main.tf b/data/data/aws/bootstrap/main.tf
index 27ae453b552..82f65ec1041 100644
--- a/data/data/aws/bootstrap/main.tf
+++ b/data/data/aws/bootstrap/main.tf
@@ -153,12 +153,24 @@ resource "aws_instance" "bootstrap" {
 resource "aws_lb_target_group_attachment" "bootstrap" {
   // Because of the issue https://github.com/hashicorp/terraform/issues/12570, the consumers cannot use a dynamic list for count
   // and therefore are force to implicitly assume that the list is of aws_lb_target_group_arns_length - 1, in case there is no api_external
-  count = local.public_endpoints ? var.target_group_arns_length : var.target_group_arns_length - 1
+  count = var.use_ipv6 == true ? 0 : (local.public_endpoints ? var.target_group_arns_length : var.target_group_arns_length - 1)
 
   target_group_arn = var.target_group_arns[count.index]
   target_id        = aws_instance.bootstrap.private_ip
 }
 
+resource "aws_elb_attachment" "bootstrap_api_internal" {
+  count    = var.use_ipv6 == true ? 1 : 0
+  elb      = var.aws_elb_api_internal_id
+  instance = aws_instance.bootstrap.id
+}
+
+resource "aws_elb_attachment" "bootstrap_api_external" {
+  count    = var.use_ipv6 == true ? 1 : 0
+  elb      = var.aws_elb_api_external_id
+  instance = aws_instance.bootstrap.id
+}
+
 resource "aws_security_group" "bootstrap" {
   vpc_id = var.vpc_id
 
diff --git a/data/data/aws/bootstrap/variables.tf b/data/data/aws/bootstrap/variables.tf
index a37e9612e3a..8b0728ac682 100644
--- a/data/data/aws/bootstrap/variables.tf
+++ b/data/data/aws/bootstrap/variables.tf
@@ -78,3 +78,18 @@ variable "publish_strategy" {
   type        = string
   description = "The publishing strategy for endpoints like load balancers"
 }
+
+variable "use_ipv6" {
+  description = "Use IPv6 instead of IPv4"
+  type        = bool
+}
+
+variable "aws_elb_api_internal_id" {
+  description = "ID of the internal IPv6 load balancer"
+  type        = string
+}
+
+variable "aws_elb_api_external_id" {
+  description = "ID of the external IPv6 load balancer"
+  type        = string
+}
diff --git a/data/data/aws/main.tf b/data/data/aws/main.tf
index d39c859bc94..ffc47d7c0e7 100644
--- a/data/data/aws/main.tf
+++ b/data/data/aws/main.tf
@@ -30,6 +30,11 @@ module "bootstrap" {
   publish_strategy         = var.aws_publish_strategy
 
   tags = local.tags
+
+  use_ipv6 = local.use_ipv6
+
+  aws_elb_api_internal_id = module.vpc.aws_elb_api_internal_id
+  aws_elb_api_external_id = module.vpc.aws_elb_api_external_id
 }
 
 module "masters" {
@@ -52,6 +57,11 @@ module "masters" {
   ec2_ami                  = aws_ami_copy.main.id
   user_data_ign            = var.ignition_master
   publish_strategy         = var.aws_publish_strategy
+
+  use_ipv6 = local.use_ipv6
+
+  aws_elb_api_internal_id = module.vpc.aws_elb_api_internal_id
+  aws_elb_api_external_id = module.vpc.aws_elb_api_external_id
 }
 
 module "iam" {
@@ -101,6 +111,8 @@ module "vpc" {
   )
 
   tags = local.tags
+
+  use_ipv6 = local.use_ipv6
 }
 
 resource "aws_ami_copy" "main" {
diff --git a/data/data/aws/master/main.tf b/data/data/aws/master/main.tf
index 6866409461e..61fae9a8870 100644
--- a/data/data/aws/master/main.tf
+++ b/data/data/aws/master/main.tf
@@ -141,9 +141,20 @@ resource "aws_instance" "master" {
 }
 
 resource "aws_lb_target_group_attachment" "master" {
-  count = var.instance_count * local.target_group_arns_length
+  count = var.use_ipv6 == false ? var.instance_count * local.target_group_arns_length : 0
 
   target_group_arn = var.target_group_arns[count.index % local.target_group_arns_length]
   target_id        = aws_instance.master[floor(count.index / local.target_group_arns_length)].private_ip
 }
 
+resource "aws_elb_attachment" "masters_internal_v6" {
+  count    = var.use_ipv6 == true ? var.instance_count : 0
+  elb      = var.aws_elb_api_internal_id
+  instance = aws_instance.master.*.id[count.index]
+}
+
+resource "aws_elb_attachment" "masters_external_v6" {
+  count    = var.use_ipv6 == true ? var.instance_count : 0
+  elb      = var.aws_elb_api_external_id
+  instance = aws_instance.master.*.id[count.index]
+}
diff --git a/data/data/aws/master/variables.tf b/data/data/aws/master/variables.tf
index 8ff122ed36c..2579c9cd503 100644
--- a/data/data/aws/master/variables.tf
+++ b/data/data/aws/master/variables.tf
@@ -80,3 +80,18 @@ and therefore are force to implicitly assume that the list is of aws_lb_target_g
 helps to decide if the target_group_arns is of length (target_group_arns_length) or (target_group_arns_length - 1)
 EOF
 }
+
+variable "use_ipv6" {
+  description = "Use IPv6 instead of IPv4"
+  type = bool
+}
+
+variable "aws_elb_api_internal_id" {
+  description = "ID of the internal IPv6 load balancer"
+  type = string
+}
+
+variable "aws_elb_api_external_id" {
+  description = "ID of the external IPv6 load balancer"
+  type = string
+}
diff --git a/data/data/aws/route53/base.tf b/data/data/aws/route53/base.tf
index 466a88f0793..63da821dbdf 100644
--- a/data/data/aws/route53/base.tf
+++ b/data/data/aws/route53/base.tf
@@ -50,7 +50,7 @@ resource "aws_route53_record" "api_external_v6" {
   type    = "AAAA"
 
   alias {
-    name                   = var.api_external_lb_dns_name
+    name                   = "ipv6.${var.api_external_lb_dns_name}"
     zone_id                = var.api_external_lb_zone_id
     evaluate_target_health = false
   }
@@ -72,14 +72,13 @@ resource "aws_route53_record" "api_internal" {
   count = var.use_ipv6 == false ? 1 : 0
 }
 
-# TODO make this conditional if IPv6 is optional
 resource "aws_route53_record" "api_internal_v6" {
   zone_id = aws_route53_zone.int.zone_id
   name    = "api-int.${var.cluster_domain}"
   type    = "AAAA"
 
   alias {
-    name                   = var.api_internal_lb_dns_name
+    name                   = "ipv6.${var.api_internal_lb_dns_name}"
     zone_id                = var.api_internal_lb_zone_id
     evaluate_target_health = false
   }
@@ -101,14 +100,13 @@ resource "aws_route53_record" "api_external_internal_zone" {
   count = var.use_ipv6 == false ? 1 : 0
 }
 
-# TODO make this conditional if IPv6 is optional
 resource "aws_route53_record" "api_external_internal_zone_v6" {
   zone_id = aws_route53_zone.int.zone_id
   name    = "api.${var.cluster_domain}"
   type    = "AAAA"
 
   alias {
-    name                   = var.api_internal_lb_dns_name
+    name                   = "ipv6.${var.api_internal_lb_dns_name}"
     zone_id                = var.api_internal_lb_zone_id
     evaluate_target_health = false
   }
@@ -133,7 +131,6 @@ resource "aws_route53_record" "etcd_a_nodes" {
   records = [var.etcd_ip_addresses[count.index]]
 }
 
-# TODO make this conditional if IPv6 is optional
 resource "aws_route53_record" "etcd_aaaa_nodes" {
   count   = var.use_ipv6 == true ? var.etcd_count : 0
   type    = "AAAA"
diff --git a/data/data/aws/vpc/master-elb.tf b/data/data/aws/vpc/master-elb.tf
index fcf9fd0ee1a..64885254d17 100644
--- a/data/data/aws/vpc/master-elb.tf
+++ b/data/data/aws/vpc/master-elb.tf
@@ -1,3 +1,6 @@
+# AWS NLBs (Network Load Balancers, or the aws_lb resource) do not support
+# IPv6.  Instead, we must use class load balancers (the aws_elb resource).
+
 resource "aws_lb" "api_internal" {
   name                             = "${var.cluster_id}-int"
   load_balancer_type               = "network"
@@ -17,10 +20,12 @@ resource "aws_lb" "api_internal" {
   }
 
   depends_on = [aws_internet_gateway.igw]
+
+  count = var.use_ipv6 == false ? 1 : 0
 }
 
 resource "aws_lb" "api_external" {
-  count = local.public_endpoints ? 1 : 0
+  count = local.public_endpoints && var.use_ipv6 == false ? 1 : 0
 
   name                             = "${var.cluster_id}-ext"
   load_balancer_type               = "network"
@@ -65,10 +70,12 @@ resource "aws_lb_target_group" "api_internal" {
     protocol            = "HTTPS"
     path                = "/readyz"
   }
+
+  count = var.use_ipv6 == false ? 1 : 0
 }
 
 resource "aws_lb_target_group" "api_external" {
-  count = local.public_endpoints ? 1 : 0
+  count = local.public_endpoints && var.use_ipv6 == false ? 1 : 0
 
   name     = "${var.cluster_id}-aext"
   protocol = "TCP"
@@ -117,32 +124,38 @@ resource "aws_lb_target_group" "services" {
     protocol            = "HTTPS"
     path                = "/healthz"
   }
+
+  count = var.use_ipv6 == false ? 1 : 0
 }
 
 resource "aws_lb_listener" "api_internal_api" {
-  load_balancer_arn = aws_lb.api_internal.arn
+  load_balancer_arn = aws_lb.api_internal[0].arn
   protocol          = "TCP"
   port              = "6443"
 
   default_action {
-    target_group_arn = aws_lb_target_group.api_internal.arn
+    target_group_arn = aws_lb_target_group.api_internal[0].arn
     type             = "forward"
   }
+
+  count = var.use_ipv6 == false ? 1 : 0
 }
 
 resource "aws_lb_listener" "api_internal_services" {
-  load_balancer_arn = aws_lb.api_internal.arn
+  load_balancer_arn = aws_lb.api_internal[0].arn
   protocol          = "TCP"
   port              = "22623"
 
   default_action {
-    target_group_arn = aws_lb_target_group.services.arn
+    target_group_arn = aws_lb_target_group.services[0].arn
     type             = "forward"
   }
+
+  count = var.use_ipv6 == false ? 1 : 0
 }
 
 resource "aws_lb_listener" "api_external_api" {
-  count = local.public_endpoints ? 1 : 0
+  count = local.public_endpoints && var.use_ipv6 == false ? 1 : 0
 
   load_balancer_arn = aws_lb.api_external[0].arn
   protocol          = "TCP"
@@ -154,3 +167,116 @@ resource "aws_lb_listener" "api_external_api" {
   }
 }
 
+#################################
+### Begin IPv6 load balancers ###
+#################################
+
+resource "aws_security_group" "api" {
+  vpc_id = "${data.aws_vpc.cluster_vpc.id}"
+
+  timeouts {
+    create = "20m"
+  }
+
+  tags = "${merge(map(
+    "Name", "${var.cluster_id}_api_sg",
+  ), var.tags)}"
+
+  count = var.use_ipv6 == true ? 1 : 0
+}
+
+resource "aws_security_group_rule" "api_ingress_api_v6" {
+  type              = "ingress"
+  security_group_id = "${aws_security_group.api[0].id}"
+
+  protocol         = "tcp"
+  ipv6_cidr_blocks = ["::/0"]
+  from_port        = 6443
+  to_port          = 6443
+
+  count = var.use_ipv6 == true ? 1 : 0
+}
+
+resource "aws_security_group_rule" "api_ingress_mcs_v6" {
+  type              = "ingress"
+  security_group_id = "${aws_security_group.api[0].id}"
+
+  protocol         = "tcp"
+  ipv6_cidr_blocks = ["::/0"]
+  from_port        = 22623
+  to_port          = 22623
+
+  count = var.use_ipv6 == true ? 1 : 0
+}
+
+resource "aws_elb" "api_internal" {
+  name            = "${var.cluster_id}-int"
+  subnets         = data.aws_subnet.private.*.id
+  internal        = true
+  security_groups = ["${aws_security_group.master.id}", "${aws_security_group.api[0].id}"]
+
+  idle_timeout                = 3600
+  connection_draining         = true
+  connection_draining_timeout = 300
+
+  listener {
+    instance_port     = 6443
+    instance_protocol = "tcp"
+    lb_port           = 6443
+    lb_protocol       = "tcp"
+  }
+
+  listener {
+    instance_port     = 22623
+    instance_protocol = "tcp"
+    lb_port           = 22623
+    lb_protocol       = "tcp"
+  }
+
+  health_check {
+    healthy_threshold   = 2
+    unhealthy_threshold = 2
+    timeout             = 3
+    target              = "SSL:6443"
+    interval            = 5
+  }
+
+  tags = "${merge(map(
+    "Name", "${var.cluster_id}-int",
+  ), var.tags)}"
+
+  count = var.use_ipv6 == true ? 1 : 0
+}
+
+resource "aws_elb" "api_external" {
+  name            = "${var.cluster_id}-ext"
+  subnets         = data.aws_subnet.public.*.id
+  internal        = false
+  security_groups = ["${aws_security_group.master.id}", "${aws_security_group.api[0].id}"]
+
+  idle_timeout                = 3600
+  connection_draining         = true
+  connection_draining_timeout = 300
+
+  listener {
+    instance_port     = 6443
+    instance_protocol = "tcp"
+    lb_port           = 6443
+    lb_protocol       = "tcp"
+  }
+
+  health_check {
+    healthy_threshold   = 2
+    unhealthy_threshold = 2
+    timeout             = 3
+    target              = "SSL:6443"
+    interval            = 5
+  }
+
+  tags = "${merge(map(
+    "Name", "${var.cluster_id}-api-external",
+  ), var.tags)}"
+
+  count = local.public_endpoints && var.use_ipv6 == true ? 1 : 0
+}
+
diff --git a/data/data/aws/vpc/outputs.tf b/data/data/aws/vpc/outputs.tf
index 8f410101aa9..b2985ce91b5 100644
--- a/data/data/aws/vpc/outputs.tf
+++ b/data/data/aws/vpc/outputs.tf
@@ -49,18 +49,25 @@ output "aws_lb_target_group_arns_length" {
 }
 
 output "aws_lb_api_external_dns_name" {
-  value = local.public_endpoints ? aws_lb.api_external[0].dns_name : null
+  value = local.public_endpoints && var.use_ipv6 == false ? aws_lb.api_external[0].dns_name : aws_elb.api_external[0].dns_name
 }
 
 output "aws_lb_api_external_zone_id" {
-  value = local.public_endpoints ? aws_lb.api_external[0].zone_id : null
+  value = local.public_endpoints && var.use_ipv6 == false ? aws_lb.api_external[0].zone_id : aws_elb.api_external[0].zone_id
 }
 
 output "aws_lb_api_internal_dns_name" {
-  value = aws_lb.api_internal.dns_name
+  value = var.use_ipv6 == false ? aws_lb.api_internal[0].dns_name : aws_elb.api_internal[0].dns_name
 }
 
 output "aws_lb_api_internal_zone_id" {
-  value = aws_lb.api_internal.zone_id
+  value = var.use_ipv6 == false ? aws_lb.api_internal[0].zone_id : aws_elb.api_internal[0].zone_id
 }
 
+output "aws_elb_api_internal_id" {
+  value = var.use_ipv6 == true ? aws_elb.api_internal[0].id : ""
+}
+
+output "aws_elb_api_external_id" {
+  value = var.use_ipv6 == true ? aws_elb.api_external[0].id : ""
+}
diff --git a/data/data/aws/vpc/variables.tf b/data/data/aws/vpc/variables.tf
index 220a21c906d..39273d91aac 100644
--- a/data/data/aws/vpc/variables.tf
+++ b/data/data/aws/vpc/variables.tf
@@ -41,3 +41,8 @@ variable "private_subnets" {
   type        = list(string)
   description = "Existing private subnets into which the cluster should be installed."
 }
+
+variable "use_ipv6" {
+  description = "Use IPv6 instead of IPv4"
+  type        = bool
+}

From 68636ab74605e8065d7c15f5d2fcc9f3249fbbfd Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Mon, 21 Oct 2019 15:57:44 -0400
Subject: [PATCH 09/34] aws: Apply use_ipv6 boolean in more places

Apply the use_ipv6 boolean in some places that were added before the
boolean existed.
---
 data/data/aws/vpc/sg-master.tf   | 3 ++-
 data/data/aws/vpc/sg-worker.tf   | 3 ++-
 data/data/aws/vpc/vpc-private.tf | 7 +++----
 data/data/aws/vpc/vpc-public.tf  | 8 +++-----
 data/data/aws/vpc/vpc.tf         | 3 +--
 5 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/data/data/aws/vpc/sg-master.tf b/data/data/aws/vpc/sg-master.tf
index 91bd564473d..0b621409d0a 100644
--- a/data/data/aws/vpc/sg-master.tf
+++ b/data/data/aws/vpc/sg-master.tf
@@ -43,7 +43,6 @@ resource "aws_security_group_rule" "master_egress" {
   cidr_blocks = ["0.0.0.0/0"]
 }
 
-# TODO make this conditional on ipv6 being enabled
 resource "aws_security_group_rule" "master_egress_v6" {
   type              = "egress"
   security_group_id = aws_security_group.master.id
@@ -52,6 +51,8 @@ resource "aws_security_group_rule" "master_egress_v6" {
   to_port          = 0
   protocol         = "-1"
   ipv6_cidr_blocks = ["::/0"]
+
+  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "master_ingress_icmp" {
diff --git a/data/data/aws/vpc/sg-worker.tf b/data/data/aws/vpc/sg-worker.tf
index 717b9415903..e55482d1060 100644
--- a/data/data/aws/vpc/sg-worker.tf
+++ b/data/data/aws/vpc/sg-worker.tf
@@ -23,7 +23,6 @@ resource "aws_security_group_rule" "worker_egress" {
   cidr_blocks = ["0.0.0.0/0"]
 }
 
-# TODO make this conditional on ipv6 being enabled
 resource "aws_security_group_rule" "worker_egress_v6" {
   type              = "egress"
   security_group_id = aws_security_group.worker.id
@@ -32,6 +31,8 @@ resource "aws_security_group_rule" "worker_egress_v6" {
   to_port          = 0
   protocol         = "-1"
   ipv6_cidr_blocks = ["::/0"]
+
+  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "worker_ingress_icmp" {
diff --git a/data/data/aws/vpc/vpc-private.tf b/data/data/aws/vpc/vpc-private.tf
index 39998f4c698..5f7519b4aa8 100644
--- a/data/data/aws/vpc/vpc-private.tf
+++ b/data/data/aws/vpc/vpc-private.tf
@@ -27,7 +27,7 @@ resource "aws_route" "to_nat_gw" {
 # We can't target the NAT gw for our "private" IPv6 subnet.  Instead, we target the internet gateway,
 # since we want our private IPv6 addresses to be able to talk out to the internet, too.
 resource "aws_route" "private_igw_v6" {
-  count = var.private_subnets == null ? length(var.availability_zones) : 0
+  count = var.use_ipv6 == true && var.private_subnets == null ? length(var.availability_zones) : 0
 
   route_table_id              = aws_route_table.private_routes[count.index].id
   destination_ipv6_cidr_block = "::/0"
@@ -48,9 +48,8 @@ resource "aws_subnet" "private_subnet" {
 
   availability_zone = var.availability_zones[count.index]
 
-  # TODO - Make this conditional if ipv6 support is made optional
-  ipv6_cidr_block                 = cidrsubnet(data.aws_vpc.cluster_vpc.ipv6_cidr_block, 8, count.index)
-  assign_ipv6_address_on_creation = true
+  ipv6_cidr_block                 = var.use_ipv6 == true ? cidrsubnet(data.aws_vpc.cluster_vpc.ipv6_cidr_block, 8, count.index) : ""
+  assign_ipv6_address_on_creation = var.use_ipv6
 
   tags = merge(
     {
diff --git a/data/data/aws/vpc/vpc-public.tf b/data/data/aws/vpc/vpc-public.tf
index e082c43214e..ecc7f346067 100644
--- a/data/data/aws/vpc/vpc-public.tf
+++ b/data/data/aws/vpc/vpc-public.tf
@@ -44,7 +44,7 @@ resource "aws_route" "igw_route" {
 }
 
 resource "aws_route" "igw_route_v6" {
-  count = var.vpc == null ? 1 : 0
+  count = var.use_ipv6 == true && var.vpc == null ? 1 : 0
 
   destination_ipv6_cidr_block = "::/0"
   route_table_id              = aws_route_table.default[0].id
@@ -62,10 +62,8 @@ resource "aws_subnet" "public_subnet" {
   cidr_block        = cidrsubnet(local.new_public_cidr_range, 3, count.index)
   availability_zone = var.availability_zones[count.index]
 
-  # TODO - Make this conditional if ipv6 support is made optional
-  # We add length(var.availability_zones) here to skip over the subnets allocated to the private_subnet (vpc-private.tf)
-  ipv6_cidr_block                 = cidrsubnet(data.aws_vpc.cluster_vpc.ipv6_cidr_block, 8, count.index + length(var.availability_zones))
-  assign_ipv6_address_on_creation = true
+  ipv6_cidr_block                 = var.use_ipv6 == true ? cidrsubnet(data.aws_vpc.cluster_vpc.ipv6_cidr_block, 8, count.index + length(var.availability_zones)) : ""
+  assign_ipv6_address_on_creation = var.use_ipv6
 
   tags = merge(
     {
diff --git a/data/data/aws/vpc/vpc.tf b/data/data/aws/vpc/vpc.tf
index 40871bb17b2..607d67968c4 100644
--- a/data/data/aws/vpc/vpc.tf
+++ b/data/data/aws/vpc/vpc.tf
@@ -10,8 +10,7 @@ resource "aws_vpc" "new_vpc" {
   enable_dns_hostnames = true
   enable_dns_support   = true
 
-  # TODO - This should probably be optional
-  assign_generated_ipv6_cidr_block = true
+  assign_generated_ipv6_cidr_block = var.use_ipv6
 
   tags = merge(
     {

From 4a64b3d8993a8be18e436a0615152c15a2f294ee Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Tue, 22 Oct 2019 09:41:15 -0400
Subject: [PATCH 10/34] aws: Re-enable api-int A record in IPv6 mode

RHCOS doesn't get an IPv6 address during boot before running ignition,
so without this record it will fail to contact the machine config
server.  Turn this record back on as a temporary workaround until we
can do IPv6 during ignition.
---
 data/data/aws/route53/base.tf | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/data/data/aws/route53/base.tf b/data/data/aws/route53/base.tf
index 63da821dbdf..5acdfafbc30 100644
--- a/data/data/aws/route53/base.tf
+++ b/data/data/aws/route53/base.tf
@@ -69,7 +69,10 @@ resource "aws_route53_record" "api_internal" {
     evaluate_target_health = false
   }
 
-  count = var.use_ipv6 == false ? 1 : 0
+  # TODO - We must leave the A record enabled, even when trying to test IPv6, because
+  # RHCOS doesn't get an IPv6 address during boot, so ignition will fail trying to reach
+  # the machine config server.
+  #count = var.use_ipv6 == false ? 1 : 0
 }
 
 resource "aws_route53_record" "api_internal_v6" {

From b1c9acf430338828f868081c488a65bf4ca5619e Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Wed, 23 Oct 2019 13:35:43 -0400
Subject: [PATCH 11/34] aws: Don't accept public connections to api-int

The ELB for IPv6 was allowing any source to reach the api-int load
balancer, instead of only the CIDRs used by the cluster.
---
 data/data/aws/vpc/master-elb.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/data/data/aws/vpc/master-elb.tf b/data/data/aws/vpc/master-elb.tf
index 64885254d17..7afc36a51b0 100644
--- a/data/data/aws/vpc/master-elb.tf
+++ b/data/data/aws/vpc/master-elb.tf
@@ -213,7 +213,7 @@ resource "aws_elb" "api_internal" {
   name            = "${var.cluster_id}-int"
   subnets         = data.aws_subnet.private.*.id
   internal        = true
-  security_groups = ["${aws_security_group.master.id}", "${aws_security_group.api[0].id}"]
+  security_groups = ["${aws_security_group.master.id}"]
 
   idle_timeout                = 3600
   connection_draining         = true

From 2a1ac7760d408bd775cb67d349ab5d6e529b13a4 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Wed, 23 Oct 2019 13:39:43 -0400
Subject: [PATCH 12/34] aws: Check local_endpoints for more IPv6 resources

Much of this was written before the code merged to allow disabling
public endpoints.  I went through and tried to honor the new settings
in various places that I missed during an earlier rebase.
---
 data/data/aws/bootstrap/main.tf      | 6 +++---
 data/data/aws/bootstrap/variables.tf | 6 ++++++
 data/data/aws/main.tf                | 1 +
 data/data/aws/master/main.tf         | 4 +++-
 data/data/aws/vpc/outputs.tf         | 6 +++++-
 5 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/data/data/aws/bootstrap/main.tf b/data/data/aws/bootstrap/main.tf
index 82f65ec1041..bc620542258 100644
--- a/data/data/aws/bootstrap/main.tf
+++ b/data/data/aws/bootstrap/main.tf
@@ -160,7 +160,7 @@ resource "aws_lb_target_group_attachment" "bootstrap" {
 }
 
 resource "aws_elb_attachment" "bootstrap_api_internal" {
-  count    = var.use_ipv6 == true ? 1 : 0
+  count    = var.use_ipv6 == true && local.public_endpoints ? 1 : 0
   elb      = var.aws_elb_api_internal_id
   instance = aws_instance.bootstrap.id
 }
@@ -201,7 +201,7 @@ resource "aws_security_group_rule" "ssh_v6" {
   security_group_id = aws_security_group.bootstrap.id
 
   protocol         = "tcp"
-  ipv6_cidr_blocks = ["::/0"]
+  ipv6_cidr_blocks = local.public_endpoints ? ["::/0"] : var.vpc_ipv6_cidrs
   from_port        = 22
   to_port          = 22
 }
@@ -221,7 +221,7 @@ resource "aws_security_group_rule" "bootstrap_journald_gateway_v6" {
   security_group_id = aws_security_group.bootstrap.id
 
   protocol         = "tcp"
-  ipv6_cidr_blocks = ["::/0"]
+  ipv6_cidr_blocks = local.public_endpoints ? ["::/0"] : var.vpc_ipv6_cidrs
   from_port        = 19531
   to_port          = 19531
 }
diff --git a/data/data/aws/bootstrap/variables.tf b/data/data/aws/bootstrap/variables.tf
index 8b0728ac682..53e45386424 100644
--- a/data/data/aws/bootstrap/variables.tf
+++ b/data/data/aws/bootstrap/variables.tf
@@ -68,6 +68,12 @@ variable "vpc_cidrs" {
   description = "VPC CIDR blocks."
 }
 
+variable "vpc_ipv6_cidrs" {
+  type        = list(string)
+  default     = []
+  description = "VPC IPv6 CIDR blocks."
+}
+
 variable "vpc_security_group_ids" {
   type        = list(string)
   default     = []
diff --git a/data/data/aws/main.tf b/data/data/aws/main.tf
index ffc47d7c0e7..199bbd9b724 100644
--- a/data/data/aws/main.tf
+++ b/data/data/aws/main.tf
@@ -26,6 +26,7 @@ module "bootstrap" {
   target_group_arns_length = module.vpc.aws_lb_target_group_arns_length
   vpc_id                   = module.vpc.vpc_id
   vpc_cidrs                = module.vpc.vpc_cidrs
+  vpc_ipv6_cidrs           = module.vpc.vpc_ipv6_cidrs
   vpc_security_group_ids   = [module.vpc.master_sg_id]
   publish_strategy         = var.aws_publish_strategy
 
diff --git a/data/data/aws/master/main.tf b/data/data/aws/master/main.tf
index 61fae9a8870..77f324a3b46 100644
--- a/data/data/aws/master/main.tf
+++ b/data/data/aws/master/main.tf
@@ -4,6 +4,8 @@ locals {
   // Because of the issue https://github.com/hashicorp/terraform/issues/12570, the consumers cannot use a dynamic list for count
   // and therefore are force to implicitly assume that the list is of aws_lb_target_group_arns_length - 1, in case there is no api_external
   target_group_arns_length = var.publish_strategy == "External" ? var.target_group_arns_length : var.target_group_arns_length - 1
+
+  public_endpoints = var.publish_strategy == "External" ? true : false
 }
 
 resource "aws_iam_instance_profile" "master" {
@@ -154,7 +156,7 @@ resource "aws_elb_attachment" "masters_internal_v6" {
 }
 
 resource "aws_elb_attachment" "masters_external_v6" {
-  count    = var.use_ipv6 == true ? var.instance_count : 0
+  count    = var.use_ipv6 == true && local.public_endpoints ? var.instance_count : 0
   elb      = var.aws_elb_api_external_id
   instance = aws_instance.master.*.id[count.index]
 }
diff --git a/data/data/aws/vpc/outputs.tf b/data/data/aws/vpc/outputs.tf
index b2985ce91b5..f6cf02f179c 100644
--- a/data/data/aws/vpc/outputs.tf
+++ b/data/data/aws/vpc/outputs.tf
@@ -6,6 +6,10 @@ output "vpc_cidrs" {
   value = [data.aws_vpc.cluster_vpc.cidr_block]
 }
 
+output "vpc_ipv6_cidrs" {
+  value = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+}
+
 output "az_to_private_subnet_id" {
   value = zipmap(data.aws_subnet.private.*.availability_zone, data.aws_subnet.private.*.id)
 }
@@ -69,5 +73,5 @@ output "aws_elb_api_internal_id" {
 }
 
 output "aws_elb_api_external_id" {
-  value = var.use_ipv6 == true ? aws_elb.api_external[0].id : ""
+  value = local.public_endpoints && var.use_ipv6 == true ? aws_elb.api_external[0].id : ""
 }

From d1f795d80ce09c95d538dbef905e35f38d178f5d Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Wed, 23 Oct 2019 13:53:43 -0400
Subject: [PATCH 13/34] aws: Disable the IPv6 dev/test env by default

Add a new terraform variable, aws_use_ipv6, for controlling the AWS
IPv6 dev/test environment.  It defaults to false, retaining existing
behavior by default.  To turn it on, you must set the
TF_VAR_aws_use_ipv6 environment variable to "true".

A future commit will expand on this with developer documentation.
---
 data/data/aws/main.tf          | 11 ++++-------
 data/data/aws/variables-aws.tf |  6 ++++++
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/data/data/aws/main.tf b/data/data/aws/main.tf
index 199bbd9b724..492511e3bfc 100644
--- a/data/data/aws/main.tf
+++ b/data/data/aws/main.tf
@@ -5,9 +5,6 @@ locals {
     },
     var.aws_extra_tags,
   )
-
-  # Use IPv6 instead of IPv4
-  use_ipv6 = true
 }
 
 provider "aws" {
@@ -32,7 +29,7 @@ module "bootstrap" {
 
   tags = local.tags
 
-  use_ipv6 = local.use_ipv6
+  use_ipv6 = var.aws_use_ipv6
 
   aws_elb_api_internal_id = module.vpc.aws_elb_api_internal_id
   aws_elb_api_external_id = module.vpc.aws_elb_api_external_id
@@ -59,7 +56,7 @@ module "masters" {
   user_data_ign            = var.ignition_master
   publish_strategy         = var.aws_publish_strategy
 
-  use_ipv6 = local.use_ipv6
+  use_ipv6 = var.aws_use_ipv6
 
   aws_elb_api_internal_id = module.vpc.aws_elb_api_internal_id
   aws_elb_api_external_id = module.vpc.aws_elb_api_external_id
@@ -90,7 +87,7 @@ module "dns" {
   vpc_id                   = module.vpc.vpc_id
   publish_strategy         = var.aws_publish_strategy
 
-  use_ipv6 = local.use_ipv6
+  use_ipv6 = var.aws_use_ipv6
 }
 
 module "vpc" {
@@ -113,7 +110,7 @@ module "vpc" {
 
   tags = local.tags
 
-  use_ipv6 = local.use_ipv6
+  use_ipv6 = var.aws_use_ipv6
 }
 
 resource "aws_ami_copy" "main" {
diff --git a/data/data/aws/variables-aws.tf b/data/data/aws/variables-aws.tf
index 07104900ffc..2234c550b99 100644
--- a/data/data/aws/variables-aws.tf
+++ b/data/data/aws/variables-aws.tf
@@ -91,3 +91,9 @@ variable "aws_publish_strategy" {
   type = string
   description = "The cluster publishing strategy, either Internal or External"
 }
+
+variable "aws_use_ipv6" {
+  type = bool
+  default = false
+  description = "Enable an experimental IPv6 environment"
+}

From 750c4b202eab34251bea60dbdead92f00bd2c683 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Wed, 23 Oct 2019 14:16:32 -0400
Subject: [PATCH 14/34] aws: Document the IPv6 dev/test env

---
 docs/dev/aws_ipv6.md | 56 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)
 create mode 100644 docs/dev/aws_ipv6.md

diff --git a/docs/dev/aws_ipv6.md b/docs/dev/aws_ipv6.md
new file mode 100644
index 00000000000..6b90cb12af2
--- /dev/null
+++ b/docs/dev/aws_ipv6.md
@@ -0,0 +1,56 @@
+# AWS IPv6 Dev and Test Environment
+
+The installer includes code to enable an experimental IPv6 dev and test
+environment on AWS.  This is off by default and currently only intended for
+those working on enabling IPv6 in OpenShift.
+
+## Enabling the Environment
+
+To enable IPv6 in your AWS environment, set the following environment variable
+before running the installer:
+
+```bash
+    export TF_VAR_aws_use_ipv6=”true”
+```
+
+## AWS Network Environment
+
+AWS does not support a single-stack (IPv6 only) environment, but it does
+support a dual-stack (IPv4 and IPv6) environment, so that’s what is enabled
+here.  This is a summary of the changes to the network environment:
+
+* The VPC has IPv6 enabled and a `/56` IPv6 CIDR will be allocated by AWS.
+* Each Subnet will have an IPv6 `/64` subnet allocated to it.
+* All IPv4 specific security group rules have corresponding IPv6 rules created.
+* AWS Network Load Balancers (NLBs) do not support IPv6, so classic Elasic Load
+  Balancers (ELBs) are created, instead.
+* The IPv4 NLBs are disabled.
+* IPv6 DNS records (AAAA) are created and the IPv4 (A) records are disabled.
+* IPv6 routing is configured.  Since all instances get global IPv6 addresses,
+  NAT is not used from the instances out to the internet.
+
+## Install Configuration
+
+Here is the suggested network configuration for `install-config.yaml`:
+
+```yaml
+networking:
+  clusterNetwork:
+  - cidr: fd01::/48
+    hostPrefix: 64
+  machineCIDR: 10.0.0.0/16
+  networkType: OVNKubernetes
+  serviceNetwork:
+  - fd02::/112
+```
+
+Note that an IPv4 CIDR is still used for `machineCIDR` since AWS will provide a
+dual-stack (IPv4 and IPv6) environment.  We must specify the IPv4 CIDR and AWS
+will automatically allocate an IPv6 CIDR.
+
+## Current Status of IPv6
+
+Note that IPv6 support is under heavy development across many components in
+OpenShift, so the use of some custom images may be needed to include fixes to
+known issues.  Coordination of work-in-progress is out of scope for this
+document.

From 3874adc15ca9e59c479c801a157ab27772718c02 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Wed, 23 Oct 2019 14:57:52 -0400
Subject: [PATCH 15/34] validate: Update tests for IPv6 CIDR validation

The tests assumed IPv6 CIDRs are not allowed.  A previous commit
started allowing them and this updates the test cases accordingly.
---
 pkg/validate/validate_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/validate/validate_test.go b/pkg/validate/validate_test.go
index 016854e971f..e0b1b511861 100644
--- a/pkg/validate/validate_test.go
+++ b/pkg/validate/validate_test.go
@@ -59,8 +59,8 @@ func TestSubnetCIDR(t *testing.T) {
 		{"1.2.3.4/1", "invalid network address. got 1.2.3.4/1, expecting 0.0.0.0/1"},
 		{"1.2.3.4/31", ""},
 		{"1.2.3.4/32", ""},
-		{"0:0:0:0:0:1:102:304/116", "must use IPv4"},
-		{"0:0:0:0:0:ffff:102:304/116", "invalid network address. got 1.2.3.4/20, expecting 1.2.0.0/20"},
+		{"0:0:0:0:0:1:102:304/116", "invalid network address. got ::1:102:304/116, expecting ::1:102:0/116"},
+		{"fd01::/48", ""},
 		{"172.17.0.0/20", "overlaps with default Docker Bridge subnet (172.17.0.0/20)"},
 		{"172.0.0.0/8", "overlaps with default Docker Bridge subnet (172.0.0.0/8)"},
 		{"255.255.255.255/1", "invalid network address. got 255.255.255.255/1, expecting 128.0.0.0/1"},

From c46de1ac4ed74b7521c96011270ea9e7ab45d7ec Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Thu, 24 Oct 2019 09:03:23 -0400
Subject: [PATCH 16/34] aws: Add env var OPENSHIFT_AWS_USE_IPV6

Instead of telling people to set TF_VAR_aws_use_ipv6, wrap it with an
OpenShift env var.
---
 docs/dev/aws_ipv6.md  | 2 +-
 pkg/tfvars/aws/aws.go | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/docs/dev/aws_ipv6.md b/docs/dev/aws_ipv6.md
index 6b90cb12af2..d9859c26300 100644
--- a/docs/dev/aws_ipv6.md
+++ b/docs/dev/aws_ipv6.md
@@ -10,7 +10,7 @@ To enable IPv6 in your AWS environment, set the following environment variable
 before running the installer:
 
 ```bash
-    export TF_VAR_aws_use_ipv6=”true”
+    export OPENSHIFT_AWS_USE_IPV6=”true”
 ```
 
 ## AWS Network Environment
diff --git a/pkg/tfvars/aws/aws.go b/pkg/tfvars/aws/aws.go
index be1115f4306..0322951c8a9 100644
--- a/pkg/tfvars/aws/aws.go
+++ b/pkg/tfvars/aws/aws.go
@@ -4,6 +4,7 @@ package aws
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 
 	"github.com/pkg/errors"
 	"sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsproviderconfig/v1beta1"
@@ -27,6 +28,7 @@ type config struct {
 	PrivateSubnets          []string          `json:"aws_private_subnets,omitempty"`
 	PublicSubnets           *[]string         `json:"aws_public_subnets,omitempty"`
 	PublishStrategy         string            `json:"aws_publish_strategy,omitempty"`
+	UseIPv6                 bool              `json:"aws_use_ipv6,omitempty"`
 }
 
 // TFVars generates AWS-specific Terraform variables launching the cluster.
@@ -76,6 +78,11 @@ func TFVars(vpc string, privateSubnets []string, publicSubnets []string, publish
 
 	instanceClass := defaults.InstanceClass(masterConfig.Placement.Region)
 
+	useIPv6 := false
+	if os.Getenv("OPENSHIFT_AWS_USE_IPV6") == "true" {
+		useIPv6 = true
+	}
+
 	cfg := &config{
 		Region:                  masterConfig.Placement.Region,
 		ExtraTags:               tags,
@@ -89,6 +96,7 @@ func TFVars(vpc string, privateSubnets []string, publicSubnets []string, publish
 		VPC:                     vpc,
 		PrivateSubnets:          privateSubnets,
 		PublishStrategy:         string(publish),
+		UseIPv6:                 useIPv6,
 	}
 
 	if len(publicSubnets) == 0 {

From b0add46178a9a98dfabb044c9f9625db2f6cf090 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Thu, 24 Oct 2019 10:23:18 -0400
Subject: [PATCH 17/34] aws: Make more IPv6 resources conditional

I missed a few more things when I started making the IPv6 support
conditional.  This was exposed by trying to run an install with IPv6
turned off (the default).  With this patch, a regular install worked
for me again.
---
 data/data/aws/vpc/sg-master.tf | 8 ++++++++
 data/data/aws/vpc/sg-worker.tf | 4 ++++
 2 files changed, 12 insertions(+)

diff --git a/data/data/aws/vpc/sg-master.tf b/data/data/aws/vpc/sg-master.tf
index 0b621409d0a..71dd49c49f7 100644
--- a/data/data/aws/vpc/sg-master.tf
+++ b/data/data/aws/vpc/sg-master.tf
@@ -31,6 +31,8 @@ resource "aws_security_group_rule" "master_mcs_v6" {
   ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
   from_port        = 22623
   to_port          = 22623
+
+  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "master_egress" {
@@ -73,6 +75,8 @@ resource "aws_security_group_rule" "master_ingress_icmp_v6" {
   ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
   from_port        = -1
   to_port          = -1
+
+  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "master_ingress_ssh" {
@@ -93,6 +97,8 @@ resource "aws_security_group_rule" "master_ingress_ssh_v6" {
   ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
   from_port        = 22
   to_port          = 22
+
+  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "master_ingress_https" {
@@ -113,6 +119,8 @@ resource "aws_security_group_rule" "master_ingress_https_v6" {
   ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
   from_port        = 6443
   to_port          = 6443
+
+  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "master_ingress_vxlan" {
diff --git a/data/data/aws/vpc/sg-worker.tf b/data/data/aws/vpc/sg-worker.tf
index e55482d1060..37055898c30 100644
--- a/data/data/aws/vpc/sg-worker.tf
+++ b/data/data/aws/vpc/sg-worker.tf
@@ -53,6 +53,8 @@ resource "aws_security_group_rule" "worker_ingress_icmp_v6" {
   ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
   from_port        = -1
   to_port          = -1
+
+  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "worker_ingress_ssh" {
@@ -73,6 +75,8 @@ resource "aws_security_group_rule" "worker_ingress_ssh_v6" {
   ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
   from_port        = 22
   to_port          = 22
+
+  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "worker_ingress_vxlan" {

From 834b8b371ac088815dd0e075f29a4a6456df5207 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Fri, 22 Nov 2019 12:45:06 -0500
Subject: [PATCH 18/34] Use TCP health checks from the IPv6 load balancers

---
 data/data/aws/vpc/master-elb.tf | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/data/data/aws/vpc/master-elb.tf b/data/data/aws/vpc/master-elb.tf
index 7afc36a51b0..030cc544169 100644
--- a/data/data/aws/vpc/master-elb.tf
+++ b/data/data/aws/vpc/master-elb.tf
@@ -237,8 +237,16 @@ resource "aws_elb" "api_internal" {
     healthy_threshold   = 2
     unhealthy_threshold = 2
     timeout             = 3
-    target              = "SSL:6443"
-    interval            = 5
+    # TODO
+    # If you use an SSL health check here, you'll get this error in the kube-apiserver log:
+    # ... http: TLS handshake error from 10.0.31.152:64414: tls: no cipher suite supported by both client and server
+    # and this results in the load balancer consider the backend down.
+    # A TCP health check completes successfully, though still results in some noise in the log:
+    # ... http: TLS handshake error from 10.0.10.44:9902: EOF
+    # Adding this cipher suite to the kube-apiserver would allow an SSL or HTTPS health check
+    # without any noise in the log: TLS_RSA_WITH_AES_256_GCM_SHA384
+    target   = "TCP:6443"
+    interval = 5
   }
 
   tags = "${merge(map(
@@ -269,8 +277,16 @@ resource "aws_elb" "api_external" {
     healthy_threshold   = 2
     unhealthy_threshold = 2
     timeout             = 3
-    target              = "SSL:6443"
-    interval            = 5
+    # TODO
+    # If you use an SSL health check here, you'll get this error in the kube-apiserver log:
+    # ... http: TLS handshake error from 10.0.31.152:64414: tls: no cipher suite supported by both client and server
+    # and this results in the load balancer consider the backend down.
+    # A TCP health check completes successfully, though still results in some noise in the log:
+    # ... http: TLS handshake error from 10.0.10.44:9902: EOF
+    # Adding this cipher suite to the kube-apiserver would allow an SSL or HTTPS health check
+    # without any noise in the log: TLS_RSA_WITH_AES_256_GCM_SHA384
+    target   = "TCP:6443"
+    interval = 5
   }
 
   tags = "${merge(map(

From 8521d1d07f56ef71cebec5eed37541a583837040 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Mon, 25 Nov 2019 11:43:45 -0500
Subject: [PATCH 19/34] aws: Allow port 5353 to masters.

For our AWS IPv6 env, we run CoreDNS with host networking so it can
reach AWS DNS over IPv4.  We need to allow traffic to CoreDNS via
security groups when run this way.  Otherwise, only DNS queries that
happen to hit CoreDNS on the same host will work.
---
 data/data/aws/vpc/sg-master.tf | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/data/data/aws/vpc/sg-master.tf b/data/data/aws/vpc/sg-master.tf
index 71dd49c49f7..b8ef5a7cd9c 100644
--- a/data/data/aws/vpc/sg-master.tf
+++ b/data/data/aws/vpc/sg-master.tf
@@ -313,3 +313,33 @@ resource "aws_security_group_rule" "master_ingress_services_udp" {
   self      = true
 }
 
+# For our AWS IPv6 environment, we run CoreDNS with host networking,
+# because it must use IPv4 to reach AWS DNS, so it can't be on our
+# IPv6 only SDN.
+resource "aws_security_group_rule" "master_dns_udp" {
+  type              = "ingress"
+  security_group_id = aws_security_group.master.id
+
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+
+  protocol  = "udp"
+  from_port = 5353
+  to_port   = 5353
+  self      = true
+
+  count = var.use_ipv6 == true ? 1 : 0
+}
+
+resource "aws_security_group_rule" "master_dns_tcp" {
+  type              = "ingress"
+  security_group_id = aws_security_group.master.id
+
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+
+  protocol  = "tcp"
+  from_port = 5353
+  to_port   = 5353
+  self      = true
+
+  count = var.use_ipv6 == true ? 1 : 0
+}

From 118af89afb3516fc8cb1f599b7dbc8d3a751975b Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Wed, 27 Nov 2019 09:24:46 -0500
Subject: [PATCH 20/34] aws: Drop custom IPv6 load balancers

This branch had configured classic load balancers for use with IPv6,
because it appeared that they supported IPv6.  Despite getting IPv6
addresses, they don't actually work.  IPv6 with classic load balancers
only works if you're using EC2 Classic, and not VPC.  They just
rejected IPv6 connections in our case, and various forum posts seem to
confirm this behavior if you try to use it this way.

The documentation for classic load balancers discusses this a bit:

https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-internet-facing-load-balancers.html

The "ipv6" hostnames exist, even if you use AWS-VPC, but they just
don't work.  The docs seem to indicate that the hostnames only exist
with EC2-Classic, but that's not true.

AWS Application Load Balancers (doing HTTPS load balancing) are
supposed to support IPv6, but that would require doing cert management
with AWS.

The simplest approach here is to just fall back to the Network Load
Balancers used with IPv4.  We'll allow API access through the load
balancer over IPv4.  Inside the cluster, API access is still all IPv6,
where it's using the kubernetes Service and Endpoints to reach the
API.  This seems good enough for our AWS dev/test environment for now.
---
 data/data/aws/bootstrap/main.tf      |  14 +--
 data/data/aws/bootstrap/variables.tf |  15 ---
 data/data/aws/main.tf                |  10 --
 data/data/aws/master/main.tf         |  14 +--
 data/data/aws/master/variables.tf    |  15 ---
 data/data/aws/route53/base.tf        |  51 +--------
 data/data/aws/vpc/master-elb.tf      | 157 ++-------------------------
 data/data/aws/vpc/outputs.tf         |  16 +--
 8 files changed, 14 insertions(+), 278 deletions(-)

diff --git a/data/data/aws/bootstrap/main.tf b/data/data/aws/bootstrap/main.tf
index bc620542258..236aecf54de 100644
--- a/data/data/aws/bootstrap/main.tf
+++ b/data/data/aws/bootstrap/main.tf
@@ -153,24 +153,12 @@ resource "aws_instance" "bootstrap" {
 resource "aws_lb_target_group_attachment" "bootstrap" {
   // Because of the issue https://github.com/hashicorp/terraform/issues/12570, the consumers cannot use a dynamic list for count
   // and therefore are force to implicitly assume that the list is of aws_lb_target_group_arns_length - 1, in case there is no api_external
-  count = var.use_ipv6 == true ? 0 : (local.public_endpoints ? var.target_group_arns_length : var.target_group_arns_length - 1)
+  count = local.public_endpoints ? var.target_group_arns_length : var.target_group_arns_length - 1
 
   target_group_arn = var.target_group_arns[count.index]
   target_id        = aws_instance.bootstrap.private_ip
 }
 
-resource "aws_elb_attachment" "bootstrap_api_internal" {
-  count    = var.use_ipv6 == true && local.public_endpoints ? 1 : 0
-  elb      = var.aws_elb_api_internal_id
-  instance = aws_instance.bootstrap.id
-}
-
-resource "aws_elb_attachment" "bootstrap_api_external" {
-  count    = var.use_ipv6 == true ? 1 : 0
-  elb      = var.aws_elb_api_external_id
-  instance = aws_instance.bootstrap.id
-}
-
 resource "aws_security_group" "bootstrap" {
   vpc_id = var.vpc_id
 
diff --git a/data/data/aws/bootstrap/variables.tf b/data/data/aws/bootstrap/variables.tf
index 53e45386424..a9ad48e69ab 100644
--- a/data/data/aws/bootstrap/variables.tf
+++ b/data/data/aws/bootstrap/variables.tf
@@ -84,18 +84,3 @@ variable "publish_strategy" {
   type        = string
   description = "The publishing strategy for endpoints like load balancers"
 }
-
-variable "use_ipv6" {
-  description = "Use IPv6 instead of IPv4"
-  type        = bool
-}
-
-variable "aws_elb_api_internal_id" {
-  description = "ID of the internal IPv6 load balancer"
-  type        = string
-}
-
-variable "aws_elb_api_external_id" {
-  description = "ID of the external IPv6 load balancer"
-  type        = string
-}
diff --git a/data/data/aws/main.tf b/data/data/aws/main.tf
index 492511e3bfc..637c8680eeb 100644
--- a/data/data/aws/main.tf
+++ b/data/data/aws/main.tf
@@ -28,11 +28,6 @@ module "bootstrap" {
   publish_strategy         = var.aws_publish_strategy
 
   tags = local.tags
-
-  use_ipv6 = var.aws_use_ipv6
-
-  aws_elb_api_internal_id = module.vpc.aws_elb_api_internal_id
-  aws_elb_api_external_id = module.vpc.aws_elb_api_external_id
 }
 
 module "masters" {
@@ -55,11 +50,6 @@ module "masters" {
   ec2_ami                  = aws_ami_copy.main.id
   user_data_ign            = var.ignition_master
   publish_strategy         = var.aws_publish_strategy
-
-  use_ipv6 = var.aws_use_ipv6
-
-  aws_elb_api_internal_id = module.vpc.aws_elb_api_internal_id
-  aws_elb_api_external_id = module.vpc.aws_elb_api_external_id
 }
 
 module "iam" {
diff --git a/data/data/aws/master/main.tf b/data/data/aws/master/main.tf
index 77f324a3b46..ed3a7de5f55 100644
--- a/data/data/aws/master/main.tf
+++ b/data/data/aws/master/main.tf
@@ -143,20 +143,8 @@ resource "aws_instance" "master" {
 }
 
 resource "aws_lb_target_group_attachment" "master" {
-  count = var.use_ipv6 == false ? var.instance_count * local.target_group_arns_length : 0
+  count = var.instance_count * local.target_group_arns_length
 
   target_group_arn = var.target_group_arns[count.index % local.target_group_arns_length]
   target_id        = aws_instance.master[floor(count.index / local.target_group_arns_length)].private_ip
 }
-
-resource "aws_elb_attachment" "masters_internal_v6" {
-  count    = var.use_ipv6 == true ? var.instance_count : 0
-  elb      = var.aws_elb_api_internal_id
-  instance = aws_instance.master.*.id[count.index]
-}
-
-resource "aws_elb_attachment" "masters_external_v6" {
-  count    = var.use_ipv6 == true && local.public_endpoints ? var.instance_count : 0
-  elb      = var.aws_elb_api_external_id
-  instance = aws_instance.master.*.id[count.index]
-}
diff --git a/data/data/aws/master/variables.tf b/data/data/aws/master/variables.tf
index 2579c9cd503..8ff122ed36c 100644
--- a/data/data/aws/master/variables.tf
+++ b/data/data/aws/master/variables.tf
@@ -80,18 +80,3 @@ and therefore are force to implicitly assume that the list is of aws_lb_target_g
 helps to decide if the target_group_arns is of length (target_group_arns_length) or (target_group_arns_length - 1)
 EOF
 }
-
-variable "use_ipv6" {
-  description = "Use IPv6 instead of IPv4"
-  type = bool
-}
-
-variable "aws_elb_api_internal_id" {
-  description = "ID of the internal IPv6 load balancer"
-  type = string
-}
-
-variable "aws_elb_api_external_id" {
-  description = "ID of the external IPv6 load balancer"
-  type = string
-}
diff --git a/data/data/aws/route53/base.tf b/data/data/aws/route53/base.tf
index 5acdfafbc30..818d80d2b11 100644
--- a/data/data/aws/route53/base.tf
+++ b/data/data/aws/route53/base.tf
@@ -31,7 +31,7 @@ resource "aws_route53_zone" "int" {
 }
 
 resource "aws_route53_record" "api_external" {
-  count = local.public_endpoints && var.use_ipv6 == false ? 1 : 0
+  count = local.public_endpoints ? 1 : 0
 
   zone_id = data.aws_route53_zone.public[0].zone_id
   name    = "api.${var.cluster_domain}"
@@ -44,20 +44,6 @@ resource "aws_route53_record" "api_external" {
   }
 }
 
-resource "aws_route53_record" "api_external_v6" {
-  zone_id = data.aws_route53_zone.public[0].zone_id
-  name    = "api.${var.cluster_domain}"
-  type    = "AAAA"
-
-  alias {
-    name                   = "ipv6.${var.api_external_lb_dns_name}"
-    zone_id                = var.api_external_lb_zone_id
-    evaluate_target_health = false
-  }
-
-  count = local.public_endpoints && var.use_ipv6 == true ? 1 : 0
-}
-
 resource "aws_route53_record" "api_internal" {
   zone_id = aws_route53_zone.int.zone_id
   name    = "api-int.${var.cluster_domain}"
@@ -68,25 +54,6 @@ resource "aws_route53_record" "api_internal" {
     zone_id                = var.api_internal_lb_zone_id
     evaluate_target_health = false
   }
-
-  # TODO - We must leave the A record enabled, even when trying to test IPv6, because
-  # RHCOS doesn't get an IPv6 address during boot, so ignition will fail trying to reach
-  # the machine config server.
-  #count = var.use_ipv6 == false ? 1 : 0
-}
-
-resource "aws_route53_record" "api_internal_v6" {
-  zone_id = aws_route53_zone.int.zone_id
-  name    = "api-int.${var.cluster_domain}"
-  type    = "AAAA"
-
-  alias {
-    name                   = "ipv6.${var.api_internal_lb_dns_name}"
-    zone_id                = var.api_internal_lb_zone_id
-    evaluate_target_health = false
-  }
-
-  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_route53_record" "api_external_internal_zone" {
@@ -99,22 +66,6 @@ resource "aws_route53_record" "api_external_internal_zone" {
     zone_id                = var.api_internal_lb_zone_id
     evaluate_target_health = false
   }
-
-  count = var.use_ipv6 == false ? 1 : 0
-}
-
-resource "aws_route53_record" "api_external_internal_zone_v6" {
-  zone_id = aws_route53_zone.int.zone_id
-  name    = "api.${var.cluster_domain}"
-  type    = "AAAA"
-
-  alias {
-    name                   = "ipv6.${var.api_internal_lb_dns_name}"
-    zone_id                = var.api_internal_lb_zone_id
-    evaluate_target_health = false
-  }
-
-  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_route53_record" "etcd_a_nodes" {
diff --git a/data/data/aws/vpc/master-elb.tf b/data/data/aws/vpc/master-elb.tf
index 030cc544169..85b5e4da7c4 100644
--- a/data/data/aws/vpc/master-elb.tf
+++ b/data/data/aws/vpc/master-elb.tf
@@ -1,6 +1,3 @@
-# AWS NLBs (Network Load Balancers, or the aws_lb resource) do not support
-# IPv6.  Instead, we must use class load balancers (the aws_elb resource).
-
 resource "aws_lb" "api_internal" {
   name                             = "${var.cluster_id}-int"
   load_balancer_type               = "network"
@@ -20,12 +17,10 @@ resource "aws_lb" "api_internal" {
   }
 
   depends_on = [aws_internet_gateway.igw]
-
-  count = var.use_ipv6 == false ? 1 : 0
 }
 
 resource "aws_lb" "api_external" {
-  count = local.public_endpoints && var.use_ipv6 == false ? 1 : 0
+  count = local.public_endpoints ? 1 : 0
 
   name                             = "${var.cluster_id}-ext"
   load_balancer_type               = "network"
@@ -70,12 +65,10 @@ resource "aws_lb_target_group" "api_internal" {
     protocol            = "HTTPS"
     path                = "/readyz"
   }
-
-  count = var.use_ipv6 == false ? 1 : 0
 }
 
 resource "aws_lb_target_group" "api_external" {
-  count = local.public_endpoints && var.use_ipv6 == false ? 1 : 0
+  count = local.public_endpoints ? 1 : 0
 
   name     = "${var.cluster_id}-aext"
   protocol = "TCP"
@@ -124,38 +117,32 @@ resource "aws_lb_target_group" "services" {
     protocol            = "HTTPS"
     path                = "/healthz"
   }
-
-  count = var.use_ipv6 == false ? 1 : 0
 }
 
 resource "aws_lb_listener" "api_internal_api" {
-  load_balancer_arn = aws_lb.api_internal[0].arn
+  load_balancer_arn = aws_lb.api_internal.arn
   protocol          = "TCP"
   port              = "6443"
 
   default_action {
-    target_group_arn = aws_lb_target_group.api_internal[0].arn
+    target_group_arn = aws_lb_target_group.api_internal.arn
     type             = "forward"
   }
-
-  count = var.use_ipv6 == false ? 1 : 0
 }
 
 resource "aws_lb_listener" "api_internal_services" {
-  load_balancer_arn = aws_lb.api_internal[0].arn
+  load_balancer_arn = aws_lb.api_internal.arn
   protocol          = "TCP"
   port              = "22623"
 
   default_action {
-    target_group_arn = aws_lb_target_group.services[0].arn
+    target_group_arn = aws_lb_target_group.services.arn
     type             = "forward"
   }
-
-  count = var.use_ipv6 == false ? 1 : 0
 }
 
 resource "aws_lb_listener" "api_external_api" {
-  count = local.public_endpoints && var.use_ipv6 == false ? 1 : 0
+  count = local.public_endpoints ? 1 : 0
 
   load_balancer_arn = aws_lb.api_external[0].arn
   protocol          = "TCP"
@@ -166,133 +153,3 @@ resource "aws_lb_listener" "api_external_api" {
     type             = "forward"
   }
 }
-
-#################################
-### Begin IPv6 load balancers ###
-#################################
-
-resource "aws_security_group" "api" {
-  vpc_id = "${data.aws_vpc.cluster_vpc.id}"
-
-  timeouts {
-    create = "20m"
-  }
-
-  tags = "${merge(map(
-    "Name", "${var.cluster_id}_api_sg",
-  ), var.tags)}"
-
-  count = var.use_ipv6 == true ? 1 : 0
-}
-
-resource "aws_security_group_rule" "api_ingress_api_v6" {
-  type              = "ingress"
-  security_group_id = "${aws_security_group.api[0].id}"
-
-  protocol         = "tcp"
-  ipv6_cidr_blocks = ["::/0"]
-  from_port        = 6443
-  to_port          = 6443
-
-  count = var.use_ipv6 == true ? 1 : 0
-}
-
-resource "aws_security_group_rule" "api_ingress_mcs_v6" {
-  type              = "ingress"
-  security_group_id = "${aws_security_group.api[0].id}"
-
-  protocol         = "tcp"
-  ipv6_cidr_blocks = ["::/0"]
-  from_port        = 22623
-  to_port          = 22623
-
-  count = var.use_ipv6 == true ? 1 : 0
-}
-
-resource "aws_elb" "api_internal" {
-  name            = "${var.cluster_id}-int"
-  subnets         = data.aws_subnet.private.*.id
-  internal        = true
-  security_groups = ["${aws_security_group.master.id}"]
-
-  idle_timeout                = 3600
-  connection_draining         = true
-  connection_draining_timeout = 300
-
-  listener {
-    instance_port     = 6443
-    instance_protocol = "tcp"
-    lb_port           = 6443
-    lb_protocol       = "tcp"
-  }
-
-  listener {
-    instance_port     = 22623
-    instance_protocol = "tcp"
-    lb_port           = 22623
-    lb_protocol       = "tcp"
-  }
-
-  health_check {
-    healthy_threshold   = 2
-    unhealthy_threshold = 2
-    timeout             = 3
-    # TODO
-    # If you use an SSL health check here, you'll get this error in the kube-apiserver log:
-    # ... http: TLS handshake error from 10.0.31.152:64414: tls: no cipher suite supported by both client and server
-    # and this results in the load balancer consider the backend down.
-    # A TCP health check completes successfully, though still results in some noise in the log:
-    # ... http: TLS handshake error from 10.0.10.44:9902: EOF
-    # Adding this cipher suite to the kube-apiserver would allow an SSL or HTTPS health check
-    # without any noise in the log: TLS_RSA_WITH_AES_256_GCM_SHA384
-    target   = "TCP:6443"
-    interval = 5
-  }
-
-  tags = "${merge(map(
-    "Name", "${var.cluster_id}-int",
-  ), var.tags)}"
-
-  count = var.use_ipv6 == true ? 1 : 0
-}
-
-resource "aws_elb" "api_external" {
-  name            = "${var.cluster_id}-ext"
-  subnets         = data.aws_subnet.public.*.id
-  internal        = false
-  security_groups = ["${aws_security_group.master.id}", "${aws_security_group.api[0].id}"]
-
-  idle_timeout                = 3600
-  connection_draining         = true
-  connection_draining_timeout = 300
-
-  listener {
-    instance_port     = 6443
-    instance_protocol = "tcp"
-    lb_port           = 6443
-    lb_protocol       = "tcp"
-  }
-
-  health_check {
-    healthy_threshold   = 2
-    unhealthy_threshold = 2
-    timeout             = 3
-    # TODO
-    # If you use an SSL health check here, you'll get this error in the kube-apiserver log:
-    # ... http: TLS handshake error from 10.0.31.152:64414: tls: no cipher suite supported by both client and server
-    # and this results in the load balancer consider the backend down.
-    # A TCP health check completes successfully, though still results in some noise in the log:
-    # ... http: TLS handshake error from 10.0.10.44:9902: EOF
-    # Adding this cipher suite to the kube-apiserver would allow an SSL or HTTPS health check
-    # without any noise in the log: TLS_RSA_WITH_AES_256_GCM_SHA384
-    target   = "TCP:6443"
-    interval = 5
-  }
-
-  tags = "${merge(map(
-    "Name", "${var.cluster_id}-api-external",
-  ), var.tags)}"
-
-  count = local.public_endpoints && var.use_ipv6 == true ? 1 : 0
-}
-
diff --git a/data/data/aws/vpc/outputs.tf b/data/data/aws/vpc/outputs.tf
index f6cf02f179c..a1f355b2475 100644
--- a/data/data/aws/vpc/outputs.tf
+++ b/data/data/aws/vpc/outputs.tf
@@ -53,25 +53,17 @@ output "aws_lb_target_group_arns_length" {
 }
 
 output "aws_lb_api_external_dns_name" {
-  value = local.public_endpoints && var.use_ipv6 == false ? aws_lb.api_external[0].dns_name : aws_elb.api_external[0].dns_name
+  value = local.public_endpoints ? aws_lb.api_external[0].dns_name : null
 }
 
 output "aws_lb_api_external_zone_id" {
-  value = local.public_endpoints && var.use_ipv6 == false ? aws_lb.api_external[0].zone_id : aws_elb.api_external[0].zone_id
+  value = local.public_endpoints ? aws_lb.api_external[0].zone_id : null
 }
 
 output "aws_lb_api_internal_dns_name" {
-  value = var.use_ipv6 == false ? aws_lb.api_internal[0].dns_name : aws_elb.api_internal[0].dns_name
+  value = aws_lb.api_internal.dns_name
 }
 
 output "aws_lb_api_internal_zone_id" {
-  value = var.use_ipv6 == false ? aws_lb.api_internal[0].zone_id : aws_elb.api_internal[0].zone_id
-}
-
-output "aws_elb_api_internal_id" {
-  value = var.use_ipv6 == true ? aws_elb.api_internal[0].id : ""
-}
-
-output "aws_elb_api_external_id" {
-  value = local.public_endpoints && var.use_ipv6 == true ? aws_elb.api_external[0].id : ""
+  value = aws_lb.api_internal.zone_id
 }

From 86c9307d1300c3c6f9fb8c6cfa833c2a3a2476ee Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Wed, 27 Nov 2019 17:48:59 -0500
Subject: [PATCH 21/34] docs: Update aws_ipv6 description of load balancer
 usage.

We reverted back to the IPv4 NLBs, so reflect that status in this doc.
---
 docs/dev/aws_ipv6.md | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/docs/dev/aws_ipv6.md b/docs/dev/aws_ipv6.md
index d9859c26300..db6badce721 100644
--- a/docs/dev/aws_ipv6.md
+++ b/docs/dev/aws_ipv6.md
@@ -22,10 +22,15 @@ here.  This is a summary of the changes to the network environment:
 * The VPC has IPv6 enabled and a `/56` IPv6 CIDR will be allocated by AWS.
 * Each Subnet will have an IPv6 `/64` subnet allocated to it.
 * All IPv4 specific security group rules have corresponding IPv6 rules created.
-* AWS Network Load Balancers (NLBs) do not support IPv6, so classic Elasic Load
-  Balancers (ELBs) are created, instead.
-* The IPv4 NLBs are disabled.
-* IPv6 DNS records (AAAA) are created and the IPv4 (A) records are disabled.
+* AWS Network Load Balancers (NLBs) do not support IPv6, so external API access
+  is still over IPv4.  AWS does not have a TCP load balancer that supports
+  IPv6, other than classic load balancers with EC2-Classic, and not EC2-VPC.
+  AWS Application Load Balancers supposedly support IPv6, but that would
+  require doing HTTPS load balancing for the API instead of just TCP load
+  balancing, so we just use the IPv4 NLBs.  API access within the cluster is
+  still exercising IPv6.
+* IPv6 DNS records (AAAA) are created and the IPv4 (A) records are disabled,
+  except for the API.
 * IPv6 routing is configured.  Since all instances get global IPv6 addresses,
   NAT is not used from the instances out to the internet.
 

From 9648a03e2e8ece9a80d9cc2b31120894ac55e393 Mon Sep 17 00:00:00 2001
From: Mark McLoughlin <markmc@redhat.com>
Date: Thu, 28 Nov 2019 14:04:34 +0000
Subject: [PATCH 22/34] aws: Allow port 5353 to workers.

The DNS pods run on workers as well as masters, so replicate the
IPv6-only rule we added to allow DNS traffic on the host network.

Also remove the unnecessary "self" rule from the masters.
---
 data/data/aws/vpc/sg-master.tf |  2 --
 data/data/aws/vpc/sg-worker.tf | 28 ++++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/data/data/aws/vpc/sg-master.tf b/data/data/aws/vpc/sg-master.tf
index b8ef5a7cd9c..3ac7335cca4 100644
--- a/data/data/aws/vpc/sg-master.tf
+++ b/data/data/aws/vpc/sg-master.tf
@@ -325,7 +325,6 @@ resource "aws_security_group_rule" "master_dns_udp" {
   protocol  = "udp"
   from_port = 5353
   to_port   = 5353
-  self      = true
 
   count = var.use_ipv6 == true ? 1 : 0
 }
@@ -339,7 +338,6 @@ resource "aws_security_group_rule" "master_dns_tcp" {
   protocol  = "tcp"
   from_port = 5353
   to_port   = 5353
-  self      = true
 
   count = var.use_ipv6 == true ? 1 : 0
 }
diff --git a/data/data/aws/vpc/sg-worker.tf b/data/data/aws/vpc/sg-worker.tf
index 37055898c30..85a878eedf9 100644
--- a/data/data/aws/vpc/sg-worker.tf
+++ b/data/data/aws/vpc/sg-worker.tf
@@ -199,3 +199,31 @@ resource "aws_security_group_rule" "worker_ingress_services_udp" {
   self      = true
 }
 
+# For our AWS IPv6 environment, we run CoreDNS with host networking,
+# because it must use IPv4 to reach AWS DNS, so it can't be on our
+# IPv6 only SDN.
+resource "aws_security_group_rule" "worker_dns_udp" {
+  type              = "ingress"
+  security_group_id = aws_security_group.worker.id
+
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+
+  protocol  = "udp"
+  from_port = 5353
+  to_port   = 5353
+
+  count = var.use_ipv6 == true ? 1 : 0
+}
+
+resource "aws_security_group_rule" "worker_dns_tcp" {
+  type              = "ingress"
+  security_group_id = aws_security_group.worker.id
+
+  ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
+
+  protocol  = "tcp"
+  from_port = 5353
+  to_port   = 5353
+
+  count = var.use_ipv6 == true ? 1 : 0
+}

From 3e066246eabe384fe2c9f66b98f5f19471be820b Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Fri, 6 Dec 2019 16:50:58 -0500
Subject: [PATCH 23/34] Rename AWS IPv6 installer support variable

Use an OPENSHIFT_INSTALL_ prefix instead of just OPENSHIFT_.
---
 docs/dev/aws_ipv6.md  | 2 +-
 pkg/tfvars/aws/aws.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/dev/aws_ipv6.md b/docs/dev/aws_ipv6.md
index db6badce721..2c53a29ed3c 100644
--- a/docs/dev/aws_ipv6.md
+++ b/docs/dev/aws_ipv6.md
@@ -10,7 +10,7 @@ To enable IPv6 in your AWS environment, set the following environment variable
 before running the installer:
 
 ```bash
-    export OPENSHIFT_AWS_USE_IPV6=”true”
+    export OPENSHIFT_INSTALL_AWS_USE_IPV6=”true”
 ```
 
 ## AWS Network Environment
diff --git a/pkg/tfvars/aws/aws.go b/pkg/tfvars/aws/aws.go
index 0322951c8a9..92947f4d23f 100644
--- a/pkg/tfvars/aws/aws.go
+++ b/pkg/tfvars/aws/aws.go
@@ -79,7 +79,7 @@ func TFVars(vpc string, privateSubnets []string, publicSubnets []string, publish
 	instanceClass := defaults.InstanceClass(masterConfig.Placement.Region)
 
 	useIPv6 := false
-	if os.Getenv("OPENSHIFT_AWS_USE_IPV6") == "true" {
+	if os.Getenv("OPENSHIFT_INSTALL_AWS_USE_IPV6") == "true" {
 		useIPv6 = true
 	}
 

From 76f1ab0fb336a48b7ce654d454d0c8381d68ef68 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Fri, 6 Dec 2019 17:06:03 -0500
Subject: [PATCH 24/34] docs: Update aws_ipv6.md with more details

Add more details that have come up in review so far.
---
 docs/dev/aws_ipv6.md | 32 ++++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/docs/dev/aws_ipv6.md b/docs/dev/aws_ipv6.md
index 2c53a29ed3c..3de6d0ef6bc 100644
--- a/docs/dev/aws_ipv6.md
+++ b/docs/dev/aws_ipv6.md
@@ -28,12 +28,38 @@ here.  This is a summary of the changes to the network environment:
   AWS Application Load Balancers supposedly support IPv6, but that would
   require doing HTTPS load balancing for the API instead of just TCP load
   balancing, so we just use the IPv4 NLBs.  API access within the cluster is
-  still exercising IPv6.
+  still exercising IPv6 when using its Service IP..
 * IPv6 DNS records (AAAA) are created and the IPv4 (A) records are disabled,
-  except for the API.
+  except for the API since the API is still accessed via an IPv4 only load
+  balancer.
 * IPv6 routing is configured.  Since all instances get global IPv6 addresses,
   NAT is not used from the instances out to the internet.
 
+## Node Addresses
+
+Each AWS instance will receive both a private IPv4 address and a globally
+routeable IPv6 address.
+
+Kubelet is configured to use the IPv6 address for the Node object.
+
+etcd and all other services running with host networking will be configured to
+use the IPv6 address.
+
+## Hack for IPv4 Access Where Necessary
+
+There are some pods that still require IPv4 access on AWS to be functional.
+For example, the CoreDNS pods must have IPv4 connectivity since the AWS DNS
+server is only available via IPv4.  This also means we have to add a security
+group rule allowing DNS traffic to our CoreDNS pods over the AWS network (they
+use port 5353).
+
+Another case where this hack is required is several pods that need to access
+AWS APIs.  The AWS APIs are IPv4-only.
+
+Since this is an AWS-IPv6 specific hack, it is currently centralized into one
+place: ovn-kubernetes.  It will automatically add a second interface with IPv4
+access to the set of affected pods.
+
 ## Install Configuration
 
 Here is the suggested network configuration for `install-config.yaml`:
@@ -53,6 +79,8 @@ Note that an IPv4 CIDR is still used for `machineCIDR` since AWS will provide a
 dual-stack (IPv4 and IPv6) environment.  We must specify the IPv4 CIDR and AWS
 will automatically allocate an IPv6 CIDR.
 
+`OVNKubernetes` is the only `networkType` supported in this environment.
+
 ## Current Status of IPv6
 
 Note that IPv6 support is under heavy development across many components in

From 2eb483e42938fd9a8c80a6c7b3c823c968b548d7 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Fri, 6 Dec 2019 17:21:55 -0500
Subject: [PATCH 25/34] aws: Drop TODO comments about a terraform upgrade

These comments were about something to check related to a terraform
upgrade.  The code is right as it is.  As recommended by the comment
itself, these TODO comments should just be removed now that it has
been checked.
---
 data/data/aws/route53/base.tf | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/data/data/aws/route53/base.tf b/data/data/aws/route53/base.tf
index 818d80d2b11..1adf09e6c47 100644
--- a/data/data/aws/route53/base.tf
+++ b/data/data/aws/route53/base.tf
@@ -74,14 +74,6 @@ resource "aws_route53_record" "etcd_a_nodes" {
   ttl     = "60"
   zone_id = aws_route53_zone.int.zone_id
   name    = "etcd-${count.index}.${var.cluster_domain}"
-  # TF-UPGRADE-TODO: In Terraform v0.10 and earlier, it was sometimes necessary to
-  # force an interpolation expression to be interpreted as a list by wrapping it
-  # in an extra set of list brackets. That form was supported for compatibilty in
-  # v0.11, but is no longer supported in Terraform v0.12.
-  #
-  # If the expression in the following list itself returns a list, remove the
-  # brackets to avoid interpretation as a list of lists. If the expression
-  # returns a single list item then leave it as-is and remove this TODO comment.
   records = [var.etcd_ip_addresses[count.index]]
 }
 
@@ -91,14 +83,6 @@ resource "aws_route53_record" "etcd_aaaa_nodes" {
   ttl     = "60"
   zone_id = aws_route53_zone.int.zone_id
   name    = "etcd-${count.index}.${var.cluster_domain}"
-  # TF-UPGRADE-TODO: In Terraform v0.10 and earlier, it was sometimes necessary to
-  # force an interpolation expression to be interpreted as a list by wrapping it
-  # in an extra set of list brackets. That form was supported for compatibilty in
-  # v0.11, but is no longer supported in Terraform v0.12.
-  #
-  # If the expression in the following list itself returns a list, remove the
-  # brackets to avoid interpretation as a list of lists. If the expression
-  # returns a single list item then leave it as-is and remove this TODO comment.
   records = [var.etcd_ipv6_addresses[count.index]]
 }
 

From 40891fb3bd30bc48be85c9a4f4e111c04578c40d Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Fri, 6 Dec 2019 19:25:34 -0500
Subject: [PATCH 26/34] aws-ipv6: Note future move to egress only gateway

Note a todo item that I will take care of soon: changing to an
egress-only internet gateway.

https://aws.amazon.com/premiumsupport/knowledge-center/configure-private-ipv6-subnet/
---
 docs/dev/aws_ipv6.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/dev/aws_ipv6.md b/docs/dev/aws_ipv6.md
index 3de6d0ef6bc..20dbd05533b 100644
--- a/docs/dev/aws_ipv6.md
+++ b/docs/dev/aws_ipv6.md
@@ -33,7 +33,11 @@ here.  This is a summary of the changes to the network environment:
   except for the API since the API is still accessed via an IPv4 only load
   balancer.
 * IPv6 routing is configured.  Since all instances get global IPv6 addresses,
-  NAT is not used from the instances out to the internet.
+  NAT is not used from the instances out to the internet.  The current
+  implementation uses security groups to block incoming traffic sent directly
+  to any of the instances, but will move to using an egress-only internet
+  gateway which will make this isolation more explicit.
+
 
 ## Node Addresses
 

From 7d7082f0ae2ee2e2168ea871b009bd2799177add Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Mon, 9 Dec 2019 09:10:09 -0500
Subject: [PATCH 27/34] aws: Remove unused variable

This variable was added in an earlier commit, but is no longer used.
---
 data/data/aws/master/main.tf | 2 --
 1 file changed, 2 deletions(-)

diff --git a/data/data/aws/master/main.tf b/data/data/aws/master/main.tf
index ed3a7de5f55..b3eeb147221 100644
--- a/data/data/aws/master/main.tf
+++ b/data/data/aws/master/main.tf
@@ -4,8 +4,6 @@ locals {
   // Because of the issue https://github.com/hashicorp/terraform/issues/12570, the consumers cannot use a dynamic list for count
   // and therefore are force to implicitly assume that the list is of aws_lb_target_group_arns_length - 1, in case there is no api_external
   target_group_arns_length = var.publish_strategy == "External" ? var.target_group_arns_length : var.target_group_arns_length - 1
-
-  public_endpoints = var.publish_strategy == "External" ? true : false
 }
 
 resource "aws_iam_instance_profile" "master" {

From 9907dd217fee6a5c10d33faf10acf094212ecc5a Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Mon, 9 Dec 2019 09:35:19 -0500
Subject: [PATCH 28/34] aws: Simplify a formatlist conditional

Move the conditional part of the expression to only apply to the part
of the formatlist() that's different between IPv4 and IPv6.
---
 data/data/aws/route53/base.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/data/data/aws/route53/base.tf b/data/data/aws/route53/base.tf
index 1adf09e6c47..6948f1aa321 100644
--- a/data/data/aws/route53/base.tf
+++ b/data/data/aws/route53/base.tf
@@ -91,6 +91,6 @@ resource "aws_route53_record" "etcd_cluster" {
   ttl     = "60"
   zone_id = aws_route53_zone.int.zone_id
   name    = "_etcd-server-ssl._tcp"
-  records = var.use_ipv6 == false ? formatlist("0 10 2380 %s", aws_route53_record.etcd_a_nodes.*.fqdn) : formatlist("0 10 2380 %s", aws_route53_record.etcd_aaaa_nodes.*.fqdn)
+  records = formatlist("0 10 2380 %s", var.use_ipv6 == false ? aws_route53_record.etcd_a_nodes.*.fqdn : aws_route53_record.etcd_aaaa_nodes.*.fqdn)
 }
 

From c21d6df5f9ba231838baf7302f6a0deba7f41b2a Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Mon, 9 Dec 2019 22:20:56 -0500
Subject: [PATCH 29/34] aws-ipv6: Adjust resource counts

This patch mostly just moves the count to the beginning of a resource
as a style change, as requested during review.

While doing this, I noticed two IPv6 security group rules that were
not conditional on IPv6 support being turned on.
---
 data/data/aws/bootstrap/main.tf      |  4 ++++
 data/data/aws/bootstrap/variables.tf |  5 +++++
 data/data/aws/main.tf                |  1 +
 data/data/aws/vpc/sg-master.tf       | 28 ++++++++++++++--------------
 data/data/aws/vpc/sg-worker.tf       | 20 ++++++++++----------
 5 files changed, 34 insertions(+), 24 deletions(-)

diff --git a/data/data/aws/bootstrap/main.tf b/data/data/aws/bootstrap/main.tf
index 236aecf54de..982bc94272b 100644
--- a/data/data/aws/bootstrap/main.tf
+++ b/data/data/aws/bootstrap/main.tf
@@ -185,6 +185,8 @@ resource "aws_security_group_rule" "ssh" {
 }
 
 resource "aws_security_group_rule" "ssh_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
   type              = "ingress"
   security_group_id = aws_security_group.bootstrap.id
 
@@ -205,6 +207,8 @@ resource "aws_security_group_rule" "bootstrap_journald_gateway" {
 }
 
 resource "aws_security_group_rule" "bootstrap_journald_gateway_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
   type              = "ingress"
   security_group_id = aws_security_group.bootstrap.id
 
diff --git a/data/data/aws/bootstrap/variables.tf b/data/data/aws/bootstrap/variables.tf
index a9ad48e69ab..000c8c84529 100644
--- a/data/data/aws/bootstrap/variables.tf
+++ b/data/data/aws/bootstrap/variables.tf
@@ -84,3 +84,8 @@ variable "publish_strategy" {
   type        = string
   description = "The publishing strategy for endpoints like load balancers"
 }
+
+variable "use_ipv6" {
+  description = "Use IPv6 instead of IPv4"
+  type        = bool
+}
diff --git a/data/data/aws/main.tf b/data/data/aws/main.tf
index 637c8680eeb..bf901c87eb8 100644
--- a/data/data/aws/main.tf
+++ b/data/data/aws/main.tf
@@ -26,6 +26,7 @@ module "bootstrap" {
   vpc_ipv6_cidrs           = module.vpc.vpc_ipv6_cidrs
   vpc_security_group_ids   = [module.vpc.master_sg_id]
   publish_strategy         = var.aws_publish_strategy
+  use_ipv6                 = var.aws_use_ipv6
 
   tags = local.tags
 }
diff --git a/data/data/aws/vpc/sg-master.tf b/data/data/aws/vpc/sg-master.tf
index 3ac7335cca4..2c7038733dc 100644
--- a/data/data/aws/vpc/sg-master.tf
+++ b/data/data/aws/vpc/sg-master.tf
@@ -24,6 +24,8 @@ resource "aws_security_group_rule" "master_mcs" {
 }
 
 resource "aws_security_group_rule" "master_mcs_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
   type              = "ingress"
   security_group_id = aws_security_group.master.id
 
@@ -31,8 +33,6 @@ resource "aws_security_group_rule" "master_mcs_v6" {
   ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
   from_port        = 22623
   to_port          = 22623
-
-  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "master_egress" {
@@ -46,6 +46,8 @@ resource "aws_security_group_rule" "master_egress" {
 }
 
 resource "aws_security_group_rule" "master_egress_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
   type              = "egress"
   security_group_id = aws_security_group.master.id
 
@@ -53,8 +55,6 @@ resource "aws_security_group_rule" "master_egress_v6" {
   to_port          = 0
   protocol         = "-1"
   ipv6_cidr_blocks = ["::/0"]
-
-  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "master_ingress_icmp" {
@@ -68,6 +68,8 @@ resource "aws_security_group_rule" "master_ingress_icmp" {
 }
 
 resource "aws_security_group_rule" "master_ingress_icmp_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
   type              = "ingress"
   security_group_id = aws_security_group.master.id
 
@@ -75,8 +77,6 @@ resource "aws_security_group_rule" "master_ingress_icmp_v6" {
   ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
   from_port        = -1
   to_port          = -1
-
-  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "master_ingress_ssh" {
@@ -90,6 +90,8 @@ resource "aws_security_group_rule" "master_ingress_ssh" {
 }
 
 resource "aws_security_group_rule" "master_ingress_ssh_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
   type              = "ingress"
   security_group_id = aws_security_group.master.id
 
@@ -97,8 +99,6 @@ resource "aws_security_group_rule" "master_ingress_ssh_v6" {
   ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
   from_port        = 22
   to_port          = 22
-
-  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "master_ingress_https" {
@@ -112,6 +112,8 @@ resource "aws_security_group_rule" "master_ingress_https" {
 }
 
 resource "aws_security_group_rule" "master_ingress_https_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
   type              = "ingress"
   security_group_id = aws_security_group.master.id
 
@@ -119,8 +121,6 @@ resource "aws_security_group_rule" "master_ingress_https_v6" {
   ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
   from_port        = 6443
   to_port          = 6443
-
-  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "master_ingress_vxlan" {
@@ -317,6 +317,8 @@ resource "aws_security_group_rule" "master_ingress_services_udp" {
 # because it must use IPv4 to reach AWS DNS, so it can't be on our
 # IPv6 only SDN.
 resource "aws_security_group_rule" "master_dns_udp" {
+  count = var.use_ipv6 == true ? 1 : 0
+
   type              = "ingress"
   security_group_id = aws_security_group.master.id
 
@@ -325,11 +327,11 @@ resource "aws_security_group_rule" "master_dns_udp" {
   protocol  = "udp"
   from_port = 5353
   to_port   = 5353
-
-  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "master_dns_tcp" {
+  count = var.use_ipv6 == true ? 1 : 0
+
   type              = "ingress"
   security_group_id = aws_security_group.master.id
 
@@ -338,6 +340,4 @@ resource "aws_security_group_rule" "master_dns_tcp" {
   protocol  = "tcp"
   from_port = 5353
   to_port   = 5353
-
-  count = var.use_ipv6 == true ? 1 : 0
 }
diff --git a/data/data/aws/vpc/sg-worker.tf b/data/data/aws/vpc/sg-worker.tf
index 85a878eedf9..e06aa472728 100644
--- a/data/data/aws/vpc/sg-worker.tf
+++ b/data/data/aws/vpc/sg-worker.tf
@@ -24,6 +24,8 @@ resource "aws_security_group_rule" "worker_egress" {
 }
 
 resource "aws_security_group_rule" "worker_egress_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
   type              = "egress"
   security_group_id = aws_security_group.worker.id
 
@@ -31,8 +33,6 @@ resource "aws_security_group_rule" "worker_egress_v6" {
   to_port          = 0
   protocol         = "-1"
   ipv6_cidr_blocks = ["::/0"]
-
-  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "worker_ingress_icmp" {
@@ -46,6 +46,8 @@ resource "aws_security_group_rule" "worker_ingress_icmp" {
 }
 
 resource "aws_security_group_rule" "worker_ingress_icmp_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
   type              = "ingress"
   security_group_id = aws_security_group.worker.id
 
@@ -53,8 +55,6 @@ resource "aws_security_group_rule" "worker_ingress_icmp_v6" {
   ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
   from_port        = -1
   to_port          = -1
-
-  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "worker_ingress_ssh" {
@@ -68,6 +68,8 @@ resource "aws_security_group_rule" "worker_ingress_ssh" {
 }
 
 resource "aws_security_group_rule" "worker_ingress_ssh_v6" {
+  count = var.use_ipv6 == true ? 1 : 0
+
   type              = "ingress"
   security_group_id = aws_security_group.worker.id
 
@@ -75,8 +77,6 @@ resource "aws_security_group_rule" "worker_ingress_ssh_v6" {
   ipv6_cidr_blocks = [data.aws_vpc.cluster_vpc.ipv6_cidr_block]
   from_port        = 22
   to_port          = 22
-
-  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "worker_ingress_vxlan" {
@@ -203,6 +203,8 @@ resource "aws_security_group_rule" "worker_ingress_services_udp" {
 # because it must use IPv4 to reach AWS DNS, so it can't be on our
 # IPv6 only SDN.
 resource "aws_security_group_rule" "worker_dns_udp" {
+  count = var.use_ipv6 == true ? 1 : 0
+
   type              = "ingress"
   security_group_id = aws_security_group.worker.id
 
@@ -211,11 +213,11 @@ resource "aws_security_group_rule" "worker_dns_udp" {
   protocol  = "udp"
   from_port = 5353
   to_port   = 5353
-
-  count = var.use_ipv6 == true ? 1 : 0
 }
 
 resource "aws_security_group_rule" "worker_dns_tcp" {
+  count = var.use_ipv6 == true ? 1 : 0
+
   type              = "ingress"
   security_group_id = aws_security_group.worker.id
 
@@ -224,6 +226,4 @@ resource "aws_security_group_rule" "worker_dns_tcp" {
   protocol  = "tcp"
   from_port = 5353
   to_port   = 5353
-
-  count = var.use_ipv6 == true ? 1 : 0
 }

From df9156f41204714742cb0bd401bde6d363f1a951 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Mon, 9 Dec 2019 22:27:29 -0500
Subject: [PATCH 30/34] aws-ipv6: Add comments to help explain subnet
 allocation

---
 data/data/aws/vpc/vpc-private.tf | 1 +
 data/data/aws/vpc/vpc-public.tf  | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/data/data/aws/vpc/vpc-private.tf b/data/data/aws/vpc/vpc-private.tf
index 5f7519b4aa8..63b08705142 100644
--- a/data/data/aws/vpc/vpc-private.tf
+++ b/data/data/aws/vpc/vpc-private.tf
@@ -48,6 +48,7 @@ resource "aws_subnet" "private_subnet" {
 
   availability_zone = var.availability_zones[count.index]
 
+  # AWS gives us a /56 and we need to carve out /64 subnets.
   ipv6_cidr_block                 = var.use_ipv6 == true ? cidrsubnet(data.aws_vpc.cluster_vpc.ipv6_cidr_block, 8, count.index) : ""
   assign_ipv6_address_on_creation = var.use_ipv6
 
diff --git a/data/data/aws/vpc/vpc-public.tf b/data/data/aws/vpc/vpc-public.tf
index ecc7f346067..eb150bd6488 100644
--- a/data/data/aws/vpc/vpc-public.tf
+++ b/data/data/aws/vpc/vpc-public.tf
@@ -62,6 +62,9 @@ resource "aws_subnet" "public_subnet" {
   cidr_block        = cidrsubnet(local.new_public_cidr_range, 3, count.index)
   availability_zone = var.availability_zones[count.index]
 
+  # AWS gives us a /56 and we need to carve out /64 subnets.
+  # We skip the first length(var.availability_zones), because
+  # that is now many were taken in vpc-private.tf.
   ipv6_cidr_block                 = var.use_ipv6 == true ? cidrsubnet(data.aws_vpc.cluster_vpc.ipv6_cidr_block, 8, count.index + length(var.availability_zones)) : ""
   assign_ipv6_address_on_creation = var.use_ipv6
 

From 9b4f14d7341c22f95499f5f84d910a696fd7a9c9 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Mon, 9 Dec 2019 22:37:24 -0500
Subject: [PATCH 31/34] aws-ipv6: Drop etcd_ipv6_addresses

Make use of the existing etcd_ip_addresses instead.
---
 data/data/aws/main.tf              | 3 +--
 data/data/aws/route53/base.tf      | 2 +-
 data/data/aws/route53/variables.tf | 8 +-------
 3 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/data/data/aws/main.tf b/data/data/aws/main.tf
index bf901c87eb8..be50c30bf15 100644
--- a/data/data/aws/main.tf
+++ b/data/data/aws/main.tf
@@ -72,8 +72,7 @@ module "dns" {
   cluster_domain           = var.cluster_domain
   cluster_id               = var.cluster_id
   etcd_count               = var.master_count
-  etcd_ip_addresses        = flatten(module.masters.ip_addresses)
-  etcd_ipv6_addresses      = flatten(module.masters.ipv6_addresses)
+  etcd_ip_addresses        = var.aws_use_ipv6 == true ? flatten(module.masters.ipv6_addresses) : flatten(module.masters.ip_addresses)
   tags                     = local.tags
   vpc_id                   = module.vpc.vpc_id
   publish_strategy         = var.aws_publish_strategy
diff --git a/data/data/aws/route53/base.tf b/data/data/aws/route53/base.tf
index 6948f1aa321..6619960b292 100644
--- a/data/data/aws/route53/base.tf
+++ b/data/data/aws/route53/base.tf
@@ -83,7 +83,7 @@ resource "aws_route53_record" "etcd_aaaa_nodes" {
   ttl     = "60"
   zone_id = aws_route53_zone.int.zone_id
   name    = "etcd-${count.index}.${var.cluster_domain}"
-  records = [var.etcd_ipv6_addresses[count.index]]
+  records = [var.etcd_ip_addresses[count.index]]
 }
 
 resource "aws_route53_record" "etcd_cluster" {
diff --git a/data/data/aws/route53/variables.tf b/data/data/aws/route53/variables.tf
index 4477e87355e..9ab0348fa0f 100644
--- a/data/data/aws/route53/variables.tf
+++ b/data/data/aws/route53/variables.tf
@@ -9,13 +9,7 @@ variable "etcd_count" {
 }
 
 variable "etcd_ip_addresses" {
-  description = "List of string IPs for machines running etcd members."
-  type        = list(string)
-  default     = []
-}
-
-variable "etcd_ipv6_addresses" {
-  description = "List of string IPs (IPv6) for machines running etcd members."
+  description = "List of string IPs (IPv4 or IPv6) for machines running etcd members."
   type        = list(string)
   default     = []
 }

From 71acc5318975b53446ea1231515b6f44b3710f77 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Tue, 10 Dec 2019 14:01:38 -0500
Subject: [PATCH 32/34] aws-ipv6: Add more validation

As requested in review:

- when aws IPv6 is on, force OVNKubernetes network type
- Force IPv6 CIDRs where needed when AWS IPv6 is on
- Re-add IPv4 CIDR enforcement everywhere else
- make sure the platform is AWS if the aws IPv6 env var is set
---
 pkg/types/validation/installconfig.go | 35 +++++++++++++++++++++------
 pkg/validate/validate.go              |  8 +++++-
 2 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/pkg/types/validation/installconfig.go b/pkg/types/validation/installconfig.go
index d2b79afcc56..55c37f165e7 100644
--- a/pkg/types/validation/installconfig.go
+++ b/pkg/types/validation/installconfig.go
@@ -3,6 +3,7 @@ package validation
 import (
 	"fmt"
 	"net"
+	"os"
 	"sort"
 	"strings"
 
@@ -75,7 +76,7 @@ func ValidateInstallConfig(c *types.InstallConfig, openStackValidValuesFetcher o
 		}
 	}
 	if c.Networking != nil {
-		allErrs = append(allErrs, validateNetworking(c.Networking, field.NewPath("networking"))...)
+		allErrs = append(allErrs, validateNetworking(c.Networking, field.NewPath("networking"), &c.Platform)...)
 	} else {
 		allErrs = append(allErrs, field.Required(field.NewPath("networking"), "networking is required"))
 	}
@@ -99,14 +100,32 @@ func ValidateInstallConfig(c *types.InstallConfig, openStackValidValuesFetcher o
 	return allErrs
 }
 
-func validateNetworking(n *types.Networking, fldPath *field.Path) field.ErrorList {
+func validateNetworking(n *types.Networking, fldPath *field.Path, platform *types.Platform) field.ErrorList {
 	allErrs := field.ErrorList{}
 	if n.NetworkType == "" {
 		allErrs = append(allErrs, field.Required(fldPath.Child("networkType"), "network provider type required"))
 	}
 
+	// IPv6 CIDRs are only allowed for:
+	//  - baremetal platform
+	//  - AWS IPv6 dev/test env, with OPENSHIFT_INSTALL_AWS_USE_IPV6 set
+	allowIPv6 := false
+	requireIPv6 := false
+	if platform.BareMetal != nil {
+		allowIPv6 = true
+	}
+	if platform.AWS != nil && os.Getenv("OPENSHIFT_INSTALL_AWS_USE_IPV6") == "true" {
+		allowIPv6 = true
+		requireIPv6 = true
+		if n.NetworkType != "OVNKubernetes" {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("networkType"), n.NetworkType, "networkType must be OVNKubernetes for AWS IPv6"))
+		}
+	}
+
 	if n.MachineCIDR != nil {
-		if err := validate.SubnetCIDR(&n.MachineCIDR.IPNet); err != nil {
+		// MachineCIDR should still be IPv4 for AWS IPv6, because AWS only does
+		// dual-stack, and the IPv6 CIDR will be assigned by AWS.
+		if err := validate.SubnetCIDR(&n.MachineCIDR.IPNet, allowIPv6, false); err != nil {
 			allErrs = append(allErrs, field.Invalid(fldPath.Child("machineCIDR"), n.MachineCIDR.String(), err.Error()))
 		}
 	} else {
@@ -114,7 +133,7 @@ func validateNetworking(n *types.Networking, fldPath *field.Path) field.ErrorLis
 	}
 
 	for i, sn := range n.ServiceNetwork {
-		if err := validate.SubnetCIDR(&sn.IPNet); err != nil {
+		if err := validate.SubnetCIDR(&sn.IPNet, allowIPv6, requireIPv6); err != nil {
 			allErrs = append(allErrs, field.Invalid(fldPath.Child("serviceNetwork").Index(i), sn.String(), err.Error()))
 		}
 		if n.MachineCIDR != nil && validate.DoCIDRsOverlap(&sn.IPNet, &n.MachineCIDR.IPNet) {
@@ -140,7 +159,7 @@ func validateNetworking(n *types.Networking, fldPath *field.Path) field.ErrorLis
 	}
 
 	for i, cn := range n.ClusterNetwork {
-		allErrs = append(allErrs, validateClusterNetwork(n, &cn, i, fldPath.Child("clusterNetwork").Index(i))...)
+		allErrs = append(allErrs, validateClusterNetwork(n, &cn, i, fldPath.Child("clusterNetwork").Index(i), allowIPv6, requireIPv6)...)
 	}
 	if len(n.ClusterNetwork) == 0 {
 		allErrs = append(allErrs, field.Required(fldPath.Child("clusterNetwork"), "cluster network required"))
@@ -148,9 +167,9 @@ func validateNetworking(n *types.Networking, fldPath *field.Path) field.ErrorLis
 	return allErrs
 }
 
-func validateClusterNetwork(n *types.Networking, cn *types.ClusterNetworkEntry, idx int, fldPath *field.Path) field.ErrorList {
+func validateClusterNetwork(n *types.Networking, cn *types.ClusterNetworkEntry, idx int, fldPath *field.Path, allowIPv6, requireIPv6 bool) field.ErrorList {
 	allErrs := field.ErrorList{}
-	if err := validate.SubnetCIDR(&cn.CIDR.IPNet); err != nil {
+	if err := validate.SubnetCIDR(&cn.CIDR.IPNet, allowIPv6, requireIPv6); err != nil {
 		allErrs = append(allErrs, field.Invalid(fldPath.Child("cidr"), cn.CIDR.IPNet.String(), err.Error()))
 	}
 	if n.MachineCIDR != nil && validate.DoCIDRsOverlap(&cn.CIDR.IPNet, &n.MachineCIDR.IPNet) {
@@ -223,6 +242,8 @@ func validatePlatform(platform *types.Platform, fldPath *field.Path, openStackVa
 	}
 	if platform.AWS != nil {
 		validate(aws.Name, platform.AWS, func(f *field.Path) field.ErrorList { return awsvalidation.ValidatePlatform(platform.AWS, f) })
+	} else if os.Getenv("OPENSHIFT_INSTALL_AWS_USE_IPV6") == "true" {
+		allErrs = append(allErrs, field.Invalid(fldPath, activePlatform, "OPENSHIFT_INSTALL_AWS_USE_IPV6 only valid for AWS"))
 	}
 	if platform.Azure != nil {
 		validate(azure.Name, platform.Azure, func(f *field.Path) field.ErrorList {
diff --git a/pkg/validate/validate.go b/pkg/validate/validate.go
index dd2c1fb9bd9..213f539d380 100644
--- a/pkg/validate/validate.go
+++ b/pkg/validate/validate.go
@@ -108,7 +108,13 @@ func ClusterName(v string) error {
 }
 
 // SubnetCIDR checks if the given IP net is a valid CIDR.
-func SubnetCIDR(cidr *net.IPNet) error {
+func SubnetCIDR(cidr *net.IPNet, allowIPv6, requireIPv6 bool) error {
+	if allowIPv6 == false && cidr.IP.To4() == nil {
+		return errors.New("must use IPv4")
+	}
+	if requireIPv6 == true && cidr.IP.To4() != nil {
+		return errors.New("must use IPv6")
+	}
 	if cidr.IP.IsUnspecified() {
 		return errors.New("address must be specified")
 	}

From 95064cf1e634a51a661189cb980c5528fd98756b Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Fri, 13 Dec 2019 14:30:58 -0500
Subject: [PATCH 33/34] aws-ipv6: Fix up SubnetCIDR() test case

I changed this code but forgot to fix the test case.
---
 pkg/validate/validate_test.go | 35 +++++++++++++++++++++--------------
 1 file changed, 21 insertions(+), 14 deletions(-)

diff --git a/pkg/validate/validate_test.go b/pkg/validate/validate_test.go
index e0b1b511861..d89c0b4c1b2 100644
--- a/pkg/validate/validate_test.go
+++ b/pkg/validate/validate_test.go
@@ -51,20 +51,27 @@ func TestClusterName(t *testing.T) {
 
 func TestSubnetCIDR(t *testing.T) {
 	cases := []struct {
-		cidr   string
-		expErr string
+		cidr        string
+		expErr      string
+		allowIPv6   bool
+		requireIPv6 bool
 	}{
-		{"0.0.0.0/32", "address must be specified"},
-		{"1.2.3.4/0", "invalid network address. got 1.2.3.4/0, expecting 0.0.0.0/0"},
-		{"1.2.3.4/1", "invalid network address. got 1.2.3.4/1, expecting 0.0.0.0/1"},
-		{"1.2.3.4/31", ""},
-		{"1.2.3.4/32", ""},
-		{"0:0:0:0:0:1:102:304/116", "invalid network address. got ::1:102:304/116, expecting ::1:102:0/116"},
-		{"fd01::/48", ""},
-		{"172.17.0.0/20", "overlaps with default Docker Bridge subnet (172.17.0.0/20)"},
-		{"172.0.0.0/8", "overlaps with default Docker Bridge subnet (172.0.0.0/8)"},
-		{"255.255.255.255/1", "invalid network address. got 255.255.255.255/1, expecting 128.0.0.0/1"},
-		{"255.255.255.255/32", ""},
+		{"0.0.0.0/32", "address must be specified", false, false},
+		{"1.2.3.4/0", "invalid network address. got 1.2.3.4/0, expecting 0.0.0.0/0", false, false},
+		{"1.2.3.4/1", "invalid network address. got 1.2.3.4/1, expecting 0.0.0.0/1", false, false},
+		{"1.2.3.4/31", "", false, false},
+		{"1.2.3.4/32", "", false, false},
+		{"0:0:0:0:0:1:102:304/116", "must use IPv4", false, false},
+		{"fd01::/48", "must use IPv4", false, false},
+		{"172.17.0.0/20", "overlaps with default Docker Bridge subnet (172.17.0.0/20)", false, false},
+		{"172.0.0.0/8", "overlaps with default Docker Bridge subnet (172.0.0.0/8)", false, false},
+		{"255.255.255.255/1", "invalid network address. got 255.255.255.255/1, expecting 128.0.0.0/1", false, false},
+		{"255.255.255.255/32", "", false, false},
+		{"fd01::/64", "must use IPv4", false, false},
+		{"fd01::/64", "", true, false},
+		{"192.168.0.0/16", "", true, false},
+		{"fd01::/64", "", true, true},
+		{"192.168.0.0/16", "must use IPv6", true, true},
 	}
 	for _, tc := range cases {
 		t.Run(tc.cidr, func(t *testing.T) {
@@ -72,7 +79,7 @@ func TestSubnetCIDR(t *testing.T) {
 			if err != nil {
 				t.Fatalf("could not parse cidr: %v", err)
 			}
-			err = SubnetCIDR(&net.IPNet{IP: ip, Mask: cidr.Mask})
+			err = SubnetCIDR(&net.IPNet{IP: ip, Mask: cidr.Mask}, tc.allowIPv6, tc.requireIPv6)
 			if tc.expErr != "" {
 				assert.EqualError(t, err, tc.expErr)
 			} else {

From b235ba1864813d5087b07788f98883c339b35402 Mon Sep 17 00:00:00 2001
From: Russell Bryant <rbryant@redhat.com>
Date: Wed, 18 Dec 2019 13:32:33 -0500
Subject: [PATCH 34/34] validation: Allow UPI deployments to use IPv6 CIDRs as
 well

---
 pkg/types/validation/installconfig.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/types/validation/installconfig.go b/pkg/types/validation/installconfig.go
index 55c37f165e7..5b0e254b457 100644
--- a/pkg/types/validation/installconfig.go
+++ b/pkg/types/validation/installconfig.go
@@ -111,7 +111,7 @@ func validateNetworking(n *types.Networking, fldPath *field.Path, platform *type
 	//  - AWS IPv6 dev/test env, with OPENSHIFT_INSTALL_AWS_USE_IPV6 set
 	allowIPv6 := false
 	requireIPv6 := false
-	if platform.BareMetal != nil {
+	if platform.BareMetal != nil || platform.None != nil {
 		allowIPv6 = true
 	}
 	if platform.AWS != nil && os.Getenv("OPENSHIFT_INSTALL_AWS_USE_IPV6") == "true" {