azure: Add full dual-stack IPv6 support to the install for Azure

The installer can now provision a dual-stack IP v4/v6 (v4 primary)
cluster if OPENSHIFT_INSTALL_AZURE_USE_IPV6=true is passed on
install. Unlike the AWS configuration, this is currently configured
to make IPv6 transparent. The machine CIDR for ipv6 is not
configurable and is currently hardcoded to fd00::/48.

A full dual stack configuration would in theory look like the
following (barring the missing machineCIDR configurability).

```
networking:
  clusterNetwork:
  - cidr: 10.128.0.0/14
    hostPrefix: 23
  - cidr: fd01::/48
    hostPrefix: 64
  machineCIDR: 10.0.0.0/16
  networkType: OVNKubernetes
  serviceNetwork:
  - 10.1.0.0/16
  - fd02::/112
```

This commit will be updated to include the ipv6 primary mode (if
all input addresses are ipv6) and to not require the use of the
environment variable.

data/data/azure/bootstrap/main.tf

@@ -1,5 +1,6 @@
 locals {
-  bootstrap_nic_ip_configuration_name = "bootstrap-nic-ip"
+  bootstrap_nic_ip_configuration_name    = "bootstrap-nic-ip"
+  bootstrap_nic_ip_v6_configuration_name = "bootstrap-nic-ip-v6"
 }
 
 data "azurerm_storage_account_sas" "ignition" {
@@ -78,16 +79,44 @@ data "azurerm_public_ip" "bootstrap_public_ip" {
   resource_group_name = var.resource_group_name
 }
 
+resource "azurerm_public_ip" "bootstrap_public_ip_v6" {
+  count = var.private || ! var.use_ipv6 ? 0 : 1
+
+  sku                 = "Standard"
+  location            = var.region
+  name                = "${var.cluster_id}-bootstrap-pip-v6"
+  resource_group_name = var.resource_group_name
+  allocation_method   = "Static"
+  ip_version          = "IPv6"
+}
+
+data "azurerm_public_ip" "bootstrap_public_ip_v6" {
+  count = var.private || ! var.use_ipv6 ? 0 : 1
+
+  name                = azurerm_public_ip.bootstrap_public_ip_v6[0].name
+  resource_group_name = var.resource_group_name
+}
+
 resource "azurerm_network_interface" "bootstrap" {
   name                = "${var.cluster_id}-bootstrap-nic"
   location            = var.region
   resource_group_name = var.resource_group_name
 
-  ip_configuration {
-    subnet_id                     = var.subnet_id
-    name                          = local.bootstrap_nic_ip_configuration_name
-    private_ip_address_allocation = "Dynamic"
-    public_ip_address_id          = var.private ? null : azurerm_public_ip.bootstrap_public_ip[0].id
+  dynamic "ip_configuration" {
+    for_each = var.use_ipv6 ? [
+      { primary : true, name : local.bootstrap_nic_ip_configuration_name, ip_address_version : "IPv4", public_ip_id : var.private ? null : azurerm_public_ip.bootstrap_public_ip[0].id },
+      { primary : false, name : local.bootstrap_nic_ip_v6_configuration_name, ip_address_version : "IPv6", public_ip_id : var.private ? null : azurerm_public_ip.bootstrap_public_ip_v6[0].id },
+      ] : [
+      { primary : true, name : local.bootstrap_nic_ip_configuration_name, ip_address_version : "IPv4", public_ip_id : var.private ? null : azurerm_public_ip.bootstrap_public_ip[0].id }
+    ]
+    content {
+      primary                       = ip_configuration.value.primary
+      name                          = ip_configuration.value.name
+      subnet_id                     = var.subnet_id
+      private_ip_address_version    = ip_configuration.value.ip_address_version
+      private_ip_address_allocation = "Dynamic"
+      public_ip_address_id          = ip_configuration.value.public_ip_id
+    }
   }
 }
 
@@ -97,12 +126,28 @@ resource "azurerm_network_interface_backend_address_pool_association" "public_lb
   ip_configuration_name   = local.bootstrap_nic_ip_configuration_name
 }
 
+resource "azurerm_network_interface_backend_address_pool_association" "public_lb_bootstrap_v6" {
+  count = var.use_ipv6 ? 1 : 0
+
+  network_interface_id    = azurerm_network_interface.bootstrap.id
+  backend_address_pool_id = var.elb_backend_pool_v6_id
+  ip_configuration_name   = local.bootstrap_nic_ip_v6_configuration_name
+}
+
 resource "azurerm_network_interface_backend_address_pool_association" "internal_lb_bootstrap" {
   network_interface_id    = azurerm_network_interface.bootstrap.id
   backend_address_pool_id = var.ilb_backend_pool_id
   ip_configuration_name   = local.bootstrap_nic_ip_configuration_name
 }
 
+resource "azurerm_network_interface_backend_address_pool_association" "internal_lb_bootstrap_v6" {
+  count = var.use_ipv6 ? 1 : 0
+
+  network_interface_id    = azurerm_network_interface.bootstrap.id
+  backend_address_pool_id = var.ilb_backend_pool_v6_id
+  ip_configuration_name   = local.bootstrap_nic_ip_v6_configuration_name
+}
+
 resource "azurerm_virtual_machine" "bootstrap" {
   name                  = "${var.cluster_id}-bootstrap"
   location              = var.region
@@ -151,7 +196,9 @@ resource "azurerm_virtual_machine" "bootstrap" {
 
   depends_on = [
     azurerm_network_interface_backend_address_pool_association.public_lb_bootstrap,
-    azurerm_network_interface_backend_address_pool_association.internal_lb_bootstrap
+    azurerm_network_interface_backend_address_pool_association.public_lb_bootstrap_v6,
+    azurerm_network_interface_backend_address_pool_association.internal_lb_bootstrap,
+    azurerm_network_interface_backend_address_pool_association.internal_lb_bootstrap_v6
   ]
 }
 

data/data/azure/bootstrap/variables.tf

@@ -43,11 +43,21 @@ variable "elb_backend_pool_id" {
   description = "The external load balancer bakend pool id. used to attach the bootstrap NIC"
 }
 
+variable "elb_backend_pool_v6_id" {
+  type        = string
+  description = "The external load balancer bakend pool id for ipv6. used to attach the bootstrap NIC"
+}
+
 variable "ilb_backend_pool_id" {
   type        = string
   description = "The internal load balancer bakend pool id. used to attach the bootstrap NIC"
 }
 
+variable "ilb_backend_pool_v6_id" {
+  type        = string
+  description = "The internal load balancer bakend pool id for ipv6. used to attach the bootstrap NIC"
+}
+
 variable "storage_account" {
   type        = any
   description = "the storage account for the cluster. It can be used for boot diagnostics."
@@ -68,3 +78,8 @@ variable "private" {
   type        = bool
   description = "This value determines if this is a private cluster or not."
 }
+
+variable "use_ipv6" {
+  description = "Use IPv6 as well as IPv4"
+  type        = bool
+}

data/data/azure/dns/dns.tf

@@ -23,6 +23,16 @@ resource "azureprivatedns_a_record" "apiint_internal" {
   records             = [var.internal_lb_ipaddress]
 }
 
+resource "azureprivatedns_aaaa_record" "apiint_internal_v6" {
+  count = var.use_ipv6 ? 1 : 0
+
+  name                = "api-int"
+  zone_name           = azureprivatedns_zone.private.name
+  resource_group_name = var.resource_group_name
+  ttl                 = 300
+  records             = [var.internal_lb_ipaddress_v6]
+}
+
 resource "azureprivatedns_a_record" "api_internal" {
   name                = "api"
   zone_name           = azureprivatedns_zone.private.name
@@ -31,6 +41,16 @@ resource "azureprivatedns_a_record" "api_internal" {
   records             = [var.internal_lb_ipaddress]
 }
 
+resource "azureprivatedns_aaaa_record" "api_internal_v6" {
+  count = var.use_ipv6 ? 1 : 0
+
+  name                = "api"
+  zone_name           = azureprivatedns_zone.private.name
+  resource_group_name = var.resource_group_name
+  ttl                 = 300
+  records             = [var.internal_lb_ipaddress_v6]
+}
+
 resource "azurerm_dns_cname_record" "api_external" {
   count = var.private ? 0 : 1
 
@@ -41,6 +61,16 @@ resource "azurerm_dns_cname_record" "api_external" {
   record              = var.external_lb_fqdn
 }
 
+resource "azurerm_dns_cname_record" "api_external_v6" {
+  count = var.private || ! var.use_ipv6 ? 0 : 1
+
+  name                = "v6-${local.api_external_name}"
+  zone_name           = var.base_domain
+  resource_group_name = var.base_domain_resource_group_name
+  ttl                 = 300
+  record              = var.external_lb_fqdn_v6
+}
+
 resource "azureprivatedns_a_record" "etcd_a_nodes" {
   count               = var.etcd_count
   name                = "etcd-${count.index}"
@@ -50,14 +80,23 @@ resource "azureprivatedns_a_record" "etcd_a_nodes" {
   records             = [var.etcd_ip_addresses[count.index]]
 }
 
+resource "azureprivatedns_aaaa_record" "etcd_aaaa_nodes" {
+  count               = var.use_ipv6 ? var.etcd_count : 0
+  name                = "etcd-${count.index}"
+  zone_name           = azureprivatedns_zone.private.name
+  resource_group_name = var.resource_group_name
+  ttl                 = 60
+  records             = [var.etcd_ip_v6_addresses[count.index]]
+}
+
 resource "azureprivatedns_srv_record" "etcd_cluster" {
   name                = "_etcd-server-ssl._tcp"
   zone_name           = azureprivatedns_zone.private.name
   resource_group_name = var.resource_group_name
   ttl                 = 60
 
   dynamic "record" {
-    for_each = azureprivatedns_a_record.etcd_a_nodes.*.name
+    for_each = var.use_ipv6 ? concat(azureprivatedns_a_record.etcd_a_nodes.*.name, azureprivatedns_aaaa_record.etcd_aaaa_nodes.*.name) : azureprivatedns_a_record.etcd_a_nodes.*.name
     iterator = name
     content {
       target   = "${name.value}.${azureprivatedns_zone.private.name}"

data/data/azure/dns/variables.tf

@@ -29,8 +29,18 @@ variable "external_lb_fqdn" {
   type        = string
 }
 
+variable "external_lb_fqdn_v6" {
+  description = "External API's LB fqdn for IPv6"
+  type        = string
+}
+
 variable "internal_lb_ipaddress" {
-  description = "External API's LB Ip address"
+  description = "External API's LB IP address"
+  type        = string
+}
+
+variable "internal_lb_ipaddress_v6" {
+  description = "External API's LB IP v6 address"
   type        = string
 }
 
@@ -50,6 +60,12 @@ variable "etcd_ip_addresses" {
   default     = []
 }
 
+variable "etcd_ip_v6_addresses" {
+  description = "List of string IPs for machines running etcd members."
+  type        = list(string)
+  default     = []
+}
+
 variable "resource_group_name" {
   type        = string
   description = "Resource group for the deployment"
@@ -59,3 +75,8 @@ variable "private" {
   type        = bool
   description = "This value determines if this is a private cluster or not."
 }
+
+variable "use_ipv6" {
+  description = "Use IPv6 as well as IPv4"
+  type        = bool
+}

data/data/azure/main.tf

@@ -22,27 +22,32 @@ provider "azureprivatedns" {
 }
 
 module "bootstrap" {
-  source              = "./bootstrap"
-  resource_group_name = azurerm_resource_group.main.name
-  region              = var.azure_region
-  vm_size             = var.azure_bootstrap_vm_type
-  vm_image            = azurerm_image.cluster.id
-  identity            = azurerm_user_assigned_identity.main.id
-  cluster_id          = var.cluster_id
-  ignition            = var.ignition_bootstrap
-  subnet_id           = module.vnet.master_subnet_id
-  elb_backend_pool_id = module.vnet.public_lb_backend_pool_id
-  ilb_backend_pool_id = module.vnet.internal_lb_backend_pool_id
-  tags                = local.tags
-  storage_account     = azurerm_storage_account.cluster
-  nsg_name            = module.vnet.master_nsg_name
-  private             = module.vnet.private
+  source                 = "./bootstrap"
+  resource_group_name    = azurerm_resource_group.main.name
+  region                 = var.azure_region
+  vm_size                = var.azure_bootstrap_vm_type
+  vm_image               = azurerm_image.cluster.id
+  identity               = azurerm_user_assigned_identity.main.id
+  cluster_id             = var.cluster_id
+  ignition               = var.ignition_bootstrap
+  subnet_id              = module.vnet.master_subnet_id
+  elb_backend_pool_id    = module.vnet.public_lb_backend_pool_id
+  elb_backend_pool_v6_id = module.vnet.public_lb_backend_pool_v6_id
+  ilb_backend_pool_id    = module.vnet.internal_lb_backend_pool_id
+  ilb_backend_pool_v6_id = module.vnet.internal_lb_backend_pool_v6_id
+  tags                   = local.tags
+  storage_account        = azurerm_storage_account.cluster
+  nsg_name               = module.vnet.master_nsg_name
+  private                = module.vnet.private
+
+  use_ipv6 = var.azure_use_ipv6
 }
 
 module "vnet" {
   source              = "./vnet"
   resource_group_name = azurerm_resource_group.main.name
   vnet_cidr           = var.machine_cidr
+  vnet_cidr_v6        = var.azure_machine_cidr_v6
   cluster_id          = var.cluster_id
   region              = var.azure_region
   dns_label           = var.cluster_id
@@ -53,27 +58,33 @@ module "vnet" {
   master_subnet               = var.azure_control_plane_subnet
   worker_subnet               = var.azure_compute_subnet
   private                     = var.azure_private
+
+  use_ipv6 = var.azure_use_ipv6
 }
 
 module "master" {
-  source              = "./master"
-  resource_group_name = azurerm_resource_group.main.name
-  cluster_id          = var.cluster_id
-  region              = var.azure_region
-  availability_zones  = var.azure_master_availability_zones
-  vm_size             = var.azure_master_vm_type
-  vm_image            = azurerm_image.cluster.id
-  identity            = azurerm_user_assigned_identity.main.id
-  ignition            = var.ignition_master
-  external_lb_id      = module.vnet.public_lb_id
-  elb_backend_pool_id = module.vnet.public_lb_backend_pool_id
-  ilb_backend_pool_id = module.vnet.internal_lb_backend_pool_id
-  subnet_id           = module.vnet.master_subnet_id
-  instance_count      = var.master_count
-  storage_account     = azurerm_storage_account.cluster
-  os_volume_type      = var.azure_master_root_volume_type
-  os_volume_size      = var.azure_master_root_volume_size
-  private             = module.vnet.private
+  source                 = "./master"
+  resource_group_name    = azurerm_resource_group.main.name
+  cluster_id             = var.cluster_id
+  region                 = var.azure_region
+  availability_zones     = var.azure_master_availability_zones
+  vm_size                = var.azure_master_vm_type
+  vm_image               = azurerm_image.cluster.id
+  identity               = azurerm_user_assigned_identity.main.id
+  ignition               = var.ignition_master
+  external_lb_id         = module.vnet.public_lb_id
+  elb_backend_pool_id    = module.vnet.public_lb_backend_pool_id
+  elb_backend_pool_v6_id = module.vnet.public_lb_backend_pool_v6_id
+  ilb_backend_pool_id    = module.vnet.internal_lb_backend_pool_id
+  ilb_backend_pool_v6_id = module.vnet.internal_lb_backend_pool_v6_id
+  subnet_id              = module.vnet.master_subnet_id
+  instance_count         = var.master_count
+  storage_account        = azurerm_storage_account.cluster
+  os_volume_type         = var.azure_master_root_volume_type
+  os_volume_size         = var.azure_master_root_volume_size
+  private                = module.vnet.private
+
+  use_ipv6 = var.azure_use_ipv6
 }
 
 module "dns" {
@@ -83,12 +94,17 @@ module "dns" {
   base_domain                     = var.base_domain
   virtual_network_id              = module.vnet.virtual_network_id
   external_lb_fqdn                = module.vnet.public_lb_pip_fqdn
+  external_lb_fqdn_v6             = module.vnet.public_lb_pip_v6_fqdn
   internal_lb_ipaddress           = module.vnet.internal_lb_ip_address
+  internal_lb_ipaddress_v6        = module.vnet.internal_lb_ip_v6_address
   resource_group_name             = azurerm_resource_group.main.name
   base_domain_resource_group_name = var.azure_base_domain_resource_group_name
   etcd_count                      = var.master_count
   etcd_ip_addresses               = module.master.ip_addresses
+  etcd_ip_v6_addresses            = module.master.ip_v6_addresses
   private                         = module.vnet.private
+
+  use_ipv6 = var.azure_use_ipv6
 }
 
 resource "random_string" "storage_suffix" {