Add experimental Fedora CoreOS arm64 support on AWS

* Add experimental `arch` variable to Fedora CoreOS AWS, accepting amd64 (default) or arm64 to support native arm64/aarch64 clusters or mixed/hybrid clusters with a worker pool of arm64 workers * Use experimental Poseidon-built aarch64 Fedora CoreOS AMIs published to us-east-1, us-east-2, and us-west-1 * WARN: Our AMIs are experimental, may be removed at any time, and will be removed when Fedora CoreOS publishes official arm64 AMIs. Do NOT use in production * Requires use of compatible CNI providers (currently, flannel)
poseidon · Nov 13, 2020 · a302daa · a302daa
1 parent 1113a22
commit a302daa
Show file tree

Hide file tree

Showing 8 changed files with 79 additions and 5 deletions.
diff --git a/aws/fedora-coreos/kubernetes/ami.tf b/aws/fedora-coreos/kubernetes/ami.tf
@@ -18,3 +18,27 @@ data "aws_ami" "fedora-coreos" {
     values = ["Fedora CoreOS ${var.os_stream} *"]
   }
 }
+
+# Experimental Fedora CoreOS arm64 / aarch64 AMIs from Poseidon
+# WARNING: These AMIs will be removed when Fedora CoreOS publishes arm64 AMIs
+# and may be removed for any reason before then as well. Do not use.
+data "aws_ami" "fedora-coreos-arm" {
+  most_recent = true
+  owners      = ["099663496933"]
+
+  filter {
+    name   = "architecture"
+    values = ["arm64"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "name"
+    values = ["fedora-coreos-*"]
+  }
+}
+
diff --git a/aws/fedora-coreos/kubernetes/controllers.tf b/aws/fedora-coreos/kubernetes/controllers.tf
@@ -22,9 +22,8 @@ resource "aws_instance" "controllers" {
   }
 
   instance_type = var.controller_type
-
-  ami       = data.aws_ami.fedora-coreos.image_id
-  user_data = data.ct_config.controller-ignitions.*.rendered[count.index]
+  ami           = var.arch == "arm64" ? data.aws_ami.fedora-coreos-arm.image_id : data.aws_ami.fedora-coreos.image_id
+  user_data     = data.ct_config.controller-ignitions.*.rendered[count.index]
 
   # storage
   root_block_device {
@@ -63,6 +62,7 @@ data "template_file" "controller-configs" {
 
   vars = {
     # Cannot use cyclic dependencies on controllers or their DNS records
+    etcd_arch   = var.arch == "arm64" ? "-arm64" : ""
     etcd_name   = "etcd${count.index}"
     etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
     # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,...

diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml
@@ -12,7 +12,7 @@ systemd:
         Wants=network-online.target network.target
         After=network-online.target
         [Service]
-        Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.4.12
+        Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.4.12${etcd_arch}
         Type=exec
         ExecStartPre=/bin/mkdir -p /var/lib/etcd
         ExecStartPre=-/usr/bin/podman rm etcd
@@ -214,6 +214,7 @@ storage:
           ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt
           ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key
           ETCD_PEER_CLIENT_CERT_AUTH=true
+          ETCD_UNSUPPORTED_ARCH=arm64
 passwd:
   users:
     - name: core

diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf
@@ -155,3 +155,14 @@ variable "cluster_domain_suffix" {
   default     = "cluster.local"
 }
 
+variable "arch" {
+  type        = string
+  description = "Container architecture (amd64 or arm64)"
+  default     = "amd64"
+
+  validation {
+    condition     = var.arch == "amd64" || var.arch == "arm64"
+    error_message = "The host arch must be amd64 or arm64."
+  }
+}
+
diff --git a/aws/fedora-coreos/kubernetes/workers.tf b/aws/fedora-coreos/kubernetes/workers.tf
@@ -9,6 +9,7 @@ module "workers" {
   worker_count    = var.worker_count
   instance_type   = var.worker_type
   os_stream       = var.os_stream
+  arch            = var.arch
   disk_size       = var.disk_size
   spot_price      = var.worker_price
   target_groups   = var.worker_target_groups

diff --git a/aws/fedora-coreos/kubernetes/workers/ami.tf b/aws/fedora-coreos/kubernetes/workers/ami.tf
@@ -18,3 +18,27 @@ data "aws_ami" "fedora-coreos" {
     values = ["Fedora CoreOS ${var.os_stream} *"]
   }
 }
+
+# Experimental Fedora CoreOS arm64 / aarch64 AMIs from Poseidon
+# WARNING: These AMIs will be removed when Fedora CoreOS publishes arm64 AMIs
+# and may be removed for any reason before then as well. Do not use.
+data "aws_ami" "fedora-coreos-arm" {
+  most_recent = true
+  owners      = ["099663496933"]
+
+  filter {
+    name   = "architecture"
+    values = ["arm64"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "name"
+    values = ["fedora-coreos-*"]
+  }
+}
+
diff --git a/aws/fedora-coreos/kubernetes/workers/variables.tf b/aws/fedora-coreos/kubernetes/workers/variables.tf
@@ -108,3 +108,16 @@ variable "node_labels" {
   description = "List of initial node labels"
   default     = []
 }
+
+# unofficial, undocumented, unsupported
+
+variable "arch" {
+  type        = string
+  description = "Container architecture (amd64 or arm64)"
+  default     = "amd64"
+
+  validation {
+    condition     = var.arch == "amd64" || var.arch == "arm64"
+    error_message = "The host arch must be amd64 or arm64."
+  }
+}
diff --git a/aws/fedora-coreos/kubernetes/workers/workers.tf b/aws/fedora-coreos/kubernetes/workers/workers.tf
@@ -44,7 +44,7 @@ resource "aws_autoscaling_group" "workers" {
 
 # Worker template
 resource "aws_launch_configuration" "worker" {
-  image_id          = data.aws_ami.fedora-coreos.image_id
+  image_id          = var.arch == "arm64" ? data.aws_ami.fedora-coreos-arm.image_id : data.aws_ami.fedora-coreos.image_id
   instance_type     = var.instance_type
   spot_price        = var.spot_price > 0 ? var.spot_price : null
   enable_monitoring = false