From a302daa66f199505aea4961c71733dd86c70d8bf Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 8 Nov 2020 10:51:42 -0800 Subject: [PATCH] Add experimental Fedora CoreOS arm64 support on AWS * Add experimental `arch` variable to Fedora CoreOS AWS, accepting amd64 (default) or arm64 to support native arm64/aarch64 clusters or mixed/hybrid clusters with a worker pool of arm64 workers * Use experimental Poseidon-built aarch64 Fedora CoreOS AMIs published to us-east-1, us-east-2, and us-west-1 * WARN: Our AMIs are experimental, may be removed at any time, and will be removed when Fedora CoreOS publishes official arm64 AMIs. Do NOT use in production * Requires use of compatible CNI providers (currently, flannel) --- aws/fedora-coreos/kubernetes/ami.tf | 24 +++++++++++++++++++ aws/fedora-coreos/kubernetes/controllers.tf | 6 ++--- .../kubernetes/fcc/controller.yaml | 3 ++- aws/fedora-coreos/kubernetes/variables.tf | 11 +++++++++ aws/fedora-coreos/kubernetes/workers.tf | 1 + aws/fedora-coreos/kubernetes/workers/ami.tf | 24 +++++++++++++++++++ .../kubernetes/workers/variables.tf | 13 ++++++++++ .../kubernetes/workers/workers.tf | 2 +- 8 files changed, 79 insertions(+), 5 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/ami.tf b/aws/fedora-coreos/kubernetes/ami.tf index a7ab184bd..2ac01d446 100644 --- a/aws/fedora-coreos/kubernetes/ami.tf +++ b/aws/fedora-coreos/kubernetes/ami.tf @@ -18,3 +18,27 @@ data "aws_ami" "fedora-coreos" { values = ["Fedora CoreOS ${var.os_stream} *"] } } + +# Experimental Fedora CoreOS arm64 / aarch64 AMIs from Poseidon +# WARNING: These AMIs will be removed when Fedora CoreOS publishes arm64 AMIs +# and may be removed for any reason before then as well. Do not use. +data "aws_ami" "fedora-coreos-arm" { + most_recent = true + owners = ["099663496933"] + + filter { + name = "architecture" + values = ["arm64"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "name" + values = ["fedora-coreos-*"] + } +} + diff --git a/aws/fedora-coreos/kubernetes/controllers.tf b/aws/fedora-coreos/kubernetes/controllers.tf index 2fd253db7..1ab26aaa6 100644 --- a/aws/fedora-coreos/kubernetes/controllers.tf +++ b/aws/fedora-coreos/kubernetes/controllers.tf @@ -22,9 +22,8 @@ resource "aws_instance" "controllers" { } instance_type = var.controller_type - - ami = data.aws_ami.fedora-coreos.image_id - user_data = data.ct_config.controller-ignitions.*.rendered[count.index] + ami = var.arch == "arm64" ? data.aws_ami.fedora-coreos-arm.image_id : data.aws_ami.fedora-coreos.image_id + user_data = data.ct_config.controller-ignitions.*.rendered[count.index] # storage root_block_device { @@ -63,6 +62,7 @@ data "template_file" "controller-configs" { vars = { # Cannot use cyclic dependencies on controllers or their DNS records + etcd_arch = var.arch == "arm64" ? "-arm64" : "" etcd_name = "etcd${count.index}" etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index eaa912ded..4147b14d8 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -12,7 +12,7 @@ systemd: Wants=network-online.target network.target After=network-online.target [Service] - Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.4.12 + Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.4.12${etcd_arch} Type=exec ExecStartPre=/bin/mkdir -p /var/lib/etcd ExecStartPre=-/usr/bin/podman rm etcd @@ -214,6 +214,7 @@ storage: ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key ETCD_PEER_CLIENT_CERT_AUTH=true + ETCD_UNSUPPORTED_ARCH=arm64 passwd: users: - name: core diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf index fa47cbe5a..c3c5ba6a8 100644 --- a/aws/fedora-coreos/kubernetes/variables.tf +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -155,3 +155,14 @@ variable "cluster_domain_suffix" { default = "cluster.local" } +variable "arch" { + type = string + description = "Container architecture (amd64 or arm64)" + default = "amd64" + + validation { + condition = var.arch == "amd64" || var.arch == "arm64" + error_message = "The host arch must be amd64 or arm64." + } +} + diff --git a/aws/fedora-coreos/kubernetes/workers.tf b/aws/fedora-coreos/kubernetes/workers.tf index dcfc05d9a..0ec9cdb6b 100644 --- a/aws/fedora-coreos/kubernetes/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers.tf @@ -9,6 +9,7 @@ module "workers" { worker_count = var.worker_count instance_type = var.worker_type os_stream = var.os_stream + arch = var.arch disk_size = var.disk_size spot_price = var.worker_price target_groups = var.worker_target_groups diff --git a/aws/fedora-coreos/kubernetes/workers/ami.tf b/aws/fedora-coreos/kubernetes/workers/ami.tf index a7ab184bd..2ac01d446 100644 --- a/aws/fedora-coreos/kubernetes/workers/ami.tf +++ b/aws/fedora-coreos/kubernetes/workers/ami.tf @@ -18,3 +18,27 @@ data "aws_ami" "fedora-coreos" { values = ["Fedora CoreOS ${var.os_stream} *"] } } + +# Experimental Fedora CoreOS arm64 / aarch64 AMIs from Poseidon +# WARNING: These AMIs will be removed when Fedora CoreOS publishes arm64 AMIs +# and may be removed for any reason before then as well. Do not use. +data "aws_ami" "fedora-coreos-arm" { + most_recent = true + owners = ["099663496933"] + + filter { + name = "architecture" + values = ["arm64"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "name" + values = ["fedora-coreos-*"] + } +} + diff --git a/aws/fedora-coreos/kubernetes/workers/variables.tf b/aws/fedora-coreos/kubernetes/workers/variables.tf index 76b33cbf9..6e0030abf 100644 --- a/aws/fedora-coreos/kubernetes/workers/variables.tf +++ b/aws/fedora-coreos/kubernetes/workers/variables.tf @@ -108,3 +108,16 @@ variable "node_labels" { description = "List of initial node labels" default = [] } + +# unofficial, undocumented, unsupported + +variable "arch" { + type = string + description = "Container architecture (amd64 or arm64)" + default = "amd64" + + validation { + condition = var.arch == "amd64" || var.arch == "arm64" + error_message = "The host arch must be amd64 or arm64." + } +} diff --git a/aws/fedora-coreos/kubernetes/workers/workers.tf b/aws/fedora-coreos/kubernetes/workers/workers.tf index 39f9a4a4e..7609c6e8c 100644 --- a/aws/fedora-coreos/kubernetes/workers/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers/workers.tf @@ -44,7 +44,7 @@ resource "aws_autoscaling_group" "workers" { # Worker template resource "aws_launch_configuration" "worker" { - image_id = data.aws_ami.fedora-coreos.image_id + image_id = var.arch == "arm64" ? data.aws_ami.fedora-coreos-arm.image_id : data.aws_ami.fedora-coreos.image_id instance_type = var.instance_type spot_price = var.spot_price > 0 ? var.spot_price : null enable_monitoring = false