Merge branch 'fix-1-30-changes' into 'main'

feat: Kubernetes 1.30 compatibility

See merge request PeeK1e/kubeadm-upgrade!1

README.md

@@ -1,26 +1,38 @@
-# Kubeadm Upgrade
+# ✨ Kubeadm Upgrade ✨
 
-⚠ **Kubernetes 1.30 is not yet supported, since there are some changes in the kubeadm upgrade steps** ⚠
+> **⚠️ Always test your upgrades on a test cluster first! ⚠️**
 
-You can upgrade your kubeadm, cri-o, Debian 11/12 based kubernetes cluster with this ansible script
+---
 
-<img src="./res/me_upgrading_the_cluster.png" alt="people praying to a server rack" width="200"/>
+Upgrade your kubeadm, cri-o, Debian 11/12 based Kubernetes cluster with this Ansible playbook.
 
-## Getting Started
+I tested it with version 1.28 through 1.30.
+It should work with versions less than 1.28, depending on the mirror availability and packages on the mirrors.
 
-### Dependencies
+## 🚀 Getting Started
 
-* ansible
-* ansible-playbook
-* python3
+### 📋 Dependencies
 
-### Settings
+- `ansible`
+- `ansible-playbook`
+- `python3`
 
-Copy the `inventory.testing` file and fill in your nodes. **Make sure to use the host name, the nodes have in the cluster, since this script uses these to drain the nodes via kubectl**
+### ⚙️ Settings
+> **⚠️ Use the exact host names as in your cluster since the script uses these to drain the nodes via kubectl ⚠️**
 
-In the `group_vars` check the global options. It's best to run the playbook once for your current version, to make sure that you're on the latest patch version.
+1. Copy `inventory.testing` and fill in your nodes.
+2. In `group_vars`, check the global options. Run the playbook once for **your current version** to ensure you're on the latest patch version.
 
-### Usage
+### ▶️ Usage
 
-Run the playbook with `ansible-playbook -i inventory --private-key=~/.ssh/ssh-key upgrade_cluster.yaml`. Grab a cup of coffee and wait.
+Run the playbook with:
 
+```bash
+ansible-playbook -i inventory --private-key=~/.ssh/ssh-key upgrade_cluster.yaml
+```
+
+There will be a prompt, where you're asked to confirm the plan. When you're satisfied with the planned output, press `ctrl+c` and `c` again to continue or `a` to abort.
+
+All that's left is to grab a cup of coffee, wait, and pray.
+
+<img src="./res/me_upgrading_the_cluster.png" alt="multiple people praying to a server rack" width="200"/>

group_vars/all/global.yaml

@@ -1,8 +1,10 @@
 ---
+# running debian version. supported values: Debian_11, Debian_12
+debianVersion: "Debian_12"
 # targeted kubernetes version in major.minor
-kubeVersion: "1.29"
+kubeVersion: "1.30"
 # targeted crio version in major.minor
-crioVersion: "1.28"
+crioVersion: "1.30"
 # time until a pod is killed
 drainGracePeriod: 30
 # time until the drain command times out
@@ -13,7 +15,15 @@ pauseDelayUncordon: 30
 # repo list to remove in /etc/apt/sources.list.d/
 repoList:
   - "repo_vanillastack_cloudical_net.list"
-  - "devel:kubic:libcontainers:stable:cri-o:1.28.list"
   - "devel:kubic:libcontainers:stable:cri-o:1.27.list"
+  - "devel:kubic:libcontainers:stable:cri-o:1.28.list"
+  - "devel:kubic:libcontainers:stable:cri-o:1.29.list"
+  - "devel:kubic:libcontainers:stable:cri-o:1.30.list"
   - "devel:kubic:libcontainers:stable.list"
+  - "kubernetes-stable.list"
   - "kubernetes.list"
+  - "kubic.list"
+  - "crio.list"
+  - "cri-o:stable:v1.28.list"
+  - "cri-o:stable:v1.29.list"
+  - "cri-o:stable:v1.30.list"

roles/common/tasks/get-versions.yaml

@@ -5,4 +5,4 @@
 
 - name: set kube version
   ansible.builtin.set_fact:
-    kube_version: "{{ kubelet_version_out.stdout | default('0.0.0-0.0')}}"
+    kube_version: "{{ kubelet_version_out.stdout | default('0.0.0-0.0') }}"

roles/common/tasks/upgrade-kube-packages.yaml

@@ -3,45 +3,23 @@
   ansible.builtin.apt:
     name: "*"
     state: latest
+    update_cache: true
 
 - name: get variables
   ansible.builtin.include_tasks: "get-versions.yaml"
 
-- name: unhold packages
-  block:
-    - name: kubeadm
-      ansible.builtin.dpkg_selections:
-        name: kubeadm
-        selection: install #equals unhold
-
-    - name: kubelet
-      ansible.builtin.dpkg_selections:
-        name: kubelet
-        selection: install #equals unhold
-
-    - name: kubectl
-      ansible.builtin.dpkg_selections:
-        name: kubectl
-        selection: install #equals unhold
-
-    - name: cri-o-runc
-      ansible.builtin.dpkg_selections:
-        name: cri-o-runc
-        selection: install #equals unhold
-
-    - name: cri-o
-      ansible.builtin.dpkg_selections:
-        name: cri-o
-        selection: install #equals unhold
-
 - name: upgrade and install
   ansible.builtin.apt:  
     pkg:
+    - jq
     - cri-o-runc
     - cri-o
     - kubelet={{ kube_version }}
     - kubeadm={{ kube_version }}
     - kubectl={{ kube_version }}
+    allow_change_held_packages: true
+    update_cache: true
+    state: latest
 
 - name: hold packages
   block:
@@ -69,3 +47,7 @@
       ansible.builtin.dpkg_selections:
         name: cri-o
         selection: hold
+
+- name: Remove dependencies that are no longer required
+  ansible.builtin.apt:
+    autoremove: yes

roles/common/tasks/upgrade-node.yaml

@@ -55,4 +55,4 @@
 - name: wait uncordon
   pause:
     prompt: Pausing after uncordon. (press ctrl+c and a to continue immediately) 
-    seconds: "{{ pauseDelayUncordon }}"
+    seconds: "{{ pauseDelayUncordon }}"

roles/common/tasks/upgrade-repo.yaml

@@ -16,24 +16,37 @@
         state: present
         filename: kubernetes
 
-- name: add crio repository
+- name: add crio repository (crio < 1.28)
   block:
     - name: crio | apt key
-      ansible.builtin.shell: "curl -fsSL https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ crioVersion }}/Debian_11/Release.key | gpg --yes --dearmor -o /usr/share/keyrings/libcontainers-crio-archive-keyring.gpg"
+      ansible.builtin.shell: "curl -fsSL https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ crioVersion }}/{{ debianVersion }}/Release.key | gpg --yes --dearmor -o /usr/share/keyrings/libcontainers-crio-archive-keyring.gpg"
 
     - name: crio | apt source
       ansible.builtin.apt_repository:
-        repo: "deb [signed-by=/usr/share/keyrings/libcontainers-crio-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ crioVersion }}/Debian_11/ /"
+        repo: "deb [signed-by=/usr/share/keyrings/libcontainers-crio-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ crioVersion }}/{{ debianVersion }}/ /"
         state: present
         filename: "devel:kubic:libcontainers:stable:cri-o:{{ crioVersion }}"
+  when: "crioVersion is version('1.28', '<', strict=true)"
+
+- name: add crio repository (crio >= 1.28)
+  block:
+    - name: crio | apt key
+      ansible.builtin.shell: "curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/stable:/v{{ crioVersion }}/deb/Release.key | gpg --yes --dearmor -o /usr/share/keyrings/libcontainers-crio-archive-keyring.gpg"
+
+    - name: crio | apt source
+      ansible.builtin.apt_repository:
+        repo: "deb [signed-by=/usr/share/keyrings/libcontainers-crio-archive-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/v{{ crioVersion }}/deb/ /"
+        state: present
+        filename: "cri-o:stable:v{{ crioVersion }}"
+  when: "crioVersion is version('1.28', '>=', strict=true)"
 
 - name: add libcontainer repository
   block:
     - name: libcontainer | apt key
-      ansible.builtin.shell: "curl -fsSL https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_11/Release.key | gpg --yes --dearmor -o /usr/share/keyrings/libcontainers-archive-keyring.gpg"
+      ansible.builtin.shell: "curl -fsSL https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{{ debianVersion }}/Release.key | gpg --yes --dearmor -o /usr/share/keyrings/libcontainers-archive-keyring.gpg"
 
     - name: libcontainer | apt source
       ansible.builtin.apt_repository:
-        repo: "deb [signed-by=/usr/share/keyrings/libcontainers-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_11/ /"
+        repo: "deb [signed-by=/usr/share/keyrings/libcontainers-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{{ debianVersion }}/ /"
         state: present
         filename: devel:kubic:libcontainers:stable

roles/upgrade-cp/tasks/controlplane-plan.yaml

@@ -12,37 +12,48 @@
         - kubeadm={{ kube_version }}
         update_cache: true
 
-    - name: unhold 
+    - name: hold 
       ansible.builtin.dpkg_selections:
         name: kubeadm
         selection: hold
-
-- name: kubadm plan
-  ansible.builtin.shell: kubeadm upgrade plan -o json | grep -iv "^\[[a-zA-Z]*].*$" | jq '.components[] | select(.name=="kubelet") | .newVersion' | tr -d '"'
-  register: new_version_out 
+
+- name: get kubeadm version
+  ansible.builtin.shell: kubeadm version -o json | jq -r '.clientVersion.gitVersion'
+  register: kubeadm_version_out
+  when: hostvars['K8S_TOKEN_HOLDER']['plan_version'] is not defined
+
+- name: set plan version
+  ansible.builtin.set_fact:
+    plan_version: "{{ kubeadm_version_out.stdout }}"
   when: hostvars['K8S_TOKEN_HOLDER']['plan_version'] is not defined
 
+- name: kubadm plan
+  ansible.builtin.shell: "kubeadm upgrade plan {{ plan_version }}"
+  register: kubeadm_plan_output
+  when: hostvars['K8S_TOKEN_HOLDER']['plan_version'] is not defined
+
 - name: output plan
   ansible.builtin.debug:
-    msg: "plan stdout {{ new_version_out.stdout }}, stderr: {{ new_version_out.stderr }}"
+    msg: 
+    - "plan stdout:"
+    - "{{ kubeadm_plan_output.stdout_lines | default('N/A') }}"
+    - "----------------------------------------------------"
+    - "stderr:"
+    - "{{ kubeadm_plan_output.stderr_lines | default('N/A') }}"
+  ignore_errors: true
   when: hostvars['K8S_TOKEN_HOLDER']['plan_version'] is not defined
 
-- name: set fact
-  ansible.builtin.set_fact:
-    new_version: "{{ new_version_out.stdout | default('v0.0.0', true) }}"
+- name: check config
+  ansible.builtin.pause:
+    prompt:
+    - "PLEASE READ THROUGH THE LOG OUTPUT ABOVE"
+    - "if you are fine with the output of 'kubeadm upgrade plan' you may continue"
+    - "this message only appears on the initial upgrade process"
+    minutes: 15
   when: hostvars['K8S_TOKEN_HOLDER']['plan_version'] is not defined
 
-- name: register plan version to dummy host
+- name: register planned version to dummy host
   add_host:
     name: "K8S_TOKEN_HOLDER"
-    plan_version: "{{ new_version }}"
+    plan_version: "{{ plan_version }}"
   when: hostvars['K8S_TOKEN_HOLDER']['plan_version'] is not defined
-
-- name: make sure installed version equals planned version
-  ansible.builtin.debug:
-    msg: "planned version: {{ hostvars['K8S_TOKEN_HOLDER']['plan_version'] }} and apt package version: {{ kube_version }}" 
-
-- name: make sure installed version equals planned version
-  ansible.builtin.fail:
-    msg: versions not equal or you are on the latest patch of {{ kubeVersion }}
-  when: hostvars['K8S_TOKEN_HOLDER']['plan_version'][1:] is version(kube_version.split('-')[0], 'ne')

roles/upgrade-cp/tasks/upgrade-other-cp.yaml

@@ -7,4 +7,4 @@
   ansible.builtin.include_tasks: "controlplane-plan.yaml"
 
 - name: upgrade node
-  ansible.builtin.shell: "kubeadm upgrade node"
+  ansible.builtin.shell: "kubeadm upgrade node"

roles/upgrade-node/tasks/main.yaml

@@ -9,4 +9,4 @@
 - name: include common
   ansible.legacy.include_role:
     name: common
-    tasks_from: upgrade-node.yaml
+    tasks_from: upgrade-node.yaml

tofu_infra/README.md

@@ -15,5 +15,6 @@ We use `cri-o`, `kubeadm` and `cilium` to setup the cluster.
 If you encounter any issue with the cilium install, just run the script again and it should work.
 
 ## Output
-You will receive an `inventory`,`kubeconfig` and `ssh-key` files in the `local/` directory which you can use to test the cluster.
-You may have to copy the files to your `~/.ssh` directory in order to connect via ssh.
+You will receive `inventory`,`kubeconfig` and `ssh-key` files in the `local/` directory which you can use to test the cluster.
+
+If you are using an SSH agent, add the generated key to the keyring with `ssh-add local/tofu-ssh` and you should be good to go.

tofu_infra/control-plane.tf

@@ -4,7 +4,7 @@ resource "hcloud_server" "control-planes" {
   image       = "debian-12"
   server_type = each.value.type
   public_net {
-    ipv6_enabled = false
+    ipv6_enabled = true
   }
   ssh_keys = [
     hcloud_ssh_key.tofu-key.name
@@ -46,11 +46,7 @@ apt:
       filename: kubic.list
 # install packages
 package_update: true
-packages:
-    - cri-o-runc
-    - cri-o
 runcmd:
-  - apt install -y kubeadm=${var.kube_version}.* kubectl=${var.kube_version}.* kubelet=${var.kube_version}.*
   - |
     cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
     overlay
@@ -62,10 +58,9 @@ runcmd:
     net.bridge.bridge-nf-call-ip6tables = 1
     net.ipv4.ip_forward                 = 1
     EOF
-  - sudo modprobe overlay
-  - sudo modprobe br_netfilter
-  - sudo sysctl --system
-  - sudo systemctl enable --now crio
+  - sudo modprobe overlay 
+  - sudo modprobe br_netfilter 
+  - sysctl -p
   - sudo rm -rf "/etc/cni/net.d/*"
 EOT
 }

tofu_infra/loadbalancer.tf

@@ -1,7 +1,18 @@
+resource "random_id" "lb" {
+  keepers = {
+    # Generate a new id each time we switch to a new AMI id
+    ami_id = hcloud_network.k8s.id
+  }
+
+  byte_length = 8
+}
+
 resource "hcloud_load_balancer" "k8s" {
-  name               = "k8s-lb"
+  name               = "k8s-lb-${random_id.lb.hex}"
   load_balancer_type = "lb11"
-  location           = "nbg1"
+  location           = "fsn1"
+
+  depends_on = [ hcloud_server.control-planes ]
 }
 
 resource "hcloud_load_balancer_network" "k8s" {

tofu_infra/ssh.tf

@@ -3,6 +3,26 @@ resource "hcloud_ssh_key" "tofu-key" {
   public_key = tls_private_key.ssh-key.public_key_openssh
 }
 
+resource "ssh_resource" "add_kube_packages" {
+  for_each = merge(hcloud_server.worker, hcloud_server.control-planes)
+
+  host = each.value.ipv4_address
+
+  user = "root"
+  when = "create"
+  agent = false
+
+  private_key = tls_private_key.ssh-key.private_key_openssh
+
+  commands = [
+    "apt -o Acquire::ForceIPv6=true update",
+    "apt -o Acquire::ForceIPv6=true install -y cri-o-runc cri-o",
+    "apt -o Acquire::ForceIPv6=true install -y kubeadm=${var.kube_version}.* kubectl=${var.kube_version}.* kubelet=${var.kube_version}.*",
+    "sudo sysctl --system",
+    "sudo systemctl enable --now crio"
+  ]
+}
+
 resource "ssh_resource" "bootstrap_first_cp" {
   host = hcloud_server.control-planes[keys(var.control-plane)[0]].ipv4_address
   # host = hcloud_server_network.cp[keys(var.control-plane)[0]].ip
@@ -26,7 +46,8 @@ resource "ssh_resource" "bootstrap_first_cp" {
     hcloud_load_balancer.k8s,
     hcloud_server.control-planes,
     hcloud_load_balancer_target.cp,
-    hcloud_load_balancer_network.k8s
+    hcloud_load_balancer_network.k8s,
+    ssh_resource.add_kube_packages
   ]
 }