diff --git a/README.md b/README.md index 0bda3b7..574992b 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,38 @@ -# Kubeadm Upgrade +# ✨ Kubeadm Upgrade ✨ -⚠ **Kubernetes 1.30 is not yet supported, since there are some changes in the kubeadm upgrade steps** ⚠ +> **⚠️ Always test your upgrades on a test cluster first! ⚠️** -You can upgrade your kubeadm, cri-o, Debian 11/12 based kubernetes cluster with this ansible script +--- -people praying to a server rack +Upgrade your kubeadm, cri-o, Debian 11/12 based Kubernetes cluster with this Ansible playbook. -## Getting Started +I tested it with version 1.28 through 1.30. +It should work with versions less than 1.28, depending on the mirror availability and packages on the mirrors. -### Dependencies +## 🚀 Getting Started -* ansible -* ansible-playbook -* python3 +### 📋 Dependencies -### Settings +- `ansible` +- `ansible-playbook` +- `python3` -Copy the `inventory.testing` file and fill in your nodes. **Make sure to use the host name, the nodes have in the cluster, since this script uses these to drain the nodes via kubectl** +### ⚙️ Settings +> **⚠️ Use the exact host names as in your cluster since the script uses these to drain the nodes via kubectl ⚠️** -In the `group_vars` check the global options. It's best to run the playbook once for your current version, to make sure that you're on the latest patch version. +1. Copy `inventory.testing` and fill in your nodes. +2. In `group_vars`, check the global options. Run the playbook once for **your current version** to ensure you're on the latest patch version. -### Usage +### ▶️ Usage -Run the playbook with `ansible-playbook -i inventory --private-key=~/.ssh/ssh-key upgrade_cluster.yaml`. Grab a cup of coffee and wait. +Run the playbook with: +```bash +ansible-playbook -i inventory --private-key=~/.ssh/ssh-key upgrade_cluster.yaml +``` + +There will be a prompt, where you're asked to confirm the plan. When you're satisfied with the planned output, press `ctrl+c` and `c` again to continue or `a` to abort. + +All that's left is to grab a cup of coffee, wait, and pray. + +multiple people praying to a server rack \ No newline at end of file diff --git a/group_vars/all/global.yaml b/group_vars/all/global.yaml index c15c908..6be9085 100644 --- a/group_vars/all/global.yaml +++ b/group_vars/all/global.yaml @@ -1,8 +1,10 @@ --- +# running debian version. supported values: Debian_11, Debian_12 +debianVersion: "Debian_12" # targeted kubernetes version in major.minor -kubeVersion: "1.29" +kubeVersion: "1.30" # targeted crio version in major.minor -crioVersion: "1.28" +crioVersion: "1.30" # time until a pod is killed drainGracePeriod: 30 # time until the drain command times out @@ -13,7 +15,15 @@ pauseDelayUncordon: 30 # repo list to remove in /etc/apt/sources.list.d/ repoList: - "repo_vanillastack_cloudical_net.list" - - "devel:kubic:libcontainers:stable:cri-o:1.28.list" - "devel:kubic:libcontainers:stable:cri-o:1.27.list" + - "devel:kubic:libcontainers:stable:cri-o:1.28.list" + - "devel:kubic:libcontainers:stable:cri-o:1.29.list" + - "devel:kubic:libcontainers:stable:cri-o:1.30.list" - "devel:kubic:libcontainers:stable.list" + - "kubernetes-stable.list" - "kubernetes.list" + - "kubic.list" + - "crio.list" + - "cri-o:stable:v1.28.list" + - "cri-o:stable:v1.29.list" + - "cri-o:stable:v1.30.list" diff --git a/roles/common/tasks/get-versions.yaml b/roles/common/tasks/get-versions.yaml index c006372..ac20447 100644 --- a/roles/common/tasks/get-versions.yaml +++ b/roles/common/tasks/get-versions.yaml @@ -5,4 +5,4 @@ - name: set kube version ansible.builtin.set_fact: - kube_version: "{{ kubelet_version_out.stdout | default('0.0.0-0.0')}}" + kube_version: "{{ kubelet_version_out.stdout | default('0.0.0-0.0') }}" diff --git a/roles/common/tasks/upgrade-kube-packages.yaml b/roles/common/tasks/upgrade-kube-packages.yaml index 28a1bff..2f48131 100644 --- a/roles/common/tasks/upgrade-kube-packages.yaml +++ b/roles/common/tasks/upgrade-kube-packages.yaml @@ -3,45 +3,23 @@ ansible.builtin.apt: name: "*" state: latest + update_cache: true - name: get variables ansible.builtin.include_tasks: "get-versions.yaml" -- name: unhold packages - block: - - name: kubeadm - ansible.builtin.dpkg_selections: - name: kubeadm - selection: install #equals unhold - - - name: kubelet - ansible.builtin.dpkg_selections: - name: kubelet - selection: install #equals unhold - - - name: kubectl - ansible.builtin.dpkg_selections: - name: kubectl - selection: install #equals unhold - - - name: cri-o-runc - ansible.builtin.dpkg_selections: - name: cri-o-runc - selection: install #equals unhold - - - name: cri-o - ansible.builtin.dpkg_selections: - name: cri-o - selection: install #equals unhold - - name: upgrade and install ansible.builtin.apt: pkg: + - jq - cri-o-runc - cri-o - kubelet={{ kube_version }} - kubeadm={{ kube_version }} - kubectl={{ kube_version }} + allow_change_held_packages: true + update_cache: true + state: latest - name: hold packages block: @@ -69,3 +47,7 @@ ansible.builtin.dpkg_selections: name: cri-o selection: hold + +- name: Remove dependencies that are no longer required + ansible.builtin.apt: + autoremove: yes diff --git a/roles/common/tasks/upgrade-node.yaml b/roles/common/tasks/upgrade-node.yaml index 718cbf2..638f0ec 100644 --- a/roles/common/tasks/upgrade-node.yaml +++ b/roles/common/tasks/upgrade-node.yaml @@ -55,4 +55,4 @@ - name: wait uncordon pause: prompt: Pausing after uncordon. (press ctrl+c and a to continue immediately) - seconds: "{{ pauseDelayUncordon }}" \ No newline at end of file + seconds: "{{ pauseDelayUncordon }}" diff --git a/roles/common/tasks/upgrade-repo.yaml b/roles/common/tasks/upgrade-repo.yaml index 6a07a1c..d5bdf27 100644 --- a/roles/common/tasks/upgrade-repo.yaml +++ b/roles/common/tasks/upgrade-repo.yaml @@ -16,24 +16,37 @@ state: present filename: kubernetes -- name: add crio repository +- name: add crio repository (crio < 1.28) block: - name: crio | apt key - ansible.builtin.shell: "curl -fsSL https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ crioVersion }}/Debian_11/Release.key | gpg --yes --dearmor -o /usr/share/keyrings/libcontainers-crio-archive-keyring.gpg" + ansible.builtin.shell: "curl -fsSL https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ crioVersion }}/{{ debianVersion }}/Release.key | gpg --yes --dearmor -o /usr/share/keyrings/libcontainers-crio-archive-keyring.gpg" - name: crio | apt source ansible.builtin.apt_repository: - repo: "deb [signed-by=/usr/share/keyrings/libcontainers-crio-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ crioVersion }}/Debian_11/ /" + repo: "deb [signed-by=/usr/share/keyrings/libcontainers-crio-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ crioVersion }}/{{ debianVersion }}/ /" state: present filename: "devel:kubic:libcontainers:stable:cri-o:{{ crioVersion }}" + when: "crioVersion is version('1.28', '<', strict=true)" + +- name: add crio repository (crio >= 1.28) + block: + - name: crio | apt key + ansible.builtin.shell: "curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/stable:/v{{ crioVersion }}/deb/Release.key | gpg --yes --dearmor -o /usr/share/keyrings/libcontainers-crio-archive-keyring.gpg" + + - name: crio | apt source + ansible.builtin.apt_repository: + repo: "deb [signed-by=/usr/share/keyrings/libcontainers-crio-archive-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/v{{ crioVersion }}/deb/ /" + state: present + filename: "cri-o:stable:v{{ crioVersion }}" + when: "crioVersion is version('1.28', '>=', strict=true)" - name: add libcontainer repository block: - name: libcontainer | apt key - ansible.builtin.shell: "curl -fsSL https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_11/Release.key | gpg --yes --dearmor -o /usr/share/keyrings/libcontainers-archive-keyring.gpg" + ansible.builtin.shell: "curl -fsSL https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{{ debianVersion }}/Release.key | gpg --yes --dearmor -o /usr/share/keyrings/libcontainers-archive-keyring.gpg" - name: libcontainer | apt source ansible.builtin.apt_repository: - repo: "deb [signed-by=/usr/share/keyrings/libcontainers-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_11/ /" + repo: "deb [signed-by=/usr/share/keyrings/libcontainers-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{{ debianVersion }}/ /" state: present filename: devel:kubic:libcontainers:stable diff --git a/roles/upgrade-cp/tasks/controlplane-plan.yaml b/roles/upgrade-cp/tasks/controlplane-plan.yaml index 02a25bf..16a5358 100644 --- a/roles/upgrade-cp/tasks/controlplane-plan.yaml +++ b/roles/upgrade-cp/tasks/controlplane-plan.yaml @@ -12,37 +12,48 @@ - kubeadm={{ kube_version }} update_cache: true - - name: unhold + - name: hold ansible.builtin.dpkg_selections: name: kubeadm selection: hold - -- name: kubadm plan - ansible.builtin.shell: kubeadm upgrade plan -o json | grep -iv "^\[[a-zA-Z]*].*$" | jq '.components[] | select(.name=="kubelet") | .newVersion' | tr -d '"' - register: new_version_out + +- name: get kubeadm version + ansible.builtin.shell: kubeadm version -o json | jq -r '.clientVersion.gitVersion' + register: kubeadm_version_out + when: hostvars['K8S_TOKEN_HOLDER']['plan_version'] is not defined + +- name: set plan version + ansible.builtin.set_fact: + plan_version: "{{ kubeadm_version_out.stdout }}" when: hostvars['K8S_TOKEN_HOLDER']['plan_version'] is not defined +- name: kubadm plan + ansible.builtin.shell: "kubeadm upgrade plan {{ plan_version }}" + register: kubeadm_plan_output + when: hostvars['K8S_TOKEN_HOLDER']['plan_version'] is not defined + - name: output plan ansible.builtin.debug: - msg: "plan stdout {{ new_version_out.stdout }}, stderr: {{ new_version_out.stderr }}" + msg: + - "plan stdout:" + - "{{ kubeadm_plan_output.stdout_lines | default('N/A') }}" + - "----------------------------------------------------" + - "stderr:" + - "{{ kubeadm_plan_output.stderr_lines | default('N/A') }}" + ignore_errors: true when: hostvars['K8S_TOKEN_HOLDER']['plan_version'] is not defined -- name: set fact - ansible.builtin.set_fact: - new_version: "{{ new_version_out.stdout | default('v0.0.0', true) }}" +- name: check config + ansible.builtin.pause: + prompt: + - "PLEASE READ THROUGH THE LOG OUTPUT ABOVE" + - "if you are fine with the output of 'kubeadm upgrade plan' you may continue" + - "this message only appears on the initial upgrade process" + minutes: 15 when: hostvars['K8S_TOKEN_HOLDER']['plan_version'] is not defined -- name: register plan version to dummy host +- name: register planned version to dummy host add_host: name: "K8S_TOKEN_HOLDER" - plan_version: "{{ new_version }}" + plan_version: "{{ plan_version }}" when: hostvars['K8S_TOKEN_HOLDER']['plan_version'] is not defined - -- name: make sure installed version equals planned version - ansible.builtin.debug: - msg: "planned version: {{ hostvars['K8S_TOKEN_HOLDER']['plan_version'] }} and apt package version: {{ kube_version }}" - -- name: make sure installed version equals planned version - ansible.builtin.fail: - msg: versions not equal or you are on the latest patch of {{ kubeVersion }} - when: hostvars['K8S_TOKEN_HOLDER']['plan_version'][1:] is version(kube_version.split('-')[0], 'ne') diff --git a/roles/upgrade-cp/tasks/upgrade-other-cp.yaml b/roles/upgrade-cp/tasks/upgrade-other-cp.yaml index 373afb4..50b399e 100644 --- a/roles/upgrade-cp/tasks/upgrade-other-cp.yaml +++ b/roles/upgrade-cp/tasks/upgrade-other-cp.yaml @@ -7,4 +7,4 @@ ansible.builtin.include_tasks: "controlplane-plan.yaml" - name: upgrade node - ansible.builtin.shell: "kubeadm upgrade node" \ No newline at end of file + ansible.builtin.shell: "kubeadm upgrade node" diff --git a/roles/upgrade-node/tasks/main.yaml b/roles/upgrade-node/tasks/main.yaml index a041097..ecf0da4 100644 --- a/roles/upgrade-node/tasks/main.yaml +++ b/roles/upgrade-node/tasks/main.yaml @@ -9,4 +9,4 @@ - name: include common ansible.legacy.include_role: name: common - tasks_from: upgrade-node.yaml \ No newline at end of file + tasks_from: upgrade-node.yaml diff --git a/tofu_infra/README.md b/tofu_infra/README.md index be02fd2..210f446 100644 --- a/tofu_infra/README.md +++ b/tofu_infra/README.md @@ -15,5 +15,6 @@ We use `cri-o`, `kubeadm` and `cilium` to setup the cluster. If you encounter any issue with the cilium install, just run the script again and it should work. ## Output -You will receive an `inventory`,`kubeconfig` and `ssh-key` files in the `local/` directory which you can use to test the cluster. -You may have to copy the files to your `~/.ssh` directory in order to connect via ssh. +You will receive `inventory`,`kubeconfig` and `ssh-key` files in the `local/` directory which you can use to test the cluster. + +If you are using an SSH agent, add the generated key to the keyring with `ssh-add local/tofu-ssh` and you should be good to go. diff --git a/tofu_infra/control-plane.tf b/tofu_infra/control-plane.tf index 2612664..c2c4058 100644 --- a/tofu_infra/control-plane.tf +++ b/tofu_infra/control-plane.tf @@ -4,7 +4,7 @@ resource "hcloud_server" "control-planes" { image = "debian-12" server_type = each.value.type public_net { - ipv6_enabled = false + ipv6_enabled = true } ssh_keys = [ hcloud_ssh_key.tofu-key.name @@ -46,11 +46,7 @@ apt: filename: kubic.list # install packages package_update: true -packages: - - cri-o-runc - - cri-o runcmd: - - apt install -y kubeadm=${var.kube_version}.* kubectl=${var.kube_version}.* kubelet=${var.kube_version}.* - | cat <