From a249fc43dfed677534f79e6bc90e577f1cb669b8 Mon Sep 17 00:00:00 2001 From: Sofer Athlan-Guyot Date: Mon, 20 Jan 2025 15:24:06 +0100 Subject: [PATCH 01/15] [DNM] Testing manual install plan. --- .../common/edpm-nodeset-values/values.yaml.j2 | 3 + .../tasks/authenticate_registry.yml | 69 +++++++++++++++++++ .../tasks/install_operators.yml | 38 ++++++++++ 3 files changed, 110 insertions(+) create mode 100644 roles/kustomize_deploy/tasks/authenticate_registry.yml diff --git a/roles/ci_gen_kustomize_values/templates/common/edpm-nodeset-values/values.yaml.j2 b/roles/ci_gen_kustomize_values/templates/common/edpm-nodeset-values/values.yaml.j2 index cf676889a4..b416a6d7a0 100644 --- a/roles/ci_gen_kustomize_values/templates/common/edpm-nodeset-values/values.yaml.j2 +++ b/roles/ci_gen_kustomize_values/templates/common/edpm-nodeset-values/values.yaml.j2 @@ -19,6 +19,9 @@ data: nodeset: ansible: ansibleUser: "zuul" + ansibleVarsFrom: + - secretRef: + name: redhat-registry ansibleVars: edpm_fips_mode: "{{ 'enabled' if cifmw_fips_enabled|default(false)|bool else 'check' }}" timesync_ntp_servers: diff --git a/roles/kustomize_deploy/tasks/authenticate_registry.yml b/roles/kustomize_deploy/tasks/authenticate_registry.yml new file mode 100644 index 0000000000..c6280feb67 --- /dev/null +++ b/roles/kustomize_deploy/tasks/authenticate_registry.yml @@ -0,0 +1,69 @@ +--- +- name: Retrieve .dockerconfigjson from OpenShift secret + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + ansible.builtin.shell: >- + oc get secret/pull-secret -n openshift-config -o json | + jq -r '.data[".dockerconfigjson"]' | base64 -d > /home/zuul/authfile.txt + +- name: Read username and password from PODMAN_LOGIN_FILE + ansible.builtin.shell: >- + read -r username password < /tmp/podman.txt + && echo "${username}" "${password}" + register: podman_login_creds + +- name: Log in to Podman with registry credentials + ansible.builtin.command: >- + podman login --authfile /home/zuul/authfile.txt + --username "{{ podman_login_creds.stdout.split()[0] }}" + --password "{{ podman_login_creds.stdout.split()[1] }}" + brew.registry.redhat.io + +- name: Update OpenShift secret with the new .dockerconfigjson + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + ansible.builtin.command: >- + oc set data secret/pull-secret -n openshift-config + --from-file=.dockerconfigjson=/home/zuul/authfile.txt + +- name: Read username and password and create Kubernetes secret + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + ansible.builtin.shell: | + read -r username password < /tmp/podman.txt + oc create secret generic redhat-registry \ + --from-literal edpm_container_registry_logins="{\"registry.redhat.io\": {\"${username}\": \"${password}\"}}" + args: + executable: /bin/bash + +- name: Create ImageContentSourcePolicy YAML + ansible.builtin.copy: + dest: "/home/zuul/brew-registry-imageContentSourcePolicy.yaml" + content: | + apiVersion: operator.openshift.io/v1alpha1 + kind: ImageContentSourcePolicy + metadata: + name: brew-registry + spec: + repositoryDigestMirrors: + - mirrors: + - brew.registry.redhat.io + source: registry.redhat.io + - mirrors: + - brew.registry.redhat.io + source: registry.stage.redhat.io + - mirrors: + - brew.registry.redhat.io + source: registry-proxy.engineering.redhat.com + +- name: Apply ImageContentSourcePolicy using oc + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + ansible.builtin.command: oc apply -f /home/zuul/brew-registry-imageContentSourcePolicy.yaml + args: + executable: /bin/bash + diff --git a/roles/kustomize_deploy/tasks/install_operators.yml b/roles/kustomize_deploy/tasks/install_operators.yml index eca39c58a2..9db17c2095 100644 --- a/roles/kustomize_deploy/tasks/install_operators.yml +++ b/roles/kustomize_deploy/tasks/install_operators.yml @@ -64,6 +64,9 @@ dest: "{{ cifmw_kustomize_deploy_olm_dest_file }}" mode: "0644" +- name: Authenticate to private registry + ansible.builtin.include_tasks: authenticate_registry.yml + - name: OLM resources when: not cifmw_kustomize_deploy_generate_crs_only block: @@ -107,6 +110,41 @@ - _cifmw_kustomize_deploy_olm_osp_operator_sub_out.resources | length == 1 - (_cifmw_kustomize_deploy_olm_osp_operator_sub_out.resources | first)['status']['installPlanRef'] is defined + - name: Retrieve the latest InstallPlan name sorted by creation date + set_fact: + installplan_name: >- + {{ + _cifmw_kustomize_deploy_olm_osp_operator_sub_out.resources + | selectattr('kind', 'equalto', 'Subscription') + | sort(attribute='metadata.creationTimestamp') + | map(attribute='status.installPlanRef.name') + | last + }} + + - name: Display debug + ansible.builtin.debug: + msg: "{{ installplan_name }}" + + - name: Accept the InstallPlan + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit)}}" + context: "{{ cifmw_openshift_context | default(omit) }}" + api_version: operators.coreos.com/v1alpha1 + kind: InstallPlan + name: "{{ installplan_name }}" + namespace: "openstack-operators" + state: present + merge_type: + - merge + definition: + spec: + approved: true + + - name: Display debug + ansible.builtin.debug: + msg: "========> INSTALLPLAN '{{ installplan_name }}' HAS BEEN APPROVED." + - name: Wait for the openstack operators InstallPlan to be finished vars: _install_plan: >- From f41fc1779d02f3f3151b5e4a15b05b0934a6773f Mon Sep 17 00:00:00 2001 From: Sofer Athlan-Guyot Date: Thu, 23 Jan 2025 06:22:54 +0100 Subject: [PATCH 02/15] Copy over auth. --- reproducer.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/reproducer.yml b/reproducer.yml index 536d159d18..ffbad6f884 100644 --- a/reproducer.yml +++ b/reproducer.yml @@ -74,6 +74,20 @@ gather_facts: false roles: - role: reproducer + tasks: + - name: Read the podman.txt file + ansible.builtin.slurp: + src: /tmp/podman.txt + register: podman_file + + - name: Decode file content + ansible.builtin.set_fact: + _podman_content: "{{ podman_file['content'] | b64decode }}" + + - name: Parse the file content + ansible.builtin.set_fact: + cifmw_update_podman_username: "{{ _podman_content.split()[0] }}" + cifmw_update_podman_password: "{{ _podman_content.split()[1] }}" post_tasks: - name: Allow traffic from OSP VMs to OSP API (needed for shiftstack) become: true @@ -94,6 +108,13 @@ name: firewalld state: restarted + - name: Recreate the podman.txt file on controller-0 + ansible.builtin.copy: + content: "{{ cifmw_update_podman_username }} {{ cifmw_update_podman_password }}" + dest: /tmp/podman.txt + mode: '0644' + delegate_to: controller-0 + - name: Run deployment if instructed to when: - cifmw_deploy_architecture | default(false) | bool From f74c7b9a7eef8e3147afc2fb6700edb1fb4f8024 Mon Sep 17 00:00:00 2001 From: Sofer Athlan-Guyot Date: Mon, 27 Jan 2025 15:42:58 +0100 Subject: [PATCH 03/15] Add missing namespace. --- roles/kustomize_deploy/tasks/authenticate_registry.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/kustomize_deploy/tasks/authenticate_registry.yml b/roles/kustomize_deploy/tasks/authenticate_registry.yml index c6280feb67..8b34061e84 100644 --- a/roles/kustomize_deploy/tasks/authenticate_registry.yml +++ b/roles/kustomize_deploy/tasks/authenticate_registry.yml @@ -34,7 +34,8 @@ PATH: "{{ cifmw_path }}" ansible.builtin.shell: | read -r username password < /tmp/podman.txt - oc create secret generic redhat-registry \ + # TODO: bug against doc: https://docs.redhat.com/en/documentation/red_hat_openstack_services_on_openshift/18.0/html-single/deploying_red_hat_openstack_services_on_openshift/index#proc_creating-the-data-plane-secrets_dataplane + oc create -n openstack secret generic redhat-registry \ --from-literal edpm_container_registry_logins="{\"registry.redhat.io\": {\"${username}\": \"${password}\"}}" args: executable: /bin/bash From eaf1e2c552f6433c653e5c744efdf0749324232b Mon Sep 17 00:00:00 2001 From: Sofer Athlan-Guyot Date: Mon, 27 Jan 2025 19:36:26 +0100 Subject: [PATCH 04/15] Fix some lint warnings. --- roles/kustomize_deploy/tasks/authenticate_registry.yml | 1 + roles/kustomize_deploy/tasks/install_operators.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/kustomize_deploy/tasks/authenticate_registry.yml b/roles/kustomize_deploy/tasks/authenticate_registry.yml index 8b34061e84..0fcde250c6 100644 --- a/roles/kustomize_deploy/tasks/authenticate_registry.yml +++ b/roles/kustomize_deploy/tasks/authenticate_registry.yml @@ -4,6 +4,7 @@ KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" ansible.builtin.shell: >- + set -o pipefail; oc get secret/pull-secret -n openshift-config -o json | jq -r '.data[".dockerconfigjson"]' | base64 -d > /home/zuul/authfile.txt diff --git a/roles/kustomize_deploy/tasks/install_operators.yml b/roles/kustomize_deploy/tasks/install_operators.yml index 9db17c2095..a8c3669867 100644 --- a/roles/kustomize_deploy/tasks/install_operators.yml +++ b/roles/kustomize_deploy/tasks/install_operators.yml @@ -111,7 +111,7 @@ - (_cifmw_kustomize_deploy_olm_osp_operator_sub_out.resources | first)['status']['installPlanRef'] is defined - name: Retrieve the latest InstallPlan name sorted by creation date - set_fact: + ansible.builtin.set_fact: installplan_name: >- {{ _cifmw_kustomize_deploy_olm_osp_operator_sub_out.resources From 1162abeac7ed62f071b8b9ad1b955ff745ae5395 Mon Sep 17 00:00:00 2001 From: Sofer Athlan-Guyot Date: Tue, 28 Jan 2025 15:05:14 +0100 Subject: [PATCH 05/15] Create openstack namespace before creating secret for registries. --- .../tasks/install_operators.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/roles/kustomize_deploy/tasks/install_operators.yml b/roles/kustomize_deploy/tasks/install_operators.yml index a8c3669867..64e29faa8d 100644 --- a/roles/kustomize_deploy/tasks/install_operators.yml +++ b/roles/kustomize_deploy/tasks/install_operators.yml @@ -64,6 +64,24 @@ dest: "{{ cifmw_kustomize_deploy_olm_dest_file }}" mode: "0644" +# The openstack namespace must be present to be able to create the +# secret generic redhat-registry in it. This is copy of what is +# generated. +- name: Create Kubernetes namespace + kubernetes.core.k8s: + state: present + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + definition: + apiVersion: v1 + kind: Namespace + metadata: + labels: + pod-security.kubernetes.io/enforce: privileged + security.openshift.io/scc.podSecurityLabelSync: "false" + name: openstack + - name: Authenticate to private registry ansible.builtin.include_tasks: authenticate_registry.yml From a548cf117ea1b7a36655b5d89533e52bf98f6f67 Mon Sep 17 00:00:00 2001 From: Sofer Athlan-Guyot Date: Wed, 29 Jan 2025 16:36:35 +0100 Subject: [PATCH 06/15] Make set_openstack_container run optional. When we update the openstack-operator we don't want to mangle with its definition. --- playbooks/update.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/update.yml b/playbooks/update.yml index 5a35158e04..98f0ef692a 100644 --- a/playbooks/update.yml +++ b/playbooks/update.yml @@ -38,6 +38,7 @@ cifmw_set_openstack_containers_openstack_final_env: "operator_env_after_update.txt" ansible.builtin.include_role: name: set_openstack_containers + when: cifmw_set_openstack_containers | default(true) | bool - name: Sync repos for controller to compute hosts: computes From b1602c5deb14a7eaea20ff64b24e8a7ffc21af7c Mon Sep 17 00:00:00 2001 From: Sofer Athlan-Guyot Date: Thu, 30 Jan 2025 11:12:58 +0100 Subject: [PATCH 07/15] read needs a newline or else it exit 1. --- reproducer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reproducer.yml b/reproducer.yml index ffbad6f884..b955fb9ae4 100644 --- a/reproducer.yml +++ b/reproducer.yml @@ -110,7 +110,7 @@ - name: Recreate the podman.txt file on controller-0 ansible.builtin.copy: - content: "{{ cifmw_update_podman_username }} {{ cifmw_update_podman_password }}" + content: "{{ cifmw_update_podman_username }} {{ cifmw_update_podman_password }}\n" dest: /tmp/podman.txt mode: '0644' delegate_to: controller-0 From 471511852a3dbed70d55f3d812f7c1cdadbfffdb Mon Sep 17 00:00:00 2001 From: Sofer Athlan-Guyot Date: Thu, 30 Jan 2025 12:45:17 +0100 Subject: [PATCH 08/15] Approve the update install plan. --- roles/update/tasks/main.yml | 57 ++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/roles/update/tasks/main.yml b/roles/update/tasks/main.yml index b8d53cf99c..50f8390249 100644 --- a/roles/update/tasks/main.yml +++ b/roles/update/tasks/main.yml @@ -36,6 +36,62 @@ ansible.builtin.shell: | {{ cifmw_update_artifacts_basedir }}/control_plane_test_start.sh +- name: Wait for Update InstallPlan creation + kubernetes.core.k8s_info: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + api_version: operators.coreos.com/v1alpha1 + kind: InstallPlan + namespace: openstack-operators + register: _cifmw_update_install_plans + until: cifmw_update_install_plans.resources | selectattr('spec', 'defined') | selectattr('spec', '!=', None) | length > 0 + retries: 30 + delay: 10 + +- name: Retrieve Update InstallPlan name + set_fact: + cifmw_update_installplan_name: >- + {{ + _cifmw_update_install_plans.resources + | selectattr('spec', 'defined') + | selectattr('spec', '!=', None) + | map(attribute='metadata.name') + | last + }} + +- name: Approve the Update InstallPlan + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + state: present + namespace: openstack-operators + definition: + apiVersion: operators.coreos.com/v1alpha1 + kind: InstallPlan + metadata: + name: "{{ cifmw_update_installplan_name }}" + spec: + approved: true + +- name: Display debug + ansible.builtin.debug: + msg: "{{ cifmw_update_installplan_name }}" + +- name: Wait for the Update InstallPlan to complete + kubernetes.core.k8s_info: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + api_version: operators.coreos.com/v1alpha1 + kind: InstallPlan + namespace: openstack-operators + name: "{{ cifmw_update_installplan_name }}" + register: _cifmw_update_installplan_status + until: _cifmw_update_installplan_status.resources[0].status.phase == 'Complete' + retries: 30 + delay: 10 - name: Set openstack_update_run Makefile environment variables tags: @@ -52,7 +108,6 @@ OPENSTACK_VERSION: {{ cifmw_update_openstack_update_run_target_version }} {% endif -%} - - name: Run make openstack_update_run vars: make_openstack_update_run_env: "{{ cifmw_install_yamls_environment | combine({'PATH': cifmw_path }) }}" From 2442c5849a2543b53e2682b3b0ea27baaf520344 Mon Sep 17 00:00:00 2001 From: Sofer Athlan-Guyot Date: Thu, 30 Jan 2025 21:35:45 +0100 Subject: [PATCH 09/15] Get the version to update to after accepting the installplan. --- roles/update/tasks/main.yml | 48 +++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/roles/update/tasks/main.yml b/roles/update/tasks/main.yml index 50f8390249..7a68fb475e 100644 --- a/roles/update/tasks/main.yml +++ b/roles/update/tasks/main.yml @@ -36,6 +36,16 @@ ansible.builtin.shell: | {{ cifmw_update_artifacts_basedir }}/control_plane_test_start.sh +- name: Create collect state script + ansible.builtin.copy: + src: "collect-state.sh" + dest: "{{ cifmw_update_artifacts_basedir }}/collect-state.sh" + mode: "0775" + +- name: Collect state before update + ansible.builtin.command: | + {{ cifmw_update_artifacts_basedir }}/collect-state.sh 1 + - name: Wait for Update InstallPlan creation kubernetes.core.k8s_info: kubeconfig: "{{ cifmw_openshift_kubeconfig }}" @@ -45,7 +55,7 @@ kind: InstallPlan namespace: openstack-operators register: _cifmw_update_install_plans - until: cifmw_update_install_plans.resources | selectattr('spec', 'defined') | selectattr('spec', '!=', None) | length > 0 + until: _cifmw_update_install_plans.resources | selectattr('spec', 'defined') | selectattr('spec', '!=', None) | length > 0 retries: 30 delay: 10 @@ -54,10 +64,9 @@ cifmw_update_installplan_name: >- {{ _cifmw_update_install_plans.resources - | selectattr('spec', 'defined') - | selectattr('spec', '!=', None) + | selectattr('spec.clusterServiceVersionNames', 'defined') | map(attribute='metadata.name') - | last + | first }} - name: Approve the Update InstallPlan @@ -93,6 +102,35 @@ retries: 30 delay: 10 +- name: Collect state after operator update + ansible.builtin.command: | + {{ cifmw_update_artifacts_basedir }}/collect-state.sh 2 + +- name: Make sure we get a new version available, block until we do. + kubernetes.core.k8s_info: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + api_version: core.openstack.org/v1beta1 + kind: OpenStackVersion + namespace: openstack + register: openstackversion_info + until: openstackversion_info.resources[0].spec.targetVersion != openstackversion_info.resources[0].status.availableVersion + retries: 20 # Adjust the number of retries as needed + delay: 15 # Adjust delay between retries as needed + +- name: Capture the .status.availableVersion in cifmw_update_available_version fact + set_fact: + cifmw_update_available_version: "{{ openstackversion_info.resources[0].status.availableVersion }}" + +- name: Display message about the update + debug: + msg: "About to update to {{ cifmw_update_available_version }}" + +- name: Collect state after we get a new version - just make sure. + ansible.builtin.command: | + {{ cifmw_update_artifacts_basedir }}/collect-state.sh 2 + - name: Set openstack_update_run Makefile environment variables tags: - always @@ -105,7 +143,7 @@ CONTAINERS_TARGET_TAG: {{ cifmw_update_openstack_update_run_containers_target_tag }} OPENSTACK_VERSION: {{ cifmw_update_openstack_update_run_target_version }} {% else -%} - OPENSTACK_VERSION: {{ cifmw_update_openstack_update_run_target_version }} + OPENSTACK_VERSION: {{ cifmw_update_available_version }} {% endif -%} - name: Run make openstack_update_run From 2eb4d384ad9726dda3a9898032e35b9ad02626d5 Mon Sep 17 00:00:00 2001 From: Sofer Athlan-Guyot Date: Thu, 30 Jan 2025 22:05:54 +0100 Subject: [PATCH 10/15] Collect state around installplan acceptation. --- roles/update/files/collect-state.sh | 93 +++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 roles/update/files/collect-state.sh diff --git a/roles/update/files/collect-state.sh b/roles/update/files/collect-state.sh new file mode 100644 index 0000000000..e7acb005eb --- /dev/null +++ b/roles/update/files/collect-state.sh @@ -0,0 +1,93 @@ +#!/usr/bin/bash + +BASE_DIR="${HOME}/ci-framework-data/tests/update/" +CI_INVENTORY="${HOME}/ci-framework-data/artifacts/zuul_inventory.yml" +OPERATOR_NAMESPACE="openstack-operators" +NAMESPACE="openstack" + +function get_current_compute_state { + local stage="${1:-}" + file_pre="${BASE_DIR}${stage:+${stage}-}" + + if [ -e "${CI_INVENTORY}" ]; then + echo "Collecting compute state ${stage:+for $stage }in ${BASE_DIR:-${PWD}}" + + # Collect all running containers an all compute nodes in ~/ci-framework-data/tests/update/ by default. + ansible -i "${CI_INVENTORY}" -m shell -a \ + "sudo podman ps -q --filter 'status=running' | xargs -I {} sudo podman inspect --format {% raw %} '{{.Name}} {{.Config.Image}} {{.State.StartedAt}}' {% endraw %} {}|sort" computes | \ + awk -vfile_pre="${file_pre}" 'BEGIN {tp=strftime("%Y%m%d%H%M%S")} /^compute/ {if (s != "") {close(s)}; s = "containers-" $1 "_" tp ".txt"; next;}; s {print > file_pre s} ' + # Collect packages list an all compute nodes in ~/ci-framework-data/tests/update/ by default. + ansible -i "${CI_INVENTORY}" -m shell -a \ + "sudo dnf list installed | sort" computes | \ + awk -vfile_pre="${file_pre}" 'BEGIN {tp=strftime("%Y%m%d%H%M%S")} /^compute/ {if (s != "") {close(s)}; s = "packages-" $1 "_" tp ".txt"; next;}; s {print > file_pre s} ' + fi +} + +function get_current_pod_state { + local stage="${1:-}" + file_pre="${BASE_DIR}${stage:+${stage}-}" + + echo "Collecting pod state ${stage:+for $stage }in ${BASE_DIR:-${PWD}}" + + local openstack_state_file="${file_pre}pods_os_state_$(date +%Y%m%d_%H%M%S).tsv" + local os_operator_state_file="${file_pre}pods_os_op_state_$(date +%Y%m%d_%H%M%S).tsv" + oc get pods -n "${OPERATOR_NAMESPACE}" -o json | jq -r '.items[] | select(.status.phase == "Running") | . as $pod | .status.containerStatuses[] | [$pod.metadata.name, $pod.status.startTime, .image, .state.running.startedAt ] | @tsv' > $os_operator_state_file + + oc get pods -n "${NAMESPACE}" -o json | jq -r '.items[] | select(.status.phase == "Running") | . as $pod | .status.containerStatuses[] | [$pod.metadata.name, $pod.status.startTime, .image, .state.running.startedAt ] | @tsv' > $openstack_state_file +} + +function get_current_state { + local stage="${1:-}" + get_current_compute_state "${stage}" + get_current_pod_state "${stage}" +} + +declare -A STAGES +STAGES=( + [1]="01_before_update" + [2]="02_after_openstack_operator_update" + [3]="03_after_ovn_controlplane_update" + [4]="04_after_ovn_dataplane_update" + [5]="05_after_controlplane_update" + [6]="06_after_update" +) + +print_stages() { + echo "Available options:" + for key in "${!STAGES[@]}"; do + echo "$key) ${STAGES[$key]}" + done +} + +if [[ $# -lt 1 ]]; then + echo "Usage: $0