From 3d4ae5bedbc40fb8084745ccf895b7e27bfdcb86 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Wed, 21 Jun 2023 16:34:30 +0200 Subject: [PATCH 01/38] added redhat_ocp_virt platform to sap_hypervisor_node_preconfigure --- ...ypervisor-redhat_ocp_virt-preconfigure.yml | 11 + ...pervisor-node-preconfigure-rh_ocp_virt.yml | 89 ++++++++ .../README.md | 209 +++++++++++++++++- .../cnv-namespace-operator-subscription.yml | 33 +++ .../sriov-enabled-unsupported-nics.sh | 5 + .../sriov-namespace-operator-subscription.yml | 26 +++ .../platform/redhat_rhel_kvm/main.yml | 7 +- .../meta/main.yml | 9 +- .../redhat_ocp_virt/99-kargs-worker.yml.j2 | 18 ++ .../configure-kargs-per-node.yml | 34 +++ .../redhat_ocp_virt/configure-worker-node.yml | 24 ++ .../redhat_ocp_virt/create-sap-bridge.yml | 49 ++++ .../redhat_ocp_virt/download-rhel-images.yml | 21 ++ .../redhat_ocp_virt/enable-cpumanager.yml | 61 +++++ .../redhat_ocp_virt/install-cnv-operator.yml | 74 +++++++ .../install-nmstate-operator.yml | 89 ++++++++ .../install-sriov-operator.yml | 55 +++++ .../redhat_ocp_virt/install-trident.yml | 48 ++++ .../redhat_ocp_virt/install-virtctl.yml | 15 ++ .../tasks/platform/redhat_ocp_virt/kargs.yml | 11 + .../redhat_ocp_virt/label-worker-invtsc.yml | 11 + .../tasks/platform/redhat_ocp_virt/main.yml | 61 +++++ .../tasks/platform/redhat_ocp_virt/mcp.yml.j2 | 47 ++++ .../platform/redhat_ocp_virt/node-network.yml | 99 +++++++++ .../platform/redhat_ocp_virt/prepare.yml | 9 + .../redhat_ocp_virt/setup-worker-node.yml | 80 +++++++ .../sriov-enabled-unsupported-nics.sh | 5 + .../templates/99-kargs-worker.yml.j2 | 18 ++ .../templates/templates/mcp.yml.j2 | 47 ++++ .../redhat_ocp_virt/trident-backend.json.j2 | 18 ++ .../redhat_ocp_virt/tuned-virtual-host.yml | 21 ++ .../redhat_rhel_kvm/assert-configuration.yml | 46 ++-- .../redhat_rhel_kvm/assert-rhv-hooks.yml | 4 + .../assert-set-tuned-profile.yml | 1 + .../redhat_rhel_kvm/configuration.yml | 65 ++---- .../tasks/platform/redhat_rhel_kvm/main.yml | 12 +- .../platform/redhat_rhel_kvm/rhv-hooks.yml | 3 +- .../redhat_rhel_kvm/set-tuned-profile.yml | 4 + .../platform_defaults_redhat_ocp_virt.yml | 19 ++ 39 files changed, 1374 insertions(+), 84 deletions(-) create mode 100644 playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml create mode 100644 playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/cnv-namespace-operator-subscription.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh create mode 100644 roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-namespace-operator-subscription.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-kargs-per-node.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/create-sap-bridge.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-cnv-operator.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-nmstate-operator.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-sriov-operator.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-virtctl.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/kargs.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/label-worker-invtsc.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/99-kargs-worker.yml.j2 create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/mcp.yml.j2 create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/trident-backend.json.j2 create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/tuned-virtual-host.yml create mode 100644 roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml diff --git a/playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml b/playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml new file mode 100644 index 000000000..285e60488 --- /dev/null +++ b/playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml @@ -0,0 +1,11 @@ +--- +- hosts: all + gather_facts: true + serial: 1 + vars: + sap_hypervisor_node_platform: redhat_ocp_virt + + tasks: + - name: Include Role + ansible.builtin.include_role: + name: sap_hypervisor_node_preconfigure diff --git a/playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml b/playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml new file mode 100644 index 000000000..a2747ac4f --- /dev/null +++ b/playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml @@ -0,0 +1,89 @@ +sap_hypervisor_node_preconfigure_cluster_config: + + # URL under which the OCP cluster is reachable + cluster_url: ocpcluster.domain.org + + # namespace under which the VMs are created, note this has to be + # openshift-sriov-network-operator in case of using SRIOV network + # devices + vm_namespace: sap + + # Optional, configuration for trident driver for Netapp NFS filer + trident: + management: management.domain.org + data: datalif.netapp.domain.org + svm: sap_svm + backend: nas_backend + aggregate: aggregate_Name + username: admin + password: xxxxx + storage_driver: ontap-nas + storage_prefix: ocpv_sap_ + + # detailed configuration for every worker that should be configured + # + workers: + - kubernetes_reserved_cpus: "0,1" # CPU cores reserved for + # kubernetes + + - name: worker-0 # name must match the node name + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov + + - bridge: # another bridge + options: + stp: + enabled: false + port: + - name: ens2f0 # network IF name + description: storage + mtu: 9000 + ipv4: + address: + - ip: 192.168.1.51 # IP config + prefix-length: 24 + auto-dns: false + auto-gateway: false + enabled: true + name: storagebridge + state: up + type: linux-bridge + - name: multi # another SRIOV device + interface: ens2f1 # network IF name + type: sriov + + - name: worker-1 # second worker configuration + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index cc26e0917..628a6a86d 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -3,7 +3,191 @@ sap_hypervisor_node_preconfigure ======================= -This role will set and check the required settings and parameters for a hypervisor running VMs for SAP HANA. +This role will configure the following hypervisors in order to run SAP workloads: +* Redhat Openshift Virtualization (OCPV) +* Redhat Enterprise Virtualization (RHV) + +Platform: Redhat Openshift Virtualization +========================================= + +This role will configure a plain vanilla Openshift cluster so it can be used for SAP workloads. + +Requirements +------------ +A freshly installed Openshift cluster. +The worker nodes should have > 96GB of memory. +Storage is required, e.g. via NFS, Openshift Data Foundation or local storage. +This role can setup access to a Netapp Filer via Trident storage connector. +Point the `KUBECONFIG` environment variable to you `kubeconfig`. + + +Install the packages stated in `requirements.txt` on the host where the role runs. +The required packages are: +``` +httpd-tools +ansible-collection-kubernetes-core +``` + + +Make the role available in case you didn't install it already in an ansible roles directory, e.g. + +``` +mkdir -p ~/.ansible/roles/ +ln -sf ~/community.sap_install/roles/sap_hypervisor_node_preconfigure ~/.ansible/roles/ +``` + +Role Variables +-------------- +General variables are defined in sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml +``` +# Install the trident NFS storage provider +sap_hypervisor_node_preconfigure_install_trident: False +# URL of the trident installer package to use +sap_hypervisor_node_preconfigure_install_trident_url: https://github.com/NetApp/trident/releases/download/v23.01.0/trident-installer-23.01.0.tar.gz + +# should SRIOV be enabled for unsupported NICs +sap_hypervisor_node_preconfigure_sriov_enable_unsupported_nics: True + +# Amount of memory [GB] to be reserved for the hypervisor on hosts >= 512GB +sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512: 64 #GB +# Amount of memory [GB] to be reserved for the hypervisor on hosts < 512GB +sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512: 32 #GB + +# Should the check for the minimal amount of be ignored? Minimal amount is 96 GB +# If ignored, the amount of $hostmemory - $reserved is allocated with a lower bound of 0 in case $reserved > $hostmemory +sap_hypervisor_node_preconfigure_ignore_minimal_memory_check: False +``` + +The following variables are describing the nodes and networks to be used. It can make sense to have them in a seperate file, e.g. see `playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml` for an example. +``` +sap_hypervisor_node_preconfigure_cluster_config: + # URL under which the OCP cluster is reachable + cluster_url: ocpcluster.domain.org + + # namespace under which the VMs are created, note this has to be + # openshift-sriov-network-operator in case of using SRIOV network + # devices + vm_namespace: sap + + # Optional, configuration for trident driver for Netapp NFS filer + trident: + management: management.domain.org + data: datalif.netapp.domain.org + svm: sap_svm + backend: nas_backend + aggregate: aggregate_Name + username: admin + password: xxxxx + storage_driver: ontap-nas + storage_prefix: ocpv_sap_ + + # detailed configuration for every worker that should be configured + workers: + kubernetes_reserved_cpus: "0,1" # CPU cores reserved for + # kubernetes + + - name: worker-0 # name must match the node name + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov + + - bridge: # another bridge + options: + stp: + enabled: false + port: + - name: ens2f0 # network IF name + description: storage + mtu: 9000 + ipv4: + address: + - ip: 192.168.1.51 # IP config + prefix-length: 24 + auto-dns: false + auto-gateway: false + enabled: true + name: storagebridge + state: up + type: linux-bridge + - name: multi # another SRIOV device + interface: ens2f1 # network IF name + type: sriov + + - name: worker-1 # second worker configuration + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov +``` + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +See `playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml` for this example: + +``` +--- +- hosts: all + gather_facts: true + serial: 1 + vars: + sap_hypervisor_node_platform: redhat_ocp_virt + + tasks: + - name: Include Role + ansible.builtin.include_role: + name: sap_hypervisor_node_preconfigure +``` + +Example Usage +------------- +Make sure to set the `KUBECONFIG` environment variable, e.g. +``` +export KUBECONFIG=~/.kubeconfig +``` +To invoke the example playbook with the example configuration using your localhost as ansible host use the following command line: +``` +ansible-playbook --connection=local -i localhost, playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml -e @s/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml +``` + + +Platform: RHEL KVM +=================== +set and check the required settings and parameters for a hypervisor running VMs for SAP HANA. Requirements ------------ @@ -48,18 +232,31 @@ Example Playbook Simple example that just sets the parameters. ``` +--- - hosts: all - roles: - - sap_hypervisor_node_preconfigure + gather_facts: true + serial: 1 + vars: + sap_hypervisor_node_platform: redhat_rhel_kvm + tasks: + - name: Include Role + ansible.builtin.include_role: + name: sap_hypervisor_node_preconfigure ``` Run in assert mode to verify that parameters have been set. ``` +--- - hosts: all - roles: - - sap_hypervisor_node_preconfigure + gather_facts: true + serial: 1 vars: - - sap_hypervisor_node_preconfigure_assert: yes + sap_hypervisor_node_platform: redhat_rhel_kvm + sap_hypervisor_node_preconfigure_assert: yes + tasks: + - name: Include Role + ansible.builtin.include_role: + name: sap_hypervisor_node_preconfigure ``` License ------- diff --git a/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/cnv-namespace-operator-subscription.yml b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/cnv-namespace-operator-subscription.yml new file mode 100644 index 000000000..2d8b3feab --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/cnv-namespace-operator-subscription.yml @@ -0,0 +1,33 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-cnv +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: kubevirt-hyperconverged-group + namespace: openshift-cnv +spec: + targetNamespaces: + - openshift-cnv +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: hco-operatorhub + namespace: openshift-cnv +spec: + source: redhat-operators + sourceNamespace: openshift-marketplace + name: kubevirt-hyperconverged +# startingCSV: kubevirt-hyperconverged-operator.v4.10.0 +# channel: "stable" +--- +apiVersion: hco.kubevirt.io/v1beta1 +kind: HyperConverged +metadata: + name: kubevirt-hyperconverged + namespace: openshift-cnv +spec: diff --git a/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh new file mode 100644 index 000000000..7732ba78e --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# in order to allow unsupported SRIOV nics such as Mellanox +oc patch sriovoperatorconfig default --type=merge -n openshift-sriov-network-operator --patch '{ "spec": { "enableOperatorWebhook": false } }' + + diff --git a/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-namespace-operator-subscription.yml b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-namespace-operator-subscription.yml new file mode 100644 index 000000000..9451b3401 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-namespace-operator-subscription.yml @@ -0,0 +1,26 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-sriov-network-operator +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: sriov-network-operators + namespace: openshift-sriov-network-operator +spec: + targetNamespaces: + - openshift-sriov-network-operator +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: sriov-network-operator-subscription + namespace: openshift-sriov-network-operator +spec: + source: redhat-operators + sourceNamespace: openshift-marketplace + name: sriov-network-operator +# startingCSV: sriov-network-operator + channel: "stable" diff --git a/roles/sap_hypervisor_node_preconfigure/handlers/platform/redhat_rhel_kvm/main.yml b/roles/sap_hypervisor_node_preconfigure/handlers/platform/redhat_rhel_kvm/main.yml index 21c4a5b4e..911776897 100644 --- a/roles/sap_hypervisor_node_preconfigure/handlers/platform/redhat_rhel_kvm/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/handlers/platform/redhat_rhel_kvm/main.yml @@ -1,5 +1,4 @@ --- - - name: "Check if server is booted in BIOS or UEFI mode" ansible.builtin.stat: path: /sys/firmware/efi @@ -40,14 +39,14 @@ - name: "Set the grub.cfg location RHEL" ansible.builtin.set_fact: __sap_hypervisor_node_preconfigure_uefi_boot_dir: /boot/efi/EFI/redhat/grub.cfg - when: + when: - ansible_distribution == 'RedHat' - name: "Set the grub.cfg location SLES" ansible.builtin.set_fact: __sap_hypervisor_node_preconfigure_uefi_boot_dir: /boot/efi/EFI/BOOT/grub.cfg - when: - - ansible_distribution == 'SLES' or ansible_distribution == 'SLES_SAP' + when: + - ansible_distribution == 'SLES' or ansible_distribution == 'SLES_SAP' - name: "Run grub-mkconfig (UEFI mode)" ansible.builtin.command: "grub2-mkconfig -o {{ __sap_hypervisor_node_preconfigure_uefi_boot_dir }}" diff --git a/roles/sap_hypervisor_node_preconfigure/meta/main.yml b/roles/sap_hypervisor_node_preconfigure/meta/main.yml index 0b4c2c801..47965cdc0 100644 --- a/roles/sap_hypervisor_node_preconfigure/meta/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/meta/main.yml @@ -3,8 +3,15 @@ galaxy_info: namespace: community role_name: sap_hypervisor_node_preconfigure author: Nils Koenig - description: Provide the configuration of SAP-certified hypervisors + description: Provide the configuration of hypervisors for SAP workloads license: Apache-2.0 min_ansible_version: 2.9 galaxy_tags: [ 'sap', 'hana', 'rhel', 'redhat', 'sles', 'suse' ] + platforms: + - name: CentOS + versions: + 8 + - name: RHEL + versions: + 8 dependencies: [] diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 new file mode 100644 index 000000000..183bfb353 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 @@ -0,0 +1,18 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + kubernetes.io/hostname: {{ __sap_hypervisor_node_preconfigure_register_worker_name }} + machineconfiguration.openshift.io/role: {{ __sap_hypervisor_node_preconfigure_register_worker_name }} + name: 99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker_name }} +spec: + config: + ignition: + version: 3.2.0 + kernelArguments: + - intel_iommu=on + - iommu=pt + - default_hugepagesz=1GB + - hugepagesz=1GB + - hugepages={{ __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages }} + - tsx=on diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-kargs-per-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-kargs-per-node.yml new file mode 100644 index 000000000..488c3eaa9 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-kargs-per-node.yml @@ -0,0 +1,34 @@ +--- +- name: Get worker name + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_name: + "{{ __sap_hypervisor_node_preconfigure_register_worker['metadata']['labels']['kubernetes.io/hostname'] }}" + +- name: Get memory of worker node + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_memory_gb: + "{{ (__sap_hypervisor_node_preconfigure_register_worker['status']['capacity']['memory'] | replace('Ki', '') | int / 1048576) | int }}" + +- name: Check if host has minimal amount of memory (96Gb) + ansible.builtin.assert: + that: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int >= 96 + fail_msg: "Not enough memory on node {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" + success_msg: "Enough memory on node {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" + ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_minimal_memory_check }}" + +# calculate memory to be allocated as hugepages +# if system < 512GB memory use 32GB as upper boundary, 64GB otherwise as upper boundary +- name: Calculate amount of hugepages to reserve (host memory < 512 Gb) + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: + "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gb | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512 }}" + when: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int < 512 + +- name: Calculate amount of hugepages to reserve (host memory >= 512 Gb) + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: + "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gb | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512 }}" + when: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int >= 512 + +- name: "Include kargs for {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" + ansible.builtin.include_tasks: kargs.yml diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml new file mode 100644 index 000000000..82fec7ac3 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml @@ -0,0 +1,24 @@ +--- +- name: Include node network + ansible.builtin.include_tasks: node-network.yml + with_items: "{{ __sap_hypervisor_node_preconfigure_register_worker.networks }}" + loop_control: + loop_var: __sap_hypervisor_node_preconfigure_register_worker_network + index_var: __sap_hypervisor_node_preconfigure_register_worker_network_nr + when: __sap_hypervisor_node_preconfigure_register_worker.networks is defined + +- name: "Create MCP for {{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + kubernetes.core.k8s: + template: + path: "mcp.yml.j2" + state: present + +- name: Pause so cluster can process config + ansible.builtin.pause: + minutes: 1 + +# How to wait for node to be scheduleable? (NodeSchedulable) +- name: Wait for all k8s nodes to be ready + ansible.builtin.command: oc wait --for=condition=Ready nodes --all --timeout=3600s + register: __sap_hypervisor_node_preconfigure_register_nodes_ready + changed_when: __sap_hypervisor_node_preconfigure_register_nodes_ready.rc != 0 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/create-sap-bridge.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/create-sap-bridge.yml new file mode 100644 index 000000000..bbdbdfffd --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/create-sap-bridge.yml @@ -0,0 +1,49 @@ +--- +- name: Create SAP bridge NodeNetworkConfigurationPolicy + kubernetes.core.k8s: + state: present + definition: + apiVersion: nmstate.io/v1 + kind: NodeNetworkConfigurationPolicy + metadata: + name: "sap-bridge-policy-{{ worker.name }}" + spec: + nodeSelector: + kubernetes.io/hostname: "{{ worker.name }}" + desiredState: + interfaces: + - name: sapbridge + description: "Linux bridge with {{ worker.sap_bridge_interface }} as physical port to access SAP network" + type: linux-bridge + state: up + ipv4: + enabled: false + bridge: + options: + stp: + enabled: false + port: + - name: "{{ worker.sap_bridge_interface }}" + + +- name: Create SAP bridge NetworkAttachmentDefinition + kubernetes.core.k8s: + state: present + definition: + apiVersion: "k8s.cni.cncf.io/v1" + kind: NetworkAttachmentDefinition + metadata: + kubernetes.io/hostname: "{{ worker.name }}" + machineconfiguration.openshift.io/role: "{{ worker.name }}" + namespace: "{{ vm_namespace }}" + name: sap-bridge-network-definition + annotations: + k8s.v1.cni.cncf.io/resourceName: bridge.network.kubevirt.io/sapbridge + spec: + config: '{ + "cniVersion": "0.3.1", + "name": "sap-bridge-network-definition", + "type": "cnv-bridge", + "bridge": "sapbridge", + "macspoofchk": true + }' diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml new file mode 100644 index 000000000..6dd050ea9 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml @@ -0,0 +1,21 @@ +--- +- name: "Download rhel 8.6 image" + kubernetes.core.k8s: + state: present + definition: + apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + namespace: openshift-virtualization-os-images + name: rhel-86 + annotations: + cdi.kubevirt.io/storage.bind.immediate.requested: 'true' + spec: + source: + registry: + url: 'docker://registry.redhat.io/rhel8/rhel-guest-image:8.6.0' + pullMethod: node + storage: + resources: + requests: + storage: 10Gi diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml new file mode 100644 index 000000000..25e88c1b2 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml @@ -0,0 +1,61 @@ +--- +- name: Label nodes + ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} \ + feature.node.kubernetes.io/cpu-feature-invtsc=true --overwrite=true" + register: __sap_hypervisor_node_preconfigure_register_label_node + changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 + +- name: Enable CPU Manager by patching MCP of "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + kubernetes.core.k8s: + state: patched + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfigPool + metadata: + name: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + labels: + custom-kubelet: "cpumanager-enabled" + +- name: Create kubletconfig for cpumanager "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is defined + kubernetes.core.k8s: + state: present + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: KubeletConfig + metadata: + name: "cpumanager-enabled" + kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + machineconfiguration.openshift.io/role: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + spec: + machineConfigPoolSelector: + matchLabels: + custom-kubelet: "cpumanager-enabled" + kubeletConfig: + cpuManagerPolicy: static + cpuManagerReconcilePeriod: 5s + reservedSystemCPUs: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus }}" + +- name: Create kubletconfig for cpumanager "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is not defined + kubernetes.core.k8s: + state: present + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: KubeletConfig + metadata: + name: "cpumanager-enabled" + kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + machineconfiguration.openshift.io/role: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + spec: + machineConfigPoolSelector: + matchLabels: + custom-kubelet: "cpumanager-enabled" + kubeletConfig: + cpuManagerPolicy: static + cpuManagerReconcilePeriod: 5s + +- name: Label nodes + ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} cpumanager=true --overwrite=true" + register: __sap_hypervisor_node_preconfigure_register_label_node + changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-cnv-operator.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-cnv-operator.yml new file mode 100644 index 000000000..1a08c9306 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-cnv-operator.yml @@ -0,0 +1,74 @@ +--- +- name: Create the CNV Operator namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: openshift-cnv + +- name: Create CNV OperatorGroup kubevirt-hyperconverged-group + kubernetes.core.k8s: + state: present + + definition: + apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + name: kubevirt-hyperconverged-group + namespace: openshift-cnv + spec: + targetNamespaces: + - openshift-cnv + +- name: Create CNV Subscription + kubernetes.core.k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + name: hco-operatorhub + namespace: openshift-cnv + spec: + source: redhat-operators + sourceNamespace: openshift-marketplace + name: kubevirt-hyperconverged + +- name: Wait + ansible.builtin.pause: + seconds: 60 + +- name: Get Install Plan Name + retries: 10 + delay: 10 + ansible.builtin.command: oc get subscriptions/hco-operatorhub --namespace openshift-cnv --output=jsonpath='{$.status.installplan.name}' + register: __sap_hypervisor_node_preconfigure_register_cnv_subscription_install_plan_name + until: __sap_hypervisor_node_preconfigure_register_cnv_subscription_install_plan_name.stdout != "" + changed_when: __sap_hypervisor_node_preconfigure_register_cnv_subscription_install_plan_name.stdout != "" + +- name: Wait for Install Plan to finish + ansible.builtin.command: "oc wait installplan \ + {{ __sap_hypervisor_node_preconfigure_register_cnv_subscription_install_plan_name.stdout }} --namespace openshift-cnv --for=condition='Installed' --timeout='5m'" + register: __sap_hypervisor_node_preconfigure_register_wait_for_installplan + changed_when: __sap_hypervisor_node_preconfigure_register_wait_for_installplan.rc != 0 + +- name: Wait + ansible.builtin.pause: + seconds: 300 + +- name: Create CNV HyperConverged + kubernetes.core.k8s: + state: present + definition: + apiVersion: hco.kubevirt.io/v1beta1 + kind: HyperConverged + metadata: + name: kubevirt-hyperconverged + namespace: openshift-cnv + spec: + +- name: Wait + ansible.builtin.pause: + seconds: 300 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-nmstate-operator.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-nmstate-operator.yml new file mode 100644 index 000000000..a961de89f --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-nmstate-operator.yml @@ -0,0 +1,89 @@ +--- +- name: Delete the nmstate operator namespace + kubernetes.core.k8s: + state: absent + definition: + apiVersion: v1 + kind: Namespace + metadata: + labels: + kubernetes.io/metadata.name: openshift-nmstate + name: openshift-nmstate + name: openshift-nmstate + spec: + finalizers: + - kubernetes + +- name: Pause to give operator a chance to uninstall + ansible.builtin.pause: + minutes: 2 + +- name: Create the nmstate operator namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + labels: + kubernetes.io/metadata.name: openshift-nmstate + name: openshift-nmstate + name: openshift-nmstate + spec: + finalizers: + - kubernetes + +- name: Create the OperatorGroup + kubernetes.core.k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + annotations: + olm.providedAPIs: NMState.v1.nmstate.io + generateName: openshift-nmstate- + name: openshift-nmstate-tn6k8 + namespace: openshift-nmstate + spec: + targetNamespaces: + - openshift-nmstate + +- name: Pause to give operator a chance to install + ansible.builtin.pause: + minutes: 2 + +- name: Subscribe to the nmstate Operator + kubernetes.core.k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + labels: + operators.coreos.com/kubernetes-nmstate-operator.openshift-nmstate: "" + name: kubernetes-nmstate-operator + namespace: openshift-nmstate + spec: + channel: stable + installPlanApproval: Automatic + name: kubernetes-nmstate-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + +- name: Pause to give operator a chance to install + ansible.builtin.pause: + minutes: 5 + +- name: Create instance of the nmstate operator + kubernetes.core.k8s: + state: present + definition: + apiVersion: nmstate.io/v1 + kind: NMState + metadata: + name: nmstate + +- name: Pause to give instance a chance to come up + ansible.builtin.pause: + minutes: 5 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-sriov-operator.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-sriov-operator.yml new file mode 100644 index 000000000..1379b1152 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-sriov-operator.yml @@ -0,0 +1,55 @@ +--- +- name: Create the CNV Operator namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: openshift-sriov-network-operator + +- name: Create the CNV Operator namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + name: sriov-network-operators + namespace: openshift-sriov-network-operator + spec: + targetNamespaces: + - openshift-sriov-network-operator +- name: Create the CNV Operator namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + name: sriov-network-operator-subscription + namespace: openshift-sriov-network-operator + spec: + source: redhat-operators + sourceNamespace: openshift-marketplace + name: sriov-network-operator + # startingCSV: sriov-network-operator + channel: "stable" + + +- name: Pause to give operator a chance to install + ansible.builtin.pause: + minutes: 3 + +- name: Copy patch to enable unsupported NICs + ansible.builtin.copy: + src: sriov-enabled-unsupported-nics.sh + dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/sriov-enabled-unsupported-nics.sh" + mode: "0755" + when: sap_hypervisor_node_preconfigure_sriov_enable_unsupported_nics + +- name: Enable unsupported NICs + ansible.builtin.command: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/sriov-enabled-unsupported-nics.sh" + when: sap_hypervisor_node_preconfigure_sriov_enable_unsupported_nics + register: __sap_hypervisor_node_preconfigure_register_enable_unsupported_nics + changed_when: __sap_hypervisor_node_preconfigure_register_enable_unsupported_nics.rc != 0 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml new file mode 100644 index 000000000..9747b55ad --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml @@ -0,0 +1,48 @@ +--- +- name: Download trident + ansible.builtin.unarchive: + remote_src: true + src: "{{ sap_hypervisor_node_preconfigure_install_trident_url }}" + dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/" + +- name: Uninstall trident + ansible.builtin.command: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/trident-installer/tridentctl uninstall -n trident" + ignore_errors: true + register: __sap_hypervisor_node_preconfigure_register_uninstall_trident + changed_when: __sap_hypervisor_node_preconfigure_register_uninstall_trident.rc != 0 + +- name: Install trident + ansible.builtin.command: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/trident-installer/tridentctl install -n trident" + register: __sap_hypervisor_node_preconfigure_register_install_trident + changed_when: __sap_hypervisor_node_preconfigure_register_install_trident.rc != 0 + +- name: Copy backend file + ansible.builtin.template: + src: "trident-backend.json.j2" + dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/trident-backend.json" + mode: "0644" + +- name: Create trident backend + ansible.builtin.command: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}\ + /trident-installer/tridentctl -n trident create backend -f\ + {{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}\ + /trident-backend.json" + register: __sap_hypervisor_node_preconfigure_register_create_trident_backend + changed_when: __sap_hypervisor_node_preconfigure_register_create_trident_backend.rc != 0 + +- name: Create storage class + kubernetes.core.k8s: + state: present + definition: + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: nas + annotations: + storageclass.kubernetes.io/is-default-class: 'true' + provisioner: csi.trident.netapp.io + parameters: + backendType: "{{ sap_hypervisor_node_preconfigure_cluster_config.trident.storage_driver }}" + snapshots: "True" + provisioningType: "thin" + encryption: "false" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-virtctl.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-virtctl.yml new file mode 100644 index 000000000..e4555f803 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-virtctl.yml @@ -0,0 +1,15 @@ +--- +- name: Create ~/bin + ansible.builtin.file: + path: ~/bin + state: directory + mode: "0755" + +- name: Get and extract virtctl +# become: yes + ansible.builtin.unarchive: + validate_certs: false + remote_src: true + src: "https://hyperconverged-cluster-cli-download-openshift-cnv.apps.\ + {{ sap_hypervisor_node_preconfigure_cluster_config.cluster_url }}/amd64/linux/virtctl.tar.gz" + dest: ~/bin diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/kargs.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/kargs.yml new file mode 100644 index 000000000..bd28ea55b --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/kargs.yml @@ -0,0 +1,11 @@ +--- +- name: Personalize template + ansible.builtin.template: + src: 99-kargs-worker.yml.j2 + dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker_name }}.yml.j2" + mode: "0644" + +- name: Enable hugepages + kubernetes.core.k8s: + state: present + src: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker_name }}.yml.j2" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/label-worker-invtsc.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/label-worker-invtsc.yml new file mode 100644 index 000000000..57a52da24 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/label-worker-invtsc.yml @@ -0,0 +1,11 @@ +--- +- name: Label worker with invtsc flag + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: default + labels: + 'feature.node.kubernetes.io/cpu-feature-invtsc': enabled diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml new file mode 100644 index 000000000..9cc470a04 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml @@ -0,0 +1,61 @@ +--- +- name: Get a list of all nodes from any namespace + kubernetes.core.k8s_info: + kind: Node + register: __sap_hypervisor_node_preconfigure_register_node_list + +- name: Generate list with worker node names + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_node_name_list: + "{{ __sap_hypervisor_node_preconfigure_register_worker_node_name_list | \ + d([]) + [__sap_hypervisor_node_preconfigure_register_worker_node.name] }}" + with_items: "{{ sap_hypervisor_node_preconfigure_cluster_config.workers }}" + loop_control: + loop_var: __sap_hypervisor_node_preconfigure_register_worker_node + +- name: Filter hosts + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_nodes: + "{{ __sap_hypervisor_node_preconfigure_register_nodes | \ + d([]) + [__sap_hypervisor_node_preconfigure_register_host] }}" + with_items: "{{ __sap_hypervisor_node_preconfigure_register_node_list['resources'] }}" + loop_control: + loop_var: __sap_hypervisor_node_preconfigure_register_host + when: __sap_hypervisor_node_preconfigure_register_host.metadata.name in __sap_hypervisor_node_preconfigure_register_worker_node_name_list + +- name: Assert that configured nodes are found + ansible.builtin.assert: + that: __sap_hypervisor_node_preconfigure_register_nodes is defined + fail_msg: No nodes found that match configuration provided in sap_hypervisor_node_preconfigure_cluster + success_msg: Configured nodes found + +# - meta: end_play + +- name: Include prepare + ansible.builtin.include_tasks: prepare.yml +- name: Include tuned virtual host + ansible.builtin.include_tasks: tuned-virtual-host.yml +- name: Include install CNV operator + ansible.builtin.include_tasks: install-cnv-operator.yml +- name: Include install sriov operator + ansible.builtin.include_tasks: install-sriov-operator.yml +- name: Include install nmstate operator + ansible.builtin.include_tasks: install-nmstate-operator.yml +- name: Include install virtctl + ansible.builtin.include_tasks: install-virtctl.yml +- name: Include setup worker node + ansible.builtin.include_tasks: setup-worker-node.yml + +# How to wait for node to be scheduleable? (NodeSchedulable) +- name: Wait for all k8s nodes to be ready + ansible.builtin.command: oc wait --for=condition=Ready nodes --all --timeout=3600s + register: __sap_hypervisor_node_preconfigure_register_nodes_ready + changed_when: __sap_hypervisor_node_preconfigure_register_nodes_ready.rc != 0 + +- name: Print nodes + ansible.builtin.debug: + var: __sap_hypervisor_node_preconfigure_register_nodes_ready.stdout_lines + +- name: Include Trident installation + ansible.builtin.include_tasks: install-trident.yml + when: sap_hypervisor_node_preconfigure_install_trident diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 new file mode 100644 index 000000000..1a39d0a06 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 @@ -0,0 +1,47 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfigPool +metadata: + labels: + machineconfiguration.openshift.io/mco-built-in: "" + name: {{ __sap_hypervisor_node_preconfigure_register_worker.name }} +spec: + configuration: + source: + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 00-worker + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 01-worker-container-runtime + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 01-worker-kubelet + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-crio-capabilities + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-crio-seccomp-use-default + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-kubelet + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-registries + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-ssh + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker.name }} + machineConfigSelector: + matchExpressions: + - key: machineconfiguration.openshift.io/role + operator: In + values: + - worker + - {{ __sap_hypervisor_node_preconfigure_register_worker.name }} + nodeSelector: + matchLabels: + kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + paused: false diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml new file mode 100644 index 000000000..088f86fe8 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml @@ -0,0 +1,99 @@ +--- +- name: Print network + ansible.builtin.debug: + var: __sap_hypervisor_node_preconfigure_register_worker_network + +- name: "Create NodeNetworkConfigurationPolicy\ + {{ __sap_hypervisor_node_preconfigure_register_worker_network.name }} on\ + {{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: nmstate.io/v1 + kind: NodeNetworkConfigurationPolicy + metadata: + name: "{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}-{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + spec: + nodeSelector: + kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + desiredState: + interfaces: + - "{{ __sap_hypervisor_node_preconfigure_register_worker_network }}" + when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'linux-bridge' + +# XXX didn't work - why? +- name: "Create NetworkAttachmentDefinition {{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: "k8s.cni.cncf.io/v1" + kind: NetworkAttachmentDefinition + metadata: + namespace: "{{ sap_hypervisor_node_preconfigure_cluster_config.vm_namespace }}" + name: "{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}-network-definition" + annotations: + k8s.v1.cni.cncf.io/resourceName: "bridge.network.kubevirt.io/{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}" + spec: + config: '{ + "cniVersion": "0.3.1", + "name": "sapbridge-network-definition", + "type": "cnv-bridge", + "bridge": "sapbridge", + "macspoofchk": true + }' + when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'linux-bridge' + +- name: Label nodes + ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }}\ + feature.node.kubernetes.io/network-sriov.capable=true --overwrite=true" + when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'sriov' + register: __sap_hypervisor_node_preconfigure_register_label_node + changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 + +- name: "Create SRIOV NodeNetworkConfigurationPolicy\ + {{ __sap_hypervisor_node_preconfigure_register_worker_network.name.name }} on\ + {{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: sriovnetwork.openshift.io/v1 + kind: SriovNetworkNodePolicy + metadata: + name: "iface-{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}-sriov-{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + namespace: openshift-sriov-network-operator + spec: + resourceName: "iface{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}sriov" + nodeSelector: + feature.node.kubernetes.io/network-sriov.capable: "true" + kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + priority: 5 + mtu: 9000 + numVfs: 8 + nicSelector: + pfNames: ['{{ __sap_hypervisor_node_preconfigure_register_worker_network.interface }}#0-7'] + deviceType: vfio-pci + isRdma: false + when: __sap_hypervisor_node_preconfigure_register_worker_network.type == "sriov" + +- name: "Create SriovNetwork Attachment Definition {{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: sriovnetwork.openshift.io/v1 + kind: SriovNetwork + metadata: + name: "iface-{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}-sriov" + namespace: openshift-sriov-network-operator + spec: + ipam: | + { + "type": "host-local", + "subnet": "192.168.1.0/24", + "rangeStart": "192.168.1.200", + "rangeEnd": "192.168.1.210" + } + networkNamespace: openshift-sriov-network-operator + resourceName: "iface{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}sriov" + spoofChk: "off" + trust: "on" + when: __sap_hypervisor_node_preconfigure_register_worker_network.type == "sriov" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml new file mode 100644 index 000000000..dd879b22c --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml @@ -0,0 +1,9 @@ +--- +- name: Gather Facts + ansible.builtin.gather_facts: + +- name: Create Tempdir + ansible.builtin.tempfile: + state: directory + suffix: "_sap_hypervisor_node_preconfigure" + register: __sap_hypervisor_node_preconfigure_register_tmpdir diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml new file mode 100644 index 000000000..2dc78034f --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml @@ -0,0 +1,80 @@ +--- +- name: Include configure kargs + ansible.builtin.include_tasks: configure-kargs-per-node.yml + with_items: "{{ __sap_hypervisor_node_preconfigure_register_nodes }}" + loop_control: + loop_var: __sap_hypervisor_node_preconfigure_register_worker + index_var: __sap_hypervisor_node_preconfigure_register_worker_nr + +- name: Include configure worker + ansible.builtin.include_tasks: configure-worker-node.yml + with_items: "{{ sap_hypervisor_node_preconfigure_cluster_config.workers }}" + loop_control: + loop_var: __sap_hypervisor_node_preconfigure_register_worker + index_var: __sap_hypervisor_node_preconfigure_register_worker_nr + +- name: Enable CPU Manager by patching MCP worker + kubernetes.core.k8s: + state: patched + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfigPool + metadata: + name: worker + labels: + custom-kubelet: cpumanager-enabled + + +- name: Create kubletconfig for cpumanager worker + when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is defined + kubernetes.core.k8s: + state: absent + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: KubeletConfig + metadata: + name: cpumanager-enabled + spec: + machineConfigPoolSelector: + matchLabels: + custom-kubelet: cpumanager-enabled + kubeletConfig: + cpuManagerPolicy: static + cpuManagerReconcilePeriod: 5s + reservedSystemCPUs: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus }}" + +- name: Create kubletconfig for cpumanager worker + when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is defined + kubernetes.core.k8s: + state: present + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: KubeletConfig + metadata: + name: cpumanager-enabled + spec: + machineConfigPoolSelector: + matchLabels: + custom-kubelet: cpumanager-enabled + kubeletConfig: + cpuManagerPolicy: static + cpuManagerReconcilePeriod: 5s + reservedSystemCPUs: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus }}" + +- name: Create kubletconfig for cpumanager worker + when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is not defined + kubernetes.core.k8s: + state: present + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: KubeletConfig + metadata: + name: cpumanager-enabled + machineconfiguration.openshift.io/role: worker + spec: + machineConfigPoolSelector: + matchLabels: + custom-kubelet: cpumanager-enabled + kubeletConfig: + cpuManagerPolicy: static + cpuManagerReconcilePeriod: 5s diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh new file mode 100644 index 000000000..7732ba78e --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# in order to allow unsupported SRIOV nics such as Mellanox +oc patch sriovoperatorconfig default --type=merge -n openshift-sriov-network-operator --patch '{ "spec": { "enableOperatorWebhook": false } }' + + diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/99-kargs-worker.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/99-kargs-worker.yml.j2 new file mode 100644 index 000000000..183bfb353 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/99-kargs-worker.yml.j2 @@ -0,0 +1,18 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + kubernetes.io/hostname: {{ __sap_hypervisor_node_preconfigure_register_worker_name }} + machineconfiguration.openshift.io/role: {{ __sap_hypervisor_node_preconfigure_register_worker_name }} + name: 99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker_name }} +spec: + config: + ignition: + version: 3.2.0 + kernelArguments: + - intel_iommu=on + - iommu=pt + - default_hugepagesz=1GB + - hugepagesz=1GB + - hugepages={{ __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages }} + - tsx=on diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/mcp.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/mcp.yml.j2 new file mode 100644 index 000000000..1a39d0a06 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/mcp.yml.j2 @@ -0,0 +1,47 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfigPool +metadata: + labels: + machineconfiguration.openshift.io/mco-built-in: "" + name: {{ __sap_hypervisor_node_preconfigure_register_worker.name }} +spec: + configuration: + source: + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 00-worker + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 01-worker-container-runtime + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 01-worker-kubelet + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-crio-capabilities + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-crio-seccomp-use-default + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-kubelet + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-generated-registries + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-worker-ssh + - apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + name: 99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker.name }} + machineConfigSelector: + matchExpressions: + - key: machineconfiguration.openshift.io/role + operator: In + values: + - worker + - {{ __sap_hypervisor_node_preconfigure_register_worker.name }} + nodeSelector: + matchLabels: + kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" + paused: false diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/trident-backend.json.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/trident-backend.json.j2 new file mode 100644 index 000000000..e422aab11 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/trident-backend.json.j2 @@ -0,0 +1,18 @@ +{ + "nfsMountOptions": "nfsvers=3", + "defaults": { + "exportPolicy": "default" + }, + "debug":false, + "managementLIF":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.management }}", + "dataLIF":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.data }}", + "svm":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.svm }}", + "backendName": "{{ sap_hypervisor_node_preconfigure_cluster_config.trident.backend }}", + "aggregate":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.aggregate }}", + "username":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.username }}", + "password":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.password }}", + "storageDriverName":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.storage_driver }}", + "storagePrefix":"{{ sap_hypervisor_node_preconfigure_cluster_config.trident.storage_prefix }}", + "version":1 +} + diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/tuned-virtual-host.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/tuned-virtual-host.yml new file mode 100644 index 000000000..e2dd4f483 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/tuned-virtual-host.yml @@ -0,0 +1,21 @@ +--- +- name: Set virtual-host for worker nodes + kubernetes.core.k8s: + state: present + definition: + apiVersion: tuned.openshift.io/v1 + kind: Tuned + metadata: + name: virtual-host + namespace: openshift-cluster-node-tuning-operator + spec: + profile: + - data: | + [main] + include=virtual-host + name: virtual-host + recommend: + - match: + - label: "node-role.kubernetes.io/worker" + priority: 10 + profile: virtual-host diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-configuration.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-configuration.yml index 52cd899ce..379ea44e4 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-configuration.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-configuration.yml @@ -4,11 +4,13 @@ - name: Get kernel command line ansible.builtin.command: cat /proc/cmdline register: __sap_hypervisor_node_preconfigure_kernelcmdline_assert + changed_when: __sap_hypervisor_node_preconfigure_kernelcmdline_assert.rc != 0 - name: "Assert - Kernel same page merging (KSM): Get status" - ansible.builtin.shell: systemctl status ksm + ansible.builtin.command: systemctl status ksm register: __sap_hypervisor_node_preconfigure_ksmstatus_assert ignore_errors: yes + changed_when: __sap_hypervisor_node_preconfigure_ksmstatus_assert.rc != 0 - name: "Assert - Kernel same page merging (KSM): Check if stopped" ansible.builtin.assert: @@ -18,9 +20,10 @@ ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - name: "Assert - Kernel same page merging (KSM) Tuning Daemon: Get status" - ansible.builtin.shell: systemctl status ksmtuned + ansible.builtin.command: systemctl status ksmtuned register: __sap_hypervisor_node_preconfigure_ksmtunedstatus_assert ignore_errors: yes + changed_when: __sap_hypervisor_node_preconfigure_ksmtunedstatus_assert.rc != 0 - name: "Assert - Kernel same page merging (KSM) Tuning Daemon: Check if stopped" ansible.builtin.assert: @@ -30,10 +33,12 @@ ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - name: Check CPU Stepping - ansible.builtin.shell: lscpu | awk '/Stepping/{print $2}' + ansible.builtin.shell: set -o pipefail && lscpu | awk '/Stepping/{print $2}' register: __sap_hypervisor_node_preconfigure_cpu_stepping_output_assert + changed_when: __sap_hypervisor_node_preconfigure_cpu_stepping_output_assert.rc != 0 -- set_fact: +- name: Register stepping as fact + ansible.builtin.set_fact: __sap_hypervisor_node_preconfigure_cpu_stepping_assert: "{{ __sap_hypervisor_node_preconfigure_cpu_stepping_output_assert.stdout }}" - name: Print CPU Stepping @@ -42,11 +47,13 @@ # skylake: - name: Assert - Check Intel Skylake CPU Platform + when: __sap_hypervisor_node_preconfigure_cpu_stepping_assert == "4" block: - name: Get ple_gap ansible.builtin.command: grep -E '^options\s+kvm_intel.*?ple_gap\s*=\s*0.*$' /etc/modprobe.d/kvm.conf register: __sap_hypervisor_node_preconfigure_skylake_plegap_assert ignore_errors: yes + changed_when: __sap_hypervisor_node_preconfigure_skylake_plegap_assert.rc != 0 - name: Assert - Check if ple_gap=0 ansible.builtin.assert: @@ -61,9 +68,9 @@ fail_msg: "FAIL: spectre_v2=retpoline is not on Kernel command line" success_msg: "PASS: spectre_v2=retpoline is on Kernel command line" ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - when: __sap_hypervisor_node_preconfigure_cpu_stepping_assert == "4" - name: Assert - check sap_hypervisor_node_preconfigure_nx_huge_pages + when: sap_hypervisor_node_preconfigure_nx_huge_pages is defined block: - name: "Assert - Check kvm.nx_huge_pages is {{ sap_hypervisor_node_preconfigure_nx_huge_pages }}" ansible.builtin.assert: @@ -72,13 +79,13 @@ success_msg: "PASS: kvm.nx_huge_pages is {{ sap_hypervisor_node_preconfigure_nx_huge_pages }}" ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - when: sap_hypervisor_node_preconfigure_nx_huge_pages is defined - - name: Assert - check seccomp_sanbox=0 block: - - command: grep -E '^seccomp_sandbox\s+=\s+0.*$' /etc/libvirt/qemu.conf + - name: Get seccomp setting + ansible.builtin.command: grep -E '^seccomp_sandbox\s+=\s+0.*$' /etc/libvirt/qemu.conf register: __sap_hypervisor_node_preconfigure_seccomp_assert ignore_errors: yes + changed_when: __sap_hypervisor_node_preconfigure_seccomp_assert.rc != 0 - name: "Assert - Check seccomp_sanbox=0 is in /etc/libvirt/qemu.conf" ansible.builtin.assert: @@ -90,38 +97,37 @@ - name: Assert - check amount of 1G hugepages block: - name: Get amount of 1G hugepages - ansible.builtin.shell: hugeadm --pool-list | grep 1073741824 | awk '{print $3}' - register: __sap_hypervisor_node_preconfigure_1Ghugepages_assert + ansible.builtin.shell: set -o pipefail && hugeadm --pool-list | grep 1073741824 | awk '{print $3}' + register: __sap_hypervisor_node_preconfigure_1g_hugepages_assert + changed_when: __sap_hypervisor_node_preconfigure_1g_hugepages_assert.rc != 0 - name: "Check that at least {{ sap_hypervisor_node_preconfigure_reserved_ram }} GB are available for the hypervisor and the rest are 1G hugepages" ansible.builtin.assert: - that: "{{ ( ansible_memtotal_mb / 1024 )|int - sap_hypervisor_node_preconfigure_reserved_ram }} >= {{ __sap_hypervisor_node_preconfigure_1Ghugepages_assert.stdout }}" + that: "{{ (ansible_memtotal_mb / 1024) | int - sap_hypervisor_node_preconfigure_reserved_ram }} >= {{ __sap_hypervisor_node_preconfigure_1g_hugepages_assert.stdout }}" fail_msg: "FAIL: Not enough memory reserved for hypervisor" success_msg: "PASS: Enough memory reserved for hypervisor" ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - name: Assert - check Kernel command line block: - - assert: + - name: Ensure iommu is enabled + ansible.builtin.assert: that: "'intel_iommu=on' in __sap_hypervisor_node_preconfigure_kernelcmdline_assert.stdout" fail_msg: "FAIL: intel_iommu=on not on Kernel command line" success_msg: "PASS: intel_iommu=on on Kernel command line" ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - - assert: + - name: Ensure iommu passthrough is enabled + ansible.builtin.assert: that: "'iommu=pt' in __sap_hypervisor_node_preconfigure_kernelcmdline_assert.stdout" fail_msg: "FAIL: iommu=pt not on Kernel command line" success_msg: "PASS: iommu=pt on Kernel command line" ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - - assert: + # XXX shouldn't tsx be on? + - name: Ensure tsx is off + ansible.builtin.assert: that: "'tsx=off' in __sap_hypervisor_node_preconfigure_kernelcmdline_assert.stdout" fail_msg: "FAIL: tsx=off not on Kernel command line" success_msg: "PASS: tsx=off on Kernel command line" ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_failed_assertion }}" - - -#- name: Trigger tuned profile sap-hana-kvm activation -# include_tasks: set-tuned-profile.yml -# -##### install hooks: HP, cpufreq diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-rhv-hooks.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-rhv-hooks.yml index 2abf6750e..4838f18fd 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-rhv-hooks.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-rhv-hooks.yml @@ -2,6 +2,7 @@ - name: Check file permissions ansible.builtin.command: "stat -c%a /usr/libexec/vdsm/hooks/before_vm_start/{{ item }}" register: __sap_hypervisor_node_preconfigure_register_file_permissions_assert + changed_when: __sap_hypervisor_node_preconfigure_register_file_permissions_assert.rc != 0 - name: Assert hook file permissions ansible.builtin.assert: @@ -14,16 +15,19 @@ ansible.builtin.file: path: /tmp/sap_hypervisor_node_preconfigure state: directory + mode: "0755" - name: Copy hook for checking ansible.builtin.copy: dest: "/tmp/sap_hypervisor_node_preconfigure/{{ item }}" src: "{{ item }}" + mode: "0755" - name: Diff hook ansible.builtin.command: "diff -uw /tmp/sap_hypervisor_node_preconfigure/{{ item }} /usr/libexec/vdsm/hooks/before_vm_start/{{ item }}" register: __sap_hypervisor_node_preconfigure_register_hook_diff_assert ignore_errors: yes + changed_when: __sap_hypervisor_node_preconfigure_register_hook_diff_assert.rc != 0 - name: Assert hook content ansible.builtin.assert: diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-set-tuned-profile.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-set-tuned-profile.yml index cb6508c2f..ab0d0c9b3 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-set-tuned-profile.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/assert-set-tuned-profile.yml @@ -4,6 +4,7 @@ - name: Get tuned profile ansible.builtin.command: tuned-adm active register: __sap_hypervisor_node_preconfigure_tuned_profile_assert + changed_when: __sap_hypervisor_node_preconfigure_tuned_profile_assert.rc != 0 - name: Verify tuned profile ansible.builtin.assert: diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/configuration.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/configuration.yml index e7ae07c18..b49399e4f 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/configuration.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/configuration.yml @@ -1,60 +1,29 @@ --- # tasks file for sap_hypervisor_node_preconfigure -- name: Test if kernel same page merging (KSM) exists - ansible.builtin.shell: systemctl cat ksm - register: ksm - ignore_errors: true - become: true - become_user: root - -- name: Test if kernel same page merging (KSM) tuning daemon exists - ansible.builtin.shell: systemctl cat ksmtuned - register: ksmtuned - ignore_errors: true - become: true - become_user: root - -- name: Stop kernel same page merging (KSM) - ansible.builtin.shell: systemctl stop ksm - when: ksm.rc == 0 - become: true - become_user: root - -- name: Disable kernel same page merging (KSM) - ansible.builtin.shell: systemctl disable ksm - when: ksm.rc == 0 - become: true - become_user: root - -- name: Stop Kernel Samepage Merging (KSM) Tuning Daemon - ansible.builtin.shell: systemctl stop ksmtuned - when: ksmtuned.rc == 0 - become: true - become_user: root - -- name: Disable Kernel Samepage Merging (KSM) Tuning Daemon - ansible.builtin.shell: systemctl disable ksmtuned - when: ksmtuned.rc == 0 - become: true - become_user: root +- name: Stop and disable kernel same page merging (KSM) + ansible.builtin.systemd: + name: ksm + state: stopped + enabled: false + +- name: Stop and disable kernel same page merging (KSM) tuning daemon + ansible.builtin.systemd: + name: ksmtuned + state: stopped + enabled: false - name: Check CPU Stepping - ansible.builtin.shell: lscpu | awk '/Stepping/{print $2}' + ansible.builtin.shell: set -o pipefail && lscpu | awk '/Stepping/{print $2}' register: cpu_stepping_output - become: true - become_user: root + changed_when: cpu_stepping_output.rc != 0 -- set_fact: +- name: Register CPU stepping as fact + ansible.builtin.set_fact: cpu_stepping: "{{ cpu_stepping_output.stdout }}" become: true become_user: root -- name: Print CPU Stepping - ansible.builtin.shell: echo "{{ cpu_stepping }}" - become: true - become_user: root - # skylake: - name: Set ple_gap=0 on Intel Skylake CPU Platform ansible.builtin.lineinfile: @@ -108,7 +77,7 @@ become: true become_user: root -- name: Trigger tuned profile sap-hana-kvm activation +- name: Include allocate hughepages at runtime ansible.builtin.include_tasks: allocate-hugepages-at-runtime.yml when: sap_hypervisor_node_preconfigure_reserve_hugepages == "runtime" @@ -123,7 +92,7 @@ with_items: - default_hugepagesz=1GB - hugepagesz=1GB - - hugepages={{ ( ansible_memtotal_mb / 1024 )|int - sap_hypervisor_node_preconfigure_reserved_ram }} + - hugepages={{ (ansible_memtotal_mb / 1024) | int - sap_hypervisor_node_preconfigure_reserved_ram }} notify: __sap_hypervisor_node_preconfigure_regenerate_grub2_conf_handler tags: grubconfig when: sap_hypervisor_node_preconfigure_reserve_hugepages == "static" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/main.yml index 213a45bca..a8c606aa1 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/main.yml @@ -13,13 +13,17 @@ assert_prefix: "assert-" when: sap_hypervisor_node_preconfigure_assert|d(false) -- include_tasks: '{{ assert_prefix }}installation.yml' +- name: Include "{{ assert_prefix }}installation.yml" + ansible.builtin.include_tasks: '{{ assert_prefix }}installation.yml' -- include_tasks: '{{ assert_prefix }}configuration.yml' +- name: Include "{{ assert_prefix }}configuration.yml" + ansible.builtin.include_tasks: '{{ assert_prefix }}configuration.yml' -- include_tasks: '{{ assert_prefix }}set-tuned-profile.yml' +- name: Include "{{ assert_prefix }}set-tuned-profile.yml" + ansible.builtin.include_tasks: '{{ assert_prefix }}set-tuned-profile.yml' -- include_tasks: "{{ assert_prefix }}rhv-hooks.yml" +- name: Include "{{ assert_prefix }}rhv-hooks.yml" + ansible.builtin.include_tasks: "{{ assert_prefix }}rhv-hooks.yml" loop: - "{{ role_path }}/tasks/platform/{{ sap_hypervisor_node_platform }}/50_hana" - "{{ role_path }}/tasks/platform/{{ sap_hypervisor_node_platform }}/50_iothread_pinning" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/rhv-hooks.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/rhv-hooks.yml index 045b55069..ee0d63a8d 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/rhv-hooks.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/rhv-hooks.yml @@ -3,6 +3,7 @@ ansible.builtin.file: path: /usr/libexec/vdsm/hooks/before_vm_start state: directory + mode: "0755" become: true become_user: root @@ -10,6 +11,6 @@ ansible.builtin.copy: dest: "/usr/libexec/vdsm/hooks/before_vm_start/{{ item }}" src: "{{ item }}" - mode: '0755' + mode: "0755" become: true become_user: root diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/set-tuned-profile.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/set-tuned-profile.yml index 415c4a194..91c3d7757 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/set-tuned-profile.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_rhel_kvm/set-tuned-profile.yml @@ -3,12 +3,14 @@ ansible.builtin.file: path: /usr/lib/tuned/sap-hana-kvm-host state: directory + mode: "0755" become: true become_user: root - name: Create sap-hana-kvm-host tuned profile ansible.builtin.copy: dest: "/usr/lib/tuned/sap-hana-kvm-host/tuned.conf" + mode: "0644" content: | # # tuned configuration @@ -36,5 +38,7 @@ - name: Activate tuned profile ansible.builtin.command: tuned-adm profile sap-hana-kvm-host + register: __sap_hypervisor_node_preconfigre_register_tuned_activation_output become: true become_user: root + changed_when: __sap_hypervisor_node_preconfigre_register_tuned_activation_output.rc != 0 diff --git a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml new file mode 100644 index 000000000..18a34fec4 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml @@ -0,0 +1,19 @@ +--- +# vars file for sap_hypervisor_node_preconfigure +# + +# Install the trident NFS storage provider +sap_hypervisor_node_preconfigure_install_trident: False +# URL of the trident installer package to use +sap_hypervisor_node_preconfigure_install_trident_url: https://github.com/NetApp/trident/releases/download/v23.01.0/trident-installer-23.01.0.tar.gz + +# should SRIOV be enabled for unsupported NICs +sap_hypervisor_node_preconfigure_sriov_enable_unsupported_nics: True + +# Amount of memory [GB] to be reserved for the hypervisor on hosts >= 512GB +sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512: 64 #GB +# Amount of memory [GB] to be reserved for the hypervisor on hosts < 512GB +sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512: 32 #GB + +# Should the check for the minimal amount of be ignored? Minimal amount is 96 GB +sap_hypervisor_node_preconfigure_ignore_minimal_memory_check: False From d455bbc3ebad2f6e2f312e793f3c9444a5342986 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 16 Oct 2023 16:45:05 +0200 Subject: [PATCH 02/38] updated documentation --- .../README.md | 87 ++++++------------- 1 file changed, 26 insertions(+), 61 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index 628a6a86d..83d92b8d4 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -1,27 +1,21 @@ `EXPERIMENTAL` -sap_hypervisor_node_preconfigure -======================= +# sap_hypervisor_node_preconfigure This role will configure the following hypervisors in order to run SAP workloads: -* Redhat Openshift Virtualization (OCPV) -* Redhat Enterprise Virtualization (RHV) +* Red Hat OpenShift Virtualization (OCPV) +* Red Hat Enterprise Virtualization (RHV) -Platform: Redhat Openshift Virtualization -========================================= +## Platform: Red Hat OpenShift Virtualization -This role will configure a plain vanilla Openshift cluster so it can be used for SAP workloads. +This role will configure a plain vanilla OpenShift cluster so it can be used for SAP workloads. -Requirements ------------- -A freshly installed Openshift cluster. -The worker nodes should have > 96GB of memory. -Storage is required, e.g. via NFS, Openshift Data Foundation or local storage. -This role can setup access to a Netapp Filer via Trident storage connector. -Point the `KUBECONFIG` environment variable to you `kubeconfig`. - - -Install the packages stated in `requirements.txt` on the host where the role runs. +### Requirements +* A freshly installed OpenShift cluster. +* The worker nodes should have > 96GB of memory. +* Storage is required, e.g. via NFS, OpenShift Data Foundation or local storage. This role can setup access to a Netapp Filer via Trident storage connector. +* `kubeconfig` Point the `KUBECONFIG` environment variable to you `kubeconfig`. +* Required packages: Install the packages stated in `requirements.txt` on the host where the role runs. The required packages are: ``` httpd-tools @@ -29,15 +23,14 @@ ansible-collection-kubernetes-core ``` -Make the role available in case you didn't install it already in an ansible roles directory, e.g. +* Make the role available in case you didn't install it already in an ansible roles directory, e.g. ``` mkdir -p ~/.ansible/roles/ ln -sf ~/community.sap_install/roles/sap_hypervisor_node_preconfigure ~/.ansible/roles/ ``` -Role Variables --------------- +### Role Variables General variables are defined in sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml ``` # Install the trident NFS storage provider @@ -149,32 +142,13 @@ sap_hypervisor_node_preconfigure_cluster_config: type: sriov ``` -Dependencies ------------- - +### Dependencies A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. -Example Playbook ----------------- - -See `playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml` for this example: - -``` ---- -- hosts: all - gather_facts: true - serial: 1 - vars: - sap_hypervisor_node_platform: redhat_ocp_virt +### Example Playbook +See `playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml` for an example. - tasks: - - name: Include Role - ansible.builtin.include_role: - name: sap_hypervisor_node_preconfigure -``` - -Example Usage -------------- +### Example Usage Make sure to set the `KUBECONFIG` environment variable, e.g. ``` export KUBECONFIG=~/.kubeconfig @@ -185,17 +159,14 @@ ansible-playbook --connection=local -i localhost, playbooks/sample-sap-hypervis ``` -Platform: RHEL KVM -=================== -set and check the required settings and parameters for a hypervisor running VMs for SAP HANA. - -Requirements ------------- -A RHV hypervisor. +## Platform: RHEL KVM +This Ansible Role allows preconfigure of Red Hat Virtualization (RHV), formerly called Red Hat Enterprise Virtualization (RHEV) prior to version 4.4 release. Red Hat Virtualization (RHV) consists of 'Red Hat Virtualization Manager (RHV-M)' and the 'Red Hat Virtualization Host (RHV-H)' hypervisor nodes that this Ansible Role preconfigures. Please note, Red Hat Virtualization is discontinued and available until mid-2024 in Maintenance support or mid-2026 in Extended Life support. +This Ansible Role does not preconfigure RHEL KVM (RHEL-KVM) hypervisor nodes. Please note that RHEL KVM is standalone, and does not have Management tooling (previously provided by RHV-M). -Role Variables --------------- +### Requirements +* A RHV hypervisor. +### Role Variables `sap_hypervisor_node_preconfigure_reserved_ram (default: 100)` Reserve memory [GB] for hypervisor host. Depending in the use case should be at least 50-100GB. `sap_hypervisor_node_preconfigure_reserve_hugepages (default: static)` Hugepage allocation method: {static|runtime}. @@ -227,9 +198,7 @@ runtime: done with hugeadm which is faster, but can in some cases not ensure all `sap_hypervisor_node_preconfigure_run_grub2_mkconfig (default: yes)` Update the grub2 config. -Example Playbook ----------------- - +### Example Playbook Simple example that just sets the parameters. ``` --- @@ -258,12 +227,8 @@ Run in assert mode to verify that parameters have been set. ansible.builtin.include_role: name: sap_hypervisor_node_preconfigure ``` -License -------- - +### License Apache 2.0 -Author Information ------------------- - +### Author Information Nils Koenig (nkoenig@redhat.com) From 5600c6ec7a593c538fec639fd242cc27560f490d Mon Sep 17 00:00:00 2001 From: newkit Date: Tue, 24 Oct 2023 15:50:19 +0200 Subject: [PATCH 03/38] Update roles/sap_hypervisor_node_preconfigure/README.md Co-authored-by: Felix Matouschek --- roles/sap_hypervisor_node_preconfigure/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index 83d92b8d4..013527e9e 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -46,7 +46,7 @@ sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512: 64 #GB # Amount of memory [GB] to be reserved for the hypervisor on hosts < 512GB sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512: 32 #GB -# Should the check for the minimal amount of be ignored? Minimal amount is 96 GB +# Should the check for the minimal amount of memory be ignored? Minimal amount is 96 GB # If ignored, the amount of $hostmemory - $reserved is allocated with a lower bound of 0 in case $reserved > $hostmemory sap_hypervisor_node_preconfigure_ignore_minimal_memory_check: False ``` From 0e1ce92379dfbeca56f86d0fcdf143d319c65ae6 Mon Sep 17 00:00:00 2001 From: newkit Date: Tue, 24 Oct 2023 15:50:59 +0200 Subject: [PATCH 04/38] Update roles/sap_hypervisor_node_preconfigure/README.md Co-authored-by: Felix Matouschek --- roles/sap_hypervisor_node_preconfigure/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index 013527e9e..1b333788c 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -14,7 +14,7 @@ This role will configure a plain vanilla OpenShift cluster so it can be used for * A freshly installed OpenShift cluster. * The worker nodes should have > 96GB of memory. * Storage is required, e.g. via NFS, OpenShift Data Foundation or local storage. This role can setup access to a Netapp Filer via Trident storage connector. -* `kubeconfig` Point the `KUBECONFIG` environment variable to you `kubeconfig`. +* Point the `KUBECONFIG` environment variable to your `kubeconfig`. * Required packages: Install the packages stated in `requirements.txt` on the host where the role runs. The required packages are: ``` From 8c80b9e8e9000e39d5ed059d0934b9b87a2b6890 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Tue, 24 Oct 2023 16:01:01 +0200 Subject: [PATCH 05/38] include hpp --- roles/sap_hypervisor_node_preconfigure/README.md | 3 --- .../tasks/platform/redhat_ocp_virt/main.yml | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index 83d92b8d4..22d09819b 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -142,9 +142,6 @@ sap_hypervisor_node_preconfigure_cluster_config: type: sriov ``` -### Dependencies -A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. - ### Example Playbook See `playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml` for an example. diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml index c3b61a5ab..fc1c7ab7a 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml @@ -62,3 +62,7 @@ - name: Include Trident installation ansible.builtin.include_tasks: install-trident.yml when: sap_hypervisor_node_preconfigure_install_trident + +- name: Include local storage creation (HPP) + ansible.builtin.include_tasks: install-hpp.yml + when: sap_hypervisor_node_preconfigure_install_hpp From 68f775d0f3f175fc4ad10b130e928bb0d1ec3b61 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Tue, 24 Oct 2023 16:05:50 +0200 Subject: [PATCH 06/38] added install-hpp.yml --- .../platform/redhat_ocp_virt/install-hpp.yml | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml new file mode 100644 index 000000000..544616d63 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml @@ -0,0 +1,93 @@ +- name: Create systemd files for local storage handling + kubernetes.core.k8s: + state: present + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + metadata: + annotations: + labels: + machineconfiguration.openshift.io/role: worker + name: 50-hpp-local + spec: + config: + ignition: + version: 2.2.0 + systemd: + units: + - contents: | + [Unit] + Description=Create mountpoint /var/localstorage and initialize filesystem + Before=var-localstorage.mount + [Service] + Type=oneshot + ExecStart=/bin/bash -c "if [[ $(lsblk -o FSTYPE {{ sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_device }} --noheadings) != 'xfs' ]]; then mkfs.xfs -f {{ sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_device }}; fi" + ExecStart=/bin/mkdir -p /var/localstorage + enabled: true + name: create-mountpoint-var-localstorage.service + - contents: | + [Unit] + After=create-mountpoint-var-localstorage.service + Requires=create-mountpoint-var-localstorage.service + Before=local-fs.target + [Mount] + What={{ sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_device }} + Where=/var/localstorage + Type=xfs + [Install] + WantedBy=local-fs.target + enabled: true + name: var-localstorage.mount + - contents: | + [Unit] + Description=Set SELinux chcon for hostpath provisioner + Before=kubelet.service + After=var-localstorage.mount + [Service] + ExecStart=/usr/bin/chcon -Rt container_file_t /var/localstorage + + [Install] + WantedBy=multi-user.target + enabled: true + name: hostpath-provisioner.service + +- name: Wait for mountpoint to be ready + ansible.builtin.pause: + minutes: 3 + +- name: Create hostpath provisioner (HPP) + kubernetes.core.k8s: + state: present + definition: + apiVersion: hostpathprovisioner.kubevirt.io/v1beta1 + kind: HostPathProvisioner + metadata: + name: hostpath-provisioner + spec: + imagePullPolicy: IfNotPresent + storagePools: + - name: localstorage + path: /var/localstorage + workload: + nodeSelector: + kubernetes.io/os: linux + machineconfiguration.openshift.io/role: worker + +- name: Create storage class for HPP + kubernetes.core.k8s: + state: present + definition: + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: local + annotations: + storageclass.kubernetes.io/is-default-class: "true" + provisioner: kubevirt.io.hostpath-provisioner + reclaimPolicy: Delete + volumeBindingMode: WaitForFirstConsumer + parameters: + storagePool: localstorage + + + From b743620702d43f81a26954cec05acc0448a30cc3 Mon Sep 17 00:00:00 2001 From: newkit Date: Tue, 24 Oct 2023 16:06:26 +0200 Subject: [PATCH 07/38] Update roles/sap_hypervisor_node_preconfigure/README.md Co-authored-by: Felix Matouschek --- roles/sap_hypervisor_node_preconfigure/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index 1b333788c..b86018e99 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -13,7 +13,7 @@ This role will configure a plain vanilla OpenShift cluster so it can be used for ### Requirements * A freshly installed OpenShift cluster. * The worker nodes should have > 96GB of memory. -* Storage is required, e.g. via NFS, OpenShift Data Foundation or local storage. This role can setup access to a Netapp Filer via Trident storage connector. +* Storage is required, e.g. via NFS, OpenShift Data Foundation or local storage. This role can setup access to a Netapp Filer via Trident storage connector. * Point the `KUBECONFIG` environment variable to your `kubeconfig`. * Required packages: Install the packages stated in `requirements.txt` on the host where the role runs. The required packages are: From 8ac014c1283c0601559d57afc9d0b0591488c62d Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Thu, 26 Oct 2023 15:00:22 +0200 Subject: [PATCH 08/38] updated doc --- .../README.md | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index 08f2b6edd..699993bb1 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -15,14 +15,7 @@ This role will configure a plain vanilla OpenShift cluster so it can be used for * The worker nodes should have > 96GB of memory. * Storage is required, e.g. via NFS, OpenShift Data Foundation or local storage. This role can setup access to a Netapp Filer via Trident storage connector. * Point the `KUBECONFIG` environment variable to your `kubeconfig`. -* Required packages: Install the packages stated in `requirements.txt` on the host where the role runs. -The required packages are: -``` -httpd-tools -ansible-collection-kubernetes-core -``` - - +* Required packages: This roles uses the kubernetes ansible module, this can be installed via the package`ansible-collection-kubernetes-core`. * Make the role available in case you didn't install it already in an ansible roles directory, e.g. ``` @@ -33,13 +26,14 @@ ln -sf ~/community.sap_install/roles/sap_hypervisor_node_preconfigure ~/.ansible ### Role Variables General variables are defined in sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml ``` -# Install the trident NFS storage provider -sap_hypervisor_node_preconfigure_install_trident: False +# Install the trident NFS storage provider. If yes, expects configuration details under +# sap_hypervisor_node_preconfigure_cluster_config.trident, see example config. +sap_hypervisor_node_preconfigure_install_trident: True|False # URL of the trident installer package to use sap_hypervisor_node_preconfigure_install_trident_url: https://github.com/NetApp/trident/releases/download/v23.01.0/trident-installer-23.01.0.tar.gz # should SRIOV be enabled for unsupported NICs -sap_hypervisor_node_preconfigure_sriov_enable_unsupported_nics: True +sap_hypervisor_node_preconfigure_sriov_enable_unsupported_nics: True|False # Amount of memory [GB] to be reserved for the hypervisor on hosts >= 512GB sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512: 64 #GB @@ -48,7 +42,12 @@ sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512: 32 #GB # Should the check for the minimal amount of memory be ignored? Minimal amount is 96 GB # If ignored, the amount of $hostmemory - $reserved is allocated with a lower bound of 0 in case $reserved > $hostmemory -sap_hypervisor_node_preconfigure_ignore_minimal_memory_check: False +sap_hypervisor_node_preconfigure_ignore_minimal_memory_check: True|False + +# Define if the host path provisioner should be installed in order to use a local disk as storage device. +# Uses the following variable to be set to the storage device to be used, e.g.: +# sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_device: /dev/sdb +sap_hypervisor_node_preconfigure_install_hpp: True|False ``` The following variables are describing the nodes and networks to be used. It can make sense to have them in a seperate file, e.g. see `playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml` for an example. @@ -74,11 +73,14 @@ sap_hypervisor_node_preconfigure_cluster_config: storage_driver: ontap-nas storage_prefix: ocpv_sap_ + # CPU cores which will be reserved for kubernetes + worker_kubernetes_reserved_cpus: "0,1" + + # Storage device used for host path provisioner as local storage. + worker_localstorage_device: /dev/vdb + # detailed configuration for every worker that should be configured workers: - kubernetes_reserved_cpus: "0,1" # CPU cores reserved for - # kubernetes - - name: worker-0 # name must match the node name networks: # Example network config - name: sapbridge # using a bridge From 1000071cfe397fd415034f616751ab80d54844c4 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Thu, 26 Oct 2023 16:37:41 +0200 Subject: [PATCH 09/38] WIP: restructure sap_hypervisor_node_preconfigure --- .../handlers/main.yml | 2 +- .../redhat_ocp_virt/99-kargs-worker.yml.j2 | 5 +- .../redhat_ocp_virt/configure-worker-node.yml | 6 -- .../redhat_ocp_virt/download-rhel-images.yml | 21 ------- .../redhat_ocp_virt/enable-cpumanager.yml | 61 ------------------- .../tasks/platform/redhat_ocp_virt/main.yml | 32 ++++++++++ .../tasks/platform/redhat_ocp_virt/mcp.yml.j2 | 47 -------------- .../redhat_ocp_virt/setup-worker-node.yml | 38 ++++-------- 8 files changed, 47 insertions(+), 165 deletions(-) delete mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml delete mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml delete mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 diff --git a/roles/sap_hypervisor_node_preconfigure/handlers/main.yml b/roles/sap_hypervisor_node_preconfigure/handlers/main.yml index f920c7196..5150d6bdd 100644 --- a/roles/sap_hypervisor_node_preconfigure/handlers/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/handlers/main.yml @@ -1,4 +1,4 @@ --- -- name: SAP certified hypervisor node preconfigure - Include Handler Tasks for {{ sap_hypervisor_node_platform }} +- name: hypervisor node preconfigure - Include Handler Tasks for {{ sap_hypervisor_node_platform }} ansible.builtin.include_tasks: "{{ role_path }}/handlers/platform/{{ sap_hypervisor_node_platform }}/main.yml" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 index 183bfb353..32064a8d7 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/99-kargs-worker.yml.j2 @@ -2,9 +2,8 @@ apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: - kubernetes.io/hostname: {{ __sap_hypervisor_node_preconfigure_register_worker_name }} - machineconfiguration.openshift.io/role: {{ __sap_hypervisor_node_preconfigure_register_worker_name }} - name: 99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker_name }} + machineconfiguration.openshift.io/role: worker + name: 99-kargs-worker spec: config: ignition: diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml index 82fec7ac3..068bbf1d8 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml @@ -7,12 +7,6 @@ index_var: __sap_hypervisor_node_preconfigure_register_worker_network_nr when: __sap_hypervisor_node_preconfigure_register_worker.networks is defined -- name: "Create MCP for {{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - kubernetes.core.k8s: - template: - path: "mcp.yml.j2" - state: present - - name: Pause so cluster can process config ansible.builtin.pause: minutes: 1 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml deleted file mode 100644 index 6dd050ea9..000000000 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/download-rhel-images.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -- name: "Download rhel 8.6 image" - kubernetes.core.k8s: - state: present - definition: - apiVersion: cdi.kubevirt.io/v1beta1 - kind: DataVolume - metadata: - namespace: openshift-virtualization-os-images - name: rhel-86 - annotations: - cdi.kubevirt.io/storage.bind.immediate.requested: 'true' - spec: - source: - registry: - url: 'docker://registry.redhat.io/rhel8/rhel-guest-image:8.6.0' - pullMethod: node - storage: - resources: - requests: - storage: 10Gi diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml deleted file mode 100644 index 25e88c1b2..000000000 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/enable-cpumanager.yml +++ /dev/null @@ -1,61 +0,0 @@ ---- -- name: Label nodes - ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} \ - feature.node.kubernetes.io/cpu-feature-invtsc=true --overwrite=true" - register: __sap_hypervisor_node_preconfigure_register_label_node - changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 - -- name: Enable CPU Manager by patching MCP of "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - kubernetes.core.k8s: - state: patched - definition: - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfigPool - metadata: - name: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - labels: - custom-kubelet: "cpumanager-enabled" - -- name: Create kubletconfig for cpumanager "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is defined - kubernetes.core.k8s: - state: present - definition: - apiVersion: machineconfiguration.openshift.io/v1 - kind: KubeletConfig - metadata: - name: "cpumanager-enabled" - kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - machineconfiguration.openshift.io/role: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - spec: - machineConfigPoolSelector: - matchLabels: - custom-kubelet: "cpumanager-enabled" - kubeletConfig: - cpuManagerPolicy: static - cpuManagerReconcilePeriod: 5s - reservedSystemCPUs: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus }}" - -- name: Create kubletconfig for cpumanager "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is not defined - kubernetes.core.k8s: - state: present - definition: - apiVersion: machineconfiguration.openshift.io/v1 - kind: KubeletConfig - metadata: - name: "cpumanager-enabled" - kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - machineconfiguration.openshift.io/role: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - spec: - machineConfigPoolSelector: - matchLabels: - custom-kubelet: "cpumanager-enabled" - kubeletConfig: - cpuManagerPolicy: static - cpuManagerReconcilePeriod: 5s - -- name: Label nodes - ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} cpumanager=true --overwrite=true" - register: __sap_hypervisor_node_preconfigure_register_label_node - changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml index fc1c7ab7a..e846f712d 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml @@ -29,6 +29,37 @@ fail_msg: No nodes found that match configuration provided in sap_hypervisor_node_preconfigure_cluster success_msg: Configured nodes found +# Determine available memory on first worker node. +# This amount will be used for all nodes, so make sure all have an identical amount. +- name: Get worker name + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_name: + "{{ __sap_hypervisor_node_preconfigure_register_nodes[0]['metadata']['labels']['kubernetes.io/hostname'] }}" + +- name: Get memory of worker node + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_memory_gb: + "{{ (__sap_hypervisor_node_preconfigure_register_nodes[0]['status']['capacity']['memory'] | replace('Ki', '') | int / 1048576) | int }}" + +- name: Check if host has minimal amount of memory (96Gb) + ansible.builtin.assert: + that: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int >= 96 + fail_msg: "Not enough memory on node {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" + success_msg: "Enough memory on node {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" + ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_minimal_memory_check }}" + +# calculate memory to be allocated as hugepages +# if system < 512GB memory use 32GB as upper boundary, 64GB otherwise as upper boundary +- name: Calculate amount of hugepages to reserve (host memory < 512 Gb) + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gb | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512 }}" + when: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int < 512 + +- name: Calculate amount of hugepages to reserve (host memory >= 512 Gb) + ansible.builtin.set_fact: + __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gb | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512 }}" + when: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int >= 512 + # - meta: end_play - name: Include prepare @@ -66,3 +97,4 @@ - name: Include local storage creation (HPP) ansible.builtin.include_tasks: install-hpp.yml when: sap_hypervisor_node_preconfigure_install_hpp + diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 deleted file mode 100644 index 1a39d0a06..000000000 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/mcp.yml.j2 +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfigPool -metadata: - labels: - machineconfiguration.openshift.io/mco-built-in: "" - name: {{ __sap_hypervisor_node_preconfigure_register_worker.name }} -spec: - configuration: - source: - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 00-worker - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 01-worker-container-runtime - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 01-worker-kubelet - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-generated-crio-capabilities - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-generated-crio-seccomp-use-default - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-generated-kubelet - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-generated-registries - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-ssh - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker.name }} - machineConfigSelector: - matchExpressions: - - key: machineconfiguration.openshift.io/role - operator: In - values: - - worker - - {{ __sap_hypervisor_node_preconfigure_register_worker.name }} - nodeSelector: - matchLabels: - kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - paused: false diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml index 2dc78034f..e144f1a9b 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml @@ -1,11 +1,4 @@ --- -- name: Include configure kargs - ansible.builtin.include_tasks: configure-kargs-per-node.yml - with_items: "{{ __sap_hypervisor_node_preconfigure_register_nodes }}" - loop_control: - loop_var: __sap_hypervisor_node_preconfigure_register_worker - index_var: __sap_hypervisor_node_preconfigure_register_worker_nr - - name: Include configure worker ansible.builtin.include_tasks: configure-worker-node.yml with_items: "{{ sap_hypervisor_node_preconfigure_cluster_config.workers }}" @@ -24,8 +17,7 @@ labels: custom-kubelet: cpumanager-enabled - -- name: Create kubletconfig for cpumanager worker +- name: Create kubletconfig for cpumanager worker with CPUs reserved for kubernetes when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is defined kubernetes.core.k8s: state: absent @@ -44,7 +36,7 @@ reservedSystemCPUs: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus }}" - name: Create kubletconfig for cpumanager worker - when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is defined + when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is not defined kubernetes.core.k8s: state: present definition: @@ -52,6 +44,7 @@ kind: KubeletConfig metadata: name: cpumanager-enabled + machineconfiguration.openshift.io/role: worker spec: machineConfigPoolSelector: matchLabels: @@ -59,22 +52,15 @@ kubeletConfig: cpuManagerPolicy: static cpuManagerReconcilePeriod: 5s - reservedSystemCPUs: "{{ sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus }}" -- name: Create kubletconfig for cpumanager worker - when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is not defined +- name: Personalize template + ansible.builtin.template: + src: 99-kargs-worker.yml.j2 + dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-worker.yml.j2" + mode: "0644" + +- name: Enable hugepages kubernetes.core.k8s: state: present - definition: - apiVersion: machineconfiguration.openshift.io/v1 - kind: KubeletConfig - metadata: - name: cpumanager-enabled - machineconfiguration.openshift.io/role: worker - spec: - machineConfigPoolSelector: - matchLabels: - custom-kubelet: cpumanager-enabled - kubeletConfig: - cpuManagerPolicy: static - cpuManagerReconcilePeriod: 5s + src: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-worker.yml.j2" + From 2edc92a0988fec62ff06ce0a19a3be4dbabee4fa Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 30 Oct 2023 16:53:38 +0100 Subject: [PATCH 10/38] variable name fix (minor) --- .../tasks/platform/redhat_ocp_virt/main.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml index e846f712d..6e0e5d0d4 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml @@ -26,7 +26,7 @@ - name: Assert that configured nodes are found ansible.builtin.assert: that: __sap_hypervisor_node_preconfigure_register_nodes is defined - fail_msg: No nodes found that match configuration provided in sap_hypervisor_node_preconfigure_cluster + fail_msg: No nodes found that match configuration provided in sap_hypervisor_node_preconfigure_cluster_config success_msg: Configured nodes found # Determine available memory on first worker node. @@ -60,8 +60,6 @@ __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gb | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512 }}" when: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int >= 512 -# - meta: end_play - - name: Include prepare ansible.builtin.include_tasks: prepare.yml - name: Include tuned virtual host From 34a8dcab698828511fe443022d3356257675a934 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 30 Oct 2023 18:03:16 +0100 Subject: [PATCH 11/38] fixed missing vars --- .../sap_hypervisor_node_preconfigure/defaults/main.yml | 10 ++++++---- .../vars/platform_defaults_redhat_ocp_virt.yml | 3 +++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/defaults/main.yml b/roles/sap_hypervisor_node_preconfigure/defaults/main.yml index 479c11670..6dc76587f 100644 --- a/roles/sap_hypervisor_node_preconfigure/defaults/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/defaults/main.yml @@ -26,12 +26,14 @@ sap_hypervisor_node_preconfigure_cluster_config: storage_driver: ontap-nas storage_prefix: ocpv_sap_ + # CPU cores reserved for kubernetes on worker node + worker_kubernetes_reserved_cpus: "0,1" + + # Storage device which should be used if host path provisioner is used + worker_localstorage_device: /dev/vdb + # detailed configuration for every worker that should be configured - # workers: - - kubernetes_reserved_cpus: "0,1" # CPU cores reserved for - # kubernetes - - name: worker-0 # name must match the node name networks: # Example network config - name: sapbridge # using a bridge diff --git a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml index eed5209cd..be422195e 100644 --- a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml +++ b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml @@ -2,6 +2,9 @@ # vars file for sap_hypervisor_node_preconfigure # +# Install and configure the host path provisioner (hpp) for a local storage disk +sap_hypervisor_node_preconfigure_install_hpp: False + # Install the trident NFS storage provider sap_hypervisor_node_preconfigure_install_trident: False # URL of the trident installer package to use From ec5b08302d5e56a51558ce0e2483d6dd03838b75 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Tue, 31 Oct 2023 15:26:19 +0100 Subject: [PATCH 12/38] removed unnecessary files --- .../templates/99-kargs-worker.yml.j2 | 18 ------- .../templates/templates/mcp.yml.j2 | 47 ------------------- 2 files changed, 65 deletions(-) delete mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/99-kargs-worker.yml.j2 delete mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/mcp.yml.j2 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/99-kargs-worker.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/99-kargs-worker.yml.j2 deleted file mode 100644 index 183bfb353..000000000 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/99-kargs-worker.yml.j2 +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - kubernetes.io/hostname: {{ __sap_hypervisor_node_preconfigure_register_worker_name }} - machineconfiguration.openshift.io/role: {{ __sap_hypervisor_node_preconfigure_register_worker_name }} - name: 99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker_name }} -spec: - config: - ignition: - version: 3.2.0 - kernelArguments: - - intel_iommu=on - - iommu=pt - - default_hugepagesz=1GB - - hugepagesz=1GB - - hugepages={{ __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages }} - - tsx=on diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/mcp.yml.j2 b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/mcp.yml.j2 deleted file mode 100644 index 1a39d0a06..000000000 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/templates/templates/mcp.yml.j2 +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfigPool -metadata: - labels: - machineconfiguration.openshift.io/mco-built-in: "" - name: {{ __sap_hypervisor_node_preconfigure_register_worker.name }} -spec: - configuration: - source: - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 00-worker - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 01-worker-container-runtime - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 01-worker-kubelet - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-generated-crio-capabilities - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-generated-crio-seccomp-use-default - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-generated-kubelet - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-generated-registries - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-worker-ssh - - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - name: 99-kargs-{{ __sap_hypervisor_node_preconfigure_register_worker.name }} - machineConfigSelector: - matchExpressions: - - key: machineconfiguration.openshift.io/role - operator: In - values: - - worker - - {{ __sap_hypervisor_node_preconfigure_register_worker.name }} - nodeSelector: - matchLabels: - kubernetes.io/hostname: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - paused: false From 4afbe8436cc89afed2a6041191eb0df8a2318fa4 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Thu, 2 Nov 2023 19:33:15 +0100 Subject: [PATCH 13/38] added label cpumanager --- .../README.md | 2 +- .../configure-kargs-per-node.yml | 34 ------------------- .../redhat_ocp_virt/configure-worker-node.yml | 4 +++ 3 files changed, 5 insertions(+), 35 deletions(-) delete mode 100644 roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-kargs-per-node.yml diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index 699993bb1..1119aaa11 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -8,7 +8,7 @@ This role will configure the following hypervisors in order to run SAP workloads ## Platform: Red Hat OpenShift Virtualization -This role will configure a plain vanilla OpenShift cluster so it can be used for SAP workloads. +Will configure a plain vanilla OpenShift cluster so it can be used for SAP workloads. ### Requirements * A freshly installed OpenShift cluster. diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-kargs-per-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-kargs-per-node.yml deleted file mode 100644 index 488c3eaa9..000000000 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-kargs-per-node.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -- name: Get worker name - ansible.builtin.set_fact: - __sap_hypervisor_node_preconfigure_register_worker_name: - "{{ __sap_hypervisor_node_preconfigure_register_worker['metadata']['labels']['kubernetes.io/hostname'] }}" - -- name: Get memory of worker node - ansible.builtin.set_fact: - __sap_hypervisor_node_preconfigure_register_worker_memory_gb: - "{{ (__sap_hypervisor_node_preconfigure_register_worker['status']['capacity']['memory'] | replace('Ki', '') | int / 1048576) | int }}" - -- name: Check if host has minimal amount of memory (96Gb) - ansible.builtin.assert: - that: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int >= 96 - fail_msg: "Not enough memory on node {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" - success_msg: "Enough memory on node {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" - ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_minimal_memory_check }}" - -# calculate memory to be allocated as hugepages -# if system < 512GB memory use 32GB as upper boundary, 64GB otherwise as upper boundary -- name: Calculate amount of hugepages to reserve (host memory < 512 Gb) - ansible.builtin.set_fact: - __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: - "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gb | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512 }}" - when: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int < 512 - -- name: Calculate amount of hugepages to reserve (host memory >= 512 Gb) - ansible.builtin.set_fact: - __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: - "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gb | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512 }}" - when: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int >= 512 - -- name: "Include kargs for {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" - ansible.builtin.include_tasks: kargs.yml diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml index 068bbf1d8..328ae3705 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml @@ -1,4 +1,8 @@ --- +- name: Label nodes + command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker }} cpumanager=true --overwrite=true" + #XXX + - name: Include node network ansible.builtin.include_tasks: node-network.yml with_items: "{{ __sap_hypervisor_node_preconfigure_register_worker.networks }}" From 1bf351b72b9bb7dd37771c6e7e135a31127d4232 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Thu, 2 Nov 2023 20:53:54 +0100 Subject: [PATCH 14/38] fixed worker node name when labeling for cpumanger --- .../tasks/platform/redhat_ocp_virt/configure-worker-node.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml index 328ae3705..14f13363b 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml @@ -1,6 +1,6 @@ --- - name: Label nodes - command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker }} cpumanager=true --overwrite=true" + command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} cpumanager=true --overwrite=true" #XXX - name: Include node network From 288b76f478c004aa5c0e4f47b3b593ea1f7183e5 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 6 Nov 2023 11:12:57 +0100 Subject: [PATCH 15/38] removed misplaced kubernetes_reserved_cpu --- ...bles-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml b/playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml index a2747ac4f..08539fb21 100644 --- a/playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml +++ b/playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml @@ -20,12 +20,15 @@ sap_hypervisor_node_preconfigure_cluster_config: storage_driver: ontap-nas storage_prefix: ocpv_sap_ + # CPU cores which will be reserved for kubernetes + worker_kubernetes_reserved_cpus: "0,1" + + # Storage device used for host path provisioner as local storage. + worker_localstorage_device: /dev/vdb + # detailed configuration for every worker that should be configured # workers: - - kubernetes_reserved_cpus: "0,1" # CPU cores reserved for - # kubernetes - - name: worker-0 # name must match the node name networks: # Example network config - name: sapbridge # using a bridge From ea875e6c31a6772ea902a2ce1766a95ebace0364 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 6 Nov 2023 11:17:17 +0100 Subject: [PATCH 16/38] added info on required dependencies --- roles/sap_hypervisor_node_preconfigure/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index 1119aaa11..70a92baef 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -23,6 +23,12 @@ mkdir -p ~/.ansible/roles/ ln -sf ~/community.sap_install/roles/sap_hypervisor_node_preconfigure ~/.ansible/roles/ ``` +### Dependencies + +Needs the ansible kubernetes module and the python3 kubernetes binding. On a RHEL based system the are named +* python3-kubernetes +* ansible-collection-kubernetes-core + ### Role Variables General variables are defined in sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml ``` From ce45a58e3cb9382f1387c88ed3c5fb80d4123ec4 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 6 Nov 2023 12:30:17 +0100 Subject: [PATCH 17/38] implemented ansible-lint suggestions --- ...pervisor-node-preconfigure-rh_ocp_virt.yml | 120 +++++++++--------- .../README.md | 5 +- .../defaults/main.yml | 116 ++++++++--------- .../cnv-namespace-operator-subscription.yml | 33 ----- .../sriov-namespace-operator-subscription.yml | 26 ---- .../redhat_ocp_virt/configure-worker-node.yml | 5 +- .../platform/redhat_ocp_virt/install-hpp.yml | 34 +++-- .../install-sriov-operator.yml | 3 +- .../tasks/platform/redhat_ocp_virt/main.yml | 9 +- .../redhat_ocp_virt/setup-worker-node.yml | 3 +- .../sriov-enabled-unsupported-nics.sh | 2 - 11 files changed, 144 insertions(+), 212 deletions(-) delete mode 100644 roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/cnv-namespace-operator-subscription.yml delete mode 100644 roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-namespace-operator-subscription.yml diff --git a/playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml b/playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml index 08539fb21..6f7f9af81 100644 --- a/playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml +++ b/playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml @@ -5,7 +5,7 @@ sap_hypervisor_node_preconfigure_cluster_config: # namespace under which the VMs are created, note this has to be # openshift-sriov-network-operator in case of using SRIOV network - # devices + # devices vm_namespace: sap # Optional, configuration for trident driver for Netapp NFS filer @@ -29,64 +29,64 @@ sap_hypervisor_node_preconfigure_cluster_config: # detailed configuration for every worker that should be configured # workers: - - name: worker-0 # name must match the node name - networks: # Example network config - - name: sapbridge # using a bridge - description: SAP bridge - state: up - type: linux-bridge - ipv4: - enabled: false - auto-gateway: false - auto-dns: false - bridge: - options: - stp: - enabled: false - port: - - name: ens1f0 # network IF name - - name: storage # an SRIOV device - interface: ens2f0 # network IF name - type: sriov + - name: worker-0 # name must match the node name + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov - - bridge: # another bridge - options: - stp: - enabled: false - port: - - name: ens2f0 # network IF name - description: storage - mtu: 9000 - ipv4: - address: - - ip: 192.168.1.51 # IP config - prefix-length: 24 - auto-dns: false - auto-gateway: false - enabled: true - name: storagebridge - state: up - type: linux-bridge - - name: multi # another SRIOV device - interface: ens2f1 # network IF name - type: sriov + - bridge: # another bridge + options: + stp: + enabled: false + port: + - name: ens2f0 # network IF name + description: storage + mtu: 9000 + ipv4: + address: + - ip: 192.168.1.51 # IP config + prefix-length: 24 + auto-dns: false + auto-gateway: false + enabled: true + name: storagebridge + state: up + type: linux-bridge + - name: multi # another SRIOV device + interface: ens2f1 # network IF name + type: sriov - - name: worker-1 # second worker configuration - networks: # Example network config - - name: sapbridge # using a bridge - description: SAP bridge - state: up - type: linux-bridge - ipv4: - enabled: false - auto-gateway: false - auto-dns: false - bridge: - options: - stp: - enabled: false - port: - - name: ens1f0 # network IF name - - name: storage # an SRIOV device - interface: ens2f0 # network IF name - type: sriov + - name: worker-1 # second worker configuration + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index 70a92baef..0a600dc6b 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -22,7 +22,6 @@ Will configure a plain vanilla OpenShift cluster so it can be used for SAP workl mkdir -p ~/.ansible/roles/ ln -sf ~/community.sap_install/roles/sap_hypervisor_node_preconfigure ~/.ansible/roles/ ``` - ### Dependencies Needs the ansible kubernetes module and the python3 kubernetes binding. On a RHEL based system the are named @@ -55,7 +54,6 @@ sap_hypervisor_node_preconfigure_ignore_minimal_memory_check: True|False # sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_device: /dev/sdb sap_hypervisor_node_preconfigure_install_hpp: True|False ``` - The following variables are describing the nodes and networks to be used. It can make sense to have them in a seperate file, e.g. see `playbooks/vars/sample-variables-sap-hypervisor-node-preconfigure-rh_ocp_virt.yml` for an example. ``` sap_hypervisor_node_preconfigure_cluster_config: @@ -163,9 +161,8 @@ To invoke the example playbook with the example configuration using your localho ansible-playbook --connection=local -i localhost, playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml -e @s/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml ``` - ## Platform: RHEL KVM -This Ansible Role allows preconfigure of Red Hat Virtualization (RHV), formerly called Red Hat Enterprise Virtualization (RHEV) prior to version 4.4 release. Red Hat Virtualization (RHV) consists of 'Red Hat Virtualization Manager (RHV-M)' and the 'Red Hat Virtualization Host (RHV-H)' hypervisor nodes that this Ansible Role preconfigures. Please note, Red Hat Virtualization is discontinued and available until mid-2024 in Maintenance support or mid-2026 in Extended Life support. +This Ansible Role allows preconfigure of Red Hat Virtualization (RHV), formerly called Red Hat Enterprise Virtualization (RHEV) prior to version 4.4 release. Red Hat Virtualization (RHV) consists of 'Red Hat Virtualization Manager (RHV-M)' and the 'Red Hat Virtualization Host (RHV-H)' hypervisor nodes that this Ansible Role preconfigures. Please note, Red Hat Virtualization is discontinued and maintenance support will end mid-2024. Extended life support for RHV ends mid-2026. This Ansible Role does not preconfigure RHEL KVM (RHEL-KVM) hypervisor nodes. Please note that RHEL KVM is standalone, and does not have Management tooling (previously provided by RHV-M). ### Requirements diff --git a/roles/sap_hypervisor_node_preconfigure/defaults/main.yml b/roles/sap_hypervisor_node_preconfigure/defaults/main.yml index 6dc76587f..a7e35e177 100644 --- a/roles/sap_hypervisor_node_preconfigure/defaults/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/defaults/main.yml @@ -11,7 +11,7 @@ sap_hypervisor_node_preconfigure_cluster_config: # namespace under which the VMs are created, note this has to be # openshift-sriov-network-operator in case of using SRIOV network - # devices + # devices vm_namespace: sap # Optional, configuration for trident driver for Netapp NFS filer @@ -34,64 +34,64 @@ sap_hypervisor_node_preconfigure_cluster_config: # detailed configuration for every worker that should be configured workers: - - name: worker-0 # name must match the node name - networks: # Example network config - - name: sapbridge # using a bridge - description: SAP bridge - state: up - type: linux-bridge - ipv4: - enabled: false - auto-gateway: false - auto-dns: false - bridge: - options: - stp: - enabled: false - port: - - name: ens1f0 # network IF name - - name: storage # an SRIOV device - interface: ens2f0 # network IF name - type: sriov + - name: worker-0 # name must match the node name + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov - - bridge: # another bridge - options: - stp: - enabled: false - port: - - name: ens2f0 # network IF name - description: storage - mtu: 9000 - ipv4: - address: + - bridge: # another bridge + options: + stp: + enabled: false + port: + - name: ens2f0 # network IF name + description: storage + mtu: 9000 + ipv4: + address: - ip: 192.168.1.51 # IP config prefix-length: 24 - auto-dns: false - auto-gateway: false - enabled: true - name: storagebridge - state: up - type: linux-bridge - - name: multi # another SRIOV device - interface: ens2f1 # network IF name - type: sriov + auto-dns: false + auto-gateway: false + enabled: true + name: storagebridge + state: up + type: linux-bridge + - name: multi # another SRIOV device + interface: ens2f1 # network IF name + type: sriov - - name: worker-1 # second worker configuration - networks: # Example network config - - name: sapbridge # using a bridge - description: SAP bridge - state: up - type: linux-bridge - ipv4: - enabled: false - auto-gateway: false - auto-dns: false - bridge: - options: - stp: - enabled: false - port: - - name: ens1f0 # network IF name - - name: storage # an SRIOV device - interface: ens2f0 # network IF name - type: sriov + - name: worker-1 # second worker configuration + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov diff --git a/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/cnv-namespace-operator-subscription.yml b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/cnv-namespace-operator-subscription.yml deleted file mode 100644 index 2d8b3feab..000000000 --- a/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/cnv-namespace-operator-subscription.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: openshift-cnv ---- -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: kubevirt-hyperconverged-group - namespace: openshift-cnv -spec: - targetNamespaces: - - openshift-cnv ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: hco-operatorhub - namespace: openshift-cnv -spec: - source: redhat-operators - sourceNamespace: openshift-marketplace - name: kubevirt-hyperconverged -# startingCSV: kubevirt-hyperconverged-operator.v4.10.0 -# channel: "stable" ---- -apiVersion: hco.kubevirt.io/v1beta1 -kind: HyperConverged -metadata: - name: kubevirt-hyperconverged - namespace: openshift-cnv -spec: diff --git a/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-namespace-operator-subscription.yml b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-namespace-operator-subscription.yml deleted file mode 100644 index 9451b3401..000000000 --- a/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-namespace-operator-subscription.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: openshift-sriov-network-operator ---- -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: sriov-network-operators - namespace: openshift-sriov-network-operator -spec: - targetNamespaces: - - openshift-sriov-network-operator ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: sriov-network-operator-subscription - namespace: openshift-sriov-network-operator -spec: - source: redhat-operators - sourceNamespace: openshift-marketplace - name: sriov-network-operator -# startingCSV: sriov-network-operator - channel: "stable" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml index 14f13363b..99b7599ec 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml @@ -1,7 +1,8 @@ --- - name: Label nodes - command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} cpumanager=true --overwrite=true" - #XXX + ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} cpumanager=true --overwrite=true" + register: __sap_hypervisor_node_preconfigure_label_node_result + changed_when: __sap_hypervisor_node_preconfigure_label_node_result.rc != 0 - name: Include node network ansible.builtin.include_tasks: node-network.yml diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml index 544616d63..04e4941bb 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml @@ -1,20 +1,21 @@ +--- - name: Create systemd files for local storage handling kubernetes.core.k8s: state: present definition: - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - metadata: - annotations: - labels: - machineconfiguration.openshift.io/role: worker - name: 50-hpp-local - spec: - config: - ignition: - version: 2.2.0 - systemd: - units: + apiVersion: machineconfiguration.openshift.io/v1 + kind: MachineConfig + metadata: + annotations: + labels: + machineconfiguration.openshift.io/role: worker + name: 50-hpp-local + spec: + config: + ignition: + version: 2.2.0 + systemd: + units: - contents: | [Unit] Description=Create mountpoint /var/localstorage and initialize filesystem @@ -66,8 +67,8 @@ spec: imagePullPolicy: IfNotPresent storagePools: - - name: localstorage - path: /var/localstorage + - name: localstorage + path: /var/localstorage workload: nodeSelector: kubernetes.io/os: linux @@ -88,6 +89,3 @@ volumeBindingMode: WaitForFirstConsumer parameters: storagePool: localstorage - - - diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-sriov-operator.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-sriov-operator.yml index 1379b1152..3eba6793c 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-sriov-operator.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-sriov-operator.yml @@ -20,6 +20,7 @@ spec: targetNamespaces: - openshift-sriov-network-operator + - name: Create the CNV Operator namespace kubernetes.core.k8s: state: present @@ -33,10 +34,8 @@ source: redhat-operators sourceNamespace: openshift-marketplace name: sriov-network-operator - # startingCSV: sriov-network-operator channel: "stable" - - name: Pause to give operator a chance to install ansible.builtin.pause: minutes: 3 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml index 6e0e5d0d4..8bb001fd8 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml @@ -29,7 +29,7 @@ fail_msg: No nodes found that match configuration provided in sap_hypervisor_node_preconfigure_cluster_config success_msg: Configured nodes found -# Determine available memory on first worker node. +# Determine available memory on first worker node. # This amount will be used for all nodes, so make sure all have an identical amount. - name: Get worker name ansible.builtin.set_fact: @@ -66,13 +66,13 @@ ansible.builtin.include_tasks: tuned-virtual-host.yml - name: Include install CNV operator ansible.builtin.include_tasks: install-cnv-operator.yml - when: sap_hypervisor_node_preconfigure_install_operators == True + when: sap_hypervisor_node_preconfigure_install_operators - name: Include install sriov operator ansible.builtin.include_tasks: install-sriov-operator.yml - when: sap_hypervisor_node_preconfigure_install_operators == True + when: sap_hypervisor_node_preconfigure_install_operators - name: Include install nmstate operator ansible.builtin.include_tasks: install-nmstate-operator.yml - when: sap_hypervisor_node_preconfigure_install_operators == True + when: sap_hypervisor_node_preconfigure_install_operators - name: Include install virtctl ansible.builtin.include_tasks: install-virtctl.yml - name: Include setup worker node @@ -95,4 +95,3 @@ - name: Include local storage creation (HPP) ansible.builtin.include_tasks: install-hpp.yml when: sap_hypervisor_node_preconfigure_install_hpp - diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml index e144f1a9b..0059a4f1e 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml @@ -17,7 +17,7 @@ labels: custom-kubelet: cpumanager-enabled -- name: Create kubletconfig for cpumanager worker with CPUs reserved for kubernetes +- name: Create kubletconfig for cpumanager worker with CPUs reserved for kubernetes when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is defined kubernetes.core.k8s: state: absent @@ -63,4 +63,3 @@ kubernetes.core.k8s: state: present src: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-worker.yml.j2" - diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh index 7732ba78e..6cec1a678 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh @@ -1,5 +1,3 @@ #!/bin/bash # in order to allow unsupported SRIOV nics such as Mellanox oc patch sriovoperatorconfig default --type=merge -n openshift-sriov-network-operator --patch '{ "spec": { "enableOperatorWebhook": false } }' - - From a98efd573b73104b9d95d04dc4e9830d941bb2ec Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 6 Nov 2023 12:33:11 +0100 Subject: [PATCH 18/38] removed obsolete file roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh --- .../redhat_ocp_virt/sriov-enabled-unsupported-nics.sh | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh diff --git a/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh b/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh deleted file mode 100644 index 7732ba78e..000000000 --- a/roles/sap_hypervisor_node_preconfigure/files/platform/redhat_ocp_virt/sriov-enabled-unsupported-nics.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# in order to allow unsupported SRIOV nics such as Mellanox -oc patch sriovoperatorconfig default --type=merge -n openshift-sriov-network-operator --patch '{ "spec": { "enableOperatorWebhook": false } }' - - From fe45274cb6379f40b4ef9da8de8ce5ce5849904a Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 6 Nov 2023 12:40:42 +0100 Subject: [PATCH 19/38] renamed setup-worker-node -> setup-worker-nodes --- roles/sap_hypervisor_node_preconfigure/meta/main.yml | 5 +---- .../tasks/platform/redhat_ocp_virt/main.yml | 4 ++-- .../{setup-worker-node.yml => setup-worker-nodes.yml} | 0 3 files changed, 3 insertions(+), 6 deletions(-) rename roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/{setup-worker-node.yml => setup-worker-nodes.yml} (100%) diff --git a/roles/sap_hypervisor_node_preconfigure/meta/main.yml b/roles/sap_hypervisor_node_preconfigure/meta/main.yml index 47965cdc0..6dd7e96da 100644 --- a/roles/sap_hypervisor_node_preconfigure/meta/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/meta/main.yml @@ -5,12 +5,9 @@ galaxy_info: author: Nils Koenig description: Provide the configuration of hypervisors for SAP workloads license: Apache-2.0 - min_ansible_version: 2.9 + min_ansible_version: "2.9" galaxy_tags: [ 'sap', 'hana', 'rhel', 'redhat', 'sles', 'suse' ] platforms: - - name: CentOS - versions: - 8 - name: RHEL versions: 8 diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml index 8bb001fd8..6a3f7764b 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml @@ -75,8 +75,8 @@ when: sap_hypervisor_node_preconfigure_install_operators - name: Include install virtctl ansible.builtin.include_tasks: install-virtctl.yml -- name: Include setup worker node - ansible.builtin.include_tasks: setup-worker-node.yml +- name: Include setup worker nodes + ansible.builtin.include_tasks: setup-worker-nodes.yml # How to wait for node to be scheduleable? (NodeSchedulable) - name: Wait for all k8s nodes to be ready diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml similarity index 100% rename from roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-node.yml rename to roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml From 8ecce64c1ef62829529fbe2c42ec1dbe883dd319 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 6 Nov 2023 15:19:41 +0100 Subject: [PATCH 20/38] updated README.md; name template.yml after instanciation --- roles/sap_hypervisor_node_preconfigure/README.md | 2 ++ .../tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index 0a600dc6b..601af3900 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -28,6 +28,8 @@ Needs the ansible kubernetes module and the python3 kubernetes binding. On a RHE * python3-kubernetes * ansible-collection-kubernetes-core +Needs `oc` binary available in path. + ### Role Variables General variables are defined in sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml ``` diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml index 0059a4f1e..17155fd38 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml @@ -56,10 +56,10 @@ - name: Personalize template ansible.builtin.template: src: 99-kargs-worker.yml.j2 - dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-worker.yml.j2" + dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-worker.yml" mode: "0644" - name: Enable hugepages kubernetes.core.k8s: state: present - src: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-worker.yml.j2" + src: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-worker.yml" From 62d34b8dbed6c87ba219fe9ccba4e425ec5b3268 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 6 Nov 2023 16:56:20 +0100 Subject: [PATCH 21/38] removed startup dependency for HPP --- .../tasks/platform/redhat_ocp_virt/install-hpp.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml index 04e4941bb..daa713a4c 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-hpp.yml @@ -30,7 +30,6 @@ [Unit] After=create-mountpoint-var-localstorage.service Requires=create-mountpoint-var-localstorage.service - Before=local-fs.target [Mount] What={{ sap_hypervisor_node_preconfigure_cluster_config.worker_localstorage_device }} Where=/var/localstorage @@ -46,7 +45,6 @@ After=var-localstorage.mount [Service] ExecStart=/usr/bin/chcon -Rt container_file_t /var/localstorage - [Install] WantedBy=multi-user.target enabled: true From 37440c172d212af4aa66ac54c84d3465bcc2c63a Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Tue, 7 Nov 2023 15:52:17 +0100 Subject: [PATCH 22/38] fixed creation of cpumanager KR --- .../redhat_ocp_virt/setup-worker-nodes.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml index 17155fd38..610032b6a 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml @@ -17,10 +17,26 @@ labels: custom-kubelet: cpumanager-enabled +- name: Delete kubletconfig for cpumanager + kubernetes.core.k8s: + state: absent + definition: + apiVersion: machineconfiguration.openshift.io/v1 + kind: KubeletConfig + metadata: + name: cpumanager-enabled + spec: + machineConfigPoolSelector: + matchLabels: + custom-kubelet: cpumanager-enabled + kubeletConfig: + cpuManagerPolicy: static + cpuManagerReconcilePeriod: 5s + - name: Create kubletconfig for cpumanager worker with CPUs reserved for kubernetes when: sap_hypervisor_node_preconfigure_cluster_config.worker_kubernetes_reserved_cpus is defined kubernetes.core.k8s: - state: absent + state: present definition: apiVersion: machineconfiguration.openshift.io/v1 kind: KubeletConfig From 3ca31d5f8b0f9d6df6c44b9c218325be66f8a8d8 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 13 Nov 2023 13:11:51 +0100 Subject: [PATCH 23/38] updated docu --- roles/sap_hypervisor_node_preconfigure/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index 601af3900..7c9f7d970 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -11,9 +11,10 @@ This role will configure the following hypervisors in order to run SAP workloads Will configure a plain vanilla OpenShift cluster so it can be used for SAP workloads. ### Requirements -* A freshly installed OpenShift cluster. +* An OpenShift cluster, best without any previous customization. * The worker nodes should have > 96GB of memory. * Storage is required, e.g. via NFS, OpenShift Data Foundation or local storage. This role can setup access to a Netapp Filer via Trident storage connector. +Local storage will be configures using host path provisioner. * Point the `KUBECONFIG` environment variable to your `kubeconfig`. * Required packages: This roles uses the kubernetes ansible module, this can be installed via the package`ansible-collection-kubernetes-core`. * Make the role available in case you didn't install it already in an ansible roles directory, e.g. From 724e1adb91a4601cea325020968a755000150c3a Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 13 Nov 2023 15:22:32 +0100 Subject: [PATCH 24/38] include defaults platform dependent --- .../defaults/main.yml | 98 +------------------ .../platform_defaults_redhat_ocp_virt.yml | 97 ++++++++++++++++++ .../meta/main.yml | 2 +- .../platform_defaults_redhat_ocp_virt.yml | 3 - .../platform_defaults_redhat_rhel_kvm.yml | 3 - 5 files changed, 100 insertions(+), 103 deletions(-) create mode 100644 roles/sap_hypervisor_node_preconfigure/defaults/platform_defaults_redhat_ocp_virt.yml diff --git a/roles/sap_hypervisor_node_preconfigure/defaults/main.yml b/roles/sap_hypervisor_node_preconfigure/defaults/main.yml index a7e35e177..7ddd6c5f6 100644 --- a/roles/sap_hypervisor_node_preconfigure/defaults/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/defaults/main.yml @@ -1,97 +1,3 @@ --- - -# ibmpower_phyp, redhat_ocp_virt, redhat_rhel_kvm, vmware_vsphere -sap_hypervisor_node_platform: - -# Example configuration -sap_hypervisor_node_preconfigure_cluster_config: - - # URL under which the OCP cluster is reachable - cluster_url: ocpcluster.domain.org - - # namespace under which the VMs are created, note this has to be - # openshift-sriov-network-operator in case of using SRIOV network - # devices - vm_namespace: sap - - # Optional, configuration for trident driver for Netapp NFS filer - trident: - management: management.domain.org - data: datalif.netapp.domain.org - svm: sap_svm - backend: nas_backend - aggregate: aggregate_Name - username: admin - password: xxxxx - storage_driver: ontap-nas - storage_prefix: ocpv_sap_ - - # CPU cores reserved for kubernetes on worker node - worker_kubernetes_reserved_cpus: "0,1" - - # Storage device which should be used if host path provisioner is used - worker_localstorage_device: /dev/vdb - - # detailed configuration for every worker that should be configured - workers: - - name: worker-0 # name must match the node name - networks: # Example network config - - name: sapbridge # using a bridge - description: SAP bridge - state: up - type: linux-bridge - ipv4: - enabled: false - auto-gateway: false - auto-dns: false - bridge: - options: - stp: - enabled: false - port: - - name: ens1f0 # network IF name - - name: storage # an SRIOV device - interface: ens2f0 # network IF name - type: sriov - - - bridge: # another bridge - options: - stp: - enabled: false - port: - - name: ens2f0 # network IF name - description: storage - mtu: 9000 - ipv4: - address: - - ip: 192.168.1.51 # IP config - prefix-length: 24 - auto-dns: false - auto-gateway: false - enabled: true - name: storagebridge - state: up - type: linux-bridge - - name: multi # another SRIOV device - interface: ens2f1 # network IF name - type: sriov - - - name: worker-1 # second worker configuration - networks: # Example network config - - name: sapbridge # using a bridge - description: SAP bridge - state: up - type: linux-bridge - ipv4: - enabled: false - auto-gateway: false - auto-dns: false - bridge: - options: - stp: - enabled: false - port: - - name: ens1f0 # network IF name - - name: storage # an SRIOV device - interface: ens2f0 # network IF name - type: sriov +- name: Include defaults for {{ sap_hypervisor_node_platform }} + ansible.builtin.include_vars: "platform_defaults_{{ sap_hypervisor_node_platform }}.yml" diff --git a/roles/sap_hypervisor_node_preconfigure/defaults/platform_defaults_redhat_ocp_virt.yml b/roles/sap_hypervisor_node_preconfigure/defaults/platform_defaults_redhat_ocp_virt.yml new file mode 100644 index 000000000..a7e35e177 --- /dev/null +++ b/roles/sap_hypervisor_node_preconfigure/defaults/platform_defaults_redhat_ocp_virt.yml @@ -0,0 +1,97 @@ +--- + +# ibmpower_phyp, redhat_ocp_virt, redhat_rhel_kvm, vmware_vsphere +sap_hypervisor_node_platform: + +# Example configuration +sap_hypervisor_node_preconfigure_cluster_config: + + # URL under which the OCP cluster is reachable + cluster_url: ocpcluster.domain.org + + # namespace under which the VMs are created, note this has to be + # openshift-sriov-network-operator in case of using SRIOV network + # devices + vm_namespace: sap + + # Optional, configuration for trident driver for Netapp NFS filer + trident: + management: management.domain.org + data: datalif.netapp.domain.org + svm: sap_svm + backend: nas_backend + aggregate: aggregate_Name + username: admin + password: xxxxx + storage_driver: ontap-nas + storage_prefix: ocpv_sap_ + + # CPU cores reserved for kubernetes on worker node + worker_kubernetes_reserved_cpus: "0,1" + + # Storage device which should be used if host path provisioner is used + worker_localstorage_device: /dev/vdb + + # detailed configuration for every worker that should be configured + workers: + - name: worker-0 # name must match the node name + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov + + - bridge: # another bridge + options: + stp: + enabled: false + port: + - name: ens2f0 # network IF name + description: storage + mtu: 9000 + ipv4: + address: + - ip: 192.168.1.51 # IP config + prefix-length: 24 + auto-dns: false + auto-gateway: false + enabled: true + name: storagebridge + state: up + type: linux-bridge + - name: multi # another SRIOV device + interface: ens2f1 # network IF name + type: sriov + + - name: worker-1 # second worker configuration + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov diff --git a/roles/sap_hypervisor_node_preconfigure/meta/main.yml b/roles/sap_hypervisor_node_preconfigure/meta/main.yml index 6dd7e96da..1891f30ca 100644 --- a/roles/sap_hypervisor_node_preconfigure/meta/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/meta/main.yml @@ -6,7 +6,7 @@ galaxy_info: description: Provide the configuration of hypervisors for SAP workloads license: Apache-2.0 min_ansible_version: "2.9" - galaxy_tags: [ 'sap', 'hana', 'rhel', 'redhat', 'sles', 'suse' ] + galaxy_tags: [ 'sap', 'hana', 'rhel', 'redhat' ] platforms: - name: RHEL versions: diff --git a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml index be422195e..69cb90442 100644 --- a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml +++ b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml @@ -1,7 +1,4 @@ --- -# vars file for sap_hypervisor_node_preconfigure -# - # Install and configure the host path provisioner (hpp) for a local storage disk sap_hypervisor_node_preconfigure_install_hpp: False diff --git a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_rhel_kvm.yml b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_rhel_kvm.yml index db5407d19..afad9216e 100644 --- a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_rhel_kvm.yml +++ b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_rhel_kvm.yml @@ -1,7 +1,4 @@ --- - -# defaults file for sap_hypervisor_node_preconfigure - # packages to install sap_hypervisor_node_preconfigure_packages: - libhugetlbfs-utils From 860f7124ea5da380090be7b716161f8f61627671 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 13 Nov 2023 16:12:00 +0100 Subject: [PATCH 25/38] Revert "include defaults platform dependent" This reverts commit 724e1adb91a4601cea325020968a755000150c3a. --- .../defaults/main.yml | 98 ++++++++++++++++++- .../platform_defaults_redhat_ocp_virt.yml | 97 ------------------ .../meta/main.yml | 2 +- .../platform_defaults_redhat_ocp_virt.yml | 3 + .../platform_defaults_redhat_rhel_kvm.yml | 3 + 5 files changed, 103 insertions(+), 100 deletions(-) delete mode 100644 roles/sap_hypervisor_node_preconfigure/defaults/platform_defaults_redhat_ocp_virt.yml diff --git a/roles/sap_hypervisor_node_preconfigure/defaults/main.yml b/roles/sap_hypervisor_node_preconfigure/defaults/main.yml index 7ddd6c5f6..a7e35e177 100644 --- a/roles/sap_hypervisor_node_preconfigure/defaults/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/defaults/main.yml @@ -1,3 +1,97 @@ --- -- name: Include defaults for {{ sap_hypervisor_node_platform }} - ansible.builtin.include_vars: "platform_defaults_{{ sap_hypervisor_node_platform }}.yml" + +# ibmpower_phyp, redhat_ocp_virt, redhat_rhel_kvm, vmware_vsphere +sap_hypervisor_node_platform: + +# Example configuration +sap_hypervisor_node_preconfigure_cluster_config: + + # URL under which the OCP cluster is reachable + cluster_url: ocpcluster.domain.org + + # namespace under which the VMs are created, note this has to be + # openshift-sriov-network-operator in case of using SRIOV network + # devices + vm_namespace: sap + + # Optional, configuration for trident driver for Netapp NFS filer + trident: + management: management.domain.org + data: datalif.netapp.domain.org + svm: sap_svm + backend: nas_backend + aggregate: aggregate_Name + username: admin + password: xxxxx + storage_driver: ontap-nas + storage_prefix: ocpv_sap_ + + # CPU cores reserved for kubernetes on worker node + worker_kubernetes_reserved_cpus: "0,1" + + # Storage device which should be used if host path provisioner is used + worker_localstorage_device: /dev/vdb + + # detailed configuration for every worker that should be configured + workers: + - name: worker-0 # name must match the node name + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov + + - bridge: # another bridge + options: + stp: + enabled: false + port: + - name: ens2f0 # network IF name + description: storage + mtu: 9000 + ipv4: + address: + - ip: 192.168.1.51 # IP config + prefix-length: 24 + auto-dns: false + auto-gateway: false + enabled: true + name: storagebridge + state: up + type: linux-bridge + - name: multi # another SRIOV device + interface: ens2f1 # network IF name + type: sriov + + - name: worker-1 # second worker configuration + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov diff --git a/roles/sap_hypervisor_node_preconfigure/defaults/platform_defaults_redhat_ocp_virt.yml b/roles/sap_hypervisor_node_preconfigure/defaults/platform_defaults_redhat_ocp_virt.yml deleted file mode 100644 index a7e35e177..000000000 --- a/roles/sap_hypervisor_node_preconfigure/defaults/platform_defaults_redhat_ocp_virt.yml +++ /dev/null @@ -1,97 +0,0 @@ ---- - -# ibmpower_phyp, redhat_ocp_virt, redhat_rhel_kvm, vmware_vsphere -sap_hypervisor_node_platform: - -# Example configuration -sap_hypervisor_node_preconfigure_cluster_config: - - # URL under which the OCP cluster is reachable - cluster_url: ocpcluster.domain.org - - # namespace under which the VMs are created, note this has to be - # openshift-sriov-network-operator in case of using SRIOV network - # devices - vm_namespace: sap - - # Optional, configuration for trident driver for Netapp NFS filer - trident: - management: management.domain.org - data: datalif.netapp.domain.org - svm: sap_svm - backend: nas_backend - aggregate: aggregate_Name - username: admin - password: xxxxx - storage_driver: ontap-nas - storage_prefix: ocpv_sap_ - - # CPU cores reserved for kubernetes on worker node - worker_kubernetes_reserved_cpus: "0,1" - - # Storage device which should be used if host path provisioner is used - worker_localstorage_device: /dev/vdb - - # detailed configuration for every worker that should be configured - workers: - - name: worker-0 # name must match the node name - networks: # Example network config - - name: sapbridge # using a bridge - description: SAP bridge - state: up - type: linux-bridge - ipv4: - enabled: false - auto-gateway: false - auto-dns: false - bridge: - options: - stp: - enabled: false - port: - - name: ens1f0 # network IF name - - name: storage # an SRIOV device - interface: ens2f0 # network IF name - type: sriov - - - bridge: # another bridge - options: - stp: - enabled: false - port: - - name: ens2f0 # network IF name - description: storage - mtu: 9000 - ipv4: - address: - - ip: 192.168.1.51 # IP config - prefix-length: 24 - auto-dns: false - auto-gateway: false - enabled: true - name: storagebridge - state: up - type: linux-bridge - - name: multi # another SRIOV device - interface: ens2f1 # network IF name - type: sriov - - - name: worker-1 # second worker configuration - networks: # Example network config - - name: sapbridge # using a bridge - description: SAP bridge - state: up - type: linux-bridge - ipv4: - enabled: false - auto-gateway: false - auto-dns: false - bridge: - options: - stp: - enabled: false - port: - - name: ens1f0 # network IF name - - name: storage # an SRIOV device - interface: ens2f0 # network IF name - type: sriov diff --git a/roles/sap_hypervisor_node_preconfigure/meta/main.yml b/roles/sap_hypervisor_node_preconfigure/meta/main.yml index 1891f30ca..6dd7e96da 100644 --- a/roles/sap_hypervisor_node_preconfigure/meta/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/meta/main.yml @@ -6,7 +6,7 @@ galaxy_info: description: Provide the configuration of hypervisors for SAP workloads license: Apache-2.0 min_ansible_version: "2.9" - galaxy_tags: [ 'sap', 'hana', 'rhel', 'redhat' ] + galaxy_tags: [ 'sap', 'hana', 'rhel', 'redhat', 'sles', 'suse' ] platforms: - name: RHEL versions: diff --git a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml index 69cb90442..be422195e 100644 --- a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml +++ b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml @@ -1,4 +1,7 @@ --- +# vars file for sap_hypervisor_node_preconfigure +# + # Install and configure the host path provisioner (hpp) for a local storage disk sap_hypervisor_node_preconfigure_install_hpp: False diff --git a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_rhel_kvm.yml b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_rhel_kvm.yml index afad9216e..db5407d19 100644 --- a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_rhel_kvm.yml +++ b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_rhel_kvm.yml @@ -1,4 +1,7 @@ --- + +# defaults file for sap_hypervisor_node_preconfigure + # packages to install sap_hypervisor_node_preconfigure_packages: - libhugetlbfs-utils From ae4456ac2be05983787786649de03b0d52067855 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Tue, 14 Nov 2023 14:07:52 +0100 Subject: [PATCH 26/38] Revert "include defaults platform dependent" This reverts commit 724e1adb91a4601cea325020968a755000150c3a. --- .../defaults/main.yml | 98 ++++++++++++++++++- .../platform_defaults_redhat_ocp_virt.yml | 97 ------------------ .../meta/main.yml | 2 +- .../platform_defaults_redhat_ocp_virt.yml | 3 + .../platform_defaults_redhat_rhel_kvm.yml | 3 + 5 files changed, 103 insertions(+), 100 deletions(-) delete mode 100644 roles/sap_hypervisor_node_preconfigure/defaults/platform_defaults_redhat_ocp_virt.yml diff --git a/roles/sap_hypervisor_node_preconfigure/defaults/main.yml b/roles/sap_hypervisor_node_preconfigure/defaults/main.yml index 7ddd6c5f6..a7e35e177 100644 --- a/roles/sap_hypervisor_node_preconfigure/defaults/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/defaults/main.yml @@ -1,3 +1,97 @@ --- -- name: Include defaults for {{ sap_hypervisor_node_platform }} - ansible.builtin.include_vars: "platform_defaults_{{ sap_hypervisor_node_platform }}.yml" + +# ibmpower_phyp, redhat_ocp_virt, redhat_rhel_kvm, vmware_vsphere +sap_hypervisor_node_platform: + +# Example configuration +sap_hypervisor_node_preconfigure_cluster_config: + + # URL under which the OCP cluster is reachable + cluster_url: ocpcluster.domain.org + + # namespace under which the VMs are created, note this has to be + # openshift-sriov-network-operator in case of using SRIOV network + # devices + vm_namespace: sap + + # Optional, configuration for trident driver for Netapp NFS filer + trident: + management: management.domain.org + data: datalif.netapp.domain.org + svm: sap_svm + backend: nas_backend + aggregate: aggregate_Name + username: admin + password: xxxxx + storage_driver: ontap-nas + storage_prefix: ocpv_sap_ + + # CPU cores reserved for kubernetes on worker node + worker_kubernetes_reserved_cpus: "0,1" + + # Storage device which should be used if host path provisioner is used + worker_localstorage_device: /dev/vdb + + # detailed configuration for every worker that should be configured + workers: + - name: worker-0 # name must match the node name + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov + + - bridge: # another bridge + options: + stp: + enabled: false + port: + - name: ens2f0 # network IF name + description: storage + mtu: 9000 + ipv4: + address: + - ip: 192.168.1.51 # IP config + prefix-length: 24 + auto-dns: false + auto-gateway: false + enabled: true + name: storagebridge + state: up + type: linux-bridge + - name: multi # another SRIOV device + interface: ens2f1 # network IF name + type: sriov + + - name: worker-1 # second worker configuration + networks: # Example network config + - name: sapbridge # using a bridge + description: SAP bridge + state: up + type: linux-bridge + ipv4: + enabled: false + auto-gateway: false + auto-dns: false + bridge: + options: + stp: + enabled: false + port: + - name: ens1f0 # network IF name + - name: storage # an SRIOV device + interface: ens2f0 # network IF name + type: sriov diff --git a/roles/sap_hypervisor_node_preconfigure/defaults/platform_defaults_redhat_ocp_virt.yml b/roles/sap_hypervisor_node_preconfigure/defaults/platform_defaults_redhat_ocp_virt.yml deleted file mode 100644 index a7e35e177..000000000 --- a/roles/sap_hypervisor_node_preconfigure/defaults/platform_defaults_redhat_ocp_virt.yml +++ /dev/null @@ -1,97 +0,0 @@ ---- - -# ibmpower_phyp, redhat_ocp_virt, redhat_rhel_kvm, vmware_vsphere -sap_hypervisor_node_platform: - -# Example configuration -sap_hypervisor_node_preconfigure_cluster_config: - - # URL under which the OCP cluster is reachable - cluster_url: ocpcluster.domain.org - - # namespace under which the VMs are created, note this has to be - # openshift-sriov-network-operator in case of using SRIOV network - # devices - vm_namespace: sap - - # Optional, configuration for trident driver for Netapp NFS filer - trident: - management: management.domain.org - data: datalif.netapp.domain.org - svm: sap_svm - backend: nas_backend - aggregate: aggregate_Name - username: admin - password: xxxxx - storage_driver: ontap-nas - storage_prefix: ocpv_sap_ - - # CPU cores reserved for kubernetes on worker node - worker_kubernetes_reserved_cpus: "0,1" - - # Storage device which should be used if host path provisioner is used - worker_localstorage_device: /dev/vdb - - # detailed configuration for every worker that should be configured - workers: - - name: worker-0 # name must match the node name - networks: # Example network config - - name: sapbridge # using a bridge - description: SAP bridge - state: up - type: linux-bridge - ipv4: - enabled: false - auto-gateway: false - auto-dns: false - bridge: - options: - stp: - enabled: false - port: - - name: ens1f0 # network IF name - - name: storage # an SRIOV device - interface: ens2f0 # network IF name - type: sriov - - - bridge: # another bridge - options: - stp: - enabled: false - port: - - name: ens2f0 # network IF name - description: storage - mtu: 9000 - ipv4: - address: - - ip: 192.168.1.51 # IP config - prefix-length: 24 - auto-dns: false - auto-gateway: false - enabled: true - name: storagebridge - state: up - type: linux-bridge - - name: multi # another SRIOV device - interface: ens2f1 # network IF name - type: sriov - - - name: worker-1 # second worker configuration - networks: # Example network config - - name: sapbridge # using a bridge - description: SAP bridge - state: up - type: linux-bridge - ipv4: - enabled: false - auto-gateway: false - auto-dns: false - bridge: - options: - stp: - enabled: false - port: - - name: ens1f0 # network IF name - - name: storage # an SRIOV device - interface: ens2f0 # network IF name - type: sriov diff --git a/roles/sap_hypervisor_node_preconfigure/meta/main.yml b/roles/sap_hypervisor_node_preconfigure/meta/main.yml index 1891f30ca..6dd7e96da 100644 --- a/roles/sap_hypervisor_node_preconfigure/meta/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/meta/main.yml @@ -6,7 +6,7 @@ galaxy_info: description: Provide the configuration of hypervisors for SAP workloads license: Apache-2.0 min_ansible_version: "2.9" - galaxy_tags: [ 'sap', 'hana', 'rhel', 'redhat' ] + galaxy_tags: [ 'sap', 'hana', 'rhel', 'redhat', 'sles', 'suse' ] platforms: - name: RHEL versions: diff --git a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml index 69cb90442..be422195e 100644 --- a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml +++ b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml @@ -1,4 +1,7 @@ --- +# vars file for sap_hypervisor_node_preconfigure +# + # Install and configure the host path provisioner (hpp) for a local storage disk sap_hypervisor_node_preconfigure_install_hpp: False diff --git a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_rhel_kvm.yml b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_rhel_kvm.yml index afad9216e..db5407d19 100644 --- a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_rhel_kvm.yml +++ b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_rhel_kvm.yml @@ -1,4 +1,7 @@ --- + +# defaults file for sap_hypervisor_node_preconfigure + # packages to install sap_hypervisor_node_preconfigure_packages: - libhugetlbfs-utils From ce82be3271003785bb5d7837d29abdfb97473187 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Thu, 16 Nov 2023 16:27:06 +0100 Subject: [PATCH 27/38] create VM namespace, remove uneeded waits --- .../defaults/main.yml | 18 ++++++++++++++++++ .../redhat_ocp_virt/configure-worker-node.yml | 4 ---- .../platform/redhat_ocp_virt/node-network.yml | 3 +-- .../tasks/platform/redhat_ocp_virt/prepare.yml | 7 +++++++ 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/defaults/main.yml b/roles/sap_hypervisor_node_preconfigure/defaults/main.yml index a7e35e177..9a48534a4 100644 --- a/roles/sap_hypervisor_node_preconfigure/defaults/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/defaults/main.yml @@ -92,6 +92,24 @@ sap_hypervisor_node_preconfigure_cluster_config: enabled: false port: - name: ens1f0 # network IF name + - bridge: # another bridge + options: + stp: + enabled: false + port: + - name: ens2f0 # network IF name + description: storage + mtu: 9000 + ipv4: + address: + - ip: 192.168.1.2 # IP config + prefix-length: 24 + auto-dns: false + auto-gateway: false + enabled: true + name: storagebridge + state: up + type: linux-bridge - name: storage # an SRIOV device interface: ens2f0 # network IF name type: sriov diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml index 99b7599ec..3af1dcf5b 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml @@ -12,10 +12,6 @@ index_var: __sap_hypervisor_node_preconfigure_register_worker_network_nr when: __sap_hypervisor_node_preconfigure_register_worker.networks is defined -- name: Pause so cluster can process config - ansible.builtin.pause: - minutes: 1 - # How to wait for node to be scheduleable? (NodeSchedulable) - name: Wait for all k8s nodes to be ready ansible.builtin.command: oc wait --for=condition=Ready nodes --all --timeout=3600s diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml index 088f86fe8..0474959c3 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml @@ -4,7 +4,7 @@ var: __sap_hypervisor_node_preconfigure_register_worker_network - name: "Create NodeNetworkConfigurationPolicy\ - {{ __sap_hypervisor_node_preconfigure_register_worker_network.name }} on\ + {{ __sap_hypervisor_node_preconfigure_register_worker_network.name }} on \ {{ __sap_hypervisor_node_preconfigure_register_worker.name }}" kubernetes.core.k8s: state: present @@ -21,7 +21,6 @@ - "{{ __sap_hypervisor_node_preconfigure_register_worker_network }}" when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'linux-bridge' -# XXX didn't work - why? - name: "Create NetworkAttachmentDefinition {{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}" kubernetes.core.k8s: state: present diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml index dd879b22c..f74e17137 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml @@ -7,3 +7,10 @@ state: directory suffix: "_sap_hypervisor_node_preconfigure" register: __sap_hypervisor_node_preconfigure_register_tmpdir + +- name: "Create VM namespace {{ sap_hypervisor_node_preconfigure_cluster_config._vm_namespace }}" + kubernetes.core.k8s: + name: "{{ sap_hypervisor_node_preconfigure_cluster_config._vm_namespace }}" + api_version: v1 + kind: Namespace + state: present From 1db86b870a8dd79468eaa3e780215d6df3258c3b Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Thu, 16 Nov 2023 16:34:12 +0100 Subject: [PATCH 28/38] fixed typo --- .../tasks/platform/redhat_ocp_virt/prepare.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml index f74e17137..0dfbfa1da 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/prepare.yml @@ -8,9 +8,9 @@ suffix: "_sap_hypervisor_node_preconfigure" register: __sap_hypervisor_node_preconfigure_register_tmpdir -- name: "Create VM namespace {{ sap_hypervisor_node_preconfigure_cluster_config._vm_namespace }}" +- name: "Create VM namespace {{ sap_hypervisor_node_preconfigure_cluster_config.vm_namespace }}" kubernetes.core.k8s: - name: "{{ sap_hypervisor_node_preconfigure_cluster_config._vm_namespace }}" + name: "{{ sap_hypervisor_node_preconfigure_cluster_config.vm_namespace }}" api_version: v1 kind: Namespace state: present From dbc634fa1e603b10558773a012941e2a3a2bb547 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Thu, 23 Nov 2023 10:34:01 +0100 Subject: [PATCH 29/38] added missing whitespace before linebreak --- .../tasks/platform/redhat_ocp_virt/main.yml | 1 + .../tasks/platform/redhat_ocp_virt/node-network.yml | 4 ++-- .../vars/platform_defaults_redhat_ocp_virt.yml | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml index 6a3f7764b..5b601bcc0 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml @@ -77,6 +77,7 @@ ansible.builtin.include_tasks: install-virtctl.yml - name: Include setup worker nodes ansible.builtin.include_tasks: setup-worker-nodes.yml + when: sap_hypervisor_node_preconfigure_setup_workers # How to wait for node to be scheduleable? (NodeSchedulable) - name: Wait for all k8s nodes to be ready diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml index 0474959c3..42dd15c5b 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml @@ -43,14 +43,14 @@ when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'linux-bridge' - name: Label nodes - ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }}\ + ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} \ feature.node.kubernetes.io/network-sriov.capable=true --overwrite=true" when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'sriov' register: __sap_hypervisor_node_preconfigure_register_label_node changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 - name: "Create SRIOV NodeNetworkConfigurationPolicy\ - {{ __sap_hypervisor_node_preconfigure_register_worker_network.name.name }} on\ + {{ __sap_hypervisor_node_preconfigure_register_worker_network.name.name }} on \ {{ __sap_hypervisor_node_preconfigure_register_worker.name }}" kubernetes.core.k8s: state: present diff --git a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml index be422195e..88245c524 100644 --- a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml +++ b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml @@ -23,3 +23,6 @@ sap_hypervisor_node_preconfigure_ignore_minimal_memory_check: False # Should the operators be installed sap_hypervisor_node_preconfigure_install_operators: True + +# Configure the workers? +sap_hypervisor_node_preconfigure_setup_workers: True From 24ccfa7ac957a71db532b8338ee581e2f3a62a41 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Thu, 23 Nov 2023 12:19:44 +0100 Subject: [PATCH 30/38] fixed typo and indentation --- .../tasks/platform/redhat_ocp_virt/node-network.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml index 42dd15c5b..79d8fe02f 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml @@ -49,8 +49,8 @@ register: __sap_hypervisor_node_preconfigure_register_label_node changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 -- name: "Create SRIOV NodeNetworkConfigurationPolicy\ - {{ __sap_hypervisor_node_preconfigure_register_worker_network.name.name }} on \ +- name: "Create SRIOV NodeNetworkConfigurationPolicy \ + {{ __sap_hypervisor_node_preconfigure_register_worker_network.name }} on \ {{ __sap_hypervisor_node_preconfigure_register_worker.name }}" kubernetes.core.k8s: state: present @@ -59,7 +59,7 @@ kind: SriovNetworkNodePolicy metadata: name: "iface-{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}-sriov-{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" - namespace: openshift-sriov-network-operator + namespace: openshift-sriov-network-operator spec: resourceName: "iface{{ __sap_hypervisor_node_preconfigure_register_worker_network.name }}sriov" nodeSelector: From bc4a43cf2404fa8a2338044c6587f618912d063f Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Wed, 6 Dec 2023 11:16:08 +0100 Subject: [PATCH 31/38] Ansible lint wants role name to start with a capital letter --- roles/sap_hypervisor_node_preconfigure/handlers/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/handlers/main.yml b/roles/sap_hypervisor_node_preconfigure/handlers/main.yml index 5150d6bdd..77d2ffde5 100644 --- a/roles/sap_hypervisor_node_preconfigure/handlers/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/handlers/main.yml @@ -1,4 +1,3 @@ --- - -- name: hypervisor node preconfigure - Include Handler Tasks for {{ sap_hypervisor_node_platform }} +- name: Hypervisor node preconfigure - Include Handler Tasks for {{ sap_hypervisor_node_platform }} ansible.builtin.include_tasks: "{{ role_path }}/handlers/platform/{{ sap_hypervisor_node_platform }}/main.yml" From a89f7da31817d64fa900292086b00b1a40eefdca Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Thu, 7 Dec 2023 13:04:22 +0100 Subject: [PATCH 32/38] Made clear that memory unit used is GiB --- .../README.md | 6 +++--- .../meta/main.yml | 2 +- .../tasks/platform/redhat_ocp_virt/main.yml | 20 +++++++++---------- .../platform_defaults_redhat_ocp_virt.yml | 10 +++++----- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/README.md b/roles/sap_hypervisor_node_preconfigure/README.md index 7c9f7d970..74e72a487 100644 --- a/roles/sap_hypervisor_node_preconfigure/README.md +++ b/roles/sap_hypervisor_node_preconfigure/README.md @@ -13,12 +13,13 @@ Will configure a plain vanilla OpenShift cluster so it can be used for SAP workl ### Requirements * An OpenShift cluster, best without any previous customization. * The worker nodes should have > 96GB of memory. +* Worker nodes need to have Intel CPUs that provide TSX feature. * Storage is required, e.g. via NFS, OpenShift Data Foundation or local storage. This role can setup access to a Netapp Filer via Trident storage connector. Local storage will be configures using host path provisioner. * Point the `KUBECONFIG` environment variable to your `kubeconfig`. -* Required packages: This roles uses the kubernetes ansible module, this can be installed via the package`ansible-collection-kubernetes-core`. * Make the role available in case you didn't install it already in an ansible roles directory, e.g. - +* Make sure to install the dependencies mentioned below are installed. +* To ensure your local checkout it found by ansible: ``` mkdir -p ~/.ansible/roles/ ln -sf ~/community.sap_install/roles/sap_hypervisor_node_preconfigure ~/.ansible/roles/ @@ -150,7 +151,6 @@ sap_hypervisor_node_preconfigure_cluster_config: interface: ens2f0 # network IF name type: sriov ``` - ### Example Playbook See `playbooks/sample-sap-hypervisor-redhat_ocp_virt-preconfigure.yml` for an example. diff --git a/roles/sap_hypervisor_node_preconfigure/meta/main.yml b/roles/sap_hypervisor_node_preconfigure/meta/main.yml index 6dd7e96da..1b04b2f41 100644 --- a/roles/sap_hypervisor_node_preconfigure/meta/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/meta/main.yml @@ -6,7 +6,7 @@ galaxy_info: description: Provide the configuration of hypervisors for SAP workloads license: Apache-2.0 min_ansible_version: "2.9" - galaxy_tags: [ 'sap', 'hana', 'rhel', 'redhat', 'sles', 'suse' ] + galaxy_tags: [ 'sap', 'hana', 'rhel', 'redhat', 'openshift' ] platforms: - name: RHEL versions: diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml index 5b601bcc0..1819d0043 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml @@ -36,28 +36,28 @@ __sap_hypervisor_node_preconfigure_register_worker_name: "{{ __sap_hypervisor_node_preconfigure_register_nodes[0]['metadata']['labels']['kubernetes.io/hostname'] }}" -- name: Get memory of worker node +- name: Get memory of first worker node (will be used for all worker nodes later on) ansible.builtin.set_fact: - __sap_hypervisor_node_preconfigure_register_worker_memory_gb: + __sap_hypervisor_node_preconfigure_register_worker_memory_gib: "{{ (__sap_hypervisor_node_preconfigure_register_nodes[0]['status']['capacity']['memory'] | replace('Ki', '') | int / 1048576) | int }}" -- name: Check if host has minimal amount of memory (96Gb) +- name: Check if host has minimal amount of memory (96GiB) ansible.builtin.assert: - that: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int >= 96 + that: __sap_hypervisor_node_preconfigure_register_worker_memory_gib >= 96 fail_msg: "Not enough memory on node {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" success_msg: "Enough memory on node {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_minimal_memory_check }}" # calculate memory to be allocated as hugepages -# if system < 512GB memory use 32GB as upper boundary, 64GB otherwise as upper boundary -- name: Calculate amount of hugepages to reserve (host memory < 512 Gb) +# if system < 512GiB memory use 32GiB as upper boundary, 64GB otherwise as upper boundary +- name: Calculate amount of hugepages to reserve (host memory < 512 GiB) ansible.builtin.set_fact: - __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gb | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512 }}" - when: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int < 512 + __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512 }}" + when: __sap_hypervisor_node_preconfigure_register_worker_memory_gib|int < 512 -- name: Calculate amount of hugepages to reserve (host memory >= 512 Gb) +- name: Calculate amount of hugepages to reserve (host memory >= 512 GiB) ansible.builtin.set_fact: - __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gb | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512 }}" + __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512 }}" when: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int >= 512 - name: Include prepare diff --git a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml index 88245c524..8cc402c9a 100644 --- a/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml +++ b/roles/sap_hypervisor_node_preconfigure/vars/platform_defaults_redhat_ocp_virt.yml @@ -13,12 +13,12 @@ sap_hypervisor_node_preconfigure_install_trident_url: https://github.com/NetApp/ # should SRIOV be enabled for unsupported NICs sap_hypervisor_node_preconfigure_sriov_enable_unsupported_nics: True -# Amount of memory [GB] to be reserved for the hypervisor on hosts >= 512GB -sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512: 64 #GB -# Amount of memory [GB] to be reserved for the hypervisor on hosts < 512GB -sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512: 32 #GB +# Amount of memory [GiB] to be reserved for the hypervisor on hosts >= 512GiB +sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512: 64 #GiB +# Amount of memory [GiB] to be reserved for the hypervisor on hosts < 512GiB +sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512: 32 #GiB -# Should the check for the minimal amount of be ignored? Minimal amount is 96 GB +# Should the check for the minimal amount of be ignored? Minimal amount is 96 GiB sap_hypervisor_node_preconfigure_ignore_minimal_memory_check: False # Should the operators be installed From a0410b67e277a375cb19329a1680c060e74be74b Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Thu, 7 Dec 2023 13:19:13 +0100 Subject: [PATCH 33/38] fixed memory variable renaming and removed cating to int --- .../tasks/platform/redhat_ocp_virt/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml index 1819d0043..c158bffa5 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml @@ -53,12 +53,12 @@ - name: Calculate amount of hugepages to reserve (host memory < 512 GiB) ansible.builtin.set_fact: __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512 }}" - when: __sap_hypervisor_node_preconfigure_register_worker_memory_gib|int < 512 + when: __sap_hypervisor_node_preconfigure_register_worker_memory_gib < 512 - name: Calculate amount of hugepages to reserve (host memory >= 512 GiB) ansible.builtin.set_fact: __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512 }}" - when: __sap_hypervisor_node_preconfigure_register_worker_memory_gb|int >= 512 + when: __sap_hypervisor_node_preconfigure_register_worker_memory_gib >= 512 - name: Include prepare ansible.builtin.include_tasks: prepare.yml From 88adb966bc520f5fec193858ea73987301d45862 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Thu, 7 Dec 2023 14:13:32 +0100 Subject: [PATCH 34/38] cast integers when used --- .../tasks/platform/redhat_ocp_virt/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml index c158bffa5..5e4cedd53 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/main.yml @@ -39,11 +39,11 @@ - name: Get memory of first worker node (will be used for all worker nodes later on) ansible.builtin.set_fact: __sap_hypervisor_node_preconfigure_register_worker_memory_gib: - "{{ (__sap_hypervisor_node_preconfigure_register_nodes[0]['status']['capacity']['memory'] | replace('Ki', '') | int / 1048576) | int }}" + "{{ (__sap_hypervisor_node_preconfigure_register_nodes[0]['status']['capacity']['memory'] | replace('Ki', '') | int / 1048576) }}" - name: Check if host has minimal amount of memory (96GiB) ansible.builtin.assert: - that: __sap_hypervisor_node_preconfigure_register_worker_memory_gib >= 96 + that: __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int >= 96 fail_msg: "Not enough memory on node {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" success_msg: "Enough memory on node {{ __sap_hypervisor_node_preconfigure_register_worker_name }}" ignore_errors: "{{ sap_hypervisor_node_preconfigure_ignore_minimal_memory_check }}" @@ -53,12 +53,12 @@ - name: Calculate amount of hugepages to reserve (host memory < 512 GiB) ansible.builtin.set_fact: __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_lt_512 }}" - when: __sap_hypervisor_node_preconfigure_register_worker_memory_gib < 512 + when: __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int < 512 - name: Calculate amount of hugepages to reserve (host memory >= 512 GiB) ansible.builtin.set_fact: __sap_hypervisor_node_preconfigure_register_worker_reserved_hugepages: "{{ __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int - sap_hypervisor_node_preconfigure_hypervisor_reserved_ram_host_ge_512 }}" - when: __sap_hypervisor_node_preconfigure_register_worker_memory_gib >= 512 + when: __sap_hypervisor_node_preconfigure_register_worker_memory_gib | int >= 512 - name: Include prepare ansible.builtin.include_tasks: prepare.yml From 7a94e6554232f894add72b2ef029f7bad9af3710 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Fri, 8 Dec 2023 20:12:33 +0100 Subject: [PATCH 35/38] label nodes via k8s --- .../redhat_ocp_virt/configure-worker-node.yml | 30 ++++++++++++++++--- .../redhat_ocp_virt/install-trident.yml | 6 ++-- .../redhat_ocp_virt/install-virtctl.yml | 2 +- .../platform/redhat_ocp_virt/node-network.yml | 22 ++++++++++---- .../redhat_ocp_virt/setup-worker-nodes.yml | 8 +---- 5 files changed, 48 insertions(+), 20 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml index 3af1dcf5b..6d4d6bc3e 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml @@ -1,8 +1,30 @@ --- -- name: Label nodes - ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} cpumanager=true --overwrite=true" - register: __sap_hypervisor_node_preconfigure_label_node_result - changed_when: __sap_hypervisor_node_preconfigure_label_node_result.rc != 0 +#- name: Label nodes +# ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} cpumanager=true --overwrite=true" +# register: __sap_hypervisor_node_preconfigure_label_node_result +# changed_when: __sap_hypervisor_node_preconfigure_label_node_result.rc != 0 + +- name: Label the node with cpumanager=true + ansible.builtin.k8s: + definition: + apiVersion: v1 + kind: Node + metadata: + name: {{ __sap_hypervisor_node_preconfigure_register_worker.name }} + labels: + cpumanager: true + state: present + +- name: Label the node with invtsc=true + ansible.builtin.k8s: + definition: + apiVersion: v1 + kind: Node + metadata: + name: {{ __sap_hypervisor_node_preconfigure_register_worker.name }} + labels: + invtsc: true + state: present - name: Include node network ansible.builtin.include_tasks: node-network.yml diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml index 9747b55ad..f1abd1fdb 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml @@ -39,10 +39,10 @@ metadata: name: nas annotations: - storageclass.kubernetes.io/is-default-class: 'true' + storageclass.kubernetes.io/is-default-class: true provisioner: csi.trident.netapp.io parameters: backendType: "{{ sap_hypervisor_node_preconfigure_cluster_config.trident.storage_driver }}" - snapshots: "True" + snapshots: true provisioningType: "thin" - encryption: "false" + encryption: false diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-virtctl.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-virtctl.yml index e4555f803..bd5dd818d 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-virtctl.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-virtctl.yml @@ -3,7 +3,7 @@ ansible.builtin.file: path: ~/bin state: directory - mode: "0755" + mode: "0700" - name: Get and extract virtctl # become: yes diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml index 79d8fe02f..a42897560 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml @@ -42,12 +42,24 @@ }' when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'linux-bridge' -- name: Label nodes - ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} \ - feature.node.kubernetes.io/network-sriov.capable=true --overwrite=true" +- name: Label the node with feature.node.kubernetes.io/network-sriov.capable=true + ansible.builtin.k8s: + definition: + apiVersion: v1 + kind: Node + metadata: + name: {{ __sap_hypervisor_node_preconfigure_register_worker.name }} + labels: + feature.node.kubernetes.io/network-sriov.capable: true + state: present + when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'sriov' - register: __sap_hypervisor_node_preconfigure_register_label_node - changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 + +# ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} \ +# feature.node.kubernetes.io/network-sriov.capable=true --overwrite=true" +# when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'sriov' +# register: __sap_hypervisor_node_preconfigure_register_label_node +# changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 - name: "Create SRIOV NodeNetworkConfigurationPolicy \ {{ __sap_hypervisor_node_preconfigure_register_worker_network.name }} on \ diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml index 610032b6a..d4d43c252 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml @@ -69,13 +69,7 @@ cpuManagerPolicy: static cpuManagerReconcilePeriod: 5s -- name: Personalize template - ansible.builtin.template: - src: 99-kargs-worker.yml.j2 - dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-worker.yml" - mode: "0644" - - name: Enable hugepages kubernetes.core.k8s: state: present - src: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-worker.yml" + src: 99-kargs-worker.yml.j2 From b225ee9a296de2aa70b4e265421aff9ad17d5b70 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Fri, 8 Dec 2023 20:53:55 +0100 Subject: [PATCH 36/38] labeling did not work --- .../redhat_ocp_virt/configure-worker-node.yml | 51 +++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml index 6d4d6bc3e..b62d3ed4d 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/configure-worker-node.yml @@ -1,30 +1,41 @@ --- -#- name: Label nodes -# ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} cpumanager=true --overwrite=true" -# register: __sap_hypervisor_node_preconfigure_label_node_result -# changed_when: __sap_hypervisor_node_preconfigure_label_node_result.rc != 0 +- name: Label nodes + ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} cpumanager=true --overwrite=true" + register: __sap_hypervisor_node_preconfigure_label_node_result + changed_when: __sap_hypervisor_node_preconfigure_label_node_result.rc != 0 -- name: Label the node with cpumanager=true - ansible.builtin.k8s: - definition: - apiVersion: v1 - kind: Node - metadata: - name: {{ __sap_hypervisor_node_preconfigure_register_worker.name }} - labels: - cpumanager: true - state: present +#- name: Label the node with cpumanager=true +# ansible.builtin.k8s: +# definition: +# apiVersion: v1 +# kind: Node +# metadata: +# name: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" +# labels: +# cpumanager: true +# state: present +# +#- name: Label the node with invtsc=true +# ansible.builtin.k8s: +# definition: +# apiVersion: v1 +# kind: Node +# metadata: +# name: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" +# labels: +# invtsc: true +# state: present -- name: Label the node with invtsc=true - ansible.builtin.k8s: +- name: Label worker with invtsc flag + kubernetes.core.k8s: + state: present definition: apiVersion: v1 - kind: Node + kind: Namespace metadata: - name: {{ __sap_hypervisor_node_preconfigure_register_worker.name }} + name: default labels: - invtsc: true - state: present + 'feature.node.kubernetes.io/cpu-feature-invtsc': enabled - name: Include node network ansible.builtin.include_tasks: node-network.yml From 53be961b91f56ff8885f85dd0b7c59f28022897e Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 11 Dec 2023 09:08:26 +0100 Subject: [PATCH 37/38] reverted not working PR suggestions: booleans in trident template; template rendering and usage in one go for HPs --- .../tasks/platform/redhat_ocp_virt/install-trident.yml | 4 ++-- .../tasks/platform/redhat_ocp_virt/node-network.yml | 2 +- .../tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml | 8 +++++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml index f1abd1fdb..a54b8715b 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/install-trident.yml @@ -43,6 +43,6 @@ provisioner: csi.trident.netapp.io parameters: backendType: "{{ sap_hypervisor_node_preconfigure_cluster_config.trident.storage_driver }}" - snapshots: true + snapshots: "true" provisioningType: "thin" - encryption: false + encryption: "false" diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml index a42897560..c942aaaaf 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml @@ -48,7 +48,7 @@ apiVersion: v1 kind: Node metadata: - name: {{ __sap_hypervisor_node_preconfigure_register_worker.name }} + name: "{{ __sap_hypervisor_node_preconfigure_register_worker.name }}" labels: feature.node.kubernetes.io/network-sriov.capable: true state: present diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml index d4d43c252..5290093d4 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/setup-worker-nodes.yml @@ -69,7 +69,13 @@ cpuManagerPolicy: static cpuManagerReconcilePeriod: 5s +- name: Render template + ansible.builtin.template: + src: 99-kargs-worker.yml.j2 + dest: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-worker.yml" + mode: "0644" + - name: Enable hugepages kubernetes.core.k8s: state: present - src: 99-kargs-worker.yml.j2 + src: "{{ __sap_hypervisor_node_preconfigure_register_tmpdir.path }}/99-kargs-worker.yml" From b072cb52c7dd2ae04540590a6770a4d78d6c3fa7 Mon Sep 17 00:00:00 2001 From: Nils Koenig Date: Mon, 11 Dec 2023 10:54:22 +0100 Subject: [PATCH 38/38] use kubernetes.core.k8s rather than ansible k8s module --- .../tasks/platform/redhat_ocp_virt/node-network.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml index c942aaaaf..014a28551 100644 --- a/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml +++ b/roles/sap_hypervisor_node_preconfigure/tasks/platform/redhat_ocp_virt/node-network.yml @@ -43,7 +43,7 @@ when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'linux-bridge' - name: Label the node with feature.node.kubernetes.io/network-sriov.capable=true - ansible.builtin.k8s: + kubernetes.core.k8s: definition: apiVersion: v1 kind: Node @@ -52,15 +52,8 @@ labels: feature.node.kubernetes.io/network-sriov.capable: true state: present - when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'sriov' -# ansible.builtin.command: "oc label node {{ __sap_hypervisor_node_preconfigure_register_worker.name }} \ -# feature.node.kubernetes.io/network-sriov.capable=true --overwrite=true" -# when: __sap_hypervisor_node_preconfigure_register_worker_network.type == 'sriov' -# register: __sap_hypervisor_node_preconfigure_register_label_node -# changed_when: __sap_hypervisor_node_preconfigure_register_label_node.rc != 0 - - name: "Create SRIOV NodeNetworkConfigurationPolicy \ {{ __sap_hypervisor_node_preconfigure_register_worker_network.name }} on \ {{ __sap_hypervisor_node_preconfigure_register_worker.name }}"