diff --git a/hack/ci-e2e.sh b/hack/ci-e2e.sh index 09416ef20a..f828f088d4 100755 --- a/hack/ci-e2e.sh +++ b/hack/ci-e2e.sh @@ -4,6 +4,10 @@ # Description: This script sets up the environment and runs E2E tests for the # BMO project. It uses either vbmc or sushy-tools based on # the BMO_E2E_EMULATOR environment variable. +# With sushy-tools, it is also possible to choose between +# redfish-virtualmedia and redfish protocols using the +# SUSHY_TOOLS_PROTOCOL environment variable. +# By default, sushy-tools and redfish-virtualmedia will be used. # Usage: export BMO_E2E_EMULATOR="vbmc" # Or "sushy-tools" # ./ci-e2e.sh # ----------------------------------------------------------------------------- @@ -16,6 +20,8 @@ cd "${REPO_ROOT}" || exit 1 # BMO_E2E_EMULATOR can be set to either "vbmc" or "sushy-tools" BMO_E2E_EMULATOR=${BMO_E2E_EMULATOR:-"sushy-tools"} +# We can choose to use redfish-virtualmedia or redfish as the protocol when using sushy-tools +SUSHY_TOOLS_PROTOCOL=${SUSHY_TOOLS_PROTOCOL:-"redfish-virtualmedia"} export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/ironic.yaml" @@ -68,6 +74,8 @@ rm /tmp/bmo-e2e.tar IP_ADDRESS="192.168.222.1" if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then + BMC_PROTOCOL="ipmi" + export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-ipmi.yaml" # Start VBMC docker run --name vbmc --network host -d \ -v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock \ @@ -76,6 +84,8 @@ if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then elif [[ "${BMO_E2E_EMULATOR}" == "sushy-tools" ]]; then + BMC_PROTOCOL=${SUSHY_TOOLS_PROTOCOL} + export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-${SUSHY_TOOLS_PROTOCOL}.yaml" # Sushy-tools variables SUSHY_EMULATOR_FILE="${REPO_ROOT}"/test/e2e/sushy-tools/sushy-emulator.conf @@ -91,7 +101,6 @@ else exit 1 fi -export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-${BMO_E2E_EMULATOR}.yaml" "${REPO_ROOT}/hack/create_bmcs.sh" "${E2E_BMCS_CONF_FILE}" baremetal-e2e # Set the number of ginkgo processes to the number of BMCs @@ -106,16 +115,42 @@ export IMAGE_URL="http://${IP_ADDRESS}/${IMAGE_FILE}" IMAGE_DIR="${REPO_ROOT}/test/e2e/images" mkdir -p "${IMAGE_DIR}" -## Download and run image server -wget --quiet -P "${IMAGE_DIR}"/ https://artifactory.nordix.org/artifactory/metal3/images/iso/"${IMAGE_FILE}" +## Download disk images +wget --quiet -P "${IMAGE_DIR}/" https://artifactory.nordix.org/artifactory/metal3/images/iso/"${IMAGE_FILE}" +wget --quiet -P "${IMAGE_DIR}/" https://fastly-cdn.system-rescue.org/releases/11.00/systemrescue-11.00-amd64.iso +## Start the image server docker run --name image-server-e2e -d \ -p 80:8080 \ -v "${IMAGE_DIR}:/usr/share/nginx/html" nginxinc/nginx-unprivileged -# Generate the key pair +# Generate ssh key pair for verifying provisioned BMHs ssh-keygen -t ed25519 -f "${IMAGE_DIR}/ssh_testkey" -q -N "" +# Build an ISO image with baked ssh key +# See https://www.system-rescue.org/scripts/sysrescue-customize/ +# We use the systemrescue ISO and their script for customizing it. +pushd "${IMAGE_DIR}" +wget -O sysrescue-customize https://gitlab.com/systemrescue/systemrescue-sources/-/raw/main/airootfs/usr/share/sysrescue/bin/sysrescue-customize?inline=false +chmod +x sysrescue-customize + +pub_ssh_key=$(cut -d " " -f "1,2" "ssh_testkey.pub") + +mkdir -p recipe/iso_add/sysrescue.d +# Reference: https://www.system-rescue.org/manual/Configuring_SystemRescue/ +cat << EOF > recipe/iso_add/sysrescue.d/90-config.yaml +--- +global: + nofirewall: true +sysconfig: + authorized_keys: + "test@example.com": "${pub_ssh_key}" +EOF + +./sysrescue-customize --auto --recipe-dir recipe --source systemrescue-11.00-amd64.iso --dest=sysrescue-out.iso +export ISO_IMAGE_URL="http://${IP_ADDRESS}/sysrescue-out.iso" +popd + # Generate credentials BMO_OVERLAYS=("${REPO_ROOT}/config/overlays/e2e" "${REPO_ROOT}/config/overlays/e2e-release-0.4" "${REPO_ROOT}/config/overlays/e2e-release-0.5") IRONIC_OVERLAY="${REPO_ROOT}/ironic-deployment/overlays/e2e" @@ -157,6 +192,6 @@ sudo sh -c "cp -r /var/log/libvirt/qemu/* ${LOGS_DIR}/qemu/" sudo chown -R "${USER}:${USER}" "${LOGS_DIR}/qemu" # Collect all artifacts -tar --directory test/e2e/ -czf "artifacts-e2e-${BMO_E2E_EMULATOR}.tar.gz" _artifacts +tar --directory test/e2e/ -czf "artifacts-e2e-${BMO_E2E_EMULATOR}-${BMC_PROTOCOL}.tar.gz" _artifacts exit "${test_status}" diff --git a/test/e2e/common.go b/test/e2e/common.go index ba76442bf6..cadd0849ce 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -202,8 +202,9 @@ func HasRootOnDisk(output string) bool { continue // Skip malformed lines } - if fields[5] == "/" && !strings.Contains(fields[0], "tmpfs") { - return true // Found a non-tmpfs root filesystem + // When booting from memory or live-ISO we can have root on tmpfs or airootfs + if fields[5] == "/" && !(strings.Contains(fields[0], "tmpfs") || strings.Contains(fields[0], "airootfs")) { + return true } } @@ -257,12 +258,8 @@ func createCirrosInstanceAndHostnameUserdata(ctx context.Context, client client. userDataContent := fmt.Sprintf(`#!/bin/sh mkdir /root/.ssh -mkdir /home/cirros/.ssh chmod 700 /root/.ssh -chmod 700 /home/cirros/.ssh -chown cirros /home/cirros/.ssh -echo "%s" >> /home/cirros/.ssh/authorized_keys -echo "%s" >> /root/.ssh/authorized_keys`, sshPubKeyData, sshPubKeyData) +echo "%s" >> /root/.ssh/authorized_keys`, sshPubKeyData) CreateSecret(ctx, client, namespace, secretName, map[string]string{"userData": userDataContent}) } @@ -271,7 +268,7 @@ echo "%s" >> /root/.ssh/authorized_keys`, sshPubKeyData, sshPubKeyData) // The `expectedBootMode` parameter should be "disk" or "memory". // The `auth` parameter is an ssh.AuthMethod for authentication. func PerformSSHBootCheck(e2eConfig *Config, expectedBootMode string, auth ssh.AuthMethod, sshAddress string) { - user := e2eConfig.GetVariable("CIRROS_USERNAME") + user := e2eConfig.GetVariable("SSH_USERNAME") client := EstablishSSHConnection(e2eConfig, auth, user, sshAddress) defer func() { diff --git a/test/e2e/config/bmcs-vbmc.yaml b/test/e2e/config/bmcs-ipmi.yaml similarity index 100% rename from test/e2e/config/bmcs-vbmc.yaml rename to test/e2e/config/bmcs-ipmi.yaml diff --git a/test/e2e/config/bmcs-redfish-virtualmedia.yaml b/test/e2e/config/bmcs-redfish-virtualmedia.yaml new file mode 100644 index 0000000000..f991a792d1 --- /dev/null +++ b/test/e2e/config/bmcs-redfish-virtualmedia.yaml @@ -0,0 +1,14 @@ +- user: admin + password: password + address: "redfish-virtualmedia+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-0" + bootMacAddress: "00:60:2f:31:81:01" + hostName: "bmo-e2e-0" + ipAddress: "192.168.222.122" + sshPort: "22" +- user: admin + password: password + address: "redfish-virtualmedia+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-1" + bootMacAddress: "00:60:2f:31:81:02" + hostName: "bmo-e2e-1" + ipAddress: "192.168.222.123" + sshPort: "22" diff --git a/test/e2e/config/bmcs-sushy-tools.yaml b/test/e2e/config/bmcs-redfish.yaml similarity index 100% rename from test/e2e/config/bmcs-sushy-tools.yaml rename to test/e2e/config/bmcs-redfish.yaml diff --git a/test/e2e/config/fixture.yaml b/test/e2e/config/fixture.yaml index 9fb6517892..e7115c1ea3 100644 --- a/test/e2e/config/fixture.yaml +++ b/test/e2e/config/fixture.yaml @@ -26,6 +26,7 @@ variables: UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/fixture-release-0.5" IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img" + ISO_IMAGE_URL: "http://192.168.222.1/cirros.iso" IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906" CERT_MANAGER_VERSION: "v1.13.1" SSH_CHECK_PROVISIONED: "false" diff --git a/test/e2e/config/ironic.yaml b/test/e2e/config/ironic.yaml index a04af65a14..137af33b5f 100644 --- a/test/e2e/config/ironic.yaml +++ b/test/e2e/config/ironic.yaml @@ -30,11 +30,11 @@ variables: UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/e2e-release-0.5" IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img" + ISO_IMAGE_URL: "http://192.168.222.1/minimal_linux_live-v2.iso" IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906" CERT_MANAGER_VERSION: "v1.13.1" SSH_CHECK_PROVISIONED: "true" - CIRROS_USERNAME: "cirros" - CIRROS_PASSWORD: "gocubsgo" + SSH_USERNAME: "root" SSH_PRIV_KEY: "./images/ssh_testkey" SSH_PUB_KEY: "./images/ssh_testkey.pub" diff --git a/test/e2e/live_iso_test.go b/test/e2e/live_iso_test.go index efd5a25014..223d470c70 100644 --- a/test/e2e/live_iso_test.go +++ b/test/e2e/live_iso_test.go @@ -3,6 +3,7 @@ package e2e import ( "context" "fmt" + "os" "golang.org/x/crypto/ssh" @@ -16,6 +17,7 @@ import ( "sigs.k8s.io/cluster-api/util/patch" metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" + metal3bmc "github.com/metal3-io/baremetal-operator/pkg/hardwareutils/bmc" capm3_e2e "github.com/metal3-io/cluster-api-provider-metal3/test/e2e" ) @@ -30,7 +32,17 @@ var _ = Describe("Live-ISO", Label("required", "live-iso"), func() { ) BeforeEach(func() { - imageURL = e2eConfig.GetVariable("IMAGE_URL") + // Check what kind of BMC we are dealing with + // It may be *possible* to boot a live-ISO over (i)PXE, but there are severe limitations. + // Therefore we skip the test if it doesn't support ISO preprovisioning images. + // See https://docs.openstack.org/ironic/latest/admin/ramdisk-boot.html + accessDetails, err := metal3bmc.NewAccessDetails(bmc.Address, false) + Expect(err).NotTo(HaveOccurred()) + if !accessDetails.SupportsISOPreprovisioningImage() { + Skip(fmt.Sprintf("BMC does not support ISO images. It does not make sense to test live-ISO here. BMC address: %s", bmc.Address)) + } + + imageURL = e2eConfig.GetVariable("ISO_IMAGE_URL") namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ Creator: clusterProxy.GetClient(), @@ -68,8 +80,9 @@ var _ = Describe("Live-ISO", Label("required", "live-iso"), func() { URL: imageURL, DiskFormat: pointer.String("live-iso"), }, - BootMode: metal3api.Legacy, - BootMACAddress: bmc.BootMacAddress, + BootMode: metal3api.Legacy, + BootMACAddress: bmc.BootMacAddress, + AutomatedCleaningMode: metal3api.CleaningModeDisabled, }, } err := clusterProxy.GetClient().Create(ctx, &bmh) @@ -92,8 +105,12 @@ var _ = Describe("Live-ISO", Label("required", "live-iso"), func() { // The ssh check is not possible in all situations (e.g. fixture) so it can be skipped if e2eConfig.GetVariable("SSH_CHECK_PROVISIONED") == "true" { By("Verifying the node booted from live ISO image") - password := e2eConfig.GetVariable("CIRROS_PASSWORD") - auth := ssh.Password(password) + keyPath := e2eConfig.GetVariable("SSH_PRIV_KEY") + key, err := os.ReadFile(keyPath) + Expect(err).NotTo(HaveOccurred(), "unable to read private key") + signer, err := ssh.ParsePrivateKey(key) + Expect(err).NotTo(HaveOccurred(), "unable to parse private key") + auth := ssh.PublicKeys(signer) PerformSSHBootCheck(e2eConfig, "memory", auth, fmt.Sprintf("%s:%s", bmc.IPAddress, bmc.SSHPort)) } else { capm3_e2e.Logf("WARNING: Skipping SSH check since SSH_CHECK_PROVISIONED != true") diff --git a/test/go.mod b/test/go.mod index 5a17752062..4285d77cc7 100644 --- a/test/go.mod +++ b/test/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/cert-manager/cert-manager v1.10.0 github.com/metal3-io/baremetal-operator/apis v0.5.1 + github.com/metal3-io/baremetal-operator/pkg/hardwareutils v0.5.1 github.com/metal3-io/cluster-api-provider-metal3/test v1.6.1 github.com/onsi/ginkgo/v2 v2.13.2 github.com/onsi/gomega v1.30.0 @@ -79,7 +80,6 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/metal3-io/baremetal-operator/pkg/hardwareutils v0.5.1 // indirect github.com/metal3-io/cluster-api-provider-metal3/api v1.5.1 // indirect github.com/metal3-io/ip-address-manager/api v1.6.2 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect diff --git a/tools/bmh_test/clean_local_bmh_test_setup.sh b/tools/bmh_test/clean_local_bmh_test_setup.sh index 17be14f9dd..641d0083e4 100755 --- a/tools/bmh_test/clean_local_bmh_test_setup.sh +++ b/tools/bmh_test/clean_local_bmh_test_setup.sh @@ -10,7 +10,7 @@ if [[ -n "${VM_LIST}" ]]; then # Loop through the list and delete each virtual machine for vm_name in ${VM_LIST}; do virsh -c qemu:///system destroy --domain "${vm_name}" - virsh -c qemu:///system undefine --domain "${vm_name}" --remove-all-storage + virsh -c qemu:///system undefine --domain "${vm_name}" --remove-all-storage --nvram kubectl delete baremetalhost "${vm_name}" done else @@ -18,7 +18,7 @@ else fi # Clear vbmc -docker rm -f vbmc +docker rm -f vbmc # Clear network virsh -c qemu:///system net-destroy baremetal-e2e