diff --git a/common/nvme b/common/nvme index 9e78f3e5..88b3efcd 100644 --- a/common/nvme +++ b/common/nvme @@ -8,10 +8,12 @@ def_traddr="127.0.0.1" def_adrfam="ipv4" def_trsvcid="4420" -def_remote_wwnn="0x10001100aa000001" -def_remote_wwpn="0x20001100aa000001" -def_local_wwnn="0x10001100aa000002" -def_local_wwpn="0x20001100aa000002" +def_anagrpid="1" +def_anastate="optimized" +def_remote_wwnn="0x10001100ab000001" +def_remote_wwpn="0x20001100ab000001" +def_local_wwnn="0x10001100aa000001" +def_local_wwpn="0x20001100aa000001" def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349" def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}" export def_subsysnqn="blktests-subsystem-1" @@ -82,18 +84,6 @@ _nvme_fcloop_add_tport() { echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port } -_setup_fcloop() { - local local_wwnn="${1:-$def_local_wwnn}" - local local_wwpn="${2:-$def_local_wwpn}" - local remote_wwnn="${3:-$def_remote_wwnn}" - local remote_wwpn="${4:-$def_remote_wwpn}" - - _nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}" - _nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}" - _nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \ - "${remote_wwnn}" "${remote_wwpn}" -} - _nvme_fcloop_del_rport() { local local_wwnn="$1" local local_wwpn="$2" @@ -129,18 +119,6 @@ _nvme_fcloop_del_tport() { echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port" } -_cleanup_fcloop() { - local local_wwnn="${1:-$def_local_wwnn}" - local local_wwpn="${2:-$def_local_wwpn}" - local remote_wwnn="${3:-$def_remote_wwnn}" - local remote_wwpn="${4:-$def_remote_wwpn}" - - _nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}" - _nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}" - _nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \ - "${remote_wwnn}" "${remote_wwpn}" -} - _cleanup_blkdev() { local blkdev local dev @@ -206,8 +184,7 @@ _cleanup_nvmet() { trap SIGINT if [[ "${nvme_trtype}" == "fc" ]]; then - _cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \ - "${def_remote_wwnn}" "${def_remote_wwpn}" + _nvme_fcloop_del_lport "${def_local_wwnn}" "${def_local_wwpn}" modprobe -rq nvme-fcloop 2>/dev/null fi modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null @@ -248,15 +225,7 @@ _setup_nvmet() { fi if [[ "${nvme_trtype}" = "fc" ]]; then modprobe -q nvme-fcloop - _setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \ - "${def_remote_wwnn}" "${def_remote_wwpn}" - - def_traddr=$(printf "nn-%s:pn-%s" \ - "${def_remote_wwnn}" \ - "${def_remote_wwpn}") - def_host_traddr=$(printf "nn-%s:pn-%s" \ - "${def_local_wwnn}" \ - "${def_local_wwpn}") + _nvme_fcloop_add_lport "${def_local_wwnn}" "${def_local_wwpn}" fi } @@ -279,10 +248,15 @@ _nvme_connect_subsys() { local reconnect_delay="" local ctrl_loss_tmo="" local no_wait=false + local port local i while [[ $# -gt 0 ]]; do case $1 in + --port) + port="$2" + shift 2 + ;; --subsysnqn) subsysnqn="$2" shift 2 @@ -338,12 +312,18 @@ _nvme_connect_subsys() { esac done - ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}") - if [[ "${nvme_trtype}" == "fc" ]] ; then - ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}") - elif [[ "${nvme_trtype}" != "loop" ]]; then - ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}") + if [[ -z "${port}" ]]; then + local ports + + read -ra ports < <(_get_nvmet_ports "${subsysnqn}") + port="${ports[0]##*/}" + if [[ -z "${port}" ]]; then + echo "WARNING: no port found" + return 1 + fi fi + read -ra ARGS < <(_get_nvmet_port_params "${port}") + ARGS+=(--nqn "${subsysnqn}") ARGS+=(--hostnqn="${hostnqn}") ARGS+=(--hostid="${hostid}") if [[ -n "${hostkey}" ]]; then @@ -370,18 +350,21 @@ _nvme_connect_subsys() { if [[ -n "${ctrl_loss_tmo}" ]]; then ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}") fi + ARGS+=(-o json) + connect=$(nvme connect "${ARGS[@]}" 2> /dev/null) - nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:" - - # Wait until device file and uuid/wwid sysfs attributes get ready for - # all namespaces. + # Wait until device file and sysfs attributes get ready if [[ ${no_wait} = false ]]; then + local ctrldev + + ctrldev=$(echo "${connect}" | sed -n 's/.*device.:.\(nvme[0-9]*\)./\1/p') udevadm settle for ((i = 0; i < 10; i++)); do - _nvme_ns_ready "${subsysnqn}" && return + _nvme_ctrl_ready "${ctrldev}" "${subsysnqn}" && return 0 sleep .1 done fi + return 1 } _nvme_disconnect_subsys() { @@ -405,28 +388,25 @@ _nvme_disconnect_subsys() { } -_nvme_ns_ready() { - local subsysnqn="${1}" - local ns_path ns_id dev - local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn" +_nvme_ctrl_ready() { + local ctrldev="${1}" + local subsysnqn="${2:-$def_subsysnqn}" + local ctrlpath="/sys/class/nvme/${ctrldev}" - dev=$(_find_nvme_dev "$subsysnqn") - for ns_path in "${cfs_path}/namespaces/"*; do - ns_id=${ns_path##*/} - if [[ ! -b /dev/${dev}n${ns_id} || - ! -e /sys/block/${dev}n${ns_id}/uuid || - ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then - return 1 - fi - done - return 0 + nqn=$(cat "${ctrlpath}/subsysnqn" 2> /dev/null) + if [[ "${nqn}" == "${subsysnqn}" && + -c /dev/${ctrldev} ]]; then + return 0 + fi + return 1 } _create_nvmet_port() { - local trtype="$1" - local traddr="${2:-$def_traddr}" - local adrfam="${3:-$def_adrfam}" - local trsvcid="${4:-$def_trsvcid}" + local trtype="${nvme_trtype}" + local traddr="${def_traddr}" + local adrfam="${def_adrfam}" + local trsvcid="${def_trsvcid}" + local portcfs local port for ((port = 0; ; port++)); do @@ -435,49 +415,199 @@ _create_nvmet_port() { fi done - mkdir "${NVMET_CFS}/ports/${port}" - echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype" - echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr" - echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam" - if [[ "${adrfam}" != "fc" ]]; then - echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid" + portcfs="${NVMET_CFS}/ports/${port}" + mkdir "${portcfs}" + if [[ "${trtype}" == "tcp" ]] || [[ "${trtype}" == "rdma" ]]; then + trsvcid=$(printf "%d" $(( trsvcid + port )) ) + elif [[ "${trtype}" == "loop" ]]; then + traddr="${port}" + adrfam="loop" + elif [[ "${trtype}" == "fc" ]]; then + local local_wwnn="${def_local_wwnn}" + local local_wwpn="${def_local_wwpn}" + local remote_wwnn + local remote_wwpn + + remote_wwnn=$(printf "0x%08x" $(( def_remote_wwnn + port )) ) + remote_wwpn=$(printf "0x%08x" $(( def_remote_wwpn + port )) ) + _nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}" + _nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \ + "${remote_wwnn}" "${remote_wwpn}" + + traddr=$(printf "nn-%s:pn-%s" \ + "${remote_wwnn}" \ + "${remote_wwpn}") + adrfam="fc" + fi + echo "${trtype}" > "${portcfs}/addr_trtype" + echo "${traddr}" > "${portcfs}/addr_traddr" + echo "${adrfam}" > "${portcfs}/addr_adrfam" + if [[ "${adrfam}" != "fc" ]] && \ + [[ "${adrfam}" != "loop" ]] ; then + echo "${trsvcid}" > "${portcfs}/addr_trsvcid" fi echo "${port}" } +_setup_nvmet_port_ana() { + local port="$1" + local anagrpid="${2:-$def_anagrpid}" + local anastate="${3:-$def_anastate}" + local cfsport="${NVMET_CFS}/ports/${port}" + local anaport="${cfsport}/ana_groups/${anagrpid}" + + if [[ ! -d "${anaport}" ]] ; then + if [[ "${anagrpid}" -eq 1 ]]; then + echo "FAIL target setup failed, ANA not supported" + exit 1 + fi + mkdir "${anaport}" + fi + echo "${anastate}" > "${anaport}/ana_state" +} + _remove_nvmet_port() { local port="$1" + local cfspath="${NVMET_CFS}/ports/${port}" + local trtype + local traddr + + trtype=$(cat "${cfspath}/addr_trtype") + traddr=$(cat "${cfspath}/addr_traddr") + if [[ "${trtype}" == "fc" ]]; then + local nn + local pn + local remote_wwnn + local remote_wwpn + local local_wwnn="${def_local_wwnn}" + local local_wwpn="${def_local_wwpn}" + + # shellcheck disable=SC2086 + nn=$(IFS=: ; set -- ${traddr} ; echo "$1" ) + # shellcheck disable=SC2086 + pn=$(IFS=: ; set -- ${traddr} ; echo "$2" ) + remote_wwnn="${nn##nn-}" + remote_wwpn="${pn##pn-}" + + _nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \ + "${remote_wwnn}" "${remote_wwpn}" + _nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}" + fi + for a in "${cfspath}/ana_groups/"*; do + [[ "${a##*/}" == "1" ]] && continue + rmdir "${a}" + done rmdir "${NVMET_CFS}/ports/${port}" } _create_nvmet_ns() { + local subsysnqn="${def_subsysnqn}" + local nsid="1" + local grpid="1" + local blkdev + local uuid + local subsys_path + local ns_path + + while [[ $# -gt 0 ]]; do + case $1 in + --subsysnqn) + subsysnqn="$2" + shift 2 + ;; + --nsid) + nsid="$2" + shift 2 + ;; + --blkdev) + blkdev="$2" + shift 2 + ;; + --uuid) + uuid="$2" + shift 2 + ;; + --grpid) + grpid="$2" + shift 2 + ;; + *) + echo "WARNING: unknown argument: $1" + shift + ;; + esac + done + + subsys_path="${NVMET_CFS}/subsystems/${subsysnqn}" + ns_path="${subsys_path}/namespaces/${nsid}" + mkdir "${ns_path}" + printf "%s" "${blkdev}" > "${ns_path}/device_path" + if [[ -n "${uuid}" ]]; then + printf "%s" "${uuid}" > "${ns_path}/device_uuid" + else + uuid=$(cat "${ns_path}/device_uuid") + fi + if (( grpid != 1 )); then + printf "%d" "${grpid}" > "${ns_path}/ana_grpid" + fi + printf 1 > "${ns_path}/enable" + echo "${uuid}" +} + +_setup_nvmet_ns_ana() { local nvmet_subsystem="$1" local nsid="$2" - local blkdev="$3" - local uuid="00000000-0000-0000-0000-000000000000" + local anagrpid="${3:-def_anagrpid}" local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" local ns_path="${subsys_path}/namespaces/${nsid}" - if [[ $# -eq 4 ]]; then - uuid="$4" + if [[ ! -d "${ns_path}" ]]; then + return fi - - mkdir "${ns_path}" - printf "%s" "${blkdev}" > "${ns_path}/device_path" - printf "%s" "${uuid}" > "${ns_path}/device_uuid" - printf 1 > "${ns_path}/enable" + echo "${anagrpid}" > "${ns_path}/anagrpid" } _create_nvmet_subsystem() { - local nvmet_subsystem="$1" - local blkdev="$2" - local uuid=$3 - local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" + local subsystem="${def_subsysnqn}" + local blkdev + local uuid="${def_subsys_uuid}" + local cfs_path + local -a ARGS + while [[ $# -gt 0 ]]; do + case $1 in + --subsysnqn) + subsystem="$2" + shift 2 + ;; + --blkdev) + blkdev="$2" + shift 2 + ;; + --uuid) + uuid="$2" + shift 2 + ;; + *) + echo "WARNING: unknown argument: $1" + shift + ;; + esac + done + + cfs_path="${NVMET_CFS}/subsystems/${subsystem}" mkdir -p "${cfs_path}" echo 0 > "${cfs_path}/attr_allow_any_host" - _create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" + if [[ -z "${blkdev}" ]]; then + return 0 + fi + ARGS+=(--subsysnqn "${subsystem}") + ARGS+=(--blkdev "${blkdev}") + if [[ -n "$uuid" ]]; then + ARGS+=(--uuid "${uuid}") + fi + _create_nvmet_ns "${ARGS[@]}" > /dev/null } _add_nvmet_allow_hosts() { @@ -525,7 +655,10 @@ _remove_nvmet_subsystem() { local nvmet_subsystem="$1" local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" - _remove_nvmet_ns "${nvmet_subsystem}" "1" + for n in "${subsys_path}/namespaces/"*; do + [ -d "${n}" ] || continue + _remove_nvmet_ns "${nvmet_subsystem}" "${n##*/}" + done rm -f "${subsys_path}"/allowed_hosts/* rmdir "${subsys_path}" } @@ -553,16 +686,47 @@ _remove_nvmet_subsystem_from_port() { } _get_nvmet_ports() { - local nvmet_subsystem="$1" - local -n nvmet_ports="$2" + local nvmet_subsysnqn="${1:-$def_subsysnqn}" + local -a nvmet_ports=() local cfs_path="${NVMET_CFS}/ports" - local sarg - sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p" - - for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do - nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")") + for port in "${cfs_path}/"*; do + if [[ -e "${port}/subsystems/${nvmet_subsysnqn}" ]]; then + nvmet_ports+=("${port##*/}") + fi done + + if (( ${#nvmet_ports[@]} )); then + echo "${nvmet_ports[@]}" + fi +} + +_get_nvmet_port_params() { + local port="$1" + local -a args=() + local cfs_path="${NVMET_CFS}/ports/${port}" + local trtype + local traddr + local trsvcid + + [[ -d "${cfs_path}" ]] || exit 1 + trtype=$(cat "${cfs_path}/addr_trtype") + args+=(--transport "${trtype}") + traddr=$(cat "${cfs_path}/addr_traddr") + args+=(--traddr "${traddr}") + if [[ "${trtype}" == "tcp" ]] || [[ "${trtype}" == "rdma" ]]; then + trsvcid=$(cat "${cfs_path}/addr_trsvcid") + args+=(--trsvcid "${trsvcid}") + elif [[ "${trtype}" == "fc" ]]; then + local host_traddr + + host_traddr=$(printf "nn-%s:pn-%s" \ + "${def_local_wwnn}" \ + "${def_local_wwpn}") + args+=(--host-traddr "${host_traddr}") + fi + + echo "${args[@]}" } _find_nvme_dev() { @@ -575,8 +739,10 @@ _find_nvme_dev() { subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)" if [[ "$subsysnqn" == "$subsys" ]]; then echo "$dev" + return 0 fi done + return 1 } _find_nvme_ns() { @@ -593,8 +759,82 @@ _find_nvme_ns() { uuid=$(cat "${ns}/uuid") if [[ "${subsys_uuid}" == "${uuid}" ]]; then basename "${ns}" + return 0 fi done + return 1 +} + +_nvmet_target_setup() { + local blkdev_type="${nvmet_blkdev_type}" + local blkdev + local ctrlkey="" + local hostkey="" + local subsysnqn="${def_subsysnqn}" + local subsys_uuid + local port + local num_ports=1 + local -a ARGS + + while [[ $# -gt 0 ]]; do + case $1 in + --blkdev) + blkdev_type="$2" + shift 2 + ;; + --ctrlkey) + ctrlkey="$2" + shift 2 + ;; + --hostkey) + hostkey="$2" + shift 2 + ;; + --subsysnqn) + subsysnqn="$2" + shift 2 + ;; + --subsys-uuid) + subsys_uuid="$2" + shift 2 + ;; + --ports) + num_ports="$2" + shift 2 + ;; + *) + echo "WARNING: unknown argument: $1" + shift + ;; + esac + done + + if [[ "${blkdev_type}" != "none" ]]; then + truncate -s "${NVME_IMG_SIZE}" "$(_nvme_def_file_path)" + if [[ "${blkdev_type}" == "device" ]]; then + blkdev="$(losetup -f --show "$(_nvme_def_file_path)")" + else + blkdev="$(_nvme_def_file_path)" + fi + fi + + ARGS=(--subsysnqn "${subsysnqn}") + if [[ -n "${blkdev}" ]]; then + ARGS+=(--blkdev "${blkdev}") + fi + if [[ -n "${subsys_uuid}" ]]; then + ARGS+=(--uuid "${subsys_uuid}") + fi + _create_nvmet_subsystem "${ARGS[@]}" + + p=0 + while (( p < num_ports )); do + port="$(_create_nvmet_port)" + _add_nvmet_subsys_to_port "${port}" "${subsysnqn}" + p=$(( p + 1 )) + done + _create_nvmet_host "${subsysnqn}" "${def_hostnqn}" \ + "${hostkey}" "${ctrlkey}" } _nvmet_target_cleanup() { @@ -621,7 +861,7 @@ _nvmet_target_cleanup() { esac done - _get_nvmet_ports "${subsysnqn}" ports + read -ra ports < <(_get_nvmet_ports "${subsysnqn}") for port in "${ports[@]}"; do _remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}" diff --git a/tests/md/001 b/tests/md/001 index 27df7b35..f1d92c8d 100755 --- a/tests/md/001 +++ b/tests/md/001 @@ -50,9 +50,9 @@ setup_nvme_over_tcp() { _setup_nvmet local port - port="$(_create_nvmet_port "${nvme_trtype}")" + port="$(_create_nvmet_port)" - _create_nvmet_subsystem "${def_subsysnqn}" "/dev/mapper/ram0_big_optio" "${def_subsys_uuid}" + _create_nvmet_subsystem --blkdev "/dev/mapper/ram0_big_optio" _add_nvmet_subsys_to_port "${port}" "${def_subsysnqn}" _create_nvmet_host "${def_subsysnqn}" "${def_hostnqn}" diff --git a/tests/nvme/002 b/tests/nvme/002 index f613c788..e5b751d2 100755 --- a/tests/nvme/002 +++ b/tests/nvme/002 @@ -26,7 +26,7 @@ test() { local iterations=1000 local port - port="$(_create_nvmet_port "${nvme_trtype}")" + port="$(_create_nvmet_port)" local loop_dev loop_dev="$(losetup -f)" @@ -34,7 +34,8 @@ test() { local genctr=1 for ((i = 0; i < iterations; i++)); do - _create_nvmet_subsystem "blktests-subsystem-$i" "${loop_dev}" + _create_nvmet_subsystem --subsysnqn "blktests-subsystem-$i" \ + --blkdev "${loop_dev}" _add_nvmet_subsys_to_port "${port}" "blktests-subsystem-$i" done diff --git a/tests/nvme/003 b/tests/nvme/003 index 36c07537..26667a39 100755 --- a/tests/nvme/003 +++ b/tests/nvme/003 @@ -26,9 +26,13 @@ test() { _setup_nvmet - _nvmet_target_setup + _create_nvmet_subsystem + port="$(_create_nvmet_port)" + _add_nvmet_subsys_to_port "${port}" "${def_subsysnqn}" + _create_nvmet_host "${def_subsysnqn}" "${def_hostnqn}" - _nvme_connect_subsys --subsysnqn "nqn.2014-08.org.nvmexpress.discovery" + _nvme_connect_subsys --port "${port}" --no-wait \ + --subsysnqn "nqn.2014-08.org.nvmexpress.discovery" # This is ugly but checking for the absence of error messages is ... sleep 10 diff --git a/tests/nvme/016 b/tests/nvme/016 index d1fdb351..33a48018 100755 --- a/tests/nvme/016 +++ b/tests/nvme/016 @@ -29,13 +29,14 @@ test() { loop_dev="$(losetup -f)" local genctr=1 - _create_nvmet_subsystem "${def_subsysnqn}" "${loop_dev}" + _create_nvmet_subsystem - for ((i = 2; i <= iterations; i++)); do - _create_nvmet_ns "${def_subsysnqn}" "${i}" "${loop_dev}" + for i in $(seq 1 "${iterations}"); do + _create_nvmet_ns --nsid "${i}" \ + --blkdev "${loop_dev}" > /dev/null done - port="$(_create_nvmet_port "${nvme_trtype}")" + port="$(_create_nvmet_port)" _add_nvmet_subsys_to_port "$port" "${def_subsysnqn}" genctr=$(_check_genctr "${genctr}" "adding a subsystem to a port") @@ -43,10 +44,6 @@ test() { _remove_nvmet_subsystem_from_port "${port}" "${def_subsysnqn}" _remove_nvmet_port "${port}" - for ((i = iterations; i > 1; i--)); do - _remove_nvmet_ns "${def_subsysnqn}" "$i" - done - _remove_nvmet_subsystem "${def_subsysnqn}" echo "Test complete" diff --git a/tests/nvme/017 b/tests/nvme/017 index 114be607..3aae8b40 100755 --- a/tests/nvme/017 +++ b/tests/nvme/017 @@ -29,15 +29,14 @@ test() { local genctr=1 - _create_nvmet_subsystem "${def_subsysnqn}" "$(_nvme_def_file_path)" \ - "${def_subsys_uuid}" + _create_nvmet_subsystem - for ((i = 2; i <= iterations; i++)); do - _create_nvmet_ns "${def_subsysnqn}" "${i}" \ - "$(_nvme_def_file_path)" + for i in $(seq 1 "${iterations}"); do + _create_nvmet_ns --nsid "${i}" \ + --blkdev "$(_nvme_def_file_path)" > /dev/null done - port="$(_create_nvmet_port "${nvme_trtype}")" + port="$(_create_nvmet_port)" _add_nvmet_subsys_to_port "${port}" "${def_subsysnqn}" genctr=$(_check_genctr "${genctr}" "adding a subsystem to a port") @@ -45,10 +44,6 @@ test() { _remove_nvmet_subsystem_from_port "${port}" "${def_subsysnqn}" _remove_nvmet_port "${port}" - for ((i = iterations; i > 1; i--)); do - _remove_nvmet_ns "${def_subsysnqn}" "$i" - done - _remove_nvmet_subsystem "${def_subsysnqn}" rm "$(_nvme_def_file_path)" diff --git a/tests/nvme/030 b/tests/nvme/030 index b1ed8bc2..1ba1274b 100755 --- a/tests/nvme/030 +++ b/tests/nvme/030 @@ -28,15 +28,17 @@ test() { local genctr local subsys="blktests-subsystem-" - port="$(_create_nvmet_port "${nvme_trtype}")" + port="$(_create_nvmet_port)" - _create_nvmet_subsystem "${subsys}1" "$(losetup -f)" + _create_nvmet_subsystem --subsysnqn "${subsys}1" \ + --blkdev "$(losetup -f)" _add_nvmet_subsys_to_port "${port}" "${subsys}1" _create_nvmet_host "${subsys}1" "${def_hostnqn}" genctr=$(_discovery_genctr) - _create_nvmet_subsystem "${subsys}2" "$(losetup -f)" + _create_nvmet_subsystem --subsysnqn "${subsys}2" \ + --blkdev "$(losetup -f)" _add_nvmet_subsys_to_port "${port}" "${subsys}2" genctr=$(_check_genctr "${genctr}" "adding a subsystem to a port") diff --git a/tests/nvme/031 b/tests/nvme/031 index 00d3d18b..3333338b 100755 --- a/tests/nvme/031 +++ b/tests/nvme/031 @@ -41,17 +41,21 @@ test() { loop_dev="$(losetup -f --show "$(_nvme_def_file_path)")" - port="$(_create_nvmet_port "${nvme_trtype}")" + port="$(_create_nvmet_port)" for ((i = 0; i < iterations; i++)); do - _nvmet_target_setup --subsysnqn "${subsys}$i" \ + subsysnqn=$(printf "blktests-subsystem-%d" "$i") + _create_nvmet_subsystem --subsysnqn "${subsysnqn}" \ --blkdev "${loop_dev}" + _add_nvmet_subsys_to_port "${port}" "${subsysnqn}" + _create_nvmet_host "${subsysnqn}" "${def_hostnqn}" - _nvme_connect_subsys --subsysnqn "${subsys}$i" - _nvme_disconnect_subsys --subsysnqn "${subsys}$i" >> "${FULL}" 2>&1 + _nvme_connect_subsys --subsysnqn "${subsysnqn}" + _nvme_disconnect_subsys --subsysnqn "${subsysnqn}" >> "${FULL}" 2>&1 - _nvmet_target_cleanup --subsysnqn "${subsys}$i" \ - --blkdev "${loop_dev}" + _remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}" + _remove_nvmet_subsystem "${subsysnqn}" + _remove_nvmet_host "${def_hostnqn}" done _remove_nvmet_port "${port}" diff --git a/tests/nvme/038 b/tests/nvme/038 index 84354153..4271ee3a 100755 --- a/tests/nvme/038 +++ b/tests/nvme/038 @@ -33,7 +33,7 @@ test() { mkdir -p "${subsys_path}" rmdir "${subsys_path}" - port=$(_create_nvmet_port loop) + port="$(_create_nvmet_port)" _remove_nvmet_port "${port}" echo "Test complete" diff --git a/tests/nvme/052 b/tests/nvme/052 index 401f043f..3f985273 100755 --- a/tests/nvme/052 +++ b/tests/nvme/052 @@ -53,23 +53,23 @@ test() { local iterations=20 - _nvmet_target_setup + _nvmet_target_setup --blkdev none _nvme_connect_subsys - # start iteration from ns-id 2 because ns-id 1 is created - # by default when nvme target is setup. Also ns-id 1 is - # deleted when nvme target is cleaned up. - for ((i = 2; i <= iterations; i++)); do { - truncate -s "${NVME_IMG_SIZE}" "$(_nvme_def_file_path).$i" - uuid="$(uuidgen -r)" + for i in $(seq 1 "${iterations}"); do + local file_path - _create_nvmet_ns "${def_subsysnqn}" "${i}" "$(_nvme_def_file_path).$i" "${uuid}" + file_path="$(_nvme_def_file_path).${i}" + truncate -s "${NVME_IMG_SIZE}" "${file_path}" + + uuid=$(_create_nvmet_ns --subsysnqn "${def_subsysnqn}" \ + --nsid "${i}" --blkdev "${file_path}") # wait until async request is processed and ns is created if ! nvmf_wait_for_ns "${uuid}" created; then echo "FAIL" - rm "$(_nvme_def_file_path).$i" + rm "${file_path}" break fi @@ -78,12 +78,11 @@ test() { # wait until async request is processed and ns is removed if ! nvmf_wait_for_ns "${uuid}" removed; then echo "FAIL" - rm "$(_nvme_def_file_path).$i" + rm "${file_path}" break fi - rm "$(_nvme_def_file_path).$i" - } + rm "${file_path}" done _nvme_disconnect_subsys >> "${FULL}" 2>&1 diff --git a/tests/nvme/054 b/tests/nvme/054 new file mode 100755 index 00000000..2a20639a --- /dev/null +++ b/tests/nvme/054 @@ -0,0 +1,97 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0+ +# Copyright (C) 2024 Hannes Reinecke (SUSE) +# +# Test nvme fabrics controller ANA failover during I/O + +. tests/nvme/rc + +DESCRIPTION="test nvme fabrics controller ANA failover during I/O" + +requires() { + _nvme_requires + _have_loop + _have_fio + _require_nvme_trtype_is_fabrics +} + +set_conditions() { + _set_nvme_trtype "$@" +} + +failback() { + local portno=0 + + for port in "$@"; do + if (( portno == 0 )); then + _setup_nvmet_port_ana "${port}" 1 "optimized" + elif (( portno == 1 )); then + _setup_nvmet_port_ana "${port}" 1 "non-optimized" + else + _setup_nvmet_port_ana "${port}" 1 "inaccessible" + fi + portno=$(( portno + 1 )) + done +} + +failover() { + local portno=0 + + for port in "$@"; do + if (( portno == 2 )); then + _setup_nvmet_port_ana "${port}" 1 "optimized" + elif (( portno == 3 )); then + _setup_nvmet_port_ana "${port}" 1 "non-optimized" + else + _setup_nvmet_port_ana "${port}" 1 "inaccessible" + fi + portno=$(( portno + 1)) + done +} + +test() { + local fio_pid + local -a ports + local port + local ns + + echo "Running ${TEST_NAME}" + + _setup_nvmet + + _nvmet_target_setup --ports 4 + + read -ra ports < <(_get_nvmet_ports) + failback "${ports[@]}" + + for port in "${ports[@]}"; do + _nvme_connect_subsys --port "${port}" + done + + # start fio job + ns=$(_find_nvme_ns "$def_subsys_uuid") + _run_fio_verify_io --filename="/dev/${ns}" \ + --group_reporting --ramp-time=5 \ + --time_based --runtime=1m &> "$FULL" & + fio_pid=$! + sleep 5 + + # switch port status + echo "ANA failover" + failover "${ports[@]}" + + sleep 10 + + # switch back to original state + echo "ANA failback" + failback "${ports[@]}" + + sleep 10 + + { kill "${fio_pid}"; wait; } &> /dev/null + + _nvme_disconnect_subsys + _nvmet_target_cleanup + + echo "Test complete" +} diff --git a/tests/nvme/054.out b/tests/nvme/054.out new file mode 100644 index 00000000..3c024ae5 --- /dev/null +++ b/tests/nvme/054.out @@ -0,0 +1,5 @@ +Running nvme/054 +ANA failover +ANA failback +disconnected 4 controller(s) +Test complete diff --git a/tests/nvme/055 b/tests/nvme/055 new file mode 100644 index 00000000..32a68c20 --- /dev/null +++ b/tests/nvme/055 @@ -0,0 +1,106 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0+ +# Copyright (C) 2024 Hannes Reinecke (SUSE) +# +# Test rapid remapping of namespaces + +. tests/nvme/rc + +DESCRIPTION="test rapid namespace remapping" + +requires() { + _nvme_requires + _have_loop + _require_nvme_trtype_is_fabrics +} + +set_conditions() { + _set_nvme_trtype "$@" +} + +_setup_ana() { + local portno=0 + + for port in "$@"; do + if (( portno == 0 )); then + _setup_nvmet_port_ana "${port}" 1 "optimized" + _setup_nvmet_port_ana "${port}" 2 "inaccessible" + _setup_nvmet_port_ana "${port}" 3 "inaccessible" + elif (( portno == 1 )); then + _setup_nvmet_port_ana "${port}" 1 "non-optimized" + _setup_nvmet_port_ana "${port}" 2 "inaccessible" + _setup_nvmet_port_ana "${port}" 3 "inaccessible" + elif (( portno == 2 )); then + _setup_nvmet_port_ana "${port}" 1 "inaccessible" + _setup_nvmet_port_ana "${port}" 2 "optimized" + _setup_nvmet_port_ana "${port}" 3 "inaccessible" + elif (( portno == 3 )); then + _setup_nvmet_port_ana "${port}" 1 "inaccessible" + _setup_nvmet_port_ana "${port}" 2 "non-optimized" + _setup_nvmet_port_ana "${port}" 3 "inaccessible" + elif (( portno == 4 )); then + _setup_nvmet_port_ana "${port}" 1 "inaccessible" + _setup_nvmet_port_ana "${port}" 2 "inaccessible" + _setup_nvmet_port_ana "${port}" 3 "optimized" + else + _setup_nvmet_port_ana "${port}" 1 "inaccessible" + _setup_nvmet_port_ana "${port}" 2 "inaccessible" + _setup_nvmet_port_ana "${port}" 3 "non-optimized" + fi + portno=$(( portno + 1 )) + done +} + +test() { + echo "Running ${TEST_NAME}" + + _setup_nvmet + + local d + local i + local -a ports + local num_namespaces=3 + + _nvmet_target_setup --ports 6 --blkdev none + + for d in $(seq 1 "${num_namespaces}"); do + local file_path + local blkdev + local uuid + + file_path="${TMPDIR}/img${d}" + truncate -s "${NVME_IMG_SIZE}" "${file_path}" + blkdev="$(losetup -f --show "${file_path}")" + uuid="$(uuidgen)" + _create_nvmet_ns --nsid "${d}" --blkdev "${blkdev}" \ + --uuid "${uuid}" --grpid "${d}" > /dev/null + done + + read -ra ports < <(_get_nvmet_ports) + _setup_ana "${ports[@]}" + + for port in "${ports[@]}"; do + _nvme_connect_subsys --port "${port}" + done + + # Simulate unmap on one cluster node and remap to anther one + for i in $(seq 0 10); do + echo "Remap namespace #${i}" + grpid=$(( (i % 3) + 1 )) + for n in $(seq 1 "${num_namespaces}" | shuf); do + _disable_nvmet_ns "${n}" + done + for n in $(seq 1 "${num_namespaces}" | shuf); do + local uuid + + uuid=$(uuidgen) + _set_nvmet_ns_uuid "${n}" "${uuid}" + _enable_nvmet_ns "${n}" + done + done + + _nvme_disconnect_subsys + _nvmet_target_cleanup + + echo "Test complete" +} diff --git a/tests/nvme/055.out b/tests/nvme/055.out new file mode 100644 index 00000000..c580c830 --- /dev/null +++ b/tests/nvme/055.out @@ -0,0 +1,14 @@ +Running nvme/055 +Remap namespace #0 +Remap namespace #1 +Remap namespace #2 +Remap namespace #3 +Remap namespace #4 +Remap namespace #5 +Remap namespace #6 +Remap namespace #7 +Remap namespace #8 +Remap namespace #9 +Remap namespace #10 +disconnected 6 controller(s) +Test complete diff --git a/tests/nvme/rc b/tests/nvme/rc index 671012ea..7eb7ca36 100644 --- a/tests/nvme/rc +++ b/tests/nvme/rc @@ -306,6 +306,37 @@ _set_nvmet_dhgroup() { "${cfs_path}/dhchap_dhgroup" } +_enable_nvmet_ns() { + local subsysnqn="${def_subsysnqn}" + local nsid="${1:-1}" + + cfs_path="${NVMET_CFS}/subsystems/${subsysnqn}" + ns_path="${cfs_path}/namespaces/${nsid}" + + echo 1 > "${ns_path}/enable" +} + +_disable_nvmet_ns() { + local subsysnqn="${def_subsysnqn}" + local nsid="${1:-1}" + + cfs_path="${NVMET_CFS}/subsystems/${subsysnqn}" + ns_path="${cfs_path}/namespaces/${nsid}" + + echo 0 > "${ns_path}/enable" +} + +_set_nvmet_ns_uuid() { + local subsysnqn="${def_subsysnqn}" + local nsid="${1:-1}" + local uuid="${2:-$(uuidgen)}" + + cfs_path="${NVMET_CFS}/subsystems/${subsysnqn}" + ns_path="${cfs_path}/namespaces/${nsid}" + + printf "%s" "${uuid}" > "${ns_path}/device_uuid" +} + _find_nvme_passthru_loop_dev() { local subsys=$1 local nsid @@ -316,59 +347,6 @@ _find_nvme_passthru_loop_dev() { echo "/dev/${dev}n${nsid}" } -_nvmet_target_setup() { - local blkdev_type="${nvmet_blkdev_type}" - local blkdev - local ctrlkey="" - local hostkey="" - local subsysnqn="${def_subsysnqn}" - local subsys_uuid="${def_subsys_uuid}" - local port - - while [[ $# -gt 0 ]]; do - case $1 in - --blkdev) - blkdev_type="$2" - shift 2 - ;; - --ctrlkey) - ctrlkey="$2" - shift 2 - ;; - --hostkey) - hostkey="$2" - shift 2 - ;; - --subsysnqn) - subsysnqn="$2" - shift 2 - ;; - --subsys-uuid) - subsys_uuid="$2" - shift 2 - ;; - *) - echo "WARNING: unknown argument: $1" - shift - ;; - esac - done - - truncate -s "${NVME_IMG_SIZE}" "$(_nvme_def_file_path)" - if [[ "${blkdev_type}" == "device" ]]; then - blkdev="$(losetup -f --show "$(_nvme_def_file_path)")" - else - blkdev="$(_nvme_def_file_path)" - fi - - _create_nvmet_subsystem "${subsysnqn}" "${blkdev}" \ - "${subsys_uuid}" - port="$(_create_nvmet_port "${nvme_trtype}")" - _add_nvmet_subsys_to_port "${port}" "${subsysnqn}" - _create_nvmet_host "${subsysnqn}" "${def_hostnqn}" \ - "${hostkey}" "${ctrlkey}" -} - _nvmet_passthru_target_setup() { local subsysnqn="$def_subsysnqn" local port @@ -387,7 +365,7 @@ _nvmet_passthru_target_setup() { done _create_nvmet_passthru "${subsysnqn}" - port="$(_create_nvmet_port "${nvme_trtype}")" + port="$(_create_nvmet_port)" _add_nvmet_subsys_to_port "${port}" "${subsysnqn}" _create_nvmet_host "${subsysnqn}" "${def_hostnqn}" } @@ -444,7 +422,7 @@ _nvmet_passthru_target_cleanup() { esac done - _get_nvmet_ports "${subsysnqn}" ports + read -ra ports < <(_get_nvmet_ports "${subsysnqn}") for port in "${ports[@]}"; do _remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}"