00:30:04.605 ========== Backtrace start: ========== 00:30:04.605 00:30:04.605 in /var/jenkins/workspace/nvmf-cvl-phy-autotest/spdk/test/nvmf/host/failover.sh:47 -> main(["--transport=rdma"]) 00:30:04.605 ... 00:30:04.605 42 00:30:04.605 43 $rpc_py nvmf_subsystem_remove_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT 00:30:04.605 44 00:30:04.605 45 sleep 3 00:30:04.605 46 00:30:04.605 => 47 $rpc_py -s $bdevperf_rpc_sock bdev_nvme_attach_controller -b NVMe0 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_THIRD_PORT -f ipv4 -n nqn.2016-06.io.spdk:cnode1 -x failover 00:30:04.605 48 $rpc_py nvmf_subsystem_remove_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_SECOND_PORT 00:30:04.605 49 00:30:04.605 50 sleep 3 00:30:04.605 51 00:30:04.605 52 # Give the admin qpair time to fail before we add the new listener in. This prevents us from trying to connect to the wrong trid. 00:30:04.605 ... 00:30:04.605 00:30:04.605 ========== Backtrace end ========== 00:30:06.248 ========== Backtrace start: ========== 00:30:06.248 00:30:06.248 in /var/jenkins/workspace/nvmf-cvl-phy-autotest/spdk/test/common/autotest_common.sh:1129 -> run_test(["nvmf_failover"],["/var/jenkins/workspace/nvmf-cvl-phy-autotest/spdk/test/nvmf/host/failover.sh"],["--transport=rdma"]) 00:30:06.248 ... 00:30:06.248 1124 timing_enter $test_name 00:30:06.248 1125 echo "************************************" 00:30:06.248 1126 echo "START TEST $test_name" 00:30:06.248 1127 echo "************************************" 00:30:06.248 1128 xtrace_restore 00:30:06.248 1129 time "$@" 00:30:06.248 1130 xtrace_disable 00:30:06.248 1131 echo "************************************" 00:30:06.248 1132 echo "END TEST $test_name" 00:30:06.248 1133 echo "************************************" 00:30:06.248 1134 timing_exit $test_name 00:30:06.248 ... 00:30:06.248 in /var/jenkins/workspace/nvmf-cvl-phy-autotest/spdk/test/nvmf/nvmf_host.sh:25 -> main(["--transport=rdma"]) 00:30:06.248 ... 00:30:06.248 20 fi 00:30:06.248 21 00:30:06.248 22 run_test "nvmf_identify" $rootdir/test/nvmf/host/identify.sh "${TEST_ARGS[@]}" 00:30:06.248 23 run_test "nvmf_perf" $rootdir/test/nvmf/host/perf.sh "${TEST_ARGS[@]}" 00:30:06.248 24 run_test "nvmf_fio_host" $rootdir/test/nvmf/host/fio.sh "${TEST_ARGS[@]}" 00:30:06.248 => 25 run_test "nvmf_failover" $rootdir/test/nvmf/host/failover.sh "${TEST_ARGS[@]}" 00:30:06.248 26 run_test "nvmf_host_discovery" $rootdir/test/nvmf/host/discovery.sh "${TEST_ARGS[@]}" 00:30:06.248 27 run_test "nvmf_host_multipath_status" $rootdir/test/nvmf/host/multipath_status.sh "${TEST_ARGS[@]}" 00:30:06.248 28 run_test "nvmf_discovery_remove_ifc" $rootdir/test/nvmf/host/discovery_remove_ifc.sh "${TEST_ARGS[@]}" 00:30:06.248 29 run_test "nvmf_identify_kernel_target" "$rootdir/test/nvmf/host/identify_kernel_nvmf.sh" "${TEST_ARGS[@]}" 00:30:06.248 30 run_test "nvmf_auth_host" "$rootdir/test/nvmf/host/auth.sh" "${TEST_ARGS[@]}" 00:30:06.248 ... 00:30:06.248 00:30:06.248 ========== Backtrace end ========== 00:30:06.248 ========== Backtrace start: ========== 00:30:06.248 00:30:06.248 in /var/jenkins/workspace/nvmf-cvl-phy-autotest/spdk/test/common/autotest_common.sh:1129 -> run_test(["nvmf_host"],["/var/jenkins/workspace/nvmf-cvl-phy-autotest/spdk/test/nvmf/nvmf_host.sh"],["--transport=rdma"]) 00:30:06.248 ... 00:30:06.248 1124 timing_enter $test_name 00:30:06.248 1125 echo "************************************" 00:30:06.248 1126 echo "START TEST $test_name" 00:30:06.248 1127 echo "************************************" 00:30:06.248 1128 xtrace_restore 00:30:06.248 1129 time "$@" 00:30:06.248 1130 xtrace_disable 00:30:06.248 1131 echo "************************************" 00:30:06.248 1132 echo "END TEST $test_name" 00:30:06.248 1133 echo "************************************" 00:30:06.248 1134 timing_exit $test_name 00:30:06.248 ... 00:30:06.248 in /var/jenkins/workspace/nvmf-cvl-phy-autotest/spdk/test/nvmf/nvmf.sh:16 -> main(["--transport=rdma"]) 00:30:06.248 ... 00:30:06.248 11 exit 0 00:30:06.248 12 fi 00:30:06.248 13 00:30:06.248 14 run_test "nvmf_target_core" $rootdir/test/nvmf/nvmf_target_core.sh --transport=$SPDK_TEST_NVMF_TRANSPORT 00:30:06.248 15 run_test "nvmf_target_extra" $rootdir/test/nvmf/nvmf_target_extra.sh --transport=$SPDK_TEST_NVMF_TRANSPORT 00:30:06.248 => 16 run_test "nvmf_host" $rootdir/test/nvmf/nvmf_host.sh --transport=$SPDK_TEST_NVMF_TRANSPORT 00:30:06.248 17 00:30:06.248 18 # Interrupt mode for now is supported only on the target, with the TCP transport and posix or ssl socket implementations. 00:30:06.248 19 if [[ "$SPDK_TEST_NVMF_TRANSPORT" = "tcp" && $SPDK_TEST_URING -eq 0 ]]; then 00:30:06.248 20 run_test "nvmf_target_core_interrupt_mode" $rootdir/test/nvmf/nvmf_target_core.sh --transport=$SPDK_TEST_NVMF_TRANSPORT --interrupt-mode 00:30:06.248 21 run_test "nvmf_interrupt" $rootdir/test/nvmf/target/interrupt.sh --transport=$SPDK_TEST_NVMF_TRANSPORT --interrupt-mode 00:30:06.248 ... 00:30:06.248 00:30:06.248 ========== Backtrace end ========== 00:30:06.248 ========== Backtrace start: ========== 00:30:06.248 00:30:06.248 in /var/jenkins/workspace/nvmf-cvl-phy-autotest/spdk/test/common/autotest_common.sh:1129 -> run_test(["nvmf_rdma"],["/var/jenkins/workspace/nvmf-cvl-phy-autotest/spdk/test/nvmf/nvmf.sh"],["--transport=rdma"]) 00:30:06.248 ... 00:30:06.248 1124 timing_enter $test_name 00:30:06.248 1125 echo "************************************" 00:30:06.248 1126 echo "START TEST $test_name" 00:30:06.248 1127 echo "************************************" 00:30:06.248 1128 xtrace_restore 00:30:06.248 1129 time "$@" 00:30:06.248 1130 xtrace_disable 00:30:06.248 1131 echo "************************************" 00:30:06.248 1132 echo "END TEST $test_name" 00:30:06.248 1133 echo "************************************" 00:30:06.248 1134 timing_exit $test_name 00:30:06.248 ... 00:30:06.248 in /var/jenkins/workspace/nvmf-cvl-phy-autotest/spdk/autotest.sh:281 -> main(["/var/jenkins/workspace/nvmf-cvl-phy-autotest/autorun-spdk.conf"]) 00:30:06.248 ... 00:30:06.248 276 if [ $SPDK_TEST_NVMF -eq 1 ]; then 00:30:06.248 277 export NET_TYPE 00:30:06.248 278 # The NVMe-oF run test cases are split out like this so that the parser that compiles the 00:30:06.248 279 # list of all tests can properly differentiate them. Please do not merge them into one line. 00:30:06.248 280 if [ "$SPDK_TEST_NVMF_TRANSPORT" = "rdma" ]; then 00:30:06.248 => 281 run_test "nvmf_rdma" $rootdir/test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT 00:30:06.248 282 run_test "spdkcli_nvmf_rdma" $rootdir/test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT 00:30:06.248 283 elif [ "$SPDK_TEST_NVMF_TRANSPORT" = "tcp" ]; then 00:30:06.248 284 run_test "nvmf_tcp" $rootdir/test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT 00:30:06.248 285 if [[ $SPDK_TEST_URING -eq 0 ]]; then 00:30:06.248 286 run_test "spdkcli_nvmf_tcp" $rootdir/test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT 00:30:06.248 ... 00:30:06.248 00:30:06.249 ========== Backtrace end ========== 00:30:32.535 ========== Backtrace start: ========== 00:30:32.535 00:30:32.535 in spdk/autorun.sh:27 -> main(["/var/jenkins/workspace/nvmf-cvl-phy-autotest/autorun-spdk.conf"]) 00:30:32.535 ... 00:30:32.535 22 trap 'timing_finish || exit 1' EXIT 00:30:32.535 23 00:30:32.535 24 # Runs agent scripts 00:30:32.535 25 $rootdir/autobuild.sh "$conf" 00:30:32.535 26 if ((SPDK_TEST_UNITTEST == 1 || SPDK_RUN_FUNCTIONAL_TEST == 1)); then 00:30:32.535 => 27 sudo -E $rootdir/autotest.sh "$conf" 00:30:32.535 28 fi 00:30:32.535 ... 00:30:32.535 00:30:32.535 ========== Backtrace end ==========