00:28:38.329  ========== Backtrace start: ==========
00:28:38.329  
00:28:38.329  in /var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/test/common/autotest_common.sh:677 -> NOT(["wait"],["4002533"])
00:28:38.329       ...
00:28:38.329     672 		elif [[ -n ${EXIT_STATUS:-} ]] && ((es != EXIT_STATUS)); then
00:28:38.329     673 			es=0
00:28:38.329     674 		fi
00:28:38.329     675 	
00:28:38.329     676 		# invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix)
00:28:38.329  => 677 		((!es == 0))
00:28:38.329     678 	}
00:28:38.329     679 	
00:28:38.329     680 	function timing() {
00:28:38.329     681 		direction="$1"
00:28:38.329     682 		testname="$2"
00:28:38.329       ...
00:28:38.329  in /var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/test/nvmf/target/shutdown.sh:158 -> nvmf_shutdown_tc4([])
00:28:38.329       ...
00:28:38.329     153 	
00:28:38.329     154 		# Kill the target half way through
00:28:38.329     155 		killprocess $nvmfpid
00:28:38.329     156 		sleep 1
00:28:38.329     157 		# Due to IOs are completed with errors, perf exits with bad status
00:28:38.329  => 158 		NOT wait $perfpid
00:28:38.329     159 		stoptarget
00:28:38.329     160 	}
00:28:38.329     161 	
00:28:38.329     162 	run_test "nvmf_shutdown_tc1" nvmf_shutdown_tc1
00:28:38.329     163 	run_test "nvmf_shutdown_tc2" nvmf_shutdown_tc2
00:28:38.329       ...
00:28:38.329  in /var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/test/common/autotest_common.sh:1125 -> run_test(["nvmf_shutdown_tc4"],["nvmf_shutdown_tc4"])
00:28:38.329       ...
00:28:38.329     1120		timing_enter $test_name
00:28:38.329     1121		echo "************************************"
00:28:38.329     1122		echo "START TEST $test_name"
00:28:38.329     1123		echo "************************************"
00:28:38.329     1124		xtrace_restore
00:28:38.329     1125		time "$@"
00:28:38.329     1126		xtrace_disable
00:28:38.330     1127		echo "************************************"
00:28:38.330     1128		echo "END TEST $test_name"
00:28:38.330     1129		echo "************************************"
00:28:38.330     1130		timing_exit $test_name
00:28:38.330       ...
00:28:38.330  in /var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/test/nvmf/target/shutdown.sh:167 -> main(["--transport=tcp"])
00:28:38.330       ...
00:28:38.330     162 	run_test "nvmf_shutdown_tc1" nvmf_shutdown_tc1
00:28:38.330     163 	run_test "nvmf_shutdown_tc2" nvmf_shutdown_tc2
00:28:38.330     164 	run_test "nvmf_shutdown_tc3" nvmf_shutdown_tc3
00:28:38.330     165 	# Temporarily disable on e810 due to issue #3523
00:28:38.330     166 	if ! [[ "$SPDK_TEST_NVMF_NICS" == "e810" && "$TEST_TRANSPORT" == "rdma" ]]; then
00:28:38.330  => 167 		run_test "nvmf_shutdown_tc4" nvmf_shutdown_tc4
00:28:38.330     168 	fi
00:28:38.330     169 	
00:28:38.330     170 	trap - SIGINT SIGTERM EXIT
00:28:38.330       ...
00:28:38.330  
00:28:38.330  ========== Backtrace end ==========
00:28:40.499  ========== Backtrace start: ==========
00:28:40.499  
00:28:40.499  in /var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/test/common/autotest_common.sh:1125 -> run_test(["nvmf_shutdown"],["/var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/test/nvmf/target/shutdown.sh"],["--transport=tcp"])
00:28:40.499       ...
00:28:40.499     1120		timing_enter $test_name
00:28:40.499     1121		echo "************************************"
00:28:40.499     1122		echo "START TEST $test_name"
00:28:40.499     1123		echo "************************************"
00:28:40.499     1124		xtrace_restore
00:28:40.499     1125		time "$@"
00:28:40.499     1126		xtrace_disable
00:28:40.499     1127		echo "************************************"
00:28:40.499     1128		echo "END TEST $test_name"
00:28:40.499     1129		echo "************************************"
00:28:40.499     1130		timing_exit $test_name
00:28:40.499       ...
00:28:40.499  in /var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/test/nvmf/nvmf_target_extra.sh:65 -> main(["--transport=tcp"])
00:28:40.499       ...
00:28:40.499     60  		elif [[ $SPDK_TEST_NVMF_TRANSPORT == "rdma" ]]; then
00:28:40.499     61  			# Disabled due to https://github.com/spdk/spdk/issues/3345
00:28:40.499     62  			# run_test "nvmf_device_removal" test/nvmf/target/device_removal.sh "${TEST_ARGS[@]}"
00:28:40.499     63  			run_test "nvmf_srq_overwhelm" "$rootdir/test/nvmf/target/srq_overwhelm.sh" "${TEST_ARGS[@]}"
00:28:40.499     64  		fi
00:28:40.499  => 65  		run_test "nvmf_shutdown" $rootdir/test/nvmf/target/shutdown.sh "${TEST_ARGS[@]}"
00:28:40.499     66  	fi
00:28:40.499     67  	
00:28:40.499     68  	trap - SIGINT SIGTERM EXIT
00:28:40.499       ...
00:28:40.499  
00:28:40.499  ========== Backtrace end ==========
00:28:40.499  ========== Backtrace start: ==========
00:28:40.499  
00:28:40.499  in /var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/test/common/autotest_common.sh:1125 -> run_test(["nvmf_target_extra"],["/var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/test/nvmf/nvmf_target_extra.sh"],["--transport=tcp"])
00:28:40.499       ...
00:28:40.499     1120		timing_enter $test_name
00:28:40.499     1121		echo "************************************"
00:28:40.499     1122		echo "START TEST $test_name"
00:28:40.499     1123		echo "************************************"
00:28:40.499     1124		xtrace_restore
00:28:40.499     1125		time "$@"
00:28:40.499     1126		xtrace_disable
00:28:40.499     1127		echo "************************************"
00:28:40.499     1128		echo "END TEST $test_name"
00:28:40.499     1129		echo "************************************"
00:28:40.499     1130		timing_exit $test_name
00:28:40.499       ...
00:28:40.499  in /var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/test/nvmf/nvmf.sh:15 -> main(["--transport=tcp"])
00:28:40.499       ...
00:28:40.499     10  	if [ ! $(uname -s) = Linux ]; then
00:28:40.499     11  		exit 0
00:28:40.499     12  	fi
00:28:40.499     13  	
00:28:40.499     14  	run_test "nvmf_target_core" $rootdir/test/nvmf/nvmf_target_core.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
00:28:40.499  => 15  	run_test "nvmf_target_extra" $rootdir/test/nvmf/nvmf_target_extra.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
00:28:40.499     16  	run_test "nvmf_host" $rootdir/test/nvmf/nvmf_host.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
00:28:40.499     17  	
00:28:40.499     18  	# Interrupt mode for now is supported only on the target, with the TCP transport and posix or ssl socket implementations.
00:28:40.499     19  	if [[ "$SPDK_TEST_NVMF_TRANSPORT" = "tcp" && $SPDK_TEST_URING -eq 0 ]]; then
00:28:40.500     20  		run_test "nvmf_target_core_interrupt_mode" $rootdir/test/nvmf/nvmf_target_core.sh --transport=$SPDK_TEST_NVMF_TRANSPORT --interrupt-mode
00:28:40.500       ...
00:28:40.500  
00:28:40.500  ========== Backtrace end ==========
00:28:40.500  ========== Backtrace start: ==========
00:28:40.500  
00:28:40.500  in /var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/test/common/autotest_common.sh:1125 -> run_test(["nvmf_tcp"],["/var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/test/nvmf/nvmf.sh"],["--transport=tcp"])
00:28:40.500       ...
00:28:40.500     1120		timing_enter $test_name
00:28:40.500     1121		echo "************************************"
00:28:40.500     1122		echo "START TEST $test_name"
00:28:40.500     1123		echo "************************************"
00:28:40.500     1124		xtrace_restore
00:28:40.500     1125		time "$@"
00:28:40.500     1126		xtrace_disable
00:28:40.500     1127		echo "************************************"
00:28:40.500     1128		echo "END TEST $test_name"
00:28:40.500     1129		echo "************************************"
00:28:40.500     1130		timing_exit $test_name
00:28:40.500       ...
00:28:40.500  in /var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/autotest.sh:280 -> main(["/var/jenkins/workspace/nvmf-tcp-phy-autotest/autorun-spdk.conf"])
00:28:40.500       ...
00:28:40.500     275 			# list of all tests can properly differentiate them. Please do not merge them into one line.
00:28:40.500     276 			if [ "$SPDK_TEST_NVMF_TRANSPORT" = "rdma" ]; then
00:28:40.500     277 				run_test "nvmf_rdma" $rootdir/test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
00:28:40.500     278 				run_test "spdkcli_nvmf_rdma" $rootdir/test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
00:28:40.500     279 			elif [ "$SPDK_TEST_NVMF_TRANSPORT" = "tcp" ]; then
00:28:40.500  => 280 				run_test "nvmf_tcp" $rootdir/test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
00:28:40.500     281 				if [[ $SPDK_TEST_URING -eq 0 ]]; then
00:28:40.500     282 					run_test "spdkcli_nvmf_tcp" $rootdir/test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
00:28:40.500     283 					run_test "nvmf_identify_passthru" $rootdir/test/nvmf/target/identify_passthru.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
00:28:40.500     284 				fi
00:28:40.500     285 				run_test "nvmf_dif" $rootdir/test/nvmf/target/dif.sh
00:28:40.500       ...
00:28:40.500  
00:28:40.500  ========== Backtrace end ==========