gdb: warning: Couldn't determine a path for the index cache directory. [New LWP 1205226] [New LWP 1205221] [New LWP 1205225] [New LWP 1205224] [New LWP 1205223] [Thread debugging using libthread_db enabled] Using host libthread_db library "/usr/lib64/libthread_db.so.1". Core was generated by `/var/jenkins/workspace/nvmf-cvl-phy-autotest/spdk/build/bin/nvmf_tgt -i 0 -e 0x'. Program terminated with signal SIGABRT, Aborted. #0 0x00007f0cff91f884 in __pthread_kill_implementation () from /usr/lib64/libc.so.6 [Current thread is 1 (Thread 0x7f0cfc8006c0 (LWP 1205226))] Thread 5 (Thread 0x7f0cfe6006c0 (LWP 1205223)): #0 0x00007f0cff9a3d72 in epoll_wait () from /usr/lib64/libc.so.6 No symbol table info available. #1 0x00007f0d00163416 in eal_intr_handle_interrupts (pfd=7, totalfds=1) at ../lib/eal/linux/eal_interrupts.c:1077 events = {{events = 0, data = {ptr = 0x0, fd = 0, u32 = 0, u64 = 0}}} nfds = 0 #2 0x00007f0d00163649 in eal_intr_thread_main (arg=0x0) at ../lib/eal/linux/eal_interrupts.c:1163 pipe_event = {events = 3, data = {ptr = 0x5, fd = 5, u32 = 5, u64 = 5}} src = 0x0 numfds = 1 pfd = 7 __func__ = "eal_intr_thread_main" #3 0x00007f0d0014242b in control_thread_start (arg=0xbf1bb0) at ../lib/eal/common/eal_common_thread.c:282 params = 0xbf1bb0 start_arg = 0x0 start_routine = 0x7f0d0016347b #4 0x00007f0d0015aa20 in thread_start_wrapper (arg=0x7fff742439f0) at ../lib/eal/unix/rte_thread.c:112 ctx = 0x7fff742439f0 thread_func = 0x7f0d001423dc thread_args = 0xbf1bb0 ret = 0 #5 0x00007f0cff91d947 in start_thread () from /usr/lib64/libc.so.6 No symbol table info available. #6 0x00007f0cff9a3970 in clone3 () from /usr/lib64/libc.so.6 No symbol table info available. Thread 4 (Thread 0x7f0cfdc006c0 (LWP 1205224)): #0 0x00007f0cff9a588b in recvmsg () from /usr/lib64/libc.so.6 No symbol table info available. #1 0x00007f0d00151331 in read_msg (fd=10, m=0x7f0cfdbfe9f0, s=0x7f0cfdbfe980) at ../lib/eal/common/eal_common_proc.c:284 msglen = 0 iov = {iov_base = 0x7f0cfdbfe9f0, iov_len = 332} msgh = {msg_name = 0x7f0cfdbfe980, msg_namelen = 110, msg_iov = 0x7f0cfdbfe940, msg_iovlen = 1, msg_control = 0x7f0cfdbfe8d0, msg_controllen = 48, msg_flags = 0} control = '\000' cmsg = 0x0 buflen = 332 #2 0x00007f0d0015183d in mp_handle (arg=0x0) at ../lib/eal/common/eal_common_proc.c:410 ret = 1664576 msg = {type = 0, msg = {name = '\000' , len_param = 0, num_fds = 0, param = '\000' , "\001\000\000\000\320\352\277\375\f\177\000\000\237\027\024\000\r\177\000\000\000\000\000\000\000\000\000\000\210\232\031\000\r\177", '\000' , "\001\000\000\000\030\000\000\000\000\000\000\000\377\377\377\377\030\000\000\000\020\353\277\375\f\177\000\000\"\030\024\000\r\177\000\000\020\353\277\375\f\177\000\000lE\025\000\r\177\000\000\020\353\277\375\f\177\000\000\312\026\024\000\377\377\377\377@\360\030\000"..., fds = {32525, 0, 0, 12524464, 0, 0, 0, 12524464}}} sa = {sun_family = 0, sun_path = '\000' } fd = 10 #3 0x00007f0d0014242b in control_thread_start (arg=0xbf1bb0) at ../lib/eal/common/eal_common_thread.c:282 params = 0xbf1bb0 start_arg = 0x0 start_routine = 0x7f0d0015180e #4 0x00007f0d0015aa20 in thread_start_wrapper (arg=0x7fff74242a00) at ../lib/eal/unix/rte_thread.c:112 ctx = 0x7fff74242a00 thread_func = 0x7f0d001423dc thread_args = 0xbf1bb0 ret = 0 #5 0x00007f0cff91d947 in start_thread () from /usr/lib64/libc.so.6 No symbol table info available. #6 0x00007f0cff9a3970 in clone3 () from /usr/lib64/libc.so.6 No symbol table info available. Thread 3 (Thread 0x7f0cfd2006c0 (LWP 1205225)): #0 0x00007f0d00daa9d4 in nvmf_rdma_poller_poll (rtransport=0xc77180, rpoller=0x7f0cf400eee0) at rdma.c:4692 wc = {{wr_id = 34359738369, status = IBV_WC_LOC_LEN_ERR, opcode = IBV_WC_SEND, vendor_err = 0, byte_len = 0, {imm_data = 1, invalidated_rkey = 1}, qp_num = 0, src_qp = 65536, wc_flags = 1, pkey_index = 1, slid = 0, sl = 30 '\036', dlid_path_bits = 0 '\000'}, {wr_id = 4294967304, status = 30, opcode = IBV_WC_RDMA_WRITE, vendor_err = 131072, byte_len = 31, {imm_data = 1, invalidated_rkey = 1}, qp_num = 32524, src_qp = 3813, wc_flags = 0, pkey_index = 58448, slid = 64799, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 35184795752664, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 8, invalidated_rkey = 8}, qp_num = 0, src_qp = 4246726544, wc_flags = 32524, pkey_index = 43520, slid = 5007, sl = 0 '\000', dlid_path_bits = 32 ' '}, {wr_id = 128849018881, status = IBV_WC_SUCCESS, opcode = 8192, vendor_err = 8, byte_len = 8, {imm_data = 8, invalidated_rkey = 8}, qp_num = 3813, src_qp = 3813, wc_flags = 0, pkey_index = 43652, slid = 5007, sl = 0 '\000', dlid_path_bits = 32 ' '}, {wr_id = 35184795752872, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 67302080, invalidated_rkey = 67302080}, qp_num = 3813, src_qp = 3813, wc_flags = 0, pkey_index = 50114, slid = 63, sl = 13 '\r', dlid_path_bits = 127 '\177'}, {wr_id = 35184795752976, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 1, invalidated_rkey = 1}, qp_num = 3813, src_qp = 3813, wc_flags = 0, pkey_index = 8, slid = 0, sl = 0 '\000', dlid_path_bits = 0 '\000'}, {wr_id = 139693763061840, status = 61865088, opcode = 8192, vendor_err = 25, byte_len = 0, {imm_data = 65536, invalidated_rkey = 65536}, qp_num = 3813, src_qp = 8, wc_flags = 8, pkey_index = 8, slid = 0, sl = 8 '\b', dlid_path_bits = 0 '\000'}, {wr_id = 8, status = IBV_WC_LOC_LEN_ERR, opcode = IBV_WC_RDMA_WRITE, vendor_err = 1, byte_len = 0, {imm_data = 1, invalidated_rkey = 1}, qp_num = 8, src_qp = 187694592, wc_flags = 8192, pkey_index = 58816, slid = 64799, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 0, status = 187694592, opcode = 8192, vendor_err = 4246726080, byte_len = 32524, {imm_data = 0, invalidated_rkey = 0}, qp_num = 0, src_qp = 187694592, wc_flags = 8192, pkey_index = 57816, slid = 64799, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 139693763060192, status = 4246725096, opcode = 32524, vendor_err = 187694592, byte_len = 8192, {imm_data = 4246726080, invalidated_rkey = 4246726080}, qp_num = 32524, src_qp = 187694592, wc_flags = 8192, pkey_index = 58816, slid = 64799, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 35184559783808, status = IBV_WC_LOC_ACCESS_ERR, opcode = IBV_WC_FLUSH, vendor_err = 8, byte_len = 0, {imm_data = 1, invalidated_rkey = 1}, qp_num = 1, src_qp = 1, wc_flags = 0, pkey_index = 1, slid = 0, sl = 8 '\b', dlid_path_bits = 0 '\000'}, {wr_id = 35184795753600, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 13688512, invalidated_rkey = 13688512}, qp_num = 3813, src_qp = 187694720, wc_flags = 8192, pkey_index = 65156, slid = 2863, sl = 0 '\000', dlid_path_bits = 32 ' '}, {wr_id = 35184795753704, status = 30, opcode = IBV_WC_RECV, vendor_err = 31, byte_len = 0, {imm_data = 65536, invalidated_rkey = 65536}, qp_num = 3813, src_qp = 5241984, wc_flags = 8192, pkey_index = 59104, slid = 64799, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 0, status = 5241984, opcode = 8192, vendor_err = 4246726368, byte_len = 32524, {imm_data = 0, invalidated_rkey = 0}, qp_num = 0, src_qp = 5241984, wc_flags = 8192, pkey_index = 59104, slid = 64799, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 0, status = 5241984, opcode = 8192, vendor_err = 4246726368, byte_len = 32524, {imm_data = 0, invalidated_rkey = 0}, qp_num = 0, src_qp = 5241984, wc_flags = 8192, pkey_index = 58104, slid = 64799, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 139693763060480, status = 4246725384, opcode = 32524, vendor_err = 5241984, byte_len = 8192, {imm_data = 27, invalidated_rkey = 27}, qp_num = 32524, src_qp = 27, wc_flags = 8192, pkey_index = 0, slid = 0, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 35184700271104, status = 4246726544, opcode = 32524, vendor_err = 0, byte_len = 0, {imm_data = 328182272, invalidated_rkey = 328182272}, qp_num = 8192, src_qp = 4246726544, wc_flags = 32524, pkey_index = 0, slid = 0, sl = 0 '\000', dlid_path_bits = 0 '\000'}, {wr_id = 139693763060800, status = 4181071, opcode = 32525, vendor_err = 0, byte_len = 0, {imm_data = 14216928, invalidated_rkey = 14216928}, qp_num = 0, src_qp = 5242112, wc_flags = 8192, pkey_index = 64772, slid = 79, sl = 0 '\000', dlid_path_bits = 32 ' '}, {wr_id = 0, status = IBV_WC_SUCCESS, opcode = IBV_WC_RDMA_WRITE, vendor_err = 67302080, byte_len = 8192, {imm_data = 12972032, invalidated_rkey = 12972032}, qp_num = 0, src_qp = 4246725696, wc_flags = 32524, pkey_index = 1, slid = 0, sl = 0 '\000', dlid_path_bits = 0 '\000'}, {wr_id = 139693763060880, status = IBV_WC_SUCCESS, opcode = 32525, vendor_err = 61865088, byte_len = 8192, {imm_data = 4246726736, invalidated_rkey = 4246726736}, qp_num = 32524, src_qp = 0, wc_flags = 0, pkey_index = 64640, slid = 943, sl = 0 '\000', dlid_path_bits = 32 ' '}, {wr_id = 139693763061840, status = IBV_WC_SUCCESS, opcode = IBV_WC_SEND, vendor_err = 1332612, byte_len = 32525, {imm_data = 57920, invalidated_rkey = 57920}, qp_num = 0, src_qp = 4, wc_flags = 0, pkey_index = 16, slid = 0, sl = 0 '\000', dlid_path_bits = 0 '\000'}, {wr_id = 139693763060944, status = 1332697, opcode = 32525, vendor_err = 328182272, byte_len = 8192, {imm_data = 4246726544, invalidated_rkey = 4246726544}, qp_num = 32524, src_qp = 0, wc_flags = 0, pkey_index = 43520, slid = 5007, sl = 0 '\000', dlid_path_bits = 32 ' '}, {wr_id = 139693763061648, status = IBV_WC_SUCCESS, opcode = IBV_WC_SEND, vendor_err = 328182272, byte_len = 8192, {imm_data = 4246725528, invalidated_rkey = 4246725528}, qp_num = 32524, src_qp = 4246725536, wc_flags = 32524, pkey_index = 58280, slid = 64799, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 139693475761056, status = 4246726000, opcode = 32524, vendor_err = 328182272, byte_len = 8192, {imm_data = 12972032, invalidated_rkey = 12972032}, qp_num = 0, src_qp = 4246725968, wc_flags = 32524, pkey_index = 12248, slid = 62, sl = 13 '\r', dlid_path_bits = 127 '\177'}, {wr_id = 139693763061104, status = 67301120, opcode = 8192, vendor_err = 4246726016, byte_len = 32524, {imm_data = 4178882, invalidated_rkey = 4178882}, qp_num = 32525, src_qp = 61865088, wc_flags = 8192, pkey_index = 59472, slid = 64799, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 0, status = 61865088, opcode = 8192, vendor_err = 4246726736, byte_len = 32524, {imm_data = 0, invalidated_rkey = 0}, qp_num = 0, src_qp = 61865088, wc_flags = 8192, pkey_index = 58424, slid = 64799, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 139693763060800, status = 4246725704, opcode = 32524, vendor_err = 4225421, byte_len = 32525, {imm_data = 12972944, invalidated_rkey = 12972944}, qp_num = 0, src_qp = 0, wc_flags = 0, pkey_index = 0, slid = 0, sl = 1 '\001', dlid_path_bits = 0 '\000'}, {wr_id = 35184439389952, status = 12972032, opcode = IBV_WC_SEND, vendor_err = 4246726112, byte_len = 32524, {imm_data = 12972032, invalidated_rkey = 12972032}, qp_num = 0, src_qp = 4246726192, wc_flags = 32524, pkey_index = 35911, slid = 64, sl = 13 '\r', dlid_path_bits = 127 '\177'}, {wr_id = 32, status = 14217008, opcode = IBV_WC_SEND, vendor_err = 0, byte_len = 0, {imm_data = 12972944, invalidated_rkey = 12972944}, qp_num = 0, src_qp = 12971120, wc_flags = 0, pkey_index = 2976, slid = 60416, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 4294967296, status = 4094262352, opcode = 32524, vendor_err = 4246726240, byte_len = 32524, {imm_data = 13238002, invalidated_rkey = 13238002}, qp_num = 32525, src_qp = 4246726320, wc_flags = 0, pkey_index = 61232, slid = 216, sl = 0 '\000', dlid_path_bits = 0 '\000'}, {wr_id = 14217008, status = 12481104, opcode = IBV_WC_SEND, vendor_err = 4246726336, byte_len = 32524, {imm_data = 13476231, invalidated_rkey = 13476231}, qp_num = 32525, src_qp = 14217008, wc_flags = 0, pkey_index = 65194, slid = 201, sl = 13 '\r', dlid_path_bits = 127 '\177'}, {wr_id = 13678816, status = 4093643776, opcode = 32524, vendor_err = 0, byte_len = 32, {imm_data = 0, invalidated_rkey = 0}, qp_num = 0, src_qp = 4093644440, wc_flags = 32524, pkey_index = 60528, slid = 197, sl = 0 '\000', dlid_path_bits = 0 '\000'}} rdma_wr = 0x0 rdma_req = 0x20000b2ffe00 rdma_recv = 0x800000001 rqpair = 0x7f0cfd1fe6e0 tmp_rqpair = 0x800000001 reaped = 0 i = 0 count = 0 rc = 0 error = false poll_tsc = 9624055863538936 __func__ = "nvmf_rdma_poller_poll" __PRETTY_FUNCTION__ = "nvmf_rdma_poller_poll" #1 0x00007f0d00dade98 in nvmf_rdma_poll_group_poll (group=0x7f0cf400ee70) at rdma.c:4898 rtransport = 0xc77180 rgroup = 0x7f0cf400ee70 rpoller = 0x7f0cf400eee0 tmp = 0x7f0cf4051260 count = 0 rc = 0 #2 0x00007f0d00d0da63 in nvmf_transport_poll_group_poll (group=0x7f0cf400ee70) at transport.c:724 No locals. #3 0x00007f0d00cc7e65 in nvmf_poll_group_poll (ctx=0x7f0cf4000c00) at nvmf.c:160 group = 0x7f0cf4000c00 rc = 0 count = 0 tgroup = 0x7f0cf400ee70 #4 0x00007f0d003f8f08 in thread_execute_poller (thread=0xc5f000, poller=0x7f0cf4000cc0) at thread.c:953 rc = 0 __PRETTY_FUNCTION__ = "thread_execute_poller" __func__ = "thread_execute_poller" #5 0x00007f0d003fac8e in thread_poll (thread=0xc5f000, max_msgs=0, now=9624055863534128) at thread.c:1079 poller_rc = 0 msg_count = 0 poller = 0x7f0cf4000cc0 tmp = 0x0 critical_msg = 0x0 rc = 0 #6 0x00007f0d003fb895 in spdk_thread_poll (thread=0xc5f000, max_msgs=0, now=9624055863534128) at thread.c:1163 orig_thread = 0x0 rc = 0 #7 0x00007f0d007b0cd2 in _reactor_run (reactor=0xc5c680) at reactor.c:914 thread = 0xc5f000 lw_thread = 0xc5f348 tmp = 0x0 now = 9624055863534128 rc = 0 #8 0x00007f0d007b1322 in reactor_run (arg=0xc5c680) at reactor.c:952 reactor = 0xc5c680 thread = 0x7f0cff9921c1 lw_thread = 0x0 tmp = 0x0 thread_name = "reactor_2\000\000\000\000\000\000\000\203\351\037\375\f\177\000\000\001\000\000\000\000\000\000" last_sched = 0 __func__ = "reactor_run" #9 0x00007f0d00142079 in eal_thread_loop (arg=0x2) at ../lib/eal/common/eal_common_thread.c:212 f = 0x7f0d007b10b1 fct_arg = 0xc5c680 lcore_id = 2 cpuset = "2", '\000' ret = 0 #10 0x00007f0d0015ca96 in eal_worker_thread_loop (arg=0x2) at ../lib/eal/linux/eal.c:916 No locals. #11 0x00007f0cff91d947 in start_thread () from /usr/lib64/libc.so.6 No symbol table info available. #12 0x00007f0cff9a3970 in clone3 () from /usr/lib64/libc.so.6 No symbol table info available. Thread 2 (Thread 0x7f0cfe772a00 (LWP 1205221)): #0 nvmf_transport_poll_group_poll (group=0xc85880) at transport.c:724 No locals. #1 0x00007f0d00cc7e65 in nvmf_poll_group_poll (ctx=0xc5fb10) at nvmf.c:160 group = 0xc5fb10 rc = 0 count = 0 tgroup = 0xc85880 #2 0x00007f0d003f8f08 in thread_execute_poller (thread=0xc5f720, poller=0xc5fbd0) at thread.c:953 rc = 0 __PRETTY_FUNCTION__ = "thread_execute_poller" __func__ = "thread_execute_poller" #3 0x00007f0d003fac8e in thread_poll (thread=0xc5f720, max_msgs=0, now=9624055863527656) at thread.c:1079 poller_rc = 0 msg_count = 0 poller = 0xc5fbd0 tmp = 0x0 critical_msg = 0x0 rc = 0 #4 0x00007f0d003fb895 in spdk_thread_poll (thread=0xc5f720, max_msgs=0, now=9624055863527656) at thread.c:1163 orig_thread = 0x0 rc = 0 #5 0x00007f0d007b0cd2 in _reactor_run (reactor=0xc5c3c0) at reactor.c:914 thread = 0xc5f720 lw_thread = 0xc5fa68 tmp = 0x0 now = 9624055863527656 rc = 0 #6 0x00007f0d007b1322 in reactor_run (arg=0xc5c3c0) at reactor.c:952 reactor = 0xc5c3c0 thread = 0x7f0d0013292d lw_thread = 0x7f003fc3c2 tmp = 0x7fff74243d90 thread_name = "reactor_1\000\000\000\200\000\000\000\300=$t\377\177\000\000%X\374\000\001\000\000" last_sched = 0 __func__ = "reactor_run" #7 0x00007f0d007b1e2f in spdk_reactors_start () at reactor.c:1068 reactor = 0xc5c3c0 i = 4294967295 current_core = 1 rc = 0 __func__ = "spdk_reactors_start" __PRETTY_FUNCTION__ = "spdk_reactors_start" #8 0x00007f0d007a6238 in spdk_app_start (opts_user=0x7fff74244110, start_fn=0x4023df , arg1=0x0) at app.c:839 rc = 0 tty = 0x0 tmp_cpumask = {str = '\000' , cpus = "\002", '\000' } g_env_was_setup = false opts_local = {name = 0x42d01d "nvmf", json_config_file = 0x0, json_config_ignore_errors = false, reserved17 = "\000\000\000\000\000\000", rpc_addr = 0x7f0d007bd224 "/var/tmp/spdk.sock", reactor_mask = 0x7fff74245032 "0xE", tpoint_group_mask = 0x7fff74245028 "0xFFFF", shm_id = 0, reserved52 = "\000\000\000", shutdown_cb = 0x0, enable_coredump = true, reserved65 = "\000\000", mem_channel = -1, main_core = -1, mem_size = -1, no_pci = false, hugepage_single_segments = false, unlink_hugepage = false, no_huge = false, reserved84 = "\000\000\000", hugedir = 0x0, print_level = SPDK_LOG_INFO, reserved100 = "\000\000\000", num_pci_addr = 0, pci_blocked = 0x0, pci_allowed = 0x0, iova_mode = 0x0, delay_subsystem_init = false, reserved137 = "\000\000\000\000\000\000", num_entries = 32768, env_context = 0x0, log = 0x0, base_virtaddr = 35184372088832, opts_size = 236, disable_signal_handlers = false, interrupt_mode = false, reserved186 = "\000\000\000\000\000", msg_mempool_size = 262143, rpc_allowlist = 0x0, vf_token = 0x0, lcore_map = 0x0, rpc_log_level = SPDK_LOG_DISABLED, rpc_log_file = 0x0} opts = 0x7fff74243e50 i = 128 __func__ = "spdk_app_start" #9 0x0000000000402563 in main (argc=7, argv=0x7fff74244338) at nvmf_main.c:47 rc = 1 opts = {name = 0x42d01d "nvmf", json_config_file = 0x0, json_config_ignore_errors = false, reserved17 = "\000\000\000\000\000\000", rpc_addr = 0x7f0d007bd224 "/var/tmp/spdk.sock", reactor_mask = 0x7fff74245032 "0xE", tpoint_group_mask = 0x7fff74245028 "0xFFFF", shm_id = 0, reserved52 = "\000\000\000", shutdown_cb = 0x0, enable_coredump = true, reserved65 = "\000\000", mem_channel = -1, main_core = -1, mem_size = -1, no_pci = false, hugepage_single_segments = false, unlink_hugepage = false, no_huge = false, reserved84 = "\000\000\000", hugedir = 0x0, print_level = SPDK_LOG_INFO, reserved100 = "\000\000\000", num_pci_addr = 0, pci_blocked = 0x0, pci_allowed = 0x0, iova_mode = 0x0, delay_subsystem_init = false, reserved137 = "\000\000\000\000\000\000", num_entries = 32768, env_context = 0x0, log = 0x0, base_virtaddr = 35184372088832, opts_size = 236, disable_signal_handlers = false, interrupt_mode = false, reserved186 = "\000\000\000\000\000", msg_mempool_size = 0, rpc_allowlist = 0x0, vf_token = 0x0, lcore_map = 0x0, rpc_log_level = SPDK_LOG_DISABLED, rpc_log_file = 0x0} Thread 1 (Thread 0x7f0cfc8006c0 (LWP 1205226)): #0 0x00007f0cff91f884 in __pthread_kill_implementation () from /usr/lib64/libc.so.6 No symbol table info available. #1 0x00007f0cff8ceafe in raise () from /usr/lib64/libc.so.6 No symbol table info available. #2 0x00007f0cff8b787f in abort () from /usr/lib64/libc.so.6 No symbol table info available. #3 0x00007f0cff8b779b in __assert_fail_base.cold () from /usr/lib64/libc.so.6 No symbol table info available. #4 0x00007f0cff8c7187 in __assert_fail () from /usr/lib64/libc.so.6 No symbol table info available. #5 0x00007f0d00dac9c1 in nvmf_rdma_poller_poll (rtransport=0xc77180, rpoller=0x7f0cec00eee0) at rdma.c:4768 wc = {{wr_id = 35184795734472, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 4236238920, invalidated_rkey = 4236238920}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 58400, slid = 64639, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 35184795734576, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 4236239904, invalidated_rkey = 4236239904}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 58448, slid = 64639, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 35184795734680, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 8, invalidated_rkey = 8}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 65024, slid = 2863, sl = 0 '\000', dlid_path_bits = 32 ' '}, {wr_id = 35184795734784, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 8, invalidated_rkey = 8}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 65156, slid = 2863, sl = 0 '\000', dlid_path_bits = 32 ' '}, {wr_id = 35184795734888, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 0, invalidated_rkey = 0}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 6, slid = 0, sl = 0 '\000', dlid_path_bits = 0 '\000'}, {wr_id = 35184795734992, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 125613824, invalidated_rkey = 125613824}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 8, slid = 0, sl = 0 '\000', dlid_path_bits = 0 '\000'}, {wr_id = 35184795735096, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 65536, invalidated_rkey = 65536}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 8, slid = 0, sl = 8 '\b', dlid_path_bits = 0 '\000'}, {wr_id = 35184795735200, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 1, invalidated_rkey = 1}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 58816, slid = 64639, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 35184795735304, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 0, invalidated_rkey = 0}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 57816, slid = 64639, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 35184795735408, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 4236240320, invalidated_rkey = 4236240320}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 58816, slid = 64639, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 35184795735512, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 1, invalidated_rkey = 1}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 1, slid = 0, sl = 8 '\b', dlid_path_bits = 0 '\000'}, {wr_id = 35184795735616, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 0, invalidated_rkey = 0}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 65156, slid = 1807, sl = 0 '\000', dlid_path_bits = 32 ' '}, {wr_id = 35184795735720, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 3960033200, invalidated_rkey = 3960033200}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 0, slid = 0, sl = 0 '\000', dlid_path_bits = 0 '\000'}, {wr_id = 35184795735824, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 12379200, invalidated_rkey = 12379200}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 32916, slid = 218, sl = 13 '\r', dlid_path_bits = 127 '\177'}, {wr_id = 35184795735928, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 4236239824, invalidated_rkey = 4236239824}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 62352, slid = 197, sl = 0 '\000', dlid_path_bits = 0 '\000'}, {wr_id = 35184795736032, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 25, invalidated_rkey = 25}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 0, slid = 0, sl = 13 '\r', dlid_path_bits = 127 '\177'}, {wr_id = 35184795736136, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 187694592, invalidated_rkey = 187694592}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 0, slid = 0, sl = 0 '\000', dlid_path_bits = 0 '\000'}, {wr_id = 35184795736240, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 14216928, invalidated_rkey = 14216928}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 63264, slid = 197, sl = 0 '\000', dlid_path_bits = 0 '\000'}, {wr_id = 35184795736344, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 12972944, invalidated_rkey = 12972944}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 1, slid = 0, sl = 0 '\000', dlid_path_bits = 0 '\000'}, {wr_id = 35184795736448, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 4236240976, invalidated_rkey = 4236240976}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 61952, slid = 143, sl = 0 '\000', dlid_path_bits = 32 ' '}, {wr_id = 35184795736552, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 13678107, invalidated_rkey = 13678107}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 48058, slid = 208, sl = 13 '\r', dlid_path_bits = 127 '\177'}, {wr_id = 35184795736656, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 4236240784, invalidated_rkey = 4236240784}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 65024, slid = 2863, sl = 0 '\000', dlid_path_bits = 32 ' '}, {wr_id = 35184795736760, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 4236239768, invalidated_rkey = 4236239768}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 58280, slid = 64639, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 35184795736864, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 12972944, invalidated_rkey = 12972944}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 12248, slid = 62, sl = 13 '\r', dlid_path_bits = 127 '\177'}, {wr_id = 35184795736968, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 4178882, invalidated_rkey = 4178882}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 59472, slid = 64639, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 35184795737072, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 0, invalidated_rkey = 0}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 58424, slid = 64639, sl = 12 '\f', dlid_path_bits = 127 '\177'}, {wr_id = 35184795737176, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 12973856, invalidated_rkey = 12973856}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 0, slid = 0, sl = 1 '\001', dlid_path_bits = 0 '\000'}, {wr_id = 35184490338538, status = IBV_WC_SUCCESS, opcode = IBV_WC_SEND, vendor_err = 4236240352, byte_len = 16, {imm_data = 12972944, invalidated_rkey = 12972944}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 35911, slid = 64, sl = 13 '\r', dlid_path_bits = 127 '\177'}, {wr_id = 35184490439769, status = IBV_WC_SUCCESS, opcode = IBV_WC_RDMA_WRITE, vendor_err = 65537, byte_len = 4096, {imm_data = 12973856, invalidated_rkey = 12973856}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 64176, slid = 197, sl = 0 '\000', dlid_path_bits = 0 '\000'}, {wr_id = 35184490439770, status = IBV_WC_SUCCESS, opcode = IBV_WC_SEND, vendor_err = 65537, byte_len = 16, {imm_data = 13238002, invalidated_rkey = 13238002}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 61232, slid = 216, sl = 0 '\000', dlid_path_bits = 0 '\000'}, {wr_id = 35184795734264, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 13476231, invalidated_rkey = 13476231}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 65194, slid = 201, sl = 13 '\r', dlid_path_bits = 127 '\177'}, {wr_id = 35184795734368, status = IBV_WC_WR_FLUSH_ERR, opcode = IBV_WC_RECV, vendor_err = 65537, byte_len = 0, {imm_data = 0, invalidated_rkey = 0}, qp_num = 3819, src_qp = 3819, wc_flags = 0, pkey_index = 60528, slid = 197, sl = 0 '\000', dlid_path_bits = 0 '\000'}} rdma_wr = 0x2000070de459 rdma_req = 0x2000070de150 rdma_recv = 0x200019405c08 rqpair = 0xc6ede0 tmp_rqpair = 0x0 reaped = 30 i = 28 count = 1 rc = 32524 error = true poll_tsc = 9624055863206010 __func__ = "nvmf_rdma_poller_poll" __PRETTY_FUNCTION__ = "nvmf_rdma_poller_poll" #6 0x00007f0d00dade98 in nvmf_rdma_poll_group_poll (group=0x7f0cec00ee70) at rdma.c:4898 rtransport = 0xc77180 rgroup = 0x7f0cec00ee70 rpoller = 0x7f0cec00eee0 tmp = 0x7f0cec051260 count = 0 rc = -1 #7 0x00007f0d00d0da63 in nvmf_transport_poll_group_poll (group=0x7f0cec00ee70) at transport.c:724 No locals. #8 0x00007f0d00cc7e65 in nvmf_poll_group_poll (ctx=0x7f0cec000c00) at nvmf.c:160 group = 0x7f0cec000c00 rc = -1 count = 0 tgroup = 0x7f0cec00ee70 #9 0x00007f0d003f8f08 in thread_execute_poller (thread=0xc5f390, poller=0x7f0cec000cc0) at thread.c:953 rc = 0 __PRETTY_FUNCTION__ = "thread_execute_poller" __func__ = "thread_execute_poller" #10 0x00007f0d003fac8e in thread_poll (thread=0xc5f390, max_msgs=0, now=9624055863197528) at thread.c:1079 poller_rc = 0 msg_count = 0 poller = 0x7f0cec000cc0 tmp = 0x0 critical_msg = 0x0 rc = 0 #11 0x00007f0d003fb895 in spdk_thread_poll (thread=0xc5f390, max_msgs=0, now=9624055863197528) at thread.c:1163 orig_thread = 0x0 rc = 0 #12 0x00007f0d007b0cd2 in _reactor_run (reactor=0xc5c940) at reactor.c:914 thread = 0xc5f390 lw_thread = 0xc5f6d8 tmp = 0x0 now = 9624055863197528 rc = 1 #13 0x00007f0d007b1322 in reactor_run (arg=0xc5c940) at reactor.c:952 reactor = 0xc5c940 thread = 0x7f0cff9921c1 lw_thread = 0x0 tmp = 0x0 thread_name = "reactor_3\000\000\000\000\000\000\000\203\351\177\374\f\177\000\000\001\000\000\000\000\000\000" last_sched = 0 __func__ = "reactor_run" #14 0x00007f0d00142079 in eal_thread_loop (arg=0x3) at ../lib/eal/common/eal_common_thread.c:212 f = 0x7f0d007b10b1 fct_arg = 0xc5c940 lcore_id = 3 cpuset = "3", '\000' ret = 0 #15 0x00007f0d0015ca96 in eal_worker_thread_loop (arg=0x3) at ../lib/eal/linux/eal.c:916 No locals. #16 0x00007f0cff91d947 in start_thread () from /usr/lib64/libc.so.6 No symbol table info available. #17 0x00007f0cff9a3970 in clone3 () from /usr/lib64/libc.so.6 No symbol table info available.