LCOV - code coverage report
Current view: top level - spdk/lib/nvme - nvme_ctrlr.c (source / functions) Hit Total Coverage
Test: Combined Lines: 2205 3071 71.8 %
Date: 2024-12-17 08:30:39 Functions: 189 209 90.4 %
Legend: Lines: hit not hit | Branches: + taken - not taken # not executed Branches: 2972 13877 21.4 %

           Branch data     Line data    Source code
       1                 :            : /*   SPDX-License-Identifier: BSD-3-Clause
       2                 :            :  *   Copyright (C) 2015 Intel Corporation. All rights reserved.
       3                 :            :  *   Copyright (c) 2019-2021 Mellanox Technologies LTD. All rights reserved.
       4                 :            :  *   Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
       5                 :            :  */
       6                 :            : 
       7                 :            : #include "spdk/stdinc.h"
       8                 :            : 
       9                 :            : #include "nvme_internal.h"
      10                 :            : #include "nvme_io_msg.h"
      11                 :            : 
      12                 :            : #include "spdk/env.h"
      13                 :            : #include "spdk/string.h"
      14                 :            : #include "spdk/endian.h"
      15                 :            : 
      16                 :            : struct nvme_active_ns_ctx;
      17                 :            : 
      18                 :            : static int nvme_ctrlr_construct_and_submit_aer(struct spdk_nvme_ctrlr *ctrlr,
      19                 :            :                 struct nvme_async_event_request *aer);
      20                 :            : static void nvme_ctrlr_identify_active_ns_async(struct nvme_active_ns_ctx *ctx);
      21                 :            : static int nvme_ctrlr_identify_ns_async(struct spdk_nvme_ns *ns);
      22                 :            : static int nvme_ctrlr_identify_ns_iocs_specific_async(struct spdk_nvme_ns *ns);
      23                 :            : static int nvme_ctrlr_identify_id_desc_async(struct spdk_nvme_ns *ns);
      24                 :            : static void nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr);
      25                 :            : static void nvme_ctrlr_set_state(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state,
      26                 :            :                                  uint64_t timeout_in_ms);
      27                 :            : 
      28                 :            : static int
      29                 :    1460598 : nvme_ns_cmp(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2)
      30                 :            : {
      31   [ +  +  +  -  :    1460598 :         if (ns1->id < ns2->id) {
          +  -  +  -  -  
                      + ]
      32                 :     495242 :                 return -1;
      33   [ +  +  +  -  :     965356 :         } else if (ns1->id > ns2->id) {
          +  -  +  -  -  
                      + ]
      34                 :     829849 :                 return 1;
      35                 :            :         } else {
      36                 :     135507 :                 return 0;
      37                 :            :         }
      38                 :       4171 : }
      39                 :            : 
      40   [ +  +  +  +  :    1901895 : RB_GENERATE_STATIC(nvme_ns_tree, spdk_nvme_ns, node, nvme_ns_cmp);
          +  +  +  +  +  
          +  +  +  +  +  
          +  +  +  +  -  
          -  +  -  +  -  
          +  +  +  +  +  
          +  +  +  +  +  
          +  +  -  +  -  
          +  +  +  +  +  
          -  +  +  +  +  
          -  +  -  +  -  
          -  +  -  -  -  
          -  +  +  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  +  -  +  
          -  +  -  +  -  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  -  +  #  
          #  #  #  #  #  
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  -  
          +  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  +  -  +  
          -  +  -  -  +  
          +  -  +  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  -  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  -  +  -  +  
          +  -  +  -  +  
          +  +  -  #  #  
          #  #  #  #  -  
          +  #  #  #  #  
          #  #  +  -  +  
          -  -  +  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  +  
          -  +  -  +  -  
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  -  +  +  
          -  +  -  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  -  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  +  -  +  
          -  +  -  +  -  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  # ]
      41                 :            : 
      42                 :            : #define CTRLR_STRING(ctrlr) \
      43                 :            :         ((ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_TCP || ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_RDMA) ? \
      44                 :            :         ctrlr->trid.subnqn : ctrlr->trid.traddr)
      45                 :            : 
      46                 :            : #define NVME_CTRLR_ERRLOG(ctrlr, format, ...) \
      47                 :            :         SPDK_ERRLOG("[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__);
      48                 :            : 
      49                 :            : #define NVME_CTRLR_WARNLOG(ctrlr, format, ...) \
      50                 :            :         SPDK_WARNLOG("[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__);
      51                 :            : 
      52                 :            : #define NVME_CTRLR_NOTICELOG(ctrlr, format, ...) \
      53                 :            :         SPDK_NOTICELOG("[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__);
      54                 :            : 
      55                 :            : #define NVME_CTRLR_INFOLOG(ctrlr, format, ...) \
      56                 :            :         SPDK_INFOLOG(nvme, "[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__);
      57                 :            : 
      58                 :            : #ifdef DEBUG
      59                 :            : #define NVME_CTRLR_DEBUGLOG(ctrlr, format, ...) \
      60                 :            :         SPDK_DEBUGLOG(nvme, "[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__);
      61                 :            : #else
      62                 :            : #define NVME_CTRLR_DEBUGLOG(ctrlr, ...) do { } while (0)
      63                 :            : #endif
      64                 :            : 
      65                 :            : #define nvme_ctrlr_get_reg_async(ctrlr, reg, sz, cb_fn, cb_arg) \
      66                 :            :         nvme_transport_ctrlr_get_reg_ ## sz ## _async(ctrlr, \
      67                 :            :                 offsetof(struct spdk_nvme_registers, reg), cb_fn, cb_arg)
      68                 :            : 
      69                 :            : #define nvme_ctrlr_set_reg_async(ctrlr, reg, sz, val, cb_fn, cb_arg) \
      70                 :            :         nvme_transport_ctrlr_set_reg_ ## sz ## _async(ctrlr, \
      71                 :            :                 offsetof(struct spdk_nvme_registers, reg), val, cb_fn, cb_arg)
      72                 :            : 
      73                 :            : #define nvme_ctrlr_get_cc_async(ctrlr, cb_fn, cb_arg) \
      74                 :            :         nvme_ctrlr_get_reg_async(ctrlr, cc, 4, cb_fn, cb_arg)
      75                 :            : 
      76                 :            : #define nvme_ctrlr_get_csts_async(ctrlr, cb_fn, cb_arg) \
      77                 :            :         nvme_ctrlr_get_reg_async(ctrlr, csts, 4, cb_fn, cb_arg)
      78                 :            : 
      79                 :            : #define nvme_ctrlr_get_cap_async(ctrlr, cb_fn, cb_arg) \
      80                 :            :         nvme_ctrlr_get_reg_async(ctrlr, cap, 8, cb_fn, cb_arg)
      81                 :            : 
      82                 :            : #define nvme_ctrlr_get_vs_async(ctrlr, cb_fn, cb_arg) \
      83                 :            :         nvme_ctrlr_get_reg_async(ctrlr, vs, 4, cb_fn, cb_arg)
      84                 :            : 
      85                 :            : #define nvme_ctrlr_set_cc_async(ctrlr, value, cb_fn, cb_arg) \
      86                 :            :         nvme_ctrlr_set_reg_async(ctrlr, cc, 4, value, cb_fn, cb_arg)
      87                 :            : 
      88                 :            : static int
      89                 :          0 : nvme_ctrlr_get_cc(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cc_register *cc)
      90                 :            : {
      91                 :          0 :         return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, cc.raw),
      92         [ #  # ]:          0 :                                               &cc->raw);
      93                 :            : }
      94                 :            : 
      95                 :            : static int
      96                 :      34917 : nvme_ctrlr_get_csts(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_csts_register *csts)
      97                 :            : {
      98                 :      34917 :         return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, csts.raw),
      99         [ +  - ]:         32 :                                               &csts->raw);
     100                 :            : }
     101                 :            : 
     102                 :            : int
     103                 :        748 : nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cap_register *cap)
     104                 :            : {
     105                 :        748 :         return nvme_transport_ctrlr_get_reg_8(ctrlr, offsetof(struct spdk_nvme_registers, cap.raw),
     106         [ +  - ]:          9 :                                               &cap->raw);
     107                 :            : }
     108                 :            : 
     109                 :            : int
     110                 :          3 : nvme_ctrlr_get_vs(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_vs_register *vs)
     111                 :            : {
     112                 :          3 :         return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, vs.raw),
     113         [ #  # ]:          0 :                                               &vs->raw);
     114                 :            : }
     115                 :            : 
     116                 :            : int
     117                 :          8 : nvme_ctrlr_get_cmbsz(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cmbsz_register *cmbsz)
     118                 :            : {
     119                 :          8 :         return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, cmbsz.raw),
     120         [ #  # ]:          0 :                                               &cmbsz->raw);
     121                 :            : }
     122                 :            : 
     123                 :            : int
     124                 :          8 : nvme_ctrlr_get_pmrcap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_pmrcap_register *pmrcap)
     125                 :            : {
     126                 :          8 :         return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, pmrcap.raw),
     127         [ #  # ]:          0 :                                               &pmrcap->raw);
     128                 :            : }
     129                 :            : 
     130                 :            : int
     131                 :          0 : nvme_ctrlr_get_bpinfo(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bpinfo_register *bpinfo)
     132                 :            : {
     133                 :          0 :         return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, bpinfo.raw),
     134         [ #  # ]:          0 :                                               &bpinfo->raw);
     135                 :            : }
     136                 :            : 
     137                 :            : int
     138                 :          0 : nvme_ctrlr_set_bprsel(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bprsel_register *bprsel)
     139                 :            : {
     140                 :          0 :         return nvme_transport_ctrlr_set_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, bprsel.raw),
     141   [ #  #  #  # ]:          0 :                                               bprsel->raw);
     142                 :            : }
     143                 :            : 
     144                 :            : int
     145                 :          0 : nvme_ctrlr_set_bpmbl(struct spdk_nvme_ctrlr *ctrlr, uint64_t bpmbl_value)
     146                 :            : {
     147                 :          0 :         return nvme_transport_ctrlr_set_reg_8(ctrlr, offsetof(struct spdk_nvme_registers, bpmbl),
     148                 :          0 :                                               bpmbl_value);
     149                 :            : }
     150                 :            : 
     151                 :            : static int
     152                 :          0 : nvme_ctrlr_set_nssr(struct spdk_nvme_ctrlr *ctrlr, uint32_t nssr_value)
     153                 :            : {
     154                 :          0 :         return nvme_transport_ctrlr_set_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, nssr),
     155                 :          0 :                                               nssr_value);
     156                 :            : }
     157                 :            : 
     158                 :            : bool
     159                 :       6708 : nvme_ctrlr_multi_iocs_enabled(struct spdk_nvme_ctrlr *ctrlr)
     160                 :            : {
     161   [ +  +  +  -  :       8688 :         return ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_IOCS &&
          +  -  +  -  +  
                      - ]
     162   [ +  +  #  #  :       1980 :                ctrlr->opts.command_set == SPDK_NVME_CC_CSS_IOCS;
                   #  # ]
     163                 :            : }
     164                 :            : 
     165                 :            : /* When the field in spdk_nvme_ctrlr_opts are changed and you change this function, please
     166                 :            :  * also update the nvme_ctrl_opts_init function in nvme_ctrlr.c
     167                 :            :  */
     168                 :            : void
     169                 :       5136 : spdk_nvme_ctrlr_get_default_ctrlr_opts(struct spdk_nvme_ctrlr_opts *opts, size_t opts_size)
     170                 :            : {
     171   [ +  +  #  # ]:       5136 :         assert(opts);
     172                 :            : 
     173   [ +  -  +  - ]:       5136 :         opts->opts_size = opts_size;
     174                 :            : 
     175                 :            : #define FIELD_OK(field) \
     176                 :            :         offsetof(struct spdk_nvme_ctrlr_opts, field) + sizeof(opts->field) <= opts_size
     177                 :            : 
     178                 :            : #define SET_FIELD(field, value) \
     179                 :            :         if (offsetof(struct spdk_nvme_ctrlr_opts, field) + sizeof(opts->field) <= opts_size) { \
     180                 :            :                 opts->field = value; \
     181                 :            :         } \
     182                 :            : 
     183   [ +  +  +  -  :       5136 :         SET_FIELD(num_io_queues, DEFAULT_MAX_IO_QUEUES);
                   +  - ]
     184   [ +  +  +  -  :       5136 :         SET_FIELD(use_cmb_sqs, false);
                   +  - ]
     185   [ +  +  +  -  :       5136 :         SET_FIELD(no_shn_notification, false);
                   +  - ]
     186   [ +  +  +  -  :       5136 :         SET_FIELD(arb_mechanism, SPDK_NVME_CC_AMS_RR);
                   +  - ]
     187   [ +  +  +  -  :       5136 :         SET_FIELD(arbitration_burst, 0);
                   +  - ]
     188   [ +  +  +  -  :       5136 :         SET_FIELD(low_priority_weight, 0);
                   +  - ]
     189   [ +  +  +  -  :       5136 :         SET_FIELD(medium_priority_weight, 0);
                   +  - ]
     190   [ +  +  +  -  :       5136 :         SET_FIELD(high_priority_weight, 0);
                   +  - ]
     191   [ +  +  +  -  :       5136 :         SET_FIELD(keep_alive_timeout_ms, MIN_KEEP_ALIVE_TIMEOUT_IN_MS);
                   +  - ]
     192   [ +  +  +  -  :       5136 :         SET_FIELD(transport_retry_count, SPDK_NVME_DEFAULT_RETRY_COUNT);
                   +  - ]
     193   [ +  +  +  -  :       5136 :         SET_FIELD(io_queue_size, DEFAULT_IO_QUEUE_SIZE);
                   +  - ]
     194                 :            : 
     195         [ +  + ]:       5136 :         if (nvme_driver_init() == 0) {
     196         [ +  + ]:       5136 :                 if (FIELD_OK(hostnqn)) {
     197         [ +  - ]:       5133 :                         nvme_get_default_hostnqn(opts->hostnqn, sizeof(opts->hostnqn));
     198                 :        834 :                 }
     199                 :            : 
     200         [ +  + ]:       5136 :                 if (FIELD_OK(extended_host_id)) {
     201   [ +  -  +  -  :       5133 :                         memcpy(opts->extended_host_id, &g_spdk_nvme_driver->default_extended_host_id,
             +  -  +  - ]
     202                 :            :                                sizeof(opts->extended_host_id));
     203                 :        834 :                 }
     204                 :            : 
     205                 :        834 :         }
     206                 :            : 
     207   [ +  +  +  -  :       5136 :         SET_FIELD(io_queue_requests, DEFAULT_IO_QUEUE_REQUESTS);
                   +  - ]
     208                 :            : 
     209         [ +  + ]:       5136 :         if (FIELD_OK(src_addr)) {
     210   [ +  +  +  - ]:       5133 :                 memset(opts->src_addr, 0, sizeof(opts->src_addr));
     211                 :        834 :         }
     212                 :            : 
     213         [ +  + ]:       5136 :         if (FIELD_OK(src_svcid)) {
     214   [ +  +  +  - ]:       5133 :                 memset(opts->src_svcid, 0, sizeof(opts->src_svcid));
     215                 :        834 :         }
     216                 :            : 
     217         [ +  + ]:       5136 :         if (FIELD_OK(host_id)) {
     218   [ +  +  +  - ]:       5133 :                 memset(opts->host_id, 0, sizeof(opts->host_id));
     219                 :        834 :         }
     220                 :            : 
     221   [ +  +  +  -  :       5136 :         SET_FIELD(command_set, CHAR_BIT);
                   +  - ]
     222   [ +  +  +  -  :       5136 :         SET_FIELD(admin_timeout_ms, NVME_MAX_ADMIN_TIMEOUT_IN_SECS * 1000);
                   +  - ]
     223   [ +  +  +  -  :       5136 :         SET_FIELD(header_digest, false);
                   +  - ]
     224   [ +  +  +  -  :       5136 :         SET_FIELD(data_digest, false);
                   +  - ]
     225   [ +  +  +  -  :       5136 :         SET_FIELD(disable_error_logging, false);
                   +  - ]
     226   [ +  +  +  -  :       5136 :         SET_FIELD(transport_ack_timeout, SPDK_NVME_DEFAULT_TRANSPORT_ACK_TIMEOUT);
                   +  - ]
     227   [ +  +  +  -  :       5136 :         SET_FIELD(admin_queue_size, DEFAULT_ADMIN_QUEUE_SIZE);
                   +  - ]
     228   [ +  +  +  -  :       5136 :         SET_FIELD(fabrics_connect_timeout_us, NVME_FABRIC_CONNECT_COMMAND_TIMEOUT);
                   +  - ]
     229   [ +  +  +  -  :       5136 :         SET_FIELD(disable_read_ana_log_page, false);
                   +  - ]
     230   [ +  +  +  -  :       5136 :         SET_FIELD(disable_read_changed_ns_list_log_page, false);
                   +  - ]
     231   [ +  +  +  -  :       5136 :         SET_FIELD(tls_psk, NULL);
                   +  - ]
     232   [ +  +  +  -  :       5136 :         SET_FIELD(dhchap_key, NULL);
                   +  - ]
     233   [ +  +  +  -  :       5136 :         SET_FIELD(dhchap_ctrlr_key, NULL);
                   +  - ]
     234   [ +  +  +  -  :       5136 :         SET_FIELD(dhchap_digests,
          +  -  +  -  +  
                -  +  - ]
     235                 :            :                   SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA256) |
     236                 :            :                   SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA384) |
     237                 :            :                   SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA512));
     238   [ +  +  +  -  :       5136 :         SET_FIELD(dhchap_dhgroups,
          +  -  +  -  +  
          -  +  -  +  -  
             +  -  +  - ]
     239                 :            :                   SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_NULL) |
     240                 :            :                   SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_2048) |
     241                 :            :                   SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_3072) |
     242                 :            :                   SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_4096) |
     243                 :            :                   SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_6144) |
     244                 :            :                   SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_8192));
     245                 :            : #undef FIELD_OK
     246                 :            : #undef SET_FIELD
     247                 :       5136 : }
     248                 :            : 
     249                 :            : const struct spdk_nvme_ctrlr_opts *
     250                 :       2981 : spdk_nvme_ctrlr_get_opts(struct spdk_nvme_ctrlr *ctrlr)
     251                 :            : {
     252         [ +  - ]:       2981 :         return &ctrlr->opts;
     253                 :            : }
     254                 :            : 
     255                 :            : /**
     256                 :            :  * This function will be called when the process allocates the IO qpair.
     257                 :            :  * Note: the ctrlr_lock must be held when calling this function.
     258                 :            :  */
     259                 :            : static void
     260                 :       5780 : nvme_ctrlr_proc_add_io_qpair(struct spdk_nvme_qpair *qpair)
     261                 :            : {
     262                 :            :         struct spdk_nvme_ctrlr_process  *active_proc;
     263   [ +  -  +  - ]:       5780 :         struct spdk_nvme_ctrlr          *ctrlr = qpair->ctrlr;
     264                 :            : 
     265                 :       5780 :         active_proc = nvme_ctrlr_get_current_process(ctrlr);
     266         [ +  + ]:       5780 :         if (active_proc) {
     267   [ +  -  +  -  :       5735 :                 TAILQ_INSERT_TAIL(&active_proc->allocated_io_qpairs, qpair, per_process_tailq);
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
     268   [ +  -  +  - ]:       5735 :                 qpair->active_proc = active_proc;
     269                 :        833 :         }
     270                 :       5780 : }
     271                 :            : 
     272                 :            : /**
     273                 :            :  * This function will be called when the process frees the IO qpair.
     274                 :            :  * Note: the ctrlr_lock must be held when calling this function.
     275                 :            :  */
     276                 :            : static void
     277                 :       5780 : nvme_ctrlr_proc_remove_io_qpair(struct spdk_nvme_qpair *qpair)
     278                 :            : {
     279                 :            :         struct spdk_nvme_ctrlr_process  *active_proc;
     280   [ +  -  +  - ]:       5780 :         struct spdk_nvme_ctrlr          *ctrlr = qpair->ctrlr;
     281                 :            :         struct spdk_nvme_qpair          *active_qpair, *tmp_qpair;
     282                 :            : 
     283                 :       5780 :         active_proc = nvme_ctrlr_get_current_process(ctrlr);
     284         [ +  + ]:       5780 :         if (!active_proc) {
     285                 :         45 :                 return;
     286                 :            :         }
     287                 :            : 
     288   [ +  -  +  -  :       6086 :         TAILQ_FOREACH_SAFE(active_qpair, &active_proc->allocated_io_qpairs,
          +  -  -  +  +  
          -  +  -  +  -  
                   -  + ]
     289                 :            :                            per_process_tailq, tmp_qpair) {
     290         [ +  + ]:       6086 :                 if (active_qpair == qpair) {
     291   [ +  +  +  -  :       5735 :                         TAILQ_REMOVE(&active_proc->allocated_io_qpairs,
          +  -  -  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  -  +  -  
          +  -  +  -  +  
          -  +  -  +  +  
          -  +  -  +  -  
          +  -  +  -  +  
                -  +  - ]
     292                 :            :                                      active_qpair, per_process_tailq);
     293                 :            : 
     294                 :       5735 :                         break;
     295                 :            :                 }
     296                 :          0 :         }
     297                 :        833 : }
     298                 :            : 
     299                 :            : void
     300                 :       8759 : spdk_nvme_ctrlr_get_default_io_qpair_opts(struct spdk_nvme_ctrlr *ctrlr,
     301                 :            :                 struct spdk_nvme_io_qpair_opts *opts,
     302                 :            :                 size_t opts_size)
     303                 :            : {
     304   [ +  +  #  # ]:       8759 :         assert(ctrlr);
     305                 :            : 
     306   [ +  +  #  # ]:       8759 :         assert(opts);
     307                 :            : 
     308         [ +  + ]:       8759 :         memset(opts, 0, opts_size);
     309                 :            : 
     310                 :            : #define FIELD_OK(field) \
     311                 :            :         offsetof(struct spdk_nvme_io_qpair_opts, field) + sizeof(opts->field) <= opts_size
     312                 :            : 
     313         [ +  + ]:       8759 :         if (FIELD_OK(qprio)) {
     314   [ +  -  +  - ]:       8759 :                 opts->qprio = SPDK_NVME_QPRIO_URGENT;
     315                 :        834 :         }
     316                 :            : 
     317         [ +  + ]:       8759 :         if (FIELD_OK(io_queue_size)) {
     318   [ +  -  +  -  :       8759 :                 opts->io_queue_size = ctrlr->opts.io_queue_size;
          +  -  +  -  +  
                      - ]
     319                 :        834 :         }
     320                 :            : 
     321         [ +  + ]:       8759 :         if (FIELD_OK(io_queue_requests)) {
     322   [ +  -  +  -  :       8756 :                 opts->io_queue_requests = ctrlr->opts.io_queue_requests;
          +  -  +  -  +  
                      - ]
     323                 :        834 :         }
     324                 :            : 
     325         [ +  + ]:       8759 :         if (FIELD_OK(delay_cmd_submit)) {
     326   [ +  -  +  -  :       8756 :                 opts->delay_cmd_submit = false;
                   +  - ]
     327                 :        834 :         }
     328                 :            : 
     329         [ +  + ]:       8759 :         if (FIELD_OK(sq.vaddr)) {
     330   [ +  -  +  -  :       8756 :                 opts->sq.vaddr = NULL;
                   +  - ]
     331                 :        834 :         }
     332                 :            : 
     333         [ +  + ]:       8759 :         if (FIELD_OK(sq.paddr)) {
     334   [ +  -  +  -  :       8756 :                 opts->sq.paddr = 0;
                   +  - ]
     335                 :        834 :         }
     336                 :            : 
     337         [ +  + ]:       8759 :         if (FIELD_OK(sq.buffer_size)) {
     338   [ +  -  +  -  :       8756 :                 opts->sq.buffer_size = 0;
                   +  - ]
     339                 :        834 :         }
     340                 :            : 
     341         [ +  + ]:       8759 :         if (FIELD_OK(cq.vaddr)) {
     342   [ +  -  +  -  :       8756 :                 opts->cq.vaddr = NULL;
                   +  - ]
     343                 :        834 :         }
     344                 :            : 
     345         [ +  + ]:       8759 :         if (FIELD_OK(cq.paddr)) {
     346   [ +  -  +  -  :       8756 :                 opts->cq.paddr = 0;
                   +  - ]
     347                 :        834 :         }
     348                 :            : 
     349         [ +  + ]:       8759 :         if (FIELD_OK(cq.buffer_size)) {
     350   [ +  -  +  -  :       8756 :                 opts->cq.buffer_size = 0;
                   +  - ]
     351                 :        834 :         }
     352                 :            : 
     353         [ +  + ]:       8759 :         if (FIELD_OK(create_only)) {
     354   [ +  -  +  - ]:       8756 :                 opts->create_only = false;
     355                 :        834 :         }
     356                 :            : 
     357         [ +  + ]:       8759 :         if (FIELD_OK(async_mode)) {
     358   [ +  -  +  - ]:       8756 :                 opts->async_mode = false;
     359                 :        834 :         }
     360                 :            : 
     361                 :            : #undef FIELD_OK
     362                 :       8759 : }
     363                 :            : 
     364                 :            : static struct spdk_nvme_qpair *
     365                 :       5802 : nvme_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
     366                 :            :                            const struct spdk_nvme_io_qpair_opts *opts)
     367                 :            : {
     368                 :            :         int32_t                                 qid;
     369                 :            :         struct spdk_nvme_qpair                  *qpair;
     370                 :            :         union spdk_nvme_cc_register             cc;
     371                 :            : 
     372         [ +  + ]:       5802 :         if (!ctrlr) {
     373                 :          0 :                 return NULL;
     374                 :            :         }
     375                 :            : 
     376                 :       5802 :         nvme_ctrlr_lock(ctrlr);
     377   [ +  -  +  -  :       5802 :         cc.raw = ctrlr->process_init_cc.raw;
                   +  - ]
     378                 :            : 
     379   [ +  +  +  -  :       5802 :         if (opts->qprio & ~SPDK_NVME_CREATE_IO_SQ_QPRIO_MASK) {
                   -  + ]
     380                 :          6 :                 nvme_ctrlr_unlock(ctrlr);
     381                 :          6 :                 return NULL;
     382                 :            :         }
     383                 :            : 
     384                 :            :         /*
     385                 :            :          * Only value SPDK_NVME_QPRIO_URGENT(0) is valid for the
     386                 :            :          * default round robin arbitration method.
     387                 :            :          */
     388   [ +  +  +  +  :       5796 :         if ((cc.bits.ams == SPDK_NVME_CC_AMS_RR) && (opts->qprio != SPDK_NVME_QPRIO_URGENT)) {
             +  -  +  - ]
     389   [ +  -  -  +  :          9 :                 NVME_CTRLR_ERRLOG(ctrlr, "invalid queue priority for default round robin arbitration method\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     390                 :          9 :                 nvme_ctrlr_unlock(ctrlr);
     391                 :          9 :                 return NULL;
     392                 :            :         }
     393                 :            : 
     394                 :       5787 :         qid = spdk_nvme_ctrlr_alloc_qid(ctrlr);
     395         [ +  + ]:       5787 :         if (qid < 0) {
     396                 :          7 :                 nvme_ctrlr_unlock(ctrlr);
     397                 :          7 :                 return NULL;
     398                 :            :         }
     399                 :            : 
     400                 :       5780 :         qpair = nvme_transport_ctrlr_create_io_qpair(ctrlr, qid, opts);
     401         [ +  + ]:       5780 :         if (qpair == NULL) {
     402   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "nvme_transport_ctrlr_create_io_qpair() failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     403                 :          0 :                 spdk_nvme_ctrlr_free_qid(ctrlr, qid);
     404                 :          0 :                 nvme_ctrlr_unlock(ctrlr);
     405                 :          0 :                 return NULL;
     406                 :            :         }
     407                 :            : 
     408   [ +  -  +  -  :       5780 :         TAILQ_INSERT_TAIL(&ctrlr->active_io_qpairs, qpair, tailq);
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
     409                 :            : 
     410                 :       5780 :         nvme_ctrlr_proc_add_io_qpair(qpair);
     411                 :            : 
     412                 :       5780 :         nvme_ctrlr_unlock(ctrlr);
     413                 :            : 
     414                 :       5780 :         return qpair;
     415                 :        833 : }
     416                 :            : 
     417                 :            : int
     418                 :       5780 : spdk_nvme_ctrlr_connect_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
     419                 :            : {
     420                 :            :         int rc;
     421                 :            : 
     422         [ -  + ]:       5780 :         if (nvme_qpair_get_state(qpair) != NVME_QPAIR_DISCONNECTED) {
     423                 :          0 :                 return -EISCONN;
     424                 :            :         }
     425                 :            : 
     426                 :       5780 :         nvme_ctrlr_lock(ctrlr);
     427                 :       5780 :         rc = nvme_transport_ctrlr_connect_qpair(ctrlr, qpair);
     428                 :       5780 :         nvme_ctrlr_unlock(ctrlr);
     429                 :            : 
     430   [ +  +  +  -  :       5780 :         if (ctrlr->quirks & NVME_QUIRK_DELAY_AFTER_QUEUE_ALLOC) {
                   +  - ]
     431                 :          0 :                 spdk_delay_us(100);
     432                 :          0 :         }
     433                 :            : 
     434                 :       5780 :         return rc;
     435                 :        833 : }
     436                 :            : 
     437                 :            : void
     438                 :       2240 : spdk_nvme_ctrlr_disconnect_io_qpair(struct spdk_nvme_qpair *qpair)
     439                 :            : {
     440   [ +  -  +  - ]:       2240 :         struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
     441                 :            : 
     442                 :       2240 :         nvme_ctrlr_lock(ctrlr);
     443                 :       2240 :         nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair);
     444                 :       2240 :         nvme_ctrlr_unlock(ctrlr);
     445                 :       2240 : }
     446                 :            : 
     447                 :            : struct spdk_nvme_qpair *
     448                 :       5805 : spdk_nvme_ctrlr_alloc_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
     449                 :            :                                const struct spdk_nvme_io_qpair_opts *user_opts,
     450                 :            :                                size_t opts_size)
     451                 :            : {
     452                 :            : 
     453                 :       5805 :         struct spdk_nvme_qpair          *qpair = NULL;
     454                 :        974 :         struct spdk_nvme_io_qpair_opts  opts;
     455                 :            :         int                             rc;
     456                 :            : 
     457                 :       5805 :         nvme_ctrlr_lock(ctrlr);
     458                 :            : 
     459   [ +  +  +  -  :       5805 :         if (spdk_unlikely(ctrlr->state != NVME_CTRLR_STATE_READY)) {
                   -  + ]
     460                 :            :                 /* When controller is resetting or initializing, free_io_qids is deleted or not created yet.
     461                 :            :                  * We can't create IO qpair in that case */
     462                 :          3 :                 goto unlock;
     463                 :            :         }
     464                 :            : 
     465                 :            :         /*
     466                 :            :          * Get the default options, then overwrite them with the user-provided options
     467                 :            :          * up to opts_size.
     468                 :            :          *
     469                 :            :          * This allows for extensions of the opts structure without breaking
     470                 :            :          * ABI compatibility.
     471                 :            :          */
     472                 :       5802 :         spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts));
     473         [ +  + ]:       5802 :         if (user_opts) {
     474   [ -  +  +  -  :       2976 :                 memcpy(&opts, user_opts, spdk_min(sizeof(opts), opts_size));
                   +  - ]
     475                 :            : 
     476                 :            :                 /* If user passes buffers, make sure they're big enough for the requested queue size */
     477   [ +  +  +  -  :       2976 :                 if (opts.sq.vaddr) {
                   +  - ]
     478   [ #  #  #  #  :          0 :                         if (opts.sq.buffer_size < (opts.io_queue_size * sizeof(struct spdk_nvme_cmd))) {
             #  #  #  # ]
     479   [ #  #  #  #  :          0 :                                 NVME_CTRLR_ERRLOG(ctrlr, "sq buffer size %" PRIx64 " is too small for sq size %zx\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
     480                 :            :                                                   opts.sq.buffer_size, (opts.io_queue_size * sizeof(struct spdk_nvme_cmd)));
     481                 :          0 :                                 goto unlock;
     482                 :            :                         }
     483                 :          0 :                 }
     484   [ +  +  +  -  :       2976 :                 if (opts.cq.vaddr) {
                   +  - ]
     485   [ #  #  #  #  :          0 :                         if (opts.cq.buffer_size < (opts.io_queue_size * sizeof(struct spdk_nvme_cpl))) {
             #  #  #  # ]
     486   [ #  #  #  #  :          0 :                                 NVME_CTRLR_ERRLOG(ctrlr, "cq buffer size %" PRIx64 " is too small for cq size %zx\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
     487                 :            :                                                   opts.cq.buffer_size, (opts.io_queue_size * sizeof(struct spdk_nvme_cpl)));
     488                 :          0 :                                 goto unlock;
     489                 :            :                         }
     490                 :          0 :                 }
     491                 :          1 :         }
     492                 :            : 
     493                 :       5802 :         qpair = nvme_ctrlr_create_io_qpair(ctrlr, &opts);
     494                 :            : 
     495   [ +  +  +  +  :       5802 :         if (qpair == NULL || opts.create_only == true) {
             +  +  +  + ]
     496                 :       2665 :                 goto unlock;
     497                 :            :         }
     498                 :            : 
     499                 :       3137 :         rc = spdk_nvme_ctrlr_connect_io_qpair(ctrlr, qpair);
     500         [ +  + ]:       3137 :         if (rc != 0) {
     501   [ +  -  -  +  :          3 :                 NVME_CTRLR_ERRLOG(ctrlr, "nvme_transport_ctrlr_connect_io_qpair() failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     502                 :          3 :                 nvme_ctrlr_proc_remove_io_qpair(qpair);
     503   [ -  +  #  #  :          3 :                 TAILQ_REMOVE(&ctrlr->active_io_qpairs, qpair, tailq);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
     504   [ #  #  #  #  :          3 :                 spdk_bit_array_set(ctrlr->free_io_qids, qpair->id);
             #  #  #  # ]
     505                 :          3 :                 nvme_transport_ctrlr_delete_io_qpair(ctrlr, qpair);
     506                 :          3 :                 qpair = NULL;
     507                 :          3 :                 goto unlock;
     508                 :            :         }
     509                 :            : 
     510                 :       3124 : unlock:
     511                 :       5805 :         nvme_ctrlr_unlock(ctrlr);
     512                 :            : 
     513                 :       5805 :         return qpair;
     514                 :            : }
     515                 :            : 
     516                 :            : int
     517                 :      14030 : spdk_nvme_ctrlr_reconnect_io_qpair(struct spdk_nvme_qpair *qpair)
     518                 :            : {
     519                 :            :         struct spdk_nvme_ctrlr *ctrlr;
     520                 :            :         enum nvme_qpair_state qpair_state;
     521                 :            :         int rc;
     522                 :            : 
     523   [ -  +  #  # ]:      14030 :         assert(qpair != NULL);
     524   [ -  +  #  # ]:      14030 :         assert(nvme_qpair_is_admin_queue(qpair) == false);
     525   [ -  +  #  #  :      14030 :         assert(qpair->ctrlr != NULL);
             #  #  #  # ]
     526                 :            : 
     527   [ #  #  #  # ]:      14030 :         ctrlr = qpair->ctrlr;
     528                 :      14030 :         nvme_ctrlr_lock(ctrlr);
     529                 :      14030 :         qpair_state = nvme_qpair_get_state(qpair);
     530                 :            : 
     531   [ +  +  +  +  :      14030 :         if (ctrlr->is_removed) {
             #  #  #  # ]
     532                 :          6 :                 rc = -ENODEV;
     533                 :          6 :                 goto out;
     534                 :            :         }
     535                 :            : 
     536   [ +  +  +  +  :      14024 :         if (ctrlr->is_resetting || qpair_state == NVME_QPAIR_DISCONNECTING) {
          -  +  #  #  #  
                      # ]
     537                 :          6 :                 rc = -EAGAIN;
     538                 :          6 :                 goto out;
     539                 :            :         }
     540                 :            : 
     541   [ +  +  +  +  :      14018 :         if (ctrlr->is_failed || qpair_state == NVME_QPAIR_DESTROYING) {
          -  +  #  #  #  
                      # ]
     542                 :          6 :                 rc = -ENXIO;
     543                 :          6 :                 goto out;
     544                 :            :         }
     545                 :            : 
     546         [ +  + ]:      14012 :         if (qpair_state != NVME_QPAIR_DISCONNECTED) {
     547                 :          3 :                 rc = 0;
     548                 :          3 :                 goto out;
     549                 :            :         }
     550                 :            : 
     551                 :      14009 :         rc = nvme_transport_ctrlr_connect_qpair(ctrlr, qpair);
     552         [ +  + ]:      14009 :         if (rc) {
     553                 :      14001 :                 rc = -EAGAIN;
     554                 :      14001 :                 goto out;
     555                 :            :         }
     556                 :            : 
     557                 :          8 : out:
     558                 :      14030 :         nvme_ctrlr_unlock(ctrlr);
     559                 :      14030 :         return rc;
     560                 :            : }
     561                 :            : 
     562                 :            : spdk_nvme_qp_failure_reason
     563                 :    1326284 : spdk_nvme_ctrlr_get_admin_qp_failure_reason(struct spdk_nvme_ctrlr *ctrlr)
     564                 :            : {
     565   [ +  -  +  -  :    1326284 :         return ctrlr->adminq->transport_failure_reason;
                   +  - ]
     566                 :            : }
     567                 :            : 
     568                 :            : /*
     569                 :            :  * This internal function will attempt to take the controller
     570                 :            :  * lock before calling disconnect on a controller qpair.
     571                 :            :  * Functions already holding the controller lock should
     572                 :            :  * call nvme_transport_ctrlr_disconnect_qpair directly.
     573                 :            :  */
     574                 :            : void
     575                 :      28865 : nvme_ctrlr_disconnect_qpair(struct spdk_nvme_qpair *qpair)
     576                 :            : {
     577   [ #  #  #  # ]:      28865 :         struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
     578                 :            : 
     579   [ -  +  #  # ]:      28865 :         assert(ctrlr != NULL);
     580                 :      28865 :         nvme_ctrlr_lock(ctrlr);
     581                 :      28865 :         nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair);
     582                 :      28865 :         nvme_ctrlr_unlock(ctrlr);
     583                 :      28865 : }
     584                 :            : 
     585                 :            : int
     586                 :       5777 : spdk_nvme_ctrlr_free_io_qpair(struct spdk_nvme_qpair *qpair)
     587                 :            : {
     588                 :            :         struct spdk_nvme_ctrlr *ctrlr;
     589                 :            : 
     590         [ +  + ]:       5777 :         if (qpair == NULL) {
     591                 :          0 :                 return 0;
     592                 :            :         }
     593                 :            : 
     594   [ +  -  +  - ]:       5777 :         ctrlr = qpair->ctrlr;
     595                 :            : 
     596   [ +  +  -  + ]:       5777 :         if (qpair->in_completion_context) {
     597                 :            :                 /*
     598                 :            :                  * There are many cases where it is convenient to delete an io qpair in the context
     599                 :            :                  *  of that qpair's completion routine.  To handle this properly, set a flag here
     600                 :            :                  *  so that the completion routine will perform an actual delete after the context
     601                 :            :                  *  unwinds.
     602                 :            :                  */
     603         [ #  # ]:          0 :                 qpair->delete_after_completion_context = 1;
     604                 :          0 :                 return 0;
     605                 :            :         }
     606                 :            : 
     607   [ +  +  +  -  :       5777 :         if (qpair->auth.cb_fn != NULL) {
             +  -  +  - ]
     608   [ #  #  #  #  :          0 :                 qpair->auth.cb_fn(qpair->auth.cb_ctx, -ECANCELED);
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     609   [ #  #  #  #  :          0 :                 qpair->auth.cb_fn = NULL;
                   #  # ]
     610                 :          0 :         }
     611                 :            : 
     612         [ +  - ]:       5777 :         qpair->destroy_in_progress = 1;
     613                 :            : 
     614                 :       5777 :         nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair);
     615                 :            : 
     616   [ +  +  +  -  :       5777 :         if (qpair->poll_group && (qpair->active_proc == nvme_ctrlr_get_current_process(ctrlr))) {
          +  +  +  -  +  
                -  +  - ]
     617   [ +  -  +  -  :       2643 :                 spdk_nvme_poll_group_remove(qpair->poll_group->group, qpair);
             +  -  +  - ]
     618                 :          1 :         }
     619                 :            : 
     620                 :            :         /* Do not retry. */
     621                 :       5777 :         nvme_qpair_set_state(qpair, NVME_QPAIR_DESTROYING);
     622                 :            : 
     623                 :            :         /* In the multi-process case, a process may call this function on a foreign
     624                 :            :          * I/O qpair (i.e. one that this process did not create) when that qpairs process
     625                 :            :          * exits unexpectedly.  In that case, we must not try to abort any reqs associated
     626                 :            :          * with that qpair, since the callbacks will also be foreign to this process.
     627                 :            :          */
     628   [ +  -  +  -  :       5777 :         if (qpair->active_proc == nvme_ctrlr_get_current_process(ctrlr)) {
                   -  + ]
     629                 :       5777 :                 nvme_qpair_abort_all_queued_reqs(qpair);
     630                 :        833 :         }
     631                 :            : 
     632                 :       5777 :         nvme_ctrlr_lock(ctrlr);
     633                 :            : 
     634                 :       5777 :         nvme_ctrlr_proc_remove_io_qpair(qpair);
     635                 :            : 
     636   [ +  +  +  -  :       5777 :         TAILQ_REMOVE(&ctrlr->active_io_qpairs, qpair, tailq);
          +  -  -  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
                -  +  - ]
     637   [ +  -  +  - ]:       5777 :         spdk_nvme_ctrlr_free_qid(ctrlr, qpair->id);
     638                 :            : 
     639                 :       5777 :         nvme_transport_ctrlr_delete_io_qpair(ctrlr, qpair);
     640                 :       5777 :         nvme_ctrlr_unlock(ctrlr);
     641                 :       5777 :         return 0;
     642                 :        833 : }
     643                 :            : 
     644                 :            : static void
     645                 :        117 : nvme_ctrlr_construct_intel_support_log_page_list(struct spdk_nvme_ctrlr *ctrlr,
     646                 :            :                 struct spdk_nvme_intel_log_page_directory *log_page_directory)
     647                 :            : {
     648         [ +  + ]:        117 :         if (log_page_directory == NULL) {
     649                 :          0 :                 return;
     650                 :            :         }
     651                 :            : 
     652   [ +  +  +  -  :        117 :         assert(ctrlr->cdata.vid == SPDK_PCI_VID_INTEL);
          +  -  +  -  #  
                      # ]
     653                 :            : 
     654   [ +  -  +  -  :        117 :         ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_PAGE_DIRECTORY] = true;
             +  -  +  - ]
     655                 :            : 
     656   [ +  +  +  -  :        117 :         if (log_page_directory->read_latency_log_len ||
             -  +  #  # ]
     657   [ +  +  #  # ]:         11 :             (ctrlr->quirks & NVME_INTEL_QUIRK_READ_LATENCY)) {
     658   [ +  -  +  -  :        109 :                 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_READ_CMD_LATENCY] = true;
             +  -  +  - ]
     659                 :          2 :         }
     660   [ +  +  +  -  :        117 :         if (log_page_directory->write_latency_log_len ||
             -  +  #  # ]
     661   [ +  +  #  # ]:         11 :             (ctrlr->quirks & NVME_INTEL_QUIRK_WRITE_LATENCY)) {
     662   [ +  -  +  -  :        109 :                 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_WRITE_CMD_LATENCY] = true;
             +  -  +  - ]
     663                 :          2 :         }
     664   [ +  +  +  -  :        117 :         if (log_page_directory->temperature_statistics_log_len) {
                   -  + ]
     665   [ +  -  +  -  :        114 :                 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_TEMPERATURE] = true;
             +  -  +  - ]
     666                 :          2 :         }
     667   [ +  +  +  -  :        117 :         if (log_page_directory->smart_log_len) {
                   -  + ]
     668   [ +  -  +  -  :        111 :                 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_SMART] = true;
             +  -  +  - ]
     669                 :          2 :         }
     670   [ +  +  +  -  :        117 :         if (log_page_directory->marketing_description_log_len) {
                   -  + ]
     671   [ +  -  +  -  :        111 :                 ctrlr->log_page_supported[SPDK_NVME_INTEL_MARKETING_DESCRIPTION] = true;
             +  -  +  - ]
     672                 :          2 :         }
     673                 :          2 : }
     674                 :            : 
     675                 :            : struct intel_log_pages_ctx {
     676                 :            :         struct spdk_nvme_intel_log_page_directory log_page_directory;
     677                 :            :         struct spdk_nvme_ctrlr *ctrlr;
     678                 :            : };
     679                 :            : 
     680                 :            : static void
     681                 :        111 : nvme_ctrlr_set_intel_support_log_pages_done(void *arg, const struct spdk_nvme_cpl *cpl)
     682                 :            : {
     683                 :        111 :         struct intel_log_pages_ctx *ctx = arg;
     684   [ +  -  +  - ]:        111 :         struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr;
     685                 :            : 
     686   [ +  -  +  -  :        111 :         if (!spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
     687         [ +  - ]:        111 :                 nvme_ctrlr_construct_intel_support_log_page_list(ctrlr, &ctx->log_page_directory);
     688                 :          2 :         }
     689                 :            : 
     690                 :        113 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES,
     691   [ +  -  +  -  :        111 :                              ctrlr->opts.admin_timeout_ms);
                   +  - ]
     692                 :        111 :         free(ctx);
     693                 :        111 : }
     694                 :            : 
     695                 :            : static int
     696                 :        111 : nvme_ctrlr_set_intel_support_log_pages(struct spdk_nvme_ctrlr *ctrlr)
     697                 :            : {
     698                 :        111 :         int rc = 0;
     699                 :            :         struct intel_log_pages_ctx *ctx;
     700                 :            : 
     701                 :        111 :         ctx = calloc(1, sizeof(*ctx));
     702         [ +  + ]:        111 :         if (!ctx) {
     703                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES,
     704   [ #  #  #  #  :          0 :                                      ctrlr->opts.admin_timeout_ms);
                   #  # ]
     705                 :          0 :                 return 0;
     706                 :            :         }
     707                 :            : 
     708   [ +  -  +  - ]:        111 :         ctx->ctrlr = ctrlr;
     709                 :            : 
     710                 :        111 :         rc = spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_INTEL_LOG_PAGE_DIRECTORY,
     711         [ +  - ]:        111 :                                               SPDK_NVME_GLOBAL_NS_TAG, &ctx->log_page_directory,
     712                 :            :                                               sizeof(struct spdk_nvme_intel_log_page_directory),
     713                 :          2 :                                               0, nvme_ctrlr_set_intel_support_log_pages_done, ctx);
     714         [ -  + ]:        111 :         if (rc != 0) {
     715                 :          0 :                 free(ctx);
     716                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES,
     717   [ #  #  #  #  :          0 :                                      ctrlr->opts.admin_timeout_ms);
                   #  # ]
     718                 :          0 :                 return 0;
     719                 :            :         }
     720                 :            : 
     721                 :        113 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES,
     722   [ -  +  -  +  :        111 :                              ctrlr->opts.admin_timeout_ms);
                   -  + ]
     723                 :            : 
     724                 :        111 :         return 0;
     725                 :          2 : }
     726                 :            : 
     727                 :            : static int
     728                 :         25 : nvme_ctrlr_alloc_ana_log_page(struct spdk_nvme_ctrlr *ctrlr)
     729                 :            : {
     730                 :            :         uint32_t ana_log_page_size;
     731                 :            : 
     732   [ #  #  #  #  :         25 :         ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + ctrlr->cdata.nanagrpid *
                   #  # ]
     733   [ #  #  #  # ]:         25 :                             sizeof(struct spdk_nvme_ana_group_descriptor) + ctrlr->active_ns_count *
     734                 :            :                             sizeof(uint32_t);
     735                 :            : 
     736                 :            :         /* Number of active namespaces may have changed.
     737                 :            :          * Check if ANA log page fits into existing buffer.
     738                 :            :          */
     739   [ +  -  #  #  :         25 :         if (ana_log_page_size > ctrlr->ana_log_page_size) {
                   #  # ]
     740                 :            :                 void *new_buffer;
     741                 :            : 
     742   [ +  +  #  #  :         25 :                 if (ctrlr->ana_log_page) {
                   #  # ]
     743   [ #  #  #  # ]:          3 :                         new_buffer = realloc(ctrlr->ana_log_page, ana_log_page_size);
     744                 :          0 :                 } else {
     745                 :         22 :                         new_buffer = calloc(1, ana_log_page_size);
     746                 :            :                 }
     747                 :            : 
     748         [ -  + ]:         25 :                 if (!new_buffer) {
     749   [ #  #  #  #  :          0 :                         NVME_CTRLR_ERRLOG(ctrlr, "could not allocate ANA log page buffer, size %u\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     750                 :            :                                           ana_log_page_size);
     751                 :          0 :                         return -ENXIO;
     752                 :            :                 }
     753                 :            : 
     754   [ #  #  #  # ]:         25 :                 ctrlr->ana_log_page = new_buffer;
     755   [ +  +  #  #  :         25 :                 if (ctrlr->copied_ana_desc) {
                   #  # ]
     756   [ #  #  #  # ]:          3 :                         new_buffer = realloc(ctrlr->copied_ana_desc, ana_log_page_size);
     757                 :          0 :                 } else {
     758                 :         22 :                         new_buffer = calloc(1, ana_log_page_size);
     759                 :            :                 }
     760                 :            : 
     761         [ -  + ]:         25 :                 if (!new_buffer) {
     762   [ #  #  #  #  :          0 :                         NVME_CTRLR_ERRLOG(ctrlr, "could not allocate a buffer to parse ANA descriptor, size %u\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     763                 :            :                                           ana_log_page_size);
     764                 :          0 :                         return -ENOMEM;
     765                 :            :                 }
     766                 :            : 
     767   [ #  #  #  # ]:         25 :                 ctrlr->copied_ana_desc = new_buffer;
     768   [ #  #  #  # ]:         25 :                 ctrlr->ana_log_page_size = ana_log_page_size;
     769                 :          0 :         }
     770                 :            : 
     771                 :         25 :         return 0;
     772                 :          0 : }
     773                 :            : 
     774                 :            : static int
     775                 :         25 : nvme_ctrlr_update_ana_log_page(struct spdk_nvme_ctrlr *ctrlr)
     776                 :            : {
     777                 :            :         struct nvme_completion_poll_status *status;
     778                 :            :         int rc;
     779                 :            : 
     780                 :         25 :         rc = nvme_ctrlr_alloc_ana_log_page(ctrlr);
     781         [ -  + ]:         25 :         if (rc != 0) {
     782                 :          0 :                 return rc;
     783                 :            :         }
     784                 :            : 
     785                 :         25 :         status = calloc(1, sizeof(*status));
     786         [ -  + ]:         25 :         if (status == NULL) {
     787   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     788                 :          0 :                 return -ENOMEM;
     789                 :            :         }
     790                 :            : 
     791                 :         25 :         rc = spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS,
     792   [ #  #  #  # ]:         25 :                                               SPDK_NVME_GLOBAL_NS_TAG, ctrlr->ana_log_page,
     793   [ #  #  #  # ]:          0 :                                               ctrlr->ana_log_page_size, 0,
     794                 :          0 :                                               nvme_completion_poll_cb, status);
     795         [ -  + ]:         25 :         if (rc != 0) {
     796                 :          0 :                 free(status);
     797                 :          0 :                 return rc;
     798                 :            :         }
     799                 :            : 
     800   [ -  +  #  #  :         25 :         if (nvme_wait_for_completion_robust_lock_timeout(ctrlr->adminq, status, &ctrlr->ctrlr_lock,
             #  #  #  # ]
     801   [ #  #  #  #  :         25 :                         ctrlr->opts.admin_timeout_ms * 1000)) {
                   #  # ]
     802   [ #  #  #  #  :          0 :                 if (!status->timed_out) {
             #  #  #  # ]
     803                 :          0 :                         free(status);
     804                 :          0 :                 }
     805                 :          0 :                 return -EIO;
     806                 :            :         }
     807                 :            : 
     808                 :         25 :         free(status);
     809                 :         25 :         return 0;
     810                 :          0 : }
     811                 :            : 
     812                 :            : static int
     813                 :         28 : nvme_ctrlr_update_ns_ana_states(const struct spdk_nvme_ana_group_descriptor *desc,
     814                 :            :                                 void *cb_arg)
     815                 :            : {
     816                 :         28 :         struct spdk_nvme_ctrlr *ctrlr = cb_arg;
     817                 :            :         struct spdk_nvme_ns *ns;
     818                 :            :         uint32_t i, nsid;
     819                 :            : 
     820   [ +  +  #  #  :         68 :         for (i = 0; i < desc->num_of_nsid; i++) {
                   #  # ]
     821   [ #  #  #  #  :         40 :                 nsid = desc->nsid[i];
                   #  # ]
     822   [ +  -  -  +  :         40 :                 if (nsid == 0 || nsid > ctrlr->cdata.nn) {
          #  #  #  #  #  
                      # ]
     823                 :          0 :                         continue;
     824                 :            :                 }
     825                 :            : 
     826                 :         40 :                 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
     827   [ -  +  #  # ]:         40 :                 assert(ns != NULL);
     828                 :            : 
     829   [ #  #  #  #  :         40 :                 ns->ana_group_id = desc->ana_group_id;
             #  #  #  # ]
     830   [ #  #  #  #  :         40 :                 ns->ana_state = desc->ana_state;
                   #  # ]
     831                 :          0 :         }
     832                 :            : 
     833                 :         28 :         return 0;
     834                 :            : }
     835                 :            : 
     836                 :            : int
     837                 :         25 : nvme_ctrlr_parse_ana_log_page(struct spdk_nvme_ctrlr *ctrlr,
     838                 :            :                               spdk_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg)
     839                 :            : {
     840                 :            :         struct spdk_nvme_ana_group_descriptor *copied_desc;
     841                 :            :         uint8_t *orig_desc;
     842                 :            :         uint32_t i, desc_size, copy_len;
     843                 :         25 :         int rc = 0;
     844                 :            : 
     845   [ -  +  #  #  :         25 :         if (ctrlr->ana_log_page == NULL) {
                   #  # ]
     846                 :          0 :                 return -EINVAL;
     847                 :            :         }
     848                 :            : 
     849   [ #  #  #  # ]:         25 :         copied_desc = ctrlr->copied_ana_desc;
     850                 :            : 
     851   [ #  #  #  #  :         25 :         orig_desc = (uint8_t *)ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page);
                   #  # ]
     852   [ #  #  #  # ]:         25 :         copy_len = ctrlr->ana_log_page_size - sizeof(struct spdk_nvme_ana_page);
     853                 :            : 
     854   [ +  +  #  #  :         53 :         for (i = 0; i < ctrlr->ana_log_page->num_ana_group_desc; i++) {
          #  #  #  #  #  
                      # ]
     855   [ -  +  -  + ]:         28 :                 memcpy(copied_desc, orig_desc, copy_len);
     856                 :            : 
     857   [ #  #  #  # ]:         28 :                 rc = cb_fn(copied_desc, cb_arg);
     858         [ -  + ]:         28 :                 if (rc != 0) {
     859                 :          0 :                         break;
     860                 :            :                 }
     861                 :            : 
     862                 :         28 :                 desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) +
     863   [ #  #  #  # ]:         28 :                             copied_desc->num_of_nsid * sizeof(uint32_t);
     864         [ #  # ]:         28 :                 orig_desc += desc_size;
     865                 :         28 :                 copy_len -= desc_size;
     866                 :          0 :         }
     867                 :            : 
     868                 :         25 :         return rc;
     869                 :          0 : }
     870                 :            : 
     871                 :            : static int
     872                 :       3248 : nvme_ctrlr_set_supported_log_pages(struct spdk_nvme_ctrlr *ctrlr)
     873                 :            : {
     874                 :       3248 :         int     rc = 0;
     875                 :            : 
     876   [ +  +  +  - ]:       3248 :         memset(ctrlr->log_page_supported, 0, sizeof(ctrlr->log_page_supported));
     877                 :            :         /* Mandatory pages */
     878   [ +  -  +  -  :       3248 :         ctrlr->log_page_supported[SPDK_NVME_LOG_ERROR] = true;
             +  -  +  - ]
     879   [ +  -  +  -  :       3248 :         ctrlr->log_page_supported[SPDK_NVME_LOG_HEALTH_INFORMATION] = true;
             +  -  +  - ]
     880   [ +  -  +  -  :       3248 :         ctrlr->log_page_supported[SPDK_NVME_LOG_FIRMWARE_SLOT] = true;
             +  -  +  - ]
     881   [ +  +  +  -  :       3248 :         if (ctrlr->cdata.lpa.celp) {
             +  -  -  + ]
     882   [ +  -  +  -  :       3203 :                 ctrlr->log_page_supported[SPDK_NVME_LOG_COMMAND_EFFECTS_LOG] = true;
             +  -  +  - ]
     883                 :        833 :         }
     884                 :            : 
     885   [ +  +  +  -  :       3248 :         if (ctrlr->cdata.cmic.ana_reporting) {
             +  -  +  - ]
     886   [ #  #  #  #  :        345 :                 ctrlr->log_page_supported[SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS] = true;
             #  #  #  # ]
     887   [ +  +  +  +  :        345 :                 if (!ctrlr->opts.disable_read_ana_log_page) {
          #  #  #  #  #  
                      # ]
     888                 :         19 :                         rc = nvme_ctrlr_update_ana_log_page(ctrlr);
     889         [ +  - ]:         19 :                         if (rc == 0) {
     890                 :         19 :                                 nvme_ctrlr_parse_ana_log_page(ctrlr, nvme_ctrlr_update_ns_ana_states,
     891                 :          0 :                                                               ctrlr);
     892                 :          0 :                         }
     893                 :          0 :                 }
     894                 :          0 :         }
     895                 :            : 
     896   [ +  +  +  -  :       3248 :         if (ctrlr->cdata.ctratt.bits.fdps) {
          +  -  +  -  +  
                      - ]
     897   [ #  #  #  #  :         26 :                 ctrlr->log_page_supported[SPDK_NVME_LOG_FDP_CONFIGURATIONS] = true;
             #  #  #  # ]
     898   [ #  #  #  #  :         26 :                 ctrlr->log_page_supported[SPDK_NVME_LOG_RECLAIM_UNIT_HANDLE_USAGE] = true;
             #  #  #  # ]
     899   [ #  #  #  #  :         26 :                 ctrlr->log_page_supported[SPDK_NVME_LOG_FDP_STATISTICS] = true;
             #  #  #  # ]
     900   [ #  #  #  #  :         26 :                 ctrlr->log_page_supported[SPDK_NVME_LOG_FDP_EVENTS] = true;
             #  #  #  # ]
     901                 :          0 :         }
     902                 :            : 
     903   [ +  +  +  -  :       3250 :         if (ctrlr->cdata.vid == SPDK_PCI_VID_INTEL &&
          +  -  +  +  -  
                      + ]
     904   [ +  +  +  -  :       2152 :             ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE &&
             +  -  +  + ]
     905   [ +  -  +  - ]:        111 :             !(ctrlr->quirks & NVME_INTEL_QUIRK_NO_LOG_PAGES)) {
     906                 :        113 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES,
     907   [ +  -  +  -  :        111 :                                      ctrlr->opts.admin_timeout_ms);
                   +  - ]
     908                 :            : 
     909                 :          2 :         } else {
     910                 :       3968 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES,
     911   [ +  -  +  -  :       3137 :                                      ctrlr->opts.admin_timeout_ms);
                   +  - ]
     912                 :            : 
     913                 :            :         }
     914                 :            : 
     915                 :       3248 :         return rc;
     916                 :            : }
     917                 :            : 
     918                 :            : static void
     919                 :       2152 : nvme_ctrlr_set_intel_supported_features(struct spdk_nvme_ctrlr *ctrlr)
     920                 :            : {
     921   [ +  -  +  -  :       2152 :         ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_MAX_LBA] = true;
             +  -  +  - ]
     922   [ +  -  +  -  :       2152 :         ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_NATIVE_MAX_LBA] = true;
             +  -  +  - ]
     923   [ +  -  +  -  :       2152 :         ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_POWER_GOVERNOR_SETTING] = true;
             +  -  +  - ]
     924   [ +  -  +  -  :       2152 :         ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_SMBUS_ADDRESS] = true;
             +  -  +  - ]
     925   [ +  -  +  -  :       2152 :         ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_LED_PATTERN] = true;
             +  -  +  - ]
     926   [ +  -  +  -  :       2152 :         ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_RESET_TIMED_WORKLOAD_COUNTERS] = true;
             +  -  +  - ]
     927   [ +  -  +  -  :       2152 :         ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING] = true;
             +  -  +  - ]
     928                 :       2152 : }
     929                 :            : 
     930                 :            : static void
     931                 :       3254 : nvme_ctrlr_set_arbitration_feature(struct spdk_nvme_ctrlr *ctrlr)
     932                 :            : {
     933                 :            :         uint32_t cdw11;
     934                 :            :         struct nvme_completion_poll_status *status;
     935                 :            : 
     936   [ +  +  +  -  :       3254 :         if (ctrlr->opts.arbitration_burst == 0) {
             +  -  -  + ]
     937                 :       3248 :                 return;
     938                 :            :         }
     939                 :            : 
     940   [ +  +  #  #  :          6 :         if (ctrlr->opts.arbitration_burst > 7) {
             #  #  #  # ]
     941   [ +  -  -  +  :          3 :                 NVME_CTRLR_WARNLOG(ctrlr, "Valid arbitration burst values is from 0-7\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     942                 :          3 :                 return;
     943                 :            :         }
     944                 :            : 
     945                 :          3 :         status = calloc(1, sizeof(*status));
     946         [ -  + ]:          3 :         if (!status) {
     947   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     948                 :          0 :                 return;
     949                 :            :         }
     950                 :            : 
     951   [ #  #  #  #  :          3 :         cdw11 = ctrlr->opts.arbitration_burst;
                   #  # ]
     952                 :            : 
     953         [ +  - ]:          3 :         if (spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_WRR_SUPPORTED) {
     954   [ #  #  #  #  :          3 :                 cdw11 |= (uint32_t)ctrlr->opts.low_priority_weight << 8;
             #  #  #  # ]
     955   [ #  #  #  #  :          3 :                 cdw11 |= (uint32_t)ctrlr->opts.medium_priority_weight << 16;
             #  #  #  # ]
     956   [ #  #  #  #  :          3 :                 cdw11 |= (uint32_t)ctrlr->opts.high_priority_weight << 24;
             #  #  #  # ]
     957                 :          0 :         }
     958                 :            : 
     959   [ -  +  #  #  :          3 :         if (spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_FEAT_ARBITRATION,
                   #  # ]
     960                 :          0 :                                             cdw11, 0, NULL, 0,
     961                 :          0 :                                             nvme_completion_poll_cb, status) < 0) {
     962   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Set arbitration feature failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     963                 :          0 :                 free(status);
     964                 :          0 :                 return;
     965                 :            :         }
     966                 :            : 
     967   [ -  +  #  #  :          3 :         if (nvme_wait_for_completion_timeout(ctrlr->adminq, status,
                   #  # ]
     968   [ #  #  #  #  :          3 :                                              ctrlr->opts.admin_timeout_ms * 1000)) {
                   #  # ]
     969   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Timeout to set arbitration feature\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     970                 :          0 :         }
     971                 :            : 
     972   [ +  +  +  -  :          3 :         if (!status->timed_out) {
             #  #  #  # ]
     973                 :          3 :                 free(status);
     974                 :          0 :         }
     975                 :        833 : }
     976                 :            : 
     977                 :            : static void
     978                 :       3248 : nvme_ctrlr_set_supported_features(struct spdk_nvme_ctrlr *ctrlr)
     979                 :            : {
     980   [ +  +  +  - ]:       3248 :         memset(ctrlr->feature_supported, 0, sizeof(ctrlr->feature_supported));
     981                 :            :         /* Mandatory features */
     982   [ +  -  +  -  :       3248 :         ctrlr->feature_supported[SPDK_NVME_FEAT_ARBITRATION] = true;
             +  -  +  - ]
     983   [ +  -  +  -  :       3248 :         ctrlr->feature_supported[SPDK_NVME_FEAT_POWER_MANAGEMENT] = true;
             +  -  +  - ]
     984   [ +  -  +  -  :       3248 :         ctrlr->feature_supported[SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD] = true;
             +  -  +  - ]
     985   [ +  -  +  -  :       3248 :         ctrlr->feature_supported[SPDK_NVME_FEAT_ERROR_RECOVERY] = true;
             +  -  +  - ]
     986   [ +  -  +  -  :       3248 :         ctrlr->feature_supported[SPDK_NVME_FEAT_NUMBER_OF_QUEUES] = true;
             +  -  +  - ]
     987   [ +  -  +  -  :       3248 :         ctrlr->feature_supported[SPDK_NVME_FEAT_INTERRUPT_COALESCING] = true;
             +  -  +  - ]
     988   [ +  -  +  -  :       3248 :         ctrlr->feature_supported[SPDK_NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION] = true;
             +  -  +  - ]
     989   [ +  -  +  -  :       3248 :         ctrlr->feature_supported[SPDK_NVME_FEAT_WRITE_ATOMICITY] = true;
             +  -  +  - ]
     990   [ +  -  +  -  :       3248 :         ctrlr->feature_supported[SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION] = true;
             +  -  +  - ]
     991                 :            :         /* Optional features */
     992   [ +  +  +  -  :       3248 :         if (ctrlr->cdata.vwc.present) {
             +  -  +  + ]
     993   [ +  -  +  -  :       3070 :                 ctrlr->feature_supported[SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE] = true;
             +  -  +  - ]
     994                 :        831 :         }
     995   [ +  +  +  -  :       3248 :         if (ctrlr->cdata.apsta.supported) {
             +  -  +  - ]
     996   [ #  #  #  #  :          0 :                 ctrlr->feature_supported[SPDK_NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION] = true;
             #  #  #  # ]
     997                 :          0 :         }
     998   [ +  +  +  -  :       3248 :         if (ctrlr->cdata.hmpre) {
             +  -  +  - ]
     999   [ #  #  #  #  :          0 :                 ctrlr->feature_supported[SPDK_NVME_FEAT_HOST_MEM_BUFFER] = true;
             #  #  #  # ]
    1000                 :          0 :         }
    1001   [ +  +  +  -  :       3248 :         if (ctrlr->cdata.vid == SPDK_PCI_VID_INTEL) {
             +  -  +  + ]
    1002                 :       2152 :                 nvme_ctrlr_set_intel_supported_features(ctrlr);
    1003                 :        826 :         }
    1004                 :            : 
    1005                 :       3248 :         nvme_ctrlr_set_arbitration_feature(ctrlr);
    1006                 :       3248 : }
    1007                 :            : 
    1008                 :            : static void
    1009                 :        665 : nvme_ctrlr_set_host_feature_done(void *arg, const struct spdk_nvme_cpl *cpl)
    1010                 :            : {
    1011                 :        665 :         struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg;
    1012                 :            : 
    1013   [ #  #  #  # ]:        665 :         spdk_free(ctrlr->tmp_ptr);
    1014   [ #  #  #  # ]:        665 :         ctrlr->tmp_ptr = NULL;
    1015                 :            : 
    1016   [ +  -  -  +  :        665 :         if (spdk_nvme_cpl_is_error(cpl)) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1017   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Set host behavior support feature failed: SC %x SCT %x\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1018                 :            :                                   cpl->status.sc, cpl->status.sct);
    1019                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    1020                 :          0 :                 return;
    1021                 :            :         }
    1022                 :            : 
    1023   [ #  #  #  #  :        665 :         ctrlr->feature_supported[SPDK_NVME_FEAT_HOST_BEHAVIOR_SUPPORT] = true;
             #  #  #  # ]
    1024                 :            : 
    1025                 :        665 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_DB_BUF_CFG,
    1026   [ #  #  #  #  :        665 :                              ctrlr->opts.admin_timeout_ms);
                   #  # ]
    1027                 :          0 : }
    1028                 :            : 
    1029                 :            : /* We do not want to do add synchronous operation anymore.
    1030                 :            :  * We set the Host Behavior Support feature asynchronousin in different states.
    1031                 :            :  */
    1032                 :            : static int
    1033                 :       3248 : nvme_ctrlr_set_host_feature(struct spdk_nvme_ctrlr *ctrlr)
    1034                 :            : {
    1035                 :            :         struct spdk_nvme_host_behavior *host;
    1036                 :            :         int rc;
    1037                 :            : 
    1038   [ +  +  +  -  :       3248 :         if (!ctrlr->cdata.ctratt.bits.elbas) {
          +  -  +  -  -  
                      + ]
    1039                 :       3416 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_DB_BUF_CFG,
    1040   [ +  -  +  -  :       2583 :                                      ctrlr->opts.admin_timeout_ms);
                   +  - ]
    1041                 :       2583 :                 return 0;
    1042                 :            :         }
    1043                 :            : 
    1044   [ #  #  #  # ]:        665 :         ctrlr->tmp_ptr = spdk_dma_zmalloc(sizeof(struct spdk_nvme_host_behavior), 4096, NULL);
    1045   [ -  +  #  #  :        665 :         if (!ctrlr->tmp_ptr) {
                   #  # ]
    1046   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate host behavior support data\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1047                 :          0 :                 rc = -ENOMEM;
    1048                 :          0 :                 goto error;
    1049                 :            :         }
    1050                 :            : 
    1051                 :        665 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_SET_HOST_FEATURE,
    1052   [ #  #  #  #  :        665 :                              ctrlr->opts.admin_timeout_ms);
                   #  # ]
    1053                 :            : 
    1054   [ #  #  #  # ]:        665 :         host = ctrlr->tmp_ptr;
    1055                 :            : 
    1056   [ #  #  #  # ]:        665 :         host->lbafee = 1;
    1057                 :            : 
    1058                 :        665 :         rc = spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_FEAT_HOST_BEHAVIOR_SUPPORT,
    1059                 :          0 :                                              0, 0, host, sizeof(struct spdk_nvme_host_behavior),
    1060                 :          0 :                                              nvme_ctrlr_set_host_feature_done, ctrlr);
    1061         [ -  + ]:        665 :         if (rc != 0) {
    1062   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Set host behavior support feature failed: %d\n", rc);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1063                 :          0 :                 goto error;
    1064                 :            :         }
    1065                 :            : 
    1066                 :        665 :         return 0;
    1067                 :            : 
    1068                 :          0 : error:
    1069   [ #  #  #  # ]:          0 :         spdk_free(ctrlr->tmp_ptr);
    1070   [ #  #  #  # ]:          0 :         ctrlr->tmp_ptr = NULL;
    1071                 :            : 
    1072                 :          0 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    1073                 :          0 :         return rc;
    1074                 :        833 : }
    1075                 :            : 
    1076                 :            : bool
    1077                 :     100497 : spdk_nvme_ctrlr_is_failed(struct spdk_nvme_ctrlr *ctrlr)
    1078                 :            : {
    1079   [ -  +  #  #  :     100497 :         return ctrlr->is_failed;
                   #  # ]
    1080                 :            : }
    1081                 :            : 
    1082                 :            : void
    1083                 :        605 : nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove)
    1084                 :            : {
    1085                 :            :         /*
    1086                 :            :          * Set the flag here and leave the work failure of qpairs to
    1087                 :            :          * spdk_nvme_qpair_process_completions().
    1088                 :            :          */
    1089   [ +  +  #  # ]:        605 :         if (hot_remove) {
    1090   [ #  #  #  # ]:         75 :                 ctrlr->is_removed = true;
    1091                 :          0 :         }
    1092                 :            : 
    1093   [ -  +  +  +  :        605 :         if (ctrlr->is_failed) {
             #  #  #  # ]
    1094   [ +  +  +  +  :         45 :                 NVME_CTRLR_NOTICELOG(ctrlr, "already in failed state\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1095                 :         45 :                 return;
    1096                 :            :         }
    1097                 :            : 
    1098   [ -  +  +  +  :        560 :         if (ctrlr->is_disconnecting) {
             #  #  #  # ]
    1099   [ -  +  -  +  :          6 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "already disconnecting\n");
          -  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    1100                 :          6 :                 return;
    1101                 :            :         }
    1102                 :            : 
    1103   [ #  #  #  # ]:        554 :         ctrlr->is_failed = true;
    1104                 :        554 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    1105   [ #  #  #  # ]:        554 :         nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq);
    1106   [ +  +  +  +  :        554 :         NVME_CTRLR_ERRLOG(ctrlr, "in failed state.\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1107                 :          0 : }
    1108                 :            : 
    1109                 :            : /**
    1110                 :            :  * This public API function will try to take the controller lock.
    1111                 :            :  * Any private functions being called from a thread already holding
    1112                 :            :  * the ctrlr lock should call nvme_ctrlr_fail directly.
    1113                 :            :  */
    1114                 :            : void
    1115                 :         43 : spdk_nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr)
    1116                 :            : {
    1117                 :         43 :         nvme_ctrlr_lock(ctrlr);
    1118                 :         43 :         nvme_ctrlr_fail(ctrlr, false);
    1119                 :         43 :         nvme_ctrlr_unlock(ctrlr);
    1120                 :         43 : }
    1121                 :            : 
    1122                 :            : static void
    1123                 :       3197 : nvme_ctrlr_shutdown_set_cc_done(void *_ctx, uint64_t value, const struct spdk_nvme_cpl *cpl)
    1124                 :            : {
    1125                 :       3197 :         struct nvme_ctrlr_detach_ctx *ctx = _ctx;
    1126   [ +  -  +  - ]:       3197 :         struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr;
    1127                 :            : 
    1128   [ +  -  +  +  :       3197 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   -  + ]
    1129   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to write CC.SHN\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1130   [ #  #  #  # ]:          0 :                 ctx->shutdown_complete = true;
    1131                 :          0 :                 return;
    1132                 :            :         }
    1133                 :            : 
    1134   [ +  +  +  +  :       3197 :         if (ctrlr->opts.no_shn_notification) {
          +  -  +  -  -  
                      + ]
    1135   [ #  #  #  # ]:          0 :                 ctx->shutdown_complete = true;
    1136                 :          0 :                 return;
    1137                 :            :         }
    1138                 :            : 
    1139                 :            :         /*
    1140                 :            :          * The NVMe specification defines RTD3E to be the time between
    1141                 :            :          *  setting SHN = 1 until the controller will set SHST = 10b.
    1142                 :            :          * If the device doesn't report RTD3 entry latency, or if it
    1143                 :            :          *  reports RTD3 entry latency less than 10 seconds, pick
    1144                 :            :          *  10 seconds as a reasonable amount of time to
    1145                 :            :          *  wait before proceeding.
    1146                 :            :          */
    1147   [ +  +  +  +  :       3197 :         NVME_CTRLR_DEBUGLOG(ctrlr, "RTD3E = %" PRIu32 " us\n", ctrlr->cdata.rtd3e);
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1148   [ +  -  +  -  :       3197 :         ctx->shutdown_timeout_ms = SPDK_CEIL_DIV(ctrlr->cdata.rtd3e, 1000);
          +  -  +  -  +  
                -  +  - ]
    1149   [ +  -  +  -  :       3197 :         ctx->shutdown_timeout_ms = spdk_max(ctx->shutdown_timeout_ms, 10000);
          -  +  #  #  #  
             #  +  -  +  
                      - ]
    1150   [ +  +  +  +  :       3197 :         NVME_CTRLR_DEBUGLOG(ctrlr, "shutdown timeout = %" PRIu32 " ms\n", ctx->shutdown_timeout_ms);
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1151                 :            : 
    1152   [ +  -  +  - ]:       3197 :         ctx->shutdown_start_tsc = spdk_get_ticks();
    1153   [ +  -  +  - ]:       3197 :         ctx->state = NVME_CTRLR_DETACH_CHECK_CSTS;
    1154                 :        833 : }
    1155                 :            : 
    1156                 :            : static void
    1157                 :       3197 : nvme_ctrlr_shutdown_get_cc_done(void *_ctx, uint64_t value, const struct spdk_nvme_cpl *cpl)
    1158                 :            : {
    1159                 :       3197 :         struct nvme_ctrlr_detach_ctx *ctx = _ctx;
    1160   [ +  -  +  - ]:       3197 :         struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr;
    1161                 :            :         union spdk_nvme_cc_register cc;
    1162                 :            :         int rc;
    1163                 :            : 
    1164   [ +  -  +  +  :       3197 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   -  + ]
    1165   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1166   [ #  #  #  # ]:          0 :                 ctx->shutdown_complete = true;
    1167                 :          0 :                 return;
    1168                 :            :         }
    1169                 :            : 
    1170   [ +  +  #  # ]:       3197 :         assert(value <= UINT32_MAX);
    1171                 :       3197 :         cc.raw = (uint32_t)value;
    1172                 :            : 
    1173   [ +  +  +  +  :       3197 :         if (ctrlr->opts.no_shn_notification) {
          +  -  +  -  -  
                      + ]
    1174   [ #  #  #  #  :          0 :                 NVME_CTRLR_INFOLOG(ctrlr, "Disable SSD without shutdown notification\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    1175         [ #  # ]:          0 :                 if (cc.bits.en == 0) {
    1176   [ #  #  #  # ]:          0 :                         ctx->shutdown_complete = true;
    1177                 :          0 :                         return;
    1178                 :            :                 }
    1179                 :            : 
    1180                 :          0 :                 cc.bits.en = 0;
    1181                 :          0 :         } else {
    1182                 :       3197 :                 cc.bits.shn = SPDK_NVME_SHN_NORMAL;
    1183                 :            :         }
    1184                 :            : 
    1185                 :       3197 :         rc = nvme_ctrlr_set_cc_async(ctrlr, cc.raw, nvme_ctrlr_shutdown_set_cc_done, ctx);
    1186         [ +  + ]:       3197 :         if (rc != 0) {
    1187   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to write CC.SHN\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1188   [ #  #  #  # ]:          0 :                 ctx->shutdown_complete = true;
    1189                 :          0 :         }
    1190                 :        833 : }
    1191                 :            : 
    1192                 :            : static void
    1193                 :       3419 : nvme_ctrlr_shutdown_async(struct spdk_nvme_ctrlr *ctrlr,
    1194                 :            :                           struct nvme_ctrlr_detach_ctx *ctx)
    1195                 :            : {
    1196                 :            :         int rc;
    1197                 :            : 
    1198   [ +  +  +  +  :       3419 :         if (ctrlr->is_removed) {
             +  -  -  + ]
    1199   [ #  #  #  # ]:         72 :                 ctx->shutdown_complete = true;
    1200                 :         72 :                 return;
    1201                 :            :         }
    1202                 :            : 
    1203   [ +  +  +  -  :       4180 :         if (ctrlr->adminq == NULL ||
             +  -  +  - ]
    1204   [ +  +  +  -  :       3326 :             ctrlr->adminq->transport_failure_reason != SPDK_NVME_QPAIR_FAILURE_NONE) {
                   +  - ]
    1205   [ -  +  -  +  :        147 :                 NVME_CTRLR_INFOLOG(ctrlr, "Adminq is not connected.\n");
          -  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    1206   [ #  #  #  # ]:        147 :                 ctx->shutdown_complete = true;
    1207                 :        147 :                 return;
    1208                 :            :         }
    1209                 :            : 
    1210   [ +  -  +  - ]:       3200 :         ctx->state = NVME_CTRLR_DETACH_SET_CC;
    1211                 :       3200 :         rc = nvme_ctrlr_get_cc_async(ctrlr, nvme_ctrlr_shutdown_get_cc_done, ctx);
    1212         [ +  + ]:       3200 :         if (rc != 0) {
    1213   [ -  +  -  -  :          3 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1214   [ #  #  #  # ]:          3 :                 ctx->shutdown_complete = true;
    1215                 :          0 :         }
    1216                 :        833 : }
    1217                 :            : 
    1218                 :            : static void
    1219                 :    6299253 : nvme_ctrlr_shutdown_get_csts_done(void *_ctx, uint64_t value, const struct spdk_nvme_cpl *cpl)
    1220                 :            : {
    1221                 :    6299253 :         struct nvme_ctrlr_detach_ctx *ctx = _ctx;
    1222                 :            : 
    1223   [ +  -  +  +  :    6299253 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   -  + ]
    1224   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctx->ctrlr, "Failed to read the CSTS register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    1225   [ #  #  #  # ]:          0 :                 ctx->shutdown_complete = true;
    1226                 :          0 :                 return;
    1227                 :            :         }
    1228                 :            : 
    1229   [ +  +  #  # ]:    6299253 :         assert(value <= UINT32_MAX);
    1230   [ +  -  +  -  :    6299253 :         ctx->csts.raw = (uint32_t)value;
                   +  - ]
    1231   [ +  -  +  - ]:    6299253 :         ctx->state = NVME_CTRLR_DETACH_GET_CSTS_DONE;
    1232                 :     405159 : }
    1233                 :            : 
    1234                 :            : static int
    1235                 :   20514678 : nvme_ctrlr_shutdown_poll_async(struct spdk_nvme_ctrlr *ctrlr,
    1236                 :            :                                struct nvme_ctrlr_detach_ctx *ctx)
    1237                 :            : {
    1238                 :            :         union spdk_nvme_csts_register   csts;
    1239                 :            :         uint32_t                        ms_waited;
    1240                 :            : 
    1241   [ +  +  +  -  :   20514678 :         switch (ctx->state) {
             -  +  +  + ]
    1242                 :    7485097 :         case NVME_CTRLR_DETACH_SET_CC:
    1243                 :            :         case NVME_CTRLR_DETACH_GET_CSTS:
    1244                 :            :                 /* We're still waiting for the register operation to complete */
    1245   [ +  -  +  - ]:    7916172 :                 spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
    1246                 :    7916172 :                 return -EAGAIN;
    1247                 :            : 
    1248                 :    5894094 :         case NVME_CTRLR_DETACH_CHECK_CSTS:
    1249   [ +  -  +  - ]:    6299253 :                 ctx->state = NVME_CTRLR_DETACH_GET_CSTS;
    1250         [ -  + ]:    6299253 :                 if (nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_shutdown_get_csts_done, ctx)) {
    1251   [ #  #  #  #  :          0 :                         NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1252                 :          0 :                         return -EIO;
    1253                 :            :                 }
    1254                 :    6299253 :                 return -EAGAIN;
    1255                 :            : 
    1256                 :    5894094 :         case NVME_CTRLR_DETACH_GET_CSTS_DONE:
    1257   [ +  -  +  - ]:    6299253 :                 ctx->state = NVME_CTRLR_DETACH_CHECK_CSTS;
    1258                 :    6299253 :                 break;
    1259                 :            : 
    1260                 :          0 :         default:
    1261         [ #  # ]:          0 :                 assert(0 && "Should never happen");
    1262                 :            :                 return -EINVAL;
    1263                 :            :         }
    1264                 :            : 
    1265   [ +  +  +  -  :    6299253 :         ms_waited = (spdk_get_ticks() - ctx->shutdown_start_tsc) * 1000 / spdk_get_ticks_hz();
                   +  - ]
    1266   [ +  -  +  -  :    6299253 :         csts.raw = ctx->csts.raw;
                   +  - ]
    1267                 :            : 
    1268         [ +  + ]:    6299253 :         if (csts.bits.shst == SPDK_NVME_SHST_COMPLETE) {
    1269   [ +  +  +  +  :       3197 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "shutdown complete in %u milliseconds\n", ms_waited);
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    1270                 :       3197 :                 return 0;
    1271                 :            :         }
    1272                 :            : 
    1273   [ +  -  +  -  :    6296056 :         if (ms_waited < ctx->shutdown_timeout_ms) {
                   +  - ]
    1274                 :    6296056 :                 return -EAGAIN;
    1275                 :            :         }
    1276                 :            : 
    1277   [ #  #  #  #  :          0 :         NVME_CTRLR_ERRLOG(ctrlr, "did not shutdown within %u milliseconds\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  # ]
    1278                 :            :                           ctx->shutdown_timeout_ms);
    1279   [ #  #  #  #  :          0 :         if (ctrlr->quirks & NVME_QUIRK_SHST_COMPLETE) {
                   #  # ]
    1280   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "likely due to shutdown handling in the VMWare emulated NVMe SSD\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1281                 :          0 :         }
    1282                 :            : 
    1283                 :          0 :         return 0;
    1284                 :    1241393 : }
    1285                 :            : 
    1286                 :            : static inline uint64_t
    1287                 :    6043292 : nvme_ctrlr_get_ready_timeout(struct spdk_nvme_ctrlr *ctrlr)
    1288                 :            : {
    1289   [ +  -  +  -  :    6043292 :         return ctrlr->cap.bits.to * 500;
             +  -  +  - ]
    1290                 :            : }
    1291                 :            : 
    1292                 :            : static void
    1293                 :       3394 : nvme_ctrlr_set_cc_en_done(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl)
    1294                 :            : {
    1295                 :       3394 :         struct spdk_nvme_ctrlr *ctrlr = ctx;
    1296                 :            : 
    1297   [ +  -  +  +  :       3394 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
    1298   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to set the CC register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1299                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    1300                 :          0 :                 return;
    1301                 :            :         }
    1302                 :            : 
    1303                 :       4227 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1,
    1304                 :        833 :                              nvme_ctrlr_get_ready_timeout(ctrlr));
    1305                 :        833 : }
    1306                 :            : 
    1307                 :            : static int
    1308                 :       3415 : nvme_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr)
    1309                 :            : {
    1310                 :            :         union spdk_nvme_cc_register     cc;
    1311                 :            :         int                             rc;
    1312                 :            : 
    1313                 :       3415 :         rc = nvme_transport_ctrlr_enable(ctrlr);
    1314         [ -  + ]:       3415 :         if (rc != 0) {
    1315   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "transport ctrlr_enable failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1316                 :          0 :                 return rc;
    1317                 :            :         }
    1318                 :            : 
    1319   [ +  -  +  -  :       3415 :         cc.raw = ctrlr->process_init_cc.raw;
                   +  - ]
    1320         [ -  + ]:       3415 :         if (cc.bits.en != 0) {
    1321   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "called with CC.EN = 1\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1322                 :          0 :                 return -EINVAL;
    1323                 :            :         }
    1324                 :            : 
    1325                 :       3415 :         cc.bits.en = 1;
    1326                 :       3415 :         cc.bits.css = 0;
    1327                 :       3415 :         cc.bits.shn = 0;
    1328                 :       3415 :         cc.bits.iosqes = 6; /* SQ entry size == 64 == 2^6 */
    1329                 :       3415 :         cc.bits.iocqes = 4; /* CQ entry size == 16 == 2^4 */
    1330                 :            : 
    1331                 :            :         /* Page size is 2 ^ (12 + mps). */
    1332   [ +  -  +  - ]:       3415 :         cc.bits.mps = spdk_u32log2(ctrlr->page_size) - 12;
    1333                 :            : 
    1334                 :            :         /*
    1335                 :            :          * Since NVMe 1.0, a controller should have at least one bit set in CAP.CSS.
    1336                 :            :          * A controller that does not have any bit set in CAP.CSS is not spec compliant.
    1337                 :            :          * Try to support such a controller regardless.
    1338                 :            :          */
    1339   [ +  +  +  -  :       3415 :         if (ctrlr->cap.bits.css == 0) {
             +  -  -  + ]
    1340   [ -  +  -  +  :         63 :                 NVME_CTRLR_INFOLOG(ctrlr, "Drive reports no command sets supported. Assuming NVM is supported.\n");
          -  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    1341   [ #  #  #  #  :         63 :                 ctrlr->cap.bits.css = SPDK_NVME_CAP_CSS_NVM;
             #  #  #  # ]
    1342                 :          0 :         }
    1343                 :            : 
    1344                 :            :         /*
    1345                 :            :          * If the user did not explicitly request a command set, or supplied a value larger than
    1346                 :            :          * what can be saved in CC.CSS, use the most reasonable default.
    1347                 :            :          */
    1348   [ +  +  +  -  :       3415 :         if (ctrlr->opts.command_set >= CHAR_BIT) {
             +  -  -  + ]
    1349   [ +  +  +  -  :       3226 :                 if (ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_IOCS) {
          +  -  +  -  -  
                      + ]
    1350   [ #  #  #  #  :        919 :                         ctrlr->opts.command_set = SPDK_NVME_CC_CSS_IOCS;
                   #  # ]
    1351   [ +  -  +  -  :       2307 :                 } else if (ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_NVM) {
          +  -  +  -  +  
                      - ]
    1352   [ +  -  +  -  :       2307 :                         ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM;
                   +  - ]
    1353   [ #  #  #  #  :        833 :                 } else if (ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_NOIO) {
          #  #  #  #  #  
                      # ]
    1354                 :            :                         /* Technically we should respond with CC_CSS_NOIO in
    1355                 :            :                          * this case, but we use NVM instead to work around
    1356                 :            :                          * buggy targets and to match Linux driver behavior.
    1357                 :            :                          */
    1358   [ #  #  #  #  :          0 :                         ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM;
                   #  # ]
    1359                 :          0 :                 } else {
    1360                 :            :                         /* Invalid supported bits detected, falling back to NVM. */
    1361   [ #  #  #  #  :          0 :                         ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM;
                   #  # ]
    1362                 :            :                 }
    1363                 :        833 :         }
    1364                 :            : 
    1365                 :            :         /* Verify that the selected command set is supported by the controller. */
    1366   [ +  +  +  +  :       3415 :         if (!(ctrlr->cap.bits.css & (1u << ctrlr->opts.command_set))) {
          +  -  +  -  +  
          -  +  -  +  -  
                   -  + ]
    1367   [ #  #  #  #  :          0 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Requested I/O command set %u but supported mask is 0x%x\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
    1368                 :            :                                     ctrlr->opts.command_set, ctrlr->cap.bits.css);
    1369   [ #  #  #  #  :          0 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Falling back to NVM. Assuming NVM is supported.\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    1370   [ #  #  #  #  :          0 :                 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM;
                   #  # ]
    1371                 :          0 :         }
    1372                 :            : 
    1373   [ +  -  +  -  :       3415 :         cc.bits.css = ctrlr->opts.command_set;
                   +  - ]
    1374                 :            : 
    1375   [ +  +  +  +  :       3415 :         switch (ctrlr->opts.arb_mechanism) {
          +  -  -  +  -  
                      - ]
    1376                 :       2549 :         case SPDK_NVME_CC_AMS_RR:
    1377                 :       3382 :                 break;
    1378                 :         12 :         case SPDK_NVME_CC_AMS_WRR:
    1379   [ +  +  #  #  :         12 :                 if (SPDK_NVME_CAP_AMS_WRR & ctrlr->cap.bits.ams) {
             #  #  #  # ]
    1380                 :          6 :                         break;
    1381                 :            :                 }
    1382                 :          6 :                 return -EINVAL;
    1383                 :         12 :         case SPDK_NVME_CC_AMS_VS:
    1384   [ +  +  #  #  :         12 :                 if (SPDK_NVME_CAP_AMS_VS & ctrlr->cap.bits.ams) {
             #  #  #  # ]
    1385                 :          6 :                         break;
    1386                 :            :                 }
    1387                 :          6 :                 return -EINVAL;
    1388                 :          9 :         default:
    1389                 :          9 :                 return -EINVAL;
    1390                 :            :         }
    1391                 :            : 
    1392   [ +  -  +  -  :       3394 :         cc.bits.ams = ctrlr->opts.arb_mechanism;
                   +  - ]
    1393   [ +  -  +  -  :       3394 :         ctrlr->process_init_cc.raw = cc.raw;
                   +  - ]
    1394                 :            : 
    1395         [ -  + ]:       3394 :         if (nvme_ctrlr_set_cc_async(ctrlr, cc.raw, nvme_ctrlr_set_cc_en_done, ctrlr)) {
    1396   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "set_cc() failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1397                 :          0 :                 return -EIO;
    1398                 :            :         }
    1399                 :            : 
    1400                 :       3394 :         return 0;
    1401                 :        833 : }
    1402                 :            : 
    1403                 :            : static const char *
    1404                 :        309 : nvme_ctrlr_state_string(enum nvme_ctrlr_state state)
    1405                 :            : {
    1406   [ -  +  +  +  :        309 :         switch (state) {
          +  +  +  +  +  
          -  -  -  -  +  
          -  +  +  +  +  
          -  +  +  +  +  
          +  +  +  +  -  
          -  -  +  +  +  
          +  +  +  +  +  
          +  -  +  -  -  
          +  +  -  +  -  
          +  -  +  +  +  
                   -  - ]
    1407                 :          0 :         case NVME_CTRLR_STATE_INIT_DELAY:
    1408                 :          0 :                 return "delay init";
    1409                 :         10 :         case NVME_CTRLR_STATE_CONNECT_ADMINQ:
    1410                 :         10 :                 return "connect adminq";
    1411                 :         10 :         case NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ:
    1412                 :         10 :                 return "wait for connect adminq";
    1413                 :         10 :         case NVME_CTRLR_STATE_READ_VS:
    1414                 :         10 :                 return "read vs";
    1415                 :         10 :         case NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS:
    1416                 :         10 :                 return "read vs wait for vs";
    1417                 :         10 :         case NVME_CTRLR_STATE_READ_CAP:
    1418                 :         10 :                 return "read cap";
    1419                 :         10 :         case NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP:
    1420                 :         10 :                 return "read cap wait for cap";
    1421                 :         10 :         case NVME_CTRLR_STATE_CHECK_EN:
    1422                 :         10 :                 return "check en";
    1423                 :         10 :         case NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC:
    1424                 :         10 :                 return "check en wait for cc";
    1425                 :          0 :         case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1:
    1426                 :          0 :                 return "disable and wait for CSTS.RDY = 1";
    1427                 :          0 :         case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS:
    1428                 :          0 :                 return "disable and wait for CSTS.RDY = 1 reg";
    1429                 :          0 :         case NVME_CTRLR_STATE_SET_EN_0:
    1430                 :          0 :                 return "set CC.EN = 0";
    1431                 :          0 :         case NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC:
    1432                 :          0 :                 return "set CC.EN = 0 wait for cc";
    1433                 :         10 :         case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0:
    1434                 :         10 :                 return "disable and wait for CSTS.RDY = 0";
    1435                 :          0 :         case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS:
    1436                 :          0 :                 return "disable and wait for CSTS.RDY = 0 reg";
    1437                 :         10 :         case NVME_CTRLR_STATE_DISABLED:
    1438                 :         10 :                 return "controller is disabled";
    1439                 :         10 :         case NVME_CTRLR_STATE_ENABLE:
    1440                 :         10 :                 return "enable controller by writing CC.EN = 1";
    1441                 :         10 :         case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC:
    1442                 :         10 :                 return "enable controller by writing CC.EN = 1 reg";
    1443                 :         10 :         case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1:
    1444                 :         10 :                 return "wait for CSTS.RDY = 1";
    1445                 :          0 :         case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS:
    1446                 :          0 :                 return "wait for CSTS.RDY = 1 reg";
    1447                 :         10 :         case NVME_CTRLR_STATE_RESET_ADMIN_QUEUE:
    1448                 :         10 :                 return "reset admin queue";
    1449                 :         10 :         case NVME_CTRLR_STATE_IDENTIFY:
    1450                 :         10 :                 return "identify controller";
    1451                 :         10 :         case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY:
    1452                 :         10 :                 return "wait for identify controller";
    1453                 :         10 :         case NVME_CTRLR_STATE_CONFIGURE_AER:
    1454                 :         10 :                 return "configure AER";
    1455                 :         10 :         case NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER:
    1456                 :         10 :                 return "wait for configure aer";
    1457                 :         10 :         case NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT:
    1458                 :         10 :                 return "set keep alive timeout";
    1459                 :         10 :         case NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT:
    1460                 :         10 :                 return "wait for set keep alive timeout";
    1461                 :          6 :         case NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC:
    1462                 :          6 :                 return "identify controller iocs specific";
    1463                 :          0 :         case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC:
    1464                 :          0 :                 return "wait for identify controller iocs specific";
    1465                 :          0 :         case NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG:
    1466                 :          0 :                 return "get zns cmd and effects log page";
    1467                 :          0 :         case NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG:
    1468                 :          0 :                 return "wait for get zns cmd and effects log page";
    1469                 :          6 :         case NVME_CTRLR_STATE_SET_NUM_QUEUES:
    1470                 :          6 :                 return "set number of queues";
    1471                 :          6 :         case NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES:
    1472                 :          6 :                 return "wait for set number of queues";
    1473                 :          6 :         case NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS:
    1474                 :          6 :                 return "identify active ns";
    1475                 :          6 :         case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS:
    1476                 :          6 :                 return "wait for identify active ns";
    1477                 :          6 :         case NVME_CTRLR_STATE_IDENTIFY_NS:
    1478                 :          6 :                 return "identify ns";
    1479                 :          6 :         case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS:
    1480                 :          6 :                 return "wait for identify ns";
    1481                 :          6 :         case NVME_CTRLR_STATE_IDENTIFY_ID_DESCS:
    1482                 :          6 :                 return "identify namespace id descriptors";
    1483                 :          6 :         case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS:
    1484                 :          6 :                 return "wait for identify namespace id descriptors";
    1485                 :          6 :         case NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC:
    1486                 :          6 :                 return "identify ns iocs specific";
    1487                 :          0 :         case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC:
    1488                 :          0 :                 return "wait for identify ns iocs specific";
    1489                 :          6 :         case NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES:
    1490                 :          6 :                 return "set supported log pages";
    1491                 :          0 :         case NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES:
    1492                 :          0 :                 return "set supported INTEL log pages";
    1493                 :          0 :         case NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES:
    1494                 :          0 :                 return "wait for supported INTEL log pages";
    1495                 :          6 :         case NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES:
    1496                 :          6 :                 return "set supported features";
    1497                 :          6 :         case NVME_CTRLR_STATE_SET_HOST_FEATURE:
    1498                 :          6 :                 return "set host behavior support feature";
    1499                 :          0 :         case NVME_CTRLR_STATE_WAIT_FOR_SET_HOST_FEATURE:
    1500                 :          0 :                 return "wait for set host behavior support feature";
    1501                 :          6 :         case NVME_CTRLR_STATE_SET_DB_BUF_CFG:
    1502                 :          6 :                 return "set doorbell buffer config";
    1503                 :          0 :         case NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG:
    1504                 :          0 :                 return "wait for doorbell buffer config";
    1505                 :          6 :         case NVME_CTRLR_STATE_SET_HOST_ID:
    1506                 :          6 :                 return "set host ID";
    1507                 :          0 :         case NVME_CTRLR_STATE_WAIT_FOR_HOST_ID:
    1508                 :          0 :                 return "wait for set host ID";
    1509                 :          6 :         case NVME_CTRLR_STATE_TRANSPORT_READY:
    1510                 :          6 :                 return "transport ready";
    1511                 :         10 :         case NVME_CTRLR_STATE_READY:
    1512                 :         10 :                 return "ready";
    1513                 :          3 :         case NVME_CTRLR_STATE_ERROR:
    1514                 :          3 :                 return "error";
    1515                 :          0 :         case NVME_CTRLR_STATE_DISCONNECTED:
    1516                 :          0 :                 return "disconnected";
    1517                 :            :         }
    1518                 :          0 :         return "unknown";
    1519                 :          0 : };
    1520                 :            : 
    1521                 :            : static void
    1522                 :     212065 : _nvme_ctrlr_set_state(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state,
    1523                 :            :                       uint64_t timeout_in_ms, bool quiet)
    1524                 :            : {
    1525                 :            :         uint64_t ticks_per_ms, timeout_in_ticks, now_ticks;
    1526                 :            : 
    1527   [ +  -  +  - ]:     212065 :         ctrlr->state = state;
    1528         [ +  + ]:     212065 :         if (timeout_in_ms == NVME_TIMEOUT_KEEP_EXISTING) {
    1529   [ +  +  +  - ]:      78465 :                 if (!quiet) {
    1530   [ #  #  #  #  :          0 :                         NVME_CTRLR_DEBUGLOG(ctrlr, "setting state to %s (keeping existing timeout)\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1531                 :            :                                             nvme_ctrlr_state_string(ctrlr->state));
    1532                 :          0 :                 }
    1533                 :      78465 :                 return;
    1534                 :            :         }
    1535                 :            : 
    1536         [ +  + ]:     133600 :         if (timeout_in_ms == NVME_TIMEOUT_INFINITE) {
    1537                 :      35408 :                 goto inf;
    1538                 :            :         }
    1539                 :            : 
    1540         [ +  - ]:      98192 :         ticks_per_ms = spdk_get_ticks_hz() / 1000;
    1541   [ +  +  -  + ]:      98192 :         if (timeout_in_ms > UINT64_MAX / ticks_per_ms) {
    1542   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr,
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1543                 :            :                                   "Specified timeout would cause integer overflow. Defaulting to no timeout.\n");
    1544                 :          0 :                 goto inf;
    1545                 :            :         }
    1546                 :            : 
    1547                 :      98192 :         now_ticks = spdk_get_ticks();
    1548                 :      98192 :         timeout_in_ticks = timeout_in_ms * ticks_per_ms;
    1549         [ +  + ]:      98192 :         if (timeout_in_ticks > UINT64_MAX - now_ticks) {
    1550   [ +  -  -  +  :          3 :                 NVME_CTRLR_ERRLOG(ctrlr,
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1551                 :            :                                   "Specified timeout would cause integer overflow. Defaulting to no timeout.\n");
    1552                 :          3 :                 goto inf;
    1553                 :            :         }
    1554                 :            : 
    1555   [ +  -  +  - ]:      98189 :         ctrlr->state_timeout_tsc = timeout_in_ticks + now_ticks;
    1556   [ +  -  -  + ]:      98189 :         if (!quiet) {
    1557   [ +  +  +  +  :      98189 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "setting state to %s (timeout %" PRIu64 " ms)\n",
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1558                 :            :                                     nvme_ctrlr_state_string(ctrlr->state), timeout_in_ms);
    1559                 :      23324 :         }
    1560                 :      98189 :         return;
    1561                 :      27912 : inf:
    1562   [ +  -  -  + ]:      35411 :         if (!quiet) {
    1563   [ +  +  +  +  :      35411 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "setting state to %s (no timeout)\n",
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1564                 :            :                                     nvme_ctrlr_state_string(ctrlr->state));
    1565                 :       7499 :         }
    1566   [ +  -  +  - ]:      35411 :         ctrlr->state_timeout_tsc = NVME_TIMEOUT_INFINITE;
    1567                 :      38375 : }
    1568                 :            : 
    1569                 :            : static void
    1570                 :     133600 : nvme_ctrlr_set_state(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state,
    1571                 :            :                      uint64_t timeout_in_ms)
    1572                 :            : {
    1573                 :     133600 :         _nvme_ctrlr_set_state(ctrlr, state, timeout_in_ms, false);
    1574                 :     133600 : }
    1575                 :            : 
    1576                 :            : static void
    1577                 :      78465 : nvme_ctrlr_set_state_quiet(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state,
    1578                 :            :                            uint64_t timeout_in_ms)
    1579                 :            : {
    1580                 :      78465 :         _nvme_ctrlr_set_state(ctrlr, state, timeout_in_ms, true);
    1581                 :      78465 : }
    1582                 :            : 
    1583                 :            : static void
    1584                 :       3934 : nvme_ctrlr_free_zns_specific_data(struct spdk_nvme_ctrlr *ctrlr)
    1585                 :            : {
    1586   [ +  -  +  - ]:       3934 :         spdk_free(ctrlr->cdata_zns);
    1587   [ +  -  +  - ]:       3934 :         ctrlr->cdata_zns = NULL;
    1588                 :       3934 : }
    1589                 :            : 
    1590                 :            : static void
    1591                 :       3934 : nvme_ctrlr_free_iocs_specific_data(struct spdk_nvme_ctrlr *ctrlr)
    1592                 :            : {
    1593                 :       3934 :         nvme_ctrlr_free_zns_specific_data(ctrlr);
    1594                 :       3934 : }
    1595                 :            : 
    1596                 :            : static void
    1597                 :       3937 : nvme_ctrlr_free_doorbell_buffer(struct spdk_nvme_ctrlr *ctrlr)
    1598                 :            : {
    1599   [ +  +  +  -  :       3937 :         if (ctrlr->shadow_doorbell) {
                   -  + ]
    1600   [ #  #  #  # ]:        661 :                 spdk_free(ctrlr->shadow_doorbell);
    1601   [ #  #  #  # ]:        661 :                 ctrlr->shadow_doorbell = NULL;
    1602                 :          0 :         }
    1603                 :            : 
    1604   [ +  +  +  -  :       3937 :         if (ctrlr->eventidx) {
                   -  + ]
    1605   [ #  #  #  # ]:        661 :                 spdk_free(ctrlr->eventidx);
    1606   [ #  #  #  # ]:        661 :                 ctrlr->eventidx = NULL;
    1607                 :          0 :         }
    1608                 :       3937 : }
    1609                 :            : 
    1610                 :            : static void
    1611                 :        665 : nvme_ctrlr_set_doorbell_buffer_config_done(void *arg, const struct spdk_nvme_cpl *cpl)
    1612                 :            : {
    1613                 :        665 :         struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg;
    1614                 :            : 
    1615   [ +  -  -  +  :        665 :         if (spdk_nvme_cpl_is_error(cpl)) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1616   [ #  #  #  #  :          0 :                 NVME_CTRLR_WARNLOG(ctrlr, "Doorbell buffer config failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1617                 :          0 :         } else {
    1618   [ -  +  -  +  :        665 :                 NVME_CTRLR_INFOLOG(ctrlr, "Doorbell buffer config enabled\n");
          -  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    1619                 :            :         }
    1620                 :        665 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID,
    1621   [ #  #  #  #  :        665 :                              ctrlr->opts.admin_timeout_ms);
                   #  # ]
    1622                 :        665 : }
    1623                 :            : 
    1624                 :            : static int
    1625                 :       3245 : nvme_ctrlr_set_doorbell_buffer_config(struct spdk_nvme_ctrlr *ctrlr)
    1626                 :            : {
    1627                 :       3245 :         int rc = 0;
    1628                 :        485 :         uint64_t prp1, prp2, len;
    1629                 :            : 
    1630   [ +  +  +  -  :       3245 :         if (!ctrlr->cdata.oacs.doorbell_buffer_config) {
             +  -  -  + ]
    1631                 :       3413 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID,
    1632   [ +  -  +  -  :       2580 :                                      ctrlr->opts.admin_timeout_ms);
                   +  - ]
    1633                 :       2580 :                 return 0;
    1634                 :            :         }
    1635                 :            : 
    1636   [ -  +  #  #  :        665 :         if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) {
             #  #  #  # ]
    1637                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID,
    1638   [ #  #  #  #  :          0 :                                      ctrlr->opts.admin_timeout_ms);
                   #  # ]
    1639                 :          0 :                 return 0;
    1640                 :            :         }
    1641                 :            : 
    1642                 :            :         /* only 1 page size for doorbell buffer */
    1643   [ #  #  #  #  :        665 :         ctrlr->shadow_doorbell = spdk_zmalloc(ctrlr->page_size, ctrlr->page_size,
          #  #  #  #  #  
                #  #  # ]
    1644                 :            :                                               NULL, SPDK_ENV_LCORE_ID_ANY,
    1645                 :            :                                               SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE);
    1646   [ -  +  #  #  :        665 :         if (ctrlr->shadow_doorbell == NULL) {
                   #  # ]
    1647                 :          0 :                 rc = -ENOMEM;
    1648                 :          0 :                 goto error;
    1649                 :            :         }
    1650                 :            : 
    1651   [ #  #  #  # ]:        665 :         len = ctrlr->page_size;
    1652   [ #  #  #  # ]:        665 :         prp1 = spdk_vtophys(ctrlr->shadow_doorbell, &len);
    1653   [ +  -  -  +  :        665 :         if (prp1 == SPDK_VTOPHYS_ERROR || len != ctrlr->page_size) {
             #  #  #  # ]
    1654                 :          0 :                 rc = -EFAULT;
    1655                 :          0 :                 goto error;
    1656                 :            :         }
    1657                 :            : 
    1658   [ #  #  #  #  :        665 :         ctrlr->eventidx = spdk_zmalloc(ctrlr->page_size, ctrlr->page_size,
          #  #  #  #  #  
                #  #  # ]
    1659                 :            :                                        NULL, SPDK_ENV_LCORE_ID_ANY,
    1660                 :            :                                        SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE);
    1661   [ -  +  #  #  :        665 :         if (ctrlr->eventidx == NULL) {
                   #  # ]
    1662                 :          0 :                 rc = -ENOMEM;
    1663                 :          0 :                 goto error;
    1664                 :            :         }
    1665                 :            : 
    1666   [ #  #  #  # ]:        665 :         len = ctrlr->page_size;
    1667   [ #  #  #  # ]:        665 :         prp2 = spdk_vtophys(ctrlr->eventidx, &len);
    1668   [ +  -  -  +  :        665 :         if (prp2 == SPDK_VTOPHYS_ERROR || len != ctrlr->page_size) {
             #  #  #  # ]
    1669                 :          0 :                 rc = -EFAULT;
    1670                 :          0 :                 goto error;
    1671                 :            :         }
    1672                 :            : 
    1673                 :        665 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG,
    1674   [ #  #  #  #  :        665 :                              ctrlr->opts.admin_timeout_ms);
                   #  # ]
    1675                 :            : 
    1676                 :        665 :         rc = nvme_ctrlr_cmd_doorbell_buffer_config(ctrlr, prp1, prp2,
    1677                 :          0 :                         nvme_ctrlr_set_doorbell_buffer_config_done, ctrlr);
    1678         [ -  + ]:        665 :         if (rc != 0) {
    1679                 :          0 :                 goto error;
    1680                 :            :         }
    1681                 :            : 
    1682                 :        665 :         return 0;
    1683                 :            : 
    1684                 :          0 : error:
    1685                 :          0 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    1686                 :          0 :         nvme_ctrlr_free_doorbell_buffer(ctrlr);
    1687                 :          0 :         return rc;
    1688                 :        833 : }
    1689                 :            : 
    1690                 :            : void
    1691                 :       6460 : nvme_ctrlr_abort_queued_aborts(struct spdk_nvme_ctrlr *ctrlr)
    1692                 :            : {
    1693                 :            :         struct nvme_request     *req, *tmp;
    1694                 :       6460 :         struct spdk_nvme_cpl    cpl = {};
    1695                 :            : 
    1696   [ +  -  +  - ]:       6460 :         cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION;
    1697   [ +  -  +  - ]:       6460 :         cpl.status.sct = SPDK_NVME_SCT_GENERIC;
    1698                 :            : 
    1699   [ +  +  +  -  :       6460 :         STAILQ_FOREACH_SAFE(req, &ctrlr->queued_aborts, stailq, tmp) {
          +  -  +  -  #  
          #  #  #  #  #  
                   +  - ]
    1700   [ #  #  #  #  :          0 :                 STAILQ_REMOVE_HEAD(&ctrlr->queued_aborts, stailq);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    1701         [ #  # ]:          0 :                 ctrlr->outstanding_aborts++;
    1702                 :            : 
    1703   [ #  #  #  #  :          0 :                 nvme_complete_request(req->cb_fn, req->cb_arg, req->qpair, req, &cpl);
          #  #  #  #  #  
                #  #  # ]
    1704                 :          0 :         }
    1705                 :       6460 : }
    1706                 :            : 
    1707                 :            : static int
    1708                 :        518 : nvme_ctrlr_disconnect(struct spdk_nvme_ctrlr *ctrlr)
    1709                 :            : {
    1710   [ +  +  +  +  :        518 :         if (ctrlr->is_resetting || ctrlr->is_removed) {
          -  +  -  +  #  
          #  #  #  #  #  
                   #  # ]
    1711                 :            :                 /*
    1712                 :            :                  * Controller is already resetting or has been removed. Return
    1713                 :            :                  *  immediately since there is no need to kick off another
    1714                 :            :                  *  reset in these cases.
    1715                 :            :                  */
    1716   [ +  +  +  -  :          3 :                 return ctrlr->is_resetting ? -EBUSY : -ENXIO;
                   #  # ]
    1717                 :            :         }
    1718                 :            : 
    1719   [ #  #  #  # ]:        515 :         ctrlr->is_resetting = true;
    1720   [ #  #  #  # ]:        515 :         ctrlr->is_failed = false;
    1721   [ #  #  #  # ]:        515 :         ctrlr->is_disconnecting = true;
    1722   [ #  #  #  # ]:        515 :         ctrlr->prepare_for_reset = true;
    1723                 :            : 
    1724   [ +  +  +  +  :        515 :         NVME_CTRLR_NOTICELOG(ctrlr, "resetting controller\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1725                 :            : 
    1726                 :            :         /* Disable keep-alive, it'll be re-enabled as part of the init process */
    1727   [ #  #  #  # ]:        515 :         ctrlr->keep_alive_interval_ticks = 0;
    1728                 :            : 
    1729                 :            :         /* Abort all of the queued abort requests */
    1730                 :        515 :         nvme_ctrlr_abort_queued_aborts(ctrlr);
    1731                 :            : 
    1732   [ #  #  #  # ]:        515 :         nvme_transport_admin_qpair_abort_aers(ctrlr->adminq);
    1733                 :            : 
    1734   [ #  #  #  #  :        515 :         ctrlr->adminq->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL;
                   #  # ]
    1735   [ #  #  #  # ]:        515 :         nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq);
    1736                 :            : 
    1737                 :        515 :         return 0;
    1738                 :          0 : }
    1739                 :            : 
    1740                 :            : static void
    1741                 :        515 : nvme_ctrlr_disconnect_done(struct spdk_nvme_ctrlr *ctrlr)
    1742                 :            : {
    1743   [ -  +  -  +  :        515 :         assert(ctrlr->is_failed == false);
          #  #  #  #  #  
                      # ]
    1744   [ #  #  #  # ]:        515 :         ctrlr->is_disconnecting = false;
    1745                 :            : 
    1746                 :            :         /* Doorbell buffer config is invalid during reset */
    1747                 :        515 :         nvme_ctrlr_free_doorbell_buffer(ctrlr);
    1748                 :            : 
    1749                 :            :         /* I/O Command Set Specific Identify Controller data is invalidated during reset */
    1750                 :        515 :         nvme_ctrlr_free_iocs_specific_data(ctrlr);
    1751                 :            : 
    1752         [ #  # ]:        515 :         spdk_bit_array_free(&ctrlr->free_io_qids);
    1753                 :            : 
    1754                 :            :         /* Set the state back to DISCONNECTED to cause a full hardware reset. */
    1755                 :        515 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISCONNECTED, NVME_TIMEOUT_INFINITE);
    1756                 :        515 : }
    1757                 :            : 
    1758                 :            : int
    1759                 :        461 : spdk_nvme_ctrlr_disconnect(struct spdk_nvme_ctrlr *ctrlr)
    1760                 :            : {
    1761                 :            :         int rc;
    1762                 :            : 
    1763                 :        461 :         nvme_ctrlr_lock(ctrlr);
    1764                 :        461 :         rc = nvme_ctrlr_disconnect(ctrlr);
    1765                 :        461 :         nvme_ctrlr_unlock(ctrlr);
    1766                 :            : 
    1767                 :        461 :         return rc;
    1768                 :            : }
    1769                 :            : 
    1770                 :            : void
    1771                 :        515 : spdk_nvme_ctrlr_reconnect_async(struct spdk_nvme_ctrlr *ctrlr)
    1772                 :            : {
    1773                 :        515 :         nvme_ctrlr_lock(ctrlr);
    1774                 :            : 
    1775   [ #  #  #  # ]:        515 :         ctrlr->prepare_for_reset = false;
    1776                 :            : 
    1777                 :            :         /* Set the state back to INIT to cause a full hardware reset. */
    1778                 :        515 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE);
    1779                 :            : 
    1780                 :            :         /* Return without releasing ctrlr_lock. ctrlr_lock will be released when
    1781                 :            :          * spdk_nvme_ctrlr_reset_poll_async() returns 0.
    1782                 :            :          */
    1783                 :        515 : }
    1784                 :            : 
    1785                 :            : int
    1786                 :         29 : nvme_ctrlr_reinitialize_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
    1787                 :            : {
    1788                 :            :         bool async;
    1789                 :            :         int rc;
    1790                 :            : 
    1791   [ +  -  +  -  :         58 :         if (nvme_ctrlr_get_current_process(ctrlr) != qpair->active_proc ||
                   #  # ]
    1792         [ -  + ]:         58 :             spdk_nvme_ctrlr_is_fabrics(ctrlr) || nvme_qpair_is_admin_queue(qpair)) {
    1793         [ #  # ]:          0 :                 assert(false);
    1794                 :            :                 return -EINVAL;
    1795                 :            :         }
    1796                 :            : 
    1797                 :            :         /* Force a synchronous connect. */
    1798         [ #  # ]:         29 :         async = qpair->async;
    1799         [ #  # ]:         29 :         qpair->async = false;
    1800                 :         29 :         rc = nvme_transport_ctrlr_connect_qpair(ctrlr, qpair);
    1801   [ #  #  #  # ]:         29 :         qpair->async = async;
    1802                 :            : 
    1803         [ -  + ]:         29 :         if (rc != 0) {
    1804         [ #  # ]:          0 :                 qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL;
    1805                 :          0 :         }
    1806                 :            : 
    1807                 :         29 :         return rc;
    1808                 :            : }
    1809                 :            : 
    1810                 :            : /**
    1811                 :            :  * This function will be called when the controller is being reinitialized.
    1812                 :            :  * Note: the ctrlr_lock must be held when calling this function.
    1813                 :            :  */
    1814                 :            : int
    1815                 :     349232 : spdk_nvme_ctrlr_reconnect_poll_async(struct spdk_nvme_ctrlr *ctrlr)
    1816                 :            : {
    1817                 :            :         struct spdk_nvme_ns *ns, *tmp_ns;
    1818                 :            :         struct spdk_nvme_qpair  *qpair;
    1819                 :     349232 :         int rc = 0, rc_tmp = 0;
    1820                 :            : 
    1821         [ +  + ]:     349232 :         if (nvme_ctrlr_process_init(ctrlr) != 0) {
    1822   [ +  +  +  -  :        386 :                 NVME_CTRLR_ERRLOG(ctrlr, "controller reinitialization failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1823                 :        386 :                 rc = -1;
    1824                 :          0 :         }
    1825   [ +  +  +  +  :     349232 :         if (ctrlr->state != NVME_CTRLR_STATE_READY && rc != -1) {
             #  #  #  # ]
    1826                 :     348717 :                 return -EAGAIN;
    1827                 :            :         }
    1828                 :            : 
    1829                 :            :         /*
    1830                 :            :          * For non-fabrics controllers, the memory locations of the transport qpair
    1831                 :            :          * don't change when the controller is reset. They simply need to be
    1832                 :            :          * re-enabled with admin commands to the controller. For fabric
    1833                 :            :          * controllers we need to disconnect and reconnect the qpair on its
    1834                 :            :          * own thread outside of the context of the reset.
    1835                 :            :          */
    1836   [ +  +  +  + ]:        515 :         if (rc == 0 && !spdk_nvme_ctrlr_is_fabrics(ctrlr)) {
    1837                 :            :                 /* Reinitialize qpairs */
    1838   [ +  +  #  #  :        118 :                 TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) {
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1839                 :            :                         /* Always clear the qid bit here, even for a foreign qpair. We need
    1840                 :            :                          * to make sure another process doesn't get the chance to grab that
    1841                 :            :                          * qid.
    1842                 :            :                          */
    1843   [ -  +  #  #  :         29 :                         assert(spdk_bit_array_get(ctrlr->free_io_qids, qpair->id));
          #  #  #  #  #  
                #  #  # ]
    1844   [ #  #  #  #  :         29 :                         spdk_bit_array_clear(ctrlr->free_io_qids, qpair->id);
             #  #  #  # ]
    1845   [ +  +  #  #  :         29 :                         if (nvme_ctrlr_get_current_process(ctrlr) != qpair->active_proc) {
                   #  # ]
    1846                 :            :                                 /*
    1847                 :            :                                  * We cannot reinitialize a foreign qpair. The qpair's owning
    1848                 :            :                                  * process will take care of it. Set failure reason to FAILURE_RESET
    1849                 :            :                                  * to ensure that happens.
    1850                 :            :                                  */
    1851         [ #  # ]:         23 :                                 qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_RESET;
    1852                 :         23 :                                 continue;
    1853                 :            :                         }
    1854                 :          6 :                         rc_tmp = nvme_ctrlr_reinitialize_io_qpair(ctrlr, qpair);
    1855         [ -  + ]:          6 :                         if (rc_tmp != 0) {
    1856                 :          0 :                                 rc = rc_tmp;
    1857                 :          0 :                         }
    1858                 :          0 :                 }
    1859                 :          0 :         }
    1860                 :            : 
    1861                 :            :         /*
    1862                 :            :          * Take this opportunity to remove inactive namespaces. During a reset namespace
    1863                 :            :          * handles can be invalidated.
    1864                 :            :          */
    1865   [ +  +  +  -  :       1095 :         RB_FOREACH_SAFE(ns, nvme_ns_tree, &ctrlr->ns, tmp_ns) {
                   #  # ]
    1866   [ +  +  +  +  :        580 :                 if (!ns->active) {
             #  #  #  # ]
    1867         [ #  # ]:          4 :                         RB_REMOVE(nvme_ns_tree, &ctrlr->ns, ns);
    1868                 :          4 :                         spdk_free(ns);
    1869                 :          0 :                 }
    1870                 :          0 :         }
    1871                 :            : 
    1872         [ +  + ]:        515 :         if (rc) {
    1873                 :        386 :                 nvme_ctrlr_fail(ctrlr, false);
    1874                 :          0 :         }
    1875   [ #  #  #  # ]:        515 :         ctrlr->is_resetting = false;
    1876                 :            : 
    1877                 :        515 :         nvme_ctrlr_unlock(ctrlr);
    1878                 :            : 
    1879   [ +  +  #  #  :        515 :         if (!ctrlr->cdata.oaes.ns_attribute_notices) {
             #  #  #  # ]
    1880                 :            :                 /*
    1881                 :            :                  * If controller doesn't support ns_attribute_notices and
    1882                 :            :                  * namespace attributes change (e.g. number of namespaces)
    1883                 :            :                  * we need to update system handling device reset.
    1884                 :            :                  */
    1885                 :         15 :                 nvme_io_msg_ctrlr_update(ctrlr);
    1886                 :          0 :         }
    1887                 :            : 
    1888                 :        515 :         return rc;
    1889                 :          0 : }
    1890                 :            : 
    1891                 :            : /*
    1892                 :            :  * For PCIe transport, spdk_nvme_ctrlr_disconnect() will do a Controller Level Reset
    1893                 :            :  * (Change CC.EN from 1 to 0) as a operation to disconnect the admin qpair.
    1894                 :            :  * The following two functions are added to do a Controller Level Reset. They have
    1895                 :            :  * to be called under the nvme controller's lock.
    1896                 :            :  */
    1897                 :            : void
    1898                 :         89 : nvme_ctrlr_disable(struct spdk_nvme_ctrlr *ctrlr)
    1899                 :            : {
    1900   [ -  +  -  +  :         89 :         assert(ctrlr->is_disconnecting == true);
          #  #  #  #  #  
                      # ]
    1901                 :            : 
    1902                 :         89 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CHECK_EN, NVME_TIMEOUT_INFINITE);
    1903                 :         89 : }
    1904                 :            : 
    1905                 :            : int
    1906                 :       1232 : nvme_ctrlr_disable_poll(struct spdk_nvme_ctrlr *ctrlr)
    1907                 :            : {
    1908                 :       1232 :         int rc = 0;
    1909                 :            : 
    1910         [ -  + ]:       1232 :         if (nvme_ctrlr_process_init(ctrlr) != 0) {
    1911   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "failed to disable controller\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1912                 :          0 :                 rc = -1;
    1913                 :          0 :         }
    1914                 :            : 
    1915   [ +  +  +  -  :       1232 :         if (ctrlr->state != NVME_CTRLR_STATE_DISABLED && rc != -1) {
             #  #  #  # ]
    1916                 :       1143 :                 return -EAGAIN;
    1917                 :            :         }
    1918                 :            : 
    1919                 :         89 :         return rc;
    1920                 :          0 : }
    1921                 :            : 
    1922                 :            : static void
    1923                 :         54 : nvme_ctrlr_fail_io_qpairs(struct spdk_nvme_ctrlr *ctrlr)
    1924                 :            : {
    1925                 :            :         struct spdk_nvme_qpair  *qpair;
    1926                 :            : 
    1927   [ +  +  #  #  :         88 :         TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) {
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1928         [ #  # ]:         34 :                 qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL;
    1929                 :          0 :         }
    1930                 :         54 : }
    1931                 :            : 
    1932                 :            : int
    1933                 :         57 : spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr)
    1934                 :            : {
    1935                 :            :         int rc;
    1936                 :            : 
    1937                 :         57 :         nvme_ctrlr_lock(ctrlr);
    1938                 :            : 
    1939                 :         57 :         rc = nvme_ctrlr_disconnect(ctrlr);
    1940         [ +  + ]:         57 :         if (rc == 0) {
    1941                 :         54 :                 nvme_ctrlr_fail_io_qpairs(ctrlr);
    1942                 :          0 :         }
    1943                 :            : 
    1944                 :         57 :         nvme_ctrlr_unlock(ctrlr);
    1945                 :            : 
    1946         [ +  + ]:         57 :         if (rc != 0) {
    1947         [ +  - ]:          3 :                 if (rc == -EBUSY) {
    1948                 :          3 :                         rc = 0;
    1949                 :          0 :                 }
    1950                 :          3 :                 return rc;
    1951                 :            :         }
    1952                 :            : 
    1953                 :          0 :         while (1) {
    1954                 :        384 :                 rc = spdk_nvme_ctrlr_process_admin_completions(ctrlr);
    1955         [ +  + ]:        384 :                 if (rc == -ENXIO) {
    1956                 :         54 :                         break;
    1957                 :            :                 }
    1958                 :            :         }
    1959                 :            : 
    1960                 :         54 :         spdk_nvme_ctrlr_reconnect_async(ctrlr);
    1961                 :            : 
    1962                 :          0 :         while (true) {
    1963                 :      31740 :                 rc = spdk_nvme_ctrlr_reconnect_poll_async(ctrlr);
    1964         [ +  + ]:      31740 :                 if (rc != -EAGAIN) {
    1965                 :         54 :                         break;
    1966                 :            :                 }
    1967                 :            :         }
    1968                 :            : 
    1969                 :         54 :         return rc;
    1970                 :          0 : }
    1971                 :            : 
    1972                 :            : int
    1973                 :          0 : spdk_nvme_ctrlr_reset_subsystem(struct spdk_nvme_ctrlr *ctrlr)
    1974                 :            : {
    1975                 :            :         union spdk_nvme_cap_register cap;
    1976                 :          0 :         int rc = 0;
    1977                 :            : 
    1978                 :          0 :         cap = spdk_nvme_ctrlr_get_regs_cap(ctrlr);
    1979         [ #  # ]:          0 :         if (cap.bits.nssrs == 0) {
    1980   [ #  #  #  #  :          0 :                 NVME_CTRLR_WARNLOG(ctrlr, "subsystem reset is not supported\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1981                 :          0 :                 return -ENOTSUP;
    1982                 :            :         }
    1983                 :            : 
    1984   [ #  #  #  #  :          0 :         NVME_CTRLR_NOTICELOG(ctrlr, "resetting subsystem\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1985                 :          0 :         nvme_ctrlr_lock(ctrlr);
    1986   [ #  #  #  # ]:          0 :         ctrlr->is_resetting = true;
    1987                 :          0 :         rc = nvme_ctrlr_set_nssr(ctrlr, SPDK_NVME_NSSR_VALUE);
    1988   [ #  #  #  # ]:          0 :         ctrlr->is_resetting = false;
    1989                 :            : 
    1990                 :          0 :         nvme_ctrlr_unlock(ctrlr);
    1991                 :            :         /*
    1992                 :            :          * No more cleanup at this point like in the ctrlr reset. A subsystem reset will cause
    1993                 :            :          * a hot remove for PCIe transport. The hot remove handling does all the necessary ctrlr cleanup.
    1994                 :            :          */
    1995                 :          0 :         return rc;
    1996                 :          0 : }
    1997                 :            : 
    1998                 :            : int
    1999                 :         29 : spdk_nvme_ctrlr_set_trid(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_transport_id *trid)
    2000                 :            : {
    2001                 :         29 :         int rc = 0;
    2002                 :            : 
    2003                 :         29 :         nvme_ctrlr_lock(ctrlr);
    2004                 :            : 
    2005   [ +  +  +  +  :         29 :         if (ctrlr->is_failed == false) {
             #  #  #  # ]
    2006                 :          3 :                 rc = -EPERM;
    2007                 :          3 :                 goto out;
    2008                 :            :         }
    2009                 :            : 
    2010   [ +  +  #  #  :         26 :         if (trid->trtype != ctrlr->trid.trtype) {
          #  #  #  #  #  
                #  #  # ]
    2011                 :          3 :                 rc = -EINVAL;
    2012                 :          3 :                 goto out;
    2013                 :            :         }
    2014                 :            : 
    2015   [ +  +  -  +  :         23 :         if (strncmp(trid->subnqn, ctrlr->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) {
          +  +  #  #  #  
                #  #  # ]
    2016                 :          3 :                 rc = -EINVAL;
    2017                 :          3 :                 goto out;
    2018                 :            :         }
    2019                 :            : 
    2020         [ #  # ]:         20 :         ctrlr->trid = *trid;
    2021                 :            : 
    2022                 :         29 : out:
    2023                 :         29 :         nvme_ctrlr_unlock(ctrlr);
    2024                 :         29 :         return rc;
    2025                 :            : }
    2026                 :            : 
    2027                 :            : void
    2028                 :       1510 : spdk_nvme_ctrlr_set_remove_cb(struct spdk_nvme_ctrlr *ctrlr,
    2029                 :            :                               spdk_nvme_remove_cb remove_cb, void *remove_ctx)
    2030                 :            : {
    2031         [ -  + ]:       1510 :         if (!spdk_process_is_primary()) {
    2032                 :          0 :                 return;
    2033                 :            :         }
    2034                 :            : 
    2035                 :       1510 :         nvme_ctrlr_lock(ctrlr);
    2036   [ -  +  -  + ]:       1510 :         ctrlr->remove_cb = remove_cb;
    2037   [ -  +  -  + ]:       1510 :         ctrlr->cb_ctx = remove_ctx;
    2038                 :       1510 :         nvme_ctrlr_unlock(ctrlr);
    2039                 :          1 : }
    2040                 :            : 
    2041                 :            : int
    2042                 :         28 : spdk_nvme_ctrlr_set_keys(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ctrlr_key_opts *opts)
    2043                 :            : {
    2044                 :         28 :         nvme_ctrlr_lock(ctrlr);
    2045   [ +  -  +  +  :         32 :         if (SPDK_GET_FIELD(opts, dhchap_key, ctrlr->opts.dhchap_key) == NULL &&
          -  +  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    2046   [ +  -  #  #  :          4 :             SPDK_GET_FIELD(opts, dhchap_ctrlr_key, ctrlr->opts.dhchap_ctrlr_key) != NULL) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    2047   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "DH-HMAC-CHAP controller key requires host key to be set\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    2048                 :          0 :                 nvme_ctrlr_unlock(ctrlr);
    2049                 :          0 :                 return -EINVAL;
    2050                 :            :         }
    2051                 :            : 
    2052   [ #  #  #  #  :         28 :         ctrlr->opts.dhchap_key =
                   #  # ]
    2053   [ +  -  #  #  :         28 :                 SPDK_GET_FIELD(opts, dhchap_key, ctrlr->opts.dhchap_key);
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    2054   [ #  #  #  #  :         28 :         ctrlr->opts.dhchap_ctrlr_key =
                   #  # ]
    2055   [ +  -  #  #  :         28 :                 SPDK_GET_FIELD(opts, dhchap_ctrlr_key, ctrlr->opts.dhchap_ctrlr_key);
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    2056                 :         28 :         nvme_ctrlr_unlock(ctrlr);
    2057                 :            : 
    2058                 :         28 :         return 0;
    2059                 :          0 : }
    2060                 :            : 
    2061                 :            : static void
    2062                 :       3400 : nvme_ctrlr_identify_done(void *arg, const struct spdk_nvme_cpl *cpl)
    2063                 :            : {
    2064                 :       3400 :         struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg;
    2065                 :            : 
    2066   [ +  -  +  +  :       3400 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   -  + ]
    2067   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "nvme_identify_controller failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    2068                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2069                 :          0 :                 return;
    2070                 :            :         }
    2071                 :            : 
    2072                 :            :         /*
    2073                 :            :          * Use MDTS to ensure our default max_xfer_size doesn't exceed what the
    2074                 :            :          *  controller supports.
    2075                 :            :          */
    2076   [ +  -  +  - ]:       3400 :         ctrlr->max_xfer_size = nvme_transport_ctrlr_get_max_xfer_size(ctrlr);
    2077   [ +  +  +  +  :       3400 :         NVME_CTRLR_DEBUGLOG(ctrlr, "transport max_xfer_size %u\n", ctrlr->max_xfer_size);
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    2078   [ +  +  +  -  :       3400 :         if (ctrlr->cdata.mdts > 0) {
             +  -  -  + ]
    2079   [ +  +  +  +  :       3102 :                 ctrlr->max_xfer_size = spdk_min(ctrlr->max_xfer_size,
          +  +  +  -  +  
          -  +  -  +  -  
          -  +  +  -  -  
          +  #  #  #  #  
          +  -  +  -  +  
          -  +  -  +  -  
          -  +  +  -  +  
                -  +  - ]
    2080                 :            :                                                 ctrlr->min_page_size * (1 << ctrlr->cdata.mdts));
    2081   [ +  +  +  +  :       3102 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "MDTS max_xfer_size %u\n", ctrlr->max_xfer_size);
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    2082                 :        833 :         }
    2083                 :            : 
    2084   [ +  +  +  +  :       3400 :         NVME_CTRLR_DEBUGLOG(ctrlr, "CNTLID 0x%04" PRIx16 "\n", ctrlr->cdata.cntlid);
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    2085   [ +  +  +  -  :       3400 :         if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
             +  -  +  + ]
    2086   [ +  -  +  -  :        795 :                 ctrlr->cntlid = ctrlr->cdata.cntlid;
          +  -  +  -  +  
                      - ]
    2087                 :          2 :         } else {
    2088                 :            :                 /*
    2089                 :            :                  * Fabrics controllers should already have CNTLID from the Connect command.
    2090                 :            :                  *
    2091                 :            :                  * If CNTLID from Connect doesn't match CNTLID in the Identify Controller data,
    2092                 :            :                  * trust the one from Connect.
    2093                 :            :                  */
    2094   [ +  +  +  -  :       2605 :                 if (ctrlr->cntlid != ctrlr->cdata.cntlid) {
          +  -  +  -  +  
                -  +  + ]
    2095   [ +  +  +  +  :         42 :                         NVME_CTRLR_DEBUGLOG(ctrlr, "Identify CNTLID 0x%04" PRIx16 " != Connect CNTLID 0x%04" PRIx16 "\n",
          +  -  -  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    2096                 :            :                                             ctrlr->cdata.cntlid, ctrlr->cntlid);
    2097                 :          7 :                 }
    2098                 :            :         }
    2099                 :            : 
    2100   [ +  +  +  -  :       3400 :         if (ctrlr->cdata.sgls.supported && !(ctrlr->quirks & NVME_QUIRK_NOT_USE_SGL)) {
          +  -  +  +  +  
             -  +  -  -  
                      + ]
    2101   [ +  +  +  -  :       3223 :                 assert(ctrlr->cdata.sgls.supported != 0x3);
          +  -  +  -  #  
                      # ]
    2102   [ +  -  +  - ]:       3223 :                 ctrlr->flags |= SPDK_NVME_CTRLR_SGL_SUPPORTED;
    2103   [ +  +  +  -  :       3223 :                 if (ctrlr->cdata.sgls.supported == 0x2) {
             +  -  +  + ]
    2104   [ +  -  +  - ]:         43 :                         ctrlr->flags |= SPDK_NVME_CTRLR_SGL_REQUIRES_DWORD_ALIGNMENT;
    2105                 :          7 :                 }
    2106                 :            : 
    2107   [ +  -  +  - ]:       3223 :                 ctrlr->max_sges = nvme_transport_ctrlr_get_max_sges(ctrlr);
    2108   [ +  +  +  +  :       3223 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "transport max_sges %u\n", ctrlr->max_sges);
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    2109                 :        831 :         }
    2110                 :            : 
    2111   [ +  +  +  -  :       3400 :         if (ctrlr->cdata.sgls.metadata_address && !(ctrlr->quirks & NVME_QUIRK_NOT_USE_SGL)) {
          +  -  -  +  #  
             #  #  #  #  
                      # ]
    2112   [ #  #  #  # ]:          1 :                 ctrlr->flags |= SPDK_NVME_CTRLR_MPTR_SGL_SUPPORTED;
    2113                 :          0 :         }
    2114                 :            : 
    2115   [ +  +  +  -  :       3400 :         if (ctrlr->cdata.oacs.security && !(ctrlr->quirks & NVME_QUIRK_OACS_SECURITY)) {
          +  -  -  +  #  
             #  #  #  #  
                      # ]
    2116   [ #  #  #  # ]:         44 :                 ctrlr->flags |= SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED;
    2117                 :          0 :         }
    2118                 :            : 
    2119   [ +  +  +  -  :       3400 :         if (ctrlr->cdata.oacs.directives) {
             +  -  +  - ]
    2120   [ #  #  #  # ]:        663 :                 ctrlr->flags |= SPDK_NVME_CTRLR_DIRECTIVES_SUPPORTED;
    2121                 :          0 :         }
    2122                 :            : 
    2123   [ +  +  +  +  :       3400 :         NVME_CTRLR_DEBUGLOG(ctrlr, "fuses compare and write: %d\n",
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    2124                 :            :                             ctrlr->cdata.fuses.compare_and_write);
    2125   [ +  +  +  -  :       3400 :         if (ctrlr->cdata.fuses.compare_and_write) {
             +  -  +  + ]
    2126   [ +  -  +  - ]:       2231 :                 ctrlr->flags |= SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED;
    2127                 :        831 :         }
    2128                 :            : 
    2129                 :       4233 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CONFIGURE_AER,
    2130   [ +  -  +  -  :       3400 :                              ctrlr->opts.admin_timeout_ms);
                   +  - ]
    2131                 :        833 : }
    2132                 :            : 
    2133                 :            : static int
    2134                 :       3400 : nvme_ctrlr_identify(struct spdk_nvme_ctrlr *ctrlr)
    2135                 :            : {
    2136                 :            :         int     rc;
    2137                 :            : 
    2138                 :       4233 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY,
    2139   [ +  -  +  -  :       3400 :                              ctrlr->opts.admin_timeout_ms);
                   +  - ]
    2140                 :            : 
    2141                 :       3400 :         rc = nvme_ctrlr_cmd_identify(ctrlr, SPDK_NVME_IDENTIFY_CTRLR, 0, 0, 0,
    2142         [ +  - ]:       3400 :                                      &ctrlr->cdata, sizeof(ctrlr->cdata),
    2143                 :        833 :                                      nvme_ctrlr_identify_done, ctrlr);
    2144         [ -  + ]:       3400 :         if (rc != 0) {
    2145                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2146                 :          0 :                 return rc;
    2147                 :            :         }
    2148                 :            : 
    2149                 :       3400 :         return 0;
    2150                 :        833 : }
    2151                 :            : 
    2152                 :            : static void
    2153                 :        987 : nvme_ctrlr_get_zns_cmd_and_effects_log_done(void *arg, const struct spdk_nvme_cpl *cpl)
    2154                 :            : {
    2155                 :            :         struct spdk_nvme_cmds_and_effect_log_page *log_page;
    2156                 :        987 :         struct spdk_nvme_ctrlr *ctrlr = arg;
    2157                 :            : 
    2158   [ +  -  -  +  :        987 :         if (spdk_nvme_cpl_is_error(cpl)) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    2159   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_get_zns_cmd_and_effects_log failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    2160   [ #  #  #  # ]:          0 :                 spdk_free(ctrlr->tmp_ptr);
    2161   [ #  #  #  # ]:          0 :                 ctrlr->tmp_ptr = NULL;
    2162                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2163                 :          0 :                 return;
    2164                 :            :         }
    2165                 :            : 
    2166   [ #  #  #  # ]:        987 :         log_page = ctrlr->tmp_ptr;
    2167                 :            : 
    2168   [ +  -  #  #  :        987 :         if (log_page->io_cmds_supported[SPDK_NVME_OPC_ZONE_APPEND].csupp) {
          #  #  #  #  #  
                      # ]
    2169   [ #  #  #  # ]:        987 :                 ctrlr->flags |= SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED;
    2170                 :          0 :         }
    2171   [ #  #  #  # ]:        987 :         spdk_free(ctrlr->tmp_ptr);
    2172   [ #  #  #  # ]:        987 :         ctrlr->tmp_ptr = NULL;
    2173                 :            : 
    2174   [ #  #  #  #  :        987 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_NUM_QUEUES, ctrlr->opts.admin_timeout_ms);
                   #  # ]
    2175                 :          0 : }
    2176                 :            : 
    2177                 :            : static int
    2178                 :        987 : nvme_ctrlr_get_zns_cmd_and_effects_log(struct spdk_nvme_ctrlr *ctrlr)
    2179                 :            : {
    2180                 :            :         int rc;
    2181                 :            : 
    2182   [ -  +  #  #  :        987 :         assert(!ctrlr->tmp_ptr);
             #  #  #  # ]
    2183   [ #  #  #  # ]:        987 :         ctrlr->tmp_ptr = spdk_zmalloc(sizeof(struct spdk_nvme_cmds_and_effect_log_page), 64, NULL,
    2184                 :            :                                       SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_SHARE | SPDK_MALLOC_DMA);
    2185   [ -  +  #  #  :        987 :         if (!ctrlr->tmp_ptr) {
                   #  # ]
    2186                 :          0 :                 rc = -ENOMEM;
    2187                 :          0 :                 goto error;
    2188                 :            :         }
    2189                 :            : 
    2190                 :        987 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG,
    2191   [ #  #  #  #  :        987 :                              ctrlr->opts.admin_timeout_ms);
                   #  # ]
    2192                 :            : 
    2193                 :        987 :         rc = spdk_nvme_ctrlr_cmd_get_log_page_ext(ctrlr, SPDK_NVME_LOG_COMMAND_EFFECTS_LOG,
    2194   [ #  #  #  # ]:          0 :                         0, ctrlr->tmp_ptr, sizeof(struct spdk_nvme_cmds_and_effect_log_page),
    2195   [ #  #  #  # ]:          0 :                         0, 0, 0, SPDK_NVME_CSI_ZNS << 24,
    2196                 :          0 :                         nvme_ctrlr_get_zns_cmd_and_effects_log_done, ctrlr);
    2197         [ -  + ]:        987 :         if (rc != 0) {
    2198                 :          0 :                 goto error;
    2199                 :            :         }
    2200                 :            : 
    2201                 :        987 :         return 0;
    2202                 :            : 
    2203                 :          0 : error:
    2204                 :          0 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2205   [ #  #  #  # ]:          0 :         spdk_free(ctrlr->tmp_ptr);
    2206   [ #  #  #  # ]:          0 :         ctrlr->tmp_ptr = NULL;
    2207                 :          0 :         return rc;
    2208                 :          0 : }
    2209                 :            : 
    2210                 :            : static void
    2211                 :        987 : nvme_ctrlr_identify_zns_specific_done(void *arg, const struct spdk_nvme_cpl *cpl)
    2212                 :            : {
    2213                 :        987 :         struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg;
    2214                 :            : 
    2215   [ +  -  -  +  :        987 :         if (spdk_nvme_cpl_is_error(cpl)) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    2216                 :            :                 /* no need to print an error, the controller simply does not support ZNS */
    2217                 :          0 :                 nvme_ctrlr_free_zns_specific_data(ctrlr);
    2218                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_NUM_QUEUES,
    2219   [ #  #  #  #  :          0 :                                      ctrlr->opts.admin_timeout_ms);
                   #  # ]
    2220                 :          0 :                 return;
    2221                 :            :         }
    2222                 :            : 
    2223                 :            :         /* A zero zasl value means use mdts */
    2224   [ -  +  #  #  :        987 :         if (ctrlr->cdata_zns->zasl) {
          #  #  #  #  #  
                      # ]
    2225   [ #  #  #  #  :          0 :                 uint32_t max_append = ctrlr->min_page_size * (1 << ctrlr->cdata_zns->zasl);
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    2226   [ #  #  #  #  :          0 :                 ctrlr->max_zone_append_size = spdk_min(ctrlr->max_xfer_size, max_append);
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    2227                 :          0 :         } else {
    2228   [ #  #  #  #  :        987 :                 ctrlr->max_zone_append_size = ctrlr->max_xfer_size;
             #  #  #  # ]
    2229                 :            :         }
    2230                 :            : 
    2231                 :        987 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG,
    2232   [ #  #  #  #  :        987 :                              ctrlr->opts.admin_timeout_ms);
                   #  # ]
    2233                 :          0 : }
    2234                 :            : 
    2235                 :            : /**
    2236                 :            :  * This function will try to fetch the I/O Command Specific Controller data structure for
    2237                 :            :  * each I/O Command Set supported by SPDK.
    2238                 :            :  *
    2239                 :            :  * If an I/O Command Set is not supported by the controller, "Invalid Field in Command"
    2240                 :            :  * will be returned. Since we are fetching in a exploratively way, getting an error back
    2241                 :            :  * from the controller should not be treated as fatal.
    2242                 :            :  *
    2243                 :            :  * I/O Command Sets not supported by SPDK will be skipped (e.g. Key Value Command Set).
    2244                 :            :  *
    2245                 :            :  * I/O Command Sets without a IOCS specific data structure (i.e. a zero-filled IOCS specific
    2246                 :            :  * data structure) will be skipped (e.g. NVM Command Set, Key Value Command Set).
    2247                 :            :  */
    2248                 :            : static int
    2249                 :       3257 : nvme_ctrlr_identify_iocs_specific(struct spdk_nvme_ctrlr *ctrlr)
    2250                 :            : {
    2251                 :            :         int     rc;
    2252                 :            : 
    2253         [ +  + ]:       3257 :         if (!nvme_ctrlr_multi_iocs_enabled(ctrlr)) {
    2254                 :       3103 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_NUM_QUEUES,
    2255   [ +  -  +  -  :       2270 :                                      ctrlr->opts.admin_timeout_ms);
                   +  - ]
    2256                 :       2270 :                 return 0;
    2257                 :            :         }
    2258                 :            : 
    2259                 :            :         /*
    2260                 :            :          * Since SPDK currently only needs to fetch a single Command Set, keep the code here,
    2261                 :            :          * instead of creating multiple NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC substates,
    2262                 :            :          * which would require additional functions and complexity for no good reason.
    2263                 :            :          */
    2264   [ -  +  #  #  :        987 :         assert(!ctrlr->cdata_zns);
             #  #  #  # ]
    2265   [ #  #  #  # ]:        987 :         ctrlr->cdata_zns = spdk_zmalloc(sizeof(*ctrlr->cdata_zns), 64, NULL, SPDK_ENV_NUMA_ID_ANY,
    2266                 :            :                                         SPDK_MALLOC_SHARE | SPDK_MALLOC_DMA);
    2267   [ -  +  #  #  :        987 :         if (!ctrlr->cdata_zns) {
                   #  # ]
    2268                 :          0 :                 rc = -ENOMEM;
    2269                 :          0 :                 goto error;
    2270                 :            :         }
    2271                 :            : 
    2272                 :        987 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC,
    2273   [ #  #  #  #  :        987 :                              ctrlr->opts.admin_timeout_ms);
                   #  # ]
    2274                 :            : 
    2275                 :        987 :         rc = nvme_ctrlr_cmd_identify(ctrlr, SPDK_NVME_IDENTIFY_CTRLR_IOCS, 0, 0, SPDK_NVME_CSI_ZNS,
    2276   [ #  #  #  # ]:        987 :                                      ctrlr->cdata_zns, sizeof(*ctrlr->cdata_zns),
    2277                 :          0 :                                      nvme_ctrlr_identify_zns_specific_done, ctrlr);
    2278         [ -  + ]:        987 :         if (rc != 0) {
    2279                 :          0 :                 goto error;
    2280                 :            :         }
    2281                 :            : 
    2282                 :        987 :         return 0;
    2283                 :            : 
    2284                 :          0 : error:
    2285                 :          0 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2286                 :          0 :         nvme_ctrlr_free_zns_specific_data(ctrlr);
    2287                 :          0 :         return rc;
    2288                 :        833 : }
    2289                 :            : 
    2290                 :            : enum nvme_active_ns_state {
    2291                 :            :         NVME_ACTIVE_NS_STATE_IDLE,
    2292                 :            :         NVME_ACTIVE_NS_STATE_PROCESSING,
    2293                 :            :         NVME_ACTIVE_NS_STATE_DONE,
    2294                 :            :         NVME_ACTIVE_NS_STATE_ERROR
    2295                 :            : };
    2296                 :            : 
    2297                 :            : typedef void (*nvme_active_ns_ctx_deleter)(struct nvme_active_ns_ctx *);
    2298                 :            : 
    2299                 :            : struct nvme_active_ns_ctx {
    2300                 :            :         struct spdk_nvme_ctrlr *ctrlr;
    2301                 :            :         uint32_t page_count;
    2302                 :            :         uint32_t next_nsid;
    2303                 :            :         uint32_t *new_ns_list;
    2304                 :            :         nvme_active_ns_ctx_deleter deleter;
    2305                 :            : 
    2306                 :            :         enum nvme_active_ns_state state;
    2307                 :            : };
    2308                 :            : 
    2309                 :            : static struct nvme_active_ns_ctx *
    2310                 :       3478 : nvme_active_ns_ctx_create(struct spdk_nvme_ctrlr *ctrlr, nvme_active_ns_ctx_deleter deleter)
    2311                 :            : {
    2312                 :            :         struct nvme_active_ns_ctx *ctx;
    2313                 :       3478 :         uint32_t *new_ns_list = NULL;
    2314                 :            : 
    2315                 :       3478 :         ctx = calloc(1, sizeof(*ctx));
    2316         [ +  + ]:       3478 :         if (!ctx) {
    2317   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate nvme_active_ns_ctx!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    2318                 :          0 :                 return NULL;
    2319                 :            :         }
    2320                 :            : 
    2321   [ +  -  +  - ]:       3478 :         new_ns_list = spdk_zmalloc(sizeof(struct spdk_nvme_ns_list), ctrlr->page_size,
    2322                 :            :                                    NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_SHARE);
    2323         [ +  + ]:       3478 :         if (!new_ns_list) {
    2324   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate active_ns_list!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    2325                 :          0 :                 free(ctx);
    2326                 :          0 :                 return NULL;
    2327                 :            :         }
    2328                 :            : 
    2329   [ +  -  +  - ]:       3478 :         ctx->page_count = 1;
    2330   [ +  -  +  - ]:       3478 :         ctx->new_ns_list = new_ns_list;
    2331   [ +  -  +  - ]:       3478 :         ctx->ctrlr = ctrlr;
    2332   [ +  -  +  - ]:       3478 :         ctx->deleter = deleter;
    2333                 :            : 
    2334                 :       3478 :         return ctx;
    2335                 :        833 : }
    2336                 :            : 
    2337                 :            : static void
    2338                 :       3478 : nvme_active_ns_ctx_destroy(struct nvme_active_ns_ctx *ctx)
    2339                 :            : {
    2340   [ +  -  +  - ]:       3478 :         spdk_free(ctx->new_ns_list);
    2341                 :       3478 :         free(ctx);
    2342                 :       3478 : }
    2343                 :            : 
    2344                 :            : static int
    2345                 :      58051 : nvme_ctrlr_destruct_namespace(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid)
    2346                 :            : {
    2347                 :      55668 :         struct spdk_nvme_ns tmp, *ns;
    2348                 :            : 
    2349   [ +  +  #  # ]:      58051 :         assert(ctrlr != NULL);
    2350                 :            : 
    2351         [ +  - ]:      58051 :         tmp.id = nsid;
    2352         [ +  - ]:      58051 :         ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp);
    2353         [ +  + ]:      58051 :         if (ns == NULL) {
    2354                 :          0 :                 return -EINVAL;
    2355                 :            :         }
    2356                 :            : 
    2357                 :      58051 :         nvme_ns_destruct(ns);
    2358   [ +  -  +  - ]:      58051 :         ns->active = false;
    2359                 :            : 
    2360                 :      58051 :         return 0;
    2361                 :        833 : }
    2362                 :            : 
    2363                 :            : static int
    2364                 :      40052 : nvme_ctrlr_construct_namespace(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid)
    2365                 :            : {
    2366                 :            :         struct spdk_nvme_ns *ns;
    2367                 :            : 
    2368   [ +  -  +  +  :      40052 :         if (nsid < 1 || nsid > ctrlr->cdata.nn) {
          +  -  +  -  -  
                      + ]
    2369                 :          0 :                 return -EINVAL;
    2370                 :            :         }
    2371                 :            : 
    2372                 :            :         /* Namespaces are constructed on demand, so simply request it. */
    2373                 :      40052 :         ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
    2374         [ +  + ]:      40052 :         if (ns == NULL) {
    2375                 :          0 :                 return -ENOMEM;
    2376                 :            :         }
    2377                 :            : 
    2378   [ +  -  +  - ]:      40052 :         ns->active = true;
    2379                 :            : 
    2380                 :      40052 :         return 0;
    2381                 :        833 : }
    2382                 :            : 
    2383                 :            : static void
    2384                 :       3475 : nvme_ctrlr_identify_active_ns_swap(struct spdk_nvme_ctrlr *ctrlr, uint32_t *new_ns_list,
    2385                 :            :                                    size_t max_entries)
    2386                 :            : {
    2387                 :       3475 :         uint32_t active_ns_count = 0;
    2388                 :            :         size_t i;
    2389                 :            :         uint32_t nsid;
    2390                 :            :         struct spdk_nvme_ns *ns, *tmp_ns;
    2391                 :            :         int rc;
    2392                 :            : 
    2393                 :            :         /* First, remove namespaces that no longer exist */
    2394   [ +  +  +  -  :      49914 :         RB_FOREACH_SAFE(ns, nvme_ns_tree, &ctrlr->ns, tmp_ns) {
                   -  + ]
    2395   [ #  #  #  # ]:      46439 :                 nsid = new_ns_list[0];
    2396                 :      46439 :                 active_ns_count = 0;
    2397         [ +  + ]:   10642987 :                 while (nsid != 0) {
    2398   [ +  +  #  #  :   10610767 :                         if (nsid == ns->id) {
                   #  # ]
    2399                 :      14219 :                                 break;
    2400                 :            :                         }
    2401                 :            : 
    2402   [ #  #  #  # ]:   10596548 :                         nsid = new_ns_list[active_ns_count++];
    2403                 :            :                 }
    2404                 :            : 
    2405   [ +  +  #  #  :      46439 :                 if (nsid != ns->id) {
                   #  # ]
    2406                 :            :                         /* Did not find this namespace id in the new list. */
    2407   [ -  +  -  +  :      32220 :                         NVME_CTRLR_DEBUGLOG(ctrlr, "Namespace %u was removed\n", ns->id);
          -  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    2408   [ #  #  #  # ]:      32220 :                         nvme_ctrlr_destruct_namespace(ctrlr, ns->id);
    2409                 :          0 :                 }
    2410                 :          0 :         }
    2411                 :            : 
    2412                 :            :         /* Next, add new namespaces */
    2413                 :       3475 :         active_ns_count = 0;
    2414         [ +  + ]:      43527 :         for (i = 0; i < max_entries; i++) {
    2415   [ +  -  +  - ]:      43527 :                 nsid = new_ns_list[active_ns_count];
    2416                 :            : 
    2417         [ +  + ]:      43527 :                 if (nsid == 0) {
    2418                 :       3475 :                         break;
    2419                 :            :                 }
    2420                 :            : 
    2421                 :            :                 /* If the namespace already exists, this will not construct it a second time. */
    2422                 :      40052 :                 rc = nvme_ctrlr_construct_namespace(ctrlr, nsid);
    2423         [ -  + ]:      40052 :                 if (rc != 0) {
    2424                 :            :                         /* We can't easily handle a failure here. But just move on. */
    2425         [ #  # ]:          0 :                         assert(false);
    2426                 :            :                         NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to allocate a namespace object.\n");
    2427                 :            :                         continue;
    2428                 :            :                 }
    2429                 :            : 
    2430                 :      40052 :                 active_ns_count++;
    2431                 :        833 :         }
    2432                 :            : 
    2433   [ +  -  +  - ]:       3475 :         ctrlr->active_ns_count = active_ns_count;
    2434                 :       3475 : }
    2435                 :            : 
    2436                 :            : static void
    2437                 :       3428 : nvme_ctrlr_identify_active_ns_async_done(void *arg, const struct spdk_nvme_cpl *cpl)
    2438                 :            : {
    2439                 :       3428 :         struct nvme_active_ns_ctx *ctx = arg;
    2440                 :       3428 :         uint32_t *new_ns_list = NULL;
    2441                 :            : 
    2442   [ +  +  +  +  :       3428 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   -  + ]
    2443   [ #  #  #  # ]:          3 :                 ctx->state = NVME_ACTIVE_NS_STATE_ERROR;
    2444                 :          3 :                 goto out;
    2445                 :            :         }
    2446                 :            : 
    2447   [ +  -  +  -  :       3425 :         ctx->next_nsid = ctx->new_ns_list[1024 * ctx->page_count - 1];
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
    2448   [ +  +  +  -  :       3425 :         if (ctx->next_nsid == 0) {
                   -  + ]
    2449   [ +  -  +  - ]:       3410 :                 ctx->state = NVME_ACTIVE_NS_STATE_DONE;
    2450                 :       3410 :                 goto out;
    2451                 :            :         }
    2452                 :            : 
    2453         [ #  # ]:         15 :         ctx->page_count++;
    2454   [ #  #  #  # ]:         15 :         new_ns_list = spdk_realloc(ctx->new_ns_list,
    2455   [ #  #  #  # ]:         15 :                                    ctx->page_count * sizeof(struct spdk_nvme_ns_list),
    2456   [ #  #  #  #  :         15 :                                    ctx->ctrlr->page_size);
             #  #  #  # ]
    2457         [ -  + ]:         15 :         if (!new_ns_list) {
    2458                 :          0 :                 SPDK_ERRLOG("Failed to reallocate active_ns_list!\n");
    2459   [ #  #  #  # ]:          0 :                 ctx->state = NVME_ACTIVE_NS_STATE_ERROR;
    2460                 :          0 :                 goto out;
    2461                 :            :         }
    2462                 :            : 
    2463   [ #  #  #  # ]:         15 :         ctx->new_ns_list = new_ns_list;
    2464                 :         15 :         nvme_ctrlr_identify_active_ns_async(ctx);
    2465                 :         15 :         return;
    2466                 :            : 
    2467                 :       2580 : out:
    2468   [ +  +  +  -  :       3413 :         if (ctx->deleter) {
                   -  + ]
    2469   [ +  -  +  -  :       3222 :                 ctx->deleter(ctx);
             -  +  +  - ]
    2470                 :        833 :         }
    2471                 :        833 : }
    2472                 :            : 
    2473                 :            : static void
    2474                 :       3493 : nvme_ctrlr_identify_active_ns_async(struct nvme_active_ns_ctx *ctx)
    2475                 :            : {
    2476   [ +  -  +  - ]:       3493 :         struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr;
    2477                 :            :         uint32_t i;
    2478                 :            :         int rc;
    2479                 :            : 
    2480   [ +  +  +  -  :       3493 :         if (ctrlr->cdata.nn == 0) {
             +  -  +  - ]
    2481   [ #  #  #  # ]:         48 :                 ctx->state = NVME_ACTIVE_NS_STATE_DONE;
    2482                 :         48 :                 goto out;
    2483                 :            :         }
    2484                 :            : 
    2485   [ +  +  +  -  :       3445 :         assert(ctx->new_ns_list != NULL);
             +  -  #  # ]
    2486                 :            : 
    2487                 :            :         /*
    2488                 :            :          * If controller doesn't support active ns list CNS 0x02 dummy up
    2489                 :            :          * an active ns list, i.e. all namespaces report as active
    2490                 :            :          */
    2491   [ +  +  +  +  :       3445 :         if (ctrlr->vs.raw < SPDK_NVME_VERSION(1, 1, 0) || ctrlr->quirks & NVME_QUIRK_IDENTIFY_CNS) {
          +  -  +  -  +  
          -  +  -  +  -  
             +  -  -  + ]
    2492                 :            :                 uint32_t *new_ns_list;
    2493                 :            : 
    2494                 :            :                 /*
    2495                 :            :                  * Active NS list must always end with zero element.
    2496                 :            :                  * So, we allocate for cdata.nn+1.
    2497                 :            :                  */
    2498   [ #  #  #  #  :         17 :                 ctx->page_count = spdk_divide_round_up(ctrlr->cdata.nn + 1,
          #  #  #  #  #  
                      # ]
    2499                 :            :                                                        sizeof(struct spdk_nvme_ns_list) / sizeof(new_ns_list[0]));
    2500   [ #  #  #  # ]:         17 :                 new_ns_list = spdk_realloc(ctx->new_ns_list,
    2501   [ #  #  #  # ]:         17 :                                            ctx->page_count * sizeof(struct spdk_nvme_ns_list),
    2502   [ #  #  #  #  :         17 :                                            ctx->ctrlr->page_size);
             #  #  #  # ]
    2503         [ -  + ]:         17 :                 if (!new_ns_list) {
    2504                 :          0 :                         SPDK_ERRLOG("Failed to reallocate active_ns_list!\n");
    2505   [ #  #  #  # ]:          0 :                         ctx->state = NVME_ACTIVE_NS_STATE_ERROR;
    2506                 :          0 :                         goto out;
    2507                 :            :                 }
    2508                 :            : 
    2509   [ #  #  #  # ]:         17 :                 ctx->new_ns_list = new_ns_list;
    2510   [ #  #  #  #  :         17 :                 ctx->new_ns_list[ctrlr->cdata.nn] = 0;
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    2511   [ +  +  #  #  :      12283 :                 for (i = 0; i < ctrlr->cdata.nn; i++) {
             #  #  #  # ]
    2512   [ #  #  #  #  :      12266 :                         ctx->new_ns_list[i] = i + 1;
             #  #  #  # ]
    2513                 :          0 :                 }
    2514                 :            : 
    2515   [ #  #  #  # ]:         17 :                 ctx->state = NVME_ACTIVE_NS_STATE_DONE;
    2516                 :         17 :                 goto out;
    2517                 :            :         }
    2518                 :            : 
    2519   [ +  -  +  - ]:       3428 :         ctx->state = NVME_ACTIVE_NS_STATE_PROCESSING;
    2520   [ +  -  +  - ]:       4261 :         rc = nvme_ctrlr_cmd_identify(ctrlr, SPDK_NVME_IDENTIFY_ACTIVE_NS_LIST, 0, ctx->next_nsid, 0,
    2521   [ +  -  +  -  :       3428 :                                      &ctx->new_ns_list[1024 * (ctx->page_count - 1)], sizeof(struct spdk_nvme_ns_list),
          +  -  +  -  +  
                      - ]
    2522                 :        833 :                                      nvme_ctrlr_identify_active_ns_async_done, ctx);
    2523         [ -  + ]:       3428 :         if (rc != 0) {
    2524   [ #  #  #  # ]:          0 :                 ctx->state = NVME_ACTIVE_NS_STATE_ERROR;
    2525                 :          0 :                 goto out;
    2526                 :            :         }
    2527                 :            : 
    2528                 :       3428 :         return;
    2529                 :            : 
    2530                 :         65 : out:
    2531   [ +  +  #  #  :         65 :         if (ctx->deleter) {
                   #  # ]
    2532   [ #  #  #  #  :         50 :                 ctx->deleter(ctx);
             #  #  #  # ]
    2533                 :          0 :         }
    2534                 :        833 : }
    2535                 :            : 
    2536                 :            : static void
    2537                 :       3272 : _nvme_active_ns_ctx_deleter(struct nvme_active_ns_ctx *ctx)
    2538                 :            : {
    2539   [ +  -  +  - ]:       3272 :         struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr;
    2540                 :            :         struct spdk_nvme_ns *ns;
    2541                 :            : 
    2542   [ +  +  +  -  :       3272 :         if (ctx->state == NVME_ACTIVE_NS_STATE_ERROR) {
                   -  + ]
    2543                 :          0 :                 nvme_active_ns_ctx_destroy(ctx);
    2544                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2545                 :          0 :                 return;
    2546                 :            :         }
    2547                 :            : 
    2548   [ +  +  +  -  :       3272 :         assert(ctx->state == NVME_ACTIVE_NS_STATE_DONE);
             +  -  #  # ]
    2549                 :            : 
    2550   [ +  +  -  + ]:       3425 :         RB_FOREACH(ns, nvme_ns_tree, &ctrlr->ns) {
    2551                 :        153 :                 nvme_ns_free_iocs_specific_data(ns);
    2552                 :          0 :         }
    2553                 :            : 
    2554   [ +  -  +  -  :       3272 :         nvme_ctrlr_identify_active_ns_swap(ctrlr, ctx->new_ns_list, ctx->page_count * 1024);
             +  -  +  - ]
    2555                 :       3272 :         nvme_active_ns_ctx_destroy(ctx);
    2556   [ +  -  +  -  :       3272 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS, ctrlr->opts.admin_timeout_ms);
                   +  - ]
    2557                 :        833 : }
    2558                 :            : 
    2559                 :            : static void
    2560                 :       3272 : _nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr)
    2561                 :            : {
    2562                 :            :         struct nvme_active_ns_ctx *ctx;
    2563                 :            : 
    2564                 :       3272 :         ctx = nvme_active_ns_ctx_create(ctrlr, _nvme_active_ns_ctx_deleter);
    2565         [ +  + ]:       3272 :         if (!ctx) {
    2566                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2567                 :          0 :                 return;
    2568                 :            :         }
    2569                 :            : 
    2570                 :       4105 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS,
    2571   [ -  +  -  +  :       3272 :                              ctrlr->opts.admin_timeout_ms);
                   -  + ]
    2572                 :       3272 :         nvme_ctrlr_identify_active_ns_async(ctx);
    2573                 :        833 : }
    2574                 :            : 
    2575                 :            : int
    2576                 :        206 : nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr)
    2577                 :            : {
    2578                 :            :         struct nvme_active_ns_ctx *ctx;
    2579                 :            :         int rc;
    2580                 :            : 
    2581                 :        206 :         ctx = nvme_active_ns_ctx_create(ctrlr, NULL);
    2582         [ -  + ]:        206 :         if (!ctx) {
    2583                 :          0 :                 return -ENOMEM;
    2584                 :            :         }
    2585                 :            : 
    2586                 :        206 :         nvme_ctrlr_identify_active_ns_async(ctx);
    2587   [ +  +  #  #  :     202184 :         while (ctx->state == NVME_ACTIVE_NS_STATE_PROCESSING) {
                   #  # ]
    2588   [ #  #  #  # ]:     201978 :                 rc = spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
    2589         [ -  + ]:     201978 :                 if (rc < 0) {
    2590   [ #  #  #  # ]:          0 :                         ctx->state = NVME_ACTIVE_NS_STATE_ERROR;
    2591                 :          0 :                         break;
    2592                 :            :                 }
    2593                 :            :         }
    2594                 :            : 
    2595   [ +  +  #  #  :        206 :         if (ctx->state == NVME_ACTIVE_NS_STATE_ERROR) {
                   #  # ]
    2596                 :          3 :                 nvme_active_ns_ctx_destroy(ctx);
    2597                 :          3 :                 return -ENXIO;
    2598                 :            :         }
    2599                 :            : 
    2600   [ -  +  #  #  :        203 :         assert(ctx->state == NVME_ACTIVE_NS_STATE_DONE);
             #  #  #  # ]
    2601   [ #  #  #  #  :        203 :         nvme_ctrlr_identify_active_ns_swap(ctrlr, ctx->new_ns_list, ctx->page_count * 1024);
             #  #  #  # ]
    2602                 :        203 :         nvme_active_ns_ctx_destroy(ctx);
    2603                 :            : 
    2604                 :        203 :         return 0;
    2605                 :          0 : }
    2606                 :            : 
    2607                 :            : static void
    2608                 :       2971 : nvme_ctrlr_identify_ns_async_done(void *arg, const struct spdk_nvme_cpl *cpl)
    2609                 :            : {
    2610                 :       2971 :         struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg;
    2611   [ +  -  +  - ]:       2971 :         struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr;
    2612                 :            :         uint32_t nsid;
    2613                 :            :         int rc;
    2614                 :            : 
    2615   [ +  -  +  +  :       2971 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   -  + ]
    2616                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2617                 :          0 :                 return;
    2618                 :            :         }
    2619                 :            : 
    2620                 :       2971 :         nvme_ns_set_identify_data(ns);
    2621                 :            : 
    2622                 :            :         /* move on to the next active NS */
    2623   [ +  -  +  - ]:       2971 :         nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, ns->id);
    2624                 :       2971 :         ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
    2625         [ +  + ]:       2971 :         if (ns == NULL) {
    2626                 :       3583 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_ID_DESCS,
    2627   [ -  +  -  +  :       2750 :                                      ctrlr->opts.admin_timeout_ms);
                   -  + ]
    2628                 :       2750 :                 return;
    2629                 :            :         }
    2630   [ #  #  #  # ]:        221 :         ns->ctrlr = ctrlr;
    2631   [ #  #  #  # ]:        221 :         ns->id = nsid;
    2632                 :            : 
    2633                 :        221 :         rc = nvme_ctrlr_identify_ns_async(ns);
    2634         [ -  + ]:        221 :         if (rc) {
    2635                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2636                 :          0 :         }
    2637                 :        833 : }
    2638                 :            : 
    2639                 :            : static int
    2640                 :       2971 : nvme_ctrlr_identify_ns_async(struct spdk_nvme_ns *ns)
    2641                 :            : {
    2642   [ +  -  +  - ]:       2971 :         struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr;
    2643                 :            :         struct spdk_nvme_ns_data *nsdata;
    2644                 :            : 
    2645         [ +  - ]:       2971 :         nsdata = &ns->nsdata;
    2646                 :            : 
    2647                 :       3804 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS,
    2648   [ +  -  +  -  :       2971 :                              ctrlr->opts.admin_timeout_ms);
                   +  - ]
    2649   [ +  -  +  -  :       2971 :         return nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS, 0, ns->id, 0,
             +  -  +  - ]
    2650                 :        833 :                                        nsdata, sizeof(*nsdata),
    2651                 :        833 :                                        nvme_ctrlr_identify_ns_async_done, ns);
    2652                 :            : }
    2653                 :            : 
    2654                 :            : static int
    2655                 :       3242 : nvme_ctrlr_identify_namespaces(struct spdk_nvme_ctrlr *ctrlr)
    2656                 :            : {
    2657                 :            :         uint32_t nsid;
    2658                 :            :         struct spdk_nvme_ns *ns;
    2659                 :            :         int rc;
    2660                 :            : 
    2661                 :       3242 :         nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
    2662                 :       3242 :         ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
    2663         [ +  + ]:       3242 :         if (ns == NULL) {
    2664                 :            :                 /* No active NS, move on to the next state */
    2665                 :        492 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_ID_DESCS,
    2666   [ #  #  #  #  :        492 :                                      ctrlr->opts.admin_timeout_ms);
                   #  # ]
    2667                 :        492 :                 return 0;
    2668                 :            :         }
    2669                 :            : 
    2670   [ +  -  +  - ]:       2750 :         ns->ctrlr = ctrlr;
    2671   [ +  -  +  - ]:       2750 :         ns->id = nsid;
    2672                 :            : 
    2673                 :       2750 :         rc = nvme_ctrlr_identify_ns_async(ns);
    2674         [ +  + ]:       2750 :         if (rc) {
    2675                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2676                 :          0 :         }
    2677                 :            : 
    2678                 :       2750 :         return rc;
    2679                 :        833 : }
    2680                 :            : 
    2681                 :            : static int
    2682                 :       1797 : nvme_ctrlr_identify_namespaces_iocs_specific_next(struct spdk_nvme_ctrlr *ctrlr, uint32_t prev_nsid)
    2683                 :            : {
    2684                 :            :         uint32_t nsid;
    2685                 :            :         struct spdk_nvme_ns *ns;
    2686                 :            :         int rc;
    2687                 :            : 
    2688         [ +  + ]:       1797 :         if (!prev_nsid) {
    2689                 :        993 :                 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
    2690                 :          0 :         } else {
    2691                 :            :                 /* move on to the next active NS */
    2692                 :        804 :                 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, prev_nsid);
    2693                 :            :         }
    2694                 :            : 
    2695                 :       1797 :         ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
    2696         [ +  + ]:       1797 :         if (ns == NULL) {
    2697                 :            :                 /* No first/next active NS, move on to the next state */
    2698                 :        665 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES,
    2699   [ #  #  #  #  :        665 :                                      ctrlr->opts.admin_timeout_ms);
                   #  # ]
    2700                 :        665 :                 return 0;
    2701                 :            :         }
    2702                 :            : 
    2703                 :            :         /* loop until we find a ns which has (supported) iocs specific data */
    2704         [ +  + ]:       1153 :         while (!nvme_ns_has_supported_iocs_specific_data(ns)) {
    2705   [ #  #  #  # ]:        349 :                 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, ns->id);
    2706                 :        349 :                 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
    2707         [ +  + ]:        349 :                 if (ns == NULL) {
    2708                 :            :                         /* no namespace with (supported) iocs specific data found */
    2709                 :        328 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES,
    2710   [ #  #  #  #  :        328 :                                              ctrlr->opts.admin_timeout_ms);
                   #  # ]
    2711                 :        328 :                         return 0;
    2712                 :            :                 }
    2713                 :            :         }
    2714                 :            : 
    2715                 :        804 :         rc = nvme_ctrlr_identify_ns_iocs_specific_async(ns);
    2716         [ +  + ]:        804 :         if (rc) {
    2717                 :          3 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2718                 :          0 :         }
    2719                 :            : 
    2720                 :        804 :         return rc;
    2721                 :          0 : }
    2722                 :            : 
    2723                 :            : static void
    2724                 :          2 : nvme_ctrlr_identify_ns_zns_specific_async_done(void *arg, const struct spdk_nvme_cpl *cpl)
    2725                 :            : {
    2726                 :          2 :         struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg;
    2727   [ #  #  #  # ]:          2 :         struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr;
    2728                 :            : 
    2729   [ +  -  -  +  :          2 :         if (spdk_nvme_cpl_is_error(cpl)) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    2730                 :          0 :                 nvme_ns_free_zns_specific_data(ns);
    2731                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2732                 :          0 :                 return;
    2733                 :            :         }
    2734                 :            : 
    2735   [ #  #  #  # ]:          2 :         nvme_ctrlr_identify_namespaces_iocs_specific_next(ctrlr, ns->id);
    2736                 :          0 : }
    2737                 :            : 
    2738                 :            : static int
    2739                 :          8 : nvme_ctrlr_identify_ns_zns_specific_async(struct spdk_nvme_ns *ns)
    2740                 :            : {
    2741   [ #  #  #  # ]:          8 :         struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr;
    2742                 :            :         int rc;
    2743                 :            : 
    2744   [ -  +  #  #  :          8 :         assert(!ns->nsdata_zns);
             #  #  #  # ]
    2745   [ #  #  #  # ]:          8 :         ns->nsdata_zns = spdk_zmalloc(sizeof(*ns->nsdata_zns), 64, NULL, SPDK_ENV_NUMA_ID_ANY,
    2746                 :            :                                       SPDK_MALLOC_SHARE);
    2747   [ -  +  #  #  :          8 :         if (!ns->nsdata_zns) {
                   #  # ]
    2748                 :          0 :                 return -ENOMEM;
    2749                 :            :         }
    2750                 :            : 
    2751                 :          8 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC,
    2752   [ #  #  #  #  :          8 :                              ctrlr->opts.admin_timeout_ms);
                   #  # ]
    2753   [ #  #  #  #  :          8 :         rc = nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS_IOCS, 0, ns->id, ns->csi,
          #  #  #  #  #  
                #  #  # ]
    2754   [ #  #  #  # ]:          8 :                                      ns->nsdata_zns, sizeof(*ns->nsdata_zns),
    2755                 :          0 :                                      nvme_ctrlr_identify_ns_zns_specific_async_done, ns);
    2756         [ +  + ]:          8 :         if (rc) {
    2757                 :          3 :                 nvme_ns_free_zns_specific_data(ns);
    2758                 :          0 :         }
    2759                 :            : 
    2760                 :          8 :         return rc;
    2761                 :          0 : }
    2762                 :            : 
    2763                 :            : static void
    2764                 :        796 : nvme_ctrlr_identify_ns_nvm_specific_async_done(void *arg, const struct spdk_nvme_cpl *cpl)
    2765                 :            : {
    2766                 :        796 :         struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg;
    2767   [ #  #  #  # ]:        796 :         struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr;
    2768                 :            : 
    2769   [ +  -  -  +  :        796 :         if (spdk_nvme_cpl_is_error(cpl)) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    2770                 :          0 :                 nvme_ns_free_nvm_specific_data(ns);
    2771                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2772                 :          0 :                 return;
    2773                 :            :         }
    2774                 :            : 
    2775   [ #  #  #  # ]:        796 :         nvme_ctrlr_identify_namespaces_iocs_specific_next(ctrlr, ns->id);
    2776                 :          0 : }
    2777                 :            : 
    2778                 :            : static int
    2779                 :        796 : nvme_ctrlr_identify_ns_nvm_specific_async(struct spdk_nvme_ns *ns)
    2780                 :            : {
    2781   [ #  #  #  # ]:        796 :         struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr;
    2782                 :            :         int rc;
    2783                 :            : 
    2784   [ -  +  #  #  :        796 :         assert(!ns->nsdata_nvm);
             #  #  #  # ]
    2785   [ #  #  #  # ]:        796 :         ns->nsdata_nvm = spdk_zmalloc(sizeof(*ns->nsdata_nvm), 64, NULL, SPDK_ENV_NUMA_ID_ANY,
    2786                 :            :                                       SPDK_MALLOC_SHARE);
    2787   [ -  +  #  #  :        796 :         if (!ns->nsdata_nvm) {
                   #  # ]
    2788                 :          0 :                 return -ENOMEM;
    2789                 :            :         }
    2790                 :            : 
    2791                 :        796 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC,
    2792   [ #  #  #  #  :        796 :                              ctrlr->opts.admin_timeout_ms);
                   #  # ]
    2793   [ #  #  #  #  :        796 :         rc = nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS_IOCS, 0, ns->id, ns->csi,
          #  #  #  #  #  
                #  #  # ]
    2794   [ #  #  #  # ]:        796 :                                      ns->nsdata_nvm, sizeof(*ns->nsdata_nvm),
    2795                 :          0 :                                      nvme_ctrlr_identify_ns_nvm_specific_async_done, ns);
    2796         [ -  + ]:        796 :         if (rc) {
    2797                 :          0 :                 nvme_ns_free_nvm_specific_data(ns);
    2798                 :          0 :         }
    2799                 :            : 
    2800                 :        796 :         return rc;
    2801                 :          0 : }
    2802                 :            : 
    2803                 :            : static int
    2804                 :        804 : nvme_ctrlr_identify_ns_iocs_specific_async(struct spdk_nvme_ns *ns)
    2805                 :            : {
    2806   [ +  +  -  #  :        804 :         switch (ns->csi) {
                #  #  # ]
    2807                 :          8 :         case SPDK_NVME_CSI_ZNS:
    2808                 :          8 :                 return nvme_ctrlr_identify_ns_zns_specific_async(ns);
    2809                 :        796 :         case SPDK_NVME_CSI_NVM:
    2810   [ +  -  #  #  :        796 :                 if (ns->ctrlr->cdata.ctratt.bits.elbas) {
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    2811                 :        796 :                         return nvme_ctrlr_identify_ns_nvm_specific_async(ns);
    2812                 :            :                 }
    2813                 :            :         /* fallthrough */
    2814                 :            :         default:
    2815                 :            :                 /*
    2816                 :            :                  * This switch must handle all cases for which
    2817                 :            :                  * nvme_ns_has_supported_iocs_specific_data() returns true,
    2818                 :            :                  * other cases should never happen.
    2819                 :            :                  */
    2820         [ #  # ]:          0 :                 assert(0);
    2821                 :            :         }
    2822                 :            : 
    2823                 :            :         return -EINVAL;
    2824                 :          0 : }
    2825                 :            : 
    2826                 :            : static int
    2827                 :       3242 : nvme_ctrlr_identify_namespaces_iocs_specific(struct spdk_nvme_ctrlr *ctrlr)
    2828                 :            : {
    2829         [ +  + ]:       3242 :         if (!nvme_ctrlr_multi_iocs_enabled(ctrlr)) {
    2830                 :            :                 /* Multi IOCS not supported/enabled, move on to the next state */
    2831                 :       3088 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES,
    2832   [ +  -  +  -  :       2255 :                                      ctrlr->opts.admin_timeout_ms);
                   +  - ]
    2833                 :       2255 :                 return 0;
    2834                 :            :         }
    2835                 :            : 
    2836                 :        987 :         return nvme_ctrlr_identify_namespaces_iocs_specific_next(ctrlr, 0);
    2837                 :        833 : }
    2838                 :            : 
    2839                 :            : static void
    2840                 :       2819 : nvme_ctrlr_identify_id_desc_async_done(void *arg, const struct spdk_nvme_cpl *cpl)
    2841                 :            : {
    2842                 :       2819 :         struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg;
    2843   [ +  -  +  - ]:       2819 :         struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr;
    2844                 :            :         uint32_t nsid;
    2845                 :            :         int rc;
    2846                 :            : 
    2847   [ +  +  +  +  :       2819 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   -  + ]
    2848                 :            :                 /*
    2849                 :            :                  * Many controllers claim to be compatible with NVMe 1.3, however,
    2850                 :            :                  * they do not implement NS ID Desc List. Therefore, instead of setting
    2851                 :            :                  * the state to NVME_CTRLR_STATE_ERROR, silently ignore the completion
    2852                 :            :                  * error and move on to the next state.
    2853                 :            :                  *
    2854                 :            :                  * The proper way is to create a new quirk for controllers that violate
    2855                 :            :                  * the NVMe 1.3 spec by not supporting NS ID Desc List.
    2856                 :            :                  * (Re-using the NVME_QUIRK_IDENTIFY_CNS quirk is not possible, since
    2857                 :            :                  * it is too generic and was added in order to handle controllers that
    2858                 :            :                  * violate the NVMe 1.1 spec by not supporting ACTIVE LIST).
    2859                 :            :                  */
    2860                 :         15 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC,
    2861   [ #  #  #  #  :         15 :                                      ctrlr->opts.admin_timeout_ms);
                   #  # ]
    2862                 :         15 :                 return;
    2863                 :            :         }
    2864                 :            : 
    2865                 :       2804 :         nvme_ns_set_id_desc_list_data(ns);
    2866                 :            : 
    2867                 :            :         /* move on to the next active NS */
    2868   [ +  -  +  - ]:       2804 :         nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, ns->id);
    2869                 :       2804 :         ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
    2870         [ +  + ]:       2804 :         if (ns == NULL) {
    2871                 :       3449 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC,
    2872   [ -  +  -  +  :       2618 :                                      ctrlr->opts.admin_timeout_ms);
                   -  + ]
    2873                 :       2618 :                 return;
    2874                 :            :         }
    2875                 :            : 
    2876                 :        186 :         rc = nvme_ctrlr_identify_id_desc_async(ns);
    2877         [ -  + ]:        186 :         if (rc) {
    2878                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2879                 :          0 :         }
    2880                 :        831 : }
    2881                 :            : 
    2882                 :            : static int
    2883                 :       2819 : nvme_ctrlr_identify_id_desc_async(struct spdk_nvme_ns *ns)
    2884                 :            : {
    2885   [ +  -  +  - ]:       2819 :         struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr;
    2886                 :            : 
    2887   [ +  +  +  - ]:       2819 :         memset(ns->id_desc_list, 0, sizeof(ns->id_desc_list));
    2888                 :            : 
    2889                 :       3650 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS,
    2890   [ +  -  +  -  :       2819 :                              ctrlr->opts.admin_timeout_ms);
                   +  - ]
    2891   [ +  -  +  - ]:       3854 :         return nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS_ID_DESCRIPTOR_LIST,
    2892   [ +  -  +  -  :       2819 :                                        0, ns->id, 0, ns->id_desc_list, sizeof(ns->id_desc_list),
                   +  - ]
    2893                 :        831 :                                        nvme_ctrlr_identify_id_desc_async_done, ns);
    2894                 :            : }
    2895                 :            : 
    2896                 :            : static int
    2897                 :       3242 : nvme_ctrlr_identify_id_desc_namespaces(struct spdk_nvme_ctrlr *ctrlr)
    2898                 :            : {
    2899                 :            :         uint32_t nsid;
    2900                 :            :         struct spdk_nvme_ns *ns;
    2901                 :            :         int rc;
    2902                 :            : 
    2903   [ +  +  +  -  :       4075 :         if ((ctrlr->vs.raw < SPDK_NVME_VERSION(1, 3, 0) &&
          +  -  +  -  +  
             -  +  +  +  
                      + ]
    2904   [ +  +  +  -  :        143 :              !(ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_IOCS)) ||
             +  -  +  - ]
    2905   [ +  +  +  - ]:       3101 :             (ctrlr->quirks & NVME_QUIRK_IDENTIFY_CNS)) {
    2906   [ +  +  +  +  :        143 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Version < 1.3; not attempting to retrieve NS ID Descriptor List\n");
          +  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    2907                 :            :                 /* NS ID Desc List not supported, move on to the next state */
    2908                 :        145 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC,
    2909   [ +  -  +  -  :        143 :                                      ctrlr->opts.admin_timeout_ms);
                   +  - ]
    2910                 :        143 :                 return 0;
    2911                 :            :         }
    2912                 :            : 
    2913                 :       3099 :         nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
    2914                 :       3099 :         ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
    2915         [ +  + ]:       3099 :         if (ns == NULL) {
    2916                 :            :                 /* No active NS, move on to the next state */
    2917                 :        466 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC,
    2918   [ #  #  #  #  :        466 :                                      ctrlr->opts.admin_timeout_ms);
                   #  # ]
    2919                 :        466 :                 return 0;
    2920                 :            :         }
    2921                 :            : 
    2922                 :       2633 :         rc = nvme_ctrlr_identify_id_desc_async(ns);
    2923         [ +  + ]:       2633 :         if (rc) {
    2924                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2925                 :          0 :         }
    2926                 :            : 
    2927                 :       2633 :         return rc;
    2928                 :        833 : }
    2929                 :            : 
    2930                 :            : static void
    2931                 :       3257 : nvme_ctrlr_update_nvmf_ioccsz(struct spdk_nvme_ctrlr *ctrlr)
    2932                 :            : {
    2933         [ +  + ]:       3257 :         if (spdk_nvme_ctrlr_is_fabrics(ctrlr)) {
    2934   [ +  +  +  -  :       2378 :                 if (ctrlr->cdata.nvmf_specific.ioccsz < 4) {
          +  -  +  -  +  
                      - ]
    2935   [ #  #  #  #  :          0 :                         NVME_CTRLR_ERRLOG(ctrlr, "Incorrect IOCCSZ %u, the minimum value should be 4\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
    2936                 :            :                                           ctrlr->cdata.nvmf_specific.ioccsz);
    2937   [ #  #  #  #  :          0 :                         ctrlr->cdata.nvmf_specific.ioccsz = 4;
             #  #  #  # ]
    2938         [ #  # ]:          0 :                         assert(0);
    2939                 :            :                 }
    2940   [ +  -  +  -  :       2378 :                 ctrlr->ioccsz_bytes = ctrlr->cdata.nvmf_specific.ioccsz * 16 - sizeof(struct spdk_nvme_cmd);
          +  -  +  -  +  
                -  +  - ]
    2941   [ +  -  +  -  :       2378 :                 ctrlr->icdoff = ctrlr->cdata.nvmf_specific.icdoff;
          +  -  +  -  +  
                -  +  - ]
    2942                 :        824 :         }
    2943                 :       3257 : }
    2944                 :            : 
    2945                 :            : static void
    2946                 :       3257 : nvme_ctrlr_set_num_queues_done(void *arg, const struct spdk_nvme_cpl *cpl)
    2947                 :            : {
    2948                 :            :         uint32_t cq_allocated, sq_allocated, min_allocated, i;
    2949                 :       3257 :         struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg;
    2950                 :            : 
    2951   [ +  -  +  +  :       3257 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   -  + ]
    2952   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Set Features - Number of Queues failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    2953   [ #  #  #  #  :          0 :                 ctrlr->opts.num_io_queues = 0;
                   #  # ]
    2954                 :          0 :         } else {
    2955                 :            :                 /*
    2956                 :            :                  * Data in cdw0 is 0-based.
    2957                 :            :                  * Lower 16-bits indicate number of submission queues allocated.
    2958                 :            :                  * Upper 16-bits indicate number of completion queues allocated.
    2959                 :            :                  */
    2960   [ +  -  +  - ]:       3257 :                 sq_allocated = (cpl->cdw0 & 0xFFFF) + 1;
    2961   [ +  -  +  -  :       3257 :                 cq_allocated = (cpl->cdw0 >> 16) + 1;
                   +  - ]
    2962                 :            : 
    2963                 :            :                 /*
    2964                 :            :                  * For 1:1 queue mapping, set number of allocated queues to be minimum of
    2965                 :            :                  * submission and completion queues.
    2966                 :            :                  */
    2967         [ -  + ]:       3257 :                 min_allocated = spdk_min(sq_allocated, cq_allocated);
    2968                 :            : 
    2969                 :            :                 /* Set number of queues to be minimum of requested and actually allocated. */
    2970   [ +  -  +  -  :       3257 :                 ctrlr->opts.num_io_queues = spdk_min(min_allocated, ctrlr->opts.num_io_queues);
          +  -  +  -  #  
          #  #  #  #  #  
          +  -  +  -  +  
                      - ]
    2971                 :            :         }
    2972                 :            : 
    2973   [ +  -  +  -  :       3257 :         ctrlr->free_io_qids = spdk_bit_array_create(ctrlr->opts.num_io_queues + 1);
          +  -  +  -  +  
                      - ]
    2974   [ +  +  +  -  :       3257 :         if (ctrlr->free_io_qids == NULL) {
                   +  - ]
    2975                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    2976                 :          0 :                 return;
    2977                 :            :         }
    2978                 :            : 
    2979                 :            :         /* Initialize list of free I/O queue IDs. QID 0 is the admin queue (implicitly allocated). */
    2980   [ +  +  +  -  :     368100 :         for (i = 1; i <= ctrlr->opts.num_io_queues; i++) {
             +  -  +  + ]
    2981                 :     364843 :                 spdk_nvme_ctrlr_free_qid(ctrlr, i);
    2982                 :     105793 :         }
    2983                 :            : 
    2984                 :       4090 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS,
    2985   [ +  -  +  -  :       3257 :                              ctrlr->opts.admin_timeout_ms);
                   +  - ]
    2986                 :        833 : }
    2987                 :            : 
    2988                 :            : static int
    2989                 :       3257 : nvme_ctrlr_set_num_queues(struct spdk_nvme_ctrlr *ctrlr)
    2990                 :            : {
    2991                 :            :         int rc;
    2992                 :            : 
    2993   [ +  +  +  -  :       3257 :         if (ctrlr->opts.num_io_queues > SPDK_NVME_MAX_IO_QUEUES) {
             +  -  -  + ]
    2994   [ #  #  #  #  :          0 :                 NVME_CTRLR_NOTICELOG(ctrlr, "Limiting requested num_io_queues %u to max %d\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    2995                 :            :                                      ctrlr->opts.num_io_queues, SPDK_NVME_MAX_IO_QUEUES);
    2996   [ #  #  #  #  :          0 :                 ctrlr->opts.num_io_queues = SPDK_NVME_MAX_IO_QUEUES;
                   #  # ]
    2997   [ +  +  +  -  :       3257 :         } else if (ctrlr->opts.num_io_queues < 1) {
             +  -  +  - ]
    2998   [ +  -  -  +  :         39 :                 NVME_CTRLR_NOTICELOG(ctrlr, "Requested num_io_queues 0, increasing to 1\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    2999   [ #  #  #  #  :         39 :                 ctrlr->opts.num_io_queues = 1;
                   #  # ]
    3000                 :          0 :         }
    3001                 :            : 
    3002                 :       4090 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES,
    3003   [ +  -  +  -  :       3257 :                              ctrlr->opts.admin_timeout_ms);
                   +  - ]
    3004                 :            : 
    3005   [ +  -  +  -  :       3257 :         rc = nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->opts.num_io_queues,
                   +  - ]
    3006                 :        833 :                                            nvme_ctrlr_set_num_queues_done, ctrlr);
    3007         [ -  + ]:       3257 :         if (rc != 0) {
    3008                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3009                 :          0 :                 return rc;
    3010                 :            :         }
    3011                 :            : 
    3012                 :       3257 :         return 0;
    3013                 :        833 : }
    3014                 :            : 
    3015                 :            : static void
    3016                 :       2569 : nvme_ctrlr_set_keep_alive_timeout_done(void *arg, const struct spdk_nvme_cpl *cpl)
    3017                 :            : {
    3018                 :            :         uint32_t keep_alive_interval_us;
    3019                 :       2569 :         struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg;
    3020                 :            : 
    3021   [ +  +  +  +  :       2569 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
    3022   [ +  -  #  #  :          6 :                 if ((cpl->status.sct == SPDK_NVME_SCT_GENERIC) &&
          #  #  #  #  #  
                      # ]
    3023   [ +  +  #  #  :          6 :                     (cpl->status.sc == SPDK_NVME_SC_INVALID_FIELD)) {
                   #  # ]
    3024   [ -  +  -  +  :          3 :                         NVME_CTRLR_DEBUGLOG(ctrlr, "Keep alive timeout Get Feature is not supported\n");
          -  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3025                 :          0 :                 } else {
    3026   [ +  -  -  +  :          3 :                         NVME_CTRLR_ERRLOG(ctrlr, "Keep alive timeout Get Feature failed: SC %x SCT %x\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    3027                 :            :                                           cpl->status.sc, cpl->status.sct);
    3028   [ #  #  #  #  :          3 :                         ctrlr->opts.keep_alive_timeout_ms = 0;
                   #  # ]
    3029                 :          3 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3030                 :          3 :                         return;
    3031                 :            :                 }
    3032                 :          0 :         } else {
    3033   [ +  +  +  -  :       2563 :                 if (ctrlr->opts.keep_alive_timeout_ms != cpl->cdw0) {
          +  -  +  -  +  
                -  +  + ]
    3034   [ +  +  +  +  :         45 :                         NVME_CTRLR_DEBUGLOG(ctrlr, "Controller adjusted keep alive timeout to %u ms\n",
          +  -  -  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3035                 :            :                                             cpl->cdw0);
    3036                 :          7 :                 }
    3037                 :            : 
    3038   [ +  -  +  -  :       2563 :                 ctrlr->opts.keep_alive_timeout_ms = cpl->cdw0;
          +  -  +  -  +  
                      - ]
    3039                 :            :         }
    3040                 :            : 
    3041   [ +  +  +  -  :       2566 :         if (ctrlr->opts.keep_alive_timeout_ms == 0) {
             +  -  +  + ]
    3042   [ +  -  +  - ]:         42 :                 ctrlr->keep_alive_interval_ticks = 0;
    3043                 :          7 :         } else {
    3044   [ +  -  +  -  :       2524 :                 keep_alive_interval_us = ctrlr->opts.keep_alive_timeout_ms * 1000 / 2;
             +  -  +  - ]
    3045                 :            : 
    3046   [ +  +  +  +  :       2524 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Sending keep alive every %u us\n", keep_alive_interval_us);
          +  +  +  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3047                 :            : 
    3048   [ +  -  +  -  :       2524 :                 ctrlr->keep_alive_interval_ticks = (keep_alive_interval_us * spdk_get_ticks_hz()) /
                   +  - ]
    3049                 :            :                                                    UINT64_C(1000000);
    3050                 :            : 
    3051                 :            :                 /* Schedule the first Keep Alive to be sent as soon as possible. */
    3052   [ +  -  +  - ]:       2524 :                 ctrlr->next_keep_alive_tick = spdk_get_ticks();
    3053                 :            :         }
    3054                 :            : 
    3055         [ +  + ]:       2566 :         if (spdk_nvme_ctrlr_is_discovery(ctrlr)) {
    3056                 :        152 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE);
    3057                 :          0 :         } else {
    3058                 :       3245 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC,
    3059   [ +  -  +  -  :       2414 :                                      ctrlr->opts.admin_timeout_ms);
                   +  - ]
    3060                 :            :         }
    3061                 :        831 : }
    3062                 :            : 
    3063                 :            : static int
    3064                 :       3418 : nvme_ctrlr_set_keep_alive_timeout(struct spdk_nvme_ctrlr *ctrlr)
    3065                 :            : {
    3066                 :            :         int rc;
    3067                 :            : 
    3068   [ +  +  +  -  :       3418 :         if (ctrlr->opts.keep_alive_timeout_ms == 0) {
             +  -  +  - ]
    3069         [ -  + ]:        143 :                 if (spdk_nvme_ctrlr_is_discovery(ctrlr)) {
    3070                 :          0 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE);
    3071                 :          0 :                 } else {
    3072                 :        143 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC,
    3073   [ #  #  #  #  :        143 :                                              ctrlr->opts.admin_timeout_ms);
                   #  # ]
    3074                 :            :                 }
    3075                 :        143 :                 return 0;
    3076                 :            :         }
    3077                 :            : 
    3078                 :            :         /* Note: Discovery controller identify data does not populate KAS according to spec. */
    3079   [ +  +  +  +  :       3275 :         if (!spdk_nvme_ctrlr_is_discovery(ctrlr) && ctrlr->cdata.kas == 0) {
          +  -  +  -  +  
                      + ]
    3080   [ +  +  +  +  :        706 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Controller KAS is 0 - not enabling Keep Alive\n");
          +  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3081   [ +  -  +  -  :        706 :                 ctrlr->opts.keep_alive_timeout_ms = 0;
                   +  - ]
    3082                 :        708 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC,
    3083   [ +  -  +  -  :        706 :                                      ctrlr->opts.admin_timeout_ms);
                   +  - ]
    3084                 :        706 :                 return 0;
    3085                 :            :         }
    3086                 :            : 
    3087                 :       3400 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT,
    3088   [ +  -  +  -  :       2569 :                              ctrlr->opts.admin_timeout_ms);
                   +  - ]
    3089                 :            : 
    3090                 :            :         /* Retrieve actual keep alive timeout, since the controller may have adjusted it. */
    3091                 :       3400 :         rc = spdk_nvme_ctrlr_cmd_get_feature(ctrlr, SPDK_NVME_FEAT_KEEP_ALIVE_TIMER, 0, NULL, 0,
    3092                 :        831 :                                              nvme_ctrlr_set_keep_alive_timeout_done, ctrlr);
    3093         [ +  + ]:       2569 :         if (rc != 0) {
    3094   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Keep alive timeout Get Feature failed: %d\n", rc);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3095   [ #  #  #  #  :          0 :                 ctrlr->opts.keep_alive_timeout_ms = 0;
                   #  # ]
    3096                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3097                 :          0 :                 return rc;
    3098                 :            :         }
    3099                 :            : 
    3100                 :       2569 :         return 0;
    3101                 :        833 : }
    3102                 :            : 
    3103                 :            : static void
    3104                 :          0 : nvme_ctrlr_set_host_id_done(void *arg, const struct spdk_nvme_cpl *cpl)
    3105                 :            : {
    3106                 :          0 :         struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg;
    3107                 :            : 
    3108   [ #  #  #  #  :          0 :         if (spdk_nvme_cpl_is_error(cpl)) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    3109                 :            :                 /*
    3110                 :            :                  * Treat Set Features - Host ID failure as non-fatal, since the Host ID feature
    3111                 :            :                  * is optional.
    3112                 :            :                  */
    3113   [ #  #  #  #  :          0 :                 NVME_CTRLR_WARNLOG(ctrlr, "Set Features - Host ID failed: SC 0x%x SCT 0x%x\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    3114                 :            :                                    cpl->status.sc, cpl->status.sct);
    3115                 :          0 :         } else {
    3116   [ #  #  #  #  :          0 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Set Features - Host ID was successful\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3117                 :            :         }
    3118                 :            : 
    3119   [ #  #  #  #  :          0 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_TRANSPORT_READY, ctrlr->opts.admin_timeout_ms);
                   #  # ]
    3120                 :          0 : }
    3121                 :            : 
    3122                 :            : static int
    3123                 :       3242 : nvme_ctrlr_set_host_id(struct spdk_nvme_ctrlr *ctrlr)
    3124                 :            : {
    3125                 :            :         uint8_t *host_id;
    3126                 :            :         uint32_t host_id_size;
    3127                 :            :         int rc;
    3128                 :            : 
    3129   [ +  +  +  -  :       3242 :         if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) {
             +  -  +  + ]
    3130                 :            :                 /*
    3131                 :            :                  * NVMe-oF sends the host ID during Connect and doesn't allow
    3132                 :            :                  * Set Features - Host Identifier after Connect, so we don't need to do anything here.
    3133                 :            :                  */
    3134   [ +  +  +  +  :       2450 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "NVMe-oF transport - not sending Set Features - Host ID\n");
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3135   [ +  -  +  -  :       2450 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_TRANSPORT_READY, ctrlr->opts.admin_timeout_ms);
                   +  - ]
    3136                 :       2450 :                 return 0;
    3137                 :            :         }
    3138                 :            : 
    3139   [ +  +  +  -  :        792 :         if (ctrlr->cdata.ctratt.bits.host_id_exhid_supported) {
          +  -  +  -  -  
                      + ]
    3140   [ #  #  #  #  :          0 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Using 128-bit extended host identifier\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3141   [ #  #  #  # ]:          0 :                 host_id = ctrlr->opts.extended_host_id;
    3142                 :          0 :                 host_id_size = sizeof(ctrlr->opts.extended_host_id);
    3143                 :          0 :         } else {
    3144   [ +  +  +  +  :        792 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Using 64-bit host identifier\n");
          +  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3145   [ +  -  +  - ]:        792 :                 host_id = ctrlr->opts.host_id;
    3146                 :        792 :                 host_id_size = sizeof(ctrlr->opts.host_id);
    3147                 :            :         }
    3148                 :            : 
    3149                 :            :         /* If the user specified an all-zeroes host identifier, don't send the command. */
    3150         [ +  - ]:        792 :         if (spdk_mem_all_zero(host_id, host_id_size)) {
    3151   [ +  +  +  +  :        792 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "User did not specify host ID - not sending Set Features - Host ID\n");
          +  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3152   [ +  -  +  -  :        792 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_TRANSPORT_READY, ctrlr->opts.admin_timeout_ms);
                   +  - ]
    3153                 :        792 :                 return 0;
    3154                 :            :         }
    3155                 :            : 
    3156   [ #  #  #  #  :          0 :         SPDK_LOGDUMP(nvme, "host_id", host_id, host_id_size);
                   #  # ]
    3157                 :            : 
    3158                 :          0 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_HOST_ID,
    3159   [ #  #  #  #  :          0 :                              ctrlr->opts.admin_timeout_ms);
                   #  # ]
    3160                 :            : 
    3161                 :          0 :         rc = nvme_ctrlr_cmd_set_host_id(ctrlr, host_id, host_id_size, nvme_ctrlr_set_host_id_done, ctrlr);
    3162         [ #  # ]:          0 :         if (rc != 0) {
    3163   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Set Features - Host ID failed: %d\n", rc);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3164                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3165                 :          0 :                 return rc;
    3166                 :            :         }
    3167                 :            : 
    3168                 :          0 :         return 0;
    3169                 :        833 : }
    3170                 :            : 
    3171                 :            : void
    3172                 :        155 : nvme_ctrlr_update_namespaces(struct spdk_nvme_ctrlr *ctrlr)
    3173                 :            : {
    3174                 :            :         uint32_t nsid;
    3175                 :            :         struct spdk_nvme_ns *ns;
    3176                 :            : 
    3177         [ #  # ]:        155 :         for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
    3178         [ +  + ]:        411 :              nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
    3179                 :        256 :                 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
    3180                 :        256 :                 nvme_ns_construct(ns, nsid, ctrlr);
    3181                 :          0 :         }
    3182                 :        155 : }
    3183                 :            : 
    3184                 :            : static int
    3185                 :        155 : nvme_ctrlr_clear_changed_ns_log(struct spdk_nvme_ctrlr *ctrlr)
    3186                 :            : {
    3187                 :            :         struct nvme_completion_poll_status      *status;
    3188                 :        155 :         int             rc = -ENOMEM;
    3189                 :        155 :         char            *buffer = NULL;
    3190                 :            :         uint32_t        nsid;
    3191                 :        155 :         size_t          buf_size = (SPDK_NVME_MAX_CHANGED_NAMESPACES * sizeof(uint32_t));
    3192                 :            : 
    3193   [ -  +  #  #  :        155 :         if (ctrlr->opts.disable_read_changed_ns_list_log_page) {
             #  #  #  # ]
    3194                 :          0 :                 return 0;
    3195                 :            :         }
    3196                 :            : 
    3197                 :        155 :         buffer = spdk_dma_zmalloc(buf_size, 4096, NULL);
    3198         [ -  + ]:        155 :         if (!buffer) {
    3199   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate buffer for getting "
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3200                 :            :                                   "changed ns log.\n");
    3201                 :          0 :                 return rc;
    3202                 :            :         }
    3203                 :            : 
    3204                 :        155 :         status = calloc(1, sizeof(*status));
    3205         [ -  + ]:        155 :         if (!status) {
    3206   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3207                 :          0 :                 goto free_buffer;
    3208                 :            :         }
    3209                 :            : 
    3210                 :        155 :         rc = spdk_nvme_ctrlr_cmd_get_log_page(ctrlr,
    3211                 :            :                                               SPDK_NVME_LOG_CHANGED_NS_LIST,
    3212                 :            :                                               SPDK_NVME_GLOBAL_NS_TAG,
    3213                 :          0 :                                               buffer, buf_size, 0,
    3214                 :          0 :                                               nvme_completion_poll_cb, status);
    3215                 :            : 
    3216         [ -  + ]:        155 :         if (rc) {
    3217   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_cmd_get_log_page() failed: rc=%d\n", rc);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3218                 :          0 :                 free(status);
    3219                 :          0 :                 goto free_buffer;
    3220                 :            :         }
    3221                 :            : 
    3222   [ #  #  #  # ]:        155 :         rc = nvme_wait_for_completion_timeout(ctrlr->adminq, status,
    3223   [ #  #  #  #  :        155 :                                               ctrlr->opts.admin_timeout_ms * 1000);
                   #  # ]
    3224   [ +  +  +  -  :        155 :         if (!status->timed_out) {
             #  #  #  # ]
    3225                 :        155 :                 free(status);
    3226                 :          0 :         }
    3227                 :            : 
    3228         [ -  + ]:        155 :         if (rc) {
    3229   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "wait for spdk_nvme_ctrlr_cmd_get_log_page failed: rc=%d\n", rc);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3230                 :          0 :                 goto free_buffer;
    3231                 :            :         }
    3232                 :            : 
    3233                 :            :         /* only check the case of overflow. */
    3234                 :        155 :         nsid = from_le32(buffer);
    3235         [ +  - ]:        155 :         if (nsid == 0xffffffffu) {
    3236   [ #  #  #  #  :          0 :                 NVME_CTRLR_WARNLOG(ctrlr, "changed ns log overflowed.\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3237                 :          0 :         }
    3238                 :            : 
    3239                 :        155 : free_buffer:
    3240                 :        155 :         spdk_dma_free(buffer);
    3241                 :        155 :         return rc;
    3242                 :          0 : }
    3243                 :            : 
    3244                 :            : static void
    3245                 :        334 : nvme_ctrlr_process_async_event(struct spdk_nvme_ctrlr *ctrlr,
    3246                 :            :                                const struct spdk_nvme_cpl *cpl)
    3247                 :            : {
    3248                 :            :         union spdk_nvme_async_event_completion event;
    3249                 :            :         struct spdk_nvme_ctrlr_process *active_proc;
    3250                 :            :         int rc;
    3251                 :            : 
    3252   [ #  #  #  # ]:        334 :         event.raw = cpl->cdw0;
    3253                 :            : 
    3254   [ +  +  #  # ]:        334 :         if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) &&
    3255         [ +  + ]:        274 :             (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) {
    3256                 :        155 :                 nvme_ctrlr_clear_changed_ns_log(ctrlr);
    3257                 :            : 
    3258                 :        155 :                 rc = nvme_ctrlr_identify_active_ns(ctrlr);
    3259         [ -  + ]:        155 :                 if (rc) {
    3260                 :          0 :                         return;
    3261                 :            :                 }
    3262                 :        155 :                 nvme_ctrlr_update_namespaces(ctrlr);
    3263                 :        155 :                 nvme_io_msg_ctrlr_update(ctrlr);
    3264                 :          0 :         }
    3265                 :            : 
    3266   [ +  +  #  # ]:        334 :         if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) &&
    3267         [ +  + ]:        274 :             (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) {
    3268   [ +  +  +  +  :        103 :                 if (!ctrlr->opts.disable_read_ana_log_page) {
          #  #  #  #  #  
                      # ]
    3269                 :          3 :                         rc = nvme_ctrlr_update_ana_log_page(ctrlr);
    3270         [ -  + ]:          3 :                         if (rc) {
    3271                 :          0 :                                 return;
    3272                 :            :                         }
    3273                 :          3 :                         nvme_ctrlr_parse_ana_log_page(ctrlr, nvme_ctrlr_update_ns_ana_states,
    3274                 :          0 :                                                       ctrlr);
    3275                 :          0 :                 }
    3276                 :          0 :         }
    3277                 :            : 
    3278                 :        334 :         active_proc = nvme_ctrlr_get_current_process(ctrlr);
    3279   [ +  -  +  +  :        334 :         if (active_proc && active_proc->aer_cb_fn) {
             #  #  #  # ]
    3280   [ #  #  #  #  :        175 :                 active_proc->aer_cb_fn(active_proc->aer_cb_arg, cpl);
          #  #  #  #  #  
                #  #  # ]
    3281                 :          0 :         }
    3282                 :          0 : }
    3283                 :            : 
    3284                 :            : static void
    3285                 :        937 : nvme_ctrlr_queue_async_event(struct spdk_nvme_ctrlr *ctrlr,
    3286                 :            :                              const struct spdk_nvme_cpl *cpl)
    3287                 :            : {
    3288                 :            :         struct  spdk_nvme_ctrlr_aer_completion_list *nvme_event;
    3289                 :            :         struct spdk_nvme_ctrlr_process *proc;
    3290                 :            : 
    3291                 :            :         /* Add async event to each process objects event list */
    3292   [ +  +  #  #  :       1909 :         TAILQ_FOREACH(proc, &ctrlr->active_procs, tailq) {
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3293                 :            :                 /* Must be shared memory so other processes can access */
    3294                 :        972 :                 nvme_event = spdk_zmalloc(sizeof(*nvme_event), 0, NULL, SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_SHARE);
    3295         [ -  + ]:        972 :                 if (!nvme_event) {
    3296   [ #  #  #  #  :          0 :                         NVME_CTRLR_ERRLOG(ctrlr, "Alloc nvme event failed, ignore the event\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3297                 :          0 :                         return;
    3298                 :            :                 }
    3299         [ #  # ]:        972 :                 nvme_event->cpl = *cpl;
    3300                 :            : 
    3301   [ #  #  #  #  :        972 :                 STAILQ_INSERT_TAIL(&proc->async_events, nvme_event, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3302                 :          0 :         }
    3303                 :          0 : }
    3304                 :            : 
    3305                 :            : static void
    3306                 :   51956874 : nvme_ctrlr_complete_queued_async_events(struct spdk_nvme_ctrlr *ctrlr)
    3307                 :            : {
    3308                 :            :         struct  spdk_nvme_ctrlr_aer_completion_list  *nvme_event, *nvme_event_tmp;
    3309                 :            :         struct spdk_nvme_ctrlr_process  *active_proc;
    3310                 :            : 
    3311                 :   51956874 :         active_proc = nvme_ctrlr_get_current_process(ctrlr);
    3312                 :            : 
    3313   [ +  +  +  -  :   51957208 :         STAILQ_FOREACH_SAFE(nvme_event, &active_proc->async_events, link, nvme_event_tmp) {
          +  -  +  -  #  
          #  #  #  #  #  
                   +  - ]
    3314   [ +  -  +  +  :        334 :                 STAILQ_REMOVE(&active_proc->async_events, nvme_event,
          -  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  # ]
    3315                 :            :                               spdk_nvme_ctrlr_aer_completion_list, link);
    3316         [ #  # ]:        334 :                 nvme_ctrlr_process_async_event(ctrlr, &nvme_event->cpl);
    3317                 :        334 :                 spdk_free(nvme_event);
    3318                 :            : 
    3319                 :          0 :         }
    3320                 :   51956874 : }
    3321                 :            : 
    3322                 :            : static void
    3323                 :      13943 : nvme_ctrlr_async_event_cb(void *arg, const struct spdk_nvme_cpl *cpl)
    3324                 :            : {
    3325                 :      13943 :         struct nvme_async_event_request *aer = arg;
    3326   [ +  -  +  - ]:      13943 :         struct spdk_nvme_ctrlr          *ctrlr = aer->ctrlr;
    3327                 :            : 
    3328   [ +  -  +  -  :      17275 :         if (cpl->status.sct == SPDK_NVME_SCT_GENERIC &&
          +  -  +  -  -  
                      + ]
    3329   [ +  +  +  -  :      13943 :             cpl->status.sc == SPDK_NVME_SC_ABORTED_SQ_DELETION) {
                   +  - ]
    3330                 :            :                 /*
    3331                 :            :                  *  This is simulated when controller is being shut down, to
    3332                 :            :                  *  effectively abort outstanding asynchronous event requests
    3333                 :            :                  *  and make sure all memory is freed.  Do not repost the
    3334                 :            :                  *  request in this case.
    3335                 :            :                  */
    3336                 :      13006 :                 return;
    3337                 :            :         }
    3338                 :            : 
    3339   [ -  +  #  #  :        937 :         if (cpl->status.sct == SPDK_NVME_SCT_COMMAND_SPECIFIC &&
          #  #  #  #  #  
                      # ]
    3340   [ #  #  #  #  :          0 :             cpl->status.sc == SPDK_NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED) {
                   #  # ]
    3341                 :            :                 /*
    3342                 :            :                  *  SPDK will only send as many AERs as the device says it supports,
    3343                 :            :                  *  so this status code indicates an out-of-spec device.  Do not repost
    3344                 :            :                  *  the request in this case.
    3345                 :            :                  */
    3346   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Controller appears out-of-spec for asynchronous event request\n"
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3347                 :            :                                   "handling.  Do not repost this AER.\n");
    3348                 :          0 :                 return;
    3349                 :            :         }
    3350                 :            : 
    3351                 :            :         /* Add the events to the list */
    3352                 :        937 :         nvme_ctrlr_queue_async_event(ctrlr, cpl);
    3353                 :            : 
    3354                 :            :         /* If the ctrlr was removed or in the destruct state, we should not send aer again */
    3355   [ +  +  +  +  :        937 :         if (ctrlr->is_removed || ctrlr->is_destructed) {
          -  +  +  +  #  
          #  #  #  #  #  
                   #  # ]
    3356                 :        290 :                 return;
    3357                 :            :         }
    3358                 :            : 
    3359                 :            :         /*
    3360                 :            :          * Repost another asynchronous event request to replace the one
    3361                 :            :          *  that just completed.
    3362                 :            :          */
    3363         [ -  + ]:        647 :         if (nvme_ctrlr_construct_and_submit_aer(ctrlr, aer)) {
    3364                 :            :                 /*
    3365                 :            :                  * We can't do anything to recover from a failure here,
    3366                 :            :                  * so just print a warning message and leave the AER unsubmitted.
    3367                 :            :                  */
    3368   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "resubmitting AER failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3369                 :          0 :         }
    3370                 :       3332 : }
    3371                 :            : 
    3372                 :            : static int
    3373                 :      14100 : nvme_ctrlr_construct_and_submit_aer(struct spdk_nvme_ctrlr *ctrlr,
    3374                 :            :                                     struct nvme_async_event_request *aer)
    3375                 :            : {
    3376                 :            :         struct nvme_request *req;
    3377                 :            : 
    3378   [ +  -  +  - ]:      14100 :         aer->ctrlr = ctrlr;
    3379   [ +  -  +  - ]:      14100 :         req = nvme_allocate_request_null(ctrlr->adminq, nvme_ctrlr_async_event_cb, aer);
    3380   [ +  -  +  - ]:      14100 :         aer->req = req;
    3381         [ +  + ]:      14100 :         if (req == NULL) {
    3382                 :          0 :                 return -1;
    3383                 :            :         }
    3384                 :            : 
    3385   [ +  -  +  - ]:      14100 :         req->cmd.opc = SPDK_NVME_OPC_ASYNC_EVENT_REQUEST;
    3386                 :      14100 :         return nvme_ctrlr_submit_admin_request(ctrlr, req);
    3387                 :       3332 : }
    3388                 :            : 
    3389                 :            : static void
    3390                 :       3409 : nvme_ctrlr_configure_aer_done(void *arg, const struct spdk_nvme_cpl *cpl)
    3391                 :            : {
    3392                 :            :         struct nvme_async_event_request         *aer;
    3393                 :            :         int                                     rc;
    3394                 :            :         uint32_t                                i;
    3395                 :       3409 :         struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg;
    3396                 :            : 
    3397   [ +  -  +  +  :       3409 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   -  + ]
    3398   [ #  #  #  #  :          0 :                 NVME_CTRLR_NOTICELOG(ctrlr, "nvme_ctrlr_configure_aer failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3399   [ #  #  #  # ]:          0 :                 ctrlr->num_aers = 0;
    3400                 :          0 :         } else {
    3401                 :            :                 /* aerl is a zero-based value, so we need to add 1 here. */
    3402   [ +  -  +  -  :       3409 :                 ctrlr->num_aers = spdk_min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl + 1));
          +  -  +  -  -  
          +  +  -  +  -  
          +  -  +  -  +  
                -  +  - ]
    3403                 :            :         }
    3404                 :            : 
    3405   [ +  +  +  -  :      16862 :         for (i = 0; i < ctrlr->num_aers; i++) {
                   +  + ]
    3406   [ +  -  +  -  :      13453 :                 aer = &ctrlr->aer[i];
                   +  - ]
    3407                 :      13453 :                 rc = nvme_ctrlr_construct_and_submit_aer(ctrlr, aer);
    3408         [ +  + ]:      13453 :                 if (rc) {
    3409   [ #  #  #  #  :          0 :                         NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_construct_and_submit_aer failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3410                 :          0 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3411                 :          0 :                         return;
    3412                 :            :                 }
    3413                 :       3332 :         }
    3414   [ +  -  +  -  :       3409 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT, ctrlr->opts.admin_timeout_ms);
                   +  - ]
    3415                 :        833 : }
    3416                 :            : 
    3417                 :            : static int
    3418                 :       3409 : nvme_ctrlr_configure_aer(struct spdk_nvme_ctrlr *ctrlr)
    3419                 :            : {
    3420                 :            :         union spdk_nvme_feat_async_event_configuration  config;
    3421                 :            :         int                                             rc;
    3422                 :            : 
    3423                 :       3409 :         config.raw = 0;
    3424                 :            : 
    3425         [ +  + ]:       3409 :         if (spdk_nvme_ctrlr_is_discovery(ctrlr)) {
    3426                 :        152 :                 config.bits.discovery_log_change_notice = 1;
    3427                 :          0 :         } else {
    3428                 :       3257 :                 config.bits.crit_warn.bits.available_spare = 1;
    3429                 :       3257 :                 config.bits.crit_warn.bits.temperature = 1;
    3430                 :       3257 :                 config.bits.crit_warn.bits.device_reliability = 1;
    3431                 :       3257 :                 config.bits.crit_warn.bits.read_only = 1;
    3432                 :       3257 :                 config.bits.crit_warn.bits.volatile_memory_backup = 1;
    3433                 :            : 
    3434   [ +  +  +  -  :       3257 :                 if (ctrlr->vs.raw >= SPDK_NVME_VERSION(1, 2, 0)) {
          +  -  +  -  +  
                -  -  + ]
    3435   [ +  +  +  -  :       3207 :                         if (ctrlr->cdata.oaes.ns_attribute_notices) {
             +  -  +  + ]
    3436                 :       3092 :                                 config.bits.ns_attr_notice = 1;
    3437                 :        831 :                         }
    3438   [ +  +  +  -  :       3207 :                         if (ctrlr->cdata.oaes.fw_activation_notices) {
             +  -  +  + ]
    3439                 :        125 :                                 config.bits.fw_activation_notice = 1;
    3440                 :          2 :                         }
    3441   [ +  +  +  -  :       3207 :                         if (ctrlr->cdata.oaes.ana_change_notices) {
             +  -  +  - ]
    3442                 :        339 :                                 config.bits.ana_change_notice = 1;
    3443                 :          0 :                         }
    3444                 :        833 :                 }
    3445   [ +  +  +  +  :       3257 :                 if (ctrlr->vs.raw >= SPDK_NVME_VERSION(1, 3, 0) && ctrlr->cdata.lpa.telemetry) {
          +  -  +  -  +  
          -  +  +  +  -  
          +  -  +  -  +  
                      - ]
    3446                 :         23 :                         config.bits.telemetry_log_notice = 1;
    3447                 :          0 :                 }
    3448                 :            :         }
    3449                 :            : 
    3450                 :       4242 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER,
    3451   [ +  -  +  -  :       3409 :                              ctrlr->opts.admin_timeout_ms);
                   +  - ]
    3452                 :            : 
    3453                 :       4242 :         rc = nvme_ctrlr_cmd_set_async_event_config(ctrlr, config,
    3454                 :            :                         nvme_ctrlr_configure_aer_done,
    3455                 :        833 :                         ctrlr);
    3456         [ -  + ]:       3409 :         if (rc != 0) {
    3457                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3458                 :          0 :                 return rc;
    3459                 :            :         }
    3460                 :            : 
    3461                 :       3409 :         return 0;
    3462                 :        833 : }
    3463                 :            : 
    3464                 :            : struct spdk_nvme_ctrlr_process *
    3465                 :  156502772 : nvme_ctrlr_get_process(struct spdk_nvme_ctrlr *ctrlr, pid_t pid)
    3466                 :            : {
    3467                 :            :         struct spdk_nvme_ctrlr_process  *active_proc;
    3468                 :            : 
    3469   [ +  +  +  -  :  288032849 :         TAILQ_FOREACH(active_proc, &ctrlr->active_procs, tailq) {
          +  -  +  +  #  
             #  #  #  #  
                      # ]
    3470   [ +  +  +  -  :  288029102 :                 if (active_proc->pid == pid) {
                   -  + ]
    3471                 :  156499025 :                         return active_proc;
    3472                 :            :                 }
    3473                 :          0 :         }
    3474                 :            : 
    3475                 :       3747 :         return NULL;
    3476                 :     551939 : }
    3477                 :            : 
    3478                 :            : struct spdk_nvme_ctrlr_process *
    3479                 :  156498960 : nvme_ctrlr_get_current_process(struct spdk_nvme_ctrlr *ctrlr)
    3480                 :            : {
    3481                 :  156498960 :         return nvme_ctrlr_get_process(ctrlr, getpid());
    3482                 :            : }
    3483                 :            : 
    3484                 :            : /**
    3485                 :            :  * This function will be called when a process is using the controller.
    3486                 :            :  *  1. For the primary process, it is called when constructing the controller.
    3487                 :            :  *  2. For the secondary process, it is called at probing the controller.
    3488                 :            :  * Note: will check whether the process is already added for the same process.
    3489                 :            :  */
    3490                 :            : int
    3491                 :       3569 : nvme_ctrlr_add_process(struct spdk_nvme_ctrlr *ctrlr, void *devhandle)
    3492                 :            : {
    3493                 :            :         struct spdk_nvme_ctrlr_process  *ctrlr_proc;
    3494                 :       3569 :         pid_t                           pid = getpid();
    3495                 :            : 
    3496                 :            :         /* Check whether the process is already added or not */
    3497         [ +  + ]:       3569 :         if (nvme_ctrlr_get_process(ctrlr, pid)) {
    3498                 :         41 :                 return 0;
    3499                 :            :         }
    3500                 :            : 
    3501                 :            :         /* Initialize the per process properties for this ctrlr */
    3502                 :       3528 :         ctrlr_proc = spdk_zmalloc(sizeof(struct spdk_nvme_ctrlr_process),
    3503                 :            :                                   64, NULL, SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_SHARE);
    3504         [ +  + ]:       3528 :         if (ctrlr_proc == NULL) {
    3505   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "failed to allocate memory to track the process props\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3506                 :            : 
    3507                 :          0 :                 return -1;
    3508                 :            :         }
    3509                 :            : 
    3510   [ +  -  +  - ]:       3528 :         ctrlr_proc->is_primary = spdk_process_is_primary();
    3511   [ +  -  +  - ]:       3528 :         ctrlr_proc->pid = pid;
    3512   [ +  -  +  -  :       3528 :         STAILQ_INIT(&ctrlr_proc->active_reqs);
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
    3513   [ +  -  +  - ]:       3528 :         ctrlr_proc->devhandle = devhandle;
    3514   [ +  -  +  - ]:       3528 :         ctrlr_proc->ref = 0;
    3515   [ +  -  +  -  :       3528 :         TAILQ_INIT(&ctrlr_proc->allocated_io_qpairs);
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
    3516   [ +  -  +  -  :       3528 :         STAILQ_INIT(&ctrlr_proc->async_events);
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
    3517                 :            : 
    3518   [ +  -  +  -  :       3528 :         TAILQ_INSERT_TAIL(&ctrlr->active_procs, ctrlr_proc, tailq);
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
    3519                 :            : 
    3520                 :       3528 :         return 0;
    3521                 :        833 : }
    3522                 :            : 
    3523                 :            : /**
    3524                 :            :  * This function will be called when the process detaches the controller.
    3525                 :            :  * Note: the ctrlr_lock must be held when calling this function.
    3526                 :            :  */
    3527                 :            : static void
    3528                 :        228 : nvme_ctrlr_remove_process(struct spdk_nvme_ctrlr *ctrlr,
    3529                 :            :                           struct spdk_nvme_ctrlr_process *proc)
    3530                 :            : {
    3531                 :            :         struct spdk_nvme_qpair  *qpair, *tmp_qpair;
    3532                 :            : 
    3533   [ -  +  #  #  :        228 :         assert(STAILQ_EMPTY(&proc->active_reqs));
          #  #  #  #  #  
                      # ]
    3534                 :            : 
    3535   [ +  +  #  #  :        248 :         TAILQ_FOREACH_SAFE(qpair, &proc->allocated_io_qpairs, per_process_tailq, tmp_qpair) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    3536                 :         20 :                 spdk_nvme_ctrlr_free_io_qpair(qpair);
    3537                 :          0 :         }
    3538                 :            : 
    3539   [ +  +  #  #  :        228 :         TAILQ_REMOVE(&ctrlr->active_procs, proc, tailq);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
    3540                 :            : 
    3541   [ +  -  #  #  :        228 :         if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
             #  #  #  # ]
    3542   [ #  #  #  # ]:        228 :                 spdk_pci_device_detach(proc->devhandle);
    3543                 :          0 :         }
    3544                 :            : 
    3545                 :        228 :         spdk_free(proc);
    3546                 :        228 : }
    3547                 :            : 
    3548                 :            : /**
    3549                 :            :  * This function will be called when the process exited unexpectedly
    3550                 :            :  *  in order to free any incomplete nvme request, allocated IO qpairs
    3551                 :            :  *  and allocated memory.
    3552                 :            :  * Note: the ctrlr_lock must be held when calling this function.
    3553                 :            :  */
    3554                 :            : static void
    3555                 :          9 : nvme_ctrlr_cleanup_process(struct spdk_nvme_ctrlr_process *proc)
    3556                 :            : {
    3557                 :            :         struct nvme_request     *req, *tmp_req;
    3558                 :            :         struct spdk_nvme_qpair  *qpair, *tmp_qpair;
    3559                 :            :         struct spdk_nvme_ctrlr_aer_completion_list *event;
    3560                 :            : 
    3561   [ -  +  #  #  :          9 :         STAILQ_FOREACH_SAFE(req, &proc->active_reqs, stailq, tmp_req) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    3562   [ #  #  #  #  :          0 :                 STAILQ_REMOVE(&proc->active_reqs, req, nvme_request, stailq);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  # ]
    3563                 :            : 
    3564   [ #  #  #  #  :          0 :                 assert(req->pid == proc->pid);
          #  #  #  #  #  
                #  #  # ]
    3565                 :          0 :                 nvme_cleanup_user_req(req);
    3566                 :          0 :                 nvme_free_request(req);
    3567                 :          0 :         }
    3568                 :            : 
    3569                 :            :         /* Remove async event from each process objects event list */
    3570   [ -  +  #  #  :          9 :         while (!STAILQ_EMPTY(&proc->async_events)) {
             #  #  #  # ]
    3571   [ #  #  #  #  :          0 :                 event = STAILQ_FIRST(&proc->async_events);
                   #  # ]
    3572   [ #  #  #  #  :          0 :                 STAILQ_REMOVE_HEAD(&proc->async_events, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3573                 :          0 :                 spdk_free(event);
    3574                 :            :         }
    3575                 :            : 
    3576   [ -  +  #  #  :          9 :         TAILQ_FOREACH_SAFE(qpair, &proc->allocated_io_qpairs, per_process_tailq, tmp_qpair) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    3577   [ #  #  #  #  :          0 :                 TAILQ_REMOVE(&proc->allocated_io_qpairs, qpair, per_process_tailq);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
    3578                 :            : 
    3579                 :            :                 /*
    3580                 :            :                  * The process may have been killed while some qpairs were in their
    3581                 :            :                  *  completion context.  Clear that flag here to allow these IO
    3582                 :            :                  *  qpairs to be deleted.
    3583                 :            :                  */
    3584         [ #  # ]:          0 :                 qpair->in_completion_context = 0;
    3585                 :            : 
    3586         [ #  # ]:          0 :                 qpair->no_deletion_notification_needed = 1;
    3587                 :            : 
    3588                 :          0 :                 spdk_nvme_ctrlr_free_io_qpair(qpair);
    3589                 :          0 :         }
    3590                 :            : 
    3591                 :          9 :         spdk_free(proc);
    3592                 :          9 : }
    3593                 :            : 
    3594                 :            : /**
    3595                 :            :  * This function will be called when destructing the controller.
    3596                 :            :  *  1. There is no more admin request on this controller.
    3597                 :            :  *  2. Clean up any left resource allocation when its associated process is gone.
    3598                 :            :  */
    3599                 :            : void
    3600                 :       3442 : nvme_ctrlr_free_processes(struct spdk_nvme_ctrlr *ctrlr)
    3601                 :            : {
    3602                 :            :         struct spdk_nvme_ctrlr_process  *active_proc, *tmp;
    3603                 :            : 
    3604                 :            :         /* Free all the processes' properties and make sure no pending admin IOs */
    3605   [ +  +  +  -  :       6729 :         TAILQ_FOREACH_SAFE(active_proc, &ctrlr->active_procs, tailq, tmp) {
          +  -  +  +  +  
          -  +  -  +  -  
                   +  + ]
    3606   [ +  +  +  -  :       3287 :                 TAILQ_REMOVE(&ctrlr->active_procs, active_proc, tailq);
          +  -  -  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
                -  +  - ]
    3607                 :            : 
    3608   [ +  +  +  -  :       3287 :                 assert(STAILQ_EMPTY(&active_proc->active_reqs));
          +  -  +  -  #  
                      # ]
    3609                 :            : 
    3610                 :       3287 :                 spdk_free(active_proc);
    3611                 :        833 :         }
    3612                 :       3442 : }
    3613                 :            : 
    3614                 :            : /**
    3615                 :            :  * This function will be called when any other process attaches or
    3616                 :            :  *  detaches the controller in order to cleanup those unexpectedly
    3617                 :            :  *  terminated processes.
    3618                 :            :  * Note: the ctrlr_lock must be held when calling this function.
    3619                 :            :  */
    3620                 :            : static int
    3621                 :      10021 : nvme_ctrlr_remove_inactive_proc(struct spdk_nvme_ctrlr *ctrlr)
    3622                 :            : {
    3623                 :            :         struct spdk_nvme_ctrlr_process  *active_proc, *tmp;
    3624                 :      10021 :         int                             active_proc_count = 0;
    3625                 :            : 
    3626   [ +  +  +  -  :      20945 :         TAILQ_FOREACH_SAFE(active_proc, &ctrlr->active_procs, tailq, tmp) {
          +  -  +  +  +  
          -  +  -  +  -  
                   +  + ]
    3627   [ +  +  +  -  :      10924 :                 if ((kill(active_proc->pid, 0) == -1) && (errno == ESRCH)) {
          -  +  #  #  #  
                      # ]
    3628   [ +  -  -  +  :          9 :                         NVME_CTRLR_ERRLOG(ctrlr, "process %d terminated unexpected\n", active_proc->pid);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  # ]
    3629                 :            : 
    3630   [ +  -  #  #  :          9 :                         TAILQ_REMOVE(&ctrlr->active_procs, active_proc, tailq);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
    3631                 :            : 
    3632                 :          9 :                         nvme_ctrlr_cleanup_process(active_proc);
    3633                 :          0 :                 } else {
    3634         [ +  - ]:      10915 :                         active_proc_count++;
    3635                 :            :                 }
    3636                 :       2499 :         }
    3637                 :            : 
    3638                 :      10021 :         return active_proc_count;
    3639                 :            : }
    3640                 :            : 
    3641                 :            : void
    3642                 :       3349 : nvme_ctrlr_proc_get_ref(struct spdk_nvme_ctrlr *ctrlr)
    3643                 :            : {
    3644                 :            :         struct spdk_nvme_ctrlr_process  *active_proc;
    3645                 :            : 
    3646                 :       3349 :         nvme_ctrlr_lock(ctrlr);
    3647                 :            : 
    3648                 :       3349 :         nvme_ctrlr_remove_inactive_proc(ctrlr);
    3649                 :            : 
    3650                 :       3349 :         active_proc = nvme_ctrlr_get_current_process(ctrlr);
    3651         [ +  - ]:       3349 :         if (active_proc) {
    3652   [ +  -  +  - ]:       3349 :                 active_proc->ref++;
    3653                 :        833 :         }
    3654                 :            : 
    3655                 :       3349 :         nvme_ctrlr_unlock(ctrlr);
    3656                 :       3349 : }
    3657                 :            : 
    3658                 :            : void
    3659                 :       3336 : nvme_ctrlr_proc_put_ref(struct spdk_nvme_ctrlr *ctrlr)
    3660                 :            : {
    3661                 :            :         struct spdk_nvme_ctrlr_process  *active_proc;
    3662                 :            :         int                             proc_count;
    3663                 :            : 
    3664                 :       3336 :         nvme_ctrlr_lock(ctrlr);
    3665                 :            : 
    3666                 :       3336 :         proc_count = nvme_ctrlr_remove_inactive_proc(ctrlr);
    3667                 :            : 
    3668                 :       3336 :         active_proc = nvme_ctrlr_get_current_process(ctrlr);
    3669         [ +  + ]:       3336 :         if (active_proc) {
    3670   [ +  -  +  - ]:       3336 :                 active_proc->ref--;
    3671   [ +  +  +  -  :       3336 :                 assert(active_proc->ref >= 0);
             +  -  #  # ]
    3672                 :            : 
    3673                 :            :                 /*
    3674                 :            :                  * The last active process will be removed at the end of
    3675                 :            :                  * the destruction of the controller.
    3676                 :            :                  */
    3677   [ +  -  +  +  :       3336 :                 if (active_proc->ref == 0 && proc_count != 1) {
             +  -  +  - ]
    3678                 :        225 :                         nvme_ctrlr_remove_process(ctrlr, active_proc);
    3679                 :          0 :                 }
    3680                 :        833 :         }
    3681                 :            : 
    3682                 :       3336 :         nvme_ctrlr_unlock(ctrlr);
    3683                 :       3336 : }
    3684                 :            : 
    3685                 :            : int
    3686                 :       3336 : nvme_ctrlr_get_ref_count(struct spdk_nvme_ctrlr *ctrlr)
    3687                 :            : {
    3688                 :            :         struct spdk_nvme_ctrlr_process  *active_proc;
    3689                 :       3336 :         int                             ref = 0;
    3690                 :            : 
    3691                 :       3336 :         nvme_ctrlr_lock(ctrlr);
    3692                 :            : 
    3693                 :       3336 :         nvme_ctrlr_remove_inactive_proc(ctrlr);
    3694                 :            : 
    3695   [ +  +  +  -  :       6967 :         TAILQ_FOREACH(active_proc, &ctrlr->active_procs, tailq) {
          +  -  +  +  +  
             -  +  -  +  
                      - ]
    3696   [ +  -  +  -  :       3631 :                 ref += active_proc->ref;
                   +  - ]
    3697                 :        833 :         }
    3698                 :            : 
    3699                 :       3336 :         nvme_ctrlr_unlock(ctrlr);
    3700                 :            : 
    3701                 :       3336 :         return ref;
    3702                 :            : }
    3703                 :            : 
    3704                 :            : /**
    3705                 :            :  *  Get the PCI device handle which is only visible to its associated process.
    3706                 :            :  */
    3707                 :            : struct spdk_pci_device *
    3708                 :        847 : nvme_ctrlr_proc_get_devhandle(struct spdk_nvme_ctrlr *ctrlr)
    3709                 :            : {
    3710                 :            :         struct spdk_nvme_ctrlr_process  *active_proc;
    3711                 :        847 :         struct spdk_pci_device          *devhandle = NULL;
    3712                 :            : 
    3713                 :        847 :         nvme_ctrlr_lock(ctrlr);
    3714                 :            : 
    3715                 :        847 :         active_proc = nvme_ctrlr_get_current_process(ctrlr);
    3716         [ +  - ]:        847 :         if (active_proc) {
    3717   [ +  -  +  - ]:        847 :                 devhandle = active_proc->devhandle;
    3718                 :          2 :         }
    3719                 :            : 
    3720                 :        847 :         nvme_ctrlr_unlock(ctrlr);
    3721                 :            : 
    3722                 :        847 :         return devhandle;
    3723                 :            : }
    3724                 :            : 
    3725                 :            : static void
    3726                 :       3415 : nvme_ctrlr_process_init_vs_done(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl)
    3727                 :            : {
    3728                 :       3415 :         struct spdk_nvme_ctrlr *ctrlr = ctx;
    3729                 :            : 
    3730   [ +  -  +  +  :       3415 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   -  + ]
    3731   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the VS register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3732                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3733                 :          0 :                 return;
    3734                 :            :         }
    3735                 :            : 
    3736   [ +  +  #  # ]:       3415 :         assert(value <= UINT32_MAX);
    3737   [ +  -  +  -  :       3415 :         ctrlr->vs.raw = (uint32_t)value;
                   +  - ]
    3738                 :       3415 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_CAP, NVME_TIMEOUT_INFINITE);
    3739                 :        833 : }
    3740                 :            : 
    3741                 :            : static void
    3742                 :       3415 : nvme_ctrlr_process_init_cap_done(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl)
    3743                 :            : {
    3744                 :       3415 :         struct spdk_nvme_ctrlr *ctrlr = ctx;
    3745                 :            : 
    3746   [ +  -  +  +  :       3415 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
    3747   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CAP register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3748                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3749                 :          0 :                 return;
    3750                 :            :         }
    3751                 :            : 
    3752   [ +  -  +  -  :       3415 :         ctrlr->cap.raw = value;
                   +  - ]
    3753                 :       3415 :         nvme_ctrlr_init_cap(ctrlr);
    3754                 :       3415 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CHECK_EN, NVME_TIMEOUT_INFINITE);
    3755                 :        833 : }
    3756                 :            : 
    3757                 :            : static void
    3758                 :       3504 : nvme_ctrlr_process_init_check_en(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl)
    3759                 :            : {
    3760                 :       3504 :         struct spdk_nvme_ctrlr *ctrlr = ctx;
    3761                 :            :         enum nvme_ctrlr_state state;
    3762                 :            : 
    3763   [ +  -  +  +  :       3504 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   -  + ]
    3764   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3765                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3766                 :          0 :                 return;
    3767                 :            :         }
    3768                 :            : 
    3769   [ +  +  #  # ]:       3504 :         assert(value <= UINT32_MAX);
    3770   [ +  -  +  -  :       3504 :         ctrlr->process_init_cc.raw = (uint32_t)value;
                   +  - ]
    3771                 :            : 
    3772   [ +  +  +  -  :       3504 :         if (ctrlr->process_init_cc.bits.en) {
             +  -  -  + ]
    3773   [ -  +  -  +  :        678 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 1\n");
          -  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3774                 :        678 :                 state = NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1;
    3775                 :          0 :         } else {
    3776                 :       2826 :                 state = NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0;
    3777                 :            :         }
    3778                 :            : 
    3779                 :       3504 :         nvme_ctrlr_set_state(ctrlr, state, nvme_ctrlr_get_ready_timeout(ctrlr));
    3780                 :        833 : }
    3781                 :            : 
    3782                 :            : static void
    3783                 :        678 : nvme_ctrlr_process_init_set_en_0(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl)
    3784                 :            : {
    3785                 :        678 :         struct spdk_nvme_ctrlr *ctrlr = ctx;
    3786                 :            : 
    3787   [ +  -  -  +  :        678 :         if (spdk_nvme_cpl_is_error(cpl)) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    3788   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to write the CC register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3789                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3790                 :          0 :                 return;
    3791                 :            :         }
    3792                 :            : 
    3793                 :            :         /*
    3794                 :            :          * Wait 2.5 seconds before accessing PCI registers.
    3795                 :            :          * Not using sleep() to avoid blocking other controller's initialization.
    3796                 :            :          */
    3797   [ -  +  #  #  :        678 :         if (ctrlr->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY) {
                   #  # ]
    3798   [ #  #  #  #  :          0 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Applying quirk: delay 2.5 seconds before reading registers\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3799   [ #  #  #  #  :          0 :                 ctrlr->sleep_timeout_tsc = spdk_get_ticks() + (2500 * spdk_get_ticks_hz() / 1000);
                   #  # ]
    3800                 :          0 :         }
    3801                 :            : 
    3802                 :        678 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0,
    3803                 :          0 :                              nvme_ctrlr_get_ready_timeout(ctrlr));
    3804                 :          0 : }
    3805                 :            : 
    3806                 :            : static void
    3807                 :        678 : nvme_ctrlr_process_init_set_en_0_read_cc(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl)
    3808                 :            : {
    3809                 :        678 :         struct spdk_nvme_ctrlr *ctrlr = ctx;
    3810                 :            :         union spdk_nvme_cc_register cc;
    3811                 :            :         int rc;
    3812                 :            : 
    3813   [ +  -  -  +  :        678 :         if (spdk_nvme_cpl_is_error(cpl)) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    3814   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3815                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3816                 :          0 :                 return;
    3817                 :            :         }
    3818                 :            : 
    3819   [ -  +  #  # ]:        678 :         assert(value <= UINT32_MAX);
    3820                 :        678 :         cc.raw = (uint32_t)value;
    3821                 :        678 :         cc.bits.en = 0;
    3822   [ #  #  #  #  :        678 :         ctrlr->process_init_cc.raw = cc.raw;
                   #  # ]
    3823                 :            : 
    3824                 :        678 :         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC,
    3825                 :          0 :                              nvme_ctrlr_get_ready_timeout(ctrlr));
    3826                 :            : 
    3827                 :        678 :         rc = nvme_ctrlr_set_cc_async(ctrlr, cc.raw, nvme_ctrlr_process_init_set_en_0, ctrlr);
    3828         [ -  + ]:        678 :         if (rc != 0) {
    3829   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "set_cc() failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3830                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3831                 :          0 :         }
    3832                 :          0 : }
    3833                 :            : 
    3834                 :            : static void
    3835                 :        678 : nvme_ctrlr_process_init_wait_for_ready_1(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl)
    3836                 :            : {
    3837                 :        678 :         struct spdk_nvme_ctrlr *ctrlr = ctx;
    3838                 :            :         union spdk_nvme_csts_register csts;
    3839                 :            : 
    3840   [ +  -  -  +  :        678 :         if (spdk_nvme_cpl_is_error(cpl)) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    3841                 :            :                 /* While a device is resetting, it may be unable to service MMIO reads
    3842                 :            :                  * temporarily. Allow for this case.
    3843                 :            :                  */
    3844   [ #  #  #  #  :          0 :                 if (!ctrlr->is_failed && ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE) {
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3845   [ #  #  #  #  :          0 :                         NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to read the CSTS register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3846                 :          0 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1,
    3847                 :            :                                              NVME_TIMEOUT_KEEP_EXISTING);
    3848                 :          0 :                 } else {
    3849   [ #  #  #  #  :          0 :                         NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3850                 :          0 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3851                 :            :                 }
    3852                 :            : 
    3853                 :          0 :                 return;
    3854                 :            :         }
    3855                 :            : 
    3856   [ -  +  #  # ]:        678 :         assert(value <= UINT32_MAX);
    3857                 :        678 :         csts.raw = (uint32_t)value;
    3858   [ -  +  -  - ]:        678 :         if (csts.bits.rdy == 1 || csts.bits.cfs == 1) {
    3859                 :        678 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_EN_0,
    3860                 :          0 :                                      nvme_ctrlr_get_ready_timeout(ctrlr));
    3861                 :          0 :         } else {
    3862   [ #  #  #  #  :          0 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 1 && CSTS.RDY = 0 - waiting for reset to complete\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3863                 :          0 :                 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1,
    3864                 :            :                                            NVME_TIMEOUT_KEEP_EXISTING);
    3865                 :            :         }
    3866                 :          0 : }
    3867                 :            : 
    3868                 :            : static void
    3869                 :       4293 : nvme_ctrlr_process_init_wait_for_ready_0(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl)
    3870                 :            : {
    3871                 :       4293 :         struct spdk_nvme_ctrlr *ctrlr = ctx;
    3872                 :            :         union spdk_nvme_csts_register csts;
    3873                 :            : 
    3874   [ +  -  +  +  :       4293 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   -  + ]
    3875                 :            :                 /* While a device is resetting, it may be unable to service MMIO reads
    3876                 :            :                  * temporarily. Allow for this case.
    3877                 :            :                  */
    3878   [ #  #  #  #  :          0 :                 if (!ctrlr->is_failed && ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE) {
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3879   [ #  #  #  #  :          0 :                         NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to read the CSTS register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3880                 :          0 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0,
    3881                 :            :                                              NVME_TIMEOUT_KEEP_EXISTING);
    3882                 :          0 :                 } else {
    3883   [ #  #  #  #  :          0 :                         NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3884                 :          0 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3885                 :            :                 }
    3886                 :            : 
    3887                 :          0 :                 return;
    3888                 :            :         }
    3889                 :            : 
    3890   [ +  +  #  # ]:       4293 :         assert(value <= UINT32_MAX);
    3891                 :       4293 :         csts.raw = (uint32_t)value;
    3892         [ +  + ]:       4293 :         if (csts.bits.rdy == 0) {
    3893   [ +  +  +  +  :       3504 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 0 && CSTS.RDY = 0\n");
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3894                 :       4337 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLED,
    3895                 :        833 :                                      nvme_ctrlr_get_ready_timeout(ctrlr));
    3896                 :        833 :         } else {
    3897                 :        789 :                 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0,
    3898                 :            :                                            NVME_TIMEOUT_KEEP_EXISTING);
    3899                 :            :         }
    3900                 :        833 : }
    3901                 :            : 
    3902                 :            : static void
    3903                 :      38042 : nvme_ctrlr_process_init_enable_wait_for_ready_1(void *ctx, uint64_t value,
    3904                 :            :                 const struct spdk_nvme_cpl *cpl)
    3905                 :            : {
    3906                 :      38042 :         struct spdk_nvme_ctrlr *ctrlr = ctx;
    3907                 :            :         union spdk_nvme_csts_register csts;
    3908                 :            : 
    3909   [ +  -  +  +  :      38042 :         if (spdk_nvme_cpl_is_error(cpl)) {
          +  -  +  -  +  
          -  +  -  +  -  
                   -  + ]
    3910                 :            :                 /* While a device is resetting, it may be unable to service MMIO reads
    3911                 :            :                  * temporarily. Allow for this case.
    3912                 :            :                  */
    3913   [ #  #  #  #  :          0 :                 if (!ctrlr->is_failed && ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE) {
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3914   [ #  #  #  #  :          0 :                         NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to read the CSTS register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3915                 :          0 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1,
    3916                 :            :                                              NVME_TIMEOUT_KEEP_EXISTING);
    3917                 :          0 :                 } else {
    3918   [ #  #  #  #  :          0 :                         NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    3919                 :          0 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3920                 :            :                 }
    3921                 :            : 
    3922                 :          0 :                 return;
    3923                 :            :         }
    3924                 :            : 
    3925   [ +  +  #  # ]:      38042 :         assert(value <= UINT32_MAX);
    3926                 :      38042 :         csts.raw = value;
    3927         [ +  + ]:      38042 :         if (csts.bits.rdy == 1) {
    3928   [ +  +  +  +  :       3379 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 1 && CSTS.RDY = 1 - controller is ready\n");
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3929                 :            :                 /*
    3930                 :            :                  * The controller has been enabled.
    3931                 :            :                  *  Perform the rest of initialization serially.
    3932                 :            :                  */
    3933                 :       4212 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_RESET_ADMIN_QUEUE,
    3934   [ +  -  +  -  :       3379 :                                      ctrlr->opts.admin_timeout_ms);
                   +  - ]
    3935                 :        833 :         } else {
    3936                 :      34663 :                 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1,
    3937                 :            :                                            NVME_TIMEOUT_KEEP_EXISTING);
    3938                 :            :         }
    3939                 :       3776 : }
    3940                 :            : 
    3941                 :            : /**
    3942                 :            :  * This function will be called repeatedly during initialization until the controller is ready.
    3943                 :            :  */
    3944                 :            : int
    3945                 :  154832443 : nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr)
    3946                 :            : {
    3947                 :            :         uint32_t ready_timeout_in_ms;
    3948                 :            :         uint64_t ticks;
    3949                 :  154832443 :         int rc = 0;
    3950                 :            : 
    3951                 :  154832443 :         ticks = spdk_get_ticks();
    3952                 :            : 
    3953                 :            :         /*
    3954                 :            :          * May need to avoid accessing any register on the target controller
    3955                 :            :          * for a while. Return early without touching the FSM.
    3956                 :            :          * Check sleep_timeout_tsc > 0 for unit test.
    3957                 :            :          */
    3958   [ +  +  +  -  :  185621344 :         if ((ctrlr->sleep_timeout_tsc > 0) &&
             +  +  +  + ]
    3959   [ +  +  +  - ]:  148801680 :             (ticks <= ctrlr->sleep_timeout_tsc)) {
    3960                 :  148801587 :                 return 0;
    3961                 :            :         }
    3962   [ +  -  +  - ]:    6030856 :         ctrlr->sleep_timeout_tsc = 0;
    3963                 :            : 
    3964                 :    6030856 :         ready_timeout_in_ms = nvme_ctrlr_get_ready_timeout(ctrlr);
    3965                 :            : 
    3966                 :            :         /*
    3967                 :            :          * Check if the current initialization step is done or has timed out.
    3968                 :            :          */
    3969   [ +  -  +  +  :    6030856 :         switch (ctrlr->state) {
          +  +  +  +  +  
          +  +  +  +  +  
          +  +  +  +  +  
          +  +  +  +  +  
          +  +  +  +  +  
          +  +  +  +  +  
             +  +  +  -  
                      - ]
    3970                 :        707 :         case NVME_CTRLR_STATE_INIT_DELAY:
    3971                 :        709 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, ready_timeout_in_ms);
    3972   [ +  +  +  -  :        709 :                 if (ctrlr->quirks & NVME_QUIRK_DELAY_BEFORE_INIT) {
                   -  + ]
    3973                 :            :                         /*
    3974                 :            :                          * Controller may need some delay before it's enabled.
    3975                 :            :                          *
    3976                 :            :                          * This is a workaround for an issue where the PCIe-attached NVMe controller
    3977                 :            :                          * is not ready after VFIO reset. We delay the initialization rather than the
    3978                 :            :                          * enabling itself, because this is required only for the very first enabling
    3979                 :            :                          * - directly after a VFIO reset.
    3980                 :            :                          */
    3981   [ +  +  +  +  :         93 :                         NVME_CTRLR_DEBUGLOG(ctrlr, "Adding 2 second delay before initializing the controller\n");
          +  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    3982   [ +  -  +  -  :         93 :                         ctrlr->sleep_timeout_tsc = ticks + (2000 * spdk_get_ticks_hz() / 1000);
                   +  - ]
    3983                 :          2 :                 }
    3984                 :        709 :                 break;
    3985                 :            : 
    3986                 :          0 :         case NVME_CTRLR_STATE_DISCONNECTED:
    3987                 :          0 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE);
    3988                 :          0 :                 break;
    3989                 :            : 
    3990                 :       3001 :         case NVME_CTRLR_STATE_CONNECT_ADMINQ: /* synonymous with NVME_CTRLR_STATE_INIT and NVME_CTRLR_STATE_DISCONNECTED */
    3991   [ +  -  +  - ]:       3834 :                 rc = nvme_transport_ctrlr_connect_qpair(ctrlr, ctrlr->adminq);
    3992         [ +  - ]:       3834 :                 if (rc == 0) {
    3993                 :       3834 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ,
    3994                 :            :                                              NVME_TIMEOUT_INFINITE);
    3995                 :        833 :                 } else {
    3996                 :          0 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    3997                 :            :                 }
    3998                 :       3834 :                 break;
    3999                 :            : 
    4000                 :    1750607 :         case NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ:
    4001   [ +  -  +  - ]:    2176513 :                 spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
    4002                 :            : 
    4003   [ +  +  +  +  :    2176513 :                 switch (nvme_qpair_get_state(ctrlr->adminq)) {
          +  -  +  +  -  
                      - ]
    4004                 :    1190790 :                 case NVME_QPAIR_CONNECTING:
    4005   [ +  +  +  +  :    1615863 :                         if (ctrlr->is_failed) {
             +  -  -  + ]
    4006   [ #  #  #  # ]:          0 :                                 nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq);
    4007                 :          0 :                                 break;
    4008                 :            :                         }
    4009                 :            : 
    4010                 :    1615863 :                         break;
    4011                 :       1843 :                 case NVME_QPAIR_CONNECTED:
    4012   [ -  +  -  + ]:       2667 :                         nvme_qpair_set_state(ctrlr->adminq, NVME_QPAIR_ENABLED);
    4013                 :            :                 /* Fall through */
    4014                 :       2582 :                 case NVME_QPAIR_ENABLED:
    4015                 :       3415 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_VS,
    4016                 :            :                                              NVME_TIMEOUT_INFINITE);
    4017                 :            :                         /* Abort any queued requests that were sent while the adminq was connecting
    4018                 :            :                          * to avoid stalling the init process during a reset, as requests don't get
    4019                 :            :                          * resubmitted while the controller is resetting and subsequent commands
    4020                 :            :                          * would get queued too.
    4021                 :            :                          */
    4022   [ +  -  +  - ]:       3415 :                         nvme_qpair_abort_queued_reqs(ctrlr->adminq);
    4023                 :       3415 :                         break;
    4024                 :     556816 :                 case NVME_QPAIR_DISCONNECTING:
    4025   [ -  +  #  #  :     556816 :                         assert(ctrlr->adminq->async == true);
          #  #  #  #  #  
                      # ]
    4026                 :     556816 :                         break;
    4027                 :        419 :                 case NVME_QPAIR_DISCONNECTED:
    4028                 :            :                 /* fallthrough */
    4029                 :            :                 default:
    4030                 :        419 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    4031                 :        419 :                         break;
    4032                 :            :                 }
    4033                 :            : 
    4034                 :    2176513 :                 break;
    4035                 :            : 
    4036                 :       2582 :         case NVME_CTRLR_STATE_READ_VS:
    4037                 :       3415 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS, NVME_TIMEOUT_INFINITE);
    4038                 :       3415 :                 rc = nvme_ctrlr_get_vs_async(ctrlr, nvme_ctrlr_process_init_vs_done, ctrlr);
    4039                 :       3415 :                 break;
    4040                 :            : 
    4041                 :       2582 :         case NVME_CTRLR_STATE_READ_CAP:
    4042                 :       3415 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP, NVME_TIMEOUT_INFINITE);
    4043                 :       3415 :                 rc = nvme_ctrlr_get_cap_async(ctrlr, nvme_ctrlr_process_init_cap_done, ctrlr);
    4044                 :       3415 :                 break;
    4045                 :            : 
    4046                 :       2671 :         case NVME_CTRLR_STATE_CHECK_EN:
    4047                 :            :                 /* Begin the hardware initialization by making sure the controller is disabled. */
    4048                 :       3504 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC, ready_timeout_in_ms);
    4049                 :       3504 :                 rc = nvme_ctrlr_get_cc_async(ctrlr, nvme_ctrlr_process_init_check_en, ctrlr);
    4050                 :       3504 :                 break;
    4051                 :            : 
    4052                 :        678 :         case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1:
    4053                 :            :                 /*
    4054                 :            :                  * Controller is currently enabled. We need to disable it to cause a reset.
    4055                 :            :                  *
    4056                 :            :                  * If CC.EN = 1 && CSTS.RDY = 0, the controller is in the process of becoming ready.
    4057                 :            :                  *  Wait for the ready bit to be 1 before disabling the controller.
    4058                 :            :                  */
    4059                 :        678 :                 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS,
    4060                 :            :                                            NVME_TIMEOUT_KEEP_EXISTING);
    4061                 :        678 :                 rc = nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_process_init_wait_for_ready_1, ctrlr);
    4062                 :        678 :                 break;
    4063                 :            : 
    4064                 :        678 :         case NVME_CTRLR_STATE_SET_EN_0:
    4065   [ -  +  -  +  :        678 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Setting CC.EN = 0\n");
          -  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    4066                 :        678 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC, ready_timeout_in_ms);
    4067                 :        678 :                 rc = nvme_ctrlr_get_cc_async(ctrlr, nvme_ctrlr_process_init_set_en_0_read_cc, ctrlr);
    4068                 :        678 :                 break;
    4069                 :            : 
    4070                 :       3460 :         case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0:
    4071                 :       4293 :                 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS,
    4072                 :            :                                            NVME_TIMEOUT_KEEP_EXISTING);
    4073                 :       4293 :                 rc = nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_process_init_wait_for_ready_0, ctrlr);
    4074                 :       4293 :                 break;
    4075                 :            : 
    4076                 :       2668 :         case NVME_CTRLR_STATE_DISABLED:
    4077   [ +  +  +  +  :       3501 :                 if (ctrlr->is_disconnecting) {
             +  -  -  + ]
    4078   [ -  +  -  +  :         86 :                         NVME_CTRLR_DEBUGLOG(ctrlr, "Ctrlr was disabled.\n");
          -  -  -  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    4079                 :          0 :                 } else {
    4080                 :       3415 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE, ready_timeout_in_ms);
    4081                 :            : 
    4082                 :            :                         /*
    4083                 :            :                          * Delay 100us before setting CC.EN = 1.  Some NVMe SSDs miss CC.EN getting
    4084                 :            :                          *  set to 1 if it is too soon after CSTS.RDY is reported as 0.
    4085                 :            :                          */
    4086                 :       3415 :                         spdk_delay_us(100);
    4087                 :            :                 }
    4088                 :       3501 :                 break;
    4089                 :            : 
    4090                 :       2582 :         case NVME_CTRLR_STATE_ENABLE:
    4091   [ +  +  +  +  :       3415 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Setting CC.EN = 1\n");
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    4092                 :       3415 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC, ready_timeout_in_ms);
    4093                 :       3415 :                 rc = nvme_ctrlr_enable(ctrlr);
    4094         [ +  + ]:       3415 :                 if (rc) {
    4095   [ +  -  -  +  :         21 :                         NVME_CTRLR_ERRLOG(ctrlr, "Ctrlr enable failed with error: %d", rc);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4096                 :          0 :                 }
    4097                 :       3415 :                 return rc;
    4098                 :            : 
    4099                 :      34266 :         case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1:
    4100                 :      38042 :                 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS,
    4101                 :            :                                            NVME_TIMEOUT_KEEP_EXISTING);
    4102                 :      38042 :                 rc = nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_process_init_enable_wait_for_ready_1,
    4103                 :            :                                                ctrlr);
    4104                 :      38042 :                 break;
    4105                 :            : 
    4106                 :       2546 :         case NVME_CTRLR_STATE_RESET_ADMIN_QUEUE:
    4107   [ -  +  -  + ]:       3379 :                 nvme_transport_qpair_reset(ctrlr->adminq);
    4108                 :       3379 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY, NVME_TIMEOUT_INFINITE);
    4109                 :       3379 :                 break;
    4110                 :            : 
    4111                 :       2567 :         case NVME_CTRLR_STATE_IDENTIFY:
    4112                 :       3400 :                 rc = nvme_ctrlr_identify(ctrlr);
    4113                 :       3400 :                 break;
    4114                 :            : 
    4115                 :       2576 :         case NVME_CTRLR_STATE_CONFIGURE_AER:
    4116                 :       3409 :                 rc = nvme_ctrlr_configure_aer(ctrlr);
    4117                 :       3409 :                 break;
    4118                 :            : 
    4119                 :       2585 :         case NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT:
    4120                 :       3418 :                 rc = nvme_ctrlr_set_keep_alive_timeout(ctrlr);
    4121                 :       3418 :                 break;
    4122                 :            : 
    4123                 :       2424 :         case NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC:
    4124                 :       3257 :                 rc = nvme_ctrlr_identify_iocs_specific(ctrlr);
    4125                 :       3257 :                 break;
    4126                 :            : 
    4127                 :        987 :         case NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG:
    4128                 :        987 :                 rc = nvme_ctrlr_get_zns_cmd_and_effects_log(ctrlr);
    4129                 :        987 :                 break;
    4130                 :            : 
    4131                 :       2424 :         case NVME_CTRLR_STATE_SET_NUM_QUEUES:
    4132                 :       3257 :                 nvme_ctrlr_update_nvmf_ioccsz(ctrlr);
    4133                 :       3257 :                 rc = nvme_ctrlr_set_num_queues(ctrlr);
    4134                 :       3257 :                 break;
    4135                 :            : 
    4136                 :       2439 :         case NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS:
    4137                 :       3272 :                 _nvme_ctrlr_identify_active_ns(ctrlr);
    4138                 :       3272 :                 break;
    4139                 :            : 
    4140                 :       2409 :         case NVME_CTRLR_STATE_IDENTIFY_NS:
    4141                 :       3242 :                 rc = nvme_ctrlr_identify_namespaces(ctrlr);
    4142                 :       3242 :                 break;
    4143                 :            : 
    4144                 :       2409 :         case NVME_CTRLR_STATE_IDENTIFY_ID_DESCS:
    4145                 :       3242 :                 rc = nvme_ctrlr_identify_id_desc_namespaces(ctrlr);
    4146                 :       3242 :                 break;
    4147                 :            : 
    4148                 :       2409 :         case NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC:
    4149                 :       3242 :                 rc = nvme_ctrlr_identify_namespaces_iocs_specific(ctrlr);
    4150                 :       3242 :                 break;
    4151                 :            : 
    4152                 :       2412 :         case NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES:
    4153                 :       3245 :                 rc = nvme_ctrlr_set_supported_log_pages(ctrlr);
    4154                 :       3245 :                 break;
    4155                 :            : 
    4156                 :        109 :         case NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES:
    4157                 :        111 :                 rc = nvme_ctrlr_set_intel_support_log_pages(ctrlr);
    4158                 :        111 :                 break;
    4159                 :            : 
    4160                 :       2409 :         case NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES:
    4161                 :       3242 :                 nvme_ctrlr_set_supported_features(ctrlr);
    4162                 :       4075 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_FEATURE,
    4163   [ +  -  +  -  :       3242 :                                      ctrlr->opts.admin_timeout_ms);
                   +  - ]
    4164                 :       3242 :                 break;
    4165                 :            : 
    4166                 :       2415 :         case NVME_CTRLR_STATE_SET_HOST_FEATURE:
    4167                 :       3248 :                 rc = nvme_ctrlr_set_host_feature(ctrlr);
    4168                 :       3248 :                 break;
    4169                 :            : 
    4170                 :       2409 :         case NVME_CTRLR_STATE_SET_DB_BUF_CFG:
    4171                 :       3242 :                 rc = nvme_ctrlr_set_doorbell_buffer_config(ctrlr);
    4172                 :       3242 :                 break;
    4173                 :            : 
    4174                 :       2409 :         case NVME_CTRLR_STATE_SET_HOST_ID:
    4175                 :       3242 :                 rc = nvme_ctrlr_set_host_id(ctrlr);
    4176                 :       3242 :                 break;
    4177                 :            : 
    4178                 :       2418 :         case NVME_CTRLR_STATE_TRANSPORT_READY:
    4179                 :       3251 :                 rc = nvme_transport_ctrlr_ready(ctrlr);
    4180         [ +  + ]:       3251 :                 if (rc) {
    4181   [ +  -  -  +  :          3 :                         NVME_CTRLR_ERRLOG(ctrlr, "Transport controller ready step failed: rc %d\n", rc);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4182                 :          3 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE);
    4183                 :          0 :                 } else {
    4184                 :       3248 :                         nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE);
    4185                 :            :                 }
    4186                 :       3251 :                 break;
    4187                 :            : 
    4188                 :         41 :         case NVME_CTRLR_STATE_READY:
    4189   [ -  +  +  +  :         41 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Ctrlr already in ready state\n");
          +  +  +  -  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    4190                 :         41 :                 return 0;
    4191                 :            : 
    4192                 :        442 :         case NVME_CTRLR_STATE_ERROR:
    4193   [ +  +  +  -  :        442 :                 NVME_CTRLR_ERRLOG(ctrlr, "Ctrlr is in error state\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4194                 :        442 :                 return -1;
    4195                 :            : 
    4196                 :    3536984 :         case NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS:
    4197                 :            :         case NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP:
    4198                 :            :         case NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC:
    4199                 :            :         case NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC:
    4200                 :            :         case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS:
    4201                 :            :         case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS:
    4202                 :            :         case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC:
    4203                 :            :         case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS:
    4204                 :            :         case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY:
    4205                 :            :         case NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER:
    4206                 :            :         case NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT:
    4207                 :            :         case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC:
    4208                 :            :         case NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG:
    4209                 :            :         case NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES:
    4210                 :            :         case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS:
    4211                 :            :         case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS:
    4212                 :            :         case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS:
    4213                 :            :         case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC:
    4214                 :            :         case NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES:
    4215                 :            :         case NVME_CTRLR_STATE_WAIT_FOR_SET_HOST_FEATURE:
    4216                 :            :         case NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG:
    4217                 :            :         case NVME_CTRLR_STATE_WAIT_FOR_HOST_ID:
    4218                 :            :                 /*
    4219                 :            :                  * nvme_ctrlr_process_init() may be called from the completion context
    4220                 :            :                  * for the admin qpair. Avoid recursive calls for this case.
    4221                 :            :                  */
    4222   [ +  +  +  -  :    3734690 :                 if (!ctrlr->adminq->in_completion_context) {
             +  -  -  + ]
    4223   [ +  -  +  - ]:    3734604 :                         spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
    4224                 :     197706 :                 }
    4225                 :    3734690 :                 break;
    4226                 :            : 
    4227                 :          0 :         default:
    4228         [ #  # ]:          0 :                 assert(0);
    4229                 :            :                 return -1;
    4230                 :            :         }
    4231                 :            : 
    4232         [ +  + ]:    6026958 :         if (rc) {
    4233   [ +  -  -  +  :          3 :                 NVME_CTRLR_ERRLOG(ctrlr, "Ctrlr operation failed with error: %d, ctrlr state: %d (%s)\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
    4234                 :            :                                   rc, ctrlr->state, nvme_ctrlr_state_string(ctrlr->state));
    4235                 :          0 :         }
    4236                 :            : 
    4237                 :            :         /* Note: we use the ticks captured when we entered this function.
    4238                 :            :          * This covers environments where the SPDK process gets swapped out after
    4239                 :            :          * we tried to advance the state but before we check the timeout here.
    4240                 :            :          * It is not normal for this to happen, but harmless to handle it in this
    4241                 :            :          * way.
    4242                 :            :          */
    4243   [ +  +  +  -  :    6225966 :         if (ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE &&
             +  +  +  - ]
    4244   [ +  +  +  - ]:    3690871 :             ticks > ctrlr->state_timeout_tsc) {
    4245   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Initialization timed out in state %d (%s)\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
    4246                 :            :                                   ctrlr->state, nvme_ctrlr_state_string(ctrlr->state));
    4247                 :          0 :                 return -1;
    4248                 :            :         }
    4249                 :            : 
    4250                 :    6026958 :         return rc;
    4251                 :   31435450 : }
    4252                 :            : 
    4253                 :            : int
    4254                 :       3437 : nvme_robust_mutex_init_recursive_shared(pthread_mutex_t *mtx)
    4255                 :            : {
    4256                 :        566 :         pthread_mutexattr_t attr;
    4257                 :       3437 :         int rc = 0;
    4258                 :            : 
    4259   [ +  +  -  + ]:       3437 :         if (pthread_mutexattr_init(&attr)) {
    4260                 :          0 :                 return -1;
    4261                 :            :         }
    4262   [ +  +  +  -  :       6874 :         if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) ||
                   +  + ]
    4263                 :            : #ifndef __FreeBSD__
    4264   [ +  +  +  - ]:       6041 :             pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST) ||
    4265   [ +  +  +  + ]:       6041 :             pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) ||
    4266                 :            : #endif
    4267         [ +  + ]:       3437 :             pthread_mutex_init(mtx, &attr)) {
    4268                 :          0 :                 rc = -1;
    4269                 :          0 :         }
    4270         [ +  + ]:       3437 :         pthread_mutexattr_destroy(&attr);
    4271                 :       3437 :         return rc;
    4272                 :        833 : }
    4273                 :            : 
    4274                 :            : int
    4275                 :       3437 : nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr)
    4276                 :            : {
    4277                 :            :         int rc;
    4278                 :            : 
    4279   [ +  +  +  -  :       3437 :         if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
             +  -  +  + ]
    4280                 :        709 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT_DELAY, NVME_TIMEOUT_INFINITE);
    4281                 :          2 :         } else {
    4282                 :       2728 :                 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE);
    4283                 :            :         }
    4284                 :            : 
    4285   [ +  +  +  -  :       3437 :         if (ctrlr->opts.admin_queue_size > SPDK_NVME_ADMIN_QUEUE_MAX_ENTRIES) {
             +  -  +  - ]
    4286   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "admin_queue_size %u exceeds max defined by NVMe spec, use max value\n",
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    4287                 :            :                                   ctrlr->opts.admin_queue_size);
    4288   [ #  #  #  #  :          0 :                 ctrlr->opts.admin_queue_size = SPDK_NVME_ADMIN_QUEUE_MAX_ENTRIES;
                   #  # ]
    4289                 :          0 :         }
    4290                 :            : 
    4291   [ +  +  +  -  :       3437 :         if (ctrlr->quirks & NVME_QUIRK_MINIMUM_ADMIN_QUEUE_SIZE &&
             -  +  #  # ]
    4292   [ #  #  #  #  :          0 :             (ctrlr->opts.admin_queue_size % SPDK_NVME_ADMIN_QUEUE_QUIRK_ENTRIES_MULTIPLE) != 0) {
             #  #  #  # ]
    4293   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr,
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    4294                 :            :                                   "admin_queue_size %u is invalid for this NVMe device, adjust to next multiple\n",
    4295                 :            :                                   ctrlr->opts.admin_queue_size);
    4296   [ #  #  #  #  :          0 :                 ctrlr->opts.admin_queue_size = SPDK_ALIGN_CEIL(ctrlr->opts.admin_queue_size,
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4297                 :            :                                                SPDK_NVME_ADMIN_QUEUE_QUIRK_ENTRIES_MULTIPLE);
    4298                 :          0 :         }
    4299                 :            : 
    4300   [ +  +  +  -  :       3437 :         if (ctrlr->opts.admin_queue_size < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES) {
             +  -  +  - ]
    4301   [ +  -  -  +  :         78 :                 NVME_CTRLR_ERRLOG(ctrlr,
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    4302                 :            :                                   "admin_queue_size %u is less than minimum defined by NVMe spec, use min value\n",
    4303                 :            :                                   ctrlr->opts.admin_queue_size);
    4304   [ #  #  #  #  :         78 :                 ctrlr->opts.admin_queue_size = SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES;
                   #  # ]
    4305                 :          0 :         }
    4306                 :            : 
    4307   [ +  -  +  - ]:       3437 :         ctrlr->flags = 0;
    4308   [ +  -  +  - ]:       3437 :         ctrlr->free_io_qids = NULL;
    4309   [ +  -  +  - ]:       3437 :         ctrlr->is_resetting = false;
    4310   [ +  -  +  - ]:       3437 :         ctrlr->is_failed = false;
    4311   [ +  -  +  - ]:       3437 :         ctrlr->is_destructed = false;
    4312                 :            : 
    4313   [ +  -  +  -  :       3437 :         TAILQ_INIT(&ctrlr->active_io_qpairs);
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
    4314   [ +  -  +  -  :       3437 :         STAILQ_INIT(&ctrlr->queued_aborts);
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
    4315   [ +  -  +  - ]:       3437 :         ctrlr->outstanding_aborts = 0;
    4316                 :            : 
    4317   [ +  -  +  - ]:       3437 :         ctrlr->ana_log_page = NULL;
    4318   [ +  -  +  - ]:       3437 :         ctrlr->ana_log_page_size = 0;
    4319                 :            : 
    4320         [ +  - ]:       3437 :         rc = nvme_robust_mutex_init_recursive_shared(&ctrlr->ctrlr_lock);
    4321         [ -  + ]:       3437 :         if (rc != 0) {
    4322                 :          0 :                 return rc;
    4323                 :            :         }
    4324                 :            : 
    4325   [ +  -  +  -  :       3437 :         TAILQ_INIT(&ctrlr->active_procs);
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
    4326   [ +  -  +  -  :       3437 :         STAILQ_INIT(&ctrlr->register_operations);
          +  -  +  -  +  
          -  +  -  +  -  
                   +  - ]
    4327                 :            : 
    4328   [ +  -  +  -  :       3437 :         RB_INIT(&ctrlr->ns);
                   +  - ]
    4329                 :            : 
    4330                 :       3437 :         return rc;
    4331                 :        833 : }
    4332                 :            : 
    4333                 :            : static void
    4334                 :       3415 : nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr)
    4335                 :            : {
    4336   [ +  +  +  -  :       3415 :         if (ctrlr->cap.bits.ams & SPDK_NVME_CAP_AMS_WRR) {
             +  -  +  + ]
    4337   [ +  -  +  - ]:        140 :                 ctrlr->flags |= SPDK_NVME_CTRLR_WRR_SUPPORTED;
    4338                 :          2 :         }
    4339                 :            : 
    4340   [ +  +  +  -  :       3415 :         ctrlr->min_page_size = 1u << (12 + ctrlr->cap.bits.mpsmin);
          +  -  +  -  +  
             -  +  -  +  
                      - ]
    4341                 :            : 
    4342                 :            :         /* For now, always select page_size == min_page_size. */
    4343   [ +  -  +  -  :       3415 :         ctrlr->page_size = ctrlr->min_page_size;
             +  -  +  - ]
    4344                 :            : 
    4345   [ +  -  +  -  :       3415 :         ctrlr->opts.io_queue_size = spdk_max(ctrlr->opts.io_queue_size, SPDK_NVME_IO_QUEUE_MIN_ENTRIES);
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
                      - ]
    4346   [ +  -  +  -  :       3415 :         ctrlr->opts.io_queue_size = spdk_min(ctrlr->opts.io_queue_size, MAX_IO_QUEUE_ENTRIES);
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  #  #  +  
             -  +  -  +  
                      - ]
    4347   [ +  +  +  -  :       3417 :         if (ctrlr->quirks & NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE &&
             +  +  -  + ]
    4348   [ +  +  +  -  :        103 :             ctrlr->opts.io_queue_size == DEFAULT_IO_QUEUE_SIZE) {
                   +  - ]
    4349                 :            :                 /* If the user specifically set an IO queue size different than the
    4350                 :            :                  * default, use that value.  Otherwise overwrite with the quirked value.
    4351                 :            :                  * This allows this quirk to be overridden when necessary.
    4352                 :            :                  * However, cap.mqes still needs to be respected.
    4353                 :            :                  */
    4354   [ +  -  +  -  :         91 :                 ctrlr->opts.io_queue_size = DEFAULT_IO_QUEUE_SIZE_FOR_QUIRK;
                   +  - ]
    4355                 :          2 :         }
    4356   [ +  -  +  -  :       3415 :         ctrlr->opts.io_queue_size = spdk_min(ctrlr->opts.io_queue_size, ctrlr->cap.bits.mqes + 1u);
          +  -  +  -  +  
          -  +  -  +  +  
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
                -  +  - ]
    4357                 :            : 
    4358   [ +  -  +  -  :       3415 :         ctrlr->opts.io_queue_requests = spdk_max(ctrlr->opts.io_queue_requests, ctrlr->opts.io_queue_size);
          +  -  +  -  +  
          -  +  -  +  +  
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
                -  +  - ]
    4359                 :       3415 : }
    4360                 :            : 
    4361                 :            : void
    4362                 :       3433 : nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr *ctrlr)
    4363                 :            : {
    4364                 :            :         int rc;
    4365                 :            : 
    4366   [ +  +  +  -  :       3433 :         if (ctrlr->lock_depth > 0) {
                   +  - ]
    4367   [ #  #  #  # ]:          0 :                 SPDK_ERRLOG("lock currently held (depth=%d)!\n", ctrlr->lock_depth);
    4368         [ #  # ]:          0 :                 assert(false);
    4369                 :            :         }
    4370                 :            : 
    4371   [ +  +  +  - ]:       3433 :         rc = pthread_mutex_destroy(&ctrlr->ctrlr_lock);
    4372         [ +  + ]:       3433 :         if (rc) {
    4373                 :          0 :                 SPDK_ERRLOG("could not destroy ctrlr_lock: %s\n", spdk_strerror(rc));
    4374         [ #  # ]:          0 :                 assert(false);
    4375                 :            :         }
    4376                 :            : 
    4377                 :       3433 :         nvme_ctrlr_free_processes(ctrlr);
    4378                 :       3433 : }
    4379                 :            : 
    4380                 :            : void
    4381                 :       3419 : nvme_ctrlr_destruct_async(struct spdk_nvme_ctrlr *ctrlr,
    4382                 :            :                           struct nvme_ctrlr_detach_ctx *ctx)
    4383                 :            : {
    4384                 :            :         struct spdk_nvme_qpair *qpair, *tmp;
    4385                 :            : 
    4386   [ +  +  +  +  :       3419 :         NVME_CTRLR_DEBUGLOG(ctrlr, "Prepare to destruct SSD\n");
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    4387                 :            : 
    4388   [ +  -  +  - ]:       3419 :         ctrlr->prepare_for_reset = false;
    4389   [ +  -  +  - ]:       3419 :         ctrlr->is_destructed = true;
    4390                 :            : 
    4391   [ +  -  +  - ]:       3419 :         spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
    4392                 :            : 
    4393                 :       3419 :         nvme_ctrlr_abort_queued_aborts(ctrlr);
    4394   [ +  -  +  - ]:       3419 :         nvme_transport_admin_qpair_abort_aers(ctrlr->adminq);
    4395                 :            : 
    4396   [ +  +  +  -  :       3585 :         TAILQ_FOREACH_SAFE(qpair, &ctrlr->active_io_qpairs, tailq, tmp) {
          +  -  +  +  +  
          -  +  -  +  -  
                   +  + ]
    4397                 :        166 :                 spdk_nvme_ctrlr_free_io_qpair(qpair);
    4398                 :          1 :         }
    4399                 :            : 
    4400                 :       3419 :         nvme_ctrlr_free_doorbell_buffer(ctrlr);
    4401                 :       3419 :         nvme_ctrlr_free_iocs_specific_data(ctrlr);
    4402                 :            : 
    4403                 :       3419 :         nvme_ctrlr_shutdown_async(ctrlr, ctx);
    4404                 :       3419 : }
    4405                 :            : 
    4406                 :            : int
    4407                 :   20514900 : nvme_ctrlr_destruct_poll_async(struct spdk_nvme_ctrlr *ctrlr,
    4408                 :            :                                struct nvme_ctrlr_detach_ctx *ctx)
    4409                 :            : {
    4410                 :            :         struct spdk_nvme_ns *ns, *tmp_ns;
    4411                 :   20514900 :         int rc = 0;
    4412                 :            : 
    4413   [ +  +  +  +  :   20514900 :         if (!ctx->shutdown_complete) {
             +  -  -  + ]
    4414                 :   20514678 :                 rc = nvme_ctrlr_shutdown_poll_async(ctrlr, ctx);
    4415         [ +  + ]:   20514678 :                 if (rc == -EAGAIN) {
    4416                 :   20511481 :                         return -EAGAIN;
    4417                 :            :                 }
    4418                 :            :                 /* Destruct ctrlr forcefully for any other error. */
    4419                 :        833 :         }
    4420                 :            : 
    4421   [ +  +  +  -  :       3419 :         if (ctx->cb_fn) {
                   +  - ]
    4422   [ +  -  +  -  :       3111 :                 ctx->cb_fn(ctrlr);
             -  +  +  - ]
    4423                 :        833 :         }
    4424                 :            : 
    4425   [ +  -  +  - ]:       3419 :         nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq);
    4426                 :            : 
    4427   [ +  +  +  +  :      29250 :         RB_FOREACH_SAFE(ns, nvme_ns_tree, &ctrlr->ns, tmp_ns) {
                   +  + ]
    4428   [ +  -  +  - ]:      25831 :                 nvme_ctrlr_destruct_namespace(ctrlr, ns->id);
    4429         [ +  - ]:      25831 :                 RB_REMOVE(nvme_ns_tree, &ctrlr->ns, ns);
    4430                 :      25831 :                 spdk_free(ns);
    4431                 :        833 :         }
    4432                 :            : 
    4433   [ +  -  +  - ]:       3419 :         ctrlr->active_ns_count = 0;
    4434                 :            : 
    4435         [ +  - ]:       3419 :         spdk_bit_array_free(&ctrlr->free_io_qids);
    4436                 :            : 
    4437   [ +  -  +  - ]:       3419 :         free(ctrlr->ana_log_page);
    4438   [ +  -  +  - ]:       3419 :         free(ctrlr->copied_ana_desc);
    4439   [ +  -  +  - ]:       3419 :         ctrlr->ana_log_page = NULL;
    4440   [ +  -  +  - ]:       3419 :         ctrlr->copied_ana_desc = NULL;
    4441   [ +  -  +  - ]:       3419 :         ctrlr->ana_log_page_size = 0;
    4442                 :            : 
    4443                 :       3419 :         nvme_transport_ctrlr_destruct(ctrlr);
    4444                 :            : 
    4445                 :       3419 :         return rc;
    4446                 :    1241393 : }
    4447                 :            : 
    4448                 :            : void
    4449                 :        308 : nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
    4450                 :            : {
    4451                 :        308 :         struct nvme_ctrlr_detach_ctx ctx = { .ctrlr = ctrlr };
    4452                 :            :         int rc;
    4453                 :            : 
    4454                 :        308 :         nvme_ctrlr_destruct_async(ctrlr, &ctx);
    4455                 :            : 
    4456                 :          0 :         while (1) {
    4457                 :       1154 :                 rc = nvme_ctrlr_destruct_poll_async(ctrlr, &ctx);
    4458         [ +  + ]:       1154 :                 if (rc != -EAGAIN) {
    4459                 :        308 :                         break;
    4460                 :            :                 }
    4461                 :        846 :                 nvme_delay(1000);
    4462                 :            :         }
    4463                 :        308 : }
    4464                 :            : 
    4465                 :            : int
    4466                 :    1753782 : nvme_ctrlr_submit_admin_request(struct spdk_nvme_ctrlr *ctrlr,
    4467                 :            :                                 struct nvme_request *req)
    4468                 :            : {
    4469   [ +  -  +  - ]:    1753782 :         return nvme_qpair_submit_request(ctrlr->adminq, req);
    4470                 :            : }
    4471                 :            : 
    4472                 :            : static void
    4473                 :       2395 : nvme_keep_alive_completion(void *cb_ctx, const struct spdk_nvme_cpl *cpl)
    4474                 :            : {
    4475                 :            :         /* Do nothing */
    4476                 :       2395 : }
    4477                 :            : 
    4478                 :            : /*
    4479                 :            :  * Check if we need to send a Keep Alive command.
    4480                 :            :  * Caller must hold ctrlr->ctrlr_lock.
    4481                 :            :  */
    4482                 :            : static int
    4483                 :    7026194 : nvme_ctrlr_keep_alive(struct spdk_nvme_ctrlr *ctrlr)
    4484                 :            : {
    4485                 :            :         uint64_t now;
    4486                 :            :         struct nvme_request *req;
    4487                 :            :         struct spdk_nvme_cmd *cmd;
    4488                 :    7026194 :         int rc = 0;
    4489                 :            : 
    4490                 :    7026194 :         now = spdk_get_ticks();
    4491   [ +  +  +  -  :    7026194 :         if (now < ctrlr->next_keep_alive_tick) {
                   +  + ]
    4492                 :    6970129 :                 return rc;
    4493                 :            :         }
    4494                 :            : 
    4495   [ +  -  +  - ]:      56065 :         req = nvme_allocate_request_null(ctrlr->adminq, nvme_keep_alive_completion, NULL);
    4496         [ +  + ]:      56065 :         if (req == NULL) {
    4497                 :      53664 :                 return rc;
    4498                 :            :         }
    4499                 :            : 
    4500         [ +  - ]:       2401 :         cmd = &req->cmd;
    4501         [ +  - ]:       2401 :         cmd->opc = SPDK_NVME_OPC_KEEP_ALIVE;
    4502                 :            : 
    4503                 :       2401 :         rc = nvme_ctrlr_submit_admin_request(ctrlr, req);
    4504         [ +  + ]:       2401 :         if (rc != 0) {
    4505   [ +  -  +  -  :          6 :                 NVME_CTRLR_ERRLOG(ctrlr, "Submitting Keep Alive failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4506                 :          6 :                 rc = -ENXIO;
    4507                 :          0 :         }
    4508                 :            : 
    4509   [ +  -  +  -  :       2401 :         ctrlr->next_keep_alive_tick = now + ctrlr->keep_alive_interval_ticks;
             +  -  +  - ]
    4510                 :       2401 :         return rc;
    4511                 :      11108 : }
    4512                 :            : 
    4513                 :            : int32_t
    4514                 :   51956868 : spdk_nvme_ctrlr_process_admin_completions(struct spdk_nvme_ctrlr *ctrlr)
    4515                 :            : {
    4516                 :            :         int32_t num_completions;
    4517                 :            :         int32_t rc;
    4518                 :            :         struct spdk_nvme_ctrlr_process  *active_proc;
    4519                 :            : 
    4520                 :   51956868 :         nvme_ctrlr_lock(ctrlr);
    4521                 :            : 
    4522   [ +  +  +  -  :   51956868 :         if (ctrlr->keep_alive_interval_ticks) {
                   +  + ]
    4523                 :    7026194 :                 rc = nvme_ctrlr_keep_alive(ctrlr);
    4524         [ +  + ]:    7026194 :                 if (rc) {
    4525                 :          6 :                         nvme_ctrlr_unlock(ctrlr);
    4526                 :          6 :                         return rc;
    4527                 :            :                 }
    4528                 :      11108 :         }
    4529                 :            : 
    4530                 :   51956862 :         rc = nvme_io_msg_process(ctrlr);
    4531         [ +  + ]:   51956862 :         if (rc < 0) {
    4532                 :          0 :                 nvme_ctrlr_unlock(ctrlr);
    4533                 :          0 :                 return rc;
    4534                 :            :         }
    4535                 :   51956862 :         num_completions = rc;
    4536                 :            : 
    4537   [ +  -  +  - ]:   51956862 :         rc = spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
    4538                 :            : 
    4539                 :            :         /* Each process has an async list, complete the ones for this process object */
    4540                 :   51956862 :         active_proc = nvme_ctrlr_get_current_process(ctrlr);
    4541         [ +  + ]:   51956862 :         if (active_proc) {
    4542                 :   51956859 :                 nvme_ctrlr_complete_queued_async_events(ctrlr);
    4543                 :      11137 :         }
    4544                 :            : 
    4545   [ +  +  +  +  :   51956862 :         if (rc == -ENXIO && ctrlr->is_disconnecting) {
          +  +  #  #  #  
                      # ]
    4546                 :        515 :                 nvme_ctrlr_disconnect_done(ctrlr);
    4547                 :          0 :         }
    4548                 :            : 
    4549                 :   51956862 :         nvme_ctrlr_unlock(ctrlr);
    4550                 :            : 
    4551         [ +  + ]:   51956862 :         if (rc < 0) {
    4552                 :       1292 :                 num_completions = rc;
    4553                 :          0 :         } else {
    4554         [ +  - ]:   51955570 :                 num_completions += rc;
    4555                 :            :         }
    4556                 :            : 
    4557                 :   51956862 :         return num_completions;
    4558                 :      11137 : }
    4559                 :            : 
    4560                 :            : const struct spdk_nvme_ctrlr_data *
    4561                 :    2415314 : spdk_nvme_ctrlr_get_data(struct spdk_nvme_ctrlr *ctrlr)
    4562                 :            : {
    4563         [ +  - ]:    2415314 :         return &ctrlr->cdata;
    4564                 :            : }
    4565                 :            : 
    4566                 :      34917 : union spdk_nvme_csts_register spdk_nvme_ctrlr_get_regs_csts(struct spdk_nvme_ctrlr *ctrlr)
    4567                 :            : {
    4568                 :      20589 :         union spdk_nvme_csts_register csts;
    4569                 :            : 
    4570         [ -  + ]:      34917 :         if (nvme_ctrlr_get_csts(ctrlr, &csts)) {
    4571                 :          0 :                 csts.raw = SPDK_NVME_INVALID_REGISTER_VALUE;
    4572                 :          0 :         }
    4573                 :      34917 :         return csts;
    4574                 :            : }
    4575                 :            : 
    4576                 :          0 : union spdk_nvme_cc_register spdk_nvme_ctrlr_get_regs_cc(struct spdk_nvme_ctrlr *ctrlr)
    4577                 :            : {
    4578                 :          0 :         union spdk_nvme_cc_register cc;
    4579                 :            : 
    4580         [ #  # ]:          0 :         if (nvme_ctrlr_get_cc(ctrlr, &cc)) {
    4581                 :          0 :                 cc.raw = SPDK_NVME_INVALID_REGISTER_VALUE;
    4582                 :          0 :         }
    4583                 :          0 :         return cc;
    4584                 :            : }
    4585                 :            : 
    4586                 :         87 : union spdk_nvme_cap_register spdk_nvme_ctrlr_get_regs_cap(struct spdk_nvme_ctrlr *ctrlr)
    4587                 :            : {
    4588         [ #  # ]:         87 :         return ctrlr->cap;
    4589                 :            : }
    4590                 :            : 
    4591                 :       1183 : union spdk_nvme_vs_register spdk_nvme_ctrlr_get_regs_vs(struct spdk_nvme_ctrlr *ctrlr)
    4592                 :            : {
    4593         [ #  # ]:       1183 :         return ctrlr->vs;
    4594                 :            : }
    4595                 :            : 
    4596                 :          8 : union spdk_nvme_cmbsz_register spdk_nvme_ctrlr_get_regs_cmbsz(struct spdk_nvme_ctrlr *ctrlr)
    4597                 :            : {
    4598                 :          0 :         union spdk_nvme_cmbsz_register cmbsz;
    4599                 :            : 
    4600         [ -  + ]:          8 :         if (nvme_ctrlr_get_cmbsz(ctrlr, &cmbsz)) {
    4601                 :          0 :                 cmbsz.raw = 0;
    4602                 :          0 :         }
    4603                 :            : 
    4604                 :          8 :         return cmbsz;
    4605                 :            : }
    4606                 :            : 
    4607                 :          8 : union spdk_nvme_pmrcap_register spdk_nvme_ctrlr_get_regs_pmrcap(struct spdk_nvme_ctrlr *ctrlr)
    4608                 :            : {
    4609                 :          0 :         union spdk_nvme_pmrcap_register pmrcap;
    4610                 :            : 
    4611         [ -  + ]:          8 :         if (nvme_ctrlr_get_pmrcap(ctrlr, &pmrcap)) {
    4612                 :          0 :                 pmrcap.raw = 0;
    4613                 :          0 :         }
    4614                 :            : 
    4615                 :          8 :         return pmrcap;
    4616                 :            : }
    4617                 :            : 
    4618                 :          0 : union spdk_nvme_bpinfo_register spdk_nvme_ctrlr_get_regs_bpinfo(struct spdk_nvme_ctrlr *ctrlr)
    4619                 :            : {
    4620                 :          0 :         union spdk_nvme_bpinfo_register bpinfo;
    4621                 :            : 
    4622         [ #  # ]:          0 :         if (nvme_ctrlr_get_bpinfo(ctrlr, &bpinfo)) {
    4623                 :          0 :                 bpinfo.raw = 0;
    4624                 :          0 :         }
    4625                 :            : 
    4626                 :          0 :         return bpinfo;
    4627                 :            : }
    4628                 :            : 
    4629                 :            : uint64_t
    4630                 :          8 : spdk_nvme_ctrlr_get_pmrsz(struct spdk_nvme_ctrlr *ctrlr)
    4631                 :            : {
    4632   [ #  #  #  # ]:          8 :         return ctrlr->pmr_size;
    4633                 :            : }
    4634                 :            : 
    4635                 :            : uint32_t
    4636                 :         31 : spdk_nvme_ctrlr_get_num_ns(struct spdk_nvme_ctrlr *ctrlr)
    4637                 :            : {
    4638   [ +  -  +  -  :         31 :         return ctrlr->cdata.nn;
                   +  - ]
    4639                 :            : }
    4640                 :            : 
    4641                 :            : bool
    4642                 :      28017 : spdk_nvme_ctrlr_is_active_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid)
    4643                 :            : {
    4644                 :      27943 :         struct spdk_nvme_ns tmp, *ns;
    4645                 :            : 
    4646         [ #  # ]:      28017 :         tmp.id = nsid;
    4647         [ #  # ]:      28017 :         ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp);
    4648                 :            : 
    4649         [ +  + ]:      28017 :         if (ns != NULL) {
    4650   [ -  +  #  #  :      27734 :                 return ns->active;
                   #  # ]
    4651                 :            :         }
    4652                 :            : 
    4653                 :        283 :         return false;
    4654                 :          0 : }
    4655                 :            : 
    4656                 :            : uint32_t
    4657                 :      12816 : spdk_nvme_ctrlr_get_first_active_ns(struct spdk_nvme_ctrlr *ctrlr)
    4658                 :            : {
    4659                 :            :         struct spdk_nvme_ns *ns;
    4660                 :            : 
    4661         [ +  - ]:      12816 :         ns = RB_MIN(nvme_ns_tree, &ctrlr->ns);
    4662         [ +  + ]:      12816 :         if (ns == NULL) {
    4663                 :       1266 :                 return 0;
    4664                 :            :         }
    4665                 :            : 
    4666         [ +  + ]:      25400 :         while (ns != NULL) {
    4667   [ +  +  +  +  :      25386 :                 if (ns->active) {
             +  -  -  + ]
    4668   [ -  +  -  + ]:      11536 :                         return ns->id;
    4669                 :            :                 }
    4670                 :            : 
    4671                 :      13850 :                 ns = RB_NEXT(nvme_ns_tree, &ctrlr->ns, ns);
    4672                 :            :         }
    4673                 :            : 
    4674                 :         14 :         return 0;
    4675                 :       1673 : }
    4676                 :            : 
    4677                 :            : uint32_t
    4678                 :      26087 : spdk_nvme_ctrlr_get_next_active_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t prev_nsid)
    4679                 :            : {
    4680                 :      15898 :         struct spdk_nvme_ns tmp, *ns;
    4681                 :            : 
    4682         [ +  - ]:      26087 :         tmp.id = prev_nsid;
    4683         [ +  - ]:      26087 :         ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp);
    4684         [ +  + ]:      26087 :         if (ns == NULL) {
    4685                 :         15 :                 return 0;
    4686                 :            :         }
    4687                 :            : 
    4688                 :      26072 :         ns = RB_NEXT(nvme_ns_tree, &ctrlr->ns, ns);
    4689         [ +  + ]:      30670 :         while (ns != NULL) {
    4690   [ +  +  +  +  :      19221 :                 if (ns->active) {
             #  #  #  # ]
    4691   [ #  #  #  # ]:      14623 :                         return ns->id;
    4692                 :            :                 }
    4693                 :            : 
    4694                 :       4598 :                 ns = RB_NEXT(nvme_ns_tree, &ctrlr->ns, ns);
    4695                 :            :         }
    4696                 :            : 
    4697                 :      11449 :         return 0;
    4698                 :       1665 : }
    4699                 :            : 
    4700                 :            : struct spdk_nvme_ns *
    4701                 :      56808 : spdk_nvme_ctrlr_get_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid)
    4702                 :            : {
    4703                 :      41066 :         struct spdk_nvme_ns tmp;
    4704                 :            :         struct spdk_nvme_ns *ns;
    4705                 :            : 
    4706   [ +  +  +  +  :      56808 :         if (nsid < 1 || nsid > ctrlr->cdata.nn) {
          +  -  +  -  -  
                      + ]
    4707                 :       7319 :                 return NULL;
    4708                 :            :         }
    4709                 :            : 
    4710                 :      49489 :         nvme_ctrlr_lock(ctrlr);
    4711                 :            : 
    4712         [ +  - ]:      49489 :         tmp.id = nsid;
    4713         [ +  - ]:      49489 :         ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp);
    4714                 :            : 
    4715         [ +  + ]:      49489 :         if (ns == NULL) {
    4716                 :      25839 :                 ns = spdk_zmalloc(sizeof(struct spdk_nvme_ns), 64, NULL, SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_SHARE);
    4717         [ +  + ]:      25839 :                 if (ns == NULL) {
    4718                 :          0 :                         nvme_ctrlr_unlock(ctrlr);
    4719                 :          0 :                         return NULL;
    4720                 :            :                 }
    4721                 :            : 
    4722   [ +  +  +  +  :      25839 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Namespace %u was added\n", nsid);
          +  +  +  +  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    4723   [ +  -  +  - ]:      25839 :                 ns->id = nsid;
    4724         [ +  - ]:      25839 :                 RB_INSERT(nvme_ns_tree, &ctrlr->ns, ns);
    4725                 :        833 :         }
    4726                 :            : 
    4727                 :      49489 :         nvme_ctrlr_unlock(ctrlr);
    4728                 :            : 
    4729                 :      49489 :         return ns;
    4730                 :       4170 : }
    4731                 :            : 
    4732                 :            : struct spdk_pci_device *
    4733                 :        145 : spdk_nvme_ctrlr_get_pci_device(struct spdk_nvme_ctrlr *ctrlr)
    4734                 :            : {
    4735         [ -  + ]:        145 :         if (ctrlr == NULL) {
    4736                 :          0 :                 return NULL;
    4737                 :            :         }
    4738                 :            : 
    4739   [ -  +  #  #  :        145 :         if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) {
             #  #  #  # ]
    4740                 :          0 :                 return NULL;
    4741                 :            :         }
    4742                 :            : 
    4743                 :        145 :         return nvme_ctrlr_proc_get_devhandle(ctrlr);
    4744                 :          0 : }
    4745                 :            : 
    4746                 :            : int32_t
    4747                 :      23033 : spdk_nvme_ctrlr_get_numa_id(struct spdk_nvme_ctrlr *ctrlr)
    4748                 :            : {
    4749   [ +  +  +  -  :      23033 :         if (ctrlr->numa.id_valid) {
                   +  + ]
    4750   [ -  +  -  + ]:      15936 :                 return ctrlr->numa.id;
    4751                 :            :         } else {
    4752                 :       7097 :                 return SPDK_ENV_NUMA_ID_ANY;
    4753                 :            :         }
    4754                 :         19 : }
    4755                 :            : 
    4756                 :            : uint32_t
    4757                 :       1393 : spdk_nvme_ctrlr_get_max_xfer_size(const struct spdk_nvme_ctrlr *ctrlr)
    4758                 :            : {
    4759   [ +  -  +  - ]:       1393 :         return ctrlr->max_xfer_size;
    4760                 :            : }
    4761                 :            : 
    4762                 :            : uint16_t
    4763                 :       1225 : spdk_nvme_ctrlr_get_max_sges(const struct spdk_nvme_ctrlr *ctrlr)
    4764                 :            : {
    4765   [ +  -  #  #  :       1225 :         if (ctrlr->flags & SPDK_NVME_CTRLR_SGL_SUPPORTED) {
                   #  # ]
    4766   [ #  #  #  # ]:       1225 :                 return ctrlr->max_sges;
    4767                 :            :         } else {
    4768                 :          0 :                 return UINT16_MAX;
    4769                 :            :         }
    4770                 :          0 : }
    4771                 :            : 
    4772                 :            : void
    4773                 :       1629 : spdk_nvme_ctrlr_register_aer_callback(struct spdk_nvme_ctrlr *ctrlr,
    4774                 :            :                                       spdk_nvme_aer_cb aer_cb_fn,
    4775                 :            :                                       void *aer_cb_arg)
    4776                 :            : {
    4777                 :            :         struct spdk_nvme_ctrlr_process *active_proc;
    4778                 :            : 
    4779                 :       1629 :         nvme_ctrlr_lock(ctrlr);
    4780                 :            : 
    4781                 :       1629 :         active_proc = nvme_ctrlr_get_current_process(ctrlr);
    4782         [ +  - ]:       1629 :         if (active_proc) {
    4783   [ +  -  +  - ]:       1629 :                 active_proc->aer_cb_fn = aer_cb_fn;
    4784   [ +  -  +  - ]:       1629 :                 active_proc->aer_cb_arg = aer_cb_arg;
    4785                 :          1 :         }
    4786                 :            : 
    4787                 :       1629 :         nvme_ctrlr_unlock(ctrlr);
    4788                 :       1629 : }
    4789                 :            : 
    4790                 :            : void
    4791                 :          0 : spdk_nvme_ctrlr_disable_read_changed_ns_list_log_page(struct spdk_nvme_ctrlr *ctrlr)
    4792                 :            : {
    4793   [ #  #  #  #  :          0 :         ctrlr->opts.disable_read_changed_ns_list_log_page = true;
                   #  # ]
    4794                 :          0 : }
    4795                 :            : 
    4796                 :            : void
    4797                 :         32 : spdk_nvme_ctrlr_register_timeout_callback(struct spdk_nvme_ctrlr *ctrlr,
    4798                 :            :                 uint64_t timeout_io_us, uint64_t timeout_admin_us,
    4799                 :            :                 spdk_nvme_timeout_cb cb_fn, void *cb_arg)
    4800                 :            : {
    4801                 :            :         struct spdk_nvme_ctrlr_process  *active_proc;
    4802                 :            : 
    4803                 :         32 :         nvme_ctrlr_lock(ctrlr);
    4804                 :            : 
    4805                 :         32 :         active_proc = nvme_ctrlr_get_current_process(ctrlr);
    4806         [ +  - ]:         32 :         if (active_proc) {
    4807   [ #  #  #  #  :         32 :                 active_proc->timeout_io_ticks = timeout_io_us * spdk_get_ticks_hz() / 1000000ULL;
                   #  # ]
    4808   [ #  #  #  #  :         32 :                 active_proc->timeout_admin_ticks = timeout_admin_us * spdk_get_ticks_hz() / 1000000ULL;
                   #  # ]
    4809   [ #  #  #  # ]:         32 :                 active_proc->timeout_cb_fn = cb_fn;
    4810   [ #  #  #  # ]:         32 :                 active_proc->timeout_cb_arg = cb_arg;
    4811                 :          0 :         }
    4812                 :            : 
    4813   [ #  #  #  # ]:         32 :         ctrlr->timeout_enabled = true;
    4814                 :            : 
    4815                 :         32 :         nvme_ctrlr_unlock(ctrlr);
    4816                 :         32 : }
    4817                 :            : 
    4818                 :            : bool
    4819                 :        357 : spdk_nvme_ctrlr_is_log_page_supported(struct spdk_nvme_ctrlr *ctrlr, uint8_t log_page)
    4820                 :            : {
    4821                 :            :         /* No bounds check necessary, since log_page is uint8_t and log_page_supported has 256 entries */
    4822                 :            :         SPDK_STATIC_ASSERT(sizeof(ctrlr->log_page_supported) == 256, "log_page_supported size mismatch");
    4823   [ -  +  #  #  :        357 :         return ctrlr->log_page_supported[log_page];
          #  #  #  #  #  
                      # ]
    4824                 :            : }
    4825                 :            : 
    4826                 :            : bool
    4827                 :         12 : spdk_nvme_ctrlr_is_feature_supported(struct spdk_nvme_ctrlr *ctrlr, uint8_t feature_code)
    4828                 :            : {
    4829                 :            :         /* No bounds check necessary, since feature_code is uint8_t and feature_supported has 256 entries */
    4830                 :            :         SPDK_STATIC_ASSERT(sizeof(ctrlr->feature_supported) == 256, "feature_supported size mismatch");
    4831   [ -  +  #  #  :         12 :         return ctrlr->feature_supported[feature_code];
          #  #  #  #  #  
                      # ]
    4832                 :            : }
    4833                 :            : 
    4834                 :            : int
    4835                 :          3 : spdk_nvme_ctrlr_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    4836                 :            :                           struct spdk_nvme_ctrlr_list *payload)
    4837                 :            : {
    4838                 :            :         struct nvme_completion_poll_status      *status;
    4839                 :            :         struct spdk_nvme_ns                     *ns;
    4840                 :            :         int                                     res;
    4841                 :            : 
    4842         [ -  + ]:          3 :         if (nsid == 0) {
    4843                 :          0 :                 return -EINVAL;
    4844                 :            :         }
    4845                 :            : 
    4846                 :          3 :         status = calloc(1, sizeof(*status));
    4847         [ -  + ]:          3 :         if (!status) {
    4848   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4849                 :          0 :                 return -ENOMEM;
    4850                 :            :         }
    4851                 :            : 
    4852                 :          3 :         res = nvme_ctrlr_cmd_attach_ns(ctrlr, nsid, payload,
    4853                 :          0 :                                        nvme_completion_poll_cb, status);
    4854         [ -  + ]:          3 :         if (res) {
    4855                 :          0 :                 free(status);
    4856                 :          0 :                 return res;
    4857                 :            :         }
    4858   [ -  +  #  #  :          3 :         if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
             #  #  #  # ]
    4859   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_attach_ns failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4860   [ #  #  #  #  :          0 :                 if (!status->timed_out) {
             #  #  #  # ]
    4861                 :          0 :                         free(status);
    4862                 :          0 :                 }
    4863                 :          0 :                 return -ENXIO;
    4864                 :            :         }
    4865                 :          3 :         free(status);
    4866                 :            : 
    4867                 :          3 :         res = nvme_ctrlr_identify_active_ns(ctrlr);
    4868         [ -  + ]:          3 :         if (res) {
    4869                 :          0 :                 return res;
    4870                 :            :         }
    4871                 :            : 
    4872                 :          3 :         ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
    4873         [ -  + ]:          3 :         if (ns == NULL) {
    4874   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_get_ns failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4875                 :          0 :                 return -ENXIO;
    4876                 :            :         }
    4877                 :            : 
    4878                 :          3 :         return nvme_ns_construct(ns, nsid, ctrlr);
    4879                 :          0 : }
    4880                 :            : 
    4881                 :            : int
    4882                 :          3 : spdk_nvme_ctrlr_detach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    4883                 :            :                           struct spdk_nvme_ctrlr_list *payload)
    4884                 :            : {
    4885                 :            :         struct nvme_completion_poll_status      *status;
    4886                 :            :         int                                     res;
    4887                 :            : 
    4888         [ -  + ]:          3 :         if (nsid == 0) {
    4889                 :          0 :                 return -EINVAL;
    4890                 :            :         }
    4891                 :            : 
    4892                 :          3 :         status = calloc(1, sizeof(*status));
    4893         [ -  + ]:          3 :         if (!status) {
    4894   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4895                 :          0 :                 return -ENOMEM;
    4896                 :            :         }
    4897                 :            : 
    4898                 :          3 :         res = nvme_ctrlr_cmd_detach_ns(ctrlr, nsid, payload,
    4899                 :          0 :                                        nvme_completion_poll_cb, status);
    4900         [ -  + ]:          3 :         if (res) {
    4901                 :          0 :                 free(status);
    4902                 :          0 :                 return res;
    4903                 :            :         }
    4904   [ -  +  #  #  :          3 :         if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
             #  #  #  # ]
    4905   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_detach_ns failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4906   [ #  #  #  #  :          0 :                 if (!status->timed_out) {
             #  #  #  # ]
    4907                 :          0 :                         free(status);
    4908                 :          0 :                 }
    4909                 :          0 :                 return -ENXIO;
    4910                 :            :         }
    4911                 :          3 :         free(status);
    4912                 :            : 
    4913                 :          3 :         return nvme_ctrlr_identify_active_ns(ctrlr);
    4914                 :          0 : }
    4915                 :            : 
    4916                 :            : uint32_t
    4917                 :          3 : spdk_nvme_ctrlr_create_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns_data *payload)
    4918                 :            : {
    4919                 :            :         struct nvme_completion_poll_status      *status;
    4920                 :            :         int                                     res;
    4921                 :            :         uint32_t                                nsid;
    4922                 :            : 
    4923                 :          3 :         status = calloc(1, sizeof(*status));
    4924         [ -  + ]:          3 :         if (!status) {
    4925   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4926                 :          0 :                 return 0;
    4927                 :            :         }
    4928                 :            : 
    4929                 :          3 :         res = nvme_ctrlr_cmd_create_ns(ctrlr, payload, nvme_completion_poll_cb, status);
    4930         [ -  + ]:          3 :         if (res) {
    4931                 :          0 :                 free(status);
    4932                 :          0 :                 return 0;
    4933                 :            :         }
    4934   [ -  +  #  #  :          3 :         if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
             #  #  #  # ]
    4935   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_create_ns failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4936   [ #  #  #  #  :          0 :                 if (!status->timed_out) {
             #  #  #  # ]
    4937                 :          0 :                         free(status);
    4938                 :          0 :                 }
    4939                 :          0 :                 return 0;
    4940                 :            :         }
    4941                 :            : 
    4942   [ #  #  #  #  :          3 :         nsid = status->cpl.cdw0;
                   #  # ]
    4943                 :          3 :         free(status);
    4944                 :            : 
    4945   [ -  +  #  # ]:          3 :         assert(nsid > 0);
    4946                 :            : 
    4947                 :            :         /* Return the namespace ID that was created */
    4948                 :          3 :         return nsid;
    4949                 :          0 : }
    4950                 :            : 
    4951                 :            : int
    4952                 :          3 : spdk_nvme_ctrlr_delete_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid)
    4953                 :            : {
    4954                 :            :         struct nvme_completion_poll_status      *status;
    4955                 :            :         int                                     res;
    4956                 :            : 
    4957         [ -  + ]:          3 :         if (nsid == 0) {
    4958                 :          0 :                 return -EINVAL;
    4959                 :            :         }
    4960                 :            : 
    4961                 :          3 :         status = calloc(1, sizeof(*status));
    4962         [ -  + ]:          3 :         if (!status) {
    4963   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4964                 :          0 :                 return -ENOMEM;
    4965                 :            :         }
    4966                 :            : 
    4967                 :          3 :         res = nvme_ctrlr_cmd_delete_ns(ctrlr, nsid, nvme_completion_poll_cb, status);
    4968         [ -  + ]:          3 :         if (res) {
    4969                 :          0 :                 free(status);
    4970                 :          0 :                 return res;
    4971                 :            :         }
    4972   [ -  +  #  #  :          3 :         if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
             #  #  #  # ]
    4973   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_delete_ns failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4974   [ #  #  #  #  :          0 :                 if (!status->timed_out) {
             #  #  #  # ]
    4975                 :          0 :                         free(status);
    4976                 :          0 :                 }
    4977                 :          0 :                 return -ENXIO;
    4978                 :            :         }
    4979                 :          3 :         free(status);
    4980                 :            : 
    4981                 :          3 :         return nvme_ctrlr_identify_active_ns(ctrlr);
    4982                 :          0 : }
    4983                 :            : 
    4984                 :            : int
    4985                 :          0 : spdk_nvme_ctrlr_format(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    4986                 :            :                        struct spdk_nvme_format *format)
    4987                 :            : {
    4988                 :            :         struct nvme_completion_poll_status      *status;
    4989                 :            :         int                                     res;
    4990                 :            : 
    4991                 :          0 :         status = calloc(1, sizeof(*status));
    4992         [ #  # ]:          0 :         if (!status) {
    4993   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    4994                 :          0 :                 return -ENOMEM;
    4995                 :            :         }
    4996                 :            : 
    4997                 :          0 :         res = nvme_ctrlr_cmd_format(ctrlr, nsid, format, nvme_completion_poll_cb,
    4998                 :          0 :                                     status);
    4999         [ #  # ]:          0 :         if (res) {
    5000                 :          0 :                 free(status);
    5001                 :          0 :                 return res;
    5002                 :            :         }
    5003   [ #  #  #  #  :          0 :         if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
             #  #  #  # ]
    5004   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_format failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5005   [ #  #  #  #  :          0 :                 if (!status->timed_out) {
             #  #  #  # ]
    5006                 :          0 :                         free(status);
    5007                 :          0 :                 }
    5008                 :          0 :                 return -ENXIO;
    5009                 :            :         }
    5010                 :          0 :         free(status);
    5011                 :            : 
    5012                 :          0 :         return spdk_nvme_ctrlr_reset(ctrlr);
    5013                 :          0 : }
    5014                 :            : 
    5015                 :            : int
    5016                 :         24 : spdk_nvme_ctrlr_update_firmware(struct spdk_nvme_ctrlr *ctrlr, void *payload, uint32_t size,
    5017                 :            :                                 int slot, enum spdk_nvme_fw_commit_action commit_action, struct spdk_nvme_status *completion_status)
    5018                 :            : {
    5019                 :         24 :         struct spdk_nvme_fw_commit              fw_commit;
    5020                 :            :         struct nvme_completion_poll_status      *status;
    5021                 :            :         int                                     res;
    5022                 :            :         unsigned int                            size_remaining;
    5023                 :            :         unsigned int                            offset;
    5024                 :            :         unsigned int                            transfer;
    5025                 :            :         uint8_t                                 *p;
    5026                 :            : 
    5027         [ -  + ]:         24 :         if (!completion_status) {
    5028                 :          0 :                 return -EINVAL;
    5029                 :            :         }
    5030         [ -  + ]:         24 :         memset(completion_status, 0, sizeof(struct spdk_nvme_status));
    5031   [ +  +  #  # ]:         24 :         if (size % 4) {
    5032   [ +  -  -  +  :          3 :                 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_update_firmware invalid size!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5033                 :          3 :                 return -1;
    5034                 :            :         }
    5035                 :            : 
    5036                 :            :         /* Current support only for SPDK_NVME_FW_COMMIT_REPLACE_IMG
    5037                 :            :          * and SPDK_NVME_FW_COMMIT_REPLACE_AND_ENABLE_IMG
    5038                 :            :          */
    5039   [ -  +  -  - ]:         21 :         if ((commit_action != SPDK_NVME_FW_COMMIT_REPLACE_IMG) &&
    5040                 :          0 :             (commit_action != SPDK_NVME_FW_COMMIT_REPLACE_AND_ENABLE_IMG)) {
    5041   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_update_firmware invalid command!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5042                 :          0 :                 return -1;
    5043                 :            :         }
    5044                 :            : 
    5045                 :         21 :         status = calloc(1, sizeof(*status));
    5046         [ -  + ]:         21 :         if (!status) {
    5047   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5048                 :          0 :                 return -ENOMEM;
    5049                 :            :         }
    5050                 :            : 
    5051                 :            :         /* Firmware download */
    5052                 :         21 :         size_remaining = size;
    5053                 :         21 :         offset = 0;
    5054                 :         21 :         p = payload;
    5055                 :            : 
    5056         [ +  + ]:         30 :         while (size_remaining > 0) {
    5057   [ #  #  #  #  :         21 :                 transfer = spdk_min(size_remaining, ctrlr->min_page_size);
          #  #  #  #  #  
                      # ]
    5058                 :            : 
    5059         [ -  + ]:         21 :                 memset(status, 0, sizeof(*status));
    5060                 :         21 :                 res = nvme_ctrlr_cmd_fw_image_download(ctrlr, transfer, offset, p,
    5061                 :            :                                                        nvme_completion_poll_cb,
    5062                 :          0 :                                                        status);
    5063         [ +  + ]:         21 :                 if (res) {
    5064                 :          6 :                         free(status);
    5065                 :          6 :                         return res;
    5066                 :            :                 }
    5067                 :            : 
    5068   [ +  +  #  #  :         15 :                 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
             #  #  #  # ]
    5069   [ +  -  -  +  :          6 :                         NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_fw_image_download failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5070   [ +  +  +  +  :          6 :                         if (!status->timed_out) {
             #  #  #  # ]
    5071                 :          3 :                                 free(status);
    5072                 :          0 :                         }
    5073                 :          6 :                         return -ENXIO;
    5074                 :            :                 }
    5075         [ #  # ]:          9 :                 p += transfer;
    5076                 :          9 :                 offset += transfer;
    5077                 :          9 :                 size_remaining -= transfer;
    5078                 :            :         }
    5079                 :            : 
    5080                 :            :         /* Firmware commit */
    5081         [ -  + ]:          9 :         memset(&fw_commit, 0, sizeof(struct spdk_nvme_fw_commit));
    5082                 :          9 :         fw_commit.fs = slot;
    5083                 :          9 :         fw_commit.ca = commit_action;
    5084                 :            : 
    5085         [ -  + ]:          9 :         memset(status, 0, sizeof(*status));
    5086                 :          9 :         res = nvme_ctrlr_cmd_fw_commit(ctrlr, &fw_commit, nvme_completion_poll_cb,
    5087                 :          0 :                                        status);
    5088         [ +  + ]:          9 :         if (res) {
    5089                 :          3 :                 free(status);
    5090                 :          3 :                 return res;
    5091                 :            :         }
    5092                 :            : 
    5093   [ #  #  #  #  :          6 :         res = nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock);
                   #  # ]
    5094                 :            : 
    5095   [ #  #  #  #  :          6 :         memcpy(completion_status, &status->cpl.status, sizeof(struct spdk_nvme_status));
          #  #  #  #  #  
                      # ]
    5096                 :            : 
    5097   [ +  +  +  -  :          6 :         if (!status->timed_out) {
             #  #  #  # ]
    5098                 :          6 :                 free(status);
    5099                 :          0 :         }
    5100                 :            : 
    5101         [ +  + ]:          6 :         if (res) {
    5102   [ -  +  #  #  :          3 :                 if (completion_status->sct != SPDK_NVME_SCT_COMMAND_SPECIFIC ||
                   #  # ]
    5103         [ #  # ]:          0 :                     completion_status->sc != SPDK_NVME_SC_FIRMWARE_REQ_NVM_RESET) {
    5104   [ -  +  #  #  :          3 :                         if (completion_status->sct == SPDK_NVME_SCT_COMMAND_SPECIFIC  &&
                   #  # ]
    5105         [ #  # ]:          0 :                             completion_status->sc == SPDK_NVME_SC_FIRMWARE_REQ_CONVENTIONAL_RESET) {
    5106   [ #  #  #  #  :          0 :                                 NVME_CTRLR_NOTICELOG(ctrlr,
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5107                 :            :                                                      "firmware activation requires conventional reset to be performed. !\n");
    5108                 :          0 :                         } else {
    5109   [ +  -  -  +  :          3 :                                 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_commit failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5110                 :            :                         }
    5111                 :          3 :                         return -ENXIO;
    5112                 :            :                 }
    5113                 :          0 :         }
    5114                 :            : 
    5115                 :          3 :         return spdk_nvme_ctrlr_reset(ctrlr);
    5116                 :          0 : }
    5117                 :            : 
    5118                 :            : int
    5119                 :          0 : spdk_nvme_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr)
    5120                 :            : {
    5121                 :            :         int rc, size;
    5122                 :            :         union spdk_nvme_cmbsz_register cmbsz;
    5123                 :            : 
    5124                 :          0 :         cmbsz = spdk_nvme_ctrlr_get_regs_cmbsz(ctrlr);
    5125                 :            : 
    5126   [ #  #  #  # ]:          0 :         if (cmbsz.bits.rds == 0 || cmbsz.bits.wds == 0) {
    5127                 :          0 :                 return -ENOTSUP;
    5128                 :            :         }
    5129                 :            : 
    5130   [ #  #  #  #  :          0 :         size = cmbsz.bits.sz * (0x1000 << (cmbsz.bits.szu * 4));
             #  #  #  # ]
    5131                 :            : 
    5132                 :          0 :         nvme_ctrlr_lock(ctrlr);
    5133                 :          0 :         rc = nvme_transport_ctrlr_reserve_cmb(ctrlr);
    5134                 :          0 :         nvme_ctrlr_unlock(ctrlr);
    5135                 :            : 
    5136         [ #  # ]:          0 :         if (rc < 0) {
    5137                 :          0 :                 return rc;
    5138                 :            :         }
    5139                 :            : 
    5140                 :          0 :         return size;
    5141                 :          0 : }
    5142                 :            : 
    5143                 :            : void *
    5144                 :         17 : spdk_nvme_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size)
    5145                 :            : {
    5146                 :            :         void *buf;
    5147                 :            : 
    5148                 :         17 :         nvme_ctrlr_lock(ctrlr);
    5149                 :         17 :         buf = nvme_transport_ctrlr_map_cmb(ctrlr, size);
    5150                 :         17 :         nvme_ctrlr_unlock(ctrlr);
    5151                 :            : 
    5152                 :         17 :         return buf;
    5153                 :            : }
    5154                 :            : 
    5155                 :            : void
    5156                 :          2 : spdk_nvme_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr)
    5157                 :            : {
    5158                 :          2 :         nvme_ctrlr_lock(ctrlr);
    5159                 :          2 :         nvme_transport_ctrlr_unmap_cmb(ctrlr);
    5160                 :          2 :         nvme_ctrlr_unlock(ctrlr);
    5161                 :          2 : }
    5162                 :            : 
    5163                 :            : int
    5164                 :         44 : spdk_nvme_ctrlr_enable_pmr(struct spdk_nvme_ctrlr *ctrlr)
    5165                 :            : {
    5166                 :            :         int rc;
    5167                 :            : 
    5168                 :         44 :         nvme_ctrlr_lock(ctrlr);
    5169                 :         44 :         rc = nvme_transport_ctrlr_enable_pmr(ctrlr);
    5170                 :         44 :         nvme_ctrlr_unlock(ctrlr);
    5171                 :            : 
    5172                 :         44 :         return rc;
    5173                 :            : }
    5174                 :            : 
    5175                 :            : int
    5176                 :         44 : spdk_nvme_ctrlr_disable_pmr(struct spdk_nvme_ctrlr *ctrlr)
    5177                 :            : {
    5178                 :            :         int rc;
    5179                 :            : 
    5180                 :         44 :         nvme_ctrlr_lock(ctrlr);
    5181                 :         44 :         rc = nvme_transport_ctrlr_disable_pmr(ctrlr);
    5182                 :         44 :         nvme_ctrlr_unlock(ctrlr);
    5183                 :            : 
    5184                 :         44 :         return rc;
    5185                 :            : }
    5186                 :            : 
    5187                 :            : void *
    5188                 :         44 : spdk_nvme_ctrlr_map_pmr(struct spdk_nvme_ctrlr *ctrlr, size_t *size)
    5189                 :            : {
    5190                 :            :         void *buf;
    5191                 :            : 
    5192                 :         44 :         nvme_ctrlr_lock(ctrlr);
    5193                 :         44 :         buf = nvme_transport_ctrlr_map_pmr(ctrlr, size);
    5194                 :         44 :         nvme_ctrlr_unlock(ctrlr);
    5195                 :            : 
    5196                 :         44 :         return buf;
    5197                 :            : }
    5198                 :            : 
    5199                 :            : int
    5200                 :         44 : spdk_nvme_ctrlr_unmap_pmr(struct spdk_nvme_ctrlr *ctrlr)
    5201                 :            : {
    5202                 :            :         int rc;
    5203                 :            : 
    5204                 :         44 :         nvme_ctrlr_lock(ctrlr);
    5205                 :         44 :         rc = nvme_transport_ctrlr_unmap_pmr(ctrlr);
    5206                 :         44 :         nvme_ctrlr_unlock(ctrlr);
    5207                 :            : 
    5208                 :         44 :         return rc;
    5209                 :            : }
    5210                 :            : 
    5211                 :            : int
    5212                 :          0 : spdk_nvme_ctrlr_read_boot_partition_start(struct spdk_nvme_ctrlr *ctrlr, void *payload,
    5213                 :            :                 uint32_t bprsz, uint32_t bprof, uint32_t bpid)
    5214                 :            : {
    5215                 :          0 :         union spdk_nvme_bprsel_register bprsel;
    5216                 :          0 :         union spdk_nvme_bpinfo_register bpinfo;
    5217                 :          0 :         uint64_t bpmbl, bpmb_size;
    5218                 :            : 
    5219   [ #  #  #  #  :          0 :         if (ctrlr->cap.bits.bps == 0) {
             #  #  #  # ]
    5220                 :          0 :                 return -ENOTSUP;
    5221                 :            :         }
    5222                 :            : 
    5223         [ #  # ]:          0 :         if (nvme_ctrlr_get_bpinfo(ctrlr, &bpinfo)) {
    5224   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "get bpinfo failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5225                 :          0 :                 return -EIO;
    5226                 :            :         }
    5227                 :            : 
    5228         [ #  # ]:          0 :         if (bpinfo.bits.brs == SPDK_NVME_BRS_READ_IN_PROGRESS) {
    5229   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Boot Partition read already initiated\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5230                 :          0 :                 return -EALREADY;
    5231                 :            :         }
    5232                 :            : 
    5233                 :          0 :         nvme_ctrlr_lock(ctrlr);
    5234                 :            : 
    5235                 :          0 :         bpmb_size = bprsz * 4096;
    5236                 :          0 :         bpmbl = spdk_vtophys(payload, &bpmb_size);
    5237         [ #  # ]:          0 :         if (bpmbl == SPDK_VTOPHYS_ERROR) {
    5238   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "spdk_vtophys of bpmbl failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5239                 :          0 :                 nvme_ctrlr_unlock(ctrlr);
    5240                 :          0 :                 return -EFAULT;
    5241                 :            :         }
    5242                 :            : 
    5243         [ #  # ]:          0 :         if (bpmb_size != bprsz * 4096) {
    5244   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Boot Partition buffer is not physically contiguous\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5245                 :          0 :                 nvme_ctrlr_unlock(ctrlr);
    5246                 :          0 :                 return -EFAULT;
    5247                 :            :         }
    5248                 :            : 
    5249         [ #  # ]:          0 :         if (nvme_ctrlr_set_bpmbl(ctrlr, bpmbl)) {
    5250   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "set_bpmbl() failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5251                 :          0 :                 nvme_ctrlr_unlock(ctrlr);
    5252                 :          0 :                 return -EIO;
    5253                 :            :         }
    5254                 :            : 
    5255                 :          0 :         bprsel.bits.bpid = bpid;
    5256                 :          0 :         bprsel.bits.bprof = bprof;
    5257                 :          0 :         bprsel.bits.bprsz = bprsz;
    5258                 :            : 
    5259         [ #  # ]:          0 :         if (nvme_ctrlr_set_bprsel(ctrlr, &bprsel)) {
    5260   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "set_bprsel() failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5261                 :          0 :                 nvme_ctrlr_unlock(ctrlr);
    5262                 :          0 :                 return -EIO;
    5263                 :            :         }
    5264                 :            : 
    5265                 :          0 :         nvme_ctrlr_unlock(ctrlr);
    5266                 :          0 :         return 0;
    5267                 :          0 : }
    5268                 :            : 
    5269                 :            : int
    5270                 :          0 : spdk_nvme_ctrlr_read_boot_partition_poll(struct spdk_nvme_ctrlr *ctrlr)
    5271                 :            : {
    5272                 :          0 :         int rc = 0;
    5273                 :          0 :         union spdk_nvme_bpinfo_register bpinfo;
    5274                 :            : 
    5275         [ #  # ]:          0 :         if (nvme_ctrlr_get_bpinfo(ctrlr, &bpinfo)) {
    5276   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "get bpinfo failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5277                 :          0 :                 return -EIO;
    5278                 :            :         }
    5279                 :            : 
    5280   [ #  #  #  #  :          0 :         switch (bpinfo.bits.brs) {
                      # ]
    5281                 :          0 :         case SPDK_NVME_BRS_NO_READ:
    5282   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Boot Partition read not initiated\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5283                 :          0 :                 rc = -EINVAL;
    5284                 :          0 :                 break;
    5285                 :          0 :         case SPDK_NVME_BRS_READ_IN_PROGRESS:
    5286   [ #  #  #  #  :          0 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition read in progress\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    5287                 :          0 :                 rc = -EAGAIN;
    5288                 :          0 :                 break;
    5289                 :          0 :         case SPDK_NVME_BRS_READ_ERROR:
    5290   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Error completing Boot Partition read\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5291                 :          0 :                 rc = -EIO;
    5292                 :          0 :                 break;
    5293                 :          0 :         case SPDK_NVME_BRS_READ_SUCCESS:
    5294   [ #  #  #  #  :          0 :                 NVME_CTRLR_INFOLOG(ctrlr, "Boot Partition read completed successfully\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    5295                 :          0 :                 break;
    5296                 :          0 :         default:
    5297   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Invalid Boot Partition read status\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5298                 :          0 :                 rc = -EINVAL;
    5299                 :          0 :         }
    5300                 :            : 
    5301                 :          0 :         return rc;
    5302                 :          0 : }
    5303                 :            : 
    5304                 :            : static void
    5305                 :          0 : nvme_write_boot_partition_cb(void *arg, const struct spdk_nvme_cpl *cpl)
    5306                 :            : {
    5307                 :            :         int res;
    5308                 :          0 :         struct spdk_nvme_ctrlr *ctrlr = arg;
    5309                 :          0 :         struct spdk_nvme_fw_commit fw_commit;
    5310                 :          0 :         struct spdk_nvme_cpl err_cpl =
    5311                 :            :         {.status = {.sct = SPDK_NVME_SCT_GENERIC, .sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR }};
    5312                 :            : 
    5313   [ #  #  #  #  :          0 :         if (spdk_nvme_cpl_is_error(cpl)) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    5314   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Write Boot Partition failed\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5315   [ #  #  #  #  :          0 :                 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, cpl);
          #  #  #  #  #  
                #  #  # ]
    5316                 :          0 :                 return;
    5317                 :            :         }
    5318                 :            : 
    5319   [ #  #  #  #  :          0 :         if (ctrlr->bp_ws == SPDK_NVME_BP_WS_DOWNLOADING) {
                   #  # ]
    5320   [ #  #  #  #  :          0 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Downloading at Offset %d Success\n", ctrlr->fw_offset);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5321   [ #  #  #  #  :          0 :                 ctrlr->fw_payload = (uint8_t *)ctrlr->fw_payload + ctrlr->fw_transfer_size;
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5322   [ #  #  #  #  :          0 :                 ctrlr->fw_offset += ctrlr->fw_transfer_size;
             #  #  #  # ]
    5323   [ #  #  #  #  :          0 :                 ctrlr->fw_size_remaining -= ctrlr->fw_transfer_size;
             #  #  #  # ]
    5324   [ #  #  #  #  :          0 :                 ctrlr->fw_transfer_size = spdk_min(ctrlr->fw_size_remaining, ctrlr->min_page_size);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
    5325   [ #  #  #  #  :          0 :                 res = nvme_ctrlr_cmd_fw_image_download(ctrlr, ctrlr->fw_transfer_size, ctrlr->fw_offset,
             #  #  #  # ]
    5326   [ #  #  #  # ]:          0 :                                                        ctrlr->fw_payload, nvme_write_boot_partition_cb, ctrlr);
    5327         [ #  # ]:          0 :                 if (res) {
    5328   [ #  #  #  #  :          0 :                         NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_image_download failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5329   [ #  #  #  #  :          0 :                         ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl);
          #  #  #  #  #  
                #  #  # ]
    5330                 :          0 :                         return;
    5331                 :            :                 }
    5332                 :            : 
    5333   [ #  #  #  #  :          0 :                 if (ctrlr->fw_transfer_size < ctrlr->min_page_size) {
          #  #  #  #  #  
                      # ]
    5334   [ #  #  #  # ]:          0 :                         ctrlr->bp_ws = SPDK_NVME_BP_WS_DOWNLOADED;
    5335                 :          0 :                 }
    5336   [ #  #  #  #  :          0 :         } else if (ctrlr->bp_ws == SPDK_NVME_BP_WS_DOWNLOADED) {
                   #  # ]
    5337   [ #  #  #  #  :          0 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Download Success\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    5338         [ #  # ]:          0 :                 memset(&fw_commit, 0, sizeof(struct spdk_nvme_fw_commit));
    5339   [ #  #  #  # ]:          0 :                 fw_commit.bpid = ctrlr->bpid;
    5340                 :          0 :                 fw_commit.ca = SPDK_NVME_FW_COMMIT_REPLACE_BOOT_PARTITION;
    5341                 :          0 :                 res = nvme_ctrlr_cmd_fw_commit(ctrlr, &fw_commit,
    5342                 :          0 :                                                nvme_write_boot_partition_cb, ctrlr);
    5343         [ #  # ]:          0 :                 if (res) {
    5344   [ #  #  #  #  :          0 :                         NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_commit failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5345   [ #  #  #  #  :          0 :                         NVME_CTRLR_ERRLOG(ctrlr, "commit action: %d\n", fw_commit.ca);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5346   [ #  #  #  #  :          0 :                         ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl);
          #  #  #  #  #  
                #  #  # ]
    5347                 :          0 :                         return;
    5348                 :            :                 }
    5349                 :            : 
    5350   [ #  #  #  # ]:          0 :                 ctrlr->bp_ws = SPDK_NVME_BP_WS_REPLACE;
    5351   [ #  #  #  #  :          0 :         } else if (ctrlr->bp_ws == SPDK_NVME_BP_WS_REPLACE) {
                   #  # ]
    5352   [ #  #  #  #  :          0 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Replacement Success\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    5353         [ #  # ]:          0 :                 memset(&fw_commit, 0, sizeof(struct spdk_nvme_fw_commit));
    5354   [ #  #  #  # ]:          0 :                 fw_commit.bpid = ctrlr->bpid;
    5355                 :          0 :                 fw_commit.ca = SPDK_NVME_FW_COMMIT_ACTIVATE_BOOT_PARTITION;
    5356                 :          0 :                 res = nvme_ctrlr_cmd_fw_commit(ctrlr, &fw_commit,
    5357                 :          0 :                                                nvme_write_boot_partition_cb, ctrlr);
    5358         [ #  # ]:          0 :                 if (res) {
    5359   [ #  #  #  #  :          0 :                         NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_commit failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5360   [ #  #  #  #  :          0 :                         NVME_CTRLR_ERRLOG(ctrlr, "commit action: %d\n", fw_commit.ca);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5361   [ #  #  #  #  :          0 :                         ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl);
          #  #  #  #  #  
                #  #  # ]
    5362                 :          0 :                         return;
    5363                 :            :                 }
    5364                 :            : 
    5365   [ #  #  #  # ]:          0 :                 ctrlr->bp_ws = SPDK_NVME_BP_WS_ACTIVATE;
    5366   [ #  #  #  #  :          0 :         } else if (ctrlr->bp_ws == SPDK_NVME_BP_WS_ACTIVATE) {
                   #  # ]
    5367   [ #  #  #  #  :          0 :                 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Activation Success\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
    5368   [ #  #  #  #  :          0 :                 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, cpl);
          #  #  #  #  #  
                #  #  # ]
    5369                 :          0 :         } else {
    5370   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Invalid Boot Partition write state\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5371   [ #  #  #  #  :          0 :                 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl);
          #  #  #  #  #  
                #  #  # ]
    5372                 :          0 :                 return;
    5373                 :            :         }
    5374                 :          0 : }
    5375                 :            : 
    5376                 :            : int
    5377                 :          0 : spdk_nvme_ctrlr_write_boot_partition(struct spdk_nvme_ctrlr *ctrlr,
    5378                 :            :                                      void *payload, uint32_t size, uint32_t bpid,
    5379                 :            :                                      spdk_nvme_cmd_cb cb_fn, void *cb_arg)
    5380                 :            : {
    5381                 :            :         int res;
    5382                 :            : 
    5383   [ #  #  #  #  :          0 :         if (ctrlr->cap.bits.bps == 0) {
             #  #  #  # ]
    5384                 :          0 :                 return -ENOTSUP;
    5385                 :            :         }
    5386                 :            : 
    5387   [ #  #  #  # ]:          0 :         ctrlr->bp_ws = SPDK_NVME_BP_WS_DOWNLOADING;
    5388   [ #  #  #  # ]:          0 :         ctrlr->bpid = bpid;
    5389   [ #  #  #  # ]:          0 :         ctrlr->bp_write_cb_fn = cb_fn;
    5390   [ #  #  #  # ]:          0 :         ctrlr->bp_write_cb_arg = cb_arg;
    5391   [ #  #  #  # ]:          0 :         ctrlr->fw_offset = 0;
    5392   [ #  #  #  # ]:          0 :         ctrlr->fw_size_remaining = size;
    5393   [ #  #  #  # ]:          0 :         ctrlr->fw_payload = payload;
    5394   [ #  #  #  #  :          0 :         ctrlr->fw_transfer_size = spdk_min(ctrlr->fw_size_remaining, ctrlr->min_page_size);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
    5395                 :            : 
    5396   [ #  #  #  #  :          0 :         res = nvme_ctrlr_cmd_fw_image_download(ctrlr, ctrlr->fw_transfer_size, ctrlr->fw_offset,
             #  #  #  # ]
    5397   [ #  #  #  # ]:          0 :                                                ctrlr->fw_payload, nvme_write_boot_partition_cb, ctrlr);
    5398                 :            : 
    5399                 :          0 :         return res;
    5400                 :          0 : }
    5401                 :            : 
    5402                 :            : bool
    5403                 :       9888 : spdk_nvme_ctrlr_is_discovery(struct spdk_nvme_ctrlr *ctrlr)
    5404                 :            : {
    5405   [ +  +  #  # ]:       9888 :         assert(ctrlr);
    5406                 :            : 
    5407   [ +  +  +  -  :       9888 :         return !strncmp(ctrlr->trid.subnqn, SPDK_NVMF_DISCOVERY_NQN,
             +  -  +  - ]
    5408                 :            :                         strlen(SPDK_NVMF_DISCOVERY_NQN));
    5409                 :            : }
    5410                 :            : 
    5411                 :            : bool
    5412                 :       4248 : spdk_nvme_ctrlr_is_fabrics(struct spdk_nvme_ctrlr *ctrlr)
    5413                 :            : {
    5414   [ +  +  #  # ]:       4248 :         assert(ctrlr);
    5415                 :            : 
    5416   [ +  -  +  -  :       4248 :         return spdk_nvme_trtype_is_fabrics(ctrlr->trid.trtype);
                   +  - ]
    5417                 :            : }
    5418                 :            : 
    5419                 :            : int
    5420                 :         64 : spdk_nvme_ctrlr_security_receive(struct spdk_nvme_ctrlr *ctrlr, uint8_t secp,
    5421                 :            :                                  uint16_t spsp, uint8_t nssf, void *payload, size_t size)
    5422                 :            : {
    5423                 :            :         struct nvme_completion_poll_status      *status;
    5424                 :            :         int                                     res;
    5425                 :            : 
    5426                 :         64 :         status = calloc(1, sizeof(*status));
    5427         [ -  + ]:         64 :         if (!status) {
    5428   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5429                 :          0 :                 return -ENOMEM;
    5430                 :            :         }
    5431                 :            : 
    5432                 :         64 :         res = spdk_nvme_ctrlr_cmd_security_receive(ctrlr, secp, spsp, nssf, payload, size,
    5433                 :          0 :                         nvme_completion_poll_cb, status);
    5434         [ -  + ]:         64 :         if (res) {
    5435                 :          0 :                 free(status);
    5436                 :          0 :                 return res;
    5437                 :            :         }
    5438   [ -  +  #  #  :         64 :         if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
             #  #  #  # ]
    5439   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_cmd_security_receive failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5440   [ #  #  #  #  :          0 :                 if (!status->timed_out) {
             #  #  #  # ]
    5441                 :          0 :                         free(status);
    5442                 :          0 :                 }
    5443                 :          0 :                 return -ENXIO;
    5444                 :            :         }
    5445                 :         64 :         free(status);
    5446                 :            : 
    5447                 :         64 :         return 0;
    5448                 :          0 : }
    5449                 :            : 
    5450                 :            : int
    5451                 :          0 : spdk_nvme_ctrlr_security_send(struct spdk_nvme_ctrlr *ctrlr, uint8_t secp,
    5452                 :            :                               uint16_t spsp, uint8_t nssf, void *payload, size_t size)
    5453                 :            : {
    5454                 :            :         struct nvme_completion_poll_status      *status;
    5455                 :            :         int                                     res;
    5456                 :            : 
    5457                 :          0 :         status = calloc(1, sizeof(*status));
    5458         [ #  # ]:          0 :         if (!status) {
    5459   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5460                 :          0 :                 return -ENOMEM;
    5461                 :            :         }
    5462                 :            : 
    5463                 :          0 :         res = spdk_nvme_ctrlr_cmd_security_send(ctrlr, secp, spsp, nssf, payload, size,
    5464                 :            :                                                 nvme_completion_poll_cb,
    5465                 :          0 :                                                 status);
    5466         [ #  # ]:          0 :         if (res) {
    5467                 :          0 :                 free(status);
    5468                 :          0 :                 return res;
    5469                 :            :         }
    5470   [ #  #  #  #  :          0 :         if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) {
             #  #  #  # ]
    5471   [ #  #  #  #  :          0 :                 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_cmd_security_send failed!\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5472   [ #  #  #  #  :          0 :                 if (!status->timed_out) {
             #  #  #  # ]
    5473                 :          0 :                         free(status);
    5474                 :          0 :                 }
    5475                 :          0 :                 return -ENXIO;
    5476                 :            :         }
    5477                 :            : 
    5478                 :          0 :         free(status);
    5479                 :            : 
    5480                 :          0 :         return 0;
    5481                 :          0 : }
    5482                 :            : 
    5483                 :            : uint64_t
    5484                 :       7950 : spdk_nvme_ctrlr_get_flags(struct spdk_nvme_ctrlr *ctrlr)
    5485                 :            : {
    5486   [ +  -  +  - ]:       7950 :         return ctrlr->flags;
    5487                 :            : }
    5488                 :            : 
    5489                 :            : const struct spdk_nvme_transport_id *
    5490                 :       2382 : spdk_nvme_ctrlr_get_transport_id(struct spdk_nvme_ctrlr *ctrlr)
    5491                 :            : {
    5492         [ #  # ]:       2382 :         return &ctrlr->trid;
    5493                 :            : }
    5494                 :            : 
    5495                 :            : int32_t
    5496                 :       5787 : spdk_nvme_ctrlr_alloc_qid(struct spdk_nvme_ctrlr *ctrlr)
    5497                 :            : {
    5498                 :            :         uint32_t qid;
    5499                 :            : 
    5500   [ +  +  +  -  :       5787 :         assert(ctrlr->free_io_qids);
             +  -  #  # ]
    5501                 :       5787 :         nvme_ctrlr_lock(ctrlr);
    5502   [ +  -  +  - ]:       5787 :         qid = spdk_bit_array_find_first_set(ctrlr->free_io_qids, 1);
    5503   [ +  +  +  -  :       5787 :         if (qid > ctrlr->opts.num_io_queues) {
             +  -  -  + ]
    5504   [ +  -  -  +  :          7 :                 NVME_CTRLR_ERRLOG(ctrlr, "No free I/O queue IDs\n");
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    5505                 :          7 :                 nvme_ctrlr_unlock(ctrlr);
    5506                 :          7 :                 return -1;
    5507                 :            :         }
    5508                 :            : 
    5509   [ +  -  +  - ]:       5780 :         spdk_bit_array_clear(ctrlr->free_io_qids, qid);
    5510                 :       5780 :         nvme_ctrlr_unlock(ctrlr);
    5511                 :       5780 :         return qid;
    5512                 :        833 : }
    5513                 :            : 
    5514                 :            : void
    5515                 :     370620 : spdk_nvme_ctrlr_free_qid(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid)
    5516                 :            : {
    5517   [ +  +  +  -  :     370620 :         assert(qid <= ctrlr->opts.num_io_queues);
          +  -  +  -  #  
                      # ]
    5518                 :            : 
    5519                 :     370620 :         nvme_ctrlr_lock(ctrlr);
    5520                 :            : 
    5521   [ +  +  +  -  :     370620 :         if (spdk_likely(ctrlr->free_io_qids)) {
                   -  + ]
    5522   [ +  -  +  - ]:     370584 :                 spdk_bit_array_set(ctrlr->free_io_qids, qid);
    5523                 :     106626 :         }
    5524                 :            : 
    5525                 :     370620 :         nvme_ctrlr_unlock(ctrlr);
    5526                 :     370620 : }
    5527                 :            : 
    5528                 :            : int
    5529                 :      10389 : spdk_nvme_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
    5530                 :            :                                    struct spdk_memory_domain **domains, int array_size)
    5531                 :            : {
    5532                 :      10389 :         return nvme_transport_ctrlr_get_memory_domains(ctrlr, domains, array_size);
    5533                 :            : }
    5534                 :            : 
    5535                 :            : int
    5536                 :         24 : spdk_nvme_ctrlr_authenticate(struct spdk_nvme_ctrlr *ctrlr,
    5537                 :            :                              spdk_nvme_authenticate_cb cb_fn, void *cb_ctx)
    5538                 :            : {
    5539   [ #  #  #  # ]:         24 :         return spdk_nvme_qpair_authenticate(ctrlr->adminq, cb_fn, cb_ctx);
    5540                 :            : }

Generated by: LCOV version 1.15