LCOV - code coverage report
Current view: top level - spdk/lib/nvme - nvme_internal.h (source / functions) Hit Total Coverage
Test: Combined Lines: 116 122 95.1 %
Date: 2024-07-15 11:21:26 Functions: 21 21 100.0 %
Legend: Lines: hit not hit | Branches: + taken - not taken # not executed Branches: 60 86 69.8 %

           Branch data     Line data    Source code
       1                 :            : /*   SPDX-License-Identifier: BSD-3-Clause
       2                 :            :  *   Copyright (C) 2015 Intel Corporation. All rights reserved.
       3                 :            :  *   Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved.
       4                 :            :  *   Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
       5                 :            :  */
       6                 :            : 
       7                 :            : #ifndef __NVME_INTERNAL_H__
       8                 :            : #define __NVME_INTERNAL_H__
       9                 :            : 
      10                 :            : #include "spdk/config.h"
      11                 :            : #include "spdk/likely.h"
      12                 :            : #include "spdk/stdinc.h"
      13                 :            : 
      14                 :            : #include "spdk/nvme.h"
      15                 :            : 
      16                 :            : #if defined(__i386__) || defined(__x86_64__)
      17                 :            : #include <x86intrin.h>
      18                 :            : #endif
      19                 :            : 
      20                 :            : #include "spdk/queue.h"
      21                 :            : #include "spdk/barrier.h"
      22                 :            : #include "spdk/bit_array.h"
      23                 :            : #include "spdk/mmio.h"
      24                 :            : #include "spdk/pci_ids.h"
      25                 :            : #include "spdk/util.h"
      26                 :            : #include "spdk/memory.h"
      27                 :            : #include "spdk/nvme_intel.h"
      28                 :            : #include "spdk/nvmf_spec.h"
      29                 :            : #include "spdk/tree.h"
      30                 :            : #include "spdk/uuid.h"
      31                 :            : 
      32                 :            : #include "spdk_internal/assert.h"
      33                 :            : #include "spdk/log.h"
      34                 :            : 
      35                 :            : extern pid_t g_spdk_nvme_pid;
      36                 :            : 
      37                 :            : extern struct spdk_nvme_transport_opts g_spdk_nvme_transport_opts;
      38                 :            : 
      39                 :            : /*
      40                 :            :  * Some Intel devices support vendor-unique read latency log page even
      41                 :            :  * though the log page directory says otherwise.
      42                 :            :  */
      43                 :            : #define NVME_INTEL_QUIRK_READ_LATENCY 0x1
      44                 :            : 
      45                 :            : /*
      46                 :            :  * Some Intel devices support vendor-unique write latency log page even
      47                 :            :  * though the log page directory says otherwise.
      48                 :            :  */
      49                 :            : #define NVME_INTEL_QUIRK_WRITE_LATENCY 0x2
      50                 :            : 
      51                 :            : /*
      52                 :            :  * The controller needs a delay before starts checking the device
      53                 :            :  * readiness, which is done by reading the NVME_CSTS_RDY bit.
      54                 :            :  */
      55                 :            : #define NVME_QUIRK_DELAY_BEFORE_CHK_RDY 0x4
      56                 :            : 
      57                 :            : /*
      58                 :            :  * The controller performs best when I/O is split on particular
      59                 :            :  * LBA boundaries.
      60                 :            :  */
      61                 :            : #define NVME_INTEL_QUIRK_STRIPING 0x8
      62                 :            : 
      63                 :            : /*
      64                 :            :  * The controller needs a delay after allocating an I/O queue pair
      65                 :            :  * before it is ready to accept I/O commands.
      66                 :            :  */
      67                 :            : #define NVME_QUIRK_DELAY_AFTER_QUEUE_ALLOC 0x10
      68                 :            : 
      69                 :            : /*
      70                 :            :  * Earlier NVMe devices do not indicate whether unmapped blocks
      71                 :            :  * will read all zeroes or not. This define indicates that the
      72                 :            :  * device does in fact read all zeroes after an unmap event
      73                 :            :  */
      74                 :            : #define NVME_QUIRK_READ_ZERO_AFTER_DEALLOCATE 0x20
      75                 :            : 
      76                 :            : /*
      77                 :            :  * The controller doesn't handle Identify value others than 0 or 1 correctly.
      78                 :            :  */
      79                 :            : #define NVME_QUIRK_IDENTIFY_CNS 0x40
      80                 :            : 
      81                 :            : /*
      82                 :            :  * The controller supports Open Channel command set if matching additional
      83                 :            :  * condition, like the first byte (value 0x1) in the vendor specific
      84                 :            :  * bits of the namespace identify structure is set.
      85                 :            :  */
      86                 :            : #define NVME_QUIRK_OCSSD 0x80
      87                 :            : 
      88                 :            : /*
      89                 :            :  * The controller has an Intel vendor ID but does not support Intel vendor-specific
      90                 :            :  * log pages.  This is primarily for QEMU emulated SSDs which report an Intel vendor
      91                 :            :  * ID but do not support these log pages.
      92                 :            :  */
      93                 :            : #define NVME_INTEL_QUIRK_NO_LOG_PAGES 0x100
      94                 :            : 
      95                 :            : /*
      96                 :            :  * The controller does not set SHST_COMPLETE in a reasonable amount of time.  This
      97                 :            :  * is primarily seen in virtual VMWare NVMe SSDs.  This quirk merely adds an additional
      98                 :            :  * error message that on VMWare NVMe SSDs, the shutdown timeout may be expected.
      99                 :            :  */
     100                 :            : #define NVME_QUIRK_SHST_COMPLETE 0x200
     101                 :            : 
     102                 :            : /*
     103                 :            :  * The controller requires an extra delay before starting the initialization process
     104                 :            :  * during attach.
     105                 :            :  */
     106                 :            : #define NVME_QUIRK_DELAY_BEFORE_INIT 0x400
     107                 :            : 
     108                 :            : /*
     109                 :            :  * Some SSDs exhibit poor performance with the default SPDK NVMe IO queue size.
     110                 :            :  * This quirk will increase the default to 1024 which matches other operating
     111                 :            :  * systems, at the cost of some extra memory usage.  Users can still override
     112                 :            :  * the increased default by changing the spdk_nvme_io_qpair_opts when allocating
     113                 :            :  * a new queue pair.
     114                 :            :  */
     115                 :            : #define NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE 0x800
     116                 :            : 
     117                 :            : /**
     118                 :            :  * The maximum access width to PCI memory space is 8 Bytes, don't use AVX2 or
     119                 :            :  * SSE instructions to optimize the memory access(memcpy or memset) larger than
     120                 :            :  * 8 Bytes.
     121                 :            :  */
     122                 :            : #define NVME_QUIRK_MAXIMUM_PCI_ACCESS_WIDTH 0x1000
     123                 :            : 
     124                 :            : /**
     125                 :            :  * The SSD does not support OPAL even through it sets the security bit in OACS.
     126                 :            :  */
     127                 :            : #define NVME_QUIRK_OACS_SECURITY 0x2000
     128                 :            : 
     129                 :            : /**
     130                 :            :  * Intel P55XX SSDs can't support Dataset Management command with SGL format,
     131                 :            :  * so use PRP with DSM command.
     132                 :            :  */
     133                 :            : #define NVME_QUIRK_NO_SGL_FOR_DSM 0x4000
     134                 :            : 
     135                 :            : /**
     136                 :            :  * Maximum Data Transfer Size(MDTS) excludes interleaved metadata.
     137                 :            :  */
     138                 :            : #define NVME_QUIRK_MDTS_EXCLUDE_MD 0x8000
     139                 :            : 
     140                 :            : /**
     141                 :            :  * Force not to use SGL even the controller report that it can
     142                 :            :  * support it.
     143                 :            :  */
     144                 :            : #define NVME_QUIRK_NOT_USE_SGL 0x10000
     145                 :            : 
     146                 :            : /*
     147                 :            :  * Some SSDs require the admin submission queue size to equate to an even
     148                 :            :  * 4KiB multiple.
     149                 :            :  */
     150                 :            : #define NVME_QUIRK_MINIMUM_ADMIN_QUEUE_SIZE 0x20000
     151                 :            : 
     152                 :            : #define NVME_MAX_ASYNC_EVENTS   (8)
     153                 :            : 
     154                 :            : #define NVME_MAX_ADMIN_TIMEOUT_IN_SECS  (30)
     155                 :            : 
     156                 :            : /* Maximum log page size to fetch for AERs. */
     157                 :            : #define NVME_MAX_AER_LOG_SIZE           (4096)
     158                 :            : 
     159                 :            : /*
     160                 :            :  * NVME_MAX_IO_QUEUES in nvme_spec.h defines the 64K spec-limit, but this
     161                 :            :  *  define specifies the maximum number of queues this driver will actually
     162                 :            :  *  try to configure, if available.
     163                 :            :  */
     164                 :            : #define DEFAULT_MAX_IO_QUEUES           (1024)
     165                 :            : #define DEFAULT_ADMIN_QUEUE_SIZE        (32)
     166                 :            : #define DEFAULT_IO_QUEUE_SIZE           (256)
     167                 :            : #define DEFAULT_IO_QUEUE_SIZE_FOR_QUIRK (1024) /* Matches Linux kernel driver */
     168                 :            : 
     169                 :            : #define DEFAULT_IO_QUEUE_REQUESTS       (512)
     170                 :            : 
     171                 :            : #define SPDK_NVME_DEFAULT_RETRY_COUNT   (4)
     172                 :            : 
     173                 :            : #define SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED        (0)
     174                 :            : #define SPDK_NVME_DEFAULT_TRANSPORT_ACK_TIMEOUT SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED
     175                 :            : 
     176                 :            : #define SPDK_NVME_TRANSPORT_TOS_DISABLED        (0)
     177                 :            : 
     178                 :            : #define MIN_KEEP_ALIVE_TIMEOUT_IN_MS    (10000)
     179                 :            : 
     180                 :            : /* We want to fit submission and completion rings each in a single 2MB
     181                 :            :  * hugepage to ensure physical address contiguity.
     182                 :            :  */
     183                 :            : #define MAX_IO_QUEUE_ENTRIES            (VALUE_2MB / spdk_max( \
     184                 :            :                                                 sizeof(struct spdk_nvme_cmd), \
     185                 :            :                                                 sizeof(struct spdk_nvme_cpl)))
     186                 :            : 
     187                 :            : /* Default timeout for fabrics connect commands. */
     188                 :            : #ifdef DEBUG
     189                 :            : #define NVME_FABRIC_CONNECT_COMMAND_TIMEOUT 0
     190                 :            : #else
     191                 :            : /* 500 millisecond timeout. */
     192                 :            : #define NVME_FABRIC_CONNECT_COMMAND_TIMEOUT 500000
     193                 :            : #endif
     194                 :            : 
     195                 :            : /* This value indicates that a read from a PCIe register is invalid. This can happen when a device is no longer present */
     196                 :            : #define SPDK_NVME_INVALID_REGISTER_VALUE 0xFFFFFFFFu
     197                 :            : 
     198                 :            : enum nvme_payload_type {
     199                 :            :         NVME_PAYLOAD_TYPE_INVALID = 0,
     200                 :            : 
     201                 :            :         /** nvme_request::u.payload.contig_buffer is valid for this request */
     202                 :            :         NVME_PAYLOAD_TYPE_CONTIG,
     203                 :            : 
     204                 :            :         /** nvme_request::u.sgl is valid for this request */
     205                 :            :         NVME_PAYLOAD_TYPE_SGL,
     206                 :            : };
     207                 :            : 
     208                 :            : /** Boot partition write states */
     209                 :            : enum nvme_bp_write_state {
     210                 :            :         SPDK_NVME_BP_WS_DOWNLOADING     = 0x0,
     211                 :            :         SPDK_NVME_BP_WS_DOWNLOADED      = 0x1,
     212                 :            :         SPDK_NVME_BP_WS_REPLACE         = 0x2,
     213                 :            :         SPDK_NVME_BP_WS_ACTIVATE        = 0x3,
     214                 :            : };
     215                 :            : 
     216                 :            : /**
     217                 :            :  * Descriptor for a request data payload.
     218                 :            :  */
     219                 :            : struct nvme_payload {
     220                 :            :         /**
     221                 :            :          * Functions for retrieving physical addresses for scattered payloads.
     222                 :            :          */
     223                 :            :         spdk_nvme_req_reset_sgl_cb reset_sgl_fn;
     224                 :            :         spdk_nvme_req_next_sge_cb next_sge_fn;
     225                 :            : 
     226                 :            :         /**
     227                 :            :          * Extended IO options passed by the user
     228                 :            :          */
     229                 :            :         struct spdk_nvme_ns_cmd_ext_io_opts *opts;
     230                 :            :         /**
     231                 :            :          * If reset_sgl_fn == NULL, this is a contig payload, and contig_or_cb_arg contains the
     232                 :            :          * virtual memory address of a single virtually contiguous buffer.
     233                 :            :          *
     234                 :            :          * If reset_sgl_fn != NULL, this is a SGL payload, and contig_or_cb_arg contains the
     235                 :            :          * cb_arg that will be passed to the SGL callback functions.
     236                 :            :          */
     237                 :            :         void *contig_or_cb_arg;
     238                 :            : 
     239                 :            :         /** Virtual memory address of a single virtually contiguous metadata buffer */
     240                 :            :         void *md;
     241                 :            : };
     242                 :            : 
     243                 :            : #define NVME_PAYLOAD_CONTIG(contig_, md_) \
     244                 :            :         (struct nvme_payload) { \
     245                 :            :                 .reset_sgl_fn = NULL, \
     246                 :            :                 .next_sge_fn = NULL, \
     247                 :            :                 .contig_or_cb_arg = (contig_), \
     248                 :            :                 .md = (md_), \
     249                 :            :         }
     250                 :            : 
     251                 :            : #define NVME_PAYLOAD_SGL(reset_sgl_fn_, next_sge_fn_, cb_arg_, md_) \
     252                 :            :         (struct nvme_payload) { \
     253                 :            :                 .reset_sgl_fn = (reset_sgl_fn_), \
     254                 :            :                 .next_sge_fn = (next_sge_fn_), \
     255                 :            :                 .contig_or_cb_arg = (cb_arg_), \
     256                 :            :                 .md = (md_), \
     257                 :            :         }
     258                 :            : 
     259                 :            : static inline enum nvme_payload_type
     260                 :  149089760 : nvme_payload_type(const struct nvme_payload *payload) {
     261         [ +  + ]:  149089760 :         return payload->reset_sgl_fn ? NVME_PAYLOAD_TYPE_SGL : NVME_PAYLOAD_TYPE_CONTIG;
     262                 :            : }
     263                 :            : 
     264                 :            : struct nvme_error_cmd {
     265                 :            :         bool                            do_not_submit;
     266                 :            :         uint64_t                        timeout_tsc;
     267                 :            :         uint32_t                        err_count;
     268                 :            :         uint8_t                         opc;
     269                 :            :         struct spdk_nvme_status         status;
     270                 :            :         TAILQ_ENTRY(nvme_error_cmd)     link;
     271                 :            : };
     272                 :            : 
     273                 :            : struct nvme_request {
     274                 :            :         struct spdk_nvme_cmd            cmd;
     275                 :            : 
     276                 :            :         uint8_t                         retries;
     277                 :            : 
     278                 :            :         uint8_t                         timed_out : 1;
     279                 :            : 
     280                 :            :         /**
     281                 :            :          * True if the request is in the queued_req list.
     282                 :            :          */
     283                 :            :         uint8_t                         queued : 1;
     284                 :            :         uint8_t                         reserved : 6;
     285                 :            : 
     286                 :            :         /**
     287                 :            :          * Number of children requests still outstanding for this
     288                 :            :          *  request which was split into multiple child requests.
     289                 :            :          */
     290                 :            :         uint16_t                        num_children;
     291                 :            : 
     292                 :            :         /**
     293                 :            :          * Offset in bytes from the beginning of payload for this request.
     294                 :            :          * This is used for I/O commands that are split into multiple requests.
     295                 :            :          */
     296                 :            :         uint32_t                        payload_offset;
     297                 :            :         uint32_t                        md_offset;
     298                 :            : 
     299                 :            :         uint32_t                        payload_size;
     300                 :            : 
     301                 :            :         /**
     302                 :            :          * Timeout ticks for error injection requests, can be extended in future
     303                 :            :          * to support per-request timeout feature.
     304                 :            :          */
     305                 :            :         uint64_t                        timeout_tsc;
     306                 :            : 
     307                 :            :         /**
     308                 :            :          * Data payload for this request's command.
     309                 :            :          */
     310                 :            :         struct nvme_payload             payload;
     311                 :            : 
     312                 :            :         spdk_nvme_cmd_cb                cb_fn;
     313                 :            :         void                            *cb_arg;
     314                 :            :         STAILQ_ENTRY(nvme_request)      stailq;
     315                 :            : 
     316                 :            :         struct spdk_nvme_qpair          *qpair;
     317                 :            : 
     318                 :            :         /*
     319                 :            :          * The value of spdk_get_ticks() when the request was submitted to the hardware.
     320                 :            :          * Only set if ctrlr->timeout_enabled is true.
     321                 :            :          */
     322                 :            :         uint64_t                        submit_tick;
     323                 :            : 
     324                 :            :         /**
     325                 :            :          * The active admin request can be moved to a per process pending
     326                 :            :          *  list based on the saved pid to tell which process it belongs
     327                 :            :          *  to. The cpl saves the original completion information which
     328                 :            :          *  is used in the completion callback.
     329                 :            :          * NOTE: these below two fields are only used for admin request.
     330                 :            :          */
     331                 :            :         pid_t                           pid;
     332                 :            :         struct spdk_nvme_cpl            cpl;
     333                 :            : 
     334                 :            :         uint32_t                        md_size;
     335                 :            : 
     336                 :            :         /**
     337                 :            :          * The following members should not be reordered with members
     338                 :            :          *  above.  These members are only needed when splitting
     339                 :            :          *  requests which is done rarely, and the driver is careful
     340                 :            :          *  to not touch the following fields until a split operation is
     341                 :            :          *  needed, to avoid touching an extra cacheline.
     342                 :            :          */
     343                 :            : 
     344                 :            :         /**
     345                 :            :          * Points to the outstanding child requests for a parent request.
     346                 :            :          *  Only valid if a request was split into multiple children
     347                 :            :          *  requests, and is not initialized for non-split requests.
     348                 :            :          */
     349                 :            :         TAILQ_HEAD(, nvme_request)      children;
     350                 :            : 
     351                 :            :         /**
     352                 :            :          * Linked-list pointers for a child request in its parent's list.
     353                 :            :          */
     354                 :            :         TAILQ_ENTRY(nvme_request)       child_tailq;
     355                 :            : 
     356                 :            :         /**
     357                 :            :          * Points to a parent request if part of a split request,
     358                 :            :          *   NULL otherwise.
     359                 :            :          */
     360                 :            :         struct nvme_request             *parent;
     361                 :            : 
     362                 :            :         /**
     363                 :            :          * Completion status for a parent request.  Initialized to all 0's
     364                 :            :          *  (SUCCESS) before child requests are submitted.  If a child
     365                 :            :          *  request completes with error, the error status is copied here,
     366                 :            :          *  to ensure that the parent request is also completed with error
     367                 :            :          *  status once all child requests are completed.
     368                 :            :          */
     369                 :            :         struct spdk_nvme_cpl            parent_status;
     370                 :            : 
     371                 :            :         /**
     372                 :            :          * The user_cb_fn and user_cb_arg fields are used for holding the original
     373                 :            :          * callback data when using nvme_allocate_request_user_copy.
     374                 :            :          */
     375                 :            :         spdk_nvme_cmd_cb                user_cb_fn;
     376                 :            :         void                            *user_cb_arg;
     377                 :            :         void                            *user_buffer;
     378                 :            : 
     379                 :            :         /** Sequence of accel operations associated with this request */
     380                 :            :         void                            *accel_sequence;
     381                 :            : };
     382                 :            : 
     383                 :            : struct nvme_completion_poll_status {
     384                 :            :         struct spdk_nvme_cpl    cpl;
     385                 :            :         uint64_t                timeout_tsc;
     386                 :            :         /**
     387                 :            :          * DMA buffer retained throughout the duration of the command.  It'll be released
     388                 :            :          * automatically if the command times out, otherwise the user is responsible for freeing it.
     389                 :            :          */
     390                 :            :         void                    *dma_data;
     391                 :            :         bool                    done;
     392                 :            :         /* This flag indicates that the request has been timed out and the memory
     393                 :            :            must be freed in a completion callback */
     394                 :            :         bool                    timed_out;
     395                 :            : };
     396                 :            : 
     397                 :            : struct nvme_async_event_request {
     398                 :            :         struct spdk_nvme_ctrlr          *ctrlr;
     399                 :            :         struct nvme_request             *req;
     400                 :            :         struct spdk_nvme_cpl            cpl;
     401                 :            : };
     402                 :            : 
     403                 :            : enum nvme_qpair_state {
     404                 :            :         NVME_QPAIR_DISCONNECTED,
     405                 :            :         NVME_QPAIR_DISCONNECTING,
     406                 :            :         NVME_QPAIR_CONNECTING,
     407                 :            :         NVME_QPAIR_CONNECTED,
     408                 :            :         NVME_QPAIR_ENABLING,
     409                 :            :         NVME_QPAIR_ENABLED,
     410                 :            :         NVME_QPAIR_DESTROYING,
     411                 :            : };
     412                 :            : 
     413                 :            : enum nvme_qpair_connect_state {
     414                 :            :         NVME_QPAIR_CONNECT_STATE_CONNECTING,
     415                 :            :         NVME_QPAIR_CONNECT_STATE_AUTHENTICATING,
     416                 :            :         NVME_QPAIR_CONNECT_STATE_CONNECTED,
     417                 :            :         NVME_QPAIR_CONNECT_STATE_FAILED,
     418                 :            : };
     419                 :            : 
     420                 :            : enum nvme_qpair_auth_state {
     421                 :            :         NVME_QPAIR_AUTH_STATE_NEGOTIATE,
     422                 :            :         NVME_QPAIR_AUTH_STATE_AWAIT_NEGOTIATE,
     423                 :            :         NVME_QPAIR_AUTH_STATE_AWAIT_CHALLENGE,
     424                 :            :         NVME_QPAIR_AUTH_STATE_AWAIT_REPLY,
     425                 :            :         NVME_QPAIR_AUTH_STATE_AWAIT_SUCCESS1,
     426                 :            :         NVME_QPAIR_AUTH_STATE_AWAIT_SUCCESS2,
     427                 :            :         NVME_QPAIR_AUTH_STATE_AWAIT_FAILURE2,
     428                 :            :         NVME_QPAIR_AUTH_STATE_DONE,
     429                 :            : };
     430                 :            : 
     431                 :            : /* Authentication transaction required (authreq.atr) */
     432                 :            : #define NVME_QPAIR_AUTH_FLAG_ATR        (1 << 0)
     433                 :            : /* Authentication and secure channel required (authreq.ascr) */
     434                 :            : #define NVME_QPAIR_AUTH_FLAG_ASCR       (1 << 1)
     435                 :            : 
     436                 :            : /* Maximum size of a digest */
     437                 :            : #define NVME_AUTH_DIGEST_MAX_SIZE       64
     438                 :            : 
     439                 :            : struct nvme_auth {
     440                 :            :         /* State of the authentication */
     441                 :            :         enum nvme_qpair_auth_state      state;
     442                 :            :         /* Status of the authentication */
     443                 :            :         int                             status;
     444                 :            :         /* Transaction ID */
     445                 :            :         uint16_t                        tid;
     446                 :            :         /* Flags */
     447                 :            :         uint32_t                        flags;
     448                 :            :         /* Selected hash function */
     449                 :            :         uint8_t                         hash;
     450                 :            :         /* Buffer used for controller challenge */
     451                 :            :         uint8_t                         challenge[NVME_AUTH_DIGEST_MAX_SIZE];
     452                 :            : };
     453                 :            : 
     454                 :            : struct spdk_nvme_qpair {
     455                 :            :         struct spdk_nvme_ctrlr                  *ctrlr;
     456                 :            : 
     457                 :            :         uint16_t                                id;
     458                 :            : 
     459                 :            :         uint8_t                                 qprio: 2;
     460                 :            : 
     461                 :            :         uint8_t                                 state: 3;
     462                 :            : 
     463                 :            :         uint8_t                                 async: 1;
     464                 :            : 
     465                 :            :         uint8_t                                 is_new_qpair: 1;
     466                 :            : 
     467                 :            :         uint8_t                                 abort_dnr: 1;
     468                 :            :         /*
     469                 :            :          * Members for handling IO qpair deletion inside of a completion context.
     470                 :            :          * These are specifically defined as single bits, so that they do not
     471                 :            :          *  push this data structure out to another cacheline.
     472                 :            :          */
     473                 :            :         uint8_t                                 in_completion_context: 1;
     474                 :            :         uint8_t                                 delete_after_completion_context: 1;
     475                 :            : 
     476                 :            :         /*
     477                 :            :          * Set when no deletion notification is needed. For example, the process
     478                 :            :          * which allocated this qpair exited unexpectedly.
     479                 :            :          */
     480                 :            :         uint8_t                                 no_deletion_notification_needed: 1;
     481                 :            : 
     482                 :            :         uint8_t                                 last_fuse: 2;
     483                 :            : 
     484                 :            :         uint8_t                                 transport_failure_reason: 3;
     485                 :            :         uint8_t                                 last_transport_failure_reason: 3;
     486                 :            : 
     487                 :            :         /* The user is destroying qpair */
     488                 :            :         uint8_t                                 destroy_in_progress: 1;
     489                 :            : 
     490                 :            :         /* Number of IO outstanding at transport level */
     491                 :            :         uint16_t                                queue_depth;
     492                 :            : 
     493                 :            :         enum spdk_nvme_transport_type           trtype;
     494                 :            : 
     495                 :            :         uint32_t                                num_outstanding_reqs;
     496                 :            : 
     497                 :            :         /* request object used only for this qpair's FABRICS/CONNECT command (if needed) */
     498                 :            :         struct nvme_request                     *reserved_req;
     499                 :            : 
     500                 :            :         STAILQ_HEAD(, nvme_request)             free_req;
     501                 :            :         STAILQ_HEAD(, nvme_request)             queued_req;
     502                 :            : 
     503                 :            :         /* List entry for spdk_nvme_transport_poll_group::qpairs */
     504                 :            :         STAILQ_ENTRY(spdk_nvme_qpair)           poll_group_stailq;
     505                 :            : 
     506                 :            :         /** Commands opcode in this list will return error */
     507                 :            :         TAILQ_HEAD(, nvme_error_cmd)            err_cmd_head;
     508                 :            :         /** Requests in this list will return error */
     509                 :            :         STAILQ_HEAD(, nvme_request)             err_req_head;
     510                 :            : 
     511                 :            :         struct spdk_nvme_ctrlr_process          *active_proc;
     512                 :            : 
     513                 :            :         struct spdk_nvme_transport_poll_group   *poll_group;
     514                 :            : 
     515                 :            :         void                                    *poll_group_tailq_head;
     516                 :            : 
     517                 :            :         const struct spdk_nvme_transport        *transport;
     518                 :            : 
     519                 :            :         /* Entries below here are not touched in the main I/O path. */
     520                 :            : 
     521                 :            :         struct nvme_completion_poll_status      *poll_status;
     522                 :            :         enum nvme_qpair_connect_state           connect_state;
     523                 :            : 
     524                 :            :         /* List entry for spdk_nvme_ctrlr::active_io_qpairs */
     525                 :            :         TAILQ_ENTRY(spdk_nvme_qpair)            tailq;
     526                 :            : 
     527                 :            :         /* List entry for spdk_nvme_ctrlr_process::allocated_io_qpairs */
     528                 :            :         TAILQ_ENTRY(spdk_nvme_qpair)            per_process_tailq;
     529                 :            : 
     530                 :            :         STAILQ_HEAD(, nvme_request)             aborting_queued_req;
     531                 :            : 
     532                 :            :         void                                    *req_buf;
     533                 :            : 
     534                 :            :         /* In-band authentication state */
     535                 :            :         struct nvme_auth                        auth;
     536                 :            : };
     537                 :            : 
     538                 :            : struct spdk_nvme_poll_group {
     539                 :            :         void                                            *ctx;
     540                 :            :         struct spdk_nvme_accel_fn_table                 accel_fn_table;
     541                 :            :         STAILQ_HEAD(, spdk_nvme_transport_poll_group)   tgroups;
     542                 :            :         bool                                            in_process_completions;
     543                 :            : };
     544                 :            : 
     545                 :            : struct spdk_nvme_transport_poll_group {
     546                 :            :         struct spdk_nvme_poll_group                     *group;
     547                 :            :         const struct spdk_nvme_transport                *transport;
     548                 :            :         STAILQ_HEAD(, spdk_nvme_qpair)                  connected_qpairs;
     549                 :            :         STAILQ_HEAD(, spdk_nvme_qpair)                  disconnected_qpairs;
     550                 :            :         STAILQ_ENTRY(spdk_nvme_transport_poll_group)    link;
     551                 :            :         uint32_t                                        num_connected_qpairs;
     552                 :            : };
     553                 :            : 
     554                 :            : struct spdk_nvme_ns {
     555                 :            :         struct spdk_nvme_ctrlr          *ctrlr;
     556                 :            :         uint32_t                        sector_size;
     557                 :            : 
     558                 :            :         /*
     559                 :            :          * Size of data transferred as part of each block,
     560                 :            :          * including metadata if FLBAS indicates the metadata is transferred
     561                 :            :          * as part of the data buffer at the end of each LBA.
     562                 :            :          */
     563                 :            :         uint32_t                        extended_lba_size;
     564                 :            : 
     565                 :            :         uint32_t                        md_size;
     566                 :            :         uint32_t                        pi_type;
     567                 :            :         uint32_t                        pi_format;
     568                 :            :         uint32_t                        sectors_per_max_io;
     569                 :            :         uint32_t                        sectors_per_max_io_no_md;
     570                 :            :         uint32_t                        sectors_per_stripe;
     571                 :            :         uint32_t                        id;
     572                 :            :         uint16_t                        flags;
     573                 :            :         bool                            active;
     574                 :            : 
     575                 :            :         /* Command Set Identifier */
     576                 :            :         enum spdk_nvme_csi              csi;
     577                 :            : 
     578                 :            :         /* Namespace Identification Descriptor List (CNS = 03h) */
     579                 :            :         uint8_t                         id_desc_list[4096];
     580                 :            : 
     581                 :            :         uint32_t                        ana_group_id;
     582                 :            :         enum spdk_nvme_ana_state        ana_state;
     583                 :            : 
     584                 :            :         /* Identify Namespace data. */
     585                 :            :         struct spdk_nvme_ns_data        nsdata;
     586                 :            : 
     587                 :            :         /* Zoned Namespace Command Set Specific Identify Namespace data. */
     588                 :            :         struct spdk_nvme_zns_ns_data    *nsdata_zns;
     589                 :            : 
     590                 :            :         struct spdk_nvme_nvm_ns_data    *nsdata_nvm;
     591                 :            : 
     592                 :            :         RB_ENTRY(spdk_nvme_ns)          node;
     593                 :            : };
     594                 :            : 
     595                 :            : /**
     596                 :            :  * State of struct spdk_nvme_ctrlr (in particular, during initialization).
     597                 :            :  */
     598                 :            : enum nvme_ctrlr_state {
     599                 :            :         /**
     600                 :            :          * Wait before initializing the controller.
     601                 :            :          */
     602                 :            :         NVME_CTRLR_STATE_INIT_DELAY,
     603                 :            : 
     604                 :            :         /**
     605                 :            :          * Connect the admin queue.
     606                 :            :          */
     607                 :            :         NVME_CTRLR_STATE_CONNECT_ADMINQ,
     608                 :            : 
     609                 :            :         /**
     610                 :            :          * Controller has not started initialized yet.
     611                 :            :          */
     612                 :            :         NVME_CTRLR_STATE_INIT = NVME_CTRLR_STATE_CONNECT_ADMINQ,
     613                 :            : 
     614                 :            :         /**
     615                 :            :          * Waiting for admin queue to connect.
     616                 :            :          */
     617                 :            :         NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ,
     618                 :            : 
     619                 :            :         /**
     620                 :            :          * Read Version (VS) register.
     621                 :            :          */
     622                 :            :         NVME_CTRLR_STATE_READ_VS,
     623                 :            : 
     624                 :            :         /**
     625                 :            :          * Waiting for Version (VS) register to be read.
     626                 :            :          */
     627                 :            :         NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS,
     628                 :            : 
     629                 :            :         /**
     630                 :            :          * Read Capabilities (CAP) register.
     631                 :            :          */
     632                 :            :         NVME_CTRLR_STATE_READ_CAP,
     633                 :            : 
     634                 :            :         /**
     635                 :            :          * Waiting for Capabilities (CAP) register to be read.
     636                 :            :          */
     637                 :            :         NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP,
     638                 :            : 
     639                 :            :         /**
     640                 :            :          * Check EN to prepare for controller initialization.
     641                 :            :          */
     642                 :            :         NVME_CTRLR_STATE_CHECK_EN,
     643                 :            : 
     644                 :            :         /**
     645                 :            :          * Waiting for CC to be read as part of EN check.
     646                 :            :          */
     647                 :            :         NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC,
     648                 :            : 
     649                 :            :         /**
     650                 :            :          * Waiting for CSTS.RDY to transition from 0 to 1 so that CC.EN may be set to 0.
     651                 :            :          */
     652                 :            :         NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1,
     653                 :            : 
     654                 :            :         /**
     655                 :            :          * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 1.
     656                 :            :          */
     657                 :            :         NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS,
     658                 :            : 
     659                 :            :         /**
     660                 :            :          * Disabling the controller by setting CC.EN to 0.
     661                 :            :          */
     662                 :            :         NVME_CTRLR_STATE_SET_EN_0,
     663                 :            : 
     664                 :            :         /**
     665                 :            :          * Waiting for the CC register to be read as part of disabling the controller.
     666                 :            :          */
     667                 :            :         NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC,
     668                 :            : 
     669                 :            :         /**
     670                 :            :          * Waiting for CSTS.RDY to transition from 1 to 0 so that CC.EN may be set to 1.
     671                 :            :          */
     672                 :            :         NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0,
     673                 :            : 
     674                 :            :         /**
     675                 :            :          * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 0.
     676                 :            :          */
     677                 :            :         NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS,
     678                 :            : 
     679                 :            :         /**
     680                 :            :          * The controller is disabled. (CC.EN and CSTS.RDY are 0.)
     681                 :            :          */
     682                 :            :         NVME_CTRLR_STATE_DISABLED,
     683                 :            : 
     684                 :            :         /**
     685                 :            :          * Enable the controller by writing CC.EN to 1
     686                 :            :          */
     687                 :            :         NVME_CTRLR_STATE_ENABLE,
     688                 :            : 
     689                 :            :         /**
     690                 :            :          * Waiting for CC register to be written as part of enabling the controller.
     691                 :            :          */
     692                 :            :         NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC,
     693                 :            : 
     694                 :            :         /**
     695                 :            :          * Waiting for CSTS.RDY to transition from 0 to 1 after enabling the controller.
     696                 :            :          */
     697                 :            :         NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1,
     698                 :            : 
     699                 :            :         /**
     700                 :            :          * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 1.
     701                 :            :          */
     702                 :            :         NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS,
     703                 :            : 
     704                 :            :         /**
     705                 :            :          * Reset the Admin queue of the controller.
     706                 :            :          */
     707                 :            :         NVME_CTRLR_STATE_RESET_ADMIN_QUEUE,
     708                 :            : 
     709                 :            :         /**
     710                 :            :          * Identify Controller command will be sent to then controller.
     711                 :            :          */
     712                 :            :         NVME_CTRLR_STATE_IDENTIFY,
     713                 :            : 
     714                 :            :         /**
     715                 :            :          * Waiting for Identify Controller command be completed.
     716                 :            :          */
     717                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY,
     718                 :            : 
     719                 :            :         /**
     720                 :            :          * Configure AER of the controller.
     721                 :            :          */
     722                 :            :         NVME_CTRLR_STATE_CONFIGURE_AER,
     723                 :            : 
     724                 :            :         /**
     725                 :            :          * Waiting for the Configure AER to be completed.
     726                 :            :          */
     727                 :            :         NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER,
     728                 :            : 
     729                 :            :         /**
     730                 :            :          * Set Keep Alive Timeout of the controller.
     731                 :            :          */
     732                 :            :         NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT,
     733                 :            : 
     734                 :            :         /**
     735                 :            :          * Waiting for Set Keep Alive Timeout to be completed.
     736                 :            :          */
     737                 :            :         NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT,
     738                 :            : 
     739                 :            :         /**
     740                 :            :          * Get Identify I/O Command Set Specific Controller data structure.
     741                 :            :          */
     742                 :            :         NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC,
     743                 :            : 
     744                 :            :         /**
     745                 :            :          * Waiting for Identify I/O Command Set Specific Controller command to be completed.
     746                 :            :          */
     747                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC,
     748                 :            : 
     749                 :            :         /**
     750                 :            :          * Get Commands Supported and Effects log page for the Zoned Namespace Command Set.
     751                 :            :          */
     752                 :            :         NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG,
     753                 :            : 
     754                 :            :         /**
     755                 :            :          * Waiting for the Get Log Page command to be completed.
     756                 :            :          */
     757                 :            :         NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG,
     758                 :            : 
     759                 :            :         /**
     760                 :            :          * Set Number of Queues of the controller.
     761                 :            :          */
     762                 :            :         NVME_CTRLR_STATE_SET_NUM_QUEUES,
     763                 :            : 
     764                 :            :         /**
     765                 :            :          * Waiting for Set Num of Queues command to be completed.
     766                 :            :          */
     767                 :            :         NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES,
     768                 :            : 
     769                 :            :         /**
     770                 :            :          * Get active Namespace list of the controller.
     771                 :            :          */
     772                 :            :         NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS,
     773                 :            : 
     774                 :            :         /**
     775                 :            :          * Waiting for the Identify Active Namespace commands to be completed.
     776                 :            :          */
     777                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS,
     778                 :            : 
     779                 :            :         /**
     780                 :            :          * Get Identify Namespace Data structure for each NS.
     781                 :            :          */
     782                 :            :         NVME_CTRLR_STATE_IDENTIFY_NS,
     783                 :            : 
     784                 :            :         /**
     785                 :            :          * Waiting for the Identify Namespace commands to be completed.
     786                 :            :          */
     787                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS,
     788                 :            : 
     789                 :            :         /**
     790                 :            :          * Get Identify Namespace Identification Descriptors.
     791                 :            :          */
     792                 :            :         NVME_CTRLR_STATE_IDENTIFY_ID_DESCS,
     793                 :            : 
     794                 :            :         /**
     795                 :            :          * Get Identify I/O Command Set Specific Namespace data structure for each NS.
     796                 :            :          */
     797                 :            :         NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC,
     798                 :            : 
     799                 :            :         /**
     800                 :            :          * Waiting for the Identify I/O Command Set Specific Namespace commands to be completed.
     801                 :            :          */
     802                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC,
     803                 :            : 
     804                 :            :         /**
     805                 :            :          * Waiting for the Identify Namespace Identification
     806                 :            :          * Descriptors to be completed.
     807                 :            :          */
     808                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS,
     809                 :            : 
     810                 :            :         /**
     811                 :            :          * Set supported log pages of the controller.
     812                 :            :          */
     813                 :            :         NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES,
     814                 :            : 
     815                 :            :         /**
     816                 :            :          * Set supported log pages of INTEL controller.
     817                 :            :          */
     818                 :            :         NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES,
     819                 :            : 
     820                 :            :         /**
     821                 :            :          * Waiting for supported log pages of INTEL controller.
     822                 :            :          */
     823                 :            :         NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES,
     824                 :            : 
     825                 :            :         /**
     826                 :            :          * Set supported features of the controller.
     827                 :            :          */
     828                 :            :         NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES,
     829                 :            : 
     830                 :            :         /**
     831                 :            :          * Set the Host Behavior Support feature of the controller.
     832                 :            :          */
     833                 :            :         NVME_CTRLR_STATE_SET_HOST_FEATURE,
     834                 :            : 
     835                 :            :         /**
     836                 :            :          * Waiting for the Host Behavior Support feature of the controller.
     837                 :            :          */
     838                 :            :         NVME_CTRLR_STATE_WAIT_FOR_SET_HOST_FEATURE,
     839                 :            : 
     840                 :            :         /**
     841                 :            :          * Set Doorbell Buffer Config of the controller.
     842                 :            :          */
     843                 :            :         NVME_CTRLR_STATE_SET_DB_BUF_CFG,
     844                 :            : 
     845                 :            :         /**
     846                 :            :          * Waiting for Doorbell Buffer Config to be completed.
     847                 :            :          */
     848                 :            :         NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG,
     849                 :            : 
     850                 :            :         /**
     851                 :            :          * Set Host ID of the controller.
     852                 :            :          */
     853                 :            :         NVME_CTRLR_STATE_SET_HOST_ID,
     854                 :            : 
     855                 :            :         /**
     856                 :            :          * Waiting for Set Host ID to be completed.
     857                 :            :          */
     858                 :            :         NVME_CTRLR_STATE_WAIT_FOR_HOST_ID,
     859                 :            : 
     860                 :            :         /**
     861                 :            :          * Let transport layer do its part of initialization.
     862                 :            :          */
     863                 :            :         NVME_CTRLR_STATE_TRANSPORT_READY,
     864                 :            : 
     865                 :            :         /**
     866                 :            :          * Controller initialization has completed and the controller is ready.
     867                 :            :          */
     868                 :            :         NVME_CTRLR_STATE_READY,
     869                 :            : 
     870                 :            :         /**
     871                 :            :          * Controller initialization has an error.
     872                 :            :          */
     873                 :            :         NVME_CTRLR_STATE_ERROR,
     874                 :            : 
     875                 :            :         /**
     876                 :            :          * Admin qpair was disconnected, controller needs to be re-initialized
     877                 :            :          */
     878                 :            :         NVME_CTRLR_STATE_DISCONNECTED,
     879                 :            : };
     880                 :            : 
     881                 :            : #define NVME_TIMEOUT_INFINITE           0
     882                 :            : #define NVME_TIMEOUT_KEEP_EXISTING      UINT64_MAX
     883                 :            : 
     884                 :            : struct spdk_nvme_ctrlr_aer_completion_list {
     885                 :            :         struct spdk_nvme_cpl    cpl;
     886                 :            :         STAILQ_ENTRY(spdk_nvme_ctrlr_aer_completion_list) link;
     887                 :            : };
     888                 :            : 
     889                 :            : /*
     890                 :            :  * Used to track properties for all processes accessing the controller.
     891                 :            :  */
     892                 :            : struct spdk_nvme_ctrlr_process {
     893                 :            :         /** Whether it is the primary process  */
     894                 :            :         bool                                            is_primary;
     895                 :            : 
     896                 :            :         /** Process ID */
     897                 :            :         pid_t                                           pid;
     898                 :            : 
     899                 :            :         /** Active admin requests to be completed */
     900                 :            :         STAILQ_HEAD(, nvme_request)                     active_reqs;
     901                 :            : 
     902                 :            :         TAILQ_ENTRY(spdk_nvme_ctrlr_process)            tailq;
     903                 :            : 
     904                 :            :         /** Per process PCI device handle */
     905                 :            :         struct spdk_pci_device                          *devhandle;
     906                 :            : 
     907                 :            :         /** Reference to track the number of attachment to this controller. */
     908                 :            :         int                                             ref;
     909                 :            : 
     910                 :            :         /** Allocated IO qpairs */
     911                 :            :         TAILQ_HEAD(, spdk_nvme_qpair)                   allocated_io_qpairs;
     912                 :            : 
     913                 :            :         spdk_nvme_aer_cb                                aer_cb_fn;
     914                 :            :         void                                            *aer_cb_arg;
     915                 :            : 
     916                 :            :         /**
     917                 :            :          * A function pointer to timeout callback function
     918                 :            :          */
     919                 :            :         spdk_nvme_timeout_cb            timeout_cb_fn;
     920                 :            :         void                            *timeout_cb_arg;
     921                 :            :         /** separate timeout values for io vs. admin reqs */
     922                 :            :         uint64_t                        timeout_io_ticks;
     923                 :            :         uint64_t                        timeout_admin_ticks;
     924                 :            : 
     925                 :            :         /** List to publish AENs to all procs in multiprocess setup */
     926                 :            :         STAILQ_HEAD(, spdk_nvme_ctrlr_aer_completion_list)      async_events;
     927                 :            : };
     928                 :            : 
     929                 :            : struct nvme_register_completion {
     930                 :            :         struct spdk_nvme_cpl                    cpl;
     931                 :            :         uint64_t                                value;
     932                 :            :         spdk_nvme_reg_cb                        cb_fn;
     933                 :            :         void                                    *cb_ctx;
     934                 :            :         STAILQ_ENTRY(nvme_register_completion)  stailq;
     935                 :            :         pid_t                                   pid;
     936                 :            : };
     937                 :            : 
     938                 :            : struct spdk_nvme_ctrlr {
     939                 :            :         /* Hot data (accessed in I/O path) starts here. */
     940                 :            : 
     941                 :            :         /* Tree of namespaces */
     942                 :            :         RB_HEAD(nvme_ns_tree, spdk_nvme_ns)     ns;
     943                 :            : 
     944                 :            :         /* The number of active namespaces */
     945                 :            :         uint32_t                        active_ns_count;
     946                 :            : 
     947                 :            :         bool                            is_removed;
     948                 :            : 
     949                 :            :         bool                            is_resetting;
     950                 :            : 
     951                 :            :         bool                            is_failed;
     952                 :            : 
     953                 :            :         bool                            is_destructed;
     954                 :            : 
     955                 :            :         bool                            timeout_enabled;
     956                 :            : 
     957                 :            :         /* The application is preparing to reset the controller.  Transports
     958                 :            :          * can use this to skip unnecessary parts of the qpair deletion process
     959                 :            :          * for example, like the DELETE_SQ/CQ commands.
     960                 :            :          */
     961                 :            :         bool                            prepare_for_reset;
     962                 :            : 
     963                 :            :         bool                            is_disconnecting;
     964                 :            : 
     965                 :            :         bool                            needs_io_msg_update;
     966                 :            : 
     967                 :            :         uint16_t                        max_sges;
     968                 :            : 
     969                 :            :         uint16_t                        cntlid;
     970                 :            : 
     971                 :            :         /** Controller support flags */
     972                 :            :         uint64_t                        flags;
     973                 :            : 
     974                 :            :         /** NVMEoF in-capsule data size in bytes */
     975                 :            :         uint32_t                        ioccsz_bytes;
     976                 :            : 
     977                 :            :         /** NVMEoF in-capsule data offset in 16 byte units */
     978                 :            :         uint16_t                        icdoff;
     979                 :            : 
     980                 :            :         /* Cold data (not accessed in normal I/O path) is after this point. */
     981                 :            : 
     982                 :            :         struct spdk_nvme_transport_id   trid;
     983                 :            : 
     984                 :            :         union spdk_nvme_cap_register    cap;
     985                 :            :         union spdk_nvme_vs_register     vs;
     986                 :            : 
     987                 :            :         int                             state;
     988                 :            :         uint64_t                        state_timeout_tsc;
     989                 :            : 
     990                 :            :         uint64_t                        next_keep_alive_tick;
     991                 :            :         uint64_t                        keep_alive_interval_ticks;
     992                 :            : 
     993                 :            :         TAILQ_ENTRY(spdk_nvme_ctrlr)    tailq;
     994                 :            : 
     995                 :            :         /** All the log pages supported */
     996                 :            :         bool                            log_page_supported[256];
     997                 :            : 
     998                 :            :         /** All the features supported */
     999                 :            :         bool                            feature_supported[256];
    1000                 :            : 
    1001                 :            :         /** maximum i/o size in bytes */
    1002                 :            :         uint32_t                        max_xfer_size;
    1003                 :            : 
    1004                 :            :         /** minimum page size supported by this controller in bytes */
    1005                 :            :         uint32_t                        min_page_size;
    1006                 :            : 
    1007                 :            :         /** selected memory page size for this controller in bytes */
    1008                 :            :         uint32_t                        page_size;
    1009                 :            : 
    1010                 :            :         uint32_t                        num_aers;
    1011                 :            :         struct nvme_async_event_request aer[NVME_MAX_ASYNC_EVENTS];
    1012                 :            : 
    1013                 :            :         /** guards access to the controller itself, including admin queues */
    1014                 :            :         pthread_mutex_t                 ctrlr_lock;
    1015                 :            : 
    1016                 :            :         struct spdk_nvme_qpair          *adminq;
    1017                 :            : 
    1018                 :            :         /** shadow doorbell buffer */
    1019                 :            :         uint32_t                        *shadow_doorbell;
    1020                 :            :         /** eventidx buffer */
    1021                 :            :         uint32_t                        *eventidx;
    1022                 :            : 
    1023                 :            :         /**
    1024                 :            :          * Identify Controller data.
    1025                 :            :          */
    1026                 :            :         struct spdk_nvme_ctrlr_data     cdata;
    1027                 :            : 
    1028                 :            :         /**
    1029                 :            :          * Zoned Namespace Command Set Specific Identify Controller data.
    1030                 :            :          */
    1031                 :            :         struct spdk_nvme_zns_ctrlr_data *cdata_zns;
    1032                 :            : 
    1033                 :            :         struct spdk_bit_array           *free_io_qids;
    1034                 :            :         TAILQ_HEAD(, spdk_nvme_qpair)   active_io_qpairs;
    1035                 :            : 
    1036                 :            :         struct spdk_nvme_ctrlr_opts     opts;
    1037                 :            : 
    1038                 :            :         uint64_t                        quirks;
    1039                 :            : 
    1040                 :            :         /* Extra sleep time during controller initialization */
    1041                 :            :         uint64_t                        sleep_timeout_tsc;
    1042                 :            : 
    1043                 :            :         /** Track all the processes manage this controller */
    1044                 :            :         TAILQ_HEAD(, spdk_nvme_ctrlr_process)   active_procs;
    1045                 :            : 
    1046                 :            : 
    1047                 :            :         STAILQ_HEAD(, nvme_request)     queued_aborts;
    1048                 :            :         uint32_t                        outstanding_aborts;
    1049                 :            : 
    1050                 :            :         uint32_t                        lock_depth;
    1051                 :            : 
    1052                 :            :         /* CB to notify the user when the ctrlr is removed/failed. */
    1053                 :            :         spdk_nvme_remove_cb                     remove_cb;
    1054                 :            :         void                                    *cb_ctx;
    1055                 :            : 
    1056                 :            :         struct spdk_nvme_qpair          *external_io_msgs_qpair;
    1057                 :            :         pthread_mutex_t                 external_io_msgs_lock;
    1058                 :            :         struct spdk_ring                *external_io_msgs;
    1059                 :            : 
    1060                 :            :         STAILQ_HEAD(, nvme_io_msg_producer) io_producers;
    1061                 :            : 
    1062                 :            :         struct spdk_nvme_ana_page               *ana_log_page;
    1063                 :            :         struct spdk_nvme_ana_group_descriptor   *copied_ana_desc;
    1064                 :            :         uint32_t                                ana_log_page_size;
    1065                 :            : 
    1066                 :            :         /* scratchpad pointer that can be used to send data between two NVME_CTRLR_STATEs */
    1067                 :            :         void                            *tmp_ptr;
    1068                 :            : 
    1069                 :            :         /* maximum zone append size in bytes */
    1070                 :            :         uint32_t                        max_zone_append_size;
    1071                 :            : 
    1072                 :            :         /* PMR size in bytes */
    1073                 :            :         uint64_t                        pmr_size;
    1074                 :            : 
    1075                 :            :         /* Boot Partition Info */
    1076                 :            :         enum nvme_bp_write_state        bp_ws;
    1077                 :            :         uint32_t                        bpid;
    1078                 :            :         spdk_nvme_cmd_cb                bp_write_cb_fn;
    1079                 :            :         void                            *bp_write_cb_arg;
    1080                 :            : 
    1081                 :            :         /* Firmware Download */
    1082                 :            :         void                            *fw_payload;
    1083                 :            :         unsigned int                    fw_size_remaining;
    1084                 :            :         unsigned int                    fw_offset;
    1085                 :            :         unsigned int                    fw_transfer_size;
    1086                 :            : 
    1087                 :            :         /* Completed register operations */
    1088                 :            :         STAILQ_HEAD(, nvme_register_completion) register_operations;
    1089                 :            : 
    1090                 :            :         union spdk_nvme_cc_register             process_init_cc;
    1091                 :            : 
    1092                 :            :         /* Authentication transaction ID */
    1093                 :            :         uint16_t                                auth_tid;
    1094                 :            :         /* Authentication sequence number */
    1095                 :            :         uint32_t                                auth_seqnum;
    1096                 :            : };
    1097                 :            : 
    1098                 :            : struct spdk_nvme_probe_ctx {
    1099                 :            :         struct spdk_nvme_transport_id           trid;
    1100                 :            :         const struct spdk_nvme_ctrlr_opts       *opts;
    1101                 :            :         void                                    *cb_ctx;
    1102                 :            :         spdk_nvme_probe_cb                      probe_cb;
    1103                 :            :         spdk_nvme_attach_cb                     attach_cb;
    1104                 :            :         spdk_nvme_remove_cb                     remove_cb;
    1105                 :            :         TAILQ_HEAD(, spdk_nvme_ctrlr)           init_ctrlrs;
    1106                 :            : };
    1107                 :            : 
    1108                 :            : typedef void (*nvme_ctrlr_detach_cb)(struct spdk_nvme_ctrlr *ctrlr);
    1109                 :            : 
    1110                 :            : enum nvme_ctrlr_detach_state {
    1111                 :            :         NVME_CTRLR_DETACH_SET_CC,
    1112                 :            :         NVME_CTRLR_DETACH_CHECK_CSTS,
    1113                 :            :         NVME_CTRLR_DETACH_GET_CSTS,
    1114                 :            :         NVME_CTRLR_DETACH_GET_CSTS_DONE,
    1115                 :            : };
    1116                 :            : 
    1117                 :            : struct nvme_ctrlr_detach_ctx {
    1118                 :            :         struct spdk_nvme_ctrlr                  *ctrlr;
    1119                 :            :         nvme_ctrlr_detach_cb                    cb_fn;
    1120                 :            :         uint64_t                                shutdown_start_tsc;
    1121                 :            :         uint32_t                                shutdown_timeout_ms;
    1122                 :            :         bool                                    shutdown_complete;
    1123                 :            :         enum nvme_ctrlr_detach_state            state;
    1124                 :            :         union spdk_nvme_csts_register           csts;
    1125                 :            :         TAILQ_ENTRY(nvme_ctrlr_detach_ctx)      link;
    1126                 :            : };
    1127                 :            : 
    1128                 :            : struct spdk_nvme_detach_ctx {
    1129                 :            :         TAILQ_HEAD(, nvme_ctrlr_detach_ctx)     head;
    1130                 :            : };
    1131                 :            : 
    1132                 :            : struct nvme_driver {
    1133                 :            :         pthread_mutex_t                 lock;
    1134                 :            : 
    1135                 :            :         /** Multi-process shared attached controller list */
    1136                 :            :         TAILQ_HEAD(, spdk_nvme_ctrlr)   shared_attached_ctrlrs;
    1137                 :            : 
    1138                 :            :         bool                            initialized;
    1139                 :            :         struct spdk_uuid                default_extended_host_id;
    1140                 :            : 
    1141                 :            :         /** netlink socket fd for hotplug messages */
    1142                 :            :         int                             hotplug_fd;
    1143                 :            : };
    1144                 :            : 
    1145                 :            : #define nvme_ns_cmd_get_ext_io_opt(opts, field, defval) \
    1146                 :            :        ((opts) != NULL && offsetof(struct spdk_nvme_ns_cmd_ext_io_opts, field) + \
    1147                 :            :         sizeof((opts)->field) <= (opts)->size ? (opts)->field : (defval))
    1148                 :            : 
    1149                 :            : extern struct nvme_driver *g_spdk_nvme_driver;
    1150                 :            : 
    1151                 :            : int nvme_driver_init(void);
    1152                 :            : 
    1153                 :            : #define nvme_delay              usleep
    1154                 :            : 
    1155                 :            : static inline bool
    1156                 : 1815821286 : nvme_qpair_is_admin_queue(struct spdk_nvme_qpair *qpair)
    1157                 :            : {
    1158                 : 1815821286 :         return qpair->id == 0;
    1159                 :            : }
    1160                 :            : 
    1161                 :            : static inline bool
    1162                 :            : nvme_qpair_is_io_queue(struct spdk_nvme_qpair *qpair)
    1163                 :            : {
    1164                 :            :         return qpair->id != 0;
    1165                 :            : }
    1166                 :            : 
    1167                 :            : static inline int
    1168                 :  154858603 : nvme_robust_mutex_lock(pthread_mutex_t *mtx)
    1169                 :            : {
    1170         [ -  + ]:  154858603 :         int rc = pthread_mutex_lock(mtx);
    1171                 :            : 
    1172                 :            : #ifndef __FreeBSD__
    1173         [ -  + ]:  154858603 :         if (rc == EOWNERDEAD) {
    1174         [ #  # ]:          0 :                 rc = pthread_mutex_consistent(mtx);
    1175                 :            :         }
    1176                 :            : #endif
    1177                 :            : 
    1178                 :  154858603 :         return rc;
    1179                 :            : }
    1180                 :            : 
    1181                 :            : static inline int
    1182                 :  154275518 : nvme_ctrlr_lock(struct spdk_nvme_ctrlr *ctrlr)
    1183                 :            : {
    1184                 :            :         int rc;
    1185                 :            : 
    1186                 :  154275518 :         rc = nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
    1187                 :  154275518 :         ctrlr->lock_depth++;
    1188                 :  154275518 :         return rc;
    1189                 :            : }
    1190                 :            : 
    1191                 :            : static inline int
    1192                 :  154858597 : nvme_robust_mutex_unlock(pthread_mutex_t *mtx)
    1193                 :            : {
    1194         [ -  + ]:  154858597 :         return pthread_mutex_unlock(mtx);
    1195                 :            : }
    1196                 :            : 
    1197                 :            : static inline int
    1198                 :  154275512 : nvme_ctrlr_unlock(struct spdk_nvme_ctrlr *ctrlr)
    1199                 :            : {
    1200                 :  154275512 :         ctrlr->lock_depth--;
    1201                 :  154275512 :         return nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
    1202                 :            : }
    1203                 :            : 
    1204                 :            : /* Poll group management functions. */
    1205                 :            : int nvme_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair);
    1206                 :            : int nvme_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair);
    1207                 :            : 
    1208                 :            : /* Admin functions */
    1209                 :            : int     nvme_ctrlr_cmd_identify(struct spdk_nvme_ctrlr *ctrlr,
    1210                 :            :                                 uint8_t cns, uint16_t cntid, uint32_t nsid,
    1211                 :            :                                 uint8_t csi, void *payload, size_t payload_size,
    1212                 :            :                                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1213                 :            : int     nvme_ctrlr_cmd_set_num_queues(struct spdk_nvme_ctrlr *ctrlr,
    1214                 :            :                                       uint32_t num_queues, spdk_nvme_cmd_cb cb_fn,
    1215                 :            :                                       void *cb_arg);
    1216                 :            : int     nvme_ctrlr_cmd_get_num_queues(struct spdk_nvme_ctrlr *ctrlr,
    1217                 :            :                                       spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1218                 :            : int     nvme_ctrlr_cmd_set_async_event_config(struct spdk_nvme_ctrlr *ctrlr,
    1219                 :            :                 union spdk_nvme_feat_async_event_configuration config,
    1220                 :            :                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1221                 :            : int     nvme_ctrlr_cmd_set_host_id(struct spdk_nvme_ctrlr *ctrlr, void *host_id, uint32_t host_id_size,
    1222                 :            :                                    spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1223                 :            : int     nvme_ctrlr_cmd_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    1224                 :            :                                  struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1225                 :            : int     nvme_ctrlr_cmd_detach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    1226                 :            :                                  struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1227                 :            : int     nvme_ctrlr_cmd_create_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns_data *payload,
    1228                 :            :                                  spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1229                 :            : int     nvme_ctrlr_cmd_doorbell_buffer_config(struct spdk_nvme_ctrlr *ctrlr,
    1230                 :            :                 uint64_t prp1, uint64_t prp2,
    1231                 :            :                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1232                 :            : int     nvme_ctrlr_cmd_delete_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, spdk_nvme_cmd_cb cb_fn,
    1233                 :            :                                  void *cb_arg);
    1234                 :            : int     nvme_ctrlr_cmd_format(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    1235                 :            :                               struct spdk_nvme_format *format, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1236                 :            : int     nvme_ctrlr_cmd_fw_commit(struct spdk_nvme_ctrlr *ctrlr,
    1237                 :            :                                  const struct spdk_nvme_fw_commit *fw_commit,
    1238                 :            :                                  spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1239                 :            : int     nvme_ctrlr_cmd_fw_image_download(struct spdk_nvme_ctrlr *ctrlr,
    1240                 :            :                 uint32_t size, uint32_t offset, void *payload,
    1241                 :            :                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1242                 :            : int     nvme_ctrlr_cmd_sanitize(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    1243                 :            :                                 struct spdk_nvme_sanitize *sanitize, uint32_t cdw11,
    1244                 :            :                                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1245                 :            : void    nvme_completion_poll_cb(void *arg, const struct spdk_nvme_cpl *cpl);
    1246                 :            : int     nvme_wait_for_completion(struct spdk_nvme_qpair *qpair,
    1247                 :            :                                  struct nvme_completion_poll_status *status);
    1248                 :            : int     nvme_wait_for_completion_robust_lock(struct spdk_nvme_qpair *qpair,
    1249                 :            :                 struct nvme_completion_poll_status *status,
    1250                 :            :                 pthread_mutex_t *robust_mutex);
    1251                 :            : int     nvme_wait_for_completion_timeout(struct spdk_nvme_qpair *qpair,
    1252                 :            :                 struct nvme_completion_poll_status *status,
    1253                 :            :                 uint64_t timeout_in_usecs);
    1254                 :            : int     nvme_wait_for_completion_robust_lock_timeout(struct spdk_nvme_qpair *qpair,
    1255                 :            :                 struct nvme_completion_poll_status *status,
    1256                 :            :                 pthread_mutex_t *robust_mutex,
    1257                 :            :                 uint64_t timeout_in_usecs);
    1258                 :            : int     nvme_wait_for_completion_robust_lock_timeout_poll(struct spdk_nvme_qpair *qpair,
    1259                 :            :                 struct nvme_completion_poll_status *status,
    1260                 :            :                 pthread_mutex_t *robust_mutex);
    1261                 :            : 
    1262                 :            : struct spdk_nvme_ctrlr_process *nvme_ctrlr_get_process(struct spdk_nvme_ctrlr *ctrlr,
    1263                 :            :                 pid_t pid);
    1264                 :            : struct spdk_nvme_ctrlr_process *nvme_ctrlr_get_current_process(struct spdk_nvme_ctrlr *ctrlr);
    1265                 :            : int     nvme_ctrlr_add_process(struct spdk_nvme_ctrlr *ctrlr, void *devhandle);
    1266                 :            : void    nvme_ctrlr_free_processes(struct spdk_nvme_ctrlr *ctrlr);
    1267                 :            : struct spdk_pci_device *nvme_ctrlr_proc_get_devhandle(struct spdk_nvme_ctrlr *ctrlr);
    1268                 :            : 
    1269                 :            : int     nvme_ctrlr_probe(const struct spdk_nvme_transport_id *trid,
    1270                 :            :                          struct spdk_nvme_probe_ctx *probe_ctx, void *devhandle);
    1271                 :            : 
    1272                 :            : int     nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr);
    1273                 :            : void    nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr *ctrlr);
    1274                 :            : void    nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr);
    1275                 :            : void    nvme_ctrlr_destruct_async(struct spdk_nvme_ctrlr *ctrlr,
    1276                 :            :                                   struct nvme_ctrlr_detach_ctx *ctx);
    1277                 :            : int     nvme_ctrlr_destruct_poll_async(struct spdk_nvme_ctrlr *ctrlr,
    1278                 :            :                                        struct nvme_ctrlr_detach_ctx *ctx);
    1279                 :            : void    nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove);
    1280                 :            : int     nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr);
    1281                 :            : void    nvme_ctrlr_disable(struct spdk_nvme_ctrlr *ctrlr);
    1282                 :            : int     nvme_ctrlr_disable_poll(struct spdk_nvme_ctrlr *ctrlr);
    1283                 :            : void    nvme_ctrlr_connected(struct spdk_nvme_probe_ctx *probe_ctx,
    1284                 :            :                              struct spdk_nvme_ctrlr *ctrlr);
    1285                 :            : 
    1286                 :            : int     nvme_ctrlr_submit_admin_request(struct spdk_nvme_ctrlr *ctrlr,
    1287                 :            :                                         struct nvme_request *req);
    1288                 :            : int     nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cap_register *cap);
    1289                 :            : int     nvme_ctrlr_get_vs(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_vs_register *vs);
    1290                 :            : int     nvme_ctrlr_get_cmbsz(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cmbsz_register *cmbsz);
    1291                 :            : int     nvme_ctrlr_get_pmrcap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_pmrcap_register *pmrcap);
    1292                 :            : int     nvme_ctrlr_get_bpinfo(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bpinfo_register *bpinfo);
    1293                 :            : int     nvme_ctrlr_set_bprsel(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bprsel_register *bprsel);
    1294                 :            : int     nvme_ctrlr_set_bpmbl(struct spdk_nvme_ctrlr *ctrlr, uint64_t bpmbl_value);
    1295                 :            : bool    nvme_ctrlr_multi_iocs_enabled(struct spdk_nvme_ctrlr *ctrlr);
    1296                 :            : void nvme_ctrlr_disconnect_qpair(struct spdk_nvme_qpair *qpair);
    1297                 :            : void nvme_ctrlr_abort_queued_aborts(struct spdk_nvme_ctrlr *ctrlr);
    1298                 :            : int nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id,
    1299                 :            :                     struct spdk_nvme_ctrlr *ctrlr,
    1300                 :            :                     enum spdk_nvme_qprio qprio,
    1301                 :            :                     uint32_t num_requests, bool async);
    1302                 :            : void    nvme_qpair_deinit(struct spdk_nvme_qpair *qpair);
    1303                 :            : void    nvme_qpair_complete_error_reqs(struct spdk_nvme_qpair *qpair);
    1304                 :            : int     nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair,
    1305                 :            :                                   struct nvme_request *req);
    1306                 :            : void    nvme_qpair_abort_all_queued_reqs(struct spdk_nvme_qpair *qpair);
    1307                 :            : uint32_t nvme_qpair_abort_queued_reqs_with_cbarg(struct spdk_nvme_qpair *qpair, void *cmd_cb_arg);
    1308                 :            : void    nvme_qpair_abort_queued_reqs(struct spdk_nvme_qpair *qpair);
    1309                 :            : void    nvme_qpair_resubmit_requests(struct spdk_nvme_qpair *qpair, uint32_t num_requests);
    1310                 :            : int     nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr);
    1311                 :            : void    nvme_ns_set_identify_data(struct spdk_nvme_ns *ns);
    1312                 :            : void    nvme_ns_set_id_desc_list_data(struct spdk_nvme_ns *ns);
    1313                 :            : void    nvme_ns_free_zns_specific_data(struct spdk_nvme_ns *ns);
    1314                 :            : void    nvme_ns_free_nvm_specific_data(struct spdk_nvme_ns *ns);
    1315                 :            : void    nvme_ns_free_iocs_specific_data(struct spdk_nvme_ns *ns);
    1316                 :            : bool    nvme_ns_has_supported_iocs_specific_data(struct spdk_nvme_ns *ns);
    1317                 :            : int     nvme_ns_construct(struct spdk_nvme_ns *ns, uint32_t id,
    1318                 :            :                           struct spdk_nvme_ctrlr *ctrlr);
    1319                 :            : void    nvme_ns_destruct(struct spdk_nvme_ns *ns);
    1320                 :            : int     nvme_ns_cmd_zone_append_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
    1321                 :            :                                         void *buffer, void *metadata, uint64_t zslba,
    1322                 :            :                                         uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,
    1323                 :            :                                         uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag);
    1324                 :            : int nvme_ns_cmd_zone_appendv_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
    1325                 :            :                                      uint64_t zslba, uint32_t lba_count,
    1326                 :            :                                      spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags,
    1327                 :            :                                      spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
    1328                 :            :                                      spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata,
    1329                 :            :                                      uint16_t apptag_mask, uint16_t apptag);
    1330                 :            : 
    1331                 :            : int     nvme_fabric_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value);
    1332                 :            : int     nvme_fabric_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value);
    1333                 :            : int     nvme_fabric_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value);
    1334                 :            : int     nvme_fabric_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value);
    1335                 :            : int     nvme_fabric_ctrlr_set_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1336                 :            :                 uint32_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1337                 :            : int     nvme_fabric_ctrlr_set_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1338                 :            :                 uint64_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1339                 :            : int     nvme_fabric_ctrlr_get_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1340                 :            :                 spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1341                 :            : int     nvme_fabric_ctrlr_get_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1342                 :            :                 spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1343                 :            : int     nvme_fabric_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, bool direct_connect);
    1344                 :            : int     nvme_fabric_ctrlr_discover(struct spdk_nvme_ctrlr *ctrlr,
    1345                 :            :                                    struct spdk_nvme_probe_ctx *probe_ctx);
    1346                 :            : int     nvme_fabric_qpair_connect(struct spdk_nvme_qpair *qpair, uint32_t num_entries);
    1347                 :            : int     nvme_fabric_qpair_connect_async(struct spdk_nvme_qpair *qpair, uint32_t num_entries);
    1348                 :            : int     nvme_fabric_qpair_connect_poll(struct spdk_nvme_qpair *qpair);
    1349                 :            : int     nvme_fabric_qpair_authenticate_async(struct spdk_nvme_qpair *qpair);
    1350                 :            : int     nvme_fabric_qpair_authenticate_poll(struct spdk_nvme_qpair *qpair);
    1351                 :            : 
    1352                 :            : typedef int (*spdk_nvme_parse_ana_log_page_cb)(
    1353                 :            :         const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg);
    1354                 :            : int     nvme_ctrlr_parse_ana_log_page(struct spdk_nvme_ctrlr *ctrlr,
    1355                 :            :                                       spdk_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg);
    1356                 :            : 
    1357                 :            : static inline void
    1358                 :   69164763 : nvme_request_clear(struct nvme_request *req)
    1359                 :            : {
    1360                 :            :         /*
    1361                 :            :          * Only memset/zero fields that need it.  All other fields
    1362                 :            :          *  will be initialized appropriately either later in this
    1363                 :            :          *  function, or before they are needed later in the
    1364                 :            :          *  submission patch.  For example, the children
    1365                 :            :          *  TAILQ_ENTRY and following members are
    1366                 :            :          *  only used as part of I/O splitting so we avoid
    1367                 :            :          *  memsetting them until it is actually needed.
    1368                 :            :          *  They will be initialized in nvme_request_add_child()
    1369                 :            :          *  if the request is split.
    1370                 :            :          */
    1371         [ -  + ]:   69164763 :         memset(req, 0, offsetof(struct nvme_request, payload_size));
    1372                 :   69164763 : }
    1373                 :            : 
    1374                 :            : #define NVME_INIT_REQUEST(req, _cb_fn, _cb_arg, _payload, _payload_size, _md_size)      \
    1375                 :            :         do {                                            \
    1376                 :            :                 nvme_request_clear(req);                \
    1377                 :            :                 req->cb_fn = _cb_fn;                 \
    1378                 :            :                 req->cb_arg = _cb_arg;                       \
    1379                 :            :                 req->payload = _payload;             \
    1380                 :            :                 req->payload_size = _payload_size;   \
    1381                 :            :                 req->md_size = _md_size;             \
    1382                 :            :                 req->pid = g_spdk_nvme_pid;          \
    1383                 :            :                 req->submit_tick = 0;                        \
    1384                 :            :                 req->accel_sequence = NULL;          \
    1385                 :            :         } while (0);
    1386                 :            : 
    1387                 :            : static inline struct nvme_request *
    1388                 :   69848687 : nvme_allocate_request(struct spdk_nvme_qpair *qpair,
    1389                 :            :                       const struct nvme_payload *payload, uint32_t payload_size, uint32_t md_size,
    1390                 :            :                       spdk_nvme_cmd_cb cb_fn, void *cb_arg)
    1391                 :            : {
    1392                 :            :         struct nvme_request *req;
    1393                 :            : 
    1394                 :   69848687 :         req = STAILQ_FIRST(&qpair->free_req);
    1395         [ +  + ]:   69848687 :         if (req == NULL) {
    1396                 :     693578 :                 return req;
    1397                 :            :         }
    1398                 :            : 
    1399         [ +  + ]:   69155109 :         STAILQ_REMOVE_HEAD(&qpair->free_req, stailq);
    1400                 :   69155109 :         qpair->num_outstanding_reqs++;
    1401                 :            : 
    1402                 :   69155109 :         NVME_INIT_REQUEST(req, cb_fn, cb_arg, *payload, payload_size, md_size);
    1403                 :            : 
    1404                 :   69155109 :         return req;
    1405                 :            : }
    1406                 :            : 
    1407                 :            : static inline struct nvme_request *
    1408                 :    3596324 : nvme_allocate_request_contig(struct spdk_nvme_qpair *qpair,
    1409                 :            :                              void *buffer, uint32_t payload_size,
    1410                 :            :                              spdk_nvme_cmd_cb cb_fn, void *cb_arg)
    1411                 :            : {
    1412                 :     336738 :         struct nvme_payload payload;
    1413                 :            : 
    1414                 :    3596324 :         payload = NVME_PAYLOAD_CONTIG(buffer, NULL);
    1415                 :            : 
    1416                 :    3596324 :         return nvme_allocate_request(qpair, &payload, payload_size, 0, cb_fn, cb_arg);
    1417                 :            : }
    1418                 :            : 
    1419                 :            : static inline struct nvme_request *
    1420                 :    1891213 : nvme_allocate_request_null(struct spdk_nvme_qpair *qpair, spdk_nvme_cmd_cb cb_fn, void *cb_arg)
    1421                 :            : {
    1422                 :    1891213 :         return nvme_allocate_request_contig(qpair, NULL, 0, cb_fn, cb_arg);
    1423                 :            : }
    1424                 :            : 
    1425                 :            : struct nvme_request *nvme_allocate_request_user_copy(struct spdk_nvme_qpair *qpair,
    1426                 :            :                 void *buffer, uint32_t payload_size,
    1427                 :            :                 spdk_nvme_cmd_cb cb_fn, void *cb_arg, bool host_to_controller);
    1428                 :            : 
    1429                 :            : static inline void
    1430                 :   69164566 : _nvme_free_request(struct nvme_request *req, struct spdk_nvme_qpair *qpair)
    1431                 :            : {
    1432         [ -  + ]:   69164566 :         assert(req != NULL);
    1433         [ -  + ]:   69164566 :         assert(req->num_children == 0);
    1434         [ -  + ]:   69164566 :         assert(qpair != NULL);
    1435                 :            : 
    1436                 :            :         /* The reserved_req does not go in the free_req STAILQ - it is
    1437                 :            :          * saved only for use with a FABRICS/CONNECT command.
    1438                 :            :          */
    1439         [ +  + ]:   69164566 :         if (spdk_likely(qpair->reserved_req != req)) {
    1440         [ +  + ]:   69154921 :                 STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq);
    1441                 :            : 
    1442         [ -  + ]:   69154921 :                 assert(qpair->num_outstanding_reqs > 0);
    1443                 :   69154921 :                 qpair->num_outstanding_reqs--;
    1444                 :            :         }
    1445                 :   69164566 : }
    1446                 :            : 
    1447                 :            : static inline void
    1448                 :   24834695 : nvme_free_request(struct nvme_request *req)
    1449                 :            : {
    1450                 :   24834695 :         _nvme_free_request(req, req->qpair);
    1451                 :   24834695 : }
    1452                 :            : 
    1453                 :            : static inline void
    1454                 :   44329869 : nvme_complete_request(spdk_nvme_cmd_cb cb_fn, void *cb_arg, struct spdk_nvme_qpair *qpair,
    1455                 :            :                       struct nvme_request *req, struct spdk_nvme_cpl *cpl)
    1456                 :            : {
    1457                 :   10794809 :         struct spdk_nvme_cpl            err_cpl;
    1458                 :            :         struct nvme_error_cmd           *cmd;
    1459                 :            : 
    1460         [ -  + ]:   44329869 :         if (spdk_unlikely(req->accel_sequence != NULL)) {
    1461                 :          0 :                 struct spdk_nvme_poll_group *pg = qpair->poll_group->group;
    1462                 :            : 
    1463                 :            :                 /* Transports are required to execuete the sequence and clear req->accel_sequence.
    1464                 :            :                  * If it's left non-NULL it must mean the request is failed. */
    1465   [ #  #  #  # ]:          0 :                 assert(spdk_nvme_cpl_is_error(cpl));
    1466                 :          0 :                 pg->accel_fn_table.abort_sequence(req->accel_sequence);
    1467                 :          0 :                 req->accel_sequence = NULL;
    1468                 :            :         }
    1469                 :            : 
    1470                 :            :         /* error injection at completion path,
    1471                 :            :          * only inject for successful completed commands
    1472                 :            :          */
    1473   [ +  +  +  +  :   44329869 :         if (spdk_unlikely(!TAILQ_EMPTY(&qpair->err_cmd_head) &&
             +  -  +  + ]
    1474                 :            :                           !spdk_nvme_cpl_is_error(cpl))) {
    1475         [ +  + ]:    1609258 :                 TAILQ_FOREACH(cmd, &qpair->err_cmd_head, link) {
    1476                 :            : 
    1477   [ +  +  +  + ]:     804634 :                         if (cmd->do_not_submit) {
    1478                 :     804611 :                                 continue;
    1479                 :            :                         }
    1480                 :            : 
    1481   [ +  -  +  + ]:         23 :                         if ((cmd->opc == req->cmd.opc) && cmd->err_count) {
    1482                 :            : 
    1483                 :         10 :                                 err_cpl = *cpl;
    1484                 :         10 :                                 err_cpl.status.sct = cmd->status.sct;
    1485                 :         10 :                                 err_cpl.status.sc = cmd->status.sc;
    1486                 :            : 
    1487                 :         10 :                                 cpl = &err_cpl;
    1488                 :         10 :                                 cmd->err_count--;
    1489                 :         10 :                                 break;
    1490                 :            :                         }
    1491                 :            :                 }
    1492                 :            :         }
    1493                 :            : 
    1494                 :            :         /* For PCIe completions, we want to avoid touching the req itself to avoid
    1495                 :            :          * dependencies on loading those cachelines. So call the internal helper
    1496                 :            :          * function instead using the qpair that was passed by the caller, instead
    1497                 :            :          * of getting it from the req.
    1498                 :            :          */
    1499                 :   44329869 :         _nvme_free_request(req, qpair);
    1500                 :            : 
    1501         [ +  + ]:   44329869 :         if (spdk_likely(cb_fn)) {
    1502                 :   44329866 :                 cb_fn(cb_arg, cpl);
    1503                 :            :         }
    1504                 :   44329869 : }
    1505                 :            : 
    1506                 :            : static inline void
    1507                 :      66719 : nvme_cleanup_user_req(struct nvme_request *req)
    1508                 :            : {
    1509   [ +  +  +  - ]:      66719 :         if (req->user_buffer && req->payload_size) {
    1510                 :      60094 :                 spdk_free(req->payload.contig_or_cb_arg);
    1511                 :      60094 :                 req->user_buffer = NULL;
    1512                 :            :         }
    1513                 :            : 
    1514                 :      66719 :         req->user_cb_arg = NULL;
    1515                 :      66719 :         req->user_cb_fn = NULL;
    1516                 :      66719 : }
    1517                 :            : 
    1518                 :            : static inline void
    1519                 :      81589 : nvme_qpair_set_state(struct spdk_nvme_qpair *qpair, enum nvme_qpair_state state)
    1520                 :            : {
    1521                 :      81589 :         qpair->state = state;
    1522         [ +  + ]:      81589 :         if (state == NVME_QPAIR_ENABLED) {
    1523                 :       7038 :                 qpair->is_new_qpair = false;
    1524                 :            :         }
    1525                 :      81589 : }
    1526                 :            : 
    1527                 :            : static inline enum nvme_qpair_state
    1528                 : 1681555415 : nvme_qpair_get_state(struct spdk_nvme_qpair *qpair) {
    1529                 : 1681555415 :         return qpair->state;
    1530                 :            : }
    1531                 :            : 
    1532                 :            : static inline void
    1533                 :   26417413 : nvme_request_remove_child(struct nvme_request *parent, struct nvme_request *child)
    1534                 :            : {
    1535         [ -  + ]:   26417413 :         assert(parent != NULL);
    1536         [ -  + ]:   26417413 :         assert(child != NULL);
    1537         [ -  + ]:   26417413 :         assert(child->parent == parent);
    1538         [ -  + ]:   26417413 :         assert(parent->num_children != 0);
    1539                 :            : 
    1540                 :   26417413 :         parent->num_children--;
    1541                 :   26417413 :         child->parent = NULL;
    1542         [ +  + ]:   26417413 :         TAILQ_REMOVE(&parent->children, child, child_tailq);
    1543                 :   26417413 : }
    1544                 :            : 
    1545                 :            : static inline void
    1546                 :    1302984 : nvme_cb_complete_child(void *child_arg, const struct spdk_nvme_cpl *cpl)
    1547                 :            : {
    1548                 :    1302984 :         struct nvme_request *child = child_arg;
    1549                 :    1302984 :         struct nvme_request *parent = child->parent;
    1550                 :            : 
    1551                 :    1302984 :         nvme_request_remove_child(parent, child);
    1552                 :            : 
    1553   [ +  -  -  + ]:    1302984 :         if (spdk_nvme_cpl_is_error(cpl)) {
    1554                 :          0 :                 memcpy(&parent->parent_status, cpl, sizeof(*cpl));
    1555                 :            :         }
    1556                 :            : 
    1557         [ +  + ]:    1302984 :         if (parent->num_children == 0) {
    1558                 :     512285 :                 nvme_complete_request(parent->cb_fn, parent->cb_arg, parent->qpair,
    1559                 :            :                                       parent, &parent->parent_status);
    1560                 :            :         }
    1561                 :    1302984 : }
    1562                 :            : 
    1563                 :            : static inline void
    1564                 :   26087560 : nvme_request_add_child(struct nvme_request *parent, struct nvme_request *child)
    1565                 :            : {
    1566         [ -  + ]:   26087560 :         assert(parent->num_children != UINT16_MAX);
    1567                 :            : 
    1568         [ +  + ]:   26087560 :         if (parent->num_children == 0) {
    1569                 :            :                 /*
    1570                 :            :                  * Defer initialization of the children TAILQ since it falls
    1571                 :            :                  *  on a separate cacheline.  This ensures we do not touch this
    1572                 :            :                  *  cacheline except on request splitting cases, which are
    1573                 :            :                  *  relatively rare.
    1574                 :            :                  */
    1575                 :     561549 :                 TAILQ_INIT(&parent->children);
    1576                 :     561549 :                 parent->parent = NULL;
    1577         [ -  + ]:     561549 :                 memset(&parent->parent_status, 0, sizeof(struct spdk_nvme_cpl));
    1578                 :            :         }
    1579                 :            : 
    1580                 :   26087560 :         parent->num_children++;
    1581                 :   26087560 :         TAILQ_INSERT_TAIL(&parent->children, child, child_tailq);
    1582                 :   26087560 :         child->parent = parent;
    1583                 :   26087560 :         child->cb_fn = nvme_cb_complete_child;
    1584                 :   26087560 :         child->cb_arg = child;
    1585                 :   26087560 : }
    1586                 :            : 
    1587                 :            : static inline void
    1588                 :   24833852 : nvme_request_free_children(struct nvme_request *req)
    1589                 :            : {
    1590                 :            :         struct nvme_request *child, *tmp;
    1591                 :            : 
    1592         [ +  + ]:   24833852 :         if (req->num_children == 0) {
    1593                 :   24784594 :                 return;
    1594                 :            :         }
    1595                 :            : 
    1596                 :            :         /* free all child nvme_request */
    1597         [ +  + ]:   24833819 :         TAILQ_FOREACH_SAFE(child, &req->children, child_tailq, tmp) {
    1598                 :   24784561 :                 nvme_request_remove_child(req, child);
    1599                 :   24784561 :                 nvme_request_free_children(child);
    1600                 :   24784561 :                 nvme_free_request(child);
    1601                 :            :         }
    1602                 :            : }
    1603                 :            : 
    1604                 :            : int     nvme_request_check_timeout(struct nvme_request *req, uint16_t cid,
    1605                 :            :                                    struct spdk_nvme_ctrlr_process *active_proc, uint64_t now_tick);
    1606                 :            : uint64_t nvme_get_quirks(const struct spdk_pci_id *id);
    1607                 :            : 
    1608                 :            : int     nvme_robust_mutex_init_shared(pthread_mutex_t *mtx);
    1609                 :            : int     nvme_robust_mutex_init_recursive_shared(pthread_mutex_t *mtx);
    1610                 :            : 
    1611                 :            : bool    nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl);
    1612                 :            : 
    1613                 :            : struct spdk_nvme_ctrlr *nvme_get_ctrlr_by_trid_unsafe(
    1614                 :            :         const struct spdk_nvme_transport_id *trid, const char *hostnqn);
    1615                 :            : 
    1616                 :            : const struct spdk_nvme_transport *nvme_get_transport(const char *transport_name);
    1617                 :            : const struct spdk_nvme_transport *nvme_get_first_transport(void);
    1618                 :            : const struct spdk_nvme_transport *nvme_get_next_transport(const struct spdk_nvme_transport
    1619                 :            :                 *transport);
    1620                 :            : void  nvme_ctrlr_update_namespaces(struct spdk_nvme_ctrlr *ctrlr);
    1621                 :            : 
    1622                 :            : /* Transport specific functions */
    1623                 :            : struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
    1624                 :            :                 const struct spdk_nvme_ctrlr_opts *opts,
    1625                 :            :                 void *devhandle);
    1626                 :            : int nvme_transport_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr);
    1627                 :            : int nvme_transport_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, bool direct_connect);
    1628                 :            : int nvme_transport_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr);
    1629                 :            : int nvme_transport_ctrlr_ready(struct spdk_nvme_ctrlr *ctrlr);
    1630                 :            : int nvme_transport_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value);
    1631                 :            : int nvme_transport_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value);
    1632                 :            : int nvme_transport_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value);
    1633                 :            : int nvme_transport_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value);
    1634                 :            : int nvme_transport_ctrlr_set_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1635                 :            :                 uint32_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1636                 :            : int nvme_transport_ctrlr_set_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1637                 :            :                 uint64_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1638                 :            : int nvme_transport_ctrlr_get_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1639                 :            :                 spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1640                 :            : int nvme_transport_ctrlr_get_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1641                 :            :                 spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1642                 :            : uint32_t nvme_transport_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr);
    1643                 :            : uint16_t nvme_transport_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr);
    1644                 :            : struct spdk_nvme_qpair *nvme_transport_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
    1645                 :            :                 uint16_t qid, const struct spdk_nvme_io_qpair_opts *opts);
    1646                 :            : int nvme_transport_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr);
    1647                 :            : void *nvme_transport_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size);
    1648                 :            : int nvme_transport_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr);
    1649                 :            : int nvme_transport_ctrlr_enable_pmr(struct spdk_nvme_ctrlr *ctrlr);
    1650                 :            : int nvme_transport_ctrlr_disable_pmr(struct spdk_nvme_ctrlr *ctrlr);
    1651                 :            : void *nvme_transport_ctrlr_map_pmr(struct spdk_nvme_ctrlr *ctrlr, size_t *size);
    1652                 :            : int nvme_transport_ctrlr_unmap_pmr(struct spdk_nvme_ctrlr *ctrlr);
    1653                 :            : void nvme_transport_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
    1654                 :            :                 struct spdk_nvme_qpair *qpair);
    1655                 :            : int nvme_transport_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr,
    1656                 :            :                                        struct spdk_nvme_qpair *qpair);
    1657                 :            : void nvme_transport_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr,
    1658                 :            :                 struct spdk_nvme_qpair *qpair);
    1659                 :            : void nvme_transport_ctrlr_disconnect_qpair_done(struct spdk_nvme_qpair *qpair);
    1660                 :            : int nvme_transport_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
    1661                 :            :                 struct spdk_memory_domain **domains, int array_size);
    1662                 :            : void nvme_transport_qpair_abort_reqs(struct spdk_nvme_qpair *qpair);
    1663                 :            : int nvme_transport_qpair_reset(struct spdk_nvme_qpair *qpair);
    1664                 :            : int nvme_transport_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req);
    1665                 :            : int32_t nvme_transport_qpair_process_completions(struct spdk_nvme_qpair *qpair,
    1666                 :            :                 uint32_t max_completions);
    1667                 :            : void nvme_transport_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair);
    1668                 :            : int nvme_transport_qpair_iterate_requests(struct spdk_nvme_qpair *qpair,
    1669                 :            :                 int (*iter_fn)(struct nvme_request *req, void *arg),
    1670                 :            :                 void *arg);
    1671                 :            : 
    1672                 :            : struct spdk_nvme_transport_poll_group *nvme_transport_poll_group_create(
    1673                 :            :         const struct spdk_nvme_transport *transport);
    1674                 :            : struct spdk_nvme_transport_poll_group *nvme_transport_qpair_get_optimal_poll_group(
    1675                 :            :         const struct spdk_nvme_transport *transport,
    1676                 :            :         struct spdk_nvme_qpair *qpair);
    1677                 :            : int nvme_transport_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup,
    1678                 :            :                                   struct spdk_nvme_qpair *qpair);
    1679                 :            : int nvme_transport_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,
    1680                 :            :                                      struct spdk_nvme_qpair *qpair);
    1681                 :            : int nvme_transport_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair);
    1682                 :            : int nvme_transport_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair);
    1683                 :            : int64_t nvme_transport_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,
    1684                 :            :                 uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb);
    1685                 :            : int nvme_transport_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup);
    1686                 :            : int nvme_transport_poll_group_get_stats(struct spdk_nvme_transport_poll_group *tgroup,
    1687                 :            :                                         struct spdk_nvme_transport_poll_group_stat **stats);
    1688                 :            : void nvme_transport_poll_group_free_stats(struct spdk_nvme_transport_poll_group *tgroup,
    1689                 :            :                 struct spdk_nvme_transport_poll_group_stat *stats);
    1690                 :            : enum spdk_nvme_transport_type nvme_transport_get_trtype(const struct spdk_nvme_transport
    1691                 :            :                 *transport);
    1692                 :            : /*
    1693                 :            :  * Below ref related functions must be called with the global
    1694                 :            :  *  driver lock held for the multi-process condition.
    1695                 :            :  *  Within these functions, the per ctrlr ctrlr_lock is also
    1696                 :            :  *  acquired for the multi-thread condition.
    1697                 :            :  */
    1698                 :            : void    nvme_ctrlr_proc_get_ref(struct spdk_nvme_ctrlr *ctrlr);
    1699                 :            : void    nvme_ctrlr_proc_put_ref(struct spdk_nvme_ctrlr *ctrlr);
    1700                 :            : int     nvme_ctrlr_get_ref_count(struct spdk_nvme_ctrlr *ctrlr);
    1701                 :            : 
    1702                 :            : int     nvme_ctrlr_reinitialize_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair);
    1703                 :            : int     nvme_parse_addr(struct sockaddr_storage *sa, int family,
    1704                 :            :                         const char *addr, const char *service, long int *port);
    1705                 :            : int     nvme_get_default_hostnqn(char *buf, int len);
    1706                 :            : 
    1707                 :            : static inline bool
    1708                 :    3136225 : _is_page_aligned(uint64_t address, uint64_t page_size)
    1709                 :            : {
    1710                 :    3136225 :         return (address & (page_size - 1)) == 0;
    1711                 :            : }
    1712                 :            : 
    1713                 :            : #endif /* __NVME_INTERNAL_H__ */

Generated by: LCOV version 1.14