LCOV - code coverage report
Current view: top level - spdk/lib/nvme - nvme_internal.h (source / functions) Hit Total Coverage
Test: Combined Lines: 114 120 95.0 %
Date: 2024-07-13 19:46:08 Functions: 20 20 100.0 %
Legend: Lines: hit not hit | Branches: + taken - not taken # not executed Branches: 60 86 69.8 %

           Branch data     Line data    Source code
       1                 :            : /*   SPDX-License-Identifier: BSD-3-Clause
       2                 :            :  *   Copyright (C) 2015 Intel Corporation. All rights reserved.
       3                 :            :  *   Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved.
       4                 :            :  *   Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
       5                 :            :  */
       6                 :            : 
       7                 :            : #ifndef __NVME_INTERNAL_H__
       8                 :            : #define __NVME_INTERNAL_H__
       9                 :            : 
      10                 :            : #include "spdk/config.h"
      11                 :            : #include "spdk/likely.h"
      12                 :            : #include "spdk/stdinc.h"
      13                 :            : 
      14                 :            : #include "spdk/nvme.h"
      15                 :            : 
      16                 :            : #if defined(__i386__) || defined(__x86_64__)
      17                 :            : #include <x86intrin.h>
      18                 :            : #endif
      19                 :            : 
      20                 :            : #include "spdk/queue.h"
      21                 :            : #include "spdk/barrier.h"
      22                 :            : #include "spdk/bit_array.h"
      23                 :            : #include "spdk/mmio.h"
      24                 :            : #include "spdk/pci_ids.h"
      25                 :            : #include "spdk/util.h"
      26                 :            : #include "spdk/memory.h"
      27                 :            : #include "spdk/nvme_intel.h"
      28                 :            : #include "spdk/nvmf_spec.h"
      29                 :            : #include "spdk/tree.h"
      30                 :            : #include "spdk/uuid.h"
      31                 :            : 
      32                 :            : #include "spdk_internal/assert.h"
      33                 :            : #include "spdk/log.h"
      34                 :            : 
      35                 :            : extern pid_t g_spdk_nvme_pid;
      36                 :            : 
      37                 :            : extern struct spdk_nvme_transport_opts g_spdk_nvme_transport_opts;
      38                 :            : 
      39                 :            : /*
      40                 :            :  * Some Intel devices support vendor-unique read latency log page even
      41                 :            :  * though the log page directory says otherwise.
      42                 :            :  */
      43                 :            : #define NVME_INTEL_QUIRK_READ_LATENCY 0x1
      44                 :            : 
      45                 :            : /*
      46                 :            :  * Some Intel devices support vendor-unique write latency log page even
      47                 :            :  * though the log page directory says otherwise.
      48                 :            :  */
      49                 :            : #define NVME_INTEL_QUIRK_WRITE_LATENCY 0x2
      50                 :            : 
      51                 :            : /*
      52                 :            :  * The controller needs a delay before starts checking the device
      53                 :            :  * readiness, which is done by reading the NVME_CSTS_RDY bit.
      54                 :            :  */
      55                 :            : #define NVME_QUIRK_DELAY_BEFORE_CHK_RDY 0x4
      56                 :            : 
      57                 :            : /*
      58                 :            :  * The controller performs best when I/O is split on particular
      59                 :            :  * LBA boundaries.
      60                 :            :  */
      61                 :            : #define NVME_INTEL_QUIRK_STRIPING 0x8
      62                 :            : 
      63                 :            : /*
      64                 :            :  * The controller needs a delay after allocating an I/O queue pair
      65                 :            :  * before it is ready to accept I/O commands.
      66                 :            :  */
      67                 :            : #define NVME_QUIRK_DELAY_AFTER_QUEUE_ALLOC 0x10
      68                 :            : 
      69                 :            : /*
      70                 :            :  * Earlier NVMe devices do not indicate whether unmapped blocks
      71                 :            :  * will read all zeroes or not. This define indicates that the
      72                 :            :  * device does in fact read all zeroes after an unmap event
      73                 :            :  */
      74                 :            : #define NVME_QUIRK_READ_ZERO_AFTER_DEALLOCATE 0x20
      75                 :            : 
      76                 :            : /*
      77                 :            :  * The controller doesn't handle Identify value others than 0 or 1 correctly.
      78                 :            :  */
      79                 :            : #define NVME_QUIRK_IDENTIFY_CNS 0x40
      80                 :            : 
      81                 :            : /*
      82                 :            :  * The controller supports Open Channel command set if matching additional
      83                 :            :  * condition, like the first byte (value 0x1) in the vendor specific
      84                 :            :  * bits of the namespace identify structure is set.
      85                 :            :  */
      86                 :            : #define NVME_QUIRK_OCSSD 0x80
      87                 :            : 
      88                 :            : /*
      89                 :            :  * The controller has an Intel vendor ID but does not support Intel vendor-specific
      90                 :            :  * log pages.  This is primarily for QEMU emulated SSDs which report an Intel vendor
      91                 :            :  * ID but do not support these log pages.
      92                 :            :  */
      93                 :            : #define NVME_INTEL_QUIRK_NO_LOG_PAGES 0x100
      94                 :            : 
      95                 :            : /*
      96                 :            :  * The controller does not set SHST_COMPLETE in a reasonable amount of time.  This
      97                 :            :  * is primarily seen in virtual VMWare NVMe SSDs.  This quirk merely adds an additional
      98                 :            :  * error message that on VMWare NVMe SSDs, the shutdown timeout may be expected.
      99                 :            :  */
     100                 :            : #define NVME_QUIRK_SHST_COMPLETE 0x200
     101                 :            : 
     102                 :            : /*
     103                 :            :  * The controller requires an extra delay before starting the initialization process
     104                 :            :  * during attach.
     105                 :            :  */
     106                 :            : #define NVME_QUIRK_DELAY_BEFORE_INIT 0x400
     107                 :            : 
     108                 :            : /*
     109                 :            :  * Some SSDs exhibit poor performance with the default SPDK NVMe IO queue size.
     110                 :            :  * This quirk will increase the default to 1024 which matches other operating
     111                 :            :  * systems, at the cost of some extra memory usage.  Users can still override
     112                 :            :  * the increased default by changing the spdk_nvme_io_qpair_opts when allocating
     113                 :            :  * a new queue pair.
     114                 :            :  */
     115                 :            : #define NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE 0x800
     116                 :            : 
     117                 :            : /**
     118                 :            :  * The maximum access width to PCI memory space is 8 Bytes, don't use AVX2 or
     119                 :            :  * SSE instructions to optimize the memory access(memcpy or memset) larger than
     120                 :            :  * 8 Bytes.
     121                 :            :  */
     122                 :            : #define NVME_QUIRK_MAXIMUM_PCI_ACCESS_WIDTH 0x1000
     123                 :            : 
     124                 :            : /**
     125                 :            :  * The SSD does not support OPAL even through it sets the security bit in OACS.
     126                 :            :  */
     127                 :            : #define NVME_QUIRK_OACS_SECURITY 0x2000
     128                 :            : 
     129                 :            : /**
     130                 :            :  * Intel P55XX SSDs can't support Dataset Management command with SGL format,
     131                 :            :  * so use PRP with DSM command.
     132                 :            :  */
     133                 :            : #define NVME_QUIRK_NO_SGL_FOR_DSM 0x4000
     134                 :            : 
     135                 :            : /**
     136                 :            :  * Maximum Data Transfer Size(MDTS) excludes interleaved metadata.
     137                 :            :  */
     138                 :            : #define NVME_QUIRK_MDTS_EXCLUDE_MD 0x8000
     139                 :            : 
     140                 :            : /**
     141                 :            :  * Force not to use SGL even the controller report that it can
     142                 :            :  * support it.
     143                 :            :  */
     144                 :            : #define NVME_QUIRK_NOT_USE_SGL 0x10000
     145                 :            : 
     146                 :            : /*
     147                 :            :  * Some SSDs require the admin submission queue size to equate to an even
     148                 :            :  * 4KiB multiple.
     149                 :            :  */
     150                 :            : #define NVME_QUIRK_MINIMUM_ADMIN_QUEUE_SIZE 0x20000
     151                 :            : 
     152                 :            : #define NVME_MAX_ASYNC_EVENTS   (8)
     153                 :            : 
     154                 :            : #define NVME_MAX_ADMIN_TIMEOUT_IN_SECS  (30)
     155                 :            : 
     156                 :            : /* Maximum log page size to fetch for AERs. */
     157                 :            : #define NVME_MAX_AER_LOG_SIZE           (4096)
     158                 :            : 
     159                 :            : /*
     160                 :            :  * NVME_MAX_IO_QUEUES in nvme_spec.h defines the 64K spec-limit, but this
     161                 :            :  *  define specifies the maximum number of queues this driver will actually
     162                 :            :  *  try to configure, if available.
     163                 :            :  */
     164                 :            : #define DEFAULT_MAX_IO_QUEUES           (1024)
     165                 :            : #define DEFAULT_ADMIN_QUEUE_SIZE        (32)
     166                 :            : #define DEFAULT_IO_QUEUE_SIZE           (256)
     167                 :            : #define DEFAULT_IO_QUEUE_SIZE_FOR_QUIRK (1024) /* Matches Linux kernel driver */
     168                 :            : 
     169                 :            : #define DEFAULT_IO_QUEUE_REQUESTS       (512)
     170                 :            : 
     171                 :            : #define SPDK_NVME_DEFAULT_RETRY_COUNT   (4)
     172                 :            : 
     173                 :            : #define SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED        (0)
     174                 :            : #define SPDK_NVME_DEFAULT_TRANSPORT_ACK_TIMEOUT SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED
     175                 :            : 
     176                 :            : #define SPDK_NVME_TRANSPORT_TOS_DISABLED        (0)
     177                 :            : 
     178                 :            : #define MIN_KEEP_ALIVE_TIMEOUT_IN_MS    (10000)
     179                 :            : 
     180                 :            : /* We want to fit submission and completion rings each in a single 2MB
     181                 :            :  * hugepage to ensure physical address contiguity.
     182                 :            :  */
     183                 :            : #define MAX_IO_QUEUE_ENTRIES            (VALUE_2MB / spdk_max( \
     184                 :            :                                                 sizeof(struct spdk_nvme_cmd), \
     185                 :            :                                                 sizeof(struct spdk_nvme_cpl)))
     186                 :            : 
     187                 :            : /* Default timeout for fabrics connect commands. */
     188                 :            : #ifdef DEBUG
     189                 :            : #define NVME_FABRIC_CONNECT_COMMAND_TIMEOUT 0
     190                 :            : #else
     191                 :            : /* 500 millisecond timeout. */
     192                 :            : #define NVME_FABRIC_CONNECT_COMMAND_TIMEOUT 500000
     193                 :            : #endif
     194                 :            : 
     195                 :            : /* This value indicates that a read from a PCIe register is invalid. This can happen when a device is no longer present */
     196                 :            : #define SPDK_NVME_INVALID_REGISTER_VALUE 0xFFFFFFFFu
     197                 :            : 
     198                 :            : enum nvme_payload_type {
     199                 :            :         NVME_PAYLOAD_TYPE_INVALID = 0,
     200                 :            : 
     201                 :            :         /** nvme_request::u.payload.contig_buffer is valid for this request */
     202                 :            :         NVME_PAYLOAD_TYPE_CONTIG,
     203                 :            : 
     204                 :            :         /** nvme_request::u.sgl is valid for this request */
     205                 :            :         NVME_PAYLOAD_TYPE_SGL,
     206                 :            : };
     207                 :            : 
     208                 :            : /** Boot partition write states */
     209                 :            : enum nvme_bp_write_state {
     210                 :            :         SPDK_NVME_BP_WS_DOWNLOADING     = 0x0,
     211                 :            :         SPDK_NVME_BP_WS_DOWNLOADED      = 0x1,
     212                 :            :         SPDK_NVME_BP_WS_REPLACE         = 0x2,
     213                 :            :         SPDK_NVME_BP_WS_ACTIVATE        = 0x3,
     214                 :            : };
     215                 :            : 
     216                 :            : /**
     217                 :            :  * Descriptor for a request data payload.
     218                 :            :  */
     219                 :            : struct nvme_payload {
     220                 :            :         /**
     221                 :            :          * Functions for retrieving physical addresses for scattered payloads.
     222                 :            :          */
     223                 :            :         spdk_nvme_req_reset_sgl_cb reset_sgl_fn;
     224                 :            :         spdk_nvme_req_next_sge_cb next_sge_fn;
     225                 :            : 
     226                 :            :         /**
     227                 :            :          * Extended IO options passed by the user
     228                 :            :          */
     229                 :            :         struct spdk_nvme_ns_cmd_ext_io_opts *opts;
     230                 :            :         /**
     231                 :            :          * If reset_sgl_fn == NULL, this is a contig payload, and contig_or_cb_arg contains the
     232                 :            :          * virtual memory address of a single virtually contiguous buffer.
     233                 :            :          *
     234                 :            :          * If reset_sgl_fn != NULL, this is a SGL payload, and contig_or_cb_arg contains the
     235                 :            :          * cb_arg that will be passed to the SGL callback functions.
     236                 :            :          */
     237                 :            :         void *contig_or_cb_arg;
     238                 :            : 
     239                 :            :         /** Virtual memory address of a single virtually contiguous metadata buffer */
     240                 :            :         void *md;
     241                 :            : };
     242                 :            : 
     243                 :            : #define NVME_PAYLOAD_CONTIG(contig_, md_) \
     244                 :            :         (struct nvme_payload) { \
     245                 :            :                 .reset_sgl_fn = NULL, \
     246                 :            :                 .next_sge_fn = NULL, \
     247                 :            :                 .contig_or_cb_arg = (contig_), \
     248                 :            :                 .md = (md_), \
     249                 :            :         }
     250                 :            : 
     251                 :            : #define NVME_PAYLOAD_SGL(reset_sgl_fn_, next_sge_fn_, cb_arg_, md_) \
     252                 :            :         (struct nvme_payload) { \
     253                 :            :                 .reset_sgl_fn = (reset_sgl_fn_), \
     254                 :            :                 .next_sge_fn = (next_sge_fn_), \
     255                 :            :                 .contig_or_cb_arg = (cb_arg_), \
     256                 :            :                 .md = (md_), \
     257                 :            :         }
     258                 :            : 
     259                 :            : static inline enum nvme_payload_type
     260                 :  188511212 : nvme_payload_type(const struct nvme_payload *payload) {
     261         [ +  + ]:  188511212 :         return payload->reset_sgl_fn ? NVME_PAYLOAD_TYPE_SGL : NVME_PAYLOAD_TYPE_CONTIG;
     262                 :            : }
     263                 :            : 
     264                 :            : struct nvme_error_cmd {
     265                 :            :         bool                            do_not_submit;
     266                 :            :         uint64_t                        timeout_tsc;
     267                 :            :         uint32_t                        err_count;
     268                 :            :         uint8_t                         opc;
     269                 :            :         struct spdk_nvme_status         status;
     270                 :            :         TAILQ_ENTRY(nvme_error_cmd)     link;
     271                 :            : };
     272                 :            : 
     273                 :            : struct nvme_request {
     274                 :            :         struct spdk_nvme_cmd            cmd;
     275                 :            : 
     276                 :            :         uint8_t                         retries;
     277                 :            : 
     278                 :            :         uint8_t                         timed_out : 1;
     279                 :            : 
     280                 :            :         /**
     281                 :            :          * True if the request is in the queued_req list.
     282                 :            :          */
     283                 :            :         uint8_t                         queued : 1;
     284                 :            :         uint8_t                         reserved : 6;
     285                 :            : 
     286                 :            :         /**
     287                 :            :          * Number of children requests still outstanding for this
     288                 :            :          *  request which was split into multiple child requests.
     289                 :            :          */
     290                 :            :         uint16_t                        num_children;
     291                 :            : 
     292                 :            :         /**
     293                 :            :          * Offset in bytes from the beginning of payload for this request.
     294                 :            :          * This is used for I/O commands that are split into multiple requests.
     295                 :            :          */
     296                 :            :         uint32_t                        payload_offset;
     297                 :            :         uint32_t                        md_offset;
     298                 :            : 
     299                 :            :         uint32_t                        payload_size;
     300                 :            : 
     301                 :            :         /**
     302                 :            :          * Timeout ticks for error injection requests, can be extended in future
     303                 :            :          * to support per-request timeout feature.
     304                 :            :          */
     305                 :            :         uint64_t                        timeout_tsc;
     306                 :            : 
     307                 :            :         /**
     308                 :            :          * Data payload for this request's command.
     309                 :            :          */
     310                 :            :         struct nvme_payload             payload;
     311                 :            : 
     312                 :            :         spdk_nvme_cmd_cb                cb_fn;
     313                 :            :         void                            *cb_arg;
     314                 :            :         STAILQ_ENTRY(nvme_request)      stailq;
     315                 :            : 
     316                 :            :         struct spdk_nvme_qpair          *qpair;
     317                 :            : 
     318                 :            :         /*
     319                 :            :          * The value of spdk_get_ticks() when the request was submitted to the hardware.
     320                 :            :          * Only set if ctrlr->timeout_enabled is true.
     321                 :            :          */
     322                 :            :         uint64_t                        submit_tick;
     323                 :            : 
     324                 :            :         /**
     325                 :            :          * The active admin request can be moved to a per process pending
     326                 :            :          *  list based on the saved pid to tell which process it belongs
     327                 :            :          *  to. The cpl saves the original completion information which
     328                 :            :          *  is used in the completion callback.
     329                 :            :          * NOTE: these below two fields are only used for admin request.
     330                 :            :          */
     331                 :            :         pid_t                           pid;
     332                 :            :         struct spdk_nvme_cpl            cpl;
     333                 :            : 
     334                 :            :         uint32_t                        md_size;
     335                 :            : 
     336                 :            :         /**
     337                 :            :          * The following members should not be reordered with members
     338                 :            :          *  above.  These members are only needed when splitting
     339                 :            :          *  requests which is done rarely, and the driver is careful
     340                 :            :          *  to not touch the following fields until a split operation is
     341                 :            :          *  needed, to avoid touching an extra cacheline.
     342                 :            :          */
     343                 :            : 
     344                 :            :         /**
     345                 :            :          * Points to the outstanding child requests for a parent request.
     346                 :            :          *  Only valid if a request was split into multiple children
     347                 :            :          *  requests, and is not initialized for non-split requests.
     348                 :            :          */
     349                 :            :         TAILQ_HEAD(, nvme_request)      children;
     350                 :            : 
     351                 :            :         /**
     352                 :            :          * Linked-list pointers for a child request in its parent's list.
     353                 :            :          */
     354                 :            :         TAILQ_ENTRY(nvme_request)       child_tailq;
     355                 :            : 
     356                 :            :         /**
     357                 :            :          * Points to a parent request if part of a split request,
     358                 :            :          *   NULL otherwise.
     359                 :            :          */
     360                 :            :         struct nvme_request             *parent;
     361                 :            : 
     362                 :            :         /**
     363                 :            :          * Completion status for a parent request.  Initialized to all 0's
     364                 :            :          *  (SUCCESS) before child requests are submitted.  If a child
     365                 :            :          *  request completes with error, the error status is copied here,
     366                 :            :          *  to ensure that the parent request is also completed with error
     367                 :            :          *  status once all child requests are completed.
     368                 :            :          */
     369                 :            :         struct spdk_nvme_cpl            parent_status;
     370                 :            : 
     371                 :            :         /**
     372                 :            :          * The user_cb_fn and user_cb_arg fields are used for holding the original
     373                 :            :          * callback data when using nvme_allocate_request_user_copy.
     374                 :            :          */
     375                 :            :         spdk_nvme_cmd_cb                user_cb_fn;
     376                 :            :         void                            *user_cb_arg;
     377                 :            :         void                            *user_buffer;
     378                 :            : 
     379                 :            :         /** Sequence of accel operations associated with this request */
     380                 :            :         void                            *accel_sequence;
     381                 :            : };
     382                 :            : 
     383                 :            : struct nvme_completion_poll_status {
     384                 :            :         struct spdk_nvme_cpl    cpl;
     385                 :            :         uint64_t                timeout_tsc;
     386                 :            :         /**
     387                 :            :          * DMA buffer retained throughout the duration of the command.  It'll be released
     388                 :            :          * automatically if the command times out, otherwise the user is responsible for freeing it.
     389                 :            :          */
     390                 :            :         void                    *dma_data;
     391                 :            :         bool                    done;
     392                 :            :         /* This flag indicates that the request has been timed out and the memory
     393                 :            :            must be freed in a completion callback */
     394                 :            :         bool                    timed_out;
     395                 :            : };
     396                 :            : 
     397                 :            : struct nvme_async_event_request {
     398                 :            :         struct spdk_nvme_ctrlr          *ctrlr;
     399                 :            :         struct nvme_request             *req;
     400                 :            :         struct spdk_nvme_cpl            cpl;
     401                 :            : };
     402                 :            : 
     403                 :            : enum nvme_qpair_state {
     404                 :            :         NVME_QPAIR_DISCONNECTED,
     405                 :            :         NVME_QPAIR_DISCONNECTING,
     406                 :            :         NVME_QPAIR_CONNECTING,
     407                 :            :         NVME_QPAIR_CONNECTED,
     408                 :            :         NVME_QPAIR_ENABLING,
     409                 :            :         NVME_QPAIR_ENABLED,
     410                 :            :         NVME_QPAIR_DESTROYING,
     411                 :            : };
     412                 :            : 
     413                 :            : struct spdk_nvme_qpair {
     414                 :            :         struct spdk_nvme_ctrlr                  *ctrlr;
     415                 :            : 
     416                 :            :         uint16_t                                id;
     417                 :            : 
     418                 :            :         uint8_t                                 qprio: 2;
     419                 :            : 
     420                 :            :         uint8_t                                 state: 3;
     421                 :            : 
     422                 :            :         uint8_t                                 async: 1;
     423                 :            : 
     424                 :            :         uint8_t                                 is_new_qpair: 1;
     425                 :            : 
     426                 :            :         uint8_t                                 abort_dnr: 1;
     427                 :            :         /*
     428                 :            :          * Members for handling IO qpair deletion inside of a completion context.
     429                 :            :          * These are specifically defined as single bits, so that they do not
     430                 :            :          *  push this data structure out to another cacheline.
     431                 :            :          */
     432                 :            :         uint8_t                                 in_completion_context: 1;
     433                 :            :         uint8_t                                 delete_after_completion_context: 1;
     434                 :            : 
     435                 :            :         /*
     436                 :            :          * Set when no deletion notification is needed. For example, the process
     437                 :            :          * which allocated this qpair exited unexpectedly.
     438                 :            :          */
     439                 :            :         uint8_t                                 no_deletion_notification_needed: 1;
     440                 :            : 
     441                 :            :         uint8_t                                 last_fuse: 2;
     442                 :            : 
     443                 :            :         uint8_t                                 transport_failure_reason: 3;
     444                 :            :         uint8_t                                 last_transport_failure_reason: 3;
     445                 :            : 
     446                 :            :         /* The user is destroying qpair */
     447                 :            :         uint8_t                                 destroy_in_progress: 1;
     448                 :            : 
     449                 :            :         enum spdk_nvme_transport_type           trtype;
     450                 :            : 
     451                 :            :         uint32_t                                num_outstanding_reqs;
     452                 :            : 
     453                 :            :         /* request object used only for this qpair's FABRICS/CONNECT command (if needed) */
     454                 :            :         struct nvme_request                     *reserved_req;
     455                 :            : 
     456                 :            :         STAILQ_HEAD(, nvme_request)             free_req;
     457                 :            :         STAILQ_HEAD(, nvme_request)             queued_req;
     458                 :            : 
     459                 :            :         /* List entry for spdk_nvme_transport_poll_group::qpairs */
     460                 :            :         STAILQ_ENTRY(spdk_nvme_qpair)           poll_group_stailq;
     461                 :            : 
     462                 :            :         /** Commands opcode in this list will return error */
     463                 :            :         TAILQ_HEAD(, nvme_error_cmd)            err_cmd_head;
     464                 :            :         /** Requests in this list will return error */
     465                 :            :         STAILQ_HEAD(, nvme_request)             err_req_head;
     466                 :            : 
     467                 :            :         struct spdk_nvme_ctrlr_process          *active_proc;
     468                 :            : 
     469                 :            :         struct spdk_nvme_transport_poll_group   *poll_group;
     470                 :            : 
     471                 :            :         void                                    *poll_group_tailq_head;
     472                 :            : 
     473                 :            :         const struct spdk_nvme_transport        *transport;
     474                 :            : 
     475                 :            :         /* Entries below here are not touched in the main I/O path. */
     476                 :            : 
     477                 :            :         struct nvme_completion_poll_status      *poll_status;
     478                 :            : 
     479                 :            :         /* List entry for spdk_nvme_ctrlr::active_io_qpairs */
     480                 :            :         TAILQ_ENTRY(spdk_nvme_qpair)            tailq;
     481                 :            : 
     482                 :            :         /* List entry for spdk_nvme_ctrlr_process::allocated_io_qpairs */
     483                 :            :         TAILQ_ENTRY(spdk_nvme_qpair)            per_process_tailq;
     484                 :            : 
     485                 :            :         STAILQ_HEAD(, nvme_request)             aborting_queued_req;
     486                 :            : 
     487                 :            :         void                                    *req_buf;
     488                 :            : };
     489                 :            : 
     490                 :            : struct spdk_nvme_poll_group {
     491                 :            :         void                                            *ctx;
     492                 :            :         struct spdk_nvme_accel_fn_table                 accel_fn_table;
     493                 :            :         STAILQ_HEAD(, spdk_nvme_transport_poll_group)   tgroups;
     494                 :            :         bool                                            in_process_completions;
     495                 :            : };
     496                 :            : 
     497                 :            : struct spdk_nvme_transport_poll_group {
     498                 :            :         struct spdk_nvme_poll_group                     *group;
     499                 :            :         const struct spdk_nvme_transport                *transport;
     500                 :            :         STAILQ_HEAD(, spdk_nvme_qpair)                  connected_qpairs;
     501                 :            :         STAILQ_HEAD(, spdk_nvme_qpair)                  disconnected_qpairs;
     502                 :            :         STAILQ_ENTRY(spdk_nvme_transport_poll_group)    link;
     503                 :            :         uint32_t                                        num_connected_qpairs;
     504                 :            : };
     505                 :            : 
     506                 :            : struct spdk_nvme_ns {
     507                 :            :         struct spdk_nvme_ctrlr          *ctrlr;
     508                 :            :         uint32_t                        sector_size;
     509                 :            : 
     510                 :            :         /*
     511                 :            :          * Size of data transferred as part of each block,
     512                 :            :          * including metadata if FLBAS indicates the metadata is transferred
     513                 :            :          * as part of the data buffer at the end of each LBA.
     514                 :            :          */
     515                 :            :         uint32_t                        extended_lba_size;
     516                 :            : 
     517                 :            :         uint32_t                        md_size;
     518                 :            :         uint32_t                        pi_type;
     519                 :            :         uint32_t                        sectors_per_max_io;
     520                 :            :         uint32_t                        sectors_per_max_io_no_md;
     521                 :            :         uint32_t                        sectors_per_stripe;
     522                 :            :         uint32_t                        id;
     523                 :            :         uint16_t                        flags;
     524                 :            :         bool                            active;
     525                 :            : 
     526                 :            :         /* Command Set Identifier */
     527                 :            :         enum spdk_nvme_csi              csi;
     528                 :            : 
     529                 :            :         /* Namespace Identification Descriptor List (CNS = 03h) */
     530                 :            :         uint8_t                         id_desc_list[4096];
     531                 :            : 
     532                 :            :         uint32_t                        ana_group_id;
     533                 :            :         enum spdk_nvme_ana_state        ana_state;
     534                 :            : 
     535                 :            :         /* Identify Namespace data. */
     536                 :            :         struct spdk_nvme_ns_data        nsdata;
     537                 :            : 
     538                 :            :         /* Zoned Namespace Command Set Specific Identify Namespace data. */
     539                 :            :         struct spdk_nvme_zns_ns_data    *nsdata_zns;
     540                 :            : 
     541                 :            :         RB_ENTRY(spdk_nvme_ns)          node;
     542                 :            : };
     543                 :            : 
     544                 :            : /**
     545                 :            :  * State of struct spdk_nvme_ctrlr (in particular, during initialization).
     546                 :            :  */
     547                 :            : enum nvme_ctrlr_state {
     548                 :            :         /**
     549                 :            :          * Wait before initializing the controller.
     550                 :            :          */
     551                 :            :         NVME_CTRLR_STATE_INIT_DELAY,
     552                 :            : 
     553                 :            :         /**
     554                 :            :          * Connect the admin queue.
     555                 :            :          */
     556                 :            :         NVME_CTRLR_STATE_CONNECT_ADMINQ,
     557                 :            : 
     558                 :            :         /**
     559                 :            :          * Controller has not started initialized yet.
     560                 :            :          */
     561                 :            :         NVME_CTRLR_STATE_INIT = NVME_CTRLR_STATE_CONNECT_ADMINQ,
     562                 :            : 
     563                 :            :         /**
     564                 :            :          * Waiting for admin queue to connect.
     565                 :            :          */
     566                 :            :         NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ,
     567                 :            : 
     568                 :            :         /**
     569                 :            :          * Read Version (VS) register.
     570                 :            :          */
     571                 :            :         NVME_CTRLR_STATE_READ_VS,
     572                 :            : 
     573                 :            :         /**
     574                 :            :          * Waiting for Version (VS) register to be read.
     575                 :            :          */
     576                 :            :         NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS,
     577                 :            : 
     578                 :            :         /**
     579                 :            :          * Read Capabilities (CAP) register.
     580                 :            :          */
     581                 :            :         NVME_CTRLR_STATE_READ_CAP,
     582                 :            : 
     583                 :            :         /**
     584                 :            :          * Waiting for Capabilities (CAP) register to be read.
     585                 :            :          */
     586                 :            :         NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP,
     587                 :            : 
     588                 :            :         /**
     589                 :            :          * Check EN to prepare for controller initialization.
     590                 :            :          */
     591                 :            :         NVME_CTRLR_STATE_CHECK_EN,
     592                 :            : 
     593                 :            :         /**
     594                 :            :          * Waiting for CC to be read as part of EN check.
     595                 :            :          */
     596                 :            :         NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC,
     597                 :            : 
     598                 :            :         /**
     599                 :            :          * Waiting for CSTS.RDY to transition from 0 to 1 so that CC.EN may be set to 0.
     600                 :            :          */
     601                 :            :         NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1,
     602                 :            : 
     603                 :            :         /**
     604                 :            :          * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 1.
     605                 :            :          */
     606                 :            :         NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS,
     607                 :            : 
     608                 :            :         /**
     609                 :            :          * Disabling the controller by setting CC.EN to 0.
     610                 :            :          */
     611                 :            :         NVME_CTRLR_STATE_SET_EN_0,
     612                 :            : 
     613                 :            :         /**
     614                 :            :          * Waiting for the CC register to be read as part of disabling the controller.
     615                 :            :          */
     616                 :            :         NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC,
     617                 :            : 
     618                 :            :         /**
     619                 :            :          * Waiting for CSTS.RDY to transition from 1 to 0 so that CC.EN may be set to 1.
     620                 :            :          */
     621                 :            :         NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0,
     622                 :            : 
     623                 :            :         /**
     624                 :            :          * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 0.
     625                 :            :          */
     626                 :            :         NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS,
     627                 :            : 
     628                 :            :         /**
     629                 :            :          * The controller is disabled. (CC.EN and CSTS.RDY are 0.)
     630                 :            :          */
     631                 :            :         NVME_CTRLR_STATE_DISABLED,
     632                 :            : 
     633                 :            :         /**
     634                 :            :          * Enable the controller by writing CC.EN to 1
     635                 :            :          */
     636                 :            :         NVME_CTRLR_STATE_ENABLE,
     637                 :            : 
     638                 :            :         /**
     639                 :            :          * Waiting for CC register to be written as part of enabling the controller.
     640                 :            :          */
     641                 :            :         NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC,
     642                 :            : 
     643                 :            :         /**
     644                 :            :          * Waiting for CSTS.RDY to transition from 0 to 1 after enabling the controller.
     645                 :            :          */
     646                 :            :         NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1,
     647                 :            : 
     648                 :            :         /**
     649                 :            :          * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 1.
     650                 :            :          */
     651                 :            :         NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS,
     652                 :            : 
     653                 :            :         /**
     654                 :            :          * Reset the Admin queue of the controller.
     655                 :            :          */
     656                 :            :         NVME_CTRLR_STATE_RESET_ADMIN_QUEUE,
     657                 :            : 
     658                 :            :         /**
     659                 :            :          * Identify Controller command will be sent to then controller.
     660                 :            :          */
     661                 :            :         NVME_CTRLR_STATE_IDENTIFY,
     662                 :            : 
     663                 :            :         /**
     664                 :            :          * Waiting for Identify Controller command be completed.
     665                 :            :          */
     666                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY,
     667                 :            : 
     668                 :            :         /**
     669                 :            :          * Configure AER of the controller.
     670                 :            :          */
     671                 :            :         NVME_CTRLR_STATE_CONFIGURE_AER,
     672                 :            : 
     673                 :            :         /**
     674                 :            :          * Waiting for the Configure AER to be completed.
     675                 :            :          */
     676                 :            :         NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER,
     677                 :            : 
     678                 :            :         /**
     679                 :            :          * Set Keep Alive Timeout of the controller.
     680                 :            :          */
     681                 :            :         NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT,
     682                 :            : 
     683                 :            :         /**
     684                 :            :          * Waiting for Set Keep Alive Timeout to be completed.
     685                 :            :          */
     686                 :            :         NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT,
     687                 :            : 
     688                 :            :         /**
     689                 :            :          * Get Identify I/O Command Set Specific Controller data structure.
     690                 :            :          */
     691                 :            :         NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC,
     692                 :            : 
     693                 :            :         /**
     694                 :            :          * Waiting for Identify I/O Command Set Specific Controller command to be completed.
     695                 :            :          */
     696                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC,
     697                 :            : 
     698                 :            :         /**
     699                 :            :          * Get Commands Supported and Effects log page for the Zoned Namespace Command Set.
     700                 :            :          */
     701                 :            :         NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG,
     702                 :            : 
     703                 :            :         /**
     704                 :            :          * Waiting for the Get Log Page command to be completed.
     705                 :            :          */
     706                 :            :         NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG,
     707                 :            : 
     708                 :            :         /**
     709                 :            :          * Set Number of Queues of the controller.
     710                 :            :          */
     711                 :            :         NVME_CTRLR_STATE_SET_NUM_QUEUES,
     712                 :            : 
     713                 :            :         /**
     714                 :            :          * Waiting for Set Num of Queues command to be completed.
     715                 :            :          */
     716                 :            :         NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES,
     717                 :            : 
     718                 :            :         /**
     719                 :            :          * Get active Namespace list of the controller.
     720                 :            :          */
     721                 :            :         NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS,
     722                 :            : 
     723                 :            :         /**
     724                 :            :          * Waiting for the Identify Active Namespace commands to be completed.
     725                 :            :          */
     726                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS,
     727                 :            : 
     728                 :            :         /**
     729                 :            :          * Get Identify Namespace Data structure for each NS.
     730                 :            :          */
     731                 :            :         NVME_CTRLR_STATE_IDENTIFY_NS,
     732                 :            : 
     733                 :            :         /**
     734                 :            :          * Waiting for the Identify Namespace commands to be completed.
     735                 :            :          */
     736                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS,
     737                 :            : 
     738                 :            :         /**
     739                 :            :          * Get Identify Namespace Identification Descriptors.
     740                 :            :          */
     741                 :            :         NVME_CTRLR_STATE_IDENTIFY_ID_DESCS,
     742                 :            : 
     743                 :            :         /**
     744                 :            :          * Get Identify I/O Command Set Specific Namespace data structure for each NS.
     745                 :            :          */
     746                 :            :         NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC,
     747                 :            : 
     748                 :            :         /**
     749                 :            :          * Waiting for the Identify I/O Command Set Specific Namespace commands to be completed.
     750                 :            :          */
     751                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC,
     752                 :            : 
     753                 :            :         /**
     754                 :            :          * Waiting for the Identify Namespace Identification
     755                 :            :          * Descriptors to be completed.
     756                 :            :          */
     757                 :            :         NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS,
     758                 :            : 
     759                 :            :         /**
     760                 :            :          * Set supported log pages of the controller.
     761                 :            :          */
     762                 :            :         NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES,
     763                 :            : 
     764                 :            :         /**
     765                 :            :          * Set supported log pages of INTEL controller.
     766                 :            :          */
     767                 :            :         NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES,
     768                 :            : 
     769                 :            :         /**
     770                 :            :          * Waiting for supported log pages of INTEL controller.
     771                 :            :          */
     772                 :            :         NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES,
     773                 :            : 
     774                 :            :         /**
     775                 :            :          * Set supported features of the controller.
     776                 :            :          */
     777                 :            :         NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES,
     778                 :            : 
     779                 :            :         /**
     780                 :            :          * Set Doorbell Buffer Config of the controller.
     781                 :            :          */
     782                 :            :         NVME_CTRLR_STATE_SET_DB_BUF_CFG,
     783                 :            : 
     784                 :            :         /**
     785                 :            :          * Waiting for Doorbell Buffer Config to be completed.
     786                 :            :          */
     787                 :            :         NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG,
     788                 :            : 
     789                 :            :         /**
     790                 :            :          * Set Host ID of the controller.
     791                 :            :          */
     792                 :            :         NVME_CTRLR_STATE_SET_HOST_ID,
     793                 :            : 
     794                 :            :         /**
     795                 :            :          * Waiting for Set Host ID to be completed.
     796                 :            :          */
     797                 :            :         NVME_CTRLR_STATE_WAIT_FOR_HOST_ID,
     798                 :            : 
     799                 :            :         /**
     800                 :            :          * Let transport layer do its part of initialization.
     801                 :            :          */
     802                 :            :         NVME_CTRLR_STATE_TRANSPORT_READY,
     803                 :            : 
     804                 :            :         /**
     805                 :            :          * Controller initialization has completed and the controller is ready.
     806                 :            :          */
     807                 :            :         NVME_CTRLR_STATE_READY,
     808                 :            : 
     809                 :            :         /**
     810                 :            :          * Controller initialization has an error.
     811                 :            :          */
     812                 :            :         NVME_CTRLR_STATE_ERROR,
     813                 :            : 
     814                 :            :         /**
     815                 :            :          * Admin qpair was disconnected, controller needs to be re-initialized
     816                 :            :          */
     817                 :            :         NVME_CTRLR_STATE_DISCONNECTED,
     818                 :            : };
     819                 :            : 
     820                 :            : #define NVME_TIMEOUT_INFINITE           0
     821                 :            : #define NVME_TIMEOUT_KEEP_EXISTING      UINT64_MAX
     822                 :            : 
     823                 :            : struct spdk_nvme_ctrlr_aer_completion_list {
     824                 :            :         struct spdk_nvme_cpl    cpl;
     825                 :            :         STAILQ_ENTRY(spdk_nvme_ctrlr_aer_completion_list) link;
     826                 :            : };
     827                 :            : 
     828                 :            : /*
     829                 :            :  * Used to track properties for all processes accessing the controller.
     830                 :            :  */
     831                 :            : struct spdk_nvme_ctrlr_process {
     832                 :            :         /** Whether it is the primary process  */
     833                 :            :         bool                                            is_primary;
     834                 :            : 
     835                 :            :         /** Process ID */
     836                 :            :         pid_t                                           pid;
     837                 :            : 
     838                 :            :         /** Active admin requests to be completed */
     839                 :            :         STAILQ_HEAD(, nvme_request)                     active_reqs;
     840                 :            : 
     841                 :            :         TAILQ_ENTRY(spdk_nvme_ctrlr_process)            tailq;
     842                 :            : 
     843                 :            :         /** Per process PCI device handle */
     844                 :            :         struct spdk_pci_device                          *devhandle;
     845                 :            : 
     846                 :            :         /** Reference to track the number of attachment to this controller. */
     847                 :            :         int                                             ref;
     848                 :            : 
     849                 :            :         /** Allocated IO qpairs */
     850                 :            :         TAILQ_HEAD(, spdk_nvme_qpair)                   allocated_io_qpairs;
     851                 :            : 
     852                 :            :         spdk_nvme_aer_cb                                aer_cb_fn;
     853                 :            :         void                                            *aer_cb_arg;
     854                 :            : 
     855                 :            :         /**
     856                 :            :          * A function pointer to timeout callback function
     857                 :            :          */
     858                 :            :         spdk_nvme_timeout_cb            timeout_cb_fn;
     859                 :            :         void                            *timeout_cb_arg;
     860                 :            :         /** separate timeout values for io vs. admin reqs */
     861                 :            :         uint64_t                        timeout_io_ticks;
     862                 :            :         uint64_t                        timeout_admin_ticks;
     863                 :            : 
     864                 :            :         /** List to publish AENs to all procs in multiprocess setup */
     865                 :            :         STAILQ_HEAD(, spdk_nvme_ctrlr_aer_completion_list)      async_events;
     866                 :            : };
     867                 :            : 
     868                 :            : struct nvme_register_completion {
     869                 :            :         struct spdk_nvme_cpl                    cpl;
     870                 :            :         uint64_t                                value;
     871                 :            :         spdk_nvme_reg_cb                        cb_fn;
     872                 :            :         void                                    *cb_ctx;
     873                 :            :         STAILQ_ENTRY(nvme_register_completion)  stailq;
     874                 :            :         pid_t                                   pid;
     875                 :            : };
     876                 :            : 
     877                 :            : struct spdk_nvme_ctrlr {
     878                 :            :         /* Hot data (accessed in I/O path) starts here. */
     879                 :            : 
     880                 :            :         /* Tree of namespaces */
     881                 :            :         RB_HEAD(nvme_ns_tree, spdk_nvme_ns)     ns;
     882                 :            : 
     883                 :            :         /* The number of active namespaces */
     884                 :            :         uint32_t                        active_ns_count;
     885                 :            : 
     886                 :            :         bool                            is_removed;
     887                 :            : 
     888                 :            :         bool                            is_resetting;
     889                 :            : 
     890                 :            :         bool                            is_failed;
     891                 :            : 
     892                 :            :         bool                            is_destructed;
     893                 :            : 
     894                 :            :         bool                            timeout_enabled;
     895                 :            : 
     896                 :            :         /* The application is preparing to reset the controller.  Transports
     897                 :            :          * can use this to skip unnecessary parts of the qpair deletion process
     898                 :            :          * for example, like the DELETE_SQ/CQ commands.
     899                 :            :          */
     900                 :            :         bool                            prepare_for_reset;
     901                 :            : 
     902                 :            :         bool                            is_disconnecting;
     903                 :            : 
     904                 :            :         bool                            needs_io_msg_update;
     905                 :            : 
     906                 :            :         uint16_t                        max_sges;
     907                 :            : 
     908                 :            :         uint16_t                        cntlid;
     909                 :            : 
     910                 :            :         /** Controller support flags */
     911                 :            :         uint64_t                        flags;
     912                 :            : 
     913                 :            :         /** NVMEoF in-capsule data size in bytes */
     914                 :            :         uint32_t                        ioccsz_bytes;
     915                 :            : 
     916                 :            :         /** NVMEoF in-capsule data offset in 16 byte units */
     917                 :            :         uint16_t                        icdoff;
     918                 :            : 
     919                 :            :         /* Cold data (not accessed in normal I/O path) is after this point. */
     920                 :            : 
     921                 :            :         struct spdk_nvme_transport_id   trid;
     922                 :            : 
     923                 :            :         union spdk_nvme_cap_register    cap;
     924                 :            :         union spdk_nvme_vs_register     vs;
     925                 :            : 
     926                 :            :         int                             state;
     927                 :            :         uint64_t                        state_timeout_tsc;
     928                 :            : 
     929                 :            :         uint64_t                        next_keep_alive_tick;
     930                 :            :         uint64_t                        keep_alive_interval_ticks;
     931                 :            : 
     932                 :            :         TAILQ_ENTRY(spdk_nvme_ctrlr)    tailq;
     933                 :            : 
     934                 :            :         /** All the log pages supported */
     935                 :            :         bool                            log_page_supported[256];
     936                 :            : 
     937                 :            :         /** All the features supported */
     938                 :            :         bool                            feature_supported[256];
     939                 :            : 
     940                 :            :         /** maximum i/o size in bytes */
     941                 :            :         uint32_t                        max_xfer_size;
     942                 :            : 
     943                 :            :         /** minimum page size supported by this controller in bytes */
     944                 :            :         uint32_t                        min_page_size;
     945                 :            : 
     946                 :            :         /** selected memory page size for this controller in bytes */
     947                 :            :         uint32_t                        page_size;
     948                 :            : 
     949                 :            :         uint32_t                        num_aers;
     950                 :            :         struct nvme_async_event_request aer[NVME_MAX_ASYNC_EVENTS];
     951                 :            : 
     952                 :            :         /** guards access to the controller itself, including admin queues */
     953                 :            :         pthread_mutex_t                 ctrlr_lock;
     954                 :            : 
     955                 :            :         struct spdk_nvme_qpair          *adminq;
     956                 :            : 
     957                 :            :         /** shadow doorbell buffer */
     958                 :            :         uint32_t                        *shadow_doorbell;
     959                 :            :         /** eventidx buffer */
     960                 :            :         uint32_t                        *eventidx;
     961                 :            : 
     962                 :            :         /**
     963                 :            :          * Identify Controller data.
     964                 :            :          */
     965                 :            :         struct spdk_nvme_ctrlr_data     cdata;
     966                 :            : 
     967                 :            :         /**
     968                 :            :          * Zoned Namespace Command Set Specific Identify Controller data.
     969                 :            :          */
     970                 :            :         struct spdk_nvme_zns_ctrlr_data *cdata_zns;
     971                 :            : 
     972                 :            :         struct spdk_bit_array           *free_io_qids;
     973                 :            :         TAILQ_HEAD(, spdk_nvme_qpair)   active_io_qpairs;
     974                 :            : 
     975                 :            :         struct spdk_nvme_ctrlr_opts     opts;
     976                 :            : 
     977                 :            :         uint64_t                        quirks;
     978                 :            : 
     979                 :            :         /* Extra sleep time during controller initialization */
     980                 :            :         uint64_t                        sleep_timeout_tsc;
     981                 :            : 
     982                 :            :         /** Track all the processes manage this controller */
     983                 :            :         TAILQ_HEAD(, spdk_nvme_ctrlr_process)   active_procs;
     984                 :            : 
     985                 :            : 
     986                 :            :         STAILQ_HEAD(, nvme_request)     queued_aborts;
     987                 :            :         uint32_t                        outstanding_aborts;
     988                 :            : 
     989                 :            :         uint32_t                        lock_depth;
     990                 :            : 
     991                 :            :         /* CB to notify the user when the ctrlr is removed/failed. */
     992                 :            :         spdk_nvme_remove_cb                     remove_cb;
     993                 :            :         void                                    *cb_ctx;
     994                 :            : 
     995                 :            :         struct spdk_nvme_qpair          *external_io_msgs_qpair;
     996                 :            :         pthread_mutex_t                 external_io_msgs_lock;
     997                 :            :         struct spdk_ring                *external_io_msgs;
     998                 :            : 
     999                 :            :         STAILQ_HEAD(, nvme_io_msg_producer) io_producers;
    1000                 :            : 
    1001                 :            :         struct spdk_nvme_ana_page               *ana_log_page;
    1002                 :            :         struct spdk_nvme_ana_group_descriptor   *copied_ana_desc;
    1003                 :            :         uint32_t                                ana_log_page_size;
    1004                 :            : 
    1005                 :            :         /* scratchpad pointer that can be used to send data between two NVME_CTRLR_STATEs */
    1006                 :            :         void                            *tmp_ptr;
    1007                 :            : 
    1008                 :            :         /* maximum zone append size in bytes */
    1009                 :            :         uint32_t                        max_zone_append_size;
    1010                 :            : 
    1011                 :            :         /* PMR size in bytes */
    1012                 :            :         uint64_t                        pmr_size;
    1013                 :            : 
    1014                 :            :         /* Boot Partition Info */
    1015                 :            :         enum nvme_bp_write_state        bp_ws;
    1016                 :            :         uint32_t                        bpid;
    1017                 :            :         spdk_nvme_cmd_cb                bp_write_cb_fn;
    1018                 :            :         void                            *bp_write_cb_arg;
    1019                 :            : 
    1020                 :            :         /* Firmware Download */
    1021                 :            :         void                            *fw_payload;
    1022                 :            :         unsigned int                    fw_size_remaining;
    1023                 :            :         unsigned int                    fw_offset;
    1024                 :            :         unsigned int                    fw_transfer_size;
    1025                 :            : 
    1026                 :            :         /* Completed register operations */
    1027                 :            :         STAILQ_HEAD(, nvme_register_completion) register_operations;
    1028                 :            : 
    1029                 :            :         union spdk_nvme_cc_register             process_init_cc;
    1030                 :            : };
    1031                 :            : 
    1032                 :            : struct spdk_nvme_probe_ctx {
    1033                 :            :         struct spdk_nvme_transport_id           trid;
    1034                 :            :         void                                    *cb_ctx;
    1035                 :            :         spdk_nvme_probe_cb                      probe_cb;
    1036                 :            :         spdk_nvme_attach_cb                     attach_cb;
    1037                 :            :         spdk_nvme_remove_cb                     remove_cb;
    1038                 :            :         TAILQ_HEAD(, spdk_nvme_ctrlr)           init_ctrlrs;
    1039                 :            : };
    1040                 :            : 
    1041                 :            : typedef void (*nvme_ctrlr_detach_cb)(struct spdk_nvme_ctrlr *ctrlr);
    1042                 :            : 
    1043                 :            : enum nvme_ctrlr_detach_state {
    1044                 :            :         NVME_CTRLR_DETACH_SET_CC,
    1045                 :            :         NVME_CTRLR_DETACH_CHECK_CSTS,
    1046                 :            :         NVME_CTRLR_DETACH_GET_CSTS,
    1047                 :            :         NVME_CTRLR_DETACH_GET_CSTS_DONE,
    1048                 :            : };
    1049                 :            : 
    1050                 :            : struct nvme_ctrlr_detach_ctx {
    1051                 :            :         struct spdk_nvme_ctrlr                  *ctrlr;
    1052                 :            :         nvme_ctrlr_detach_cb                    cb_fn;
    1053                 :            :         uint64_t                                shutdown_start_tsc;
    1054                 :            :         uint32_t                                shutdown_timeout_ms;
    1055                 :            :         bool                                    shutdown_complete;
    1056                 :            :         enum nvme_ctrlr_detach_state            state;
    1057                 :            :         union spdk_nvme_csts_register           csts;
    1058                 :            :         TAILQ_ENTRY(nvme_ctrlr_detach_ctx)      link;
    1059                 :            : };
    1060                 :            : 
    1061                 :            : struct spdk_nvme_detach_ctx {
    1062                 :            :         TAILQ_HEAD(, nvme_ctrlr_detach_ctx)     head;
    1063                 :            : };
    1064                 :            : 
    1065                 :            : struct nvme_driver {
    1066                 :            :         pthread_mutex_t                 lock;
    1067                 :            : 
    1068                 :            :         /** Multi-process shared attached controller list */
    1069                 :            :         TAILQ_HEAD(, spdk_nvme_ctrlr)   shared_attached_ctrlrs;
    1070                 :            : 
    1071                 :            :         bool                            initialized;
    1072                 :            :         struct spdk_uuid                default_extended_host_id;
    1073                 :            : 
    1074                 :            :         /** netlink socket fd for hotplug messages */
    1075                 :            :         int                             hotplug_fd;
    1076                 :            : };
    1077                 :            : 
    1078                 :            : #define nvme_ns_cmd_get_ext_io_opt(opts, field, defval) \
    1079                 :            :        ((opts) != NULL && offsetof(struct spdk_nvme_ns_cmd_ext_io_opts, field) + \
    1080                 :            :         sizeof((opts)->field) <= (opts)->size ? (opts)->field : (defval))
    1081                 :            : 
    1082                 :            : extern struct nvme_driver *g_spdk_nvme_driver;
    1083                 :            : 
    1084                 :            : int nvme_driver_init(void);
    1085                 :            : 
    1086                 :            : #define nvme_delay              usleep
    1087                 :            : 
    1088                 :            : static inline bool
    1089                 : 4045802830 : nvme_qpair_is_admin_queue(struct spdk_nvme_qpair *qpair)
    1090                 :            : {
    1091                 : 4045802830 :         return qpair->id == 0;
    1092                 :            : }
    1093                 :            : 
    1094                 :            : static inline bool
    1095                 :            : nvme_qpair_is_io_queue(struct spdk_nvme_qpair *qpair)
    1096                 :            : {
    1097                 :            :         return qpair->id != 0;
    1098                 :            : }
    1099                 :            : 
    1100                 :            : static inline int
    1101                 :  144943944 : nvme_robust_mutex_lock(pthread_mutex_t *mtx)
    1102                 :            : {
    1103         [ -  + ]:  144943944 :         int rc = pthread_mutex_lock(mtx);
    1104                 :            : 
    1105                 :            : #ifndef __FreeBSD__
    1106         [ -  + ]:  144943944 :         if (rc == EOWNERDEAD) {
    1107         [ #  # ]:          0 :                 rc = pthread_mutex_consistent(mtx);
    1108                 :            :         }
    1109                 :            : #endif
    1110                 :            : 
    1111                 :  144943944 :         return rc;
    1112                 :            : }
    1113                 :            : 
    1114                 :            : static inline int
    1115                 :  144304777 : nvme_ctrlr_lock(struct spdk_nvme_ctrlr *ctrlr)
    1116                 :            : {
    1117                 :            :         int rc;
    1118                 :            : 
    1119                 :  144304777 :         rc = nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
    1120                 :  144304777 :         ctrlr->lock_depth++;
    1121                 :  144304777 :         return rc;
    1122                 :            : }
    1123                 :            : 
    1124                 :            : static inline int
    1125                 :  144943932 : nvme_robust_mutex_unlock(pthread_mutex_t *mtx)
    1126                 :            : {
    1127         [ -  + ]:  144943932 :         return pthread_mutex_unlock(mtx);
    1128                 :            : }
    1129                 :            : 
    1130                 :            : static inline int
    1131                 :  144304765 : nvme_ctrlr_unlock(struct spdk_nvme_ctrlr *ctrlr)
    1132                 :            : {
    1133                 :  144304765 :         ctrlr->lock_depth--;
    1134                 :  144304765 :         return nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
    1135                 :            : }
    1136                 :            : 
    1137                 :            : /* Poll group management functions. */
    1138                 :            : int nvme_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair);
    1139                 :            : int nvme_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair);
    1140                 :            : 
    1141                 :            : /* Admin functions */
    1142                 :            : int     nvme_ctrlr_cmd_identify(struct spdk_nvme_ctrlr *ctrlr,
    1143                 :            :                                 uint8_t cns, uint16_t cntid, uint32_t nsid,
    1144                 :            :                                 uint8_t csi, void *payload, size_t payload_size,
    1145                 :            :                                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1146                 :            : int     nvme_ctrlr_cmd_set_num_queues(struct spdk_nvme_ctrlr *ctrlr,
    1147                 :            :                                       uint32_t num_queues, spdk_nvme_cmd_cb cb_fn,
    1148                 :            :                                       void *cb_arg);
    1149                 :            : int     nvme_ctrlr_cmd_get_num_queues(struct spdk_nvme_ctrlr *ctrlr,
    1150                 :            :                                       spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1151                 :            : int     nvme_ctrlr_cmd_set_async_event_config(struct spdk_nvme_ctrlr *ctrlr,
    1152                 :            :                 union spdk_nvme_feat_async_event_configuration config,
    1153                 :            :                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1154                 :            : int     nvme_ctrlr_cmd_set_host_id(struct spdk_nvme_ctrlr *ctrlr, void *host_id, uint32_t host_id_size,
    1155                 :            :                                    spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1156                 :            : int     nvme_ctrlr_cmd_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    1157                 :            :                                  struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1158                 :            : int     nvme_ctrlr_cmd_detach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    1159                 :            :                                  struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1160                 :            : int     nvme_ctrlr_cmd_create_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns_data *payload,
    1161                 :            :                                  spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1162                 :            : int     nvme_ctrlr_cmd_doorbell_buffer_config(struct spdk_nvme_ctrlr *ctrlr,
    1163                 :            :                 uint64_t prp1, uint64_t prp2,
    1164                 :            :                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1165                 :            : int     nvme_ctrlr_cmd_delete_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, spdk_nvme_cmd_cb cb_fn,
    1166                 :            :                                  void *cb_arg);
    1167                 :            : int     nvme_ctrlr_cmd_format(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    1168                 :            :                               struct spdk_nvme_format *format, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1169                 :            : int     nvme_ctrlr_cmd_fw_commit(struct spdk_nvme_ctrlr *ctrlr,
    1170                 :            :                                  const struct spdk_nvme_fw_commit *fw_commit,
    1171                 :            :                                  spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1172                 :            : int     nvme_ctrlr_cmd_fw_image_download(struct spdk_nvme_ctrlr *ctrlr,
    1173                 :            :                 uint32_t size, uint32_t offset, void *payload,
    1174                 :            :                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1175                 :            : int     nvme_ctrlr_cmd_sanitize(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
    1176                 :            :                                 struct spdk_nvme_sanitize *sanitize, uint32_t cdw11,
    1177                 :            :                                 spdk_nvme_cmd_cb cb_fn, void *cb_arg);
    1178                 :            : void    nvme_completion_poll_cb(void *arg, const struct spdk_nvme_cpl *cpl);
    1179                 :            : int     nvme_wait_for_completion(struct spdk_nvme_qpair *qpair,
    1180                 :            :                                  struct nvme_completion_poll_status *status);
    1181                 :            : int     nvme_wait_for_completion_robust_lock(struct spdk_nvme_qpair *qpair,
    1182                 :            :                 struct nvme_completion_poll_status *status,
    1183                 :            :                 pthread_mutex_t *robust_mutex);
    1184                 :            : int     nvme_wait_for_completion_timeout(struct spdk_nvme_qpair *qpair,
    1185                 :            :                 struct nvme_completion_poll_status *status,
    1186                 :            :                 uint64_t timeout_in_usecs);
    1187                 :            : int     nvme_wait_for_completion_robust_lock_timeout(struct spdk_nvme_qpair *qpair,
    1188                 :            :                 struct nvme_completion_poll_status *status,
    1189                 :            :                 pthread_mutex_t *robust_mutex,
    1190                 :            :                 uint64_t timeout_in_usecs);
    1191                 :            : int     nvme_wait_for_completion_robust_lock_timeout_poll(struct spdk_nvme_qpair *qpair,
    1192                 :            :                 struct nvme_completion_poll_status *status,
    1193                 :            :                 pthread_mutex_t *robust_mutex);
    1194                 :            : 
    1195                 :            : struct spdk_nvme_ctrlr_process *nvme_ctrlr_get_process(struct spdk_nvme_ctrlr *ctrlr,
    1196                 :            :                 pid_t pid);
    1197                 :            : struct spdk_nvme_ctrlr_process *nvme_ctrlr_get_current_process(struct spdk_nvme_ctrlr *ctrlr);
    1198                 :            : int     nvme_ctrlr_add_process(struct spdk_nvme_ctrlr *ctrlr, void *devhandle);
    1199                 :            : void    nvme_ctrlr_free_processes(struct spdk_nvme_ctrlr *ctrlr);
    1200                 :            : struct spdk_pci_device *nvme_ctrlr_proc_get_devhandle(struct spdk_nvme_ctrlr *ctrlr);
    1201                 :            : 
    1202                 :            : int     nvme_ctrlr_probe(const struct spdk_nvme_transport_id *trid,
    1203                 :            :                          struct spdk_nvme_probe_ctx *probe_ctx, void *devhandle);
    1204                 :            : 
    1205                 :            : int     nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr);
    1206                 :            : void    nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr *ctrlr);
    1207                 :            : void    nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr);
    1208                 :            : void    nvme_ctrlr_destruct_async(struct spdk_nvme_ctrlr *ctrlr,
    1209                 :            :                                   struct nvme_ctrlr_detach_ctx *ctx);
    1210                 :            : int     nvme_ctrlr_destruct_poll_async(struct spdk_nvme_ctrlr *ctrlr,
    1211                 :            :                                        struct nvme_ctrlr_detach_ctx *ctx);
    1212                 :            : void    nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove);
    1213                 :            : int     nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr);
    1214                 :            : void    nvme_ctrlr_disable(struct spdk_nvme_ctrlr *ctrlr);
    1215                 :            : int     nvme_ctrlr_disable_poll(struct spdk_nvme_ctrlr *ctrlr);
    1216                 :            : void    nvme_ctrlr_connected(struct spdk_nvme_probe_ctx *probe_ctx,
    1217                 :            :                              struct spdk_nvme_ctrlr *ctrlr);
    1218                 :            : 
    1219                 :            : int     nvme_ctrlr_submit_admin_request(struct spdk_nvme_ctrlr *ctrlr,
    1220                 :            :                                         struct nvme_request *req);
    1221                 :            : int     nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cap_register *cap);
    1222                 :            : int     nvme_ctrlr_get_vs(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_vs_register *vs);
    1223                 :            : int     nvme_ctrlr_get_cmbsz(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cmbsz_register *cmbsz);
    1224                 :            : int     nvme_ctrlr_get_pmrcap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_pmrcap_register *pmrcap);
    1225                 :            : int     nvme_ctrlr_get_bpinfo(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bpinfo_register *bpinfo);
    1226                 :            : int     nvme_ctrlr_set_bprsel(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bprsel_register *bprsel);
    1227                 :            : int     nvme_ctrlr_set_bpmbl(struct spdk_nvme_ctrlr *ctrlr, uint64_t bpmbl_value);
    1228                 :            : bool    nvme_ctrlr_multi_iocs_enabled(struct spdk_nvme_ctrlr *ctrlr);
    1229                 :            : void    nvme_ctrlr_process_async_event(struct spdk_nvme_ctrlr *ctrlr,
    1230                 :            :                                        const struct spdk_nvme_cpl *cpl);
    1231                 :            : void nvme_ctrlr_disconnect_qpair(struct spdk_nvme_qpair *qpair);
    1232                 :            : void nvme_ctrlr_complete_queued_async_events(struct spdk_nvme_ctrlr *ctrlr);
    1233                 :            : void nvme_ctrlr_abort_queued_aborts(struct spdk_nvme_ctrlr *ctrlr);
    1234                 :            : int nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id,
    1235                 :            :                     struct spdk_nvme_ctrlr *ctrlr,
    1236                 :            :                     enum spdk_nvme_qprio qprio,
    1237                 :            :                     uint32_t num_requests, bool async);
    1238                 :            : void    nvme_qpair_deinit(struct spdk_nvme_qpair *qpair);
    1239                 :            : void    nvme_qpair_complete_error_reqs(struct spdk_nvme_qpair *qpair);
    1240                 :            : int     nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair,
    1241                 :            :                                   struct nvme_request *req);
    1242                 :            : void    nvme_qpair_abort_all_queued_reqs(struct spdk_nvme_qpair *qpair);
    1243                 :            : uint32_t nvme_qpair_abort_queued_reqs_with_cbarg(struct spdk_nvme_qpair *qpair, void *cmd_cb_arg);
    1244                 :            : void    nvme_qpair_abort_queued_reqs(struct spdk_nvme_qpair *qpair);
    1245                 :            : void    nvme_qpair_resubmit_requests(struct spdk_nvme_qpair *qpair, uint32_t num_requests);
    1246                 :            : int     nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr);
    1247                 :            : void    nvme_ns_set_identify_data(struct spdk_nvme_ns *ns);
    1248                 :            : void    nvme_ns_set_id_desc_list_data(struct spdk_nvme_ns *ns);
    1249                 :            : void    nvme_ns_free_zns_specific_data(struct spdk_nvme_ns *ns);
    1250                 :            : void    nvme_ns_free_iocs_specific_data(struct spdk_nvme_ns *ns);
    1251                 :            : bool    nvme_ns_has_supported_iocs_specific_data(struct spdk_nvme_ns *ns);
    1252                 :            : int     nvme_ns_construct(struct spdk_nvme_ns *ns, uint32_t id,
    1253                 :            :                           struct spdk_nvme_ctrlr *ctrlr);
    1254                 :            : void    nvme_ns_destruct(struct spdk_nvme_ns *ns);
    1255                 :            : int     nvme_ns_cmd_zone_append_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
    1256                 :            :                                         void *buffer, void *metadata, uint64_t zslba,
    1257                 :            :                                         uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,
    1258                 :            :                                         uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag);
    1259                 :            : int nvme_ns_cmd_zone_appendv_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
    1260                 :            :                                      uint64_t zslba, uint32_t lba_count,
    1261                 :            :                                      spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags,
    1262                 :            :                                      spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
    1263                 :            :                                      spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata,
    1264                 :            :                                      uint16_t apptag_mask, uint16_t apptag);
    1265                 :            : 
    1266                 :            : int     nvme_fabric_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value);
    1267                 :            : int     nvme_fabric_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value);
    1268                 :            : int     nvme_fabric_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value);
    1269                 :            : int     nvme_fabric_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value);
    1270                 :            : int     nvme_fabric_ctrlr_set_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1271                 :            :                 uint32_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1272                 :            : int     nvme_fabric_ctrlr_set_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1273                 :            :                 uint64_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1274                 :            : int     nvme_fabric_ctrlr_get_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1275                 :            :                 spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1276                 :            : int     nvme_fabric_ctrlr_get_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1277                 :            :                 spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1278                 :            : int     nvme_fabric_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, bool direct_connect);
    1279                 :            : int     nvme_fabric_ctrlr_discover(struct spdk_nvme_ctrlr *ctrlr,
    1280                 :            :                                    struct spdk_nvme_probe_ctx *probe_ctx);
    1281                 :            : int     nvme_fabric_qpair_connect(struct spdk_nvme_qpair *qpair, uint32_t num_entries);
    1282                 :            : int     nvme_fabric_qpair_connect_async(struct spdk_nvme_qpair *qpair, uint32_t num_entries);
    1283                 :            : int     nvme_fabric_qpair_connect_poll(struct spdk_nvme_qpair *qpair);
    1284                 :            : 
    1285                 :            : typedef int (*spdk_nvme_parse_ana_log_page_cb)(
    1286                 :            :         const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg);
    1287                 :            : int     nvme_ctrlr_parse_ana_log_page(struct spdk_nvme_ctrlr *ctrlr,
    1288                 :            :                                       spdk_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg);
    1289                 :            : 
    1290                 :            : #define NVME_INIT_REQUEST(req, _cb_fn, _cb_arg, _payload, _payload_size, _md_size)      \
    1291                 :            :         do {                                            \
    1292                 :            :                 req->cb_fn = _cb_fn;                 \
    1293                 :            :                 req->cb_arg = _cb_arg;                       \
    1294                 :            :                 req->payload = _payload;             \
    1295                 :            :                 req->payload_size = _payload_size;   \
    1296                 :            :                 req->md_size = _md_size;             \
    1297                 :            :                 req->pid = g_spdk_nvme_pid;          \
    1298                 :            :                 req->submit_tick = 0;                        \
    1299                 :            :                 req->accel_sequence = NULL;          \
    1300                 :            :         } while (0);
    1301                 :            : 
    1302                 :            : static inline struct nvme_request *
    1303                 :   90570120 : nvme_allocate_request(struct spdk_nvme_qpair *qpair,
    1304                 :            :                       const struct nvme_payload *payload, uint32_t payload_size, uint32_t md_size,
    1305                 :            :                       spdk_nvme_cmd_cb cb_fn, void *cb_arg)
    1306                 :            : {
    1307                 :            :         struct nvme_request *req;
    1308                 :            : 
    1309                 :   90570120 :         req = STAILQ_FIRST(&qpair->free_req);
    1310         [ +  + ]:   90570120 :         if (req == NULL) {
    1311                 :     651560 :                 return req;
    1312                 :            :         }
    1313                 :            : 
    1314         [ +  + ]:   89918560 :         STAILQ_REMOVE_HEAD(&qpair->free_req, stailq);
    1315                 :   89918560 :         qpair->num_outstanding_reqs++;
    1316                 :            : 
    1317                 :            :         /*
    1318                 :            :          * Only memset/zero fields that need it.  All other fields
    1319                 :            :          *  will be initialized appropriately either later in this
    1320                 :            :          *  function, or before they are needed later in the
    1321                 :            :          *  submission patch.  For example, the children
    1322                 :            :          *  TAILQ_ENTRY and following members are
    1323                 :            :          *  only used as part of I/O splitting so we avoid
    1324                 :            :          *  memsetting them until it is actually needed.
    1325                 :            :          *  They will be initialized in nvme_request_add_child()
    1326                 :            :          *  if the request is split.
    1327                 :            :          */
    1328         [ -  + ]:   89918560 :         memset(req, 0, offsetof(struct nvme_request, payload_size));
    1329                 :            : 
    1330                 :   89918560 :         NVME_INIT_REQUEST(req, cb_fn, cb_arg, *payload, payload_size, md_size);
    1331                 :            : 
    1332                 :   89918560 :         return req;
    1333                 :            : }
    1334                 :            : 
    1335                 :            : static inline struct nvme_request *
    1336                 :    4918473 : nvme_allocate_request_contig(struct spdk_nvme_qpair *qpair,
    1337                 :            :                              void *buffer, uint32_t payload_size,
    1338                 :            :                              spdk_nvme_cmd_cb cb_fn, void *cb_arg)
    1339                 :            : {
    1340                 :     527594 :         struct nvme_payload payload;
    1341                 :            : 
    1342                 :    4918473 :         payload = NVME_PAYLOAD_CONTIG(buffer, NULL);
    1343                 :            : 
    1344                 :    4918473 :         return nvme_allocate_request(qpair, &payload, payload_size, 0, cb_fn, cb_arg);
    1345                 :            : }
    1346                 :            : 
    1347                 :            : static inline struct nvme_request *
    1348                 :    1774319 : nvme_allocate_request_null(struct spdk_nvme_qpair *qpair, spdk_nvme_cmd_cb cb_fn, void *cb_arg)
    1349                 :            : {
    1350                 :    1774319 :         return nvme_allocate_request_contig(qpair, NULL, 0, cb_fn, cb_arg);
    1351                 :            : }
    1352                 :            : 
    1353                 :            : struct nvme_request *nvme_allocate_request_user_copy(struct spdk_nvme_qpair *qpair,
    1354                 :            :                 void *buffer, uint32_t payload_size,
    1355                 :            :                 spdk_nvme_cmd_cb cb_fn, void *cb_arg, bool host_to_controller);
    1356                 :            : 
    1357                 :            : static inline void
    1358                 :   89922519 : _nvme_free_request(struct nvme_request *req, struct spdk_nvme_qpair *qpair)
    1359                 :            : {
    1360         [ -  + ]:   89922519 :         assert(req != NULL);
    1361         [ -  + ]:   89922519 :         assert(req->num_children == 0);
    1362         [ -  + ]:   89922519 :         assert(qpair != NULL);
    1363                 :            : 
    1364                 :            :         /* The reserved_req does not go in the free_req STAILQ - it is
    1365                 :            :          * saved only for use with a FABRICS/CONNECT command.
    1366                 :            :          */
    1367         [ +  + ]:   89922519 :         if (spdk_likely(qpair->reserved_req != req)) {
    1368         [ +  + ]:   89918196 :                 STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq);
    1369                 :            : 
    1370         [ -  + ]:   89918196 :                 assert(qpair->num_outstanding_reqs > 0);
    1371                 :   89918196 :                 qpair->num_outstanding_reqs--;
    1372                 :            :         }
    1373                 :   89922519 : }
    1374                 :            : 
    1375                 :            : static inline void
    1376                 :   25524963 : nvme_free_request(struct nvme_request *req)
    1377                 :            : {
    1378                 :   25524963 :         _nvme_free_request(req, req->qpair);
    1379                 :   25524963 : }
    1380                 :            : 
    1381                 :            : static inline void
    1382                 :   64397554 : nvme_complete_request(spdk_nvme_cmd_cb cb_fn, void *cb_arg, struct spdk_nvme_qpair *qpair,
    1383                 :            :                       struct nvme_request *req, struct spdk_nvme_cpl *cpl)
    1384                 :            : {
    1385                 :   20687724 :         struct spdk_nvme_cpl            err_cpl;
    1386                 :            :         struct nvme_error_cmd           *cmd;
    1387                 :            : 
    1388         [ -  + ]:   64397554 :         if (spdk_unlikely(req->accel_sequence != NULL)) {
    1389                 :          0 :                 struct spdk_nvme_poll_group *pg = qpair->poll_group->group;
    1390                 :            : 
    1391                 :            :                 /* Transports are required to execuete the sequence and clear req->accel_sequence.
    1392                 :            :                  * If it's left non-NULL it must mean the request is failed. */
    1393   [ #  #  #  # ]:          0 :                 assert(spdk_nvme_cpl_is_error(cpl));
    1394                 :          0 :                 pg->accel_fn_table.abort_sequence(req->accel_sequence);
    1395                 :          0 :                 req->accel_sequence = NULL;
    1396                 :            :         }
    1397                 :            : 
    1398                 :            :         /* error injection at completion path,
    1399                 :            :          * only inject for successful completed commands
    1400                 :            :          */
    1401   [ +  +  +  +  :   64397554 :         if (spdk_unlikely(!TAILQ_EMPTY(&qpair->err_cmd_head) &&
             +  -  +  + ]
    1402                 :            :                           !spdk_nvme_cpl_is_error(cpl))) {
    1403         [ +  + ]:     926482 :                 TAILQ_FOREACH(cmd, &qpair->err_cmd_head, link) {
    1404                 :            : 
    1405   [ +  +  +  + ]:     463246 :                         if (cmd->do_not_submit) {
    1406                 :     463220 :                                 continue;
    1407                 :            :                         }
    1408                 :            : 
    1409   [ +  -  +  + ]:         26 :                         if ((cmd->opc == req->cmd.opc) && cmd->err_count) {
    1410                 :            : 
    1411                 :         10 :                                 err_cpl = *cpl;
    1412                 :         10 :                                 err_cpl.status.sct = cmd->status.sct;
    1413                 :         10 :                                 err_cpl.status.sc = cmd->status.sc;
    1414                 :            : 
    1415                 :         10 :                                 cpl = &err_cpl;
    1416                 :         10 :                                 cmd->err_count--;
    1417                 :         10 :                                 break;
    1418                 :            :                         }
    1419                 :            :                 }
    1420                 :            :         }
    1421                 :            : 
    1422                 :            :         /* For PCIe completions, we want to avoid touching the req itself to avoid
    1423                 :            :          * dependencies on loading those cachelines. So call the internal helper
    1424                 :            :          * function instead using the qpair that was passed by the caller, instead
    1425                 :            :          * of getting it from the req.
    1426                 :            :          */
    1427                 :   64397554 :         _nvme_free_request(req, qpair);
    1428                 :            : 
    1429         [ +  + ]:   64397554 :         if (spdk_likely(cb_fn)) {
    1430                 :   64397548 :                 cb_fn(cb_arg, cpl);
    1431                 :            :         }
    1432                 :   64397554 : }
    1433                 :            : 
    1434                 :            : static inline void
    1435                 :     264060 : nvme_cleanup_user_req(struct nvme_request *req)
    1436                 :            : {
    1437   [ +  +  +  - ]:     264060 :         if (req->user_buffer && req->payload_size) {
    1438                 :     259099 :                 spdk_free(req->payload.contig_or_cb_arg);
    1439                 :     259099 :                 req->user_buffer = NULL;
    1440                 :            :         }
    1441                 :            : 
    1442                 :     264060 :         req->user_cb_arg = NULL;
    1443                 :     264060 :         req->user_cb_fn = NULL;
    1444                 :     264060 : }
    1445                 :            : 
    1446                 :            : static inline void
    1447                 :      62205 : nvme_qpair_set_state(struct spdk_nvme_qpair *qpair, enum nvme_qpair_state state)
    1448                 :            : {
    1449                 :      62205 :         qpair->state = state;
    1450         [ +  + ]:      62205 :         if (state == NVME_QPAIR_ENABLED) {
    1451                 :       5759 :                 qpair->is_new_qpair = false;
    1452                 :            :         }
    1453                 :      62205 : }
    1454                 :            : 
    1455                 :            : static inline enum nvme_qpair_state
    1456                 : 3458103304 : nvme_qpair_get_state(struct spdk_nvme_qpair *qpair) {
    1457                 : 3458103304 :         return qpair->state;
    1458                 :            : }
    1459                 :            : 
    1460                 :            : static inline void
    1461                 :   28680601 : nvme_request_remove_child(struct nvme_request *parent, struct nvme_request *child)
    1462                 :            : {
    1463         [ -  + ]:   28680601 :         assert(parent != NULL);
    1464         [ -  + ]:   28680601 :         assert(child != NULL);
    1465         [ -  + ]:   28680601 :         assert(child->parent == parent);
    1466         [ -  + ]:   28680601 :         assert(parent->num_children != 0);
    1467                 :            : 
    1468                 :   28680601 :         parent->num_children--;
    1469                 :   28680601 :         child->parent = NULL;
    1470         [ +  + ]:   28680601 :         TAILQ_REMOVE(&parent->children, child, child_tailq);
    1471                 :   28680601 : }
    1472                 :            : 
    1473                 :            : static inline void
    1474                 :    2950121 : nvme_cb_complete_child(void *child_arg, const struct spdk_nvme_cpl *cpl)
    1475                 :            : {
    1476                 :    2950121 :         struct nvme_request *child = child_arg;
    1477                 :    2950121 :         struct nvme_request *parent = child->parent;
    1478                 :            : 
    1479                 :    2950121 :         nvme_request_remove_child(parent, child);
    1480                 :            : 
    1481   [ +  -  -  + ]:    2950121 :         if (spdk_nvme_cpl_is_error(cpl)) {
    1482                 :          0 :                 memcpy(&parent->parent_status, cpl, sizeof(*cpl));
    1483                 :            :         }
    1484                 :            : 
    1485         [ +  + ]:    2950121 :         if (parent->num_children == 0) {
    1486                 :    1227496 :                 nvme_complete_request(parent->cb_fn, parent->cb_arg, parent->qpair,
    1487                 :            :                                       parent, &parent->parent_status);
    1488                 :            :         }
    1489                 :    2950121 : }
    1490                 :            : 
    1491                 :            : static inline void
    1492                 :   28423258 : nvme_request_add_child(struct nvme_request *parent, struct nvme_request *child)
    1493                 :            : {
    1494         [ -  + ]:   28423258 :         assert(parent->num_children != UINT16_MAX);
    1495                 :            : 
    1496         [ +  + ]:   28423258 :         if (parent->num_children == 0) {
    1497                 :            :                 /*
    1498                 :            :                  * Defer initialization of the children TAILQ since it falls
    1499                 :            :                  *  on a separate cacheline.  This ensures we do not touch this
    1500                 :            :                  *  cacheline except on request splitting cases, which are
    1501                 :            :                  *  relatively rare.
    1502                 :            :                  */
    1503                 :    1278161 :                 TAILQ_INIT(&parent->children);
    1504                 :    1278161 :                 parent->parent = NULL;
    1505         [ -  + ]:    1278161 :                 memset(&parent->parent_status, 0, sizeof(struct spdk_nvme_cpl));
    1506                 :            :         }
    1507                 :            : 
    1508                 :   28423258 :         parent->num_children++;
    1509                 :   28423258 :         TAILQ_INSERT_TAIL(&parent->children, child, child_tailq);
    1510                 :   28423258 :         child->parent = parent;
    1511                 :   28423258 :         child->cb_fn = nvme_cb_complete_child;
    1512                 :   28423258 :         child->cb_arg = child;
    1513                 :   28423258 : }
    1514                 :            : 
    1515                 :            : static inline void
    1516                 :   25523886 : nvme_request_free_children(struct nvme_request *req)
    1517                 :            : {
    1518                 :            :         struct nvme_request *child, *tmp;
    1519                 :            : 
    1520         [ +  + ]:   25523886 :         if (req->num_children == 0) {
    1521                 :   25473233 :                 return;
    1522                 :            :         }
    1523                 :            : 
    1524                 :            :         /* free all child nvme_request */
    1525         [ +  + ]:   25523760 :         TAILQ_FOREACH_SAFE(child, &req->children, child_tailq, tmp) {
    1526                 :   25473107 :                 nvme_request_remove_child(req, child);
    1527                 :   25473107 :                 nvme_request_free_children(child);
    1528                 :   25473107 :                 nvme_free_request(child);
    1529                 :            :         }
    1530                 :            : }
    1531                 :            : 
    1532                 :            : int     nvme_request_check_timeout(struct nvme_request *req, uint16_t cid,
    1533                 :            :                                    struct spdk_nvme_ctrlr_process *active_proc, uint64_t now_tick);
    1534                 :            : uint64_t nvme_get_quirks(const struct spdk_pci_id *id);
    1535                 :            : 
    1536                 :            : int     nvme_robust_mutex_init_shared(pthread_mutex_t *mtx);
    1537                 :            : int     nvme_robust_mutex_init_recursive_shared(pthread_mutex_t *mtx);
    1538                 :            : 
    1539                 :            : bool    nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl);
    1540                 :            : 
    1541                 :            : struct spdk_nvme_ctrlr *nvme_get_ctrlr_by_trid_unsafe(
    1542                 :            :         const struct spdk_nvme_transport_id *trid);
    1543                 :            : 
    1544                 :            : const struct spdk_nvme_transport *nvme_get_transport(const char *transport_name);
    1545                 :            : const struct spdk_nvme_transport *nvme_get_first_transport(void);
    1546                 :            : const struct spdk_nvme_transport *nvme_get_next_transport(const struct spdk_nvme_transport
    1547                 :            :                 *transport);
    1548                 :            : void  nvme_ctrlr_update_namespaces(struct spdk_nvme_ctrlr *ctrlr);
    1549                 :            : 
    1550                 :            : /* Transport specific functions */
    1551                 :            : struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
    1552                 :            :                 const struct spdk_nvme_ctrlr_opts *opts,
    1553                 :            :                 void *devhandle);
    1554                 :            : int nvme_transport_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr);
    1555                 :            : int nvme_transport_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, bool direct_connect);
    1556                 :            : int nvme_transport_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr);
    1557                 :            : int nvme_transport_ctrlr_ready(struct spdk_nvme_ctrlr *ctrlr);
    1558                 :            : int nvme_transport_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value);
    1559                 :            : int nvme_transport_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value);
    1560                 :            : int nvme_transport_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value);
    1561                 :            : int nvme_transport_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value);
    1562                 :            : int nvme_transport_ctrlr_set_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1563                 :            :                 uint32_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1564                 :            : int nvme_transport_ctrlr_set_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1565                 :            :                 uint64_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1566                 :            : int nvme_transport_ctrlr_get_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1567                 :            :                 spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1568                 :            : int nvme_transport_ctrlr_get_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
    1569                 :            :                 spdk_nvme_reg_cb cb_fn, void *cb_arg);
    1570                 :            : uint32_t nvme_transport_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr);
    1571                 :            : uint16_t nvme_transport_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr);
    1572                 :            : struct spdk_nvme_qpair *nvme_transport_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
    1573                 :            :                 uint16_t qid, const struct spdk_nvme_io_qpair_opts *opts);
    1574                 :            : int nvme_transport_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr);
    1575                 :            : void *nvme_transport_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size);
    1576                 :            : int nvme_transport_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr);
    1577                 :            : int nvme_transport_ctrlr_enable_pmr(struct spdk_nvme_ctrlr *ctrlr);
    1578                 :            : int nvme_transport_ctrlr_disable_pmr(struct spdk_nvme_ctrlr *ctrlr);
    1579                 :            : void *nvme_transport_ctrlr_map_pmr(struct spdk_nvme_ctrlr *ctrlr, size_t *size);
    1580                 :            : int nvme_transport_ctrlr_unmap_pmr(struct spdk_nvme_ctrlr *ctrlr);
    1581                 :            : void nvme_transport_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
    1582                 :            :                 struct spdk_nvme_qpair *qpair);
    1583                 :            : int nvme_transport_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr,
    1584                 :            :                                        struct spdk_nvme_qpair *qpair);
    1585                 :            : void nvme_transport_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr,
    1586                 :            :                 struct spdk_nvme_qpair *qpair);
    1587                 :            : void nvme_transport_ctrlr_disconnect_qpair_done(struct spdk_nvme_qpair *qpair);
    1588                 :            : int nvme_transport_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
    1589                 :            :                 struct spdk_memory_domain **domains, int array_size);
    1590                 :            : void nvme_transport_qpair_abort_reqs(struct spdk_nvme_qpair *qpair);
    1591                 :            : int nvme_transport_qpair_reset(struct spdk_nvme_qpair *qpair);
    1592                 :            : int nvme_transport_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req);
    1593                 :            : int32_t nvme_transport_qpair_process_completions(struct spdk_nvme_qpair *qpair,
    1594                 :            :                 uint32_t max_completions);
    1595                 :            : void nvme_transport_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair);
    1596                 :            : int nvme_transport_qpair_iterate_requests(struct spdk_nvme_qpair *qpair,
    1597                 :            :                 int (*iter_fn)(struct nvme_request *req, void *arg),
    1598                 :            :                 void *arg);
    1599                 :            : 
    1600                 :            : struct spdk_nvme_transport_poll_group *nvme_transport_poll_group_create(
    1601                 :            :         const struct spdk_nvme_transport *transport);
    1602                 :            : struct spdk_nvme_transport_poll_group *nvme_transport_qpair_get_optimal_poll_group(
    1603                 :            :         const struct spdk_nvme_transport *transport,
    1604                 :            :         struct spdk_nvme_qpair *qpair);
    1605                 :            : int nvme_transport_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup,
    1606                 :            :                                   struct spdk_nvme_qpair *qpair);
    1607                 :            : int nvme_transport_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,
    1608                 :            :                                      struct spdk_nvme_qpair *qpair);
    1609                 :            : int nvme_transport_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair);
    1610                 :            : int nvme_transport_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair);
    1611                 :            : int64_t nvme_transport_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,
    1612                 :            :                 uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb);
    1613                 :            : int nvme_transport_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup);
    1614                 :            : int nvme_transport_poll_group_get_stats(struct spdk_nvme_transport_poll_group *tgroup,
    1615                 :            :                                         struct spdk_nvme_transport_poll_group_stat **stats);
    1616                 :            : void nvme_transport_poll_group_free_stats(struct spdk_nvme_transport_poll_group *tgroup,
    1617                 :            :                 struct spdk_nvme_transport_poll_group_stat *stats);
    1618                 :            : enum spdk_nvme_transport_type nvme_transport_get_trtype(const struct spdk_nvme_transport
    1619                 :            :                 *transport);
    1620                 :            : /*
    1621                 :            :  * Below ref related functions must be called with the global
    1622                 :            :  *  driver lock held for the multi-process condition.
    1623                 :            :  *  Within these functions, the per ctrlr ctrlr_lock is also
    1624                 :            :  *  acquired for the multi-thread condition.
    1625                 :            :  */
    1626                 :            : void    nvme_ctrlr_proc_get_ref(struct spdk_nvme_ctrlr *ctrlr);
    1627                 :            : void    nvme_ctrlr_proc_put_ref(struct spdk_nvme_ctrlr *ctrlr);
    1628                 :            : int     nvme_ctrlr_get_ref_count(struct spdk_nvme_ctrlr *ctrlr);
    1629                 :            : 
    1630                 :            : int     nvme_ctrlr_reinitialize_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair);
    1631                 :            : int     nvme_parse_addr(struct sockaddr_storage *sa, int family,
    1632                 :            :                         const char *addr, const char *service, long int *port);
    1633                 :            : 
    1634                 :            : static inline bool
    1635                 :   40249224 : _is_page_aligned(uint64_t address, uint64_t page_size)
    1636                 :            : {
    1637                 :   40249224 :         return (address & (page_size - 1)) == 0;
    1638                 :            : }
    1639                 :            : 
    1640                 :            : #endif /* __NVME_INTERNAL_H__ */

Generated by: LCOV version 1.14