Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright (C) 2015 Intel Corporation. All rights reserved.
3 : : * Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved.
4 : : * Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : : */
6 : :
7 : : #ifndef __NVME_INTERNAL_H__
8 : : #define __NVME_INTERNAL_H__
9 : :
10 : : #include "spdk/config.h"
11 : : #include "spdk/likely.h"
12 : : #include "spdk/stdinc.h"
13 : :
14 : : #include "spdk/nvme.h"
15 : :
16 : : #if defined(__i386__) || defined(__x86_64__)
17 : : #include <x86intrin.h>
18 : : #endif
19 : :
20 : : #include "spdk/queue.h"
21 : : #include "spdk/barrier.h"
22 : : #include "spdk/bit_array.h"
23 : : #include "spdk/mmio.h"
24 : : #include "spdk/pci_ids.h"
25 : : #include "spdk/util.h"
26 : : #include "spdk/memory.h"
27 : : #include "spdk/nvme_intel.h"
28 : : #include "spdk/nvmf_spec.h"
29 : : #include "spdk/tree.h"
30 : : #include "spdk/uuid.h"
31 : : #include "spdk/fd_group.h"
32 : :
33 : : #include "spdk_internal/assert.h"
34 : : #include "spdk/log.h"
35 : :
36 : : extern pid_t g_spdk_nvme_pid;
37 : :
38 : : extern struct spdk_nvme_transport_opts g_spdk_nvme_transport_opts;
39 : :
40 : : /*
41 : : * Some Intel devices support vendor-unique read latency log page even
42 : : * though the log page directory says otherwise.
43 : : */
44 : : #define NVME_INTEL_QUIRK_READ_LATENCY 0x1
45 : :
46 : : /*
47 : : * Some Intel devices support vendor-unique write latency log page even
48 : : * though the log page directory says otherwise.
49 : : */
50 : : #define NVME_INTEL_QUIRK_WRITE_LATENCY 0x2
51 : :
52 : : /*
53 : : * The controller needs a delay before starts checking the device
54 : : * readiness, which is done by reading the NVME_CSTS_RDY bit.
55 : : */
56 : : #define NVME_QUIRK_DELAY_BEFORE_CHK_RDY 0x4
57 : :
58 : : /*
59 : : * The controller performs best when I/O is split on particular
60 : : * LBA boundaries.
61 : : */
62 : : #define NVME_INTEL_QUIRK_STRIPING 0x8
63 : :
64 : : /*
65 : : * The controller needs a delay after allocating an I/O queue pair
66 : : * before it is ready to accept I/O commands.
67 : : */
68 : : #define NVME_QUIRK_DELAY_AFTER_QUEUE_ALLOC 0x10
69 : :
70 : : /*
71 : : * Earlier NVMe devices do not indicate whether unmapped blocks
72 : : * will read all zeroes or not. This define indicates that the
73 : : * device does in fact read all zeroes after an unmap event
74 : : */
75 : : #define NVME_QUIRK_READ_ZERO_AFTER_DEALLOCATE 0x20
76 : :
77 : : /*
78 : : * The controller doesn't handle Identify value others than 0 or 1 correctly.
79 : : */
80 : : #define NVME_QUIRK_IDENTIFY_CNS 0x40
81 : :
82 : : /*
83 : : * The controller supports Open Channel command set if matching additional
84 : : * condition, like the first byte (value 0x1) in the vendor specific
85 : : * bits of the namespace identify structure is set.
86 : : */
87 : : #define NVME_QUIRK_OCSSD 0x80
88 : :
89 : : /*
90 : : * The controller has an Intel vendor ID but does not support Intel vendor-specific
91 : : * log pages. This is primarily for QEMU emulated SSDs which report an Intel vendor
92 : : * ID but do not support these log pages.
93 : : */
94 : : #define NVME_INTEL_QUIRK_NO_LOG_PAGES 0x100
95 : :
96 : : /*
97 : : * The controller does not set SHST_COMPLETE in a reasonable amount of time. This
98 : : * is primarily seen in virtual VMWare NVMe SSDs. This quirk merely adds an additional
99 : : * error message that on VMWare NVMe SSDs, the shutdown timeout may be expected.
100 : : */
101 : : #define NVME_QUIRK_SHST_COMPLETE 0x200
102 : :
103 : : /*
104 : : * The controller requires an extra delay before starting the initialization process
105 : : * during attach.
106 : : */
107 : : #define NVME_QUIRK_DELAY_BEFORE_INIT 0x400
108 : :
109 : : /*
110 : : * Some SSDs exhibit poor performance with the default SPDK NVMe IO queue size.
111 : : * This quirk will increase the default to 1024 which matches other operating
112 : : * systems, at the cost of some extra memory usage. Users can still override
113 : : * the increased default by changing the spdk_nvme_io_qpair_opts when allocating
114 : : * a new queue pair.
115 : : */
116 : : #define NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE 0x800
117 : :
118 : : /**
119 : : * The maximum access width to PCI memory space is 8 Bytes, don't use AVX2 or
120 : : * SSE instructions to optimize the memory access(memcpy or memset) larger than
121 : : * 8 Bytes.
122 : : */
123 : : #define NVME_QUIRK_MAXIMUM_PCI_ACCESS_WIDTH 0x1000
124 : :
125 : : /**
126 : : * The SSD does not support OPAL even through it sets the security bit in OACS.
127 : : */
128 : : #define NVME_QUIRK_OACS_SECURITY 0x2000
129 : :
130 : : /**
131 : : * Intel P55XX SSDs can't support Dataset Management command with SGL format,
132 : : * so use PRP with DSM command.
133 : : */
134 : : #define NVME_QUIRK_NO_SGL_FOR_DSM 0x4000
135 : :
136 : : /**
137 : : * Maximum Data Transfer Size(MDTS) excludes interleaved metadata.
138 : : */
139 : : #define NVME_QUIRK_MDTS_EXCLUDE_MD 0x8000
140 : :
141 : : /**
142 : : * Force not to use SGL even the controller report that it can
143 : : * support it.
144 : : */
145 : : #define NVME_QUIRK_NOT_USE_SGL 0x10000
146 : :
147 : : /*
148 : : * Some SSDs require the admin submission queue size to equate to an even
149 : : * 4KiB multiple.
150 : : */
151 : : #define NVME_QUIRK_MINIMUM_ADMIN_QUEUE_SIZE 0x20000
152 : :
153 : : #define NVME_MAX_ASYNC_EVENTS (8)
154 : :
155 : : #define NVME_MAX_ADMIN_TIMEOUT_IN_SECS (30)
156 : :
157 : : /* Maximum log page size to fetch for AERs. */
158 : : #define NVME_MAX_AER_LOG_SIZE (4096)
159 : :
160 : : /*
161 : : * NVME_MAX_IO_QUEUES in nvme_spec.h defines the 64K spec-limit, but this
162 : : * define specifies the maximum number of queues this driver will actually
163 : : * try to configure, if available.
164 : : */
165 : : #define DEFAULT_MAX_IO_QUEUES (1024)
166 : : #define MAX_IO_QUEUES_WITH_INTERRUPTS (256)
167 : : #define DEFAULT_ADMIN_QUEUE_SIZE (32)
168 : : #define DEFAULT_IO_QUEUE_SIZE (256)
169 : : #define DEFAULT_IO_QUEUE_SIZE_FOR_QUIRK (1024) /* Matches Linux kernel driver */
170 : :
171 : : #define DEFAULT_IO_QUEUE_REQUESTS (512)
172 : :
173 : : #define SPDK_NVME_DEFAULT_RETRY_COUNT (4)
174 : :
175 : : #define SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED (0)
176 : : #define SPDK_NVME_DEFAULT_TRANSPORT_ACK_TIMEOUT SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED
177 : :
178 : : #define SPDK_NVME_TRANSPORT_TOS_DISABLED (0)
179 : :
180 : : #define MIN_KEEP_ALIVE_TIMEOUT_IN_MS (10000)
181 : :
182 : : /* We want to fit submission and completion rings each in a single 2MB
183 : : * hugepage to ensure physical address contiguity.
184 : : */
185 : : #define MAX_IO_QUEUE_ENTRIES (VALUE_2MB / spdk_max( \
186 : : sizeof(struct spdk_nvme_cmd), \
187 : : sizeof(struct spdk_nvme_cpl)))
188 : :
189 : : /* Default timeout for fabrics connect commands. */
190 : : #ifdef DEBUG
191 : : #define NVME_FABRIC_CONNECT_COMMAND_TIMEOUT 0
192 : : #else
193 : : /* 500 millisecond timeout. */
194 : : #define NVME_FABRIC_CONNECT_COMMAND_TIMEOUT 500000
195 : : #endif
196 : :
197 : : /* This value indicates that a read from a PCIe register is invalid. This can happen when a device is no longer present */
198 : : #define SPDK_NVME_INVALID_REGISTER_VALUE 0xFFFFFFFFu
199 : :
200 : : enum nvme_payload_type {
201 : : NVME_PAYLOAD_TYPE_INVALID = 0,
202 : :
203 : : /** nvme_request::u.payload.contig_buffer is valid for this request */
204 : : NVME_PAYLOAD_TYPE_CONTIG,
205 : :
206 : : /** nvme_request::u.sgl is valid for this request */
207 : : NVME_PAYLOAD_TYPE_SGL,
208 : : };
209 : :
210 : : /** Boot partition write states */
211 : : enum nvme_bp_write_state {
212 : : SPDK_NVME_BP_WS_DOWNLOADING = 0x0,
213 : : SPDK_NVME_BP_WS_DOWNLOADED = 0x1,
214 : : SPDK_NVME_BP_WS_REPLACE = 0x2,
215 : : SPDK_NVME_BP_WS_ACTIVATE = 0x3,
216 : : };
217 : :
218 : : /**
219 : : * Descriptor for a request data payload.
220 : : */
221 : : struct nvme_payload {
222 : : /**
223 : : * Functions for retrieving physical addresses for scattered payloads.
224 : : */
225 : : spdk_nvme_req_reset_sgl_cb reset_sgl_fn;
226 : : spdk_nvme_req_next_sge_cb next_sge_fn;
227 : :
228 : : /**
229 : : * Extended IO options passed by the user
230 : : */
231 : : struct spdk_nvme_ns_cmd_ext_io_opts *opts;
232 : : /**
233 : : * If reset_sgl_fn == NULL, this is a contig payload, and contig_or_cb_arg contains the
234 : : * virtual memory address of a single virtually contiguous buffer.
235 : : *
236 : : * If reset_sgl_fn != NULL, this is a SGL payload, and contig_or_cb_arg contains the
237 : : * cb_arg that will be passed to the SGL callback functions.
238 : : */
239 : : void *contig_or_cb_arg;
240 : :
241 : : /** Virtual memory address of a single virtually contiguous metadata buffer */
242 : : void *md;
243 : : };
244 : :
245 : : #define NVME_PAYLOAD_CONTIG(contig_, md_) \
246 : : (struct nvme_payload) { \
247 : : .reset_sgl_fn = NULL, \
248 : : .next_sge_fn = NULL, \
249 : : .contig_or_cb_arg = (contig_), \
250 : : .md = (md_), \
251 : : }
252 : :
253 : : #define NVME_PAYLOAD_SGL(reset_sgl_fn_, next_sge_fn_, cb_arg_, md_) \
254 : : (struct nvme_payload) { \
255 : : .reset_sgl_fn = (reset_sgl_fn_), \
256 : : .next_sge_fn = (next_sge_fn_), \
257 : : .contig_or_cb_arg = (cb_arg_), \
258 : : .md = (md_), \
259 : : }
260 : :
261 : : static inline enum nvme_payload_type
262 : 190773252 : nvme_payload_type(const struct nvme_payload *payload) {
263 [ + + + - ]: 190773252 : return payload->reset_sgl_fn ? NVME_PAYLOAD_TYPE_SGL : NVME_PAYLOAD_TYPE_CONTIG;
264 : : }
265 : :
266 : : struct nvme_error_cmd {
267 : : bool do_not_submit;
268 : : uint64_t timeout_tsc;
269 : : uint32_t err_count;
270 : : uint8_t opc;
271 : : struct spdk_nvme_status status;
272 : : TAILQ_ENTRY(nvme_error_cmd) link;
273 : : };
274 : :
275 : : struct nvme_request {
276 : : struct spdk_nvme_cmd cmd;
277 : :
278 : : uint8_t retries;
279 : :
280 : : uint8_t timed_out : 1;
281 : :
282 : : /**
283 : : * True if the request is in the queued_req list.
284 : : */
285 : : uint8_t queued : 1;
286 : : uint8_t reserved : 6;
287 : :
288 : : /**
289 : : * Number of children requests still outstanding for this
290 : : * request which was split into multiple child requests.
291 : : */
292 : : uint16_t num_children;
293 : :
294 : : /**
295 : : * Offset in bytes from the beginning of payload for this request.
296 : : * This is used for I/O commands that are split into multiple requests.
297 : : */
298 : : uint32_t payload_offset;
299 : : uint32_t md_offset;
300 : :
301 : : uint32_t payload_size;
302 : :
303 : : /**
304 : : * Timeout ticks for error injection requests, can be extended in future
305 : : * to support per-request timeout feature.
306 : : */
307 : : uint64_t timeout_tsc;
308 : :
309 : : /**
310 : : * Data payload for this request's command.
311 : : */
312 : : struct nvme_payload payload;
313 : :
314 : : spdk_nvme_cmd_cb cb_fn;
315 : : void *cb_arg;
316 : : STAILQ_ENTRY(nvme_request) stailq;
317 : :
318 : : struct spdk_nvme_qpair *qpair;
319 : :
320 : : /*
321 : : * The value of spdk_get_ticks() when the request was submitted to the hardware.
322 : : * Only set if ctrlr->timeout_enabled is true.
323 : : */
324 : : uint64_t submit_tick;
325 : :
326 : : /**
327 : : * The active admin request can be moved to a per process pending
328 : : * list based on the saved pid to tell which process it belongs
329 : : * to. The cpl saves the original completion information which
330 : : * is used in the completion callback.
331 : : * NOTE: these below two fields are only used for admin request.
332 : : */
333 : : pid_t pid;
334 : : struct spdk_nvme_cpl cpl;
335 : :
336 : : uint32_t md_size;
337 : :
338 : : /**
339 : : * The following members should not be reordered with members
340 : : * above. These members are only needed when splitting
341 : : * requests which is done rarely, and the driver is careful
342 : : * to not touch the following fields until a split operation is
343 : : * needed, to avoid touching an extra cacheline.
344 : : */
345 : :
346 : : /**
347 : : * Points to the outstanding child requests for a parent request.
348 : : * Only valid if a request was split into multiple children
349 : : * requests, and is not initialized for non-split requests.
350 : : */
351 : : TAILQ_HEAD(, nvme_request) children;
352 : :
353 : : /**
354 : : * Linked-list pointers for a child request in its parent's list.
355 : : */
356 : : TAILQ_ENTRY(nvme_request) child_tailq;
357 : :
358 : : /**
359 : : * Points to a parent request if part of a split request,
360 : : * NULL otherwise.
361 : : */
362 : : struct nvme_request *parent;
363 : :
364 : : /**
365 : : * Completion status for a parent request. Initialized to all 0's
366 : : * (SUCCESS) before child requests are submitted. If a child
367 : : * request completes with error, the error status is copied here,
368 : : * to ensure that the parent request is also completed with error
369 : : * status once all child requests are completed.
370 : : */
371 : : struct spdk_nvme_cpl parent_status;
372 : :
373 : : /**
374 : : * The user_cb_fn and user_cb_arg fields are used for holding the original
375 : : * callback data when using nvme_allocate_request_user_copy.
376 : : */
377 : : spdk_nvme_cmd_cb user_cb_fn;
378 : : void *user_cb_arg;
379 : : void *user_buffer;
380 : :
381 : : /** Sequence of accel operations associated with this request */
382 : : void *accel_sequence;
383 : : };
384 : :
385 : : struct nvme_completion_poll_status {
386 : : struct spdk_nvme_cpl cpl;
387 : : uint64_t timeout_tsc;
388 : : /**
389 : : * DMA buffer retained throughout the duration of the command. It'll be released
390 : : * automatically if the command times out, otherwise the user is responsible for freeing it.
391 : : */
392 : : void *dma_data;
393 : : bool done;
394 : : /* This flag indicates that the request has been timed out and the memory
395 : : must be freed in a completion callback */
396 : : bool timed_out;
397 : : };
398 : :
399 : : struct nvme_async_event_request {
400 : : struct spdk_nvme_ctrlr *ctrlr;
401 : : struct nvme_request *req;
402 : : struct spdk_nvme_cpl cpl;
403 : : };
404 : :
405 : : enum nvme_qpair_state {
406 : : NVME_QPAIR_DISCONNECTED,
407 : : NVME_QPAIR_DISCONNECTING,
408 : : NVME_QPAIR_CONNECTING,
409 : : NVME_QPAIR_CONNECTED,
410 : : NVME_QPAIR_ENABLING,
411 : : NVME_QPAIR_ENABLED,
412 : : NVME_QPAIR_DESTROYING,
413 : : };
414 : :
415 : : enum nvme_qpair_auth_state {
416 : : NVME_QPAIR_AUTH_STATE_NEGOTIATE,
417 : : NVME_QPAIR_AUTH_STATE_AWAIT_NEGOTIATE,
418 : : NVME_QPAIR_AUTH_STATE_AWAIT_CHALLENGE,
419 : : NVME_QPAIR_AUTH_STATE_AWAIT_REPLY,
420 : : NVME_QPAIR_AUTH_STATE_AWAIT_SUCCESS1,
421 : : NVME_QPAIR_AUTH_STATE_AWAIT_SUCCESS2,
422 : : NVME_QPAIR_AUTH_STATE_AWAIT_FAILURE2,
423 : : NVME_QPAIR_AUTH_STATE_DONE,
424 : : };
425 : :
426 : : /* Authentication transaction required (authreq.atr) */
427 : : #define NVME_QPAIR_AUTH_FLAG_ATR (1 << 0)
428 : : /* Authentication and secure channel required (authreq.ascr) */
429 : : #define NVME_QPAIR_AUTH_FLAG_ASCR (1 << 1)
430 : :
431 : : /* Maximum size of a digest */
432 : : #define NVME_AUTH_DIGEST_MAX_SIZE 64
433 : :
434 : : struct nvme_auth {
435 : : /* State of the authentication */
436 : : enum nvme_qpair_auth_state state;
437 : : /* Status of the authentication */
438 : : int status;
439 : : /* Transaction ID */
440 : : uint16_t tid;
441 : : /* Flags */
442 : : uint32_t flags;
443 : : /* Selected hash function */
444 : : uint8_t hash;
445 : : /* Buffer used for controller challenge */
446 : : uint8_t challenge[NVME_AUTH_DIGEST_MAX_SIZE];
447 : : /* User's auth cb fn/ctx */
448 : : spdk_nvme_authenticate_cb cb_fn;
449 : : void *cb_ctx;
450 : : };
451 : :
452 : : struct spdk_nvme_qpair {
453 : : struct spdk_nvme_ctrlr *ctrlr;
454 : :
455 : : uint16_t id;
456 : :
457 : : uint8_t qprio: 2;
458 : :
459 : : uint8_t state: 3;
460 : :
461 : : uint8_t async: 1;
462 : :
463 : : uint8_t is_new_qpair: 1;
464 : :
465 : : uint8_t abort_dnr: 1;
466 : : /*
467 : : * Members for handling IO qpair deletion inside of a completion context.
468 : : * These are specifically defined as single bits, so that they do not
469 : : * push this data structure out to another cacheline.
470 : : */
471 : : uint8_t in_completion_context: 1;
472 : : uint8_t delete_after_completion_context: 1;
473 : :
474 : : /*
475 : : * Set when no deletion notification is needed. For example, the process
476 : : * which allocated this qpair exited unexpectedly.
477 : : */
478 : : uint8_t no_deletion_notification_needed: 1;
479 : :
480 : : uint8_t last_fuse: 2;
481 : :
482 : : uint8_t transport_failure_reason: 3;
483 : : uint8_t last_transport_failure_reason: 3;
484 : :
485 : : /* The user is destroying qpair */
486 : : uint8_t destroy_in_progress: 1;
487 : :
488 : : /* Number of IO outstanding at transport level */
489 : : uint16_t queue_depth;
490 : :
491 : : enum spdk_nvme_transport_type trtype;
492 : :
493 : : uint32_t num_outstanding_reqs;
494 : :
495 : : /* request object used only for this qpair's FABRICS/CONNECT command (if needed) */
496 : : struct nvme_request *reserved_req;
497 : :
498 : : STAILQ_HEAD(, nvme_request) free_req;
499 : : STAILQ_HEAD(, nvme_request) queued_req;
500 : :
501 : : /* List entry for spdk_nvme_transport_poll_group::qpairs */
502 : : STAILQ_ENTRY(spdk_nvme_qpair) poll_group_stailq;
503 : :
504 : : /** Commands opcode in this list will return error */
505 : : TAILQ_HEAD(, nvme_error_cmd) err_cmd_head;
506 : : /** Requests in this list will return error */
507 : : STAILQ_HEAD(, nvme_request) err_req_head;
508 : :
509 : : struct spdk_nvme_ctrlr_process *active_proc;
510 : :
511 : : struct spdk_nvme_transport_poll_group *poll_group;
512 : :
513 : : void *poll_group_tailq_head;
514 : :
515 : : const struct spdk_nvme_transport *transport;
516 : :
517 : : /* Entries below here are not touched in the main I/O path. */
518 : :
519 : : struct nvme_completion_poll_status *poll_status;
520 : :
521 : : /* List entry for spdk_nvme_ctrlr::active_io_qpairs */
522 : : TAILQ_ENTRY(spdk_nvme_qpair) tailq;
523 : :
524 : : /* List entry for spdk_nvme_ctrlr_process::allocated_io_qpairs */
525 : : TAILQ_ENTRY(spdk_nvme_qpair) per_process_tailq;
526 : :
527 : : STAILQ_HEAD(, nvme_request) aborting_queued_req;
528 : :
529 : : void *req_buf;
530 : :
531 : : /* In-band authentication state */
532 : : struct nvme_auth auth;
533 : : };
534 : :
535 : : struct spdk_nvme_poll_group {
536 : : void *ctx;
537 : : struct spdk_nvme_accel_fn_table accel_fn_table;
538 : : STAILQ_HEAD(, spdk_nvme_transport_poll_group) tgroups;
539 : : bool in_process_completions;
540 : : bool enable_interrupts;
541 : : bool enable_interrupts_is_valid;
542 : : int disconnect_qpair_fd;
543 : : struct spdk_fd_group *fgrp;
544 : : };
545 : :
546 : : struct spdk_nvme_transport_poll_group {
547 : : struct spdk_nvme_poll_group *group;
548 : : const struct spdk_nvme_transport *transport;
549 : : STAILQ_HEAD(, spdk_nvme_qpair) connected_qpairs;
550 : : STAILQ_HEAD(, spdk_nvme_qpair) disconnected_qpairs;
551 : : STAILQ_ENTRY(spdk_nvme_transport_poll_group) link;
552 : : uint32_t num_connected_qpairs;
553 : : };
554 : :
555 : : struct spdk_nvme_ns {
556 : : struct spdk_nvme_ctrlr *ctrlr;
557 : : uint32_t sector_size;
558 : :
559 : : /*
560 : : * Size of data transferred as part of each block,
561 : : * including metadata if FLBAS indicates the metadata is transferred
562 : : * as part of the data buffer at the end of each LBA.
563 : : */
564 : : uint32_t extended_lba_size;
565 : :
566 : : uint32_t md_size;
567 : : uint32_t pi_type;
568 : : uint32_t pi_format;
569 : : uint32_t sectors_per_max_io;
570 : : uint32_t sectors_per_max_io_no_md;
571 : : uint32_t sectors_per_stripe;
572 : : uint32_t id;
573 : : uint16_t flags;
574 : : bool active;
575 : :
576 : : /* Command Set Identifier */
577 : : enum spdk_nvme_csi csi;
578 : :
579 : : /* Namespace Identification Descriptor List (CNS = 03h) */
580 : : uint8_t id_desc_list[4096];
581 : :
582 : : uint32_t ana_group_id;
583 : : enum spdk_nvme_ana_state ana_state;
584 : :
585 : : /* Identify Namespace data. */
586 : : struct spdk_nvme_ns_data nsdata;
587 : :
588 : : /* Zoned Namespace Command Set Specific Identify Namespace data. */
589 : : struct spdk_nvme_zns_ns_data *nsdata_zns;
590 : :
591 : : struct spdk_nvme_nvm_ns_data *nsdata_nvm;
592 : :
593 : : RB_ENTRY(spdk_nvme_ns) node;
594 : : };
595 : :
596 : : #define CTRLR_STRING(ctrlr) \
597 : : (spdk_nvme_trtype_is_fabrics(ctrlr->trid.trtype) ? \
598 : : ctrlr->trid.subnqn : ctrlr->trid.traddr)
599 : :
600 : : #define NVME_CTRLR_ERRLOG(ctrlr, format, ...) \
601 : : SPDK_ERRLOG("[%s, %u] " format, CTRLR_STRING(ctrlr), ctrlr->cntlid, ##__VA_ARGS__);
602 : :
603 : : #define NVME_CTRLR_WARNLOG(ctrlr, format, ...) \
604 : : SPDK_WARNLOG("[%s, %u] " format, CTRLR_STRING(ctrlr), ctrlr->cntlid, ##__VA_ARGS__);
605 : :
606 : : #define NVME_CTRLR_NOTICELOG(ctrlr, format, ...) \
607 : : SPDK_NOTICELOG("[%s, %u] " format, CTRLR_STRING(ctrlr), ctrlr->cntlid, ##__VA_ARGS__);
608 : :
609 : : #define NVME_CTRLR_INFOLOG(ctrlr, format, ...) \
610 : : SPDK_INFOLOG(nvme, "[%s, %u] " format, CTRLR_STRING(ctrlr), ctrlr->cntlid, ##__VA_ARGS__);
611 : :
612 : : #ifdef DEBUG
613 : : #define NVME_CTRLR_DEBUGLOG(ctrlr, format, ...) \
614 : : SPDK_DEBUGLOG(nvme, "[%s, %u] " format, CTRLR_STRING(ctrlr), ctrlr->cntlid, ##__VA_ARGS__);
615 : : #else
616 : : #define NVME_CTRLR_DEBUGLOG(ctrlr, ...) do { } while (0)
617 : : #endif
618 : :
619 : : /**
620 : : * State of struct spdk_nvme_ctrlr (in particular, during initialization).
621 : : */
622 : : enum nvme_ctrlr_state {
623 : : /**
624 : : * Wait before initializing the controller.
625 : : */
626 : : NVME_CTRLR_STATE_INIT_DELAY,
627 : :
628 : : /**
629 : : * Connect the admin queue.
630 : : */
631 : : NVME_CTRLR_STATE_CONNECT_ADMINQ,
632 : :
633 : : /**
634 : : * Controller has not started initialized yet.
635 : : */
636 : : NVME_CTRLR_STATE_INIT = NVME_CTRLR_STATE_CONNECT_ADMINQ,
637 : :
638 : : /**
639 : : * Waiting for admin queue to connect.
640 : : */
641 : : NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ,
642 : :
643 : : /**
644 : : * Read Version (VS) register.
645 : : */
646 : : NVME_CTRLR_STATE_READ_VS,
647 : :
648 : : /**
649 : : * Waiting for Version (VS) register to be read.
650 : : */
651 : : NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS,
652 : :
653 : : /**
654 : : * Read Capabilities (CAP) register.
655 : : */
656 : : NVME_CTRLR_STATE_READ_CAP,
657 : :
658 : : /**
659 : : * Waiting for Capabilities (CAP) register to be read.
660 : : */
661 : : NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP,
662 : :
663 : : /**
664 : : * Check EN to prepare for controller initialization.
665 : : */
666 : : NVME_CTRLR_STATE_CHECK_EN,
667 : :
668 : : /**
669 : : * Waiting for CC to be read as part of EN check.
670 : : */
671 : : NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC,
672 : :
673 : : /**
674 : : * Waiting for CSTS.RDY to transition from 0 to 1 so that CC.EN may be set to 0.
675 : : */
676 : : NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1,
677 : :
678 : : /**
679 : : * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 1.
680 : : */
681 : : NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS,
682 : :
683 : : /**
684 : : * Disabling the controller by setting CC.EN to 0.
685 : : */
686 : : NVME_CTRLR_STATE_SET_EN_0,
687 : :
688 : : /**
689 : : * Waiting for the CC register to be read as part of disabling the controller.
690 : : */
691 : : NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC,
692 : :
693 : : /**
694 : : * Waiting for CSTS.RDY to transition from 1 to 0 so that CC.EN may be set to 1.
695 : : */
696 : : NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0,
697 : :
698 : : /**
699 : : * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 0.
700 : : */
701 : : NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS,
702 : :
703 : : /**
704 : : * The controller is disabled. (CC.EN and CSTS.RDY are 0.)
705 : : */
706 : : NVME_CTRLR_STATE_DISABLED,
707 : :
708 : : /**
709 : : * Enable the controller by writing CC.EN to 1
710 : : */
711 : : NVME_CTRLR_STATE_ENABLE,
712 : :
713 : : /**
714 : : * Waiting for CC register to be written as part of enabling the controller.
715 : : */
716 : : NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC,
717 : :
718 : : /**
719 : : * Waiting for CSTS.RDY to transition from 0 to 1 after enabling the controller.
720 : : */
721 : : NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1,
722 : :
723 : : /**
724 : : * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 1.
725 : : */
726 : : NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS,
727 : :
728 : : /**
729 : : * Reset the Admin queue of the controller.
730 : : */
731 : : NVME_CTRLR_STATE_RESET_ADMIN_QUEUE,
732 : :
733 : : /**
734 : : * Identify Controller command will be sent to then controller.
735 : : */
736 : : NVME_CTRLR_STATE_IDENTIFY,
737 : :
738 : : /**
739 : : * Waiting for Identify Controller command be completed.
740 : : */
741 : : NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY,
742 : :
743 : : /**
744 : : * Configure AER of the controller.
745 : : */
746 : : NVME_CTRLR_STATE_CONFIGURE_AER,
747 : :
748 : : /**
749 : : * Waiting for the Configure AER to be completed.
750 : : */
751 : : NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER,
752 : :
753 : : /**
754 : : * Set Keep Alive Timeout of the controller.
755 : : */
756 : : NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT,
757 : :
758 : : /**
759 : : * Waiting for Set Keep Alive Timeout to be completed.
760 : : */
761 : : NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT,
762 : :
763 : : /**
764 : : * Get Identify I/O Command Set Specific Controller data structure.
765 : : */
766 : : NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC,
767 : :
768 : : /**
769 : : * Waiting for Identify I/O Command Set Specific Controller command to be completed.
770 : : */
771 : : NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC,
772 : :
773 : : /**
774 : : * Get Commands Supported and Effects log page for the Zoned Namespace Command Set.
775 : : */
776 : : NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG,
777 : :
778 : : /**
779 : : * Waiting for the Get Log Page command to be completed.
780 : : */
781 : : NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG,
782 : :
783 : : /**
784 : : * Set Number of Queues of the controller.
785 : : */
786 : : NVME_CTRLR_STATE_SET_NUM_QUEUES,
787 : :
788 : : /**
789 : : * Waiting for Set Num of Queues command to be completed.
790 : : */
791 : : NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES,
792 : :
793 : : /**
794 : : * Get active Namespace list of the controller.
795 : : */
796 : : NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS,
797 : :
798 : : /**
799 : : * Waiting for the Identify Active Namespace commands to be completed.
800 : : */
801 : : NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS,
802 : :
803 : : /**
804 : : * Get Identify Namespace Data structure for each NS.
805 : : */
806 : : NVME_CTRLR_STATE_IDENTIFY_NS,
807 : :
808 : : /**
809 : : * Waiting for the Identify Namespace commands to be completed.
810 : : */
811 : : NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS,
812 : :
813 : : /**
814 : : * Get Identify Namespace Identification Descriptors.
815 : : */
816 : : NVME_CTRLR_STATE_IDENTIFY_ID_DESCS,
817 : :
818 : : /**
819 : : * Get Identify I/O Command Set Specific Namespace data structure for each NS.
820 : : */
821 : : NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC,
822 : :
823 : : /**
824 : : * Waiting for the Identify I/O Command Set Specific Namespace commands to be completed.
825 : : */
826 : : NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC,
827 : :
828 : : /**
829 : : * Waiting for the Identify Namespace Identification
830 : : * Descriptors to be completed.
831 : : */
832 : : NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS,
833 : :
834 : : /**
835 : : * Set supported log pages of the controller.
836 : : */
837 : : NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES,
838 : :
839 : : /**
840 : : * Set supported log pages of INTEL controller.
841 : : */
842 : : NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES,
843 : :
844 : : /**
845 : : * Waiting for supported log pages of INTEL controller.
846 : : */
847 : : NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES,
848 : :
849 : : /**
850 : : * Set supported features of the controller.
851 : : */
852 : : NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES,
853 : :
854 : : /**
855 : : * Set the Host Behavior Support feature of the controller.
856 : : */
857 : : NVME_CTRLR_STATE_SET_HOST_FEATURE,
858 : :
859 : : /**
860 : : * Waiting for the Host Behavior Support feature of the controller.
861 : : */
862 : : NVME_CTRLR_STATE_WAIT_FOR_SET_HOST_FEATURE,
863 : :
864 : : /**
865 : : * Set Doorbell Buffer Config of the controller.
866 : : */
867 : : NVME_CTRLR_STATE_SET_DB_BUF_CFG,
868 : :
869 : : /**
870 : : * Waiting for Doorbell Buffer Config to be completed.
871 : : */
872 : : NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG,
873 : :
874 : : /**
875 : : * Set Host ID of the controller.
876 : : */
877 : : NVME_CTRLR_STATE_SET_HOST_ID,
878 : :
879 : : /**
880 : : * Waiting for Set Host ID to be completed.
881 : : */
882 : : NVME_CTRLR_STATE_WAIT_FOR_HOST_ID,
883 : :
884 : : /**
885 : : * Let transport layer do its part of initialization.
886 : : */
887 : : NVME_CTRLR_STATE_TRANSPORT_READY,
888 : :
889 : : /**
890 : : * Controller initialization has completed and the controller is ready.
891 : : */
892 : : NVME_CTRLR_STATE_READY,
893 : :
894 : : /**
895 : : * Controller initialization has an error.
896 : : */
897 : : NVME_CTRLR_STATE_ERROR,
898 : :
899 : : /**
900 : : * Admin qpair was disconnected, controller needs to be re-initialized
901 : : */
902 : : NVME_CTRLR_STATE_DISCONNECTED,
903 : : };
904 : :
905 : : #define NVME_TIMEOUT_INFINITE 0
906 : : #define NVME_TIMEOUT_KEEP_EXISTING UINT64_MAX
907 : :
908 : : struct spdk_nvme_ctrlr_aer_completion {
909 : : struct spdk_nvme_cpl cpl;
910 : : STAILQ_ENTRY(spdk_nvme_ctrlr_aer_completion) link;
911 : : };
912 : :
913 : : /*
914 : : * Used to track properties for all processes accessing the controller.
915 : : */
916 : : struct spdk_nvme_ctrlr_process {
917 : : /** Whether it is the primary process */
918 : : bool is_primary;
919 : :
920 : : /** Process ID */
921 : : pid_t pid;
922 : :
923 : : /** Active admin requests to be completed */
924 : : STAILQ_HEAD(, nvme_request) active_reqs;
925 : :
926 : : TAILQ_ENTRY(spdk_nvme_ctrlr_process) tailq;
927 : :
928 : : /** Per process PCI device handle */
929 : : struct spdk_pci_device *devhandle;
930 : :
931 : : /** Reference to track the number of attachment to this controller. */
932 : : int ref;
933 : :
934 : : /** Allocated IO qpairs */
935 : : TAILQ_HEAD(, spdk_nvme_qpair) allocated_io_qpairs;
936 : :
937 : : spdk_nvme_aer_cb aer_cb_fn;
938 : : void *aer_cb_arg;
939 : :
940 : : /**
941 : : * A function pointer to timeout callback function
942 : : */
943 : : spdk_nvme_timeout_cb timeout_cb_fn;
944 : : void *timeout_cb_arg;
945 : : /** separate timeout values for io vs. admin reqs */
946 : : uint64_t timeout_io_ticks;
947 : : uint64_t timeout_admin_ticks;
948 : :
949 : : /** List to publish AENs to all procs in multiprocess setup */
950 : : STAILQ_HEAD(, spdk_nvme_ctrlr_aer_completion) async_events;
951 : : };
952 : :
953 : : struct nvme_register_completion {
954 : : struct spdk_nvme_cpl cpl;
955 : : uint64_t value;
956 : : spdk_nvme_reg_cb cb_fn;
957 : : void *cb_ctx;
958 : : STAILQ_ENTRY(nvme_register_completion) stailq;
959 : : pid_t pid;
960 : : };
961 : :
962 : : struct spdk_nvme_ctrlr {
963 : : /* Hot data (accessed in I/O path) starts here. */
964 : :
965 : : /* Tree of namespaces */
966 : : RB_HEAD(nvme_ns_tree, spdk_nvme_ns) ns;
967 : :
968 : : /* The number of active namespaces */
969 : : uint32_t active_ns_count;
970 : :
971 : : bool is_removed;
972 : :
973 : : bool is_resetting;
974 : :
975 : : bool is_failed;
976 : :
977 : : bool is_destructed;
978 : :
979 : : bool timeout_enabled;
980 : :
981 : : /* The application is preparing to reset the controller. Transports
982 : : * can use this to skip unnecessary parts of the qpair deletion process
983 : : * for example, like the DELETE_SQ/CQ commands.
984 : : */
985 : : bool prepare_for_reset;
986 : :
987 : : bool is_disconnecting;
988 : :
989 : : bool needs_io_msg_update;
990 : :
991 : : uint16_t max_sges;
992 : :
993 : : uint16_t cntlid;
994 : :
995 : : /** Controller support flags */
996 : : uint64_t flags;
997 : :
998 : : /** NVMEoF in-capsule data size in bytes */
999 : : uint32_t ioccsz_bytes;
1000 : :
1001 : : /** NVMEoF in-capsule data offset in 16 byte units */
1002 : : uint16_t icdoff;
1003 : :
1004 : : /* Cold data (not accessed in normal I/O path) is after this point. */
1005 : :
1006 : : struct spdk_nvme_transport_id trid;
1007 : :
1008 : : struct {
1009 : : /** Is numa.id valid? Ensures numa.id == 0 is interpreted correctly. */
1010 : : uint32_t id_valid : 1;
1011 : : int32_t id : 31;
1012 : : } numa;
1013 : :
1014 : : union spdk_nvme_cap_register cap;
1015 : : union spdk_nvme_vs_register vs;
1016 : :
1017 : : int state;
1018 : : uint64_t state_timeout_tsc;
1019 : :
1020 : : uint64_t next_keep_alive_tick;
1021 : : uint64_t keep_alive_interval_ticks;
1022 : :
1023 : : TAILQ_ENTRY(spdk_nvme_ctrlr) tailq;
1024 : :
1025 : : /** All the log pages supported */
1026 : : bool log_page_supported[256];
1027 : :
1028 : : /** All the features supported */
1029 : : bool feature_supported[256];
1030 : :
1031 : : /** maximum i/o size in bytes */
1032 : : uint32_t max_xfer_size;
1033 : :
1034 : : /** minimum page size supported by this controller in bytes */
1035 : : uint32_t min_page_size;
1036 : :
1037 : : /** selected memory page size for this controller in bytes */
1038 : : uint32_t page_size;
1039 : :
1040 : : uint32_t num_aers;
1041 : : struct nvme_async_event_request aer[NVME_MAX_ASYNC_EVENTS];
1042 : :
1043 : : /** guards access to the controller itself, including admin queues */
1044 : : pthread_mutex_t ctrlr_lock;
1045 : :
1046 : : struct spdk_nvme_qpair *adminq;
1047 : :
1048 : : /** shadow doorbell buffer */
1049 : : uint32_t *shadow_doorbell;
1050 : : /** eventidx buffer */
1051 : : uint32_t *eventidx;
1052 : :
1053 : : /**
1054 : : * Identify Controller data.
1055 : : */
1056 : : struct spdk_nvme_ctrlr_data cdata;
1057 : :
1058 : : /**
1059 : : * Zoned Namespace Command Set Specific Identify Controller data.
1060 : : */
1061 : : struct spdk_nvme_zns_ctrlr_data *cdata_zns;
1062 : :
1063 : : struct spdk_bit_array *free_io_qids;
1064 : : TAILQ_HEAD(, spdk_nvme_qpair) active_io_qpairs;
1065 : :
1066 : : struct spdk_nvme_ctrlr_opts opts;
1067 : :
1068 : : uint64_t quirks;
1069 : :
1070 : : /* Extra sleep time during controller initialization */
1071 : : uint64_t sleep_timeout_tsc;
1072 : :
1073 : : /** Track all the processes manage this controller */
1074 : : TAILQ_HEAD(, spdk_nvme_ctrlr_process) active_procs;
1075 : :
1076 : :
1077 : : STAILQ_HEAD(, nvme_request) queued_aborts;
1078 : : uint32_t outstanding_aborts;
1079 : :
1080 : : uint32_t lock_depth;
1081 : :
1082 : : /* CB to notify the user when the ctrlr is removed/failed. */
1083 : : spdk_nvme_remove_cb remove_cb;
1084 : : void *cb_ctx;
1085 : :
1086 : : struct spdk_nvme_qpair *external_io_msgs_qpair;
1087 : : pthread_mutex_t external_io_msgs_lock;
1088 : : struct spdk_ring *external_io_msgs;
1089 : :
1090 : : STAILQ_HEAD(, nvme_io_msg_producer) io_producers;
1091 : :
1092 : : struct spdk_nvme_ana_page *ana_log_page;
1093 : : struct spdk_nvme_ana_group_descriptor *copied_ana_desc;
1094 : : uint32_t ana_log_page_size;
1095 : :
1096 : : /* scratchpad pointer that can be used to send data between two NVME_CTRLR_STATEs */
1097 : : void *tmp_ptr;
1098 : :
1099 : : /* maximum zone append size in bytes */
1100 : : uint32_t max_zone_append_size;
1101 : :
1102 : : /* PMR size in bytes */
1103 : : uint64_t pmr_size;
1104 : :
1105 : : /* Boot Partition Info */
1106 : : enum nvme_bp_write_state bp_ws;
1107 : : uint32_t bpid;
1108 : : spdk_nvme_cmd_cb bp_write_cb_fn;
1109 : : void *bp_write_cb_arg;
1110 : :
1111 : : /* Firmware Download */
1112 : : void *fw_payload;
1113 : : unsigned int fw_size_remaining;
1114 : : unsigned int fw_offset;
1115 : : unsigned int fw_transfer_size;
1116 : :
1117 : : /* Completed register operations */
1118 : : STAILQ_HEAD(, nvme_register_completion) register_operations;
1119 : :
1120 : : union spdk_nvme_cc_register process_init_cc;
1121 : :
1122 : : /* Authentication transaction ID */
1123 : : uint16_t auth_tid;
1124 : : /* Authentication sequence number */
1125 : : uint32_t auth_seqnum;
1126 : : };
1127 : :
1128 : : struct spdk_nvme_probe_ctx {
1129 : : struct spdk_nvme_transport_id trid;
1130 : : const struct spdk_nvme_ctrlr_opts *opts;
1131 : : void *cb_ctx;
1132 : : spdk_nvme_probe_cb probe_cb;
1133 : : spdk_nvme_attach_cb attach_cb;
1134 : : spdk_nvme_attach_fail_cb attach_fail_cb;
1135 : : spdk_nvme_remove_cb remove_cb;
1136 : : TAILQ_HEAD(, spdk_nvme_ctrlr) init_ctrlrs;
1137 : : };
1138 : :
1139 : : typedef void (*nvme_ctrlr_detach_cb)(struct spdk_nvme_ctrlr *ctrlr);
1140 : :
1141 : : enum nvme_ctrlr_detach_state {
1142 : : NVME_CTRLR_DETACH_SET_CC,
1143 : : NVME_CTRLR_DETACH_CHECK_CSTS,
1144 : : NVME_CTRLR_DETACH_GET_CSTS,
1145 : : NVME_CTRLR_DETACH_GET_CSTS_DONE,
1146 : : };
1147 : :
1148 : : struct nvme_ctrlr_detach_ctx {
1149 : : struct spdk_nvme_ctrlr *ctrlr;
1150 : : nvme_ctrlr_detach_cb cb_fn;
1151 : : uint64_t shutdown_start_tsc;
1152 : : uint32_t shutdown_timeout_ms;
1153 : : bool shutdown_complete;
1154 : : enum nvme_ctrlr_detach_state state;
1155 : : union spdk_nvme_csts_register csts;
1156 : : TAILQ_ENTRY(nvme_ctrlr_detach_ctx) link;
1157 : : };
1158 : :
1159 : : struct spdk_nvme_detach_ctx {
1160 : : TAILQ_HEAD(, nvme_ctrlr_detach_ctx) head;
1161 : : };
1162 : :
1163 : : struct nvme_driver {
1164 : : pthread_mutex_t lock;
1165 : :
1166 : : /** Multi-process shared attached controller list */
1167 : : TAILQ_HEAD(, spdk_nvme_ctrlr) shared_attached_ctrlrs;
1168 : :
1169 : : bool initialized;
1170 : : struct spdk_uuid default_extended_host_id;
1171 : :
1172 : : /** netlink socket fd for hotplug messages */
1173 : : int hotplug_fd;
1174 : : };
1175 : :
1176 : : #define nvme_ns_cmd_get_ext_io_opt(opts, field, defval) \
1177 : : ((opts) != NULL && offsetof(struct spdk_nvme_ns_cmd_ext_io_opts, field) + \
1178 : : sizeof((opts)->field) <= (opts)->size ? (opts)->field : (defval))
1179 : :
1180 : : extern struct nvme_driver *g_spdk_nvme_driver;
1181 : :
1182 : : int nvme_driver_init(void);
1183 : :
1184 : : #define nvme_delay usleep
1185 : :
1186 : : static inline bool
1187 : 2529531473 : nvme_qpair_is_admin_queue(struct spdk_nvme_qpair *qpair)
1188 : : {
1189 [ + - + - ]: 2529531473 : return qpair->id == 0;
1190 : : }
1191 : :
1192 : : static inline bool
1193 : : nvme_qpair_is_io_queue(struct spdk_nvme_qpair *qpair)
1194 : : {
1195 : : return qpair->id != 0;
1196 : : }
1197 : :
1198 : : static inline int
1199 : 205651167 : nvme_robust_mutex_lock(pthread_mutex_t *mtx)
1200 : : {
1201 [ + + ]: 205651167 : int rc = pthread_mutex_lock(mtx);
1202 : :
1203 : : #ifndef __FreeBSD__
1204 [ + + ]: 205562604 : if (rc == EOWNERDEAD) {
1205 [ # # ]: 0 : rc = pthread_mutex_consistent(mtx);
1206 : 0 : }
1207 : : #endif
1208 : :
1209 : 205651167 : return rc;
1210 : : }
1211 : :
1212 : : static inline int
1213 : 204658163 : nvme_ctrlr_lock(struct spdk_nvme_ctrlr *ctrlr)
1214 : : {
1215 : : int rc;
1216 : :
1217 [ + - ]: 204658163 : rc = nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
1218 [ + - ]: 204658163 : ctrlr->lock_depth++;
1219 : 204658163 : return rc;
1220 : : }
1221 : :
1222 : : static inline int
1223 : 205651159 : nvme_robust_mutex_unlock(pthread_mutex_t *mtx)
1224 : : {
1225 [ + + ]: 205651159 : return pthread_mutex_unlock(mtx);
1226 : : }
1227 : :
1228 : : static inline int
1229 : 204658155 : nvme_ctrlr_unlock(struct spdk_nvme_ctrlr *ctrlr)
1230 : : {
1231 [ + - ]: 204658155 : ctrlr->lock_depth--;
1232 [ + - ]: 204658155 : return nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
1233 : : }
1234 : :
1235 : : /* Poll group management functions. */
1236 : : int nvme_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair);
1237 : : int nvme_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair);
1238 : : void nvme_poll_group_write_disconnect_qpair_fd(struct spdk_nvme_poll_group *group);
1239 : :
1240 : : /* Admin functions */
1241 : : int nvme_ctrlr_cmd_identify(struct spdk_nvme_ctrlr *ctrlr,
1242 : : uint8_t cns, uint16_t cntid, uint32_t nsid,
1243 : : uint8_t csi, void *payload, size_t payload_size,
1244 : : spdk_nvme_cmd_cb cb_fn, void *cb_arg);
1245 : : int nvme_ctrlr_cmd_set_num_queues(struct spdk_nvme_ctrlr *ctrlr,
1246 : : uint32_t num_queues, spdk_nvme_cmd_cb cb_fn,
1247 : : void *cb_arg);
1248 : : int nvme_ctrlr_cmd_get_num_queues(struct spdk_nvme_ctrlr *ctrlr,
1249 : : spdk_nvme_cmd_cb cb_fn, void *cb_arg);
1250 : : int nvme_ctrlr_cmd_set_async_event_config(struct spdk_nvme_ctrlr *ctrlr,
1251 : : union spdk_nvme_feat_async_event_configuration config,
1252 : : spdk_nvme_cmd_cb cb_fn, void *cb_arg);
1253 : : int nvme_ctrlr_cmd_set_host_id(struct spdk_nvme_ctrlr *ctrlr, void *host_id, uint32_t host_id_size,
1254 : : spdk_nvme_cmd_cb cb_fn, void *cb_arg);
1255 : : int nvme_ctrlr_cmd_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
1256 : : struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
1257 : : int nvme_ctrlr_cmd_detach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
1258 : : struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
1259 : : int nvme_ctrlr_cmd_create_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns_data *payload,
1260 : : spdk_nvme_cmd_cb cb_fn, void *cb_arg);
1261 : : int nvme_ctrlr_cmd_doorbell_buffer_config(struct spdk_nvme_ctrlr *ctrlr,
1262 : : uint64_t prp1, uint64_t prp2,
1263 : : spdk_nvme_cmd_cb cb_fn, void *cb_arg);
1264 : : int nvme_ctrlr_cmd_delete_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, spdk_nvme_cmd_cb cb_fn,
1265 : : void *cb_arg);
1266 : : int nvme_ctrlr_cmd_format(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
1267 : : struct spdk_nvme_format *format, spdk_nvme_cmd_cb cb_fn, void *cb_arg);
1268 : : int nvme_ctrlr_cmd_fw_commit(struct spdk_nvme_ctrlr *ctrlr,
1269 : : const struct spdk_nvme_fw_commit *fw_commit,
1270 : : spdk_nvme_cmd_cb cb_fn, void *cb_arg);
1271 : : int nvme_ctrlr_cmd_fw_image_download(struct spdk_nvme_ctrlr *ctrlr,
1272 : : uint32_t size, uint32_t offset, void *payload,
1273 : : spdk_nvme_cmd_cb cb_fn, void *cb_arg);
1274 : : int nvme_ctrlr_cmd_sanitize(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid,
1275 : : struct spdk_nvme_sanitize *sanitize, uint32_t cdw11,
1276 : : spdk_nvme_cmd_cb cb_fn, void *cb_arg);
1277 : : void nvme_completion_poll_cb(void *arg, const struct spdk_nvme_cpl *cpl);
1278 : : int nvme_wait_for_completion(struct spdk_nvme_qpair *qpair,
1279 : : struct nvme_completion_poll_status *status);
1280 : : int nvme_wait_for_completion_robust_lock(struct spdk_nvme_qpair *qpair,
1281 : : struct nvme_completion_poll_status *status,
1282 : : pthread_mutex_t *robust_mutex);
1283 : : int nvme_wait_for_completion_timeout(struct spdk_nvme_qpair *qpair,
1284 : : struct nvme_completion_poll_status *status,
1285 : : uint64_t timeout_in_usecs);
1286 : : int nvme_wait_for_completion_robust_lock_timeout(struct spdk_nvme_qpair *qpair,
1287 : : struct nvme_completion_poll_status *status,
1288 : : pthread_mutex_t *robust_mutex,
1289 : : uint64_t timeout_in_usecs);
1290 : : int nvme_wait_for_completion_robust_lock_timeout_poll(struct spdk_nvme_qpair *qpair,
1291 : : struct nvme_completion_poll_status *status,
1292 : : pthread_mutex_t *robust_mutex);
1293 : :
1294 : : struct spdk_nvme_ctrlr_process *nvme_ctrlr_get_process(struct spdk_nvme_ctrlr *ctrlr,
1295 : : pid_t pid);
1296 : : struct spdk_nvme_ctrlr_process *nvme_ctrlr_get_current_process(struct spdk_nvme_ctrlr *ctrlr);
1297 : : int nvme_ctrlr_add_process(struct spdk_nvme_ctrlr *ctrlr, void *devhandle);
1298 : : void nvme_ctrlr_free_processes(struct spdk_nvme_ctrlr *ctrlr);
1299 : : struct spdk_pci_device *nvme_ctrlr_proc_get_devhandle(struct spdk_nvme_ctrlr *ctrlr);
1300 : :
1301 : : int nvme_ctrlr_probe(const struct spdk_nvme_transport_id *trid,
1302 : : struct spdk_nvme_probe_ctx *probe_ctx, void *devhandle);
1303 : :
1304 : : int nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr);
1305 : : void nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr *ctrlr);
1306 : : void nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr);
1307 : : void nvme_ctrlr_destruct_async(struct spdk_nvme_ctrlr *ctrlr,
1308 : : struct nvme_ctrlr_detach_ctx *ctx);
1309 : : int nvme_ctrlr_destruct_poll_async(struct spdk_nvme_ctrlr *ctrlr,
1310 : : struct nvme_ctrlr_detach_ctx *ctx);
1311 : : void nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove);
1312 : : int nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr);
1313 : : void nvme_ctrlr_disable(struct spdk_nvme_ctrlr *ctrlr);
1314 : : int nvme_ctrlr_disable_poll(struct spdk_nvme_ctrlr *ctrlr);
1315 : : void nvme_ctrlr_connected(struct spdk_nvme_probe_ctx *probe_ctx,
1316 : : struct spdk_nvme_ctrlr *ctrlr);
1317 : :
1318 : : int nvme_ctrlr_submit_admin_request(struct spdk_nvme_ctrlr *ctrlr,
1319 : : struct nvme_request *req);
1320 : : int nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cap_register *cap);
1321 : : int nvme_ctrlr_get_vs(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_vs_register *vs);
1322 : : int nvme_ctrlr_get_cmbsz(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cmbsz_register *cmbsz);
1323 : : int nvme_ctrlr_get_pmrcap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_pmrcap_register *pmrcap);
1324 : : int nvme_ctrlr_get_bpinfo(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bpinfo_register *bpinfo);
1325 : : int nvme_ctrlr_set_bprsel(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bprsel_register *bprsel);
1326 : : int nvme_ctrlr_set_bpmbl(struct spdk_nvme_ctrlr *ctrlr, uint64_t bpmbl_value);
1327 : : bool nvme_ctrlr_multi_iocs_enabled(struct spdk_nvme_ctrlr *ctrlr);
1328 : : void nvme_ctrlr_disconnect_qpair(struct spdk_nvme_qpair *qpair);
1329 : : void nvme_ctrlr_abort_queued_aborts(struct spdk_nvme_ctrlr *ctrlr);
1330 : : int nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id,
1331 : : struct spdk_nvme_ctrlr *ctrlr,
1332 : : enum spdk_nvme_qprio qprio,
1333 : : uint32_t num_requests, bool async);
1334 : : void nvme_qpair_deinit(struct spdk_nvme_qpair *qpair);
1335 : : void nvme_qpair_complete_error_reqs(struct spdk_nvme_qpair *qpair);
1336 : : int nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair,
1337 : : struct nvme_request *req);
1338 : : void nvme_qpair_abort_all_queued_reqs(struct spdk_nvme_qpair *qpair);
1339 : : uint32_t nvme_qpair_abort_queued_reqs_with_cbarg(struct spdk_nvme_qpair *qpair, void *cmd_cb_arg);
1340 : : void nvme_qpair_abort_queued_reqs(struct spdk_nvme_qpair *qpair);
1341 : : void nvme_qpair_resubmit_requests(struct spdk_nvme_qpair *qpair, uint32_t num_requests);
1342 : : int nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr);
1343 : : void nvme_ns_set_identify_data(struct spdk_nvme_ns *ns);
1344 : : void nvme_ns_set_id_desc_list_data(struct spdk_nvme_ns *ns);
1345 : : void nvme_ns_free_zns_specific_data(struct spdk_nvme_ns *ns);
1346 : : void nvme_ns_free_nvm_specific_data(struct spdk_nvme_ns *ns);
1347 : : void nvme_ns_free_iocs_specific_data(struct spdk_nvme_ns *ns);
1348 : : bool nvme_ns_has_supported_iocs_specific_data(struct spdk_nvme_ns *ns);
1349 : : int nvme_ns_construct(struct spdk_nvme_ns *ns, uint32_t id,
1350 : : struct spdk_nvme_ctrlr *ctrlr);
1351 : : void nvme_ns_destruct(struct spdk_nvme_ns *ns);
1352 : : int nvme_ns_cmd_zone_append_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
1353 : : void *buffer, void *metadata, uint64_t zslba,
1354 : : uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,
1355 : : uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag);
1356 : : int nvme_ns_cmd_zone_appendv_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
1357 : : uint64_t zslba, uint32_t lba_count,
1358 : : spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags,
1359 : : spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
1360 : : spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata,
1361 : : uint16_t apptag_mask, uint16_t apptag);
1362 : :
1363 : : int nvme_fabric_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value);
1364 : : int nvme_fabric_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value);
1365 : : int nvme_fabric_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value);
1366 : : int nvme_fabric_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value);
1367 : : int nvme_fabric_ctrlr_set_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
1368 : : uint32_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
1369 : : int nvme_fabric_ctrlr_set_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
1370 : : uint64_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
1371 : : int nvme_fabric_ctrlr_get_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
1372 : : spdk_nvme_reg_cb cb_fn, void *cb_arg);
1373 : : int nvme_fabric_ctrlr_get_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
1374 : : spdk_nvme_reg_cb cb_fn, void *cb_arg);
1375 : : int nvme_fabric_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, bool direct_connect);
1376 : : int nvme_fabric_ctrlr_discover(struct spdk_nvme_ctrlr *ctrlr,
1377 : : struct spdk_nvme_probe_ctx *probe_ctx);
1378 : : int nvme_fabric_qpair_connect(struct spdk_nvme_qpair *qpair, uint32_t num_entries);
1379 : : int nvme_fabric_qpair_connect_async(struct spdk_nvme_qpair *qpair, uint32_t num_entries);
1380 : : int nvme_fabric_qpair_connect_poll(struct spdk_nvme_qpair *qpair);
1381 : : bool nvme_fabric_qpair_auth_required(struct spdk_nvme_qpair *qpair);
1382 : : int nvme_fabric_qpair_authenticate_async(struct spdk_nvme_qpair *qpair);
1383 : : int nvme_fabric_qpair_authenticate_poll(struct spdk_nvme_qpair *qpair);
1384 : :
1385 : : typedef int (*spdk_nvme_parse_ana_log_page_cb)(
1386 : : const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg);
1387 : : int nvme_ctrlr_parse_ana_log_page(struct spdk_nvme_ctrlr *ctrlr,
1388 : : spdk_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg);
1389 : :
1390 : : static inline void
1391 : 87279424 : nvme_request_clear(struct nvme_request *req)
1392 : : {
1393 : : /*
1394 : : * Only memset/zero fields that need it. All other fields
1395 : : * will be initialized appropriately either later in this
1396 : : * function, or before they are needed later in the
1397 : : * submission patch. For example, the children
1398 : : * TAILQ_ENTRY and following members are
1399 : : * only used as part of I/O splitting so we avoid
1400 : : * memsetting them until it is actually needed.
1401 : : * They will be initialized in nvme_request_add_child()
1402 : : * if the request is split.
1403 : : */
1404 [ + + ]: 87279424 : memset(req, 0, offsetof(struct nvme_request, payload_size));
1405 : 87279424 : }
1406 : :
1407 : : #define NVME_INIT_REQUEST(req, _cb_fn, _cb_arg, _payload, _payload_size, _md_size) \
1408 : : do { \
1409 : : nvme_request_clear(req); \
1410 : : req->cb_fn = _cb_fn; \
1411 : : req->cb_arg = _cb_arg; \
1412 : : req->payload = _payload; \
1413 : : req->payload_size = _payload_size; \
1414 : : req->md_size = _md_size; \
1415 : : req->pid = g_spdk_nvme_pid; \
1416 : : req->submit_tick = 0; \
1417 : : req->accel_sequence = NULL; \
1418 : : } while (0);
1419 : :
1420 : : static inline struct nvme_request *
1421 : 88119251 : nvme_allocate_request(struct spdk_nvme_qpair *qpair,
1422 : : const struct nvme_payload *payload, uint32_t payload_size, uint32_t md_size,
1423 : : spdk_nvme_cmd_cb cb_fn, void *cb_arg)
1424 : : {
1425 : : struct nvme_request *req;
1426 : :
1427 [ + - + - : 88119251 : req = STAILQ_FIRST(&qpair->free_req);
+ - ]
1428 [ + + ]: 88119251 : if (req == NULL) {
1429 : 851638 : return req;
1430 : : }
1431 : :
1432 [ + + + - : 87267613 : STAILQ_REMOVE_HEAD(&qpair->free_req, stailq);
+ - + - +
- + - + -
+ - + - -
+ # # # #
# # # # #
# ]
1433 [ + - ]: 87267613 : qpair->num_outstanding_reqs++;
1434 : :
1435 [ + - + - : 87267613 : NVME_INIT_REQUEST(req, cb_fn, cb_arg, *payload, payload_size, md_size);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- ]
1436 : :
1437 : 87267613 : return req;
1438 : 1663198 : }
1439 : :
1440 : : static inline struct nvme_request *
1441 : 5240325 : nvme_allocate_request_contig(struct spdk_nvme_qpair *qpair,
1442 : : void *buffer, uint32_t payload_size,
1443 : : spdk_nvme_cmd_cb cb_fn, void *cb_arg)
1444 : : {
1445 : 387184 : struct nvme_payload payload;
1446 : :
1447 : 5240325 : payload = NVME_PAYLOAD_CONTIG(buffer, NULL);
1448 : :
1449 : 5240325 : return nvme_allocate_request(qpair, &payload, payload_size, 0, cb_fn, cb_arg);
1450 : : }
1451 : :
1452 : : static inline struct nvme_request *
1453 : 2445136 : nvme_allocate_request_null(struct spdk_nvme_qpair *qpair, spdk_nvme_cmd_cb cb_fn, void *cb_arg)
1454 : : {
1455 : 2445136 : return nvme_allocate_request_contig(qpair, NULL, 0, cb_fn, cb_arg);
1456 : : }
1457 : :
1458 : : struct nvme_request *nvme_allocate_request_user_copy(struct spdk_nvme_qpair *qpair,
1459 : : void *buffer, uint32_t payload_size,
1460 : : spdk_nvme_cmd_cb cb_fn, void *cb_arg, bool host_to_controller);
1461 : :
1462 : : static inline void
1463 : 87279140 : _nvme_free_request(struct nvme_request *req, struct spdk_nvme_qpair *qpair)
1464 : : {
1465 [ + + # # ]: 87279140 : assert(req != NULL);
1466 [ + + + - : 87279140 : assert(req->num_children == 0);
+ - # # ]
1467 [ + + # # ]: 87279140 : assert(qpair != NULL);
1468 : :
1469 : : /* The reserved_req does not go in the free_req STAILQ - it is
1470 : : * saved only for use with a FABRICS/CONNECT command.
1471 : : */
1472 [ + + + - : 87279140 : if (spdk_likely(qpair->reserved_req != req)) {
+ + ]
1473 [ + + + - : 87267286 : STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq);
+ - + - +
- + - + -
# # # # #
# # # # #
+ - + - +
- ]
1474 : :
1475 [ + + + - : 87267286 : assert(qpair->num_outstanding_reqs > 0);
+ - # # ]
1476 [ + - ]: 87267286 : qpair->num_outstanding_reqs--;
1477 : 1663056 : }
1478 : 87279140 : }
1479 : :
1480 : : static inline void
1481 : 29122129 : nvme_free_request(struct nvme_request *req)
1482 : : {
1483 [ # # # # ]: 29122129 : _nvme_free_request(req, req->qpair);
1484 : 29122129 : }
1485 : :
1486 : : static inline void
1487 : 58157096 : nvme_complete_request(spdk_nvme_cmd_cb cb_fn, void *cb_arg, struct spdk_nvme_qpair *qpair,
1488 : : struct nvme_request *req, struct spdk_nvme_cpl *cpl)
1489 : : {
1490 : 11713948 : struct spdk_nvme_cpl err_cpl;
1491 : : struct nvme_error_cmd *cmd;
1492 : :
1493 [ + + + - : 58157096 : if (spdk_unlikely(req->accel_sequence != NULL)) {
+ - ]
1494 [ # # # # : 0 : struct spdk_nvme_poll_group *pg = qpair->poll_group->group;
# # # # ]
1495 : :
1496 : : /* Transports are required to execute the sequence and clear req->accel_sequence.
1497 : : * If it's left non-NULL it must mean the request is failed. */
1498 [ # # # # : 0 : assert(spdk_nvme_cpl_is_error(cpl));
# # # # #
# # # # #
# # # # ]
1499 [ # # # # : 0 : pg->accel_fn_table.abort_sequence(req->accel_sequence);
# # # # #
# # # #
# ]
1500 [ # # # # ]: 0 : req->accel_sequence = NULL;
1501 : 0 : }
1502 : :
1503 : : /* error injection at completion path,
1504 : : * only inject for successful completed commands
1505 : : */
1506 [ + + + + : 58157096 : if (spdk_unlikely(!TAILQ_EMPTY(&qpair->err_cmd_head) &&
+ + + + #
# # # # #
# # # # #
# # # +
- ]
1507 : : !spdk_nvme_cpl_is_error(cpl))) {
1508 [ + + # # : 1662215 : TAILQ_FOREACH(cmd, &qpair->err_cmd_head, link) {
# # # # #
# # # #
# ]
1509 : :
1510 [ + + + + : 831113 : if (cmd->do_not_submit) {
# # # # ]
1511 : 831087 : continue;
1512 : : }
1513 : :
1514 [ + - + + : 26 : if ((cmd->opc == req->cmd.opc) && cmd->err_count) {
# # # # #
# # # # #
# # ]
1515 : :
1516 : 11 : err_cpl = *cpl;
1517 [ # # # # : 11 : err_cpl.status.sct = cmd->status.sct;
# # # # ]
1518 [ # # # # : 11 : err_cpl.status.sc = cmd->status.sc;
# # # # ]
1519 : :
1520 : 11 : cpl = &err_cpl;
1521 [ # # ]: 11 : cmd->err_count--;
1522 : 11 : break;
1523 : : }
1524 : 2 : }
1525 : 17 : }
1526 : :
1527 : : /* For PCIe completions, we want to avoid touching the req itself to avoid
1528 : : * dependencies on loading those cachelines. So call the internal helper
1529 : : * function instead using the qpair that was passed by the caller, instead
1530 : : * of getting it from the req.
1531 : : */
1532 : 58157096 : _nvme_free_request(req, qpair);
1533 : :
1534 [ + + ]: 58157096 : if (spdk_likely(cb_fn)) {
1535 [ - + + - ]: 58156947 : cb_fn(cb_arg, cpl);
1536 : 1664418 : }
1537 : 58157096 : }
1538 : :
1539 : : static inline void
1540 : 97154 : nvme_cleanup_user_req(struct nvme_request *req)
1541 : : {
1542 [ + + + - : 97154 : if (req->user_buffer && req->payload_size) {
+ + + - +
- + - ]
1543 [ + - + - : 85853 : spdk_free(req->payload.contig_or_cb_arg);
+ - ]
1544 [ + - + - ]: 85853 : req->user_buffer = NULL;
1545 : 3425 : }
1546 : :
1547 [ + - + - ]: 97154 : req->user_cb_arg = NULL;
1548 [ + - + - ]: 97154 : req->user_cb_fn = NULL;
1549 : 97154 : }
1550 : :
1551 : : static inline bool
1552 : 33139997 : nvme_request_abort_match(struct nvme_request *req, void *cmd_cb_arg)
1553 : : {
1554 [ + + # # : 33139999 : return req->cb_arg == cmd_cb_arg ||
# # ]
1555 [ + + + - : 65703937 : req->user_cb_arg == cmd_cb_arg ||
# # ]
1556 [ + + - - : 32563942 : (req->parent != NULL && req->parent->cb_arg == cmd_cb_arg);
# # # # #
# # # #
# ]
1557 : : }
1558 : :
1559 : : static inline void
1560 : 86195 : nvme_qpair_set_state(struct spdk_nvme_qpair *qpair, enum nvme_qpair_state state)
1561 : : {
1562 [ + - ]: 86195 : qpair->state = state;
1563 [ + + ]: 86195 : if (state == NVME_QPAIR_ENABLED) {
1564 [ + - ]: 10110 : qpair->is_new_qpair = false;
1565 : 2482 : }
1566 : 86195 : }
1567 : :
1568 : : static inline enum nvme_qpair_state
1569 : 2333241631 : nvme_qpair_get_state(struct spdk_nvme_qpair *qpair) {
1570 [ + - ]: 2333241631 : return qpair->state;
1571 : : }
1572 : :
1573 : : static inline void
1574 : 30853723 : nvme_request_remove_child(struct nvme_request *parent, struct nvme_request *child)
1575 : : {
1576 [ + + # # ]: 30853723 : assert(parent != NULL);
1577 [ + + # # ]: 30853723 : assert(child != NULL);
1578 [ + + # # : 30853723 : assert(child->parent == parent);
# # # # ]
1579 [ - + # # : 30853723 : assert(parent->num_children != 0);
# # # # ]
1580 : :
1581 [ # # ]: 30853723 : parent->num_children--;
1582 [ # # # # ]: 30853723 : child->parent = NULL;
1583 [ + + # # : 30853723 : TAILQ_REMOVE(&parent->children, child, child_tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1584 : 30853723 : }
1585 : :
1586 : : static inline void
1587 : 1346255 : nvme_cb_complete_child(void *child_arg, const struct spdk_nvme_cpl *cpl)
1588 : : {
1589 : 1346255 : struct nvme_request *child = child_arg;
1590 [ # # # # ]: 1346255 : struct nvme_request *parent = child->parent;
1591 : :
1592 : 1346255 : nvme_request_remove_child(parent, child);
1593 : :
1594 [ + - - + : 1346255 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
1595 [ # # # # : 0 : memcpy(&parent->parent_status, cpl, sizeof(*cpl));
# # ]
1596 : 0 : }
1597 : :
1598 [ + + # # : 1346255 : if (parent->num_children == 0) {
# # ]
1599 [ # # # # : 519667 : nvme_complete_request(parent->cb_fn, parent->cb_arg, parent->qpair,
# # # # #
# # # ]
1600 [ # # ]: 3 : parent, &parent->parent_status);
1601 : 3 : }
1602 : 1346255 : }
1603 : :
1604 : : static inline void
1605 : 30408030 : nvme_request_add_child(struct nvme_request *parent, struct nvme_request *child)
1606 : : {
1607 [ + + # # : 30408030 : assert(parent->num_children != UINT16_MAX);
# # # # ]
1608 : :
1609 [ + + # # : 30408030 : if (parent->num_children == 0) {
# # ]
1610 : : /*
1611 : : * Defer initialization of the children TAILQ since it falls
1612 : : * on a separate cacheline. This ensures we do not touch this
1613 : : * cacheline except on request splitting cases, which are
1614 : : * relatively rare.
1615 : : */
1616 [ # # # # : 577268 : TAILQ_INIT(&parent->children);
# # # # #
# # # # #
# # ]
1617 [ # # # # ]: 577268 : parent->parent = NULL;
1618 [ - + # # ]: 577268 : memset(&parent->parent_status, 0, sizeof(struct spdk_nvme_cpl));
1619 : 18 : }
1620 : :
1621 [ # # ]: 30408030 : parent->num_children++;
1622 [ # # # # : 30408030 : TAILQ_INSERT_TAIL(&parent->children, child, child_tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1623 [ # # # # ]: 30408030 : child->parent = parent;
1624 [ # # # # ]: 30408030 : child->cb_fn = nvme_cb_complete_child;
1625 [ # # # # ]: 30408030 : child->cb_arg = child;
1626 : 30408030 : }
1627 : :
1628 : : static inline void
1629 : 29119401 : nvme_request_free_children(struct nvme_request *req)
1630 : : {
1631 : : struct nvme_request *child, *tmp;
1632 : :
1633 [ + + # # : 29119401 : if (req->num_children == 0) {
# # ]
1634 : 29061809 : return;
1635 : : }
1636 : :
1637 : : /* free all child nvme_request */
1638 [ + + + + : 29119339 : TAILQ_FOREACH_SAFE(child, &req->children, child_tailq, tmp) {
# # # # #
# # # # #
# # ]
1639 : 29061747 : nvme_request_remove_child(req, child);
1640 : 29061747 : nvme_request_free_children(child);
1641 : 29061747 : nvme_free_request(child);
1642 : 50 : }
1643 : 69 : }
1644 : :
1645 : : int nvme_request_check_timeout(struct nvme_request *req, uint16_t cid,
1646 : : struct spdk_nvme_ctrlr_process *active_proc, uint64_t now_tick);
1647 : : uint64_t nvme_get_quirks(const struct spdk_pci_id *id);
1648 : :
1649 : : int nvme_robust_mutex_init_shared(pthread_mutex_t *mtx);
1650 : : int nvme_robust_mutex_init_recursive_shared(pthread_mutex_t *mtx);
1651 : :
1652 : : bool nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl);
1653 : :
1654 : : struct spdk_nvme_ctrlr *nvme_get_ctrlr_by_trid_unsafe(
1655 : : const struct spdk_nvme_transport_id *trid, const char *hostnqn);
1656 : :
1657 : : const struct spdk_nvme_transport *nvme_get_transport(const char *transport_name);
1658 : : const struct spdk_nvme_transport *nvme_get_first_transport(void);
1659 : : const struct spdk_nvme_transport *nvme_get_next_transport(const struct spdk_nvme_transport
1660 : : *transport);
1661 : : void nvme_ctrlr_update_namespaces(struct spdk_nvme_ctrlr *ctrlr);
1662 : :
1663 : : /* Transport specific functions */
1664 : : struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
1665 : : const struct spdk_nvme_ctrlr_opts *opts,
1666 : : void *devhandle);
1667 : : int nvme_transport_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr);
1668 : : int nvme_transport_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, bool direct_connect);
1669 : : int nvme_transport_ctrlr_scan_attached(struct spdk_nvme_probe_ctx *probe_ctx);
1670 : : int nvme_transport_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr);
1671 : : int nvme_transport_ctrlr_ready(struct spdk_nvme_ctrlr *ctrlr);
1672 : : int nvme_transport_ctrlr_enable_interrupts(struct spdk_nvme_ctrlr *ctrlr);
1673 : : int nvme_transport_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value);
1674 : : int nvme_transport_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value);
1675 : : int nvme_transport_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value);
1676 : : int nvme_transport_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value);
1677 : : int nvme_transport_ctrlr_set_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
1678 : : uint32_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
1679 : : int nvme_transport_ctrlr_set_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
1680 : : uint64_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg);
1681 : : int nvme_transport_ctrlr_get_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
1682 : : spdk_nvme_reg_cb cb_fn, void *cb_arg);
1683 : : int nvme_transport_ctrlr_get_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
1684 : : spdk_nvme_reg_cb cb_fn, void *cb_arg);
1685 : : uint32_t nvme_transport_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr);
1686 : : uint16_t nvme_transport_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr);
1687 : : struct spdk_nvme_qpair *nvme_transport_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
1688 : : uint16_t qid, const struct spdk_nvme_io_qpair_opts *opts);
1689 : : int nvme_transport_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr);
1690 : : void *nvme_transport_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size);
1691 : : int nvme_transport_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr);
1692 : : int nvme_transport_ctrlr_enable_pmr(struct spdk_nvme_ctrlr *ctrlr);
1693 : : int nvme_transport_ctrlr_disable_pmr(struct spdk_nvme_ctrlr *ctrlr);
1694 : : void *nvme_transport_ctrlr_map_pmr(struct spdk_nvme_ctrlr *ctrlr, size_t *size);
1695 : : int nvme_transport_ctrlr_unmap_pmr(struct spdk_nvme_ctrlr *ctrlr);
1696 : : void nvme_transport_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr,
1697 : : struct spdk_nvme_qpair *qpair);
1698 : : int nvme_transport_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr,
1699 : : struct spdk_nvme_qpair *qpair);
1700 : : void nvme_transport_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr,
1701 : : struct spdk_nvme_qpair *qpair);
1702 : : void nvme_transport_ctrlr_disconnect_qpair_done(struct spdk_nvme_qpair *qpair);
1703 : : int nvme_transport_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
1704 : : struct spdk_memory_domain **domains, int array_size);
1705 : : void nvme_transport_qpair_abort_reqs(struct spdk_nvme_qpair *qpair);
1706 : : int nvme_transport_qpair_reset(struct spdk_nvme_qpair *qpair);
1707 : : int nvme_transport_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req);
1708 : : int nvme_transport_qpair_get_fd(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
1709 : : struct spdk_event_handler_opts *opts);
1710 : : int32_t nvme_transport_qpair_process_completions(struct spdk_nvme_qpair *qpair,
1711 : : uint32_t max_completions);
1712 : : void nvme_transport_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair);
1713 : : int nvme_transport_qpair_iterate_requests(struct spdk_nvme_qpair *qpair,
1714 : : int (*iter_fn)(struct nvme_request *req, void *arg),
1715 : : void *arg);
1716 : : int nvme_transport_qpair_authenticate(struct spdk_nvme_qpair *qpair);
1717 : :
1718 : : struct spdk_nvme_transport_poll_group *nvme_transport_poll_group_create(
1719 : : const struct spdk_nvme_transport *transport);
1720 : : struct spdk_nvme_transport_poll_group *nvme_transport_qpair_get_optimal_poll_group(
1721 : : const struct spdk_nvme_transport *transport,
1722 : : struct spdk_nvme_qpair *qpair);
1723 : : int nvme_transport_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup,
1724 : : struct spdk_nvme_qpair *qpair);
1725 : : int nvme_transport_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,
1726 : : struct spdk_nvme_qpair *qpair);
1727 : : int nvme_transport_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair);
1728 : : int nvme_transport_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair);
1729 : : int64_t nvme_transport_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,
1730 : : uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb);
1731 : : void nvme_transport_poll_group_check_disconnected_qpairs(
1732 : : struct spdk_nvme_transport_poll_group *tgroup,
1733 : : spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb);
1734 : : int nvme_transport_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup);
1735 : : int nvme_transport_poll_group_get_stats(struct spdk_nvme_transport_poll_group *tgroup,
1736 : : struct spdk_nvme_transport_poll_group_stat **stats);
1737 : : void nvme_transport_poll_group_free_stats(struct spdk_nvme_transport_poll_group *tgroup,
1738 : : struct spdk_nvme_transport_poll_group_stat *stats);
1739 : : enum spdk_nvme_transport_type nvme_transport_get_trtype(const struct spdk_nvme_transport
1740 : : *transport);
1741 : : /*
1742 : : * Below ref related functions must be called with the global
1743 : : * driver lock held for the multi-process condition.
1744 : : * Within these functions, the per ctrlr ctrlr_lock is also
1745 : : * acquired for the multi-thread condition.
1746 : : */
1747 : : void nvme_ctrlr_proc_get_ref(struct spdk_nvme_ctrlr *ctrlr);
1748 : : void nvme_ctrlr_proc_put_ref(struct spdk_nvme_ctrlr *ctrlr);
1749 : : int nvme_ctrlr_get_ref_count(struct spdk_nvme_ctrlr *ctrlr);
1750 : :
1751 : : int nvme_ctrlr_reinitialize_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair);
1752 : : int nvme_parse_addr(struct sockaddr_storage *sa, int family,
1753 : : const char *addr, const char *service, long int *port);
1754 : : int nvme_get_default_hostnqn(char *buf, int len);
1755 : :
1756 : : static inline bool
1757 : 2710370 : _is_page_aligned(uint64_t address, uint64_t page_size)
1758 : : {
1759 : 2710370 : return (address & (page_size - 1)) == 0;
1760 : : }
1761 : :
1762 : : #endif /* __NVME_INTERNAL_H__ */
|