Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2021 Intel Corporation. All rights reserved.
3 : * Copyright (c) 2021 Mellanox Technologies LTD. All rights reserved.
4 : * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : */
6 :
7 : /*
8 : * NVMe over PCIe common library
9 : */
10 :
11 : #include "spdk/stdinc.h"
12 : #include "spdk/likely.h"
13 : #include "spdk/string.h"
14 : #include "nvme_internal.h"
15 : #include "nvme_pcie_internal.h"
16 : #include "spdk/trace.h"
17 :
18 : #include "spdk_internal/trace_defs.h"
19 :
20 : __thread struct nvme_pcie_ctrlr *g_thread_mmio_ctrlr = NULL;
21 :
22 : static struct spdk_nvme_pcie_stat g_dummy_stat = {};
23 :
24 : static void nvme_pcie_fail_request_bad_vtophys(struct spdk_nvme_qpair *qpair,
25 : struct nvme_tracker *tr);
26 :
27 : static inline uint64_t
28 2093 : nvme_pcie_vtophys(struct spdk_nvme_ctrlr *ctrlr, const void *buf, uint64_t *size)
29 : {
30 2093 : if (spdk_likely(ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE)) {
31 2086 : return spdk_vtophys(buf, size);
32 : } else {
33 : /* vfio-user address translation with IOVA=VA mode */
34 7 : return (uint64_t)(uintptr_t)buf;
35 : }
36 2093 : }
37 :
38 : int
39 6 : nvme_pcie_qpair_reset(struct spdk_nvme_qpair *qpair)
40 : {
41 6 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
42 6 : uint32_t i;
43 :
44 : /* all head/tail vals are set to 0 */
45 6 : pqpair->last_sq_tail = pqpair->sq_tail = pqpair->sq_head = pqpair->cq_head = 0;
46 :
47 : /*
48 : * First time through the completion queue, HW will set phase
49 : * bit on completions to 1. So set this to 1 here, indicating
50 : * we're looking for a 1 to know which entries have completed.
51 : * we'll toggle the bit each time when the completion queue
52 : * rolls over.
53 : */
54 6 : pqpair->flags.phase = 1;
55 46 : for (i = 0; i < pqpair->num_entries; i++) {
56 40 : pqpair->cpl[i].status.p = 0;
57 40 : }
58 :
59 6 : return 0;
60 6 : }
61 :
62 : int
63 0 : nvme_pcie_qpair_get_fd(struct spdk_nvme_qpair *qpair, struct spdk_event_handler_opts *opts)
64 : {
65 0 : struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
66 0 : struct spdk_pci_device *devhandle = nvme_ctrlr_proc_get_devhandle(ctrlr);
67 :
68 0 : assert(devhandle != NULL);
69 0 : if (!ctrlr->opts.enable_interrupts) {
70 0 : return -1;
71 : }
72 :
73 0 : if (!opts) {
74 0 : return spdk_pci_device_get_interrupt_efd_by_index(devhandle, qpair->id);
75 : }
76 :
77 0 : if (!SPDK_FIELD_VALID(opts, fd_type, opts->opts_size)) {
78 0 : return -EINVAL;
79 : }
80 :
81 0 : spdk_fd_group_get_default_event_handler_opts(opts, opts->opts_size);
82 0 : opts->fd_type = SPDK_FD_TYPE_EVENTFD;
83 :
84 0 : return spdk_pci_device_get_interrupt_efd_by_index(devhandle, qpair->id);
85 0 : }
86 :
87 : static void
88 27 : nvme_qpair_construct_tracker(struct nvme_tracker *tr, uint16_t cid, uint64_t phys_addr)
89 : {
90 27 : tr->prp_sgl_bus_addr = phys_addr + offsetof(struct nvme_tracker, u.prp);
91 27 : tr->cid = cid;
92 27 : tr->req = NULL;
93 27 : }
94 :
95 : static void *
96 4 : nvme_pcie_ctrlr_alloc_cmb(struct spdk_nvme_ctrlr *ctrlr, uint64_t size, uint64_t alignment,
97 : uint64_t *phys_addr)
98 : {
99 4 : struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
100 4 : uintptr_t addr;
101 :
102 4 : if (pctrlr->cmb.mem_register_addr != NULL) {
103 : /* BAR is mapped for data */
104 1 : return NULL;
105 : }
106 :
107 3 : addr = (uintptr_t)pctrlr->cmb.bar_va + pctrlr->cmb.current_offset;
108 3 : addr = (addr + (alignment - 1)) & ~(alignment - 1);
109 :
110 : /* CMB may only consume part of the BAR, calculate accordingly */
111 3 : if (addr + size > ((uintptr_t)pctrlr->cmb.bar_va + pctrlr->cmb.size)) {
112 1 : SPDK_ERRLOG("Tried to allocate past valid CMB range!\n");
113 1 : return NULL;
114 : }
115 2 : *phys_addr = pctrlr->cmb.bar_pa + addr - (uintptr_t)pctrlr->cmb.bar_va;
116 :
117 2 : pctrlr->cmb.current_offset = (addr + size) - (uintptr_t)pctrlr->cmb.bar_va;
118 :
119 2 : return (void *)addr;
120 4 : }
121 :
122 : int
123 4 : nvme_pcie_qpair_construct(struct spdk_nvme_qpair *qpair,
124 : const struct spdk_nvme_io_qpair_opts *opts)
125 : {
126 4 : struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
127 4 : struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
128 4 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
129 4 : struct nvme_tracker *tr;
130 4 : uint16_t i;
131 4 : uint16_t num_trackers;
132 4 : size_t page_align = sysconf(_SC_PAGESIZE);
133 4 : size_t queue_align, queue_len;
134 4 : uint32_t flags = SPDK_MALLOC_DMA;
135 4 : int32_t numa_id;
136 4 : uint64_t sq_paddr = 0;
137 4 : uint64_t cq_paddr = 0;
138 :
139 4 : if (opts) {
140 2 : pqpair->sq_vaddr = opts->sq.vaddr;
141 2 : pqpair->cq_vaddr = opts->cq.vaddr;
142 2 : pqpair->flags.disable_pcie_sgl_merge = opts->disable_pcie_sgl_merge;
143 2 : sq_paddr = opts->sq.paddr;
144 2 : cq_paddr = opts->cq.paddr;
145 2 : }
146 :
147 4 : pqpair->retry_count = ctrlr->opts.transport_retry_count;
148 :
149 : /*
150 : * Limit the maximum number of completions to return per call to prevent wraparound,
151 : * and calculate how many trackers can be submitted at once without overflowing the
152 : * completion queue.
153 : */
154 4 : pqpair->max_completions_cap = pqpair->num_entries / 4;
155 4 : pqpair->max_completions_cap = spdk_max(pqpair->max_completions_cap, NVME_MIN_COMPLETIONS);
156 4 : pqpair->max_completions_cap = spdk_min(pqpair->max_completions_cap, NVME_MAX_COMPLETIONS);
157 4 : num_trackers = pqpair->num_entries - pqpair->max_completions_cap;
158 :
159 4 : SPDK_INFOLOG(nvme, "max_completions_cap = %" PRIu16 " num_trackers = %" PRIu16 "\n",
160 : pqpair->max_completions_cap, num_trackers);
161 :
162 4 : assert(num_trackers != 0);
163 :
164 4 : pqpair->sq_in_cmb = false;
165 :
166 4 : if (nvme_qpair_is_admin_queue(&pqpair->qpair)) {
167 1 : flags |= SPDK_MALLOC_SHARE;
168 1 : }
169 :
170 : /* cmd and cpl rings must be aligned on page size boundaries. */
171 4 : if (ctrlr->opts.use_cmb_sqs) {
172 1 : pqpair->cmd = nvme_pcie_ctrlr_alloc_cmb(ctrlr, pqpair->num_entries * sizeof(struct spdk_nvme_cmd),
173 1 : page_align, &pqpair->cmd_bus_addr);
174 1 : if (pqpair->cmd != NULL) {
175 1 : pqpair->sq_in_cmb = true;
176 1 : }
177 1 : }
178 :
179 4 : if (pqpair->sq_in_cmb == false) {
180 3 : if (pqpair->sq_vaddr) {
181 1 : pqpair->cmd = pqpair->sq_vaddr;
182 1 : } else {
183 : /* To ensure physical address contiguity we make each ring occupy
184 : * a single hugepage only. See MAX_IO_QUEUE_ENTRIES.
185 : */
186 2 : queue_len = pqpair->num_entries * sizeof(struct spdk_nvme_cmd);
187 2 : queue_align = spdk_max(spdk_align32pow2(queue_len), page_align);
188 2 : pqpair->cmd = spdk_zmalloc(queue_len, queue_align, NULL, SPDK_ENV_NUMA_ID_ANY, flags);
189 2 : if (pqpair->cmd == NULL) {
190 0 : SPDK_ERRLOG("alloc qpair_cmd failed\n");
191 0 : return -ENOMEM;
192 : }
193 : }
194 3 : if (sq_paddr) {
195 1 : assert(pqpair->sq_vaddr != NULL);
196 1 : pqpair->cmd_bus_addr = sq_paddr;
197 1 : } else {
198 2 : pqpair->cmd_bus_addr = nvme_pcie_vtophys(ctrlr, pqpair->cmd, NULL);
199 2 : if (pqpair->cmd_bus_addr == SPDK_VTOPHYS_ERROR) {
200 0 : SPDK_ERRLOG("spdk_vtophys(pqpair->cmd) failed\n");
201 0 : return -EFAULT;
202 : }
203 : }
204 3 : }
205 :
206 4 : if (pqpair->cq_vaddr) {
207 2 : pqpair->cpl = pqpair->cq_vaddr;
208 2 : } else {
209 2 : queue_len = pqpair->num_entries * sizeof(struct spdk_nvme_cpl);
210 2 : queue_align = spdk_max(spdk_align32pow2(queue_len), page_align);
211 2 : numa_id = spdk_nvme_ctrlr_get_numa_id(ctrlr);
212 2 : pqpair->cpl = spdk_zmalloc(queue_len, queue_align, NULL, numa_id, flags);
213 2 : if (pqpair->cpl == NULL) {
214 0 : SPDK_ERRLOG("alloc qpair_cpl failed\n");
215 0 : return -ENOMEM;
216 : }
217 : }
218 4 : if (cq_paddr) {
219 2 : assert(pqpair->cq_vaddr != NULL);
220 2 : pqpair->cpl_bus_addr = cq_paddr;
221 2 : } else {
222 2 : pqpair->cpl_bus_addr = nvme_pcie_vtophys(ctrlr, pqpair->cpl, NULL);
223 2 : if (pqpair->cpl_bus_addr == SPDK_VTOPHYS_ERROR) {
224 0 : SPDK_ERRLOG("spdk_vtophys(pqpair->cpl) failed\n");
225 0 : return -EFAULT;
226 : }
227 : }
228 :
229 4 : pqpair->sq_tdbl = pctrlr->doorbell_base + (2 * qpair->id + 0) * pctrlr->doorbell_stride_u32;
230 4 : pqpair->cq_hdbl = pctrlr->doorbell_base + (2 * qpair->id + 1) * pctrlr->doorbell_stride_u32;
231 :
232 : /*
233 : * Reserve space for all of the trackers in a single allocation.
234 : * struct nvme_tracker must be padded so that its size is already a power of 2.
235 : * This ensures the PRP list embedded in the nvme_tracker object will not span a
236 : * 4KB boundary, while allowing access to trackers in tr[] via normal array indexing.
237 : */
238 4 : pqpair->tr = spdk_zmalloc(num_trackers * sizeof(*tr), sizeof(*tr), NULL,
239 : SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_SHARE);
240 4 : if (pqpair->tr == NULL) {
241 0 : SPDK_ERRLOG("nvme_tr failed\n");
242 0 : return -ENOMEM;
243 : }
244 :
245 4 : TAILQ_INIT(&pqpair->free_tr);
246 4 : TAILQ_INIT(&pqpair->outstanding_tr);
247 4 : pqpair->qpair.queue_depth = 0;
248 :
249 31 : for (i = 0; i < num_trackers; i++) {
250 27 : tr = &pqpair->tr[i];
251 27 : nvme_qpair_construct_tracker(tr, i, nvme_pcie_vtophys(ctrlr, tr, NULL));
252 27 : TAILQ_INSERT_HEAD(&pqpair->free_tr, tr, tq_list);
253 27 : }
254 :
255 4 : nvme_pcie_qpair_reset(qpair);
256 :
257 4 : return 0;
258 4 : }
259 :
260 : int
261 1 : nvme_pcie_ctrlr_construct_admin_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t num_entries)
262 : {
263 1 : struct nvme_pcie_qpair *pqpair;
264 1 : int rc;
265 :
266 1 : pqpair = spdk_zmalloc(sizeof(*pqpair), 64, NULL, SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_SHARE);
267 1 : if (pqpair == NULL) {
268 0 : return -ENOMEM;
269 : }
270 :
271 1 : pqpair->num_entries = num_entries;
272 1 : pqpair->flags.delay_cmd_submit = 0;
273 1 : pqpair->pcie_state = NVME_PCIE_QPAIR_READY;
274 :
275 1 : ctrlr->adminq = &pqpair->qpair;
276 :
277 1 : rc = nvme_qpair_init(ctrlr->adminq,
278 : 0, /* qpair ID */
279 1 : ctrlr,
280 : SPDK_NVME_QPRIO_URGENT,
281 1 : num_entries,
282 : false);
283 1 : if (rc != 0) {
284 0 : return rc;
285 : }
286 :
287 1 : pqpair->stat = spdk_zmalloc(sizeof(*pqpair->stat), 64, NULL, SPDK_ENV_NUMA_ID_ANY,
288 : SPDK_MALLOC_SHARE);
289 1 : if (!pqpair->stat) {
290 0 : SPDK_ERRLOG("Failed to allocate admin qpair statistics\n");
291 0 : return -ENOMEM;
292 : }
293 :
294 1 : return nvme_pcie_qpair_construct(ctrlr->adminq, NULL);
295 1 : }
296 :
297 : /**
298 : * Note: the ctrlr_lock must be held when calling this function.
299 : */
300 : void
301 0 : nvme_pcie_qpair_insert_pending_admin_request(struct spdk_nvme_qpair *qpair,
302 : struct nvme_request *req, struct spdk_nvme_cpl *cpl)
303 : {
304 0 : struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
305 0 : struct nvme_request *active_req = req;
306 0 : struct spdk_nvme_ctrlr_process *active_proc;
307 :
308 : /*
309 : * The admin request is from another process. Move to the per
310 : * process list for that process to handle it later.
311 : */
312 0 : assert(nvme_qpair_is_admin_queue(qpair));
313 0 : assert(active_req->pid != getpid());
314 :
315 0 : active_proc = nvme_ctrlr_get_process(ctrlr, active_req->pid);
316 0 : if (active_proc) {
317 : /* Save the original completion information */
318 0 : memcpy(&active_req->cpl, cpl, sizeof(*cpl));
319 0 : STAILQ_INSERT_TAIL(&active_proc->active_reqs, active_req, stailq);
320 0 : } else {
321 0 : SPDK_ERRLOG("The owning process (pid %d) is not found. Dropping the request.\n",
322 : active_req->pid);
323 0 : nvme_cleanup_user_req(active_req);
324 0 : nvme_free_request(active_req);
325 : }
326 0 : }
327 :
328 : /**
329 : * Note: the ctrlr_lock must be held when calling this function.
330 : */
331 : void
332 0 : nvme_pcie_qpair_complete_pending_admin_request(struct spdk_nvme_qpair *qpair)
333 : {
334 0 : struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
335 0 : struct nvme_request *req, *tmp_req;
336 0 : pid_t pid = getpid();
337 0 : struct spdk_nvme_ctrlr_process *proc;
338 :
339 : /*
340 : * Check whether there is any pending admin request from
341 : * other active processes.
342 : */
343 0 : assert(nvme_qpair_is_admin_queue(qpair));
344 :
345 0 : proc = nvme_ctrlr_get_current_process(ctrlr);
346 0 : if (!proc) {
347 0 : SPDK_ERRLOG("the active process (pid %d) is not found for this controller.\n", pid);
348 0 : assert(proc);
349 0 : return;
350 : }
351 :
352 0 : STAILQ_FOREACH_SAFE(req, &proc->active_reqs, stailq, tmp_req) {
353 0 : STAILQ_REMOVE(&proc->active_reqs, req, nvme_request, stailq);
354 :
355 0 : assert(req->pid == pid);
356 :
357 0 : nvme_complete_request(req->cb_fn, req->cb_arg, qpair, req, &req->cpl);
358 0 : }
359 0 : }
360 :
361 : int
362 7 : nvme_pcie_ctrlr_cmd_create_io_cq(struct spdk_nvme_ctrlr *ctrlr,
363 : struct spdk_nvme_qpair *io_que, spdk_nvme_cmd_cb cb_fn,
364 : void *cb_arg)
365 : {
366 7 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(io_que);
367 7 : struct nvme_request *req;
368 7 : struct spdk_nvme_cmd *cmd;
369 7 : bool ien = ctrlr->opts.enable_interrupts;
370 :
371 7 : req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
372 7 : if (req == NULL) {
373 2 : return -ENOMEM;
374 : }
375 :
376 5 : cmd = &req->cmd;
377 5 : cmd->opc = SPDK_NVME_OPC_CREATE_IO_CQ;
378 :
379 5 : cmd->cdw10_bits.create_io_q.qid = io_que->id;
380 5 : cmd->cdw10_bits.create_io_q.qsize = pqpair->num_entries - 1;
381 :
382 5 : cmd->cdw11_bits.create_io_cq.pc = 1;
383 5 : if (ien) {
384 0 : cmd->cdw11_bits.create_io_cq.ien = 1;
385 : /* The interrupt vector offset starts from 1. We directly map the
386 : * queue id to interrupt vector.
387 : */
388 0 : cmd->cdw11_bits.create_io_cq.iv = io_que->id;
389 0 : }
390 :
391 5 : cmd->dptr.prp.prp1 = pqpair->cpl_bus_addr;
392 :
393 5 : return nvme_ctrlr_submit_admin_request(ctrlr, req);
394 7 : }
395 :
396 : int
397 5 : nvme_pcie_ctrlr_cmd_create_io_sq(struct spdk_nvme_ctrlr *ctrlr,
398 : struct spdk_nvme_qpair *io_que, spdk_nvme_cmd_cb cb_fn, void *cb_arg)
399 : {
400 5 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(io_que);
401 5 : struct nvme_request *req;
402 5 : struct spdk_nvme_cmd *cmd;
403 :
404 5 : req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
405 5 : if (req == NULL) {
406 1 : return -ENOMEM;
407 : }
408 :
409 4 : cmd = &req->cmd;
410 4 : cmd->opc = SPDK_NVME_OPC_CREATE_IO_SQ;
411 :
412 4 : cmd->cdw10_bits.create_io_q.qid = io_que->id;
413 4 : cmd->cdw10_bits.create_io_q.qsize = pqpair->num_entries - 1;
414 4 : cmd->cdw11_bits.create_io_sq.pc = 1;
415 4 : cmd->cdw11_bits.create_io_sq.qprio = io_que->qprio;
416 4 : cmd->cdw11_bits.create_io_sq.cqid = io_que->id;
417 4 : cmd->dptr.prp.prp1 = pqpair->cmd_bus_addr;
418 :
419 4 : return nvme_ctrlr_submit_admin_request(ctrlr, req);
420 5 : }
421 :
422 : int
423 3 : nvme_pcie_ctrlr_cmd_delete_io_cq(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
424 : spdk_nvme_cmd_cb cb_fn, void *cb_arg)
425 : {
426 3 : struct nvme_request *req;
427 3 : struct spdk_nvme_cmd *cmd;
428 :
429 3 : req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
430 3 : if (req == NULL) {
431 1 : return -ENOMEM;
432 : }
433 :
434 2 : cmd = &req->cmd;
435 2 : cmd->opc = SPDK_NVME_OPC_DELETE_IO_CQ;
436 2 : cmd->cdw10_bits.delete_io_q.qid = qpair->id;
437 :
438 2 : return nvme_ctrlr_submit_admin_request(ctrlr, req);
439 3 : }
440 :
441 : int
442 2 : nvme_pcie_ctrlr_cmd_delete_io_sq(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
443 : spdk_nvme_cmd_cb cb_fn, void *cb_arg)
444 : {
445 2 : struct nvme_request *req;
446 2 : struct spdk_nvme_cmd *cmd;
447 :
448 2 : req = nvme_allocate_request_null(ctrlr->adminq, cb_fn, cb_arg);
449 2 : if (req == NULL) {
450 1 : return -ENOMEM;
451 : }
452 :
453 1 : cmd = &req->cmd;
454 1 : cmd->opc = SPDK_NVME_OPC_DELETE_IO_SQ;
455 1 : cmd->cdw10_bits.delete_io_q.qid = qpair->id;
456 :
457 1 : return nvme_ctrlr_submit_admin_request(ctrlr, req);
458 2 : }
459 :
460 : static void
461 1 : nvme_completion_sq_error_delete_cq_cb(void *arg, const struct spdk_nvme_cpl *cpl)
462 : {
463 1 : struct spdk_nvme_qpair *qpair = arg;
464 1 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
465 :
466 1 : if (spdk_nvme_cpl_is_error(cpl)) {
467 0 : SPDK_ERRLOG("delete_io_cq failed!\n");
468 0 : }
469 :
470 1 : pqpair->pcie_state = NVME_PCIE_QPAIR_FAILED;
471 1 : }
472 :
473 : static void
474 3 : nvme_completion_create_sq_cb(void *arg, const struct spdk_nvme_cpl *cpl)
475 : {
476 3 : struct spdk_nvme_qpair *qpair = arg;
477 3 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
478 3 : struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
479 3 : struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
480 3 : int rc;
481 :
482 3 : if (pqpair->flags.defer_destruction) {
483 : /* This qpair was deleted by the application while the
484 : * connection was still in progress. We had to wait
485 : * to free the qpair resources until this outstanding
486 : * command was completed. Now that we have the completion
487 : * free it now.
488 : */
489 0 : nvme_pcie_qpair_destroy(qpair);
490 0 : return;
491 : }
492 :
493 3 : if (spdk_nvme_cpl_is_error(cpl)) {
494 1 : SPDK_ERRLOG("nvme_create_io_sq failed, deleting cq!\n");
495 1 : rc = nvme_pcie_ctrlr_cmd_delete_io_cq(qpair->ctrlr, qpair, nvme_completion_sq_error_delete_cq_cb,
496 1 : qpair);
497 1 : if (rc != 0) {
498 0 : SPDK_ERRLOG("Failed to send request to delete_io_cq with rc=%d\n", rc);
499 0 : pqpair->pcie_state = NVME_PCIE_QPAIR_FAILED;
500 0 : }
501 1 : return;
502 : }
503 2 : pqpair->pcie_state = NVME_PCIE_QPAIR_READY;
504 2 : if (ctrlr->shadow_doorbell) {
505 2 : pqpair->shadow_doorbell.sq_tdbl = ctrlr->shadow_doorbell + (2 * qpair->id + 0) *
506 1 : pctrlr->doorbell_stride_u32;
507 2 : pqpair->shadow_doorbell.cq_hdbl = ctrlr->shadow_doorbell + (2 * qpair->id + 1) *
508 1 : pctrlr->doorbell_stride_u32;
509 2 : pqpair->shadow_doorbell.sq_eventidx = ctrlr->eventidx + (2 * qpair->id + 0) *
510 1 : pctrlr->doorbell_stride_u32;
511 2 : pqpair->shadow_doorbell.cq_eventidx = ctrlr->eventidx + (2 * qpair->id + 1) *
512 1 : pctrlr->doorbell_stride_u32;
513 1 : pqpair->flags.has_shadow_doorbell = 1;
514 1 : } else {
515 1 : pqpair->flags.has_shadow_doorbell = 0;
516 : }
517 2 : nvme_pcie_qpair_reset(qpair);
518 :
519 3 : }
520 :
521 : static void
522 4 : nvme_completion_create_cq_cb(void *arg, const struct spdk_nvme_cpl *cpl)
523 : {
524 4 : struct spdk_nvme_qpair *qpair = arg;
525 4 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
526 4 : int rc;
527 :
528 4 : if (pqpair->flags.defer_destruction) {
529 : /* This qpair was deleted by the application while the
530 : * connection was still in progress. We had to wait
531 : * to free the qpair resources until this outstanding
532 : * command was completed. Now that we have the completion
533 : * free it now.
534 : */
535 0 : nvme_pcie_qpair_destroy(qpair);
536 0 : return;
537 : }
538 :
539 4 : if (spdk_nvme_cpl_is_error(cpl)) {
540 1 : pqpair->pcie_state = NVME_PCIE_QPAIR_FAILED;
541 1 : SPDK_ERRLOG("nvme_create_io_cq failed!\n");
542 1 : return;
543 : }
544 :
545 3 : rc = nvme_pcie_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair, nvme_completion_create_sq_cb, qpair);
546 :
547 3 : if (rc != 0) {
548 0 : SPDK_ERRLOG("Failed to send request to create_io_sq, deleting cq!\n");
549 0 : rc = nvme_pcie_ctrlr_cmd_delete_io_cq(qpair->ctrlr, qpair, nvme_completion_sq_error_delete_cq_cb,
550 0 : qpair);
551 0 : if (rc != 0) {
552 0 : SPDK_ERRLOG("Failed to send request to delete_io_cq with rc=%d\n", rc);
553 0 : pqpair->pcie_state = NVME_PCIE_QPAIR_FAILED;
554 0 : }
555 0 : return;
556 : }
557 3 : pqpair->pcie_state = NVME_PCIE_QPAIR_WAIT_FOR_SQ;
558 4 : }
559 :
560 : static int
561 5 : _nvme_pcie_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
562 : uint16_t qid)
563 : {
564 5 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
565 5 : int rc;
566 :
567 : /* Statistics may already be allocated in the case of controller reset */
568 5 : if (qpair->poll_group) {
569 5 : struct nvme_pcie_poll_group *group = SPDK_CONTAINEROF(qpair->poll_group,
570 : struct nvme_pcie_poll_group, group);
571 :
572 5 : pqpair->stat = &group->stats;
573 5 : pqpair->shared_stats = true;
574 5 : } else {
575 0 : if (pqpair->stat == NULL) {
576 0 : pqpair->stat = calloc(1, sizeof(*pqpair->stat));
577 0 : if (!pqpair->stat) {
578 0 : SPDK_ERRLOG("Failed to allocate qpair statistics\n");
579 0 : nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTED);
580 0 : return -ENOMEM;
581 : }
582 0 : }
583 : }
584 :
585 5 : rc = nvme_pcie_ctrlr_cmd_create_io_cq(ctrlr, qpair, nvme_completion_create_cq_cb, qpair);
586 :
587 5 : if (rc != 0) {
588 1 : SPDK_ERRLOG("Failed to send request to create_io_cq\n");
589 1 : nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTED);
590 1 : return rc;
591 : }
592 4 : pqpair->pcie_state = NVME_PCIE_QPAIR_WAIT_FOR_CQ;
593 4 : return 0;
594 5 : }
595 :
596 : int
597 5 : nvme_pcie_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
598 : {
599 5 : int rc = 0;
600 :
601 5 : if (!nvme_qpair_is_admin_queue(qpair)) {
602 5 : rc = _nvme_pcie_ctrlr_create_io_qpair(ctrlr, qpair, qpair->id);
603 5 : } else {
604 0 : nvme_qpair_set_state(qpair, NVME_QPAIR_CONNECTED);
605 : }
606 :
607 10 : return rc;
608 5 : }
609 :
610 : void
611 0 : nvme_pcie_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
612 : {
613 0 : if (!nvme_qpair_is_admin_queue(qpair) || !ctrlr->is_disconnecting) {
614 0 : nvme_transport_ctrlr_disconnect_qpair_done(qpair);
615 0 : } else {
616 : /* If this function is called for the admin qpair via spdk_nvme_ctrlr_reset()
617 : * or spdk_nvme_ctrlr_disconnect(), initiate a Controller Level Reset.
618 : * Then we can abort trackers safely because the Controller Level Reset deletes
619 : * all I/O SQ/CQs.
620 : */
621 0 : nvme_ctrlr_disable(ctrlr);
622 : }
623 0 : }
624 :
625 : /* Used when dst points to MMIO (i.e. CMB) in a virtual machine - in these cases we must
626 : * not use wide instructions because QEMU will not emulate such instructions to MMIO space.
627 : * So this function ensures we only copy 8 bytes at a time.
628 : */
629 : static inline void
630 0 : nvme_pcie_copy_command_mmio(struct spdk_nvme_cmd *dst, const struct spdk_nvme_cmd *src)
631 : {
632 0 : uint64_t *dst64 = (uint64_t *)dst;
633 0 : const uint64_t *src64 = (const uint64_t *)src;
634 0 : uint32_t i;
635 :
636 0 : for (i = 0; i < sizeof(*dst) / 8; i++) {
637 0 : dst64[i] = src64[i];
638 0 : }
639 0 : }
640 :
641 : static inline void
642 0 : nvme_pcie_copy_command(struct spdk_nvme_cmd *dst, const struct spdk_nvme_cmd *src)
643 : {
644 : /* dst and src are known to be non-overlapping and 64-byte aligned. */
645 : #if defined(__SSE2__)
646 0 : __m128i *d128 = (__m128i *)dst;
647 0 : const __m128i *s128 = (const __m128i *)src;
648 :
649 0 : _mm_stream_si128(&d128[0], _mm_load_si128(&s128[0]));
650 0 : _mm_stream_si128(&d128[1], _mm_load_si128(&s128[1]));
651 0 : _mm_stream_si128(&d128[2], _mm_load_si128(&s128[2]));
652 0 : _mm_stream_si128(&d128[3], _mm_load_si128(&s128[3]));
653 : #else
654 : *dst = *src;
655 : #endif
656 0 : }
657 :
658 : void
659 0 : nvme_pcie_qpair_submit_tracker(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr)
660 : {
661 0 : struct nvme_request *req;
662 0 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
663 0 : struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
664 :
665 0 : req = tr->req;
666 0 : assert(req != NULL);
667 :
668 0 : spdk_trace_record(TRACE_NVME_PCIE_SUBMIT, qpair->id, 0, (uintptr_t)req, req->cb_arg,
669 : (uint32_t)req->cmd.cid, (uint32_t)req->cmd.opc,
670 : req->cmd.cdw10, req->cmd.cdw11, req->cmd.cdw12,
671 : pqpair->qpair.queue_depth);
672 :
673 0 : if (req->cmd.fuse) {
674 : /*
675 : * Keep track of the fuse operation sequence so that we ring the doorbell only
676 : * after the second fuse is submitted.
677 : */
678 0 : qpair->last_fuse = req->cmd.fuse;
679 0 : }
680 :
681 : /* Don't use wide instructions to copy NVMe command, this is limited by QEMU
682 : * virtual NVMe controller, the maximum access width is 8 Bytes for one time.
683 : */
684 0 : if (spdk_unlikely((ctrlr->quirks & NVME_QUIRK_MAXIMUM_PCI_ACCESS_WIDTH) && pqpair->sq_in_cmb)) {
685 0 : nvme_pcie_copy_command_mmio(&pqpair->cmd[pqpair->sq_tail], &req->cmd);
686 0 : } else {
687 : /* Copy the command from the tracker to the submission queue. */
688 0 : nvme_pcie_copy_command(&pqpair->cmd[pqpair->sq_tail], &req->cmd);
689 : }
690 :
691 0 : if (spdk_unlikely(++pqpair->sq_tail == pqpair->num_entries)) {
692 0 : pqpair->sq_tail = 0;
693 0 : }
694 :
695 0 : if (spdk_unlikely(pqpair->sq_tail == pqpair->sq_head)) {
696 0 : SPDK_ERRLOG("sq_tail is passing sq_head!\n");
697 0 : }
698 :
699 0 : if (!pqpair->flags.delay_cmd_submit) {
700 0 : nvme_pcie_qpair_ring_sq_doorbell(qpair);
701 0 : }
702 0 : }
703 :
704 : void
705 0 : nvme_pcie_qpair_complete_tracker(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr,
706 : struct spdk_nvme_cpl *cpl, bool print_on_error)
707 : {
708 0 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
709 0 : struct nvme_request *req;
710 0 : bool retry, error;
711 0 : bool print_error;
712 :
713 0 : req = tr->req;
714 :
715 0 : spdk_trace_record(TRACE_NVME_PCIE_COMPLETE, qpair->id, 0, (uintptr_t)req, req->cb_arg,
716 : (uint32_t)req->cmd.cid, (uint32_t)cpl->status_raw, pqpair->qpair.queue_depth);
717 :
718 0 : assert(req != NULL);
719 :
720 0 : error = spdk_nvme_cpl_is_error(cpl);
721 0 : retry = error && nvme_completion_is_retry(cpl) &&
722 0 : req->retries < pqpair->retry_count;
723 0 : print_error = error && print_on_error && !qpair->ctrlr->opts.disable_error_logging;
724 :
725 0 : if (print_error) {
726 0 : spdk_nvme_qpair_print_command(qpair, &req->cmd);
727 0 : }
728 :
729 0 : if (print_error || SPDK_DEBUGLOG_FLAG_ENABLED("nvme")) {
730 0 : spdk_nvme_qpair_print_completion(qpair, cpl);
731 0 : }
732 :
733 0 : assert(cpl->cid == req->cmd.cid);
734 :
735 0 : if (retry) {
736 0 : req->retries++;
737 0 : nvme_pcie_qpair_submit_tracker(qpair, tr);
738 0 : } else {
739 0 : TAILQ_REMOVE(&pqpair->outstanding_tr, tr, tq_list);
740 0 : pqpair->qpair.queue_depth--;
741 :
742 : /* Only check admin requests from different processes. */
743 0 : if (nvme_qpair_is_admin_queue(qpair) && req->pid != getpid()) {
744 0 : nvme_pcie_qpair_insert_pending_admin_request(qpair, req, cpl);
745 0 : } else {
746 0 : nvme_complete_request(tr->cb_fn, tr->cb_arg, qpair, req, cpl);
747 : }
748 :
749 0 : tr->req = NULL;
750 :
751 0 : TAILQ_INSERT_HEAD(&pqpair->free_tr, tr, tq_list);
752 : }
753 0 : }
754 :
755 : void
756 0 : nvme_pcie_qpair_manual_complete_tracker(struct spdk_nvme_qpair *qpair,
757 : struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr,
758 : bool print_on_error)
759 : {
760 0 : struct spdk_nvme_cpl cpl;
761 :
762 0 : memset(&cpl, 0, sizeof(cpl));
763 0 : cpl.sqid = qpair->id;
764 0 : cpl.cid = tr->cid;
765 0 : cpl.status.sct = sct;
766 0 : cpl.status.sc = sc;
767 0 : cpl.status.dnr = dnr;
768 0 : nvme_pcie_qpair_complete_tracker(qpair, tr, &cpl, print_on_error);
769 0 : }
770 :
771 : void
772 0 : nvme_pcie_qpair_abort_trackers(struct spdk_nvme_qpair *qpair, uint32_t dnr)
773 : {
774 0 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
775 0 : struct nvme_tracker *tr, *temp, *last;
776 :
777 0 : last = TAILQ_LAST(&pqpair->outstanding_tr, nvme_outstanding_tr_head);
778 :
779 : /* Abort previously submitted (outstanding) trs */
780 0 : TAILQ_FOREACH_SAFE(tr, &pqpair->outstanding_tr, tq_list, temp) {
781 0 : if (!qpair->ctrlr->opts.disable_error_logging) {
782 0 : SPDK_ERRLOG("aborting outstanding command\n");
783 0 : }
784 0 : nvme_pcie_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC,
785 0 : SPDK_NVME_SC_ABORTED_BY_REQUEST, dnr, true);
786 :
787 0 : if (tr == last) {
788 0 : break;
789 : }
790 0 : }
791 0 : }
792 :
793 : void
794 1 : nvme_pcie_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair)
795 : {
796 1 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
797 1 : struct nvme_tracker *tr;
798 :
799 1 : tr = TAILQ_FIRST(&pqpair->outstanding_tr);
800 1 : while (tr != NULL) {
801 0 : assert(tr->req != NULL);
802 0 : if (tr->req->cmd.opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST) {
803 0 : nvme_pcie_qpair_manual_complete_tracker(qpair, tr,
804 : SPDK_NVME_SCT_GENERIC, SPDK_NVME_SC_ABORTED_SQ_DELETION, 0,
805 : false);
806 0 : tr = TAILQ_FIRST(&pqpair->outstanding_tr);
807 0 : } else {
808 0 : tr = TAILQ_NEXT(tr, tq_list);
809 : }
810 : }
811 1 : }
812 :
813 : void
814 1 : nvme_pcie_admin_qpair_destroy(struct spdk_nvme_qpair *qpair)
815 : {
816 1 : nvme_pcie_admin_qpair_abort_aers(qpair);
817 1 : }
818 :
819 : void
820 0 : nvme_pcie_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr)
821 : {
822 0 : nvme_pcie_qpair_abort_trackers(qpair, dnr);
823 0 : }
824 :
825 : static void
826 0 : nvme_pcie_qpair_check_timeout(struct spdk_nvme_qpair *qpair)
827 : {
828 0 : uint64_t t02;
829 0 : struct nvme_tracker *tr, *tmp;
830 0 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
831 0 : struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
832 0 : struct spdk_nvme_ctrlr_process *active_proc;
833 :
834 : /* Don't check timeouts during controller initialization. */
835 0 : if (ctrlr->state != NVME_CTRLR_STATE_READY) {
836 0 : return;
837 : }
838 :
839 0 : if (nvme_qpair_is_admin_queue(qpair)) {
840 0 : active_proc = nvme_ctrlr_get_current_process(ctrlr);
841 0 : } else {
842 0 : active_proc = qpair->active_proc;
843 : }
844 :
845 : /* Only check timeouts if the current process has a timeout callback. */
846 0 : if (active_proc == NULL || active_proc->timeout_cb_fn == NULL) {
847 0 : return;
848 : }
849 :
850 0 : t02 = spdk_get_ticks();
851 0 : TAILQ_FOREACH_SAFE(tr, &pqpair->outstanding_tr, tq_list, tmp) {
852 0 : assert(tr->req != NULL);
853 :
854 0 : if (nvme_request_check_timeout(tr->req, tr->cid, active_proc, t02)) {
855 : /*
856 : * The requests are in order, so as soon as one has not timed out,
857 : * stop iterating.
858 : */
859 0 : break;
860 : }
861 0 : }
862 0 : }
863 :
864 : int32_t
865 0 : nvme_pcie_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
866 : {
867 0 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
868 0 : struct nvme_tracker *tr;
869 0 : struct spdk_nvme_cpl *cpl, *next_cpl;
870 0 : uint32_t num_completions = 0;
871 0 : struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
872 0 : uint16_t next_cq_head;
873 0 : uint8_t next_phase;
874 0 : bool next_is_valid = false;
875 0 : int rc;
876 :
877 0 : if (spdk_unlikely(pqpair->pcie_state == NVME_PCIE_QPAIR_FAILED)) {
878 0 : return -ENXIO;
879 : }
880 :
881 0 : if (spdk_unlikely(nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING)) {
882 0 : if (pqpair->pcie_state == NVME_PCIE_QPAIR_READY) {
883 : /* It is possible that another thread set the pcie_state to
884 : * QPAIR_READY, if it polled the adminq and processed the SQ
885 : * completion for this qpair. So check for that condition
886 : * here and then update the qpair's state to CONNECTED, since
887 : * we can only set the qpair state from the qpair's thread.
888 : * (Note: this fixed issue #2157.)
889 : */
890 0 : nvme_qpair_set_state(qpair, NVME_QPAIR_CONNECTED);
891 0 : } else if (pqpair->pcie_state == NVME_PCIE_QPAIR_FAILED) {
892 0 : nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTED);
893 0 : return -ENXIO;
894 : } else {
895 0 : rc = spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
896 0 : if (rc < 0) {
897 0 : return rc;
898 0 : } else if (pqpair->pcie_state == NVME_PCIE_QPAIR_FAILED) {
899 0 : nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTED);
900 0 : return -ENXIO;
901 : }
902 : }
903 0 : return 0;
904 : }
905 :
906 0 : if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {
907 0 : nvme_ctrlr_lock(ctrlr);
908 0 : }
909 :
910 0 : if (max_completions == 0 || max_completions > pqpair->max_completions_cap) {
911 : /*
912 : * max_completions == 0 means unlimited, but complete at most
913 : * max_completions_cap batch of I/O at a time so that the completion
914 : * queue doorbells don't wrap around.
915 : */
916 0 : max_completions = pqpair->max_completions_cap;
917 0 : }
918 :
919 0 : pqpair->stat->polls++;
920 :
921 0 : while (1) {
922 0 : cpl = &pqpair->cpl[pqpair->cq_head];
923 :
924 0 : if (!next_is_valid && cpl->status.p != pqpair->flags.phase) {
925 0 : break;
926 : }
927 :
928 0 : if (spdk_likely(pqpair->cq_head + 1 != pqpair->num_entries)) {
929 0 : next_cq_head = pqpair->cq_head + 1;
930 0 : next_phase = pqpair->flags.phase;
931 0 : } else {
932 0 : next_cq_head = 0;
933 0 : next_phase = !pqpair->flags.phase;
934 : }
935 0 : next_cpl = &pqpair->cpl[next_cq_head];
936 0 : next_is_valid = (next_cpl->status.p == next_phase);
937 0 : if (next_is_valid) {
938 0 : __builtin_prefetch(&pqpair->tr[next_cpl->cid]);
939 0 : }
940 :
941 : #if defined(__PPC64__) || defined(__riscv) || defined(__loongarch__)
942 : /*
943 : * This memory barrier prevents reordering of:
944 : * - load after store from/to tr
945 : * - load after load cpl phase and cpl cid
946 : */
947 : spdk_mb();
948 : #elif defined(__aarch64__)
949 : __asm volatile("dmb oshld" ::: "memory");
950 : #endif
951 :
952 0 : if (spdk_unlikely(++pqpair->cq_head == pqpair->num_entries)) {
953 0 : pqpair->cq_head = 0;
954 0 : pqpair->flags.phase = !pqpair->flags.phase;
955 0 : }
956 :
957 0 : tr = &pqpair->tr[cpl->cid];
958 0 : pqpair->sq_head = cpl->sqhd;
959 :
960 0 : if (tr->req) {
961 : /* Prefetch the req's STAILQ_ENTRY since we'll need to access it
962 : * as part of putting the req back on the qpair's free list.
963 : */
964 0 : __builtin_prefetch(&tr->req->stailq);
965 0 : nvme_pcie_qpair_complete_tracker(qpair, tr, cpl, true);
966 0 : } else {
967 0 : SPDK_ERRLOG("cpl does not map to outstanding cmd\n");
968 0 : spdk_nvme_qpair_print_completion(qpair, cpl);
969 0 : assert(0);
970 : }
971 :
972 0 : if (++num_completions == max_completions) {
973 0 : break;
974 : }
975 : }
976 :
977 0 : if (num_completions > 0) {
978 0 : pqpair->stat->completions += num_completions;
979 0 : nvme_pcie_qpair_ring_cq_doorbell(qpair);
980 0 : } else {
981 0 : pqpair->stat->idle_polls++;
982 : }
983 :
984 0 : if (pqpair->flags.delay_cmd_submit) {
985 0 : if (pqpair->last_sq_tail != pqpair->sq_tail) {
986 0 : nvme_pcie_qpair_ring_sq_doorbell(qpair);
987 0 : pqpair->last_sq_tail = pqpair->sq_tail;
988 0 : }
989 0 : }
990 :
991 0 : if (spdk_unlikely(ctrlr->timeout_enabled)) {
992 : /*
993 : * User registered for timeout callback
994 : */
995 0 : nvme_pcie_qpair_check_timeout(qpair);
996 0 : }
997 :
998 : /* Before returning, complete any pending admin request or
999 : * process the admin qpair disconnection.
1000 : */
1001 0 : if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {
1002 0 : nvme_pcie_qpair_complete_pending_admin_request(qpair);
1003 :
1004 0 : if (nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTING) {
1005 0 : rc = nvme_ctrlr_disable_poll(qpair->ctrlr);
1006 0 : if (rc != -EAGAIN) {
1007 0 : nvme_transport_ctrlr_disconnect_qpair_done(qpair);
1008 0 : }
1009 0 : }
1010 :
1011 0 : nvme_ctrlr_unlock(ctrlr);
1012 0 : }
1013 :
1014 0 : if (spdk_unlikely(pqpair->flags.has_pending_vtophys_failures)) {
1015 0 : struct nvme_tracker *tr, *tmp;
1016 :
1017 0 : TAILQ_FOREACH_SAFE(tr, &pqpair->outstanding_tr, tq_list, tmp) {
1018 0 : if (tr->bad_vtophys) {
1019 0 : tr->bad_vtophys = 0;
1020 0 : nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1021 0 : }
1022 0 : }
1023 0 : pqpair->flags.has_pending_vtophys_failures = 0;
1024 0 : }
1025 :
1026 0 : return num_completions;
1027 0 : }
1028 :
1029 : int
1030 4 : nvme_pcie_qpair_destroy(struct spdk_nvme_qpair *qpair)
1031 : {
1032 4 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
1033 :
1034 4 : if (nvme_qpair_is_admin_queue(qpair)) {
1035 1 : nvme_pcie_admin_qpair_destroy(qpair);
1036 1 : }
1037 : /*
1038 : * We check sq_vaddr and cq_vaddr to see if the user specified the memory
1039 : * buffers when creating the I/O queue.
1040 : * If the user specified them, we cannot free that memory.
1041 : * Nor do we free it if it's in the CMB.
1042 : */
1043 4 : if (!pqpair->sq_vaddr && pqpair->cmd && !pqpair->sq_in_cmb) {
1044 2 : spdk_free(pqpair->cmd);
1045 2 : }
1046 4 : if (!pqpair->cq_vaddr && pqpair->cpl) {
1047 2 : spdk_free(pqpair->cpl);
1048 2 : }
1049 4 : if (pqpair->tr) {
1050 4 : spdk_free(pqpair->tr);
1051 4 : }
1052 :
1053 4 : nvme_qpair_deinit(qpair);
1054 :
1055 4 : if (!pqpair->shared_stats && (!qpair->active_proc ||
1056 0 : qpair->active_proc == nvme_ctrlr_get_current_process(qpair->ctrlr))) {
1057 4 : if (qpair->id) {
1058 3 : free(pqpair->stat);
1059 3 : } else {
1060 : /* statistics of admin qpair are allocates from huge pages because
1061 : * admin qpair is shared for multi-process */
1062 1 : spdk_free(pqpair->stat);
1063 : }
1064 :
1065 4 : }
1066 :
1067 4 : spdk_free(pqpair);
1068 :
1069 4 : return 0;
1070 4 : }
1071 :
1072 : struct spdk_nvme_qpair *
1073 0 : nvme_pcie_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid,
1074 : const struct spdk_nvme_io_qpair_opts *opts)
1075 : {
1076 0 : struct nvme_pcie_qpair *pqpair;
1077 0 : struct spdk_nvme_qpair *qpair;
1078 0 : int rc;
1079 :
1080 0 : assert(ctrlr != NULL);
1081 :
1082 0 : pqpair = spdk_zmalloc(sizeof(*pqpair), 64, NULL,
1083 : SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_SHARE);
1084 0 : if (pqpair == NULL) {
1085 0 : return NULL;
1086 : }
1087 :
1088 0 : pqpair->num_entries = opts->io_queue_size;
1089 0 : pqpair->flags.delay_cmd_submit = opts->delay_cmd_submit;
1090 :
1091 0 : qpair = &pqpair->qpair;
1092 :
1093 0 : rc = nvme_qpair_init(qpair, qid, ctrlr, opts->qprio, opts->io_queue_requests, opts->async_mode);
1094 0 : if (rc != 0) {
1095 0 : nvme_pcie_qpair_destroy(qpair);
1096 0 : return NULL;
1097 : }
1098 :
1099 0 : rc = nvme_pcie_qpair_construct(qpair, opts);
1100 :
1101 0 : if (rc != 0) {
1102 0 : nvme_pcie_qpair_destroy(qpair);
1103 0 : return NULL;
1104 : }
1105 :
1106 0 : return qpair;
1107 0 : }
1108 :
1109 : int
1110 0 : nvme_pcie_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
1111 : {
1112 0 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
1113 0 : struct nvme_completion_poll_status *status;
1114 0 : int rc;
1115 :
1116 0 : assert(ctrlr != NULL);
1117 :
1118 0 : if (ctrlr->is_removed) {
1119 0 : goto free;
1120 : }
1121 :
1122 0 : if (ctrlr->prepare_for_reset) {
1123 0 : if (nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING) {
1124 0 : pqpair->flags.defer_destruction = true;
1125 0 : }
1126 0 : goto clear_shadow_doorbells;
1127 : }
1128 :
1129 : /* If attempting to delete a qpair that's still being connected, we have to wait until it's
1130 : * finished, so that we don't free it while it's waiting for the create cq/sq callbacks.
1131 : */
1132 0 : while (pqpair->pcie_state == NVME_PCIE_QPAIR_WAIT_FOR_CQ ||
1133 0 : pqpair->pcie_state == NVME_PCIE_QPAIR_WAIT_FOR_SQ) {
1134 0 : rc = spdk_nvme_qpair_process_completions(ctrlr->adminq, 0);
1135 0 : if (rc < 0) {
1136 0 : break;
1137 : }
1138 : }
1139 :
1140 0 : status = calloc(1, sizeof(*status));
1141 0 : if (!status) {
1142 0 : SPDK_ERRLOG("Failed to allocate status tracker\n");
1143 0 : goto free;
1144 : }
1145 :
1146 : /* Delete the I/O submission queue */
1147 0 : rc = nvme_pcie_ctrlr_cmd_delete_io_sq(ctrlr, qpair, nvme_completion_poll_cb, status);
1148 0 : if (rc != 0) {
1149 0 : SPDK_ERRLOG("Failed to send request to delete_io_sq with rc=%d\n", rc);
1150 0 : free(status);
1151 0 : goto free;
1152 : }
1153 0 : if (nvme_wait_for_completion(ctrlr->adminq, status)) {
1154 0 : if (!status->timed_out) {
1155 0 : free(status);
1156 0 : }
1157 0 : goto free;
1158 : }
1159 :
1160 : /* Now that the submission queue is deleted, the device is supposed to have
1161 : * completed any outstanding I/O. Try to complete them. If they don't complete,
1162 : * they'll be marked as aborted and completed below. */
1163 0 : if (qpair->active_proc == nvme_ctrlr_get_current_process(ctrlr)) {
1164 0 : nvme_pcie_qpair_process_completions(qpair, 0);
1165 0 : }
1166 :
1167 0 : memset(status, 0, sizeof(*status));
1168 : /* Delete the completion queue */
1169 0 : rc = nvme_pcie_ctrlr_cmd_delete_io_cq(ctrlr, qpair, nvme_completion_poll_cb, status);
1170 0 : if (rc != 0) {
1171 0 : SPDK_ERRLOG("Failed to send request to delete_io_cq with rc=%d\n", rc);
1172 0 : free(status);
1173 0 : goto free;
1174 : }
1175 0 : if (nvme_wait_for_completion(ctrlr->adminq, status)) {
1176 0 : if (!status->timed_out) {
1177 0 : free(status);
1178 0 : }
1179 0 : goto free;
1180 : }
1181 0 : free(status);
1182 :
1183 : clear_shadow_doorbells:
1184 0 : if (pqpair->flags.has_shadow_doorbell && ctrlr->shadow_doorbell) {
1185 0 : *pqpair->shadow_doorbell.sq_tdbl = 0;
1186 0 : *pqpair->shadow_doorbell.cq_hdbl = 0;
1187 0 : *pqpair->shadow_doorbell.sq_eventidx = 0;
1188 0 : *pqpair->shadow_doorbell.cq_eventidx = 0;
1189 0 : }
1190 : free:
1191 0 : if (qpair->no_deletion_notification_needed == 0) {
1192 : /* Abort the rest of the I/O */
1193 0 : nvme_pcie_qpair_abort_trackers(qpair, 1);
1194 0 : }
1195 :
1196 0 : if (!pqpair->flags.defer_destruction) {
1197 0 : nvme_pcie_qpair_destroy(qpair);
1198 0 : }
1199 0 : return 0;
1200 0 : }
1201 :
1202 : static void
1203 3 : nvme_pcie_fail_request_bad_vtophys(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr)
1204 : {
1205 3 : if (!qpair->in_completion_context) {
1206 3 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
1207 :
1208 3 : tr->bad_vtophys = 1;
1209 3 : pqpair->flags.has_pending_vtophys_failures = 1;
1210 : return;
1211 3 : }
1212 :
1213 : /*
1214 : * Bad vtophys translation, so abort this request and return
1215 : * immediately.
1216 : */
1217 0 : SPDK_ERRLOG("vtophys or other payload buffer related error\n");
1218 0 : nvme_pcie_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC,
1219 : SPDK_NVME_SC_INVALID_FIELD,
1220 : 1 /* do not retry */, true);
1221 3 : }
1222 :
1223 : /*
1224 : * Append PRP list entries to describe a virtually contiguous buffer starting at virt_addr of len bytes.
1225 : *
1226 : * *prp_index will be updated to account for the number of PRP entries used.
1227 : */
1228 : static inline int
1229 25 : nvme_pcie_prp_list_append(struct spdk_nvme_ctrlr *ctrlr, struct nvme_tracker *tr,
1230 : uint32_t *prp_index, void *virt_addr, size_t len,
1231 : uint32_t page_size)
1232 : {
1233 25 : struct spdk_nvme_cmd *cmd = &tr->req->cmd;
1234 25 : uintptr_t page_mask = page_size - 1;
1235 25 : uint64_t phys_addr;
1236 25 : uint32_t i;
1237 :
1238 25 : SPDK_DEBUGLOG(nvme, "prp_index:%u virt_addr:%p len:%u\n",
1239 : *prp_index, virt_addr, (uint32_t)len);
1240 :
1241 25 : if (spdk_unlikely(((uintptr_t)virt_addr & 3) != 0)) {
1242 2 : SPDK_ERRLOG("virt_addr %p not dword aligned\n", virt_addr);
1243 2 : return -EFAULT;
1244 : }
1245 :
1246 23 : i = *prp_index;
1247 2070 : while (len) {
1248 2051 : uint32_t seg_len;
1249 :
1250 : /*
1251 : * prp_index 0 is stored in prp1, and the rest are stored in the prp[] array,
1252 : * so prp_index == count is valid.
1253 : */
1254 2051 : if (spdk_unlikely(i > SPDK_COUNTOF(tr->u.prp))) {
1255 2 : SPDK_ERRLOG("out of PRP entries\n");
1256 2 : return -EFAULT;
1257 : }
1258 :
1259 2049 : phys_addr = nvme_pcie_vtophys(ctrlr, virt_addr, NULL);
1260 2049 : if (spdk_unlikely(phys_addr == SPDK_VTOPHYS_ERROR)) {
1261 1 : SPDK_ERRLOG("vtophys(%p) failed\n", virt_addr);
1262 1 : return -EFAULT;
1263 : }
1264 :
1265 2048 : if (i == 0) {
1266 19 : SPDK_DEBUGLOG(nvme, "prp1 = %p\n", (void *)phys_addr);
1267 19 : cmd->dptr.prp.prp1 = phys_addr;
1268 19 : seg_len = page_size - ((uintptr_t)virt_addr & page_mask);
1269 19 : } else {
1270 2029 : if ((phys_addr & page_mask) != 0) {
1271 1 : SPDK_ERRLOG("PRP %u not page aligned (%p)\n", i, virt_addr);
1272 1 : return -EFAULT;
1273 : }
1274 :
1275 2028 : SPDK_DEBUGLOG(nvme, "prp[%u] = %p\n", i - 1, (void *)phys_addr);
1276 2028 : tr->u.prp[i - 1] = phys_addr;
1277 2028 : seg_len = page_size;
1278 : }
1279 :
1280 2047 : seg_len = spdk_min(seg_len, len);
1281 2047 : virt_addr = (uint8_t *)virt_addr + seg_len;
1282 2047 : len -= seg_len;
1283 2047 : i++;
1284 2051 : }
1285 :
1286 19 : cmd->psdt = SPDK_NVME_PSDT_PRP;
1287 19 : if (i <= 1) {
1288 6 : cmd->dptr.prp.prp2 = 0;
1289 19 : } else if (i == 2) {
1290 6 : cmd->dptr.prp.prp2 = tr->u.prp[0];
1291 6 : SPDK_DEBUGLOG(nvme, "prp2 = %p\n", (void *)cmd->dptr.prp.prp2);
1292 6 : } else {
1293 7 : cmd->dptr.prp.prp2 = tr->prp_sgl_bus_addr;
1294 7 : SPDK_DEBUGLOG(nvme, "prp2 = %p (PRP list)\n", (void *)cmd->dptr.prp.prp2);
1295 : }
1296 :
1297 19 : *prp_index = i;
1298 19 : return 0;
1299 25 : }
1300 :
1301 : static int
1302 0 : nvme_pcie_qpair_build_request_invalid(struct spdk_nvme_qpair *qpair,
1303 : struct nvme_request *req, struct nvme_tracker *tr, bool dword_aligned)
1304 : {
1305 0 : assert(0);
1306 : nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1307 : return -EINVAL;
1308 : }
1309 :
1310 : /**
1311 : * Build PRP list describing physically contiguous payload buffer.
1312 : */
1313 : static int
1314 4 : nvme_pcie_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
1315 : struct nvme_tracker *tr, bool dword_aligned)
1316 : {
1317 4 : uint32_t prp_index = 0;
1318 4 : int rc;
1319 :
1320 8 : rc = nvme_pcie_prp_list_append(qpair->ctrlr, tr, &prp_index,
1321 4 : (uint8_t *)req->payload.contig_or_cb_arg + req->payload_offset,
1322 4 : req->payload_size, qpair->ctrlr->page_size);
1323 4 : if (rc) {
1324 1 : nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1325 1 : } else {
1326 3 : SPDK_DEBUGLOG(nvme, "Number of PRP entries: %" PRIu32 "\n", prp_index);
1327 : }
1328 :
1329 8 : return rc;
1330 4 : }
1331 :
1332 : /**
1333 : * Build an SGL describing a physically contiguous payload buffer.
1334 : *
1335 : * This is more efficient than using PRP because large buffers can be
1336 : * described this way.
1337 : */
1338 : static int
1339 3 : nvme_pcie_qpair_build_contig_hw_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
1340 : struct nvme_tracker *tr, bool dword_aligned)
1341 : {
1342 3 : uint8_t *virt_addr;
1343 3 : uint64_t phys_addr, mapping_length;
1344 3 : uint32_t length;
1345 3 : struct spdk_nvme_sgl_descriptor *sgl;
1346 3 : uint32_t nseg = 0;
1347 :
1348 3 : assert(req->payload_size != 0);
1349 3 : assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_CONTIG);
1350 :
1351 3 : sgl = tr->u.sgl;
1352 3 : req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;
1353 3 : req->cmd.dptr.sgl1.unkeyed.subtype = 0;
1354 :
1355 3 : length = req->payload_size;
1356 : /* ubsan complains about applying zero offset to null pointer if contig_or_cb_arg is NULL,
1357 : * so just double cast it to make it go away */
1358 3 : virt_addr = (uint8_t *)((uintptr_t)req->payload.contig_or_cb_arg + req->payload_offset);
1359 :
1360 7 : while (length > 0) {
1361 4 : if (nseg >= NVME_MAX_SGL_DESCRIPTORS) {
1362 0 : nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1363 0 : return -EFAULT;
1364 : }
1365 :
1366 4 : if (dword_aligned && ((uintptr_t)virt_addr & 3)) {
1367 0 : SPDK_ERRLOG("virt_addr %p not dword aligned\n", virt_addr);
1368 0 : nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1369 0 : return -EFAULT;
1370 : }
1371 :
1372 4 : mapping_length = length;
1373 4 : phys_addr = nvme_pcie_vtophys(qpair->ctrlr, virt_addr, &mapping_length);
1374 4 : if (phys_addr == SPDK_VTOPHYS_ERROR) {
1375 0 : nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1376 0 : return -EFAULT;
1377 : }
1378 :
1379 4 : mapping_length = spdk_min(length, mapping_length);
1380 :
1381 4 : length -= mapping_length;
1382 4 : virt_addr += mapping_length;
1383 :
1384 4 : sgl->unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
1385 4 : sgl->unkeyed.length = mapping_length;
1386 4 : sgl->address = phys_addr;
1387 4 : sgl->unkeyed.subtype = 0;
1388 :
1389 4 : sgl++;
1390 4 : nseg++;
1391 : }
1392 :
1393 3 : if (nseg == 1) {
1394 : /*
1395 : * The whole transfer can be described by a single SGL descriptor.
1396 : * Use the special case described by the spec where SGL1's type is Data Block.
1397 : * This means the SGL in the tracker is not used at all, so copy the first (and only)
1398 : * SGL element into SGL1.
1399 : */
1400 2 : req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
1401 2 : req->cmd.dptr.sgl1.address = tr->u.sgl[0].address;
1402 2 : req->cmd.dptr.sgl1.unkeyed.length = tr->u.sgl[0].unkeyed.length;
1403 2 : } else {
1404 : /* SPDK NVMe driver supports only 1 SGL segment for now, it is enough because
1405 : * NVME_MAX_SGL_DESCRIPTORS * 16 is less than one page.
1406 : */
1407 1 : req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT;
1408 1 : req->cmd.dptr.sgl1.address = tr->prp_sgl_bus_addr;
1409 1 : req->cmd.dptr.sgl1.unkeyed.length = nseg * sizeof(struct spdk_nvme_sgl_descriptor);
1410 : }
1411 :
1412 3 : SPDK_DEBUGLOG(nvme, "Number of SGL descriptors: %" PRIu32 "\n", nseg);
1413 3 : return 0;
1414 3 : }
1415 :
1416 : /**
1417 : * Build SGL list describing scattered payload buffer.
1418 : */
1419 : static int
1420 2 : nvme_pcie_qpair_build_hw_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
1421 : struct nvme_tracker *tr, bool dword_aligned)
1422 : {
1423 2 : int rc;
1424 2 : void *virt_addr;
1425 2 : uint64_t phys_addr, mapping_length;
1426 2 : uint32_t remaining_transfer_len, remaining_user_sge_len, length;
1427 2 : struct spdk_nvme_sgl_descriptor *sgl;
1428 2 : uint32_t nseg = 0;
1429 2 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
1430 :
1431 : /*
1432 : * Build scattered payloads.
1433 : */
1434 2 : assert(req->payload_size != 0);
1435 2 : assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_SGL);
1436 2 : assert(req->payload.reset_sgl_fn != NULL);
1437 2 : assert(req->payload.next_sge_fn != NULL);
1438 2 : req->payload.reset_sgl_fn(req->payload.contig_or_cb_arg, req->payload_offset);
1439 :
1440 2 : sgl = tr->u.sgl;
1441 2 : req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;
1442 2 : req->cmd.dptr.sgl1.unkeyed.subtype = 0;
1443 :
1444 2 : remaining_transfer_len = req->payload_size;
1445 :
1446 6 : while (remaining_transfer_len > 0) {
1447 4 : rc = req->payload.next_sge_fn(req->payload.contig_or_cb_arg,
1448 : &virt_addr, &remaining_user_sge_len);
1449 4 : if (rc) {
1450 0 : nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1451 0 : return -EFAULT;
1452 : }
1453 :
1454 : /* Bit Bucket SGL descriptor */
1455 4 : if ((uint64_t)virt_addr == UINT64_MAX) {
1456 : /* TODO: enable WRITE and COMPARE when necessary */
1457 0 : if (req->cmd.opc != SPDK_NVME_OPC_READ) {
1458 0 : SPDK_ERRLOG("Only READ command can be supported\n");
1459 0 : goto exit;
1460 : }
1461 0 : if (nseg >= NVME_MAX_SGL_DESCRIPTORS) {
1462 0 : SPDK_ERRLOG("Too many SGL entries\n");
1463 0 : goto exit;
1464 : }
1465 :
1466 0 : sgl->unkeyed.type = SPDK_NVME_SGL_TYPE_BIT_BUCKET;
1467 : /* If the SGL describes a destination data buffer, the length of data
1468 : * buffer shall be discarded by controller, and the length is included
1469 : * in Number of Logical Blocks (NLB) parameter. Otherwise, the length
1470 : * is not included in the NLB parameter.
1471 : */
1472 0 : remaining_user_sge_len = spdk_min(remaining_user_sge_len, remaining_transfer_len);
1473 0 : remaining_transfer_len -= remaining_user_sge_len;
1474 :
1475 0 : sgl->unkeyed.length = remaining_user_sge_len;
1476 0 : sgl->address = 0;
1477 0 : sgl->unkeyed.subtype = 0;
1478 :
1479 0 : sgl++;
1480 0 : nseg++;
1481 :
1482 0 : continue;
1483 : }
1484 :
1485 4 : remaining_user_sge_len = spdk_min(remaining_user_sge_len, remaining_transfer_len);
1486 4 : remaining_transfer_len -= remaining_user_sge_len;
1487 8 : while (remaining_user_sge_len > 0) {
1488 4 : if (nseg >= NVME_MAX_SGL_DESCRIPTORS) {
1489 0 : SPDK_ERRLOG("Too many SGL entries\n");
1490 0 : goto exit;
1491 : }
1492 :
1493 4 : if (dword_aligned && ((uintptr_t)virt_addr & 3)) {
1494 0 : SPDK_ERRLOG("virt_addr %p not dword aligned\n", virt_addr);
1495 0 : goto exit;
1496 : }
1497 :
1498 4 : mapping_length = remaining_user_sge_len;
1499 4 : phys_addr = nvme_pcie_vtophys(qpair->ctrlr, virt_addr, &mapping_length);
1500 4 : if (phys_addr == SPDK_VTOPHYS_ERROR) {
1501 0 : goto exit;
1502 : }
1503 :
1504 4 : length = spdk_min(remaining_user_sge_len, mapping_length);
1505 4 : remaining_user_sge_len -= length;
1506 4 : virt_addr = (uint8_t *)virt_addr + length;
1507 :
1508 6 : if (!pqpair->flags.disable_pcie_sgl_merge && nseg > 0 &&
1509 2 : phys_addr == (*(sgl - 1)).address + (*(sgl - 1)).unkeyed.length) {
1510 : /* extend previous entry */
1511 0 : (*(sgl - 1)).unkeyed.length += length;
1512 0 : continue;
1513 : }
1514 :
1515 4 : sgl->unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
1516 4 : sgl->unkeyed.length = length;
1517 4 : sgl->address = phys_addr;
1518 4 : sgl->unkeyed.subtype = 0;
1519 :
1520 4 : sgl++;
1521 4 : nseg++;
1522 : }
1523 : }
1524 :
1525 2 : if (nseg == 1) {
1526 : /*
1527 : * The whole transfer can be described by a single SGL descriptor.
1528 : * Use the special case described by the spec where SGL1's type is Data Block.
1529 : * This means the SGL in the tracker is not used at all, so copy the first (and only)
1530 : * SGL element into SGL1.
1531 : */
1532 1 : req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
1533 1 : req->cmd.dptr.sgl1.address = tr->u.sgl[0].address;
1534 1 : req->cmd.dptr.sgl1.unkeyed.length = tr->u.sgl[0].unkeyed.length;
1535 1 : } else {
1536 : /* SPDK NVMe driver supports only 1 SGL segment for now, it is enough because
1537 : * NVME_MAX_SGL_DESCRIPTORS * 16 is less than one page.
1538 : */
1539 1 : req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT;
1540 1 : req->cmd.dptr.sgl1.address = tr->prp_sgl_bus_addr;
1541 1 : req->cmd.dptr.sgl1.unkeyed.length = nseg * sizeof(struct spdk_nvme_sgl_descriptor);
1542 : }
1543 :
1544 2 : SPDK_DEBUGLOG(nvme, "Number of SGL descriptors: %" PRIu32 "\n", nseg);
1545 2 : return 0;
1546 :
1547 : exit:
1548 0 : nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1549 0 : return -EFAULT;
1550 2 : }
1551 :
1552 : /**
1553 : * Build PRP list describing scattered payload buffer.
1554 : */
1555 : static int
1556 1 : nvme_pcie_qpair_build_prps_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
1557 : struct nvme_tracker *tr, bool dword_aligned)
1558 : {
1559 1 : int rc;
1560 1 : void *virt_addr;
1561 1 : uint32_t remaining_transfer_len, length;
1562 1 : uint32_t prp_index = 0;
1563 1 : uint32_t page_size = qpair->ctrlr->page_size;
1564 :
1565 : /*
1566 : * Build scattered payloads.
1567 : */
1568 1 : assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_SGL);
1569 1 : assert(req->payload.reset_sgl_fn != NULL);
1570 1 : req->payload.reset_sgl_fn(req->payload.contig_or_cb_arg, req->payload_offset);
1571 :
1572 1 : remaining_transfer_len = req->payload_size;
1573 2 : while (remaining_transfer_len > 0) {
1574 1 : assert(req->payload.next_sge_fn != NULL);
1575 1 : rc = req->payload.next_sge_fn(req->payload.contig_or_cb_arg, &virt_addr, &length);
1576 1 : if (rc) {
1577 0 : nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1578 0 : return -EFAULT;
1579 : }
1580 :
1581 1 : length = spdk_min(remaining_transfer_len, length);
1582 :
1583 : /*
1584 : * Any incompatible sges should have been handled up in the splitting routine,
1585 : * but assert here as an additional check.
1586 : *
1587 : * All SGEs except last must end on a page boundary.
1588 : */
1589 1 : assert((length == remaining_transfer_len) ||
1590 : _is_page_aligned((uintptr_t)virt_addr + length, page_size));
1591 :
1592 1 : rc = nvme_pcie_prp_list_append(qpair->ctrlr, tr, &prp_index, virt_addr, length, page_size);
1593 1 : if (rc) {
1594 0 : nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1595 0 : return rc;
1596 : }
1597 :
1598 1 : remaining_transfer_len -= length;
1599 : }
1600 :
1601 1 : SPDK_DEBUGLOG(nvme, "Number of PRP entries: %" PRIu32 "\n", prp_index);
1602 1 : return 0;
1603 1 : }
1604 :
1605 : typedef int(*build_req_fn)(struct spdk_nvme_qpair *, struct nvme_request *, struct nvme_tracker *,
1606 : bool);
1607 :
1608 : static build_req_fn const g_nvme_pcie_build_req_table[][2] = {
1609 : [NVME_PAYLOAD_TYPE_INVALID] = {
1610 : nvme_pcie_qpair_build_request_invalid, /* PRP */
1611 : nvme_pcie_qpair_build_request_invalid /* SGL */
1612 : },
1613 : [NVME_PAYLOAD_TYPE_CONTIG] = {
1614 : nvme_pcie_qpair_build_contig_request, /* PRP */
1615 : nvme_pcie_qpair_build_contig_hw_sgl_request /* SGL */
1616 : },
1617 : [NVME_PAYLOAD_TYPE_SGL] = {
1618 : nvme_pcie_qpair_build_prps_sgl_request, /* PRP */
1619 : nvme_pcie_qpair_build_hw_sgl_request /* SGL */
1620 : }
1621 : };
1622 :
1623 : static int
1624 5 : nvme_pcie_qpair_build_metadata(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr,
1625 : bool sgl_supported, bool mptr_sgl_supported, bool dword_aligned)
1626 : {
1627 5 : void *md_payload;
1628 5 : struct nvme_request *req = tr->req;
1629 5 : uint64_t mapping_length;
1630 :
1631 5 : if (req->payload.md) {
1632 5 : md_payload = (uint8_t *)req->payload.md + req->md_offset;
1633 5 : if (dword_aligned && ((uintptr_t)md_payload & 3)) {
1634 0 : SPDK_ERRLOG("virt_addr %p not dword aligned\n", md_payload);
1635 0 : goto exit;
1636 : }
1637 :
1638 5 : mapping_length = req->md_size;
1639 5 : if (sgl_supported && mptr_sgl_supported && dword_aligned) {
1640 2 : assert(req->cmd.psdt == SPDK_NVME_PSDT_SGL_MPTR_CONTIG);
1641 2 : req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_SGL;
1642 :
1643 2 : tr->meta_sgl.address = nvme_pcie_vtophys(qpair->ctrlr, md_payload, &mapping_length);
1644 2 : if (tr->meta_sgl.address == SPDK_VTOPHYS_ERROR || mapping_length != req->md_size) {
1645 1 : goto exit;
1646 : }
1647 1 : tr->meta_sgl.unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
1648 1 : tr->meta_sgl.unkeyed.length = req->md_size;
1649 1 : tr->meta_sgl.unkeyed.subtype = 0;
1650 1 : req->cmd.mptr = tr->prp_sgl_bus_addr - sizeof(struct spdk_nvme_sgl_descriptor);
1651 1 : } else {
1652 3 : req->cmd.mptr = nvme_pcie_vtophys(qpair->ctrlr, md_payload, &mapping_length);
1653 3 : if (req->cmd.mptr == SPDK_VTOPHYS_ERROR || mapping_length != req->md_size) {
1654 1 : goto exit;
1655 : }
1656 : }
1657 3 : }
1658 :
1659 3 : return 0;
1660 :
1661 : exit:
1662 2 : nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1663 2 : return -EINVAL;
1664 5 : }
1665 :
1666 : int
1667 0 : nvme_pcie_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
1668 : {
1669 0 : struct nvme_tracker *tr;
1670 0 : int rc = 0;
1671 0 : struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
1672 0 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
1673 0 : enum nvme_payload_type payload_type;
1674 0 : bool sgl_supported;
1675 0 : bool mptr_sgl_supported;
1676 0 : bool dword_aligned = true;
1677 :
1678 0 : if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {
1679 0 : nvme_ctrlr_lock(ctrlr);
1680 0 : }
1681 :
1682 0 : tr = TAILQ_FIRST(&pqpair->free_tr);
1683 :
1684 0 : if (tr == NULL) {
1685 0 : pqpair->stat->queued_requests++;
1686 : /* Inform the upper layer to try again later. */
1687 0 : rc = -EAGAIN;
1688 0 : goto exit;
1689 : }
1690 :
1691 0 : pqpair->stat->submitted_requests++;
1692 0 : TAILQ_REMOVE(&pqpair->free_tr, tr, tq_list); /* remove tr from free_tr */
1693 0 : TAILQ_INSERT_TAIL(&pqpair->outstanding_tr, tr, tq_list);
1694 0 : pqpair->qpair.queue_depth++;
1695 0 : tr->req = req;
1696 0 : tr->cb_fn = req->cb_fn;
1697 0 : tr->cb_arg = req->cb_arg;
1698 0 : req->cmd.cid = tr->cid;
1699 : /* Use PRP by default. This bit will be overridden below if needed. */
1700 0 : req->cmd.psdt = SPDK_NVME_PSDT_PRP;
1701 :
1702 0 : if (req->payload_size != 0) {
1703 0 : payload_type = nvme_payload_type(&req->payload);
1704 : /* According to the specification, PRPs shall be used for all
1705 : * Admin commands for NVMe over PCIe implementations.
1706 : */
1707 0 : sgl_supported = (ctrlr->flags & SPDK_NVME_CTRLR_SGL_SUPPORTED) != 0 &&
1708 0 : !nvme_qpair_is_admin_queue(qpair);
1709 0 : mptr_sgl_supported = (ctrlr->flags & SPDK_NVME_CTRLR_MPTR_SGL_SUPPORTED) != 0 &&
1710 0 : !nvme_qpair_is_admin_queue(qpair);
1711 :
1712 0 : if (sgl_supported) {
1713 : /* Don't use SGL for DSM command */
1714 0 : if (spdk_unlikely((ctrlr->quirks & NVME_QUIRK_NO_SGL_FOR_DSM) &&
1715 : (req->cmd.opc == SPDK_NVME_OPC_DATASET_MANAGEMENT))) {
1716 0 : sgl_supported = false;
1717 0 : }
1718 0 : }
1719 :
1720 0 : if (sgl_supported && !(ctrlr->flags & SPDK_NVME_CTRLR_SGL_REQUIRES_DWORD_ALIGNMENT)) {
1721 0 : dword_aligned = false;
1722 0 : }
1723 :
1724 : /* If we fail to build the request or the metadata, do not return the -EFAULT back up
1725 : * the stack. This ensures that we always fail these types of requests via a
1726 : * completion callback, and never in the context of the submission.
1727 : */
1728 0 : rc = g_nvme_pcie_build_req_table[payload_type][sgl_supported](qpair, req, tr, dword_aligned);
1729 0 : if (rc < 0) {
1730 0 : assert(rc == -EFAULT);
1731 0 : rc = 0;
1732 0 : goto exit;
1733 : }
1734 :
1735 0 : rc = nvme_pcie_qpair_build_metadata(qpair, tr, sgl_supported, mptr_sgl_supported, dword_aligned);
1736 0 : if (rc < 0) {
1737 0 : assert(rc == -EFAULT);
1738 0 : rc = 0;
1739 0 : goto exit;
1740 : }
1741 0 : }
1742 :
1743 0 : nvme_pcie_qpair_submit_tracker(qpair, tr);
1744 :
1745 : exit:
1746 0 : if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {
1747 0 : nvme_ctrlr_unlock(ctrlr);
1748 0 : }
1749 :
1750 0 : return rc;
1751 0 : }
1752 :
1753 : struct spdk_nvme_transport_poll_group *
1754 1 : nvme_pcie_poll_group_create(void)
1755 : {
1756 1 : struct nvme_pcie_poll_group *group = calloc(1, sizeof(*group));
1757 :
1758 1 : if (group == NULL) {
1759 0 : SPDK_ERRLOG("Unable to allocate poll group.\n");
1760 0 : return NULL;
1761 : }
1762 :
1763 1 : return &group->group;
1764 1 : }
1765 :
1766 : int
1767 0 : nvme_pcie_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
1768 : {
1769 0 : return 0;
1770 : }
1771 :
1772 : int
1773 0 : nvme_pcie_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)
1774 : {
1775 0 : return 0;
1776 : }
1777 :
1778 : int
1779 0 : nvme_pcie_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup,
1780 : struct spdk_nvme_qpair *qpair)
1781 : {
1782 0 : return 0;
1783 : }
1784 :
1785 : int
1786 0 : nvme_pcie_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,
1787 : struct spdk_nvme_qpair *qpair)
1788 : {
1789 0 : struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
1790 :
1791 0 : pqpair->stat = &g_dummy_stat;
1792 0 : return 0;
1793 0 : }
1794 :
1795 : int64_t
1796 0 : nvme_pcie_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,
1797 : uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
1798 : {
1799 0 : struct spdk_nvme_qpair *qpair, *tmp_qpair;
1800 0 : int32_t local_completions = 0;
1801 0 : int64_t total_completions = 0;
1802 :
1803 0 : STAILQ_FOREACH_SAFE(qpair, &tgroup->disconnected_qpairs, poll_group_stailq, tmp_qpair) {
1804 0 : disconnected_qpair_cb(qpair, tgroup->group->ctx);
1805 0 : }
1806 :
1807 0 : STAILQ_FOREACH_SAFE(qpair, &tgroup->connected_qpairs, poll_group_stailq, tmp_qpair) {
1808 0 : local_completions = spdk_nvme_qpair_process_completions(qpair, completions_per_qpair);
1809 0 : if (spdk_unlikely(local_completions < 0)) {
1810 0 : disconnected_qpair_cb(qpair, tgroup->group->ctx);
1811 0 : total_completions = -ENXIO;
1812 0 : } else if (spdk_likely(total_completions >= 0)) {
1813 0 : total_completions += local_completions;
1814 0 : }
1815 0 : }
1816 :
1817 0 : return total_completions;
1818 0 : }
1819 :
1820 : void
1821 0 : nvme_pcie_poll_group_check_disconnected_qpairs(struct spdk_nvme_transport_poll_group *tgroup,
1822 : spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
1823 : {
1824 0 : struct spdk_nvme_qpair *qpair, *tmp_qpair;
1825 :
1826 0 : STAILQ_FOREACH_SAFE(qpair, &tgroup->disconnected_qpairs, poll_group_stailq, tmp_qpair) {
1827 0 : disconnected_qpair_cb(qpair, tgroup->group->ctx);
1828 0 : }
1829 0 : }
1830 :
1831 : int
1832 1 : nvme_pcie_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup)
1833 : {
1834 1 : if (!STAILQ_EMPTY(&tgroup->connected_qpairs) || !STAILQ_EMPTY(&tgroup->disconnected_qpairs)) {
1835 0 : return -EBUSY;
1836 : }
1837 :
1838 1 : free(tgroup);
1839 :
1840 1 : return 0;
1841 1 : }
1842 :
1843 : int
1844 3 : nvme_pcie_poll_group_get_stats(struct spdk_nvme_transport_poll_group *tgroup,
1845 : struct spdk_nvme_transport_poll_group_stat **_stats)
1846 : {
1847 3 : struct nvme_pcie_poll_group *group;
1848 3 : struct spdk_nvme_transport_poll_group_stat *stats;
1849 :
1850 3 : if (tgroup == NULL || _stats == NULL) {
1851 2 : SPDK_ERRLOG("Invalid stats or group pointer\n");
1852 2 : return -EINVAL;
1853 : }
1854 :
1855 1 : stats = calloc(1, sizeof(*stats));
1856 1 : if (!stats) {
1857 0 : SPDK_ERRLOG("Can't allocate memory for stats\n");
1858 0 : return -ENOMEM;
1859 : }
1860 1 : stats->trtype = SPDK_NVME_TRANSPORT_PCIE;
1861 1 : group = SPDK_CONTAINEROF(tgroup, struct nvme_pcie_poll_group, group);
1862 1 : memcpy(&stats->pcie, &group->stats, sizeof(group->stats));
1863 :
1864 1 : *_stats = stats;
1865 :
1866 1 : return 0;
1867 3 : }
1868 :
1869 : void
1870 1 : nvme_pcie_poll_group_free_stats(struct spdk_nvme_transport_poll_group *tgroup,
1871 : struct spdk_nvme_transport_poll_group_stat *stats)
1872 : {
1873 1 : free(stats);
1874 1 : }
1875 :
1876 : static void
1877 0 : nvme_pcie_trace(void)
1878 : {
1879 0 : struct spdk_trace_tpoint_opts opts[] = {
1880 : {
1881 : "NVME_PCIE_SUBMIT", TRACE_NVME_PCIE_SUBMIT,
1882 : OWNER_TYPE_NVME_PCIE_QP, OBJECT_NVME_PCIE_REQ, 1,
1883 : { { "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 },
1884 : { "cid", SPDK_TRACE_ARG_TYPE_INT, 4 },
1885 : { "opc", SPDK_TRACE_ARG_TYPE_INT, 4 },
1886 : { "dw10", SPDK_TRACE_ARG_TYPE_PTR, 4 },
1887 : { "dw11", SPDK_TRACE_ARG_TYPE_PTR, 4 },
1888 : { "dw12", SPDK_TRACE_ARG_TYPE_PTR, 4 },
1889 : { "qd", SPDK_TRACE_ARG_TYPE_INT, 4 }
1890 : }
1891 : },
1892 : {
1893 : "NVME_PCIE_COMPLETE", TRACE_NVME_PCIE_COMPLETE,
1894 : OWNER_TYPE_NVME_PCIE_QP, OBJECT_NVME_PCIE_REQ, 0,
1895 : { { "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 },
1896 : { "cid", SPDK_TRACE_ARG_TYPE_INT, 4 },
1897 : { "cpl", SPDK_TRACE_ARG_TYPE_PTR, 4 },
1898 : { "qd", SPDK_TRACE_ARG_TYPE_INT, 4 }
1899 : }
1900 : },
1901 : };
1902 :
1903 0 : spdk_trace_register_object(OBJECT_NVME_PCIE_REQ, 'p');
1904 0 : spdk_trace_register_owner_type(OWNER_TYPE_NVME_PCIE_QP, 'q');
1905 0 : spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts));
1906 0 : }
1907 2 : SPDK_TRACE_REGISTER_FN(nvme_pcie_trace, "nvme_pcie", TRACE_GROUP_NVME_PCIE)
|
