Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2017 Intel Corporation.
3 : * All rights reserved.
4 : */
5 :
6 : #include "spdk/stdinc.h"
7 :
8 : #include "spdk/bdev.h"
9 : #include "spdk/endian.h"
10 : #include "spdk/env.h"
11 : #include "spdk/thread.h"
12 : #include "spdk/scsi_spec.h"
13 : #include "spdk/string.h"
14 : #include "spdk/util.h"
15 : #include "spdk/json.h"
16 :
17 : #include "spdk/bdev_module.h"
18 : #include "spdk/log.h"
19 : #include "spdk_internal/virtio.h"
20 : #include "spdk_internal/vhost_user.h"
21 :
22 : #include <linux/virtio_scsi.h>
23 : #include <linux/virtio_ids.h>
24 :
25 : #include "bdev_virtio.h"
26 :
27 : #define BDEV_VIRTIO_MAX_TARGET 64
28 : #define BDEV_VIRTIO_SCAN_PAYLOAD_SIZE 256
29 : #define MGMT_POLL_PERIOD_US (1000 * 5)
30 : #define CTRLQ_RING_SIZE 16
31 : #define SCAN_REQUEST_RETRIES 5
32 :
33 : /* Number of non-request queues - eventq and controlq */
34 : #define SPDK_VIRTIO_SCSI_QUEUE_NUM_FIXED 2
35 :
36 : #define VIRTIO_SCSI_EVENTQ_BUFFER_COUNT 16
37 :
38 : #define VIRTIO_SCSI_CONTROLQ 0
39 : #define VIRTIO_SCSI_EVENTQ 1
40 : #define VIRTIO_SCSI_REQUESTQ 2
41 :
42 : static int bdev_virtio_initialize(void);
43 : static void bdev_virtio_finish(void);
44 :
45 : struct virtio_scsi_dev {
46 : /* Generic virtio device data. */
47 : struct virtio_dev vdev;
48 :
49 : /** Detected SCSI LUNs */
50 : TAILQ_HEAD(, virtio_scsi_disk) luns;
51 :
52 : /** Context for the SCSI target scan. */
53 : struct virtio_scsi_scan_base *scan_ctx;
54 :
55 : /** Controlq poller. */
56 : struct spdk_poller *mgmt_poller;
57 :
58 : /** Controlq messages to be sent. */
59 : struct spdk_ring *ctrlq_ring;
60 :
61 : /** Buffers for the eventq. */
62 : struct virtio_scsi_eventq_io *eventq_ios;
63 :
64 : /** Device marked for removal. */
65 : bool removed;
66 :
67 : /** Callback to be called after vdev removal. */
68 : bdev_virtio_remove_cb remove_cb;
69 :
70 : /** Context for the `remove_cb`. */
71 : void *remove_ctx;
72 :
73 : TAILQ_ENTRY(virtio_scsi_dev) tailq;
74 : };
75 :
76 : struct virtio_scsi_io_ctx {
77 : struct iovec iov_req;
78 : struct iovec iov_resp;
79 : union {
80 : struct virtio_scsi_cmd_req req;
81 : struct virtio_scsi_ctrl_tmf_req tmf_req;
82 : };
83 : union {
84 : struct virtio_scsi_cmd_resp resp;
85 : struct virtio_scsi_ctrl_tmf_resp tmf_resp;
86 : };
87 : };
88 :
89 : struct virtio_scsi_eventq_io {
90 : struct iovec iov;
91 : struct virtio_scsi_event ev;
92 : };
93 :
94 : struct virtio_scsi_scan_info {
95 : uint64_t num_blocks;
96 : uint32_t block_size;
97 : uint8_t target;
98 : bool unmap_supported;
99 : TAILQ_ENTRY(virtio_scsi_scan_info) tailq;
100 : };
101 :
102 : struct virtio_scsi_scan_base {
103 : struct virtio_scsi_dev *svdev;
104 :
105 : /** I/O channel used for the scan I/O. */
106 : struct bdev_virtio_io_channel *channel;
107 :
108 : bdev_virtio_create_cb cb_fn;
109 : void *cb_arg;
110 :
111 : /** Scan all targets on the device. */
112 : bool full_scan;
113 :
114 : /** Start a full rescan after receiving next scan I/O response. */
115 : bool restart;
116 :
117 : /** Additional targets to be (re)scanned. */
118 : TAILQ_HEAD(, virtio_scsi_scan_info) scan_queue;
119 :
120 : /** Remaining attempts for sending the current request. */
121 : unsigned retries;
122 :
123 : /** If set, the last scan I/O needs to be resent */
124 : bool needs_resend;
125 :
126 : struct virtio_scsi_io_ctx io_ctx;
127 : struct iovec iov;
128 : uint8_t payload[BDEV_VIRTIO_SCAN_PAYLOAD_SIZE];
129 :
130 : /** Scan results for the current target. */
131 : struct virtio_scsi_scan_info info;
132 : };
133 :
134 : struct virtio_scsi_disk {
135 : struct spdk_bdev bdev;
136 : struct virtio_scsi_dev *svdev;
137 : struct virtio_scsi_scan_info info;
138 :
139 : /** Descriptor opened just to be notified of external bdev hotremove. */
140 : struct spdk_bdev_desc *notify_desc;
141 :
142 : /** Disk marked for removal. */
143 : bool removed;
144 : TAILQ_ENTRY(virtio_scsi_disk) link;
145 : };
146 :
147 : struct bdev_virtio_io_channel {
148 : struct virtio_scsi_dev *svdev;
149 :
150 : /** Virtqueue exclusively assigned to this channel. */
151 : struct virtqueue *vq;
152 :
153 : /** Virtio response poller. */
154 : struct spdk_poller *poller;
155 : };
156 :
157 : static TAILQ_HEAD(, virtio_scsi_dev) g_virtio_scsi_devs =
158 : TAILQ_HEAD_INITIALIZER(g_virtio_scsi_devs);
159 :
160 : static pthread_mutex_t g_virtio_scsi_mutex = PTHREAD_MUTEX_INITIALIZER;
161 :
162 : /** Module finish in progress */
163 : static bool g_bdev_virtio_finish = false;
164 :
165 : /* Features desired/implemented by this driver. */
166 : #define VIRTIO_SCSI_DEV_SUPPORTED_FEATURES \
167 : (1ULL << VIRTIO_SCSI_F_INOUT | \
168 : 1ULL << VIRTIO_SCSI_F_HOTPLUG | \
169 : 1ULL << VIRTIO_RING_F_EVENT_IDX)
170 :
171 : static void virtio_scsi_dev_unregister_cb(void *io_device);
172 : static void virtio_scsi_dev_remove(struct virtio_scsi_dev *svdev,
173 : bdev_virtio_remove_cb cb_fn, void *cb_arg);
174 : static int bdev_virtio_scsi_ch_create_cb(void *io_device, void *ctx_buf);
175 : static void bdev_virtio_scsi_ch_destroy_cb(void *io_device, void *ctx_buf);
176 : static void process_scan_resp(struct virtio_scsi_scan_base *base);
177 : static int bdev_virtio_mgmt_poll(void *arg);
178 :
179 : static int
180 0 : virtio_scsi_dev_send_eventq_io(struct virtqueue *vq, struct virtio_scsi_eventq_io *io)
181 : {
182 : int rc;
183 :
184 0 : rc = virtqueue_req_start(vq, io, 1);
185 0 : if (rc != 0) {
186 0 : return -1;
187 : }
188 :
189 0 : virtqueue_req_add_iovs(vq, &io->iov, 1, SPDK_VIRTIO_DESC_WR);
190 0 : virtqueue_req_flush(vq);
191 :
192 0 : return 0;
193 : }
194 :
195 : static int
196 0 : virtio_scsi_dev_init(struct virtio_scsi_dev *svdev, uint16_t max_queues, uint64_t feature_bits)
197 : {
198 0 : struct virtio_dev *vdev = &svdev->vdev;
199 : struct spdk_ring *ctrlq_ring;
200 : struct virtio_scsi_eventq_io *eventq_io;
201 : struct virtqueue *eventq;
202 : uint16_t i, num_events;
203 : int rc;
204 :
205 0 : rc = virtio_dev_reset(vdev, feature_bits);
206 0 : if (rc != 0) {
207 0 : return rc;
208 : }
209 :
210 0 : rc = virtio_dev_start(vdev, max_queues, SPDK_VIRTIO_SCSI_QUEUE_NUM_FIXED);
211 0 : if (rc != 0) {
212 0 : return rc;
213 : }
214 :
215 0 : ctrlq_ring = spdk_ring_create(SPDK_RING_TYPE_MP_SC, CTRLQ_RING_SIZE,
216 : SPDK_ENV_SOCKET_ID_ANY);
217 0 : if (ctrlq_ring == NULL) {
218 0 : SPDK_ERRLOG("Failed to allocate send ring for the controlq.\n");
219 0 : return -1;
220 : }
221 :
222 0 : rc = virtio_dev_acquire_queue(vdev, VIRTIO_SCSI_CONTROLQ);
223 0 : if (rc != 0) {
224 0 : SPDK_ERRLOG("Failed to acquire the controlq.\n");
225 0 : spdk_ring_free(ctrlq_ring);
226 0 : return -1;
227 : }
228 :
229 0 : rc = virtio_dev_acquire_queue(vdev, VIRTIO_SCSI_EVENTQ);
230 0 : if (rc != 0) {
231 0 : SPDK_ERRLOG("Failed to acquire the eventq.\n");
232 0 : virtio_dev_release_queue(vdev, VIRTIO_SCSI_CONTROLQ);
233 0 : spdk_ring_free(ctrlq_ring);
234 0 : return -1;
235 : }
236 :
237 0 : eventq = vdev->vqs[VIRTIO_SCSI_EVENTQ];
238 0 : num_events = spdk_min(eventq->vq_nentries, VIRTIO_SCSI_EVENTQ_BUFFER_COUNT);
239 0 : svdev->eventq_ios = spdk_zmalloc(sizeof(*svdev->eventq_ios) * num_events,
240 : 0, NULL, SPDK_ENV_LCORE_ID_ANY,
241 : SPDK_MALLOC_DMA);
242 0 : if (svdev->eventq_ios == NULL) {
243 0 : SPDK_ERRLOG("cannot allocate memory for %"PRIu16" eventq buffers\n",
244 : num_events);
245 0 : virtio_dev_release_queue(vdev, VIRTIO_SCSI_EVENTQ);
246 0 : virtio_dev_release_queue(vdev, VIRTIO_SCSI_CONTROLQ);
247 0 : spdk_ring_free(ctrlq_ring);
248 0 : return -1;
249 : }
250 :
251 0 : for (i = 0; i < num_events; i++) {
252 0 : eventq_io = &svdev->eventq_ios[i];
253 0 : eventq_io->iov.iov_base = &eventq_io->ev;
254 0 : eventq_io->iov.iov_len = sizeof(eventq_io->ev);
255 0 : virtio_scsi_dev_send_eventq_io(eventq, eventq_io);
256 : }
257 :
258 0 : svdev->ctrlq_ring = ctrlq_ring;
259 :
260 0 : svdev->mgmt_poller = SPDK_POLLER_REGISTER(bdev_virtio_mgmt_poll, svdev,
261 : MGMT_POLL_PERIOD_US);
262 :
263 0 : TAILQ_INIT(&svdev->luns);
264 0 : svdev->scan_ctx = NULL;
265 0 : svdev->removed = false;
266 0 : svdev->remove_cb = NULL;
267 0 : svdev->remove_ctx = NULL;
268 :
269 0 : spdk_io_device_register(svdev, bdev_virtio_scsi_ch_create_cb,
270 : bdev_virtio_scsi_ch_destroy_cb,
271 : sizeof(struct bdev_virtio_io_channel),
272 0 : svdev->vdev.name);
273 :
274 0 : pthread_mutex_lock(&g_virtio_scsi_mutex);
275 0 : TAILQ_INSERT_TAIL(&g_virtio_scsi_devs, svdev, tailq);
276 0 : pthread_mutex_unlock(&g_virtio_scsi_mutex);
277 0 : return 0;
278 : }
279 :
280 : static struct virtio_scsi_dev *
281 0 : virtio_pci_scsi_dev_create(const char *name, struct virtio_pci_ctx *pci_ctx)
282 : {
283 : static int pci_dev_counter = 0;
284 : struct virtio_scsi_dev *svdev;
285 : struct virtio_dev *vdev;
286 0 : char *default_name = NULL;
287 0 : uint32_t num_queues;
288 : int rc;
289 :
290 0 : svdev = calloc(1, sizeof(*svdev));
291 0 : if (svdev == NULL) {
292 0 : SPDK_ERRLOG("virtio device calloc failed\n");
293 0 : return NULL;
294 : }
295 :
296 0 : vdev = &svdev->vdev;
297 0 : if (name == NULL) {
298 0 : default_name = spdk_sprintf_alloc("VirtioScsi%"PRIu32, pci_dev_counter++);
299 0 : if (default_name == NULL) {
300 0 : free(vdev);
301 0 : return NULL;
302 : }
303 0 : name = default_name;
304 : }
305 :
306 0 : rc = virtio_pci_dev_init(vdev, name, pci_ctx);
307 0 : free(default_name);
308 :
309 0 : if (rc != 0) {
310 0 : free(svdev);
311 0 : return NULL;
312 : }
313 :
314 0 : rc = virtio_dev_read_dev_config(vdev, offsetof(struct virtio_scsi_config, num_queues),
315 : &num_queues, sizeof(num_queues));
316 0 : if (rc) {
317 0 : SPDK_ERRLOG("%s: config read failed: %s\n", vdev->name, spdk_strerror(-rc));
318 0 : goto fail;
319 : }
320 :
321 0 : rc = virtio_scsi_dev_init(svdev, num_queues, VIRTIO_SCSI_DEV_SUPPORTED_FEATURES);
322 0 : if (rc != 0) {
323 0 : goto fail;
324 : }
325 :
326 0 : return svdev;
327 :
328 0 : fail:
329 0 : vdev->ctx = NULL;
330 0 : virtio_dev_destruct(vdev);
331 0 : free(svdev);
332 0 : return NULL;
333 : }
334 :
335 : static struct virtio_scsi_dev *
336 0 : virtio_user_scsi_dev_create(const char *name, const char *path,
337 : uint16_t num_queues, uint32_t queue_size)
338 : {
339 : struct virtio_scsi_dev *svdev;
340 : struct virtio_dev *vdev;
341 : uint64_t feature_bits;
342 : int rc;
343 :
344 0 : svdev = calloc(1, sizeof(*svdev));
345 0 : if (svdev == NULL) {
346 0 : SPDK_ERRLOG("calloc failed for virtio device %s: %s\n", name, path);
347 0 : return NULL;
348 : }
349 :
350 0 : vdev = &svdev->vdev;
351 0 : rc = virtio_user_dev_init(vdev, name, path, queue_size);
352 0 : if (rc != 0) {
353 0 : SPDK_ERRLOG("Failed to create virito device %s: %s\n", name, path);
354 0 : free(svdev);
355 0 : return NULL;
356 : }
357 :
358 0 : feature_bits = VIRTIO_SCSI_DEV_SUPPORTED_FEATURES;
359 0 : feature_bits |= (1ULL << VHOST_USER_F_PROTOCOL_FEATURES);
360 0 : rc = virtio_scsi_dev_init(svdev, num_queues + SPDK_VIRTIO_SCSI_QUEUE_NUM_FIXED, feature_bits);
361 0 : if (rc != 0) {
362 0 : virtio_dev_destruct(vdev);
363 0 : free(svdev);
364 0 : return NULL;
365 : }
366 :
367 0 : return svdev;
368 : }
369 :
370 : static struct virtio_scsi_disk *
371 0 : virtio_scsi_dev_get_disk_by_id(struct virtio_scsi_dev *svdev, uint8_t target_id)
372 : {
373 : struct virtio_scsi_disk *disk;
374 :
375 0 : TAILQ_FOREACH(disk, &svdev->luns, link) {
376 0 : if (disk->info.target == target_id) {
377 0 : return disk;
378 : }
379 : }
380 :
381 0 : return NULL;
382 : }
383 :
384 : static int virtio_scsi_dev_scan(struct virtio_scsi_dev *svdev,
385 : bdev_virtio_create_cb cb_fn, void *cb_arg);
386 : static int send_scan_io(struct virtio_scsi_scan_base *base);
387 : static void _virtio_scsi_dev_scan_tgt(struct virtio_scsi_scan_base *base, uint8_t target);
388 : static int _virtio_scsi_dev_scan_next(struct virtio_scsi_scan_base *base, int rc);
389 : static void _virtio_scsi_dev_scan_finish(struct virtio_scsi_scan_base *base, int errnum);
390 : static int virtio_scsi_dev_scan_tgt(struct virtio_scsi_dev *svdev, uint8_t target);
391 :
392 : static int
393 0 : bdev_virtio_get_ctx_size(void)
394 : {
395 0 : return sizeof(struct virtio_scsi_io_ctx);
396 : }
397 :
398 : static int
399 0 : bdev_virtio_scsi_config_json(struct spdk_json_write_ctx *w)
400 : {
401 : struct virtio_scsi_dev *svdev;
402 :
403 0 : pthread_mutex_lock(&g_virtio_scsi_mutex);
404 0 : TAILQ_FOREACH(svdev, &g_virtio_scsi_devs, tailq) {
405 0 : spdk_json_write_object_begin(w);
406 :
407 0 : spdk_json_write_named_string(w, "method", "bdev_virtio_attach_controller");
408 :
409 0 : spdk_json_write_named_object_begin(w, "params");
410 0 : spdk_json_write_named_string(w, "name", svdev->vdev.name);
411 0 : spdk_json_write_named_string(w, "dev_type", "scsi");
412 :
413 : /* Write transport specific parameters. */
414 0 : svdev->vdev.backend_ops->write_json_config(&svdev->vdev, w);
415 :
416 0 : spdk_json_write_object_end(w);
417 :
418 0 : spdk_json_write_object_end(w);
419 :
420 : }
421 0 : pthread_mutex_unlock(&g_virtio_scsi_mutex);
422 :
423 0 : return 0;
424 : }
425 :
426 :
427 : static struct spdk_bdev_module virtio_scsi_if = {
428 : .name = "virtio_scsi",
429 : .module_init = bdev_virtio_initialize,
430 : .module_fini = bdev_virtio_finish,
431 : .get_ctx_size = bdev_virtio_get_ctx_size,
432 : .config_json = bdev_virtio_scsi_config_json,
433 : .async_fini = true,
434 : };
435 :
436 0 : SPDK_BDEV_MODULE_REGISTER(virtio_scsi, &virtio_scsi_if)
437 :
438 : static struct virtio_scsi_io_ctx *
439 0 : bdev_virtio_init_io_vreq(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
440 : {
441 : struct virtio_scsi_cmd_req *req;
442 : struct virtio_scsi_cmd_resp *resp;
443 0 : struct virtio_scsi_disk *disk = (struct virtio_scsi_disk *)bdev_io->bdev;
444 0 : struct virtio_scsi_io_ctx *io_ctx = (struct virtio_scsi_io_ctx *)bdev_io->driver_ctx;
445 :
446 0 : req = &io_ctx->req;
447 0 : resp = &io_ctx->resp;
448 :
449 0 : io_ctx->iov_req.iov_base = req;
450 0 : io_ctx->iov_req.iov_len = sizeof(*req);
451 :
452 0 : io_ctx->iov_resp.iov_base = resp;
453 0 : io_ctx->iov_resp.iov_len = sizeof(*resp);
454 :
455 0 : memset(req, 0, sizeof(*req));
456 0 : req->lun[0] = 1;
457 0 : req->lun[1] = disk->info.target;
458 :
459 0 : return io_ctx;
460 : }
461 :
462 : static struct virtio_scsi_io_ctx *
463 0 : bdev_virtio_init_tmf_vreq(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
464 : {
465 : struct virtio_scsi_ctrl_tmf_req *tmf_req;
466 : struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
467 0 : struct virtio_scsi_disk *disk = SPDK_CONTAINEROF(bdev_io->bdev, struct virtio_scsi_disk, bdev);
468 0 : struct virtio_scsi_io_ctx *io_ctx = (struct virtio_scsi_io_ctx *)bdev_io->driver_ctx;
469 :
470 0 : tmf_req = &io_ctx->tmf_req;
471 0 : tmf_resp = &io_ctx->tmf_resp;
472 :
473 0 : io_ctx->iov_req.iov_base = tmf_req;
474 0 : io_ctx->iov_req.iov_len = sizeof(*tmf_req);
475 0 : io_ctx->iov_resp.iov_base = tmf_resp;
476 0 : io_ctx->iov_resp.iov_len = sizeof(*tmf_resp);
477 :
478 0 : memset(tmf_req, 0, sizeof(*tmf_req));
479 0 : tmf_req->lun[0] = 1;
480 0 : tmf_req->lun[1] = disk->info.target;
481 :
482 0 : return io_ctx;
483 : }
484 :
485 : static void
486 0 : bdev_virtio_send_io(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
487 : {
488 0 : struct bdev_virtio_io_channel *virtio_channel = spdk_io_channel_get_ctx(ch);
489 0 : struct virtqueue *vq = virtio_channel->vq;
490 0 : struct virtio_scsi_io_ctx *io_ctx = (struct virtio_scsi_io_ctx *)bdev_io->driver_ctx;
491 : int rc;
492 :
493 0 : rc = virtqueue_req_start(vq, bdev_io, bdev_io->u.bdev.iovcnt + 2);
494 0 : if (rc == -ENOMEM) {
495 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM);
496 0 : return;
497 0 : } else if (rc != 0) {
498 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
499 0 : return;
500 : }
501 :
502 0 : virtqueue_req_add_iovs(vq, &io_ctx->iov_req, 1, SPDK_VIRTIO_DESC_RO);
503 0 : if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ) {
504 0 : virtqueue_req_add_iovs(vq, &io_ctx->iov_resp, 1, SPDK_VIRTIO_DESC_WR);
505 0 : virtqueue_req_add_iovs(vq, bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
506 : SPDK_VIRTIO_DESC_WR);
507 : } else {
508 0 : virtqueue_req_add_iovs(vq, bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
509 : SPDK_VIRTIO_DESC_RO);
510 0 : virtqueue_req_add_iovs(vq, &io_ctx->iov_resp, 1, SPDK_VIRTIO_DESC_WR);
511 : }
512 :
513 0 : virtqueue_req_flush(vq);
514 : }
515 :
516 : static void
517 0 : bdev_virtio_rw(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
518 : {
519 0 : struct virtio_scsi_disk *disk = SPDK_CONTAINEROF(bdev_io->bdev, struct virtio_scsi_disk, bdev);
520 0 : struct virtio_scsi_io_ctx *io_ctx = bdev_virtio_init_io_vreq(ch, bdev_io);
521 0 : struct virtio_scsi_cmd_req *req = &io_ctx->req;
522 0 : bool is_write = bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE;
523 :
524 0 : if (disk->info.num_blocks > (1ULL << 32)) {
525 0 : req->cdb[0] = is_write ? SPDK_SBC_WRITE_16 : SPDK_SBC_READ_16;
526 0 : to_be64(&req->cdb[2], bdev_io->u.bdev.offset_blocks);
527 0 : to_be32(&req->cdb[10], bdev_io->u.bdev.num_blocks);
528 : } else {
529 0 : req->cdb[0] = is_write ? SPDK_SBC_WRITE_10 : SPDK_SBC_READ_10;
530 0 : to_be32(&req->cdb[2], bdev_io->u.bdev.offset_blocks);
531 0 : to_be16(&req->cdb[7], bdev_io->u.bdev.num_blocks);
532 : }
533 :
534 0 : bdev_virtio_send_io(ch, bdev_io);
535 0 : }
536 :
537 : static void
538 0 : bdev_virtio_reset(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
539 : {
540 0 : struct bdev_virtio_io_channel *virtio_ch = spdk_io_channel_get_ctx(ch);
541 0 : struct virtio_scsi_io_ctx *io_ctx = bdev_virtio_init_tmf_vreq(ch, bdev_io);
542 0 : struct virtio_scsi_ctrl_tmf_req *tmf_req = &io_ctx->tmf_req;
543 0 : struct virtio_scsi_dev *svdev = virtio_ch->svdev;
544 : size_t enqueued_count;
545 :
546 0 : tmf_req->type = VIRTIO_SCSI_T_TMF;
547 0 : tmf_req->subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET;
548 :
549 0 : enqueued_count = spdk_ring_enqueue(svdev->ctrlq_ring, (void **)&bdev_io, 1, NULL);
550 0 : if (spdk_likely(enqueued_count == 1)) {
551 0 : return;
552 : } else {
553 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM);
554 : }
555 : }
556 :
557 : static void
558 0 : bdev_virtio_unmap(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bool success)
559 : {
560 0 : struct virtio_scsi_io_ctx *io_ctx = bdev_virtio_init_io_vreq(ch, bdev_io);
561 0 : struct virtio_scsi_cmd_req *req = &io_ctx->req;
562 : struct spdk_scsi_unmap_bdesc *desc, *first_desc;
563 : uint8_t *buf;
564 : uint64_t offset_blocks, num_blocks;
565 : uint16_t cmd_len;
566 :
567 0 : if (!success) {
568 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
569 0 : return;
570 : }
571 :
572 0 : buf = bdev_io->u.bdev.iovs[0].iov_base;
573 :
574 0 : offset_blocks = bdev_io->u.bdev.offset_blocks;
575 0 : num_blocks = bdev_io->u.bdev.num_blocks;
576 :
577 : /* (n-1) * 16-byte descriptors */
578 0 : first_desc = desc = (struct spdk_scsi_unmap_bdesc *)&buf[8];
579 0 : while (num_blocks > UINT32_MAX) {
580 0 : to_be64(&desc->lba, offset_blocks);
581 0 : to_be32(&desc->block_count, UINT32_MAX);
582 0 : memset(&desc->reserved, 0, sizeof(desc->reserved));
583 0 : offset_blocks += UINT32_MAX;
584 0 : num_blocks -= UINT32_MAX;
585 0 : desc++;
586 : }
587 :
588 : /* The last descriptor with block_count <= UINT32_MAX */
589 0 : to_be64(&desc->lba, offset_blocks);
590 0 : to_be32(&desc->block_count, num_blocks);
591 0 : memset(&desc->reserved, 0, sizeof(desc->reserved));
592 :
593 : /* 8-byte header + n * 16-byte block descriptor */
594 0 : cmd_len = 8 + (desc - first_desc + 1) * sizeof(struct spdk_scsi_unmap_bdesc);
595 :
596 0 : req->cdb[0] = SPDK_SBC_UNMAP;
597 0 : to_be16(&req->cdb[7], cmd_len);
598 :
599 : /* 8-byte header */
600 0 : to_be16(&buf[0], cmd_len - 2); /* total length (excluding the length field) */
601 0 : to_be16(&buf[2], cmd_len - 8); /* length of block descriptors */
602 0 : memset(&buf[4], 0, 4); /* reserved */
603 :
604 0 : bdev_virtio_send_io(ch, bdev_io);
605 : }
606 :
607 : static void
608 0 : bdev_virtio_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,
609 : bool success)
610 : {
611 0 : if (!success) {
612 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
613 0 : return;
614 : }
615 :
616 0 : bdev_virtio_rw(ch, bdev_io);
617 : }
618 :
619 : static int
620 0 : _bdev_virtio_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
621 : {
622 0 : struct virtio_scsi_disk *disk = SPDK_CONTAINEROF(bdev_io->bdev, struct virtio_scsi_disk, bdev);
623 :
624 0 : switch (bdev_io->type) {
625 0 : case SPDK_BDEV_IO_TYPE_READ:
626 0 : spdk_bdev_io_get_buf(bdev_io, bdev_virtio_get_buf_cb,
627 0 : bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen);
628 0 : return 0;
629 0 : case SPDK_BDEV_IO_TYPE_WRITE:
630 0 : bdev_virtio_rw(ch, bdev_io);
631 0 : return 0;
632 0 : case SPDK_BDEV_IO_TYPE_RESET:
633 0 : bdev_virtio_reset(ch, bdev_io);
634 0 : return 0;
635 0 : case SPDK_BDEV_IO_TYPE_UNMAP: {
636 0 : uint64_t buf_len = 8 /* header size */ +
637 0 : (bdev_io->u.bdev.num_blocks + UINT32_MAX - 1) /
638 0 : UINT32_MAX * sizeof(struct spdk_scsi_unmap_bdesc);
639 :
640 0 : if (!disk->info.unmap_supported) {
641 0 : return -1;
642 : }
643 :
644 0 : if (buf_len > SPDK_BDEV_LARGE_BUF_MAX_SIZE) {
645 0 : SPDK_ERRLOG("Trying to UNMAP too many blocks: %"PRIu64"\n",
646 : bdev_io->u.bdev.num_blocks);
647 0 : return -1;
648 : }
649 0 : spdk_bdev_io_get_buf(bdev_io, bdev_virtio_unmap, buf_len);
650 0 : return 0;
651 : }
652 0 : case SPDK_BDEV_IO_TYPE_FLUSH:
653 : default:
654 0 : return -1;
655 : }
656 : return 0;
657 : }
658 :
659 : static void
660 0 : bdev_virtio_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
661 : {
662 0 : if (_bdev_virtio_submit_request(ch, bdev_io) < 0) {
663 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
664 : }
665 0 : }
666 :
667 : static bool
668 0 : bdev_virtio_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type)
669 : {
670 0 : struct virtio_scsi_disk *disk = ctx;
671 :
672 0 : switch (io_type) {
673 0 : case SPDK_BDEV_IO_TYPE_READ:
674 : case SPDK_BDEV_IO_TYPE_WRITE:
675 : case SPDK_BDEV_IO_TYPE_FLUSH:
676 : case SPDK_BDEV_IO_TYPE_RESET:
677 0 : return true;
678 :
679 0 : case SPDK_BDEV_IO_TYPE_UNMAP:
680 0 : return disk->info.unmap_supported;
681 :
682 0 : default:
683 0 : return false;
684 : }
685 : }
686 :
687 : static struct spdk_io_channel *
688 0 : bdev_virtio_get_io_channel(void *ctx)
689 : {
690 0 : struct virtio_scsi_disk *disk = ctx;
691 :
692 0 : return spdk_get_io_channel(disk->svdev);
693 : }
694 :
695 : static int
696 0 : bdev_virtio_disk_destruct(void *ctx)
697 : {
698 0 : struct virtio_scsi_disk *disk = ctx;
699 0 : struct virtio_scsi_dev *svdev = disk->svdev;
700 :
701 0 : TAILQ_REMOVE(&svdev->luns, disk, link);
702 0 : free(disk->bdev.name);
703 0 : free(disk);
704 :
705 0 : if (svdev->removed && TAILQ_EMPTY(&svdev->luns)) {
706 0 : spdk_io_device_unregister(svdev, virtio_scsi_dev_unregister_cb);
707 : }
708 :
709 0 : return 0;
710 : }
711 :
712 : static int
713 0 : bdev_virtio_dump_info_json(void *ctx, struct spdk_json_write_ctx *w)
714 : {
715 0 : struct virtio_scsi_disk *disk = ctx;
716 :
717 0 : virtio_dev_dump_json_info(&disk->svdev->vdev, w);
718 0 : return 0;
719 : }
720 :
721 : static void
722 0 : bdev_virtio_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
723 : {
724 : /* SCSI targets and LUNS are discovered during scan process so nothing
725 : * to save here.
726 : */
727 0 : }
728 :
729 : static const struct spdk_bdev_fn_table virtio_fn_table = {
730 : .destruct = bdev_virtio_disk_destruct,
731 : .submit_request = bdev_virtio_submit_request,
732 : .io_type_supported = bdev_virtio_io_type_supported,
733 : .get_io_channel = bdev_virtio_get_io_channel,
734 : .dump_info_json = bdev_virtio_dump_info_json,
735 : .write_config_json = bdev_virtio_write_config_json,
736 : };
737 :
738 : static void
739 0 : get_scsi_status(struct virtio_scsi_cmd_resp *resp, int *sk, int *asc, int *ascq)
740 : {
741 : /* see spdk_scsi_task_build_sense_data() for sense data details */
742 0 : *sk = 0;
743 0 : *asc = 0;
744 0 : *ascq = 0;
745 :
746 0 : if (resp->sense_len < 3) {
747 0 : return;
748 : }
749 :
750 0 : *sk = resp->sense[2] & 0xf;
751 :
752 0 : if (resp->sense_len < 13) {
753 0 : return;
754 : }
755 :
756 0 : *asc = resp->sense[12];
757 :
758 0 : if (resp->sense_len < 14) {
759 0 : return;
760 : }
761 :
762 0 : *ascq = resp->sense[13];
763 : }
764 :
765 : static void
766 0 : bdev_virtio_io_cpl(struct spdk_bdev_io *bdev_io)
767 : {
768 0 : struct virtio_scsi_io_ctx *io_ctx = (struct virtio_scsi_io_ctx *)bdev_io->driver_ctx;
769 0 : int sk, asc, ascq;
770 :
771 0 : get_scsi_status(&io_ctx->resp, &sk, &asc, &ascq);
772 0 : spdk_bdev_io_complete_scsi_status(bdev_io, io_ctx->resp.status, sk, asc, ascq);
773 0 : }
774 :
775 : static int
776 0 : bdev_virtio_poll(void *arg)
777 : {
778 0 : struct bdev_virtio_io_channel *ch = arg;
779 0 : struct virtio_scsi_dev *svdev = ch->svdev;
780 0 : struct virtio_scsi_scan_base *scan_ctx = svdev->scan_ctx;
781 0 : void *io[32];
782 0 : uint32_t io_len[32];
783 : uint16_t i, cnt;
784 : int rc;
785 :
786 0 : cnt = virtio_recv_pkts(ch->vq, (void **)io, io_len, SPDK_COUNTOF(io));
787 0 : for (i = 0; i < cnt; ++i) {
788 0 : if (spdk_unlikely(scan_ctx && io[i] == &scan_ctx->io_ctx)) {
789 0 : if (svdev->removed) {
790 0 : _virtio_scsi_dev_scan_finish(scan_ctx, -EINTR);
791 0 : return SPDK_POLLER_BUSY;
792 : }
793 :
794 0 : if (scan_ctx->restart) {
795 0 : scan_ctx->restart = false;
796 0 : scan_ctx->full_scan = true;
797 0 : _virtio_scsi_dev_scan_tgt(scan_ctx, 0);
798 0 : continue;
799 : }
800 :
801 0 : process_scan_resp(scan_ctx);
802 0 : continue;
803 : }
804 :
805 0 : bdev_virtio_io_cpl(io[i]);
806 : }
807 :
808 : /* scan_ctx could have been freed while processing completions above, so
809 : * we need to re-read the value again here into the local variable before
810 : * using it.
811 : */
812 0 : scan_ctx = svdev->scan_ctx;
813 0 : if (spdk_unlikely(scan_ctx && scan_ctx->needs_resend)) {
814 0 : if (svdev->removed) {
815 0 : _virtio_scsi_dev_scan_finish(scan_ctx, -EINTR);
816 0 : return SPDK_POLLER_BUSY;
817 0 : } else if (cnt == 0) {
818 0 : return SPDK_POLLER_IDLE;
819 : }
820 :
821 0 : rc = send_scan_io(scan_ctx);
822 0 : if (rc != 0) {
823 0 : assert(scan_ctx->retries > 0);
824 0 : scan_ctx->retries--;
825 0 : if (scan_ctx->retries == 0) {
826 0 : SPDK_ERRLOG("Target scan failed unrecoverably with rc = %d.\n", rc);
827 0 : _virtio_scsi_dev_scan_finish(scan_ctx, rc);
828 : }
829 : }
830 : }
831 :
832 0 : return cnt;
833 : }
834 :
835 : static void
836 0 : bdev_virtio_tmf_cpl_cb(void *ctx)
837 : {
838 0 : struct spdk_bdev_io *bdev_io = ctx;
839 0 : struct virtio_scsi_io_ctx *io_ctx = (struct virtio_scsi_io_ctx *)bdev_io->driver_ctx;
840 :
841 0 : if (io_ctx->tmf_resp.response == VIRTIO_SCSI_S_OK) {
842 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS);
843 : } else {
844 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
845 : }
846 0 : }
847 :
848 : static void
849 0 : bdev_virtio_tmf_cpl(struct spdk_bdev_io *bdev_io)
850 : {
851 0 : spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), bdev_virtio_tmf_cpl_cb, bdev_io);
852 0 : }
853 :
854 : static void
855 0 : bdev_virtio_eventq_io_cpl(struct virtio_scsi_dev *svdev, struct virtio_scsi_eventq_io *io)
856 : {
857 0 : struct virtio_scsi_event *ev = &io->ev;
858 : struct virtio_scsi_disk *disk;
859 :
860 0 : if (ev->lun[0] != 1) {
861 0 : SPDK_WARNLOG("Received an event with invalid data layout.\n");
862 0 : goto out;
863 : }
864 :
865 0 : if (ev->event & VIRTIO_SCSI_T_EVENTS_MISSED) {
866 0 : ev->event &= ~VIRTIO_SCSI_T_EVENTS_MISSED;
867 0 : virtio_scsi_dev_scan(svdev, NULL, NULL);
868 : }
869 :
870 0 : switch (ev->event) {
871 0 : case VIRTIO_SCSI_T_NO_EVENT:
872 0 : break;
873 0 : case VIRTIO_SCSI_T_TRANSPORT_RESET:
874 0 : switch (ev->reason) {
875 0 : case VIRTIO_SCSI_EVT_RESET_RESCAN:
876 0 : virtio_scsi_dev_scan_tgt(svdev, ev->lun[1]);
877 0 : break;
878 0 : case VIRTIO_SCSI_EVT_RESET_REMOVED:
879 0 : disk = virtio_scsi_dev_get_disk_by_id(svdev, ev->lun[1]);
880 0 : if (disk != NULL) {
881 0 : spdk_bdev_unregister(&disk->bdev, NULL, NULL);
882 : }
883 0 : break;
884 0 : default:
885 0 : break;
886 : }
887 0 : break;
888 0 : default:
889 0 : break;
890 : }
891 :
892 0 : out:
893 0 : virtio_scsi_dev_send_eventq_io(svdev->vdev.vqs[VIRTIO_SCSI_EVENTQ], io);
894 0 : }
895 :
896 : static void
897 0 : bdev_virtio_tmf_abort_nomem_cb(void *ctx)
898 : {
899 0 : struct spdk_bdev_io *bdev_io = ctx;
900 :
901 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM);
902 0 : }
903 :
904 : static void
905 0 : bdev_virtio_tmf_abort_ioerr_cb(void *ctx)
906 : {
907 0 : struct spdk_bdev_io *bdev_io = ctx;
908 :
909 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
910 0 : }
911 :
912 : static void
913 0 : bdev_virtio_tmf_abort(struct spdk_bdev_io *bdev_io, int status)
914 : {
915 : spdk_msg_fn fn;
916 :
917 0 : if (status == -ENOMEM) {
918 0 : fn = bdev_virtio_tmf_abort_nomem_cb;
919 : } else {
920 0 : fn = bdev_virtio_tmf_abort_ioerr_cb;
921 : }
922 :
923 0 : spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), fn, bdev_io);
924 0 : }
925 :
926 : static int
927 0 : bdev_virtio_send_tmf_io(struct virtqueue *ctrlq, struct spdk_bdev_io *bdev_io)
928 : {
929 0 : struct virtio_scsi_io_ctx *io_ctx = (struct virtio_scsi_io_ctx *)bdev_io->driver_ctx;
930 : int rc;
931 :
932 0 : rc = virtqueue_req_start(ctrlq, bdev_io, 2);
933 0 : if (rc != 0) {
934 0 : return rc;
935 : }
936 :
937 0 : virtqueue_req_add_iovs(ctrlq, &io_ctx->iov_req, 1, SPDK_VIRTIO_DESC_RO);
938 0 : virtqueue_req_add_iovs(ctrlq, &io_ctx->iov_resp, 1, SPDK_VIRTIO_DESC_WR);
939 :
940 0 : virtqueue_req_flush(ctrlq);
941 0 : return 0;
942 : }
943 :
944 : static int
945 0 : bdev_virtio_mgmt_poll(void *arg)
946 : {
947 0 : struct virtio_scsi_dev *svdev = arg;
948 0 : struct virtio_dev *vdev = &svdev->vdev;
949 0 : struct virtqueue *eventq = vdev->vqs[VIRTIO_SCSI_EVENTQ];
950 0 : struct virtqueue *ctrlq = vdev->vqs[VIRTIO_SCSI_CONTROLQ];
951 0 : struct spdk_ring *send_ring = svdev->ctrlq_ring;
952 0 : void *io[16];
953 0 : uint32_t io_len[16];
954 : uint16_t i, cnt;
955 : int rc;
956 0 : int total = 0;
957 :
958 0 : cnt = spdk_ring_dequeue(send_ring, io, SPDK_COUNTOF(io));
959 0 : total += cnt;
960 0 : for (i = 0; i < cnt; ++i) {
961 0 : rc = bdev_virtio_send_tmf_io(ctrlq, io[i]);
962 0 : if (rc != 0) {
963 0 : bdev_virtio_tmf_abort(io[i], rc);
964 : }
965 : }
966 :
967 0 : cnt = virtio_recv_pkts(ctrlq, io, io_len, SPDK_COUNTOF(io));
968 0 : total += cnt;
969 0 : for (i = 0; i < cnt; ++i) {
970 0 : bdev_virtio_tmf_cpl(io[i]);
971 : }
972 :
973 0 : cnt = virtio_recv_pkts(eventq, io, io_len, SPDK_COUNTOF(io));
974 0 : total += cnt;
975 0 : for (i = 0; i < cnt; ++i) {
976 0 : bdev_virtio_eventq_io_cpl(svdev, io[i]);
977 : }
978 :
979 0 : return total;
980 : }
981 :
982 : static int
983 0 : bdev_virtio_scsi_ch_create_cb(void *io_device, void *ctx_buf)
984 : {
985 0 : struct virtio_scsi_dev *svdev = io_device;
986 0 : struct virtio_dev *vdev = &svdev->vdev;
987 0 : struct bdev_virtio_io_channel *ch = ctx_buf;
988 : struct virtqueue *vq;
989 : int32_t queue_idx;
990 :
991 0 : queue_idx = virtio_dev_find_and_acquire_queue(vdev, VIRTIO_SCSI_REQUESTQ);
992 0 : if (queue_idx < 0) {
993 0 : SPDK_ERRLOG("Couldn't get an unused queue for the io_channel.\n");
994 0 : return -1;
995 : }
996 :
997 0 : vq = vdev->vqs[queue_idx];
998 :
999 0 : ch->svdev = svdev;
1000 0 : ch->vq = vq;
1001 :
1002 0 : ch->poller = SPDK_POLLER_REGISTER(bdev_virtio_poll, ch, 0);
1003 :
1004 0 : return 0;
1005 : }
1006 :
1007 : static void
1008 0 : bdev_virtio_scsi_ch_destroy_cb(void *io_device, void *ctx_buf)
1009 : {
1010 0 : struct bdev_virtio_io_channel *ch = ctx_buf;
1011 0 : struct virtio_scsi_dev *svdev = ch->svdev;
1012 0 : struct virtio_dev *vdev = &svdev->vdev;
1013 0 : struct virtqueue *vq = ch->vq;
1014 :
1015 0 : spdk_poller_unregister(&ch->poller);
1016 0 : virtio_dev_release_queue(vdev, vq->vq_queue_index);
1017 0 : }
1018 :
1019 : static void
1020 0 : _virtio_scsi_dev_scan_finish(struct virtio_scsi_scan_base *base, int errnum)
1021 : {
1022 0 : struct virtio_scsi_dev *svdev = base->svdev;
1023 : size_t bdevs_cnt;
1024 0 : struct spdk_bdev *bdevs[BDEV_VIRTIO_MAX_TARGET];
1025 : struct virtio_scsi_disk *disk;
1026 : struct virtio_scsi_scan_info *tgt, *next_tgt;
1027 :
1028 0 : spdk_put_io_channel(spdk_io_channel_from_ctx(base->channel));
1029 0 : base->svdev->scan_ctx = NULL;
1030 :
1031 0 : TAILQ_FOREACH_SAFE(tgt, &base->scan_queue, tailq, next_tgt) {
1032 0 : TAILQ_REMOVE(&base->scan_queue, tgt, tailq);
1033 0 : free(tgt);
1034 : }
1035 :
1036 0 : if (base->cb_fn == NULL) {
1037 0 : spdk_free(base);
1038 0 : return;
1039 : }
1040 :
1041 0 : bdevs_cnt = 0;
1042 0 : if (errnum == 0) {
1043 0 : TAILQ_FOREACH(disk, &svdev->luns, link) {
1044 0 : bdevs[bdevs_cnt] = &disk->bdev;
1045 0 : bdevs_cnt++;
1046 : }
1047 : }
1048 :
1049 0 : base->cb_fn(base->cb_arg, errnum, bdevs, bdevs_cnt);
1050 0 : spdk_free(base);
1051 : }
1052 :
1053 : static int
1054 0 : send_scan_io(struct virtio_scsi_scan_base *base)
1055 : {
1056 0 : struct virtio_scsi_io_ctx *io_ctx = &base->io_ctx;
1057 0 : struct virtio_scsi_cmd_req *req = &base->io_ctx.req;
1058 0 : struct virtqueue *vq = base->channel->vq;
1059 0 : int payload_iov_cnt = base->iov.iov_len > 0 ? 1 : 0;
1060 : int rc;
1061 :
1062 0 : req->lun[0] = 1;
1063 0 : req->lun[1] = base->info.target;
1064 :
1065 0 : rc = virtqueue_req_start(vq, io_ctx, 2 + payload_iov_cnt);
1066 0 : if (rc != 0) {
1067 0 : base->needs_resend = true;
1068 0 : return -1;
1069 : }
1070 :
1071 0 : virtqueue_req_add_iovs(vq, &io_ctx->iov_req, 1, SPDK_VIRTIO_DESC_RO);
1072 0 : virtqueue_req_add_iovs(vq, &io_ctx->iov_resp, 1, SPDK_VIRTIO_DESC_WR);
1073 0 : virtqueue_req_add_iovs(vq, &base->iov, payload_iov_cnt, SPDK_VIRTIO_DESC_WR);
1074 :
1075 0 : virtqueue_req_flush(vq);
1076 0 : return 0;
1077 : }
1078 :
1079 : static int
1080 0 : send_inquiry(struct virtio_scsi_scan_base *base)
1081 : {
1082 0 : struct virtio_scsi_cmd_req *req = &base->io_ctx.req;
1083 : struct spdk_scsi_cdb_inquiry *cdb;
1084 :
1085 0 : memset(req, 0, sizeof(*req));
1086 :
1087 0 : base->iov.iov_len = BDEV_VIRTIO_SCAN_PAYLOAD_SIZE;
1088 0 : cdb = (struct spdk_scsi_cdb_inquiry *)req->cdb;
1089 0 : cdb->opcode = SPDK_SPC_INQUIRY;
1090 0 : to_be16(cdb->alloc_len, BDEV_VIRTIO_SCAN_PAYLOAD_SIZE);
1091 :
1092 0 : return send_scan_io(base);
1093 : }
1094 :
1095 : static int
1096 0 : send_inquiry_vpd(struct virtio_scsi_scan_base *base, uint8_t page_code)
1097 : {
1098 0 : struct virtio_scsi_cmd_req *req = &base->io_ctx.req;
1099 0 : struct spdk_scsi_cdb_inquiry *inquiry_cdb = (struct spdk_scsi_cdb_inquiry *)req->cdb;
1100 :
1101 0 : memset(req, 0, sizeof(*req));
1102 :
1103 0 : base->iov.iov_len = BDEV_VIRTIO_SCAN_PAYLOAD_SIZE;
1104 0 : inquiry_cdb->opcode = SPDK_SPC_INQUIRY;
1105 0 : inquiry_cdb->evpd = 1;
1106 0 : inquiry_cdb->page_code = page_code;
1107 0 : to_be16(inquiry_cdb->alloc_len, base->iov.iov_len);
1108 :
1109 0 : return send_scan_io(base);
1110 : }
1111 :
1112 : static int
1113 0 : send_read_cap_10(struct virtio_scsi_scan_base *base)
1114 : {
1115 0 : struct virtio_scsi_cmd_req *req = &base->io_ctx.req;
1116 :
1117 0 : memset(req, 0, sizeof(*req));
1118 :
1119 0 : base->iov.iov_len = 8;
1120 0 : req->cdb[0] = SPDK_SBC_READ_CAPACITY_10;
1121 :
1122 0 : return send_scan_io(base);
1123 : }
1124 :
1125 : static int
1126 0 : send_read_cap_16(struct virtio_scsi_scan_base *base)
1127 : {
1128 0 : struct virtio_scsi_cmd_req *req = &base->io_ctx.req;
1129 :
1130 0 : memset(req, 0, sizeof(*req));
1131 :
1132 0 : base->iov.iov_len = 32;
1133 0 : req->cdb[0] = SPDK_SPC_SERVICE_ACTION_IN_16;
1134 0 : req->cdb[1] = SPDK_SBC_SAI_READ_CAPACITY_16;
1135 0 : to_be32(&req->cdb[10], base->iov.iov_len);
1136 :
1137 0 : return send_scan_io(base);
1138 : }
1139 :
1140 : static int
1141 0 : send_test_unit_ready(struct virtio_scsi_scan_base *base)
1142 : {
1143 0 : struct virtio_scsi_cmd_req *req = &base->io_ctx.req;
1144 :
1145 0 : memset(req, 0, sizeof(*req));
1146 0 : req->cdb[0] = SPDK_SPC_TEST_UNIT_READY;
1147 0 : base->iov.iov_len = 0;
1148 :
1149 0 : return send_scan_io(base);
1150 : }
1151 :
1152 : static int
1153 0 : send_start_stop_unit(struct virtio_scsi_scan_base *base)
1154 : {
1155 0 : struct virtio_scsi_cmd_req *req = &base->io_ctx.req;
1156 :
1157 0 : memset(req, 0, sizeof(*req));
1158 0 : req->cdb[0] = SPDK_SBC_START_STOP_UNIT;
1159 0 : req->cdb[4] = SPDK_SBC_START_STOP_UNIT_START_BIT;
1160 0 : base->iov.iov_len = 0;
1161 :
1162 0 : return send_scan_io(base);
1163 : }
1164 :
1165 : static int
1166 0 : process_scan_start_stop_unit(struct virtio_scsi_scan_base *base)
1167 : {
1168 0 : struct virtio_scsi_cmd_resp *resp = &base->io_ctx.resp;
1169 :
1170 0 : if (resp->status == SPDK_SCSI_STATUS_GOOD) {
1171 0 : return send_inquiry_vpd(base, SPDK_SPC_VPD_SUPPORTED_VPD_PAGES);
1172 : }
1173 :
1174 0 : return -1;
1175 : }
1176 :
1177 : static int
1178 0 : process_scan_test_unit_ready(struct virtio_scsi_scan_base *base)
1179 : {
1180 0 : struct virtio_scsi_cmd_resp *resp = &base->io_ctx.resp;
1181 0 : int sk, asc, ascq;
1182 :
1183 0 : get_scsi_status(resp, &sk, &asc, &ascq);
1184 :
1185 : /* check response, get VPD if spun up otherwise send SSU */
1186 0 : if (resp->status == SPDK_SCSI_STATUS_GOOD) {
1187 0 : return send_inquiry_vpd(base, SPDK_SPC_VPD_SUPPORTED_VPD_PAGES);
1188 0 : } else if (resp->response == VIRTIO_SCSI_S_OK &&
1189 0 : resp->status == SPDK_SCSI_STATUS_CHECK_CONDITION &&
1190 0 : sk == SPDK_SCSI_SENSE_UNIT_ATTENTION &&
1191 0 : asc == SPDK_SCSI_ASC_LOGICAL_UNIT_NOT_READY) {
1192 0 : return send_start_stop_unit(base);
1193 : } else {
1194 0 : return -1;
1195 : }
1196 : }
1197 :
1198 : static int
1199 0 : process_scan_inquiry_standard(struct virtio_scsi_scan_base *base)
1200 : {
1201 0 : struct virtio_scsi_cmd_resp *resp = &base->io_ctx.resp;
1202 0 : struct spdk_scsi_cdb_inquiry_data *inquiry_data =
1203 : (struct spdk_scsi_cdb_inquiry_data *)base->payload;
1204 :
1205 0 : if (resp->status != SPDK_SCSI_STATUS_GOOD) {
1206 0 : return -1;
1207 : }
1208 :
1209 : /* check to make sure its a supported device */
1210 0 : if (inquiry_data->peripheral_device_type != SPDK_SPC_PERIPHERAL_DEVICE_TYPE_DISK ||
1211 0 : inquiry_data->peripheral_qualifier != SPDK_SPC_PERIPHERAL_QUALIFIER_CONNECTED) {
1212 0 : SPDK_WARNLOG("Unsupported peripheral device type 0x%02x (qualifier 0x%02x)\n",
1213 : inquiry_data->peripheral_device_type,
1214 : inquiry_data->peripheral_qualifier);
1215 0 : return -1;
1216 : }
1217 :
1218 0 : return send_test_unit_ready(base);
1219 : }
1220 :
1221 : static int
1222 0 : process_scan_inquiry_vpd_supported_vpd_pages(struct virtio_scsi_scan_base *base)
1223 : {
1224 0 : struct virtio_scsi_cmd_resp *resp = &base->io_ctx.resp;
1225 0 : bool block_provisioning_page_supported = false;
1226 :
1227 0 : if (resp->status == SPDK_SCSI_STATUS_GOOD) {
1228 0 : const uint8_t *vpd_data = base->payload;
1229 0 : const uint8_t *supported_vpd_pages = vpd_data + 4;
1230 : uint16_t page_length;
1231 : uint16_t num_supported_pages;
1232 : uint16_t i;
1233 :
1234 0 : page_length = from_be16(vpd_data + 2);
1235 0 : num_supported_pages = spdk_min(page_length, base->iov.iov_len - 4);
1236 :
1237 0 : for (i = 0; i < num_supported_pages; i++) {
1238 0 : if (supported_vpd_pages[i] == SPDK_SPC_VPD_BLOCK_THIN_PROVISION) {
1239 0 : block_provisioning_page_supported = true;
1240 0 : break;
1241 : }
1242 : }
1243 : }
1244 :
1245 0 : if (block_provisioning_page_supported) {
1246 0 : return send_inquiry_vpd(base, SPDK_SPC_VPD_BLOCK_THIN_PROVISION);
1247 : } else {
1248 0 : return send_read_cap_10(base);
1249 : }
1250 : }
1251 :
1252 : static int
1253 0 : process_scan_inquiry_vpd_block_thin_provision(struct virtio_scsi_scan_base *base)
1254 : {
1255 0 : struct virtio_scsi_cmd_resp *resp = &base->io_ctx.resp;
1256 :
1257 0 : base->info.unmap_supported = false;
1258 :
1259 0 : if (resp->status == SPDK_SCSI_STATUS_GOOD) {
1260 0 : uint8_t *vpd_data = base->payload;
1261 :
1262 0 : base->info.unmap_supported = !!(vpd_data[5] & SPDK_SCSI_UNMAP_LBPU);
1263 : }
1264 :
1265 0 : SPDK_INFOLOG(virtio, "Target %u: unmap supported = %d\n",
1266 : base->info.target, (int)base->info.unmap_supported);
1267 :
1268 0 : return send_read_cap_10(base);
1269 : }
1270 :
1271 : static int
1272 0 : process_scan_inquiry(struct virtio_scsi_scan_base *base)
1273 : {
1274 0 : struct virtio_scsi_cmd_req *req = &base->io_ctx.req;
1275 0 : struct spdk_scsi_cdb_inquiry *inquiry_cdb = (struct spdk_scsi_cdb_inquiry *)req->cdb;
1276 :
1277 0 : if ((inquiry_cdb->evpd & 1) == 0) {
1278 0 : return process_scan_inquiry_standard(base);
1279 : }
1280 :
1281 0 : switch (inquiry_cdb->page_code) {
1282 0 : case SPDK_SPC_VPD_SUPPORTED_VPD_PAGES:
1283 0 : return process_scan_inquiry_vpd_supported_vpd_pages(base);
1284 0 : case SPDK_SPC_VPD_BLOCK_THIN_PROVISION:
1285 0 : return process_scan_inquiry_vpd_block_thin_provision(base);
1286 0 : default:
1287 0 : SPDK_DEBUGLOG(virtio, "Unexpected VPD page 0x%02x\n", inquiry_cdb->page_code);
1288 0 : return -1;
1289 : }
1290 : }
1291 :
1292 : static void
1293 0 : bdev_virtio_disk_notify_remove(struct virtio_scsi_disk *disk)
1294 : {
1295 0 : disk->removed = true;
1296 0 : spdk_bdev_close(disk->notify_desc);
1297 0 : }
1298 :
1299 : static void
1300 0 : bdev_virtio_disk_notify_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev,
1301 : void *event_ctx)
1302 : {
1303 0 : switch (type) {
1304 0 : case SPDK_BDEV_EVENT_REMOVE:
1305 0 : bdev_virtio_disk_notify_remove(event_ctx);
1306 0 : break;
1307 0 : default:
1308 0 : SPDK_NOTICELOG("Unsupported bdev event: type %d\n", type);
1309 0 : break;
1310 : }
1311 0 : }
1312 :
1313 : /* To be called only from the thread performing target scan */
1314 : static int
1315 0 : virtio_scsi_dev_add_tgt(struct virtio_scsi_dev *svdev, struct virtio_scsi_scan_info *info)
1316 : {
1317 : struct virtio_scsi_disk *disk;
1318 : struct spdk_bdev *bdev;
1319 : int rc;
1320 :
1321 0 : TAILQ_FOREACH(disk, &svdev->luns, link) {
1322 0 : if (disk->info.target == info->target) {
1323 : /* Target is already attached and param change is not supported */
1324 0 : return 0;
1325 : }
1326 : }
1327 :
1328 0 : if (info->block_size == 0 || info->num_blocks == 0) {
1329 0 : SPDK_ERRLOG("%s: invalid target %u: bs=%"PRIu32" blocks=%"PRIu64"\n",
1330 : svdev->vdev.name, info->target, info->block_size, info->num_blocks);
1331 0 : return -EINVAL;
1332 : }
1333 :
1334 0 : disk = calloc(1, sizeof(*disk));
1335 0 : if (disk == NULL) {
1336 0 : SPDK_ERRLOG("could not allocate disk\n");
1337 0 : return -ENOMEM;
1338 : }
1339 :
1340 0 : disk->svdev = svdev;
1341 0 : memcpy(&disk->info, info, sizeof(*info));
1342 :
1343 0 : bdev = &disk->bdev;
1344 0 : bdev->name = spdk_sprintf_alloc("%st%"PRIu8, svdev->vdev.name, info->target);
1345 0 : if (bdev->name == NULL) {
1346 0 : SPDK_ERRLOG("Couldn't alloc memory for the bdev name.\n");
1347 0 : free(disk);
1348 0 : return -ENOMEM;
1349 : }
1350 :
1351 0 : bdev->product_name = "Virtio SCSI Disk";
1352 0 : bdev->write_cache = 0;
1353 0 : bdev->blocklen = disk->info.block_size;
1354 0 : bdev->blockcnt = disk->info.num_blocks;
1355 :
1356 0 : bdev->ctxt = disk;
1357 0 : bdev->fn_table = &virtio_fn_table;
1358 0 : bdev->module = &virtio_scsi_if;
1359 :
1360 0 : rc = spdk_bdev_register(&disk->bdev);
1361 0 : if (rc) {
1362 0 : SPDK_ERRLOG("Failed to register bdev name=%s\n", disk->bdev.name);
1363 0 : free(bdev->name);
1364 0 : free(disk);
1365 0 : return rc;
1366 : }
1367 :
1368 0 : rc = spdk_bdev_open_ext(bdev->name, false, bdev_virtio_disk_notify_event_cb,
1369 : disk, &disk->notify_desc);
1370 0 : if (rc) {
1371 0 : assert(false);
1372 : }
1373 :
1374 0 : TAILQ_INSERT_TAIL(&svdev->luns, disk, link);
1375 0 : return 0;
1376 : }
1377 :
1378 : static int
1379 0 : process_read_cap_10(struct virtio_scsi_scan_base *base)
1380 : {
1381 0 : struct virtio_scsi_cmd_req *req = &base->io_ctx.req;
1382 0 : struct virtio_scsi_cmd_resp *resp = &base->io_ctx.resp;
1383 : uint64_t max_block;
1384 : uint32_t block_size;
1385 0 : uint8_t target_id = req->lun[1];
1386 : int rc;
1387 :
1388 0 : if (resp->response != VIRTIO_SCSI_S_OK || resp->status != SPDK_SCSI_STATUS_GOOD) {
1389 0 : SPDK_ERRLOG("READ CAPACITY (10) failed for target %"PRIu8".\n", target_id);
1390 0 : return -1;
1391 : }
1392 :
1393 0 : block_size = from_be32(base->payload + 4);
1394 0 : max_block = from_be32(base->payload);
1395 :
1396 0 : if (max_block == 0xffffffff) {
1397 0 : return send_read_cap_16(base);
1398 : }
1399 :
1400 0 : base->info.num_blocks = (uint64_t)max_block + 1;
1401 0 : base->info.block_size = block_size;
1402 :
1403 0 : rc = virtio_scsi_dev_add_tgt(base->svdev, &base->info);
1404 0 : if (rc != 0) {
1405 0 : return rc;
1406 : }
1407 :
1408 0 : return _virtio_scsi_dev_scan_next(base, 0);
1409 : }
1410 :
1411 : static int
1412 0 : process_read_cap_16(struct virtio_scsi_scan_base *base)
1413 : {
1414 0 : struct virtio_scsi_cmd_req *req = &base->io_ctx.req;
1415 0 : struct virtio_scsi_cmd_resp *resp = &base->io_ctx.resp;
1416 0 : uint8_t target_id = req->lun[1];
1417 : int rc;
1418 :
1419 0 : if (resp->response != VIRTIO_SCSI_S_OK || resp->status != SPDK_SCSI_STATUS_GOOD) {
1420 0 : SPDK_ERRLOG("READ CAPACITY (16) failed for target %"PRIu8".\n", target_id);
1421 0 : return -1;
1422 : }
1423 :
1424 0 : base->info.num_blocks = from_be64(base->payload) + 1;
1425 0 : base->info.block_size = from_be32(base->payload + 8);
1426 0 : rc = virtio_scsi_dev_add_tgt(base->svdev, &base->info);
1427 0 : if (rc != 0) {
1428 0 : return rc;
1429 : }
1430 :
1431 0 : return _virtio_scsi_dev_scan_next(base, 0);
1432 : }
1433 :
1434 : static void
1435 0 : process_scan_resp(struct virtio_scsi_scan_base *base)
1436 : {
1437 0 : struct virtio_scsi_cmd_req *req = &base->io_ctx.req;
1438 0 : struct virtio_scsi_cmd_resp *resp = &base->io_ctx.resp;
1439 0 : int rc, sk, asc, ascq;
1440 : uint8_t target_id;
1441 :
1442 0 : if (base->io_ctx.iov_req.iov_len < sizeof(struct virtio_scsi_cmd_req) ||
1443 0 : base->io_ctx.iov_resp.iov_len < sizeof(struct virtio_scsi_cmd_resp)) {
1444 0 : SPDK_ERRLOG("Received target scan message with invalid length.\n");
1445 0 : _virtio_scsi_dev_scan_next(base, -EIO);
1446 0 : return;
1447 : }
1448 :
1449 0 : get_scsi_status(resp, &sk, &asc, &ascq);
1450 0 : target_id = req->lun[1];
1451 :
1452 0 : if (resp->response == VIRTIO_SCSI_S_BAD_TARGET ||
1453 0 : resp->response == VIRTIO_SCSI_S_INCORRECT_LUN) {
1454 0 : _virtio_scsi_dev_scan_next(base, -ENODEV);
1455 0 : return;
1456 : }
1457 :
1458 0 : if (resp->response != VIRTIO_SCSI_S_OK ||
1459 0 : (resp->status == SPDK_SCSI_STATUS_CHECK_CONDITION &&
1460 0 : sk != SPDK_SCSI_SENSE_ILLEGAL_REQUEST)) {
1461 0 : assert(base->retries > 0);
1462 0 : base->retries--;
1463 0 : if (base->retries == 0) {
1464 0 : SPDK_NOTICELOG("Target %"PRIu8" is present, but unavailable.\n", target_id);
1465 0 : SPDK_LOGDUMP(virtio, "CDB", req->cdb, sizeof(req->cdb));
1466 0 : SPDK_LOGDUMP(virtio, "SENSE DATA", resp->sense, sizeof(resp->sense));
1467 0 : _virtio_scsi_dev_scan_next(base, -EBUSY);
1468 0 : return;
1469 : }
1470 :
1471 : /* resend the same request */
1472 0 : rc = send_scan_io(base);
1473 : if (rc != 0) {
1474 : /* Let response poller do the resend */
1475 : }
1476 0 : return;
1477 : }
1478 :
1479 0 : base->retries = SCAN_REQUEST_RETRIES;
1480 :
1481 0 : switch (req->cdb[0]) {
1482 0 : case SPDK_SPC_INQUIRY:
1483 0 : rc = process_scan_inquiry(base);
1484 0 : break;
1485 0 : case SPDK_SPC_TEST_UNIT_READY:
1486 0 : rc = process_scan_test_unit_ready(base);
1487 0 : break;
1488 0 : case SPDK_SBC_START_STOP_UNIT:
1489 0 : rc = process_scan_start_stop_unit(base);
1490 0 : break;
1491 0 : case SPDK_SBC_READ_CAPACITY_10:
1492 0 : rc = process_read_cap_10(base);
1493 0 : break;
1494 0 : case SPDK_SPC_SERVICE_ACTION_IN_16:
1495 0 : rc = process_read_cap_16(base);
1496 0 : break;
1497 0 : default:
1498 0 : SPDK_ERRLOG("Received invalid target scan message: cdb[0] = %"PRIu8".\n", req->cdb[0]);
1499 0 : rc = -1;
1500 0 : break;
1501 : }
1502 :
1503 0 : if (rc != 0) {
1504 0 : if (base->needs_resend) {
1505 0 : return; /* Let response poller do the resend */
1506 : }
1507 :
1508 0 : _virtio_scsi_dev_scan_next(base, rc);
1509 : }
1510 : }
1511 :
1512 : static int
1513 0 : _virtio_scsi_dev_scan_next(struct virtio_scsi_scan_base *base, int rc)
1514 : {
1515 : struct virtio_scsi_scan_info *next;
1516 : struct virtio_scsi_disk *disk;
1517 : uint8_t target_id;
1518 :
1519 0 : if (base->full_scan) {
1520 0 : if (rc != 0) {
1521 0 : disk = virtio_scsi_dev_get_disk_by_id(base->svdev,
1522 0 : base->info.target);
1523 0 : if (disk != NULL) {
1524 0 : spdk_bdev_unregister(&disk->bdev, NULL, NULL);
1525 : }
1526 : }
1527 :
1528 0 : target_id = base->info.target + 1;
1529 0 : if (target_id < BDEV_VIRTIO_MAX_TARGET) {
1530 0 : _virtio_scsi_dev_scan_tgt(base, target_id);
1531 0 : return 0;
1532 : }
1533 :
1534 0 : base->full_scan = false;
1535 : }
1536 :
1537 0 : next = TAILQ_FIRST(&base->scan_queue);
1538 0 : if (next == NULL) {
1539 0 : _virtio_scsi_dev_scan_finish(base, 0);
1540 0 : return 0;
1541 : }
1542 :
1543 0 : TAILQ_REMOVE(&base->scan_queue, next, tailq);
1544 0 : target_id = next->target;
1545 0 : free(next);
1546 :
1547 0 : _virtio_scsi_dev_scan_tgt(base, target_id);
1548 0 : return 0;
1549 : }
1550 :
1551 : static int
1552 0 : _virtio_scsi_dev_scan_init(struct virtio_scsi_dev *svdev)
1553 : {
1554 : struct virtio_scsi_scan_base *base;
1555 : struct spdk_io_channel *io_ch;
1556 : struct virtio_scsi_io_ctx *io_ctx;
1557 : struct virtio_scsi_cmd_req *req;
1558 : struct virtio_scsi_cmd_resp *resp;
1559 :
1560 0 : io_ch = spdk_get_io_channel(svdev);
1561 0 : if (io_ch == NULL) {
1562 0 : return -EBUSY;
1563 : }
1564 :
1565 0 : base = spdk_zmalloc(sizeof(*base), 64, NULL,
1566 : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
1567 0 : if (base == NULL) {
1568 0 : SPDK_ERRLOG("couldn't allocate memory for scsi target scan.\n");
1569 0 : return -ENOMEM;
1570 : }
1571 :
1572 0 : base->svdev = svdev;
1573 :
1574 0 : base->channel = spdk_io_channel_get_ctx(io_ch);
1575 0 : TAILQ_INIT(&base->scan_queue);
1576 0 : svdev->scan_ctx = base;
1577 :
1578 0 : base->iov.iov_base = base->payload;
1579 0 : io_ctx = &base->io_ctx;
1580 0 : req = &io_ctx->req;
1581 0 : resp = &io_ctx->resp;
1582 0 : io_ctx->iov_req.iov_base = req;
1583 0 : io_ctx->iov_req.iov_len = sizeof(*req);
1584 0 : io_ctx->iov_resp.iov_base = resp;
1585 0 : io_ctx->iov_resp.iov_len = sizeof(*resp);
1586 :
1587 0 : base->retries = SCAN_REQUEST_RETRIES;
1588 0 : return 0;
1589 : }
1590 :
1591 : static void
1592 0 : _virtio_scsi_dev_scan_tgt(struct virtio_scsi_scan_base *base, uint8_t target)
1593 : {
1594 : int rc;
1595 :
1596 0 : memset(&base->info, 0, sizeof(base->info));
1597 0 : base->info.target = target;
1598 :
1599 0 : rc = send_inquiry(base);
1600 : if (rc) {
1601 : /* Let response poller do the resend */
1602 : }
1603 0 : }
1604 :
1605 : static int
1606 0 : virtio_scsi_dev_scan(struct virtio_scsi_dev *svdev, bdev_virtio_create_cb cb_fn,
1607 : void *cb_arg)
1608 : {
1609 : struct virtio_scsi_scan_base *base;
1610 : struct virtio_scsi_scan_info *tgt, *next_tgt;
1611 : int rc;
1612 :
1613 0 : if (svdev->scan_ctx) {
1614 0 : if (svdev->scan_ctx->full_scan) {
1615 0 : return -EEXIST;
1616 : }
1617 :
1618 : /* We're about to start a full rescan, so there's no need
1619 : * to scan particular targets afterwards.
1620 : */
1621 0 : TAILQ_FOREACH_SAFE(tgt, &svdev->scan_ctx->scan_queue, tailq, next_tgt) {
1622 0 : TAILQ_REMOVE(&svdev->scan_ctx->scan_queue, tgt, tailq);
1623 0 : free(tgt);
1624 : }
1625 :
1626 0 : svdev->scan_ctx->cb_fn = cb_fn;
1627 0 : svdev->scan_ctx->cb_arg = cb_arg;
1628 0 : svdev->scan_ctx->restart = true;
1629 0 : return 0;
1630 : }
1631 :
1632 0 : rc = _virtio_scsi_dev_scan_init(svdev);
1633 0 : if (rc != 0) {
1634 0 : return rc;
1635 : }
1636 :
1637 0 : base = svdev->scan_ctx;
1638 0 : base->cb_fn = cb_fn;
1639 0 : base->cb_arg = cb_arg;
1640 0 : base->full_scan = true;
1641 :
1642 0 : _virtio_scsi_dev_scan_tgt(base, 0);
1643 0 : return 0;
1644 : }
1645 :
1646 : static int
1647 0 : virtio_scsi_dev_scan_tgt(struct virtio_scsi_dev *svdev, uint8_t target)
1648 : {
1649 : struct virtio_scsi_scan_base *base;
1650 : struct virtio_scsi_scan_info *info;
1651 : int rc;
1652 :
1653 0 : base = svdev->scan_ctx;
1654 0 : if (base) {
1655 0 : info = calloc(1, sizeof(*info));
1656 0 : if (info == NULL) {
1657 0 : SPDK_ERRLOG("calloc failed\n");
1658 0 : return -ENOMEM;
1659 : }
1660 :
1661 0 : info->target = target;
1662 0 : TAILQ_INSERT_TAIL(&base->scan_queue, info, tailq);
1663 0 : return 0;
1664 : }
1665 :
1666 0 : rc = _virtio_scsi_dev_scan_init(svdev);
1667 0 : if (rc != 0) {
1668 0 : return rc;
1669 : }
1670 :
1671 0 : base = svdev->scan_ctx;
1672 0 : base->full_scan = true;
1673 0 : _virtio_scsi_dev_scan_tgt(base, target);
1674 0 : return 0;
1675 : }
1676 :
1677 : static int
1678 0 : bdev_virtio_initialize(void)
1679 : {
1680 0 : return 0;
1681 : }
1682 :
1683 : static void
1684 0 : _virtio_scsi_dev_unregister_cb(void *io_device)
1685 : {
1686 0 : struct virtio_scsi_dev *svdev = io_device;
1687 0 : struct virtio_dev *vdev = &svdev->vdev;
1688 : bool finish_module;
1689 : bdev_virtio_remove_cb remove_cb;
1690 : void *remove_ctx;
1691 :
1692 0 : assert(spdk_ring_count(svdev->ctrlq_ring) == 0);
1693 0 : spdk_ring_free(svdev->ctrlq_ring);
1694 0 : spdk_poller_unregister(&svdev->mgmt_poller);
1695 :
1696 0 : virtio_dev_release_queue(vdev, VIRTIO_SCSI_EVENTQ);
1697 0 : virtio_dev_release_queue(vdev, VIRTIO_SCSI_CONTROLQ);
1698 :
1699 0 : virtio_dev_stop(vdev);
1700 0 : virtio_dev_destruct(vdev);
1701 :
1702 0 : pthread_mutex_lock(&g_virtio_scsi_mutex);
1703 0 : TAILQ_REMOVE(&g_virtio_scsi_devs, svdev, tailq);
1704 0 : pthread_mutex_unlock(&g_virtio_scsi_mutex);
1705 :
1706 0 : remove_cb = svdev->remove_cb;
1707 0 : remove_ctx = svdev->remove_ctx;
1708 0 : spdk_free(svdev->eventq_ios);
1709 0 : free(svdev);
1710 :
1711 0 : if (remove_cb) {
1712 0 : remove_cb(remove_ctx, 0);
1713 : }
1714 :
1715 0 : finish_module = TAILQ_EMPTY(&g_virtio_scsi_devs);
1716 :
1717 0 : if (g_bdev_virtio_finish && finish_module) {
1718 0 : spdk_bdev_module_fini_done();
1719 : }
1720 0 : }
1721 :
1722 : static void
1723 0 : virtio_scsi_dev_unregister_cb(void *io_device)
1724 : {
1725 0 : struct virtio_scsi_dev *svdev = io_device;
1726 : struct spdk_thread *thread;
1727 :
1728 0 : thread = virtio_dev_queue_get_thread(&svdev->vdev, VIRTIO_SCSI_CONTROLQ);
1729 0 : spdk_thread_send_msg(thread, _virtio_scsi_dev_unregister_cb, io_device);
1730 0 : }
1731 :
1732 : static void
1733 0 : virtio_scsi_dev_remove(struct virtio_scsi_dev *svdev,
1734 : bdev_virtio_remove_cb cb_fn, void *cb_arg)
1735 : {
1736 : struct virtio_scsi_disk *disk, *disk_tmp;
1737 0 : bool do_remove = true;
1738 :
1739 0 : if (svdev->removed) {
1740 0 : if (cb_fn) {
1741 0 : cb_fn(cb_arg, -EBUSY);
1742 : }
1743 0 : return;
1744 : }
1745 :
1746 0 : svdev->remove_cb = cb_fn;
1747 0 : svdev->remove_ctx = cb_arg;
1748 0 : svdev->removed = true;
1749 :
1750 0 : if (svdev->scan_ctx) {
1751 : /* The removal will continue after we receive a pending scan I/O. */
1752 0 : return;
1753 : }
1754 :
1755 0 : TAILQ_FOREACH_SAFE(disk, &svdev->luns, link, disk_tmp) {
1756 0 : if (!disk->removed) {
1757 0 : spdk_bdev_unregister(&disk->bdev, NULL, NULL);
1758 : }
1759 0 : do_remove = false;
1760 : }
1761 :
1762 0 : if (do_remove) {
1763 0 : spdk_io_device_unregister(svdev, virtio_scsi_dev_unregister_cb);
1764 : }
1765 : }
1766 :
1767 : static void
1768 0 : bdev_virtio_finish(void)
1769 : {
1770 : struct virtio_scsi_dev *svdev, *next;
1771 :
1772 0 : g_bdev_virtio_finish = true;
1773 :
1774 0 : pthread_mutex_lock(&g_virtio_scsi_mutex);
1775 0 : if (TAILQ_EMPTY(&g_virtio_scsi_devs)) {
1776 0 : pthread_mutex_unlock(&g_virtio_scsi_mutex);
1777 0 : spdk_bdev_module_fini_done();
1778 0 : return;
1779 : }
1780 :
1781 : /* Defer module finish until all controllers are removed. */
1782 0 : TAILQ_FOREACH_SAFE(svdev, &g_virtio_scsi_devs, tailq, next) {
1783 0 : virtio_scsi_dev_remove(svdev, NULL, NULL);
1784 : }
1785 0 : pthread_mutex_unlock(&g_virtio_scsi_mutex);
1786 : }
1787 :
1788 : int
1789 0 : bdev_virtio_user_scsi_dev_create(const char *base_name, const char *path,
1790 : unsigned num_queues, unsigned queue_size,
1791 : bdev_virtio_create_cb cb_fn, void *cb_arg)
1792 : {
1793 : struct virtio_scsi_dev *svdev;
1794 : int rc;
1795 :
1796 0 : svdev = virtio_user_scsi_dev_create(base_name, path, num_queues, queue_size);
1797 0 : if (svdev == NULL) {
1798 0 : return -1;
1799 : }
1800 :
1801 0 : rc = virtio_scsi_dev_scan(svdev, cb_fn, cb_arg);
1802 0 : if (rc) {
1803 0 : virtio_scsi_dev_remove(svdev, NULL, NULL);
1804 : }
1805 :
1806 0 : return rc;
1807 : }
1808 :
1809 : int
1810 0 : bdev_vfio_user_scsi_dev_create(const char *base_name, const char *path,
1811 : bdev_virtio_create_cb cb_fn, void *cb_arg)
1812 : {
1813 : struct virtio_scsi_dev *svdev;
1814 0 : uint32_t num_queues = 0;
1815 : int rc;
1816 :
1817 0 : svdev = calloc(1, sizeof(*svdev));
1818 0 : if (svdev == NULL) {
1819 0 : SPDK_ERRLOG("calloc failed for virtio device %s: %s\n", base_name, path);
1820 0 : return -ENOMEM;
1821 : }
1822 :
1823 0 : rc = virtio_vfio_user_dev_init(&svdev->vdev, base_name, path);
1824 0 : if (rc != 0) {
1825 0 : SPDK_ERRLOG("Failed to create %s as virtio device\n", path);
1826 0 : free(svdev);
1827 0 : return -EFAULT;
1828 : }
1829 :
1830 0 : rc = virtio_dev_read_dev_config(&svdev->vdev, offsetof(struct virtio_scsi_config, num_queues),
1831 : &num_queues, sizeof(num_queues));
1832 0 : if (rc) {
1833 0 : SPDK_ERRLOG("%s: config read failed: %s\n", base_name, spdk_strerror(-rc));
1834 0 : virtio_dev_destruct(&svdev->vdev);
1835 0 : free(svdev);
1836 0 : return rc;
1837 : }
1838 :
1839 0 : if (num_queues < SPDK_VIRTIO_SCSI_QUEUE_NUM_FIXED) {
1840 0 : SPDK_ERRLOG("%s: invalid num_queues %u\n", base_name, num_queues);
1841 0 : virtio_dev_destruct(&svdev->vdev);
1842 0 : free(svdev);
1843 0 : return -EINVAL;
1844 : }
1845 :
1846 0 : rc = virtio_scsi_dev_init(svdev, num_queues, VIRTIO_SCSI_DEV_SUPPORTED_FEATURES);
1847 0 : if (rc != 0) {
1848 0 : virtio_dev_destruct(&svdev->vdev);
1849 0 : free(svdev);
1850 0 : return -EFAULT;
1851 : }
1852 :
1853 0 : rc = virtio_scsi_dev_scan(svdev, cb_fn, cb_arg);
1854 0 : if (rc) {
1855 0 : virtio_scsi_dev_remove(svdev, NULL, NULL);
1856 : }
1857 :
1858 0 : return rc;
1859 : }
1860 :
1861 : struct bdev_virtio_pci_dev_create_ctx {
1862 : const char *name;
1863 : bdev_virtio_create_cb cb_fn;
1864 : void *cb_arg;
1865 : };
1866 :
1867 : static int
1868 0 : bdev_virtio_pci_scsi_dev_create_cb(struct virtio_pci_ctx *pci_ctx, void *ctx)
1869 : {
1870 : struct virtio_scsi_dev *svdev;
1871 0 : struct bdev_virtio_pci_dev_create_ctx *create_ctx = ctx;
1872 : int rc;
1873 :
1874 0 : svdev = virtio_pci_scsi_dev_create(create_ctx->name, pci_ctx);
1875 0 : if (svdev == NULL) {
1876 0 : return -1;
1877 : }
1878 :
1879 0 : rc = virtio_scsi_dev_scan(svdev, create_ctx->cb_fn, create_ctx->cb_arg);
1880 0 : if (rc) {
1881 0 : svdev->vdev.ctx = NULL;
1882 0 : virtio_scsi_dev_remove(svdev, NULL, NULL);
1883 : }
1884 :
1885 0 : return rc;
1886 : }
1887 :
1888 : int
1889 0 : bdev_virtio_pci_scsi_dev_create(const char *name, struct spdk_pci_addr *pci_addr,
1890 : bdev_virtio_create_cb cb_fn, void *cb_arg)
1891 : {
1892 0 : struct bdev_virtio_pci_dev_create_ctx create_ctx;
1893 :
1894 0 : create_ctx.name = name;
1895 0 : create_ctx.cb_fn = cb_fn;
1896 0 : create_ctx.cb_arg = cb_arg;
1897 :
1898 0 : return virtio_pci_dev_attach(bdev_virtio_pci_scsi_dev_create_cb, &create_ctx,
1899 : VIRTIO_ID_SCSI, pci_addr);
1900 : }
1901 :
1902 : int
1903 0 : bdev_virtio_scsi_dev_remove(const char *name, bdev_virtio_remove_cb cb_fn, void *cb_arg)
1904 : {
1905 : struct virtio_scsi_dev *svdev;
1906 :
1907 0 : pthread_mutex_lock(&g_virtio_scsi_mutex);
1908 0 : TAILQ_FOREACH(svdev, &g_virtio_scsi_devs, tailq) {
1909 0 : if (strcmp(svdev->vdev.name, name) == 0) {
1910 0 : break;
1911 : }
1912 : }
1913 :
1914 0 : if (svdev == NULL) {
1915 0 : pthread_mutex_unlock(&g_virtio_scsi_mutex);
1916 0 : SPDK_ERRLOG("Cannot find Virtio-SCSI device named '%s'\n", name);
1917 0 : return -ENODEV;
1918 : }
1919 :
1920 0 : virtio_scsi_dev_remove(svdev, cb_fn, cb_arg);
1921 0 : pthread_mutex_unlock(&g_virtio_scsi_mutex);
1922 :
1923 0 : return 0;
1924 : }
1925 :
1926 : void
1927 0 : bdev_virtio_scsi_dev_list(struct spdk_json_write_ctx *w)
1928 : {
1929 : struct virtio_scsi_dev *svdev;
1930 :
1931 0 : spdk_json_write_array_begin(w);
1932 :
1933 0 : pthread_mutex_lock(&g_virtio_scsi_mutex);
1934 0 : TAILQ_FOREACH(svdev, &g_virtio_scsi_devs, tailq) {
1935 0 : spdk_json_write_object_begin(w);
1936 :
1937 0 : spdk_json_write_named_string(w, "name", svdev->vdev.name);
1938 :
1939 0 : virtio_dev_dump_json_info(&svdev->vdev, w);
1940 :
1941 0 : spdk_json_write_object_end(w);
1942 : }
1943 0 : pthread_mutex_unlock(&g_virtio_scsi_mutex);
1944 :
1945 0 : spdk_json_write_array_end(w);
1946 0 : }
1947 :
1948 0 : SPDK_LOG_REGISTER_COMPONENT(virtio)
|