Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2017 Intel Corporation.
3 : * All rights reserved.
4 : */
5 :
6 : #include "spdk/stdinc.h"
7 :
8 : #include "spdk/bdev.h"
9 : #include "spdk/endian.h"
10 : #include "spdk/env.h"
11 : #include "spdk/thread.h"
12 : #include "spdk/string.h"
13 : #include "spdk/util.h"
14 : #include "spdk/json.h"
15 :
16 : #include "spdk_internal/assert.h"
17 : #include "spdk/bdev_module.h"
18 : #include "spdk/log.h"
19 : #include "spdk_internal/virtio.h"
20 : #include "spdk_internal/vhost_user.h"
21 :
22 : #include <linux/virtio_blk.h>
23 : #include <linux/virtio_ids.h>
24 :
25 : #include "bdev_virtio.h"
26 :
27 : struct virtio_blk_dev {
28 : struct virtio_dev vdev;
29 : struct spdk_bdev bdev;
30 : bool readonly;
31 : bool unmap;
32 : };
33 :
34 : struct virtio_blk_io_ctx {
35 : struct iovec iov_req;
36 : struct iovec iov_resp;
37 : struct iovec iov_unmap;
38 : struct virtio_blk_outhdr req;
39 : struct virtio_blk_discard_write_zeroes unmap;
40 : uint8_t resp;
41 : };
42 :
43 : struct bdev_virtio_blk_io_channel {
44 : struct virtio_dev *vdev;
45 :
46 : /** Virtqueue exclusively assigned to this channel. */
47 : struct virtqueue *vq;
48 :
49 : /** Virtio response poller. */
50 : struct spdk_poller *poller;
51 : };
52 :
53 : /* Features desired/implemented by this driver. */
54 : #define VIRTIO_BLK_DEV_SUPPORTED_FEATURES \
55 : (1ULL << VIRTIO_BLK_F_SIZE_MAX | \
56 : 1ULL << VIRTIO_BLK_F_SEG_MAX | \
57 : 1ULL << VIRTIO_BLK_F_BLK_SIZE | \
58 : 1ULL << VIRTIO_BLK_F_TOPOLOGY | \
59 : 1ULL << VIRTIO_BLK_F_MQ | \
60 : 1ULL << VIRTIO_BLK_F_RO | \
61 : 1ULL << VIRTIO_BLK_F_DISCARD | \
62 : 1ULL << VIRTIO_RING_F_EVENT_IDX)
63 :
64 : /* 10 sec for max poll period */
65 : #define VIRTIO_BLK_HOTPLUG_POLL_PERIOD_MAX 10000000ULL
66 : /* Default poll period is 100ms */
67 : #define VIRTIO_BLK_HOTPLUG_POLL_PERIOD_DEFAULT 100000ULL
68 :
69 : static struct spdk_poller *g_blk_hotplug_poller = NULL;
70 : static int g_blk_hotplug_fd = -1;
71 :
72 : static int bdev_virtio_initialize(void);
73 : static int bdev_virtio_blk_get_ctx_size(void);
74 :
75 : static struct spdk_bdev_module virtio_blk_if = {
76 : .name = "virtio_blk",
77 : .module_init = bdev_virtio_initialize,
78 : .get_ctx_size = bdev_virtio_blk_get_ctx_size,
79 : };
80 :
81 0 : SPDK_BDEV_MODULE_REGISTER(virtio_blk, &virtio_blk_if)
82 :
83 : static int bdev_virtio_blk_ch_create_cb(void *io_device, void *ctx_buf);
84 : static void bdev_virtio_blk_ch_destroy_cb(void *io_device, void *ctx_buf);
85 :
86 : static struct virtio_blk_io_ctx *
87 0 : bdev_virtio_blk_init_io_vreq(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
88 : {
89 : struct virtio_blk_outhdr *req;
90 : uint8_t *resp;
91 : struct virtio_blk_discard_write_zeroes *desc;
92 :
93 0 : struct virtio_blk_io_ctx *io_ctx = (struct virtio_blk_io_ctx *)bdev_io->driver_ctx;
94 :
95 0 : req = &io_ctx->req;
96 0 : resp = &io_ctx->resp;
97 0 : desc = &io_ctx->unmap;
98 :
99 0 : io_ctx->iov_req.iov_base = req;
100 0 : io_ctx->iov_req.iov_len = sizeof(*req);
101 :
102 0 : io_ctx->iov_resp.iov_base = resp;
103 0 : io_ctx->iov_resp.iov_len = sizeof(*resp);
104 :
105 0 : io_ctx->iov_unmap.iov_base = desc;
106 0 : io_ctx->iov_unmap.iov_len = sizeof(*desc);
107 :
108 0 : memset(req, 0, sizeof(*req));
109 0 : return io_ctx;
110 : }
111 :
112 : static void
113 0 : bdev_virtio_blk_send_io(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
114 : {
115 0 : struct bdev_virtio_blk_io_channel *virtio_channel = spdk_io_channel_get_ctx(ch);
116 0 : struct virtqueue *vq = virtio_channel->vq;
117 0 : struct virtio_blk_io_ctx *io_ctx = (struct virtio_blk_io_ctx *)bdev_io->driver_ctx;
118 : int rc;
119 :
120 0 : rc = virtqueue_req_start(vq, bdev_io, bdev_io->u.bdev.iovcnt + 2);
121 0 : if (rc == -ENOMEM) {
122 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM);
123 0 : return;
124 0 : } else if (rc != 0) {
125 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
126 0 : return;
127 : }
128 :
129 0 : virtqueue_req_add_iovs(vq, &io_ctx->iov_req, 1, SPDK_VIRTIO_DESC_RO);
130 0 : if (bdev_io->type == SPDK_BDEV_IO_TYPE_UNMAP) {
131 0 : virtqueue_req_add_iovs(vq, &io_ctx->iov_unmap, 1, SPDK_VIRTIO_DESC_RO);
132 : } else {
133 0 : virtqueue_req_add_iovs(vq, bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
134 0 : bdev_io->type == SPDK_BDEV_IO_TYPE_READ ?
135 : SPDK_VIRTIO_DESC_WR : SPDK_VIRTIO_DESC_RO);
136 : }
137 0 : virtqueue_req_add_iovs(vq, &io_ctx->iov_resp, 1, SPDK_VIRTIO_DESC_WR);
138 :
139 0 : virtqueue_req_flush(vq);
140 : }
141 :
142 : static void
143 0 : bdev_virtio_command(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
144 : {
145 0 : struct virtio_blk_io_ctx *io_ctx = bdev_virtio_blk_init_io_vreq(ch, bdev_io);
146 0 : struct virtio_blk_outhdr *req = &io_ctx->req;
147 0 : struct virtio_blk_discard_write_zeroes *desc = &io_ctx->unmap;
148 :
149 0 : if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ) {
150 0 : req->type = VIRTIO_BLK_T_IN;
151 0 : } else if (bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) {
152 0 : req->type = VIRTIO_BLK_T_OUT;
153 0 : } else if (bdev_io->type == SPDK_BDEV_IO_TYPE_UNMAP) {
154 0 : req->type = VIRTIO_BLK_T_DISCARD;
155 0 : desc->sector = bdev_io->u.bdev.offset_blocks *
156 0 : spdk_bdev_get_block_size(bdev_io->bdev) / 512;
157 0 : desc->num_sectors = bdev_io->u.bdev.num_blocks *
158 0 : spdk_bdev_get_block_size(bdev_io->bdev) / 512;
159 0 : desc->flags = 0;
160 : }
161 :
162 0 : req->sector = bdev_io->u.bdev.offset_blocks *
163 0 : spdk_bdev_get_block_size(bdev_io->bdev) / 512;
164 :
165 0 : bdev_virtio_blk_send_io(ch, bdev_io);
166 0 : }
167 :
168 : static void
169 0 : bdev_virtio_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,
170 : bool success)
171 : {
172 0 : if (!success) {
173 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
174 0 : return;
175 : }
176 :
177 0 : bdev_virtio_command(ch, bdev_io);
178 : }
179 :
180 : static int
181 0 : _bdev_virtio_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
182 : {
183 0 : struct virtio_blk_dev *bvdev = bdev_io->bdev->ctxt;
184 :
185 0 : switch (bdev_io->type) {
186 0 : case SPDK_BDEV_IO_TYPE_READ:
187 0 : spdk_bdev_io_get_buf(bdev_io, bdev_virtio_get_buf_cb,
188 0 : bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen);
189 0 : return 0;
190 0 : case SPDK_BDEV_IO_TYPE_WRITE:
191 0 : if (bvdev->readonly) {
192 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
193 : } else {
194 0 : bdev_virtio_command(ch, bdev_io);
195 : }
196 0 : return 0;
197 0 : case SPDK_BDEV_IO_TYPE_RESET:
198 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS);
199 0 : return 0;
200 0 : case SPDK_BDEV_IO_TYPE_UNMAP:
201 0 : if (bvdev->unmap) {
202 0 : bdev_virtio_command(ch, bdev_io);
203 : } else {
204 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
205 : }
206 0 : return 0;
207 0 : case SPDK_BDEV_IO_TYPE_FLUSH:
208 : default:
209 0 : return -1;
210 : }
211 :
212 : SPDK_UNREACHABLE();
213 : }
214 :
215 : static void
216 0 : bdev_virtio_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
217 : {
218 0 : if (_bdev_virtio_submit_request(ch, bdev_io) < 0) {
219 0 : spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
220 : }
221 0 : }
222 :
223 : static bool
224 0 : bdev_virtio_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type)
225 : {
226 0 : struct virtio_blk_dev *bvdev = ctx;
227 :
228 0 : switch (io_type) {
229 0 : case SPDK_BDEV_IO_TYPE_READ:
230 : case SPDK_BDEV_IO_TYPE_RESET:
231 0 : return true;
232 0 : case SPDK_BDEV_IO_TYPE_WRITE:
233 0 : return !bvdev->readonly;
234 0 : case SPDK_BDEV_IO_TYPE_UNMAP:
235 0 : return bvdev->unmap;
236 0 : case SPDK_BDEV_IO_TYPE_FLUSH:
237 : default:
238 0 : return false;
239 : }
240 : }
241 :
242 : static struct spdk_io_channel *
243 0 : bdev_virtio_get_io_channel(void *ctx)
244 : {
245 0 : struct virtio_blk_dev *bvdev = ctx;
246 :
247 0 : return spdk_get_io_channel(bvdev);
248 : }
249 :
250 : static void
251 0 : virtio_blk_dev_unregister_cb(void *io_device)
252 : {
253 0 : struct virtio_blk_dev *bvdev = io_device;
254 0 : struct virtio_dev *vdev = &bvdev->vdev;
255 :
256 0 : virtio_dev_stop(vdev);
257 0 : virtio_dev_destruct(vdev);
258 0 : spdk_bdev_destruct_done(&bvdev->bdev, 0);
259 0 : free(bvdev);
260 0 : }
261 :
262 : static int
263 0 : bdev_virtio_disk_destruct(void *ctx)
264 : {
265 0 : struct virtio_blk_dev *bvdev = ctx;
266 :
267 0 : spdk_io_device_unregister(bvdev, virtio_blk_dev_unregister_cb);
268 0 : return 1;
269 : }
270 :
271 : int
272 0 : bdev_virtio_blk_dev_remove(const char *name, bdev_virtio_remove_cb cb_fn, void *cb_arg)
273 : {
274 0 : return spdk_bdev_unregister_by_name(name, &virtio_blk_if, cb_fn, cb_arg);
275 : }
276 :
277 : static int
278 0 : bdev_virtio_dump_json_config(void *ctx, struct spdk_json_write_ctx *w)
279 : {
280 0 : struct virtio_blk_dev *bvdev = ctx;
281 :
282 0 : virtio_dev_dump_json_info(&bvdev->vdev, w);
283 0 : return 0;
284 : }
285 :
286 : static void
287 0 : bdev_virtio_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
288 : {
289 0 : struct virtio_blk_dev *bvdev = bdev->ctxt;
290 :
291 0 : spdk_json_write_object_begin(w);
292 :
293 0 : spdk_json_write_named_string(w, "method", "bdev_virtio_attach_controller");
294 :
295 0 : spdk_json_write_named_object_begin(w, "params");
296 0 : spdk_json_write_named_string(w, "name", bvdev->vdev.name);
297 0 : spdk_json_write_named_string(w, "dev_type", "blk");
298 :
299 : /* Write transport specific parameters. */
300 0 : bvdev->vdev.backend_ops->write_json_config(&bvdev->vdev, w);
301 :
302 0 : spdk_json_write_object_end(w);
303 :
304 0 : spdk_json_write_object_end(w);
305 0 : }
306 :
307 : static const struct spdk_bdev_fn_table virtio_fn_table = {
308 : .destruct = bdev_virtio_disk_destruct,
309 : .submit_request = bdev_virtio_submit_request,
310 : .io_type_supported = bdev_virtio_io_type_supported,
311 : .get_io_channel = bdev_virtio_get_io_channel,
312 : .dump_info_json = bdev_virtio_dump_json_config,
313 : .write_config_json = bdev_virtio_write_config_json,
314 : };
315 :
316 : static void
317 0 : bdev_virtio_io_cpl(struct spdk_bdev_io *bdev_io)
318 : {
319 0 : struct virtio_blk_io_ctx *io_ctx = (struct virtio_blk_io_ctx *)bdev_io->driver_ctx;
320 :
321 0 : spdk_bdev_io_complete(bdev_io, io_ctx->resp == VIRTIO_BLK_S_OK ?
322 : SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED);
323 0 : }
324 :
325 : static int
326 0 : bdev_virtio_poll(void *arg)
327 : {
328 0 : struct bdev_virtio_blk_io_channel *ch = arg;
329 0 : void *io[32];
330 0 : uint32_t io_len[32];
331 : uint16_t i, cnt;
332 :
333 0 : cnt = virtio_recv_pkts(ch->vq, io, io_len, SPDK_COUNTOF(io));
334 0 : for (i = 0; i < cnt; ++i) {
335 0 : bdev_virtio_io_cpl(io[i]);
336 : }
337 :
338 0 : return cnt;
339 : }
340 :
341 : static int
342 0 : bdev_virtio_blk_ch_create_cb(void *io_device, void *ctx_buf)
343 : {
344 0 : struct virtio_blk_dev *bvdev = io_device;
345 0 : struct virtio_dev *vdev = &bvdev->vdev;
346 0 : struct bdev_virtio_blk_io_channel *ch = ctx_buf;
347 : struct virtqueue *vq;
348 : int32_t queue_idx;
349 :
350 0 : queue_idx = virtio_dev_find_and_acquire_queue(vdev, 0);
351 0 : if (queue_idx < 0) {
352 0 : SPDK_ERRLOG("Couldn't get an unused queue for the io_channel.\n");
353 0 : return -1;
354 : }
355 :
356 0 : vq = vdev->vqs[queue_idx];
357 :
358 0 : ch->vdev = vdev;
359 0 : ch->vq = vq;
360 :
361 0 : ch->poller = SPDK_POLLER_REGISTER(bdev_virtio_poll, ch, 0);
362 0 : return 0;
363 : }
364 :
365 : static void
366 0 : bdev_virtio_blk_ch_destroy_cb(void *io_device, void *ctx_buf)
367 : {
368 0 : struct virtio_blk_dev *bvdev = io_device;
369 0 : struct virtio_dev *vdev = &bvdev->vdev;
370 0 : struct bdev_virtio_blk_io_channel *ch = ctx_buf;
371 0 : struct virtqueue *vq = ch->vq;
372 :
373 0 : spdk_poller_unregister(&ch->poller);
374 0 : virtio_dev_release_queue(vdev, vq->vq_queue_index);
375 0 : }
376 :
377 : static int
378 0 : virtio_blk_dev_init(struct virtio_blk_dev *bvdev, uint16_t max_queues)
379 : {
380 0 : struct virtio_dev *vdev = &bvdev->vdev;
381 0 : struct spdk_bdev *bdev = &bvdev->bdev;
382 0 : uint64_t capacity, num_blocks;
383 0 : uint32_t block_size, size_max, seg_max;
384 0 : uint16_t host_max_queues;
385 : int rc;
386 :
387 0 : if (virtio_dev_has_feature(vdev, VIRTIO_BLK_F_BLK_SIZE)) {
388 0 : rc = virtio_dev_read_dev_config(vdev, offsetof(struct virtio_blk_config, blk_size),
389 : &block_size, sizeof(block_size));
390 0 : if (rc) {
391 0 : SPDK_ERRLOG("%s: config read failed: %s\n", vdev->name, spdk_strerror(-rc));
392 0 : return rc;
393 : }
394 :
395 0 : if (block_size == 0 || block_size % 512 != 0) {
396 0 : SPDK_ERRLOG("%s: invalid block size (%"PRIu32"). Must be "
397 : "a multiple of 512.\n", vdev->name, block_size);
398 0 : return -EIO;
399 : }
400 : } else {
401 0 : block_size = 512;
402 : }
403 :
404 0 : rc = virtio_dev_read_dev_config(vdev, offsetof(struct virtio_blk_config, capacity),
405 : &capacity, sizeof(capacity));
406 0 : if (rc) {
407 0 : SPDK_ERRLOG("%s: config read failed: %s\n", vdev->name, spdk_strerror(-rc));
408 0 : return rc;
409 : }
410 :
411 : /* `capacity` is a number of 512-byte sectors. */
412 0 : num_blocks = capacity * 512 / block_size;
413 0 : if (num_blocks == 0) {
414 0 : SPDK_ERRLOG("%s: size too small (size: %"PRIu64", blocksize: %"PRIu32").\n",
415 : vdev->name, capacity * 512, block_size);
416 0 : return -EIO;
417 : }
418 :
419 0 : if ((capacity * 512) % block_size != 0) {
420 0 : SPDK_WARNLOG("%s: size has been rounded down to the nearest block size boundary. "
421 : "(block size: %"PRIu32", previous size: %"PRIu64", new size: %"PRIu64")\n",
422 : vdev->name, block_size, capacity * 512, num_blocks * block_size);
423 : }
424 :
425 0 : if (virtio_dev_has_feature(vdev, VIRTIO_BLK_F_MQ)) {
426 0 : rc = virtio_dev_read_dev_config(vdev, offsetof(struct virtio_blk_config, num_queues),
427 : &host_max_queues, sizeof(host_max_queues));
428 0 : if (rc) {
429 0 : SPDK_ERRLOG("%s: config read failed: %s\n", vdev->name, spdk_strerror(-rc));
430 0 : return rc;
431 : }
432 : } else {
433 0 : host_max_queues = 1;
434 : }
435 :
436 0 : if (virtio_dev_has_feature(vdev, VIRTIO_BLK_F_SIZE_MAX)) {
437 0 : rc = virtio_dev_read_dev_config(vdev, offsetof(struct virtio_blk_config, size_max),
438 : &size_max, sizeof(size_max));
439 0 : if (rc) {
440 0 : SPDK_ERRLOG("%s: config read failed: %s\n", vdev->name, spdk_strerror(-rc));
441 0 : return rc;
442 : }
443 :
444 0 : if (spdk_unlikely(size_max < block_size)) {
445 0 : SPDK_WARNLOG("%s: minimum segment size is set to block size %u forcefully.\n",
446 : vdev->name, block_size);
447 0 : size_max = block_size;
448 : }
449 :
450 0 : bdev->max_segment_size = size_max;
451 : }
452 :
453 0 : if (virtio_dev_has_feature(vdev, VIRTIO_BLK_F_SEG_MAX)) {
454 0 : rc = virtio_dev_read_dev_config(vdev, offsetof(struct virtio_blk_config, seg_max),
455 : &seg_max, sizeof(seg_max));
456 0 : if (rc) {
457 0 : SPDK_ERRLOG("%s: config read failed: %s\n", vdev->name, spdk_strerror(-rc));
458 0 : return rc;
459 : }
460 :
461 0 : if (spdk_unlikely(seg_max == 0)) {
462 0 : SPDK_ERRLOG("%s: virtio blk SEG_MAX can't be 0\n", vdev->name);
463 0 : return -EINVAL;
464 : }
465 :
466 0 : bdev->max_num_segments = seg_max;
467 : }
468 :
469 0 : if (virtio_dev_has_feature(vdev, VIRTIO_BLK_F_RO)) {
470 0 : bvdev->readonly = true;
471 : }
472 :
473 0 : if (virtio_dev_has_feature(vdev, VIRTIO_BLK_F_DISCARD)) {
474 0 : bvdev->unmap = true;
475 : }
476 :
477 0 : if (max_queues == 0) {
478 0 : SPDK_ERRLOG("%s: requested 0 request queues (%"PRIu16" available).\n",
479 : vdev->name, host_max_queues);
480 0 : return -EINVAL;
481 : }
482 :
483 0 : if (max_queues > host_max_queues) {
484 0 : SPDK_WARNLOG("%s: requested %"PRIu16" request queues "
485 : "but only %"PRIu16" available.\n",
486 : vdev->name, max_queues, host_max_queues);
487 0 : max_queues = host_max_queues;
488 : }
489 :
490 : /* bdev is tied with the virtio device; we can reuse the name */
491 0 : bdev->name = vdev->name;
492 0 : rc = virtio_dev_start(vdev, max_queues, 0);
493 0 : if (rc != 0) {
494 0 : return rc;
495 : }
496 :
497 0 : bdev->product_name = "VirtioBlk Disk";
498 0 : bdev->write_cache = 0;
499 0 : bdev->blocklen = block_size;
500 0 : bdev->blockcnt = num_blocks;
501 :
502 0 : bdev->ctxt = bvdev;
503 0 : bdev->fn_table = &virtio_fn_table;
504 0 : bdev->module = &virtio_blk_if;
505 :
506 0 : spdk_io_device_register(bvdev, bdev_virtio_blk_ch_create_cb,
507 : bdev_virtio_blk_ch_destroy_cb,
508 : sizeof(struct bdev_virtio_blk_io_channel),
509 0 : vdev->name);
510 :
511 0 : rc = spdk_bdev_register(bdev);
512 0 : if (rc) {
513 0 : SPDK_ERRLOG("Failed to register bdev name=%s\n", bdev->name);
514 0 : spdk_io_device_unregister(bvdev, NULL);
515 0 : virtio_dev_stop(vdev);
516 0 : return rc;
517 : }
518 :
519 0 : return 0;
520 : }
521 :
522 : static struct virtio_blk_dev *
523 0 : virtio_pci_blk_dev_create(const char *name, struct virtio_pci_ctx *pci_ctx)
524 : {
525 : static int pci_dev_counter = 0;
526 : struct virtio_blk_dev *bvdev;
527 : struct virtio_dev *vdev;
528 0 : char *default_name = NULL;
529 0 : uint16_t num_queues;
530 : int rc;
531 :
532 0 : bvdev = calloc(1, sizeof(*bvdev));
533 0 : if (bvdev == NULL) {
534 0 : SPDK_ERRLOG("virtio device calloc failed\n");
535 0 : return NULL;
536 : }
537 0 : vdev = &bvdev->vdev;
538 :
539 0 : if (name == NULL) {
540 0 : default_name = spdk_sprintf_alloc("VirtioBlk%"PRIu32, pci_dev_counter++);
541 0 : if (default_name == NULL) {
542 0 : free(vdev);
543 0 : return NULL;
544 : }
545 0 : name = default_name;
546 : }
547 :
548 0 : rc = virtio_pci_dev_init(vdev, name, pci_ctx);
549 0 : free(default_name);
550 :
551 0 : if (rc != 0) {
552 0 : free(bvdev);
553 0 : return NULL;
554 : }
555 :
556 0 : rc = virtio_dev_reset(vdev, VIRTIO_BLK_DEV_SUPPORTED_FEATURES);
557 0 : if (rc != 0) {
558 0 : goto fail;
559 : }
560 :
561 : /* TODO: add a way to limit usable virtqueues */
562 0 : if (virtio_dev_has_feature(vdev, VIRTIO_BLK_F_MQ)) {
563 0 : rc = virtio_dev_read_dev_config(vdev, offsetof(struct virtio_blk_config, num_queues),
564 : &num_queues, sizeof(num_queues));
565 0 : if (rc) {
566 0 : SPDK_ERRLOG("%s: config read failed: %s\n", vdev->name, spdk_strerror(-rc));
567 0 : goto fail;
568 : }
569 : } else {
570 0 : num_queues = 1;
571 : }
572 :
573 0 : rc = virtio_blk_dev_init(bvdev, num_queues);
574 0 : if (rc != 0) {
575 0 : goto fail;
576 : }
577 :
578 0 : return bvdev;
579 :
580 0 : fail:
581 0 : vdev->ctx = NULL;
582 0 : virtio_dev_destruct(vdev);
583 0 : free(bvdev);
584 0 : return NULL;
585 : }
586 :
587 : static struct virtio_blk_dev *
588 0 : virtio_user_blk_dev_create(const char *name, const char *path,
589 : uint16_t num_queues, uint32_t queue_size)
590 : {
591 : struct virtio_blk_dev *bvdev;
592 : uint64_t feature_bits;
593 : int rc;
594 :
595 0 : bvdev = calloc(1, sizeof(*bvdev));
596 0 : if (bvdev == NULL) {
597 0 : SPDK_ERRLOG("calloc failed for virtio device %s: %s\n", name, path);
598 0 : return NULL;
599 : }
600 :
601 0 : rc = virtio_user_dev_init(&bvdev->vdev, name, path, queue_size);
602 0 : if (rc != 0) {
603 0 : SPDK_ERRLOG("Failed to create virito device %s: %s\n", name, path);
604 0 : free(bvdev);
605 0 : return NULL;
606 : }
607 :
608 0 : feature_bits = VIRTIO_BLK_DEV_SUPPORTED_FEATURES;
609 0 : feature_bits |= (1ULL << VHOST_USER_F_PROTOCOL_FEATURES);
610 0 : rc = virtio_dev_reset(&bvdev->vdev, feature_bits);
611 0 : if (rc != 0) {
612 0 : virtio_dev_destruct(&bvdev->vdev);
613 0 : free(bvdev);
614 0 : return NULL;
615 : }
616 :
617 0 : rc = virtio_blk_dev_init(bvdev, num_queues);
618 0 : if (rc != 0) {
619 0 : virtio_dev_destruct(&bvdev->vdev);
620 0 : free(bvdev);
621 0 : return NULL;
622 : }
623 :
624 0 : return bvdev;
625 : }
626 :
627 : struct bdev_virtio_pci_dev_create_ctx {
628 : const char *name;
629 : struct virtio_blk_dev *ret;
630 : };
631 :
632 : static int
633 0 : bdev_virtio_pci_blk_dev_create_cb(struct virtio_pci_ctx *pci_ctx, void *ctx)
634 : {
635 0 : struct bdev_virtio_pci_dev_create_ctx *create_ctx = ctx;
636 :
637 0 : create_ctx->ret = virtio_pci_blk_dev_create(create_ctx->name, pci_ctx);
638 0 : if (create_ctx->ret == NULL) {
639 0 : return -1;
640 : }
641 :
642 0 : return 0;
643 : }
644 :
645 : struct spdk_bdev *
646 0 : bdev_virtio_pci_blk_dev_create(const char *name, struct spdk_pci_addr *pci_addr)
647 : {
648 0 : struct bdev_virtio_pci_dev_create_ctx create_ctx;
649 :
650 0 : create_ctx.name = name;
651 0 : create_ctx.ret = NULL;
652 :
653 0 : virtio_pci_dev_attach(bdev_virtio_pci_blk_dev_create_cb, &create_ctx,
654 : VIRTIO_ID_BLOCK, pci_addr);
655 :
656 0 : if (create_ctx.ret == NULL) {
657 0 : return NULL;
658 : }
659 :
660 0 : return &create_ctx.ret->bdev;
661 : }
662 :
663 : static int
664 0 : bdev_virtio_pci_blk_monitor(void *arg)
665 : {
666 : const char *vdev_name;
667 0 : struct bdev_virtio_pci_dev_create_ctx create_ctx;
668 :
669 0 : while ((vdev_name = virtio_pci_dev_event_process(g_blk_hotplug_fd, VIRTIO_ID_BLOCK)) != NULL) {
670 0 : bdev_virtio_blk_dev_remove(vdev_name, NULL, NULL);
671 : }
672 :
673 : /* Enumerate virtio pci_blk device */
674 0 : memset(&create_ctx, 0, sizeof(create_ctx));
675 0 : virtio_pci_dev_enumerate(bdev_virtio_pci_blk_dev_create_cb, &create_ctx,
676 : VIRTIO_ID_BLOCK);
677 :
678 0 : return SPDK_POLLER_BUSY;
679 : }
680 :
681 : int
682 0 : bdev_virtio_pci_blk_set_hotplug(bool enabled, uint64_t period_us)
683 : {
684 0 : if (enabled == true && !spdk_process_is_primary()) {
685 0 : return -EPERM;
686 : }
687 :
688 0 : if (g_blk_hotplug_poller) {
689 0 : close(g_blk_hotplug_fd);
690 0 : spdk_poller_unregister(&g_blk_hotplug_poller);
691 : }
692 :
693 0 : if (!enabled) {
694 0 : return 0;
695 : }
696 :
697 0 : g_blk_hotplug_fd = spdk_pci_event_listen();
698 0 : if (g_blk_hotplug_fd < 0) {
699 0 : return g_blk_hotplug_fd;
700 : }
701 :
702 0 : period_us = period_us ? period_us : VIRTIO_BLK_HOTPLUG_POLL_PERIOD_DEFAULT;
703 0 : period_us = spdk_min(period_us, VIRTIO_BLK_HOTPLUG_POLL_PERIOD_MAX);
704 0 : g_blk_hotplug_poller = spdk_poller_register(bdev_virtio_pci_blk_monitor, NULL, period_us);
705 0 : if (!g_blk_hotplug_poller) {
706 0 : close(g_blk_hotplug_fd);
707 0 : return -1;
708 : }
709 :
710 0 : return 0;
711 : }
712 :
713 : static int
714 0 : bdev_virtio_initialize(void)
715 : {
716 0 : return 0;
717 : }
718 :
719 : struct spdk_bdev *
720 0 : bdev_virtio_user_blk_dev_create(const char *name, const char *path,
721 : unsigned num_queues, unsigned queue_size)
722 : {
723 : struct virtio_blk_dev *bvdev;
724 :
725 0 : bvdev = virtio_user_blk_dev_create(name, path, num_queues, queue_size);
726 0 : if (bvdev == NULL) {
727 0 : return NULL;
728 : }
729 :
730 0 : return &bvdev->bdev;
731 : }
732 :
733 : struct spdk_bdev *
734 0 : bdev_virtio_vfio_user_blk_dev_create(const char *name, const char *path)
735 : {
736 : struct virtio_blk_dev *bvdev;
737 0 : uint16_t num_queues = 0;
738 : int rc;
739 :
740 0 : bvdev = calloc(1, sizeof(*bvdev));
741 0 : if (bvdev == NULL) {
742 0 : SPDK_ERRLOG("calloc failed for virtio device %s: %s\n", name, path);
743 0 : return NULL;
744 : }
745 :
746 0 : rc = virtio_vfio_user_dev_init(&bvdev->vdev, name, path);
747 0 : if (rc != 0) {
748 0 : SPDK_ERRLOG("Failed to create %s as virtio device\n", path);
749 0 : free(bvdev);
750 0 : return NULL;
751 : }
752 :
753 0 : rc = virtio_dev_reset(&bvdev->vdev, VIRTIO_BLK_DEV_SUPPORTED_FEATURES);
754 0 : if (rc != 0) {
755 0 : SPDK_ERRLOG("Failed to reset %s as virtio device\n", path);
756 0 : virtio_dev_destruct(&bvdev->vdev);
757 0 : free(bvdev);
758 0 : return NULL;
759 : }
760 :
761 0 : if (virtio_dev_has_feature(&bvdev->vdev, VIRTIO_BLK_F_MQ)) {
762 0 : rc = virtio_dev_read_dev_config(&bvdev->vdev, offsetof(struct virtio_blk_config, num_queues),
763 : &num_queues, sizeof(num_queues));
764 0 : if (rc) {
765 0 : SPDK_ERRLOG("%s: config read failed: %s\n", name, spdk_strerror(-rc));
766 0 : virtio_dev_destruct(&bvdev->vdev);
767 0 : free(bvdev);
768 0 : return NULL;
769 : }
770 : } else {
771 0 : num_queues = 1;
772 : }
773 :
774 0 : rc = virtio_blk_dev_init(bvdev, num_queues);
775 0 : if (rc != 0) {
776 0 : SPDK_ERRLOG("Failed to initialize %s as virtio device\n", path);
777 0 : virtio_dev_destruct(&bvdev->vdev);
778 0 : free(bvdev);
779 0 : return NULL;
780 : }
781 :
782 0 : return &bvdev->bdev;
783 : }
784 :
785 : static int
786 0 : bdev_virtio_blk_get_ctx_size(void)
787 : {
788 0 : return sizeof(struct virtio_blk_io_ctx);
789 : }
790 :
791 0 : SPDK_LOG_REGISTER_COMPONENT(virtio_blk)
|