Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2010-2016 Intel Corporation. All rights reserved.
3 : * All rights reserved.
4 : */
5 :
6 : #include "spdk/stdinc.h"
7 :
8 : #include "spdk/env.h"
9 : #include "spdk/util.h"
10 : #include "spdk/barrier.h"
11 :
12 : #include "spdk_internal/virtio.h"
13 :
14 : /* We use SMP memory barrier variants as all virtio_pci devices
15 : * are purely virtual. All MMIO is executed on a CPU core, so
16 : * there's no need to do full MMIO synchronization.
17 : */
18 : #define virtio_mb() spdk_smp_mb()
19 : #define virtio_rmb() spdk_smp_rmb()
20 : #define virtio_wmb() spdk_smp_wmb()
21 :
22 : /* Chain all the descriptors in the ring with an END */
23 : static inline void
24 0 : vring_desc_init(struct vring_desc *dp, uint16_t n)
25 : {
26 0 : uint16_t i;
27 :
28 0 : for (i = 0; i < n - 1; i++) {
29 0 : dp[i].next = (uint16_t)(i + 1);
30 0 : }
31 0 : dp[i].next = VQ_RING_DESC_CHAIN_END;
32 0 : }
33 :
34 : static void
35 0 : virtio_init_vring(struct virtqueue *vq)
36 : {
37 0 : int size = vq->vq_nentries;
38 0 : struct vring *vr = &vq->vq_ring;
39 0 : uint8_t *ring_mem = vq->vq_ring_virt_mem;
40 :
41 : /*
42 : * Reinitialise since virtio port might have been stopped and restarted
43 : */
44 0 : memset(ring_mem, 0, vq->vq_ring_size);
45 0 : vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN);
46 0 : vq->vq_used_cons_idx = 0;
47 0 : vq->vq_desc_head_idx = 0;
48 0 : vq->vq_avail_idx = 0;
49 0 : vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
50 0 : vq->vq_free_cnt = vq->vq_nentries;
51 0 : vq->req_start = VQ_RING_DESC_CHAIN_END;
52 0 : vq->req_end = VQ_RING_DESC_CHAIN_END;
53 0 : vq->reqs_finished = 0;
54 0 : memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
55 :
56 0 : vring_desc_init(vr->desc, size);
57 :
58 : /* Tell the backend not to interrupt us.
59 : * If F_EVENT_IDX is negotiated, we will always set incredibly high
60 : * used event idx, so that we will practically never receive an
61 : * interrupt. See virtqueue_req_flush()
62 : */
63 0 : if (vq->vdev->negotiated_features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {
64 0 : vring_used_event(&vq->vq_ring) = UINT16_MAX;
65 0 : } else {
66 0 : vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
67 : }
68 0 : }
69 :
70 : static int
71 0 : virtio_init_queue(struct virtio_dev *dev, uint16_t vtpci_queue_idx)
72 : {
73 0 : unsigned int vq_size, size;
74 0 : struct virtqueue *vq;
75 0 : int rc;
76 :
77 0 : SPDK_DEBUGLOG(virtio_dev, "setting up queue: %"PRIu16"\n", vtpci_queue_idx);
78 :
79 : /*
80 : * Read the virtqueue size from the Queue Size field
81 : * Always power of 2 and if 0 virtqueue does not exist
82 : */
83 0 : vq_size = virtio_dev_backend_ops(dev)->get_queue_size(dev, vtpci_queue_idx);
84 0 : SPDK_DEBUGLOG(virtio_dev, "vq_size: %u\n", vq_size);
85 0 : if (vq_size == 0) {
86 0 : SPDK_ERRLOG("virtqueue %"PRIu16" does not exist\n", vtpci_queue_idx);
87 0 : return -EINVAL;
88 : }
89 :
90 0 : if (!spdk_u32_is_pow2(vq_size)) {
91 0 : SPDK_ERRLOG("virtqueue %"PRIu16" size (%u) is not powerof 2\n",
92 : vtpci_queue_idx, vq_size);
93 0 : return -EINVAL;
94 : }
95 :
96 0 : size = sizeof(*vq) + vq_size * sizeof(struct vq_desc_extra);
97 :
98 0 : if (posix_memalign((void **)&vq, SPDK_CACHE_LINE_SIZE, size)) {
99 0 : SPDK_ERRLOG("can not allocate vq\n");
100 0 : return -ENOMEM;
101 : }
102 0 : memset(vq, 0, size);
103 0 : dev->vqs[vtpci_queue_idx] = vq;
104 :
105 0 : vq->vdev = dev;
106 0 : vq->vq_queue_index = vtpci_queue_idx;
107 0 : vq->vq_nentries = vq_size;
108 :
109 : /*
110 : * Reserve a memzone for vring elements
111 : */
112 0 : size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
113 0 : vq->vq_ring_size = SPDK_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
114 0 : SPDK_DEBUGLOG(virtio_dev, "vring_size: %u, rounded_vring_size: %u\n",
115 : size, vq->vq_ring_size);
116 :
117 0 : vq->owner_thread = NULL;
118 :
119 0 : rc = virtio_dev_backend_ops(dev)->setup_queue(dev, vq);
120 0 : if (rc < 0) {
121 0 : SPDK_ERRLOG("setup_queue failed\n");
122 0 : free(vq);
123 0 : dev->vqs[vtpci_queue_idx] = NULL;
124 0 : return rc;
125 : }
126 :
127 0 : SPDK_DEBUGLOG(virtio_dev, "vq->vq_ring_mem: 0x%" PRIx64 "\n",
128 : vq->vq_ring_mem);
129 0 : SPDK_DEBUGLOG(virtio_dev, "vq->vq_ring_virt_mem: 0x%" PRIx64 "\n",
130 : (uint64_t)(uintptr_t)vq->vq_ring_virt_mem);
131 :
132 0 : virtio_init_vring(vq);
133 0 : return 0;
134 0 : }
135 :
136 : static void
137 0 : virtio_free_queues(struct virtio_dev *dev)
138 : {
139 0 : uint16_t nr_vq = dev->max_queues;
140 0 : struct virtqueue *vq;
141 0 : uint16_t i;
142 :
143 0 : if (dev->vqs == NULL) {
144 0 : return;
145 : }
146 :
147 0 : for (i = 0; i < nr_vq; i++) {
148 0 : vq = dev->vqs[i];
149 0 : if (!vq) {
150 0 : continue;
151 : }
152 :
153 0 : virtio_dev_backend_ops(dev)->del_queue(dev, vq);
154 :
155 0 : free(vq);
156 0 : dev->vqs[i] = NULL;
157 0 : }
158 :
159 0 : free(dev->vqs);
160 0 : dev->vqs = NULL;
161 0 : }
162 :
163 : static int
164 0 : virtio_alloc_queues(struct virtio_dev *dev, uint16_t max_queues, uint16_t fixed_vq_num)
165 : {
166 0 : uint16_t i;
167 0 : int ret;
168 :
169 0 : if (max_queues == 0) {
170 : /* perfectly fine to have a device with no virtqueues. */
171 0 : return 0;
172 : }
173 :
174 0 : assert(dev->vqs == NULL);
175 0 : dev->vqs = calloc(1, sizeof(struct virtqueue *) * max_queues);
176 0 : if (!dev->vqs) {
177 0 : SPDK_ERRLOG("failed to allocate %"PRIu16" vqs\n", max_queues);
178 0 : return -ENOMEM;
179 : }
180 :
181 0 : for (i = 0; i < max_queues; i++) {
182 0 : ret = virtio_init_queue(dev, i);
183 0 : if (ret < 0) {
184 0 : virtio_free_queues(dev);
185 0 : return ret;
186 : }
187 0 : }
188 :
189 0 : dev->max_queues = max_queues;
190 0 : dev->fixed_queues_num = fixed_vq_num;
191 0 : return 0;
192 0 : }
193 :
194 : /**
195 : * Negotiate virtio features. For virtio_user this will also set
196 : * dev->modern flag if VIRTIO_F_VERSION_1 flag is negotiated.
197 : */
198 : static int
199 0 : virtio_negotiate_features(struct virtio_dev *dev, uint64_t req_features)
200 : {
201 0 : uint64_t host_features = virtio_dev_backend_ops(dev)->get_features(dev);
202 0 : int rc;
203 :
204 0 : SPDK_DEBUGLOG(virtio_dev, "guest features = %" PRIx64 "\n", req_features);
205 0 : SPDK_DEBUGLOG(virtio_dev, "device features = %" PRIx64 "\n", host_features);
206 :
207 0 : rc = virtio_dev_backend_ops(dev)->set_features(dev, req_features & host_features);
208 0 : if (rc != 0) {
209 0 : SPDK_ERRLOG("failed to negotiate device features.\n");
210 0 : return rc;
211 : }
212 :
213 0 : SPDK_DEBUGLOG(virtio_dev, "negotiated features = %" PRIx64 "\n",
214 : dev->negotiated_features);
215 :
216 0 : virtio_dev_set_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
217 0 : if (!(virtio_dev_get_status(dev) & VIRTIO_CONFIG_S_FEATURES_OK)) {
218 0 : SPDK_ERRLOG("failed to set FEATURES_OK status!\n");
219 : /* either the device failed, or we offered some features that
220 : * depend on other, not offered features.
221 : */
222 0 : return -EINVAL;
223 : }
224 :
225 0 : return 0;
226 0 : }
227 :
228 : int
229 0 : virtio_dev_construct(struct virtio_dev *vdev, const char *name,
230 : const struct virtio_dev_ops *ops, void *ctx)
231 : {
232 0 : int rc;
233 :
234 0 : vdev->name = strdup(name);
235 0 : if (vdev->name == NULL) {
236 0 : return -ENOMEM;
237 : }
238 :
239 0 : rc = pthread_mutex_init(&vdev->mutex, NULL);
240 0 : if (rc != 0) {
241 0 : free(vdev->name);
242 0 : return -rc;
243 : }
244 :
245 0 : vdev->backend_ops = ops;
246 0 : vdev->ctx = ctx;
247 :
248 0 : return 0;
249 0 : }
250 :
251 : int
252 0 : virtio_dev_reset(struct virtio_dev *dev, uint64_t req_features)
253 : {
254 0 : req_features |= (1ULL << VIRTIO_F_VERSION_1);
255 :
256 0 : virtio_dev_stop(dev);
257 :
258 0 : virtio_dev_set_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
259 0 : if (!(virtio_dev_get_status(dev) & VIRTIO_CONFIG_S_ACKNOWLEDGE)) {
260 0 : SPDK_ERRLOG("Failed to set VIRTIO_CONFIG_S_ACKNOWLEDGE status.\n");
261 0 : return -EIO;
262 : }
263 :
264 0 : virtio_dev_set_status(dev, VIRTIO_CONFIG_S_DRIVER);
265 0 : if (!(virtio_dev_get_status(dev) & VIRTIO_CONFIG_S_DRIVER)) {
266 0 : SPDK_ERRLOG("Failed to set VIRTIO_CONFIG_S_DRIVER status.\n");
267 0 : return -EIO;
268 : }
269 :
270 0 : return virtio_negotiate_features(dev, req_features);
271 0 : }
272 :
273 : int
274 0 : virtio_dev_start(struct virtio_dev *vdev, uint16_t max_queues, uint16_t fixed_queue_num)
275 : {
276 0 : int ret;
277 :
278 0 : ret = virtio_alloc_queues(vdev, max_queues, fixed_queue_num);
279 0 : if (ret < 0) {
280 0 : return ret;
281 : }
282 :
283 0 : virtio_dev_set_status(vdev, VIRTIO_CONFIG_S_DRIVER_OK);
284 0 : if (!(virtio_dev_get_status(vdev) & VIRTIO_CONFIG_S_DRIVER_OK)) {
285 0 : SPDK_ERRLOG("Failed to set VIRTIO_CONFIG_S_DRIVER_OK status.\n");
286 0 : return -1;
287 : }
288 :
289 0 : return 0;
290 0 : }
291 :
292 : void
293 0 : virtio_dev_destruct(struct virtio_dev *dev)
294 : {
295 0 : virtio_dev_backend_ops(dev)->destruct_dev(dev);
296 0 : pthread_mutex_destroy(&dev->mutex);
297 0 : free(dev->name);
298 0 : }
299 :
300 : static void
301 0 : vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
302 : {
303 0 : struct vring_desc *dp, *dp_tail;
304 0 : struct vq_desc_extra *dxp;
305 0 : uint16_t desc_idx_last = desc_idx;
306 :
307 0 : dp = &vq->vq_ring.desc[desc_idx];
308 0 : dxp = &vq->vq_descx[desc_idx];
309 0 : vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);
310 0 : if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
311 0 : while (dp->flags & VRING_DESC_F_NEXT) {
312 0 : desc_idx_last = dp->next;
313 0 : dp = &vq->vq_ring.desc[dp->next];
314 : }
315 0 : }
316 0 : dxp->ndescs = 0;
317 :
318 : /*
319 : * We must append the existing free chain, if any, to the end of
320 : * newly freed chain. If the virtqueue was completely used, then
321 : * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
322 : */
323 0 : if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {
324 0 : vq->vq_desc_head_idx = desc_idx;
325 0 : } else {
326 0 : dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx];
327 0 : dp_tail->next = desc_idx;
328 : }
329 :
330 0 : vq->vq_desc_tail_idx = desc_idx_last;
331 0 : dp->next = VQ_RING_DESC_CHAIN_END;
332 0 : }
333 :
334 : static uint16_t
335 0 : virtqueue_dequeue_burst_rx(struct virtqueue *vq, void **rx_pkts,
336 : uint32_t *len, uint16_t num)
337 : {
338 0 : struct vring_used_elem *uep;
339 0 : void *cookie;
340 0 : uint16_t used_idx, desc_idx;
341 0 : uint16_t i;
342 :
343 : /* Caller does the check */
344 0 : for (i = 0; i < num ; i++) {
345 0 : used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
346 0 : uep = &vq->vq_ring.used->ring[used_idx];
347 0 : desc_idx = (uint16_t) uep->id;
348 0 : len[i] = uep->len;
349 0 : cookie = vq->vq_descx[desc_idx].cookie;
350 :
351 0 : if (spdk_unlikely(cookie == NULL)) {
352 0 : SPDK_WARNLOG("vring descriptor with no mbuf cookie at %"PRIu16"\n",
353 : vq->vq_used_cons_idx);
354 0 : break;
355 : }
356 :
357 0 : __builtin_prefetch(cookie);
358 :
359 0 : rx_pkts[i] = cookie;
360 0 : vq->vq_used_cons_idx++;
361 0 : vq_ring_free_chain(vq, desc_idx);
362 0 : vq->vq_descx[desc_idx].cookie = NULL;
363 0 : }
364 :
365 0 : return i;
366 0 : }
367 :
368 : static void
369 0 : finish_req(struct virtqueue *vq)
370 : {
371 0 : struct vring_desc *desc;
372 0 : uint16_t avail_idx;
373 :
374 0 : desc = &vq->vq_ring.desc[vq->req_end];
375 0 : desc->flags &= ~VRING_DESC_F_NEXT;
376 :
377 : /*
378 : * Place the head of the descriptor chain into the next slot and make
379 : * it usable to the host. The chain is made available now rather than
380 : * deferring to virtqueue_req_flush() in the hopes that if the host is
381 : * currently running on another CPU, we can keep it processing the new
382 : * descriptor.
383 : */
384 0 : avail_idx = (uint16_t)(vq->vq_avail_idx & (vq->vq_nentries - 1));
385 0 : vq->vq_ring.avail->ring[avail_idx] = vq->req_start;
386 0 : vq->vq_avail_idx++;
387 0 : vq->req_end = VQ_RING_DESC_CHAIN_END;
388 0 : virtio_wmb();
389 0 : vq->vq_ring.avail->idx = vq->vq_avail_idx;
390 0 : vq->reqs_finished++;
391 0 : }
392 :
393 : int
394 0 : virtqueue_req_start(struct virtqueue *vq, void *cookie, int iovcnt)
395 : {
396 0 : struct vq_desc_extra *dxp;
397 :
398 : /* Reserve enough entries to handle iov split */
399 0 : if (2 * iovcnt > vq->vq_free_cnt) {
400 0 : return iovcnt > vq->vq_nentries ? -EINVAL : -ENOMEM;
401 : }
402 :
403 0 : if (vq->req_end != VQ_RING_DESC_CHAIN_END) {
404 0 : finish_req(vq);
405 0 : }
406 :
407 0 : vq->req_start = vq->vq_desc_head_idx;
408 0 : dxp = &vq->vq_descx[vq->req_start];
409 0 : dxp->cookie = cookie;
410 0 : dxp->ndescs = 0;
411 :
412 0 : return 0;
413 0 : }
414 :
415 : void
416 0 : virtqueue_req_flush(struct virtqueue *vq)
417 : {
418 0 : uint16_t reqs_finished;
419 :
420 0 : if (vq->req_end == VQ_RING_DESC_CHAIN_END) {
421 : /* no non-empty requests have been started */
422 0 : return;
423 : }
424 :
425 0 : finish_req(vq);
426 0 : virtio_mb();
427 :
428 0 : reqs_finished = vq->reqs_finished;
429 0 : vq->reqs_finished = 0;
430 :
431 0 : if (vq->vdev->negotiated_features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {
432 : /* Set used event idx to a value the device will never reach.
433 : * This effectively disables interrupts.
434 : */
435 0 : vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx - vq->vq_nentries - 1;
436 :
437 0 : if (!vring_need_event(vring_avail_event(&vq->vq_ring),
438 0 : vq->vq_avail_idx,
439 0 : vq->vq_avail_idx - reqs_finished)) {
440 0 : return;
441 : }
442 0 : } else if (vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) {
443 0 : return;
444 : }
445 :
446 0 : virtio_dev_backend_ops(vq->vdev)->notify_queue(vq->vdev, vq);
447 0 : SPDK_DEBUGLOG(virtio_dev, "Notified backend after xmit\n");
448 0 : }
449 :
450 : void
451 0 : virtqueue_req_abort(struct virtqueue *vq)
452 : {
453 0 : struct vring_desc *desc;
454 :
455 0 : if (vq->req_start == VQ_RING_DESC_CHAIN_END) {
456 : /* no requests have been started */
457 0 : return;
458 : }
459 :
460 0 : desc = &vq->vq_ring.desc[vq->req_end];
461 0 : desc->flags &= ~VRING_DESC_F_NEXT;
462 :
463 0 : vq_ring_free_chain(vq, vq->req_start);
464 0 : vq->req_start = VQ_RING_DESC_CHAIN_END;
465 0 : }
466 :
467 : void
468 0 : virtqueue_req_add_iovs(struct virtqueue *vq, struct iovec *iovs, uint16_t iovcnt,
469 : enum spdk_virtio_desc_type desc_type)
470 : {
471 0 : struct vring_desc *desc;
472 0 : struct vq_desc_extra *dxp;
473 0 : uint16_t i, prev_head, new_head;
474 0 : uint64_t processed_length, iovec_length, current_length;
475 0 : void *current_base;
476 0 : uint16_t used_desc_count = 0;
477 :
478 0 : assert(vq->req_start != VQ_RING_DESC_CHAIN_END);
479 0 : assert(iovcnt <= vq->vq_free_cnt);
480 :
481 : /* TODO use indirect descriptors if iovcnt is high enough
482 : * or the caller specifies SPDK_VIRTIO_DESC_F_INDIRECT
483 : */
484 :
485 0 : prev_head = vq->req_end;
486 0 : new_head = vq->vq_desc_head_idx;
487 0 : for (i = 0; i < iovcnt; ++i) {
488 0 : processed_length = 0;
489 0 : iovec_length = iovs[i].iov_len;
490 0 : current_base = iovs[i].iov_base;
491 :
492 0 : while (processed_length < iovec_length) {
493 0 : desc = &vq->vq_ring.desc[new_head];
494 0 : current_length = iovec_length - processed_length;
495 :
496 0 : if (!vq->vdev->is_hw) {
497 0 : desc->addr = (uintptr_t)current_base;
498 0 : } else {
499 0 : desc->addr = spdk_vtophys(current_base, ¤t_length);
500 : }
501 :
502 0 : desc->len = current_length;
503 : /* always set NEXT flag. unset it on the last descriptor
504 : * in the request-ending function.
505 : */
506 0 : desc->flags = desc_type | VRING_DESC_F_NEXT;
507 :
508 0 : prev_head = new_head;
509 0 : new_head = desc->next;
510 0 : used_desc_count++;
511 :
512 0 : processed_length += current_length;
513 0 : current_base += current_length;
514 : }
515 0 : }
516 :
517 0 : dxp = &vq->vq_descx[vq->req_start];
518 0 : dxp->ndescs += used_desc_count;
519 :
520 0 : vq->req_end = prev_head;
521 0 : vq->vq_desc_head_idx = new_head;
522 0 : vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - used_desc_count);
523 0 : if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) {
524 0 : assert(vq->vq_free_cnt == 0);
525 0 : vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
526 0 : }
527 0 : }
528 :
529 : #define DESC_PER_CACHELINE (SPDK_CACHE_LINE_SIZE / sizeof(struct vring_desc))
530 : uint16_t
531 0 : virtio_recv_pkts(struct virtqueue *vq, void **io, uint32_t *len, uint16_t nb_pkts)
532 : {
533 0 : uint16_t nb_used, num;
534 :
535 0 : nb_used = vq->vq_ring.used->idx - vq->vq_used_cons_idx;
536 0 : virtio_rmb();
537 :
538 0 : num = (uint16_t)(spdk_likely(nb_used <= nb_pkts) ? nb_used : nb_pkts);
539 0 : if (spdk_likely(num > DESC_PER_CACHELINE)) {
540 0 : num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
541 0 : }
542 :
543 0 : return virtqueue_dequeue_burst_rx(vq, io, len, num);
544 0 : }
545 :
546 : int
547 0 : virtio_dev_acquire_queue(struct virtio_dev *vdev, uint16_t index)
548 : {
549 0 : struct virtqueue *vq = NULL;
550 :
551 0 : if (index >= vdev->max_queues) {
552 0 : SPDK_ERRLOG("requested vq index %"PRIu16" exceeds max queue count %"PRIu16".\n",
553 : index, vdev->max_queues);
554 0 : return -1;
555 : }
556 :
557 0 : pthread_mutex_lock(&vdev->mutex);
558 0 : vq = vdev->vqs[index];
559 0 : if (vq == NULL || vq->owner_thread != NULL) {
560 0 : pthread_mutex_unlock(&vdev->mutex);
561 0 : return -1;
562 : }
563 :
564 0 : vq->owner_thread = spdk_get_thread();
565 0 : pthread_mutex_unlock(&vdev->mutex);
566 0 : return 0;
567 0 : }
568 :
569 : int32_t
570 0 : virtio_dev_find_and_acquire_queue(struct virtio_dev *vdev, uint16_t start_index)
571 : {
572 0 : struct virtqueue *vq = NULL;
573 0 : uint16_t i;
574 :
575 0 : pthread_mutex_lock(&vdev->mutex);
576 0 : for (i = start_index; i < vdev->max_queues; ++i) {
577 0 : vq = vdev->vqs[i];
578 0 : if (vq != NULL && vq->owner_thread == NULL) {
579 0 : break;
580 : }
581 0 : }
582 :
583 0 : if (vq == NULL || i == vdev->max_queues) {
584 0 : SPDK_ERRLOG("no more unused virtio queues with idx >= %"PRIu16".\n", start_index);
585 0 : pthread_mutex_unlock(&vdev->mutex);
586 0 : return -1;
587 : }
588 :
589 0 : vq->owner_thread = spdk_get_thread();
590 0 : pthread_mutex_unlock(&vdev->mutex);
591 0 : return i;
592 0 : }
593 :
594 : struct spdk_thread *
595 0 : virtio_dev_queue_get_thread(struct virtio_dev *vdev, uint16_t index)
596 : {
597 0 : struct spdk_thread *thread = NULL;
598 :
599 0 : if (index >= vdev->max_queues) {
600 0 : SPDK_ERRLOG("given vq index %"PRIu16" exceeds max queue count %"PRIu16"\n",
601 : index, vdev->max_queues);
602 0 : abort(); /* This is not recoverable */
603 : }
604 :
605 0 : pthread_mutex_lock(&vdev->mutex);
606 0 : thread = vdev->vqs[index]->owner_thread;
607 0 : pthread_mutex_unlock(&vdev->mutex);
608 :
609 0 : return thread;
610 0 : }
611 :
612 : bool
613 0 : virtio_dev_queue_is_acquired(struct virtio_dev *vdev, uint16_t index)
614 : {
615 0 : return virtio_dev_queue_get_thread(vdev, index) != NULL;
616 : }
617 :
618 : void
619 0 : virtio_dev_release_queue(struct virtio_dev *vdev, uint16_t index)
620 : {
621 0 : struct virtqueue *vq = NULL;
622 :
623 0 : if (index >= vdev->max_queues) {
624 0 : SPDK_ERRLOG("given vq index %"PRIu16" exceeds max queue count %"PRIu16".\n",
625 : index, vdev->max_queues);
626 0 : return;
627 : }
628 :
629 0 : pthread_mutex_lock(&vdev->mutex);
630 0 : vq = vdev->vqs[index];
631 0 : if (vq == NULL) {
632 0 : SPDK_ERRLOG("virtqueue at index %"PRIu16" is not initialized.\n", index);
633 0 : pthread_mutex_unlock(&vdev->mutex);
634 0 : return;
635 : }
636 :
637 0 : assert(vq->owner_thread == spdk_get_thread());
638 0 : vq->owner_thread = NULL;
639 0 : pthread_mutex_unlock(&vdev->mutex);
640 0 : }
641 :
642 : int
643 0 : virtio_dev_read_dev_config(struct virtio_dev *dev, size_t offset,
644 : void *dst, int length)
645 : {
646 0 : return virtio_dev_backend_ops(dev)->read_dev_cfg(dev, offset, dst, length);
647 : }
648 :
649 : int
650 0 : virtio_dev_write_dev_config(struct virtio_dev *dev, size_t offset,
651 : const void *src, int length)
652 : {
653 0 : return virtio_dev_backend_ops(dev)->write_dev_cfg(dev, offset, src, length);
654 : }
655 :
656 : void
657 0 : virtio_dev_stop(struct virtio_dev *dev)
658 : {
659 0 : virtio_dev_backend_ops(dev)->set_status(dev, VIRTIO_CONFIG_S_RESET);
660 : /* flush status write */
661 0 : virtio_dev_backend_ops(dev)->get_status(dev);
662 0 : virtio_free_queues(dev);
663 0 : }
664 :
665 : void
666 0 : virtio_dev_set_status(struct virtio_dev *dev, uint8_t status)
667 : {
668 0 : if (status != VIRTIO_CONFIG_S_RESET) {
669 0 : status |= virtio_dev_backend_ops(dev)->get_status(dev);
670 0 : }
671 :
672 0 : virtio_dev_backend_ops(dev)->set_status(dev, status);
673 0 : }
674 :
675 : uint8_t
676 0 : virtio_dev_get_status(struct virtio_dev *dev)
677 : {
678 0 : return virtio_dev_backend_ops(dev)->get_status(dev);
679 : }
680 :
681 : const struct virtio_dev_ops *
682 0 : virtio_dev_backend_ops(struct virtio_dev *dev)
683 : {
684 0 : return dev->backend_ops;
685 : }
686 :
687 : void
688 0 : virtio_dev_dump_json_info(struct virtio_dev *hw, struct spdk_json_write_ctx *w)
689 : {
690 0 : spdk_json_write_named_object_begin(w, "virtio");
691 :
692 0 : spdk_json_write_named_uint32(w, "vq_count", hw->max_queues);
693 :
694 0 : spdk_json_write_named_uint32(w, "vq_size",
695 0 : virtio_dev_backend_ops(hw)->get_queue_size(hw, 0));
696 :
697 0 : virtio_dev_backend_ops(hw)->dump_json_info(hw, w);
698 :
699 0 : spdk_json_write_object_end(w);
700 0 : }
701 :
702 0 : SPDK_LOG_REGISTER_COMPONENT(virtio_dev)
|
