Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
3 : : * All rights reserved.
4 : : */
5 : :
6 : : #ifndef SPDK_VIRTIO_H
7 : : #define SPDK_VIRTIO_H
8 : :
9 : : #include "spdk/stdinc.h"
10 : :
11 : : #include <linux/virtio_ring.h>
12 : : #include <linux/virtio_pci.h>
13 : : #include <linux/virtio_config.h>
14 : :
15 : : #include "spdk/log.h"
16 : : #include "spdk/likely.h"
17 : : #include "spdk/queue.h"
18 : : #include "spdk/json.h"
19 : : #include "spdk/thread.h"
20 : : #include "spdk/pci_ids.h"
21 : : #include "spdk/env.h"
22 : :
23 : : /**
24 : : * The maximum virtqueue size is 2^15. Use that value as the end of
25 : : * descriptor chain terminator since it will never be a valid index
26 : : * in the descriptor table. This is used to verify we are correctly
27 : : * handling vq_free_cnt.
28 : : */
29 : : #define VQ_RING_DESC_CHAIN_END 32768
30 : :
31 : : #define SPDK_VIRTIO_MAX_VIRTQUEUES 0x100
32 : :
33 : : /* Extra status define for readability */
34 : : #define VIRTIO_CONFIG_S_RESET 0
35 : :
36 : : struct virtio_dev_ops;
37 : :
38 : : struct virtio_dev {
39 : : struct virtqueue **vqs;
40 : :
41 : : /** Name of this virtio dev set by backend */
42 : : char *name;
43 : :
44 : : /** Fixed number of backend-specific non-I/O virtqueues. */
45 : : uint16_t fixed_queues_num;
46 : :
47 : : /** Max number of virtqueues the host supports. */
48 : : uint16_t max_queues;
49 : :
50 : : /** Common device & guest features. */
51 : : uint64_t negotiated_features;
52 : :
53 : : int is_hw;
54 : :
55 : : /** Modern/legacy virtio device flag. */
56 : : uint8_t modern;
57 : :
58 : : /** Mutex for asynchronous virtqueue-changing operations. */
59 : : pthread_mutex_t mutex;
60 : :
61 : : /** Backend-specific callbacks. */
62 : : const struct virtio_dev_ops *backend_ops;
63 : :
64 : : /** Context for the backend ops */
65 : : void *ctx;
66 : : };
67 : :
68 : : struct virtio_dev_ops {
69 : : int (*read_dev_cfg)(struct virtio_dev *hw, size_t offset,
70 : : void *dst, int len);
71 : : int (*write_dev_cfg)(struct virtio_dev *hw, size_t offset,
72 : : const void *src, int len);
73 : : uint8_t (*get_status)(struct virtio_dev *hw);
74 : : void (*set_status)(struct virtio_dev *hw, uint8_t status);
75 : :
76 : : /**
77 : : * Get device features. The features might be already
78 : : * negotiated with driver (guest) features.
79 : : */
80 : : uint64_t (*get_features)(struct virtio_dev *vdev);
81 : :
82 : : /**
83 : : * Negotiate and set device features.
84 : : * The negotiation can fail with return code -1.
85 : : * This function should also set vdev->negotiated_features field.
86 : : */
87 : : int (*set_features)(struct virtio_dev *vdev, uint64_t features);
88 : :
89 : : /** Destruct virtio device */
90 : : void (*destruct_dev)(struct virtio_dev *vdev);
91 : :
92 : : uint16_t (*get_queue_size)(struct virtio_dev *vdev, uint16_t queue_id);
93 : : int (*setup_queue)(struct virtio_dev *hw, struct virtqueue *vq);
94 : : void (*del_queue)(struct virtio_dev *hw, struct virtqueue *vq);
95 : : void (*notify_queue)(struct virtio_dev *hw, struct virtqueue *vq);
96 : :
97 : : void (*dump_json_info)(struct virtio_dev *hw, struct spdk_json_write_ctx *w);
98 : : void (*write_json_config)(struct virtio_dev *hw, struct spdk_json_write_ctx *w);
99 : : };
100 : :
101 : : struct vq_desc_extra {
102 : : void *cookie;
103 : : uint16_t ndescs;
104 : : };
105 : :
106 : : struct virtqueue {
107 : : struct virtio_dev *vdev; /**< owner of this virtqueue */
108 : : struct vring vq_ring; /**< vring keeping desc, used and avail */
109 : : /**
110 : : * Last consumed descriptor in the used table,
111 : : * trails vq_ring.used->idx.
112 : : */
113 : : uint16_t vq_used_cons_idx;
114 : : uint16_t vq_nentries; /**< vring desc numbers */
115 : : uint16_t vq_free_cnt; /**< num of desc available */
116 : : uint16_t vq_avail_idx; /**< sync until needed */
117 : :
118 : : void *vq_ring_virt_mem; /**< virtual address of vring */
119 : : unsigned int vq_ring_size;
120 : :
121 : : uint64_t vq_ring_mem; /**< physical address of vring */
122 : :
123 : : /**
124 : : * Head of the free chain in the descriptor table. If
125 : : * there are no free descriptors, this will be set to
126 : : * VQ_RING_DESC_CHAIN_END.
127 : : */
128 : : uint16_t vq_desc_head_idx;
129 : :
130 : : /**
131 : : * Tail of the free chain in desc table. If
132 : : * there are no free descriptors, this will be set to
133 : : * VQ_RING_DESC_CHAIN_END.
134 : : */
135 : : uint16_t vq_desc_tail_idx;
136 : : uint16_t vq_queue_index; /**< PCI queue index */
137 : : uint16_t *notify_addr;
138 : :
139 : : /** Thread that's polling this queue. */
140 : : struct spdk_thread *owner_thread;
141 : :
142 : : uint16_t req_start;
143 : : uint16_t req_end;
144 : : uint16_t reqs_finished;
145 : :
146 : : struct vq_desc_extra vq_descx[0];
147 : : };
148 : :
149 : : enum spdk_virtio_desc_type {
150 : : SPDK_VIRTIO_DESC_RO = 0, /**< Read only */
151 : : SPDK_VIRTIO_DESC_WR = VRING_DESC_F_WRITE, /**< Write only */
152 : : /* TODO VIRTIO_DESC_INDIRECT */
153 : : };
154 : :
155 : : /** Context for creating PCI virtio_devs */
156 : : struct virtio_pci_ctx;
157 : :
158 : : /**
159 : : * Callback for creating virtio_dev from a PCI device.
160 : : * \param pci_ctx PCI context to be associated with a virtio_dev
161 : : * \param ctx context provided by the user
162 : : * \return 0 on success, -1 on error.
163 : : */
164 : : typedef int (*virtio_pci_create_cb)(struct virtio_pci_ctx *pci_ctx, void *ctx);
165 : :
166 : : uint16_t virtio_recv_pkts(struct virtqueue *vq, void **io, uint32_t *len, uint16_t io_cnt);
167 : :
168 : : /**
169 : : * Start a new request on the current vring head position and associate it
170 : : * with an opaque cookie object. The previous request in given vq will be
171 : : * made visible to the device in hopes it can be processed early, but there's
172 : : * no guarantee it will be until the device is notified with \c
173 : : * virtqueue_req_flush. This behavior is simply an optimization and virtqueues
174 : : * must always be flushed. Empty requests (with no descriptors added) will be
175 : : * ignored. The device owning given virtqueue must be started.
176 : : *
177 : : * \param vq virtio queue
178 : : * \param cookie opaque object to associate with this request. Once the request
179 : : * is sent, processed and a response is received, the same object will be
180 : : * returned to the user after calling the virtio poll API.
181 : : * \param iovcnt number of required iovectors for the request. This can be
182 : : * higher than than the actual number of iovectors to be added.
183 : : * \return 0 on success or negative errno otherwise. If the `iovcnt` is
184 : : * greater than virtqueue depth, -EINVAL is returned. If simply not enough
185 : : * iovectors are available, -ENOMEM is returned.
186 : : */
187 : : int virtqueue_req_start(struct virtqueue *vq, void *cookie, int iovcnt);
188 : :
189 : : /**
190 : : * Flush a virtqueue. This will notify the device if it's required.
191 : : * The device owning given virtqueue must be started.
192 : : *
193 : : * \param vq virtio queue
194 : : */
195 : : void virtqueue_req_flush(struct virtqueue *vq);
196 : :
197 : : /**
198 : : * Abort the very last request in a virtqueue. This will restore virtqueue
199 : : * state to the point before the last request was created. Note that this
200 : : * is only effective if a queue hasn't been flushed yet. The device owning
201 : : * given virtqueue must be started.
202 : : *
203 : : * \param vq virtio queue
204 : : */
205 : : void virtqueue_req_abort(struct virtqueue *vq);
206 : :
207 : : /**
208 : : * Add iovec chain to the last created request. This call does not provide any
209 : : * error-checking. The caller has to ensure that he doesn't add more iovs than
210 : : * what was specified during request creation. The device owning given virtqueue
211 : : * must be started.
212 : : *
213 : : * \param vq virtio queue
214 : : * \param iovs iovec array
215 : : * \param iovcnt number of iovs in iovec array
216 : : * \param desc_type type of all given iovectors
217 : : */
218 : : void virtqueue_req_add_iovs(struct virtqueue *vq, struct iovec *iovs, uint16_t iovcnt,
219 : : enum spdk_virtio_desc_type desc_type);
220 : :
221 : : /**
222 : : * Construct a virtio device. The device will be in stopped state by default.
223 : : * Before doing any I/O, it has to be manually started via \c virtio_dev_restart.
224 : : *
225 : : * \param vdev memory for virtio device, must be zeroed
226 : : * \param name name for the virtio device
227 : : * \param ops backend callbacks
228 : : * \param ops_ctx argument for the backend callbacks
229 : : * \return zero on success, or negative error code otherwise
230 : : */
231 : : int virtio_dev_construct(struct virtio_dev *vdev, const char *name,
232 : : const struct virtio_dev_ops *ops, void *ops_ctx);
233 : :
234 : : /**
235 : : * Reset the device and prepare it to be `virtio_dev_start`ed. This call
236 : : * will also renegotiate feature flags.
237 : : *
238 : : * \param vdev virtio device
239 : : * \param req_features features this driver supports. A VIRTIO_F_VERSION_1
240 : : * flag will be automatically appended, as legacy devices are not supported.
241 : : */
242 : : int virtio_dev_reset(struct virtio_dev *vdev, uint64_t req_features);
243 : :
244 : : /**
245 : : * Notify the host to start processing this virtio device. This is
246 : : * a blocking call that won't return until the host has started.
247 : : * This will also allocate virtqueues.
248 : : *
249 : : * \param vdev virtio device
250 : : * \param max_queues number of queues to allocate. The max number of
251 : : * usable I/O queues is also limited by the host device. `vdev` will be
252 : : * started successfully even if the host supports less queues than requested.
253 : : * \param fixed_queue_num number of queues preceding the first
254 : : * request queue. For Virtio-SCSI this is equal to 2, as there are
255 : : * additional event and control queues.
256 : : */
257 : : int virtio_dev_start(struct virtio_dev *vdev, uint16_t max_queues,
258 : : uint16_t fixed_queues_num);
259 : :
260 : : /**
261 : : * Stop the host from processing the device. This is a blocking call
262 : : * that won't return until all outstanding I/O has been processed on
263 : : * the host (virtio device) side. In order to re-start the device, it
264 : : * has to be `virtio_dev_reset` first.
265 : : *
266 : : * \param vdev virtio device
267 : : */
268 : : void virtio_dev_stop(struct virtio_dev *vdev);
269 : :
270 : : /**
271 : : * Destruct a virtio device. Note that it must be in the stopped state.
272 : : * The virtio_dev should be manually freed afterwards.
273 : : *
274 : : * \param vdev virtio device
275 : : */
276 : : void virtio_dev_destruct(struct virtio_dev *vdev);
277 : :
278 : : /**
279 : : * Bind a virtqueue with given index to the current thread;
280 : : *
281 : : * This function is thread-safe.
282 : : *
283 : : * \param vdev vhost device
284 : : * \param index virtqueue index
285 : : * \return 0 on success, -1 in case a virtqueue with given index either
286 : : * does not exists or is already acquired.
287 : : */
288 : : int virtio_dev_acquire_queue(struct virtio_dev *vdev, uint16_t index);
289 : :
290 : : /**
291 : : * Look for unused queue and bind it to the current thread. This will
292 : : * scan the queues in range from *start_index* (inclusive) up to
293 : : * vdev->max_queues (exclusive).
294 : : *
295 : : * This function is thread-safe.
296 : : *
297 : : * \param vdev vhost device
298 : : * \param start_index virtqueue index to start looking from
299 : : * \return index of acquired queue or -1 in case no unused queue in given range
300 : : * has been found
301 : : */
302 : : int32_t virtio_dev_find_and_acquire_queue(struct virtio_dev *vdev, uint16_t start_index);
303 : :
304 : : /**
305 : : * Get thread that acquired given virtqueue.
306 : : *
307 : : * This function is thread-safe.
308 : : *
309 : : * \param vdev vhost device
310 : : * \param index index of virtqueue
311 : : * \return thread that acquired given virtqueue. If the queue is unused
312 : : * or doesn't exist a NULL is returned.
313 : : */
314 : : struct spdk_thread *virtio_dev_queue_get_thread(struct virtio_dev *vdev, uint16_t index);
315 : :
316 : : /**
317 : : * Check if virtqueue with given index is acquired.
318 : : *
319 : : * This function is thread-safe.
320 : : *
321 : : * \param vdev vhost device
322 : : * \param index index of virtqueue
323 : : * \return virtqueue acquire status. in case of invalid index *false* is returned.
324 : : */
325 : : bool virtio_dev_queue_is_acquired(struct virtio_dev *vdev, uint16_t index);
326 : :
327 : : /**
328 : : * Release previously acquired queue.
329 : : *
330 : : * This function must be called from the thread that acquired the queue.
331 : : *
332 : : * \param vdev vhost device
333 : : * \param index index of virtqueue to release
334 : : */
335 : : void virtio_dev_release_queue(struct virtio_dev *vdev, uint16_t index);
336 : :
337 : : /**
338 : : * Get Virtio status flags.
339 : : *
340 : : * \param vdev virtio device
341 : : */
342 : : uint8_t virtio_dev_get_status(struct virtio_dev *vdev);
343 : :
344 : : /**
345 : : * Set Virtio status flag. The flags have to be set in very specific order
346 : : * defined the VIRTIO 1.0 spec section 3.1.1. To unset the flags, stop the
347 : : * device or set \c VIRTIO_CONFIG_S_RESET status flag. There is no way to
348 : : * unset only particular flags.
349 : : *
350 : : * \param vdev virtio device
351 : : * \param flag flag to set
352 : : */
353 : : void virtio_dev_set_status(struct virtio_dev *vdev, uint8_t flag);
354 : :
355 : : /**
356 : : * Write raw data into the device config at given offset. This call does not
357 : : * provide any error checking.
358 : : *
359 : : * \param vdev virtio device
360 : : * \param offset offset in bytes
361 : : * \param src pointer to data to copy from
362 : : * \param len length of data to copy in bytes
363 : : * \return 0 on success, negative errno otherwise
364 : : */
365 : : int virtio_dev_write_dev_config(struct virtio_dev *vdev, size_t offset, const void *src, int len);
366 : :
367 : : /**
368 : : * Read raw data from the device config at given offset. This call does not
369 : : * provide any error checking.
370 : : *
371 : : * \param vdev virtio device
372 : : * \param offset offset in bytes
373 : : * \param dst pointer to buffer to copy data into
374 : : * \param len length of data to copy in bytes
375 : : * \return 0 on success, negative errno otherwise
376 : : */
377 : : int virtio_dev_read_dev_config(struct virtio_dev *vdev, size_t offset, void *dst, int len);
378 : :
379 : : /**
380 : : * Get backend-specific ops for given device.
381 : : *
382 : : * \param vdev virtio device
383 : : */
384 : : const struct virtio_dev_ops *virtio_dev_backend_ops(struct virtio_dev *vdev);
385 : :
386 : : /**
387 : : * Check if the device has negotiated given feature bit.
388 : : *
389 : : * \param vdev virtio device
390 : : * \param bit feature bit
391 : : */
392 : : static inline bool
393 : 0 : virtio_dev_has_feature(struct virtio_dev *vdev, uint64_t bit)
394 : : {
395 : 0 : return !!(vdev->negotiated_features & (1ULL << bit));
396 : : }
397 : :
398 : : /**
399 : : * Dump all device specific information into given json stream.
400 : : *
401 : : * \param vdev virtio device
402 : : * \param w json stream
403 : : */
404 : : void virtio_dev_dump_json_info(struct virtio_dev *vdev, struct spdk_json_write_ctx *w);
405 : :
406 : : /**
407 : : * Enumerate all PCI Virtio devices of given type on the system.
408 : : *
409 : : * \param enum_cb a function to be called for each valid PCI device.
410 : : * If a virtio_dev is has been created, the callback should return 0.
411 : : * Returning any other value will cause the PCI context to be freed,
412 : : * making it unusable.
413 : : * \param enum_ctx additional opaque context to be passed into `enum_cb`
414 : : * \param pci_device_id PCI Device ID of devices to iterate through
415 : : */
416 : : int virtio_pci_dev_enumerate(virtio_pci_create_cb enum_cb, void *enum_ctx,
417 : : uint16_t pci_device_id);
418 : :
419 : : /**
420 : : * Attach a PCI Virtio device of given type.
421 : : *
422 : : * \param create_cb callback to create a virtio_dev.
423 : : * If virtio_dev is has been created, the callback should return 0.
424 : : * Returning any other value will cause the PCI context to be freed,
425 : : * making it unusable.
426 : : * \param enum_ctx additional opaque context to be passed into `enum_cb`
427 : : * \param device_id Device ID of devices to iterate through
428 : : * \param pci_addr PCI address of the device to attach
429 : : */
430 : : int virtio_pci_dev_attach(virtio_pci_create_cb create_cb, void *enum_ctx,
431 : : uint16_t device_id, struct spdk_pci_addr *pci_addr);
432 : :
433 : : /**
434 : : * Connect to a vhost-user device and init corresponding virtio_dev struct.
435 : : * The virtio_dev will have to be freed with \c virtio_dev_free.
436 : : *
437 : : * \param vdev preallocated vhost device struct to operate on
438 : : * \param name name of this virtio device
439 : : * \param path path to the Unix domain socket of the vhost-user device
440 : : * \param queue_size size of each of the queues
441 : : * \return virtio device
442 : : */
443 : : int virtio_user_dev_init(struct virtio_dev *vdev, const char *name, const char *path,
444 : : uint32_t queue_size);
445 : :
446 : : /**
447 : : * Connect to a vfio-user device and init corresponding virtio_dev struct.
448 : : * The virtio_dev will have to be freed with \c virtio_dev_free.
449 : : *
450 : : * \param vdev preallocated vhost device struct to operate on
451 : : * \param name name of this virtio device
452 : : * \param path path to the Unix domain socket of the vhost-user device
453 : : * \return virtio device
454 : : */
455 : : int virtio_vfio_user_dev_init(struct virtio_dev *vdev, const char *name, const char *path);
456 : :
457 : : /**
458 : : * Initialize virtio_dev for a given PCI device.
459 : : * The virtio_dev has to be freed with \c virtio_dev_destruct.
460 : : *
461 : : * \param vdev preallocated vhost device struct to operate on
462 : : * \param name name of this virtio device
463 : : * \param pci_ctx context of the PCI device
464 : : * \return 0 on success, -1 on error.
465 : : */
466 : : int virtio_pci_dev_init(struct virtio_dev *vdev, const char *name,
467 : : struct virtio_pci_ctx *pci_ctx);
468 : :
469 : : /**
470 : : * Process the uevent which is accepted from the kernel and the
471 : : * uevent descript the physical device hot add or remove action.
472 : : *
473 : : * \param fd the file descriptor of the kobject netlink socket
474 : : * \param device_id virtio device ID used to represent virtio-blk or other device.
475 : : * \return the name of the virtio device on success, NULL means it
476 : : * is not a suitable uevent.
477 : : */
478 : : const char *virtio_pci_dev_event_process(int fd, uint16_t device_id);
479 : :
480 : : #endif /* SPDK_VIRTIO_H */
|