Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2022 Intel Corporation. All rights reserved.
3 : */
4 :
5 : #include "spdk/stdinc.h"
6 : #include "spdk/memory.h"
7 : #include "spdk/vfio_user_pci.h"
8 :
9 : #include "spdk_internal/virtio.h"
10 :
11 : #include <linux/vfio.h>
12 :
13 : struct virtio_vfio_user_dev {
14 : struct vfio_device *ctx;
15 : char path[PATH_MAX];
16 :
17 : uint32_t pci_cap_region;
18 : uint32_t pci_cap_common_cfg_offset;
19 : uint32_t pci_cap_common_cfg_length;
20 : uint32_t pci_cap_device_specific_offset;
21 : uint32_t pci_cap_device_specific_length;
22 : uint32_t pci_cap_notifications_offset;
23 : uint32_t pci_cap_notifications_length;
24 : };
25 :
26 : static int
27 0 : virtio_vfio_user_read_dev_config(struct virtio_dev *vdev, size_t offset,
28 : void *dst, int length)
29 : {
30 0 : struct virtio_vfio_user_dev *dev = vdev->ctx;
31 :
32 0 : SPDK_DEBUGLOG(virtio_vfio_user, "offset 0x%lx, length 0x%x\n", offset, length);
33 0 : return spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
34 0 : dev->pci_cap_device_specific_offset + offset,
35 0 : length, dst, false);
36 0 : }
37 :
38 : static int
39 0 : virtio_vfio_user_write_dev_config(struct virtio_dev *vdev, size_t offset,
40 : const void *src, int length)
41 : {
42 0 : struct virtio_vfio_user_dev *dev = vdev->ctx;
43 :
44 0 : SPDK_DEBUGLOG(virtio_vfio_user, "offset 0x%lx, length 0x%x\n", offset, length);
45 0 : return spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
46 0 : dev->pci_cap_device_specific_offset + offset,
47 0 : length, (void *)src, true);
48 0 : }
49 :
50 : static uint8_t
51 0 : virtio_vfio_user_get_status(struct virtio_dev *vdev)
52 : {
53 0 : struct virtio_vfio_user_dev *dev = vdev->ctx;
54 0 : uint64_t offset;
55 0 : uint8_t status = 0;
56 0 : int rc;
57 :
58 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_STATUS;
59 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
60 0 : offset, 1, &status, false);
61 0 : if (rc) {
62 0 : SPDK_ERRLOG("Failed to get device status\n");
63 0 : }
64 :
65 0 : SPDK_DEBUGLOG(virtio_vfio_user, "device status %x\n", status);
66 :
67 0 : return status;
68 0 : }
69 :
70 : static void
71 0 : virtio_vfio_user_set_status(struct virtio_dev *vdev, uint8_t status)
72 : {
73 0 : struct virtio_vfio_user_dev *dev = vdev->ctx;
74 0 : uint64_t offset;
75 0 : int rc;
76 :
77 0 : SPDK_DEBUGLOG(virtio_vfio_user, "device status %x\n", status);
78 :
79 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_STATUS;
80 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
81 0 : offset, 1, &status, true);
82 0 : if (rc) {
83 0 : SPDK_ERRLOG("Failed to set device status\n");
84 0 : }
85 0 : }
86 :
87 : static uint64_t
88 0 : virtio_vfio_user_get_features(struct virtio_dev *vdev)
89 : {
90 0 : struct virtio_vfio_user_dev *dev = vdev->ctx;
91 0 : uint64_t offset;
92 0 : uint32_t features_lo, features_hi, feature_select;
93 0 : int rc;
94 :
95 0 : feature_select = 0;
96 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DFSELECT;
97 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
98 0 : offset, 4, &feature_select, true);
99 0 : if (rc) {
100 0 : SPDK_ERRLOG("Failed to set device feature select\n");
101 0 : }
102 :
103 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DF;
104 0 : features_lo = 0;
105 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
106 0 : offset, 4, &features_lo, false);
107 0 : if (rc) {
108 0 : SPDK_ERRLOG("Failed to get device feature low\n");
109 0 : }
110 :
111 0 : feature_select = 1;
112 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DFSELECT;
113 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
114 0 : offset, 4, &feature_select, true);
115 0 : if (rc) {
116 0 : SPDK_ERRLOG("Failed to set device feature select\n");
117 0 : }
118 :
119 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DF;
120 0 : features_hi = 0;
121 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
122 0 : offset, 4, &features_hi, false);
123 0 : if (rc) {
124 0 : SPDK_ERRLOG("Failed to get device feature high\n");
125 0 : }
126 :
127 0 : SPDK_DEBUGLOG(virtio_vfio_user, "feature_hi 0x%x, feature_low 0x%x\n", features_hi, features_lo);
128 :
129 0 : return (((uint64_t)features_hi << 32) | ((uint64_t)features_lo));
130 0 : }
131 :
132 : static int
133 0 : virtio_vfio_user_set_features(struct virtio_dev *vdev, uint64_t features)
134 : {
135 0 : struct virtio_vfio_user_dev *dev = vdev->ctx;
136 0 : uint64_t offset;
137 0 : uint32_t features_lo, features_hi, feature_select;
138 0 : int rc;
139 :
140 0 : feature_select = 0;
141 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GFSELECT;
142 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
143 0 : offset, 4, &feature_select, true);
144 0 : if (rc) {
145 0 : SPDK_ERRLOG("Failed to set Guest feature select\n");
146 0 : return rc;
147 : }
148 :
149 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GF;
150 0 : features_lo = (uint32_t)features;
151 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
152 0 : offset, 4, &features_lo, true);
153 0 : if (rc) {
154 0 : SPDK_ERRLOG("Failed to set Guest feature low\n");
155 0 : return rc;
156 : }
157 :
158 0 : feature_select = 1;
159 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GFSELECT;
160 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
161 0 : offset, 4, &feature_select, true);
162 0 : if (rc) {
163 0 : SPDK_ERRLOG("Failed to set Guest feature select\n");
164 0 : return rc;
165 : }
166 :
167 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GF;
168 0 : features_hi = (uint32_t)(features >> 32);
169 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
170 0 : offset, 4, &features_hi, true);
171 0 : if (rc) {
172 0 : SPDK_ERRLOG("Failed to set Guest feature high\n");
173 0 : }
174 :
175 0 : vdev->negotiated_features = features;
176 0 : SPDK_DEBUGLOG(virtio_vfio_user, "features 0x%"PRIx64"\n", features);
177 :
178 0 : return rc;
179 0 : }
180 :
181 : static void
182 0 : virtio_vfio_user_destruct_dev(struct virtio_dev *vdev)
183 : {
184 0 : struct virtio_vfio_user_dev *dev = vdev->ctx;
185 :
186 0 : if (dev) {
187 0 : spdk_vfio_user_release(dev->ctx);
188 0 : free(dev);
189 0 : }
190 0 : }
191 :
192 : static uint16_t
193 0 : virtio_vfio_user_get_queue_size(struct virtio_dev *vdev, uint16_t queue_id)
194 : {
195 0 : struct virtio_vfio_user_dev *dev = vdev->ctx;
196 0 : uint64_t offset;
197 0 : uint16_t qsize = 0;
198 0 : int rc;
199 :
200 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SELECT;
201 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
202 0 : offset, 2, &queue_id, true);
203 0 : if (rc) {
204 0 : SPDK_ERRLOG("Failed to set queue select\n");
205 0 : return 0;
206 : }
207 :
208 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SIZE;
209 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
210 0 : offset, 2, &qsize, false);
211 0 : if (rc) {
212 0 : SPDK_ERRLOG("Failed to get queue size\n");
213 0 : return 0;
214 : }
215 :
216 0 : SPDK_DEBUGLOG(virtio_vfio_user, "queue %u, size %u\n", queue_id, qsize);
217 :
218 0 : return qsize;
219 0 : }
220 :
221 : static int
222 0 : virtio_vfio_user_setup_queue(struct virtio_dev *vdev, struct virtqueue *vq)
223 : {
224 0 : struct virtio_vfio_user_dev *dev = vdev->ctx;
225 0 : uint64_t desc_addr, avail_addr, used_addr, offset;
226 0 : uint32_t addr_lo, addr_hi;
227 0 : uint16_t notify_off, queue_enable;
228 0 : void *queue_mem;
229 0 : uint64_t queue_mem_phys_addr;
230 0 : int rc;
231 :
232 0 : queue_mem = spdk_zmalloc(vq->vq_ring_size, VIRTIO_PCI_VRING_ALIGN, NULL,
233 : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
234 0 : if (queue_mem == NULL) {
235 0 : return -ENOMEM;
236 : }
237 :
238 0 : queue_mem_phys_addr = spdk_vtophys(queue_mem, NULL);
239 0 : if (queue_mem_phys_addr == SPDK_VTOPHYS_ERROR) {
240 0 : spdk_free(queue_mem);
241 0 : return -EFAULT;
242 : }
243 :
244 0 : vq->vq_ring_mem = queue_mem_phys_addr;
245 0 : vq->vq_ring_virt_mem = queue_mem;
246 :
247 0 : desc_addr = vq->vq_ring_mem;
248 0 : avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc);
249 0 : used_addr = (avail_addr + offsetof(struct vring_avail, ring[vq->vq_nentries])
250 0 : + VIRTIO_PCI_VRING_ALIGN - 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1);
251 :
252 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SELECT;
253 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
254 0 : offset, 2, &vq->vq_queue_index, true);
255 0 : if (rc) {
256 0 : SPDK_ERRLOG("Failed to set queue select\n");
257 0 : goto err;
258 : }
259 :
260 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_DESCLO;
261 0 : addr_lo = (uint32_t)desc_addr;
262 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
263 0 : offset, 4, &addr_lo, true);
264 0 : if (rc) {
265 0 : SPDK_ERRLOG("Failed to set desc addr low\n");
266 0 : goto err;
267 : }
268 :
269 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_DESCHI;
270 0 : addr_hi = (uint32_t)(desc_addr >> 32);
271 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
272 0 : offset, 4, &addr_hi, true);
273 0 : if (rc) {
274 0 : SPDK_ERRLOG("Failed to set desc addr high\n");
275 0 : goto err;
276 : }
277 :
278 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_AVAILLO;
279 0 : addr_lo = (uint32_t)avail_addr;
280 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
281 0 : offset, 4, &addr_lo, true);
282 0 : if (rc) {
283 0 : SPDK_ERRLOG("Failed to set avail addr low\n");
284 0 : goto err;
285 : }
286 :
287 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_AVAILHI;
288 0 : addr_hi = (uint32_t)(avail_addr >> 32);
289 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
290 0 : offset, 4, &addr_hi, true);
291 0 : if (rc) {
292 0 : SPDK_ERRLOG("Failed to set avail addr high\n");
293 0 : goto err;
294 : }
295 :
296 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_USEDLO;
297 0 : addr_lo = (uint32_t)used_addr;
298 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
299 0 : offset, 4, &addr_lo, true);
300 0 : if (rc) {
301 0 : SPDK_ERRLOG("Failed to set used addr low\n");
302 0 : goto err;
303 : }
304 :
305 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_USEDHI;
306 0 : addr_hi = (uint32_t)(used_addr >> 32);
307 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
308 0 : offset, 4, &addr_hi, true);
309 0 : if (rc) {
310 0 : SPDK_ERRLOG("Failed to set used addr high\n");
311 0 : goto err;
312 : }
313 :
314 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_NOFF;
315 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
316 0 : offset, 2, ¬ify_off, false);
317 0 : if (rc) {
318 0 : SPDK_ERRLOG("Failed to get queue notify off\n");
319 0 : goto err;
320 : }
321 :
322 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_ENABLE;
323 0 : queue_enable = 1;
324 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
325 0 : offset, 2, &queue_enable, true);
326 0 : if (rc) {
327 0 : SPDK_ERRLOG("Failed to enable queue %u\n", vq->vq_queue_index);
328 0 : goto err;
329 : }
330 :
331 0 : SPDK_DEBUGLOG(virtio_vfio_user, "queue %"PRIu16" addresses:\n", vq->vq_queue_index);
332 0 : SPDK_DEBUGLOG(virtio_vfio_user, "\t desc_addr: %" PRIx64 "\n", desc_addr);
333 0 : SPDK_DEBUGLOG(virtio_vfio_user, "\t aval_addr: %" PRIx64 "\n", avail_addr);
334 0 : SPDK_DEBUGLOG(virtio_vfio_user, "\t used_addr: %" PRIx64 "\n", used_addr);
335 :
336 0 : return 0;
337 : err:
338 0 : spdk_free(queue_mem);
339 0 : return rc;
340 0 : }
341 :
342 : static void
343 0 : virtio_vfio_user_del_queue(struct virtio_dev *vdev, struct virtqueue *vq)
344 : {
345 0 : struct virtio_vfio_user_dev *dev = vdev->ctx;
346 0 : uint64_t offset;
347 0 : uint16_t queue_enable = 0;
348 0 : int rc;
349 :
350 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SELECT;
351 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
352 0 : offset, 2, &vq->vq_queue_index, true);
353 0 : if (rc) {
354 0 : SPDK_ERRLOG("Failed to select queue %u\n", vq->vq_queue_index);
355 0 : spdk_free(vq->vq_ring_virt_mem);
356 0 : return;
357 : }
358 :
359 0 : offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_ENABLE;
360 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region,
361 0 : offset, 2, &queue_enable, true);
362 0 : if (rc) {
363 0 : SPDK_ERRLOG("Failed to enable queue %u\n", vq->vq_queue_index);
364 0 : }
365 :
366 0 : spdk_free(vq->vq_ring_virt_mem);
367 : /* TODO: clear desc/avail/used address */
368 0 : }
369 :
370 : static void
371 0 : virtio_vfio_user_notify_queue(struct virtio_dev *vdev, struct virtqueue *vq)
372 : {
373 : /* we're running in polling mode, no need to write doorbells */
374 0 : }
375 :
376 : static const struct virtio_dev_ops virtio_vfio_user_ops = {
377 : .read_dev_cfg = virtio_vfio_user_read_dev_config,
378 : .write_dev_cfg = virtio_vfio_user_write_dev_config,
379 : .get_status = virtio_vfio_user_get_status,
380 : .set_status = virtio_vfio_user_set_status,
381 : .get_features = virtio_vfio_user_get_features,
382 : .set_features = virtio_vfio_user_set_features,
383 : .destruct_dev = virtio_vfio_user_destruct_dev,
384 : .get_queue_size = virtio_vfio_user_get_queue_size,
385 : .setup_queue = virtio_vfio_user_setup_queue,
386 : .del_queue = virtio_vfio_user_del_queue,
387 : .notify_queue = virtio_vfio_user_notify_queue
388 : };
389 :
390 : int
391 0 : virtio_vfio_user_dev_init(struct virtio_dev *vdev, const char *name, const char *path)
392 : {
393 0 : struct virtio_vfio_user_dev *dev;
394 0 : uint16_t cmd_reg;
395 0 : int rc;
396 :
397 0 : if (name == NULL) {
398 0 : SPDK_ERRLOG("No name given for controller: %s\n", path);
399 0 : return -EINVAL;
400 : }
401 :
402 0 : rc = access(path, F_OK);
403 0 : if (rc != 0) {
404 0 : SPDK_ERRLOG("Access path %s failed\n", path);
405 0 : return -EACCES;
406 : }
407 :
408 0 : dev = calloc(1, sizeof(*dev));
409 0 : if (dev == NULL) {
410 0 : return -ENOMEM;
411 : }
412 :
413 0 : rc = virtio_dev_construct(vdev, name, &virtio_vfio_user_ops, dev);
414 0 : if (rc != 0) {
415 0 : SPDK_ERRLOG("Failed to init device: %s\n", path);
416 0 : free(dev);
417 0 : return rc;
418 : }
419 :
420 0 : snprintf(dev->path, PATH_MAX, "%s", path);
421 0 : dev->ctx = spdk_vfio_user_setup(path);
422 0 : if (!dev->ctx) {
423 0 : SPDK_ERRLOG("Error to setup %s as vfio device\n", path);
424 0 : virtio_dev_destruct(vdev);
425 0 : return -EINVAL;
426 : }
427 :
428 : /* Enable PCI busmaster and disable INTx */
429 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, VFIO_PCI_CONFIG_REGION_INDEX, 4, 2,
430 : &cmd_reg, false);
431 0 : if (rc != 0) {
432 0 : SPDK_ERRLOG("Read PCI CMD REG failed\n");
433 0 : virtio_dev_destruct(vdev);
434 0 : return rc;
435 : }
436 0 : cmd_reg |= 0x404;
437 0 : rc = spdk_vfio_user_pci_bar_access(dev->ctx, VFIO_PCI_CONFIG_REGION_INDEX, 4, 2,
438 : &cmd_reg, true);
439 0 : if (rc != 0) {
440 0 : SPDK_ERRLOG("Write PCI CMD REG failed\n");
441 0 : virtio_dev_destruct(vdev);
442 0 : return rc;
443 : }
444 :
445 : /* TODO: we cat get virtio device PCI common space layout via
446 : * iterating vendor capabilities in PCI Configuration space,
447 : * while here we use hardcoded layout first, this feature can
448 : * be added in future.
449 : *
450 : * vfio-user emulated virtio device layout in Target:
451 : *
452 : * region 1: MSI-X Table
453 : * region 2: MSI-X PBA
454 : * region 4: virtio modern memory 64bits BAR
455 : * Common configuration 0x0 - 0x1000
456 : * ISR access 0x1000 - 0x2000
457 : * Device specific configuration 0x2000 - 0x3000
458 : * Notifications 0x3000 - 0x4000
459 : */
460 0 : dev->pci_cap_region = VFIO_PCI_BAR4_REGION_INDEX;
461 0 : dev->pci_cap_common_cfg_offset = 0x0;
462 0 : dev->pci_cap_common_cfg_length = 0x1000;
463 0 : dev->pci_cap_device_specific_offset = 0x2000;
464 0 : dev->pci_cap_device_specific_length = 0x1000;
465 0 : dev->pci_cap_notifications_offset = 0x3000;
466 0 : dev->pci_cap_notifications_length = 0x1000;
467 :
468 0 : return 0;
469 0 : }
470 :
471 0 : SPDK_LOG_REGISTER_COMPONENT(virtio_vfio_user)
|
