Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2019 Intel Corporation.
3 : * All rights reserved.
4 : */
5 :
6 : #include "vmd_internal.h"
7 :
8 : #include "spdk/stdinc.h"
9 : #include "spdk/string.h"
10 : #include "spdk/likely.h"
11 :
12 : static unsigned char *device_type[] = {
13 : "PCI Express Endpoint",
14 : "Legacy PCI Express Endpoint",
15 : "Reserved 1",
16 : "Reserved 2",
17 : "Root Port of PCI Express Root Complex",
18 : "Upstream Port of PCI Express Switch",
19 : "Downstream Port of PCI Express Switch",
20 : "PCI Express to PCI/PCI-X Bridge",
21 : "PCI/PCI-X to PCI Express Bridge",
22 : "Root Complex Integrated Endpoint",
23 : "Root Complex Event Collector",
24 : "Reserved Capability"
25 : };
26 :
27 : /*
28 : * Container for all VMD adapter probed in the system.
29 : */
30 : struct vmd_container {
31 : uint32_t count;
32 : struct vmd_adapter vmd[MAX_VMD_SUPPORTED];
33 : };
34 :
35 : static struct vmd_container g_vmd_container;
36 : static uint8_t g_end_device_count;
37 :
38 : static bool
39 0 : vmd_is_valid_cfg_addr(struct vmd_pci_bus *bus, uint64_t addr)
40 : {
41 0 : return addr >= (uint64_t)bus->vmd->cfg_vaddr &&
42 0 : addr < bus->vmd->cfgbar_size + (uint64_t)bus->vmd->cfg_vaddr;
43 : }
44 :
45 : static void
46 0 : vmd_align_base_addrs(struct vmd_adapter *vmd, uint32_t alignment)
47 : {
48 : uint32_t pad;
49 :
50 : /*
51 : * Device is not in hot plug path, align the base address remaining from membar 1.
52 : */
53 0 : if (vmd->physical_addr & (alignment - 1)) {
54 0 : pad = alignment - (vmd->physical_addr & (alignment - 1));
55 0 : vmd->physical_addr += pad;
56 0 : vmd->current_addr_size -= pad;
57 0 : }
58 0 : }
59 :
60 : static bool
61 0 : vmd_device_is_enumerated(volatile struct pci_header *header)
62 : {
63 0 : return header->one.prefetch_base_upper == VMD_UPPER_BASE_SIGNATURE &&
64 0 : header->one.prefetch_limit_upper == VMD_UPPER_LIMIT_SIGNATURE;
65 : }
66 :
67 : static bool
68 0 : vmd_device_is_root_port(volatile struct pci_header *header)
69 : {
70 0 : return header->common.vendor_id == SPDK_PCI_VID_INTEL &&
71 0 : (header->common.device_id == PCI_ROOT_PORT_A_INTEL_SKX ||
72 0 : header->common.device_id == PCI_ROOT_PORT_B_INTEL_SKX ||
73 0 : header->common.device_id == PCI_ROOT_PORT_C_INTEL_SKX ||
74 0 : header->common.device_id == PCI_ROOT_PORT_D_INTEL_SKX ||
75 0 : header->common.device_id == PCI_ROOT_PORT_A_INTEL_ICX ||
76 0 : header->common.device_id == PCI_ROOT_PORT_B_INTEL_ICX ||
77 0 : header->common.device_id == PCI_ROOT_PORT_C_INTEL_ICX ||
78 0 : header->common.device_id == PCI_ROOT_PORT_D_INTEL_ICX);
79 : }
80 :
81 : static void
82 0 : vmd_hotplug_coalesce_regions(struct vmd_hot_plug *hp)
83 : {
84 : struct pci_mem_mgr *region, *prev;
85 :
86 0 : do {
87 0 : prev = NULL;
88 0 : TAILQ_FOREACH(region, &hp->free_mem_queue, tailq) {
89 0 : if (prev != NULL && (prev->addr + prev->size == region->addr)) {
90 0 : break;
91 : }
92 :
93 0 : prev = region;
94 0 : }
95 :
96 0 : if (region != NULL) {
97 0 : prev->size += region->size;
98 0 : TAILQ_REMOVE(&hp->free_mem_queue, region, tailq);
99 0 : TAILQ_INSERT_TAIL(&hp->unused_mem_queue, region, tailq);
100 0 : }
101 0 : } while (region != NULL);
102 0 : }
103 :
104 : static void
105 0 : vmd_hotplug_free_region(struct vmd_hot_plug *hp, struct pci_mem_mgr *region)
106 : {
107 0 : struct pci_mem_mgr *current, *prev = NULL;
108 :
109 0 : assert(region->addr >= hp->bar.start && region->addr < hp->bar.start + hp->bar.size);
110 :
111 0 : TAILQ_FOREACH(current, &hp->free_mem_queue, tailq) {
112 0 : if (current->addr > region->addr) {
113 0 : break;
114 : }
115 :
116 0 : prev = current;
117 0 : }
118 :
119 0 : if (prev != NULL) {
120 0 : assert(prev->addr + prev->size <= region->addr);
121 0 : assert(current == NULL || (region->addr + region->size <= current->addr));
122 0 : TAILQ_INSERT_AFTER(&hp->free_mem_queue, prev, region, tailq);
123 0 : } else {
124 0 : TAILQ_INSERT_HEAD(&hp->free_mem_queue, region, tailq);
125 : }
126 :
127 0 : vmd_hotplug_coalesce_regions(hp);
128 0 : }
129 :
130 : static void
131 0 : vmd_hotplug_free_addr(struct vmd_hot_plug *hp, uint64_t addr)
132 : {
133 : struct pci_mem_mgr *region;
134 :
135 0 : TAILQ_FOREACH(region, &hp->alloc_mem_queue, tailq) {
136 0 : if (region->addr == addr) {
137 0 : break;
138 : }
139 0 : }
140 :
141 0 : assert(region != NULL);
142 0 : TAILQ_REMOVE(&hp->alloc_mem_queue, region, tailq);
143 :
144 0 : vmd_hotplug_free_region(hp, region);
145 0 : }
146 :
147 : static uint64_t
148 0 : vmd_hotplug_allocate_base_addr(struct vmd_hot_plug *hp, uint32_t size)
149 : {
150 0 : struct pci_mem_mgr *region = NULL, *free_region;
151 :
152 0 : TAILQ_FOREACH(region, &hp->free_mem_queue, tailq) {
153 0 : if (region->size >= size) {
154 0 : break;
155 : }
156 0 : }
157 :
158 0 : if (region == NULL) {
159 0 : SPDK_INFOLOG(vmd, "Unable to find free hotplug memory region of size:"
160 : "%"PRIx32"\n", size);
161 0 : return 0;
162 : }
163 :
164 0 : TAILQ_REMOVE(&hp->free_mem_queue, region, tailq);
165 0 : if (size < region->size) {
166 0 : free_region = TAILQ_FIRST(&hp->unused_mem_queue);
167 0 : if (free_region == NULL) {
168 0 : SPDK_INFOLOG(vmd, "Unable to find unused descriptor to store the "
169 : "free region of size: %"PRIu32"\n", region->size - size);
170 0 : } else {
171 0 : TAILQ_REMOVE(&hp->unused_mem_queue, free_region, tailq);
172 0 : free_region->size = region->size - size;
173 0 : free_region->addr = region->addr + size;
174 0 : region->size = size;
175 0 : vmd_hotplug_free_region(hp, free_region);
176 : }
177 0 : }
178 :
179 0 : TAILQ_INSERT_TAIL(&hp->alloc_mem_queue, region, tailq);
180 :
181 0 : return region->addr;
182 0 : }
183 :
184 : /*
185 : * Allocates an address from vmd membar for the input memory size
186 : * vmdAdapter - vmd adapter object
187 : * dev - vmd_pci_device to allocate a base address for.
188 : * size - size of the memory window requested.
189 : * Size must be an integral multiple of 2. Addresses are returned on the size boundary.
190 : * Returns physical address within the VMD membar window, or 0x0 if cannot allocate window.
191 : * Consider increasing the size of vmd membar if 0x0 is returned.
192 : */
193 : static uint64_t
194 0 : vmd_allocate_base_addr(struct vmd_adapter *vmd, struct vmd_pci_device *dev, uint32_t size)
195 : {
196 0 : uint64_t base_address = 0, padding = 0;
197 : struct vmd_pci_bus *hp_bus;
198 :
199 0 : if (size && ((size & (~size + 1)) != size)) {
200 0 : return base_address;
201 : }
202 :
203 : /*
204 : * If device is downstream of a hot plug port, allocate address from the
205 : * range dedicated for the hot plug slot. Search the list of addresses allocated to determine
206 : * if a free range exists that satisfy the input request. If a free range cannot be found,
207 : * get a buffer from the unused chunk. First fit algorithm, is used.
208 : */
209 0 : if (dev) {
210 0 : hp_bus = dev->parent;
211 0 : if (hp_bus && hp_bus->self && hp_bus->self->hotplug_capable) {
212 0 : return vmd_hotplug_allocate_base_addr(&hp_bus->self->hp, size);
213 : }
214 0 : }
215 :
216 : /* Ensure physical membar allocated is size aligned */
217 0 : if (vmd->physical_addr & (size - 1)) {
218 0 : padding = size - (vmd->physical_addr & (size - 1));
219 0 : }
220 :
221 : /* Allocate from membar if enough memory is left */
222 0 : if (vmd->current_addr_size >= size + padding) {
223 0 : base_address = vmd->physical_addr + padding;
224 0 : vmd->physical_addr += size + padding;
225 0 : vmd->current_addr_size -= size + padding;
226 0 : }
227 :
228 0 : SPDK_INFOLOG(vmd, "allocated(size) %" PRIx64 " (%x)\n", base_address, size);
229 :
230 0 : return base_address;
231 0 : }
232 :
233 : static bool
234 0 : vmd_is_end_device(struct vmd_pci_device *dev)
235 : {
236 0 : return (dev && dev->header) &&
237 0 : ((dev->header->common.header_type & ~PCI_MULTI_FUNCTION) == PCI_HEADER_TYPE_NORMAL);
238 : }
239 :
240 : static void
241 0 : vmd_update_base_limit_register(struct vmd_pci_device *dev, uint16_t base, uint16_t limit)
242 : {
243 : struct vmd_pci_bus *bus;
244 : struct vmd_pci_device *bridge;
245 :
246 0 : if (base == 0 || limit == 0) {
247 0 : return;
248 : }
249 :
250 0 : if (dev->header->common.header_type == PCI_HEADER_TYPE_BRIDGE) {
251 0 : bus = dev->bus_object;
252 0 : } else {
253 0 : bus = dev->parent;
254 : }
255 :
256 0 : bridge = bus->self;
257 0 : SPDK_INFOLOG(vmd, "base:limit = %x:%x\n", bridge->header->one.mem_base,
258 : bridge->header->one.mem_limit);
259 :
260 0 : if (dev->bus->vmd->scan_completed) {
261 0 : return;
262 : }
263 :
264 0 : while (bus && bus->self != NULL) {
265 0 : bridge = bus->self;
266 :
267 : /* This is only for 32-bit memory space, need to revisit to support 64-bit */
268 0 : if (bridge->header->one.mem_base > base) {
269 0 : bridge->header->one.mem_base = base;
270 0 : base = bridge->header->one.mem_base;
271 0 : }
272 :
273 0 : if (bridge->header->one.mem_limit < limit) {
274 0 : bridge->header->one.mem_limit = limit;
275 0 : limit = bridge->header->one.mem_limit;
276 0 : }
277 :
278 0 : bus = bus->parent;
279 : }
280 0 : }
281 :
282 : static uint64_t
283 0 : vmd_get_base_addr(struct vmd_pci_device *dev, uint32_t index, uint32_t size)
284 : {
285 0 : struct vmd_pci_bus *bus = dev->parent;
286 :
287 0 : if (dev->header_type == PCI_HEADER_TYPE_BRIDGE) {
288 0 : return dev->header->zero.BAR[index] & ~0xf;
289 : } else {
290 0 : if (bus->self->hotplug_capable) {
291 0 : return vmd_hotplug_allocate_base_addr(&bus->self->hp, size);
292 : } else {
293 0 : return (uint64_t)bus->self->header->one.mem_base << 16;
294 : }
295 : }
296 0 : }
297 :
298 : static bool
299 0 : vmd_assign_base_addrs(struct vmd_pci_device *dev)
300 : {
301 0 : uint16_t mem_base = 0, mem_limit = 0;
302 0 : unsigned char mem_attr = 0;
303 : int last;
304 0 : struct vmd_adapter *vmd = NULL;
305 0 : bool ret_val = false;
306 : uint32_t bar_value;
307 : uint32_t table_offset;
308 :
309 0 : if (dev && dev->bus) {
310 0 : vmd = dev->bus->vmd;
311 0 : }
312 :
313 0 : if (!vmd) {
314 0 : return 0;
315 : }
316 :
317 0 : vmd_align_base_addrs(vmd, ONE_MB);
318 :
319 0 : last = dev->header_type ? 2 : 6;
320 0 : for (int i = 0; i < last; i++) {
321 0 : bar_value = dev->header->zero.BAR[i];
322 0 : dev->header->zero.BAR[i] = ~(0U);
323 0 : dev->bar[i].size = dev->header->zero.BAR[i];
324 0 : dev->header->zero.BAR[i] = bar_value;
325 :
326 0 : if (dev->bar[i].size == ~(0U) || dev->bar[i].size == 0 ||
327 0 : dev->header->zero.BAR[i] & 1) {
328 0 : dev->bar[i].size = 0;
329 0 : continue;
330 : }
331 0 : mem_attr = dev->bar[i].size & PCI_BASE_ADDR_MASK;
332 0 : dev->bar[i].size = TWOS_COMPLEMENT(dev->bar[i].size & PCI_BASE_ADDR_MASK);
333 :
334 0 : if (vmd->scan_completed) {
335 0 : dev->bar[i].start = vmd_get_base_addr(dev, i, dev->bar[i].size);
336 0 : } else {
337 0 : dev->bar[i].start = vmd_allocate_base_addr(vmd, dev, dev->bar[i].size);
338 : }
339 :
340 0 : dev->header->zero.BAR[i] = (uint32_t)dev->bar[i].start;
341 :
342 0 : if (!dev->bar[i].start) {
343 0 : if (mem_attr == (PCI_BAR_MEMORY_PREFETCH | PCI_BAR_MEMORY_TYPE_64)) {
344 0 : i++;
345 0 : }
346 0 : continue;
347 : }
348 :
349 0 : dev->bar[i].vaddr = ((uint64_t)vmd->mem_vaddr + (dev->bar[i].start - vmd->membar));
350 0 : mem_limit = BRIDGE_BASEREG(dev->header->zero.BAR[i]) +
351 0 : BRIDGE_BASEREG(dev->bar[i].size - 1);
352 0 : if (!mem_base) {
353 0 : mem_base = BRIDGE_BASEREG(dev->header->zero.BAR[i]);
354 0 : }
355 :
356 0 : ret_val = true;
357 :
358 0 : if (mem_attr == (PCI_BAR_MEMORY_PREFETCH | PCI_BAR_MEMORY_TYPE_64)) {
359 0 : i++;
360 0 : if (i < last) {
361 0 : dev->header->zero.BAR[i] = (uint32_t)(dev->bar[i].start >> PCI_DWORD_SHIFT);
362 0 : }
363 0 : }
364 0 : }
365 :
366 : /* Enable device MEM and bus mastering */
367 0 : dev->header->zero.command |= (PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
368 : /*
369 : * Writes to the pci config space is posted write. To ensure transaction reaches its destination
370 : * before another write is posed, an immediate read of the written value should be performed.
371 : */
372 0 : { uint16_t cmd = dev->header->zero.command; (void)cmd; }
373 :
374 0 : if (dev->msix_cap && ret_val) {
375 0 : table_offset = ((volatile struct pci_msix_cap *)dev->msix_cap)->msix_table_offset;
376 0 : if (dev->bar[table_offset & 0x3].vaddr) {
377 0 : dev->msix_table = (volatile struct pci_msix_table_entry *)
378 0 : (dev->bar[table_offset & 0x3].vaddr + (table_offset & 0xfff8));
379 0 : }
380 0 : }
381 :
382 0 : if (ret_val && vmd_is_end_device(dev)) {
383 0 : vmd_update_base_limit_register(dev, mem_base, mem_limit);
384 0 : }
385 :
386 0 : return ret_val;
387 0 : }
388 :
389 : static void
390 0 : vmd_get_device_capabilities(struct vmd_pci_device *dev)
391 :
392 : {
393 : volatile uint8_t *config_space;
394 : uint8_t capabilities_offset;
395 : struct pci_capabilities_header *capabilities_hdr;
396 :
397 0 : config_space = (volatile uint8_t *)dev->header;
398 0 : if ((dev->header->common.status & PCI_CAPABILITIES_LIST) == 0) {
399 0 : return;
400 : }
401 :
402 0 : capabilities_offset = dev->header->zero.cap_pointer;
403 0 : if (dev->header->common.header_type & PCI_HEADER_TYPE_BRIDGE) {
404 0 : capabilities_offset = dev->header->one.cap_pointer;
405 0 : }
406 :
407 0 : while (capabilities_offset > 0) {
408 0 : capabilities_hdr = (struct pci_capabilities_header *)
409 0 : &config_space[capabilities_offset];
410 0 : switch (capabilities_hdr->capability_id) {
411 : case CAPABILITY_ID_PCI_EXPRESS:
412 0 : dev->pcie_cap = (volatile struct pci_express_cap *)(capabilities_hdr);
413 0 : break;
414 :
415 : case CAPABILITY_ID_MSI:
416 0 : dev->msi_cap = (volatile struct pci_msi_cap *)capabilities_hdr;
417 0 : break;
418 :
419 : case CAPABILITY_ID_MSIX:
420 0 : dev->msix_cap = (volatile struct pci_msix_capability *)capabilities_hdr;
421 0 : dev->msix_table_size = dev->msix_cap->message_control.bit.table_size + 1;
422 0 : break;
423 :
424 : default:
425 0 : break;
426 : }
427 0 : capabilities_offset = capabilities_hdr->next;
428 : }
429 0 : }
430 :
431 : static volatile struct pci_enhanced_capability_header *
432 0 : vmd_get_enhanced_capabilities(struct vmd_pci_device *dev, uint16_t capability_id)
433 : {
434 : uint8_t *data;
435 0 : uint16_t cap_offset = EXTENDED_CAPABILITY_OFFSET;
436 0 : volatile struct pci_enhanced_capability_header *cap_hdr = NULL;
437 :
438 0 : data = (uint8_t *)dev->header;
439 0 : while (cap_offset >= EXTENDED_CAPABILITY_OFFSET) {
440 0 : cap_hdr = (volatile struct pci_enhanced_capability_header *) &data[cap_offset];
441 0 : if (cap_hdr->capability_id == capability_id) {
442 0 : return cap_hdr;
443 : }
444 0 : cap_offset = cap_hdr->next;
445 0 : if (cap_offset == 0 || cap_offset < EXTENDED_CAPABILITY_OFFSET) {
446 0 : break;
447 : }
448 : }
449 :
450 0 : return NULL;
451 0 : }
452 :
453 : static void
454 0 : vmd_read_config_space(struct vmd_pci_device *dev)
455 : {
456 : /*
457 : * Writes to the pci config space is posted weite. To ensure transaction reaches its destination
458 : * before another write is posed, an immediate read of the written value should be performed.
459 : */
460 0 : dev->header->common.command |= (BUS_MASTER_ENABLE | MEMORY_SPACE_ENABLE);
461 0 : { uint16_t cmd = dev->header->common.command; (void)cmd; }
462 :
463 0 : vmd_get_device_capabilities(dev);
464 0 : dev->sn_cap = (struct serial_number_capability *)vmd_get_enhanced_capabilities(dev,
465 : DEVICE_SERIAL_NUMBER_CAP_ID);
466 0 : }
467 :
468 : static void
469 0 : vmd_update_scan_info(struct vmd_pci_device *dev)
470 : {
471 0 : struct vmd_adapter *vmd_adapter = dev->bus->vmd;
472 :
473 0 : if (vmd_adapter->root_port_updated) {
474 0 : return;
475 : }
476 :
477 0 : if (dev->header_type == PCI_HEADER_TYPE_NORMAL) {
478 0 : return;
479 : }
480 :
481 0 : if (vmd_device_is_root_port(dev->header)) {
482 0 : vmd_adapter->root_port_updated = 1;
483 0 : SPDK_INFOLOG(vmd, "root_port_updated = %d\n",
484 : vmd_adapter->root_port_updated);
485 0 : SPDK_INFOLOG(vmd, "upper:limit = %x : %x\n",
486 : dev->header->one.prefetch_base_upper,
487 : dev->header->one.prefetch_limit_upper);
488 0 : if (vmd_device_is_enumerated(dev->header)) {
489 0 : vmd_adapter->scan_completed = 1;
490 0 : SPDK_INFOLOG(vmd, "scan_completed = %d\n",
491 : vmd_adapter->scan_completed);
492 0 : }
493 0 : }
494 0 : }
495 :
496 : static void
497 0 : vmd_reset_base_limit_registers(volatile struct pci_header *header)
498 : {
499 : uint32_t reg __attribute__((unused));
500 :
501 : /*
502 : * Writes to the pci config space are posted writes.
503 : * To ensure transaction reaches its destination
504 : * before another write is posted, an immediate read
505 : * of the written value should be performed.
506 : */
507 0 : header->one.mem_base = 0xfff0;
508 0 : reg = header->one.mem_base;
509 0 : header->one.mem_limit = 0x0;
510 0 : reg = header->one.mem_limit;
511 0 : header->one.prefetch_base = 0x0;
512 0 : reg = header->one.prefetch_base;
513 0 : header->one.prefetch_limit = 0x0;
514 0 : reg = header->one.prefetch_limit;
515 0 : header->one.prefetch_base_upper = 0x0;
516 0 : reg = header->one.prefetch_base_upper;
517 0 : header->one.prefetch_limit_upper = 0x0;
518 0 : reg = header->one.prefetch_limit_upper;
519 0 : header->one.io_base_upper = 0x0;
520 0 : reg = header->one.io_base_upper;
521 0 : header->one.io_limit_upper = 0x0;
522 0 : reg = header->one.io_limit_upper;
523 0 : header->one.primary = 0;
524 0 : reg = header->one.primary;
525 0 : header->one.secondary = 0;
526 0 : reg = header->one.secondary;
527 0 : header->one.subordinate = 0;
528 0 : reg = header->one.subordinate;
529 0 : }
530 :
531 : static void
532 0 : vmd_init_hotplug(struct vmd_pci_device *dev, struct vmd_pci_bus *bus)
533 : {
534 0 : struct vmd_adapter *vmd = bus->vmd;
535 0 : struct vmd_hot_plug *hp = &dev->hp;
536 : size_t mem_id;
537 :
538 0 : dev->hotplug_capable = true;
539 0 : hp->bar.size = 1 << 20;
540 :
541 0 : if (!vmd->scan_completed) {
542 0 : hp->bar.start = vmd_allocate_base_addr(vmd, NULL, hp->bar.size);
543 0 : bus->self->header->one.mem_base = BRIDGE_BASEREG(hp->bar.start);
544 0 : bus->self->header->one.mem_limit =
545 0 : bus->self->header->one.mem_base + BRIDGE_BASEREG(hp->bar.size - 1);
546 0 : } else {
547 0 : hp->bar.start = (uint64_t)bus->self->header->one.mem_base << 16;
548 : }
549 :
550 0 : hp->bar.vaddr = (uint64_t)vmd->mem_vaddr + (hp->bar.start - vmd->membar);
551 :
552 0 : TAILQ_INIT(&hp->free_mem_queue);
553 0 : TAILQ_INIT(&hp->unused_mem_queue);
554 0 : TAILQ_INIT(&hp->alloc_mem_queue);
555 :
556 0 : hp->mem[0].size = hp->bar.size;
557 0 : hp->mem[0].addr = hp->bar.start;
558 :
559 0 : TAILQ_INSERT_TAIL(&hp->free_mem_queue, &hp->mem[0], tailq);
560 :
561 0 : for (mem_id = 1; mem_id < ADDR_ELEM_COUNT; ++mem_id) {
562 0 : TAILQ_INSERT_TAIL(&hp->unused_mem_queue, &hp->mem[mem_id], tailq);
563 0 : }
564 :
565 0 : SPDK_INFOLOG(vmd, "%s: mem_base:mem_limit = %x : %x\n", __func__,
566 : bus->self->header->one.mem_base, bus->self->header->one.mem_limit);
567 0 : }
568 :
569 : static bool
570 0 : vmd_bus_device_present(struct vmd_pci_bus *bus, uint32_t devfn)
571 : {
572 : volatile struct pci_header *header;
573 :
574 0 : header = (volatile struct pci_header *)(bus->vmd->cfg_vaddr +
575 0 : CONFIG_OFFSET_ADDR(bus->config_bus_number, devfn, 0, 0));
576 0 : if (!vmd_is_valid_cfg_addr(bus, (uint64_t)header)) {
577 0 : return false;
578 : }
579 :
580 0 : if (header->common.vendor_id == PCI_INVALID_VENDORID || header->common.vendor_id == 0x0) {
581 0 : return false;
582 : }
583 :
584 0 : return true;
585 0 : }
586 :
587 : static struct vmd_pci_device *
588 0 : vmd_alloc_dev(struct vmd_pci_bus *bus, uint32_t devfn)
589 : {
590 0 : struct vmd_pci_device *dev = NULL;
591 : struct pci_header volatile *header;
592 : uint8_t header_type;
593 : uint32_t rev_class;
594 :
595 : /* Make sure we're not creating two devices on the same dev/fn */
596 0 : TAILQ_FOREACH(dev, &bus->dev_list, tailq) {
597 0 : if (dev->devfn == devfn) {
598 0 : return NULL;
599 : }
600 0 : }
601 :
602 0 : if (!vmd_bus_device_present(bus, devfn)) {
603 0 : return NULL;
604 : }
605 :
606 0 : header = (struct pci_header * volatile)(bus->vmd->cfg_vaddr +
607 0 : CONFIG_OFFSET_ADDR(bus->config_bus_number, devfn, 0, 0));
608 :
609 0 : SPDK_INFOLOG(vmd, "PCI device found: %04x:%04x ***\n",
610 : header->common.vendor_id, header->common.device_id);
611 :
612 0 : dev = calloc(1, sizeof(*dev));
613 0 : if (!dev) {
614 0 : return NULL;
615 : }
616 :
617 0 : dev->header = header;
618 0 : dev->vid = dev->header->common.vendor_id;
619 0 : dev->did = dev->header->common.device_id;
620 0 : dev->bus = bus;
621 0 : dev->parent = bus;
622 0 : dev->devfn = devfn;
623 0 : header_type = dev->header->common.header_type;
624 0 : rev_class = dev->header->common.rev_class;
625 0 : dev->class = rev_class >> 8;
626 0 : dev->header_type = header_type & 0x7;
627 :
628 0 : if (header_type == PCI_HEADER_TYPE_BRIDGE) {
629 0 : vmd_update_scan_info(dev);
630 0 : if (!dev->bus->vmd->scan_completed) {
631 0 : vmd_reset_base_limit_registers(dev->header);
632 0 : }
633 0 : }
634 :
635 0 : vmd_read_config_space(dev);
636 :
637 0 : return dev;
638 0 : }
639 :
640 : static struct vmd_pci_bus *
641 0 : vmd_create_new_bus(struct vmd_pci_bus *parent, struct vmd_pci_device *bridge, uint8_t bus_number)
642 : {
643 : struct vmd_pci_bus *new_bus;
644 :
645 0 : new_bus = calloc(1, sizeof(*new_bus));
646 0 : if (!new_bus) {
647 0 : return NULL;
648 : }
649 :
650 0 : new_bus->parent = parent;
651 0 : new_bus->domain = parent->domain;
652 0 : new_bus->bus_number = bus_number;
653 0 : new_bus->secondary_bus = new_bus->subordinate_bus = bus_number;
654 0 : new_bus->self = bridge;
655 0 : new_bus->vmd = parent->vmd;
656 0 : new_bus->config_bus_number = new_bus->bus_number - new_bus->vmd->vmd_bus.bus_start;
657 0 : TAILQ_INIT(&new_bus->dev_list);
658 :
659 0 : bridge->subordinate = new_bus;
660 :
661 0 : bridge->pci.addr.bus = new_bus->bus_number;
662 0 : bridge->pci.addr.dev = bridge->devfn;
663 0 : bridge->pci.addr.func = 0;
664 0 : bridge->pci.addr.domain = parent->vmd->pci->addr.domain;
665 :
666 0 : return new_bus;
667 0 : }
668 :
669 : static uint8_t
670 0 : vmd_get_next_bus_number(struct vmd_adapter *vmd)
671 : {
672 0 : uint8_t bus = 0xff;
673 :
674 0 : if ((vmd->next_bus_number + 1) < vmd->max_pci_bus) {
675 0 : bus = vmd->next_bus_number;
676 0 : vmd->next_bus_number++;
677 0 : }
678 :
679 0 : return bus;
680 : }
681 :
682 : static uint8_t
683 0 : vmd_get_hotplug_bus_numbers(struct vmd_pci_device *dev)
684 : {
685 0 : uint8_t bus_number = 0xff;
686 :
687 0 : if (dev && dev->bus && dev->bus->vmd &&
688 0 : ((dev->bus->vmd->next_bus_number + RESERVED_HOTPLUG_BUSES) < dev->bus->vmd->max_pci_bus)) {
689 0 : bus_number = RESERVED_HOTPLUG_BUSES;
690 0 : dev->bus->vmd->next_bus_number += RESERVED_HOTPLUG_BUSES;
691 0 : }
692 :
693 0 : return bus_number;
694 : }
695 :
696 : static void
697 0 : vmd_enable_msix(struct vmd_pci_device *dev)
698 : {
699 : volatile uint16_t control;
700 :
701 0 : control = dev->msix_cap->message_control.as_uint16_t | (1 << 14);
702 0 : dev->msix_cap->message_control.as_uint16_t = control;
703 0 : control = dev->msix_cap->message_control.as_uint16_t;
704 0 : dev->msix_cap->message_control.as_uint16_t = (control | (1 << 15));
705 0 : control = dev->msix_cap->message_control.as_uint16_t;
706 0 : control = control & ~(1 << 14);
707 0 : dev->msix_cap->message_control.as_uint16_t = control;
708 0 : control = dev->msix_cap->message_control.as_uint16_t;
709 0 : }
710 :
711 : static void
712 0 : vmd_disable_msix(struct vmd_pci_device *dev)
713 : {
714 : volatile uint16_t control;
715 :
716 0 : control = dev->msix_cap->message_control.as_uint16_t | (1 << 14);
717 0 : dev->msix_cap->message_control.as_uint16_t = control;
718 0 : control = dev->msix_cap->message_control.as_uint16_t & ~(1 << 15);
719 0 : dev->msix_cap->message_control.as_uint16_t = control;
720 0 : control = dev->msix_cap->message_control.as_uint16_t;
721 0 : }
722 :
723 : /*
724 : * Set up MSI-X table entries for the port. Vmd MSIX vector 0 is used for
725 : * port interrupt, so vector 0 is mapped to all MSIX entries for the port.
726 : */
727 : static void
728 0 : vmd_setup_msix(struct vmd_pci_device *dev, volatile struct pci_msix_table_entry *vmdEntry)
729 : {
730 : int entry;
731 :
732 0 : if (!dev || !vmdEntry || !dev->msix_cap) {
733 0 : return;
734 : }
735 :
736 0 : vmd_disable_msix(dev);
737 0 : if (dev->msix_table == NULL || dev->msix_table_size > MAX_MSIX_TABLE_SIZE) {
738 0 : return;
739 : }
740 :
741 0 : for (entry = 0; entry < dev->msix_table_size; ++entry) {
742 0 : dev->msix_table[entry].vector_control = 1;
743 0 : }
744 0 : vmd_enable_msix(dev);
745 0 : }
746 :
747 : static void
748 0 : vmd_bus_update_bridge_info(struct vmd_pci_device *bridge)
749 : {
750 : /* Update the subordinate bus of all bridges above this bridge */
751 0 : volatile struct vmd_pci_device *dev = bridge;
752 : uint8_t subordinate_bus;
753 :
754 0 : if (!dev) {
755 0 : return;
756 : }
757 0 : subordinate_bus = bridge->header->one.subordinate;
758 0 : while (dev->parent_bridge != NULL) {
759 0 : dev = dev->parent_bridge;
760 0 : if (dev->header->one.subordinate < subordinate_bus) {
761 0 : dev->header->one.subordinate = subordinate_bus;
762 0 : subordinate_bus = dev->header->one.subordinate;
763 0 : }
764 : }
765 0 : }
766 :
767 : static bool
768 0 : vmd_is_supported_device(struct vmd_pci_device *dev)
769 : {
770 0 : return dev->class == PCI_CLASS_STORAGE_EXPRESS;
771 : }
772 :
773 : static int
774 0 : vmd_dev_map_bar(struct spdk_pci_device *pci_dev, uint32_t bar,
775 : void **mapped_addr, uint64_t *phys_addr, uint64_t *size)
776 : {
777 0 : struct vmd_pci_device *dev = SPDK_CONTAINEROF(pci_dev, struct vmd_pci_device, pci);
778 :
779 0 : *size = dev->bar[bar].size;
780 0 : *phys_addr = dev->bar[bar].start;
781 0 : *mapped_addr = (void *)dev->bar[bar].vaddr;
782 :
783 0 : return 0;
784 : }
785 :
786 : static int
787 0 : vmd_dev_unmap_bar(struct spdk_pci_device *_dev, uint32_t bar, void *addr)
788 : {
789 0 : return 0;
790 : }
791 :
792 : static int
793 0 : vmd_dev_cfg_read(struct spdk_pci_device *_dev, void *value, uint32_t len,
794 : uint32_t offset)
795 : {
796 0 : struct vmd_pci_device *dev = SPDK_CONTAINEROF(_dev, struct vmd_pci_device, pci);
797 0 : volatile uint8_t *src = (volatile uint8_t *)dev->header;
798 0 : uint8_t *dst = value;
799 : size_t i;
800 :
801 0 : if (len + offset > PCI_MAX_CFG_SIZE) {
802 0 : return -1;
803 : }
804 :
805 0 : for (i = 0; i < len; ++i) {
806 0 : dst[i] = src[offset + i];
807 0 : }
808 :
809 0 : return 0;
810 0 : }
811 :
812 : static int
813 0 : vmd_dev_cfg_write(struct spdk_pci_device *_dev, void *value,
814 : uint32_t len, uint32_t offset)
815 : {
816 0 : struct vmd_pci_device *dev = SPDK_CONTAINEROF(_dev, struct vmd_pci_device, pci);
817 0 : volatile uint8_t *dst = (volatile uint8_t *)dev->header;
818 0 : uint8_t *src = value;
819 : size_t i;
820 :
821 0 : if ((len + offset) > PCI_MAX_CFG_SIZE) {
822 0 : return -1;
823 : }
824 :
825 0 : for (i = 0; i < len; ++i) {
826 0 : dst[offset + i] = src[i];
827 0 : }
828 :
829 0 : return 0;
830 0 : }
831 :
832 : static void
833 0 : vmd_dev_free(struct vmd_pci_device *dev)
834 : {
835 0 : struct vmd_pci_device *bus_device = dev->bus->self;
836 0 : size_t i, num_bars = dev->header_type ? 2 : 6;
837 :
838 : /* Release the hotplug region if the device is under hotplug-capable bus */
839 0 : if (bus_device && bus_device->hotplug_capable) {
840 0 : for (i = 0; i < num_bars; ++i) {
841 0 : if (dev->bar[i].start != 0) {
842 0 : vmd_hotplug_free_addr(&bus_device->hp, dev->bar[i].start);
843 0 : }
844 0 : }
845 0 : }
846 :
847 0 : free(dev);
848 0 : }
849 :
850 : static void
851 0 : vmd_dev_detach(struct spdk_pci_device *dev)
852 : {
853 0 : struct vmd_pci_device *vmd_device = (struct vmd_pci_device *)dev;
854 0 : struct vmd_pci_bus *bus = vmd_device->bus;
855 :
856 0 : spdk_pci_unhook_device(dev);
857 0 : TAILQ_REMOVE(&bus->dev_list, vmd_device, tailq);
858 :
859 0 : vmd_dev_free(vmd_device);
860 0 : }
861 :
862 : static void
863 0 : vmd_dev_init(struct vmd_pci_device *dev)
864 : {
865 0 : dev->pci.addr.domain = dev->bus->vmd->domain;
866 0 : dev->pci.addr.bus = dev->bus->bus_number;
867 0 : dev->pci.addr.dev = dev->devfn;
868 0 : dev->pci.addr.func = 0;
869 0 : dev->pci.numa_id = spdk_pci_device_get_numa_id(dev->bus->vmd->pci);
870 0 : dev->pci.id.vendor_id = dev->header->common.vendor_id;
871 0 : dev->pci.id.device_id = dev->header->common.device_id;
872 0 : dev->pci.type = "vmd";
873 0 : dev->pci.map_bar = vmd_dev_map_bar;
874 0 : dev->pci.unmap_bar = vmd_dev_unmap_bar;
875 0 : dev->pci.cfg_read = vmd_dev_cfg_read;
876 0 : dev->pci.cfg_write = vmd_dev_cfg_write;
877 0 : dev->hotplug_capable = false;
878 0 : if (dev->pcie_cap != NULL) {
879 0 : dev->cached_slot_control = dev->pcie_cap->slot_control;
880 0 : }
881 0 : }
882 :
883 : static int
884 0 : vmd_init_end_device(struct vmd_pci_device *dev)
885 : {
886 0 : struct vmd_pci_bus *bus = dev->bus;
887 : struct vmd_adapter *vmd;
888 : struct spdk_pci_driver *driver;
889 : uint8_t bdf[32];
890 : int rc;
891 :
892 0 : if (!vmd_assign_base_addrs(dev)) {
893 0 : SPDK_ERRLOG("Failed to allocate BARs for device: %p\n", dev);
894 0 : return -1;
895 : }
896 :
897 0 : vmd_setup_msix(dev, &bus->vmd->msix_table[0]);
898 0 : vmd_dev_init(dev);
899 :
900 0 : if (vmd_is_supported_device(dev)) {
901 0 : spdk_pci_addr_fmt(bdf, sizeof(bdf), &dev->pci.addr);
902 0 : SPDK_INFOLOG(vmd, "Initializing NVMe device at %s\n", bdf);
903 0 : dev->pci.parent = dev->bus->vmd->pci;
904 :
905 0 : driver = spdk_pci_nvme_get_driver();
906 0 : assert(driver != NULL);
907 0 : rc = spdk_pci_hook_device(driver, &dev->pci);
908 0 : if (rc != 0) {
909 0 : SPDK_ERRLOG("Failed to hook device %s: %s\n", bdf, spdk_strerror(-rc));
910 0 : return -1;
911 : }
912 :
913 0 : vmd = bus->vmd;
914 0 : vmd->target[vmd->nvme_count] = dev;
915 0 : vmd->nvme_count++;
916 0 : }
917 :
918 : /* Attach the device to the current bus and assign base addresses */
919 0 : TAILQ_INSERT_TAIL(&bus->dev_list, dev, tailq);
920 0 : g_end_device_count++;
921 :
922 0 : return 0;
923 0 : }
924 :
925 : /*
926 : * Scans a single bus for all devices attached and return a count of
927 : * how many devices found. In the VMD topology, it is assume there are no multi-
928 : * function devices. Hence a bus(bridge) will not have multi function with both type
929 : * 0 and 1 header.
930 : *
931 : * The other option for implementing this function is the bus is an int and
932 : * create a new device PciBridge. PciBridge would inherit from PciDevice with extra fields,
933 : * sub/pri/sec bus. The input becomes PciPort, bus number and parent_bridge.
934 : *
935 : * The bus number is scanned and if a device is found, based on the header_type, create
936 : * either PciBridge(1) or PciDevice(0).
937 : *
938 : * If a PciBridge, assign bus numbers and rescan new bus. The currently PciBridge being
939 : * scanned becomes the passed in parent_bridge with the new bus number.
940 : *
941 : * The linked list becomes list of pciBridges with PciDevices attached.
942 : *
943 : * Return count of how many devices found(type1 + type 0 header devices)
944 : */
945 : static uint8_t
946 0 : vmd_scan_single_bus(struct vmd_pci_bus *bus, struct vmd_pci_device *parent_bridge, bool hotplug)
947 : {
948 : /* assuming only single function devices are on the bus */
949 : struct vmd_pci_device *new_dev;
950 : union express_slot_capabilities_register slot_cap;
951 : struct vmd_pci_bus *new_bus;
952 0 : uint8_t device_number, dev_cnt = 0;
953 : uint8_t new_bus_num;
954 : int rc;
955 :
956 0 : for (device_number = 0; device_number < 32; device_number++) {
957 0 : new_dev = vmd_alloc_dev(bus, device_number);
958 0 : if (new_dev == NULL) {
959 0 : continue;
960 : }
961 :
962 0 : if (new_dev->header->common.header_type & PCI_HEADER_TYPE_BRIDGE) {
963 0 : if (hotplug) {
964 0 : free(new_dev);
965 0 : continue;
966 : }
967 :
968 0 : slot_cap.as_uint32_t = 0;
969 0 : if (new_dev->pcie_cap != NULL) {
970 0 : slot_cap.as_uint32_t = new_dev->pcie_cap->slot_cap.as_uint32_t;
971 0 : }
972 :
973 0 : new_bus_num = vmd_get_next_bus_number(bus->vmd);
974 0 : if (new_bus_num == 0xff) {
975 0 : vmd_dev_free(new_dev);
976 0 : return dev_cnt;
977 : }
978 0 : new_bus = vmd_create_new_bus(bus, new_dev, new_bus_num);
979 0 : if (!new_bus) {
980 0 : vmd_dev_free(new_dev);
981 0 : return dev_cnt;
982 : }
983 0 : new_bus->primary_bus = bus->secondary_bus;
984 0 : new_bus->self = new_dev;
985 0 : new_dev->bus_object = new_bus;
986 :
987 0 : if (slot_cap.bit_field.hotplug_capable && new_dev->pcie_cap != NULL &&
988 0 : new_dev->pcie_cap->express_cap_register.bit_field.slot_implemented) {
989 0 : new_bus->hotplug_buses = vmd_get_hotplug_bus_numbers(new_dev);
990 0 : new_bus->subordinate_bus += new_bus->hotplug_buses;
991 :
992 : /* Attach hot plug instance if HP is supported */
993 : /* Hot inserted SSDs can be assigned port bus of sub-ordinate + 1 */
994 0 : SPDK_INFOLOG(vmd, "hotplug_capable/slot_implemented = "
995 : "%x:%x\n", slot_cap.bit_field.hotplug_capable,
996 : new_dev->pcie_cap->express_cap_register.bit_field.slot_implemented);
997 0 : }
998 :
999 0 : new_dev->parent_bridge = parent_bridge;
1000 0 : new_dev->header->one.primary = new_bus->primary_bus;
1001 0 : new_dev->header->one.secondary = new_bus->secondary_bus;
1002 0 : new_dev->header->one.subordinate = new_bus->subordinate_bus;
1003 :
1004 0 : vmd_bus_update_bridge_info(new_dev);
1005 0 : TAILQ_INSERT_TAIL(&bus->vmd->bus_list, new_bus, tailq);
1006 :
1007 0 : vmd_dev_init(new_dev);
1008 0 : dev_cnt++;
1009 :
1010 0 : if (slot_cap.bit_field.hotplug_capable && new_dev->pcie_cap != NULL &&
1011 0 : new_dev->pcie_cap->express_cap_register.bit_field.slot_implemented) {
1012 0 : vmd_init_hotplug(new_dev, new_bus);
1013 0 : }
1014 :
1015 0 : dev_cnt += vmd_scan_single_bus(new_bus, new_dev, hotplug);
1016 0 : if (new_dev->pcie_cap != NULL) {
1017 0 : if (new_dev->pcie_cap->express_cap_register.bit_field.device_type == SwitchUpstreamPort) {
1018 0 : return dev_cnt;
1019 : }
1020 0 : }
1021 0 : } else {
1022 0 : rc = vmd_init_end_device(new_dev);
1023 0 : if (rc != 0) {
1024 0 : vmd_dev_free(new_dev);
1025 0 : } else {
1026 0 : dev_cnt++;
1027 : }
1028 : }
1029 0 : }
1030 :
1031 0 : return dev_cnt;
1032 0 : }
1033 :
1034 : static void
1035 0 : vmd_print_pci_info(struct vmd_pci_device *dev)
1036 : {
1037 0 : if (!dev) {
1038 0 : return;
1039 : }
1040 :
1041 0 : if (dev->pcie_cap != NULL) {
1042 0 : SPDK_INFOLOG(vmd, "PCI DEVICE: [%04X:%04X] type(%x) : %s\n",
1043 : dev->header->common.vendor_id, dev->header->common.device_id,
1044 : dev->pcie_cap->express_cap_register.bit_field.device_type,
1045 : device_type[dev->pcie_cap->express_cap_register.bit_field.device_type]);
1046 0 : } else {
1047 0 : SPDK_INFOLOG(vmd, "PCI DEVICE: [%04X:%04X]\n",
1048 : dev->header->common.vendor_id, dev->header->common.device_id);
1049 : }
1050 :
1051 0 : SPDK_INFOLOG(vmd, "\tDOMAIN:BDF: %04x:%02x:%02x:%x\n", dev->pci.addr.domain,
1052 : dev->pci.addr.bus, dev->pci.addr.dev, dev->pci.addr.func);
1053 :
1054 0 : if (!(dev->header_type & PCI_HEADER_TYPE_BRIDGE) && dev->bus) {
1055 0 : SPDK_INFOLOG(vmd, "\tbase addr: %x : %p\n",
1056 : dev->header->zero.BAR[0], (void *)dev->bar[0].vaddr);
1057 0 : }
1058 :
1059 0 : if ((dev->header_type & PCI_HEADER_TYPE_BRIDGE)) {
1060 0 : SPDK_INFOLOG(vmd, "\tPrimary = %d, Secondary = %d, Subordinate = %d\n",
1061 : dev->header->one.primary, dev->header->one.secondary, dev->header->one.subordinate);
1062 0 : if (dev->pcie_cap && dev->pcie_cap->express_cap_register.bit_field.slot_implemented) {
1063 0 : SPDK_INFOLOG(vmd, "\tSlot implemented on this device.\n");
1064 0 : if (dev->pcie_cap->slot_cap.bit_field.hotplug_capable) {
1065 0 : SPDK_INFOLOG(vmd, "Device has HOT-PLUG capable slot.\n");
1066 0 : }
1067 0 : }
1068 0 : }
1069 :
1070 0 : if (dev->sn_cap != NULL) {
1071 0 : uint8_t *snLow = (uint8_t *)&dev->sn_cap->sn_low;
1072 0 : uint8_t *snHi = (uint8_t *)&dev->sn_cap->sn_hi;
1073 :
1074 0 : SPDK_INFOLOG(vmd, "\tSN: %02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x\n",
1075 : snHi[3], snHi[2], snHi[1], snHi[0], snLow[3], snLow[2], snLow[1], snLow[0]);
1076 0 : }
1077 0 : }
1078 :
1079 : static void
1080 0 : vmd_cache_scan_info(struct vmd_pci_device *dev)
1081 : {
1082 : uint32_t reg __attribute__((unused));
1083 :
1084 0 : if (dev->header_type == PCI_HEADER_TYPE_NORMAL) {
1085 0 : return;
1086 : }
1087 :
1088 0 : SPDK_INFOLOG(vmd, "vendor/device id:%x:%x\n", dev->header->common.vendor_id,
1089 : dev->header->common.device_id);
1090 :
1091 0 : if (vmd_device_is_root_port(dev->header)) {
1092 0 : dev->header->one.prefetch_base_upper = VMD_UPPER_BASE_SIGNATURE;
1093 0 : reg = dev->header->one.prefetch_base_upper;
1094 0 : dev->header->one.prefetch_limit_upper = VMD_UPPER_LIMIT_SIGNATURE;
1095 0 : reg = dev->header->one.prefetch_limit_upper;
1096 :
1097 0 : SPDK_INFOLOG(vmd, "prefetch: %x:%x\n",
1098 : dev->header->one.prefetch_base_upper,
1099 : dev->header->one.prefetch_limit_upper);
1100 0 : }
1101 0 : }
1102 :
1103 : static void
1104 0 : vmd_reset_root_ports(struct vmd_pci_bus *bus)
1105 : {
1106 : volatile struct pci_header *header;
1107 : uint32_t devfn;
1108 :
1109 : /*
1110 : * The root ports might have been configured by some other driver (e.g. Linux kernel) prior
1111 : * to loading the SPDK one, so we need to clear it. We need to do it before starting the
1112 : * scanning process, as it's depth-first, so when initial root ports are scanned, the
1113 : * latter ones might still be using stale configuration. This can lead to two bridges
1114 : * having the same secondary/subordinate bus configuration, which, of course, isn't correct.
1115 : * (Note: this fixed issue #2413.)
1116 : */
1117 0 : for (devfn = 0; devfn < 32; ++devfn) {
1118 0 : if (!vmd_bus_device_present(bus, devfn)) {
1119 0 : continue;
1120 : }
1121 :
1122 0 : header = (volatile void *)(bus->vmd->cfg_vaddr +
1123 0 : CONFIG_OFFSET_ADDR(bus->config_bus_number, devfn, 0, 0));
1124 0 : if (vmd_device_is_root_port(header) && !vmd_device_is_enumerated(header)) {
1125 0 : vmd_reset_base_limit_registers(header);
1126 0 : }
1127 0 : }
1128 0 : }
1129 :
1130 : static uint8_t
1131 0 : vmd_scan_pcibus(struct vmd_pci_bus *bus)
1132 : {
1133 : struct vmd_pci_bus *bus_entry;
1134 : struct vmd_pci_device *dev;
1135 : uint8_t dev_cnt;
1136 :
1137 0 : vmd_reset_root_ports(bus);
1138 :
1139 0 : g_end_device_count = 0;
1140 0 : TAILQ_INSERT_TAIL(&bus->vmd->bus_list, bus, tailq);
1141 0 : bus->vmd->next_bus_number = bus->bus_number + 1;
1142 0 : dev_cnt = vmd_scan_single_bus(bus, NULL, false);
1143 :
1144 0 : SPDK_INFOLOG(vmd, "VMD scan found %u devices\n", dev_cnt);
1145 0 : SPDK_INFOLOG(vmd, "VMD scan found %u END DEVICES\n", g_end_device_count);
1146 :
1147 0 : SPDK_INFOLOG(vmd, "PCIe devices attached to VMD %04x:%02x:%02x:%x...\n",
1148 : bus->vmd->pci->addr.domain, bus->vmd->pci->addr.bus,
1149 : bus->vmd->pci->addr.dev, bus->vmd->pci->addr.func);
1150 :
1151 0 : TAILQ_FOREACH(bus_entry, &bus->vmd->bus_list, tailq) {
1152 0 : if (bus_entry->self != NULL) {
1153 0 : vmd_print_pci_info(bus_entry->self);
1154 0 : vmd_cache_scan_info(bus_entry->self);
1155 0 : }
1156 :
1157 0 : TAILQ_FOREACH(dev, &bus_entry->dev_list, tailq) {
1158 0 : vmd_print_pci_info(dev);
1159 0 : }
1160 0 : }
1161 :
1162 0 : return dev_cnt;
1163 : }
1164 :
1165 : static int
1166 0 : vmd_domain_map_bar(struct vmd_adapter *vmd, uint32_t bar,
1167 : void **vaddr, uint64_t *paddr, uint64_t *size)
1168 : {
1169 : uint64_t unused;
1170 : int rc;
1171 :
1172 0 : rc = spdk_pci_device_map_bar(vmd->pci, bar, vaddr, &unused, size);
1173 0 : if (rc != 0) {
1174 0 : return rc;
1175 : }
1176 :
1177 : /* Depending on the IOVA configuration, the physical address of the BAR returned by
1178 : * spdk_pci_device_map_bar() can be either an actual physical address or a virtual one (if
1179 : * IOMMU is enabled). Since we do need an actual physical address to fill out the
1180 : * base/limit registers and the BARs of the devices behind the VMD, read the config space to
1181 : * get the correct address, regardless of IOVA configuration. */
1182 0 : rc = spdk_pci_device_cfg_read(vmd->pci, paddr, sizeof(*paddr),
1183 0 : PCI_BAR0_OFFSET + bar * PCI_BAR_SIZE);
1184 0 : if (rc != 0) {
1185 0 : return rc;
1186 : }
1187 :
1188 0 : *paddr &= PCI_BAR_MEMORY_ADDR_OFFSET;
1189 :
1190 0 : return 0;
1191 0 : }
1192 :
1193 : static int
1194 0 : vmd_domain_map_bars(struct vmd_adapter *vmd)
1195 : {
1196 : int rc;
1197 :
1198 0 : rc = vmd_domain_map_bar(vmd, 0, (void **)&vmd->cfg_vaddr,
1199 0 : &vmd->cfgbar, &vmd->cfgbar_size);
1200 0 : if (rc != 0) {
1201 0 : SPDK_ERRLOG("Failed to map config bar: %s\n", spdk_strerror(-rc));
1202 0 : return rc;
1203 : }
1204 :
1205 0 : rc = vmd_domain_map_bar(vmd, 2, (void **)&vmd->mem_vaddr,
1206 0 : &vmd->membar, &vmd->membar_size);
1207 0 : if (rc != 0) {
1208 0 : SPDK_ERRLOG("Failed to map memory bar: %s\n", spdk_strerror(-rc));
1209 0 : return rc;
1210 : }
1211 :
1212 0 : vmd->physical_addr = vmd->membar;
1213 0 : vmd->current_addr_size = vmd->membar_size;
1214 :
1215 0 : return 0;
1216 0 : }
1217 :
1218 : static void
1219 0 : vmd_set_starting_bus_number(struct vmd_adapter *vmd, uint8_t *bus_start,
1220 : uint8_t *max_bus)
1221 : {
1222 0 : uint32_t vmd_cap = 0, vmd_config = 0;
1223 : uint8_t bus_restrict_cap, bus_restrictions;
1224 :
1225 0 : spdk_pci_device_cfg_read32(vmd->pci, &vmd_cap, PCI_VMD_VMCAP);
1226 0 : spdk_pci_device_cfg_read32(vmd->pci, &vmd_config, PCI_VMD_VMCONFIG);
1227 :
1228 0 : bus_restrict_cap = vmd_cap & 0x1; /* bit 0 */
1229 0 : bus_restrictions = (vmd_config >> 8) & 0x3; /* bits 8-9 */
1230 0 : if ((bus_restrict_cap == 0x1) && (bus_restrictions == 0x1)) {
1231 0 : *bus_start = 128;
1232 0 : *max_bus = 255;
1233 0 : } else {
1234 0 : *bus_start = 0;
1235 0 : *max_bus = 127;
1236 : }
1237 0 : }
1238 :
1239 : static int
1240 0 : vmd_enumerate_devices(struct vmd_adapter *vmd)
1241 : {
1242 : uint8_t max_bus, bus_start;
1243 :
1244 0 : vmd->vmd_bus.vmd = vmd;
1245 0 : vmd->vmd_bus.domain = vmd->pci->addr.domain;
1246 :
1247 0 : if (vmd->pci->id.device_id == PCI_DEVICE_ID_INTEL_VMD_ICX) {
1248 0 : vmd_set_starting_bus_number(vmd, &bus_start, &max_bus);
1249 0 : vmd->vmd_bus.bus_start = bus_start;
1250 0 : vmd->vmd_bus.secondary_bus = vmd->vmd_bus.subordinate_bus = vmd->vmd_bus.bus_start;
1251 0 : vmd->vmd_bus.primary_bus = vmd->vmd_bus.bus_number = vmd->vmd_bus.bus_start;
1252 0 : vmd->max_pci_bus = max_bus;
1253 0 : } else {
1254 0 : vmd->vmd_bus.bus_start = 0;
1255 0 : vmd->vmd_bus.secondary_bus = vmd->vmd_bus.subordinate_bus = 0;
1256 0 : vmd->vmd_bus.primary_bus = vmd->vmd_bus.bus_number = 0;
1257 0 : vmd->max_pci_bus = PCI_MAX_BUS_NUMBER;
1258 : }
1259 :
1260 0 : return vmd_scan_pcibus(&vmd->vmd_bus);
1261 : }
1262 :
1263 : struct vmd_pci_device *
1264 0 : vmd_find_device(const struct spdk_pci_addr *addr)
1265 : {
1266 : struct vmd_pci_bus *bus;
1267 : struct vmd_pci_device *dev;
1268 : uint32_t i;
1269 :
1270 0 : for (i = 0; i < g_vmd_container.count; ++i) {
1271 0 : TAILQ_FOREACH(bus, &g_vmd_container.vmd[i].bus_list, tailq) {
1272 0 : if (bus->self) {
1273 0 : if (spdk_pci_addr_compare(&bus->self->pci.addr, addr) == 0) {
1274 0 : return bus->self;
1275 : }
1276 0 : }
1277 :
1278 0 : TAILQ_FOREACH(dev, &bus->dev_list, tailq) {
1279 0 : if (spdk_pci_addr_compare(&dev->pci.addr, addr) == 0) {
1280 0 : return dev;
1281 : }
1282 0 : }
1283 0 : }
1284 0 : }
1285 :
1286 0 : return NULL;
1287 0 : }
1288 :
1289 : static int
1290 0 : vmd_enum_cb(void *ctx, struct spdk_pci_device *pci_dev)
1291 : {
1292 0 : uint32_t cmd_reg = 0;
1293 0 : char bdf[32] = {0};
1294 0 : struct vmd_container *vmd_c = ctx;
1295 0 : struct vmd_adapter *vmd = &vmd_c->vmd[vmd_c->count];
1296 :
1297 0 : spdk_pci_device_cfg_read32(pci_dev, &cmd_reg, 4);
1298 0 : cmd_reg |= 0x6; /* PCI bus master/memory enable. */
1299 0 : spdk_pci_device_cfg_write32(pci_dev, cmd_reg, 4);
1300 :
1301 0 : spdk_pci_addr_fmt(bdf, sizeof(bdf), &pci_dev->addr);
1302 0 : SPDK_INFOLOG(vmd, "Found a VMD[ %d ] at %s\n", vmd_c->count, bdf);
1303 :
1304 : /* map vmd bars */
1305 0 : vmd->pci = pci_dev;
1306 0 : vmd->vmd_index = vmd_c->count;
1307 0 : vmd->domain = (pci_dev->addr.bus << 16) | (pci_dev->addr.dev << 8) | pci_dev->addr.func;
1308 0 : TAILQ_INIT(&vmd->bus_list);
1309 :
1310 0 : if (vmd_domain_map_bars(vmd) != 0) {
1311 0 : return -1;
1312 : }
1313 :
1314 0 : SPDK_INFOLOG(vmd, "vmd config bar(%p) vaddr(%p) size(%x)\n",
1315 : (void *)vmd->cfgbar, (void *)vmd->cfg_vaddr,
1316 : (uint32_t)vmd->cfgbar_size);
1317 0 : SPDK_INFOLOG(vmd, "vmd mem bar(%p) vaddr(%p) size(%x)\n",
1318 : (void *)vmd->membar, (void *)vmd->mem_vaddr,
1319 : (uint32_t)vmd->membar_size);
1320 :
1321 0 : vmd_c->count++;
1322 0 : vmd_enumerate_devices(vmd);
1323 :
1324 0 : return 0;
1325 0 : }
1326 :
1327 : int
1328 0 : spdk_vmd_pci_device_list(struct spdk_pci_addr vmd_addr, struct spdk_pci_device *nvme_list)
1329 : {
1330 0 : int cnt = 0;
1331 : struct vmd_pci_bus *bus;
1332 : struct vmd_pci_device *dev;
1333 : uint32_t i;
1334 :
1335 0 : if (!nvme_list) {
1336 0 : return -1;
1337 : }
1338 :
1339 0 : for (i = 0; i < g_vmd_container.count; ++i) {
1340 0 : if (spdk_pci_addr_compare(&vmd_addr, &g_vmd_container.vmd[i].pci->addr) == 0) {
1341 0 : TAILQ_FOREACH(bus, &g_vmd_container.vmd[i].bus_list, tailq) {
1342 0 : TAILQ_FOREACH(dev, &bus->dev_list, tailq) {
1343 0 : nvme_list[cnt++] = dev->pci;
1344 0 : if (!dev->is_hooked) {
1345 0 : vmd_dev_init(dev);
1346 0 : dev->is_hooked = 1;
1347 0 : }
1348 0 : }
1349 0 : }
1350 0 : }
1351 0 : }
1352 :
1353 0 : return cnt;
1354 0 : }
1355 :
1356 : static void
1357 0 : vmd_clear_hotplug_status(struct vmd_pci_bus *bus)
1358 : {
1359 0 : struct vmd_pci_device *device = bus->self;
1360 : uint16_t status __attribute__((unused));
1361 :
1362 0 : status = device->pcie_cap->slot_status.as_uint16_t;
1363 0 : device->pcie_cap->slot_status.as_uint16_t = status;
1364 0 : status = device->pcie_cap->slot_status.as_uint16_t;
1365 :
1366 0 : status = device->pcie_cap->link_status.as_uint16_t;
1367 0 : device->pcie_cap->link_status.as_uint16_t = status;
1368 0 : status = device->pcie_cap->link_status.as_uint16_t;
1369 0 : }
1370 :
1371 : static void
1372 0 : vmd_bus_handle_hotplug(struct vmd_pci_bus *bus)
1373 : {
1374 : uint8_t num_devices, sleep_count;
1375 :
1376 0 : for (sleep_count = 0; sleep_count < 20; ++sleep_count) {
1377 : /* Scan until a new device is found */
1378 0 : num_devices = vmd_scan_single_bus(bus, bus->self, true);
1379 0 : if (num_devices > 0) {
1380 0 : break;
1381 : }
1382 :
1383 0 : spdk_delay_us(200000);
1384 0 : }
1385 :
1386 0 : if (num_devices == 0) {
1387 0 : SPDK_ERRLOG("Timed out while scanning for hotplugged devices\n");
1388 0 : }
1389 0 : }
1390 :
1391 : static void
1392 0 : vmd_remove_device(struct vmd_pci_device *device)
1393 : {
1394 0 : device->pci.internal.pending_removal = true;
1395 :
1396 : /* If the device isn't attached, remove it immediately */
1397 0 : if (!device->pci.internal.attached) {
1398 0 : vmd_dev_detach(&device->pci);
1399 0 : }
1400 0 : }
1401 :
1402 : static void
1403 0 : vmd_bus_handle_hotremove(struct vmd_pci_bus *bus)
1404 : {
1405 : struct vmd_pci_device *device, *tmpdev;
1406 :
1407 0 : TAILQ_FOREACH_SAFE(device, &bus->dev_list, tailq, tmpdev) {
1408 0 : if (!vmd_bus_device_present(bus, device->devfn)) {
1409 0 : vmd_remove_device(device);
1410 0 : }
1411 0 : }
1412 0 : }
1413 :
1414 : int
1415 0 : spdk_vmd_hotplug_monitor(void)
1416 : {
1417 : struct vmd_pci_bus *bus;
1418 : struct vmd_pci_device *device;
1419 0 : int num_hotplugs = 0;
1420 : uint32_t i;
1421 :
1422 0 : for (i = 0; i < g_vmd_container.count; ++i) {
1423 0 : TAILQ_FOREACH(bus, &g_vmd_container.vmd[i].bus_list, tailq) {
1424 0 : device = bus->self;
1425 0 : if (device == NULL || !device->hotplug_capable) {
1426 0 : continue;
1427 : }
1428 :
1429 0 : if (device->pcie_cap->slot_status.bit_field.datalink_state_changed != 1) {
1430 0 : continue;
1431 : }
1432 :
1433 0 : if (device->pcie_cap->link_status.bit_field.datalink_layer_active == 1) {
1434 0 : SPDK_INFOLOG(vmd, "Device hotplug detected on bus "
1435 : "%"PRIu32"\n", bus->bus_number);
1436 0 : vmd_bus_handle_hotplug(bus);
1437 0 : } else {
1438 0 : SPDK_INFOLOG(vmd, "Device hotremove detected on bus "
1439 : "%"PRIu32"\n", bus->bus_number);
1440 0 : vmd_bus_handle_hotremove(bus);
1441 : }
1442 :
1443 0 : vmd_clear_hotplug_status(bus);
1444 0 : num_hotplugs++;
1445 0 : }
1446 0 : }
1447 :
1448 0 : return num_hotplugs;
1449 : }
1450 :
1451 : int
1452 0 : spdk_vmd_remove_device(const struct spdk_pci_addr *addr)
1453 : {
1454 : struct vmd_pci_device *device;
1455 :
1456 0 : device = vmd_find_device(addr);
1457 0 : if (device == NULL) {
1458 0 : return -ENODEV;
1459 : }
1460 :
1461 0 : assert(strcmp(spdk_pci_device_get_type(&device->pci), "vmd") == 0);
1462 0 : vmd_remove_device(device);
1463 :
1464 0 : return 0;
1465 0 : }
1466 :
1467 : int
1468 0 : spdk_vmd_rescan(void)
1469 : {
1470 : struct vmd_pci_bus *bus;
1471 : uint32_t i;
1472 0 : int rc = 0;
1473 :
1474 0 : for (i = 0; i < g_vmd_container.count; ++i) {
1475 0 : TAILQ_FOREACH(bus, &g_vmd_container.vmd[i].bus_list, tailq) {
1476 0 : rc += vmd_scan_single_bus(bus, bus->self, true);
1477 0 : }
1478 0 : }
1479 :
1480 0 : return rc;
1481 : }
1482 :
1483 : static int
1484 0 : vmd_attach_device(const struct spdk_pci_addr *addr)
1485 : {
1486 : struct vmd_pci_bus *bus;
1487 : struct vmd_adapter *vmd;
1488 : struct vmd_pci_device *dev;
1489 : uint32_t i;
1490 : int rc;
1491 :
1492 : /* VMD always sets function to zero */
1493 0 : if (addr->func != 0) {
1494 0 : return -ENODEV;
1495 : }
1496 :
1497 0 : for (i = 0; i < g_vmd_container.count; ++i) {
1498 0 : vmd = &g_vmd_container.vmd[i];
1499 0 : if (vmd->domain != addr->domain) {
1500 0 : continue;
1501 : }
1502 :
1503 0 : TAILQ_FOREACH(bus, &vmd->bus_list, tailq) {
1504 0 : if (bus->bus_number != addr->bus) {
1505 0 : continue;
1506 : }
1507 :
1508 0 : dev = vmd_alloc_dev(bus, addr->dev);
1509 0 : if (dev == NULL) {
1510 0 : return -ENODEV;
1511 : }
1512 :
1513 : /* Only allow attaching endpoint devices */
1514 0 : if (dev->header->common.header_type & PCI_HEADER_TYPE_BRIDGE) {
1515 0 : free(dev);
1516 0 : return -ENODEV;
1517 : }
1518 :
1519 0 : rc = vmd_init_end_device(dev);
1520 0 : if (rc != 0) {
1521 0 : free(dev);
1522 0 : return -ENODEV;
1523 : }
1524 :
1525 0 : return 0;
1526 : }
1527 0 : }
1528 :
1529 0 : return -ENODEV;
1530 0 : }
1531 :
1532 : static void
1533 0 : vmd_detach_device(struct spdk_pci_device *pci_dev)
1534 : {
1535 0 : struct vmd_pci_device *dev = SPDK_CONTAINEROF(pci_dev, struct vmd_pci_device, pci);
1536 :
1537 0 : assert(strcmp(spdk_pci_device_get_type(pci_dev), "vmd") == 0);
1538 0 : assert(vmd_find_device(&pci_dev->addr) != NULL);
1539 :
1540 0 : vmd_remove_device(dev);
1541 0 : }
1542 :
1543 : static struct spdk_pci_device_provider g_vmd_device_provider = {
1544 : .name = "vmd",
1545 : .attach_cb = vmd_attach_device,
1546 : .detach_cb = vmd_detach_device,
1547 : };
1548 :
1549 0 : SPDK_PCI_REGISTER_DEVICE_PROVIDER(vmd, &g_vmd_device_provider);
1550 :
1551 : int
1552 0 : spdk_vmd_init(void)
1553 : {
1554 0 : return spdk_pci_enumerate(spdk_pci_vmd_get_driver(), vmd_enum_cb, &g_vmd_container);
1555 : }
1556 :
1557 : void
1558 0 : spdk_vmd_fini(void)
1559 : {
1560 : uint32_t i;
1561 :
1562 0 : for (i = 0; i < g_vmd_container.count; ++i) {
1563 0 : spdk_pci_device_detach(g_vmd_container.vmd[i].pci);
1564 0 : }
1565 0 : }
1566 :
1567 0 : SPDK_LOG_REGISTER_COMPONENT(vmd)
|
