Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause 2 : * Copyright (C) 2021 Intel Corporation. 3 : * All rights reserved. 4 : */ 5 : 6 : #include "spdk/stdinc.h" 7 : 8 : #include <accel-config/libaccel_config.h> 9 : 10 : #include "spdk/env.h" 11 : #include "spdk/util.h" 12 : #include "spdk/memory.h" 13 : #include "spdk/likely.h" 14 : 15 : #include "spdk/log.h" 16 : #include "spdk_internal/idxd.h" 17 : 18 : #include "idxd_internal.h" 19 : 20 : struct spdk_kernel_idxd_device { 21 : struct spdk_idxd_device idxd; 22 : struct accfg_ctx *ctx; 23 : 24 : unsigned int max_batch_size; 25 : unsigned int max_xfer_size; 26 : unsigned int max_xfer_bits; 27 : 28 : /* We only use a single WQ */ 29 : struct accfg_wq *wq; 30 : int fd; 31 : void *portal; 32 : }; 33 : 34 : #define __kernel_idxd(idxd) SPDK_CONTAINEROF(idxd, struct spdk_kernel_idxd_device, idxd) 35 : 36 : static void 37 0 : kernel_idxd_device_destruct(struct spdk_idxd_device *idxd) 38 : { 39 0 : struct spdk_kernel_idxd_device *kernel_idxd = __kernel_idxd(idxd); 40 : 41 0 : if (kernel_idxd->portal != NULL) { 42 0 : munmap(kernel_idxd->portal, 0x1000); 43 : } 44 : 45 0 : if (kernel_idxd->fd >= 0) { 46 0 : close(kernel_idxd->fd); 47 : } 48 : 49 0 : accfg_unref(kernel_idxd->ctx); 50 0 : free(kernel_idxd); 51 0 : } 52 : 53 : static struct spdk_idxd_impl g_kernel_idxd_impl; 54 : 55 : static int 56 0 : kernel_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb, spdk_idxd_probe_cb probe_cb) 57 : { 58 : int rc; 59 0 : struct accfg_ctx *ctx; 60 : struct accfg_device *device; 61 : 62 0 : rc = accfg_new(&ctx); 63 0 : if (rc < 0) { 64 0 : SPDK_ERRLOG("Unable to allocate accel-config context\n"); 65 0 : return rc; 66 : } 67 : 68 : /* Loop over each IDXD device */ 69 0 : accfg_device_foreach(ctx, device) { 70 : enum accfg_device_state dstate; 71 : struct spdk_kernel_idxd_device *kernel_idxd; 72 : struct accfg_wq *wq; 73 : bool pasid_enabled; 74 : 75 : /* Make sure that the device is enabled */ 76 0 : dstate = accfg_device_get_state(device); 77 0 : if (dstate != ACCFG_DEVICE_ENABLED) { 78 0 : continue; 79 : } 80 : 81 0 : pasid_enabled = accfg_device_get_pasid_enabled(device); 82 0 : if (!pasid_enabled && spdk_iommu_is_enabled()) { 83 : /* 84 : * If the IOMMU is enabled but shared memory mode is not on, 85 : * then we have no way to get the IOVA from userspace to use this 86 : * device or any kernel device. Return an error. 87 : */ 88 0 : SPDK_ERRLOG("Found kernel IDXD device, but cannot use it when IOMMU is enabled but SM is disabled\n"); 89 0 : return -ENOTSUP; 90 : } 91 : 92 0 : kernel_idxd = calloc(1, sizeof(struct spdk_kernel_idxd_device)); 93 0 : if (kernel_idxd == NULL) { 94 0 : SPDK_ERRLOG("Failed to allocate memory for kernel_idxd device.\n"); 95 : /* TODO: Goto error cleanup */ 96 0 : return -ENOMEM; 97 : } 98 : 99 0 : kernel_idxd->max_batch_size = accfg_device_get_max_batch_size(device); 100 0 : kernel_idxd->max_xfer_size = accfg_device_get_max_transfer_size(device); 101 0 : kernel_idxd->idxd.socket_id = accfg_device_get_numa_node(device); 102 0 : kernel_idxd->idxd.impl = &g_kernel_idxd_impl; 103 0 : kernel_idxd->fd = -1; 104 0 : kernel_idxd->idxd.version = accfg_device_get_version(device); 105 0 : kernel_idxd->idxd.pasid_enabled = pasid_enabled; 106 : 107 0 : accfg_wq_foreach(device, wq) { 108 : enum accfg_wq_state wstate; 109 : enum accfg_wq_mode mode; 110 : enum accfg_wq_type type; 111 : int major, minor; 112 0 : char path[1024]; 113 : 114 0 : wstate = accfg_wq_get_state(wq); 115 0 : if (wstate != ACCFG_WQ_ENABLED) { 116 0 : continue; 117 : } 118 : 119 0 : type = accfg_wq_get_type(wq); 120 0 : if (type != ACCFG_WQT_USER) { 121 0 : continue; 122 : } 123 : 124 : /* TODO: For now, only support dedicated WQ */ 125 0 : mode = accfg_wq_get_mode(wq); 126 0 : if (mode != ACCFG_WQ_DEDICATED) { 127 0 : continue; 128 : } 129 : 130 0 : major = accfg_device_get_cdev_major(device); 131 0 : if (major < 0) { 132 0 : continue; 133 : } 134 : 135 0 : minor = accfg_wq_get_cdev_minor(wq); 136 0 : if (minor < 0) { 137 0 : continue; 138 : } 139 : 140 : /* Map the portal */ 141 0 : snprintf(path, sizeof(path), "/dev/char/%u:%u", major, minor); 142 0 : kernel_idxd->fd = open(path, O_RDWR); 143 0 : if (kernel_idxd->fd < 0) { 144 0 : SPDK_ERRLOG("Can not open the WQ file descriptor on path=%s\n", 145 : path); 146 0 : continue; 147 : } 148 : 149 0 : kernel_idxd->portal = mmap(NULL, 0x1000, PROT_WRITE, 150 : MAP_SHARED | MAP_POPULATE, kernel_idxd->fd, 0); 151 0 : if (kernel_idxd->portal == MAP_FAILED) { 152 0 : perror("mmap"); 153 0 : continue; 154 : } 155 : 156 0 : kernel_idxd->wq = wq; 157 : 158 : /* Since we only use a single WQ, the total size is the size of this WQ */ 159 0 : kernel_idxd->idxd.total_wq_size = accfg_wq_get_size(wq); 160 0 : kernel_idxd->idxd.chan_per_device = (kernel_idxd->idxd.total_wq_size >= 128) ? 8 : 4; 161 : 162 : /* We only use a single WQ, so once we've found one we can stop looking. */ 163 0 : break; 164 : } 165 : 166 0 : if (kernel_idxd->idxd.total_wq_size > 0) { 167 : /* This device has at least 1 WQ available, so ask the user if they want to use it. */ 168 0 : attach_cb(cb_ctx, &kernel_idxd->idxd); 169 : } else { 170 0 : kernel_idxd_device_destruct(&kernel_idxd->idxd); 171 : } 172 : } 173 : 174 0 : return 0; 175 : } 176 : 177 : static void 178 0 : kernel_idxd_dump_sw_error(struct spdk_idxd_device *idxd, void *portal) 179 : { 180 : /* Need to be enhanced later */ 181 0 : } 182 : 183 : static char * 184 0 : kernel_idxd_portal_get_addr(struct spdk_idxd_device *idxd) 185 : { 186 0 : struct spdk_kernel_idxd_device *kernel_idxd = __kernel_idxd(idxd); 187 : 188 0 : return kernel_idxd->portal; 189 : } 190 : 191 : static struct spdk_idxd_impl g_kernel_idxd_impl = { 192 : .name = "kernel", 193 : .probe = kernel_idxd_probe, 194 : .destruct = kernel_idxd_device_destruct, 195 : .dump_sw_error = kernel_idxd_dump_sw_error, 196 : .portal_get_addr = kernel_idxd_portal_get_addr, 197 : }; 198 : 199 0 : SPDK_IDXD_IMPL_REGISTER(kernel, &g_kernel_idxd_impl);