Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2018 Intel Corporation.
3 : * All rights reserved.
4 : * Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : */
6 :
7 : #include "accel_dpdk_compressdev.h"
8 : #include "spdk/accel_module.h"
9 :
10 : #include "spdk/stdinc.h"
11 : #include "spdk/rpc.h"
12 : #include "spdk/env.h"
13 : #include "spdk/endian.h"
14 : #include "spdk/string.h"
15 : #include "spdk/thread.h"
16 : #include "spdk/util.h"
17 : #include "spdk/likely.h"
18 :
19 : #include "spdk/log.h"
20 :
21 : #include <rte_config.h>
22 : #include <rte_bus_vdev.h>
23 : #include <rte_compressdev.h>
24 : #include <rte_comp.h>
25 : #include <rte_mbuf_dyn.h>
26 :
27 : /* Used to store IO context in mbuf */
28 : static const struct rte_mbuf_dynfield rte_mbuf_dynfield_io_context = {
29 : .name = "context_accel_comp",
30 : .size = sizeof(uint64_t),
31 : .align = __alignof__(uint64_t),
32 : .flags = 0,
33 : };
34 : static int g_mbuf_offset;
35 : static enum compress_pmd g_opts;
36 : static bool g_compressdev_enable = false;
37 : static bool g_compressdev_initialized = false;
38 :
39 : #define NUM_MAX_XFORMS 2
40 : #define NUM_MAX_INFLIGHT_OPS 128
41 : #define DEFAULT_WINDOW_SIZE 15
42 : #define MBUF_SPLIT (1UL << DEFAULT_WINDOW_SIZE)
43 : #define QAT_PMD "compress_qat"
44 : #define MLX5_PMD "mlx5_pci"
45 : #define NUM_MBUFS 65536
46 : #define POOL_CACHE_SIZE 256
47 :
48 : /* Global list of available compression devices. */
49 : struct compress_dev {
50 : struct rte_compressdev_info cdev_info; /* includes device friendly name */
51 : uint8_t cdev_id; /* identifier for the device */
52 : void *comp_xform; /* shared private xform for comp on this PMD */
53 : void *decomp_xform; /* shared private xform for decomp on this PMD */
54 : bool sgl_in;
55 : bool sgl_out;
56 : TAILQ_ENTRY(compress_dev) link;
57 : };
58 : static TAILQ_HEAD(, compress_dev) g_compress_devs = TAILQ_HEAD_INITIALIZER(g_compress_devs);
59 :
60 : #define MAX_NUM_QP 48
61 : /* Global list and lock for unique device/queue pair combos */
62 : struct comp_device_qp {
63 : struct compress_dev *device; /* ptr to compression device */
64 : uint8_t qp; /* queue pair for this node */
65 : struct compress_io_channel *chan;
66 : TAILQ_ENTRY(comp_device_qp) link;
67 : };
68 : static TAILQ_HEAD(, comp_device_qp) g_comp_device_qp = TAILQ_HEAD_INITIALIZER(g_comp_device_qp);
69 : static pthread_mutex_t g_comp_device_qp_lock = PTHREAD_MUTEX_INITIALIZER;
70 :
71 : struct compress_io_channel {
72 : char *drv_name; /* name of the compression device driver */
73 : struct comp_device_qp *device_qp;
74 : struct spdk_poller *poller;
75 : struct rte_mbuf **src_mbufs;
76 : struct rte_mbuf **dst_mbufs;
77 : STAILQ_HEAD(, spdk_accel_task) queued_tasks;
78 : };
79 :
80 : /* Shared mempools between all devices on this system */
81 : static struct rte_mempool *g_mbuf_mp = NULL; /* mbuf mempool */
82 : static struct rte_mempool *g_comp_op_mp = NULL; /* comp operations, must be rte* mempool */
83 : static struct rte_mbuf_ext_shared_info g_shinfo = {}; /* used by DPDK mbuf macros */
84 : static bool g_qat_available = false;
85 : static bool g_mlx5_pci_available = false;
86 :
87 : /* Create shared (between all ops per PMD) compress xforms. */
88 : static struct rte_comp_xform g_comp_xform = {
89 : .type = RTE_COMP_COMPRESS,
90 : .compress = {
91 : .algo = RTE_COMP_ALGO_DEFLATE,
92 : .deflate.huffman = RTE_COMP_HUFFMAN_DEFAULT,
93 : .level = RTE_COMP_LEVEL_MAX,
94 : .window_size = DEFAULT_WINDOW_SIZE,
95 : .chksum = RTE_COMP_CHECKSUM_NONE,
96 : .hash_algo = RTE_COMP_HASH_ALGO_NONE
97 : }
98 : };
99 : /* Create shared (between all ops per PMD) decompress xforms. */
100 : static struct rte_comp_xform g_decomp_xform = {
101 : .type = RTE_COMP_DECOMPRESS,
102 : .decompress = {
103 : .algo = RTE_COMP_ALGO_DEFLATE,
104 : .chksum = RTE_COMP_CHECKSUM_NONE,
105 : .window_size = DEFAULT_WINDOW_SIZE,
106 : .hash_algo = RTE_COMP_HASH_ALGO_NONE
107 : }
108 : };
109 :
110 : /* Dummy function used by DPDK to free ext attached buffers
111 : * to mbufs, we free them ourselves but this callback has to
112 : * be here.
113 : */
114 : static void
115 0 : shinfo_free_cb(void *arg1, void *arg2)
116 : {
117 0 : }
118 :
119 : /* Called by accel_init_compress_drivers() to init each discovered compression device */
120 : static int
121 6 : create_compress_dev(uint8_t index)
122 : {
123 6 : struct compress_dev *device;
124 6 : uint16_t q_pairs;
125 6 : uint8_t cdev_id;
126 6 : int rc, i;
127 6 : struct comp_device_qp *dev_qp;
128 6 : struct comp_device_qp *tmp_qp;
129 :
130 6 : device = calloc(1, sizeof(struct compress_dev));
131 6 : if (!device) {
132 0 : return -ENOMEM;
133 : }
134 :
135 : /* Get details about this device. */
136 6 : rte_compressdev_info_get(index, &device->cdev_info);
137 :
138 6 : cdev_id = device->cdev_id = index;
139 :
140 : /* Zero means no limit so choose number of lcores. */
141 6 : if (device->cdev_info.max_nb_queue_pairs == 0) {
142 5 : q_pairs = MAX_NUM_QP;
143 5 : } else {
144 1 : q_pairs = spdk_min(device->cdev_info.max_nb_queue_pairs, MAX_NUM_QP);
145 : }
146 :
147 : /* Configure the compression device. */
148 18 : struct rte_compressdev_config config = {
149 6 : .socket_id = rte_socket_id(),
150 6 : .nb_queue_pairs = q_pairs,
151 : .max_nb_priv_xforms = NUM_MAX_XFORMS,
152 : .max_nb_streams = 0
153 : };
154 6 : rc = rte_compressdev_configure(cdev_id, &config);
155 6 : if (rc < 0) {
156 2 : SPDK_ERRLOG("Failed to configure compressdev %u\n", cdev_id);
157 2 : goto err;
158 : }
159 :
160 : /* Pre-setup all potential qpairs now and assign them in the channel
161 : * callback.
162 : */
163 148 : for (i = 0; i < q_pairs; i++) {
164 290 : rc = rte_compressdev_queue_pair_setup(cdev_id, i,
165 : NUM_MAX_INFLIGHT_OPS,
166 145 : rte_socket_id());
167 145 : if (rc) {
168 1 : if (i > 0) {
169 0 : q_pairs = i;
170 0 : SPDK_NOTICELOG("FYI failed to setup a queue pair on "
171 : "compressdev %u with error %u "
172 : "so limiting to %u qpairs\n",
173 : cdev_id, rc, q_pairs);
174 0 : break;
175 : } else {
176 1 : SPDK_ERRLOG("Failed to setup queue pair on "
177 : "compressdev %u with error %u\n", cdev_id, rc);
178 1 : rc = -EINVAL;
179 1 : goto err;
180 : }
181 : }
182 144 : }
183 :
184 3 : rc = rte_compressdev_start(cdev_id);
185 3 : if (rc < 0) {
186 1 : SPDK_ERRLOG("Failed to start device %u: error %d\n",
187 : cdev_id, rc);
188 1 : goto err;
189 : }
190 :
191 2 : if (device->cdev_info.capabilities->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
192 2 : rc = rte_compressdev_private_xform_create(cdev_id, &g_comp_xform,
193 2 : &device->comp_xform);
194 2 : if (rc < 0) {
195 1 : SPDK_ERRLOG("Failed to create private comp xform device %u: error %d\n",
196 : cdev_id, rc);
197 1 : goto err;
198 : }
199 :
200 1 : rc = rte_compressdev_private_xform_create(cdev_id, &g_decomp_xform,
201 1 : &device->decomp_xform);
202 1 : if (rc) {
203 0 : SPDK_ERRLOG("Failed to create private decomp xform device %u: error %d\n",
204 : cdev_id, rc);
205 0 : goto err;
206 : }
207 1 : } else {
208 0 : SPDK_ERRLOG("PMD does not support shared transforms\n");
209 0 : goto err;
210 : }
211 :
212 : /* Build up list of device/qp combinations */
213 49 : for (i = 0; i < q_pairs; i++) {
214 48 : dev_qp = calloc(1, sizeof(struct comp_device_qp));
215 48 : if (!dev_qp) {
216 0 : rc = -ENOMEM;
217 0 : goto err;
218 : }
219 48 : dev_qp->device = device;
220 48 : dev_qp->qp = i;
221 48 : dev_qp->chan = NULL;
222 48 : TAILQ_INSERT_TAIL(&g_comp_device_qp, dev_qp, link);
223 48 : }
224 :
225 1 : TAILQ_INSERT_TAIL(&g_compress_devs, device, link);
226 :
227 1 : if (strcmp(device->cdev_info.driver_name, QAT_PMD) == 0) {
228 0 : g_qat_available = true;
229 0 : }
230 :
231 1 : if (strcmp(device->cdev_info.driver_name, MLX5_PMD) == 0) {
232 0 : g_mlx5_pci_available = true;
233 0 : }
234 :
235 1 : return 0;
236 :
237 : err:
238 5 : TAILQ_FOREACH_SAFE(dev_qp, &g_comp_device_qp, link, tmp_qp) {
239 0 : TAILQ_REMOVE(&g_comp_device_qp, dev_qp, link);
240 0 : free(dev_qp);
241 0 : }
242 5 : free(device);
243 5 : return rc;
244 6 : }
245 :
246 : /* Called from driver init entry point, accel_compress_init() */
247 : static int
248 9 : accel_init_compress_drivers(void)
249 : {
250 9 : uint8_t cdev_count, i;
251 9 : struct compress_dev *tmp_dev;
252 9 : struct compress_dev *device;
253 9 : int rc;
254 :
255 : /* If we have no compression devices, there's no reason to continue. */
256 9 : cdev_count = rte_compressdev_count();
257 9 : if (cdev_count == 0) {
258 1 : return 0;
259 : }
260 8 : if (cdev_count > RTE_COMPRESS_MAX_DEVS) {
261 1 : SPDK_ERRLOG("invalid device count from rte_compressdev_count()\n");
262 1 : return -EINVAL;
263 : }
264 :
265 7 : g_mbuf_offset = rte_mbuf_dynfield_register(&rte_mbuf_dynfield_io_context);
266 7 : if (g_mbuf_offset < 0) {
267 1 : SPDK_ERRLOG("error registering dynamic field with DPDK\n");
268 1 : return -EINVAL;
269 : }
270 :
271 6 : g_mbuf_mp = rte_pktmbuf_pool_create("comp_mbuf_mp", NUM_MBUFS, POOL_CACHE_SIZE,
272 6 : sizeof(struct rte_mbuf), 0, rte_socket_id());
273 6 : if (g_mbuf_mp == NULL) {
274 0 : SPDK_ERRLOG("Cannot create mbuf pool\n");
275 0 : rc = -ENOMEM;
276 0 : goto error_create_mbuf;
277 : }
278 :
279 6 : g_comp_op_mp = rte_comp_op_pool_create("comp_op_pool", NUM_MBUFS, POOL_CACHE_SIZE,
280 6 : 0, rte_socket_id());
281 6 : if (g_comp_op_mp == NULL) {
282 0 : SPDK_ERRLOG("Cannot create comp op pool\n");
283 0 : rc = -ENOMEM;
284 0 : goto error_create_op;
285 : }
286 :
287 : /* Init all devices */
288 7 : for (i = 0; i < cdev_count; i++) {
289 6 : rc = create_compress_dev(i);
290 6 : if (rc != 0) {
291 5 : goto error_create_compress_devs;
292 : }
293 1 : }
294 :
295 1 : if (g_qat_available == true) {
296 0 : SPDK_NOTICELOG("initialized QAT PMD\n");
297 0 : }
298 :
299 1 : g_shinfo.free_cb = shinfo_free_cb;
300 :
301 1 : return 0;
302 :
303 : /* Error cleanup paths. */
304 : error_create_compress_devs:
305 5 : TAILQ_FOREACH_SAFE(device, &g_compress_devs, link, tmp_dev) {
306 0 : TAILQ_REMOVE(&g_compress_devs, device, link);
307 0 : free(device);
308 5 : }
309 : error_create_op:
310 : error_create_mbuf:
311 5 : rte_mempool_free(g_mbuf_mp);
312 :
313 5 : return rc;
314 9 : }
315 :
316 : int
317 0 : accel_compressdev_enable_probe(enum compress_pmd *opts)
318 : {
319 0 : g_opts = *opts;
320 0 : g_compressdev_enable = true;
321 :
322 0 : return 0;
323 : }
324 :
325 : static int
326 23 : _setup_compress_mbuf(struct rte_mbuf **mbufs, int *mbuf_total, uint64_t *total_length,
327 : struct iovec *iovs, int iovcnt, struct spdk_accel_task *task)
328 : {
329 23 : uint64_t iovec_length, updated_length, phys_addr;
330 23 : uint64_t processed, mbuf_length, remainder;
331 23 : uint8_t *current_base = NULL;
332 23 : int iov_index, mbuf_index;
333 23 : int rc = 0;
334 :
335 : /* Setup mbufs */
336 23 : iov_index = mbuf_index = 0;
337 86 : while (iov_index < iovcnt) {
338 :
339 63 : processed = 0;
340 63 : iovec_length = iovs[iov_index].iov_len;
341 :
342 63 : current_base = iovs[iov_index].iov_base;
343 63 : if (total_length) {
344 36 : *total_length += iovec_length;
345 36 : }
346 :
347 63 : assert(mbufs[mbuf_index] != NULL);
348 63 : *RTE_MBUF_DYNFIELD(mbufs[mbuf_index], g_mbuf_offset, uint64_t *) = (uint64_t)task;
349 :
350 63 : do {
351 : /* new length is min of remaining left or max mbuf size of MBUF_SPLIT */
352 66 : mbuf_length = updated_length = spdk_min(MBUF_SPLIT, iovec_length - processed);
353 :
354 66 : phys_addr = spdk_vtophys((void *)current_base, &updated_length);
355 :
356 66 : rte_pktmbuf_attach_extbuf(mbufs[mbuf_index],
357 66 : current_base,
358 66 : phys_addr,
359 66 : updated_length,
360 : &g_shinfo);
361 66 : rte_pktmbuf_append(mbufs[mbuf_index], updated_length);
362 66 : remainder = mbuf_length - updated_length;
363 :
364 : /* although the mbufs were preallocated, we still need to chain them */
365 66 : if (mbuf_index > 0) {
366 43 : rte_pktmbuf_chain(mbufs[0], mbufs[mbuf_index]);
367 43 : }
368 :
369 : /* keep track of the total we've put into the mbuf chain */
370 66 : processed += updated_length;
371 : /* bump the base by what was previously added */
372 66 : current_base += updated_length;
373 :
374 : /* If we crossed 2MB boundary we need another mbuf for the remainder */
375 66 : if (remainder > 0) {
376 :
377 5 : assert(remainder <= MBUF_SPLIT);
378 :
379 : /* allocate an mbuf at the end of the array */
380 10 : rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp,
381 5 : (struct rte_mbuf **)&mbufs[*mbuf_total], 1);
382 5 : if (rc) {
383 0 : SPDK_ERRLOG("ERROR trying to get an extra mbuf!\n");
384 0 : return -1;
385 : }
386 5 : (*mbuf_total)++;
387 5 : mbuf_index++;
388 5 : *RTE_MBUF_DYNFIELD(mbufs[mbuf_index], g_mbuf_offset, uint64_t *) = (uint64_t)task;
389 :
390 : /* bump the base by what was previously added */
391 5 : current_base += updated_length;
392 :
393 5 : updated_length = remainder;
394 5 : phys_addr = spdk_vtophys((void *)current_base, &updated_length);
395 :
396 : /* assert we don't cross another */
397 5 : assert(remainder == updated_length);
398 :
399 5 : rte_pktmbuf_attach_extbuf(mbufs[mbuf_index],
400 5 : current_base,
401 5 : phys_addr,
402 5 : remainder,
403 : &g_shinfo);
404 5 : rte_pktmbuf_append(mbufs[mbuf_index], remainder);
405 5 : rte_pktmbuf_chain(mbufs[0], mbufs[mbuf_index]);
406 :
407 : /* keep track of the total we've put into the mbuf chain */
408 5 : processed += remainder;
409 5 : }
410 :
411 66 : mbuf_index++;
412 :
413 66 : } while (processed < iovec_length);
414 :
415 63 : assert(processed == iovec_length);
416 63 : iov_index++;
417 : }
418 :
419 23 : return 0;
420 23 : }
421 :
422 : static int
423 13 : _compress_operation(struct compress_io_channel *chan, struct spdk_accel_task *task)
424 : {
425 13 : int dst_iovcnt = task->d.iovcnt;
426 13 : struct iovec *dst_iovs = task->d.iovs;
427 13 : int src_iovcnt = task->s.iovcnt;
428 13 : struct iovec *src_iovs = task->s.iovs;
429 13 : struct rte_comp_op *comp_op;
430 13 : uint8_t cdev_id;
431 13 : uint64_t total_length = 0;
432 13 : int rc = 0, i;
433 13 : int src_mbuf_total = 0;
434 13 : int dst_mbuf_total = 0;
435 13 : bool device_error = false;
436 13 : bool compress = (task->op_code == SPDK_ACCEL_OPC_COMPRESS);
437 :
438 13 : assert(chan->device_qp->device != NULL);
439 13 : cdev_id = chan->device_qp->device->cdev_id;
440 :
441 : /* calc our mbuf totals based on max MBUF size allowed so we can pre-alloc mbufs in bulk */
442 52 : for (i = 0 ; i < src_iovcnt; i++) {
443 39 : src_mbuf_total += spdk_divide_round_up(src_iovs[i].iov_len, MBUF_SPLIT);
444 39 : }
445 52 : for (i = 0 ; i < dst_iovcnt; i++) {
446 39 : dst_mbuf_total += spdk_divide_round_up(dst_iovs[i].iov_len, MBUF_SPLIT);
447 39 : }
448 :
449 13 : comp_op = rte_comp_op_alloc(g_comp_op_mp);
450 13 : if (!comp_op) {
451 1 : SPDK_ERRLOG("trying to get a comp op!\n");
452 1 : rc = -ENOMEM;
453 1 : goto error_get_op;
454 : }
455 :
456 : /* get an mbuf per iov, src and dst */
457 12 : rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, chan->src_mbufs, src_mbuf_total);
458 12 : if (rc) {
459 1 : SPDK_ERRLOG("ERROR trying to get src_mbufs!\n");
460 1 : rc = -ENOMEM;
461 1 : goto error_get_src;
462 : }
463 11 : assert(chan->src_mbufs[0]);
464 :
465 11 : rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, chan->dst_mbufs, dst_mbuf_total);
466 11 : if (rc) {
467 0 : SPDK_ERRLOG("ERROR trying to get dst_mbufs!\n");
468 0 : rc = -ENOMEM;
469 0 : goto error_get_dst;
470 : }
471 11 : assert(chan->dst_mbufs[0]);
472 :
473 11 : rc = _setup_compress_mbuf(chan->src_mbufs, &src_mbuf_total, &total_length,
474 11 : src_iovs, src_iovcnt, task);
475 :
476 11 : if (rc < 0) {
477 0 : goto error_src_dst;
478 : }
479 11 : if (!chan->device_qp->device->sgl_in && src_mbuf_total > 1) {
480 2 : SPDK_ERRLOG("Src buffer uses chained mbufs but driver %s doesn't support SGL input\n",
481 : chan->drv_name);
482 2 : rc = -EINVAL;
483 2 : goto error_src_dst;
484 : }
485 :
486 9 : comp_op->m_src = chan->src_mbufs[0];
487 9 : comp_op->src.offset = 0;
488 9 : comp_op->src.length = total_length;
489 :
490 9 : rc = _setup_compress_mbuf(chan->dst_mbufs, &dst_mbuf_total, NULL,
491 9 : dst_iovs, dst_iovcnt, task);
492 9 : if (rc < 0) {
493 0 : goto error_src_dst;
494 : }
495 9 : if (!chan->device_qp->device->sgl_out && dst_mbuf_total > 1) {
496 2 : SPDK_ERRLOG("Dst buffer uses chained mbufs but driver %s doesn't support SGL output\n",
497 : chan->drv_name);
498 2 : rc = -EINVAL;
499 2 : goto error_src_dst;
500 : }
501 :
502 7 : comp_op->m_dst = chan->dst_mbufs[0];
503 7 : comp_op->dst.offset = 0;
504 :
505 7 : if (compress == true) {
506 7 : comp_op->private_xform = chan->device_qp->device->comp_xform;
507 7 : } else {
508 0 : comp_op->private_xform = chan->device_qp->device->decomp_xform;
509 : }
510 :
511 7 : comp_op->op_type = RTE_COMP_OP_STATELESS;
512 7 : comp_op->flush_flag = RTE_COMP_FLUSH_FINAL;
513 :
514 7 : rc = rte_compressdev_enqueue_burst(cdev_id, chan->device_qp->qp, &comp_op, 1);
515 7 : assert(rc <= 1);
516 :
517 : /* We always expect 1 got queued, if 0 then we need to queue it up. */
518 9 : if (rc == 1) {
519 5 : return 0;
520 2 : } else if (comp_op->status == RTE_COMP_OP_STATUS_NOT_PROCESSED) {
521 1 : rc = -EAGAIN;
522 1 : } else {
523 1 : device_error = true;
524 : }
525 :
526 : /* Error cleanup paths. */
527 : error_src_dst:
528 6 : rte_pktmbuf_free_bulk(chan->dst_mbufs, dst_iovcnt);
529 : error_get_dst:
530 6 : rte_pktmbuf_free_bulk(chan->src_mbufs, src_iovcnt);
531 : error_get_src:
532 7 : rte_comp_op_free(comp_op);
533 : error_get_op:
534 :
535 8 : if (device_error == true) {
536 : /* There was an error sending the op to the device, most
537 : * likely with the parameters.
538 : */
539 1 : SPDK_ERRLOG("Compression API returned 0x%x\n", comp_op->status);
540 1 : return -EINVAL;
541 : }
542 7 : if (rc != -ENOMEM && rc != -EAGAIN) {
543 4 : return rc;
544 : }
545 :
546 3 : STAILQ_INSERT_TAIL(&chan->queued_tasks, task, link);
547 3 : return 0;
548 13 : }
549 :
550 : /* Poller for the DPDK compression driver. */
551 : static int
552 3 : comp_dev_poller(void *args)
553 : {
554 3 : struct compress_io_channel *chan = args;
555 3 : uint8_t cdev_id;
556 3 : struct rte_comp_op *deq_ops[NUM_MAX_INFLIGHT_OPS];
557 3 : uint16_t num_deq;
558 3 : struct spdk_accel_task *task, *task_to_resubmit;
559 3 : int rc, i, status;
560 :
561 3 : assert(chan->device_qp->device != NULL);
562 3 : cdev_id = chan->device_qp->device->cdev_id;
563 :
564 3 : num_deq = rte_compressdev_dequeue_burst(cdev_id, chan->device_qp->qp, deq_ops,
565 : NUM_MAX_INFLIGHT_OPS);
566 7 : for (i = 0; i < num_deq; i++) {
567 :
568 : /* We store this off regardless of success/error so we know how to contruct the
569 : * next task
570 : */
571 4 : task = (struct spdk_accel_task *)*RTE_MBUF_DYNFIELD(deq_ops[i]->m_src, g_mbuf_offset,
572 : uint64_t *);
573 4 : status = deq_ops[i]->status;
574 :
575 4 : if (spdk_likely(status == RTE_COMP_OP_STATUS_SUCCESS)) {
576 3 : if (task->output_size != NULL) {
577 3 : *task->output_size = deq_ops[i]->produced;
578 3 : }
579 3 : } else {
580 1 : SPDK_NOTICELOG("Deque status %u\n", status);
581 : }
582 :
583 4 : spdk_accel_task_complete(task, status);
584 :
585 : /* Now free both mbufs and the compress operation. The rte_pktmbuf_free()
586 : * call takes care of freeing all of the mbufs in the chain back to their
587 : * original pool.
588 : */
589 4 : rte_pktmbuf_free(deq_ops[i]->m_src);
590 4 : rte_pktmbuf_free(deq_ops[i]->m_dst);
591 :
592 : /* There is no bulk free for com ops so we have to free them one at a time
593 : * here however it would be rare that we'd ever have more than 1 at a time
594 : * anyways.
595 : */
596 4 : rte_comp_op_free(deq_ops[i]);
597 :
598 : /* Check if there are any pending comp ops to process, only pull one
599 : * at a time off as _compress_operation() may re-queue the op.
600 : */
601 4 : if (!STAILQ_EMPTY(&chan->queued_tasks)) {
602 1 : task_to_resubmit = STAILQ_FIRST(&chan->queued_tasks);
603 1 : rc = _compress_operation(chan, task_to_resubmit);
604 1 : if (rc == 0) {
605 1 : STAILQ_REMOVE_HEAD(&chan->queued_tasks, link);
606 1 : }
607 1 : }
608 4 : }
609 :
610 6 : return num_deq == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY;
611 3 : }
612 :
613 : static int
614 0 : _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task)
615 : {
616 0 : struct compress_io_channel *chan = spdk_io_channel_get_ctx(ch);
617 0 : int rc;
618 :
619 0 : rc = _compress_operation(chan, task);
620 0 : if (rc) {
621 0 : SPDK_ERRLOG("Error (%d) in comrpess operation\n", rc);
622 0 : assert(false);
623 : }
624 :
625 0 : return rc;
626 0 : }
627 :
628 : static int
629 0 : compress_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *first_task)
630 : {
631 0 : struct compress_io_channel *chan = spdk_io_channel_get_ctx(ch);
632 0 : struct spdk_accel_task *task, *tmp;
633 0 : int rc = 0;
634 :
635 0 : task = first_task;
636 :
637 0 : if (!STAILQ_EMPTY(&chan->queued_tasks)) {
638 0 : goto queue_tasks;
639 : }
640 :
641 : /* The caller will either submit a single task or a group of tasks that are
642 : * linked together but they cannot be on a list. For example, see poller
643 : * where a list of queued tasks is being resubmitted, the list they are on
644 : * is initialized after saving off the first task from the list which is then
645 : * passed in here. Similar thing is done in the accel framework.
646 : */
647 0 : while (task) {
648 0 : tmp = STAILQ_NEXT(task, link);
649 0 : rc = _process_single_task(ch, task);
650 :
651 0 : if (rc == -EBUSY) {
652 0 : goto queue_tasks;
653 0 : } else if (rc) {
654 0 : spdk_accel_task_complete(task, rc);
655 0 : }
656 0 : task = tmp;
657 : }
658 :
659 0 : return 0;
660 :
661 : queue_tasks:
662 0 : while (task != NULL) {
663 0 : tmp = STAILQ_NEXT(task, link);
664 0 : STAILQ_INSERT_TAIL(&chan->queued_tasks, task, link);
665 0 : task = tmp;
666 : }
667 0 : return 0;
668 0 : }
669 :
670 : static bool
671 0 : _set_pmd(struct compress_io_channel *chan)
672 : {
673 :
674 : /* Note: the compress_isal PMD is not supported as accel_fw supports native ISAL
675 : * using the accel_sw module */
676 0 : if (g_opts == COMPRESS_PMD_AUTO) {
677 0 : if (g_qat_available) {
678 0 : chan->drv_name = QAT_PMD;
679 0 : } else if (g_mlx5_pci_available) {
680 0 : chan->drv_name = MLX5_PMD;
681 0 : }
682 0 : } else if (g_opts == COMPRESS_PMD_QAT_ONLY && g_qat_available) {
683 0 : chan->drv_name = QAT_PMD;
684 0 : } else if (g_opts == COMPRESS_PMD_MLX5_PCI_ONLY && g_mlx5_pci_available) {
685 0 : chan->drv_name = MLX5_PMD;
686 0 : } else {
687 0 : SPDK_ERRLOG("Requested PMD is not available.\n");
688 0 : return false;
689 : }
690 0 : SPDK_NOTICELOG("Channel %p PMD being used: %s\n", chan, chan->drv_name);
691 0 : return true;
692 0 : }
693 :
694 : static int compress_create_cb(void *io_device, void *ctx_buf);
695 : static void compress_destroy_cb(void *io_device, void *ctx_buf);
696 : static struct spdk_accel_module_if g_compress_module;
697 : static int
698 0 : accel_compress_init(void)
699 : {
700 0 : int rc;
701 :
702 0 : if (!g_compressdev_enable) {
703 0 : return -EINVAL;
704 : }
705 :
706 0 : rc = accel_init_compress_drivers();
707 0 : if (rc) {
708 0 : assert(TAILQ_EMPTY(&g_compress_devs));
709 0 : SPDK_NOTICELOG("no available compression devices\n");
710 0 : return -EINVAL;
711 : }
712 :
713 0 : g_compressdev_initialized = true;
714 0 : spdk_io_device_register(&g_compress_module, compress_create_cb, compress_destroy_cb,
715 : sizeof(struct compress_io_channel), "compressdev_accel_module");
716 0 : return 0;
717 0 : }
718 :
719 : static int
720 0 : compress_create_cb(void *io_device, void *ctx_buf)
721 : {
722 0 : struct compress_io_channel *chan = ctx_buf;
723 0 : const struct rte_compressdev_capabilities *capab;
724 0 : struct comp_device_qp *device_qp;
725 0 : size_t length;
726 :
727 0 : if (_set_pmd(chan) == false) {
728 0 : assert(false);
729 : return -ENODEV;
730 : }
731 :
732 : /* The following variable length arrays of mbuf pointers are required to submit to compressdev */
733 0 : length = NUM_MBUFS * sizeof(void *);
734 0 : chan->src_mbufs = spdk_zmalloc(length, 0x40, NULL,
735 : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
736 0 : if (chan->src_mbufs == NULL) {
737 0 : return -ENOMEM;
738 : }
739 0 : chan->dst_mbufs = spdk_zmalloc(length, 0x40, NULL,
740 : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
741 0 : if (chan->dst_mbufs == NULL) {
742 0 : free(chan->src_mbufs);
743 0 : return -ENOMEM;
744 : }
745 :
746 0 : chan->poller = SPDK_POLLER_REGISTER(comp_dev_poller, chan, 0);
747 0 : STAILQ_INIT(&chan->queued_tasks);
748 :
749 0 : pthread_mutex_lock(&g_comp_device_qp_lock);
750 0 : TAILQ_FOREACH(device_qp, &g_comp_device_qp, link) {
751 0 : if (strcmp(device_qp->device->cdev_info.driver_name, chan->drv_name) == 0) {
752 0 : if (device_qp->chan == NULL) {
753 0 : chan->device_qp = device_qp;
754 0 : device_qp->chan = chan;
755 0 : break;
756 : }
757 0 : }
758 0 : }
759 0 : pthread_mutex_unlock(&g_comp_device_qp_lock);
760 :
761 0 : if (chan->device_qp == NULL) {
762 0 : SPDK_ERRLOG("out of qpairs, cannot assign one\n");
763 0 : assert(false);
764 : return -ENOMEM;
765 : } else {
766 0 : capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
767 :
768 0 : if (capab->comp_feature_flags & (RTE_COMP_FF_OOP_SGL_IN_SGL_OUT | RTE_COMP_FF_OOP_SGL_IN_LB_OUT)) {
769 0 : chan->device_qp->device->sgl_in = true;
770 0 : }
771 :
772 0 : if (capab->comp_feature_flags & (RTE_COMP_FF_OOP_SGL_IN_SGL_OUT | RTE_COMP_FF_OOP_LB_IN_SGL_OUT)) {
773 0 : chan->device_qp->device->sgl_out = true;
774 0 : }
775 : }
776 :
777 0 : return 0;
778 0 : }
779 :
780 : static void
781 0 : accel_compress_write_config_json(struct spdk_json_write_ctx *w)
782 : {
783 0 : if (g_compressdev_enable) {
784 0 : spdk_json_write_object_begin(w);
785 0 : spdk_json_write_named_string(w, "method", "compressdev_scan_accel_module");
786 0 : spdk_json_write_named_object_begin(w, "params");
787 0 : spdk_json_write_named_uint32(w, "pmd", g_opts);
788 0 : spdk_json_write_object_end(w);
789 0 : spdk_json_write_object_end(w);
790 0 : }
791 0 : }
792 :
793 : static void
794 0 : compress_destroy_cb(void *io_device, void *ctx_buf)
795 : {
796 0 : struct compress_io_channel *chan = ctx_buf;
797 0 : struct comp_device_qp *device_qp = chan->device_qp;
798 :
799 0 : spdk_free(chan->src_mbufs);
800 0 : spdk_free(chan->dst_mbufs);
801 :
802 0 : spdk_poller_unregister(&chan->poller);
803 :
804 0 : pthread_mutex_lock(&g_comp_device_qp_lock);
805 0 : chan->device_qp = NULL;
806 0 : device_qp->chan = NULL;
807 0 : pthread_mutex_unlock(&g_comp_device_qp_lock);
808 0 : }
809 :
810 : static size_t
811 0 : accel_compress_get_ctx_size(void)
812 : {
813 0 : return 0;
814 : }
815 :
816 : static bool
817 0 : compress_supports_opcode(enum spdk_accel_opcode opc)
818 : {
819 0 : if (g_mlx5_pci_available || g_qat_available) {
820 0 : switch (opc) {
821 : case SPDK_ACCEL_OPC_COMPRESS:
822 : case SPDK_ACCEL_OPC_DECOMPRESS:
823 0 : return true;
824 : default:
825 0 : break;
826 : }
827 0 : }
828 :
829 0 : return false;
830 0 : }
831 :
832 : static struct spdk_io_channel *
833 0 : compress_get_io_channel(void)
834 : {
835 0 : return spdk_get_io_channel(&g_compress_module);
836 : }
837 :
838 : static void accel_compress_exit(void *ctx);
839 : static struct spdk_accel_module_if g_compress_module = {
840 : .module_init = accel_compress_init,
841 : .module_fini = accel_compress_exit,
842 : .write_config_json = accel_compress_write_config_json,
843 : .get_ctx_size = accel_compress_get_ctx_size,
844 : .name = "dpdk_compressdev",
845 : .supports_opcode = compress_supports_opcode,
846 : .get_io_channel = compress_get_io_channel,
847 : .submit_tasks = compress_submit_tasks
848 : };
849 :
850 : void
851 0 : accel_dpdk_compressdev_enable(void)
852 : {
853 0 : spdk_accel_module_list_add(&g_compress_module);
854 0 : }
855 :
856 : /* Callback for unregistering the IO device. */
857 : static void
858 0 : _device_unregister_cb(void *io_device)
859 : {
860 0 : struct comp_device_qp *dev_qp;
861 0 : struct compress_dev *device;
862 :
863 0 : while ((device = TAILQ_FIRST(&g_compress_devs))) {
864 0 : TAILQ_REMOVE(&g_compress_devs, device, link);
865 0 : free(device);
866 : }
867 :
868 0 : while ((dev_qp = TAILQ_FIRST(&g_comp_device_qp))) {
869 0 : TAILQ_REMOVE(&g_comp_device_qp, dev_qp, link);
870 0 : free(dev_qp);
871 : }
872 :
873 0 : pthread_mutex_destroy(&g_comp_device_qp_lock);
874 :
875 0 : rte_mempool_free(g_comp_op_mp);
876 0 : rte_mempool_free(g_mbuf_mp);
877 :
878 0 : spdk_accel_module_finish();
879 0 : }
880 :
881 : static void
882 0 : accel_compress_exit(void *ctx)
883 : {
884 0 : if (g_compressdev_initialized) {
885 0 : spdk_io_device_unregister(&g_compress_module, _device_unregister_cb);
886 0 : g_compressdev_initialized = false;
887 0 : } else {
888 0 : spdk_accel_module_finish();
889 : }
890 0 : }
|