LCOV - code coverage report
Current view: top level - module/accel/dpdk_compressdev - accel_dpdk_compressdev.c (source / functions) Hit Total Coverage
Test: ut_cov_unit.info Lines: 262 461 56.8 %
Date: 2024-11-17 05:04:02 Functions: 5 20 25.0 %

          Line data    Source code
       1             : /*   SPDX-License-Identifier: BSD-3-Clause
       2             :  *   Copyright (C) 2018 Intel Corporation.
       3             :  *   All rights reserved.
       4             :  *   Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
       5             :  */
       6             : 
       7             : #include "accel_dpdk_compressdev.h"
       8             : #include "spdk/accel_module.h"
       9             : 
      10             : #include "spdk/stdinc.h"
      11             : #include "spdk/rpc.h"
      12             : #include "spdk/env.h"
      13             : #include "spdk/endian.h"
      14             : #include "spdk/string.h"
      15             : #include "spdk/thread.h"
      16             : #include "spdk/util.h"
      17             : #include "spdk/likely.h"
      18             : 
      19             : #include "spdk/log.h"
      20             : 
      21             : #include <rte_config.h>
      22             : #include <rte_bus_vdev.h>
      23             : #include <rte_compressdev.h>
      24             : #include <rte_comp.h>
      25             : #include <rte_mbuf_dyn.h>
      26             : 
      27             : /* Used to store IO context in mbuf */
      28             : static const struct rte_mbuf_dynfield rte_mbuf_dynfield_io_context = {
      29             :         .name = "context_accel_comp",
      30             :         .size = sizeof(uint64_t),
      31             :         .align = __alignof__(uint64_t),
      32             :         .flags = 0,
      33             : };
      34             : static int g_mbuf_offset;
      35             : static enum compress_pmd g_opts;
      36             : static bool g_compressdev_enable = false;
      37             : static bool g_compressdev_initialized = false;
      38             : 
      39             : #define NUM_MAX_XFORMS          2
      40             : #define NUM_MAX_INFLIGHT_OPS    128
      41             : #define DEFAULT_WINDOW_SIZE     15
      42             : #define MBUF_SPLIT              (1UL << DEFAULT_WINDOW_SIZE)
      43             : #define QAT_PMD                 "compress_qat"
      44             : #define MLX5_PMD                "mlx5_pci"
      45             : #define NUM_MBUFS               65536
      46             : #define POOL_CACHE_SIZE         256
      47             : 
      48             : /* Global list of available compression devices. */
      49             : struct compress_dev {
      50             :         struct rte_compressdev_info     cdev_info;      /* includes device friendly name */
      51             :         uint8_t                         cdev_id;        /* identifier for the device */
      52             :         void                            *comp_xform;    /* shared private xform for comp on this PMD */
      53             :         void                            *decomp_xform;  /* shared private xform for decomp on this PMD */
      54             :         bool                            sgl_in;
      55             :         bool                            sgl_out;
      56             :         TAILQ_ENTRY(compress_dev)       link;
      57             : };
      58             : static TAILQ_HEAD(, compress_dev) g_compress_devs = TAILQ_HEAD_INITIALIZER(g_compress_devs);
      59             : 
      60             : #define MAX_NUM_QP 48
      61             : /* Global list and lock for unique device/queue pair combos */
      62             : struct comp_device_qp {
      63             :         struct compress_dev             *device;        /* ptr to compression device */
      64             :         uint8_t                         qp;             /* queue pair for this node */
      65             :         struct compress_io_channel      *chan;
      66             :         TAILQ_ENTRY(comp_device_qp)     link;
      67             : };
      68             : static TAILQ_HEAD(, comp_device_qp) g_comp_device_qp = TAILQ_HEAD_INITIALIZER(g_comp_device_qp);
      69             : static pthread_mutex_t g_comp_device_qp_lock = PTHREAD_MUTEX_INITIALIZER;
      70             : 
      71             : struct compress_io_channel {
      72             :         char                            *drv_name;      /* name of the compression device driver */
      73             :         struct comp_device_qp           *device_qp;
      74             :         struct spdk_poller              *poller;
      75             :         struct rte_mbuf                 **src_mbufs;
      76             :         struct rte_mbuf                 **dst_mbufs;
      77             :         STAILQ_HEAD(, spdk_accel_task)  queued_tasks;
      78             : };
      79             : 
      80             : /* Shared mempools between all devices on this system */
      81             : static struct rte_mempool *g_mbuf_mp = NULL;            /* mbuf mempool */
      82             : static struct rte_mempool *g_comp_op_mp = NULL;         /* comp operations, must be rte* mempool */
      83             : static struct rte_mbuf_ext_shared_info g_shinfo = {};   /* used by DPDK mbuf macros */
      84             : static bool g_qat_available = false;
      85             : static bool g_mlx5_pci_available = false;
      86             : 
      87             : /* Create shared (between all ops per PMD) compress xforms. */
      88             : static struct rte_comp_xform g_comp_xform = {
      89             :         .type = RTE_COMP_COMPRESS,
      90             :         .compress = {
      91             :                 .algo = RTE_COMP_ALGO_DEFLATE,
      92             :                 .deflate.huffman = RTE_COMP_HUFFMAN_DEFAULT,
      93             :                 .level = RTE_COMP_LEVEL_MAX,
      94             :                 .window_size = DEFAULT_WINDOW_SIZE,
      95             :                 .chksum = RTE_COMP_CHECKSUM_NONE,
      96             :                 .hash_algo = RTE_COMP_HASH_ALGO_NONE
      97             :         }
      98             : };
      99             : /* Create shared (between all ops per PMD) decompress xforms. */
     100             : static struct rte_comp_xform g_decomp_xform = {
     101             :         .type = RTE_COMP_DECOMPRESS,
     102             :         .decompress = {
     103             :                 .algo = RTE_COMP_ALGO_DEFLATE,
     104             :                 .chksum = RTE_COMP_CHECKSUM_NONE,
     105             :                 .window_size = DEFAULT_WINDOW_SIZE,
     106             :                 .hash_algo = RTE_COMP_HASH_ALGO_NONE
     107             :         }
     108             : };
     109             : 
     110             : /* Dummy function used by DPDK to free ext attached buffers
     111             :  * to mbufs, we free them ourselves but this callback has to
     112             :  * be here.
     113             :  */
     114             : static void
     115           0 : shinfo_free_cb(void *arg1, void *arg2)
     116             : {
     117           0 : }
     118             : 
     119             : /* Called by accel_init_compress_drivers() to init each discovered compression device */
     120             : static int
     121           6 : create_compress_dev(uint8_t index)
     122             : {
     123           6 :         struct compress_dev *device;
     124           6 :         uint16_t q_pairs;
     125           6 :         uint8_t cdev_id;
     126           6 :         int rc, i;
     127           6 :         struct comp_device_qp *dev_qp;
     128           6 :         struct comp_device_qp *tmp_qp;
     129             : 
     130           6 :         device = calloc(1, sizeof(struct compress_dev));
     131           6 :         if (!device) {
     132           0 :                 return -ENOMEM;
     133             :         }
     134             : 
     135             :         /* Get details about this device. */
     136           6 :         rte_compressdev_info_get(index, &device->cdev_info);
     137             : 
     138           6 :         cdev_id = device->cdev_id = index;
     139             : 
     140             :         /* Zero means no limit so choose number of lcores. */
     141           6 :         if (device->cdev_info.max_nb_queue_pairs == 0) {
     142           5 :                 q_pairs = MAX_NUM_QP;
     143           5 :         } else {
     144           1 :                 q_pairs = spdk_min(device->cdev_info.max_nb_queue_pairs, MAX_NUM_QP);
     145             :         }
     146             : 
     147             :         /* Configure the compression device. */
     148          18 :         struct rte_compressdev_config config = {
     149           6 :                 .socket_id = rte_socket_id(),
     150           6 :                 .nb_queue_pairs = q_pairs,
     151             :                 .max_nb_priv_xforms = NUM_MAX_XFORMS,
     152             :                 .max_nb_streams = 0
     153             :         };
     154           6 :         rc = rte_compressdev_configure(cdev_id, &config);
     155           6 :         if (rc < 0) {
     156           2 :                 SPDK_ERRLOG("Failed to configure compressdev %u\n", cdev_id);
     157           2 :                 goto err;
     158             :         }
     159             : 
     160             :         /* Pre-setup all potential qpairs now and assign them in the channel
     161             :          * callback.
     162             :          */
     163         148 :         for (i = 0; i < q_pairs; i++) {
     164         290 :                 rc = rte_compressdev_queue_pair_setup(cdev_id, i,
     165             :                                                       NUM_MAX_INFLIGHT_OPS,
     166         145 :                                                       rte_socket_id());
     167         145 :                 if (rc) {
     168           1 :                         if (i > 0) {
     169           0 :                                 q_pairs = i;
     170           0 :                                 SPDK_NOTICELOG("FYI failed to setup a queue pair on "
     171             :                                                "compressdev %u with error %u "
     172             :                                                "so limiting to %u qpairs\n",
     173             :                                                cdev_id, rc, q_pairs);
     174           0 :                                 break;
     175             :                         } else {
     176           1 :                                 SPDK_ERRLOG("Failed to setup queue pair on "
     177             :                                             "compressdev %u with error %u\n", cdev_id, rc);
     178           1 :                                 rc = -EINVAL;
     179           1 :                                 goto err;
     180             :                         }
     181             :                 }
     182         144 :         }
     183             : 
     184           3 :         rc = rte_compressdev_start(cdev_id);
     185           3 :         if (rc < 0) {
     186           1 :                 SPDK_ERRLOG("Failed to start device %u: error %d\n",
     187             :                             cdev_id, rc);
     188           1 :                 goto err;
     189             :         }
     190             : 
     191           2 :         if (device->cdev_info.capabilities->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
     192           2 :                 rc = rte_compressdev_private_xform_create(cdev_id, &g_comp_xform,
     193           2 :                                 &device->comp_xform);
     194           2 :                 if (rc < 0) {
     195           1 :                         SPDK_ERRLOG("Failed to create private comp xform device %u: error %d\n",
     196             :                                     cdev_id, rc);
     197           1 :                         goto err;
     198             :                 }
     199             : 
     200           1 :                 rc = rte_compressdev_private_xform_create(cdev_id, &g_decomp_xform,
     201           1 :                                 &device->decomp_xform);
     202           1 :                 if (rc) {
     203           0 :                         SPDK_ERRLOG("Failed to create private decomp xform device %u: error %d\n",
     204             :                                     cdev_id, rc);
     205           0 :                         goto err;
     206             :                 }
     207           1 :         } else {
     208           0 :                 SPDK_ERRLOG("PMD does not support shared transforms\n");
     209           0 :                 goto err;
     210             :         }
     211             : 
     212             :         /* Build up list of device/qp combinations */
     213          49 :         for (i = 0; i < q_pairs; i++) {
     214          48 :                 dev_qp = calloc(1, sizeof(struct comp_device_qp));
     215          48 :                 if (!dev_qp) {
     216           0 :                         rc = -ENOMEM;
     217           0 :                         goto err;
     218             :                 }
     219          48 :                 dev_qp->device = device;
     220          48 :                 dev_qp->qp = i;
     221          48 :                 dev_qp->chan = NULL;
     222          48 :                 TAILQ_INSERT_TAIL(&g_comp_device_qp, dev_qp, link);
     223          48 :         }
     224             : 
     225           1 :         TAILQ_INSERT_TAIL(&g_compress_devs, device, link);
     226             : 
     227           1 :         if (strcmp(device->cdev_info.driver_name, QAT_PMD) == 0) {
     228           0 :                 g_qat_available = true;
     229           0 :         }
     230             : 
     231           1 :         if (strcmp(device->cdev_info.driver_name, MLX5_PMD) == 0) {
     232           0 :                 g_mlx5_pci_available = true;
     233           0 :         }
     234             : 
     235           1 :         return 0;
     236             : 
     237             : err:
     238           5 :         TAILQ_FOREACH_SAFE(dev_qp, &g_comp_device_qp, link, tmp_qp) {
     239           0 :                 TAILQ_REMOVE(&g_comp_device_qp, dev_qp, link);
     240           0 :                 free(dev_qp);
     241           0 :         }
     242           5 :         free(device);
     243           5 :         return rc;
     244           6 : }
     245             : 
     246             : /* Called from driver init entry point, accel_compress_init() */
     247             : static int
     248           9 : accel_init_compress_drivers(void)
     249             : {
     250           9 :         uint8_t cdev_count, i;
     251           9 :         struct compress_dev *tmp_dev;
     252           9 :         struct compress_dev *device;
     253           9 :         int rc;
     254             : 
     255             :         /* If we have no compression devices, there's no reason to continue. */
     256           9 :         cdev_count = rte_compressdev_count();
     257           9 :         if (cdev_count == 0) {
     258           1 :                 return 0;
     259             :         }
     260           8 :         if (cdev_count > RTE_COMPRESS_MAX_DEVS) {
     261           1 :                 SPDK_ERRLOG("invalid device count from rte_compressdev_count()\n");
     262           1 :                 return -EINVAL;
     263             :         }
     264             : 
     265           7 :         g_mbuf_offset = rte_mbuf_dynfield_register(&rte_mbuf_dynfield_io_context);
     266           7 :         if (g_mbuf_offset < 0) {
     267           1 :                 SPDK_ERRLOG("error registering dynamic field with DPDK\n");
     268           1 :                 return -EINVAL;
     269             :         }
     270             : 
     271           6 :         g_mbuf_mp = rte_pktmbuf_pool_create("comp_mbuf_mp", NUM_MBUFS, POOL_CACHE_SIZE,
     272           6 :                                             sizeof(struct rte_mbuf), 0, rte_socket_id());
     273           6 :         if (g_mbuf_mp == NULL) {
     274           0 :                 SPDK_ERRLOG("Cannot create mbuf pool\n");
     275           0 :                 rc = -ENOMEM;
     276           0 :                 goto error_create_mbuf;
     277             :         }
     278             : 
     279           6 :         g_comp_op_mp = rte_comp_op_pool_create("comp_op_pool", NUM_MBUFS, POOL_CACHE_SIZE,
     280           6 :                                                0, rte_socket_id());
     281           6 :         if (g_comp_op_mp == NULL) {
     282           0 :                 SPDK_ERRLOG("Cannot create comp op pool\n");
     283           0 :                 rc = -ENOMEM;
     284           0 :                 goto error_create_op;
     285             :         }
     286             : 
     287             :         /* Init all devices */
     288           7 :         for (i = 0; i < cdev_count; i++) {
     289           6 :                 rc = create_compress_dev(i);
     290           6 :                 if (rc != 0) {
     291           5 :                         goto error_create_compress_devs;
     292             :                 }
     293           1 :         }
     294             : 
     295           1 :         if (g_qat_available == true) {
     296           0 :                 SPDK_NOTICELOG("initialized QAT PMD\n");
     297           0 :         }
     298             : 
     299           1 :         g_shinfo.free_cb = shinfo_free_cb;
     300             : 
     301           1 :         return 0;
     302             : 
     303             :         /* Error cleanup paths. */
     304             : error_create_compress_devs:
     305           5 :         TAILQ_FOREACH_SAFE(device, &g_compress_devs, link, tmp_dev) {
     306           0 :                 TAILQ_REMOVE(&g_compress_devs, device, link);
     307           0 :                 free(device);
     308           5 :         }
     309             : error_create_op:
     310             : error_create_mbuf:
     311           5 :         rte_mempool_free(g_mbuf_mp);
     312             : 
     313           5 :         return rc;
     314           9 : }
     315             : 
     316             : int
     317           0 : accel_compressdev_enable_probe(enum compress_pmd *opts)
     318             : {
     319           0 :         g_opts = *opts;
     320           0 :         g_compressdev_enable = true;
     321             : 
     322           0 :         return 0;
     323             : }
     324             : 
     325             : static int
     326          23 : _setup_compress_mbuf(struct rte_mbuf **mbufs, int *mbuf_total, uint64_t *total_length,
     327             :                      struct iovec *iovs, int iovcnt, struct spdk_accel_task *task)
     328             : {
     329          23 :         uint64_t iovec_length, updated_length, phys_addr;
     330          23 :         uint64_t processed, mbuf_length, remainder;
     331          23 :         uint8_t *current_base = NULL;
     332          23 :         int iov_index, mbuf_index;
     333          23 :         int rc = 0;
     334             : 
     335             :         /* Setup mbufs */
     336          23 :         iov_index = mbuf_index = 0;
     337          86 :         while (iov_index < iovcnt) {
     338             : 
     339          63 :                 processed = 0;
     340          63 :                 iovec_length = iovs[iov_index].iov_len;
     341             : 
     342          63 :                 current_base = iovs[iov_index].iov_base;
     343          63 :                 if (total_length) {
     344          36 :                         *total_length += iovec_length;
     345          36 :                 }
     346             : 
     347          63 :                 assert(mbufs[mbuf_index] != NULL);
     348          63 :                 *RTE_MBUF_DYNFIELD(mbufs[mbuf_index], g_mbuf_offset, uint64_t *) = (uint64_t)task;
     349             : 
     350          63 :                 do {
     351             :                         /* new length is min of remaining left or max mbuf size of MBUF_SPLIT */
     352          66 :                         mbuf_length = updated_length = spdk_min(MBUF_SPLIT, iovec_length - processed);
     353             : 
     354          66 :                         phys_addr = spdk_vtophys((void *)current_base, &updated_length);
     355             : 
     356          66 :                         rte_pktmbuf_attach_extbuf(mbufs[mbuf_index],
     357          66 :                                                   current_base,
     358          66 :                                                   phys_addr,
     359          66 :                                                   updated_length,
     360             :                                                   &g_shinfo);
     361          66 :                         rte_pktmbuf_append(mbufs[mbuf_index], updated_length);
     362          66 :                         remainder = mbuf_length - updated_length;
     363             : 
     364             :                         /* although the mbufs were preallocated, we still need to chain them */
     365          66 :                         if (mbuf_index > 0) {
     366          43 :                                 rte_pktmbuf_chain(mbufs[0], mbufs[mbuf_index]);
     367          43 :                         }
     368             : 
     369             :                         /* keep track of the total we've put into the mbuf chain */
     370          66 :                         processed += updated_length;
     371             :                         /* bump the base by what was previously added */
     372          66 :                         current_base += updated_length;
     373             : 
     374             :                         /* If we crossed 2MB boundary we need another mbuf for the remainder */
     375          66 :                         if (remainder > 0) {
     376             : 
     377           5 :                                 assert(remainder <= MBUF_SPLIT);
     378             : 
     379             :                                 /* allocate an mbuf at the end of the array */
     380          10 :                                 rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp,
     381           5 :                                                             (struct rte_mbuf **)&mbufs[*mbuf_total], 1);
     382           5 :                                 if (rc) {
     383           0 :                                         SPDK_ERRLOG("ERROR trying to get an extra mbuf!\n");
     384           0 :                                         return -1;
     385             :                                 }
     386           5 :                                 (*mbuf_total)++;
     387           5 :                                 mbuf_index++;
     388           5 :                                 *RTE_MBUF_DYNFIELD(mbufs[mbuf_index], g_mbuf_offset, uint64_t *) = (uint64_t)task;
     389             : 
     390             :                                 /* bump the base by what was previously added */
     391           5 :                                 current_base += updated_length;
     392             : 
     393           5 :                                 updated_length = remainder;
     394           5 :                                 phys_addr = spdk_vtophys((void *)current_base, &updated_length);
     395             : 
     396             :                                 /* assert we don't cross another */
     397           5 :                                 assert(remainder == updated_length);
     398             : 
     399           5 :                                 rte_pktmbuf_attach_extbuf(mbufs[mbuf_index],
     400           5 :                                                           current_base,
     401           5 :                                                           phys_addr,
     402           5 :                                                           remainder,
     403             :                                                           &g_shinfo);
     404           5 :                                 rte_pktmbuf_append(mbufs[mbuf_index], remainder);
     405           5 :                                 rte_pktmbuf_chain(mbufs[0], mbufs[mbuf_index]);
     406             : 
     407             :                                 /* keep track of the total we've put into the mbuf chain */
     408           5 :                                 processed += remainder;
     409           5 :                         }
     410             : 
     411          66 :                         mbuf_index++;
     412             : 
     413          66 :                 } while (processed < iovec_length);
     414             : 
     415          63 :                 assert(processed == iovec_length);
     416          63 :                 iov_index++;
     417             :         }
     418             : 
     419          23 :         return 0;
     420          23 : }
     421             : 
     422             : static int
     423          13 : _compress_operation(struct compress_io_channel *chan,  struct spdk_accel_task *task)
     424             : {
     425          13 :         int dst_iovcnt = task->d.iovcnt;
     426          13 :         struct iovec *dst_iovs = task->d.iovs;
     427          13 :         int src_iovcnt = task->s.iovcnt;
     428          13 :         struct iovec *src_iovs = task->s.iovs;
     429          13 :         struct rte_comp_op *comp_op;
     430          13 :         uint8_t cdev_id;
     431          13 :         uint64_t total_length = 0;
     432          13 :         int rc = 0, i;
     433          13 :         int src_mbuf_total = 0;
     434          13 :         int dst_mbuf_total = 0;
     435          13 :         bool device_error = false;
     436          13 :         bool compress = (task->op_code == SPDK_ACCEL_OPC_COMPRESS);
     437             : 
     438          13 :         assert(chan->device_qp->device != NULL);
     439          13 :         cdev_id = chan->device_qp->device->cdev_id;
     440             : 
     441             :         /* calc our mbuf totals based on max MBUF size allowed so we can pre-alloc mbufs in bulk */
     442          52 :         for (i = 0 ; i < src_iovcnt; i++) {
     443          39 :                 src_mbuf_total += spdk_divide_round_up(src_iovs[i].iov_len, MBUF_SPLIT);
     444          39 :         }
     445          52 :         for (i = 0 ; i < dst_iovcnt; i++) {
     446          39 :                 dst_mbuf_total += spdk_divide_round_up(dst_iovs[i].iov_len, MBUF_SPLIT);
     447          39 :         }
     448             : 
     449          13 :         comp_op = rte_comp_op_alloc(g_comp_op_mp);
     450          13 :         if (!comp_op) {
     451           1 :                 SPDK_ERRLOG("trying to get a comp op!\n");
     452           1 :                 rc = -ENOMEM;
     453           1 :                 goto error_get_op;
     454             :         }
     455             : 
     456             :         /* get an mbuf per iov, src and dst */
     457          12 :         rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, chan->src_mbufs, src_mbuf_total);
     458          12 :         if (rc) {
     459           1 :                 SPDK_ERRLOG("ERROR trying to get src_mbufs!\n");
     460           1 :                 rc = -ENOMEM;
     461           1 :                 goto error_get_src;
     462             :         }
     463          11 :         assert(chan->src_mbufs[0]);
     464             : 
     465          11 :         rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, chan->dst_mbufs, dst_mbuf_total);
     466          11 :         if (rc) {
     467           0 :                 SPDK_ERRLOG("ERROR trying to get dst_mbufs!\n");
     468           0 :                 rc = -ENOMEM;
     469           0 :                 goto error_get_dst;
     470             :         }
     471          11 :         assert(chan->dst_mbufs[0]);
     472             : 
     473          11 :         rc = _setup_compress_mbuf(chan->src_mbufs, &src_mbuf_total, &total_length,
     474          11 :                                   src_iovs, src_iovcnt, task);
     475             : 
     476          11 :         if (rc < 0) {
     477           0 :                 goto error_src_dst;
     478             :         }
     479          11 :         if (!chan->device_qp->device->sgl_in && src_mbuf_total > 1) {
     480           2 :                 SPDK_ERRLOG("Src buffer uses chained mbufs but driver %s doesn't support SGL input\n",
     481             :                             chan->drv_name);
     482           2 :                 rc = -EINVAL;
     483           2 :                 goto error_src_dst;
     484             :         }
     485             : 
     486           9 :         comp_op->m_src = chan->src_mbufs[0];
     487           9 :         comp_op->src.offset = 0;
     488           9 :         comp_op->src.length = total_length;
     489             : 
     490           9 :         rc = _setup_compress_mbuf(chan->dst_mbufs, &dst_mbuf_total, NULL,
     491           9 :                                   dst_iovs, dst_iovcnt, task);
     492           9 :         if (rc < 0) {
     493           0 :                 goto error_src_dst;
     494             :         }
     495           9 :         if (!chan->device_qp->device->sgl_out && dst_mbuf_total > 1) {
     496           2 :                 SPDK_ERRLOG("Dst buffer uses chained mbufs but driver %s doesn't support SGL output\n",
     497             :                             chan->drv_name);
     498           2 :                 rc = -EINVAL;
     499           2 :                 goto error_src_dst;
     500             :         }
     501             : 
     502           7 :         comp_op->m_dst = chan->dst_mbufs[0];
     503           7 :         comp_op->dst.offset = 0;
     504             : 
     505           7 :         if (compress == true) {
     506           7 :                 comp_op->private_xform = chan->device_qp->device->comp_xform;
     507           7 :         } else {
     508           0 :                 comp_op->private_xform = chan->device_qp->device->decomp_xform;
     509             :         }
     510             : 
     511           7 :         comp_op->op_type = RTE_COMP_OP_STATELESS;
     512           7 :         comp_op->flush_flag = RTE_COMP_FLUSH_FINAL;
     513             : 
     514           7 :         rc = rte_compressdev_enqueue_burst(cdev_id, chan->device_qp->qp, &comp_op, 1);
     515           7 :         assert(rc <= 1);
     516             : 
     517             :         /* We always expect 1 got queued, if 0 then we need to queue it up. */
     518           9 :         if (rc == 1) {
     519           5 :                 return 0;
     520           2 :         } else if (comp_op->status == RTE_COMP_OP_STATUS_NOT_PROCESSED) {
     521           1 :                 rc = -EAGAIN;
     522           1 :         } else {
     523           1 :                 device_error = true;
     524             :         }
     525             : 
     526             :         /* Error cleanup paths. */
     527             : error_src_dst:
     528           6 :         rte_pktmbuf_free_bulk(chan->dst_mbufs, dst_iovcnt);
     529             : error_get_dst:
     530           6 :         rte_pktmbuf_free_bulk(chan->src_mbufs, src_iovcnt);
     531             : error_get_src:
     532           7 :         rte_comp_op_free(comp_op);
     533             : error_get_op:
     534             : 
     535           8 :         if (device_error == true) {
     536             :                 /* There was an error sending the op to the device, most
     537             :                  * likely with the parameters.
     538             :                  */
     539           1 :                 SPDK_ERRLOG("Compression API returned 0x%x\n", comp_op->status);
     540           1 :                 return -EINVAL;
     541             :         }
     542           7 :         if (rc != -ENOMEM && rc != -EAGAIN) {
     543           4 :                 return rc;
     544             :         }
     545             : 
     546           3 :         STAILQ_INSERT_TAIL(&chan->queued_tasks, task, link);
     547           3 :         return 0;
     548          13 : }
     549             : 
     550             : /* Poller for the DPDK compression driver. */
     551             : static int
     552           3 : comp_dev_poller(void *args)
     553             : {
     554           3 :         struct compress_io_channel *chan = args;
     555           3 :         uint8_t cdev_id;
     556           3 :         struct rte_comp_op *deq_ops[NUM_MAX_INFLIGHT_OPS];
     557           3 :         uint16_t num_deq;
     558           3 :         struct spdk_accel_task *task, *task_to_resubmit;
     559           3 :         int rc, i, status;
     560             : 
     561           3 :         assert(chan->device_qp->device != NULL);
     562           3 :         cdev_id = chan->device_qp->device->cdev_id;
     563             : 
     564           3 :         num_deq = rte_compressdev_dequeue_burst(cdev_id, chan->device_qp->qp, deq_ops,
     565             :                                                 NUM_MAX_INFLIGHT_OPS);
     566           7 :         for (i = 0; i < num_deq; i++) {
     567             : 
     568             :                 /* We store this off regardless of success/error so we know how to contruct the
     569             :                  * next task
     570             :                  */
     571           4 :                 task = (struct spdk_accel_task *)*RTE_MBUF_DYNFIELD(deq_ops[i]->m_src, g_mbuf_offset,
     572             :                                 uint64_t *);
     573           4 :                 status = deq_ops[i]->status;
     574             : 
     575           4 :                 if (spdk_likely(status == RTE_COMP_OP_STATUS_SUCCESS)) {
     576           3 :                         if (task->output_size != NULL) {
     577           3 :                                 *task->output_size = deq_ops[i]->produced;
     578           3 :                         }
     579           3 :                 } else {
     580           1 :                         SPDK_NOTICELOG("Deque status %u\n", status);
     581             :                 }
     582             : 
     583           4 :                 spdk_accel_task_complete(task, status);
     584             : 
     585             :                 /* Now free both mbufs and the compress operation. The rte_pktmbuf_free()
     586             :                  * call takes care of freeing all of the mbufs in the chain back to their
     587             :                  * original pool.
     588             :                  */
     589           4 :                 rte_pktmbuf_free(deq_ops[i]->m_src);
     590           4 :                 rte_pktmbuf_free(deq_ops[i]->m_dst);
     591             : 
     592             :                 /* There is no bulk free for com ops so we have to free them one at a time
     593             :                  * here however it would be rare that we'd ever have more than 1 at a time
     594             :                  * anyways.
     595             :                  */
     596           4 :                 rte_comp_op_free(deq_ops[i]);
     597             : 
     598             :                 /* Check if there are any pending comp ops to process, only pull one
     599             :                  * at a time off as _compress_operation() may re-queue the op.
     600             :                  */
     601           4 :                 if (!STAILQ_EMPTY(&chan->queued_tasks)) {
     602           1 :                         task_to_resubmit = STAILQ_FIRST(&chan->queued_tasks);
     603           1 :                         rc = _compress_operation(chan, task_to_resubmit);
     604           1 :                         if (rc == 0) {
     605           1 :                                 STAILQ_REMOVE_HEAD(&chan->queued_tasks, link);
     606           1 :                         }
     607           1 :                 }
     608           4 :         }
     609             : 
     610           6 :         return num_deq == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY;
     611           3 : }
     612             : 
     613             : static int
     614           0 : _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task)
     615             : {
     616           0 :         struct compress_io_channel *chan = spdk_io_channel_get_ctx(ch);
     617           0 :         int rc;
     618             : 
     619           0 :         rc = _compress_operation(chan, task);
     620           0 :         if (rc) {
     621           0 :                 SPDK_ERRLOG("Error (%d) in comrpess operation\n", rc);
     622           0 :                 assert(false);
     623             :         }
     624             : 
     625           0 :         return rc;
     626           0 : }
     627             : 
     628             : static int
     629           0 : compress_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *first_task)
     630             : {
     631           0 :         struct compress_io_channel *chan = spdk_io_channel_get_ctx(ch);
     632           0 :         struct spdk_accel_task *task, *tmp;
     633           0 :         int rc = 0;
     634             : 
     635           0 :         task = first_task;
     636             : 
     637           0 :         if (!STAILQ_EMPTY(&chan->queued_tasks)) {
     638           0 :                 goto queue_tasks;
     639             :         }
     640             : 
     641             :         /* The caller will either submit a single task or a group of tasks that are
     642             :          * linked together but they cannot be on a list. For example, see poller
     643             :          * where a list of queued tasks is being resubmitted, the list they are on
     644             :          * is initialized after saving off the first task from the list which is then
     645             :          * passed in here.  Similar thing is done in the accel framework.
     646             :          */
     647           0 :         while (task) {
     648           0 :                 tmp = STAILQ_NEXT(task, link);
     649           0 :                 rc = _process_single_task(ch, task);
     650             : 
     651           0 :                 if (rc == -EBUSY) {
     652           0 :                         goto queue_tasks;
     653           0 :                 } else if (rc) {
     654           0 :                         spdk_accel_task_complete(task, rc);
     655           0 :                 }
     656           0 :                 task = tmp;
     657             :         }
     658             : 
     659           0 :         return 0;
     660             : 
     661             : queue_tasks:
     662           0 :         while (task != NULL) {
     663           0 :                 tmp = STAILQ_NEXT(task, link);
     664           0 :                 STAILQ_INSERT_TAIL(&chan->queued_tasks, task, link);
     665           0 :                 task = tmp;
     666             :         }
     667           0 :         return 0;
     668           0 : }
     669             : 
     670             : static bool
     671           0 : _set_pmd(struct compress_io_channel *chan)
     672             : {
     673             : 
     674             :         /* Note: the compress_isal PMD is not supported as accel_fw supports native ISAL
     675             :          * using the accel_sw module */
     676           0 :         if (g_opts == COMPRESS_PMD_AUTO) {
     677           0 :                 if (g_qat_available) {
     678           0 :                         chan->drv_name = QAT_PMD;
     679           0 :                 } else if (g_mlx5_pci_available) {
     680           0 :                         chan->drv_name = MLX5_PMD;
     681           0 :                 }
     682           0 :         } else if (g_opts == COMPRESS_PMD_QAT_ONLY && g_qat_available) {
     683           0 :                 chan->drv_name = QAT_PMD;
     684           0 :         } else if (g_opts == COMPRESS_PMD_MLX5_PCI_ONLY && g_mlx5_pci_available) {
     685           0 :                 chan->drv_name = MLX5_PMD;
     686           0 :         } else {
     687           0 :                 SPDK_ERRLOG("Requested PMD is not available.\n");
     688           0 :                 return false;
     689             :         }
     690           0 :         SPDK_NOTICELOG("Channel %p PMD being used: %s\n", chan, chan->drv_name);
     691           0 :         return true;
     692           0 : }
     693             : 
     694             : static int compress_create_cb(void *io_device, void *ctx_buf);
     695             : static void compress_destroy_cb(void *io_device, void *ctx_buf);
     696             : static struct spdk_accel_module_if g_compress_module;
     697             : static int
     698           0 : accel_compress_init(void)
     699             : {
     700           0 :         int rc;
     701             : 
     702           0 :         if (!g_compressdev_enable) {
     703           0 :                 return -EINVAL;
     704             :         }
     705             : 
     706           0 :         rc = accel_init_compress_drivers();
     707           0 :         if (rc) {
     708           0 :                 assert(TAILQ_EMPTY(&g_compress_devs));
     709           0 :                 SPDK_NOTICELOG("no available compression devices\n");
     710           0 :                 return -EINVAL;
     711             :         }
     712             : 
     713           0 :         g_compressdev_initialized = true;
     714           0 :         spdk_io_device_register(&g_compress_module, compress_create_cb, compress_destroy_cb,
     715             :                                 sizeof(struct compress_io_channel), "compressdev_accel_module");
     716           0 :         return 0;
     717           0 : }
     718             : 
     719             : static int
     720           0 : compress_create_cb(void *io_device, void *ctx_buf)
     721             : {
     722           0 :         struct compress_io_channel *chan = ctx_buf;
     723           0 :         const struct rte_compressdev_capabilities *capab;
     724           0 :         struct comp_device_qp *device_qp;
     725           0 :         size_t length;
     726             : 
     727           0 :         if (_set_pmd(chan) == false) {
     728           0 :                 assert(false);
     729             :                 return -ENODEV;
     730             :         }
     731             : 
     732             :         /* The following variable length arrays of mbuf pointers are required to submit to compressdev */
     733           0 :         length = NUM_MBUFS * sizeof(void *);
     734           0 :         chan->src_mbufs = spdk_zmalloc(length, 0x40, NULL,
     735             :                                        SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
     736           0 :         if (chan->src_mbufs == NULL) {
     737           0 :                 return -ENOMEM;
     738             :         }
     739           0 :         chan->dst_mbufs = spdk_zmalloc(length, 0x40, NULL,
     740             :                                        SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
     741           0 :         if (chan->dst_mbufs == NULL) {
     742           0 :                 free(chan->src_mbufs);
     743           0 :                 return -ENOMEM;
     744             :         }
     745             : 
     746           0 :         chan->poller = SPDK_POLLER_REGISTER(comp_dev_poller, chan, 0);
     747           0 :         STAILQ_INIT(&chan->queued_tasks);
     748             : 
     749           0 :         pthread_mutex_lock(&g_comp_device_qp_lock);
     750           0 :         TAILQ_FOREACH(device_qp, &g_comp_device_qp, link) {
     751           0 :                 if (strcmp(device_qp->device->cdev_info.driver_name, chan->drv_name) == 0) {
     752           0 :                         if (device_qp->chan == NULL) {
     753           0 :                                 chan->device_qp = device_qp;
     754           0 :                                 device_qp->chan = chan;
     755           0 :                                 break;
     756             :                         }
     757           0 :                 }
     758           0 :         }
     759           0 :         pthread_mutex_unlock(&g_comp_device_qp_lock);
     760             : 
     761           0 :         if (chan->device_qp == NULL) {
     762           0 :                 SPDK_ERRLOG("out of qpairs, cannot assign one\n");
     763           0 :                 assert(false);
     764             :                 return -ENOMEM;
     765             :         } else {
     766           0 :                 capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
     767             : 
     768           0 :                 if (capab->comp_feature_flags & (RTE_COMP_FF_OOP_SGL_IN_SGL_OUT | RTE_COMP_FF_OOP_SGL_IN_LB_OUT)) {
     769           0 :                         chan->device_qp->device->sgl_in = true;
     770           0 :                 }
     771             : 
     772           0 :                 if (capab->comp_feature_flags & (RTE_COMP_FF_OOP_SGL_IN_SGL_OUT | RTE_COMP_FF_OOP_LB_IN_SGL_OUT)) {
     773           0 :                         chan->device_qp->device->sgl_out = true;
     774           0 :                 }
     775             :         }
     776             : 
     777           0 :         return 0;
     778           0 : }
     779             : 
     780             : static void
     781           0 : accel_compress_write_config_json(struct spdk_json_write_ctx *w)
     782             : {
     783           0 :         if (g_compressdev_enable) {
     784           0 :                 spdk_json_write_object_begin(w);
     785           0 :                 spdk_json_write_named_string(w, "method", "compressdev_scan_accel_module");
     786           0 :                 spdk_json_write_named_object_begin(w, "params");
     787           0 :                 spdk_json_write_named_uint32(w, "pmd", g_opts);
     788           0 :                 spdk_json_write_object_end(w);
     789           0 :                 spdk_json_write_object_end(w);
     790           0 :         }
     791           0 : }
     792             : 
     793             : static void
     794           0 : compress_destroy_cb(void *io_device, void *ctx_buf)
     795             : {
     796           0 :         struct compress_io_channel *chan = ctx_buf;
     797           0 :         struct comp_device_qp *device_qp = chan->device_qp;
     798             : 
     799           0 :         spdk_free(chan->src_mbufs);
     800           0 :         spdk_free(chan->dst_mbufs);
     801             : 
     802           0 :         spdk_poller_unregister(&chan->poller);
     803             : 
     804           0 :         pthread_mutex_lock(&g_comp_device_qp_lock);
     805           0 :         chan->device_qp = NULL;
     806           0 :         device_qp->chan = NULL;
     807           0 :         pthread_mutex_unlock(&g_comp_device_qp_lock);
     808           0 : }
     809             : 
     810             : static size_t
     811           0 : accel_compress_get_ctx_size(void)
     812             : {
     813           0 :         return 0;
     814             : }
     815             : 
     816             : static bool
     817           0 : compress_supports_opcode(enum spdk_accel_opcode opc)
     818             : {
     819           0 :         if (g_mlx5_pci_available || g_qat_available) {
     820           0 :                 switch (opc) {
     821             :                 case SPDK_ACCEL_OPC_COMPRESS:
     822             :                 case SPDK_ACCEL_OPC_DECOMPRESS:
     823           0 :                         return true;
     824             :                 default:
     825           0 :                         break;
     826             :                 }
     827           0 :         }
     828             : 
     829           0 :         return false;
     830           0 : }
     831             : 
     832             : static struct spdk_io_channel *
     833           0 : compress_get_io_channel(void)
     834             : {
     835           0 :         return spdk_get_io_channel(&g_compress_module);
     836             : }
     837             : 
     838             : static void accel_compress_exit(void *ctx);
     839             : static struct spdk_accel_module_if g_compress_module = {
     840             :         .module_init            = accel_compress_init,
     841             :         .module_fini            = accel_compress_exit,
     842             :         .write_config_json      = accel_compress_write_config_json,
     843             :         .get_ctx_size           = accel_compress_get_ctx_size,
     844             :         .name                   = "dpdk_compressdev",
     845             :         .supports_opcode        = compress_supports_opcode,
     846             :         .get_io_channel         = compress_get_io_channel,
     847             :         .submit_tasks           = compress_submit_tasks
     848             : };
     849             : 
     850             : void
     851           0 : accel_dpdk_compressdev_enable(void)
     852             : {
     853           0 :         spdk_accel_module_list_add(&g_compress_module);
     854           0 : }
     855             : 
     856             : /* Callback for unregistering the IO device. */
     857             : static void
     858           0 : _device_unregister_cb(void *io_device)
     859             : {
     860           0 :         struct comp_device_qp *dev_qp;
     861           0 :         struct compress_dev *device;
     862             : 
     863           0 :         while ((device = TAILQ_FIRST(&g_compress_devs))) {
     864           0 :                 TAILQ_REMOVE(&g_compress_devs, device, link);
     865           0 :                 free(device);
     866             :         }
     867             : 
     868           0 :         while ((dev_qp = TAILQ_FIRST(&g_comp_device_qp))) {
     869           0 :                 TAILQ_REMOVE(&g_comp_device_qp, dev_qp, link);
     870           0 :                 free(dev_qp);
     871             :         }
     872             : 
     873           0 :         pthread_mutex_destroy(&g_comp_device_qp_lock);
     874             : 
     875           0 :         rte_mempool_free(g_comp_op_mp);
     876           0 :         rte_mempool_free(g_mbuf_mp);
     877             : 
     878           0 :         spdk_accel_module_finish();
     879           0 : }
     880             : 
     881             : static void
     882           0 : accel_compress_exit(void *ctx)
     883             : {
     884           0 :         if (g_compressdev_initialized) {
     885           0 :                 spdk_io_device_unregister(&g_compress_module, _device_unregister_cb);
     886           0 :                 g_compressdev_initialized = false;
     887           0 :         } else {
     888           0 :                 spdk_accel_module_finish();
     889             :         }
     890           0 : }

Generated by: LCOV version 1.15