LCOV - code coverage report
Current view: top level - lib/accel - accel.c (source / functions) Hit Total Coverage
Test: ut_cov_unit.info Lines: 980 1608 60.9 %
Date: 2024-07-13 20:56:32 Functions: 71 110 64.5 %

          Line data    Source code
       1             : /*   SPDX-License-Identifier: BSD-3-Clause
       2             :  *   Copyright (C) 2020 Intel Corporation.
       3             :  *   Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES.
       4             :  *   All rights reserved.
       5             :  */
       6             : 
       7             : #include "spdk/stdinc.h"
       8             : 
       9             : #include "spdk/accel_module.h"
      10             : 
      11             : #include "accel_internal.h"
      12             : 
      13             : #include "spdk/dma.h"
      14             : #include "spdk/env.h"
      15             : #include "spdk/likely.h"
      16             : #include "spdk/log.h"
      17             : #include "spdk/thread.h"
      18             : #include "spdk/json.h"
      19             : #include "spdk/crc32.h"
      20             : #include "spdk/util.h"
      21             : #include "spdk/hexlify.h"
      22             : #include "spdk/string.h"
      23             : 
      24             : /* Accelerator Framework: The following provides a top level
      25             :  * generic API for the accelerator functions defined here. Modules,
      26             :  * such as the one in /module/accel/ioat, supply the implementation
      27             :  * with the exception of the pure software implementation contained
      28             :  * later in this file.
      29             :  */
      30             : 
      31             : #define ALIGN_4K                        0x1000
      32             : #define MAX_TASKS_PER_CHANNEL           0x800
      33             : #define ACCEL_SMALL_CACHE_SIZE          128
      34             : #define ACCEL_LARGE_CACHE_SIZE          16
      35             : /* Set MSB, so we don't return NULL pointers as buffers */
      36             : #define ACCEL_BUFFER_BASE               ((void *)(1ull << 63))
      37             : #define ACCEL_BUFFER_OFFSET_MASK        ((uintptr_t)ACCEL_BUFFER_BASE - 1)
      38             : 
      39             : #define ACCEL_CRYPTO_TWEAK_MODE_DEFAULT SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA
      40             : 
      41             : struct accel_module {
      42             :         struct spdk_accel_module_if     *module;
      43             :         bool                            supports_memory_domains;
      44             : };
      45             : 
      46             : /* Largest context size for all accel modules */
      47             : static size_t g_max_accel_module_size = sizeof(struct spdk_accel_task);
      48             : 
      49             : static struct spdk_accel_module_if *g_accel_module = NULL;
      50             : static spdk_accel_fini_cb g_fini_cb_fn = NULL;
      51             : static void *g_fini_cb_arg = NULL;
      52             : static bool g_modules_started = false;
      53             : static struct spdk_memory_domain *g_accel_domain;
      54             : 
      55             : /* Global list of registered accelerator modules */
      56             : static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list =
      57             :         TAILQ_HEAD_INITIALIZER(spdk_accel_module_list);
      58             : 
      59             : /* Crypto keyring */
      60             : static TAILQ_HEAD(, spdk_accel_crypto_key) g_keyring = TAILQ_HEAD_INITIALIZER(g_keyring);
      61             : static struct spdk_spinlock g_keyring_spin;
      62             : 
      63             : /* Global array mapping capabilities to modules */
      64             : static struct accel_module g_modules_opc[SPDK_ACCEL_OPC_LAST] = {};
      65             : static char *g_modules_opc_override[SPDK_ACCEL_OPC_LAST] = {};
      66             : TAILQ_HEAD(, spdk_accel_driver) g_accel_drivers = TAILQ_HEAD_INITIALIZER(g_accel_drivers);
      67             : static struct spdk_accel_driver *g_accel_driver;
      68             : static struct spdk_accel_opts g_opts = {
      69             :         .small_cache_size = ACCEL_SMALL_CACHE_SIZE,
      70             :         .large_cache_size = ACCEL_LARGE_CACHE_SIZE,
      71             :         .task_count = MAX_TASKS_PER_CHANNEL,
      72             :         .sequence_count = MAX_TASKS_PER_CHANNEL,
      73             :         .buf_count = MAX_TASKS_PER_CHANNEL,
      74             : };
      75             : static struct accel_stats g_stats;
      76             : static struct spdk_spinlock g_stats_lock;
      77             : 
      78             : static const char *g_opcode_strings[SPDK_ACCEL_OPC_LAST] = {
      79             :         "copy", "fill", "dualcast", "compare", "crc32c", "copy_crc32c",
      80             :         "compress", "decompress", "encrypt", "decrypt", "xor",
      81             :         "dif_verify", "dif_generate", "dif_generate_copy"
      82             : };
      83             : 
      84             : enum accel_sequence_state {
      85             :         ACCEL_SEQUENCE_STATE_INIT,
      86             :         ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF,
      87             :         ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF,
      88             :         ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF,
      89             :         ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF,
      90             :         ACCEL_SEQUENCE_STATE_PULL_DATA,
      91             :         ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA,
      92             :         ACCEL_SEQUENCE_STATE_EXEC_TASK,
      93             :         ACCEL_SEQUENCE_STATE_AWAIT_TASK,
      94             :         ACCEL_SEQUENCE_STATE_COMPLETE_TASK,
      95             :         ACCEL_SEQUENCE_STATE_NEXT_TASK,
      96             :         ACCEL_SEQUENCE_STATE_PUSH_DATA,
      97             :         ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA,
      98             :         ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS,
      99             :         ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS,
     100             :         ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS,
     101             :         ACCEL_SEQUENCE_STATE_ERROR,
     102             :         ACCEL_SEQUENCE_STATE_MAX,
     103             : };
     104             : 
     105             : static const char *g_seq_states[]
     106             : __attribute__((unused)) = {
     107             :         [ACCEL_SEQUENCE_STATE_INIT] = "init",
     108             :         [ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF] = "check-virtbuf",
     109             :         [ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF] = "await-virtbuf",
     110             :         [ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF] = "check-bouncebuf",
     111             :         [ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF] = "await-bouncebuf",
     112             :         [ACCEL_SEQUENCE_STATE_PULL_DATA] = "pull-data",
     113             :         [ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA] = "await-pull-data",
     114             :         [ACCEL_SEQUENCE_STATE_EXEC_TASK] = "exec-task",
     115             :         [ACCEL_SEQUENCE_STATE_AWAIT_TASK] = "await-task",
     116             :         [ACCEL_SEQUENCE_STATE_COMPLETE_TASK] = "complete-task",
     117             :         [ACCEL_SEQUENCE_STATE_NEXT_TASK] = "next-task",
     118             :         [ACCEL_SEQUENCE_STATE_PUSH_DATA] = "push-data",
     119             :         [ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA] = "await-push-data",
     120             :         [ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS] = "driver-exec-tasks",
     121             :         [ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS] = "driver-await-tasks",
     122             :         [ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS] = "driver-complete-tasks",
     123             :         [ACCEL_SEQUENCE_STATE_ERROR] = "error",
     124             :         [ACCEL_SEQUENCE_STATE_MAX] = "",
     125             : };
     126             : 
     127             : #define ACCEL_SEQUENCE_STATE_STRING(s) \
     128             :         (((s) >= ACCEL_SEQUENCE_STATE_INIT && (s) < ACCEL_SEQUENCE_STATE_MAX) \
     129             :          ? g_seq_states[s] : "unknown")
     130             : 
     131             : struct accel_buffer {
     132             :         struct spdk_accel_sequence      *seq;
     133             :         void                            *buf;
     134             :         uint64_t                        len;
     135             :         struct spdk_iobuf_entry         iobuf;
     136             :         spdk_accel_sequence_get_buf_cb  cb_fn;
     137             :         void                            *cb_ctx;
     138             :         SLIST_ENTRY(accel_buffer)       link;
     139             : };
     140             : 
     141             : struct accel_io_channel {
     142             :         struct spdk_io_channel                  *module_ch[SPDK_ACCEL_OPC_LAST];
     143             :         struct spdk_io_channel                  *driver_channel;
     144             :         void                                    *task_pool_base;
     145             :         struct spdk_accel_sequence              *seq_pool_base;
     146             :         struct accel_buffer                     *buf_pool_base;
     147             :         struct spdk_accel_task_aux_data         *task_aux_data_base;
     148             :         STAILQ_HEAD(, spdk_accel_task)          task_pool;
     149             :         SLIST_HEAD(, spdk_accel_task_aux_data)  task_aux_data_pool;
     150             :         SLIST_HEAD(, spdk_accel_sequence)       seq_pool;
     151             :         SLIST_HEAD(, accel_buffer)              buf_pool;
     152             :         struct spdk_iobuf_channel               iobuf;
     153             :         struct accel_stats                      stats;
     154             : };
     155             : 
     156             : TAILQ_HEAD(accel_sequence_tasks, spdk_accel_task);
     157             : 
     158             : struct spdk_accel_sequence {
     159             :         struct accel_io_channel                 *ch;
     160             :         struct accel_sequence_tasks             tasks;
     161             :         SLIST_HEAD(, accel_buffer)              bounce_bufs;
     162             :         int                                     status;
     163             :         /* state uses enum accel_sequence_state */
     164             :         uint8_t                                 state;
     165             :         bool                                    in_process_sequence;
     166             :         spdk_accel_completion_cb                cb_fn;
     167             :         void                                    *cb_arg;
     168             :         SLIST_ENTRY(spdk_accel_sequence)        link;
     169             : };
     170             : SPDK_STATIC_ASSERT(sizeof(struct spdk_accel_sequence) == 64, "invalid size");
     171             : 
     172             : #define accel_update_stats(ch, event, v) \
     173             :         do { \
     174             :                 (ch)->stats.event += (v); \
     175             :         } while (0)
     176             : 
     177             : #define accel_update_task_stats(ch, task, event, v) \
     178             :         accel_update_stats(ch, operations[(task)->op_code].event, v)
     179             : 
     180             : static inline void accel_sequence_task_cb(void *cb_arg, int status);
     181             : 
     182             : static inline void
     183         701 : accel_sequence_set_state(struct spdk_accel_sequence *seq, enum accel_sequence_state state)
     184             : {
     185         701 :         SPDK_DEBUGLOG(accel, "seq=%p, setting state: %s -> %s\n", seq,
     186             :                       ACCEL_SEQUENCE_STATE_STRING(seq->state), ACCEL_SEQUENCE_STATE_STRING(state));
     187         701 :         assert(seq->state != ACCEL_SEQUENCE_STATE_ERROR || state == ACCEL_SEQUENCE_STATE_ERROR);
     188         701 :         seq->state = state;
     189         701 : }
     190             : 
     191             : static void
     192           9 : accel_sequence_set_fail(struct spdk_accel_sequence *seq, int status)
     193             : {
     194           9 :         accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR);
     195           9 :         assert(status != 0);
     196           9 :         seq->status = status;
     197           9 : }
     198             : 
     199             : int
     200          14 : spdk_accel_get_opc_module_name(enum spdk_accel_opcode opcode, const char **module_name)
     201             : {
     202          14 :         if (opcode >= SPDK_ACCEL_OPC_LAST) {
     203             :                 /* invalid opcode */
     204           0 :                 return -EINVAL;
     205             :         }
     206             : 
     207          14 :         if (g_modules_opc[opcode].module) {
     208          14 :                 *module_name = g_modules_opc[opcode].module->name;
     209             :         } else {
     210           0 :                 return -ENOENT;
     211             :         }
     212             : 
     213          14 :         return 0;
     214             : }
     215             : 
     216             : void
     217           0 : _accel_for_each_module(struct module_info *info, _accel_for_each_module_fn fn)
     218             : {
     219             :         struct spdk_accel_module_if *accel_module;
     220             :         enum spdk_accel_opcode opcode;
     221           0 :         int j = 0;
     222             : 
     223           0 :         TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
     224           0 :                 for (opcode = 0; opcode < SPDK_ACCEL_OPC_LAST; opcode++) {
     225           0 :                         if (accel_module->supports_opcode(opcode)) {
     226           0 :                                 info->ops[j] = opcode;
     227           0 :                                 j++;
     228             :                         }
     229             :                 }
     230           0 :                 info->name = accel_module->name;
     231           0 :                 info->num_ops = j;
     232           0 :                 fn(info);
     233           0 :                 j = 0;
     234             :         }
     235           0 : }
     236             : 
     237             : const char *
     238           0 : spdk_accel_get_opcode_name(enum spdk_accel_opcode opcode)
     239             : {
     240           0 :         if (opcode < SPDK_ACCEL_OPC_LAST) {
     241           0 :                 return g_opcode_strings[opcode];
     242             :         }
     243             : 
     244           0 :         return NULL;
     245             : }
     246             : 
     247             : int
     248           0 : spdk_accel_assign_opc(enum spdk_accel_opcode opcode, const char *name)
     249             : {
     250             :         char *copy;
     251             : 
     252           0 :         if (g_modules_started == true) {
     253             :                 /* we don't allow re-assignment once things have started */
     254           0 :                 return -EINVAL;
     255             :         }
     256             : 
     257           0 :         if (opcode >= SPDK_ACCEL_OPC_LAST) {
     258             :                 /* invalid opcode */
     259           0 :                 return -EINVAL;
     260             :         }
     261             : 
     262           0 :         copy = strdup(name);
     263           0 :         if (copy == NULL) {
     264           0 :                 return -ENOMEM;
     265             :         }
     266             : 
     267             :         /* module selection will be validated after the framework starts. */
     268           0 :         free(g_modules_opc_override[opcode]);
     269           0 :         g_modules_opc_override[opcode] = copy;
     270             : 
     271           0 :         return 0;
     272             : }
     273             : 
     274             : void
     275          94 : spdk_accel_task_complete(struct spdk_accel_task *accel_task, int status)
     276             : {
     277          94 :         struct accel_io_channel         *accel_ch = accel_task->accel_ch;
     278             :         spdk_accel_completion_cb        cb_fn;
     279             :         void                            *cb_arg;
     280             : 
     281          94 :         accel_update_task_stats(accel_ch, accel_task, executed, 1);
     282          94 :         accel_update_task_stats(accel_ch, accel_task, num_bytes, accel_task->nbytes);
     283          94 :         if (spdk_unlikely(status != 0)) {
     284           3 :                 accel_update_task_stats(accel_ch, accel_task, failed, 1);
     285             :         }
     286             : 
     287          94 :         if (accel_task->seq) {
     288          93 :                 accel_sequence_task_cb(accel_task->seq, status);
     289          93 :                 return;
     290             :         }
     291             : 
     292           1 :         cb_fn = accel_task->cb_fn;
     293           1 :         cb_arg = accel_task->cb_arg;
     294             : 
     295           1 :         if (accel_task->has_aux) {
     296           0 :                 SLIST_INSERT_HEAD(&accel_ch->task_aux_data_pool, accel_task->aux, link);
     297           0 :                 accel_task->aux = NULL;
     298           0 :                 accel_task->has_aux = false;
     299             :         }
     300             : 
     301             :         /* We should put the accel_task into the list firstly in order to avoid
     302             :          * the accel task list is exhausted when there is recursive call to
     303             :          * allocate accel_task in user's call back function (cb_fn)
     304             :          */
     305           1 :         STAILQ_INSERT_HEAD(&accel_ch->task_pool, accel_task, link);
     306             : 
     307           1 :         cb_fn(cb_arg, status);
     308             : }
     309             : 
     310             : inline static struct spdk_accel_task *
     311         151 : _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, void *cb_arg)
     312             : {
     313             :         struct spdk_accel_task *accel_task;
     314             : 
     315         151 :         accel_task = STAILQ_FIRST(&accel_ch->task_pool);
     316         151 :         if (spdk_unlikely(accel_task == NULL)) {
     317          11 :                 accel_update_stats(accel_ch, retry.task, 1);
     318          11 :                 return NULL;
     319             :         }
     320             : 
     321         140 :         STAILQ_REMOVE_HEAD(&accel_ch->task_pool, link);
     322         140 :         accel_task->link.stqe_next = NULL;
     323             : 
     324         140 :         accel_task->cb_fn = cb_fn;
     325         140 :         accel_task->cb_arg = cb_arg;
     326         140 :         accel_task->accel_ch = accel_ch;
     327         140 :         accel_task->s.iovs = NULL;
     328         140 :         accel_task->d.iovs = NULL;
     329             : 
     330         140 :         return accel_task;
     331             : }
     332             : 
     333             : static inline int
     334          91 : accel_submit_task(struct accel_io_channel *accel_ch, struct spdk_accel_task *task)
     335             : {
     336          91 :         struct spdk_io_channel *module_ch = accel_ch->module_ch[task->op_code];
     337          91 :         struct spdk_accel_module_if *module = g_modules_opc[task->op_code].module;
     338             :         int rc;
     339             : 
     340          91 :         rc = module->submit_tasks(module_ch, task);
     341          91 :         if (spdk_unlikely(rc != 0)) {
     342           2 :                 accel_update_task_stats(accel_ch, task, failed, 1);
     343             :         }
     344             : 
     345          91 :         return rc;
     346             : }
     347             : 
     348             : static inline uint64_t
     349         114 : accel_get_iovlen(struct iovec *iovs, uint32_t iovcnt)
     350             : {
     351         114 :         uint64_t result = 0;
     352             :         uint32_t i;
     353             : 
     354         259 :         for (i = 0; i < iovcnt; ++i) {
     355         145 :                 result += iovs[i].iov_len;
     356             :         }
     357             : 
     358         114 :         return result;
     359             : }
     360             : 
     361             : #define ACCEL_TASK_ALLOC_AUX_BUF(task)                                          \
     362             : do {                                                                            \
     363             :         (task)->aux = SLIST_FIRST(&(task)->accel_ch->task_aux_data_pool);  \
     364             :         if (spdk_unlikely(!(task)->aux)) {                                   \
     365             :                 SPDK_ERRLOG("Fatal problem, aux data was not allocated\n");   \
     366             :                 STAILQ_INSERT_HEAD(&(task)->accel_ch->task_pool, (task), link);       \
     367             :                 assert(0);                                                      \
     368             :                 return -ENOMEM;                                                 \
     369             :         }                                                                       \
     370             :         SLIST_REMOVE_HEAD(&(task)->accel_ch->task_aux_data_pool, link);               \
     371             :         (task)->has_aux = true;                                                      \
     372             : } while (0)
     373             : 
     374           1 : SPDK_LOG_DEPRECATION_REGISTER(accel_flags,
     375             :                               "The flags parameter is unused and deprecated",
     376             :                               "v24.05", 0);
     377             : 
     378             : /* \b `flags` is int in API, since it is not used anywahere. we narrowed it down to uint8_t internally
     379             :  * To prevent possible problems in the future, add a macro which checks that the value of `flags` passed in the API
     380             :  * doesn't exceed 1 byte. */
     381             : #define ACCEL_ASSIGN_FLAGS(task, flags)                                                 \
     382             : do {                                                                                    \
     383             :         assert(((flags) & (~0xff)) == 0 && "task::flags needs to be extended");           \
     384             :         (task)->flags = (uint8_t)(flags);                                            \
     385             :         if ((task)->flags) {                                                         \
     386             :                 SPDK_LOG_DEPRECATED(accel_flags);                                       \
     387             :         }                                                                               \
     388             : } while (0)                                                                             \
     389             : 
     390             : /* Accel framework public API for copy function */
     391             : int
     392           2 : spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src,
     393             :                        uint64_t nbytes, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)
     394             : {
     395           2 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     396             :         struct spdk_accel_task *accel_task;
     397             : 
     398           2 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     399           2 :         if (spdk_unlikely(accel_task == NULL)) {
     400           1 :                 return -ENOMEM;
     401             :         }
     402             : 
     403           1 :         ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
     404             : 
     405           1 :         accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
     406           1 :         accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
     407           1 :         accel_task->d.iovs[0].iov_base = dst;
     408           1 :         accel_task->d.iovs[0].iov_len = nbytes;
     409           1 :         accel_task->d.iovcnt = 1;
     410           1 :         accel_task->s.iovs[0].iov_base = src;
     411           1 :         accel_task->s.iovs[0].iov_len = nbytes;
     412           1 :         accel_task->s.iovcnt = 1;
     413           1 :         accel_task->nbytes = nbytes;
     414           1 :         accel_task->op_code = SPDK_ACCEL_OPC_COPY;
     415           1 :         accel_task->src_domain = NULL;
     416           1 :         accel_task->dst_domain = NULL;
     417           1 :         ACCEL_ASSIGN_FLAGS(accel_task, flags);
     418             : 
     419           1 :         return accel_submit_task(accel_ch, accel_task);
     420             : }
     421             : 
     422             : /* Accel framework public API for dual cast copy function */
     423             : int
     424           4 : spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1,
     425             :                            void *dst2, void *src, uint64_t nbytes, int flags,
     426             :                            spdk_accel_completion_cb cb_fn, void *cb_arg)
     427             : {
     428           4 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     429             :         struct spdk_accel_task *accel_task;
     430             : 
     431           4 :         if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {
     432           2 :                 SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
     433           2 :                 return -EINVAL;
     434             :         }
     435             : 
     436           2 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     437           2 :         if (spdk_unlikely(accel_task == NULL)) {
     438           1 :                 return -ENOMEM;
     439             :         }
     440             : 
     441           1 :         ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
     442             : 
     443           1 :         accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
     444           1 :         accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
     445           1 :         accel_task->d2.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST2];
     446           1 :         accel_task->d.iovs[0].iov_base = dst1;
     447           1 :         accel_task->d.iovs[0].iov_len = nbytes;
     448           1 :         accel_task->d.iovcnt = 1;
     449           1 :         accel_task->d2.iovs[0].iov_base = dst2;
     450           1 :         accel_task->d2.iovs[0].iov_len = nbytes;
     451           1 :         accel_task->d2.iovcnt = 1;
     452           1 :         accel_task->s.iovs[0].iov_base = src;
     453           1 :         accel_task->s.iovs[0].iov_len = nbytes;
     454           1 :         accel_task->s.iovcnt = 1;
     455           1 :         accel_task->nbytes = nbytes;
     456           1 :         accel_task->op_code = SPDK_ACCEL_OPC_DUALCAST;
     457           1 :         accel_task->src_domain = NULL;
     458           1 :         accel_task->dst_domain = NULL;
     459           1 :         ACCEL_ASSIGN_FLAGS(accel_task, flags);
     460             : 
     461           1 :         return accel_submit_task(accel_ch, accel_task);
     462             : }
     463             : 
     464             : /* Accel framework public API for compare function */
     465             : 
     466             : int
     467           2 : spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1,
     468             :                           void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
     469             :                           void *cb_arg)
     470             : {
     471           2 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     472             :         struct spdk_accel_task *accel_task;
     473             : 
     474           2 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     475           2 :         if (spdk_unlikely(accel_task == NULL)) {
     476           1 :                 return -ENOMEM;
     477             :         }
     478             : 
     479           1 :         ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
     480             : 
     481           1 :         accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
     482           1 :         accel_task->s2.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC2];
     483           1 :         accel_task->s.iovs[0].iov_base = src1;
     484           1 :         accel_task->s.iovs[0].iov_len = nbytes;
     485           1 :         accel_task->s.iovcnt = 1;
     486           1 :         accel_task->s2.iovs[0].iov_base = src2;
     487           1 :         accel_task->s2.iovs[0].iov_len = nbytes;
     488           1 :         accel_task->s2.iovcnt = 1;
     489           1 :         accel_task->nbytes = nbytes;
     490           1 :         accel_task->op_code = SPDK_ACCEL_OPC_COMPARE;
     491           1 :         accel_task->src_domain = NULL;
     492           1 :         accel_task->dst_domain = NULL;
     493           1 :         accel_task->flags = 0;
     494             : 
     495           1 :         return accel_submit_task(accel_ch, accel_task);
     496             : }
     497             : 
     498             : /* Accel framework public API for fill function */
     499             : int
     500           2 : spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst,
     501             :                        uint8_t fill, uint64_t nbytes, int flags,
     502             :                        spdk_accel_completion_cb cb_fn, void *cb_arg)
     503             : {
     504           2 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     505             :         struct spdk_accel_task *accel_task;
     506             : 
     507           2 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     508           2 :         if (spdk_unlikely(accel_task == NULL)) {
     509           1 :                 return -ENOMEM;
     510             :         }
     511             : 
     512           1 :         ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
     513             : 
     514           1 :         accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
     515           1 :         accel_task->d.iovs[0].iov_base = dst;
     516           1 :         accel_task->d.iovs[0].iov_len = nbytes;
     517           1 :         accel_task->d.iovcnt = 1;
     518           1 :         accel_task->nbytes = nbytes;
     519           1 :         memset(&accel_task->fill_pattern, fill, sizeof(uint64_t));
     520           1 :         accel_task->op_code = SPDK_ACCEL_OPC_FILL;
     521           1 :         accel_task->src_domain = NULL;
     522           1 :         accel_task->dst_domain = NULL;
     523           1 :         ACCEL_ASSIGN_FLAGS(accel_task, flags);
     524             : 
     525           1 :         return accel_submit_task(accel_ch, accel_task);
     526             : }
     527             : 
     528             : /* Accel framework public API for CRC-32C function */
     529             : int
     530           2 : spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst,
     531             :                          void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
     532             :                          void *cb_arg)
     533             : {
     534           2 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     535             :         struct spdk_accel_task *accel_task;
     536             : 
     537           2 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     538           2 :         if (spdk_unlikely(accel_task == NULL)) {
     539           1 :                 return -ENOMEM;
     540             :         }
     541             : 
     542           1 :         ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
     543             : 
     544           1 :         accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
     545           1 :         accel_task->s.iovs[0].iov_base = src;
     546           1 :         accel_task->s.iovs[0].iov_len = nbytes;
     547           1 :         accel_task->s.iovcnt = 1;
     548           1 :         accel_task->nbytes = nbytes;
     549           1 :         accel_task->crc_dst = crc_dst;
     550           1 :         accel_task->seed = seed;
     551           1 :         accel_task->op_code = SPDK_ACCEL_OPC_CRC32C;
     552           1 :         accel_task->src_domain = NULL;
     553           1 :         accel_task->dst_domain = NULL;
     554           1 :         accel_task->flags = 0;
     555             : 
     556           1 :         return accel_submit_task(accel_ch, accel_task);
     557             : }
     558             : 
     559             : /* Accel framework public API for chained CRC-32C function */
     560             : int
     561           1 : spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst,
     562             :                           struct iovec *iov, uint32_t iov_cnt, uint32_t seed,
     563             :                           spdk_accel_completion_cb cb_fn, void *cb_arg)
     564             : {
     565           1 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     566             :         struct spdk_accel_task *accel_task;
     567             : 
     568           1 :         if (iov == NULL) {
     569           0 :                 SPDK_ERRLOG("iov should not be NULL");
     570           0 :                 return -EINVAL;
     571             :         }
     572             : 
     573           1 :         if (!iov_cnt) {
     574           0 :                 SPDK_ERRLOG("iovcnt should not be zero value\n");
     575           0 :                 return -EINVAL;
     576             :         }
     577             : 
     578           1 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     579           1 :         if (spdk_unlikely(accel_task == NULL)) {
     580           0 :                 SPDK_ERRLOG("no memory\n");
     581           0 :                 assert(0);
     582             :                 return -ENOMEM;
     583             :         }
     584             : 
     585           1 :         accel_task->s.iovs = iov;
     586           1 :         accel_task->s.iovcnt = iov_cnt;
     587           1 :         accel_task->nbytes = accel_get_iovlen(iov, iov_cnt);
     588           1 :         accel_task->crc_dst = crc_dst;
     589           1 :         accel_task->seed = seed;
     590           1 :         accel_task->op_code = SPDK_ACCEL_OPC_CRC32C;
     591           1 :         accel_task->src_domain = NULL;
     592           1 :         accel_task->dst_domain = NULL;
     593           1 :         accel_task->flags = 0;
     594             : 
     595           1 :         return accel_submit_task(accel_ch, accel_task);
     596             : }
     597             : 
     598             : /* Accel framework public API for copy with CRC-32C function */
     599             : int
     600           2 : spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst,
     601             :                               void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes,
     602             :                               int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)
     603             : {
     604           2 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     605             :         struct spdk_accel_task *accel_task;
     606             : 
     607           2 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     608           2 :         if (spdk_unlikely(accel_task == NULL)) {
     609           1 :                 return -ENOMEM;
     610             :         }
     611             : 
     612           1 :         ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
     613             : 
     614           1 :         accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
     615           1 :         accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
     616           1 :         accel_task->d.iovs[0].iov_base = dst;
     617           1 :         accel_task->d.iovs[0].iov_len = nbytes;
     618           1 :         accel_task->d.iovcnt = 1;
     619           1 :         accel_task->s.iovs[0].iov_base = src;
     620           1 :         accel_task->s.iovs[0].iov_len = nbytes;
     621           1 :         accel_task->s.iovcnt = 1;
     622           1 :         accel_task->nbytes = nbytes;
     623           1 :         accel_task->crc_dst = crc_dst;
     624           1 :         accel_task->seed = seed;
     625           1 :         accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C;
     626           1 :         accel_task->src_domain = NULL;
     627           1 :         accel_task->dst_domain = NULL;
     628           1 :         ACCEL_ASSIGN_FLAGS(accel_task, flags);
     629             : 
     630           1 :         return accel_submit_task(accel_ch, accel_task);
     631             : }
     632             : 
     633             : /* Accel framework public API for chained copy + CRC-32C function */
     634             : int
     635           0 : spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst,
     636             :                                struct iovec *src_iovs, uint32_t iov_cnt, uint32_t *crc_dst,
     637             :                                uint32_t seed, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)
     638             : {
     639           0 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     640             :         struct spdk_accel_task *accel_task;
     641             :         uint64_t nbytes;
     642             : 
     643           0 :         if (src_iovs == NULL) {
     644           0 :                 SPDK_ERRLOG("iov should not be NULL");
     645           0 :                 return -EINVAL;
     646             :         }
     647             : 
     648           0 :         if (!iov_cnt) {
     649           0 :                 SPDK_ERRLOG("iovcnt should not be zero value\n");
     650           0 :                 return -EINVAL;
     651             :         }
     652             : 
     653           0 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     654           0 :         if (spdk_unlikely(accel_task == NULL)) {
     655           0 :                 SPDK_ERRLOG("no memory\n");
     656           0 :                 assert(0);
     657             :                 return -ENOMEM;
     658             :         }
     659             : 
     660           0 :         nbytes = accel_get_iovlen(src_iovs, iov_cnt);
     661             : 
     662           0 :         ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
     663             : 
     664           0 :         accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
     665           0 :         accel_task->d.iovs[0].iov_base = dst;
     666           0 :         accel_task->d.iovs[0].iov_len = nbytes;
     667           0 :         accel_task->d.iovcnt = 1;
     668           0 :         accel_task->s.iovs = src_iovs;
     669           0 :         accel_task->s.iovcnt = iov_cnt;
     670           0 :         accel_task->nbytes = nbytes;
     671           0 :         accel_task->crc_dst = crc_dst;
     672           0 :         accel_task->seed = seed;
     673           0 :         accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C;
     674           0 :         accel_task->src_domain = NULL;
     675           0 :         accel_task->dst_domain = NULL;
     676           0 :         ACCEL_ASSIGN_FLAGS(accel_task, flags);
     677             : 
     678           0 :         return accel_submit_task(accel_ch, accel_task);
     679             : }
     680             : 
     681             : int
     682           0 : spdk_accel_submit_compress(struct spdk_io_channel *ch, void *dst, uint64_t nbytes,
     683             :                            struct iovec *src_iovs, size_t src_iovcnt, uint32_t *output_size, int flags,
     684             :                            spdk_accel_completion_cb cb_fn, void *cb_arg)
     685             : {
     686           0 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     687             :         struct spdk_accel_task *accel_task;
     688             : 
     689           0 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     690           0 :         if (spdk_unlikely(accel_task == NULL)) {
     691           0 :                 return -ENOMEM;
     692             :         }
     693             : 
     694           0 :         ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
     695             : 
     696           0 :         accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
     697           0 :         accel_task->d.iovs[0].iov_base = dst;
     698           0 :         accel_task->d.iovs[0].iov_len = nbytes;
     699           0 :         accel_task->d.iovcnt = 1;
     700           0 :         accel_task->output_size = output_size;
     701           0 :         accel_task->s.iovs = src_iovs;
     702           0 :         accel_task->s.iovcnt = src_iovcnt;
     703           0 :         accel_task->nbytes = nbytes;
     704           0 :         accel_task->op_code = SPDK_ACCEL_OPC_COMPRESS;
     705           0 :         accel_task->src_domain = NULL;
     706           0 :         accel_task->dst_domain = NULL;
     707           0 :         ACCEL_ASSIGN_FLAGS(accel_task, flags);
     708             : 
     709           0 :         return accel_submit_task(accel_ch, accel_task);
     710             : }
     711             : 
     712             : int
     713           0 : spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs,
     714             :                              size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt,
     715             :                              uint32_t *output_size, int flags, spdk_accel_completion_cb cb_fn,
     716             :                              void *cb_arg)
     717             : {
     718           0 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     719             :         struct spdk_accel_task *accel_task;
     720             : 
     721           0 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     722           0 :         if (spdk_unlikely(accel_task == NULL)) {
     723           0 :                 return -ENOMEM;
     724             :         }
     725             : 
     726           0 :         accel_task->output_size = output_size;
     727           0 :         accel_task->s.iovs = src_iovs;
     728           0 :         accel_task->s.iovcnt = src_iovcnt;
     729           0 :         accel_task->d.iovs = dst_iovs;
     730           0 :         accel_task->d.iovcnt = dst_iovcnt;
     731           0 :         accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
     732           0 :         accel_task->op_code = SPDK_ACCEL_OPC_DECOMPRESS;
     733           0 :         accel_task->src_domain = NULL;
     734           0 :         accel_task->dst_domain = NULL;
     735           0 :         ACCEL_ASSIGN_FLAGS(accel_task, flags);
     736             : 
     737           0 :         return accel_submit_task(accel_ch, accel_task);
     738             : }
     739             : 
     740             : int
     741           0 : spdk_accel_submit_encrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key,
     742             :                           struct iovec *dst_iovs, uint32_t dst_iovcnt,
     743             :                           struct iovec *src_iovs, uint32_t src_iovcnt,
     744             :                           uint64_t iv, uint32_t block_size, int flags,
     745             :                           spdk_accel_completion_cb cb_fn, void *cb_arg)
     746             : {
     747           0 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     748             :         struct spdk_accel_task *accel_task;
     749             : 
     750           0 :         if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) {
     751           0 :                 return -EINVAL;
     752             :         }
     753             : 
     754           0 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     755           0 :         if (spdk_unlikely(accel_task == NULL)) {
     756           0 :                 return -ENOMEM;
     757             :         }
     758             : 
     759           0 :         accel_task->crypto_key = key;
     760           0 :         accel_task->s.iovs = src_iovs;
     761           0 :         accel_task->s.iovcnt = src_iovcnt;
     762           0 :         accel_task->d.iovs = dst_iovs;
     763           0 :         accel_task->d.iovcnt = dst_iovcnt;
     764           0 :         accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
     765           0 :         accel_task->iv = iv;
     766           0 :         accel_task->block_size = block_size;
     767           0 :         accel_task->op_code = SPDK_ACCEL_OPC_ENCRYPT;
     768           0 :         accel_task->src_domain = NULL;
     769           0 :         accel_task->dst_domain = NULL;
     770           0 :         ACCEL_ASSIGN_FLAGS(accel_task, flags);
     771             : 
     772           0 :         return accel_submit_task(accel_ch, accel_task);
     773             : }
     774             : 
     775             : int
     776           0 : spdk_accel_submit_decrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key,
     777             :                           struct iovec *dst_iovs, uint32_t dst_iovcnt,
     778             :                           struct iovec *src_iovs, uint32_t src_iovcnt,
     779             :                           uint64_t iv, uint32_t block_size, int flags,
     780             :                           spdk_accel_completion_cb cb_fn, void *cb_arg)
     781             : {
     782           0 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     783             :         struct spdk_accel_task *accel_task;
     784             : 
     785           0 :         if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) {
     786           0 :                 return -EINVAL;
     787             :         }
     788             : 
     789           0 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     790           0 :         if (spdk_unlikely(accel_task == NULL)) {
     791           0 :                 return -ENOMEM;
     792             :         }
     793             : 
     794           0 :         accel_task->crypto_key = key;
     795           0 :         accel_task->s.iovs = src_iovs;
     796           0 :         accel_task->s.iovcnt = src_iovcnt;
     797           0 :         accel_task->d.iovs = dst_iovs;
     798           0 :         accel_task->d.iovcnt = dst_iovcnt;
     799           0 :         accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
     800           0 :         accel_task->iv = iv;
     801           0 :         accel_task->block_size = block_size;
     802           0 :         accel_task->op_code = SPDK_ACCEL_OPC_DECRYPT;
     803           0 :         accel_task->src_domain = NULL;
     804           0 :         accel_task->dst_domain = NULL;
     805           0 :         ACCEL_ASSIGN_FLAGS(accel_task, flags);
     806             : 
     807           0 :         return accel_submit_task(accel_ch, accel_task);
     808             : }
     809             : 
     810             : int
     811           2 : spdk_accel_submit_xor(struct spdk_io_channel *ch, void *dst, void **sources, uint32_t nsrcs,
     812             :                       uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
     813             : {
     814           2 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     815             :         struct spdk_accel_task *accel_task;
     816             : 
     817           2 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     818           2 :         if (spdk_unlikely(accel_task == NULL)) {
     819           1 :                 return -ENOMEM;
     820             :         }
     821             : 
     822           1 :         ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
     823             : 
     824           1 :         accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
     825           1 :         accel_task->nsrcs.srcs = sources;
     826           1 :         accel_task->nsrcs.cnt = nsrcs;
     827           1 :         accel_task->d.iovs[0].iov_base = dst;
     828           1 :         accel_task->d.iovs[0].iov_len = nbytes;
     829           1 :         accel_task->d.iovcnt = 1;
     830           1 :         accel_task->nbytes = nbytes;
     831           1 :         accel_task->op_code = SPDK_ACCEL_OPC_XOR;
     832           1 :         accel_task->src_domain = NULL;
     833           1 :         accel_task->dst_domain = NULL;
     834             : 
     835           1 :         return accel_submit_task(accel_ch, accel_task);
     836             : }
     837             : 
     838             : int
     839           0 : spdk_accel_submit_dif_verify(struct spdk_io_channel *ch,
     840             :                              struct iovec *iovs, size_t iovcnt, uint32_t num_blocks,
     841             :                              const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
     842             :                              spdk_accel_completion_cb cb_fn, void *cb_arg)
     843             : {
     844           0 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     845             :         struct spdk_accel_task *accel_task;
     846             : 
     847           0 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     848           0 :         if (accel_task == NULL) {
     849           0 :                 return -ENOMEM;
     850             :         }
     851             : 
     852           0 :         accel_task->s.iovs = iovs;
     853           0 :         accel_task->s.iovcnt = iovcnt;
     854           0 :         accel_task->dif.ctx = ctx;
     855           0 :         accel_task->dif.err = err;
     856           0 :         accel_task->dif.num_blocks = num_blocks;
     857           0 :         accel_task->nbytes = num_blocks * ctx->block_size;
     858           0 :         accel_task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY;
     859           0 :         accel_task->src_domain = NULL;
     860           0 :         accel_task->dst_domain = NULL;
     861             : 
     862           0 :         return accel_submit_task(accel_ch, accel_task);
     863             : }
     864             : 
     865             : int
     866           0 : spdk_accel_submit_dif_generate(struct spdk_io_channel *ch,
     867             :                                struct iovec *iovs, size_t iovcnt, uint32_t num_blocks,
     868             :                                const struct spdk_dif_ctx *ctx,
     869             :                                spdk_accel_completion_cb cb_fn, void *cb_arg)
     870             : {
     871           0 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     872             :         struct spdk_accel_task *accel_task;
     873             : 
     874           0 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     875           0 :         if (accel_task == NULL) {
     876           0 :                 return -ENOMEM;
     877             :         }
     878             : 
     879           0 :         accel_task->s.iovs = iovs;
     880           0 :         accel_task->s.iovcnt = iovcnt;
     881           0 :         accel_task->dif.ctx = ctx;
     882           0 :         accel_task->dif.num_blocks = num_blocks;
     883           0 :         accel_task->nbytes = num_blocks * ctx->block_size;
     884           0 :         accel_task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE;
     885           0 :         accel_task->src_domain = NULL;
     886           0 :         accel_task->dst_domain = NULL;
     887             : 
     888           0 :         return accel_submit_task(accel_ch, accel_task);
     889             : }
     890             : 
     891             : int
     892           0 : spdk_accel_submit_dif_generate_copy(struct spdk_io_channel *ch, struct iovec *dst_iovs,
     893             :                                     size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt,
     894             :                                     uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
     895             :                                     spdk_accel_completion_cb cb_fn, void *cb_arg)
     896             : {
     897           0 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
     898             :         struct spdk_accel_task *accel_task;
     899             : 
     900           0 :         accel_task = _get_task(accel_ch, cb_fn, cb_arg);
     901           0 :         if (accel_task == NULL) {
     902           0 :                 return -ENOMEM;
     903             :         }
     904             : 
     905           0 :         accel_task->s.iovs = src_iovs;
     906           0 :         accel_task->s.iovcnt = src_iovcnt;
     907           0 :         accel_task->d.iovs = dst_iovs;
     908           0 :         accel_task->d.iovcnt = dst_iovcnt;
     909           0 :         accel_task->dif.ctx = ctx;
     910           0 :         accel_task->dif.num_blocks = num_blocks;
     911           0 :         accel_task->nbytes = num_blocks * ctx->block_size;
     912           0 :         accel_task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE_COPY;
     913           0 :         accel_task->src_domain = NULL;
     914           0 :         accel_task->dst_domain = NULL;
     915             : 
     916           0 :         return accel_submit_task(accel_ch, accel_task);
     917             : }
     918             : 
     919             : static inline struct accel_buffer *
     920          33 : accel_get_buf(struct accel_io_channel *ch, uint64_t len)
     921             : {
     922             :         struct accel_buffer *buf;
     923             : 
     924          33 :         buf = SLIST_FIRST(&ch->buf_pool);
     925          33 :         if (spdk_unlikely(buf == NULL)) {
     926           0 :                 accel_update_stats(ch, retry.bufdesc, 1);
     927           0 :                 return NULL;
     928             :         }
     929             : 
     930          33 :         SLIST_REMOVE_HEAD(&ch->buf_pool, link);
     931          33 :         buf->len = len;
     932          33 :         buf->buf = NULL;
     933          33 :         buf->seq = NULL;
     934          33 :         buf->cb_fn = NULL;
     935             : 
     936          33 :         return buf;
     937             : }
     938             : 
     939             : static inline void
     940          33 : accel_put_buf(struct accel_io_channel *ch, struct accel_buffer *buf)
     941             : {
     942          33 :         if (buf->buf != NULL) {
     943          29 :                 spdk_iobuf_put(&ch->iobuf, buf->buf, buf->len);
     944             :         }
     945             : 
     946          33 :         SLIST_INSERT_HEAD(&ch->buf_pool, buf, link);
     947          33 : }
     948             : 
     949             : static inline struct spdk_accel_sequence *
     950          60 : accel_sequence_get(struct accel_io_channel *ch)
     951             : {
     952             :         struct spdk_accel_sequence *seq;
     953             : 
     954          60 :         seq = SLIST_FIRST(&ch->seq_pool);
     955          60 :         if (spdk_unlikely(seq == NULL)) {
     956           3 :                 accel_update_stats(ch, retry.sequence, 1);
     957           3 :                 return NULL;
     958             :         }
     959             : 
     960          57 :         SLIST_REMOVE_HEAD(&ch->seq_pool, link);
     961             : 
     962          57 :         TAILQ_INIT(&seq->tasks);
     963          57 :         SLIST_INIT(&seq->bounce_bufs);
     964             : 
     965          57 :         seq->ch = ch;
     966          57 :         seq->status = 0;
     967          57 :         seq->state = ACCEL_SEQUENCE_STATE_INIT;
     968          57 :         seq->in_process_sequence = false;
     969             : 
     970          57 :         return seq;
     971             : }
     972             : 
     973             : static inline void
     974          57 : accel_sequence_put(struct spdk_accel_sequence *seq)
     975             : {
     976          57 :         struct accel_io_channel *ch = seq->ch;
     977             :         struct accel_buffer *buf;
     978             : 
     979          76 :         while (!SLIST_EMPTY(&seq->bounce_bufs)) {
     980          19 :                 buf = SLIST_FIRST(&seq->bounce_bufs);
     981          19 :                 SLIST_REMOVE_HEAD(&seq->bounce_bufs, link);
     982          19 :                 accel_put_buf(seq->ch, buf);
     983             :         }
     984             : 
     985          57 :         assert(TAILQ_EMPTY(&seq->tasks));
     986          57 :         seq->ch = NULL;
     987             : 
     988          57 :         SLIST_INSERT_HEAD(&ch->seq_pool, seq, link);
     989          57 : }
     990             : 
     991             : static void accel_sequence_task_cb(void *cb_arg, int status);
     992             : 
     993             : static inline struct spdk_accel_task *
     994         134 : accel_sequence_get_task(struct accel_io_channel *ch, struct spdk_accel_sequence *seq,
     995             :                         spdk_accel_step_cb cb_fn, void *cb_arg)
     996             : {
     997             :         struct spdk_accel_task *task;
     998             : 
     999         134 :         task = _get_task(ch, NULL, NULL);
    1000         134 :         if (spdk_unlikely(task == NULL)) {
    1001           3 :                 return task;
    1002             :         }
    1003             : 
    1004         131 :         task->step_cb_fn = cb_fn;
    1005         131 :         task->cb_arg = cb_arg;
    1006         131 :         task->seq = seq;
    1007             : 
    1008         131 :         return task;
    1009             : }
    1010             : 
    1011             : int
    1012          35 : spdk_accel_append_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
    1013             :                        struct iovec *dst_iovs, uint32_t dst_iovcnt,
    1014             :                        struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
    1015             :                        struct iovec *src_iovs, uint32_t src_iovcnt,
    1016             :                        struct spdk_memory_domain *src_domain, void *src_domain_ctx,
    1017             :                        int flags, spdk_accel_step_cb cb_fn, void *cb_arg)
    1018             : {
    1019          35 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
    1020             :         struct spdk_accel_task *task;
    1021          35 :         struct spdk_accel_sequence *seq = *pseq;
    1022             : 
    1023          35 :         if (seq == NULL) {
    1024          12 :                 seq = accel_sequence_get(accel_ch);
    1025          12 :                 if (spdk_unlikely(seq == NULL)) {
    1026           1 :                         return -ENOMEM;
    1027             :                 }
    1028             :         }
    1029             : 
    1030          34 :         assert(seq->ch == accel_ch);
    1031          34 :         task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
    1032          34 :         if (spdk_unlikely(task == NULL)) {
    1033           1 :                 if (*pseq == NULL) {
    1034           1 :                         accel_sequence_put(seq);
    1035             :                 }
    1036             : 
    1037           1 :                 return -ENOMEM;
    1038             :         }
    1039             : 
    1040          33 :         task->dst_domain = dst_domain;
    1041          33 :         task->dst_domain_ctx = dst_domain_ctx;
    1042          33 :         task->d.iovs = dst_iovs;
    1043          33 :         task->d.iovcnt = dst_iovcnt;
    1044          33 :         task->src_domain = src_domain;
    1045          33 :         task->src_domain_ctx = src_domain_ctx;
    1046          33 :         task->s.iovs = src_iovs;
    1047          33 :         task->s.iovcnt = src_iovcnt;
    1048          33 :         task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
    1049          33 :         task->op_code = SPDK_ACCEL_OPC_COPY;
    1050          33 :         ACCEL_ASSIGN_FLAGS(task, flags);
    1051             : 
    1052          33 :         TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
    1053          33 :         *pseq = seq;
    1054             : 
    1055          33 :         return 0;
    1056             : }
    1057             : 
    1058             : int
    1059          39 : spdk_accel_append_fill(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
    1060             :                        void *buf, uint64_t len,
    1061             :                        struct spdk_memory_domain *domain, void *domain_ctx, uint8_t pattern,
    1062             :                        int flags, spdk_accel_step_cb cb_fn, void *cb_arg)
    1063             : {
    1064          39 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
    1065             :         struct spdk_accel_task *task;
    1066          39 :         struct spdk_accel_sequence *seq = *pseq;
    1067             : 
    1068          39 :         if (seq == NULL) {
    1069          23 :                 seq = accel_sequence_get(accel_ch);
    1070          23 :                 if (spdk_unlikely(seq == NULL)) {
    1071           1 :                         return -ENOMEM;
    1072             :                 }
    1073             :         }
    1074             : 
    1075          38 :         assert(seq->ch == accel_ch);
    1076          38 :         task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
    1077          38 :         if (spdk_unlikely(task == NULL)) {
    1078           1 :                 if (*pseq == NULL) {
    1079           1 :                         accel_sequence_put(seq);
    1080             :                 }
    1081             : 
    1082           1 :                 return -ENOMEM;
    1083             :         }
    1084             : 
    1085          37 :         memset(&task->fill_pattern, pattern, sizeof(uint64_t));
    1086             : 
    1087          37 :         task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
    1088          37 :         if (spdk_unlikely(!task->aux)) {
    1089           0 :                 SPDK_ERRLOG("Fatal problem, aux data was not allocated\n");
    1090           0 :                 if (*pseq == NULL) {
    1091           0 :                         accel_sequence_put((seq));
    1092             :                 }
    1093           0 :                 STAILQ_INSERT_HEAD(&task->accel_ch->task_pool, task, link);
    1094           0 :                 task->seq = NULL;
    1095           0 :                 assert(0);
    1096             :                 return -ENOMEM;
    1097             :         }
    1098          37 :         SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
    1099          37 :         task->has_aux = true;
    1100             : 
    1101          37 :         task->d.iovs = &task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
    1102          37 :         task->d.iovs[0].iov_base = buf;
    1103          37 :         task->d.iovs[0].iov_len = len;
    1104          37 :         task->d.iovcnt = 1;
    1105          37 :         task->nbytes = len;
    1106          37 :         task->src_domain = NULL;
    1107          37 :         task->dst_domain = domain;
    1108          37 :         task->dst_domain_ctx = domain_ctx;
    1109          37 :         task->op_code = SPDK_ACCEL_OPC_FILL;
    1110          37 :         ACCEL_ASSIGN_FLAGS(task, flags);
    1111             : 
    1112          37 :         TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
    1113          37 :         *pseq = seq;
    1114             : 
    1115          37 :         return 0;
    1116             : }
    1117             : 
    1118             : int
    1119          40 : spdk_accel_append_decompress(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
    1120             :                              struct iovec *dst_iovs, size_t dst_iovcnt,
    1121             :                              struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
    1122             :                              struct iovec *src_iovs, size_t src_iovcnt,
    1123             :                              struct spdk_memory_domain *src_domain, void *src_domain_ctx,
    1124             :                              int flags, spdk_accel_step_cb cb_fn, void *cb_arg)
    1125             : {
    1126          40 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
    1127             :         struct spdk_accel_task *task;
    1128          40 :         struct spdk_accel_sequence *seq = *pseq;
    1129             : 
    1130          40 :         if (seq == NULL) {
    1131          17 :                 seq = accel_sequence_get(accel_ch);
    1132          17 :                 if (spdk_unlikely(seq == NULL)) {
    1133           1 :                         return -ENOMEM;
    1134             :                 }
    1135             :         }
    1136             : 
    1137          39 :         assert(seq->ch == accel_ch);
    1138          39 :         task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
    1139          39 :         if (spdk_unlikely(task == NULL)) {
    1140           1 :                 if (*pseq == NULL) {
    1141           1 :                         accel_sequence_put(seq);
    1142             :                 }
    1143             : 
    1144           1 :                 return -ENOMEM;
    1145             :         }
    1146             : 
    1147             :         /* TODO: support output_size for chaining */
    1148          38 :         task->output_size = NULL;
    1149          38 :         task->dst_domain = dst_domain;
    1150          38 :         task->dst_domain_ctx = dst_domain_ctx;
    1151          38 :         task->d.iovs = dst_iovs;
    1152          38 :         task->d.iovcnt = dst_iovcnt;
    1153          38 :         task->src_domain = src_domain;
    1154          38 :         task->src_domain_ctx = src_domain_ctx;
    1155          38 :         task->s.iovs = src_iovs;
    1156          38 :         task->s.iovcnt = src_iovcnt;
    1157          38 :         task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
    1158          38 :         task->op_code = SPDK_ACCEL_OPC_DECOMPRESS;
    1159          38 :         ACCEL_ASSIGN_FLAGS(task, flags);
    1160             : 
    1161          38 :         TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
    1162          38 :         *pseq = seq;
    1163             : 
    1164          38 :         return 0;
    1165             : }
    1166             : 
    1167             : int
    1168           8 : spdk_accel_append_encrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
    1169             :                           struct spdk_accel_crypto_key *key,
    1170             :                           struct iovec *dst_iovs, uint32_t dst_iovcnt,
    1171             :                           struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
    1172             :                           struct iovec *src_iovs, uint32_t src_iovcnt,
    1173             :                           struct spdk_memory_domain *src_domain, void *src_domain_ctx,
    1174             :                           uint64_t iv, uint32_t block_size, int flags,
    1175             :                           spdk_accel_step_cb cb_fn, void *cb_arg)
    1176             : {
    1177           8 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
    1178             :         struct spdk_accel_task *task;
    1179           8 :         struct spdk_accel_sequence *seq = *pseq;
    1180             : 
    1181           8 :         assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size);
    1182             : 
    1183           8 :         if (seq == NULL) {
    1184           6 :                 seq = accel_sequence_get(accel_ch);
    1185           6 :                 if (spdk_unlikely(seq == NULL)) {
    1186           0 :                         return -ENOMEM;
    1187             :                 }
    1188             :         }
    1189             : 
    1190           8 :         assert(seq->ch == accel_ch);
    1191           8 :         task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
    1192           8 :         if (spdk_unlikely(task == NULL)) {
    1193           0 :                 if (*pseq == NULL) {
    1194           0 :                         accel_sequence_put(seq);
    1195             :                 }
    1196             : 
    1197           0 :                 return -ENOMEM;
    1198             :         }
    1199             : 
    1200           8 :         task->crypto_key = key;
    1201           8 :         task->src_domain = src_domain;
    1202           8 :         task->src_domain_ctx = src_domain_ctx;
    1203           8 :         task->s.iovs = src_iovs;
    1204           8 :         task->s.iovcnt = src_iovcnt;
    1205           8 :         task->dst_domain = dst_domain;
    1206           8 :         task->dst_domain_ctx = dst_domain_ctx;
    1207           8 :         task->d.iovs = dst_iovs;
    1208           8 :         task->d.iovcnt = dst_iovcnt;
    1209           8 :         task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
    1210           8 :         task->iv = iv;
    1211           8 :         task->block_size = block_size;
    1212           8 :         task->op_code = SPDK_ACCEL_OPC_ENCRYPT;
    1213           8 :         ACCEL_ASSIGN_FLAGS(task, flags);
    1214             : 
    1215           8 :         TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
    1216           8 :         *pseq = seq;
    1217             : 
    1218           8 :         return 0;
    1219             : }
    1220             : 
    1221             : int
    1222           9 : spdk_accel_append_decrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
    1223             :                           struct spdk_accel_crypto_key *key,
    1224             :                           struct iovec *dst_iovs, uint32_t dst_iovcnt,
    1225             :                           struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
    1226             :                           struct iovec *src_iovs, uint32_t src_iovcnt,
    1227             :                           struct spdk_memory_domain *src_domain, void *src_domain_ctx,
    1228             :                           uint64_t iv, uint32_t block_size, int flags,
    1229             :                           spdk_accel_step_cb cb_fn, void *cb_arg)
    1230             : {
    1231           9 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
    1232             :         struct spdk_accel_task *task;
    1233           9 :         struct spdk_accel_sequence *seq = *pseq;
    1234             : 
    1235           9 :         assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size);
    1236             : 
    1237           9 :         if (seq == NULL) {
    1238           0 :                 seq = accel_sequence_get(accel_ch);
    1239           0 :                 if (spdk_unlikely(seq == NULL)) {
    1240           0 :                         return -ENOMEM;
    1241             :                 }
    1242             :         }
    1243             : 
    1244           9 :         assert(seq->ch == accel_ch);
    1245           9 :         task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
    1246           9 :         if (spdk_unlikely(task == NULL)) {
    1247           0 :                 if (*pseq == NULL) {
    1248           0 :                         accel_sequence_put(seq);
    1249             :                 }
    1250             : 
    1251           0 :                 return -ENOMEM;
    1252             :         }
    1253             : 
    1254           9 :         task->crypto_key = key;
    1255           9 :         task->src_domain = src_domain;
    1256           9 :         task->src_domain_ctx = src_domain_ctx;
    1257           9 :         task->s.iovs = src_iovs;
    1258           9 :         task->s.iovcnt = src_iovcnt;
    1259           9 :         task->dst_domain = dst_domain;
    1260           9 :         task->dst_domain_ctx = dst_domain_ctx;
    1261           9 :         task->d.iovs = dst_iovs;
    1262           9 :         task->d.iovcnt = dst_iovcnt;
    1263           9 :         task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
    1264           9 :         task->iv = iv;
    1265           9 :         task->block_size = block_size;
    1266           9 :         task->op_code = SPDK_ACCEL_OPC_DECRYPT;
    1267           9 :         ACCEL_ASSIGN_FLAGS(task, flags);
    1268             : 
    1269           9 :         TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
    1270           9 :         *pseq = seq;
    1271             : 
    1272           9 :         return 0;
    1273             : }
    1274             : 
    1275             : int
    1276           6 : spdk_accel_append_crc32c(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
    1277             :                          uint32_t *dst, struct iovec *iovs, uint32_t iovcnt,
    1278             :                          struct spdk_memory_domain *domain, void *domain_ctx,
    1279             :                          uint32_t seed, spdk_accel_step_cb cb_fn, void *cb_arg)
    1280             : {
    1281           6 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
    1282             :         struct spdk_accel_task *task;
    1283           6 :         struct spdk_accel_sequence *seq = *pseq;
    1284             : 
    1285           6 :         if (seq == NULL) {
    1286           2 :                 seq = accel_sequence_get(accel_ch);
    1287           2 :                 if (spdk_unlikely(seq == NULL)) {
    1288           0 :                         return -ENOMEM;
    1289             :                 }
    1290             :         }
    1291             : 
    1292           6 :         assert(seq->ch == accel_ch);
    1293           6 :         task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
    1294           6 :         if (spdk_unlikely(task == NULL)) {
    1295           0 :                 if (*pseq == NULL) {
    1296           0 :                         accel_sequence_put(seq);
    1297             :                 }
    1298             : 
    1299           0 :                 return -ENOMEM;
    1300             :         }
    1301             : 
    1302           6 :         task->s.iovs = iovs;
    1303           6 :         task->s.iovcnt = iovcnt;
    1304           6 :         task->src_domain = domain;
    1305           6 :         task->src_domain_ctx = domain_ctx;
    1306           6 :         task->nbytes = accel_get_iovlen(iovs, iovcnt);
    1307           6 :         task->crc_dst = dst;
    1308           6 :         task->seed = seed;
    1309           6 :         task->op_code = SPDK_ACCEL_OPC_CRC32C;
    1310           6 :         task->dst_domain = NULL;
    1311             : 
    1312           6 :         TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
    1313           6 :         *pseq = seq;
    1314             : 
    1315           6 :         return 0;
    1316             : }
    1317             : 
    1318             : int
    1319          14 : spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf,
    1320             :                    struct spdk_memory_domain **domain, void **domain_ctx)
    1321             : {
    1322          14 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
    1323             :         struct accel_buffer *accel_buf;
    1324             : 
    1325          14 :         accel_buf = accel_get_buf(accel_ch, len);
    1326          14 :         if (spdk_unlikely(accel_buf == NULL)) {
    1327           0 :                 return -ENOMEM;
    1328             :         }
    1329             : 
    1330             :         /* We always return the same pointer and identify the buffers through domain_ctx */
    1331          14 :         *buf = ACCEL_BUFFER_BASE;
    1332          14 :         *domain_ctx = accel_buf;
    1333          14 :         *domain = g_accel_domain;
    1334             : 
    1335          14 :         return 0;
    1336             : }
    1337             : 
    1338             : void
    1339          14 : spdk_accel_put_buf(struct spdk_io_channel *ch, void *buf,
    1340             :                    struct spdk_memory_domain *domain, void *domain_ctx)
    1341             : {
    1342          14 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
    1343          14 :         struct accel_buffer *accel_buf = domain_ctx;
    1344             : 
    1345          14 :         assert(domain == g_accel_domain);
    1346          14 :         assert(buf == ACCEL_BUFFER_BASE);
    1347             : 
    1348          14 :         accel_put_buf(accel_ch, accel_buf);
    1349          14 : }
    1350             : 
    1351             : static void
    1352         131 : accel_sequence_complete_task(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
    1353             : {
    1354         131 :         struct accel_io_channel *ch = seq->ch;
    1355             :         spdk_accel_step_cb cb_fn;
    1356             :         void *cb_arg;
    1357             : 
    1358         131 :         TAILQ_REMOVE(&seq->tasks, task, seq_link);
    1359         131 :         cb_fn = task->step_cb_fn;
    1360         131 :         cb_arg = task->cb_arg;
    1361         131 :         task->seq = NULL;
    1362         131 :         if (task->has_aux) {
    1363          58 :                 SLIST_INSERT_HEAD(&ch->task_aux_data_pool, task->aux, link);
    1364          58 :                 task->aux = NULL;
    1365          58 :                 task->has_aux = false;
    1366             :         }
    1367         131 :         STAILQ_INSERT_HEAD(&ch->task_pool, task, link);
    1368         131 :         if (cb_fn != NULL) {
    1369         131 :                 cb_fn(cb_arg);
    1370             :         }
    1371         131 : }
    1372             : 
    1373             : static void
    1374          54 : accel_sequence_complete_tasks(struct spdk_accel_sequence *seq)
    1375             : {
    1376             :         struct spdk_accel_task *task;
    1377             : 
    1378          72 :         while (!TAILQ_EMPTY(&seq->tasks)) {
    1379          18 :                 task = TAILQ_FIRST(&seq->tasks);
    1380          18 :                 accel_sequence_complete_task(seq, task);
    1381             :         }
    1382          54 : }
    1383             : 
    1384             : static void
    1385          52 : accel_sequence_complete(struct spdk_accel_sequence *seq)
    1386             : {
    1387          52 :         SPDK_DEBUGLOG(accel, "Completed sequence: %p with status: %d\n", seq, seq->status);
    1388             : 
    1389          52 :         accel_update_stats(seq->ch, sequence_executed, 1);
    1390          52 :         if (spdk_unlikely(seq->status != 0)) {
    1391          10 :                 accel_update_stats(seq->ch, sequence_failed, 1);
    1392             :         }
    1393             : 
    1394             :         /* First notify all users that appended operations to this sequence */
    1395          52 :         accel_sequence_complete_tasks(seq);
    1396             : 
    1397             :         /* Then notify the user that finished the sequence */
    1398          52 :         seq->cb_fn(seq->cb_arg, seq->status);
    1399             : 
    1400          52 :         accel_sequence_put(seq);
    1401          52 : }
    1402             : 
    1403             : static void
    1404          28 : accel_update_virt_iov(struct iovec *diov, struct iovec *siov, struct accel_buffer *accel_buf)
    1405             : {
    1406             :         uintptr_t offset;
    1407             : 
    1408          28 :         offset = (uintptr_t)siov->iov_base & ACCEL_BUFFER_OFFSET_MASK;
    1409          28 :         assert(offset < accel_buf->len);
    1410             : 
    1411          28 :         diov->iov_base = (char *)accel_buf->buf + offset;
    1412          28 :         diov->iov_len = siov->iov_len;
    1413          28 : }
    1414             : 
    1415             : static void
    1416          10 : accel_sequence_set_virtbuf(struct spdk_accel_sequence *seq, struct accel_buffer *buf)
    1417             : {
    1418             :         struct spdk_accel_task *task;
    1419             :         struct iovec *iov;
    1420             : 
    1421             :         /* Now that we've allocated the actual data buffer for this accel_buffer, update all tasks
    1422             :          * in a sequence that were using it.
    1423             :          */
    1424          38 :         TAILQ_FOREACH(task, &seq->tasks, seq_link) {
    1425          28 :                 if (task->src_domain == g_accel_domain && task->src_domain_ctx == buf) {
    1426          11 :                         if (!task->has_aux) {
    1427          11 :                                 task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
    1428          11 :                                 assert(task->aux && "Can't allocate aux data structure");
    1429          11 :                                 task->has_aux = true;
    1430          11 :                                 SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
    1431             :                         }
    1432             : 
    1433          11 :                         iov = &task->aux->iovs[SPDK_ACCEL_AXU_IOV_VIRT_SRC];
    1434          11 :                         assert(task->s.iovcnt == 1);
    1435          11 :                         accel_update_virt_iov(iov, &task->s.iovs[0], buf);
    1436          11 :                         task->src_domain = NULL;
    1437          11 :                         task->s.iovs = iov;
    1438             :                 }
    1439          28 :                 if (task->dst_domain == g_accel_domain && task->dst_domain_ctx == buf) {
    1440          17 :                         if (!task->has_aux) {
    1441           2 :                                 task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
    1442           2 :                                 assert(task->aux && "Can't allocate aux data structure");
    1443           2 :                                 task->has_aux = true;
    1444           2 :                                 SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
    1445             :                         }
    1446             : 
    1447          17 :                         iov = &task->aux->iovs[SPDK_ACCEL_AXU_IOV_VIRT_DST];
    1448          17 :                         assert(task->d.iovcnt == 1);
    1449          17 :                         accel_update_virt_iov(iov, &task->d.iovs[0], buf);
    1450          17 :                         task->dst_domain = NULL;
    1451          17 :                         task->d.iovs = iov;
    1452             :                 }
    1453             :         }
    1454          10 : }
    1455             : 
    1456             : static void accel_process_sequence(struct spdk_accel_sequence *seq);
    1457             : 
    1458             : static void
    1459           3 : accel_iobuf_get_virtbuf_cb(struct spdk_iobuf_entry *entry, void *buf)
    1460             : {
    1461             :         struct accel_buffer *accel_buf;
    1462             : 
    1463           3 :         accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
    1464             : 
    1465           3 :         assert(accel_buf->seq != NULL);
    1466           3 :         assert(accel_buf->buf == NULL);
    1467           3 :         accel_buf->buf = buf;
    1468             : 
    1469           3 :         assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF);
    1470           3 :         accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF);
    1471           3 :         accel_sequence_set_virtbuf(accel_buf->seq, accel_buf);
    1472           3 :         accel_process_sequence(accel_buf->seq);
    1473           3 : }
    1474             : 
    1475             : static bool
    1476          29 : accel_sequence_alloc_buf(struct spdk_accel_sequence *seq, struct accel_buffer *buf,
    1477             :                          spdk_iobuf_get_cb cb_fn)
    1478             : {
    1479          29 :         struct accel_io_channel *ch = seq->ch;
    1480             : 
    1481          29 :         assert(buf->buf == NULL);
    1482          29 :         assert(buf->seq == NULL);
    1483             : 
    1484          29 :         buf->seq = seq;
    1485          29 :         buf->buf = spdk_iobuf_get(&ch->iobuf, buf->len, &buf->iobuf, cb_fn);
    1486          29 :         if (buf->buf == NULL) {
    1487           5 :                 accel_update_stats(ch, retry.iobuf, 1);
    1488           5 :                 return false;
    1489             :         }
    1490             : 
    1491          24 :         return true;
    1492             : }
    1493             : 
    1494             : static bool
    1495          88 : accel_sequence_check_virtbuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
    1496             : {
    1497             :         /* If a task doesn't have dst/src (e.g. fill, crc32), its dst/src domain should be set to
    1498             :          * NULL */
    1499          88 :         if (task->src_domain == g_accel_domain) {
    1500           0 :                 if (!accel_sequence_alloc_buf(seq, task->src_domain_ctx,
    1501             :                                               accel_iobuf_get_virtbuf_cb)) {
    1502           0 :                         return false;
    1503             :                 }
    1504             : 
    1505           0 :                 accel_sequence_set_virtbuf(seq, task->src_domain_ctx);
    1506             :         }
    1507             : 
    1508          88 :         if (task->dst_domain == g_accel_domain) {
    1509          10 :                 if (!accel_sequence_alloc_buf(seq, task->dst_domain_ctx,
    1510             :                                               accel_iobuf_get_virtbuf_cb)) {
    1511           3 :                         return false;
    1512             :                 }
    1513             : 
    1514           7 :                 accel_sequence_set_virtbuf(seq, task->dst_domain_ctx);
    1515             :         }
    1516             : 
    1517          85 :         return true;
    1518             : }
    1519             : 
    1520             : static void
    1521           0 : accel_sequence_get_buf_cb(struct spdk_iobuf_entry *entry, void *buf)
    1522             : {
    1523             :         struct accel_buffer *accel_buf;
    1524             : 
    1525           0 :         accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
    1526             : 
    1527           0 :         assert(accel_buf->seq != NULL);
    1528           0 :         assert(accel_buf->buf == NULL);
    1529           0 :         accel_buf->buf = buf;
    1530             : 
    1531           0 :         accel_sequence_set_virtbuf(accel_buf->seq, accel_buf);
    1532           0 :         accel_buf->cb_fn(accel_buf->seq, accel_buf->cb_ctx);
    1533           0 : }
    1534             : 
    1535             : bool
    1536           0 : spdk_accel_alloc_sequence_buf(struct spdk_accel_sequence *seq, void *buf,
    1537             :                               struct spdk_memory_domain *domain, void *domain_ctx,
    1538             :                               spdk_accel_sequence_get_buf_cb cb_fn, void *cb_ctx)
    1539             : {
    1540           0 :         struct accel_buffer *accel_buf = domain_ctx;
    1541             : 
    1542           0 :         assert(domain == g_accel_domain);
    1543           0 :         accel_buf->cb_fn = cb_fn;
    1544           0 :         accel_buf->cb_ctx = cb_ctx;
    1545             : 
    1546           0 :         if (!accel_sequence_alloc_buf(seq, accel_buf, accel_sequence_get_buf_cb)) {
    1547           0 :                 return false;
    1548             :         }
    1549             : 
    1550           0 :         accel_sequence_set_virtbuf(seq, accel_buf);
    1551             : 
    1552           0 :         return true;
    1553             : }
    1554             : 
    1555             : struct spdk_accel_task *
    1556          24 : spdk_accel_sequence_first_task(struct spdk_accel_sequence *seq)
    1557             : {
    1558          24 :         return TAILQ_FIRST(&seq->tasks);
    1559             : }
    1560             : 
    1561             : struct spdk_accel_task *
    1562           0 : spdk_accel_sequence_next_task(struct spdk_accel_task *task)
    1563             : {
    1564           0 :         return TAILQ_NEXT(task, seq_link);
    1565             : }
    1566             : 
    1567             : static inline void
    1568          19 : accel_set_bounce_buffer(struct spdk_accel_bounce_buffer *bounce, struct iovec **iovs,
    1569             :                         uint32_t *iovcnt, struct spdk_memory_domain **domain, void **domain_ctx,
    1570             :                         struct accel_buffer *buf)
    1571             : {
    1572          19 :         bounce->orig_iovs = *iovs;
    1573          19 :         bounce->orig_iovcnt = *iovcnt;
    1574          19 :         bounce->orig_domain = *domain;
    1575          19 :         bounce->orig_domain_ctx = *domain_ctx;
    1576          19 :         bounce->iov.iov_base = buf->buf;
    1577          19 :         bounce->iov.iov_len = buf->len;
    1578             : 
    1579          19 :         *iovs = &bounce->iov;
    1580          19 :         *iovcnt = 1;
    1581          19 :         *domain = NULL;
    1582          19 : }
    1583             : 
    1584             : static void
    1585           1 : accel_iobuf_get_src_bounce_cb(struct spdk_iobuf_entry *entry, void *buf)
    1586             : {
    1587             :         struct spdk_accel_task *task;
    1588             :         struct accel_buffer *accel_buf;
    1589             : 
    1590           1 :         accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
    1591           1 :         assert(accel_buf->buf == NULL);
    1592           1 :         accel_buf->buf = buf;
    1593             : 
    1594           1 :         task = TAILQ_FIRST(&accel_buf->seq->tasks);
    1595           1 :         assert(task != NULL);
    1596             : 
    1597           1 :         assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
    1598           1 :         accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
    1599           1 :         assert(task->aux);
    1600           1 :         assert(task->has_aux);
    1601           1 :         accel_set_bounce_buffer(&task->aux->bounce.s, &task->s.iovs, &task->s.iovcnt, &task->src_domain,
    1602             :                                 &task->src_domain_ctx, accel_buf);
    1603           1 :         accel_process_sequence(accel_buf->seq);
    1604           1 : }
    1605             : 
    1606             : static void
    1607           1 : accel_iobuf_get_dst_bounce_cb(struct spdk_iobuf_entry *entry, void *buf)
    1608             : {
    1609             :         struct spdk_accel_task *task;
    1610             :         struct accel_buffer *accel_buf;
    1611             : 
    1612           1 :         accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
    1613           1 :         assert(accel_buf->buf == NULL);
    1614           1 :         accel_buf->buf = buf;
    1615             : 
    1616           1 :         task = TAILQ_FIRST(&accel_buf->seq->tasks);
    1617           1 :         assert(task != NULL);
    1618             : 
    1619           1 :         assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
    1620           1 :         accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
    1621           1 :         assert(task->aux);
    1622           1 :         assert(task->has_aux);
    1623           1 :         accel_set_bounce_buffer(&task->aux->bounce.d, &task->d.iovs, &task->d.iovcnt, &task->dst_domain,
    1624             :                                 &task->dst_domain_ctx, accel_buf);
    1625           1 :         accel_process_sequence(accel_buf->seq);
    1626           1 : }
    1627             : 
    1628             : static int
    1629          72 : accel_sequence_check_bouncebuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
    1630             : {
    1631             :         struct accel_buffer *buf;
    1632             : 
    1633          72 :         if (task->src_domain != NULL) {
    1634             :                 /* By the time we're here, accel buffers should have been allocated */
    1635           9 :                 assert(task->src_domain != g_accel_domain);
    1636             : 
    1637           9 :                 if (!task->has_aux) {
    1638           8 :                         task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
    1639           8 :                         if (spdk_unlikely(!task->aux)) {
    1640           0 :                                 SPDK_ERRLOG("Can't allocate aux data structure\n");
    1641           0 :                                 assert(0);
    1642             :                                 return -EAGAIN;
    1643             :                         }
    1644           8 :                         task->has_aux = true;
    1645           8 :                         SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
    1646             :                 }
    1647           9 :                 buf = accel_get_buf(seq->ch, accel_get_iovlen(task->s.iovs, task->s.iovcnt));
    1648           9 :                 if (buf == NULL) {
    1649           0 :                         SPDK_ERRLOG("Couldn't allocate buffer descriptor\n");
    1650           0 :                         return -ENOMEM;
    1651             :                 }
    1652             : 
    1653           9 :                 SLIST_INSERT_HEAD(&seq->bounce_bufs, buf, link);
    1654           9 :                 if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_src_bounce_cb)) {
    1655           1 :                         return -EAGAIN;
    1656             :                 }
    1657             : 
    1658           8 :                 accel_set_bounce_buffer(&task->aux->bounce.s, &task->s.iovs, &task->s.iovcnt,
    1659             :                                         &task->src_domain, &task->src_domain_ctx, buf);
    1660             :         }
    1661             : 
    1662          71 :         if (task->dst_domain != NULL) {
    1663             :                 /* By the time we're here, accel buffers should have been allocated */
    1664          10 :                 assert(task->dst_domain != g_accel_domain);
    1665             : 
    1666          10 :                 if (!task->has_aux) {
    1667           0 :                         task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
    1668           0 :                         if (spdk_unlikely(!task->aux)) {
    1669           0 :                                 SPDK_ERRLOG("Can't allocate aux data structure\n");
    1670           0 :                                 assert(0);
    1671             :                                 return -EAGAIN;
    1672             :                         }
    1673           0 :                         task->has_aux = true;
    1674           0 :                         SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
    1675             :                 }
    1676          10 :                 buf = accel_get_buf(seq->ch, accel_get_iovlen(task->d.iovs, task->d.iovcnt));
    1677          10 :                 if (buf == NULL) {
    1678             :                         /* The src buffer will be released when a sequence is completed */
    1679           0 :                         SPDK_ERRLOG("Couldn't allocate buffer descriptor\n");
    1680           0 :                         return -ENOMEM;
    1681             :                 }
    1682             : 
    1683          10 :                 SLIST_INSERT_HEAD(&seq->bounce_bufs, buf, link);
    1684          10 :                 if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_dst_bounce_cb)) {
    1685           1 :                         return -EAGAIN;
    1686             :                 }
    1687             : 
    1688           9 :                 accel_set_bounce_buffer(&task->aux->bounce.d, &task->d.iovs, &task->d.iovcnt,
    1689             :                                         &task->dst_domain, &task->dst_domain_ctx, buf);
    1690             :         }
    1691             : 
    1692          70 :         return 0;
    1693             : }
    1694             : 
    1695             : static void
    1696           8 : accel_task_pull_data_cb(void *ctx, int status)
    1697             : {
    1698           8 :         struct spdk_accel_sequence *seq = ctx;
    1699             : 
    1700           8 :         assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA);
    1701           8 :         if (spdk_likely(status == 0)) {
    1702           7 :                 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
    1703             :         } else {
    1704           1 :                 accel_sequence_set_fail(seq, status);
    1705             :         }
    1706             : 
    1707           8 :         accel_process_sequence(seq);
    1708           8 : }
    1709             : 
    1710             : static void
    1711           9 : accel_task_pull_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
    1712             : {
    1713             :         int rc;
    1714             : 
    1715           9 :         assert(task->has_aux);
    1716           9 :         assert(task->aux);
    1717           9 :         assert(task->aux->bounce.s.orig_iovs != NULL);
    1718           9 :         assert(task->aux->bounce.s.orig_domain != NULL);
    1719           9 :         assert(task->aux->bounce.s.orig_domain != g_accel_domain);
    1720           9 :         assert(!g_modules_opc[task->op_code].supports_memory_domains);
    1721             : 
    1722           9 :         rc = spdk_memory_domain_pull_data(task->aux->bounce.s.orig_domain,
    1723           9 :                                           task->aux->bounce.s.orig_domain_ctx,
    1724           9 :                                           task->aux->bounce.s.orig_iovs, task->aux->bounce.s.orig_iovcnt,
    1725             :                                           task->s.iovs, task->s.iovcnt,
    1726             :                                           accel_task_pull_data_cb, seq);
    1727           9 :         if (spdk_unlikely(rc != 0)) {
    1728           1 :                 SPDK_ERRLOG("Failed to pull data from memory domain: %s, rc: %d\n",
    1729             :                             spdk_memory_domain_get_dma_device_id(task->aux->bounce.s.orig_domain), rc);
    1730           1 :                 accel_sequence_set_fail(seq, rc);
    1731             :         }
    1732           9 : }
    1733             : 
    1734             : static void
    1735           7 : accel_task_push_data_cb(void *ctx, int status)
    1736             : {
    1737           7 :         struct spdk_accel_sequence *seq = ctx;
    1738             : 
    1739           7 :         assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA);
    1740           7 :         if (spdk_likely(status == 0)) {
    1741           6 :                 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK);
    1742             :         } else {
    1743           1 :                 accel_sequence_set_fail(seq, status);
    1744             :         }
    1745             : 
    1746           7 :         accel_process_sequence(seq);
    1747           7 : }
    1748             : 
    1749             : static void
    1750           8 : accel_task_push_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
    1751             : {
    1752             :         int rc;
    1753             : 
    1754           8 :         assert(task->has_aux);
    1755           8 :         assert(task->aux);
    1756           8 :         assert(task->aux->bounce.d.orig_iovs != NULL);
    1757           8 :         assert(task->aux->bounce.d.orig_domain != NULL);
    1758           8 :         assert(task->aux->bounce.d.orig_domain != g_accel_domain);
    1759           8 :         assert(!g_modules_opc[task->op_code].supports_memory_domains);
    1760             : 
    1761           8 :         rc = spdk_memory_domain_push_data(task->aux->bounce.d.orig_domain,
    1762           8 :                                           task->aux->bounce.d.orig_domain_ctx,
    1763           8 :                                           task->aux->bounce.d.orig_iovs, task->aux->bounce.d.orig_iovcnt,
    1764             :                                           task->d.iovs, task->d.iovcnt,
    1765             :                                           accel_task_push_data_cb, seq);
    1766           8 :         if (spdk_unlikely(rc != 0)) {
    1767           1 :                 SPDK_ERRLOG("Failed to push data to memory domain: %s, rc: %d\n",
    1768             :                             spdk_memory_domain_get_dma_device_id(task->aux->bounce.s.orig_domain), rc);
    1769           1 :                 accel_sequence_set_fail(seq, rc);
    1770             :         }
    1771           8 : }
    1772             : 
    1773             : static void
    1774         165 : accel_process_sequence(struct spdk_accel_sequence *seq)
    1775             : {
    1776         165 :         struct accel_io_channel *accel_ch = seq->ch;
    1777             :         struct spdk_accel_task *task;
    1778             :         enum accel_sequence_state state;
    1779             :         int rc;
    1780             : 
    1781             :         /* Prevent recursive calls to this function */
    1782         165 :         if (spdk_unlikely(seq->in_process_sequence)) {
    1783          74 :                 return;
    1784             :         }
    1785          91 :         seq->in_process_sequence = true;
    1786             : 
    1787          91 :         task = TAILQ_FIRST(&seq->tasks);
    1788             :         do {
    1789         371 :                 state = seq->state;
    1790         371 :                 switch (state) {
    1791          90 :                 case ACCEL_SEQUENCE_STATE_INIT:
    1792          90 :                         if (g_accel_driver != NULL) {
    1793          13 :                                 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS);
    1794          13 :                                 break;
    1795             :                         }
    1796             :                 /* Fall through */
    1797             :                 case ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF:
    1798          88 :                         accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF);
    1799          88 :                         if (!accel_sequence_check_virtbuf(seq, task)) {
    1800             :                                 /* We couldn't allocate a buffer, wait until one is available */
    1801           3 :                                 break;
    1802             :                         }
    1803          85 :                         accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
    1804             :                 /* Fall through */
    1805          87 :                 case ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF:
    1806             :                         /* If a module supports memory domains, we don't need to allocate bounce
    1807             :                          * buffers */
    1808          87 :                         if (g_modules_opc[task->op_code].supports_memory_domains) {
    1809          15 :                                 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
    1810          15 :                                 break;
    1811             :                         }
    1812          72 :                         accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
    1813          72 :                         rc = accel_sequence_check_bouncebuf(seq, task);
    1814          72 :                         if (spdk_unlikely(rc != 0)) {
    1815             :                                 /* We couldn't allocate a buffer, wait until one is available */
    1816           2 :                                 if (rc == -EAGAIN) {
    1817           2 :                                         break;
    1818             :                                 }
    1819           0 :                                 accel_sequence_set_fail(seq, rc);
    1820           0 :                                 break;
    1821             :                         }
    1822          70 :                         if (task->has_aux && task->s.iovs == &task->aux->bounce.s.iov) {
    1823           9 :                                 assert(task->aux->bounce.s.orig_iovs);
    1824           9 :                                 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PULL_DATA);
    1825           9 :                                 break;
    1826             :                         }
    1827          61 :                         accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
    1828             :                 /* Fall through */
    1829          83 :                 case ACCEL_SEQUENCE_STATE_EXEC_TASK:
    1830          83 :                         SPDK_DEBUGLOG(accel, "Executing %s operation, sequence: %p\n",
    1831             :                                       g_opcode_strings[task->op_code], seq);
    1832             : 
    1833          83 :                         accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_TASK);
    1834          83 :                         rc = accel_submit_task(accel_ch, task);
    1835          83 :                         if (spdk_unlikely(rc != 0)) {
    1836           2 :                                 SPDK_ERRLOG("Failed to submit %s operation, sequence: %p\n",
    1837             :                                             g_opcode_strings[task->op_code], seq);
    1838           2 :                                 accel_sequence_set_fail(seq, rc);
    1839             :                         }
    1840          83 :                         break;
    1841           9 :                 case ACCEL_SEQUENCE_STATE_PULL_DATA:
    1842           9 :                         accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA);
    1843           9 :                         accel_task_pull_data(seq, task);
    1844           9 :                         break;
    1845          79 :                 case ACCEL_SEQUENCE_STATE_COMPLETE_TASK:
    1846          79 :                         if (task->has_aux && task->d.iovs == &task->aux->bounce.d.iov) {
    1847           8 :                                 assert(task->aux->bounce.d.orig_iovs);
    1848           8 :                                 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PUSH_DATA);
    1849           8 :                                 break;
    1850             :                         }
    1851          71 :                         accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK);
    1852          71 :                         break;
    1853           8 :                 case ACCEL_SEQUENCE_STATE_PUSH_DATA:
    1854           8 :                         accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA);
    1855           8 :                         accel_task_push_data(seq, task);
    1856           8 :                         break;
    1857          77 :                 case ACCEL_SEQUENCE_STATE_NEXT_TASK:
    1858          77 :                         accel_sequence_complete_task(seq, task);
    1859             :                         /* Check if there are any remaining tasks */
    1860          77 :                         task = TAILQ_FIRST(&seq->tasks);
    1861          77 :                         if (task == NULL) {
    1862             :                                 /* Immediately return here to make sure we don't touch the sequence
    1863             :                                  * after it's completed */
    1864          39 :                                 accel_sequence_complete(seq);
    1865          39 :                                 return;
    1866             :                         }
    1867          38 :                         accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_INIT);
    1868          38 :                         break;
    1869          13 :                 case ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS:
    1870          13 :                         assert(!TAILQ_EMPTY(&seq->tasks));
    1871             : 
    1872          13 :                         accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS);
    1873          13 :                         rc = g_accel_driver->execute_sequence(accel_ch->driver_channel, seq);
    1874          13 :                         if (spdk_unlikely(rc != 0)) {
    1875           1 :                                 SPDK_ERRLOG("Failed to execute sequence: %p using driver: %s\n",
    1876             :                                             seq, g_accel_driver->name);
    1877           1 :                                 accel_sequence_set_fail(seq, rc);
    1878             :                         }
    1879          13 :                         break;
    1880          11 :                 case ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS:
    1881             :                         /* Get the task again, as the driver might have completed some tasks
    1882             :                          * synchronously */
    1883          11 :                         task = TAILQ_FIRST(&seq->tasks);
    1884          11 :                         if (task == NULL) {
    1885             :                                 /* Immediately return here to make sure we don't touch the sequence
    1886             :                                  * after it's completed */
    1887           3 :                                 accel_sequence_complete(seq);
    1888           3 :                                 return;
    1889             :                         }
    1890             :                         /* We don't want to execute the next task through the driver, so we
    1891             :                          * explicitly omit the INIT state here */
    1892           8 :                         accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF);
    1893           8 :                         break;
    1894          10 :                 case ACCEL_SEQUENCE_STATE_ERROR:
    1895             :                         /* Immediately return here to make sure we don't touch the sequence
    1896             :                          * after it's completed */
    1897          10 :                         assert(seq->status != 0);
    1898          10 :                         accel_sequence_complete(seq);
    1899          10 :                         return;
    1900          39 :                 case ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF:
    1901             :                 case ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF:
    1902             :                 case ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA:
    1903             :                 case ACCEL_SEQUENCE_STATE_AWAIT_TASK:
    1904             :                 case ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA:
    1905             :                 case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS:
    1906          39 :                         break;
    1907           0 :                 default:
    1908           0 :                         assert(0 && "bad state");
    1909             :                         break;
    1910             :                 }
    1911         319 :         } while (seq->state != state);
    1912             : 
    1913          39 :         seq->in_process_sequence = false;
    1914             : }
    1915             : 
    1916             : static void
    1917          93 : accel_sequence_task_cb(void *cb_arg, int status)
    1918             : {
    1919          93 :         struct spdk_accel_sequence *seq = cb_arg;
    1920          93 :         struct spdk_accel_task *task = TAILQ_FIRST(&seq->tasks);
    1921             : 
    1922          93 :         switch (seq->state) {
    1923          81 :         case ACCEL_SEQUENCE_STATE_AWAIT_TASK:
    1924          81 :                 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_COMPLETE_TASK);
    1925          81 :                 if (spdk_unlikely(status != 0)) {
    1926           2 :                         SPDK_ERRLOG("Failed to execute %s operation, sequence: %p\n",
    1927             :                                     g_opcode_strings[task->op_code], seq);
    1928           2 :                         accel_sequence_set_fail(seq, status);
    1929             :                 }
    1930             : 
    1931          81 :                 accel_process_sequence(seq);
    1932          81 :                 break;
    1933          12 :         case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS:
    1934          12 :                 assert(g_accel_driver != NULL);
    1935             :                 /* Immediately remove the task from the outstanding list to make sure the next call
    1936             :                  * to spdk_accel_sequence_first_task() doesn't return it */
    1937          12 :                 accel_sequence_complete_task(seq, task);
    1938          12 :                 if (spdk_unlikely(status != 0)) {
    1939           1 :                         SPDK_ERRLOG("Failed to execute %s operation, sequence: %p through "
    1940             :                                     "driver: %s\n", g_opcode_strings[task->op_code], seq,
    1941             :                                     g_accel_driver->name);
    1942             :                         /* Update status without using accel_sequence_set_fail() to avoid changing
    1943             :                          * seq's state to ERROR until driver calls spdk_accel_sequence_continue() */
    1944           1 :                         seq->status = status;
    1945             :                 }
    1946          12 :                 break;
    1947           0 :         default:
    1948           0 :                 assert(0 && "bad state");
    1949             :                 break;
    1950             :         }
    1951          93 : }
    1952             : 
    1953             : void
    1954          12 : spdk_accel_sequence_continue(struct spdk_accel_sequence *seq)
    1955             : {
    1956          12 :         assert(g_accel_driver != NULL);
    1957          12 :         assert(seq->state == ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS);
    1958             : 
    1959          12 :         if (spdk_likely(seq->status == 0)) {
    1960          11 :                 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS);
    1961             :         } else {
    1962           1 :                 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR);
    1963             :         }
    1964             : 
    1965          12 :         accel_process_sequence(seq);
    1966          12 : }
    1967             : 
    1968             : static bool
    1969          29 : accel_compare_iovs(struct iovec *iova, uint32_t iovacnt, struct iovec *iovb, uint32_t iovbcnt)
    1970             : {
    1971             :         /* For now, just do a dumb check that the iovecs arrays are exactly the same */
    1972          29 :         if (iovacnt != iovbcnt) {
    1973           0 :                 return false;
    1974             :         }
    1975             : 
    1976          29 :         return memcmp(iova, iovb, sizeof(*iova) * iovacnt) == 0;
    1977             : }
    1978             : 
    1979             : static bool
    1980          22 : accel_task_set_dstbuf(struct spdk_accel_task *task, struct spdk_accel_task *next)
    1981             : {
    1982             :         struct spdk_accel_task *prev;
    1983             : 
    1984          22 :         switch (task->op_code) {
    1985          18 :         case SPDK_ACCEL_OPC_DECOMPRESS:
    1986             :         case SPDK_ACCEL_OPC_FILL:
    1987             :         case SPDK_ACCEL_OPC_ENCRYPT:
    1988             :         case SPDK_ACCEL_OPC_DECRYPT:
    1989          18 :                 if (task->dst_domain != next->src_domain) {
    1990           0 :                         return false;
    1991             :                 }
    1992          18 :                 if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt,
    1993             :                                         next->s.iovs, next->s.iovcnt)) {
    1994           1 :                         return false;
    1995             :                 }
    1996          17 :                 task->d.iovs = next->d.iovs;
    1997          17 :                 task->d.iovcnt = next->d.iovcnt;
    1998          17 :                 task->dst_domain = next->dst_domain;
    1999          17 :                 task->dst_domain_ctx = next->dst_domain_ctx;
    2000          17 :                 break;
    2001           4 :         case SPDK_ACCEL_OPC_CRC32C:
    2002             :                 /* crc32 is special, because it doesn't have a dst buffer */
    2003           4 :                 if (task->src_domain != next->src_domain) {
    2004           0 :                         return false;
    2005             :                 }
    2006           4 :                 if (!accel_compare_iovs(task->s.iovs, task->s.iovcnt,
    2007             :                                         next->s.iovs, next->s.iovcnt)) {
    2008           1 :                         return false;
    2009             :                 }
    2010             :                 /* We can only change crc32's buffer if we can change previous task's buffer */
    2011           3 :                 prev = TAILQ_PREV(task, accel_sequence_tasks, seq_link);
    2012           3 :                 if (prev == NULL) {
    2013           1 :                         return false;
    2014             :                 }
    2015           2 :                 if (!accel_task_set_dstbuf(prev, next)) {
    2016           0 :                         return false;
    2017             :                 }
    2018           2 :                 task->s.iovs = next->d.iovs;
    2019           2 :                 task->s.iovcnt = next->d.iovcnt;
    2020           2 :                 task->src_domain = next->dst_domain;
    2021           2 :                 task->src_domain_ctx = next->dst_domain_ctx;
    2022           2 :                 break;
    2023           0 :         default:
    2024           0 :                 return false;
    2025             :         }
    2026             : 
    2027          19 :         return true;
    2028             : }
    2029             : 
    2030             : static void
    2031          72 : accel_sequence_merge_tasks(struct spdk_accel_sequence *seq, struct spdk_accel_task *task,
    2032             :                            struct spdk_accel_task **next_task)
    2033             : {
    2034          72 :         struct spdk_accel_task *next = *next_task;
    2035             : 
    2036          72 :         switch (task->op_code) {
    2037           8 :         case SPDK_ACCEL_OPC_COPY:
    2038             :                 /* We only allow changing src of operations that actually have a src, e.g. we never
    2039             :                  * do it for fill.  Theoretically, it is possible, but we'd have to be careful to
    2040             :                  * change the src of the operation after fill (which in turn could also be a fill).
    2041             :                  * So, for the sake of simplicity, skip this type of operations for now.
    2042             :                  */
    2043           8 :                 if (next->op_code != SPDK_ACCEL_OPC_DECOMPRESS &&
    2044           6 :                     next->op_code != SPDK_ACCEL_OPC_COPY &&
    2045           5 :                     next->op_code != SPDK_ACCEL_OPC_ENCRYPT &&
    2046           4 :                     next->op_code != SPDK_ACCEL_OPC_DECRYPT &&
    2047           2 :                     next->op_code != SPDK_ACCEL_OPC_CRC32C) {
    2048           1 :                         break;
    2049             :                 }
    2050           7 :                 if (task->dst_domain != next->src_domain) {
    2051           0 :                         break;
    2052             :                 }
    2053           7 :                 if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt,
    2054             :                                         next->s.iovs, next->s.iovcnt)) {
    2055           0 :                         break;
    2056             :                 }
    2057           7 :                 next->s.iovs = task->s.iovs;
    2058           7 :                 next->s.iovcnt = task->s.iovcnt;
    2059           7 :                 next->src_domain = task->src_domain;
    2060           7 :                 next->src_domain_ctx = task->src_domain_ctx;
    2061           7 :                 accel_sequence_complete_task(seq, task);
    2062           7 :                 break;
    2063          64 :         case SPDK_ACCEL_OPC_DECOMPRESS:
    2064             :         case SPDK_ACCEL_OPC_FILL:
    2065             :         case SPDK_ACCEL_OPC_ENCRYPT:
    2066             :         case SPDK_ACCEL_OPC_DECRYPT:
    2067             :         case SPDK_ACCEL_OPC_CRC32C:
    2068             :                 /* We can only merge tasks when one of them is a copy */
    2069          64 :                 if (next->op_code != SPDK_ACCEL_OPC_COPY) {
    2070          44 :                         break;
    2071             :                 }
    2072          20 :                 if (!accel_task_set_dstbuf(task, next)) {
    2073           3 :                         break;
    2074             :                 }
    2075             :                 /* We're removing next_task from the tasks queue, so we need to update its pointer,
    2076             :                  * so that the TAILQ_FOREACH_SAFE() loop below works correctly */
    2077          17 :                 *next_task = TAILQ_NEXT(next, seq_link);
    2078          17 :                 accel_sequence_complete_task(seq, next);
    2079          17 :                 break;
    2080           0 :         default:
    2081           0 :                 assert(0 && "bad opcode");
    2082             :                 break;
    2083             :         }
    2084          72 : }
    2085             : 
    2086             : void
    2087          52 : spdk_accel_sequence_finish(struct spdk_accel_sequence *seq,
    2088             :                            spdk_accel_completion_cb cb_fn, void *cb_arg)
    2089             : {
    2090          52 :         struct spdk_accel_task *task, *next;
    2091             : 
    2092             :         /* Try to remove any copy operations if possible */
    2093         124 :         TAILQ_FOREACH_SAFE(task, &seq->tasks, seq_link, next) {
    2094         109 :                 if (next == NULL) {
    2095          37 :                         break;
    2096             :                 }
    2097          72 :                 accel_sequence_merge_tasks(seq, task, &next);
    2098             :         }
    2099             : 
    2100          52 :         seq->cb_fn = cb_fn;
    2101          52 :         seq->cb_arg = cb_arg;
    2102             : 
    2103          52 :         accel_process_sequence(seq);
    2104          52 : }
    2105             : 
    2106             : void
    2107           0 : spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq)
    2108             : {
    2109           0 :         struct accel_sequence_tasks tasks = TAILQ_HEAD_INITIALIZER(tasks);
    2110             :         struct spdk_accel_task *task;
    2111             : 
    2112           0 :         TAILQ_SWAP(&tasks, &seq->tasks, spdk_accel_task, seq_link);
    2113             : 
    2114           0 :         while (!TAILQ_EMPTY(&tasks)) {
    2115           0 :                 task = TAILQ_FIRST(&tasks);
    2116           0 :                 TAILQ_REMOVE(&tasks, task, seq_link);
    2117           0 :                 TAILQ_INSERT_HEAD(&seq->tasks, task, seq_link);
    2118             :         }
    2119           0 : }
    2120             : 
    2121             : void
    2122           3 : spdk_accel_sequence_abort(struct spdk_accel_sequence *seq)
    2123             : {
    2124           3 :         if (seq == NULL) {
    2125           1 :                 return;
    2126             :         }
    2127             : 
    2128           2 :         accel_sequence_complete_tasks(seq);
    2129           2 :         accel_sequence_put(seq);
    2130             : }
    2131             : 
    2132             : struct spdk_memory_domain *
    2133           0 : spdk_accel_get_memory_domain(void)
    2134             : {
    2135           0 :         return g_accel_domain;
    2136             : }
    2137             : 
    2138             : static struct spdk_accel_module_if *
    2139           7 : _module_find_by_name(const char *name)
    2140             : {
    2141           7 :         struct spdk_accel_module_if *accel_module = NULL;
    2142             : 
    2143          16 :         TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
    2144          10 :                 if (strcmp(name, accel_module->name) == 0) {
    2145           1 :                         break;
    2146             :                 }
    2147             :         }
    2148             : 
    2149           7 :         return accel_module;
    2150             : }
    2151             : 
    2152             : static inline struct spdk_accel_crypto_key *
    2153           0 : _accel_crypto_key_get(const char *name)
    2154             : {
    2155             :         struct spdk_accel_crypto_key *key;
    2156             : 
    2157           0 :         assert(spdk_spin_held(&g_keyring_spin));
    2158             : 
    2159           0 :         TAILQ_FOREACH(key, &g_keyring, link) {
    2160           0 :                 if (strcmp(name, key->param.key_name) == 0) {
    2161           0 :                         return key;
    2162             :                 }
    2163             :         }
    2164             : 
    2165           0 :         return NULL;
    2166             : }
    2167             : 
    2168             : static void
    2169           0 : accel_crypto_key_free_mem(struct spdk_accel_crypto_key *key)
    2170             : {
    2171           0 :         if (key->param.hex_key) {
    2172           0 :                 spdk_memset_s(key->param.hex_key, key->key_size * 2, 0, key->key_size * 2);
    2173           0 :                 free(key->param.hex_key);
    2174             :         }
    2175           0 :         if (key->param.hex_key2) {
    2176           0 :                 spdk_memset_s(key->param.hex_key2, key->key2_size * 2, 0, key->key2_size * 2);
    2177           0 :                 free(key->param.hex_key2);
    2178             :         }
    2179           0 :         free(key->param.tweak_mode);
    2180           0 :         free(key->param.key_name);
    2181           0 :         free(key->param.cipher);
    2182           0 :         if (key->key) {
    2183           0 :                 spdk_memset_s(key->key, key->key_size, 0, key->key_size);
    2184           0 :                 free(key->key);
    2185             :         }
    2186           0 :         if (key->key2) {
    2187           0 :                 spdk_memset_s(key->key2, key->key2_size, 0, key->key2_size);
    2188           0 :                 free(key->key2);
    2189             :         }
    2190           0 :         free(key);
    2191           0 : }
    2192             : 
    2193             : static void
    2194           0 : accel_crypto_key_destroy_unsafe(struct spdk_accel_crypto_key *key)
    2195             : {
    2196           0 :         assert(key->module_if);
    2197           0 :         assert(key->module_if->crypto_key_deinit);
    2198             : 
    2199           0 :         key->module_if->crypto_key_deinit(key);
    2200           0 :         accel_crypto_key_free_mem(key);
    2201           0 : }
    2202             : 
    2203             : /*
    2204             :  * This function mitigates a timing side channel which could be caused by using strcmp()
    2205             :  * Please refer to chapter "Mitigating Information Leakage Based on Variable Timing" in
    2206             :  * the article [1] for more details
    2207             :  * [1] https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/secure-coding/mitigate-timing-side-channel-crypto-implementation.html
    2208             :  */
    2209             : static bool
    2210           0 : accel_aes_xts_keys_equal(const char *k1, size_t k1_len, const char *k2, size_t k2_len)
    2211             : {
    2212             :         size_t i;
    2213           0 :         volatile size_t x = k1_len ^ k2_len;
    2214             : 
    2215           0 :         for (i = 0; ((i < k1_len) & (i < k2_len)); i++) {
    2216           0 :                 x |= k1[i] ^ k2[i];
    2217             :         }
    2218             : 
    2219           0 :         return x == 0;
    2220             : }
    2221             : 
    2222             : static const char *g_tweak_modes[] = {
    2223             :         [SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA] = "SIMPLE_LBA",
    2224             :         [SPDK_ACCEL_CRYPTO_TWEAK_MODE_JOIN_NEG_LBA_WITH_LBA] = "JOIN_NEG_LBA_WITH_LBA",
    2225             :         [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_FULL_LBA] = "INCR_512_FULL_LBA",
    2226             :         [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_UPPER_LBA] = "INCR_512_UPPER_LBA",
    2227             : };
    2228             : 
    2229             : static const char *g_ciphers[] = {
    2230             :         [SPDK_ACCEL_CIPHER_AES_CBC] = "AES_CBC",
    2231             :         [SPDK_ACCEL_CIPHER_AES_XTS] = "AES_XTS",
    2232             : };
    2233             : 
    2234             : int
    2235           0 : spdk_accel_crypto_key_create(const struct spdk_accel_crypto_key_create_param *param)
    2236             : {
    2237             :         struct spdk_accel_module_if *module;
    2238             :         struct spdk_accel_crypto_key *key;
    2239             :         size_t hex_key_size, hex_key2_size;
    2240           0 :         bool found = false;
    2241             :         size_t i;
    2242             :         int rc;
    2243             : 
    2244           0 :         if (!param || !param->hex_key || !param->cipher || !param->key_name) {
    2245           0 :                 return -EINVAL;
    2246             :         }
    2247             : 
    2248           0 :         if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) {
    2249             :                 /* hardly ever possible, but let's check and warn the user */
    2250           0 :                 SPDK_ERRLOG("Different accel modules are used for encryption and decryption\n");
    2251             :         }
    2252           0 :         module = g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module;
    2253             : 
    2254           0 :         if (!module) {
    2255           0 :                 SPDK_ERRLOG("No accel module found assigned for crypto operation\n");
    2256           0 :                 return -ENOENT;
    2257             :         }
    2258             : 
    2259           0 :         if (!module->crypto_key_init || !module->crypto_supports_cipher) {
    2260           0 :                 SPDK_ERRLOG("Module %s doesn't support crypto operations\n", module->name);
    2261           0 :                 return -ENOTSUP;
    2262             :         }
    2263             : 
    2264           0 :         key = calloc(1, sizeof(*key));
    2265           0 :         if (!key) {
    2266           0 :                 return -ENOMEM;
    2267             :         }
    2268             : 
    2269           0 :         key->param.key_name = strdup(param->key_name);
    2270           0 :         if (!key->param.key_name) {
    2271           0 :                 rc = -ENOMEM;
    2272           0 :                 goto error;
    2273             :         }
    2274             : 
    2275           0 :         for (i = 0; i < SPDK_COUNTOF(g_ciphers); ++i) {
    2276           0 :                 assert(g_ciphers[i]);
    2277             : 
    2278           0 :                 if (strncmp(param->cipher, g_ciphers[i], strlen(g_ciphers[i])) == 0) {
    2279           0 :                         key->cipher = i;
    2280           0 :                         found = true;
    2281           0 :                         break;
    2282             :                 }
    2283             :         }
    2284             : 
    2285           0 :         if (!found) {
    2286           0 :                 SPDK_ERRLOG("Failed to parse cipher\n");
    2287           0 :                 rc = -EINVAL;
    2288           0 :                 goto error;
    2289             :         }
    2290             : 
    2291           0 :         key->param.cipher = strdup(param->cipher);
    2292           0 :         if (!key->param.cipher) {
    2293           0 :                 rc = -ENOMEM;
    2294           0 :                 goto error;
    2295             :         }
    2296             : 
    2297           0 :         hex_key_size = strnlen(param->hex_key, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
    2298           0 :         if (hex_key_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) {
    2299           0 :                 SPDK_ERRLOG("key1 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
    2300           0 :                 rc = -EINVAL;
    2301           0 :                 goto error;
    2302             :         }
    2303             : 
    2304           0 :         if (hex_key_size == 0) {
    2305           0 :                 SPDK_ERRLOG("key1 size cannot be 0\n");
    2306           0 :                 rc = -EINVAL;
    2307           0 :                 goto error;
    2308             :         }
    2309             : 
    2310           0 :         key->param.hex_key = strdup(param->hex_key);
    2311           0 :         if (!key->param.hex_key) {
    2312           0 :                 rc = -ENOMEM;
    2313           0 :                 goto error;
    2314             :         }
    2315             : 
    2316           0 :         key->key_size = hex_key_size / 2;
    2317           0 :         key->key = spdk_unhexlify(key->param.hex_key);
    2318           0 :         if (!key->key) {
    2319           0 :                 SPDK_ERRLOG("Failed to unhexlify key1\n");
    2320           0 :                 rc = -EINVAL;
    2321           0 :                 goto error;
    2322             :         }
    2323             : 
    2324           0 :         if (param->hex_key2) {
    2325           0 :                 hex_key2_size = strnlen(param->hex_key2, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
    2326           0 :                 if (hex_key2_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) {
    2327           0 :                         SPDK_ERRLOG("key2 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
    2328           0 :                         rc = -EINVAL;
    2329           0 :                         goto error;
    2330             :                 }
    2331             : 
    2332           0 :                 if (hex_key2_size == 0) {
    2333           0 :                         SPDK_ERRLOG("key2 size cannot be 0\n");
    2334           0 :                         rc = -EINVAL;
    2335           0 :                         goto error;
    2336             :                 }
    2337             : 
    2338           0 :                 key->param.hex_key2 = strdup(param->hex_key2);
    2339           0 :                 if (!key->param.hex_key2) {
    2340           0 :                         rc = -ENOMEM;
    2341           0 :                         goto error;
    2342             :                 }
    2343             : 
    2344           0 :                 key->key2_size = hex_key2_size / 2;
    2345           0 :                 key->key2 = spdk_unhexlify(key->param.hex_key2);
    2346           0 :                 if (!key->key2) {
    2347           0 :                         SPDK_ERRLOG("Failed to unhexlify key2\n");
    2348           0 :                         rc = -EINVAL;
    2349           0 :                         goto error;
    2350             :                 }
    2351             :         }
    2352             : 
    2353           0 :         key->tweak_mode = ACCEL_CRYPTO_TWEAK_MODE_DEFAULT;
    2354           0 :         if (param->tweak_mode) {
    2355           0 :                 found = false;
    2356             : 
    2357           0 :                 key->param.tweak_mode = strdup(param->tweak_mode);
    2358           0 :                 if (!key->param.tweak_mode) {
    2359           0 :                         rc = -ENOMEM;
    2360           0 :                         goto error;
    2361             :                 }
    2362             : 
    2363           0 :                 for (i = 0; i < SPDK_COUNTOF(g_tweak_modes); ++i) {
    2364           0 :                         assert(g_tweak_modes[i]);
    2365             : 
    2366           0 :                         if (strncmp(param->tweak_mode, g_tweak_modes[i], strlen(g_tweak_modes[i])) == 0) {
    2367           0 :                                 key->tweak_mode = i;
    2368           0 :                                 found = true;
    2369           0 :                                 break;
    2370             :                         }
    2371             :                 }
    2372             : 
    2373           0 :                 if (!found) {
    2374           0 :                         SPDK_ERRLOG("Failed to parse tweak mode\n");
    2375           0 :                         rc = -EINVAL;
    2376           0 :                         goto error;
    2377             :                 }
    2378             :         }
    2379             : 
    2380           0 :         if ((!module->crypto_supports_tweak_mode && key->tweak_mode != ACCEL_CRYPTO_TWEAK_MODE_DEFAULT) ||
    2381           0 :             (module->crypto_supports_tweak_mode && !module->crypto_supports_tweak_mode(key->tweak_mode))) {
    2382           0 :                 SPDK_ERRLOG("Module %s doesn't support %s tweak mode\n", module->name,
    2383             :                             g_tweak_modes[key->tweak_mode]);
    2384           0 :                 rc = -EINVAL;
    2385           0 :                 goto error;
    2386             :         }
    2387             : 
    2388           0 :         if (!module->crypto_supports_cipher(key->cipher, key->key_size)) {
    2389           0 :                 SPDK_ERRLOG("Module %s doesn't support %s cipher with %zu key size\n", module->name,
    2390             :                             g_ciphers[key->cipher], key->key_size);
    2391           0 :                 rc = -EINVAL;
    2392           0 :                 goto error;
    2393             :         }
    2394             : 
    2395           0 :         if (key->cipher == SPDK_ACCEL_CIPHER_AES_XTS) {
    2396           0 :                 if (!key->key2) {
    2397           0 :                         SPDK_ERRLOG("%s key2 is missing\n", g_ciphers[key->cipher]);
    2398           0 :                         rc = -EINVAL;
    2399           0 :                         goto error;
    2400             :                 }
    2401             : 
    2402           0 :                 if (key->key_size != key->key2_size) {
    2403           0 :                         SPDK_ERRLOG("%s key size %zu is not equal to key2 size %zu\n", g_ciphers[key->cipher],
    2404             :                                     key->key_size,
    2405             :                                     key->key2_size);
    2406           0 :                         rc = -EINVAL;
    2407           0 :                         goto error;
    2408             :                 }
    2409             : 
    2410           0 :                 if (accel_aes_xts_keys_equal(key->key, key->key_size, key->key2, key->key2_size)) {
    2411           0 :                         SPDK_ERRLOG("%s identical keys are not secure\n", g_ciphers[key->cipher]);
    2412           0 :                         rc = -EINVAL;
    2413           0 :                         goto error;
    2414             :                 }
    2415             :         }
    2416             : 
    2417           0 :         if (key->cipher == SPDK_ACCEL_CIPHER_AES_CBC) {
    2418           0 :                 if (key->key2_size) {
    2419           0 :                         SPDK_ERRLOG("%s doesn't use key2\n", g_ciphers[key->cipher]);
    2420           0 :                         rc = -EINVAL;
    2421           0 :                         goto error;
    2422             :                 }
    2423             :         }
    2424             : 
    2425           0 :         key->module_if = module;
    2426             : 
    2427           0 :         spdk_spin_lock(&g_keyring_spin);
    2428           0 :         if (_accel_crypto_key_get(param->key_name)) {
    2429           0 :                 rc = -EEXIST;
    2430             :         } else {
    2431           0 :                 rc = module->crypto_key_init(key);
    2432           0 :                 if (rc) {
    2433           0 :                         SPDK_ERRLOG("Module %s failed to initialize crypto key\n", module->name);
    2434             :                 } else {
    2435           0 :                         TAILQ_INSERT_TAIL(&g_keyring, key, link);
    2436             :                 }
    2437             :         }
    2438           0 :         spdk_spin_unlock(&g_keyring_spin);
    2439             : 
    2440           0 :         if (rc) {
    2441           0 :                 goto error;
    2442             :         }
    2443             : 
    2444           0 :         return 0;
    2445             : 
    2446           0 : error:
    2447           0 :         accel_crypto_key_free_mem(key);
    2448           0 :         return rc;
    2449             : }
    2450             : 
    2451             : int
    2452           0 : spdk_accel_crypto_key_destroy(struct spdk_accel_crypto_key *key)
    2453             : {
    2454           0 :         if (!key || !key->module_if) {
    2455           0 :                 return -EINVAL;
    2456             :         }
    2457             : 
    2458           0 :         spdk_spin_lock(&g_keyring_spin);
    2459           0 :         if (!_accel_crypto_key_get(key->param.key_name)) {
    2460           0 :                 spdk_spin_unlock(&g_keyring_spin);
    2461           0 :                 return -ENOENT;
    2462             :         }
    2463           0 :         TAILQ_REMOVE(&g_keyring, key, link);
    2464           0 :         spdk_spin_unlock(&g_keyring_spin);
    2465             : 
    2466           0 :         accel_crypto_key_destroy_unsafe(key);
    2467             : 
    2468           0 :         return 0;
    2469             : }
    2470             : 
    2471             : struct spdk_accel_crypto_key *
    2472           0 : spdk_accel_crypto_key_get(const char *name)
    2473             : {
    2474             :         struct spdk_accel_crypto_key *key;
    2475             : 
    2476           0 :         spdk_spin_lock(&g_keyring_spin);
    2477           0 :         key = _accel_crypto_key_get(name);
    2478           0 :         spdk_spin_unlock(&g_keyring_spin);
    2479             : 
    2480           0 :         return key;
    2481             : }
    2482             : 
    2483             : /* Helper function when accel modules register with the framework. */
    2484             : void
    2485           5 : spdk_accel_module_list_add(struct spdk_accel_module_if *accel_module)
    2486             : {
    2487             :         struct spdk_accel_module_if *tmp;
    2488             : 
    2489           5 :         if (_module_find_by_name(accel_module->name)) {
    2490           0 :                 SPDK_NOTICELOG("Module %s already registered\n", accel_module->name);
    2491           0 :                 assert(false);
    2492             :                 return;
    2493             :         }
    2494             : 
    2495           8 :         TAILQ_FOREACH(tmp, &spdk_accel_module_list, tailq) {
    2496           5 :                 if (accel_module->priority < tmp->priority) {
    2497           2 :                         break;
    2498             :                 }
    2499             :         }
    2500             : 
    2501           5 :         if (tmp != NULL) {
    2502           2 :                 TAILQ_INSERT_BEFORE(tmp, accel_module, tailq);
    2503             :         } else {
    2504           3 :                 TAILQ_INSERT_TAIL(&spdk_accel_module_list, accel_module, tailq);
    2505             :         }
    2506             : }
    2507             : 
    2508             : /* Framework level channel create callback. */
    2509             : static int
    2510          11 : accel_create_channel(void *io_device, void *ctx_buf)
    2511             : {
    2512          11 :         struct accel_io_channel *accel_ch = ctx_buf;
    2513             :         struct spdk_accel_task *accel_task;
    2514             :         struct spdk_accel_task_aux_data *accel_task_aux;
    2515             :         struct spdk_accel_sequence *seq;
    2516             :         struct accel_buffer *buf;
    2517             :         size_t task_size_aligned;
    2518             :         uint8_t *task_mem;
    2519          11 :         uint32_t i = 0, j;
    2520             :         int rc;
    2521             : 
    2522          11 :         task_size_aligned = SPDK_ALIGN_CEIL(g_max_accel_module_size, SPDK_CACHE_LINE_SIZE);
    2523          11 :         accel_ch->task_pool_base = aligned_alloc(SPDK_CACHE_LINE_SIZE,
    2524          11 :                                    g_opts.task_count * task_size_aligned);
    2525          11 :         if (!accel_ch->task_pool_base) {
    2526           0 :                 return -ENOMEM;
    2527             :         }
    2528          11 :         memset(accel_ch->task_pool_base, 0, g_opts.task_count * task_size_aligned);
    2529             : 
    2530          11 :         accel_ch->seq_pool_base = aligned_alloc(SPDK_CACHE_LINE_SIZE,
    2531          11 :                                                 g_opts.sequence_count * sizeof(struct spdk_accel_sequence));
    2532          11 :         if (accel_ch->seq_pool_base == NULL) {
    2533           0 :                 goto err;
    2534             :         }
    2535          11 :         memset(accel_ch->seq_pool_base, 0, g_opts.sequence_count * sizeof(struct spdk_accel_sequence));
    2536             : 
    2537          11 :         accel_ch->task_aux_data_base = calloc(g_opts.task_count, sizeof(struct spdk_accel_task_aux_data));
    2538          11 :         if (accel_ch->task_aux_data_base == NULL) {
    2539           0 :                 goto err;
    2540             :         }
    2541             : 
    2542          11 :         accel_ch->buf_pool_base = calloc(g_opts.buf_count, sizeof(struct accel_buffer));
    2543          11 :         if (accel_ch->buf_pool_base == NULL) {
    2544           0 :                 goto err;
    2545             :         }
    2546             : 
    2547          11 :         STAILQ_INIT(&accel_ch->task_pool);
    2548          11 :         SLIST_INIT(&accel_ch->task_aux_data_pool);
    2549          11 :         SLIST_INIT(&accel_ch->seq_pool);
    2550          11 :         SLIST_INIT(&accel_ch->buf_pool);
    2551             : 
    2552          11 :         task_mem = accel_ch->task_pool_base;
    2553       22539 :         for (i = 0; i < g_opts.task_count; i++) {
    2554       22528 :                 accel_task = (struct spdk_accel_task *)task_mem;
    2555       22528 :                 accel_task->aux = NULL;
    2556       22528 :                 STAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link);
    2557       22528 :                 task_mem += task_size_aligned;
    2558       22528 :                 accel_task_aux = &accel_ch->task_aux_data_base[i];
    2559       22528 :                 SLIST_INSERT_HEAD(&accel_ch->task_aux_data_pool, accel_task_aux, link);
    2560             :         }
    2561       22539 :         for (i = 0; i < g_opts.sequence_count; i++) {
    2562       22528 :                 seq = &accel_ch->seq_pool_base[i];
    2563       22528 :                 SLIST_INSERT_HEAD(&accel_ch->seq_pool, seq, link);
    2564             :         }
    2565       22539 :         for (i = 0; i < g_opts.buf_count; i++) {
    2566       22528 :                 buf = &accel_ch->buf_pool_base[i];
    2567       22528 :                 SLIST_INSERT_HEAD(&accel_ch->buf_pool, buf, link);
    2568             :         }
    2569             : 
    2570             :         /* Assign modules and get IO channels for each */
    2571         165 :         for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
    2572         154 :                 accel_ch->module_ch[i] = g_modules_opc[i].module->get_io_channel();
    2573             :                 /* This can happen if idxd runs out of channels. */
    2574         154 :                 if (accel_ch->module_ch[i] == NULL) {
    2575           0 :                         SPDK_ERRLOG("Module %s failed to get io channel\n", g_modules_opc[i].module->name);
    2576           0 :                         goto err;
    2577             :                 }
    2578             :         }
    2579             : 
    2580          11 :         if (g_accel_driver != NULL) {
    2581           0 :                 accel_ch->driver_channel = g_accel_driver->get_io_channel();
    2582           0 :                 if (accel_ch->driver_channel == NULL) {
    2583           0 :                         SPDK_ERRLOG("Failed to get driver's IO channel\n");
    2584           0 :                         goto err;
    2585             :                 }
    2586             :         }
    2587             : 
    2588          11 :         rc = spdk_iobuf_channel_init(&accel_ch->iobuf, "accel", g_opts.small_cache_size,
    2589             :                                      g_opts.large_cache_size);
    2590          11 :         if (rc != 0) {
    2591           0 :                 SPDK_ERRLOG("Failed to initialize iobuf accel channel\n");
    2592           0 :                 goto err;
    2593             :         }
    2594             : 
    2595          11 :         return 0;
    2596           0 : err:
    2597           0 :         if (accel_ch->driver_channel != NULL) {
    2598           0 :                 spdk_put_io_channel(accel_ch->driver_channel);
    2599             :         }
    2600           0 :         for (j = 0; j < i; j++) {
    2601           0 :                 spdk_put_io_channel(accel_ch->module_ch[j]);
    2602             :         }
    2603           0 :         free(accel_ch->task_pool_base);
    2604           0 :         free(accel_ch->task_aux_data_base);
    2605           0 :         free(accel_ch->seq_pool_base);
    2606           0 :         free(accel_ch->buf_pool_base);
    2607             : 
    2608           0 :         return -ENOMEM;
    2609             : }
    2610             : 
    2611             : static void
    2612          11 : accel_add_stats(struct accel_stats *total, struct accel_stats *stats)
    2613             : {
    2614             :         int i;
    2615             : 
    2616          11 :         total->sequence_executed += stats->sequence_executed;
    2617          11 :         total->sequence_failed += stats->sequence_failed;
    2618          11 :         total->retry.task += stats->retry.task;
    2619          11 :         total->retry.sequence += stats->retry.sequence;
    2620          11 :         total->retry.iobuf += stats->retry.iobuf;
    2621          11 :         total->retry.bufdesc += stats->retry.bufdesc;
    2622         165 :         for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) {
    2623         154 :                 total->operations[i].executed += stats->operations[i].executed;
    2624         154 :                 total->operations[i].failed += stats->operations[i].failed;
    2625         154 :                 total->operations[i].num_bytes += stats->operations[i].num_bytes;
    2626             :         }
    2627          11 : }
    2628             : 
    2629             : /* Framework level channel destroy callback. */
    2630             : static void
    2631          11 : accel_destroy_channel(void *io_device, void *ctx_buf)
    2632             : {
    2633          11 :         struct accel_io_channel *accel_ch = ctx_buf;
    2634             :         int i;
    2635             : 
    2636          11 :         spdk_iobuf_channel_fini(&accel_ch->iobuf);
    2637             : 
    2638          11 :         if (accel_ch->driver_channel != NULL) {
    2639           0 :                 spdk_put_io_channel(accel_ch->driver_channel);
    2640             :         }
    2641             : 
    2642         165 :         for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
    2643         154 :                 assert(accel_ch->module_ch[i] != NULL);
    2644         154 :                 spdk_put_io_channel(accel_ch->module_ch[i]);
    2645         154 :                 accel_ch->module_ch[i] = NULL;
    2646             :         }
    2647             : 
    2648             :         /* Update global stats to make sure channel's stats aren't lost after a channel is gone */
    2649          11 :         spdk_spin_lock(&g_stats_lock);
    2650          11 :         accel_add_stats(&g_stats, &accel_ch->stats);
    2651          11 :         spdk_spin_unlock(&g_stats_lock);
    2652             : 
    2653          11 :         free(accel_ch->task_pool_base);
    2654          11 :         free(accel_ch->task_aux_data_base);
    2655          11 :         free(accel_ch->seq_pool_base);
    2656          11 :         free(accel_ch->buf_pool_base);
    2657          11 : }
    2658             : 
    2659             : struct spdk_io_channel *
    2660          11 : spdk_accel_get_io_channel(void)
    2661             : {
    2662          11 :         return spdk_get_io_channel(&spdk_accel_module_list);
    2663             : }
    2664             : 
    2665             : static int
    2666           2 : accel_module_initialize(void)
    2667             : {
    2668             :         struct spdk_accel_module_if *accel_module, *tmp_module;
    2669           2 :         int rc = 0, module_rc;
    2670             : 
    2671           7 :         TAILQ_FOREACH_SAFE(accel_module, &spdk_accel_module_list, tailq, tmp_module) {
    2672           5 :                 module_rc = accel_module->module_init();
    2673           5 :                 if (module_rc) {
    2674           0 :                         SPDK_ERRLOG("Module %s initialization failed with %d\n", accel_module->name, module_rc);
    2675           0 :                         TAILQ_REMOVE(&spdk_accel_module_list, accel_module, tailq);
    2676           0 :                         if (!rc) {
    2677           0 :                                 rc = module_rc;
    2678             :                         }
    2679             :                 }
    2680             : 
    2681           5 :                 SPDK_DEBUGLOG(accel, "Module %s initialized.\n", accel_module->name);
    2682             :         }
    2683             : 
    2684           2 :         return rc;
    2685             : }
    2686             : 
    2687             : static void
    2688          28 : accel_module_init_opcode(enum spdk_accel_opcode opcode)
    2689             : {
    2690          28 :         struct accel_module *module = &g_modules_opc[opcode];
    2691          28 :         struct spdk_accel_module_if *module_if = module->module;
    2692             : 
    2693          28 :         if (module_if->get_memory_domains != NULL) {
    2694           0 :                 module->supports_memory_domains = module_if->get_memory_domains(NULL, 0) > 0;
    2695             :         }
    2696          28 : }
    2697             : 
    2698             : int
    2699           2 : spdk_accel_initialize(void)
    2700             : {
    2701             :         enum spdk_accel_opcode op;
    2702           2 :         struct spdk_accel_module_if *accel_module = NULL;
    2703             :         int rc;
    2704             : 
    2705             :         /*
    2706             :          * We need a unique identifier for the accel framework, so use the
    2707             :          * spdk_accel_module_list address for this purpose.
    2708             :          */
    2709           2 :         spdk_io_device_register(&spdk_accel_module_list, accel_create_channel, accel_destroy_channel,
    2710             :                                 sizeof(struct accel_io_channel), "accel");
    2711             : 
    2712           2 :         spdk_spin_init(&g_keyring_spin);
    2713           2 :         spdk_spin_init(&g_stats_lock);
    2714             : 
    2715           2 :         rc = spdk_memory_domain_create(&g_accel_domain, SPDK_DMA_DEVICE_TYPE_ACCEL, NULL,
    2716             :                                        "SPDK_ACCEL_DMA_DEVICE");
    2717           2 :         if (rc != 0) {
    2718           0 :                 SPDK_ERRLOG("Failed to create accel memory domain\n");
    2719           0 :                 return rc;
    2720             :         }
    2721             : 
    2722           2 :         g_modules_started = true;
    2723           2 :         rc = accel_module_initialize();
    2724           2 :         if (rc) {
    2725           0 :                 return rc;
    2726             :         }
    2727             : 
    2728           2 :         if (g_accel_driver != NULL && g_accel_driver->init != NULL) {
    2729           0 :                 rc = g_accel_driver->init();
    2730           0 :                 if (rc != 0) {
    2731           0 :                         SPDK_ERRLOG("Failed to initialize driver %s: %s\n", g_accel_driver->name,
    2732             :                                     spdk_strerror(-rc));
    2733           0 :                         return rc;
    2734             :                 }
    2735             :         }
    2736             : 
    2737             :         /* The module list is order by priority, with the highest priority modules being at the end
    2738             :          * of the list.  The software module should be somewhere at the beginning of the list,
    2739             :          * before all HW modules.
    2740             :          * NOTE: all opcodes must be supported by software in the event that no HW modules are
    2741             :          * initialized to support the operation.
    2742             :          */
    2743           7 :         TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
    2744          75 :                 for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
    2745          70 :                         if (accel_module->supports_opcode(op)) {
    2746          70 :                                 g_modules_opc[op].module = accel_module;
    2747          70 :                                 SPDK_DEBUGLOG(accel, "OPC 0x%x now assigned to %s\n", op, accel_module->name);
    2748             :                         }
    2749             :                 }
    2750             : 
    2751           5 :                 if (accel_module->get_ctx_size != NULL) {
    2752           1 :                         g_max_accel_module_size = spdk_max(g_max_accel_module_size,
    2753             :                                                            accel_module->get_ctx_size());
    2754             :                 }
    2755             :         }
    2756             : 
    2757             :         /* Now lets check for overrides and apply all that exist */
    2758          30 :         for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
    2759          28 :                 if (g_modules_opc_override[op] != NULL) {
    2760           0 :                         accel_module = _module_find_by_name(g_modules_opc_override[op]);
    2761           0 :                         if (accel_module == NULL) {
    2762           0 :                                 SPDK_ERRLOG("Invalid module name of %s\n", g_modules_opc_override[op]);
    2763           0 :                                 return -EINVAL;
    2764             :                         }
    2765           0 :                         if (accel_module->supports_opcode(op) == false) {
    2766           0 :                                 SPDK_ERRLOG("Module %s does not support op code %d\n", accel_module->name, op);
    2767           0 :                                 return -EINVAL;
    2768             :                         }
    2769           0 :                         g_modules_opc[op].module = accel_module;
    2770             :                 }
    2771             :         }
    2772             : 
    2773           2 :         if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) {
    2774           0 :                 SPDK_ERRLOG("Different accel modules are assigned to encrypt and decrypt operations");
    2775           0 :                 return -EINVAL;
    2776             :         }
    2777             : 
    2778          30 :         for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
    2779          28 :                 assert(g_modules_opc[op].module != NULL);
    2780          28 :                 accel_module_init_opcode(op);
    2781             :         }
    2782             : 
    2783           2 :         rc = spdk_iobuf_register_module("accel");
    2784           2 :         if (rc != 0) {
    2785           0 :                 SPDK_ERRLOG("Failed to register accel iobuf module\n");
    2786           0 :                 return rc;
    2787             :         }
    2788             : 
    2789           2 :         return 0;
    2790             : }
    2791             : 
    2792             : static void
    2793           2 : accel_module_finish_cb(void)
    2794             : {
    2795           2 :         spdk_accel_fini_cb cb_fn = g_fini_cb_fn;
    2796             : 
    2797           2 :         cb_fn(g_fini_cb_arg);
    2798           2 :         g_fini_cb_fn = NULL;
    2799           2 :         g_fini_cb_arg = NULL;
    2800           2 : }
    2801             : 
    2802             : static void
    2803           0 : accel_write_overridden_opc(struct spdk_json_write_ctx *w, const char *opc_str,
    2804             :                            const char *module_str)
    2805             : {
    2806           0 :         spdk_json_write_object_begin(w);
    2807           0 :         spdk_json_write_named_string(w, "method", "accel_assign_opc");
    2808           0 :         spdk_json_write_named_object_begin(w, "params");
    2809           0 :         spdk_json_write_named_string(w, "opname", opc_str);
    2810           0 :         spdk_json_write_named_string(w, "module", module_str);
    2811           0 :         spdk_json_write_object_end(w);
    2812           0 :         spdk_json_write_object_end(w);
    2813           0 : }
    2814             : 
    2815             : static void
    2816           0 : __accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key)
    2817             : {
    2818           0 :         spdk_json_write_named_string(w, "name", key->param.key_name);
    2819           0 :         spdk_json_write_named_string(w, "cipher", key->param.cipher);
    2820           0 :         spdk_json_write_named_string(w, "key", key->param.hex_key);
    2821           0 :         if (key->param.hex_key2) {
    2822           0 :                 spdk_json_write_named_string(w, "key2", key->param.hex_key2);
    2823             :         }
    2824             : 
    2825           0 :         if (key->param.tweak_mode) {
    2826           0 :                 spdk_json_write_named_string(w, "tweak_mode", key->param.tweak_mode);
    2827             :         }
    2828           0 : }
    2829             : 
    2830             : void
    2831           0 : _accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key)
    2832             : {
    2833           0 :         spdk_json_write_object_begin(w);
    2834           0 :         __accel_crypto_key_dump_param(w, key);
    2835           0 :         spdk_json_write_object_end(w);
    2836           0 : }
    2837             : 
    2838             : static void
    2839           0 : _accel_crypto_key_write_config_json(struct spdk_json_write_ctx *w,
    2840             :                                     struct spdk_accel_crypto_key *key)
    2841             : {
    2842           0 :         spdk_json_write_object_begin(w);
    2843           0 :         spdk_json_write_named_string(w, "method", "accel_crypto_key_create");
    2844           0 :         spdk_json_write_named_object_begin(w, "params");
    2845           0 :         __accel_crypto_key_dump_param(w, key);
    2846           0 :         spdk_json_write_object_end(w);
    2847           0 :         spdk_json_write_object_end(w);
    2848           0 : }
    2849             : 
    2850             : static void
    2851           0 : accel_write_options(struct spdk_json_write_ctx *w)
    2852             : {
    2853           0 :         spdk_json_write_object_begin(w);
    2854           0 :         spdk_json_write_named_string(w, "method", "accel_set_options");
    2855           0 :         spdk_json_write_named_object_begin(w, "params");
    2856           0 :         spdk_json_write_named_uint32(w, "small_cache_size", g_opts.small_cache_size);
    2857           0 :         spdk_json_write_named_uint32(w, "large_cache_size", g_opts.large_cache_size);
    2858           0 :         spdk_json_write_named_uint32(w, "task_count", g_opts.task_count);
    2859           0 :         spdk_json_write_named_uint32(w, "sequence_count", g_opts.sequence_count);
    2860           0 :         spdk_json_write_named_uint32(w, "buf_count", g_opts.buf_count);
    2861           0 :         spdk_json_write_object_end(w);
    2862           0 :         spdk_json_write_object_end(w);
    2863           0 : }
    2864             : 
    2865             : static void
    2866           0 : _accel_crypto_keys_write_config_json(struct spdk_json_write_ctx *w, bool full_dump)
    2867             : {
    2868             :         struct spdk_accel_crypto_key *key;
    2869             : 
    2870           0 :         spdk_spin_lock(&g_keyring_spin);
    2871           0 :         TAILQ_FOREACH(key, &g_keyring, link) {
    2872           0 :                 if (full_dump) {
    2873           0 :                         _accel_crypto_key_write_config_json(w, key);
    2874             :                 } else {
    2875           0 :                         _accel_crypto_key_dump_param(w, key);
    2876             :                 }
    2877             :         }
    2878           0 :         spdk_spin_unlock(&g_keyring_spin);
    2879           0 : }
    2880             : 
    2881             : void
    2882           0 : _accel_crypto_keys_dump_param(struct spdk_json_write_ctx *w)
    2883             : {
    2884           0 :         _accel_crypto_keys_write_config_json(w, false);
    2885           0 : }
    2886             : 
    2887             : void
    2888           0 : spdk_accel_write_config_json(struct spdk_json_write_ctx *w)
    2889             : {
    2890             :         struct spdk_accel_module_if *accel_module;
    2891             :         int i;
    2892             : 
    2893           0 :         spdk_json_write_array_begin(w);
    2894           0 :         accel_write_options(w);
    2895             : 
    2896           0 :         TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
    2897           0 :                 if (accel_module->write_config_json) {
    2898           0 :                         accel_module->write_config_json(w);
    2899             :                 }
    2900             :         }
    2901           0 :         for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
    2902           0 :                 if (g_modules_opc_override[i]) {
    2903           0 :                         accel_write_overridden_opc(w, g_opcode_strings[i], g_modules_opc_override[i]);
    2904             :                 }
    2905             :         }
    2906             : 
    2907           0 :         _accel_crypto_keys_write_config_json(w, true);
    2908             : 
    2909           0 :         spdk_json_write_array_end(w);
    2910           0 : }
    2911             : 
    2912             : void
    2913           7 : spdk_accel_module_finish(void)
    2914             : {
    2915           7 :         if (!g_accel_module) {
    2916           2 :                 g_accel_module = TAILQ_FIRST(&spdk_accel_module_list);
    2917             :         } else {
    2918           5 :                 g_accel_module = TAILQ_NEXT(g_accel_module, tailq);
    2919             :         }
    2920             : 
    2921           7 :         if (!g_accel_module) {
    2922           2 :                 if (g_accel_driver != NULL && g_accel_driver->fini != NULL) {
    2923           0 :                         g_accel_driver->fini();
    2924             :                 }
    2925             : 
    2926           2 :                 spdk_spin_destroy(&g_keyring_spin);
    2927           2 :                 spdk_spin_destroy(&g_stats_lock);
    2928           2 :                 if (g_accel_domain) {
    2929           2 :                         spdk_memory_domain_destroy(g_accel_domain);
    2930           2 :                         g_accel_domain = NULL;
    2931             :                 }
    2932           2 :                 accel_module_finish_cb();
    2933           2 :                 return;
    2934             :         }
    2935             : 
    2936           5 :         if (g_accel_module->module_fini) {
    2937           1 :                 spdk_thread_send_msg(spdk_get_thread(), g_accel_module->module_fini, NULL);
    2938             :         } else {
    2939           4 :                 spdk_accel_module_finish();
    2940             :         }
    2941             : }
    2942             : 
    2943             : static void
    2944           2 : accel_io_device_unregister_cb(void *io_device)
    2945             : {
    2946             :         struct spdk_accel_crypto_key *key, *key_tmp;
    2947             :         enum spdk_accel_opcode op;
    2948             : 
    2949           2 :         spdk_spin_lock(&g_keyring_spin);
    2950           2 :         TAILQ_FOREACH_SAFE(key, &g_keyring, link, key_tmp) {
    2951           0 :                 accel_crypto_key_destroy_unsafe(key);
    2952             :         }
    2953           2 :         spdk_spin_unlock(&g_keyring_spin);
    2954             : 
    2955          30 :         for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
    2956          28 :                 if (g_modules_opc_override[op] != NULL) {
    2957           0 :                         free(g_modules_opc_override[op]);
    2958           0 :                         g_modules_opc_override[op] = NULL;
    2959             :                 }
    2960          28 :                 g_modules_opc[op].module = NULL;
    2961             :         }
    2962             : 
    2963           2 :         spdk_accel_module_finish();
    2964           2 : }
    2965             : 
    2966             : void
    2967           2 : spdk_accel_finish(spdk_accel_fini_cb cb_fn, void *cb_arg)
    2968             : {
    2969           2 :         assert(cb_fn != NULL);
    2970             : 
    2971           2 :         g_fini_cb_fn = cb_fn;
    2972           2 :         g_fini_cb_arg = cb_arg;
    2973             : 
    2974           2 :         spdk_io_device_unregister(&spdk_accel_module_list, accel_io_device_unregister_cb);
    2975           2 : }
    2976             : 
    2977             : static struct spdk_accel_driver *
    2978           2 : accel_find_driver(const char *name)
    2979             : {
    2980             :         struct spdk_accel_driver *driver;
    2981             : 
    2982           2 :         TAILQ_FOREACH(driver, &g_accel_drivers, tailq) {
    2983           1 :                 if (strcmp(driver->name, name) == 0) {
    2984           1 :                         return driver;
    2985             :                 }
    2986             :         }
    2987             : 
    2988           1 :         return NULL;
    2989             : }
    2990             : 
    2991             : int
    2992           1 : spdk_accel_set_driver(const char *name)
    2993             : {
    2994             :         struct spdk_accel_driver *driver;
    2995             : 
    2996           1 :         driver = accel_find_driver(name);
    2997           1 :         if (driver == NULL) {
    2998           0 :                 SPDK_ERRLOG("Couldn't find driver named '%s'\n", name);
    2999           0 :                 return -ENODEV;
    3000             :         }
    3001             : 
    3002           1 :         g_accel_driver = driver;
    3003             : 
    3004           1 :         return 0;
    3005             : }
    3006             : 
    3007             : void
    3008           1 : spdk_accel_driver_register(struct spdk_accel_driver *driver)
    3009             : {
    3010           1 :         if (accel_find_driver(driver->name)) {
    3011           0 :                 SPDK_ERRLOG("Driver named '%s' has already been registered\n", driver->name);
    3012           0 :                 assert(0);
    3013             :                 return;
    3014             :         }
    3015             : 
    3016           1 :         TAILQ_INSERT_TAIL(&g_accel_drivers, driver, tailq);
    3017             : }
    3018             : 
    3019             : int
    3020           0 : spdk_accel_set_opts(const struct spdk_accel_opts *opts)
    3021             : {
    3022           0 :         if (opts->size > sizeof(*opts)) {
    3023           0 :                 return -EINVAL;
    3024             :         }
    3025             : 
    3026           0 :         memcpy(&g_opts, opts, opts->size);
    3027             : 
    3028           0 :         return 0;
    3029             : }
    3030             : 
    3031             : void
    3032           0 : spdk_accel_get_opts(struct spdk_accel_opts *opts)
    3033             : {
    3034           0 :         size_t size = opts->size;
    3035             : 
    3036           0 :         assert(size <= sizeof(*opts));
    3037             : 
    3038           0 :         memcpy(opts, &g_opts, spdk_min(sizeof(*opts), size));
    3039           0 :         opts->size = size;
    3040           0 : }
    3041             : 
    3042             : struct accel_get_stats_ctx {
    3043             :         struct accel_stats      stats;
    3044             :         accel_get_stats_cb      cb_fn;
    3045             :         void                    *cb_arg;
    3046             : };
    3047             : 
    3048             : static void
    3049           0 : accel_get_channel_stats_done(struct spdk_io_channel_iter *iter, int status)
    3050             : {
    3051           0 :         struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter);
    3052             : 
    3053           0 :         ctx->cb_fn(&ctx->stats, ctx->cb_arg);
    3054           0 :         free(ctx);
    3055           0 : }
    3056             : 
    3057             : static void
    3058           0 : accel_get_channel_stats(struct spdk_io_channel_iter *iter)
    3059             : {
    3060           0 :         struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(iter);
    3061           0 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
    3062           0 :         struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter);
    3063             : 
    3064           0 :         accel_add_stats(&ctx->stats, &accel_ch->stats);
    3065           0 :         spdk_for_each_channel_continue(iter, 0);
    3066           0 : }
    3067             : 
    3068             : int
    3069           0 : accel_get_stats(accel_get_stats_cb cb_fn, void *cb_arg)
    3070             : {
    3071             :         struct accel_get_stats_ctx *ctx;
    3072             : 
    3073           0 :         ctx = calloc(1, sizeof(*ctx));
    3074           0 :         if (ctx == NULL) {
    3075           0 :                 return -ENOMEM;
    3076             :         }
    3077             : 
    3078           0 :         spdk_spin_lock(&g_stats_lock);
    3079           0 :         accel_add_stats(&ctx->stats, &g_stats);
    3080           0 :         spdk_spin_unlock(&g_stats_lock);
    3081             : 
    3082           0 :         ctx->cb_fn = cb_fn;
    3083           0 :         ctx->cb_arg = cb_arg;
    3084             : 
    3085           0 :         spdk_for_each_channel(&spdk_accel_module_list, accel_get_channel_stats, ctx,
    3086             :                               accel_get_channel_stats_done);
    3087             : 
    3088           0 :         return 0;
    3089             : }
    3090             : 
    3091             : void
    3092           0 : spdk_accel_get_opcode_stats(struct spdk_io_channel *ch, enum spdk_accel_opcode opcode,
    3093             :                             struct spdk_accel_opcode_stats *stats, size_t size)
    3094             : {
    3095           0 :         struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
    3096             : 
    3097             : #define FIELD_OK(field) \
    3098             :         offsetof(struct spdk_accel_opcode_stats, field) + sizeof(stats->field) <= size
    3099             : 
    3100             : #define SET_FIELD(field, value) \
    3101             :         if (FIELD_OK(field)) { \
    3102             :                 stats->field = value; \
    3103             :         }
    3104             : 
    3105           0 :         SET_FIELD(executed, accel_ch->stats.operations[opcode].executed);
    3106           0 :         SET_FIELD(failed, accel_ch->stats.operations[opcode].failed);
    3107           0 :         SET_FIELD(num_bytes, accel_ch->stats.operations[opcode].num_bytes);
    3108             : 
    3109             : #undef FIELD_OK
    3110             : #undef SET_FIELD
    3111           0 : }
    3112             : 
    3113             : uint8_t
    3114           0 : spdk_accel_get_buf_align(enum spdk_accel_opcode opcode,
    3115             :                          const struct spdk_accel_operation_exec_ctx *ctx)
    3116             : {
    3117           0 :         struct spdk_accel_module_if *module = g_modules_opc[opcode].module;
    3118           0 :         struct spdk_accel_opcode_info modinfo = {}, drvinfo = {};
    3119             : 
    3120           0 :         if (g_accel_driver != NULL && g_accel_driver->get_operation_info != NULL) {
    3121           0 :                 g_accel_driver->get_operation_info(opcode, ctx, &drvinfo);
    3122             :         }
    3123             : 
    3124           0 :         if (module->get_operation_info != NULL) {
    3125           0 :                 module->get_operation_info(opcode, ctx, &modinfo);
    3126             :         }
    3127             : 
    3128             :         /* If a driver is set, it'll execute most of the operations, while the rest will usually
    3129             :          * fall back to accel_sw, which doesn't have any alignment requiremenets.  However, to be
    3130             :          * extra safe, return the max(driver, module) if a driver delegates some operations to a
    3131             :          * hardware module. */
    3132           0 :         return spdk_max(modinfo.required_alignment, drvinfo.required_alignment);
    3133             : }
    3134             : 
    3135             : struct spdk_accel_module_if *
    3136           0 : spdk_accel_get_module(const char *name)
    3137             : {
    3138             :         struct spdk_accel_module_if *module;
    3139             : 
    3140           0 :         TAILQ_FOREACH(module, &spdk_accel_module_list, tailq) {
    3141           0 :                 if (strcmp(module->name, name) == 0) {
    3142           0 :                         return module;
    3143             :                 }
    3144             :         }
    3145             : 
    3146           0 :         return NULL;
    3147             : }
    3148             : 
    3149           1 : SPDK_LOG_REGISTER_COMPONENT(accel)

Generated by: LCOV version 1.15