Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2020 Intel Corporation.
3 : * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES.
4 : * All rights reserved.
5 : */
6 :
7 : #include "spdk/stdinc.h"
8 :
9 : #include "spdk/accel_module.h"
10 :
11 : #include "accel_internal.h"
12 :
13 : #include "spdk/dma.h"
14 : #include "spdk/env.h"
15 : #include "spdk/likely.h"
16 : #include "spdk/log.h"
17 : #include "spdk/thread.h"
18 : #include "spdk/json.h"
19 : #include "spdk/crc32.h"
20 : #include "spdk/util.h"
21 : #include "spdk/hexlify.h"
22 : #include "spdk/string.h"
23 :
24 : /* Accelerator Framework: The following provides a top level
25 : * generic API for the accelerator functions defined here. Modules,
26 : * such as the one in /module/accel/ioat, supply the implementation
27 : * with the exception of the pure software implementation contained
28 : * later in this file.
29 : */
30 :
31 : #define ALIGN_4K 0x1000
32 : #define ACCEL_TASKS_PER_CHANNEL 2048
33 : #define ACCEL_SMALL_CACHE_SIZE 128
34 : #define ACCEL_LARGE_CACHE_SIZE 16
35 : /* Set MSB, so we don't return NULL pointers as buffers */
36 : #define ACCEL_BUFFER_BASE ((void *)(1ull << 63))
37 : #define ACCEL_BUFFER_OFFSET_MASK ((uintptr_t)ACCEL_BUFFER_BASE - 1)
38 :
39 : #define ACCEL_CRYPTO_TWEAK_MODE_DEFAULT SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA
40 : #define ACCEL_TASKS_IN_SEQUENCE_LIMIT 8
41 :
42 : struct accel_module {
43 : struct spdk_accel_module_if *module;
44 : bool supports_memory_domains;
45 : };
46 :
47 : /* Largest context size for all accel modules */
48 : static size_t g_max_accel_module_size = sizeof(struct spdk_accel_task);
49 :
50 : static struct spdk_accel_module_if *g_accel_module = NULL;
51 : static spdk_accel_fini_cb g_fini_cb_fn = NULL;
52 : static void *g_fini_cb_arg = NULL;
53 : static bool g_modules_started = false;
54 : static struct spdk_memory_domain *g_accel_domain;
55 :
56 : /* Global list of registered accelerator modules */
57 : static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list =
58 : TAILQ_HEAD_INITIALIZER(spdk_accel_module_list);
59 :
60 : /* Crypto keyring */
61 : static TAILQ_HEAD(, spdk_accel_crypto_key) g_keyring = TAILQ_HEAD_INITIALIZER(g_keyring);
62 : static struct spdk_spinlock g_keyring_spin;
63 :
64 : /* Global array mapping capabilities to modules */
65 : static struct accel_module g_modules_opc[SPDK_ACCEL_OPC_LAST] = {};
66 : static char *g_modules_opc_override[SPDK_ACCEL_OPC_LAST] = {};
67 : TAILQ_HEAD(, spdk_accel_driver) g_accel_drivers = TAILQ_HEAD_INITIALIZER(g_accel_drivers);
68 : static struct spdk_accel_driver *g_accel_driver;
69 : static struct spdk_accel_opts g_opts = {
70 : .small_cache_size = ACCEL_SMALL_CACHE_SIZE,
71 : .large_cache_size = ACCEL_LARGE_CACHE_SIZE,
72 : .task_count = ACCEL_TASKS_PER_CHANNEL,
73 : .sequence_count = ACCEL_TASKS_PER_CHANNEL,
74 : .buf_count = ACCEL_TASKS_PER_CHANNEL,
75 : };
76 : static struct accel_stats g_stats;
77 : static struct spdk_spinlock g_stats_lock;
78 :
79 : static const char *g_opcode_strings[SPDK_ACCEL_OPC_LAST] = {
80 : "copy", "fill", "dualcast", "compare", "crc32c", "copy_crc32c",
81 : "compress", "decompress", "encrypt", "decrypt", "xor",
82 : "dif_verify", "dif_verify_copy", "dif_generate", "dif_generate_copy",
83 : "dix_generate", "dix_verify"
84 : };
85 :
86 : enum accel_sequence_state {
87 : ACCEL_SEQUENCE_STATE_INIT,
88 : ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF,
89 : ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF,
90 : ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF,
91 : ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF,
92 : ACCEL_SEQUENCE_STATE_PULL_DATA,
93 : ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA,
94 : ACCEL_SEQUENCE_STATE_EXEC_TASK,
95 : ACCEL_SEQUENCE_STATE_AWAIT_TASK,
96 : ACCEL_SEQUENCE_STATE_COMPLETE_TASK,
97 : ACCEL_SEQUENCE_STATE_NEXT_TASK,
98 : ACCEL_SEQUENCE_STATE_PUSH_DATA,
99 : ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA,
100 : ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS,
101 : ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS,
102 : ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS,
103 : ACCEL_SEQUENCE_STATE_ERROR,
104 : ACCEL_SEQUENCE_STATE_MAX,
105 : };
106 :
107 : static const char *g_seq_states[]
108 : __attribute__((unused)) = {
109 : [ACCEL_SEQUENCE_STATE_INIT] = "init",
110 : [ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF] = "check-virtbuf",
111 : [ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF] = "await-virtbuf",
112 : [ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF] = "check-bouncebuf",
113 : [ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF] = "await-bouncebuf",
114 : [ACCEL_SEQUENCE_STATE_PULL_DATA] = "pull-data",
115 : [ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA] = "await-pull-data",
116 : [ACCEL_SEQUENCE_STATE_EXEC_TASK] = "exec-task",
117 : [ACCEL_SEQUENCE_STATE_AWAIT_TASK] = "await-task",
118 : [ACCEL_SEQUENCE_STATE_COMPLETE_TASK] = "complete-task",
119 : [ACCEL_SEQUENCE_STATE_NEXT_TASK] = "next-task",
120 : [ACCEL_SEQUENCE_STATE_PUSH_DATA] = "push-data",
121 : [ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA] = "await-push-data",
122 : [ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS] = "driver-exec-tasks",
123 : [ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS] = "driver-await-tasks",
124 : [ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS] = "driver-complete-tasks",
125 : [ACCEL_SEQUENCE_STATE_ERROR] = "error",
126 : [ACCEL_SEQUENCE_STATE_MAX] = "",
127 : };
128 :
129 : #define ACCEL_SEQUENCE_STATE_STRING(s) \
130 : (((s) >= ACCEL_SEQUENCE_STATE_INIT && (s) < ACCEL_SEQUENCE_STATE_MAX) \
131 : ? g_seq_states[s] : "unknown")
132 :
133 : struct accel_buffer {
134 : struct spdk_accel_sequence *seq;
135 : void *buf;
136 : uint64_t len;
137 : struct spdk_iobuf_entry iobuf;
138 : spdk_accel_sequence_get_buf_cb cb_fn;
139 : void *cb_ctx;
140 : SLIST_ENTRY(accel_buffer) link;
141 : struct accel_io_channel *ch;
142 : };
143 :
144 : struct accel_io_channel {
145 : struct spdk_io_channel *module_ch[SPDK_ACCEL_OPC_LAST];
146 : struct spdk_io_channel *driver_channel;
147 : void *task_pool_base;
148 : struct spdk_accel_sequence *seq_pool_base;
149 : struct accel_buffer *buf_pool_base;
150 : struct spdk_accel_task_aux_data *task_aux_data_base;
151 : STAILQ_HEAD(, spdk_accel_task) task_pool;
152 : SLIST_HEAD(, spdk_accel_task_aux_data) task_aux_data_pool;
153 : SLIST_HEAD(, spdk_accel_sequence) seq_pool;
154 : SLIST_HEAD(, accel_buffer) buf_pool;
155 : struct spdk_iobuf_channel iobuf;
156 : struct accel_stats stats;
157 : };
158 :
159 : TAILQ_HEAD(accel_sequence_tasks, spdk_accel_task);
160 :
161 : struct spdk_accel_sequence {
162 : struct accel_io_channel *ch;
163 : struct accel_sequence_tasks tasks;
164 : SLIST_HEAD(, accel_buffer) bounce_bufs;
165 : int status;
166 : /* state uses enum accel_sequence_state */
167 : uint8_t state;
168 : bool in_process_sequence;
169 : spdk_accel_completion_cb cb_fn;
170 : void *cb_arg;
171 : SLIST_ENTRY(spdk_accel_sequence) link;
172 : };
173 : SPDK_STATIC_ASSERT(sizeof(struct spdk_accel_sequence) == 64, "invalid size");
174 :
175 : #define accel_update_stats(ch, event, v) \
176 : do { \
177 : (ch)->stats.event += (v); \
178 : } while (0)
179 :
180 : #define accel_update_task_stats(ch, task, event, v) \
181 : accel_update_stats(ch, operations[(task)->op_code].event, v)
182 :
183 : static inline void accel_sequence_task_cb(struct spdk_accel_sequence *seq,
184 : struct spdk_accel_task *task, int status);
185 :
186 : static inline void
187 888 : accel_sequence_set_state(struct spdk_accel_sequence *seq, enum accel_sequence_state state)
188 : {
189 888 : SPDK_DEBUGLOG(accel, "seq=%p, setting state: %s -> %s\n", seq,
190 : ACCEL_SEQUENCE_STATE_STRING(seq->state), ACCEL_SEQUENCE_STATE_STRING(state));
191 888 : assert(seq->state != ACCEL_SEQUENCE_STATE_ERROR || state == ACCEL_SEQUENCE_STATE_ERROR);
192 888 : seq->state = state;
193 888 : }
194 :
195 : static void
196 9 : accel_sequence_set_fail(struct spdk_accel_sequence *seq, int status)
197 : {
198 9 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR);
199 9 : assert(status != 0);
200 9 : seq->status = status;
201 9 : }
202 :
203 : int
204 17 : spdk_accel_get_opc_module_name(enum spdk_accel_opcode opcode, const char **module_name)
205 : {
206 17 : if (opcode >= SPDK_ACCEL_OPC_LAST) {
207 : /* invalid opcode */
208 0 : return -EINVAL;
209 : }
210 :
211 17 : if (g_modules_opc[opcode].module) {
212 17 : *module_name = g_modules_opc[opcode].module->name;
213 : } else {
214 0 : return -ENOENT;
215 : }
216 :
217 17 : return 0;
218 : }
219 :
220 : void
221 0 : _accel_for_each_module(struct module_info *info, _accel_for_each_module_fn fn)
222 : {
223 : struct spdk_accel_module_if *accel_module;
224 : enum spdk_accel_opcode opcode;
225 0 : int j = 0;
226 :
227 0 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
228 0 : for (opcode = 0; opcode < SPDK_ACCEL_OPC_LAST; opcode++) {
229 0 : if (accel_module->supports_opcode(opcode)) {
230 0 : info->ops[j] = opcode;
231 0 : j++;
232 : }
233 : }
234 0 : info->name = accel_module->name;
235 0 : info->num_ops = j;
236 0 : fn(info);
237 0 : j = 0;
238 : }
239 0 : }
240 :
241 : const char *
242 0 : spdk_accel_get_opcode_name(enum spdk_accel_opcode opcode)
243 : {
244 0 : if (opcode < SPDK_ACCEL_OPC_LAST) {
245 0 : return g_opcode_strings[opcode];
246 : }
247 :
248 0 : return NULL;
249 : }
250 :
251 : int
252 0 : spdk_accel_assign_opc(enum spdk_accel_opcode opcode, const char *name)
253 : {
254 : char *copy;
255 :
256 0 : if (g_modules_started == true) {
257 : /* we don't allow re-assignment once things have started */
258 0 : return -EINVAL;
259 : }
260 :
261 0 : if (opcode >= SPDK_ACCEL_OPC_LAST) {
262 : /* invalid opcode */
263 0 : return -EINVAL;
264 : }
265 :
266 0 : copy = strdup(name);
267 0 : if (copy == NULL) {
268 0 : return -ENOMEM;
269 : }
270 :
271 : /* module selection will be validated after the framework starts. */
272 0 : free(g_modules_opc_override[opcode]);
273 0 : g_modules_opc_override[opcode] = copy;
274 :
275 0 : return 0;
276 : }
277 :
278 : inline static struct spdk_accel_task *
279 191 : _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, void *cb_arg)
280 : {
281 : struct spdk_accel_task *accel_task;
282 :
283 191 : accel_task = STAILQ_FIRST(&accel_ch->task_pool);
284 191 : if (spdk_unlikely(accel_task == NULL)) {
285 11 : accel_update_stats(accel_ch, retry.task, 1);
286 11 : return NULL;
287 : }
288 :
289 180 : accel_update_stats(accel_ch, task_outstanding, 1);
290 180 : STAILQ_REMOVE_HEAD(&accel_ch->task_pool, link);
291 180 : accel_task->link.stqe_next = NULL;
292 :
293 180 : accel_task->cb_fn = cb_fn;
294 180 : accel_task->cb_arg = cb_arg;
295 180 : accel_task->accel_ch = accel_ch;
296 180 : accel_task->s.iovs = NULL;
297 180 : accel_task->d.iovs = NULL;
298 :
299 180 : return accel_task;
300 : }
301 :
302 : static void
303 172 : _put_task(struct accel_io_channel *ch, struct spdk_accel_task *task)
304 : {
305 172 : STAILQ_INSERT_HEAD(&ch->task_pool, task, link);
306 172 : accel_update_stats(ch, task_outstanding, -1);
307 172 : }
308 :
309 : void
310 122 : spdk_accel_task_complete(struct spdk_accel_task *accel_task, int status)
311 : {
312 122 : struct accel_io_channel *accel_ch = accel_task->accel_ch;
313 : spdk_accel_completion_cb cb_fn;
314 : void *cb_arg;
315 :
316 122 : accel_update_task_stats(accel_ch, accel_task, executed, 1);
317 122 : accel_update_task_stats(accel_ch, accel_task, num_bytes, accel_task->nbytes);
318 122 : if (spdk_unlikely(status != 0)) {
319 3 : accel_update_task_stats(accel_ch, accel_task, failed, 1);
320 : }
321 :
322 122 : if (accel_task->seq) {
323 118 : accel_sequence_task_cb(accel_task->seq, accel_task, status);
324 118 : return;
325 : }
326 :
327 4 : cb_fn = accel_task->cb_fn;
328 4 : cb_arg = accel_task->cb_arg;
329 :
330 4 : if (accel_task->has_aux) {
331 2 : SLIST_INSERT_HEAD(&accel_ch->task_aux_data_pool, accel_task->aux, link);
332 2 : accel_task->aux = NULL;
333 2 : accel_task->has_aux = false;
334 : }
335 :
336 : /* We should put the accel_task into the list firstly in order to avoid
337 : * the accel task list is exhausted when there is recursive call to
338 : * allocate accel_task in user's call back function (cb_fn)
339 : */
340 4 : _put_task(accel_ch, accel_task);
341 :
342 4 : cb_fn(cb_arg, status);
343 : }
344 :
345 : static inline int
346 119 : accel_submit_task(struct accel_io_channel *accel_ch, struct spdk_accel_task *task)
347 : {
348 119 : struct spdk_io_channel *module_ch = accel_ch->module_ch[task->op_code];
349 119 : struct spdk_accel_module_if *module = g_modules_opc[task->op_code].module;
350 : int rc;
351 :
352 119 : rc = module->submit_tasks(module_ch, task);
353 119 : if (spdk_unlikely(rc != 0)) {
354 2 : accel_update_task_stats(accel_ch, task, failed, 1);
355 : }
356 :
357 119 : return rc;
358 : }
359 :
360 : static inline uint64_t
361 141 : accel_get_iovlen(struct iovec *iovs, uint32_t iovcnt)
362 : {
363 141 : uint64_t result = 0;
364 : uint32_t i;
365 :
366 313 : for (i = 0; i < iovcnt; ++i) {
367 172 : result += iovs[i].iov_len;
368 : }
369 :
370 141 : return result;
371 : }
372 :
373 : #define ACCEL_TASK_ALLOC_AUX_BUF(task) \
374 : do { \
375 : (task)->aux = SLIST_FIRST(&(task)->accel_ch->task_aux_data_pool); \
376 : if (spdk_unlikely(!(task)->aux)) { \
377 : SPDK_ERRLOG("Fatal problem, aux data was not allocated\n"); \
378 : _put_task(task->accel_ch, task); \
379 : assert(0); \
380 : return -ENOMEM; \
381 : } \
382 : SLIST_REMOVE_HEAD(&(task)->accel_ch->task_aux_data_pool, link); \
383 : (task)->has_aux = true; \
384 : } while (0)
385 :
386 : /* Accel framework public API for copy function */
387 : int
388 2 : spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src,
389 : uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
390 : {
391 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
392 : struct spdk_accel_task *accel_task;
393 :
394 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
395 2 : if (spdk_unlikely(accel_task == NULL)) {
396 1 : return -ENOMEM;
397 : }
398 :
399 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
400 :
401 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
402 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
403 1 : accel_task->d.iovs[0].iov_base = dst;
404 1 : accel_task->d.iovs[0].iov_len = nbytes;
405 1 : accel_task->d.iovcnt = 1;
406 1 : accel_task->s.iovs[0].iov_base = src;
407 1 : accel_task->s.iovs[0].iov_len = nbytes;
408 1 : accel_task->s.iovcnt = 1;
409 1 : accel_task->nbytes = nbytes;
410 1 : accel_task->op_code = SPDK_ACCEL_OPC_COPY;
411 1 : accel_task->src_domain = NULL;
412 1 : accel_task->dst_domain = NULL;
413 :
414 1 : return accel_submit_task(accel_ch, accel_task);
415 : }
416 :
417 : /* Accel framework public API for dual cast copy function */
418 : int
419 4 : spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1,
420 : void *dst2, void *src, uint64_t nbytes,
421 : spdk_accel_completion_cb cb_fn, void *cb_arg)
422 : {
423 4 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
424 : struct spdk_accel_task *accel_task;
425 :
426 4 : if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {
427 2 : SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
428 2 : return -EINVAL;
429 : }
430 :
431 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
432 2 : if (spdk_unlikely(accel_task == NULL)) {
433 1 : return -ENOMEM;
434 : }
435 :
436 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
437 :
438 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
439 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
440 1 : accel_task->d2.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST2];
441 1 : accel_task->d.iovs[0].iov_base = dst1;
442 1 : accel_task->d.iovs[0].iov_len = nbytes;
443 1 : accel_task->d.iovcnt = 1;
444 1 : accel_task->d2.iovs[0].iov_base = dst2;
445 1 : accel_task->d2.iovs[0].iov_len = nbytes;
446 1 : accel_task->d2.iovcnt = 1;
447 1 : accel_task->s.iovs[0].iov_base = src;
448 1 : accel_task->s.iovs[0].iov_len = nbytes;
449 1 : accel_task->s.iovcnt = 1;
450 1 : accel_task->nbytes = nbytes;
451 1 : accel_task->op_code = SPDK_ACCEL_OPC_DUALCAST;
452 1 : accel_task->src_domain = NULL;
453 1 : accel_task->dst_domain = NULL;
454 :
455 1 : return accel_submit_task(accel_ch, accel_task);
456 : }
457 :
458 : /* Accel framework public API for compare function */
459 :
460 : int
461 2 : spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1,
462 : void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
463 : void *cb_arg)
464 : {
465 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
466 : struct spdk_accel_task *accel_task;
467 :
468 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
469 2 : if (spdk_unlikely(accel_task == NULL)) {
470 1 : return -ENOMEM;
471 : }
472 :
473 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
474 :
475 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
476 1 : accel_task->s2.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC2];
477 1 : accel_task->s.iovs[0].iov_base = src1;
478 1 : accel_task->s.iovs[0].iov_len = nbytes;
479 1 : accel_task->s.iovcnt = 1;
480 1 : accel_task->s2.iovs[0].iov_base = src2;
481 1 : accel_task->s2.iovs[0].iov_len = nbytes;
482 1 : accel_task->s2.iovcnt = 1;
483 1 : accel_task->nbytes = nbytes;
484 1 : accel_task->op_code = SPDK_ACCEL_OPC_COMPARE;
485 1 : accel_task->src_domain = NULL;
486 1 : accel_task->dst_domain = NULL;
487 :
488 1 : return accel_submit_task(accel_ch, accel_task);
489 : }
490 :
491 : /* Accel framework public API for fill function */
492 : int
493 2 : spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst,
494 : uint8_t fill, uint64_t nbytes,
495 : spdk_accel_completion_cb cb_fn, void *cb_arg)
496 : {
497 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
498 : struct spdk_accel_task *accel_task;
499 :
500 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
501 2 : if (spdk_unlikely(accel_task == NULL)) {
502 1 : return -ENOMEM;
503 : }
504 :
505 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
506 :
507 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
508 1 : accel_task->d.iovs[0].iov_base = dst;
509 1 : accel_task->d.iovs[0].iov_len = nbytes;
510 1 : accel_task->d.iovcnt = 1;
511 1 : accel_task->nbytes = nbytes;
512 1 : memset(&accel_task->fill_pattern, fill, sizeof(uint64_t));
513 1 : accel_task->op_code = SPDK_ACCEL_OPC_FILL;
514 1 : accel_task->src_domain = NULL;
515 1 : accel_task->dst_domain = NULL;
516 :
517 1 : return accel_submit_task(accel_ch, accel_task);
518 : }
519 :
520 : /* Accel framework public API for CRC-32C function */
521 : int
522 2 : spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst,
523 : void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
524 : void *cb_arg)
525 : {
526 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
527 : struct spdk_accel_task *accel_task;
528 :
529 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
530 2 : if (spdk_unlikely(accel_task == NULL)) {
531 1 : return -ENOMEM;
532 : }
533 :
534 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
535 :
536 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
537 1 : accel_task->s.iovs[0].iov_base = src;
538 1 : accel_task->s.iovs[0].iov_len = nbytes;
539 1 : accel_task->s.iovcnt = 1;
540 1 : accel_task->nbytes = nbytes;
541 1 : accel_task->crc_dst = crc_dst;
542 1 : accel_task->seed = seed;
543 1 : accel_task->op_code = SPDK_ACCEL_OPC_CRC32C;
544 1 : accel_task->src_domain = NULL;
545 1 : accel_task->dst_domain = NULL;
546 :
547 1 : return accel_submit_task(accel_ch, accel_task);
548 : }
549 :
550 : /* Accel framework public API for chained CRC-32C function */
551 : int
552 1 : spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst,
553 : struct iovec *iov, uint32_t iov_cnt, uint32_t seed,
554 : spdk_accel_completion_cb cb_fn, void *cb_arg)
555 : {
556 1 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
557 : struct spdk_accel_task *accel_task;
558 :
559 1 : if (iov == NULL) {
560 0 : SPDK_ERRLOG("iov should not be NULL");
561 0 : return -EINVAL;
562 : }
563 :
564 1 : if (!iov_cnt) {
565 0 : SPDK_ERRLOG("iovcnt should not be zero value\n");
566 0 : return -EINVAL;
567 : }
568 :
569 1 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
570 1 : if (spdk_unlikely(accel_task == NULL)) {
571 0 : SPDK_ERRLOG("no memory\n");
572 0 : assert(0);
573 : return -ENOMEM;
574 : }
575 :
576 1 : accel_task->s.iovs = iov;
577 1 : accel_task->s.iovcnt = iov_cnt;
578 1 : accel_task->nbytes = accel_get_iovlen(iov, iov_cnt);
579 1 : accel_task->crc_dst = crc_dst;
580 1 : accel_task->seed = seed;
581 1 : accel_task->op_code = SPDK_ACCEL_OPC_CRC32C;
582 1 : accel_task->src_domain = NULL;
583 1 : accel_task->dst_domain = NULL;
584 :
585 1 : return accel_submit_task(accel_ch, accel_task);
586 : }
587 :
588 : /* Accel framework public API for copy with CRC-32C function */
589 : int
590 2 : spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst,
591 : void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes,
592 : spdk_accel_completion_cb cb_fn, void *cb_arg)
593 : {
594 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
595 : struct spdk_accel_task *accel_task;
596 :
597 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
598 2 : if (spdk_unlikely(accel_task == NULL)) {
599 1 : return -ENOMEM;
600 : }
601 :
602 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
603 :
604 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
605 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
606 1 : accel_task->d.iovs[0].iov_base = dst;
607 1 : accel_task->d.iovs[0].iov_len = nbytes;
608 1 : accel_task->d.iovcnt = 1;
609 1 : accel_task->s.iovs[0].iov_base = src;
610 1 : accel_task->s.iovs[0].iov_len = nbytes;
611 1 : accel_task->s.iovcnt = 1;
612 1 : accel_task->nbytes = nbytes;
613 1 : accel_task->crc_dst = crc_dst;
614 1 : accel_task->seed = seed;
615 1 : accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C;
616 1 : accel_task->src_domain = NULL;
617 1 : accel_task->dst_domain = NULL;
618 :
619 1 : return accel_submit_task(accel_ch, accel_task);
620 : }
621 :
622 : /* Accel framework public API for chained copy + CRC-32C function */
623 : int
624 0 : spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst,
625 : struct iovec *src_iovs, uint32_t iov_cnt, uint32_t *crc_dst,
626 : uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg)
627 : {
628 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
629 : struct spdk_accel_task *accel_task;
630 : uint64_t nbytes;
631 :
632 0 : if (src_iovs == NULL) {
633 0 : SPDK_ERRLOG("iov should not be NULL");
634 0 : return -EINVAL;
635 : }
636 :
637 0 : if (!iov_cnt) {
638 0 : SPDK_ERRLOG("iovcnt should not be zero value\n");
639 0 : return -EINVAL;
640 : }
641 :
642 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
643 0 : if (spdk_unlikely(accel_task == NULL)) {
644 0 : SPDK_ERRLOG("no memory\n");
645 0 : assert(0);
646 : return -ENOMEM;
647 : }
648 :
649 0 : nbytes = accel_get_iovlen(src_iovs, iov_cnt);
650 :
651 0 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
652 :
653 0 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
654 0 : accel_task->d.iovs[0].iov_base = dst;
655 0 : accel_task->d.iovs[0].iov_len = nbytes;
656 0 : accel_task->d.iovcnt = 1;
657 0 : accel_task->s.iovs = src_iovs;
658 0 : accel_task->s.iovcnt = iov_cnt;
659 0 : accel_task->nbytes = nbytes;
660 0 : accel_task->crc_dst = crc_dst;
661 0 : accel_task->seed = seed;
662 0 : accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C;
663 0 : accel_task->src_domain = NULL;
664 0 : accel_task->dst_domain = NULL;
665 :
666 0 : return accel_submit_task(accel_ch, accel_task);
667 : }
668 :
669 : int
670 0 : spdk_accel_get_compress_level_range(enum spdk_accel_comp_algo comp_algo,
671 : uint32_t *min_level, uint32_t *max_level)
672 : {
673 0 : struct spdk_accel_module_if *module = g_modules_opc[SPDK_ACCEL_OPC_COMPRESS].module;
674 :
675 0 : if (module->get_compress_level_range == NULL) {
676 0 : SPDK_ERRLOG("Module %s doesn't implement callback fn get_compress_level_range.\n", module->name);
677 0 : return -ENOTSUP;
678 : }
679 :
680 0 : return module->get_compress_level_range(comp_algo, min_level, max_level);
681 : }
682 :
683 : static int
684 49 : _accel_check_comp_algo(enum spdk_accel_comp_algo comp_algo)
685 : {
686 49 : struct spdk_accel_module_if *module = g_modules_opc[SPDK_ACCEL_OPC_COMPRESS].module;
687 :
688 49 : if (!module->compress_supports_algo || !module->compress_supports_algo(comp_algo)) {
689 0 : SPDK_ERRLOG("Module %s doesn't support compression algo %d\n", module->name, comp_algo);
690 0 : return -ENOTSUP;
691 : }
692 :
693 49 : return 0;
694 : }
695 :
696 : int
697 2 : spdk_accel_submit_compress_ext(struct spdk_io_channel *ch, void *dst, uint64_t nbytes,
698 : struct iovec *src_iovs, size_t src_iovcnt,
699 : enum spdk_accel_comp_algo comp_algo, uint32_t comp_level,
700 : uint32_t *output_size, spdk_accel_completion_cb cb_fn, void *cb_arg)
701 : {
702 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
703 : struct spdk_accel_task *accel_task;
704 : int rc;
705 :
706 2 : rc = _accel_check_comp_algo(comp_algo);
707 2 : if (spdk_unlikely(rc != 0)) {
708 0 : return rc;
709 : }
710 :
711 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
712 2 : if (spdk_unlikely(accel_task == NULL)) {
713 0 : return -ENOMEM;
714 : }
715 :
716 2 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
717 :
718 2 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
719 2 : accel_task->d.iovs[0].iov_base = dst;
720 2 : accel_task->d.iovs[0].iov_len = nbytes;
721 2 : accel_task->d.iovcnt = 1;
722 2 : accel_task->output_size = output_size;
723 2 : accel_task->s.iovs = src_iovs;
724 2 : accel_task->s.iovcnt = src_iovcnt;
725 2 : accel_task->nbytes = nbytes;
726 2 : accel_task->op_code = SPDK_ACCEL_OPC_COMPRESS;
727 2 : accel_task->src_domain = NULL;
728 2 : accel_task->dst_domain = NULL;
729 2 : accel_task->comp.algo = comp_algo;
730 2 : accel_task->comp.level = comp_level;
731 :
732 2 : return accel_submit_task(accel_ch, accel_task);
733 : }
734 :
735 : int
736 0 : spdk_accel_submit_decompress_ext(struct spdk_io_channel *ch, struct iovec *dst_iovs,
737 : size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt,
738 : enum spdk_accel_comp_algo decomp_algo, uint32_t *output_size,
739 : spdk_accel_completion_cb cb_fn, void *cb_arg)
740 : {
741 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
742 : struct spdk_accel_task *accel_task;
743 : int rc;
744 :
745 0 : rc = _accel_check_comp_algo(decomp_algo);
746 0 : if (spdk_unlikely(rc != 0)) {
747 0 : return rc;
748 : }
749 :
750 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
751 0 : if (spdk_unlikely(accel_task == NULL)) {
752 0 : return -ENOMEM;
753 : }
754 :
755 0 : accel_task->output_size = output_size;
756 0 : accel_task->s.iovs = src_iovs;
757 0 : accel_task->s.iovcnt = src_iovcnt;
758 0 : accel_task->d.iovs = dst_iovs;
759 0 : accel_task->d.iovcnt = dst_iovcnt;
760 0 : accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
761 0 : accel_task->op_code = SPDK_ACCEL_OPC_DECOMPRESS;
762 0 : accel_task->src_domain = NULL;
763 0 : accel_task->dst_domain = NULL;
764 0 : accel_task->comp.algo = decomp_algo;
765 :
766 0 : return accel_submit_task(accel_ch, accel_task);
767 : }
768 :
769 : int
770 2 : spdk_accel_submit_compress(struct spdk_io_channel *ch, void *dst, uint64_t nbytes,
771 : struct iovec *src_iovs, size_t src_iovcnt, uint32_t *output_size,
772 : spdk_accel_completion_cb cb_fn, void *cb_arg)
773 : {
774 2 : return spdk_accel_submit_compress_ext(ch, dst, nbytes, src_iovs, src_iovcnt,
775 : SPDK_ACCEL_COMP_ALGO_DEFLATE, 1, output_size, cb_fn, cb_arg);
776 : }
777 :
778 : int
779 0 : spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs,
780 : size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt,
781 : uint32_t *output_size, spdk_accel_completion_cb cb_fn,
782 : void *cb_arg)
783 : {
784 0 : return spdk_accel_submit_decompress_ext(ch, dst_iovs, dst_iovcnt, src_iovs, src_iovcnt,
785 : SPDK_ACCEL_COMP_ALGO_DEFLATE, output_size, cb_fn, cb_arg);
786 : }
787 :
788 : int
789 1 : spdk_accel_submit_encrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key,
790 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
791 : struct iovec *src_iovs, uint32_t src_iovcnt,
792 : uint64_t iv, uint32_t block_size,
793 : spdk_accel_completion_cb cb_fn, void *cb_arg)
794 : {
795 1 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
796 : struct spdk_accel_task *accel_task;
797 :
798 1 : if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) {
799 0 : return -EINVAL;
800 : }
801 :
802 1 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
803 1 : if (spdk_unlikely(accel_task == NULL)) {
804 0 : return -ENOMEM;
805 : }
806 :
807 1 : accel_task->crypto_key = key;
808 1 : accel_task->s.iovs = src_iovs;
809 1 : accel_task->s.iovcnt = src_iovcnt;
810 1 : accel_task->d.iovs = dst_iovs;
811 1 : accel_task->d.iovcnt = dst_iovcnt;
812 1 : accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
813 1 : accel_task->iv = iv;
814 1 : accel_task->block_size = block_size;
815 1 : accel_task->op_code = SPDK_ACCEL_OPC_ENCRYPT;
816 1 : accel_task->src_domain = NULL;
817 1 : accel_task->dst_domain = NULL;
818 :
819 1 : return accel_submit_task(accel_ch, accel_task);
820 : }
821 :
822 : int
823 0 : spdk_accel_submit_decrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key,
824 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
825 : struct iovec *src_iovs, uint32_t src_iovcnt,
826 : uint64_t iv, uint32_t block_size,
827 : spdk_accel_completion_cb cb_fn, void *cb_arg)
828 : {
829 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
830 : struct spdk_accel_task *accel_task;
831 :
832 0 : if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) {
833 0 : return -EINVAL;
834 : }
835 :
836 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
837 0 : if (spdk_unlikely(accel_task == NULL)) {
838 0 : return -ENOMEM;
839 : }
840 :
841 0 : accel_task->crypto_key = key;
842 0 : accel_task->s.iovs = src_iovs;
843 0 : accel_task->s.iovcnt = src_iovcnt;
844 0 : accel_task->d.iovs = dst_iovs;
845 0 : accel_task->d.iovcnt = dst_iovcnt;
846 0 : accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
847 0 : accel_task->iv = iv;
848 0 : accel_task->block_size = block_size;
849 0 : accel_task->op_code = SPDK_ACCEL_OPC_DECRYPT;
850 0 : accel_task->src_domain = NULL;
851 0 : accel_task->dst_domain = NULL;
852 :
853 0 : return accel_submit_task(accel_ch, accel_task);
854 : }
855 :
856 : int
857 2 : spdk_accel_submit_xor(struct spdk_io_channel *ch, void *dst, void **sources, uint32_t nsrcs,
858 : uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
859 : {
860 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
861 : struct spdk_accel_task *accel_task;
862 :
863 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
864 2 : if (spdk_unlikely(accel_task == NULL)) {
865 1 : return -ENOMEM;
866 : }
867 :
868 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
869 :
870 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
871 1 : accel_task->nsrcs.srcs = sources;
872 1 : accel_task->nsrcs.cnt = nsrcs;
873 1 : accel_task->d.iovs[0].iov_base = dst;
874 1 : accel_task->d.iovs[0].iov_len = nbytes;
875 1 : accel_task->d.iovcnt = 1;
876 1 : accel_task->nbytes = nbytes;
877 1 : accel_task->op_code = SPDK_ACCEL_OPC_XOR;
878 1 : accel_task->src_domain = NULL;
879 1 : accel_task->dst_domain = NULL;
880 :
881 1 : return accel_submit_task(accel_ch, accel_task);
882 : }
883 :
884 : int
885 0 : spdk_accel_submit_dif_verify(struct spdk_io_channel *ch,
886 : struct iovec *iovs, size_t iovcnt, uint32_t num_blocks,
887 : const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
888 : spdk_accel_completion_cb cb_fn, void *cb_arg)
889 : {
890 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
891 : struct spdk_accel_task *accel_task;
892 :
893 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
894 0 : if (accel_task == NULL) {
895 0 : return -ENOMEM;
896 : }
897 :
898 0 : accel_task->s.iovs = iovs;
899 0 : accel_task->s.iovcnt = iovcnt;
900 0 : accel_task->dif.ctx = ctx;
901 0 : accel_task->dif.err = err;
902 0 : accel_task->dif.num_blocks = num_blocks;
903 0 : accel_task->nbytes = num_blocks * ctx->block_size;
904 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY;
905 0 : accel_task->src_domain = NULL;
906 0 : accel_task->dst_domain = NULL;
907 :
908 0 : return accel_submit_task(accel_ch, accel_task);
909 : }
910 :
911 : int
912 0 : spdk_accel_submit_dif_generate(struct spdk_io_channel *ch,
913 : struct iovec *iovs, size_t iovcnt, uint32_t num_blocks,
914 : const struct spdk_dif_ctx *ctx,
915 : spdk_accel_completion_cb cb_fn, void *cb_arg)
916 : {
917 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
918 : struct spdk_accel_task *accel_task;
919 :
920 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
921 0 : if (accel_task == NULL) {
922 0 : return -ENOMEM;
923 : }
924 :
925 0 : accel_task->s.iovs = iovs;
926 0 : accel_task->s.iovcnt = iovcnt;
927 0 : accel_task->dif.ctx = ctx;
928 0 : accel_task->dif.num_blocks = num_blocks;
929 0 : accel_task->nbytes = num_blocks * ctx->block_size;
930 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE;
931 0 : accel_task->src_domain = NULL;
932 0 : accel_task->dst_domain = NULL;
933 :
934 0 : return accel_submit_task(accel_ch, accel_task);
935 : }
936 :
937 : int
938 0 : spdk_accel_submit_dif_generate_copy(struct spdk_io_channel *ch, struct iovec *dst_iovs,
939 : size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt,
940 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
941 : spdk_accel_completion_cb cb_fn, void *cb_arg)
942 : {
943 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
944 : struct spdk_accel_task *accel_task;
945 :
946 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
947 0 : if (accel_task == NULL) {
948 0 : return -ENOMEM;
949 : }
950 :
951 0 : accel_task->s.iovs = src_iovs;
952 0 : accel_task->s.iovcnt = src_iovcnt;
953 0 : accel_task->d.iovs = dst_iovs;
954 0 : accel_task->d.iovcnt = dst_iovcnt;
955 0 : accel_task->dif.ctx = ctx;
956 0 : accel_task->dif.num_blocks = num_blocks;
957 0 : accel_task->nbytes = num_blocks * ctx->block_size;
958 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE_COPY;
959 0 : accel_task->src_domain = NULL;
960 0 : accel_task->dst_domain = NULL;
961 :
962 0 : return accel_submit_task(accel_ch, accel_task);
963 : }
964 :
965 : int
966 0 : spdk_accel_submit_dif_verify_copy(struct spdk_io_channel *ch,
967 : struct iovec *dst_iovs, size_t dst_iovcnt,
968 : struct iovec *src_iovs, size_t src_iovcnt, uint32_t num_blocks,
969 : const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
970 : spdk_accel_completion_cb cb_fn, void *cb_arg)
971 : {
972 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
973 : struct spdk_accel_task *accel_task;
974 :
975 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
976 0 : if (accel_task == NULL) {
977 0 : return -ENOMEM;
978 : }
979 :
980 0 : accel_task->s.iovs = src_iovs;
981 0 : accel_task->s.iovcnt = src_iovcnt;
982 0 : accel_task->d.iovs = dst_iovs;
983 0 : accel_task->d.iovcnt = dst_iovcnt;
984 0 : accel_task->dif.ctx = ctx;
985 0 : accel_task->dif.err = err;
986 0 : accel_task->dif.num_blocks = num_blocks;
987 0 : accel_task->nbytes = num_blocks * ctx->block_size;
988 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY_COPY;
989 0 : accel_task->src_domain = NULL;
990 0 : accel_task->dst_domain = NULL;
991 :
992 0 : return accel_submit_task(accel_ch, accel_task);
993 : }
994 :
995 : int
996 0 : spdk_accel_submit_dix_generate(struct spdk_io_channel *ch, struct iovec *iovs,
997 : size_t iovcnt, struct iovec *md_iov, uint32_t num_blocks,
998 : const struct spdk_dif_ctx *ctx, spdk_accel_completion_cb cb_fn,
999 : void *cb_arg)
1000 : {
1001 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1002 : struct spdk_accel_task *accel_task;
1003 :
1004 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
1005 0 : if (accel_task == NULL) {
1006 0 : return -ENOMEM;
1007 : }
1008 :
1009 0 : accel_task->s.iovs = iovs;
1010 0 : accel_task->s.iovcnt = iovcnt;
1011 0 : accel_task->d.iovs = md_iov;
1012 0 : accel_task->d.iovcnt = 1;
1013 0 : accel_task->dif.ctx = ctx;
1014 0 : accel_task->dif.num_blocks = num_blocks;
1015 0 : accel_task->nbytes = num_blocks * ctx->block_size;
1016 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIX_GENERATE;
1017 0 : accel_task->src_domain = NULL;
1018 0 : accel_task->dst_domain = NULL;
1019 :
1020 0 : return accel_submit_task(accel_ch, accel_task);
1021 : }
1022 :
1023 : int
1024 0 : spdk_accel_submit_dix_verify(struct spdk_io_channel *ch, struct iovec *iovs,
1025 : size_t iovcnt, struct iovec *md_iov, uint32_t num_blocks,
1026 : const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
1027 : spdk_accel_completion_cb cb_fn, void *cb_arg)
1028 : {
1029 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1030 : struct spdk_accel_task *accel_task;
1031 :
1032 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
1033 0 : if (accel_task == NULL) {
1034 0 : return -ENOMEM;
1035 : }
1036 :
1037 0 : accel_task->s.iovs = iovs;
1038 0 : accel_task->s.iovcnt = iovcnt;
1039 0 : accel_task->d.iovs = md_iov;
1040 0 : accel_task->d.iovcnt = 1;
1041 0 : accel_task->dif.ctx = ctx;
1042 0 : accel_task->dif.err = err;
1043 0 : accel_task->dif.num_blocks = num_blocks;
1044 0 : accel_task->nbytes = num_blocks * ctx->block_size;
1045 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIX_VERIFY;
1046 0 : accel_task->src_domain = NULL;
1047 0 : accel_task->dst_domain = NULL;
1048 :
1049 0 : return accel_submit_task(accel_ch, accel_task);
1050 : }
1051 :
1052 : static inline struct accel_buffer *
1053 33 : accel_get_buf(struct accel_io_channel *ch, uint64_t len)
1054 : {
1055 : struct accel_buffer *buf;
1056 :
1057 33 : buf = SLIST_FIRST(&ch->buf_pool);
1058 33 : if (spdk_unlikely(buf == NULL)) {
1059 0 : accel_update_stats(ch, retry.bufdesc, 1);
1060 0 : return NULL;
1061 : }
1062 :
1063 33 : SLIST_REMOVE_HEAD(&ch->buf_pool, link);
1064 33 : buf->len = len;
1065 33 : buf->buf = NULL;
1066 33 : buf->seq = NULL;
1067 33 : buf->cb_fn = NULL;
1068 :
1069 33 : return buf;
1070 : }
1071 :
1072 : static inline void
1073 33 : accel_put_buf(struct accel_io_channel *ch, struct accel_buffer *buf)
1074 : {
1075 33 : if (buf->buf != NULL) {
1076 29 : spdk_iobuf_put(&ch->iobuf, buf->buf, buf->len);
1077 : }
1078 :
1079 33 : SLIST_INSERT_HEAD(&ch->buf_pool, buf, link);
1080 33 : }
1081 :
1082 : static inline struct spdk_accel_sequence *
1083 73 : accel_sequence_get(struct accel_io_channel *ch)
1084 : {
1085 : struct spdk_accel_sequence *seq;
1086 :
1087 73 : assert(g_opts.task_count >= ch->stats.task_outstanding);
1088 :
1089 : /* Sequence cannot be allocated if number of available task objects cannot satisfy required limit.
1090 : * This is to prevent potential dead lock when few requests are pending task resource and none can
1091 : * advance the processing. This solution should work only if there is single async operation after
1092 : * sequence obj obtained, so assume that is possible to happen with io buffer allocation now, if
1093 : * there are more async operations then solution should be improved. */
1094 73 : if (spdk_unlikely(g_opts.task_count - ch->stats.task_outstanding < ACCEL_TASKS_IN_SEQUENCE_LIMIT)) {
1095 0 : return NULL;
1096 : }
1097 :
1098 73 : seq = SLIST_FIRST(&ch->seq_pool);
1099 73 : if (spdk_unlikely(seq == NULL)) {
1100 3 : accel_update_stats(ch, retry.sequence, 1);
1101 3 : return NULL;
1102 : }
1103 :
1104 70 : accel_update_stats(ch, sequence_outstanding, 1);
1105 70 : SLIST_REMOVE_HEAD(&ch->seq_pool, link);
1106 :
1107 70 : TAILQ_INIT(&seq->tasks);
1108 70 : SLIST_INIT(&seq->bounce_bufs);
1109 :
1110 70 : seq->ch = ch;
1111 70 : seq->status = 0;
1112 70 : seq->state = ACCEL_SEQUENCE_STATE_INIT;
1113 70 : seq->in_process_sequence = false;
1114 :
1115 70 : return seq;
1116 : }
1117 :
1118 : static inline void
1119 70 : accel_sequence_put(struct spdk_accel_sequence *seq)
1120 : {
1121 70 : struct accel_io_channel *ch = seq->ch;
1122 : struct accel_buffer *buf;
1123 :
1124 89 : while (!SLIST_EMPTY(&seq->bounce_bufs)) {
1125 19 : buf = SLIST_FIRST(&seq->bounce_bufs);
1126 19 : SLIST_REMOVE_HEAD(&seq->bounce_bufs, link);
1127 19 : accel_put_buf(seq->ch, buf);
1128 : }
1129 :
1130 70 : assert(TAILQ_EMPTY(&seq->tasks));
1131 70 : seq->ch = NULL;
1132 :
1133 70 : SLIST_INSERT_HEAD(&ch->seq_pool, seq, link);
1134 70 : accel_update_stats(ch, sequence_outstanding, -1);
1135 70 : }
1136 :
1137 : static inline struct spdk_accel_task *
1138 171 : accel_sequence_get_task(struct accel_io_channel *ch, struct spdk_accel_sequence *seq,
1139 : spdk_accel_step_cb cb_fn, void *cb_arg)
1140 : {
1141 : struct spdk_accel_task *task;
1142 :
1143 171 : task = _get_task(ch, NULL, NULL);
1144 171 : if (spdk_unlikely(task == NULL)) {
1145 3 : return task;
1146 : }
1147 :
1148 168 : task->step_cb_fn = cb_fn;
1149 168 : task->cb_arg = cb_arg;
1150 168 : task->seq = seq;
1151 :
1152 168 : return task;
1153 : }
1154 :
1155 : int
1156 48 : spdk_accel_append_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1157 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
1158 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1159 : struct iovec *src_iovs, uint32_t src_iovcnt,
1160 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1161 : spdk_accel_step_cb cb_fn, void *cb_arg)
1162 : {
1163 48 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1164 : struct spdk_accel_task *task;
1165 48 : struct spdk_accel_sequence *seq = *pseq;
1166 :
1167 48 : if (seq == NULL) {
1168 17 : seq = accel_sequence_get(accel_ch);
1169 17 : if (spdk_unlikely(seq == NULL)) {
1170 1 : return -ENOMEM;
1171 : }
1172 : }
1173 :
1174 47 : assert(seq->ch == accel_ch);
1175 47 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1176 47 : if (spdk_unlikely(task == NULL)) {
1177 1 : if (*pseq == NULL) {
1178 1 : accel_sequence_put(seq);
1179 : }
1180 :
1181 1 : return -ENOMEM;
1182 : }
1183 :
1184 46 : task->dst_domain = dst_domain;
1185 46 : task->dst_domain_ctx = dst_domain_ctx;
1186 46 : task->d.iovs = dst_iovs;
1187 46 : task->d.iovcnt = dst_iovcnt;
1188 46 : task->src_domain = src_domain;
1189 46 : task->src_domain_ctx = src_domain_ctx;
1190 46 : task->s.iovs = src_iovs;
1191 46 : task->s.iovcnt = src_iovcnt;
1192 46 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1193 46 : task->op_code = SPDK_ACCEL_OPC_COPY;
1194 :
1195 46 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1196 46 : *pseq = seq;
1197 :
1198 46 : return 0;
1199 : }
1200 :
1201 : int
1202 42 : spdk_accel_append_fill(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1203 : void *buf, uint64_t len,
1204 : struct spdk_memory_domain *domain, void *domain_ctx, uint8_t pattern,
1205 : spdk_accel_step_cb cb_fn, void *cb_arg)
1206 : {
1207 42 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1208 : struct spdk_accel_task *task;
1209 42 : struct spdk_accel_sequence *seq = *pseq;
1210 :
1211 42 : if (seq == NULL) {
1212 24 : seq = accel_sequence_get(accel_ch);
1213 24 : if (spdk_unlikely(seq == NULL)) {
1214 1 : return -ENOMEM;
1215 : }
1216 : }
1217 :
1218 41 : assert(seq->ch == accel_ch);
1219 41 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1220 41 : if (spdk_unlikely(task == NULL)) {
1221 1 : if (*pseq == NULL) {
1222 1 : accel_sequence_put(seq);
1223 : }
1224 :
1225 1 : return -ENOMEM;
1226 : }
1227 :
1228 40 : memset(&task->fill_pattern, pattern, sizeof(uint64_t));
1229 :
1230 40 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1231 40 : if (spdk_unlikely(!task->aux)) {
1232 0 : SPDK_ERRLOG("Fatal problem, aux data was not allocated\n");
1233 0 : if (*pseq == NULL) {
1234 0 : accel_sequence_put((seq));
1235 : }
1236 :
1237 0 : task->seq = NULL;
1238 0 : _put_task(task->accel_ch, task);
1239 0 : assert(0);
1240 : return -ENOMEM;
1241 : }
1242 40 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1243 40 : task->has_aux = true;
1244 :
1245 40 : task->d.iovs = &task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
1246 40 : task->d.iovs[0].iov_base = buf;
1247 40 : task->d.iovs[0].iov_len = len;
1248 40 : task->d.iovcnt = 1;
1249 40 : task->nbytes = len;
1250 40 : task->src_domain = NULL;
1251 40 : task->dst_domain = domain;
1252 40 : task->dst_domain_ctx = domain_ctx;
1253 40 : task->op_code = SPDK_ACCEL_OPC_FILL;
1254 :
1255 40 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1256 40 : *pseq = seq;
1257 :
1258 40 : return 0;
1259 : }
1260 :
1261 : int
1262 47 : spdk_accel_append_decompress(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1263 : struct iovec *dst_iovs, size_t dst_iovcnt,
1264 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1265 : struct iovec *src_iovs, size_t src_iovcnt,
1266 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1267 : spdk_accel_step_cb cb_fn, void *cb_arg)
1268 : {
1269 47 : return spdk_accel_append_decompress_ext(pseq, ch, dst_iovs, dst_iovcnt, dst_domain,
1270 : dst_domain_ctx, src_iovs, src_iovcnt, src_domain,
1271 : src_domain_ctx, SPDK_ACCEL_COMP_ALGO_DEFLATE,
1272 : cb_fn, cb_arg);
1273 : }
1274 :
1275 : int
1276 47 : spdk_accel_append_decompress_ext(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1277 : struct iovec *dst_iovs, size_t dst_iovcnt,
1278 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1279 : struct iovec *src_iovs, size_t src_iovcnt,
1280 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1281 : enum spdk_accel_comp_algo decomp_algo,
1282 : spdk_accel_step_cb cb_fn, void *cb_arg)
1283 : {
1284 47 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1285 : struct spdk_accel_task *task;
1286 47 : struct spdk_accel_sequence *seq = *pseq;
1287 : int rc;
1288 :
1289 47 : rc = _accel_check_comp_algo(decomp_algo);
1290 47 : if (spdk_unlikely(rc != 0)) {
1291 0 : return rc;
1292 : }
1293 :
1294 47 : if (seq == NULL) {
1295 21 : seq = accel_sequence_get(accel_ch);
1296 21 : if (spdk_unlikely(seq == NULL)) {
1297 1 : return -ENOMEM;
1298 : }
1299 : }
1300 :
1301 46 : assert(seq->ch == accel_ch);
1302 46 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1303 46 : if (spdk_unlikely(task == NULL)) {
1304 1 : if (*pseq == NULL) {
1305 1 : accel_sequence_put(seq);
1306 : }
1307 :
1308 1 : return -ENOMEM;
1309 : }
1310 :
1311 : /* TODO: support output_size for chaining */
1312 45 : task->output_size = NULL;
1313 45 : task->dst_domain = dst_domain;
1314 45 : task->dst_domain_ctx = dst_domain_ctx;
1315 45 : task->d.iovs = dst_iovs;
1316 45 : task->d.iovcnt = dst_iovcnt;
1317 45 : task->src_domain = src_domain;
1318 45 : task->src_domain_ctx = src_domain_ctx;
1319 45 : task->s.iovs = src_iovs;
1320 45 : task->s.iovcnt = src_iovcnt;
1321 45 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1322 45 : task->op_code = SPDK_ACCEL_OPC_DECOMPRESS;
1323 45 : task->comp.algo = decomp_algo;
1324 :
1325 45 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1326 45 : *pseq = seq;
1327 :
1328 45 : return 0;
1329 : }
1330 :
1331 : int
1332 11 : spdk_accel_append_encrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1333 : struct spdk_accel_crypto_key *key,
1334 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
1335 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1336 : struct iovec *src_iovs, uint32_t src_iovcnt,
1337 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1338 : uint64_t iv, uint32_t block_size,
1339 : spdk_accel_step_cb cb_fn, void *cb_arg)
1340 : {
1341 11 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1342 : struct spdk_accel_task *task;
1343 11 : struct spdk_accel_sequence *seq = *pseq;
1344 :
1345 11 : assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size);
1346 :
1347 11 : if (seq == NULL) {
1348 7 : seq = accel_sequence_get(accel_ch);
1349 7 : if (spdk_unlikely(seq == NULL)) {
1350 0 : return -ENOMEM;
1351 : }
1352 : }
1353 :
1354 11 : assert(seq->ch == accel_ch);
1355 11 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1356 11 : if (spdk_unlikely(task == NULL)) {
1357 0 : if (*pseq == NULL) {
1358 0 : accel_sequence_put(seq);
1359 : }
1360 :
1361 0 : return -ENOMEM;
1362 : }
1363 :
1364 11 : task->crypto_key = key;
1365 11 : task->src_domain = src_domain;
1366 11 : task->src_domain_ctx = src_domain_ctx;
1367 11 : task->s.iovs = src_iovs;
1368 11 : task->s.iovcnt = src_iovcnt;
1369 11 : task->dst_domain = dst_domain;
1370 11 : task->dst_domain_ctx = dst_domain_ctx;
1371 11 : task->d.iovs = dst_iovs;
1372 11 : task->d.iovcnt = dst_iovcnt;
1373 11 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1374 11 : task->iv = iv;
1375 11 : task->block_size = block_size;
1376 11 : task->op_code = SPDK_ACCEL_OPC_ENCRYPT;
1377 :
1378 11 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1379 11 : *pseq = seq;
1380 :
1381 11 : return 0;
1382 : }
1383 :
1384 : int
1385 12 : spdk_accel_append_decrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1386 : struct spdk_accel_crypto_key *key,
1387 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
1388 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1389 : struct iovec *src_iovs, uint32_t src_iovcnt,
1390 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1391 : uint64_t iv, uint32_t block_size,
1392 : spdk_accel_step_cb cb_fn, void *cb_arg)
1393 : {
1394 12 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1395 : struct spdk_accel_task *task;
1396 12 : struct spdk_accel_sequence *seq = *pseq;
1397 :
1398 12 : assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size);
1399 :
1400 12 : if (seq == NULL) {
1401 1 : seq = accel_sequence_get(accel_ch);
1402 1 : if (spdk_unlikely(seq == NULL)) {
1403 0 : return -ENOMEM;
1404 : }
1405 : }
1406 :
1407 12 : assert(seq->ch == accel_ch);
1408 12 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1409 12 : if (spdk_unlikely(task == NULL)) {
1410 0 : if (*pseq == NULL) {
1411 0 : accel_sequence_put(seq);
1412 : }
1413 :
1414 0 : return -ENOMEM;
1415 : }
1416 :
1417 12 : task->crypto_key = key;
1418 12 : task->src_domain = src_domain;
1419 12 : task->src_domain_ctx = src_domain_ctx;
1420 12 : task->s.iovs = src_iovs;
1421 12 : task->s.iovcnt = src_iovcnt;
1422 12 : task->dst_domain = dst_domain;
1423 12 : task->dst_domain_ctx = dst_domain_ctx;
1424 12 : task->d.iovs = dst_iovs;
1425 12 : task->d.iovcnt = dst_iovcnt;
1426 12 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1427 12 : task->iv = iv;
1428 12 : task->block_size = block_size;
1429 12 : task->op_code = SPDK_ACCEL_OPC_DECRYPT;
1430 :
1431 12 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1432 12 : *pseq = seq;
1433 :
1434 12 : return 0;
1435 : }
1436 :
1437 : int
1438 6 : spdk_accel_append_crc32c(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1439 : uint32_t *dst, struct iovec *iovs, uint32_t iovcnt,
1440 : struct spdk_memory_domain *domain, void *domain_ctx,
1441 : uint32_t seed, spdk_accel_step_cb cb_fn, void *cb_arg)
1442 : {
1443 6 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1444 : struct spdk_accel_task *task;
1445 6 : struct spdk_accel_sequence *seq = *pseq;
1446 :
1447 6 : if (seq == NULL) {
1448 2 : seq = accel_sequence_get(accel_ch);
1449 2 : if (spdk_unlikely(seq == NULL)) {
1450 0 : return -ENOMEM;
1451 : }
1452 : }
1453 :
1454 6 : assert(seq->ch == accel_ch);
1455 6 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1456 6 : if (spdk_unlikely(task == NULL)) {
1457 0 : if (*pseq == NULL) {
1458 0 : accel_sequence_put(seq);
1459 : }
1460 :
1461 0 : return -ENOMEM;
1462 : }
1463 :
1464 6 : task->s.iovs = iovs;
1465 6 : task->s.iovcnt = iovcnt;
1466 6 : task->src_domain = domain;
1467 6 : task->src_domain_ctx = domain_ctx;
1468 6 : task->nbytes = accel_get_iovlen(iovs, iovcnt);
1469 6 : task->crc_dst = dst;
1470 6 : task->seed = seed;
1471 6 : task->op_code = SPDK_ACCEL_OPC_CRC32C;
1472 6 : task->dst_domain = NULL;
1473 :
1474 6 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1475 6 : *pseq = seq;
1476 :
1477 6 : return 0;
1478 : }
1479 :
1480 : int
1481 0 : spdk_accel_append_dif_verify(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1482 : struct iovec *iovs, size_t iovcnt,
1483 : struct spdk_memory_domain *domain, void *domain_ctx,
1484 : uint32_t num_blocks,
1485 : const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
1486 : spdk_accel_step_cb cb_fn, void *cb_arg)
1487 : {
1488 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1489 : struct spdk_accel_task *task;
1490 0 : struct spdk_accel_sequence *seq = *pseq;
1491 :
1492 0 : if (seq == NULL) {
1493 0 : seq = accel_sequence_get(accel_ch);
1494 0 : if (spdk_unlikely(seq == NULL)) {
1495 0 : return -ENOMEM;
1496 : }
1497 : }
1498 :
1499 0 : assert(seq->ch == accel_ch);
1500 0 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1501 0 : if (spdk_unlikely(task == NULL)) {
1502 0 : if (*pseq == NULL) {
1503 0 : accel_sequence_put(seq);
1504 : }
1505 :
1506 0 : return -ENOMEM;
1507 : }
1508 :
1509 0 : task->s.iovs = iovs;
1510 0 : task->s.iovcnt = iovcnt;
1511 0 : task->src_domain = domain;
1512 0 : task->src_domain_ctx = domain_ctx;
1513 0 : task->dst_domain = NULL;
1514 0 : task->dif.ctx = ctx;
1515 0 : task->dif.err = err;
1516 0 : task->dif.num_blocks = num_blocks;
1517 0 : task->nbytes = num_blocks * ctx->block_size;
1518 0 : task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY;
1519 :
1520 0 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1521 0 : *pseq = seq;
1522 :
1523 0 : return 0;
1524 : }
1525 :
1526 : int
1527 0 : spdk_accel_append_dif_verify_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1528 : struct iovec *dst_iovs, size_t dst_iovcnt,
1529 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1530 : struct iovec *src_iovs, size_t src_iovcnt,
1531 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1532 : uint32_t num_blocks,
1533 : const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
1534 : spdk_accel_step_cb cb_fn, void *cb_arg)
1535 : {
1536 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1537 : struct spdk_accel_task *task;
1538 0 : struct spdk_accel_sequence *seq = *pseq;
1539 :
1540 0 : if (seq == NULL) {
1541 0 : seq = accel_sequence_get(accel_ch);
1542 0 : if (spdk_unlikely(seq == NULL)) {
1543 0 : return -ENOMEM;
1544 : }
1545 : }
1546 :
1547 0 : assert(seq->ch == accel_ch);
1548 0 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1549 0 : if (spdk_unlikely(task == NULL)) {
1550 0 : if (*pseq == NULL) {
1551 0 : accel_sequence_put(seq);
1552 : }
1553 :
1554 0 : return -ENOMEM;
1555 : }
1556 :
1557 0 : task->dst_domain = dst_domain;
1558 0 : task->dst_domain_ctx = dst_domain_ctx;
1559 0 : task->d.iovs = dst_iovs;
1560 0 : task->d.iovcnt = dst_iovcnt;
1561 0 : task->src_domain = src_domain;
1562 0 : task->src_domain_ctx = src_domain_ctx;
1563 0 : task->s.iovs = src_iovs;
1564 0 : task->s.iovcnt = src_iovcnt;
1565 0 : task->dif.ctx = ctx;
1566 0 : task->dif.err = err;
1567 0 : task->dif.num_blocks = num_blocks;
1568 0 : task->nbytes = num_blocks * ctx->block_size;
1569 0 : task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY_COPY;
1570 :
1571 0 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1572 0 : *pseq = seq;
1573 :
1574 0 : return 0;
1575 : }
1576 :
1577 : int
1578 0 : spdk_accel_append_dif_generate(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1579 : struct iovec *iovs, size_t iovcnt,
1580 : struct spdk_memory_domain *domain, void *domain_ctx,
1581 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
1582 : spdk_accel_step_cb cb_fn, void *cb_arg)
1583 : {
1584 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1585 : struct spdk_accel_task *task;
1586 0 : struct spdk_accel_sequence *seq = *pseq;
1587 :
1588 0 : if (seq == NULL) {
1589 0 : seq = accel_sequence_get(accel_ch);
1590 0 : if (spdk_unlikely(seq == NULL)) {
1591 0 : return -ENOMEM;
1592 : }
1593 : }
1594 :
1595 0 : assert(seq->ch == accel_ch);
1596 0 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1597 0 : if (spdk_unlikely(task == NULL)) {
1598 0 : if (*pseq == NULL) {
1599 0 : accel_sequence_put(seq);
1600 : }
1601 :
1602 0 : return -ENOMEM;
1603 : }
1604 :
1605 0 : task->s.iovs = iovs;
1606 0 : task->s.iovcnt = iovcnt;
1607 0 : task->src_domain = domain;
1608 0 : task->src_domain_ctx = domain_ctx;
1609 0 : task->dst_domain = NULL;
1610 0 : task->dif.ctx = ctx;
1611 0 : task->dif.num_blocks = num_blocks;
1612 0 : task->nbytes = num_blocks * ctx->block_size;
1613 0 : task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE;
1614 :
1615 0 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1616 0 : *pseq = seq;
1617 :
1618 0 : return 0;
1619 : }
1620 :
1621 : int
1622 0 : spdk_accel_append_dif_generate_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1623 : struct iovec *dst_iovs, size_t dst_iovcnt,
1624 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1625 : struct iovec *src_iovs, size_t src_iovcnt,
1626 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1627 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
1628 : spdk_accel_step_cb cb_fn, void *cb_arg)
1629 : {
1630 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1631 : struct spdk_accel_task *task;
1632 0 : struct spdk_accel_sequence *seq = *pseq;
1633 :
1634 0 : if (seq == NULL) {
1635 0 : seq = accel_sequence_get(accel_ch);
1636 0 : if (spdk_unlikely(seq == NULL)) {
1637 0 : return -ENOMEM;
1638 : }
1639 : }
1640 :
1641 0 : assert(seq->ch == accel_ch);
1642 0 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1643 0 : if (spdk_unlikely(task == NULL)) {
1644 0 : if (*pseq == NULL) {
1645 0 : accel_sequence_put(seq);
1646 : }
1647 :
1648 0 : return -ENOMEM;
1649 : }
1650 :
1651 0 : task->dst_domain = dst_domain;
1652 0 : task->dst_domain_ctx = dst_domain_ctx;
1653 0 : task->d.iovs = dst_iovs;
1654 0 : task->d.iovcnt = dst_iovcnt;
1655 0 : task->src_domain = src_domain;
1656 0 : task->src_domain_ctx = src_domain_ctx;
1657 0 : task->s.iovs = src_iovs;
1658 0 : task->s.iovcnt = src_iovcnt;
1659 0 : task->dif.ctx = ctx;
1660 0 : task->dif.num_blocks = num_blocks;
1661 0 : task->nbytes = num_blocks * ctx->block_size;
1662 0 : task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE_COPY;
1663 :
1664 0 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1665 0 : *pseq = seq;
1666 :
1667 0 : return 0;
1668 : }
1669 :
1670 : int
1671 4 : spdk_accel_append_dix_generate(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,
1672 : struct iovec *iovs, size_t iovcnt, struct spdk_memory_domain *domain,
1673 : void *domain_ctx, struct iovec *md_iov,
1674 : struct spdk_memory_domain *md_domain, void *md_domain_ctx,
1675 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
1676 : spdk_accel_step_cb cb_fn, void *cb_arg)
1677 : {
1678 4 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1679 : struct spdk_accel_task *task;
1680 4 : struct spdk_accel_sequence *pseq = *seq;
1681 :
1682 4 : if (pseq == NULL) {
1683 1 : pseq = accel_sequence_get(accel_ch);
1684 1 : if (spdk_unlikely(pseq == NULL)) {
1685 0 : return -ENOMEM;
1686 : }
1687 : }
1688 :
1689 4 : assert(pseq->ch == accel_ch);
1690 4 : task = accel_sequence_get_task(accel_ch, pseq, cb_fn, cb_arg);
1691 4 : if (spdk_unlikely(task == NULL)) {
1692 0 : if (*seq == NULL) {
1693 0 : accel_sequence_put(pseq);
1694 : }
1695 :
1696 0 : return -ENOMEM;
1697 : }
1698 :
1699 4 : task->d.iovs = md_iov;
1700 4 : task->d.iovcnt = 1;
1701 4 : task->dst_domain = md_domain;
1702 4 : task->dst_domain_ctx = md_domain_ctx;
1703 4 : task->s.iovs = iovs;
1704 4 : task->s.iovcnt = iovcnt;
1705 4 : task->src_domain = domain;
1706 4 : task->src_domain_ctx = domain_ctx;
1707 4 : task->dif.ctx = ctx;
1708 4 : task->dif.num_blocks = num_blocks;
1709 4 : task->nbytes = num_blocks * ctx->block_size;
1710 4 : task->op_code = SPDK_ACCEL_OPC_DIX_GENERATE;
1711 :
1712 4 : TAILQ_INSERT_TAIL(&pseq->tasks, task, seq_link);
1713 4 : *seq = pseq;
1714 :
1715 4 : return 0;
1716 : }
1717 :
1718 : int
1719 4 : spdk_accel_append_dix_verify(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,
1720 : struct iovec *iovs, size_t iovcnt, struct spdk_memory_domain *domain,
1721 : void *domain_ctx, struct iovec *md_iov,
1722 : struct spdk_memory_domain *md_domain, void *md_domain_ctx,
1723 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
1724 : struct spdk_dif_error *err, spdk_accel_step_cb cb_fn, void *cb_arg)
1725 : {
1726 4 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1727 : struct spdk_accel_task *task;
1728 4 : struct spdk_accel_sequence *pseq = *seq;
1729 :
1730 4 : if (pseq == NULL) {
1731 0 : pseq = accel_sequence_get(accel_ch);
1732 0 : if (spdk_unlikely(pseq == NULL)) {
1733 0 : return -ENOMEM;
1734 : }
1735 : }
1736 :
1737 4 : assert(pseq->ch == accel_ch);
1738 4 : task = accel_sequence_get_task(accel_ch, pseq, cb_fn, cb_arg);
1739 4 : if (spdk_unlikely(task == NULL)) {
1740 0 : if (*seq == NULL) {
1741 0 : accel_sequence_put(pseq);
1742 : }
1743 :
1744 0 : return -ENOMEM;
1745 : }
1746 :
1747 4 : task->d.iovs = md_iov;
1748 4 : task->d.iovcnt = 1;
1749 4 : task->dst_domain = md_domain;
1750 4 : task->dst_domain_ctx = md_domain_ctx;
1751 4 : task->s.iovs = iovs;
1752 4 : task->s.iovcnt = iovcnt;
1753 4 : task->src_domain = domain;
1754 4 : task->src_domain_ctx = domain_ctx;
1755 4 : task->dif.ctx = ctx;
1756 4 : task->dif.err = err;
1757 4 : task->dif.num_blocks = num_blocks;
1758 4 : task->nbytes = num_blocks * ctx->block_size;
1759 4 : task->op_code = SPDK_ACCEL_OPC_DIX_VERIFY;
1760 :
1761 4 : TAILQ_INSERT_TAIL(&pseq->tasks, task, seq_link);
1762 4 : *seq = pseq;
1763 :
1764 4 : return 0;
1765 : }
1766 :
1767 : int
1768 14 : spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf,
1769 : struct spdk_memory_domain **domain, void **domain_ctx)
1770 : {
1771 14 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1772 : struct accel_buffer *accel_buf;
1773 :
1774 14 : accel_buf = accel_get_buf(accel_ch, len);
1775 14 : if (spdk_unlikely(accel_buf == NULL)) {
1776 0 : return -ENOMEM;
1777 : }
1778 :
1779 14 : accel_buf->ch = accel_ch;
1780 :
1781 : /* We always return the same pointer and identify the buffers through domain_ctx */
1782 14 : *buf = ACCEL_BUFFER_BASE;
1783 14 : *domain_ctx = accel_buf;
1784 14 : *domain = g_accel_domain;
1785 :
1786 14 : return 0;
1787 : }
1788 :
1789 : void
1790 14 : spdk_accel_put_buf(struct spdk_io_channel *ch, void *buf,
1791 : struct spdk_memory_domain *domain, void *domain_ctx)
1792 : {
1793 14 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1794 14 : struct accel_buffer *accel_buf = domain_ctx;
1795 :
1796 14 : assert(domain == g_accel_domain);
1797 14 : assert(buf == ACCEL_BUFFER_BASE);
1798 :
1799 14 : accel_put_buf(accel_ch, accel_buf);
1800 14 : }
1801 :
1802 : static void
1803 168 : accel_sequence_complete_task(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
1804 : {
1805 168 : struct accel_io_channel *ch = seq->ch;
1806 : spdk_accel_step_cb cb_fn;
1807 : void *cb_arg;
1808 :
1809 168 : TAILQ_REMOVE(&seq->tasks, task, seq_link);
1810 168 : cb_fn = task->step_cb_fn;
1811 168 : cb_arg = task->cb_arg;
1812 168 : task->seq = NULL;
1813 168 : if (task->has_aux) {
1814 61 : SLIST_INSERT_HEAD(&ch->task_aux_data_pool, task->aux, link);
1815 61 : task->aux = NULL;
1816 61 : task->has_aux = false;
1817 : }
1818 :
1819 168 : _put_task(ch, task);
1820 :
1821 168 : if (cb_fn != NULL) {
1822 168 : cb_fn(cb_arg);
1823 : }
1824 168 : }
1825 :
1826 : static void
1827 67 : accel_sequence_complete_tasks(struct spdk_accel_sequence *seq)
1828 : {
1829 : struct spdk_accel_task *task;
1830 :
1831 85 : while (!TAILQ_EMPTY(&seq->tasks)) {
1832 18 : task = TAILQ_FIRST(&seq->tasks);
1833 18 : accel_sequence_complete_task(seq, task);
1834 : }
1835 67 : }
1836 :
1837 : static void
1838 65 : accel_sequence_complete(struct spdk_accel_sequence *seq)
1839 : {
1840 65 : spdk_accel_completion_cb cb_fn = seq->cb_fn;
1841 65 : void *cb_arg = seq->cb_arg;
1842 65 : int status = seq->status;
1843 :
1844 65 : SPDK_DEBUGLOG(accel, "Completed sequence: %p with status: %d\n", seq, status);
1845 :
1846 65 : accel_update_stats(seq->ch, sequence_executed, 1);
1847 65 : if (spdk_unlikely(status != 0)) {
1848 10 : accel_update_stats(seq->ch, sequence_failed, 1);
1849 : }
1850 :
1851 : /* First notify all users that appended operations to this sequence */
1852 65 : accel_sequence_complete_tasks(seq);
1853 65 : accel_sequence_put(seq);
1854 :
1855 : /* Then notify the user that finished the sequence */
1856 65 : cb_fn(cb_arg, status);
1857 65 : }
1858 :
1859 : static void
1860 28 : accel_update_virt_iov(struct iovec *diov, struct iovec *siov, struct accel_buffer *accel_buf)
1861 : {
1862 : uintptr_t offset;
1863 :
1864 28 : offset = (uintptr_t)siov->iov_base & ACCEL_BUFFER_OFFSET_MASK;
1865 28 : assert(offset < accel_buf->len);
1866 :
1867 28 : diov->iov_base = (char *)accel_buf->buf + offset;
1868 28 : diov->iov_len = siov->iov_len;
1869 28 : }
1870 :
1871 : static void
1872 10 : accel_sequence_set_virtbuf(struct spdk_accel_sequence *seq, struct accel_buffer *buf)
1873 : {
1874 : struct spdk_accel_task *task;
1875 : struct iovec *iov;
1876 :
1877 : /* Now that we've allocated the actual data buffer for this accel_buffer, update all tasks
1878 : * in a sequence that were using it.
1879 : */
1880 38 : TAILQ_FOREACH(task, &seq->tasks, seq_link) {
1881 28 : if (task->src_domain == g_accel_domain && task->src_domain_ctx == buf) {
1882 11 : if (!task->has_aux) {
1883 11 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1884 11 : assert(task->aux && "Can't allocate aux data structure");
1885 11 : task->has_aux = true;
1886 11 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1887 : }
1888 :
1889 11 : iov = &task->aux->iovs[SPDK_ACCEL_AXU_IOV_VIRT_SRC];
1890 11 : assert(task->s.iovcnt == 1);
1891 11 : accel_update_virt_iov(iov, &task->s.iovs[0], buf);
1892 11 : task->src_domain = NULL;
1893 11 : task->s.iovs = iov;
1894 : }
1895 28 : if (task->dst_domain == g_accel_domain && task->dst_domain_ctx == buf) {
1896 17 : if (!task->has_aux) {
1897 2 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1898 2 : assert(task->aux && "Can't allocate aux data structure");
1899 2 : task->has_aux = true;
1900 2 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1901 : }
1902 :
1903 17 : iov = &task->aux->iovs[SPDK_ACCEL_AXU_IOV_VIRT_DST];
1904 17 : assert(task->d.iovcnt == 1);
1905 17 : accel_update_virt_iov(iov, &task->d.iovs[0], buf);
1906 17 : task->dst_domain = NULL;
1907 17 : task->d.iovs = iov;
1908 : }
1909 : }
1910 10 : }
1911 :
1912 : static void accel_process_sequence(struct spdk_accel_sequence *seq);
1913 :
1914 : static void
1915 3 : accel_iobuf_get_virtbuf_cb(struct spdk_iobuf_entry *entry, void *buf)
1916 : {
1917 : struct accel_buffer *accel_buf;
1918 :
1919 3 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
1920 :
1921 3 : assert(accel_buf->seq != NULL);
1922 3 : assert(accel_buf->buf == NULL);
1923 3 : accel_buf->buf = buf;
1924 :
1925 3 : assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF);
1926 3 : accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF);
1927 3 : accel_sequence_set_virtbuf(accel_buf->seq, accel_buf);
1928 3 : accel_process_sequence(accel_buf->seq);
1929 3 : }
1930 :
1931 : static bool
1932 29 : accel_sequence_alloc_buf(struct spdk_accel_sequence *seq, struct accel_buffer *buf,
1933 : spdk_iobuf_get_cb cb_fn)
1934 : {
1935 29 : struct accel_io_channel *ch = seq->ch;
1936 :
1937 29 : assert(buf->seq == NULL);
1938 :
1939 29 : buf->seq = seq;
1940 :
1941 : /* Buffer might be already allocated by memory domain translation. */
1942 29 : if (buf->buf) {
1943 0 : return true;
1944 : }
1945 :
1946 29 : buf->buf = spdk_iobuf_get(&ch->iobuf, buf->len, &buf->iobuf, cb_fn);
1947 29 : if (spdk_unlikely(buf->buf == NULL)) {
1948 5 : accel_update_stats(ch, retry.iobuf, 1);
1949 5 : return false;
1950 : }
1951 :
1952 24 : return true;
1953 : }
1954 :
1955 : static bool
1956 113 : accel_sequence_check_virtbuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
1957 : {
1958 : /* If a task doesn't have dst/src (e.g. fill, crc32), its dst/src domain should be set to
1959 : * NULL */
1960 113 : if (task->src_domain == g_accel_domain) {
1961 0 : if (!accel_sequence_alloc_buf(seq, task->src_domain_ctx,
1962 : accel_iobuf_get_virtbuf_cb)) {
1963 0 : return false;
1964 : }
1965 :
1966 0 : accel_sequence_set_virtbuf(seq, task->src_domain_ctx);
1967 : }
1968 :
1969 113 : if (task->dst_domain == g_accel_domain) {
1970 10 : if (!accel_sequence_alloc_buf(seq, task->dst_domain_ctx,
1971 : accel_iobuf_get_virtbuf_cb)) {
1972 3 : return false;
1973 : }
1974 :
1975 7 : accel_sequence_set_virtbuf(seq, task->dst_domain_ctx);
1976 : }
1977 :
1978 110 : return true;
1979 : }
1980 :
1981 : static void
1982 0 : accel_sequence_get_buf_cb(struct spdk_iobuf_entry *entry, void *buf)
1983 : {
1984 : struct accel_buffer *accel_buf;
1985 :
1986 0 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
1987 :
1988 0 : assert(accel_buf->seq != NULL);
1989 0 : assert(accel_buf->buf == NULL);
1990 0 : accel_buf->buf = buf;
1991 :
1992 0 : accel_sequence_set_virtbuf(accel_buf->seq, accel_buf);
1993 0 : accel_buf->cb_fn(accel_buf->seq, accel_buf->cb_ctx);
1994 0 : }
1995 :
1996 : bool
1997 0 : spdk_accel_alloc_sequence_buf(struct spdk_accel_sequence *seq, void *buf,
1998 : struct spdk_memory_domain *domain, void *domain_ctx,
1999 : spdk_accel_sequence_get_buf_cb cb_fn, void *cb_ctx)
2000 : {
2001 0 : struct accel_buffer *accel_buf = domain_ctx;
2002 :
2003 0 : assert(domain == g_accel_domain);
2004 0 : accel_buf->cb_fn = cb_fn;
2005 0 : accel_buf->cb_ctx = cb_ctx;
2006 :
2007 0 : if (!accel_sequence_alloc_buf(seq, accel_buf, accel_sequence_get_buf_cb)) {
2008 0 : return false;
2009 : }
2010 :
2011 0 : accel_sequence_set_virtbuf(seq, accel_buf);
2012 :
2013 0 : return true;
2014 : }
2015 :
2016 : struct spdk_accel_task *
2017 24 : spdk_accel_sequence_first_task(struct spdk_accel_sequence *seq)
2018 : {
2019 24 : return TAILQ_FIRST(&seq->tasks);
2020 : }
2021 :
2022 : struct spdk_accel_task *
2023 0 : spdk_accel_sequence_next_task(struct spdk_accel_task *task)
2024 : {
2025 0 : return TAILQ_NEXT(task, seq_link);
2026 : }
2027 :
2028 : static inline void
2029 19 : accel_set_bounce_buffer(struct spdk_accel_bounce_buffer *bounce, struct iovec **iovs,
2030 : uint32_t *iovcnt, struct spdk_memory_domain **domain, void **domain_ctx,
2031 : struct accel_buffer *buf)
2032 : {
2033 19 : bounce->orig_iovs = *iovs;
2034 19 : bounce->orig_iovcnt = *iovcnt;
2035 19 : bounce->orig_domain = *domain;
2036 19 : bounce->orig_domain_ctx = *domain_ctx;
2037 19 : bounce->iov.iov_base = buf->buf;
2038 19 : bounce->iov.iov_len = buf->len;
2039 :
2040 19 : *iovs = &bounce->iov;
2041 19 : *iovcnt = 1;
2042 19 : *domain = NULL;
2043 19 : }
2044 :
2045 : static void
2046 1 : accel_iobuf_get_src_bounce_cb(struct spdk_iobuf_entry *entry, void *buf)
2047 : {
2048 : struct spdk_accel_task *task;
2049 : struct accel_buffer *accel_buf;
2050 :
2051 1 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
2052 1 : assert(accel_buf->buf == NULL);
2053 1 : accel_buf->buf = buf;
2054 :
2055 1 : task = TAILQ_FIRST(&accel_buf->seq->tasks);
2056 1 : assert(task != NULL);
2057 :
2058 1 : assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
2059 1 : accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
2060 1 : assert(task->aux);
2061 1 : assert(task->has_aux);
2062 1 : accel_set_bounce_buffer(&task->aux->bounce.s, &task->s.iovs, &task->s.iovcnt, &task->src_domain,
2063 : &task->src_domain_ctx, accel_buf);
2064 1 : accel_process_sequence(accel_buf->seq);
2065 1 : }
2066 :
2067 : static void
2068 1 : accel_iobuf_get_dst_bounce_cb(struct spdk_iobuf_entry *entry, void *buf)
2069 : {
2070 : struct spdk_accel_task *task;
2071 : struct accel_buffer *accel_buf;
2072 :
2073 1 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
2074 1 : assert(accel_buf->buf == NULL);
2075 1 : accel_buf->buf = buf;
2076 :
2077 1 : task = TAILQ_FIRST(&accel_buf->seq->tasks);
2078 1 : assert(task != NULL);
2079 :
2080 1 : assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
2081 1 : accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
2082 1 : assert(task->aux);
2083 1 : assert(task->has_aux);
2084 1 : accel_set_bounce_buffer(&task->aux->bounce.d, &task->d.iovs, &task->d.iovcnt, &task->dst_domain,
2085 : &task->dst_domain_ctx, accel_buf);
2086 1 : accel_process_sequence(accel_buf->seq);
2087 1 : }
2088 :
2089 : static int
2090 97 : accel_sequence_check_bouncebuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
2091 : {
2092 : struct accel_buffer *buf;
2093 :
2094 97 : if (task->src_domain != NULL) {
2095 : /* By the time we're here, accel buffers should have been allocated */
2096 9 : assert(task->src_domain != g_accel_domain);
2097 :
2098 9 : if (!task->has_aux) {
2099 8 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
2100 8 : if (spdk_unlikely(!task->aux)) {
2101 0 : SPDK_ERRLOG("Can't allocate aux data structure\n");
2102 0 : assert(0);
2103 : return -EAGAIN;
2104 : }
2105 8 : task->has_aux = true;
2106 8 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
2107 : }
2108 9 : buf = accel_get_buf(seq->ch, accel_get_iovlen(task->s.iovs, task->s.iovcnt));
2109 9 : if (buf == NULL) {
2110 0 : SPDK_ERRLOG("Couldn't allocate buffer descriptor\n");
2111 0 : return -ENOMEM;
2112 : }
2113 :
2114 9 : SLIST_INSERT_HEAD(&seq->bounce_bufs, buf, link);
2115 9 : if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_src_bounce_cb)) {
2116 1 : return -EAGAIN;
2117 : }
2118 :
2119 8 : accel_set_bounce_buffer(&task->aux->bounce.s, &task->s.iovs, &task->s.iovcnt,
2120 : &task->src_domain, &task->src_domain_ctx, buf);
2121 : }
2122 :
2123 96 : if (task->dst_domain != NULL) {
2124 : /* By the time we're here, accel buffers should have been allocated */
2125 10 : assert(task->dst_domain != g_accel_domain);
2126 :
2127 10 : if (!task->has_aux) {
2128 0 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
2129 0 : if (spdk_unlikely(!task->aux)) {
2130 0 : SPDK_ERRLOG("Can't allocate aux data structure\n");
2131 0 : assert(0);
2132 : return -EAGAIN;
2133 : }
2134 0 : task->has_aux = true;
2135 0 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
2136 : }
2137 10 : buf = accel_get_buf(seq->ch, accel_get_iovlen(task->d.iovs, task->d.iovcnt));
2138 10 : if (buf == NULL) {
2139 : /* The src buffer will be released when a sequence is completed */
2140 0 : SPDK_ERRLOG("Couldn't allocate buffer descriptor\n");
2141 0 : return -ENOMEM;
2142 : }
2143 :
2144 10 : SLIST_INSERT_HEAD(&seq->bounce_bufs, buf, link);
2145 10 : if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_dst_bounce_cb)) {
2146 1 : return -EAGAIN;
2147 : }
2148 :
2149 9 : accel_set_bounce_buffer(&task->aux->bounce.d, &task->d.iovs, &task->d.iovcnt,
2150 : &task->dst_domain, &task->dst_domain_ctx, buf);
2151 : }
2152 :
2153 95 : return 0;
2154 : }
2155 :
2156 : static void
2157 8 : accel_task_pull_data_cb(void *ctx, int status)
2158 : {
2159 8 : struct spdk_accel_sequence *seq = ctx;
2160 :
2161 8 : assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA);
2162 8 : if (spdk_likely(status == 0)) {
2163 7 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
2164 : } else {
2165 1 : accel_sequence_set_fail(seq, status);
2166 : }
2167 :
2168 8 : accel_process_sequence(seq);
2169 8 : }
2170 :
2171 : static void
2172 9 : accel_task_pull_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
2173 : {
2174 : int rc;
2175 :
2176 9 : assert(task->has_aux);
2177 9 : assert(task->aux);
2178 9 : assert(task->aux->bounce.s.orig_iovs != NULL);
2179 9 : assert(task->aux->bounce.s.orig_domain != NULL);
2180 9 : assert(task->aux->bounce.s.orig_domain != g_accel_domain);
2181 9 : assert(!g_modules_opc[task->op_code].supports_memory_domains);
2182 :
2183 36 : rc = spdk_memory_domain_pull_data(task->aux->bounce.s.orig_domain,
2184 9 : task->aux->bounce.s.orig_domain_ctx,
2185 18 : task->aux->bounce.s.orig_iovs, task->aux->bounce.s.orig_iovcnt,
2186 : task->s.iovs, task->s.iovcnt,
2187 : accel_task_pull_data_cb, seq);
2188 9 : if (spdk_unlikely(rc != 0)) {
2189 1 : SPDK_ERRLOG("Failed to pull data from memory domain: %s, rc: %d\n",
2190 : spdk_memory_domain_get_dma_device_id(task->aux->bounce.s.orig_domain), rc);
2191 1 : accel_sequence_set_fail(seq, rc);
2192 : }
2193 9 : }
2194 :
2195 : static void
2196 7 : accel_task_push_data_cb(void *ctx, int status)
2197 : {
2198 7 : struct spdk_accel_sequence *seq = ctx;
2199 :
2200 7 : assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA);
2201 7 : if (spdk_likely(status == 0)) {
2202 6 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK);
2203 : } else {
2204 1 : accel_sequence_set_fail(seq, status);
2205 : }
2206 :
2207 7 : accel_process_sequence(seq);
2208 7 : }
2209 :
2210 : static void
2211 8 : accel_task_push_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
2212 : {
2213 : int rc;
2214 :
2215 8 : assert(task->has_aux);
2216 8 : assert(task->aux);
2217 8 : assert(task->aux->bounce.d.orig_iovs != NULL);
2218 8 : assert(task->aux->bounce.d.orig_domain != NULL);
2219 8 : assert(task->aux->bounce.d.orig_domain != g_accel_domain);
2220 8 : assert(!g_modules_opc[task->op_code].supports_memory_domains);
2221 :
2222 32 : rc = spdk_memory_domain_push_data(task->aux->bounce.d.orig_domain,
2223 8 : task->aux->bounce.d.orig_domain_ctx,
2224 16 : task->aux->bounce.d.orig_iovs, task->aux->bounce.d.orig_iovcnt,
2225 : task->d.iovs, task->d.iovcnt,
2226 : accel_task_push_data_cb, seq);
2227 8 : if (spdk_unlikely(rc != 0)) {
2228 1 : SPDK_ERRLOG("Failed to push data to memory domain: %s, rc: %d\n",
2229 : spdk_memory_domain_get_dma_device_id(task->aux->bounce.s.orig_domain), rc);
2230 1 : accel_sequence_set_fail(seq, rc);
2231 : }
2232 8 : }
2233 :
2234 : static void
2235 203 : accel_process_sequence(struct spdk_accel_sequence *seq)
2236 : {
2237 203 : struct accel_io_channel *accel_ch = seq->ch;
2238 : struct spdk_accel_task *task;
2239 : enum accel_sequence_state state;
2240 : int rc;
2241 :
2242 : /* Prevent recursive calls to this function */
2243 203 : if (spdk_unlikely(seq->in_process_sequence)) {
2244 78 : return;
2245 : }
2246 125 : seq->in_process_sequence = true;
2247 :
2248 125 : task = TAILQ_FIRST(&seq->tasks);
2249 : do {
2250 467 : state = seq->state;
2251 467 : switch (state) {
2252 115 : case ACCEL_SEQUENCE_STATE_INIT:
2253 115 : if (g_accel_driver != NULL) {
2254 13 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS);
2255 13 : break;
2256 : }
2257 : /* Fall through */
2258 : case ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF:
2259 113 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF);
2260 113 : if (!accel_sequence_check_virtbuf(seq, task)) {
2261 : /* We couldn't allocate a buffer, wait until one is available */
2262 3 : break;
2263 : }
2264 110 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
2265 : /* Fall through */
2266 112 : case ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF:
2267 : /* If a module supports memory domains, we don't need to allocate bounce
2268 : * buffers */
2269 112 : if (g_modules_opc[task->op_code].supports_memory_domains) {
2270 15 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
2271 15 : break;
2272 : }
2273 97 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
2274 97 : rc = accel_sequence_check_bouncebuf(seq, task);
2275 97 : if (spdk_unlikely(rc != 0)) {
2276 : /* We couldn't allocate a buffer, wait until one is available */
2277 2 : if (rc == -EAGAIN) {
2278 2 : break;
2279 : }
2280 0 : accel_sequence_set_fail(seq, rc);
2281 0 : break;
2282 : }
2283 95 : if (task->has_aux && task->s.iovs == &task->aux->bounce.s.iov) {
2284 9 : assert(task->aux->bounce.s.orig_iovs);
2285 9 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PULL_DATA);
2286 9 : break;
2287 : }
2288 86 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
2289 : /* Fall through */
2290 108 : case ACCEL_SEQUENCE_STATE_EXEC_TASK:
2291 108 : SPDK_DEBUGLOG(accel, "Executing %s operation, sequence: %p\n",
2292 : g_opcode_strings[task->op_code], seq);
2293 :
2294 108 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_TASK);
2295 108 : rc = accel_submit_task(accel_ch, task);
2296 108 : if (spdk_unlikely(rc != 0)) {
2297 2 : SPDK_ERRLOG("Failed to submit %s operation, sequence: %p\n",
2298 : g_opcode_strings[task->op_code], seq);
2299 2 : accel_sequence_set_fail(seq, rc);
2300 : }
2301 108 : break;
2302 9 : case ACCEL_SEQUENCE_STATE_PULL_DATA:
2303 9 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA);
2304 9 : accel_task_pull_data(seq, task);
2305 9 : break;
2306 104 : case ACCEL_SEQUENCE_STATE_COMPLETE_TASK:
2307 104 : if (task->has_aux && task->d.iovs == &task->aux->bounce.d.iov) {
2308 8 : assert(task->aux->bounce.d.orig_iovs);
2309 8 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PUSH_DATA);
2310 8 : break;
2311 : }
2312 96 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK);
2313 96 : break;
2314 8 : case ACCEL_SEQUENCE_STATE_PUSH_DATA:
2315 8 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA);
2316 8 : accel_task_push_data(seq, task);
2317 8 : break;
2318 102 : case ACCEL_SEQUENCE_STATE_NEXT_TASK:
2319 102 : accel_sequence_complete_task(seq, task);
2320 : /* Check if there are any remaining tasks */
2321 102 : task = TAILQ_FIRST(&seq->tasks);
2322 102 : if (task == NULL) {
2323 : /* Immediately return here to make sure we don't touch the sequence
2324 : * after it's completed */
2325 52 : accel_sequence_complete(seq);
2326 52 : return;
2327 : }
2328 50 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_INIT);
2329 50 : break;
2330 13 : case ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS:
2331 13 : assert(!TAILQ_EMPTY(&seq->tasks));
2332 :
2333 13 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS);
2334 13 : rc = g_accel_driver->execute_sequence(accel_ch->driver_channel, seq);
2335 13 : if (spdk_unlikely(rc != 0)) {
2336 1 : SPDK_ERRLOG("Failed to execute sequence: %p using driver: %s\n",
2337 : seq, g_accel_driver->name);
2338 1 : accel_sequence_set_fail(seq, rc);
2339 : }
2340 13 : break;
2341 11 : case ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS:
2342 : /* Get the task again, as the driver might have completed some tasks
2343 : * synchronously */
2344 11 : task = TAILQ_FIRST(&seq->tasks);
2345 11 : if (task == NULL) {
2346 : /* Immediately return here to make sure we don't touch the sequence
2347 : * after it's completed */
2348 3 : accel_sequence_complete(seq);
2349 3 : return;
2350 : }
2351 : /* We don't want to execute the next task through the driver, so we
2352 : * explicitly omit the INIT state here */
2353 8 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF);
2354 8 : break;
2355 10 : case ACCEL_SEQUENCE_STATE_ERROR:
2356 : /* Immediately return here to make sure we don't touch the sequence
2357 : * after it's completed */
2358 10 : assert(seq->status != 0);
2359 10 : accel_sequence_complete(seq);
2360 10 : return;
2361 60 : case ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF:
2362 : case ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF:
2363 : case ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA:
2364 : case ACCEL_SEQUENCE_STATE_AWAIT_TASK:
2365 : case ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA:
2366 : case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS:
2367 60 : break;
2368 0 : default:
2369 0 : assert(0 && "bad state");
2370 : break;
2371 : }
2372 402 : } while (seq->state != state);
2373 :
2374 60 : seq->in_process_sequence = false;
2375 : }
2376 :
2377 : static void
2378 118 : accel_sequence_task_cb(struct spdk_accel_sequence *seq, struct spdk_accel_task *task, int status)
2379 : {
2380 118 : switch (seq->state) {
2381 106 : case ACCEL_SEQUENCE_STATE_AWAIT_TASK:
2382 106 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_COMPLETE_TASK);
2383 106 : if (spdk_unlikely(status != 0)) {
2384 2 : SPDK_ERRLOG("Failed to execute %s operation, sequence: %p\n",
2385 : g_opcode_strings[task->op_code], seq);
2386 2 : accel_sequence_set_fail(seq, status);
2387 : }
2388 :
2389 106 : accel_process_sequence(seq);
2390 106 : break;
2391 12 : case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS:
2392 12 : assert(g_accel_driver != NULL);
2393 : /* Immediately remove the task from the outstanding list to make sure the next call
2394 : * to spdk_accel_sequence_first_task() doesn't return it */
2395 12 : accel_sequence_complete_task(seq, task);
2396 12 : if (spdk_unlikely(status != 0)) {
2397 1 : SPDK_ERRLOG("Failed to execute %s operation, sequence: %p through "
2398 : "driver: %s\n", g_opcode_strings[task->op_code], seq,
2399 : g_accel_driver->name);
2400 : /* Update status without using accel_sequence_set_fail() to avoid changing
2401 : * seq's state to ERROR until driver calls spdk_accel_sequence_continue() */
2402 1 : seq->status = status;
2403 : }
2404 12 : break;
2405 0 : default:
2406 0 : assert(0 && "bad state");
2407 : break;
2408 : }
2409 118 : }
2410 :
2411 : void
2412 12 : spdk_accel_sequence_continue(struct spdk_accel_sequence *seq)
2413 : {
2414 12 : assert(g_accel_driver != NULL);
2415 12 : assert(seq->state == ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS);
2416 :
2417 12 : if (spdk_likely(seq->status == 0)) {
2418 11 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS);
2419 : } else {
2420 1 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR);
2421 : }
2422 :
2423 12 : accel_process_sequence(seq);
2424 12 : }
2425 :
2426 : static bool
2427 43 : accel_compare_iovs(struct iovec *iova, uint32_t iovacnt, struct iovec *iovb, uint32_t iovbcnt)
2428 : {
2429 : /* For now, just do a dumb check that the iovecs arrays are exactly the same */
2430 43 : if (iovacnt != iovbcnt) {
2431 0 : return false;
2432 : }
2433 :
2434 43 : return memcmp(iova, iovb, sizeof(*iova) * iovacnt) == 0;
2435 : }
2436 :
2437 : static bool
2438 30 : accel_task_set_dstbuf(struct spdk_accel_task *task, struct spdk_accel_task *next)
2439 : {
2440 : struct spdk_accel_task *prev;
2441 :
2442 30 : switch (task->op_code) {
2443 24 : case SPDK_ACCEL_OPC_DECOMPRESS:
2444 : case SPDK_ACCEL_OPC_FILL:
2445 : case SPDK_ACCEL_OPC_ENCRYPT:
2446 : case SPDK_ACCEL_OPC_DECRYPT:
2447 : case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:
2448 : case SPDK_ACCEL_OPC_DIF_VERIFY_COPY:
2449 24 : if (task->dst_domain != next->src_domain) {
2450 0 : return false;
2451 : }
2452 24 : if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt,
2453 : next->s.iovs, next->s.iovcnt)) {
2454 1 : return false;
2455 : }
2456 23 : task->d.iovs = next->d.iovs;
2457 23 : task->d.iovcnt = next->d.iovcnt;
2458 23 : task->dst_domain = next->dst_domain;
2459 23 : task->dst_domain_ctx = next->dst_domain_ctx;
2460 23 : break;
2461 6 : case SPDK_ACCEL_OPC_CRC32C:
2462 : case SPDK_ACCEL_OPC_DIX_GENERATE:
2463 : case SPDK_ACCEL_OPC_DIX_VERIFY:
2464 : /* crc32 and dix_generate/verify are special, because they do not have a dst buffer */
2465 6 : if (task->src_domain != next->src_domain) {
2466 0 : return false;
2467 : }
2468 6 : if (!accel_compare_iovs(task->s.iovs, task->s.iovcnt,
2469 : next->s.iovs, next->s.iovcnt)) {
2470 1 : return false;
2471 : }
2472 : /* We can only change operation's buffer if we can change previous task's buffer */
2473 5 : prev = TAILQ_PREV(task, accel_sequence_tasks, seq_link);
2474 5 : if (prev == NULL) {
2475 1 : return false;
2476 : }
2477 4 : if (!accel_task_set_dstbuf(prev, next)) {
2478 0 : return false;
2479 : }
2480 4 : task->s.iovs = next->d.iovs;
2481 4 : task->s.iovcnt = next->d.iovcnt;
2482 4 : task->src_domain = next->dst_domain;
2483 4 : task->src_domain_ctx = next->dst_domain_ctx;
2484 4 : break;
2485 0 : default:
2486 0 : return false;
2487 : }
2488 :
2489 27 : return true;
2490 : }
2491 :
2492 : static void
2493 96 : accel_sequence_merge_tasks(struct spdk_accel_sequence *seq, struct spdk_accel_task *task,
2494 : struct spdk_accel_task **next_task)
2495 : {
2496 96 : struct spdk_accel_task *next = *next_task;
2497 :
2498 96 : switch (task->op_code) {
2499 15 : case SPDK_ACCEL_OPC_COPY:
2500 : /* We only allow changing src of operations that actually have a src, e.g. we never
2501 : * do it for fill. Theoretically, it is possible, but we'd have to be careful to
2502 : * change the src of the operation after fill (which in turn could also be a fill).
2503 : * So, for the sake of simplicity, skip this type of operations for now.
2504 : */
2505 15 : if (next->op_code != SPDK_ACCEL_OPC_DECOMPRESS &&
2506 9 : next->op_code != SPDK_ACCEL_OPC_COPY &&
2507 8 : next->op_code != SPDK_ACCEL_OPC_ENCRYPT &&
2508 5 : next->op_code != SPDK_ACCEL_OPC_DECRYPT &&
2509 2 : next->op_code != SPDK_ACCEL_OPC_COPY_CRC32C &&
2510 2 : next->op_code != SPDK_ACCEL_OPC_DIF_GENERATE_COPY &&
2511 2 : next->op_code != SPDK_ACCEL_OPC_DIF_VERIFY_COPY) {
2512 2 : break;
2513 : }
2514 13 : if (task->dst_domain != next->src_domain) {
2515 0 : break;
2516 : }
2517 13 : if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt,
2518 : next->s.iovs, next->s.iovcnt)) {
2519 0 : break;
2520 : }
2521 13 : next->s.iovs = task->s.iovs;
2522 13 : next->s.iovcnt = task->s.iovcnt;
2523 13 : next->src_domain = task->src_domain;
2524 13 : next->src_domain_ctx = task->src_domain_ctx;
2525 13 : accel_sequence_complete_task(seq, task);
2526 13 : break;
2527 81 : case SPDK_ACCEL_OPC_DECOMPRESS:
2528 : case SPDK_ACCEL_OPC_FILL:
2529 : case SPDK_ACCEL_OPC_ENCRYPT:
2530 : case SPDK_ACCEL_OPC_DECRYPT:
2531 : case SPDK_ACCEL_OPC_CRC32C:
2532 : case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:
2533 : case SPDK_ACCEL_OPC_DIF_VERIFY_COPY:
2534 : case SPDK_ACCEL_OPC_DIX_GENERATE:
2535 : case SPDK_ACCEL_OPC_DIX_VERIFY:
2536 : /* We can only merge tasks when one of them is a copy */
2537 81 : if (next->op_code != SPDK_ACCEL_OPC_COPY) {
2538 55 : break;
2539 : }
2540 26 : if (!accel_task_set_dstbuf(task, next)) {
2541 3 : break;
2542 : }
2543 : /* We're removing next_task from the tasks queue, so we need to update its pointer,
2544 : * so that the TAILQ_FOREACH_SAFE() loop below works correctly */
2545 23 : *next_task = TAILQ_NEXT(next, seq_link);
2546 23 : accel_sequence_complete_task(seq, next);
2547 23 : break;
2548 0 : default:
2549 0 : assert(0 && "bad opcode");
2550 : break;
2551 : }
2552 96 : }
2553 :
2554 : void
2555 65 : spdk_accel_sequence_finish(struct spdk_accel_sequence *seq,
2556 : spdk_accel_completion_cb cb_fn, void *cb_arg)
2557 : {
2558 65 : struct spdk_accel_task *task, *next;
2559 :
2560 : /* Try to remove any copy operations if possible */
2561 161 : TAILQ_FOREACH_SAFE(task, &seq->tasks, seq_link, next) {
2562 140 : if (next == NULL) {
2563 44 : break;
2564 : }
2565 96 : accel_sequence_merge_tasks(seq, task, &next);
2566 : }
2567 :
2568 65 : seq->cb_fn = cb_fn;
2569 65 : seq->cb_arg = cb_arg;
2570 :
2571 65 : accel_process_sequence(seq);
2572 65 : }
2573 :
2574 : void
2575 5 : spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq)
2576 : {
2577 5 : struct accel_sequence_tasks tasks = TAILQ_HEAD_INITIALIZER(tasks);
2578 : struct spdk_accel_task *task;
2579 :
2580 5 : TAILQ_SWAP(&tasks, &seq->tasks, spdk_accel_task, seq_link);
2581 :
2582 17 : while (!TAILQ_EMPTY(&tasks)) {
2583 12 : task = TAILQ_FIRST(&tasks);
2584 12 : TAILQ_REMOVE(&tasks, task, seq_link);
2585 12 : TAILQ_INSERT_HEAD(&seq->tasks, task, seq_link);
2586 : }
2587 5 : }
2588 :
2589 : void
2590 3 : spdk_accel_sequence_abort(struct spdk_accel_sequence *seq)
2591 : {
2592 3 : if (seq == NULL) {
2593 1 : return;
2594 : }
2595 :
2596 2 : accel_sequence_complete_tasks(seq);
2597 2 : accel_sequence_put(seq);
2598 : }
2599 :
2600 : struct spdk_memory_domain *
2601 0 : spdk_accel_get_memory_domain(void)
2602 : {
2603 0 : return g_accel_domain;
2604 : }
2605 :
2606 : static struct spdk_accel_module_if *
2607 7 : _module_find_by_name(const char *name)
2608 : {
2609 7 : struct spdk_accel_module_if *accel_module = NULL;
2610 :
2611 16 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
2612 10 : if (strcmp(name, accel_module->name) == 0) {
2613 1 : break;
2614 : }
2615 : }
2616 :
2617 7 : return accel_module;
2618 : }
2619 :
2620 : static inline struct spdk_accel_crypto_key *
2621 6 : _accel_crypto_key_get(const char *name)
2622 : {
2623 : struct spdk_accel_crypto_key *key;
2624 :
2625 6 : assert(spdk_spin_held(&g_keyring_spin));
2626 :
2627 6 : TAILQ_FOREACH(key, &g_keyring, link) {
2628 4 : if (strcmp(name, key->param.key_name) == 0) {
2629 4 : return key;
2630 : }
2631 : }
2632 :
2633 2 : return NULL;
2634 : }
2635 :
2636 : static void
2637 2 : accel_crypto_key_free_mem(struct spdk_accel_crypto_key *key)
2638 : {
2639 2 : if (key->param.hex_key) {
2640 2 : spdk_memset_s(key->param.hex_key, key->key_size * 2, 0, key->key_size * 2);
2641 2 : free(key->param.hex_key);
2642 : }
2643 2 : if (key->param.hex_key2) {
2644 2 : spdk_memset_s(key->param.hex_key2, key->key2_size * 2, 0, key->key2_size * 2);
2645 2 : free(key->param.hex_key2);
2646 : }
2647 2 : free(key->param.tweak_mode);
2648 2 : free(key->param.key_name);
2649 2 : free(key->param.cipher);
2650 2 : if (key->key) {
2651 2 : spdk_memset_s(key->key, key->key_size, 0, key->key_size);
2652 2 : free(key->key);
2653 : }
2654 2 : if (key->key2) {
2655 2 : spdk_memset_s(key->key2, key->key2_size, 0, key->key2_size);
2656 2 : free(key->key2);
2657 : }
2658 2 : free(key);
2659 2 : }
2660 :
2661 : static void
2662 2 : accel_crypto_key_destroy_unsafe(struct spdk_accel_crypto_key *key)
2663 : {
2664 2 : assert(key->module_if);
2665 2 : assert(key->module_if->crypto_key_deinit);
2666 :
2667 2 : key->module_if->crypto_key_deinit(key);
2668 2 : accel_crypto_key_free_mem(key);
2669 2 : }
2670 :
2671 : /*
2672 : * This function mitigates a timing side channel which could be caused by using strcmp()
2673 : * Please refer to chapter "Mitigating Information Leakage Based on Variable Timing" in
2674 : * the article [1] for more details
2675 : * [1] https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/secure-coding/mitigate-timing-side-channel-crypto-implementation.html
2676 : */
2677 : static bool
2678 2 : accel_aes_xts_keys_equal(const char *k1, size_t k1_len, const char *k2, size_t k2_len)
2679 : {
2680 : size_t i;
2681 2 : volatile size_t x = k1_len ^ k2_len;
2682 :
2683 34 : for (i = 0; ((i < k1_len) & (i < k2_len)); i++) {
2684 32 : x |= k1[i] ^ k2[i];
2685 : }
2686 :
2687 2 : return x == 0;
2688 : }
2689 :
2690 : static const char *g_tweak_modes[] = {
2691 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA] = "SIMPLE_LBA",
2692 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_JOIN_NEG_LBA_WITH_LBA] = "JOIN_NEG_LBA_WITH_LBA",
2693 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_FULL_LBA] = "INCR_512_FULL_LBA",
2694 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_UPPER_LBA] = "INCR_512_UPPER_LBA",
2695 : };
2696 :
2697 : static const char *g_ciphers[] = {
2698 : [SPDK_ACCEL_CIPHER_AES_CBC] = "AES_CBC",
2699 : [SPDK_ACCEL_CIPHER_AES_XTS] = "AES_XTS",
2700 : };
2701 :
2702 : int
2703 2 : spdk_accel_crypto_key_create(const struct spdk_accel_crypto_key_create_param *param)
2704 : {
2705 : struct spdk_accel_module_if *module;
2706 : struct spdk_accel_crypto_key *key;
2707 : size_t hex_key_size, hex_key2_size;
2708 2 : bool found = false;
2709 : size_t i;
2710 : int rc;
2711 :
2712 2 : if (!param || !param->hex_key || !param->cipher || !param->key_name) {
2713 0 : return -EINVAL;
2714 : }
2715 :
2716 2 : if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) {
2717 : /* hardly ever possible, but let's check and warn the user */
2718 0 : SPDK_ERRLOG("Different accel modules are used for encryption and decryption\n");
2719 : }
2720 2 : module = g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module;
2721 :
2722 2 : if (!module) {
2723 0 : SPDK_ERRLOG("No accel module found assigned for crypto operation\n");
2724 0 : return -ENOENT;
2725 : }
2726 :
2727 2 : if (!module->crypto_key_init || !module->crypto_supports_cipher) {
2728 0 : SPDK_ERRLOG("Module %s doesn't support crypto operations\n", module->name);
2729 0 : return -ENOTSUP;
2730 : }
2731 :
2732 2 : key = calloc(1, sizeof(*key));
2733 2 : if (!key) {
2734 0 : return -ENOMEM;
2735 : }
2736 :
2737 2 : key->param.key_name = strdup(param->key_name);
2738 2 : if (!key->param.key_name) {
2739 0 : rc = -ENOMEM;
2740 0 : goto error;
2741 : }
2742 :
2743 4 : for (i = 0; i < SPDK_COUNTOF(g_ciphers); ++i) {
2744 4 : assert(g_ciphers[i]);
2745 :
2746 4 : if (strncmp(param->cipher, g_ciphers[i], strlen(g_ciphers[i])) == 0) {
2747 2 : key->cipher = i;
2748 2 : found = true;
2749 2 : break;
2750 : }
2751 : }
2752 :
2753 2 : if (!found) {
2754 0 : SPDK_ERRLOG("Failed to parse cipher\n");
2755 0 : rc = -EINVAL;
2756 0 : goto error;
2757 : }
2758 :
2759 2 : key->param.cipher = strdup(param->cipher);
2760 2 : if (!key->param.cipher) {
2761 0 : rc = -ENOMEM;
2762 0 : goto error;
2763 : }
2764 :
2765 2 : hex_key_size = strnlen(param->hex_key, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2766 2 : if (hex_key_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) {
2767 0 : SPDK_ERRLOG("key1 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2768 0 : rc = -EINVAL;
2769 0 : goto error;
2770 : }
2771 :
2772 2 : if (hex_key_size == 0) {
2773 0 : SPDK_ERRLOG("key1 size cannot be 0\n");
2774 0 : rc = -EINVAL;
2775 0 : goto error;
2776 : }
2777 :
2778 2 : key->param.hex_key = strdup(param->hex_key);
2779 2 : if (!key->param.hex_key) {
2780 0 : rc = -ENOMEM;
2781 0 : goto error;
2782 : }
2783 :
2784 2 : key->key_size = hex_key_size / 2;
2785 2 : key->key = spdk_unhexlify(key->param.hex_key);
2786 2 : if (!key->key) {
2787 0 : SPDK_ERRLOG("Failed to unhexlify key1\n");
2788 0 : rc = -EINVAL;
2789 0 : goto error;
2790 : }
2791 :
2792 2 : if (param->hex_key2) {
2793 2 : hex_key2_size = strnlen(param->hex_key2, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2794 2 : if (hex_key2_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) {
2795 0 : SPDK_ERRLOG("key2 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2796 0 : rc = -EINVAL;
2797 0 : goto error;
2798 : }
2799 :
2800 2 : if (hex_key2_size == 0) {
2801 0 : SPDK_ERRLOG("key2 size cannot be 0\n");
2802 0 : rc = -EINVAL;
2803 0 : goto error;
2804 : }
2805 :
2806 2 : key->param.hex_key2 = strdup(param->hex_key2);
2807 2 : if (!key->param.hex_key2) {
2808 0 : rc = -ENOMEM;
2809 0 : goto error;
2810 : }
2811 :
2812 2 : key->key2_size = hex_key2_size / 2;
2813 2 : key->key2 = spdk_unhexlify(key->param.hex_key2);
2814 2 : if (!key->key2) {
2815 0 : SPDK_ERRLOG("Failed to unhexlify key2\n");
2816 0 : rc = -EINVAL;
2817 0 : goto error;
2818 : }
2819 : }
2820 :
2821 2 : key->tweak_mode = ACCEL_CRYPTO_TWEAK_MODE_DEFAULT;
2822 2 : if (param->tweak_mode) {
2823 0 : found = false;
2824 :
2825 0 : key->param.tweak_mode = strdup(param->tweak_mode);
2826 0 : if (!key->param.tweak_mode) {
2827 0 : rc = -ENOMEM;
2828 0 : goto error;
2829 : }
2830 :
2831 0 : for (i = 0; i < SPDK_COUNTOF(g_tweak_modes); ++i) {
2832 0 : assert(g_tweak_modes[i]);
2833 :
2834 0 : if (strncmp(param->tweak_mode, g_tweak_modes[i], strlen(g_tweak_modes[i])) == 0) {
2835 0 : key->tweak_mode = i;
2836 0 : found = true;
2837 0 : break;
2838 : }
2839 : }
2840 :
2841 0 : if (!found) {
2842 0 : SPDK_ERRLOG("Failed to parse tweak mode\n");
2843 0 : rc = -EINVAL;
2844 0 : goto error;
2845 : }
2846 : }
2847 :
2848 2 : if ((!module->crypto_supports_tweak_mode && key->tweak_mode != ACCEL_CRYPTO_TWEAK_MODE_DEFAULT) ||
2849 2 : (module->crypto_supports_tweak_mode && !module->crypto_supports_tweak_mode(key->tweak_mode))) {
2850 0 : SPDK_ERRLOG("Module %s doesn't support %s tweak mode\n", module->name,
2851 : g_tweak_modes[key->tweak_mode]);
2852 0 : rc = -EINVAL;
2853 0 : goto error;
2854 : }
2855 :
2856 2 : if (!module->crypto_supports_cipher(key->cipher, key->key_size)) {
2857 0 : SPDK_ERRLOG("Module %s doesn't support %s cipher with %zu key size\n", module->name,
2858 : g_ciphers[key->cipher], key->key_size);
2859 0 : rc = -EINVAL;
2860 0 : goto error;
2861 : }
2862 :
2863 2 : if (key->cipher == SPDK_ACCEL_CIPHER_AES_XTS) {
2864 2 : if (!key->key2) {
2865 0 : SPDK_ERRLOG("%s key2 is missing\n", g_ciphers[key->cipher]);
2866 0 : rc = -EINVAL;
2867 0 : goto error;
2868 : }
2869 :
2870 2 : if (key->key_size != key->key2_size) {
2871 0 : SPDK_ERRLOG("%s key size %zu is not equal to key2 size %zu\n", g_ciphers[key->cipher],
2872 : key->key_size,
2873 : key->key2_size);
2874 0 : rc = -EINVAL;
2875 0 : goto error;
2876 : }
2877 :
2878 2 : if (accel_aes_xts_keys_equal(key->key, key->key_size, key->key2, key->key2_size)) {
2879 0 : SPDK_ERRLOG("%s identical keys are not secure\n", g_ciphers[key->cipher]);
2880 0 : rc = -EINVAL;
2881 0 : goto error;
2882 : }
2883 : }
2884 :
2885 2 : if (key->cipher == SPDK_ACCEL_CIPHER_AES_CBC) {
2886 0 : if (key->key2_size) {
2887 0 : SPDK_ERRLOG("%s doesn't use key2\n", g_ciphers[key->cipher]);
2888 0 : rc = -EINVAL;
2889 0 : goto error;
2890 : }
2891 : }
2892 :
2893 2 : key->module_if = module;
2894 :
2895 2 : spdk_spin_lock(&g_keyring_spin);
2896 2 : if (_accel_crypto_key_get(param->key_name)) {
2897 0 : rc = -EEXIST;
2898 : } else {
2899 2 : rc = module->crypto_key_init(key);
2900 2 : if (rc) {
2901 0 : SPDK_ERRLOG("Module %s failed to initialize crypto key\n", module->name);
2902 : } else {
2903 2 : TAILQ_INSERT_TAIL(&g_keyring, key, link);
2904 : }
2905 : }
2906 2 : spdk_spin_unlock(&g_keyring_spin);
2907 :
2908 2 : if (rc) {
2909 0 : goto error;
2910 : }
2911 :
2912 2 : return 0;
2913 :
2914 0 : error:
2915 0 : accel_crypto_key_free_mem(key);
2916 0 : return rc;
2917 : }
2918 :
2919 : int
2920 2 : spdk_accel_crypto_key_destroy(struct spdk_accel_crypto_key *key)
2921 : {
2922 2 : if (!key || !key->module_if) {
2923 0 : return -EINVAL;
2924 : }
2925 :
2926 2 : spdk_spin_lock(&g_keyring_spin);
2927 2 : if (!_accel_crypto_key_get(key->param.key_name)) {
2928 0 : spdk_spin_unlock(&g_keyring_spin);
2929 0 : return -ENOENT;
2930 : }
2931 2 : TAILQ_REMOVE(&g_keyring, key, link);
2932 2 : spdk_spin_unlock(&g_keyring_spin);
2933 :
2934 2 : accel_crypto_key_destroy_unsafe(key);
2935 :
2936 2 : return 0;
2937 : }
2938 :
2939 : struct spdk_accel_crypto_key *
2940 2 : spdk_accel_crypto_key_get(const char *name)
2941 : {
2942 : struct spdk_accel_crypto_key *key;
2943 :
2944 2 : spdk_spin_lock(&g_keyring_spin);
2945 2 : key = _accel_crypto_key_get(name);
2946 2 : spdk_spin_unlock(&g_keyring_spin);
2947 :
2948 2 : return key;
2949 : }
2950 :
2951 : /* Helper function when accel modules register with the framework. */
2952 : void
2953 5 : spdk_accel_module_list_add(struct spdk_accel_module_if *accel_module)
2954 : {
2955 : struct spdk_accel_module_if *tmp;
2956 :
2957 5 : if (_module_find_by_name(accel_module->name)) {
2958 0 : SPDK_NOTICELOG("Module %s already registered\n", accel_module->name);
2959 0 : assert(false);
2960 : return;
2961 : }
2962 :
2963 8 : TAILQ_FOREACH(tmp, &spdk_accel_module_list, tailq) {
2964 5 : if (accel_module->priority < tmp->priority) {
2965 2 : break;
2966 : }
2967 : }
2968 :
2969 5 : if (tmp != NULL) {
2970 2 : TAILQ_INSERT_BEFORE(tmp, accel_module, tailq);
2971 : } else {
2972 3 : TAILQ_INSERT_TAIL(&spdk_accel_module_list, accel_module, tailq);
2973 : }
2974 : }
2975 :
2976 : /* Framework level channel create callback. */
2977 : static int
2978 16 : accel_create_channel(void *io_device, void *ctx_buf)
2979 : {
2980 16 : struct accel_io_channel *accel_ch = ctx_buf;
2981 : struct spdk_accel_task *accel_task;
2982 : struct spdk_accel_task_aux_data *accel_task_aux;
2983 : struct spdk_accel_sequence *seq;
2984 : struct accel_buffer *buf;
2985 : size_t task_size_aligned;
2986 : uint8_t *task_mem;
2987 16 : uint32_t i = 0, j;
2988 : int rc;
2989 :
2990 16 : task_size_aligned = SPDK_ALIGN_CEIL(g_max_accel_module_size, SPDK_CACHE_LINE_SIZE);
2991 16 : accel_ch->task_pool_base = aligned_alloc(SPDK_CACHE_LINE_SIZE,
2992 16 : g_opts.task_count * task_size_aligned);
2993 16 : if (!accel_ch->task_pool_base) {
2994 0 : return -ENOMEM;
2995 : }
2996 16 : memset(accel_ch->task_pool_base, 0, g_opts.task_count * task_size_aligned);
2997 :
2998 16 : accel_ch->seq_pool_base = aligned_alloc(SPDK_CACHE_LINE_SIZE,
2999 16 : g_opts.sequence_count * sizeof(struct spdk_accel_sequence));
3000 16 : if (accel_ch->seq_pool_base == NULL) {
3001 0 : goto err;
3002 : }
3003 16 : memset(accel_ch->seq_pool_base, 0, g_opts.sequence_count * sizeof(struct spdk_accel_sequence));
3004 :
3005 16 : accel_ch->task_aux_data_base = calloc(g_opts.task_count, sizeof(struct spdk_accel_task_aux_data));
3006 16 : if (accel_ch->task_aux_data_base == NULL) {
3007 0 : goto err;
3008 : }
3009 :
3010 16 : accel_ch->buf_pool_base = calloc(g_opts.buf_count, sizeof(struct accel_buffer));
3011 16 : if (accel_ch->buf_pool_base == NULL) {
3012 0 : goto err;
3013 : }
3014 :
3015 16 : STAILQ_INIT(&accel_ch->task_pool);
3016 16 : SLIST_INIT(&accel_ch->task_aux_data_pool);
3017 16 : SLIST_INIT(&accel_ch->seq_pool);
3018 16 : SLIST_INIT(&accel_ch->buf_pool);
3019 :
3020 16 : task_mem = accel_ch->task_pool_base;
3021 32784 : for (i = 0; i < g_opts.task_count; i++) {
3022 32768 : accel_task = (struct spdk_accel_task *)task_mem;
3023 32768 : accel_task->aux = NULL;
3024 32768 : STAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link);
3025 32768 : task_mem += task_size_aligned;
3026 32768 : accel_task_aux = &accel_ch->task_aux_data_base[i];
3027 32768 : SLIST_INSERT_HEAD(&accel_ch->task_aux_data_pool, accel_task_aux, link);
3028 : }
3029 32784 : for (i = 0; i < g_opts.sequence_count; i++) {
3030 32768 : seq = &accel_ch->seq_pool_base[i];
3031 32768 : SLIST_INSERT_HEAD(&accel_ch->seq_pool, seq, link);
3032 : }
3033 32784 : for (i = 0; i < g_opts.buf_count; i++) {
3034 32768 : buf = &accel_ch->buf_pool_base[i];
3035 32768 : SLIST_INSERT_HEAD(&accel_ch->buf_pool, buf, link);
3036 : }
3037 :
3038 : /* Assign modules and get IO channels for each */
3039 288 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
3040 272 : accel_ch->module_ch[i] = g_modules_opc[i].module->get_io_channel();
3041 : /* This can happen if idxd runs out of channels. */
3042 272 : if (accel_ch->module_ch[i] == NULL) {
3043 0 : SPDK_ERRLOG("Module %s failed to get io channel\n", g_modules_opc[i].module->name);
3044 0 : goto err;
3045 : }
3046 : }
3047 :
3048 16 : if (g_accel_driver != NULL) {
3049 0 : accel_ch->driver_channel = g_accel_driver->get_io_channel();
3050 0 : if (accel_ch->driver_channel == NULL) {
3051 0 : SPDK_ERRLOG("Failed to get driver's IO channel\n");
3052 0 : goto err;
3053 : }
3054 : }
3055 :
3056 16 : rc = spdk_iobuf_channel_init(&accel_ch->iobuf, "accel", g_opts.small_cache_size,
3057 : g_opts.large_cache_size);
3058 16 : if (rc != 0) {
3059 0 : SPDK_ERRLOG("Failed to initialize iobuf accel channel\n");
3060 0 : goto err;
3061 : }
3062 :
3063 16 : return 0;
3064 0 : err:
3065 0 : if (accel_ch->driver_channel != NULL) {
3066 0 : spdk_put_io_channel(accel_ch->driver_channel);
3067 : }
3068 0 : for (j = 0; j < i; j++) {
3069 0 : spdk_put_io_channel(accel_ch->module_ch[j]);
3070 : }
3071 0 : free(accel_ch->task_pool_base);
3072 0 : free(accel_ch->task_aux_data_base);
3073 0 : free(accel_ch->seq_pool_base);
3074 0 : free(accel_ch->buf_pool_base);
3075 :
3076 0 : return -ENOMEM;
3077 : }
3078 :
3079 : static void
3080 16 : accel_add_stats(struct accel_stats *total, struct accel_stats *stats)
3081 : {
3082 : int i;
3083 :
3084 16 : total->sequence_executed += stats->sequence_executed;
3085 16 : total->sequence_failed += stats->sequence_failed;
3086 16 : total->sequence_outstanding += stats->sequence_outstanding;
3087 16 : total->task_outstanding += stats->task_outstanding;
3088 16 : total->retry.task += stats->retry.task;
3089 16 : total->retry.sequence += stats->retry.sequence;
3090 16 : total->retry.iobuf += stats->retry.iobuf;
3091 16 : total->retry.bufdesc += stats->retry.bufdesc;
3092 288 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) {
3093 272 : total->operations[i].executed += stats->operations[i].executed;
3094 272 : total->operations[i].failed += stats->operations[i].failed;
3095 272 : total->operations[i].num_bytes += stats->operations[i].num_bytes;
3096 : }
3097 16 : }
3098 :
3099 : /* Framework level channel destroy callback. */
3100 : static void
3101 16 : accel_destroy_channel(void *io_device, void *ctx_buf)
3102 : {
3103 16 : struct accel_io_channel *accel_ch = ctx_buf;
3104 : int i;
3105 :
3106 16 : spdk_iobuf_channel_fini(&accel_ch->iobuf);
3107 :
3108 16 : if (accel_ch->driver_channel != NULL) {
3109 0 : spdk_put_io_channel(accel_ch->driver_channel);
3110 : }
3111 :
3112 288 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
3113 272 : assert(accel_ch->module_ch[i] != NULL);
3114 272 : spdk_put_io_channel(accel_ch->module_ch[i]);
3115 272 : accel_ch->module_ch[i] = NULL;
3116 : }
3117 :
3118 : /* Update global stats to make sure channel's stats aren't lost after a channel is gone */
3119 16 : spdk_spin_lock(&g_stats_lock);
3120 16 : accel_add_stats(&g_stats, &accel_ch->stats);
3121 16 : spdk_spin_unlock(&g_stats_lock);
3122 :
3123 16 : free(accel_ch->task_pool_base);
3124 16 : free(accel_ch->task_aux_data_base);
3125 16 : free(accel_ch->seq_pool_base);
3126 16 : free(accel_ch->buf_pool_base);
3127 16 : }
3128 :
3129 : struct spdk_io_channel *
3130 16 : spdk_accel_get_io_channel(void)
3131 : {
3132 16 : return spdk_get_io_channel(&spdk_accel_module_list);
3133 : }
3134 :
3135 : static int
3136 2 : accel_module_initialize(void)
3137 : {
3138 : struct spdk_accel_module_if *accel_module, *tmp_module;
3139 2 : int rc = 0, module_rc;
3140 :
3141 7 : TAILQ_FOREACH_SAFE(accel_module, &spdk_accel_module_list, tailq, tmp_module) {
3142 5 : module_rc = accel_module->module_init();
3143 5 : if (module_rc) {
3144 0 : TAILQ_REMOVE(&spdk_accel_module_list, accel_module, tailq);
3145 0 : if (module_rc == -ENODEV) {
3146 0 : SPDK_NOTICELOG("No devices for module %s, skipping\n", accel_module->name);
3147 0 : } else if (!rc) {
3148 0 : SPDK_ERRLOG("Module %s initialization failed with %d\n", accel_module->name, module_rc);
3149 0 : rc = module_rc;
3150 : }
3151 0 : continue;
3152 : }
3153 :
3154 5 : SPDK_DEBUGLOG(accel, "Module %s initialized.\n", accel_module->name);
3155 : }
3156 :
3157 2 : return rc;
3158 : }
3159 :
3160 : static void
3161 34 : accel_module_init_opcode(enum spdk_accel_opcode opcode)
3162 : {
3163 34 : struct accel_module *module = &g_modules_opc[opcode];
3164 34 : struct spdk_accel_module_if *module_if = module->module;
3165 :
3166 34 : if (module_if->get_memory_domains != NULL) {
3167 0 : module->supports_memory_domains = module_if->get_memory_domains(NULL, 0) > 0;
3168 : }
3169 34 : }
3170 :
3171 : static int
3172 0 : accel_memory_domain_translate(struct spdk_memory_domain *src_domain, void *src_domain_ctx,
3173 : struct spdk_memory_domain *dst_domain, struct spdk_memory_domain_translation_ctx *dst_domain_ctx,
3174 : void *addr, size_t len, struct spdk_memory_domain_translation_result *result)
3175 : {
3176 0 : struct accel_buffer *buf = src_domain_ctx;
3177 :
3178 0 : SPDK_DEBUGLOG(accel, "translate addr %p, len %zu\n", addr, len);
3179 :
3180 0 : assert(g_accel_domain == src_domain);
3181 0 : assert(spdk_memory_domain_get_system_domain() == dst_domain);
3182 0 : assert(buf->buf == NULL);
3183 0 : assert(addr == ACCEL_BUFFER_BASE);
3184 0 : assert(len == buf->len);
3185 :
3186 0 : buf->buf = spdk_iobuf_get(&buf->ch->iobuf, buf->len, NULL, NULL);
3187 0 : if (spdk_unlikely(buf->buf == NULL)) {
3188 0 : return -ENOMEM;
3189 : }
3190 :
3191 0 : result->iov_count = 1;
3192 0 : result->iov.iov_base = buf->buf;
3193 0 : result->iov.iov_len = buf->len;
3194 0 : SPDK_DEBUGLOG(accel, "translated addr %p\n", result->iov.iov_base);
3195 0 : return 0;
3196 : }
3197 :
3198 : static void
3199 0 : accel_memory_domain_invalidate(struct spdk_memory_domain *domain, void *domain_ctx,
3200 : struct iovec *iov, uint32_t iovcnt)
3201 : {
3202 0 : struct accel_buffer *buf = domain_ctx;
3203 :
3204 0 : SPDK_DEBUGLOG(accel, "invalidate addr %p, len %zu\n", iov[0].iov_base, iov[0].iov_len);
3205 :
3206 0 : assert(g_accel_domain == domain);
3207 0 : assert(iovcnt == 1);
3208 0 : assert(buf->buf != NULL);
3209 0 : assert(iov[0].iov_base == buf->buf);
3210 0 : assert(iov[0].iov_len == buf->len);
3211 :
3212 0 : spdk_iobuf_put(&buf->ch->iobuf, buf->buf, buf->len);
3213 0 : buf->buf = NULL;
3214 0 : }
3215 :
3216 : int
3217 2 : spdk_accel_initialize(void)
3218 : {
3219 : enum spdk_accel_opcode op;
3220 2 : struct spdk_accel_module_if *accel_module = NULL;
3221 : int rc;
3222 :
3223 : /*
3224 : * We need a unique identifier for the accel framework, so use the
3225 : * spdk_accel_module_list address for this purpose.
3226 : */
3227 2 : spdk_io_device_register(&spdk_accel_module_list, accel_create_channel, accel_destroy_channel,
3228 : sizeof(struct accel_io_channel), "accel");
3229 :
3230 2 : spdk_spin_init(&g_keyring_spin);
3231 2 : spdk_spin_init(&g_stats_lock);
3232 :
3233 2 : rc = spdk_memory_domain_create(&g_accel_domain, SPDK_DMA_DEVICE_TYPE_ACCEL, NULL,
3234 : "SPDK_ACCEL_DMA_DEVICE");
3235 2 : if (rc != 0) {
3236 0 : SPDK_ERRLOG("Failed to create accel memory domain\n");
3237 0 : return rc;
3238 : }
3239 :
3240 2 : spdk_memory_domain_set_translation(g_accel_domain, accel_memory_domain_translate);
3241 2 : spdk_memory_domain_set_invalidate(g_accel_domain, accel_memory_domain_invalidate);
3242 :
3243 2 : g_modules_started = true;
3244 2 : rc = accel_module_initialize();
3245 2 : if (rc) {
3246 0 : return rc;
3247 : }
3248 :
3249 2 : if (g_accel_driver != NULL && g_accel_driver->init != NULL) {
3250 0 : rc = g_accel_driver->init();
3251 0 : if (rc != 0) {
3252 0 : SPDK_ERRLOG("Failed to initialize driver %s: %s\n", g_accel_driver->name,
3253 : spdk_strerror(-rc));
3254 0 : return rc;
3255 : }
3256 : }
3257 :
3258 : /* The module list is order by priority, with the highest priority modules being at the end
3259 : * of the list. The software module should be somewhere at the beginning of the list,
3260 : * before all HW modules.
3261 : * NOTE: all opcodes must be supported by software in the event that no HW modules are
3262 : * initialized to support the operation.
3263 : */
3264 7 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
3265 90 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
3266 85 : if (accel_module->supports_opcode(op)) {
3267 85 : g_modules_opc[op].module = accel_module;
3268 85 : SPDK_DEBUGLOG(accel, "OPC 0x%x now assigned to %s\n", op, accel_module->name);
3269 : }
3270 : }
3271 :
3272 5 : if (accel_module->get_ctx_size != NULL) {
3273 1 : g_max_accel_module_size = spdk_max(g_max_accel_module_size,
3274 : accel_module->get_ctx_size());
3275 : }
3276 : }
3277 :
3278 : /* Now lets check for overrides and apply all that exist */
3279 36 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
3280 34 : if (g_modules_opc_override[op] != NULL) {
3281 0 : accel_module = _module_find_by_name(g_modules_opc_override[op]);
3282 0 : if (accel_module == NULL) {
3283 0 : SPDK_ERRLOG("Invalid module name of %s\n", g_modules_opc_override[op]);
3284 0 : return -EINVAL;
3285 : }
3286 0 : if (accel_module->supports_opcode(op) == false) {
3287 0 : SPDK_ERRLOG("Module %s does not support op code %d\n", accel_module->name, op);
3288 0 : return -EINVAL;
3289 : }
3290 0 : g_modules_opc[op].module = accel_module;
3291 : }
3292 : }
3293 :
3294 2 : if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) {
3295 0 : SPDK_ERRLOG("Different accel modules are assigned to encrypt and decrypt operations");
3296 0 : return -EINVAL;
3297 : }
3298 2 : if (g_modules_opc[SPDK_ACCEL_OPC_COMPRESS].module !=
3299 2 : g_modules_opc[SPDK_ACCEL_OPC_DECOMPRESS].module) {
3300 0 : SPDK_ERRLOG("Different accel modules are assigned to compress and decompress operations");
3301 0 : return -EINVAL;
3302 : }
3303 :
3304 36 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
3305 34 : assert(g_modules_opc[op].module != NULL);
3306 34 : accel_module_init_opcode(op);
3307 : }
3308 :
3309 2 : rc = spdk_iobuf_register_module("accel");
3310 2 : if (rc != 0) {
3311 0 : SPDK_ERRLOG("Failed to register accel iobuf module\n");
3312 0 : return rc;
3313 : }
3314 :
3315 2 : return 0;
3316 : }
3317 :
3318 : static void
3319 2 : accel_module_finish_cb(void)
3320 : {
3321 2 : spdk_accel_fini_cb cb_fn = g_fini_cb_fn;
3322 :
3323 2 : cb_fn(g_fini_cb_arg);
3324 2 : g_fini_cb_fn = NULL;
3325 2 : g_fini_cb_arg = NULL;
3326 2 : }
3327 :
3328 : static void
3329 0 : accel_write_overridden_opc(struct spdk_json_write_ctx *w, const char *opc_str,
3330 : const char *module_str)
3331 : {
3332 0 : spdk_json_write_object_begin(w);
3333 0 : spdk_json_write_named_string(w, "method", "accel_assign_opc");
3334 0 : spdk_json_write_named_object_begin(w, "params");
3335 0 : spdk_json_write_named_string(w, "opname", opc_str);
3336 0 : spdk_json_write_named_string(w, "module", module_str);
3337 0 : spdk_json_write_object_end(w);
3338 0 : spdk_json_write_object_end(w);
3339 0 : }
3340 :
3341 : static void
3342 0 : __accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key)
3343 : {
3344 0 : spdk_json_write_named_string(w, "name", key->param.key_name);
3345 0 : spdk_json_write_named_string(w, "cipher", key->param.cipher);
3346 0 : spdk_json_write_named_string(w, "key", key->param.hex_key);
3347 0 : if (key->param.hex_key2) {
3348 0 : spdk_json_write_named_string(w, "key2", key->param.hex_key2);
3349 : }
3350 :
3351 0 : if (key->param.tweak_mode) {
3352 0 : spdk_json_write_named_string(w, "tweak_mode", key->param.tweak_mode);
3353 : }
3354 0 : }
3355 :
3356 : void
3357 0 : _accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key)
3358 : {
3359 0 : spdk_json_write_object_begin(w);
3360 0 : __accel_crypto_key_dump_param(w, key);
3361 0 : spdk_json_write_object_end(w);
3362 0 : }
3363 :
3364 : static void
3365 0 : _accel_crypto_key_write_config_json(struct spdk_json_write_ctx *w,
3366 : struct spdk_accel_crypto_key *key)
3367 : {
3368 0 : spdk_json_write_object_begin(w);
3369 0 : spdk_json_write_named_string(w, "method", "accel_crypto_key_create");
3370 0 : spdk_json_write_named_object_begin(w, "params");
3371 0 : __accel_crypto_key_dump_param(w, key);
3372 0 : spdk_json_write_object_end(w);
3373 0 : spdk_json_write_object_end(w);
3374 0 : }
3375 :
3376 : static void
3377 0 : accel_write_options(struct spdk_json_write_ctx *w)
3378 : {
3379 0 : spdk_json_write_object_begin(w);
3380 0 : spdk_json_write_named_string(w, "method", "accel_set_options");
3381 0 : spdk_json_write_named_object_begin(w, "params");
3382 0 : spdk_json_write_named_uint32(w, "small_cache_size", g_opts.small_cache_size);
3383 0 : spdk_json_write_named_uint32(w, "large_cache_size", g_opts.large_cache_size);
3384 0 : spdk_json_write_named_uint32(w, "task_count", g_opts.task_count);
3385 0 : spdk_json_write_named_uint32(w, "sequence_count", g_opts.sequence_count);
3386 0 : spdk_json_write_named_uint32(w, "buf_count", g_opts.buf_count);
3387 0 : spdk_json_write_object_end(w);
3388 0 : spdk_json_write_object_end(w);
3389 0 : }
3390 :
3391 : static void
3392 0 : _accel_crypto_keys_write_config_json(struct spdk_json_write_ctx *w, bool full_dump)
3393 : {
3394 : struct spdk_accel_crypto_key *key;
3395 :
3396 0 : spdk_spin_lock(&g_keyring_spin);
3397 0 : TAILQ_FOREACH(key, &g_keyring, link) {
3398 0 : if (full_dump) {
3399 0 : _accel_crypto_key_write_config_json(w, key);
3400 : } else {
3401 0 : _accel_crypto_key_dump_param(w, key);
3402 : }
3403 : }
3404 0 : spdk_spin_unlock(&g_keyring_spin);
3405 0 : }
3406 :
3407 : void
3408 0 : _accel_crypto_keys_dump_param(struct spdk_json_write_ctx *w)
3409 : {
3410 0 : _accel_crypto_keys_write_config_json(w, false);
3411 0 : }
3412 :
3413 : void
3414 0 : spdk_accel_write_config_json(struct spdk_json_write_ctx *w)
3415 : {
3416 : struct spdk_accel_module_if *accel_module;
3417 : int i;
3418 :
3419 0 : spdk_json_write_array_begin(w);
3420 0 : accel_write_options(w);
3421 :
3422 0 : if (g_accel_driver != NULL) {
3423 0 : spdk_json_write_object_begin(w);
3424 0 : spdk_json_write_named_string(w, "method", "accel_set_driver");
3425 0 : spdk_json_write_named_object_begin(w, "params");
3426 0 : spdk_json_write_named_string(w, "name", g_accel_driver->name);
3427 0 : spdk_json_write_object_end(w);
3428 0 : spdk_json_write_object_end(w);
3429 : }
3430 :
3431 0 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
3432 0 : if (accel_module->write_config_json) {
3433 0 : accel_module->write_config_json(w);
3434 : }
3435 : }
3436 0 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
3437 0 : if (g_modules_opc_override[i]) {
3438 0 : accel_write_overridden_opc(w, g_opcode_strings[i], g_modules_opc_override[i]);
3439 : }
3440 : }
3441 :
3442 0 : _accel_crypto_keys_write_config_json(w, true);
3443 :
3444 0 : spdk_json_write_array_end(w);
3445 0 : }
3446 :
3447 : void
3448 7 : spdk_accel_module_finish(void)
3449 : {
3450 7 : if (!g_accel_module) {
3451 2 : g_accel_module = TAILQ_FIRST(&spdk_accel_module_list);
3452 : } else {
3453 5 : g_accel_module = TAILQ_NEXT(g_accel_module, tailq);
3454 : }
3455 :
3456 7 : if (!g_accel_module) {
3457 2 : if (g_accel_driver != NULL && g_accel_driver->fini != NULL) {
3458 0 : g_accel_driver->fini();
3459 : }
3460 :
3461 2 : spdk_spin_destroy(&g_keyring_spin);
3462 2 : spdk_spin_destroy(&g_stats_lock);
3463 2 : if (g_accel_domain) {
3464 2 : spdk_memory_domain_destroy(g_accel_domain);
3465 2 : g_accel_domain = NULL;
3466 : }
3467 2 : accel_module_finish_cb();
3468 2 : return;
3469 : }
3470 :
3471 5 : if (g_accel_module->module_fini) {
3472 1 : spdk_thread_send_msg(spdk_get_thread(), g_accel_module->module_fini, NULL);
3473 : } else {
3474 4 : spdk_accel_module_finish();
3475 : }
3476 : }
3477 :
3478 : static void
3479 2 : accel_io_device_unregister_cb(void *io_device)
3480 : {
3481 : struct spdk_accel_crypto_key *key, *key_tmp;
3482 : enum spdk_accel_opcode op;
3483 :
3484 2 : spdk_spin_lock(&g_keyring_spin);
3485 2 : TAILQ_FOREACH_SAFE(key, &g_keyring, link, key_tmp) {
3486 0 : accel_crypto_key_destroy_unsafe(key);
3487 : }
3488 2 : spdk_spin_unlock(&g_keyring_spin);
3489 :
3490 36 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
3491 34 : if (g_modules_opc_override[op] != NULL) {
3492 0 : free(g_modules_opc_override[op]);
3493 0 : g_modules_opc_override[op] = NULL;
3494 : }
3495 34 : g_modules_opc[op].module = NULL;
3496 : }
3497 :
3498 2 : spdk_accel_module_finish();
3499 2 : }
3500 :
3501 : void
3502 2 : spdk_accel_finish(spdk_accel_fini_cb cb_fn, void *cb_arg)
3503 : {
3504 2 : assert(cb_fn != NULL);
3505 :
3506 2 : g_fini_cb_fn = cb_fn;
3507 2 : g_fini_cb_arg = cb_arg;
3508 :
3509 2 : spdk_io_device_unregister(&spdk_accel_module_list, accel_io_device_unregister_cb);
3510 2 : }
3511 :
3512 : static struct spdk_accel_driver *
3513 2 : accel_find_driver(const char *name)
3514 : {
3515 : struct spdk_accel_driver *driver;
3516 :
3517 2 : TAILQ_FOREACH(driver, &g_accel_drivers, tailq) {
3518 1 : if (strcmp(driver->name, name) == 0) {
3519 1 : return driver;
3520 : }
3521 : }
3522 :
3523 1 : return NULL;
3524 : }
3525 :
3526 : int
3527 1 : spdk_accel_set_driver(const char *name)
3528 : {
3529 1 : struct spdk_accel_driver *driver = NULL;
3530 :
3531 1 : if (name != NULL && name[0] != '\0') {
3532 1 : driver = accel_find_driver(name);
3533 1 : if (driver == NULL) {
3534 0 : SPDK_ERRLOG("Couldn't find driver named '%s'\n", name);
3535 0 : return -ENODEV;
3536 : }
3537 : }
3538 :
3539 1 : g_accel_driver = driver;
3540 :
3541 1 : return 0;
3542 : }
3543 :
3544 : const char *
3545 0 : spdk_accel_get_driver_name(void)
3546 : {
3547 0 : if (!g_accel_driver) {
3548 0 : return NULL;
3549 : }
3550 :
3551 0 : return g_accel_driver->name;
3552 : }
3553 :
3554 : void
3555 1 : spdk_accel_driver_register(struct spdk_accel_driver *driver)
3556 : {
3557 1 : if (accel_find_driver(driver->name)) {
3558 0 : SPDK_ERRLOG("Driver named '%s' has already been registered\n", driver->name);
3559 0 : assert(0);
3560 : return;
3561 : }
3562 :
3563 1 : TAILQ_INSERT_TAIL(&g_accel_drivers, driver, tailq);
3564 : }
3565 :
3566 : int
3567 0 : spdk_accel_set_opts(const struct spdk_accel_opts *opts)
3568 : {
3569 0 : if (!opts) {
3570 0 : SPDK_ERRLOG("opts cannot be NULL\n");
3571 0 : return -1;
3572 : }
3573 :
3574 0 : if (!opts->opts_size) {
3575 0 : SPDK_ERRLOG("opts_size inside opts cannot be zero value\n");
3576 0 : return -1;
3577 : }
3578 :
3579 0 : if (SPDK_GET_FIELD(opts, task_count, g_opts.task_count,
3580 0 : opts->opts_size) < ACCEL_TASKS_IN_SEQUENCE_LIMIT) {
3581 0 : return -EINVAL;
3582 : }
3583 :
3584 : #define SET_FIELD(field) \
3585 : if (offsetof(struct spdk_accel_opts, field) + sizeof(opts->field) <= opts->opts_size) { \
3586 : g_opts.field = opts->field; \
3587 : } \
3588 :
3589 0 : SET_FIELD(small_cache_size);
3590 0 : SET_FIELD(large_cache_size);
3591 0 : SET_FIELD(task_count);
3592 0 : SET_FIELD(sequence_count);
3593 0 : SET_FIELD(buf_count);
3594 :
3595 0 : g_opts.opts_size = opts->opts_size;
3596 :
3597 : #undef SET_FIELD
3598 :
3599 0 : return 0;
3600 : }
3601 :
3602 : void
3603 0 : spdk_accel_get_opts(struct spdk_accel_opts *opts, size_t opts_size)
3604 : {
3605 0 : if (!opts) {
3606 0 : SPDK_ERRLOG("opts should not be NULL\n");
3607 0 : return;
3608 : }
3609 :
3610 0 : if (!opts_size) {
3611 0 : SPDK_ERRLOG("opts_size should not be zero value\n");
3612 0 : return;
3613 : }
3614 :
3615 0 : opts->opts_size = opts_size;
3616 :
3617 : #define SET_FIELD(field) \
3618 : if (offsetof(struct spdk_accel_opts, field) + sizeof(opts->field) <= opts_size) { \
3619 : opts->field = g_opts.field; \
3620 : } \
3621 :
3622 0 : SET_FIELD(small_cache_size);
3623 0 : SET_FIELD(large_cache_size);
3624 0 : SET_FIELD(task_count);
3625 0 : SET_FIELD(sequence_count);
3626 0 : SET_FIELD(buf_count);
3627 :
3628 : #undef SET_FIELD
3629 :
3630 : /* Do not remove this statement, you should always update this statement when you adding a new field,
3631 : * and do not forget to add the SET_FIELD statement for your added field. */
3632 : SPDK_STATIC_ASSERT(sizeof(struct spdk_accel_opts) == 28, "Incorrect size");
3633 : }
3634 :
3635 : struct accel_get_stats_ctx {
3636 : struct accel_stats stats;
3637 : accel_get_stats_cb cb_fn;
3638 : void *cb_arg;
3639 : };
3640 :
3641 : static void
3642 0 : accel_get_channel_stats_done(struct spdk_io_channel_iter *iter, int status)
3643 : {
3644 0 : struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter);
3645 :
3646 0 : ctx->cb_fn(&ctx->stats, ctx->cb_arg);
3647 0 : free(ctx);
3648 0 : }
3649 :
3650 : static void
3651 0 : accel_get_channel_stats(struct spdk_io_channel_iter *iter)
3652 : {
3653 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(iter);
3654 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
3655 0 : struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter);
3656 :
3657 0 : accel_add_stats(&ctx->stats, &accel_ch->stats);
3658 0 : spdk_for_each_channel_continue(iter, 0);
3659 0 : }
3660 :
3661 : int
3662 0 : accel_get_stats(accel_get_stats_cb cb_fn, void *cb_arg)
3663 : {
3664 : struct accel_get_stats_ctx *ctx;
3665 :
3666 0 : ctx = calloc(1, sizeof(*ctx));
3667 0 : if (ctx == NULL) {
3668 0 : return -ENOMEM;
3669 : }
3670 :
3671 0 : spdk_spin_lock(&g_stats_lock);
3672 0 : accel_add_stats(&ctx->stats, &g_stats);
3673 0 : spdk_spin_unlock(&g_stats_lock);
3674 :
3675 0 : ctx->cb_fn = cb_fn;
3676 0 : ctx->cb_arg = cb_arg;
3677 :
3678 0 : spdk_for_each_channel(&spdk_accel_module_list, accel_get_channel_stats, ctx,
3679 : accel_get_channel_stats_done);
3680 :
3681 0 : return 0;
3682 : }
3683 :
3684 : void
3685 0 : spdk_accel_get_opcode_stats(struct spdk_io_channel *ch, enum spdk_accel_opcode opcode,
3686 : struct spdk_accel_opcode_stats *stats, size_t size)
3687 : {
3688 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
3689 :
3690 : #define FIELD_OK(field) \
3691 : offsetof(struct spdk_accel_opcode_stats, field) + sizeof(stats->field) <= size
3692 :
3693 : #define SET_FIELD(field, value) \
3694 : if (FIELD_OK(field)) { \
3695 : stats->field = value; \
3696 : }
3697 :
3698 0 : SET_FIELD(executed, accel_ch->stats.operations[opcode].executed);
3699 0 : SET_FIELD(failed, accel_ch->stats.operations[opcode].failed);
3700 0 : SET_FIELD(num_bytes, accel_ch->stats.operations[opcode].num_bytes);
3701 :
3702 : #undef FIELD_OK
3703 : #undef SET_FIELD
3704 0 : }
3705 :
3706 : uint8_t
3707 0 : spdk_accel_get_buf_align(enum spdk_accel_opcode opcode,
3708 : const struct spdk_accel_operation_exec_ctx *ctx)
3709 : {
3710 0 : struct spdk_accel_module_if *module = g_modules_opc[opcode].module;
3711 0 : struct spdk_accel_opcode_info modinfo = {}, drvinfo = {};
3712 :
3713 0 : if (g_accel_driver != NULL && g_accel_driver->get_operation_info != NULL) {
3714 0 : g_accel_driver->get_operation_info(opcode, ctx, &drvinfo);
3715 : }
3716 :
3717 0 : if (module->get_operation_info != NULL) {
3718 0 : module->get_operation_info(opcode, ctx, &modinfo);
3719 : }
3720 :
3721 : /* If a driver is set, it'll execute most of the operations, while the rest will usually
3722 : * fall back to accel_sw, which doesn't have any alignment requirements. However, to be
3723 : * extra safe, return the max(driver, module) if a driver delegates some operations to a
3724 : * hardware module. */
3725 0 : return spdk_max(modinfo.required_alignment, drvinfo.required_alignment);
3726 : }
3727 :
3728 : struct spdk_accel_module_if *
3729 0 : spdk_accel_get_module(const char *name)
3730 : {
3731 : struct spdk_accel_module_if *module;
3732 :
3733 0 : TAILQ_FOREACH(module, &spdk_accel_module_list, tailq) {
3734 0 : if (strcmp(module->name, name) == 0) {
3735 0 : return module;
3736 : }
3737 : }
3738 :
3739 0 : return NULL;
3740 : }
3741 :
3742 : int
3743 0 : spdk_accel_get_opc_memory_domains(enum spdk_accel_opcode opcode,
3744 : struct spdk_memory_domain **domains,
3745 : int array_size)
3746 : {
3747 0 : assert(opcode < SPDK_ACCEL_OPC_LAST);
3748 :
3749 0 : if (g_modules_opc[opcode].module->get_memory_domains) {
3750 0 : return g_modules_opc[opcode].module->get_memory_domains(domains, array_size);
3751 : }
3752 :
3753 0 : return 0;
3754 : }
3755 :
3756 1 : SPDK_LOG_REGISTER_COMPONENT(accel)
|
