Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2020 Intel Corporation.
3 : * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES.
4 : * All rights reserved.
5 : */
6 :
7 : #include "spdk/stdinc.h"
8 :
9 : #include "spdk/accel_module.h"
10 :
11 : #include "accel_internal.h"
12 :
13 : #include "spdk/dma.h"
14 : #include "spdk/env.h"
15 : #include "spdk/likely.h"
16 : #include "spdk/log.h"
17 : #include "spdk/thread.h"
18 : #include "spdk/json.h"
19 : #include "spdk/crc32.h"
20 : #include "spdk/util.h"
21 : #include "spdk/hexlify.h"
22 : #include "spdk/string.h"
23 :
24 : /* Accelerator Framework: The following provides a top level
25 : * generic API for the accelerator functions defined here. Modules,
26 : * such as the one in /module/accel/ioat, supply the implementation
27 : * with the exception of the pure software implementation contained
28 : * later in this file.
29 : */
30 :
31 : #define ALIGN_4K 0x1000
32 : #define MAX_TASKS_PER_CHANNEL 0x800
33 : #define ACCEL_SMALL_CACHE_SIZE 128
34 : #define ACCEL_LARGE_CACHE_SIZE 16
35 : /* Set MSB, so we don't return NULL pointers as buffers */
36 : #define ACCEL_BUFFER_BASE ((void *)(1ull << 63))
37 : #define ACCEL_BUFFER_OFFSET_MASK ((uintptr_t)ACCEL_BUFFER_BASE - 1)
38 :
39 : #define ACCEL_CRYPTO_TWEAK_MODE_DEFAULT SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA
40 :
41 : struct accel_module {
42 : struct spdk_accel_module_if *module;
43 : bool supports_memory_domains;
44 : };
45 :
46 : /* Largest context size for all accel modules */
47 : static size_t g_max_accel_module_size = sizeof(struct spdk_accel_task);
48 :
49 : static struct spdk_accel_module_if *g_accel_module = NULL;
50 : static spdk_accel_fini_cb g_fini_cb_fn = NULL;
51 : static void *g_fini_cb_arg = NULL;
52 : static bool g_modules_started = false;
53 : static struct spdk_memory_domain *g_accel_domain;
54 :
55 : /* Global list of registered accelerator modules */
56 : static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list =
57 : TAILQ_HEAD_INITIALIZER(spdk_accel_module_list);
58 :
59 : /* Crypto keyring */
60 : static TAILQ_HEAD(, spdk_accel_crypto_key) g_keyring = TAILQ_HEAD_INITIALIZER(g_keyring);
61 : static struct spdk_spinlock g_keyring_spin;
62 :
63 : /* Global array mapping capabilities to modules */
64 : static struct accel_module g_modules_opc[SPDK_ACCEL_OPC_LAST] = {};
65 : static char *g_modules_opc_override[SPDK_ACCEL_OPC_LAST] = {};
66 : TAILQ_HEAD(, spdk_accel_driver) g_accel_drivers = TAILQ_HEAD_INITIALIZER(g_accel_drivers);
67 : static struct spdk_accel_driver *g_accel_driver;
68 : static struct spdk_accel_opts g_opts = {
69 : .small_cache_size = ACCEL_SMALL_CACHE_SIZE,
70 : .large_cache_size = ACCEL_LARGE_CACHE_SIZE,
71 : .task_count = MAX_TASKS_PER_CHANNEL,
72 : .sequence_count = MAX_TASKS_PER_CHANNEL,
73 : .buf_count = MAX_TASKS_PER_CHANNEL,
74 : };
75 : static struct accel_stats g_stats;
76 : static struct spdk_spinlock g_stats_lock;
77 :
78 : static const char *g_opcode_strings[SPDK_ACCEL_OPC_LAST] = {
79 : "copy", "fill", "dualcast", "compare", "crc32c", "copy_crc32c",
80 : "compress", "decompress", "encrypt", "decrypt", "xor",
81 : "dif_verify", "dif_verify_copy", "dif_generate", "dif_generate_copy"
82 : };
83 :
84 : enum accel_sequence_state {
85 : ACCEL_SEQUENCE_STATE_INIT,
86 : ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF,
87 : ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF,
88 : ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF,
89 : ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF,
90 : ACCEL_SEQUENCE_STATE_PULL_DATA,
91 : ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA,
92 : ACCEL_SEQUENCE_STATE_EXEC_TASK,
93 : ACCEL_SEQUENCE_STATE_AWAIT_TASK,
94 : ACCEL_SEQUENCE_STATE_COMPLETE_TASK,
95 : ACCEL_SEQUENCE_STATE_NEXT_TASK,
96 : ACCEL_SEQUENCE_STATE_PUSH_DATA,
97 : ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA,
98 : ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS,
99 : ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS,
100 : ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS,
101 : ACCEL_SEQUENCE_STATE_ERROR,
102 : ACCEL_SEQUENCE_STATE_MAX,
103 : };
104 :
105 : static const char *g_seq_states[]
106 : __attribute__((unused)) = {
107 : [ACCEL_SEQUENCE_STATE_INIT] = "init",
108 : [ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF] = "check-virtbuf",
109 : [ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF] = "await-virtbuf",
110 : [ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF] = "check-bouncebuf",
111 : [ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF] = "await-bouncebuf",
112 : [ACCEL_SEQUENCE_STATE_PULL_DATA] = "pull-data",
113 : [ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA] = "await-pull-data",
114 : [ACCEL_SEQUENCE_STATE_EXEC_TASK] = "exec-task",
115 : [ACCEL_SEQUENCE_STATE_AWAIT_TASK] = "await-task",
116 : [ACCEL_SEQUENCE_STATE_COMPLETE_TASK] = "complete-task",
117 : [ACCEL_SEQUENCE_STATE_NEXT_TASK] = "next-task",
118 : [ACCEL_SEQUENCE_STATE_PUSH_DATA] = "push-data",
119 : [ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA] = "await-push-data",
120 : [ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS] = "driver-exec-tasks",
121 : [ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS] = "driver-await-tasks",
122 : [ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS] = "driver-complete-tasks",
123 : [ACCEL_SEQUENCE_STATE_ERROR] = "error",
124 : [ACCEL_SEQUENCE_STATE_MAX] = "",
125 : };
126 :
127 : #define ACCEL_SEQUENCE_STATE_STRING(s) \
128 : (((s) >= ACCEL_SEQUENCE_STATE_INIT && (s) < ACCEL_SEQUENCE_STATE_MAX) \
129 : ? g_seq_states[s] : "unknown")
130 :
131 : struct accel_buffer {
132 : struct spdk_accel_sequence *seq;
133 : void *buf;
134 : uint64_t len;
135 : struct spdk_iobuf_entry iobuf;
136 : spdk_accel_sequence_get_buf_cb cb_fn;
137 : void *cb_ctx;
138 : SLIST_ENTRY(accel_buffer) link;
139 : struct accel_io_channel *ch;
140 : };
141 :
142 : struct accel_io_channel {
143 : struct spdk_io_channel *module_ch[SPDK_ACCEL_OPC_LAST];
144 : struct spdk_io_channel *driver_channel;
145 : void *task_pool_base;
146 : struct spdk_accel_sequence *seq_pool_base;
147 : struct accel_buffer *buf_pool_base;
148 : struct spdk_accel_task_aux_data *task_aux_data_base;
149 : STAILQ_HEAD(, spdk_accel_task) task_pool;
150 : SLIST_HEAD(, spdk_accel_task_aux_data) task_aux_data_pool;
151 : SLIST_HEAD(, spdk_accel_sequence) seq_pool;
152 : SLIST_HEAD(, accel_buffer) buf_pool;
153 : struct spdk_iobuf_channel iobuf;
154 : struct accel_stats stats;
155 : };
156 :
157 : TAILQ_HEAD(accel_sequence_tasks, spdk_accel_task);
158 :
159 : struct spdk_accel_sequence {
160 : struct accel_io_channel *ch;
161 : struct accel_sequence_tasks tasks;
162 : SLIST_HEAD(, accel_buffer) bounce_bufs;
163 : int status;
164 : /* state uses enum accel_sequence_state */
165 : uint8_t state;
166 : bool in_process_sequence;
167 : spdk_accel_completion_cb cb_fn;
168 : void *cb_arg;
169 : SLIST_ENTRY(spdk_accel_sequence) link;
170 : };
171 : SPDK_STATIC_ASSERT(sizeof(struct spdk_accel_sequence) == 64, "invalid size");
172 :
173 : #define accel_update_stats(ch, event, v) \
174 : do { \
175 : (ch)->stats.event += (v); \
176 : } while (0)
177 :
178 : #define accel_update_task_stats(ch, task, event, v) \
179 : accel_update_stats(ch, operations[(task)->op_code].event, v)
180 :
181 : static inline void accel_sequence_task_cb(void *cb_arg, int status);
182 :
183 : static inline void
184 709 : accel_sequence_set_state(struct spdk_accel_sequence *seq, enum accel_sequence_state state)
185 : {
186 709 : SPDK_DEBUGLOG(accel, "seq=%p, setting state: %s -> %s\n", seq,
187 : ACCEL_SEQUENCE_STATE_STRING(seq->state), ACCEL_SEQUENCE_STATE_STRING(state));
188 709 : assert(seq->state != ACCEL_SEQUENCE_STATE_ERROR || state == ACCEL_SEQUENCE_STATE_ERROR);
189 709 : seq->state = state;
190 709 : }
191 :
192 : static void
193 9 : accel_sequence_set_fail(struct spdk_accel_sequence *seq, int status)
194 : {
195 9 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR);
196 9 : assert(status != 0);
197 9 : seq->status = status;
198 9 : }
199 :
200 : int
201 15 : spdk_accel_get_opc_module_name(enum spdk_accel_opcode opcode, const char **module_name)
202 : {
203 15 : if (opcode >= SPDK_ACCEL_OPC_LAST) {
204 : /* invalid opcode */
205 0 : return -EINVAL;
206 : }
207 :
208 15 : if (g_modules_opc[opcode].module) {
209 15 : *module_name = g_modules_opc[opcode].module->name;
210 : } else {
211 0 : return -ENOENT;
212 : }
213 :
214 15 : return 0;
215 : }
216 :
217 : void
218 0 : _accel_for_each_module(struct module_info *info, _accel_for_each_module_fn fn)
219 : {
220 : struct spdk_accel_module_if *accel_module;
221 : enum spdk_accel_opcode opcode;
222 0 : int j = 0;
223 :
224 0 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
225 0 : for (opcode = 0; opcode < SPDK_ACCEL_OPC_LAST; opcode++) {
226 0 : if (accel_module->supports_opcode(opcode)) {
227 0 : info->ops[j] = opcode;
228 0 : j++;
229 : }
230 : }
231 0 : info->name = accel_module->name;
232 0 : info->num_ops = j;
233 0 : fn(info);
234 0 : j = 0;
235 : }
236 0 : }
237 :
238 : const char *
239 0 : spdk_accel_get_opcode_name(enum spdk_accel_opcode opcode)
240 : {
241 0 : if (opcode < SPDK_ACCEL_OPC_LAST) {
242 0 : return g_opcode_strings[opcode];
243 : }
244 :
245 0 : return NULL;
246 : }
247 :
248 : int
249 0 : spdk_accel_assign_opc(enum spdk_accel_opcode opcode, const char *name)
250 : {
251 : char *copy;
252 :
253 0 : if (g_modules_started == true) {
254 : /* we don't allow re-assignment once things have started */
255 0 : return -EINVAL;
256 : }
257 :
258 0 : if (opcode >= SPDK_ACCEL_OPC_LAST) {
259 : /* invalid opcode */
260 0 : return -EINVAL;
261 : }
262 :
263 0 : copy = strdup(name);
264 0 : if (copy == NULL) {
265 0 : return -ENOMEM;
266 : }
267 :
268 : /* module selection will be validated after the framework starts. */
269 0 : free(g_modules_opc_override[opcode]);
270 0 : g_modules_opc_override[opcode] = copy;
271 :
272 0 : return 0;
273 : }
274 :
275 : inline static struct spdk_accel_task *
276 151 : _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, void *cb_arg)
277 : {
278 : struct spdk_accel_task *accel_task;
279 :
280 151 : accel_task = STAILQ_FIRST(&accel_ch->task_pool);
281 151 : if (spdk_unlikely(accel_task == NULL)) {
282 11 : accel_update_stats(accel_ch, retry.task, 1);
283 11 : return NULL;
284 : }
285 :
286 140 : STAILQ_REMOVE_HEAD(&accel_ch->task_pool, link);
287 140 : accel_task->link.stqe_next = NULL;
288 :
289 140 : accel_task->cb_fn = cb_fn;
290 140 : accel_task->cb_arg = cb_arg;
291 140 : accel_task->accel_ch = accel_ch;
292 140 : accel_task->s.iovs = NULL;
293 140 : accel_task->d.iovs = NULL;
294 :
295 140 : return accel_task;
296 : }
297 :
298 : static void
299 132 : _put_task(struct accel_io_channel *ch, struct spdk_accel_task *task)
300 : {
301 132 : STAILQ_INSERT_HEAD(&ch->task_pool, task, link);
302 132 : }
303 :
304 : void
305 95 : spdk_accel_task_complete(struct spdk_accel_task *accel_task, int status)
306 : {
307 95 : struct accel_io_channel *accel_ch = accel_task->accel_ch;
308 : spdk_accel_completion_cb cb_fn;
309 : void *cb_arg;
310 :
311 95 : accel_update_task_stats(accel_ch, accel_task, executed, 1);
312 95 : accel_update_task_stats(accel_ch, accel_task, num_bytes, accel_task->nbytes);
313 95 : if (spdk_unlikely(status != 0)) {
314 3 : accel_update_task_stats(accel_ch, accel_task, failed, 1);
315 : }
316 :
317 95 : if (accel_task->seq) {
318 94 : accel_sequence_task_cb(accel_task->seq, status);
319 94 : return;
320 : }
321 :
322 1 : cb_fn = accel_task->cb_fn;
323 1 : cb_arg = accel_task->cb_arg;
324 :
325 1 : if (accel_task->has_aux) {
326 0 : SLIST_INSERT_HEAD(&accel_ch->task_aux_data_pool, accel_task->aux, link);
327 0 : accel_task->aux = NULL;
328 0 : accel_task->has_aux = false;
329 : }
330 :
331 : /* We should put the accel_task into the list firstly in order to avoid
332 : * the accel task list is exhausted when there is recursive call to
333 : * allocate accel_task in user's call back function (cb_fn)
334 : */
335 1 : _put_task(accel_ch, accel_task);
336 :
337 1 : cb_fn(cb_arg, status);
338 : }
339 :
340 : static inline int
341 92 : accel_submit_task(struct accel_io_channel *accel_ch, struct spdk_accel_task *task)
342 : {
343 92 : struct spdk_io_channel *module_ch = accel_ch->module_ch[task->op_code];
344 92 : struct spdk_accel_module_if *module = g_modules_opc[task->op_code].module;
345 : int rc;
346 :
347 92 : rc = module->submit_tasks(module_ch, task);
348 92 : if (spdk_unlikely(rc != 0)) {
349 2 : accel_update_task_stats(accel_ch, task, failed, 1);
350 : }
351 :
352 92 : return rc;
353 : }
354 :
355 : static inline uint64_t
356 114 : accel_get_iovlen(struct iovec *iovs, uint32_t iovcnt)
357 : {
358 114 : uint64_t result = 0;
359 : uint32_t i;
360 :
361 259 : for (i = 0; i < iovcnt; ++i) {
362 145 : result += iovs[i].iov_len;
363 : }
364 :
365 114 : return result;
366 : }
367 :
368 : #define ACCEL_TASK_ALLOC_AUX_BUF(task) \
369 : do { \
370 : (task)->aux = SLIST_FIRST(&(task)->accel_ch->task_aux_data_pool); \
371 : if (spdk_unlikely(!(task)->aux)) { \
372 : SPDK_ERRLOG("Fatal problem, aux data was not allocated\n"); \
373 : _put_task(task->accel_ch, task); \
374 : assert(0); \
375 : return -ENOMEM; \
376 : } \
377 : SLIST_REMOVE_HEAD(&(task)->accel_ch->task_aux_data_pool, link); \
378 : (task)->has_aux = true; \
379 : } while (0)
380 :
381 : /* Accel framework public API for copy function */
382 : int
383 2 : spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src,
384 : uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
385 : {
386 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
387 : struct spdk_accel_task *accel_task;
388 :
389 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
390 2 : if (spdk_unlikely(accel_task == NULL)) {
391 1 : return -ENOMEM;
392 : }
393 :
394 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
395 :
396 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
397 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
398 1 : accel_task->d.iovs[0].iov_base = dst;
399 1 : accel_task->d.iovs[0].iov_len = nbytes;
400 1 : accel_task->d.iovcnt = 1;
401 1 : accel_task->s.iovs[0].iov_base = src;
402 1 : accel_task->s.iovs[0].iov_len = nbytes;
403 1 : accel_task->s.iovcnt = 1;
404 1 : accel_task->nbytes = nbytes;
405 1 : accel_task->op_code = SPDK_ACCEL_OPC_COPY;
406 1 : accel_task->src_domain = NULL;
407 1 : accel_task->dst_domain = NULL;
408 :
409 1 : return accel_submit_task(accel_ch, accel_task);
410 : }
411 :
412 : /* Accel framework public API for dual cast copy function */
413 : int
414 4 : spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1,
415 : void *dst2, void *src, uint64_t nbytes,
416 : spdk_accel_completion_cb cb_fn, void *cb_arg)
417 : {
418 4 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
419 : struct spdk_accel_task *accel_task;
420 :
421 4 : if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {
422 2 : SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
423 2 : return -EINVAL;
424 : }
425 :
426 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
427 2 : if (spdk_unlikely(accel_task == NULL)) {
428 1 : return -ENOMEM;
429 : }
430 :
431 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
432 :
433 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
434 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
435 1 : accel_task->d2.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST2];
436 1 : accel_task->d.iovs[0].iov_base = dst1;
437 1 : accel_task->d.iovs[0].iov_len = nbytes;
438 1 : accel_task->d.iovcnt = 1;
439 1 : accel_task->d2.iovs[0].iov_base = dst2;
440 1 : accel_task->d2.iovs[0].iov_len = nbytes;
441 1 : accel_task->d2.iovcnt = 1;
442 1 : accel_task->s.iovs[0].iov_base = src;
443 1 : accel_task->s.iovs[0].iov_len = nbytes;
444 1 : accel_task->s.iovcnt = 1;
445 1 : accel_task->nbytes = nbytes;
446 1 : accel_task->op_code = SPDK_ACCEL_OPC_DUALCAST;
447 1 : accel_task->src_domain = NULL;
448 1 : accel_task->dst_domain = NULL;
449 :
450 1 : return accel_submit_task(accel_ch, accel_task);
451 : }
452 :
453 : /* Accel framework public API for compare function */
454 :
455 : int
456 2 : spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1,
457 : void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
458 : void *cb_arg)
459 : {
460 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
461 : struct spdk_accel_task *accel_task;
462 :
463 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
464 2 : if (spdk_unlikely(accel_task == NULL)) {
465 1 : return -ENOMEM;
466 : }
467 :
468 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
469 :
470 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
471 1 : accel_task->s2.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC2];
472 1 : accel_task->s.iovs[0].iov_base = src1;
473 1 : accel_task->s.iovs[0].iov_len = nbytes;
474 1 : accel_task->s.iovcnt = 1;
475 1 : accel_task->s2.iovs[0].iov_base = src2;
476 1 : accel_task->s2.iovs[0].iov_len = nbytes;
477 1 : accel_task->s2.iovcnt = 1;
478 1 : accel_task->nbytes = nbytes;
479 1 : accel_task->op_code = SPDK_ACCEL_OPC_COMPARE;
480 1 : accel_task->src_domain = NULL;
481 1 : accel_task->dst_domain = NULL;
482 :
483 1 : return accel_submit_task(accel_ch, accel_task);
484 : }
485 :
486 : /* Accel framework public API for fill function */
487 : int
488 2 : spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst,
489 : uint8_t fill, uint64_t nbytes,
490 : spdk_accel_completion_cb cb_fn, void *cb_arg)
491 : {
492 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
493 : struct spdk_accel_task *accel_task;
494 :
495 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
496 2 : if (spdk_unlikely(accel_task == NULL)) {
497 1 : return -ENOMEM;
498 : }
499 :
500 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
501 :
502 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
503 1 : accel_task->d.iovs[0].iov_base = dst;
504 1 : accel_task->d.iovs[0].iov_len = nbytes;
505 1 : accel_task->d.iovcnt = 1;
506 1 : accel_task->nbytes = nbytes;
507 1 : memset(&accel_task->fill_pattern, fill, sizeof(uint64_t));
508 1 : accel_task->op_code = SPDK_ACCEL_OPC_FILL;
509 1 : accel_task->src_domain = NULL;
510 1 : accel_task->dst_domain = NULL;
511 :
512 1 : return accel_submit_task(accel_ch, accel_task);
513 : }
514 :
515 : /* Accel framework public API for CRC-32C function */
516 : int
517 2 : spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst,
518 : void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
519 : void *cb_arg)
520 : {
521 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
522 : struct spdk_accel_task *accel_task;
523 :
524 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
525 2 : if (spdk_unlikely(accel_task == NULL)) {
526 1 : return -ENOMEM;
527 : }
528 :
529 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
530 :
531 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
532 1 : accel_task->s.iovs[0].iov_base = src;
533 1 : accel_task->s.iovs[0].iov_len = nbytes;
534 1 : accel_task->s.iovcnt = 1;
535 1 : accel_task->nbytes = nbytes;
536 1 : accel_task->crc_dst = crc_dst;
537 1 : accel_task->seed = seed;
538 1 : accel_task->op_code = SPDK_ACCEL_OPC_CRC32C;
539 1 : accel_task->src_domain = NULL;
540 1 : accel_task->dst_domain = NULL;
541 :
542 1 : return accel_submit_task(accel_ch, accel_task);
543 : }
544 :
545 : /* Accel framework public API for chained CRC-32C function */
546 : int
547 1 : spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst,
548 : struct iovec *iov, uint32_t iov_cnt, uint32_t seed,
549 : spdk_accel_completion_cb cb_fn, void *cb_arg)
550 : {
551 1 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
552 : struct spdk_accel_task *accel_task;
553 :
554 1 : if (iov == NULL) {
555 0 : SPDK_ERRLOG("iov should not be NULL");
556 0 : return -EINVAL;
557 : }
558 :
559 1 : if (!iov_cnt) {
560 0 : SPDK_ERRLOG("iovcnt should not be zero value\n");
561 0 : return -EINVAL;
562 : }
563 :
564 1 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
565 1 : if (spdk_unlikely(accel_task == NULL)) {
566 0 : SPDK_ERRLOG("no memory\n");
567 0 : assert(0);
568 : return -ENOMEM;
569 : }
570 :
571 1 : accel_task->s.iovs = iov;
572 1 : accel_task->s.iovcnt = iov_cnt;
573 1 : accel_task->nbytes = accel_get_iovlen(iov, iov_cnt);
574 1 : accel_task->crc_dst = crc_dst;
575 1 : accel_task->seed = seed;
576 1 : accel_task->op_code = SPDK_ACCEL_OPC_CRC32C;
577 1 : accel_task->src_domain = NULL;
578 1 : accel_task->dst_domain = NULL;
579 :
580 1 : return accel_submit_task(accel_ch, accel_task);
581 : }
582 :
583 : /* Accel framework public API for copy with CRC-32C function */
584 : int
585 2 : spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst,
586 : void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes,
587 : spdk_accel_completion_cb cb_fn, void *cb_arg)
588 : {
589 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
590 : struct spdk_accel_task *accel_task;
591 :
592 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
593 2 : if (spdk_unlikely(accel_task == NULL)) {
594 1 : return -ENOMEM;
595 : }
596 :
597 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
598 :
599 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
600 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
601 1 : accel_task->d.iovs[0].iov_base = dst;
602 1 : accel_task->d.iovs[0].iov_len = nbytes;
603 1 : accel_task->d.iovcnt = 1;
604 1 : accel_task->s.iovs[0].iov_base = src;
605 1 : accel_task->s.iovs[0].iov_len = nbytes;
606 1 : accel_task->s.iovcnt = 1;
607 1 : accel_task->nbytes = nbytes;
608 1 : accel_task->crc_dst = crc_dst;
609 1 : accel_task->seed = seed;
610 1 : accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C;
611 1 : accel_task->src_domain = NULL;
612 1 : accel_task->dst_domain = NULL;
613 :
614 1 : return accel_submit_task(accel_ch, accel_task);
615 : }
616 :
617 : /* Accel framework public API for chained copy + CRC-32C function */
618 : int
619 0 : spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst,
620 : struct iovec *src_iovs, uint32_t iov_cnt, uint32_t *crc_dst,
621 : uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg)
622 : {
623 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
624 : struct spdk_accel_task *accel_task;
625 : uint64_t nbytes;
626 :
627 0 : if (src_iovs == NULL) {
628 0 : SPDK_ERRLOG("iov should not be NULL");
629 0 : return -EINVAL;
630 : }
631 :
632 0 : if (!iov_cnt) {
633 0 : SPDK_ERRLOG("iovcnt should not be zero value\n");
634 0 : return -EINVAL;
635 : }
636 :
637 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
638 0 : if (spdk_unlikely(accel_task == NULL)) {
639 0 : SPDK_ERRLOG("no memory\n");
640 0 : assert(0);
641 : return -ENOMEM;
642 : }
643 :
644 0 : nbytes = accel_get_iovlen(src_iovs, iov_cnt);
645 :
646 0 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
647 :
648 0 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
649 0 : accel_task->d.iovs[0].iov_base = dst;
650 0 : accel_task->d.iovs[0].iov_len = nbytes;
651 0 : accel_task->d.iovcnt = 1;
652 0 : accel_task->s.iovs = src_iovs;
653 0 : accel_task->s.iovcnt = iov_cnt;
654 0 : accel_task->nbytes = nbytes;
655 0 : accel_task->crc_dst = crc_dst;
656 0 : accel_task->seed = seed;
657 0 : accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C;
658 0 : accel_task->src_domain = NULL;
659 0 : accel_task->dst_domain = NULL;
660 :
661 0 : return accel_submit_task(accel_ch, accel_task);
662 : }
663 :
664 : int
665 0 : spdk_accel_submit_compress(struct spdk_io_channel *ch, void *dst, uint64_t nbytes,
666 : struct iovec *src_iovs, size_t src_iovcnt, uint32_t *output_size,
667 : spdk_accel_completion_cb cb_fn, void *cb_arg)
668 : {
669 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
670 : struct spdk_accel_task *accel_task;
671 :
672 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
673 0 : if (spdk_unlikely(accel_task == NULL)) {
674 0 : return -ENOMEM;
675 : }
676 :
677 0 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
678 :
679 0 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
680 0 : accel_task->d.iovs[0].iov_base = dst;
681 0 : accel_task->d.iovs[0].iov_len = nbytes;
682 0 : accel_task->d.iovcnt = 1;
683 0 : accel_task->output_size = output_size;
684 0 : accel_task->s.iovs = src_iovs;
685 0 : accel_task->s.iovcnt = src_iovcnt;
686 0 : accel_task->nbytes = nbytes;
687 0 : accel_task->op_code = SPDK_ACCEL_OPC_COMPRESS;
688 0 : accel_task->src_domain = NULL;
689 0 : accel_task->dst_domain = NULL;
690 :
691 0 : return accel_submit_task(accel_ch, accel_task);
692 : }
693 :
694 : int
695 0 : spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs,
696 : size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt,
697 : uint32_t *output_size, spdk_accel_completion_cb cb_fn,
698 : void *cb_arg)
699 : {
700 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
701 : struct spdk_accel_task *accel_task;
702 :
703 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
704 0 : if (spdk_unlikely(accel_task == NULL)) {
705 0 : return -ENOMEM;
706 : }
707 :
708 0 : accel_task->output_size = output_size;
709 0 : accel_task->s.iovs = src_iovs;
710 0 : accel_task->s.iovcnt = src_iovcnt;
711 0 : accel_task->d.iovs = dst_iovs;
712 0 : accel_task->d.iovcnt = dst_iovcnt;
713 0 : accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
714 0 : accel_task->op_code = SPDK_ACCEL_OPC_DECOMPRESS;
715 0 : accel_task->src_domain = NULL;
716 0 : accel_task->dst_domain = NULL;
717 :
718 0 : return accel_submit_task(accel_ch, accel_task);
719 : }
720 :
721 : int
722 0 : spdk_accel_submit_encrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key,
723 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
724 : struct iovec *src_iovs, uint32_t src_iovcnt,
725 : uint64_t iv, uint32_t block_size,
726 : spdk_accel_completion_cb cb_fn, void *cb_arg)
727 : {
728 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
729 : struct spdk_accel_task *accel_task;
730 :
731 0 : if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) {
732 0 : return -EINVAL;
733 : }
734 :
735 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
736 0 : if (spdk_unlikely(accel_task == NULL)) {
737 0 : return -ENOMEM;
738 : }
739 :
740 0 : accel_task->crypto_key = key;
741 0 : accel_task->s.iovs = src_iovs;
742 0 : accel_task->s.iovcnt = src_iovcnt;
743 0 : accel_task->d.iovs = dst_iovs;
744 0 : accel_task->d.iovcnt = dst_iovcnt;
745 0 : accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
746 0 : accel_task->iv = iv;
747 0 : accel_task->block_size = block_size;
748 0 : accel_task->op_code = SPDK_ACCEL_OPC_ENCRYPT;
749 0 : accel_task->src_domain = NULL;
750 0 : accel_task->dst_domain = NULL;
751 :
752 0 : return accel_submit_task(accel_ch, accel_task);
753 : }
754 :
755 : int
756 0 : spdk_accel_submit_decrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key,
757 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
758 : struct iovec *src_iovs, uint32_t src_iovcnt,
759 : uint64_t iv, uint32_t block_size,
760 : spdk_accel_completion_cb cb_fn, void *cb_arg)
761 : {
762 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
763 : struct spdk_accel_task *accel_task;
764 :
765 0 : if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) {
766 0 : return -EINVAL;
767 : }
768 :
769 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
770 0 : if (spdk_unlikely(accel_task == NULL)) {
771 0 : return -ENOMEM;
772 : }
773 :
774 0 : accel_task->crypto_key = key;
775 0 : accel_task->s.iovs = src_iovs;
776 0 : accel_task->s.iovcnt = src_iovcnt;
777 0 : accel_task->d.iovs = dst_iovs;
778 0 : accel_task->d.iovcnt = dst_iovcnt;
779 0 : accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
780 0 : accel_task->iv = iv;
781 0 : accel_task->block_size = block_size;
782 0 : accel_task->op_code = SPDK_ACCEL_OPC_DECRYPT;
783 0 : accel_task->src_domain = NULL;
784 0 : accel_task->dst_domain = NULL;
785 :
786 0 : return accel_submit_task(accel_ch, accel_task);
787 : }
788 :
789 : int
790 2 : spdk_accel_submit_xor(struct spdk_io_channel *ch, void *dst, void **sources, uint32_t nsrcs,
791 : uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
792 : {
793 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
794 : struct spdk_accel_task *accel_task;
795 :
796 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
797 2 : if (spdk_unlikely(accel_task == NULL)) {
798 1 : return -ENOMEM;
799 : }
800 :
801 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
802 :
803 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
804 1 : accel_task->nsrcs.srcs = sources;
805 1 : accel_task->nsrcs.cnt = nsrcs;
806 1 : accel_task->d.iovs[0].iov_base = dst;
807 1 : accel_task->d.iovs[0].iov_len = nbytes;
808 1 : accel_task->d.iovcnt = 1;
809 1 : accel_task->nbytes = nbytes;
810 1 : accel_task->op_code = SPDK_ACCEL_OPC_XOR;
811 1 : accel_task->src_domain = NULL;
812 1 : accel_task->dst_domain = NULL;
813 :
814 1 : return accel_submit_task(accel_ch, accel_task);
815 : }
816 :
817 : int
818 0 : spdk_accel_submit_dif_verify(struct spdk_io_channel *ch,
819 : struct iovec *iovs, size_t iovcnt, uint32_t num_blocks,
820 : const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
821 : spdk_accel_completion_cb cb_fn, void *cb_arg)
822 : {
823 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
824 : struct spdk_accel_task *accel_task;
825 :
826 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
827 0 : if (accel_task == NULL) {
828 0 : return -ENOMEM;
829 : }
830 :
831 0 : accel_task->s.iovs = iovs;
832 0 : accel_task->s.iovcnt = iovcnt;
833 0 : accel_task->dif.ctx = ctx;
834 0 : accel_task->dif.err = err;
835 0 : accel_task->dif.num_blocks = num_blocks;
836 0 : accel_task->nbytes = num_blocks * ctx->block_size;
837 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY;
838 0 : accel_task->src_domain = NULL;
839 0 : accel_task->dst_domain = NULL;
840 :
841 0 : return accel_submit_task(accel_ch, accel_task);
842 : }
843 :
844 : int
845 0 : spdk_accel_submit_dif_generate(struct spdk_io_channel *ch,
846 : struct iovec *iovs, size_t iovcnt, uint32_t num_blocks,
847 : const struct spdk_dif_ctx *ctx,
848 : spdk_accel_completion_cb cb_fn, void *cb_arg)
849 : {
850 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
851 : struct spdk_accel_task *accel_task;
852 :
853 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
854 0 : if (accel_task == NULL) {
855 0 : return -ENOMEM;
856 : }
857 :
858 0 : accel_task->s.iovs = iovs;
859 0 : accel_task->s.iovcnt = iovcnt;
860 0 : accel_task->dif.ctx = ctx;
861 0 : accel_task->dif.num_blocks = num_blocks;
862 0 : accel_task->nbytes = num_blocks * ctx->block_size;
863 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE;
864 0 : accel_task->src_domain = NULL;
865 0 : accel_task->dst_domain = NULL;
866 :
867 0 : return accel_submit_task(accel_ch, accel_task);
868 : }
869 :
870 : int
871 0 : spdk_accel_submit_dif_generate_copy(struct spdk_io_channel *ch, struct iovec *dst_iovs,
872 : size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt,
873 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
874 : spdk_accel_completion_cb cb_fn, void *cb_arg)
875 : {
876 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
877 : struct spdk_accel_task *accel_task;
878 :
879 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
880 0 : if (accel_task == NULL) {
881 0 : return -ENOMEM;
882 : }
883 :
884 0 : accel_task->s.iovs = src_iovs;
885 0 : accel_task->s.iovcnt = src_iovcnt;
886 0 : accel_task->d.iovs = dst_iovs;
887 0 : accel_task->d.iovcnt = dst_iovcnt;
888 0 : accel_task->dif.ctx = ctx;
889 0 : accel_task->dif.num_blocks = num_blocks;
890 0 : accel_task->nbytes = num_blocks * ctx->block_size;
891 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE_COPY;
892 0 : accel_task->src_domain = NULL;
893 0 : accel_task->dst_domain = NULL;
894 :
895 0 : return accel_submit_task(accel_ch, accel_task);
896 : }
897 :
898 : int
899 0 : spdk_accel_submit_dif_verify_copy(struct spdk_io_channel *ch,
900 : struct iovec *dst_iovs, size_t dst_iovcnt,
901 : struct iovec *src_iovs, size_t src_iovcnt, uint32_t num_blocks,
902 : const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
903 : spdk_accel_completion_cb cb_fn, void *cb_arg)
904 : {
905 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
906 : struct spdk_accel_task *accel_task;
907 :
908 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
909 0 : if (accel_task == NULL) {
910 0 : return -ENOMEM;
911 : }
912 :
913 0 : accel_task->s.iovs = src_iovs;
914 0 : accel_task->s.iovcnt = src_iovcnt;
915 0 : accel_task->d.iovs = dst_iovs;
916 0 : accel_task->d.iovcnt = dst_iovcnt;
917 0 : accel_task->dif.ctx = ctx;
918 0 : accel_task->dif.err = err;
919 0 : accel_task->dif.num_blocks = num_blocks;
920 0 : accel_task->nbytes = num_blocks * ctx->block_size;
921 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY_COPY;
922 0 : accel_task->src_domain = NULL;
923 0 : accel_task->dst_domain = NULL;
924 :
925 0 : return accel_submit_task(accel_ch, accel_task);
926 : }
927 :
928 : static inline struct accel_buffer *
929 33 : accel_get_buf(struct accel_io_channel *ch, uint64_t len)
930 : {
931 : struct accel_buffer *buf;
932 :
933 33 : buf = SLIST_FIRST(&ch->buf_pool);
934 33 : if (spdk_unlikely(buf == NULL)) {
935 0 : accel_update_stats(ch, retry.bufdesc, 1);
936 0 : return NULL;
937 : }
938 :
939 33 : SLIST_REMOVE_HEAD(&ch->buf_pool, link);
940 33 : buf->len = len;
941 33 : buf->buf = NULL;
942 33 : buf->seq = NULL;
943 33 : buf->cb_fn = NULL;
944 :
945 33 : return buf;
946 : }
947 :
948 : static inline void
949 33 : accel_put_buf(struct accel_io_channel *ch, struct accel_buffer *buf)
950 : {
951 33 : if (buf->buf != NULL) {
952 29 : spdk_iobuf_put(&ch->iobuf, buf->buf, buf->len);
953 : }
954 :
955 33 : SLIST_INSERT_HEAD(&ch->buf_pool, buf, link);
956 33 : }
957 :
958 : static inline struct spdk_accel_sequence *
959 60 : accel_sequence_get(struct accel_io_channel *ch)
960 : {
961 : struct spdk_accel_sequence *seq;
962 :
963 60 : seq = SLIST_FIRST(&ch->seq_pool);
964 60 : if (spdk_unlikely(seq == NULL)) {
965 3 : accel_update_stats(ch, retry.sequence, 1);
966 3 : return NULL;
967 : }
968 :
969 57 : accel_update_stats(ch, sequence_outstanding, 1);
970 57 : SLIST_REMOVE_HEAD(&ch->seq_pool, link);
971 :
972 57 : TAILQ_INIT(&seq->tasks);
973 57 : SLIST_INIT(&seq->bounce_bufs);
974 :
975 57 : seq->ch = ch;
976 57 : seq->status = 0;
977 57 : seq->state = ACCEL_SEQUENCE_STATE_INIT;
978 57 : seq->in_process_sequence = false;
979 :
980 57 : return seq;
981 : }
982 :
983 : static inline void
984 57 : accel_sequence_put(struct spdk_accel_sequence *seq)
985 : {
986 57 : struct accel_io_channel *ch = seq->ch;
987 : struct accel_buffer *buf;
988 :
989 76 : while (!SLIST_EMPTY(&seq->bounce_bufs)) {
990 19 : buf = SLIST_FIRST(&seq->bounce_bufs);
991 19 : SLIST_REMOVE_HEAD(&seq->bounce_bufs, link);
992 19 : accel_put_buf(seq->ch, buf);
993 : }
994 :
995 57 : assert(TAILQ_EMPTY(&seq->tasks));
996 57 : seq->ch = NULL;
997 :
998 57 : SLIST_INSERT_HEAD(&ch->seq_pool, seq, link);
999 57 : accel_update_stats(ch, sequence_outstanding, -1);
1000 57 : }
1001 :
1002 : static void accel_sequence_task_cb(void *cb_arg, int status);
1003 :
1004 : static inline struct spdk_accel_task *
1005 134 : accel_sequence_get_task(struct accel_io_channel *ch, struct spdk_accel_sequence *seq,
1006 : spdk_accel_step_cb cb_fn, void *cb_arg)
1007 : {
1008 : struct spdk_accel_task *task;
1009 :
1010 134 : task = _get_task(ch, NULL, NULL);
1011 134 : if (spdk_unlikely(task == NULL)) {
1012 3 : return task;
1013 : }
1014 :
1015 131 : task->step_cb_fn = cb_fn;
1016 131 : task->cb_arg = cb_arg;
1017 131 : task->seq = seq;
1018 :
1019 131 : return task;
1020 : }
1021 :
1022 : int
1023 35 : spdk_accel_append_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1024 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
1025 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1026 : struct iovec *src_iovs, uint32_t src_iovcnt,
1027 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1028 : spdk_accel_step_cb cb_fn, void *cb_arg)
1029 : {
1030 35 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1031 : struct spdk_accel_task *task;
1032 35 : struct spdk_accel_sequence *seq = *pseq;
1033 :
1034 35 : if (seq == NULL) {
1035 12 : seq = accel_sequence_get(accel_ch);
1036 12 : if (spdk_unlikely(seq == NULL)) {
1037 1 : return -ENOMEM;
1038 : }
1039 : }
1040 :
1041 34 : assert(seq->ch == accel_ch);
1042 34 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1043 34 : if (spdk_unlikely(task == NULL)) {
1044 1 : if (*pseq == NULL) {
1045 1 : accel_sequence_put(seq);
1046 : }
1047 :
1048 1 : return -ENOMEM;
1049 : }
1050 :
1051 33 : task->dst_domain = dst_domain;
1052 33 : task->dst_domain_ctx = dst_domain_ctx;
1053 33 : task->d.iovs = dst_iovs;
1054 33 : task->d.iovcnt = dst_iovcnt;
1055 33 : task->src_domain = src_domain;
1056 33 : task->src_domain_ctx = src_domain_ctx;
1057 33 : task->s.iovs = src_iovs;
1058 33 : task->s.iovcnt = src_iovcnt;
1059 33 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1060 33 : task->op_code = SPDK_ACCEL_OPC_COPY;
1061 :
1062 33 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1063 33 : *pseq = seq;
1064 :
1065 33 : return 0;
1066 : }
1067 :
1068 : int
1069 39 : spdk_accel_append_fill(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1070 : void *buf, uint64_t len,
1071 : struct spdk_memory_domain *domain, void *domain_ctx, uint8_t pattern,
1072 : spdk_accel_step_cb cb_fn, void *cb_arg)
1073 : {
1074 39 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1075 : struct spdk_accel_task *task;
1076 39 : struct spdk_accel_sequence *seq = *pseq;
1077 :
1078 39 : if (seq == NULL) {
1079 23 : seq = accel_sequence_get(accel_ch);
1080 23 : if (spdk_unlikely(seq == NULL)) {
1081 1 : return -ENOMEM;
1082 : }
1083 : }
1084 :
1085 38 : assert(seq->ch == accel_ch);
1086 38 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1087 38 : if (spdk_unlikely(task == NULL)) {
1088 1 : if (*pseq == NULL) {
1089 1 : accel_sequence_put(seq);
1090 : }
1091 :
1092 1 : return -ENOMEM;
1093 : }
1094 :
1095 37 : memset(&task->fill_pattern, pattern, sizeof(uint64_t));
1096 :
1097 37 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1098 37 : if (spdk_unlikely(!task->aux)) {
1099 0 : SPDK_ERRLOG("Fatal problem, aux data was not allocated\n");
1100 0 : if (*pseq == NULL) {
1101 0 : accel_sequence_put((seq));
1102 : }
1103 :
1104 0 : task->seq = NULL;
1105 0 : _put_task(task->accel_ch, task);
1106 0 : assert(0);
1107 : return -ENOMEM;
1108 : }
1109 37 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1110 37 : task->has_aux = true;
1111 :
1112 37 : task->d.iovs = &task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
1113 37 : task->d.iovs[0].iov_base = buf;
1114 37 : task->d.iovs[0].iov_len = len;
1115 37 : task->d.iovcnt = 1;
1116 37 : task->nbytes = len;
1117 37 : task->src_domain = NULL;
1118 37 : task->dst_domain = domain;
1119 37 : task->dst_domain_ctx = domain_ctx;
1120 37 : task->op_code = SPDK_ACCEL_OPC_FILL;
1121 :
1122 37 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1123 37 : *pseq = seq;
1124 :
1125 37 : return 0;
1126 : }
1127 :
1128 : int
1129 40 : spdk_accel_append_decompress(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1130 : struct iovec *dst_iovs, size_t dst_iovcnt,
1131 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1132 : struct iovec *src_iovs, size_t src_iovcnt,
1133 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1134 : spdk_accel_step_cb cb_fn, void *cb_arg)
1135 : {
1136 40 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1137 : struct spdk_accel_task *task;
1138 40 : struct spdk_accel_sequence *seq = *pseq;
1139 :
1140 40 : if (seq == NULL) {
1141 17 : seq = accel_sequence_get(accel_ch);
1142 17 : if (spdk_unlikely(seq == NULL)) {
1143 1 : return -ENOMEM;
1144 : }
1145 : }
1146 :
1147 39 : assert(seq->ch == accel_ch);
1148 39 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1149 39 : if (spdk_unlikely(task == NULL)) {
1150 1 : if (*pseq == NULL) {
1151 1 : accel_sequence_put(seq);
1152 : }
1153 :
1154 1 : return -ENOMEM;
1155 : }
1156 :
1157 : /* TODO: support output_size for chaining */
1158 38 : task->output_size = NULL;
1159 38 : task->dst_domain = dst_domain;
1160 38 : task->dst_domain_ctx = dst_domain_ctx;
1161 38 : task->d.iovs = dst_iovs;
1162 38 : task->d.iovcnt = dst_iovcnt;
1163 38 : task->src_domain = src_domain;
1164 38 : task->src_domain_ctx = src_domain_ctx;
1165 38 : task->s.iovs = src_iovs;
1166 38 : task->s.iovcnt = src_iovcnt;
1167 38 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1168 38 : task->op_code = SPDK_ACCEL_OPC_DECOMPRESS;
1169 :
1170 38 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1171 38 : *pseq = seq;
1172 :
1173 38 : return 0;
1174 : }
1175 :
1176 : int
1177 8 : spdk_accel_append_encrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1178 : struct spdk_accel_crypto_key *key,
1179 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
1180 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1181 : struct iovec *src_iovs, uint32_t src_iovcnt,
1182 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1183 : uint64_t iv, uint32_t block_size,
1184 : spdk_accel_step_cb cb_fn, void *cb_arg)
1185 : {
1186 8 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1187 : struct spdk_accel_task *task;
1188 8 : struct spdk_accel_sequence *seq = *pseq;
1189 :
1190 8 : assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size);
1191 :
1192 8 : if (seq == NULL) {
1193 6 : seq = accel_sequence_get(accel_ch);
1194 6 : if (spdk_unlikely(seq == NULL)) {
1195 0 : return -ENOMEM;
1196 : }
1197 : }
1198 :
1199 8 : assert(seq->ch == accel_ch);
1200 8 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1201 8 : if (spdk_unlikely(task == NULL)) {
1202 0 : if (*pseq == NULL) {
1203 0 : accel_sequence_put(seq);
1204 : }
1205 :
1206 0 : return -ENOMEM;
1207 : }
1208 :
1209 8 : task->crypto_key = key;
1210 8 : task->src_domain = src_domain;
1211 8 : task->src_domain_ctx = src_domain_ctx;
1212 8 : task->s.iovs = src_iovs;
1213 8 : task->s.iovcnt = src_iovcnt;
1214 8 : task->dst_domain = dst_domain;
1215 8 : task->dst_domain_ctx = dst_domain_ctx;
1216 8 : task->d.iovs = dst_iovs;
1217 8 : task->d.iovcnt = dst_iovcnt;
1218 8 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1219 8 : task->iv = iv;
1220 8 : task->block_size = block_size;
1221 8 : task->op_code = SPDK_ACCEL_OPC_ENCRYPT;
1222 :
1223 8 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1224 8 : *pseq = seq;
1225 :
1226 8 : return 0;
1227 : }
1228 :
1229 : int
1230 9 : spdk_accel_append_decrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1231 : struct spdk_accel_crypto_key *key,
1232 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
1233 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1234 : struct iovec *src_iovs, uint32_t src_iovcnt,
1235 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1236 : uint64_t iv, uint32_t block_size,
1237 : spdk_accel_step_cb cb_fn, void *cb_arg)
1238 : {
1239 9 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1240 : struct spdk_accel_task *task;
1241 9 : struct spdk_accel_sequence *seq = *pseq;
1242 :
1243 9 : assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size);
1244 :
1245 9 : if (seq == NULL) {
1246 0 : seq = accel_sequence_get(accel_ch);
1247 0 : if (spdk_unlikely(seq == NULL)) {
1248 0 : return -ENOMEM;
1249 : }
1250 : }
1251 :
1252 9 : assert(seq->ch == accel_ch);
1253 9 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1254 9 : if (spdk_unlikely(task == NULL)) {
1255 0 : if (*pseq == NULL) {
1256 0 : accel_sequence_put(seq);
1257 : }
1258 :
1259 0 : return -ENOMEM;
1260 : }
1261 :
1262 9 : task->crypto_key = key;
1263 9 : task->src_domain = src_domain;
1264 9 : task->src_domain_ctx = src_domain_ctx;
1265 9 : task->s.iovs = src_iovs;
1266 9 : task->s.iovcnt = src_iovcnt;
1267 9 : task->dst_domain = dst_domain;
1268 9 : task->dst_domain_ctx = dst_domain_ctx;
1269 9 : task->d.iovs = dst_iovs;
1270 9 : task->d.iovcnt = dst_iovcnt;
1271 9 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1272 9 : task->iv = iv;
1273 9 : task->block_size = block_size;
1274 9 : task->op_code = SPDK_ACCEL_OPC_DECRYPT;
1275 :
1276 9 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1277 9 : *pseq = seq;
1278 :
1279 9 : return 0;
1280 : }
1281 :
1282 : int
1283 6 : spdk_accel_append_crc32c(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1284 : uint32_t *dst, struct iovec *iovs, uint32_t iovcnt,
1285 : struct spdk_memory_domain *domain, void *domain_ctx,
1286 : uint32_t seed, spdk_accel_step_cb cb_fn, void *cb_arg)
1287 : {
1288 6 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1289 : struct spdk_accel_task *task;
1290 6 : struct spdk_accel_sequence *seq = *pseq;
1291 :
1292 6 : if (seq == NULL) {
1293 2 : seq = accel_sequence_get(accel_ch);
1294 2 : if (spdk_unlikely(seq == NULL)) {
1295 0 : return -ENOMEM;
1296 : }
1297 : }
1298 :
1299 6 : assert(seq->ch == accel_ch);
1300 6 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1301 6 : if (spdk_unlikely(task == NULL)) {
1302 0 : if (*pseq == NULL) {
1303 0 : accel_sequence_put(seq);
1304 : }
1305 :
1306 0 : return -ENOMEM;
1307 : }
1308 :
1309 6 : task->s.iovs = iovs;
1310 6 : task->s.iovcnt = iovcnt;
1311 6 : task->src_domain = domain;
1312 6 : task->src_domain_ctx = domain_ctx;
1313 6 : task->nbytes = accel_get_iovlen(iovs, iovcnt);
1314 6 : task->crc_dst = dst;
1315 6 : task->seed = seed;
1316 6 : task->op_code = SPDK_ACCEL_OPC_CRC32C;
1317 6 : task->dst_domain = NULL;
1318 :
1319 6 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1320 6 : *pseq = seq;
1321 :
1322 6 : return 0;
1323 : }
1324 :
1325 : int
1326 14 : spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf,
1327 : struct spdk_memory_domain **domain, void **domain_ctx)
1328 : {
1329 14 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1330 : struct accel_buffer *accel_buf;
1331 :
1332 14 : accel_buf = accel_get_buf(accel_ch, len);
1333 14 : if (spdk_unlikely(accel_buf == NULL)) {
1334 0 : return -ENOMEM;
1335 : }
1336 :
1337 14 : accel_buf->ch = accel_ch;
1338 :
1339 : /* We always return the same pointer and identify the buffers through domain_ctx */
1340 14 : *buf = ACCEL_BUFFER_BASE;
1341 14 : *domain_ctx = accel_buf;
1342 14 : *domain = g_accel_domain;
1343 :
1344 14 : return 0;
1345 : }
1346 :
1347 : void
1348 14 : spdk_accel_put_buf(struct spdk_io_channel *ch, void *buf,
1349 : struct spdk_memory_domain *domain, void *domain_ctx)
1350 : {
1351 14 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1352 14 : struct accel_buffer *accel_buf = domain_ctx;
1353 :
1354 14 : assert(domain == g_accel_domain);
1355 14 : assert(buf == ACCEL_BUFFER_BASE);
1356 :
1357 14 : accel_put_buf(accel_ch, accel_buf);
1358 14 : }
1359 :
1360 : static void
1361 131 : accel_sequence_complete_task(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
1362 : {
1363 131 : struct accel_io_channel *ch = seq->ch;
1364 : spdk_accel_step_cb cb_fn;
1365 : void *cb_arg;
1366 :
1367 131 : TAILQ_REMOVE(&seq->tasks, task, seq_link);
1368 131 : cb_fn = task->step_cb_fn;
1369 131 : cb_arg = task->cb_arg;
1370 131 : task->seq = NULL;
1371 131 : if (task->has_aux) {
1372 58 : SLIST_INSERT_HEAD(&ch->task_aux_data_pool, task->aux, link);
1373 58 : task->aux = NULL;
1374 58 : task->has_aux = false;
1375 : }
1376 :
1377 131 : _put_task(ch, task);
1378 :
1379 131 : if (cb_fn != NULL) {
1380 131 : cb_fn(cb_arg);
1381 : }
1382 131 : }
1383 :
1384 : static void
1385 54 : accel_sequence_complete_tasks(struct spdk_accel_sequence *seq)
1386 : {
1387 : struct spdk_accel_task *task;
1388 :
1389 72 : while (!TAILQ_EMPTY(&seq->tasks)) {
1390 18 : task = TAILQ_FIRST(&seq->tasks);
1391 18 : accel_sequence_complete_task(seq, task);
1392 : }
1393 54 : }
1394 :
1395 : static void
1396 52 : accel_sequence_complete(struct spdk_accel_sequence *seq)
1397 : {
1398 52 : spdk_accel_completion_cb cb_fn = seq->cb_fn;
1399 52 : void *cb_arg = seq->cb_arg;
1400 52 : int status = seq->status;
1401 :
1402 52 : SPDK_DEBUGLOG(accel, "Completed sequence: %p with status: %d\n", seq, status);
1403 :
1404 52 : accel_update_stats(seq->ch, sequence_executed, 1);
1405 52 : if (spdk_unlikely(status != 0)) {
1406 10 : accel_update_stats(seq->ch, sequence_failed, 1);
1407 : }
1408 :
1409 : /* First notify all users that appended operations to this sequence */
1410 52 : accel_sequence_complete_tasks(seq);
1411 52 : accel_sequence_put(seq);
1412 :
1413 : /* Then notify the user that finished the sequence */
1414 52 : cb_fn(cb_arg, status);
1415 52 : }
1416 :
1417 : static void
1418 28 : accel_update_virt_iov(struct iovec *diov, struct iovec *siov, struct accel_buffer *accel_buf)
1419 : {
1420 : uintptr_t offset;
1421 :
1422 28 : offset = (uintptr_t)siov->iov_base & ACCEL_BUFFER_OFFSET_MASK;
1423 28 : assert(offset < accel_buf->len);
1424 :
1425 28 : diov->iov_base = (char *)accel_buf->buf + offset;
1426 28 : diov->iov_len = siov->iov_len;
1427 28 : }
1428 :
1429 : static void
1430 10 : accel_sequence_set_virtbuf(struct spdk_accel_sequence *seq, struct accel_buffer *buf)
1431 : {
1432 : struct spdk_accel_task *task;
1433 : struct iovec *iov;
1434 :
1435 : /* Now that we've allocated the actual data buffer for this accel_buffer, update all tasks
1436 : * in a sequence that were using it.
1437 : */
1438 38 : TAILQ_FOREACH(task, &seq->tasks, seq_link) {
1439 28 : if (task->src_domain == g_accel_domain && task->src_domain_ctx == buf) {
1440 11 : if (!task->has_aux) {
1441 11 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1442 11 : assert(task->aux && "Can't allocate aux data structure");
1443 11 : task->has_aux = true;
1444 11 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1445 : }
1446 :
1447 11 : iov = &task->aux->iovs[SPDK_ACCEL_AXU_IOV_VIRT_SRC];
1448 11 : assert(task->s.iovcnt == 1);
1449 11 : accel_update_virt_iov(iov, &task->s.iovs[0], buf);
1450 11 : task->src_domain = NULL;
1451 11 : task->s.iovs = iov;
1452 : }
1453 28 : if (task->dst_domain == g_accel_domain && task->dst_domain_ctx == buf) {
1454 17 : if (!task->has_aux) {
1455 2 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1456 2 : assert(task->aux && "Can't allocate aux data structure");
1457 2 : task->has_aux = true;
1458 2 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1459 : }
1460 :
1461 17 : iov = &task->aux->iovs[SPDK_ACCEL_AXU_IOV_VIRT_DST];
1462 17 : assert(task->d.iovcnt == 1);
1463 17 : accel_update_virt_iov(iov, &task->d.iovs[0], buf);
1464 17 : task->dst_domain = NULL;
1465 17 : task->d.iovs = iov;
1466 : }
1467 : }
1468 10 : }
1469 :
1470 : static void accel_process_sequence(struct spdk_accel_sequence *seq);
1471 :
1472 : static void
1473 3 : accel_iobuf_get_virtbuf_cb(struct spdk_iobuf_entry *entry, void *buf)
1474 : {
1475 : struct accel_buffer *accel_buf;
1476 :
1477 3 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
1478 :
1479 3 : assert(accel_buf->seq != NULL);
1480 3 : assert(accel_buf->buf == NULL);
1481 3 : accel_buf->buf = buf;
1482 :
1483 3 : assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF);
1484 3 : accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF);
1485 3 : accel_sequence_set_virtbuf(accel_buf->seq, accel_buf);
1486 3 : accel_process_sequence(accel_buf->seq);
1487 3 : }
1488 :
1489 : static bool
1490 29 : accel_sequence_alloc_buf(struct spdk_accel_sequence *seq, struct accel_buffer *buf,
1491 : spdk_iobuf_get_cb cb_fn)
1492 : {
1493 29 : struct accel_io_channel *ch = seq->ch;
1494 :
1495 29 : assert(buf->seq == NULL);
1496 :
1497 29 : buf->seq = seq;
1498 :
1499 : /* Buffer might be already allocated by memory domain translation. */
1500 29 : if (buf->buf) {
1501 0 : return true;
1502 : }
1503 :
1504 29 : buf->buf = spdk_iobuf_get(&ch->iobuf, buf->len, &buf->iobuf, cb_fn);
1505 29 : if (spdk_unlikely(buf->buf == NULL)) {
1506 5 : accel_update_stats(ch, retry.iobuf, 1);
1507 5 : return false;
1508 : }
1509 :
1510 24 : return true;
1511 : }
1512 :
1513 : static bool
1514 89 : accel_sequence_check_virtbuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
1515 : {
1516 : /* If a task doesn't have dst/src (e.g. fill, crc32), its dst/src domain should be set to
1517 : * NULL */
1518 89 : if (task->src_domain == g_accel_domain) {
1519 0 : if (!accel_sequence_alloc_buf(seq, task->src_domain_ctx,
1520 : accel_iobuf_get_virtbuf_cb)) {
1521 0 : return false;
1522 : }
1523 :
1524 0 : accel_sequence_set_virtbuf(seq, task->src_domain_ctx);
1525 : }
1526 :
1527 89 : if (task->dst_domain == g_accel_domain) {
1528 10 : if (!accel_sequence_alloc_buf(seq, task->dst_domain_ctx,
1529 : accel_iobuf_get_virtbuf_cb)) {
1530 3 : return false;
1531 : }
1532 :
1533 7 : accel_sequence_set_virtbuf(seq, task->dst_domain_ctx);
1534 : }
1535 :
1536 86 : return true;
1537 : }
1538 :
1539 : static void
1540 0 : accel_sequence_get_buf_cb(struct spdk_iobuf_entry *entry, void *buf)
1541 : {
1542 : struct accel_buffer *accel_buf;
1543 :
1544 0 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
1545 :
1546 0 : assert(accel_buf->seq != NULL);
1547 0 : assert(accel_buf->buf == NULL);
1548 0 : accel_buf->buf = buf;
1549 :
1550 0 : accel_sequence_set_virtbuf(accel_buf->seq, accel_buf);
1551 0 : accel_buf->cb_fn(accel_buf->seq, accel_buf->cb_ctx);
1552 0 : }
1553 :
1554 : bool
1555 0 : spdk_accel_alloc_sequence_buf(struct spdk_accel_sequence *seq, void *buf,
1556 : struct spdk_memory_domain *domain, void *domain_ctx,
1557 : spdk_accel_sequence_get_buf_cb cb_fn, void *cb_ctx)
1558 : {
1559 0 : struct accel_buffer *accel_buf = domain_ctx;
1560 :
1561 0 : assert(domain == g_accel_domain);
1562 0 : accel_buf->cb_fn = cb_fn;
1563 0 : accel_buf->cb_ctx = cb_ctx;
1564 :
1565 0 : if (!accel_sequence_alloc_buf(seq, accel_buf, accel_sequence_get_buf_cb)) {
1566 0 : return false;
1567 : }
1568 :
1569 0 : accel_sequence_set_virtbuf(seq, accel_buf);
1570 :
1571 0 : return true;
1572 : }
1573 :
1574 : struct spdk_accel_task *
1575 24 : spdk_accel_sequence_first_task(struct spdk_accel_sequence *seq)
1576 : {
1577 24 : return TAILQ_FIRST(&seq->tasks);
1578 : }
1579 :
1580 : struct spdk_accel_task *
1581 0 : spdk_accel_sequence_next_task(struct spdk_accel_task *task)
1582 : {
1583 0 : return TAILQ_NEXT(task, seq_link);
1584 : }
1585 :
1586 : static inline void
1587 19 : accel_set_bounce_buffer(struct spdk_accel_bounce_buffer *bounce, struct iovec **iovs,
1588 : uint32_t *iovcnt, struct spdk_memory_domain **domain, void **domain_ctx,
1589 : struct accel_buffer *buf)
1590 : {
1591 19 : bounce->orig_iovs = *iovs;
1592 19 : bounce->orig_iovcnt = *iovcnt;
1593 19 : bounce->orig_domain = *domain;
1594 19 : bounce->orig_domain_ctx = *domain_ctx;
1595 19 : bounce->iov.iov_base = buf->buf;
1596 19 : bounce->iov.iov_len = buf->len;
1597 :
1598 19 : *iovs = &bounce->iov;
1599 19 : *iovcnt = 1;
1600 19 : *domain = NULL;
1601 19 : }
1602 :
1603 : static void
1604 1 : accel_iobuf_get_src_bounce_cb(struct spdk_iobuf_entry *entry, void *buf)
1605 : {
1606 : struct spdk_accel_task *task;
1607 : struct accel_buffer *accel_buf;
1608 :
1609 1 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
1610 1 : assert(accel_buf->buf == NULL);
1611 1 : accel_buf->buf = buf;
1612 :
1613 1 : task = TAILQ_FIRST(&accel_buf->seq->tasks);
1614 1 : assert(task != NULL);
1615 :
1616 1 : assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
1617 1 : accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
1618 1 : assert(task->aux);
1619 1 : assert(task->has_aux);
1620 1 : accel_set_bounce_buffer(&task->aux->bounce.s, &task->s.iovs, &task->s.iovcnt, &task->src_domain,
1621 : &task->src_domain_ctx, accel_buf);
1622 1 : accel_process_sequence(accel_buf->seq);
1623 1 : }
1624 :
1625 : static void
1626 1 : accel_iobuf_get_dst_bounce_cb(struct spdk_iobuf_entry *entry, void *buf)
1627 : {
1628 : struct spdk_accel_task *task;
1629 : struct accel_buffer *accel_buf;
1630 :
1631 1 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
1632 1 : assert(accel_buf->buf == NULL);
1633 1 : accel_buf->buf = buf;
1634 :
1635 1 : task = TAILQ_FIRST(&accel_buf->seq->tasks);
1636 1 : assert(task != NULL);
1637 :
1638 1 : assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
1639 1 : accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
1640 1 : assert(task->aux);
1641 1 : assert(task->has_aux);
1642 1 : accel_set_bounce_buffer(&task->aux->bounce.d, &task->d.iovs, &task->d.iovcnt, &task->dst_domain,
1643 : &task->dst_domain_ctx, accel_buf);
1644 1 : accel_process_sequence(accel_buf->seq);
1645 1 : }
1646 :
1647 : static int
1648 73 : accel_sequence_check_bouncebuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
1649 : {
1650 : struct accel_buffer *buf;
1651 :
1652 73 : if (task->src_domain != NULL) {
1653 : /* By the time we're here, accel buffers should have been allocated */
1654 9 : assert(task->src_domain != g_accel_domain);
1655 :
1656 9 : if (!task->has_aux) {
1657 8 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1658 8 : if (spdk_unlikely(!task->aux)) {
1659 0 : SPDK_ERRLOG("Can't allocate aux data structure\n");
1660 0 : assert(0);
1661 : return -EAGAIN;
1662 : }
1663 8 : task->has_aux = true;
1664 8 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1665 : }
1666 9 : buf = accel_get_buf(seq->ch, accel_get_iovlen(task->s.iovs, task->s.iovcnt));
1667 9 : if (buf == NULL) {
1668 0 : SPDK_ERRLOG("Couldn't allocate buffer descriptor\n");
1669 0 : return -ENOMEM;
1670 : }
1671 :
1672 9 : SLIST_INSERT_HEAD(&seq->bounce_bufs, buf, link);
1673 9 : if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_src_bounce_cb)) {
1674 1 : return -EAGAIN;
1675 : }
1676 :
1677 8 : accel_set_bounce_buffer(&task->aux->bounce.s, &task->s.iovs, &task->s.iovcnt,
1678 : &task->src_domain, &task->src_domain_ctx, buf);
1679 : }
1680 :
1681 72 : if (task->dst_domain != NULL) {
1682 : /* By the time we're here, accel buffers should have been allocated */
1683 10 : assert(task->dst_domain != g_accel_domain);
1684 :
1685 10 : if (!task->has_aux) {
1686 0 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1687 0 : if (spdk_unlikely(!task->aux)) {
1688 0 : SPDK_ERRLOG("Can't allocate aux data structure\n");
1689 0 : assert(0);
1690 : return -EAGAIN;
1691 : }
1692 0 : task->has_aux = true;
1693 0 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1694 : }
1695 10 : buf = accel_get_buf(seq->ch, accel_get_iovlen(task->d.iovs, task->d.iovcnt));
1696 10 : if (buf == NULL) {
1697 : /* The src buffer will be released when a sequence is completed */
1698 0 : SPDK_ERRLOG("Couldn't allocate buffer descriptor\n");
1699 0 : return -ENOMEM;
1700 : }
1701 :
1702 10 : SLIST_INSERT_HEAD(&seq->bounce_bufs, buf, link);
1703 10 : if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_dst_bounce_cb)) {
1704 1 : return -EAGAIN;
1705 : }
1706 :
1707 9 : accel_set_bounce_buffer(&task->aux->bounce.d, &task->d.iovs, &task->d.iovcnt,
1708 : &task->dst_domain, &task->dst_domain_ctx, buf);
1709 : }
1710 :
1711 71 : return 0;
1712 : }
1713 :
1714 : static void
1715 8 : accel_task_pull_data_cb(void *ctx, int status)
1716 : {
1717 8 : struct spdk_accel_sequence *seq = ctx;
1718 :
1719 8 : assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA);
1720 8 : if (spdk_likely(status == 0)) {
1721 7 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
1722 : } else {
1723 1 : accel_sequence_set_fail(seq, status);
1724 : }
1725 :
1726 8 : accel_process_sequence(seq);
1727 8 : }
1728 :
1729 : static void
1730 9 : accel_task_pull_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
1731 : {
1732 : int rc;
1733 :
1734 9 : assert(task->has_aux);
1735 9 : assert(task->aux);
1736 9 : assert(task->aux->bounce.s.orig_iovs != NULL);
1737 9 : assert(task->aux->bounce.s.orig_domain != NULL);
1738 9 : assert(task->aux->bounce.s.orig_domain != g_accel_domain);
1739 9 : assert(!g_modules_opc[task->op_code].supports_memory_domains);
1740 :
1741 36 : rc = spdk_memory_domain_pull_data(task->aux->bounce.s.orig_domain,
1742 9 : task->aux->bounce.s.orig_domain_ctx,
1743 18 : task->aux->bounce.s.orig_iovs, task->aux->bounce.s.orig_iovcnt,
1744 : task->s.iovs, task->s.iovcnt,
1745 : accel_task_pull_data_cb, seq);
1746 9 : if (spdk_unlikely(rc != 0)) {
1747 1 : SPDK_ERRLOG("Failed to pull data from memory domain: %s, rc: %d\n",
1748 : spdk_memory_domain_get_dma_device_id(task->aux->bounce.s.orig_domain), rc);
1749 1 : accel_sequence_set_fail(seq, rc);
1750 : }
1751 9 : }
1752 :
1753 : static void
1754 7 : accel_task_push_data_cb(void *ctx, int status)
1755 : {
1756 7 : struct spdk_accel_sequence *seq = ctx;
1757 :
1758 7 : assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA);
1759 7 : if (spdk_likely(status == 0)) {
1760 6 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK);
1761 : } else {
1762 1 : accel_sequence_set_fail(seq, status);
1763 : }
1764 :
1765 7 : accel_process_sequence(seq);
1766 7 : }
1767 :
1768 : static void
1769 8 : accel_task_push_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
1770 : {
1771 : int rc;
1772 :
1773 8 : assert(task->has_aux);
1774 8 : assert(task->aux);
1775 8 : assert(task->aux->bounce.d.orig_iovs != NULL);
1776 8 : assert(task->aux->bounce.d.orig_domain != NULL);
1777 8 : assert(task->aux->bounce.d.orig_domain != g_accel_domain);
1778 8 : assert(!g_modules_opc[task->op_code].supports_memory_domains);
1779 :
1780 32 : rc = spdk_memory_domain_push_data(task->aux->bounce.d.orig_domain,
1781 8 : task->aux->bounce.d.orig_domain_ctx,
1782 16 : task->aux->bounce.d.orig_iovs, task->aux->bounce.d.orig_iovcnt,
1783 : task->d.iovs, task->d.iovcnt,
1784 : accel_task_push_data_cb, seq);
1785 8 : if (spdk_unlikely(rc != 0)) {
1786 1 : SPDK_ERRLOG("Failed to push data to memory domain: %s, rc: %d\n",
1787 : spdk_memory_domain_get_dma_device_id(task->aux->bounce.s.orig_domain), rc);
1788 1 : accel_sequence_set_fail(seq, rc);
1789 : }
1790 8 : }
1791 :
1792 : static void
1793 166 : accel_process_sequence(struct spdk_accel_sequence *seq)
1794 : {
1795 166 : struct accel_io_channel *accel_ch = seq->ch;
1796 : struct spdk_accel_task *task;
1797 : enum accel_sequence_state state;
1798 : int rc;
1799 :
1800 : /* Prevent recursive calls to this function */
1801 166 : if (spdk_unlikely(seq->in_process_sequence)) {
1802 74 : return;
1803 : }
1804 92 : seq->in_process_sequence = true;
1805 :
1806 92 : task = TAILQ_FIRST(&seq->tasks);
1807 : do {
1808 375 : state = seq->state;
1809 375 : switch (state) {
1810 91 : case ACCEL_SEQUENCE_STATE_INIT:
1811 91 : if (g_accel_driver != NULL) {
1812 13 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS);
1813 13 : break;
1814 : }
1815 : /* Fall through */
1816 : case ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF:
1817 89 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF);
1818 89 : if (!accel_sequence_check_virtbuf(seq, task)) {
1819 : /* We couldn't allocate a buffer, wait until one is available */
1820 3 : break;
1821 : }
1822 86 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
1823 : /* Fall through */
1824 88 : case ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF:
1825 : /* If a module supports memory domains, we don't need to allocate bounce
1826 : * buffers */
1827 88 : if (g_modules_opc[task->op_code].supports_memory_domains) {
1828 15 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
1829 15 : break;
1830 : }
1831 73 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
1832 73 : rc = accel_sequence_check_bouncebuf(seq, task);
1833 73 : if (spdk_unlikely(rc != 0)) {
1834 : /* We couldn't allocate a buffer, wait until one is available */
1835 2 : if (rc == -EAGAIN) {
1836 2 : break;
1837 : }
1838 0 : accel_sequence_set_fail(seq, rc);
1839 0 : break;
1840 : }
1841 71 : if (task->has_aux && task->s.iovs == &task->aux->bounce.s.iov) {
1842 9 : assert(task->aux->bounce.s.orig_iovs);
1843 9 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PULL_DATA);
1844 9 : break;
1845 : }
1846 62 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
1847 : /* Fall through */
1848 84 : case ACCEL_SEQUENCE_STATE_EXEC_TASK:
1849 84 : SPDK_DEBUGLOG(accel, "Executing %s operation, sequence: %p\n",
1850 : g_opcode_strings[task->op_code], seq);
1851 :
1852 84 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_TASK);
1853 84 : rc = accel_submit_task(accel_ch, task);
1854 84 : if (spdk_unlikely(rc != 0)) {
1855 2 : SPDK_ERRLOG("Failed to submit %s operation, sequence: %p\n",
1856 : g_opcode_strings[task->op_code], seq);
1857 2 : accel_sequence_set_fail(seq, rc);
1858 : }
1859 84 : break;
1860 9 : case ACCEL_SEQUENCE_STATE_PULL_DATA:
1861 9 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA);
1862 9 : accel_task_pull_data(seq, task);
1863 9 : break;
1864 80 : case ACCEL_SEQUENCE_STATE_COMPLETE_TASK:
1865 80 : if (task->has_aux && task->d.iovs == &task->aux->bounce.d.iov) {
1866 8 : assert(task->aux->bounce.d.orig_iovs);
1867 8 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PUSH_DATA);
1868 8 : break;
1869 : }
1870 72 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK);
1871 72 : break;
1872 8 : case ACCEL_SEQUENCE_STATE_PUSH_DATA:
1873 8 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA);
1874 8 : accel_task_push_data(seq, task);
1875 8 : break;
1876 78 : case ACCEL_SEQUENCE_STATE_NEXT_TASK:
1877 78 : accel_sequence_complete_task(seq, task);
1878 : /* Check if there are any remaining tasks */
1879 78 : task = TAILQ_FIRST(&seq->tasks);
1880 78 : if (task == NULL) {
1881 : /* Immediately return here to make sure we don't touch the sequence
1882 : * after it's completed */
1883 39 : accel_sequence_complete(seq);
1884 39 : return;
1885 : }
1886 39 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_INIT);
1887 39 : break;
1888 13 : case ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS:
1889 13 : assert(!TAILQ_EMPTY(&seq->tasks));
1890 :
1891 13 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS);
1892 13 : rc = g_accel_driver->execute_sequence(accel_ch->driver_channel, seq);
1893 13 : if (spdk_unlikely(rc != 0)) {
1894 1 : SPDK_ERRLOG("Failed to execute sequence: %p using driver: %s\n",
1895 : seq, g_accel_driver->name);
1896 1 : accel_sequence_set_fail(seq, rc);
1897 : }
1898 13 : break;
1899 11 : case ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS:
1900 : /* Get the task again, as the driver might have completed some tasks
1901 : * synchronously */
1902 11 : task = TAILQ_FIRST(&seq->tasks);
1903 11 : if (task == NULL) {
1904 : /* Immediately return here to make sure we don't touch the sequence
1905 : * after it's completed */
1906 3 : accel_sequence_complete(seq);
1907 3 : return;
1908 : }
1909 : /* We don't want to execute the next task through the driver, so we
1910 : * explicitly omit the INIT state here */
1911 8 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF);
1912 8 : break;
1913 10 : case ACCEL_SEQUENCE_STATE_ERROR:
1914 : /* Immediately return here to make sure we don't touch the sequence
1915 : * after it's completed */
1916 10 : assert(seq->status != 0);
1917 10 : accel_sequence_complete(seq);
1918 10 : return;
1919 40 : case ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF:
1920 : case ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF:
1921 : case ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA:
1922 : case ACCEL_SEQUENCE_STATE_AWAIT_TASK:
1923 : case ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA:
1924 : case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS:
1925 40 : break;
1926 0 : default:
1927 0 : assert(0 && "bad state");
1928 : break;
1929 : }
1930 323 : } while (seq->state != state);
1931 :
1932 40 : seq->in_process_sequence = false;
1933 : }
1934 :
1935 : static void
1936 94 : accel_sequence_task_cb(void *cb_arg, int status)
1937 : {
1938 94 : struct spdk_accel_sequence *seq = cb_arg;
1939 94 : struct spdk_accel_task *task = TAILQ_FIRST(&seq->tasks);
1940 :
1941 94 : switch (seq->state) {
1942 82 : case ACCEL_SEQUENCE_STATE_AWAIT_TASK:
1943 82 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_COMPLETE_TASK);
1944 82 : if (spdk_unlikely(status != 0)) {
1945 2 : SPDK_ERRLOG("Failed to execute %s operation, sequence: %p\n",
1946 : g_opcode_strings[task->op_code], seq);
1947 2 : accel_sequence_set_fail(seq, status);
1948 : }
1949 :
1950 82 : accel_process_sequence(seq);
1951 82 : break;
1952 12 : case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS:
1953 12 : assert(g_accel_driver != NULL);
1954 : /* Immediately remove the task from the outstanding list to make sure the next call
1955 : * to spdk_accel_sequence_first_task() doesn't return it */
1956 12 : accel_sequence_complete_task(seq, task);
1957 12 : if (spdk_unlikely(status != 0)) {
1958 1 : SPDK_ERRLOG("Failed to execute %s operation, sequence: %p through "
1959 : "driver: %s\n", g_opcode_strings[task->op_code], seq,
1960 : g_accel_driver->name);
1961 : /* Update status without using accel_sequence_set_fail() to avoid changing
1962 : * seq's state to ERROR until driver calls spdk_accel_sequence_continue() */
1963 1 : seq->status = status;
1964 : }
1965 12 : break;
1966 0 : default:
1967 0 : assert(0 && "bad state");
1968 : break;
1969 : }
1970 94 : }
1971 :
1972 : void
1973 12 : spdk_accel_sequence_continue(struct spdk_accel_sequence *seq)
1974 : {
1975 12 : assert(g_accel_driver != NULL);
1976 12 : assert(seq->state == ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS);
1977 :
1978 12 : if (spdk_likely(seq->status == 0)) {
1979 11 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS);
1980 : } else {
1981 1 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR);
1982 : }
1983 :
1984 12 : accel_process_sequence(seq);
1985 12 : }
1986 :
1987 : static bool
1988 28 : accel_compare_iovs(struct iovec *iova, uint32_t iovacnt, struct iovec *iovb, uint32_t iovbcnt)
1989 : {
1990 : /* For now, just do a dumb check that the iovecs arrays are exactly the same */
1991 28 : if (iovacnt != iovbcnt) {
1992 0 : return false;
1993 : }
1994 :
1995 28 : return memcmp(iova, iovb, sizeof(*iova) * iovacnt) == 0;
1996 : }
1997 :
1998 : static bool
1999 22 : accel_task_set_dstbuf(struct spdk_accel_task *task, struct spdk_accel_task *next)
2000 : {
2001 : struct spdk_accel_task *prev;
2002 :
2003 22 : switch (task->op_code) {
2004 18 : case SPDK_ACCEL_OPC_DECOMPRESS:
2005 : case SPDK_ACCEL_OPC_FILL:
2006 : case SPDK_ACCEL_OPC_ENCRYPT:
2007 : case SPDK_ACCEL_OPC_DECRYPT:
2008 18 : if (task->dst_domain != next->src_domain) {
2009 0 : return false;
2010 : }
2011 18 : if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt,
2012 : next->s.iovs, next->s.iovcnt)) {
2013 1 : return false;
2014 : }
2015 17 : task->d.iovs = next->d.iovs;
2016 17 : task->d.iovcnt = next->d.iovcnt;
2017 17 : task->dst_domain = next->dst_domain;
2018 17 : task->dst_domain_ctx = next->dst_domain_ctx;
2019 17 : break;
2020 4 : case SPDK_ACCEL_OPC_CRC32C:
2021 : /* crc32 is special, because it doesn't have a dst buffer */
2022 4 : if (task->src_domain != next->src_domain) {
2023 0 : return false;
2024 : }
2025 4 : if (!accel_compare_iovs(task->s.iovs, task->s.iovcnt,
2026 : next->s.iovs, next->s.iovcnt)) {
2027 1 : return false;
2028 : }
2029 : /* We can only change crc32's buffer if we can change previous task's buffer */
2030 3 : prev = TAILQ_PREV(task, accel_sequence_tasks, seq_link);
2031 3 : if (prev == NULL) {
2032 1 : return false;
2033 : }
2034 2 : if (!accel_task_set_dstbuf(prev, next)) {
2035 0 : return false;
2036 : }
2037 2 : task->s.iovs = next->d.iovs;
2038 2 : task->s.iovcnt = next->d.iovcnt;
2039 2 : task->src_domain = next->dst_domain;
2040 2 : task->src_domain_ctx = next->dst_domain_ctx;
2041 2 : break;
2042 0 : default:
2043 0 : return false;
2044 : }
2045 :
2046 19 : return true;
2047 : }
2048 :
2049 : static void
2050 72 : accel_sequence_merge_tasks(struct spdk_accel_sequence *seq, struct spdk_accel_task *task,
2051 : struct spdk_accel_task **next_task)
2052 : {
2053 72 : struct spdk_accel_task *next = *next_task;
2054 :
2055 72 : switch (task->op_code) {
2056 8 : case SPDK_ACCEL_OPC_COPY:
2057 : /* We only allow changing src of operations that actually have a src, e.g. we never
2058 : * do it for fill. Theoretically, it is possible, but we'd have to be careful to
2059 : * change the src of the operation after fill (which in turn could also be a fill).
2060 : * So, for the sake of simplicity, skip this type of operations for now.
2061 : */
2062 8 : if (next->op_code != SPDK_ACCEL_OPC_DECOMPRESS &&
2063 6 : next->op_code != SPDK_ACCEL_OPC_COPY &&
2064 5 : next->op_code != SPDK_ACCEL_OPC_ENCRYPT &&
2065 4 : next->op_code != SPDK_ACCEL_OPC_DECRYPT &&
2066 2 : next->op_code != SPDK_ACCEL_OPC_COPY_CRC32C) {
2067 2 : break;
2068 : }
2069 6 : if (task->dst_domain != next->src_domain) {
2070 0 : break;
2071 : }
2072 6 : if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt,
2073 : next->s.iovs, next->s.iovcnt)) {
2074 0 : break;
2075 : }
2076 6 : next->s.iovs = task->s.iovs;
2077 6 : next->s.iovcnt = task->s.iovcnt;
2078 6 : next->src_domain = task->src_domain;
2079 6 : next->src_domain_ctx = task->src_domain_ctx;
2080 6 : accel_sequence_complete_task(seq, task);
2081 6 : break;
2082 64 : case SPDK_ACCEL_OPC_DECOMPRESS:
2083 : case SPDK_ACCEL_OPC_FILL:
2084 : case SPDK_ACCEL_OPC_ENCRYPT:
2085 : case SPDK_ACCEL_OPC_DECRYPT:
2086 : case SPDK_ACCEL_OPC_CRC32C:
2087 : /* We can only merge tasks when one of them is a copy */
2088 64 : if (next->op_code != SPDK_ACCEL_OPC_COPY) {
2089 44 : break;
2090 : }
2091 20 : if (!accel_task_set_dstbuf(task, next)) {
2092 3 : break;
2093 : }
2094 : /* We're removing next_task from the tasks queue, so we need to update its pointer,
2095 : * so that the TAILQ_FOREACH_SAFE() loop below works correctly */
2096 17 : *next_task = TAILQ_NEXT(next, seq_link);
2097 17 : accel_sequence_complete_task(seq, next);
2098 17 : break;
2099 0 : default:
2100 0 : assert(0 && "bad opcode");
2101 : break;
2102 : }
2103 72 : }
2104 :
2105 : void
2106 52 : spdk_accel_sequence_finish(struct spdk_accel_sequence *seq,
2107 : spdk_accel_completion_cb cb_fn, void *cb_arg)
2108 : {
2109 52 : struct spdk_accel_task *task, *next;
2110 :
2111 : /* Try to remove any copy operations if possible */
2112 124 : TAILQ_FOREACH_SAFE(task, &seq->tasks, seq_link, next) {
2113 109 : if (next == NULL) {
2114 37 : break;
2115 : }
2116 72 : accel_sequence_merge_tasks(seq, task, &next);
2117 : }
2118 :
2119 52 : seq->cb_fn = cb_fn;
2120 52 : seq->cb_arg = cb_arg;
2121 :
2122 52 : accel_process_sequence(seq);
2123 52 : }
2124 :
2125 : void
2126 0 : spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq)
2127 : {
2128 0 : struct accel_sequence_tasks tasks = TAILQ_HEAD_INITIALIZER(tasks);
2129 : struct spdk_accel_task *task;
2130 :
2131 0 : TAILQ_SWAP(&tasks, &seq->tasks, spdk_accel_task, seq_link);
2132 :
2133 0 : while (!TAILQ_EMPTY(&tasks)) {
2134 0 : task = TAILQ_FIRST(&tasks);
2135 0 : TAILQ_REMOVE(&tasks, task, seq_link);
2136 0 : TAILQ_INSERT_HEAD(&seq->tasks, task, seq_link);
2137 : }
2138 0 : }
2139 :
2140 : void
2141 3 : spdk_accel_sequence_abort(struct spdk_accel_sequence *seq)
2142 : {
2143 3 : if (seq == NULL) {
2144 1 : return;
2145 : }
2146 :
2147 2 : accel_sequence_complete_tasks(seq);
2148 2 : accel_sequence_put(seq);
2149 : }
2150 :
2151 : struct spdk_memory_domain *
2152 0 : spdk_accel_get_memory_domain(void)
2153 : {
2154 0 : return g_accel_domain;
2155 : }
2156 :
2157 : static struct spdk_accel_module_if *
2158 7 : _module_find_by_name(const char *name)
2159 : {
2160 7 : struct spdk_accel_module_if *accel_module = NULL;
2161 :
2162 16 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
2163 10 : if (strcmp(name, accel_module->name) == 0) {
2164 1 : break;
2165 : }
2166 : }
2167 :
2168 7 : return accel_module;
2169 : }
2170 :
2171 : static inline struct spdk_accel_crypto_key *
2172 0 : _accel_crypto_key_get(const char *name)
2173 : {
2174 : struct spdk_accel_crypto_key *key;
2175 :
2176 0 : assert(spdk_spin_held(&g_keyring_spin));
2177 :
2178 0 : TAILQ_FOREACH(key, &g_keyring, link) {
2179 0 : if (strcmp(name, key->param.key_name) == 0) {
2180 0 : return key;
2181 : }
2182 : }
2183 :
2184 0 : return NULL;
2185 : }
2186 :
2187 : static void
2188 0 : accel_crypto_key_free_mem(struct spdk_accel_crypto_key *key)
2189 : {
2190 0 : if (key->param.hex_key) {
2191 0 : spdk_memset_s(key->param.hex_key, key->key_size * 2, 0, key->key_size * 2);
2192 0 : free(key->param.hex_key);
2193 : }
2194 0 : if (key->param.hex_key2) {
2195 0 : spdk_memset_s(key->param.hex_key2, key->key2_size * 2, 0, key->key2_size * 2);
2196 0 : free(key->param.hex_key2);
2197 : }
2198 0 : free(key->param.tweak_mode);
2199 0 : free(key->param.key_name);
2200 0 : free(key->param.cipher);
2201 0 : if (key->key) {
2202 0 : spdk_memset_s(key->key, key->key_size, 0, key->key_size);
2203 0 : free(key->key);
2204 : }
2205 0 : if (key->key2) {
2206 0 : spdk_memset_s(key->key2, key->key2_size, 0, key->key2_size);
2207 0 : free(key->key2);
2208 : }
2209 0 : free(key);
2210 0 : }
2211 :
2212 : static void
2213 0 : accel_crypto_key_destroy_unsafe(struct spdk_accel_crypto_key *key)
2214 : {
2215 0 : assert(key->module_if);
2216 0 : assert(key->module_if->crypto_key_deinit);
2217 :
2218 0 : key->module_if->crypto_key_deinit(key);
2219 0 : accel_crypto_key_free_mem(key);
2220 0 : }
2221 :
2222 : /*
2223 : * This function mitigates a timing side channel which could be caused by using strcmp()
2224 : * Please refer to chapter "Mitigating Information Leakage Based on Variable Timing" in
2225 : * the article [1] for more details
2226 : * [1] https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/secure-coding/mitigate-timing-side-channel-crypto-implementation.html
2227 : */
2228 : static bool
2229 0 : accel_aes_xts_keys_equal(const char *k1, size_t k1_len, const char *k2, size_t k2_len)
2230 : {
2231 : size_t i;
2232 0 : volatile size_t x = k1_len ^ k2_len;
2233 :
2234 0 : for (i = 0; ((i < k1_len) & (i < k2_len)); i++) {
2235 0 : x |= k1[i] ^ k2[i];
2236 : }
2237 :
2238 0 : return x == 0;
2239 : }
2240 :
2241 : static const char *g_tweak_modes[] = {
2242 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA] = "SIMPLE_LBA",
2243 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_JOIN_NEG_LBA_WITH_LBA] = "JOIN_NEG_LBA_WITH_LBA",
2244 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_FULL_LBA] = "INCR_512_FULL_LBA",
2245 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_UPPER_LBA] = "INCR_512_UPPER_LBA",
2246 : };
2247 :
2248 : static const char *g_ciphers[] = {
2249 : [SPDK_ACCEL_CIPHER_AES_CBC] = "AES_CBC",
2250 : [SPDK_ACCEL_CIPHER_AES_XTS] = "AES_XTS",
2251 : };
2252 :
2253 : int
2254 0 : spdk_accel_crypto_key_create(const struct spdk_accel_crypto_key_create_param *param)
2255 : {
2256 : struct spdk_accel_module_if *module;
2257 : struct spdk_accel_crypto_key *key;
2258 : size_t hex_key_size, hex_key2_size;
2259 0 : bool found = false;
2260 : size_t i;
2261 : int rc;
2262 :
2263 0 : if (!param || !param->hex_key || !param->cipher || !param->key_name) {
2264 0 : return -EINVAL;
2265 : }
2266 :
2267 0 : if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) {
2268 : /* hardly ever possible, but let's check and warn the user */
2269 0 : SPDK_ERRLOG("Different accel modules are used for encryption and decryption\n");
2270 : }
2271 0 : module = g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module;
2272 :
2273 0 : if (!module) {
2274 0 : SPDK_ERRLOG("No accel module found assigned for crypto operation\n");
2275 0 : return -ENOENT;
2276 : }
2277 :
2278 0 : if (!module->crypto_key_init || !module->crypto_supports_cipher) {
2279 0 : SPDK_ERRLOG("Module %s doesn't support crypto operations\n", module->name);
2280 0 : return -ENOTSUP;
2281 : }
2282 :
2283 0 : key = calloc(1, sizeof(*key));
2284 0 : if (!key) {
2285 0 : return -ENOMEM;
2286 : }
2287 :
2288 0 : key->param.key_name = strdup(param->key_name);
2289 0 : if (!key->param.key_name) {
2290 0 : rc = -ENOMEM;
2291 0 : goto error;
2292 : }
2293 :
2294 0 : for (i = 0; i < SPDK_COUNTOF(g_ciphers); ++i) {
2295 0 : assert(g_ciphers[i]);
2296 :
2297 0 : if (strncmp(param->cipher, g_ciphers[i], strlen(g_ciphers[i])) == 0) {
2298 0 : key->cipher = i;
2299 0 : found = true;
2300 0 : break;
2301 : }
2302 : }
2303 :
2304 0 : if (!found) {
2305 0 : SPDK_ERRLOG("Failed to parse cipher\n");
2306 0 : rc = -EINVAL;
2307 0 : goto error;
2308 : }
2309 :
2310 0 : key->param.cipher = strdup(param->cipher);
2311 0 : if (!key->param.cipher) {
2312 0 : rc = -ENOMEM;
2313 0 : goto error;
2314 : }
2315 :
2316 0 : hex_key_size = strnlen(param->hex_key, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2317 0 : if (hex_key_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) {
2318 0 : SPDK_ERRLOG("key1 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2319 0 : rc = -EINVAL;
2320 0 : goto error;
2321 : }
2322 :
2323 0 : if (hex_key_size == 0) {
2324 0 : SPDK_ERRLOG("key1 size cannot be 0\n");
2325 0 : rc = -EINVAL;
2326 0 : goto error;
2327 : }
2328 :
2329 0 : key->param.hex_key = strdup(param->hex_key);
2330 0 : if (!key->param.hex_key) {
2331 0 : rc = -ENOMEM;
2332 0 : goto error;
2333 : }
2334 :
2335 0 : key->key_size = hex_key_size / 2;
2336 0 : key->key = spdk_unhexlify(key->param.hex_key);
2337 0 : if (!key->key) {
2338 0 : SPDK_ERRLOG("Failed to unhexlify key1\n");
2339 0 : rc = -EINVAL;
2340 0 : goto error;
2341 : }
2342 :
2343 0 : if (param->hex_key2) {
2344 0 : hex_key2_size = strnlen(param->hex_key2, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2345 0 : if (hex_key2_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) {
2346 0 : SPDK_ERRLOG("key2 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2347 0 : rc = -EINVAL;
2348 0 : goto error;
2349 : }
2350 :
2351 0 : if (hex_key2_size == 0) {
2352 0 : SPDK_ERRLOG("key2 size cannot be 0\n");
2353 0 : rc = -EINVAL;
2354 0 : goto error;
2355 : }
2356 :
2357 0 : key->param.hex_key2 = strdup(param->hex_key2);
2358 0 : if (!key->param.hex_key2) {
2359 0 : rc = -ENOMEM;
2360 0 : goto error;
2361 : }
2362 :
2363 0 : key->key2_size = hex_key2_size / 2;
2364 0 : key->key2 = spdk_unhexlify(key->param.hex_key2);
2365 0 : if (!key->key2) {
2366 0 : SPDK_ERRLOG("Failed to unhexlify key2\n");
2367 0 : rc = -EINVAL;
2368 0 : goto error;
2369 : }
2370 : }
2371 :
2372 0 : key->tweak_mode = ACCEL_CRYPTO_TWEAK_MODE_DEFAULT;
2373 0 : if (param->tweak_mode) {
2374 0 : found = false;
2375 :
2376 0 : key->param.tweak_mode = strdup(param->tweak_mode);
2377 0 : if (!key->param.tweak_mode) {
2378 0 : rc = -ENOMEM;
2379 0 : goto error;
2380 : }
2381 :
2382 0 : for (i = 0; i < SPDK_COUNTOF(g_tweak_modes); ++i) {
2383 0 : assert(g_tweak_modes[i]);
2384 :
2385 0 : if (strncmp(param->tweak_mode, g_tweak_modes[i], strlen(g_tweak_modes[i])) == 0) {
2386 0 : key->tweak_mode = i;
2387 0 : found = true;
2388 0 : break;
2389 : }
2390 : }
2391 :
2392 0 : if (!found) {
2393 0 : SPDK_ERRLOG("Failed to parse tweak mode\n");
2394 0 : rc = -EINVAL;
2395 0 : goto error;
2396 : }
2397 : }
2398 :
2399 0 : if ((!module->crypto_supports_tweak_mode && key->tweak_mode != ACCEL_CRYPTO_TWEAK_MODE_DEFAULT) ||
2400 0 : (module->crypto_supports_tweak_mode && !module->crypto_supports_tweak_mode(key->tweak_mode))) {
2401 0 : SPDK_ERRLOG("Module %s doesn't support %s tweak mode\n", module->name,
2402 : g_tweak_modes[key->tweak_mode]);
2403 0 : rc = -EINVAL;
2404 0 : goto error;
2405 : }
2406 :
2407 0 : if (!module->crypto_supports_cipher(key->cipher, key->key_size)) {
2408 0 : SPDK_ERRLOG("Module %s doesn't support %s cipher with %zu key size\n", module->name,
2409 : g_ciphers[key->cipher], key->key_size);
2410 0 : rc = -EINVAL;
2411 0 : goto error;
2412 : }
2413 :
2414 0 : if (key->cipher == SPDK_ACCEL_CIPHER_AES_XTS) {
2415 0 : if (!key->key2) {
2416 0 : SPDK_ERRLOG("%s key2 is missing\n", g_ciphers[key->cipher]);
2417 0 : rc = -EINVAL;
2418 0 : goto error;
2419 : }
2420 :
2421 0 : if (key->key_size != key->key2_size) {
2422 0 : SPDK_ERRLOG("%s key size %zu is not equal to key2 size %zu\n", g_ciphers[key->cipher],
2423 : key->key_size,
2424 : key->key2_size);
2425 0 : rc = -EINVAL;
2426 0 : goto error;
2427 : }
2428 :
2429 0 : if (accel_aes_xts_keys_equal(key->key, key->key_size, key->key2, key->key2_size)) {
2430 0 : SPDK_ERRLOG("%s identical keys are not secure\n", g_ciphers[key->cipher]);
2431 0 : rc = -EINVAL;
2432 0 : goto error;
2433 : }
2434 : }
2435 :
2436 0 : if (key->cipher == SPDK_ACCEL_CIPHER_AES_CBC) {
2437 0 : if (key->key2_size) {
2438 0 : SPDK_ERRLOG("%s doesn't use key2\n", g_ciphers[key->cipher]);
2439 0 : rc = -EINVAL;
2440 0 : goto error;
2441 : }
2442 : }
2443 :
2444 0 : key->module_if = module;
2445 :
2446 0 : spdk_spin_lock(&g_keyring_spin);
2447 0 : if (_accel_crypto_key_get(param->key_name)) {
2448 0 : rc = -EEXIST;
2449 : } else {
2450 0 : rc = module->crypto_key_init(key);
2451 0 : if (rc) {
2452 0 : SPDK_ERRLOG("Module %s failed to initialize crypto key\n", module->name);
2453 : } else {
2454 0 : TAILQ_INSERT_TAIL(&g_keyring, key, link);
2455 : }
2456 : }
2457 0 : spdk_spin_unlock(&g_keyring_spin);
2458 :
2459 0 : if (rc) {
2460 0 : goto error;
2461 : }
2462 :
2463 0 : return 0;
2464 :
2465 0 : error:
2466 0 : accel_crypto_key_free_mem(key);
2467 0 : return rc;
2468 : }
2469 :
2470 : int
2471 0 : spdk_accel_crypto_key_destroy(struct spdk_accel_crypto_key *key)
2472 : {
2473 0 : if (!key || !key->module_if) {
2474 0 : return -EINVAL;
2475 : }
2476 :
2477 0 : spdk_spin_lock(&g_keyring_spin);
2478 0 : if (!_accel_crypto_key_get(key->param.key_name)) {
2479 0 : spdk_spin_unlock(&g_keyring_spin);
2480 0 : return -ENOENT;
2481 : }
2482 0 : TAILQ_REMOVE(&g_keyring, key, link);
2483 0 : spdk_spin_unlock(&g_keyring_spin);
2484 :
2485 0 : accel_crypto_key_destroy_unsafe(key);
2486 :
2487 0 : return 0;
2488 : }
2489 :
2490 : struct spdk_accel_crypto_key *
2491 0 : spdk_accel_crypto_key_get(const char *name)
2492 : {
2493 : struct spdk_accel_crypto_key *key;
2494 :
2495 0 : spdk_spin_lock(&g_keyring_spin);
2496 0 : key = _accel_crypto_key_get(name);
2497 0 : spdk_spin_unlock(&g_keyring_spin);
2498 :
2499 0 : return key;
2500 : }
2501 :
2502 : /* Helper function when accel modules register with the framework. */
2503 : void
2504 5 : spdk_accel_module_list_add(struct spdk_accel_module_if *accel_module)
2505 : {
2506 : struct spdk_accel_module_if *tmp;
2507 :
2508 5 : if (_module_find_by_name(accel_module->name)) {
2509 0 : SPDK_NOTICELOG("Module %s already registered\n", accel_module->name);
2510 0 : assert(false);
2511 : return;
2512 : }
2513 :
2514 8 : TAILQ_FOREACH(tmp, &spdk_accel_module_list, tailq) {
2515 5 : if (accel_module->priority < tmp->priority) {
2516 2 : break;
2517 : }
2518 : }
2519 :
2520 5 : if (tmp != NULL) {
2521 2 : TAILQ_INSERT_BEFORE(tmp, accel_module, tailq);
2522 : } else {
2523 3 : TAILQ_INSERT_TAIL(&spdk_accel_module_list, accel_module, tailq);
2524 : }
2525 : }
2526 :
2527 : /* Framework level channel create callback. */
2528 : static int
2529 11 : accel_create_channel(void *io_device, void *ctx_buf)
2530 : {
2531 11 : struct accel_io_channel *accel_ch = ctx_buf;
2532 : struct spdk_accel_task *accel_task;
2533 : struct spdk_accel_task_aux_data *accel_task_aux;
2534 : struct spdk_accel_sequence *seq;
2535 : struct accel_buffer *buf;
2536 : size_t task_size_aligned;
2537 : uint8_t *task_mem;
2538 11 : uint32_t i = 0, j;
2539 : int rc;
2540 :
2541 11 : task_size_aligned = SPDK_ALIGN_CEIL(g_max_accel_module_size, SPDK_CACHE_LINE_SIZE);
2542 11 : accel_ch->task_pool_base = aligned_alloc(SPDK_CACHE_LINE_SIZE,
2543 11 : g_opts.task_count * task_size_aligned);
2544 11 : if (!accel_ch->task_pool_base) {
2545 0 : return -ENOMEM;
2546 : }
2547 11 : memset(accel_ch->task_pool_base, 0, g_opts.task_count * task_size_aligned);
2548 :
2549 11 : accel_ch->seq_pool_base = aligned_alloc(SPDK_CACHE_LINE_SIZE,
2550 11 : g_opts.sequence_count * sizeof(struct spdk_accel_sequence));
2551 11 : if (accel_ch->seq_pool_base == NULL) {
2552 0 : goto err;
2553 : }
2554 11 : memset(accel_ch->seq_pool_base, 0, g_opts.sequence_count * sizeof(struct spdk_accel_sequence));
2555 :
2556 11 : accel_ch->task_aux_data_base = calloc(g_opts.task_count, sizeof(struct spdk_accel_task_aux_data));
2557 11 : if (accel_ch->task_aux_data_base == NULL) {
2558 0 : goto err;
2559 : }
2560 :
2561 11 : accel_ch->buf_pool_base = calloc(g_opts.buf_count, sizeof(struct accel_buffer));
2562 11 : if (accel_ch->buf_pool_base == NULL) {
2563 0 : goto err;
2564 : }
2565 :
2566 11 : STAILQ_INIT(&accel_ch->task_pool);
2567 11 : SLIST_INIT(&accel_ch->task_aux_data_pool);
2568 11 : SLIST_INIT(&accel_ch->seq_pool);
2569 11 : SLIST_INIT(&accel_ch->buf_pool);
2570 :
2571 11 : task_mem = accel_ch->task_pool_base;
2572 22539 : for (i = 0; i < g_opts.task_count; i++) {
2573 22528 : accel_task = (struct spdk_accel_task *)task_mem;
2574 22528 : accel_task->aux = NULL;
2575 22528 : STAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link);
2576 22528 : task_mem += task_size_aligned;
2577 22528 : accel_task_aux = &accel_ch->task_aux_data_base[i];
2578 22528 : SLIST_INSERT_HEAD(&accel_ch->task_aux_data_pool, accel_task_aux, link);
2579 : }
2580 22539 : for (i = 0; i < g_opts.sequence_count; i++) {
2581 22528 : seq = &accel_ch->seq_pool_base[i];
2582 22528 : SLIST_INSERT_HEAD(&accel_ch->seq_pool, seq, link);
2583 : }
2584 22539 : for (i = 0; i < g_opts.buf_count; i++) {
2585 22528 : buf = &accel_ch->buf_pool_base[i];
2586 22528 : SLIST_INSERT_HEAD(&accel_ch->buf_pool, buf, link);
2587 : }
2588 :
2589 : /* Assign modules and get IO channels for each */
2590 176 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
2591 165 : accel_ch->module_ch[i] = g_modules_opc[i].module->get_io_channel();
2592 : /* This can happen if idxd runs out of channels. */
2593 165 : if (accel_ch->module_ch[i] == NULL) {
2594 0 : SPDK_ERRLOG("Module %s failed to get io channel\n", g_modules_opc[i].module->name);
2595 0 : goto err;
2596 : }
2597 : }
2598 :
2599 11 : if (g_accel_driver != NULL) {
2600 0 : accel_ch->driver_channel = g_accel_driver->get_io_channel();
2601 0 : if (accel_ch->driver_channel == NULL) {
2602 0 : SPDK_ERRLOG("Failed to get driver's IO channel\n");
2603 0 : goto err;
2604 : }
2605 : }
2606 :
2607 11 : rc = spdk_iobuf_channel_init(&accel_ch->iobuf, "accel", g_opts.small_cache_size,
2608 : g_opts.large_cache_size);
2609 11 : if (rc != 0) {
2610 0 : SPDK_ERRLOG("Failed to initialize iobuf accel channel\n");
2611 0 : goto err;
2612 : }
2613 :
2614 11 : return 0;
2615 0 : err:
2616 0 : if (accel_ch->driver_channel != NULL) {
2617 0 : spdk_put_io_channel(accel_ch->driver_channel);
2618 : }
2619 0 : for (j = 0; j < i; j++) {
2620 0 : spdk_put_io_channel(accel_ch->module_ch[j]);
2621 : }
2622 0 : free(accel_ch->task_pool_base);
2623 0 : free(accel_ch->task_aux_data_base);
2624 0 : free(accel_ch->seq_pool_base);
2625 0 : free(accel_ch->buf_pool_base);
2626 :
2627 0 : return -ENOMEM;
2628 : }
2629 :
2630 : static void
2631 11 : accel_add_stats(struct accel_stats *total, struct accel_stats *stats)
2632 : {
2633 : int i;
2634 :
2635 11 : total->sequence_executed += stats->sequence_executed;
2636 11 : total->sequence_failed += stats->sequence_failed;
2637 11 : total->sequence_outstanding += stats->sequence_outstanding;
2638 11 : total->retry.task += stats->retry.task;
2639 11 : total->retry.sequence += stats->retry.sequence;
2640 11 : total->retry.iobuf += stats->retry.iobuf;
2641 11 : total->retry.bufdesc += stats->retry.bufdesc;
2642 176 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) {
2643 165 : total->operations[i].executed += stats->operations[i].executed;
2644 165 : total->operations[i].failed += stats->operations[i].failed;
2645 165 : total->operations[i].num_bytes += stats->operations[i].num_bytes;
2646 : }
2647 11 : }
2648 :
2649 : /* Framework level channel destroy callback. */
2650 : static void
2651 11 : accel_destroy_channel(void *io_device, void *ctx_buf)
2652 : {
2653 11 : struct accel_io_channel *accel_ch = ctx_buf;
2654 : int i;
2655 :
2656 11 : spdk_iobuf_channel_fini(&accel_ch->iobuf);
2657 :
2658 11 : if (accel_ch->driver_channel != NULL) {
2659 0 : spdk_put_io_channel(accel_ch->driver_channel);
2660 : }
2661 :
2662 176 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
2663 165 : assert(accel_ch->module_ch[i] != NULL);
2664 165 : spdk_put_io_channel(accel_ch->module_ch[i]);
2665 165 : accel_ch->module_ch[i] = NULL;
2666 : }
2667 :
2668 : /* Update global stats to make sure channel's stats aren't lost after a channel is gone */
2669 11 : spdk_spin_lock(&g_stats_lock);
2670 11 : accel_add_stats(&g_stats, &accel_ch->stats);
2671 11 : spdk_spin_unlock(&g_stats_lock);
2672 :
2673 11 : free(accel_ch->task_pool_base);
2674 11 : free(accel_ch->task_aux_data_base);
2675 11 : free(accel_ch->seq_pool_base);
2676 11 : free(accel_ch->buf_pool_base);
2677 11 : }
2678 :
2679 : struct spdk_io_channel *
2680 11 : spdk_accel_get_io_channel(void)
2681 : {
2682 11 : return spdk_get_io_channel(&spdk_accel_module_list);
2683 : }
2684 :
2685 : static int
2686 2 : accel_module_initialize(void)
2687 : {
2688 : struct spdk_accel_module_if *accel_module, *tmp_module;
2689 2 : int rc = 0, module_rc;
2690 :
2691 7 : TAILQ_FOREACH_SAFE(accel_module, &spdk_accel_module_list, tailq, tmp_module) {
2692 5 : module_rc = accel_module->module_init();
2693 5 : if (module_rc) {
2694 0 : TAILQ_REMOVE(&spdk_accel_module_list, accel_module, tailq);
2695 0 : if (module_rc == -ENODEV) {
2696 0 : SPDK_NOTICELOG("No devices for module %s, skipping\n", accel_module->name);
2697 0 : } else if (!rc) {
2698 0 : SPDK_ERRLOG("Module %s initialization failed with %d\n", accel_module->name, module_rc);
2699 0 : rc = module_rc;
2700 : }
2701 0 : continue;
2702 : }
2703 :
2704 5 : SPDK_DEBUGLOG(accel, "Module %s initialized.\n", accel_module->name);
2705 : }
2706 :
2707 2 : return rc;
2708 : }
2709 :
2710 : static void
2711 30 : accel_module_init_opcode(enum spdk_accel_opcode opcode)
2712 : {
2713 30 : struct accel_module *module = &g_modules_opc[opcode];
2714 30 : struct spdk_accel_module_if *module_if = module->module;
2715 :
2716 30 : if (module_if->get_memory_domains != NULL) {
2717 0 : module->supports_memory_domains = module_if->get_memory_domains(NULL, 0) > 0;
2718 : }
2719 30 : }
2720 :
2721 : static int
2722 0 : accel_memory_domain_translate(struct spdk_memory_domain *src_domain, void *src_domain_ctx,
2723 : struct spdk_memory_domain *dst_domain, struct spdk_memory_domain_translation_ctx *dst_domain_ctx,
2724 : void *addr, size_t len, struct spdk_memory_domain_translation_result *result)
2725 : {
2726 0 : struct accel_buffer *buf = src_domain_ctx;
2727 :
2728 0 : SPDK_DEBUGLOG(accel, "translate addr %p, len %zu\n", addr, len);
2729 :
2730 0 : assert(g_accel_domain == src_domain);
2731 0 : assert(spdk_memory_domain_get_system_domain() == dst_domain);
2732 0 : assert(buf->buf == NULL);
2733 0 : assert(addr == ACCEL_BUFFER_BASE);
2734 0 : assert(len == buf->len);
2735 :
2736 0 : buf->buf = spdk_iobuf_get(&buf->ch->iobuf, buf->len, NULL, NULL);
2737 0 : if (spdk_unlikely(buf->buf == NULL)) {
2738 0 : return -ENOMEM;
2739 : }
2740 :
2741 0 : result->iov_count = 1;
2742 0 : result->iov.iov_base = buf->buf;
2743 0 : result->iov.iov_len = buf->len;
2744 0 : SPDK_DEBUGLOG(accel, "translated addr %p\n", result->iov.iov_base);
2745 0 : return 0;
2746 : }
2747 :
2748 : static void
2749 0 : accel_memory_domain_invalidate(struct spdk_memory_domain *domain, void *domain_ctx,
2750 : struct iovec *iov, uint32_t iovcnt)
2751 : {
2752 0 : struct accel_buffer *buf = domain_ctx;
2753 :
2754 0 : SPDK_DEBUGLOG(accel, "invalidate addr %p, len %zu\n", iov[0].iov_base, iov[0].iov_len);
2755 :
2756 0 : assert(g_accel_domain == domain);
2757 0 : assert(iovcnt == 1);
2758 0 : assert(buf->buf != NULL);
2759 0 : assert(iov[0].iov_base == buf->buf);
2760 0 : assert(iov[0].iov_len == buf->len);
2761 :
2762 0 : spdk_iobuf_put(&buf->ch->iobuf, buf->buf, buf->len);
2763 0 : buf->buf = NULL;
2764 0 : }
2765 :
2766 : int
2767 2 : spdk_accel_initialize(void)
2768 : {
2769 : enum spdk_accel_opcode op;
2770 2 : struct spdk_accel_module_if *accel_module = NULL;
2771 : int rc;
2772 :
2773 : /*
2774 : * We need a unique identifier for the accel framework, so use the
2775 : * spdk_accel_module_list address for this purpose.
2776 : */
2777 2 : spdk_io_device_register(&spdk_accel_module_list, accel_create_channel, accel_destroy_channel,
2778 : sizeof(struct accel_io_channel), "accel");
2779 :
2780 2 : spdk_spin_init(&g_keyring_spin);
2781 2 : spdk_spin_init(&g_stats_lock);
2782 :
2783 2 : rc = spdk_memory_domain_create(&g_accel_domain, SPDK_DMA_DEVICE_TYPE_ACCEL, NULL,
2784 : "SPDK_ACCEL_DMA_DEVICE");
2785 2 : if (rc != 0) {
2786 0 : SPDK_ERRLOG("Failed to create accel memory domain\n");
2787 0 : return rc;
2788 : }
2789 :
2790 2 : spdk_memory_domain_set_translation(g_accel_domain, accel_memory_domain_translate);
2791 2 : spdk_memory_domain_set_invalidate(g_accel_domain, accel_memory_domain_invalidate);
2792 :
2793 2 : g_modules_started = true;
2794 2 : rc = accel_module_initialize();
2795 2 : if (rc) {
2796 0 : return rc;
2797 : }
2798 :
2799 2 : if (g_accel_driver != NULL && g_accel_driver->init != NULL) {
2800 0 : rc = g_accel_driver->init();
2801 0 : if (rc != 0) {
2802 0 : SPDK_ERRLOG("Failed to initialize driver %s: %s\n", g_accel_driver->name,
2803 : spdk_strerror(-rc));
2804 0 : return rc;
2805 : }
2806 : }
2807 :
2808 : /* The module list is order by priority, with the highest priority modules being at the end
2809 : * of the list. The software module should be somewhere at the beginning of the list,
2810 : * before all HW modules.
2811 : * NOTE: all opcodes must be supported by software in the event that no HW modules are
2812 : * initialized to support the operation.
2813 : */
2814 7 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
2815 80 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
2816 75 : if (accel_module->supports_opcode(op)) {
2817 75 : g_modules_opc[op].module = accel_module;
2818 75 : SPDK_DEBUGLOG(accel, "OPC 0x%x now assigned to %s\n", op, accel_module->name);
2819 : }
2820 : }
2821 :
2822 5 : if (accel_module->get_ctx_size != NULL) {
2823 1 : g_max_accel_module_size = spdk_max(g_max_accel_module_size,
2824 : accel_module->get_ctx_size());
2825 : }
2826 : }
2827 :
2828 : /* Now lets check for overrides and apply all that exist */
2829 32 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
2830 30 : if (g_modules_opc_override[op] != NULL) {
2831 0 : accel_module = _module_find_by_name(g_modules_opc_override[op]);
2832 0 : if (accel_module == NULL) {
2833 0 : SPDK_ERRLOG("Invalid module name of %s\n", g_modules_opc_override[op]);
2834 0 : return -EINVAL;
2835 : }
2836 0 : if (accel_module->supports_opcode(op) == false) {
2837 0 : SPDK_ERRLOG("Module %s does not support op code %d\n", accel_module->name, op);
2838 0 : return -EINVAL;
2839 : }
2840 0 : g_modules_opc[op].module = accel_module;
2841 : }
2842 : }
2843 :
2844 2 : if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) {
2845 0 : SPDK_ERRLOG("Different accel modules are assigned to encrypt and decrypt operations");
2846 0 : return -EINVAL;
2847 : }
2848 :
2849 32 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
2850 30 : assert(g_modules_opc[op].module != NULL);
2851 30 : accel_module_init_opcode(op);
2852 : }
2853 :
2854 2 : rc = spdk_iobuf_register_module("accel");
2855 2 : if (rc != 0) {
2856 0 : SPDK_ERRLOG("Failed to register accel iobuf module\n");
2857 0 : return rc;
2858 : }
2859 :
2860 2 : return 0;
2861 : }
2862 :
2863 : static void
2864 2 : accel_module_finish_cb(void)
2865 : {
2866 2 : spdk_accel_fini_cb cb_fn = g_fini_cb_fn;
2867 :
2868 2 : cb_fn(g_fini_cb_arg);
2869 2 : g_fini_cb_fn = NULL;
2870 2 : g_fini_cb_arg = NULL;
2871 2 : }
2872 :
2873 : static void
2874 0 : accel_write_overridden_opc(struct spdk_json_write_ctx *w, const char *opc_str,
2875 : const char *module_str)
2876 : {
2877 0 : spdk_json_write_object_begin(w);
2878 0 : spdk_json_write_named_string(w, "method", "accel_assign_opc");
2879 0 : spdk_json_write_named_object_begin(w, "params");
2880 0 : spdk_json_write_named_string(w, "opname", opc_str);
2881 0 : spdk_json_write_named_string(w, "module", module_str);
2882 0 : spdk_json_write_object_end(w);
2883 0 : spdk_json_write_object_end(w);
2884 0 : }
2885 :
2886 : static void
2887 0 : __accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key)
2888 : {
2889 0 : spdk_json_write_named_string(w, "name", key->param.key_name);
2890 0 : spdk_json_write_named_string(w, "cipher", key->param.cipher);
2891 0 : spdk_json_write_named_string(w, "key", key->param.hex_key);
2892 0 : if (key->param.hex_key2) {
2893 0 : spdk_json_write_named_string(w, "key2", key->param.hex_key2);
2894 : }
2895 :
2896 0 : if (key->param.tweak_mode) {
2897 0 : spdk_json_write_named_string(w, "tweak_mode", key->param.tweak_mode);
2898 : }
2899 0 : }
2900 :
2901 : void
2902 0 : _accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key)
2903 : {
2904 0 : spdk_json_write_object_begin(w);
2905 0 : __accel_crypto_key_dump_param(w, key);
2906 0 : spdk_json_write_object_end(w);
2907 0 : }
2908 :
2909 : static void
2910 0 : _accel_crypto_key_write_config_json(struct spdk_json_write_ctx *w,
2911 : struct spdk_accel_crypto_key *key)
2912 : {
2913 0 : spdk_json_write_object_begin(w);
2914 0 : spdk_json_write_named_string(w, "method", "accel_crypto_key_create");
2915 0 : spdk_json_write_named_object_begin(w, "params");
2916 0 : __accel_crypto_key_dump_param(w, key);
2917 0 : spdk_json_write_object_end(w);
2918 0 : spdk_json_write_object_end(w);
2919 0 : }
2920 :
2921 : static void
2922 0 : accel_write_options(struct spdk_json_write_ctx *w)
2923 : {
2924 0 : spdk_json_write_object_begin(w);
2925 0 : spdk_json_write_named_string(w, "method", "accel_set_options");
2926 0 : spdk_json_write_named_object_begin(w, "params");
2927 0 : spdk_json_write_named_uint32(w, "small_cache_size", g_opts.small_cache_size);
2928 0 : spdk_json_write_named_uint32(w, "large_cache_size", g_opts.large_cache_size);
2929 0 : spdk_json_write_named_uint32(w, "task_count", g_opts.task_count);
2930 0 : spdk_json_write_named_uint32(w, "sequence_count", g_opts.sequence_count);
2931 0 : spdk_json_write_named_uint32(w, "buf_count", g_opts.buf_count);
2932 0 : spdk_json_write_object_end(w);
2933 0 : spdk_json_write_object_end(w);
2934 0 : }
2935 :
2936 : static void
2937 0 : _accel_crypto_keys_write_config_json(struct spdk_json_write_ctx *w, bool full_dump)
2938 : {
2939 : struct spdk_accel_crypto_key *key;
2940 :
2941 0 : spdk_spin_lock(&g_keyring_spin);
2942 0 : TAILQ_FOREACH(key, &g_keyring, link) {
2943 0 : if (full_dump) {
2944 0 : _accel_crypto_key_write_config_json(w, key);
2945 : } else {
2946 0 : _accel_crypto_key_dump_param(w, key);
2947 : }
2948 : }
2949 0 : spdk_spin_unlock(&g_keyring_spin);
2950 0 : }
2951 :
2952 : void
2953 0 : _accel_crypto_keys_dump_param(struct spdk_json_write_ctx *w)
2954 : {
2955 0 : _accel_crypto_keys_write_config_json(w, false);
2956 0 : }
2957 :
2958 : void
2959 0 : spdk_accel_write_config_json(struct spdk_json_write_ctx *w)
2960 : {
2961 : struct spdk_accel_module_if *accel_module;
2962 : int i;
2963 :
2964 0 : spdk_json_write_array_begin(w);
2965 0 : accel_write_options(w);
2966 :
2967 0 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
2968 0 : if (accel_module->write_config_json) {
2969 0 : accel_module->write_config_json(w);
2970 : }
2971 : }
2972 0 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
2973 0 : if (g_modules_opc_override[i]) {
2974 0 : accel_write_overridden_opc(w, g_opcode_strings[i], g_modules_opc_override[i]);
2975 : }
2976 : }
2977 :
2978 0 : _accel_crypto_keys_write_config_json(w, true);
2979 :
2980 0 : spdk_json_write_array_end(w);
2981 0 : }
2982 :
2983 : void
2984 7 : spdk_accel_module_finish(void)
2985 : {
2986 7 : if (!g_accel_module) {
2987 2 : g_accel_module = TAILQ_FIRST(&spdk_accel_module_list);
2988 : } else {
2989 5 : g_accel_module = TAILQ_NEXT(g_accel_module, tailq);
2990 : }
2991 :
2992 7 : if (!g_accel_module) {
2993 2 : if (g_accel_driver != NULL && g_accel_driver->fini != NULL) {
2994 0 : g_accel_driver->fini();
2995 : }
2996 :
2997 2 : spdk_spin_destroy(&g_keyring_spin);
2998 2 : spdk_spin_destroy(&g_stats_lock);
2999 2 : if (g_accel_domain) {
3000 2 : spdk_memory_domain_destroy(g_accel_domain);
3001 2 : g_accel_domain = NULL;
3002 : }
3003 2 : accel_module_finish_cb();
3004 2 : return;
3005 : }
3006 :
3007 5 : if (g_accel_module->module_fini) {
3008 1 : spdk_thread_send_msg(spdk_get_thread(), g_accel_module->module_fini, NULL);
3009 : } else {
3010 4 : spdk_accel_module_finish();
3011 : }
3012 : }
3013 :
3014 : static void
3015 2 : accel_io_device_unregister_cb(void *io_device)
3016 : {
3017 : struct spdk_accel_crypto_key *key, *key_tmp;
3018 : enum spdk_accel_opcode op;
3019 :
3020 2 : spdk_spin_lock(&g_keyring_spin);
3021 2 : TAILQ_FOREACH_SAFE(key, &g_keyring, link, key_tmp) {
3022 0 : accel_crypto_key_destroy_unsafe(key);
3023 : }
3024 2 : spdk_spin_unlock(&g_keyring_spin);
3025 :
3026 32 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
3027 30 : if (g_modules_opc_override[op] != NULL) {
3028 0 : free(g_modules_opc_override[op]);
3029 0 : g_modules_opc_override[op] = NULL;
3030 : }
3031 30 : g_modules_opc[op].module = NULL;
3032 : }
3033 :
3034 2 : spdk_accel_module_finish();
3035 2 : }
3036 :
3037 : void
3038 2 : spdk_accel_finish(spdk_accel_fini_cb cb_fn, void *cb_arg)
3039 : {
3040 2 : assert(cb_fn != NULL);
3041 :
3042 2 : g_fini_cb_fn = cb_fn;
3043 2 : g_fini_cb_arg = cb_arg;
3044 :
3045 2 : spdk_io_device_unregister(&spdk_accel_module_list, accel_io_device_unregister_cb);
3046 2 : }
3047 :
3048 : static struct spdk_accel_driver *
3049 2 : accel_find_driver(const char *name)
3050 : {
3051 : struct spdk_accel_driver *driver;
3052 :
3053 2 : TAILQ_FOREACH(driver, &g_accel_drivers, tailq) {
3054 1 : if (strcmp(driver->name, name) == 0) {
3055 1 : return driver;
3056 : }
3057 : }
3058 :
3059 1 : return NULL;
3060 : }
3061 :
3062 : int
3063 1 : spdk_accel_set_driver(const char *name)
3064 : {
3065 : struct spdk_accel_driver *driver;
3066 :
3067 1 : driver = accel_find_driver(name);
3068 1 : if (driver == NULL) {
3069 0 : SPDK_ERRLOG("Couldn't find driver named '%s'\n", name);
3070 0 : return -ENODEV;
3071 : }
3072 :
3073 1 : g_accel_driver = driver;
3074 :
3075 1 : return 0;
3076 : }
3077 :
3078 : const char *
3079 0 : spdk_accel_get_driver_name(void)
3080 : {
3081 0 : if (!g_accel_driver) {
3082 0 : return NULL;
3083 : }
3084 :
3085 0 : return g_accel_driver->name;
3086 : }
3087 :
3088 : void
3089 1 : spdk_accel_driver_register(struct spdk_accel_driver *driver)
3090 : {
3091 1 : if (accel_find_driver(driver->name)) {
3092 0 : SPDK_ERRLOG("Driver named '%s' has already been registered\n", driver->name);
3093 0 : assert(0);
3094 : return;
3095 : }
3096 :
3097 1 : TAILQ_INSERT_TAIL(&g_accel_drivers, driver, tailq);
3098 : }
3099 :
3100 : int
3101 0 : spdk_accel_set_opts(const struct spdk_accel_opts *opts)
3102 : {
3103 0 : if (!opts) {
3104 0 : SPDK_ERRLOG("opts cannot be NULL\n");
3105 0 : return -1;
3106 : }
3107 :
3108 0 : if (!opts->opts_size) {
3109 0 : SPDK_ERRLOG("opts_size inside opts cannot be zero value\n");
3110 0 : return -1;
3111 : }
3112 :
3113 : #define SET_FIELD(field) \
3114 : if (offsetof(struct spdk_accel_opts, field) + sizeof(opts->field) <= opts->opts_size) { \
3115 : g_opts.field = opts->field; \
3116 : } \
3117 :
3118 0 : SET_FIELD(small_cache_size);
3119 0 : SET_FIELD(large_cache_size);
3120 0 : SET_FIELD(task_count);
3121 0 : SET_FIELD(sequence_count);
3122 0 : SET_FIELD(buf_count);
3123 :
3124 0 : g_opts.opts_size = opts->opts_size;
3125 :
3126 : #undef SET_FIELD
3127 :
3128 0 : return 0;
3129 : }
3130 :
3131 : void
3132 0 : spdk_accel_get_opts(struct spdk_accel_opts *opts, size_t opts_size)
3133 : {
3134 0 : if (!opts) {
3135 0 : SPDK_ERRLOG("opts should not be NULL\n");
3136 0 : return;
3137 : }
3138 :
3139 0 : if (!opts_size) {
3140 0 : SPDK_ERRLOG("opts_size should not be zero value\n");
3141 0 : return;
3142 : }
3143 :
3144 0 : opts->opts_size = opts_size;
3145 :
3146 : #define SET_FIELD(field) \
3147 : if (offsetof(struct spdk_accel_opts, field) + sizeof(opts->field) <= opts_size) { \
3148 : opts->field = g_opts.field; \
3149 : } \
3150 :
3151 0 : SET_FIELD(small_cache_size);
3152 0 : SET_FIELD(large_cache_size);
3153 0 : SET_FIELD(task_count);
3154 0 : SET_FIELD(sequence_count);
3155 0 : SET_FIELD(buf_count);
3156 :
3157 : #undef SET_FIELD
3158 :
3159 : /* Do not remove this statement, you should always update this statement when you adding a new field,
3160 : * and do not forget to add the SET_FIELD statement for your added field. */
3161 : SPDK_STATIC_ASSERT(sizeof(struct spdk_accel_opts) == 28, "Incorrect size");
3162 : }
3163 :
3164 : struct accel_get_stats_ctx {
3165 : struct accel_stats stats;
3166 : accel_get_stats_cb cb_fn;
3167 : void *cb_arg;
3168 : };
3169 :
3170 : static void
3171 0 : accel_get_channel_stats_done(struct spdk_io_channel_iter *iter, int status)
3172 : {
3173 0 : struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter);
3174 :
3175 0 : ctx->cb_fn(&ctx->stats, ctx->cb_arg);
3176 0 : free(ctx);
3177 0 : }
3178 :
3179 : static void
3180 0 : accel_get_channel_stats(struct spdk_io_channel_iter *iter)
3181 : {
3182 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(iter);
3183 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
3184 0 : struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter);
3185 :
3186 0 : accel_add_stats(&ctx->stats, &accel_ch->stats);
3187 0 : spdk_for_each_channel_continue(iter, 0);
3188 0 : }
3189 :
3190 : int
3191 0 : accel_get_stats(accel_get_stats_cb cb_fn, void *cb_arg)
3192 : {
3193 : struct accel_get_stats_ctx *ctx;
3194 :
3195 0 : ctx = calloc(1, sizeof(*ctx));
3196 0 : if (ctx == NULL) {
3197 0 : return -ENOMEM;
3198 : }
3199 :
3200 0 : spdk_spin_lock(&g_stats_lock);
3201 0 : accel_add_stats(&ctx->stats, &g_stats);
3202 0 : spdk_spin_unlock(&g_stats_lock);
3203 :
3204 0 : ctx->cb_fn = cb_fn;
3205 0 : ctx->cb_arg = cb_arg;
3206 :
3207 0 : spdk_for_each_channel(&spdk_accel_module_list, accel_get_channel_stats, ctx,
3208 : accel_get_channel_stats_done);
3209 :
3210 0 : return 0;
3211 : }
3212 :
3213 : void
3214 0 : spdk_accel_get_opcode_stats(struct spdk_io_channel *ch, enum spdk_accel_opcode opcode,
3215 : struct spdk_accel_opcode_stats *stats, size_t size)
3216 : {
3217 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
3218 :
3219 : #define FIELD_OK(field) \
3220 : offsetof(struct spdk_accel_opcode_stats, field) + sizeof(stats->field) <= size
3221 :
3222 : #define SET_FIELD(field, value) \
3223 : if (FIELD_OK(field)) { \
3224 : stats->field = value; \
3225 : }
3226 :
3227 0 : SET_FIELD(executed, accel_ch->stats.operations[opcode].executed);
3228 0 : SET_FIELD(failed, accel_ch->stats.operations[opcode].failed);
3229 0 : SET_FIELD(num_bytes, accel_ch->stats.operations[opcode].num_bytes);
3230 :
3231 : #undef FIELD_OK
3232 : #undef SET_FIELD
3233 0 : }
3234 :
3235 : uint8_t
3236 0 : spdk_accel_get_buf_align(enum spdk_accel_opcode opcode,
3237 : const struct spdk_accel_operation_exec_ctx *ctx)
3238 : {
3239 0 : struct spdk_accel_module_if *module = g_modules_opc[opcode].module;
3240 0 : struct spdk_accel_opcode_info modinfo = {}, drvinfo = {};
3241 :
3242 0 : if (g_accel_driver != NULL && g_accel_driver->get_operation_info != NULL) {
3243 0 : g_accel_driver->get_operation_info(opcode, ctx, &drvinfo);
3244 : }
3245 :
3246 0 : if (module->get_operation_info != NULL) {
3247 0 : module->get_operation_info(opcode, ctx, &modinfo);
3248 : }
3249 :
3250 : /* If a driver is set, it'll execute most of the operations, while the rest will usually
3251 : * fall back to accel_sw, which doesn't have any alignment requiremenets. However, to be
3252 : * extra safe, return the max(driver, module) if a driver delegates some operations to a
3253 : * hardware module. */
3254 0 : return spdk_max(modinfo.required_alignment, drvinfo.required_alignment);
3255 : }
3256 :
3257 : struct spdk_accel_module_if *
3258 0 : spdk_accel_get_module(const char *name)
3259 : {
3260 : struct spdk_accel_module_if *module;
3261 :
3262 0 : TAILQ_FOREACH(module, &spdk_accel_module_list, tailq) {
3263 0 : if (strcmp(module->name, name) == 0) {
3264 0 : return module;
3265 : }
3266 : }
3267 :
3268 0 : return NULL;
3269 : }
3270 :
3271 : int
3272 0 : spdk_accel_get_opc_memory_domains(enum spdk_accel_opcode opcode,
3273 : struct spdk_memory_domain **domains,
3274 : int array_size)
3275 : {
3276 0 : assert(opcode < SPDK_ACCEL_OPC_LAST);
3277 :
3278 0 : if (g_modules_opc[opcode].module->get_memory_domains) {
3279 0 : return g_modules_opc[opcode].module->get_memory_domains(domains, array_size);
3280 : }
3281 :
3282 0 : return 0;
3283 : }
3284 :
3285 1 : SPDK_LOG_REGISTER_COMPONENT(accel)
|