Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2020 Intel Corporation.
3 : * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES.
4 : * All rights reserved.
5 : */
6 :
7 : #include "spdk/stdinc.h"
8 :
9 : #include "spdk/accel_module.h"
10 :
11 : #include "accel_internal.h"
12 :
13 : #include "spdk/dma.h"
14 : #include "spdk/env.h"
15 : #include "spdk/likely.h"
16 : #include "spdk/log.h"
17 : #include "spdk/thread.h"
18 : #include "spdk/json.h"
19 : #include "spdk/crc32.h"
20 : #include "spdk/util.h"
21 : #include "spdk/hexlify.h"
22 : #include "spdk/string.h"
23 :
24 : /* Accelerator Framework: The following provides a top level
25 : * generic API for the accelerator functions defined here. Modules,
26 : * such as the one in /module/accel/ioat, supply the implementation
27 : * with the exception of the pure software implementation contained
28 : * later in this file.
29 : */
30 :
31 : #define ALIGN_4K 0x1000
32 : #define MAX_TASKS_PER_CHANNEL 0x800
33 : #define ACCEL_SMALL_CACHE_SIZE 128
34 : #define ACCEL_LARGE_CACHE_SIZE 16
35 : /* Set MSB, so we don't return NULL pointers as buffers */
36 : #define ACCEL_BUFFER_BASE ((void *)(1ull << 63))
37 : #define ACCEL_BUFFER_OFFSET_MASK ((uintptr_t)ACCEL_BUFFER_BASE - 1)
38 :
39 : #define ACCEL_CRYPTO_TWEAK_MODE_DEFAULT SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA
40 : #define ACCEL_TASKS_IN_SEQUENCE_LIMIT 8
41 :
42 : struct accel_module {
43 : struct spdk_accel_module_if *module;
44 : bool supports_memory_domains;
45 : };
46 :
47 : /* Largest context size for all accel modules */
48 : static size_t g_max_accel_module_size = sizeof(struct spdk_accel_task);
49 :
50 : static struct spdk_accel_module_if *g_accel_module = NULL;
51 : static spdk_accel_fini_cb g_fini_cb_fn = NULL;
52 : static void *g_fini_cb_arg = NULL;
53 : static bool g_modules_started = false;
54 : static struct spdk_memory_domain *g_accel_domain;
55 :
56 : /* Global list of registered accelerator modules */
57 : static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list =
58 : TAILQ_HEAD_INITIALIZER(spdk_accel_module_list);
59 :
60 : /* Crypto keyring */
61 : static TAILQ_HEAD(, spdk_accel_crypto_key) g_keyring = TAILQ_HEAD_INITIALIZER(g_keyring);
62 : static struct spdk_spinlock g_keyring_spin;
63 :
64 : /* Global array mapping capabilities to modules */
65 : static struct accel_module g_modules_opc[SPDK_ACCEL_OPC_LAST] = {};
66 : static char *g_modules_opc_override[SPDK_ACCEL_OPC_LAST] = {};
67 : TAILQ_HEAD(, spdk_accel_driver) g_accel_drivers = TAILQ_HEAD_INITIALIZER(g_accel_drivers);
68 : static struct spdk_accel_driver *g_accel_driver;
69 : static struct spdk_accel_opts g_opts = {
70 : .small_cache_size = ACCEL_SMALL_CACHE_SIZE,
71 : .large_cache_size = ACCEL_LARGE_CACHE_SIZE,
72 : .task_count = MAX_TASKS_PER_CHANNEL,
73 : .sequence_count = MAX_TASKS_PER_CHANNEL,
74 : .buf_count = MAX_TASKS_PER_CHANNEL,
75 : };
76 : static struct accel_stats g_stats;
77 : static struct spdk_spinlock g_stats_lock;
78 :
79 : static const char *g_opcode_strings[SPDK_ACCEL_OPC_LAST] = {
80 : "copy", "fill", "dualcast", "compare", "crc32c", "copy_crc32c",
81 : "compress", "decompress", "encrypt", "decrypt", "xor",
82 : "dif_verify", "dif_verify_copy", "dif_generate", "dif_generate_copy"
83 : };
84 :
85 : enum accel_sequence_state {
86 : ACCEL_SEQUENCE_STATE_INIT,
87 : ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF,
88 : ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF,
89 : ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF,
90 : ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF,
91 : ACCEL_SEQUENCE_STATE_PULL_DATA,
92 : ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA,
93 : ACCEL_SEQUENCE_STATE_EXEC_TASK,
94 : ACCEL_SEQUENCE_STATE_AWAIT_TASK,
95 : ACCEL_SEQUENCE_STATE_COMPLETE_TASK,
96 : ACCEL_SEQUENCE_STATE_NEXT_TASK,
97 : ACCEL_SEQUENCE_STATE_PUSH_DATA,
98 : ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA,
99 : ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS,
100 : ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS,
101 : ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS,
102 : ACCEL_SEQUENCE_STATE_ERROR,
103 : ACCEL_SEQUENCE_STATE_MAX,
104 : };
105 :
106 : static const char *g_seq_states[]
107 : __attribute__((unused)) = {
108 : [ACCEL_SEQUENCE_STATE_INIT] = "init",
109 : [ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF] = "check-virtbuf",
110 : [ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF] = "await-virtbuf",
111 : [ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF] = "check-bouncebuf",
112 : [ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF] = "await-bouncebuf",
113 : [ACCEL_SEQUENCE_STATE_PULL_DATA] = "pull-data",
114 : [ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA] = "await-pull-data",
115 : [ACCEL_SEQUENCE_STATE_EXEC_TASK] = "exec-task",
116 : [ACCEL_SEQUENCE_STATE_AWAIT_TASK] = "await-task",
117 : [ACCEL_SEQUENCE_STATE_COMPLETE_TASK] = "complete-task",
118 : [ACCEL_SEQUENCE_STATE_NEXT_TASK] = "next-task",
119 : [ACCEL_SEQUENCE_STATE_PUSH_DATA] = "push-data",
120 : [ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA] = "await-push-data",
121 : [ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS] = "driver-exec-tasks",
122 : [ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS] = "driver-await-tasks",
123 : [ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS] = "driver-complete-tasks",
124 : [ACCEL_SEQUENCE_STATE_ERROR] = "error",
125 : [ACCEL_SEQUENCE_STATE_MAX] = "",
126 : };
127 :
128 : #define ACCEL_SEQUENCE_STATE_STRING(s) \
129 : (((s) >= ACCEL_SEQUENCE_STATE_INIT && (s) < ACCEL_SEQUENCE_STATE_MAX) \
130 : ? g_seq_states[s] : "unknown")
131 :
132 : struct accel_buffer {
133 : struct spdk_accel_sequence *seq;
134 : void *buf;
135 : uint64_t len;
136 : struct spdk_iobuf_entry iobuf;
137 : spdk_accel_sequence_get_buf_cb cb_fn;
138 : void *cb_ctx;
139 : SLIST_ENTRY(accel_buffer) link;
140 : struct accel_io_channel *ch;
141 : };
142 :
143 : struct accel_io_channel {
144 : struct spdk_io_channel *module_ch[SPDK_ACCEL_OPC_LAST];
145 : struct spdk_io_channel *driver_channel;
146 : void *task_pool_base;
147 : struct spdk_accel_sequence *seq_pool_base;
148 : struct accel_buffer *buf_pool_base;
149 : struct spdk_accel_task_aux_data *task_aux_data_base;
150 : STAILQ_HEAD(, spdk_accel_task) task_pool;
151 : SLIST_HEAD(, spdk_accel_task_aux_data) task_aux_data_pool;
152 : SLIST_HEAD(, spdk_accel_sequence) seq_pool;
153 : SLIST_HEAD(, accel_buffer) buf_pool;
154 : struct spdk_iobuf_channel iobuf;
155 : struct accel_stats stats;
156 : };
157 :
158 : TAILQ_HEAD(accel_sequence_tasks, spdk_accel_task);
159 :
160 : struct spdk_accel_sequence {
161 : struct accel_io_channel *ch;
162 : struct accel_sequence_tasks tasks;
163 : SLIST_HEAD(, accel_buffer) bounce_bufs;
164 : int status;
165 : /* state uses enum accel_sequence_state */
166 : uint8_t state;
167 : bool in_process_sequence;
168 : spdk_accel_completion_cb cb_fn;
169 : void *cb_arg;
170 : SLIST_ENTRY(spdk_accel_sequence) link;
171 : };
172 : SPDK_STATIC_ASSERT(sizeof(struct spdk_accel_sequence) == 64, "invalid size");
173 :
174 : #define accel_update_stats(ch, event, v) \
175 : do { \
176 : (ch)->stats.event += (v); \
177 : } while (0)
178 :
179 : #define accel_update_task_stats(ch, task, event, v) \
180 : accel_update_stats(ch, operations[(task)->op_code].event, v)
181 :
182 : static inline void accel_sequence_task_cb(void *cb_arg, int status);
183 :
184 : static inline void
185 709 : accel_sequence_set_state(struct spdk_accel_sequence *seq, enum accel_sequence_state state)
186 : {
187 709 : SPDK_DEBUGLOG(accel, "seq=%p, setting state: %s -> %s\n", seq,
188 : ACCEL_SEQUENCE_STATE_STRING(seq->state), ACCEL_SEQUENCE_STATE_STRING(state));
189 709 : assert(seq->state != ACCEL_SEQUENCE_STATE_ERROR || state == ACCEL_SEQUENCE_STATE_ERROR);
190 709 : seq->state = state;
191 709 : }
192 :
193 : static void
194 9 : accel_sequence_set_fail(struct spdk_accel_sequence *seq, int status)
195 : {
196 9 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR);
197 9 : assert(status != 0);
198 9 : seq->status = status;
199 9 : }
200 :
201 : int
202 15 : spdk_accel_get_opc_module_name(enum spdk_accel_opcode opcode, const char **module_name)
203 : {
204 15 : if (opcode >= SPDK_ACCEL_OPC_LAST) {
205 : /* invalid opcode */
206 0 : return -EINVAL;
207 : }
208 :
209 15 : if (g_modules_opc[opcode].module) {
210 15 : *module_name = g_modules_opc[opcode].module->name;
211 : } else {
212 0 : return -ENOENT;
213 : }
214 :
215 15 : return 0;
216 : }
217 :
218 : void
219 0 : _accel_for_each_module(struct module_info *info, _accel_for_each_module_fn fn)
220 : {
221 : struct spdk_accel_module_if *accel_module;
222 : enum spdk_accel_opcode opcode;
223 0 : int j = 0;
224 :
225 0 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
226 0 : for (opcode = 0; opcode < SPDK_ACCEL_OPC_LAST; opcode++) {
227 0 : if (accel_module->supports_opcode(opcode)) {
228 0 : info->ops[j] = opcode;
229 0 : j++;
230 : }
231 : }
232 0 : info->name = accel_module->name;
233 0 : info->num_ops = j;
234 0 : fn(info);
235 0 : j = 0;
236 : }
237 0 : }
238 :
239 : const char *
240 0 : spdk_accel_get_opcode_name(enum spdk_accel_opcode opcode)
241 : {
242 0 : if (opcode < SPDK_ACCEL_OPC_LAST) {
243 0 : return g_opcode_strings[opcode];
244 : }
245 :
246 0 : return NULL;
247 : }
248 :
249 : int
250 0 : spdk_accel_assign_opc(enum spdk_accel_opcode opcode, const char *name)
251 : {
252 : char *copy;
253 :
254 0 : if (g_modules_started == true) {
255 : /* we don't allow re-assignment once things have started */
256 0 : return -EINVAL;
257 : }
258 :
259 0 : if (opcode >= SPDK_ACCEL_OPC_LAST) {
260 : /* invalid opcode */
261 0 : return -EINVAL;
262 : }
263 :
264 0 : copy = strdup(name);
265 0 : if (copy == NULL) {
266 0 : return -ENOMEM;
267 : }
268 :
269 : /* module selection will be validated after the framework starts. */
270 0 : free(g_modules_opc_override[opcode]);
271 0 : g_modules_opc_override[opcode] = copy;
272 :
273 0 : return 0;
274 : }
275 :
276 : inline static struct spdk_accel_task *
277 151 : _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, void *cb_arg)
278 : {
279 : struct spdk_accel_task *accel_task;
280 :
281 151 : accel_task = STAILQ_FIRST(&accel_ch->task_pool);
282 151 : if (spdk_unlikely(accel_task == NULL)) {
283 11 : accel_update_stats(accel_ch, retry.task, 1);
284 11 : return NULL;
285 : }
286 :
287 140 : accel_update_stats(accel_ch, task_outstanding, 1);
288 140 : STAILQ_REMOVE_HEAD(&accel_ch->task_pool, link);
289 140 : accel_task->link.stqe_next = NULL;
290 :
291 140 : accel_task->cb_fn = cb_fn;
292 140 : accel_task->cb_arg = cb_arg;
293 140 : accel_task->accel_ch = accel_ch;
294 140 : accel_task->s.iovs = NULL;
295 140 : accel_task->d.iovs = NULL;
296 :
297 140 : return accel_task;
298 : }
299 :
300 : static void
301 132 : _put_task(struct accel_io_channel *ch, struct spdk_accel_task *task)
302 : {
303 132 : STAILQ_INSERT_HEAD(&ch->task_pool, task, link);
304 132 : accel_update_stats(ch, task_outstanding, -1);
305 132 : }
306 :
307 : void
308 95 : spdk_accel_task_complete(struct spdk_accel_task *accel_task, int status)
309 : {
310 95 : struct accel_io_channel *accel_ch = accel_task->accel_ch;
311 : spdk_accel_completion_cb cb_fn;
312 : void *cb_arg;
313 :
314 95 : accel_update_task_stats(accel_ch, accel_task, executed, 1);
315 95 : accel_update_task_stats(accel_ch, accel_task, num_bytes, accel_task->nbytes);
316 95 : if (spdk_unlikely(status != 0)) {
317 3 : accel_update_task_stats(accel_ch, accel_task, failed, 1);
318 : }
319 :
320 95 : if (accel_task->seq) {
321 94 : accel_sequence_task_cb(accel_task->seq, status);
322 94 : return;
323 : }
324 :
325 1 : cb_fn = accel_task->cb_fn;
326 1 : cb_arg = accel_task->cb_arg;
327 :
328 1 : if (accel_task->has_aux) {
329 0 : SLIST_INSERT_HEAD(&accel_ch->task_aux_data_pool, accel_task->aux, link);
330 0 : accel_task->aux = NULL;
331 0 : accel_task->has_aux = false;
332 : }
333 :
334 : /* We should put the accel_task into the list firstly in order to avoid
335 : * the accel task list is exhausted when there is recursive call to
336 : * allocate accel_task in user's call back function (cb_fn)
337 : */
338 1 : _put_task(accel_ch, accel_task);
339 :
340 1 : cb_fn(cb_arg, status);
341 : }
342 :
343 : static inline int
344 92 : accel_submit_task(struct accel_io_channel *accel_ch, struct spdk_accel_task *task)
345 : {
346 92 : struct spdk_io_channel *module_ch = accel_ch->module_ch[task->op_code];
347 92 : struct spdk_accel_module_if *module = g_modules_opc[task->op_code].module;
348 : int rc;
349 :
350 92 : rc = module->submit_tasks(module_ch, task);
351 92 : if (spdk_unlikely(rc != 0)) {
352 2 : accel_update_task_stats(accel_ch, task, failed, 1);
353 : }
354 :
355 92 : return rc;
356 : }
357 :
358 : static inline uint64_t
359 114 : accel_get_iovlen(struct iovec *iovs, uint32_t iovcnt)
360 : {
361 114 : uint64_t result = 0;
362 : uint32_t i;
363 :
364 259 : for (i = 0; i < iovcnt; ++i) {
365 145 : result += iovs[i].iov_len;
366 : }
367 :
368 114 : return result;
369 : }
370 :
371 : #define ACCEL_TASK_ALLOC_AUX_BUF(task) \
372 : do { \
373 : (task)->aux = SLIST_FIRST(&(task)->accel_ch->task_aux_data_pool); \
374 : if (spdk_unlikely(!(task)->aux)) { \
375 : SPDK_ERRLOG("Fatal problem, aux data was not allocated\n"); \
376 : _put_task(task->accel_ch, task); \
377 : assert(0); \
378 : return -ENOMEM; \
379 : } \
380 : SLIST_REMOVE_HEAD(&(task)->accel_ch->task_aux_data_pool, link); \
381 : (task)->has_aux = true; \
382 : } while (0)
383 :
384 : /* Accel framework public API for copy function */
385 : int
386 2 : spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src,
387 : uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
388 : {
389 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
390 : struct spdk_accel_task *accel_task;
391 :
392 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
393 2 : if (spdk_unlikely(accel_task == NULL)) {
394 1 : return -ENOMEM;
395 : }
396 :
397 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
398 :
399 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
400 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
401 1 : accel_task->d.iovs[0].iov_base = dst;
402 1 : accel_task->d.iovs[0].iov_len = nbytes;
403 1 : accel_task->d.iovcnt = 1;
404 1 : accel_task->s.iovs[0].iov_base = src;
405 1 : accel_task->s.iovs[0].iov_len = nbytes;
406 1 : accel_task->s.iovcnt = 1;
407 1 : accel_task->nbytes = nbytes;
408 1 : accel_task->op_code = SPDK_ACCEL_OPC_COPY;
409 1 : accel_task->src_domain = NULL;
410 1 : accel_task->dst_domain = NULL;
411 :
412 1 : return accel_submit_task(accel_ch, accel_task);
413 : }
414 :
415 : /* Accel framework public API for dual cast copy function */
416 : int
417 4 : spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1,
418 : void *dst2, void *src, uint64_t nbytes,
419 : spdk_accel_completion_cb cb_fn, void *cb_arg)
420 : {
421 4 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
422 : struct spdk_accel_task *accel_task;
423 :
424 4 : if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {
425 2 : SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
426 2 : return -EINVAL;
427 : }
428 :
429 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
430 2 : if (spdk_unlikely(accel_task == NULL)) {
431 1 : return -ENOMEM;
432 : }
433 :
434 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
435 :
436 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
437 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
438 1 : accel_task->d2.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST2];
439 1 : accel_task->d.iovs[0].iov_base = dst1;
440 1 : accel_task->d.iovs[0].iov_len = nbytes;
441 1 : accel_task->d.iovcnt = 1;
442 1 : accel_task->d2.iovs[0].iov_base = dst2;
443 1 : accel_task->d2.iovs[0].iov_len = nbytes;
444 1 : accel_task->d2.iovcnt = 1;
445 1 : accel_task->s.iovs[0].iov_base = src;
446 1 : accel_task->s.iovs[0].iov_len = nbytes;
447 1 : accel_task->s.iovcnt = 1;
448 1 : accel_task->nbytes = nbytes;
449 1 : accel_task->op_code = SPDK_ACCEL_OPC_DUALCAST;
450 1 : accel_task->src_domain = NULL;
451 1 : accel_task->dst_domain = NULL;
452 :
453 1 : return accel_submit_task(accel_ch, accel_task);
454 : }
455 :
456 : /* Accel framework public API for compare function */
457 :
458 : int
459 2 : spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1,
460 : void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
461 : void *cb_arg)
462 : {
463 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
464 : struct spdk_accel_task *accel_task;
465 :
466 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
467 2 : if (spdk_unlikely(accel_task == NULL)) {
468 1 : return -ENOMEM;
469 : }
470 :
471 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
472 :
473 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
474 1 : accel_task->s2.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC2];
475 1 : accel_task->s.iovs[0].iov_base = src1;
476 1 : accel_task->s.iovs[0].iov_len = nbytes;
477 1 : accel_task->s.iovcnt = 1;
478 1 : accel_task->s2.iovs[0].iov_base = src2;
479 1 : accel_task->s2.iovs[0].iov_len = nbytes;
480 1 : accel_task->s2.iovcnt = 1;
481 1 : accel_task->nbytes = nbytes;
482 1 : accel_task->op_code = SPDK_ACCEL_OPC_COMPARE;
483 1 : accel_task->src_domain = NULL;
484 1 : accel_task->dst_domain = NULL;
485 :
486 1 : return accel_submit_task(accel_ch, accel_task);
487 : }
488 :
489 : /* Accel framework public API for fill function */
490 : int
491 2 : spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst,
492 : uint8_t fill, uint64_t nbytes,
493 : spdk_accel_completion_cb cb_fn, void *cb_arg)
494 : {
495 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
496 : struct spdk_accel_task *accel_task;
497 :
498 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
499 2 : if (spdk_unlikely(accel_task == NULL)) {
500 1 : return -ENOMEM;
501 : }
502 :
503 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
504 :
505 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
506 1 : accel_task->d.iovs[0].iov_base = dst;
507 1 : accel_task->d.iovs[0].iov_len = nbytes;
508 1 : accel_task->d.iovcnt = 1;
509 1 : accel_task->nbytes = nbytes;
510 1 : memset(&accel_task->fill_pattern, fill, sizeof(uint64_t));
511 1 : accel_task->op_code = SPDK_ACCEL_OPC_FILL;
512 1 : accel_task->src_domain = NULL;
513 1 : accel_task->dst_domain = NULL;
514 :
515 1 : return accel_submit_task(accel_ch, accel_task);
516 : }
517 :
518 : /* Accel framework public API for CRC-32C function */
519 : int
520 2 : spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst,
521 : void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
522 : void *cb_arg)
523 : {
524 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
525 : struct spdk_accel_task *accel_task;
526 :
527 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
528 2 : if (spdk_unlikely(accel_task == NULL)) {
529 1 : return -ENOMEM;
530 : }
531 :
532 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
533 :
534 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
535 1 : accel_task->s.iovs[0].iov_base = src;
536 1 : accel_task->s.iovs[0].iov_len = nbytes;
537 1 : accel_task->s.iovcnt = 1;
538 1 : accel_task->nbytes = nbytes;
539 1 : accel_task->crc_dst = crc_dst;
540 1 : accel_task->seed = seed;
541 1 : accel_task->op_code = SPDK_ACCEL_OPC_CRC32C;
542 1 : accel_task->src_domain = NULL;
543 1 : accel_task->dst_domain = NULL;
544 :
545 1 : return accel_submit_task(accel_ch, accel_task);
546 : }
547 :
548 : /* Accel framework public API for chained CRC-32C function */
549 : int
550 1 : spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst,
551 : struct iovec *iov, uint32_t iov_cnt, uint32_t seed,
552 : spdk_accel_completion_cb cb_fn, void *cb_arg)
553 : {
554 1 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
555 : struct spdk_accel_task *accel_task;
556 :
557 1 : if (iov == NULL) {
558 0 : SPDK_ERRLOG("iov should not be NULL");
559 0 : return -EINVAL;
560 : }
561 :
562 1 : if (!iov_cnt) {
563 0 : SPDK_ERRLOG("iovcnt should not be zero value\n");
564 0 : return -EINVAL;
565 : }
566 :
567 1 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
568 1 : if (spdk_unlikely(accel_task == NULL)) {
569 0 : SPDK_ERRLOG("no memory\n");
570 0 : assert(0);
571 : return -ENOMEM;
572 : }
573 :
574 1 : accel_task->s.iovs = iov;
575 1 : accel_task->s.iovcnt = iov_cnt;
576 1 : accel_task->nbytes = accel_get_iovlen(iov, iov_cnt);
577 1 : accel_task->crc_dst = crc_dst;
578 1 : accel_task->seed = seed;
579 1 : accel_task->op_code = SPDK_ACCEL_OPC_CRC32C;
580 1 : accel_task->src_domain = NULL;
581 1 : accel_task->dst_domain = NULL;
582 :
583 1 : return accel_submit_task(accel_ch, accel_task);
584 : }
585 :
586 : /* Accel framework public API for copy with CRC-32C function */
587 : int
588 2 : spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst,
589 : void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes,
590 : spdk_accel_completion_cb cb_fn, void *cb_arg)
591 : {
592 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
593 : struct spdk_accel_task *accel_task;
594 :
595 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
596 2 : if (spdk_unlikely(accel_task == NULL)) {
597 1 : return -ENOMEM;
598 : }
599 :
600 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
601 :
602 1 : accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC];
603 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
604 1 : accel_task->d.iovs[0].iov_base = dst;
605 1 : accel_task->d.iovs[0].iov_len = nbytes;
606 1 : accel_task->d.iovcnt = 1;
607 1 : accel_task->s.iovs[0].iov_base = src;
608 1 : accel_task->s.iovs[0].iov_len = nbytes;
609 1 : accel_task->s.iovcnt = 1;
610 1 : accel_task->nbytes = nbytes;
611 1 : accel_task->crc_dst = crc_dst;
612 1 : accel_task->seed = seed;
613 1 : accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C;
614 1 : accel_task->src_domain = NULL;
615 1 : accel_task->dst_domain = NULL;
616 :
617 1 : return accel_submit_task(accel_ch, accel_task);
618 : }
619 :
620 : /* Accel framework public API for chained copy + CRC-32C function */
621 : int
622 0 : spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst,
623 : struct iovec *src_iovs, uint32_t iov_cnt, uint32_t *crc_dst,
624 : uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg)
625 : {
626 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
627 : struct spdk_accel_task *accel_task;
628 : uint64_t nbytes;
629 :
630 0 : if (src_iovs == NULL) {
631 0 : SPDK_ERRLOG("iov should not be NULL");
632 0 : return -EINVAL;
633 : }
634 :
635 0 : if (!iov_cnt) {
636 0 : SPDK_ERRLOG("iovcnt should not be zero value\n");
637 0 : return -EINVAL;
638 : }
639 :
640 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
641 0 : if (spdk_unlikely(accel_task == NULL)) {
642 0 : SPDK_ERRLOG("no memory\n");
643 0 : assert(0);
644 : return -ENOMEM;
645 : }
646 :
647 0 : nbytes = accel_get_iovlen(src_iovs, iov_cnt);
648 :
649 0 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
650 :
651 0 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
652 0 : accel_task->d.iovs[0].iov_base = dst;
653 0 : accel_task->d.iovs[0].iov_len = nbytes;
654 0 : accel_task->d.iovcnt = 1;
655 0 : accel_task->s.iovs = src_iovs;
656 0 : accel_task->s.iovcnt = iov_cnt;
657 0 : accel_task->nbytes = nbytes;
658 0 : accel_task->crc_dst = crc_dst;
659 0 : accel_task->seed = seed;
660 0 : accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C;
661 0 : accel_task->src_domain = NULL;
662 0 : accel_task->dst_domain = NULL;
663 :
664 0 : return accel_submit_task(accel_ch, accel_task);
665 : }
666 :
667 : int
668 0 : spdk_accel_submit_compress(struct spdk_io_channel *ch, void *dst, uint64_t nbytes,
669 : struct iovec *src_iovs, size_t src_iovcnt, uint32_t *output_size,
670 : spdk_accel_completion_cb cb_fn, void *cb_arg)
671 : {
672 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
673 : struct spdk_accel_task *accel_task;
674 :
675 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
676 0 : if (spdk_unlikely(accel_task == NULL)) {
677 0 : return -ENOMEM;
678 : }
679 :
680 0 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
681 :
682 0 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
683 0 : accel_task->d.iovs[0].iov_base = dst;
684 0 : accel_task->d.iovs[0].iov_len = nbytes;
685 0 : accel_task->d.iovcnt = 1;
686 0 : accel_task->output_size = output_size;
687 0 : accel_task->s.iovs = src_iovs;
688 0 : accel_task->s.iovcnt = src_iovcnt;
689 0 : accel_task->nbytes = nbytes;
690 0 : accel_task->op_code = SPDK_ACCEL_OPC_COMPRESS;
691 0 : accel_task->src_domain = NULL;
692 0 : accel_task->dst_domain = NULL;
693 :
694 0 : return accel_submit_task(accel_ch, accel_task);
695 : }
696 :
697 : int
698 0 : spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs,
699 : size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt,
700 : uint32_t *output_size, spdk_accel_completion_cb cb_fn,
701 : void *cb_arg)
702 : {
703 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
704 : struct spdk_accel_task *accel_task;
705 :
706 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
707 0 : if (spdk_unlikely(accel_task == NULL)) {
708 0 : return -ENOMEM;
709 : }
710 :
711 0 : accel_task->output_size = output_size;
712 0 : accel_task->s.iovs = src_iovs;
713 0 : accel_task->s.iovcnt = src_iovcnt;
714 0 : accel_task->d.iovs = dst_iovs;
715 0 : accel_task->d.iovcnt = dst_iovcnt;
716 0 : accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
717 0 : accel_task->op_code = SPDK_ACCEL_OPC_DECOMPRESS;
718 0 : accel_task->src_domain = NULL;
719 0 : accel_task->dst_domain = NULL;
720 :
721 0 : return accel_submit_task(accel_ch, accel_task);
722 : }
723 :
724 : int
725 0 : spdk_accel_submit_encrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key,
726 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
727 : struct iovec *src_iovs, uint32_t src_iovcnt,
728 : uint64_t iv, uint32_t block_size,
729 : spdk_accel_completion_cb cb_fn, void *cb_arg)
730 : {
731 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
732 : struct spdk_accel_task *accel_task;
733 :
734 0 : if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) {
735 0 : return -EINVAL;
736 : }
737 :
738 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
739 0 : if (spdk_unlikely(accel_task == NULL)) {
740 0 : return -ENOMEM;
741 : }
742 :
743 0 : accel_task->crypto_key = key;
744 0 : accel_task->s.iovs = src_iovs;
745 0 : accel_task->s.iovcnt = src_iovcnt;
746 0 : accel_task->d.iovs = dst_iovs;
747 0 : accel_task->d.iovcnt = dst_iovcnt;
748 0 : accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
749 0 : accel_task->iv = iv;
750 0 : accel_task->block_size = block_size;
751 0 : accel_task->op_code = SPDK_ACCEL_OPC_ENCRYPT;
752 0 : accel_task->src_domain = NULL;
753 0 : accel_task->dst_domain = NULL;
754 :
755 0 : return accel_submit_task(accel_ch, accel_task);
756 : }
757 :
758 : int
759 0 : spdk_accel_submit_decrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key,
760 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
761 : struct iovec *src_iovs, uint32_t src_iovcnt,
762 : uint64_t iv, uint32_t block_size,
763 : spdk_accel_completion_cb cb_fn, void *cb_arg)
764 : {
765 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
766 : struct spdk_accel_task *accel_task;
767 :
768 0 : if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) {
769 0 : return -EINVAL;
770 : }
771 :
772 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
773 0 : if (spdk_unlikely(accel_task == NULL)) {
774 0 : return -ENOMEM;
775 : }
776 :
777 0 : accel_task->crypto_key = key;
778 0 : accel_task->s.iovs = src_iovs;
779 0 : accel_task->s.iovcnt = src_iovcnt;
780 0 : accel_task->d.iovs = dst_iovs;
781 0 : accel_task->d.iovcnt = dst_iovcnt;
782 0 : accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
783 0 : accel_task->iv = iv;
784 0 : accel_task->block_size = block_size;
785 0 : accel_task->op_code = SPDK_ACCEL_OPC_DECRYPT;
786 0 : accel_task->src_domain = NULL;
787 0 : accel_task->dst_domain = NULL;
788 :
789 0 : return accel_submit_task(accel_ch, accel_task);
790 : }
791 :
792 : int
793 2 : spdk_accel_submit_xor(struct spdk_io_channel *ch, void *dst, void **sources, uint32_t nsrcs,
794 : uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
795 : {
796 2 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
797 : struct spdk_accel_task *accel_task;
798 :
799 2 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
800 2 : if (spdk_unlikely(accel_task == NULL)) {
801 1 : return -ENOMEM;
802 : }
803 :
804 1 : ACCEL_TASK_ALLOC_AUX_BUF(accel_task);
805 :
806 1 : accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
807 1 : accel_task->nsrcs.srcs = sources;
808 1 : accel_task->nsrcs.cnt = nsrcs;
809 1 : accel_task->d.iovs[0].iov_base = dst;
810 1 : accel_task->d.iovs[0].iov_len = nbytes;
811 1 : accel_task->d.iovcnt = 1;
812 1 : accel_task->nbytes = nbytes;
813 1 : accel_task->op_code = SPDK_ACCEL_OPC_XOR;
814 1 : accel_task->src_domain = NULL;
815 1 : accel_task->dst_domain = NULL;
816 :
817 1 : return accel_submit_task(accel_ch, accel_task);
818 : }
819 :
820 : int
821 0 : spdk_accel_submit_dif_verify(struct spdk_io_channel *ch,
822 : struct iovec *iovs, size_t iovcnt, uint32_t num_blocks,
823 : const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
824 : spdk_accel_completion_cb cb_fn, void *cb_arg)
825 : {
826 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
827 : struct spdk_accel_task *accel_task;
828 :
829 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
830 0 : if (accel_task == NULL) {
831 0 : return -ENOMEM;
832 : }
833 :
834 0 : accel_task->s.iovs = iovs;
835 0 : accel_task->s.iovcnt = iovcnt;
836 0 : accel_task->dif.ctx = ctx;
837 0 : accel_task->dif.err = err;
838 0 : accel_task->dif.num_blocks = num_blocks;
839 0 : accel_task->nbytes = num_blocks * ctx->block_size;
840 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY;
841 0 : accel_task->src_domain = NULL;
842 0 : accel_task->dst_domain = NULL;
843 :
844 0 : return accel_submit_task(accel_ch, accel_task);
845 : }
846 :
847 : int
848 0 : spdk_accel_submit_dif_generate(struct spdk_io_channel *ch,
849 : struct iovec *iovs, size_t iovcnt, uint32_t num_blocks,
850 : const struct spdk_dif_ctx *ctx,
851 : spdk_accel_completion_cb cb_fn, void *cb_arg)
852 : {
853 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
854 : struct spdk_accel_task *accel_task;
855 :
856 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
857 0 : if (accel_task == NULL) {
858 0 : return -ENOMEM;
859 : }
860 :
861 0 : accel_task->s.iovs = iovs;
862 0 : accel_task->s.iovcnt = iovcnt;
863 0 : accel_task->dif.ctx = ctx;
864 0 : accel_task->dif.num_blocks = num_blocks;
865 0 : accel_task->nbytes = num_blocks * ctx->block_size;
866 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE;
867 0 : accel_task->src_domain = NULL;
868 0 : accel_task->dst_domain = NULL;
869 :
870 0 : return accel_submit_task(accel_ch, accel_task);
871 : }
872 :
873 : int
874 0 : spdk_accel_submit_dif_generate_copy(struct spdk_io_channel *ch, struct iovec *dst_iovs,
875 : size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt,
876 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
877 : spdk_accel_completion_cb cb_fn, void *cb_arg)
878 : {
879 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
880 : struct spdk_accel_task *accel_task;
881 :
882 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
883 0 : if (accel_task == NULL) {
884 0 : return -ENOMEM;
885 : }
886 :
887 0 : accel_task->s.iovs = src_iovs;
888 0 : accel_task->s.iovcnt = src_iovcnt;
889 0 : accel_task->d.iovs = dst_iovs;
890 0 : accel_task->d.iovcnt = dst_iovcnt;
891 0 : accel_task->dif.ctx = ctx;
892 0 : accel_task->dif.num_blocks = num_blocks;
893 0 : accel_task->nbytes = num_blocks * ctx->block_size;
894 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE_COPY;
895 0 : accel_task->src_domain = NULL;
896 0 : accel_task->dst_domain = NULL;
897 :
898 0 : return accel_submit_task(accel_ch, accel_task);
899 : }
900 :
901 : int
902 0 : spdk_accel_submit_dif_verify_copy(struct spdk_io_channel *ch,
903 : struct iovec *dst_iovs, size_t dst_iovcnt,
904 : struct iovec *src_iovs, size_t src_iovcnt, uint32_t num_blocks,
905 : const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,
906 : spdk_accel_completion_cb cb_fn, void *cb_arg)
907 : {
908 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
909 : struct spdk_accel_task *accel_task;
910 :
911 0 : accel_task = _get_task(accel_ch, cb_fn, cb_arg);
912 0 : if (accel_task == NULL) {
913 0 : return -ENOMEM;
914 : }
915 :
916 0 : accel_task->s.iovs = src_iovs;
917 0 : accel_task->s.iovcnt = src_iovcnt;
918 0 : accel_task->d.iovs = dst_iovs;
919 0 : accel_task->d.iovcnt = dst_iovcnt;
920 0 : accel_task->dif.ctx = ctx;
921 0 : accel_task->dif.err = err;
922 0 : accel_task->dif.num_blocks = num_blocks;
923 0 : accel_task->nbytes = num_blocks * ctx->block_size;
924 0 : accel_task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY_COPY;
925 0 : accel_task->src_domain = NULL;
926 0 : accel_task->dst_domain = NULL;
927 :
928 0 : return accel_submit_task(accel_ch, accel_task);
929 : }
930 :
931 : static inline struct accel_buffer *
932 33 : accel_get_buf(struct accel_io_channel *ch, uint64_t len)
933 : {
934 : struct accel_buffer *buf;
935 :
936 33 : buf = SLIST_FIRST(&ch->buf_pool);
937 33 : if (spdk_unlikely(buf == NULL)) {
938 0 : accel_update_stats(ch, retry.bufdesc, 1);
939 0 : return NULL;
940 : }
941 :
942 33 : SLIST_REMOVE_HEAD(&ch->buf_pool, link);
943 33 : buf->len = len;
944 33 : buf->buf = NULL;
945 33 : buf->seq = NULL;
946 33 : buf->cb_fn = NULL;
947 :
948 33 : return buf;
949 : }
950 :
951 : static inline void
952 33 : accel_put_buf(struct accel_io_channel *ch, struct accel_buffer *buf)
953 : {
954 33 : if (buf->buf != NULL) {
955 29 : spdk_iobuf_put(&ch->iobuf, buf->buf, buf->len);
956 : }
957 :
958 33 : SLIST_INSERT_HEAD(&ch->buf_pool, buf, link);
959 33 : }
960 :
961 : static inline struct spdk_accel_sequence *
962 60 : accel_sequence_get(struct accel_io_channel *ch)
963 : {
964 : struct spdk_accel_sequence *seq;
965 :
966 : /* Sequence cannot be allocated if number of available task objects cannot satisfy required limit.
967 : * This is to prevent potential dead lock when few requests are pending task resource and none can
968 : * advance the processing. This solution should work only if there is single async operation after
969 : * sequence obj obtained, so assume that is possible to happen with io buffer allocation now, if
970 : * there are more async operations then solution should be improved. */
971 60 : if (spdk_unlikely(g_opts.task_count - ch->stats.task_outstanding < ACCEL_TASKS_IN_SEQUENCE_LIMIT)) {
972 0 : return NULL;
973 : }
974 :
975 60 : seq = SLIST_FIRST(&ch->seq_pool);
976 60 : if (spdk_unlikely(seq == NULL)) {
977 3 : accel_update_stats(ch, retry.sequence, 1);
978 3 : return NULL;
979 : }
980 :
981 57 : accel_update_stats(ch, sequence_outstanding, 1);
982 57 : SLIST_REMOVE_HEAD(&ch->seq_pool, link);
983 :
984 57 : TAILQ_INIT(&seq->tasks);
985 57 : SLIST_INIT(&seq->bounce_bufs);
986 :
987 57 : seq->ch = ch;
988 57 : seq->status = 0;
989 57 : seq->state = ACCEL_SEQUENCE_STATE_INIT;
990 57 : seq->in_process_sequence = false;
991 :
992 57 : return seq;
993 : }
994 :
995 : static inline void
996 57 : accel_sequence_put(struct spdk_accel_sequence *seq)
997 : {
998 57 : struct accel_io_channel *ch = seq->ch;
999 : struct accel_buffer *buf;
1000 :
1001 76 : while (!SLIST_EMPTY(&seq->bounce_bufs)) {
1002 19 : buf = SLIST_FIRST(&seq->bounce_bufs);
1003 19 : SLIST_REMOVE_HEAD(&seq->bounce_bufs, link);
1004 19 : accel_put_buf(seq->ch, buf);
1005 : }
1006 :
1007 57 : assert(TAILQ_EMPTY(&seq->tasks));
1008 57 : seq->ch = NULL;
1009 :
1010 57 : SLIST_INSERT_HEAD(&ch->seq_pool, seq, link);
1011 57 : accel_update_stats(ch, sequence_outstanding, -1);
1012 57 : }
1013 :
1014 : static void accel_sequence_task_cb(void *cb_arg, int status);
1015 :
1016 : static inline struct spdk_accel_task *
1017 134 : accel_sequence_get_task(struct accel_io_channel *ch, struct spdk_accel_sequence *seq,
1018 : spdk_accel_step_cb cb_fn, void *cb_arg)
1019 : {
1020 : struct spdk_accel_task *task;
1021 :
1022 134 : task = _get_task(ch, NULL, NULL);
1023 134 : if (spdk_unlikely(task == NULL)) {
1024 3 : return task;
1025 : }
1026 :
1027 131 : task->step_cb_fn = cb_fn;
1028 131 : task->cb_arg = cb_arg;
1029 131 : task->seq = seq;
1030 :
1031 131 : return task;
1032 : }
1033 :
1034 : int
1035 35 : spdk_accel_append_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1036 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
1037 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1038 : struct iovec *src_iovs, uint32_t src_iovcnt,
1039 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1040 : spdk_accel_step_cb cb_fn, void *cb_arg)
1041 : {
1042 35 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1043 : struct spdk_accel_task *task;
1044 35 : struct spdk_accel_sequence *seq = *pseq;
1045 :
1046 35 : if (seq == NULL) {
1047 12 : seq = accel_sequence_get(accel_ch);
1048 12 : if (spdk_unlikely(seq == NULL)) {
1049 1 : return -ENOMEM;
1050 : }
1051 : }
1052 :
1053 34 : assert(seq->ch == accel_ch);
1054 34 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1055 34 : if (spdk_unlikely(task == NULL)) {
1056 1 : if (*pseq == NULL) {
1057 1 : accel_sequence_put(seq);
1058 : }
1059 :
1060 1 : return -ENOMEM;
1061 : }
1062 :
1063 33 : task->dst_domain = dst_domain;
1064 33 : task->dst_domain_ctx = dst_domain_ctx;
1065 33 : task->d.iovs = dst_iovs;
1066 33 : task->d.iovcnt = dst_iovcnt;
1067 33 : task->src_domain = src_domain;
1068 33 : task->src_domain_ctx = src_domain_ctx;
1069 33 : task->s.iovs = src_iovs;
1070 33 : task->s.iovcnt = src_iovcnt;
1071 33 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1072 33 : task->op_code = SPDK_ACCEL_OPC_COPY;
1073 :
1074 33 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1075 33 : *pseq = seq;
1076 :
1077 33 : return 0;
1078 : }
1079 :
1080 : int
1081 39 : spdk_accel_append_fill(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1082 : void *buf, uint64_t len,
1083 : struct spdk_memory_domain *domain, void *domain_ctx, uint8_t pattern,
1084 : spdk_accel_step_cb cb_fn, void *cb_arg)
1085 : {
1086 39 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1087 : struct spdk_accel_task *task;
1088 39 : struct spdk_accel_sequence *seq = *pseq;
1089 :
1090 39 : if (seq == NULL) {
1091 23 : seq = accel_sequence_get(accel_ch);
1092 23 : if (spdk_unlikely(seq == NULL)) {
1093 1 : return -ENOMEM;
1094 : }
1095 : }
1096 :
1097 38 : assert(seq->ch == accel_ch);
1098 38 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1099 38 : if (spdk_unlikely(task == NULL)) {
1100 1 : if (*pseq == NULL) {
1101 1 : accel_sequence_put(seq);
1102 : }
1103 :
1104 1 : return -ENOMEM;
1105 : }
1106 :
1107 37 : memset(&task->fill_pattern, pattern, sizeof(uint64_t));
1108 :
1109 37 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1110 37 : if (spdk_unlikely(!task->aux)) {
1111 0 : SPDK_ERRLOG("Fatal problem, aux data was not allocated\n");
1112 0 : if (*pseq == NULL) {
1113 0 : accel_sequence_put((seq));
1114 : }
1115 :
1116 0 : task->seq = NULL;
1117 0 : _put_task(task->accel_ch, task);
1118 0 : assert(0);
1119 : return -ENOMEM;
1120 : }
1121 37 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1122 37 : task->has_aux = true;
1123 :
1124 37 : task->d.iovs = &task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST];
1125 37 : task->d.iovs[0].iov_base = buf;
1126 37 : task->d.iovs[0].iov_len = len;
1127 37 : task->d.iovcnt = 1;
1128 37 : task->nbytes = len;
1129 37 : task->src_domain = NULL;
1130 37 : task->dst_domain = domain;
1131 37 : task->dst_domain_ctx = domain_ctx;
1132 37 : task->op_code = SPDK_ACCEL_OPC_FILL;
1133 :
1134 37 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1135 37 : *pseq = seq;
1136 :
1137 37 : return 0;
1138 : }
1139 :
1140 : int
1141 40 : spdk_accel_append_decompress(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1142 : struct iovec *dst_iovs, size_t dst_iovcnt,
1143 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1144 : struct iovec *src_iovs, size_t src_iovcnt,
1145 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1146 : spdk_accel_step_cb cb_fn, void *cb_arg)
1147 : {
1148 40 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1149 : struct spdk_accel_task *task;
1150 40 : struct spdk_accel_sequence *seq = *pseq;
1151 :
1152 40 : if (seq == NULL) {
1153 17 : seq = accel_sequence_get(accel_ch);
1154 17 : if (spdk_unlikely(seq == NULL)) {
1155 1 : return -ENOMEM;
1156 : }
1157 : }
1158 :
1159 39 : assert(seq->ch == accel_ch);
1160 39 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1161 39 : if (spdk_unlikely(task == NULL)) {
1162 1 : if (*pseq == NULL) {
1163 1 : accel_sequence_put(seq);
1164 : }
1165 :
1166 1 : return -ENOMEM;
1167 : }
1168 :
1169 : /* TODO: support output_size for chaining */
1170 38 : task->output_size = NULL;
1171 38 : task->dst_domain = dst_domain;
1172 38 : task->dst_domain_ctx = dst_domain_ctx;
1173 38 : task->d.iovs = dst_iovs;
1174 38 : task->d.iovcnt = dst_iovcnt;
1175 38 : task->src_domain = src_domain;
1176 38 : task->src_domain_ctx = src_domain_ctx;
1177 38 : task->s.iovs = src_iovs;
1178 38 : task->s.iovcnt = src_iovcnt;
1179 38 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1180 38 : task->op_code = SPDK_ACCEL_OPC_DECOMPRESS;
1181 :
1182 38 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1183 38 : *pseq = seq;
1184 :
1185 38 : return 0;
1186 : }
1187 :
1188 : int
1189 8 : spdk_accel_append_encrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1190 : struct spdk_accel_crypto_key *key,
1191 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
1192 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1193 : struct iovec *src_iovs, uint32_t src_iovcnt,
1194 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1195 : uint64_t iv, uint32_t block_size,
1196 : spdk_accel_step_cb cb_fn, void *cb_arg)
1197 : {
1198 8 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1199 : struct spdk_accel_task *task;
1200 8 : struct spdk_accel_sequence *seq = *pseq;
1201 :
1202 8 : assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size);
1203 :
1204 8 : if (seq == NULL) {
1205 6 : seq = accel_sequence_get(accel_ch);
1206 6 : if (spdk_unlikely(seq == NULL)) {
1207 0 : return -ENOMEM;
1208 : }
1209 : }
1210 :
1211 8 : assert(seq->ch == accel_ch);
1212 8 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1213 8 : if (spdk_unlikely(task == NULL)) {
1214 0 : if (*pseq == NULL) {
1215 0 : accel_sequence_put(seq);
1216 : }
1217 :
1218 0 : return -ENOMEM;
1219 : }
1220 :
1221 8 : task->crypto_key = key;
1222 8 : task->src_domain = src_domain;
1223 8 : task->src_domain_ctx = src_domain_ctx;
1224 8 : task->s.iovs = src_iovs;
1225 8 : task->s.iovcnt = src_iovcnt;
1226 8 : task->dst_domain = dst_domain;
1227 8 : task->dst_domain_ctx = dst_domain_ctx;
1228 8 : task->d.iovs = dst_iovs;
1229 8 : task->d.iovcnt = dst_iovcnt;
1230 8 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1231 8 : task->iv = iv;
1232 8 : task->block_size = block_size;
1233 8 : task->op_code = SPDK_ACCEL_OPC_ENCRYPT;
1234 :
1235 8 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1236 8 : *pseq = seq;
1237 :
1238 8 : return 0;
1239 : }
1240 :
1241 : int
1242 9 : spdk_accel_append_decrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1243 : struct spdk_accel_crypto_key *key,
1244 : struct iovec *dst_iovs, uint32_t dst_iovcnt,
1245 : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
1246 : struct iovec *src_iovs, uint32_t src_iovcnt,
1247 : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
1248 : uint64_t iv, uint32_t block_size,
1249 : spdk_accel_step_cb cb_fn, void *cb_arg)
1250 : {
1251 9 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1252 : struct spdk_accel_task *task;
1253 9 : struct spdk_accel_sequence *seq = *pseq;
1254 :
1255 9 : assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size);
1256 :
1257 9 : if (seq == NULL) {
1258 0 : seq = accel_sequence_get(accel_ch);
1259 0 : if (spdk_unlikely(seq == NULL)) {
1260 0 : return -ENOMEM;
1261 : }
1262 : }
1263 :
1264 9 : assert(seq->ch == accel_ch);
1265 9 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1266 9 : if (spdk_unlikely(task == NULL)) {
1267 0 : if (*pseq == NULL) {
1268 0 : accel_sequence_put(seq);
1269 : }
1270 :
1271 0 : return -ENOMEM;
1272 : }
1273 :
1274 9 : task->crypto_key = key;
1275 9 : task->src_domain = src_domain;
1276 9 : task->src_domain_ctx = src_domain_ctx;
1277 9 : task->s.iovs = src_iovs;
1278 9 : task->s.iovcnt = src_iovcnt;
1279 9 : task->dst_domain = dst_domain;
1280 9 : task->dst_domain_ctx = dst_domain_ctx;
1281 9 : task->d.iovs = dst_iovs;
1282 9 : task->d.iovcnt = dst_iovcnt;
1283 9 : task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt);
1284 9 : task->iv = iv;
1285 9 : task->block_size = block_size;
1286 9 : task->op_code = SPDK_ACCEL_OPC_DECRYPT;
1287 :
1288 9 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1289 9 : *pseq = seq;
1290 :
1291 9 : return 0;
1292 : }
1293 :
1294 : int
1295 6 : spdk_accel_append_crc32c(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,
1296 : uint32_t *dst, struct iovec *iovs, uint32_t iovcnt,
1297 : struct spdk_memory_domain *domain, void *domain_ctx,
1298 : uint32_t seed, spdk_accel_step_cb cb_fn, void *cb_arg)
1299 : {
1300 6 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1301 : struct spdk_accel_task *task;
1302 6 : struct spdk_accel_sequence *seq = *pseq;
1303 :
1304 6 : if (seq == NULL) {
1305 2 : seq = accel_sequence_get(accel_ch);
1306 2 : if (spdk_unlikely(seq == NULL)) {
1307 0 : return -ENOMEM;
1308 : }
1309 : }
1310 :
1311 6 : assert(seq->ch == accel_ch);
1312 6 : task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);
1313 6 : if (spdk_unlikely(task == NULL)) {
1314 0 : if (*pseq == NULL) {
1315 0 : accel_sequence_put(seq);
1316 : }
1317 :
1318 0 : return -ENOMEM;
1319 : }
1320 :
1321 6 : task->s.iovs = iovs;
1322 6 : task->s.iovcnt = iovcnt;
1323 6 : task->src_domain = domain;
1324 6 : task->src_domain_ctx = domain_ctx;
1325 6 : task->nbytes = accel_get_iovlen(iovs, iovcnt);
1326 6 : task->crc_dst = dst;
1327 6 : task->seed = seed;
1328 6 : task->op_code = SPDK_ACCEL_OPC_CRC32C;
1329 6 : task->dst_domain = NULL;
1330 :
1331 6 : TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);
1332 6 : *pseq = seq;
1333 :
1334 6 : return 0;
1335 : }
1336 :
1337 : int
1338 14 : spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf,
1339 : struct spdk_memory_domain **domain, void **domain_ctx)
1340 : {
1341 14 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1342 : struct accel_buffer *accel_buf;
1343 :
1344 14 : accel_buf = accel_get_buf(accel_ch, len);
1345 14 : if (spdk_unlikely(accel_buf == NULL)) {
1346 0 : return -ENOMEM;
1347 : }
1348 :
1349 14 : accel_buf->ch = accel_ch;
1350 :
1351 : /* We always return the same pointer and identify the buffers through domain_ctx */
1352 14 : *buf = ACCEL_BUFFER_BASE;
1353 14 : *domain_ctx = accel_buf;
1354 14 : *domain = g_accel_domain;
1355 :
1356 14 : return 0;
1357 : }
1358 :
1359 : void
1360 14 : spdk_accel_put_buf(struct spdk_io_channel *ch, void *buf,
1361 : struct spdk_memory_domain *domain, void *domain_ctx)
1362 : {
1363 14 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
1364 14 : struct accel_buffer *accel_buf = domain_ctx;
1365 :
1366 14 : assert(domain == g_accel_domain);
1367 14 : assert(buf == ACCEL_BUFFER_BASE);
1368 :
1369 14 : accel_put_buf(accel_ch, accel_buf);
1370 14 : }
1371 :
1372 : static void
1373 131 : accel_sequence_complete_task(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
1374 : {
1375 131 : struct accel_io_channel *ch = seq->ch;
1376 : spdk_accel_step_cb cb_fn;
1377 : void *cb_arg;
1378 :
1379 131 : TAILQ_REMOVE(&seq->tasks, task, seq_link);
1380 131 : cb_fn = task->step_cb_fn;
1381 131 : cb_arg = task->cb_arg;
1382 131 : task->seq = NULL;
1383 131 : if (task->has_aux) {
1384 58 : SLIST_INSERT_HEAD(&ch->task_aux_data_pool, task->aux, link);
1385 58 : task->aux = NULL;
1386 58 : task->has_aux = false;
1387 : }
1388 :
1389 131 : _put_task(ch, task);
1390 :
1391 131 : if (cb_fn != NULL) {
1392 131 : cb_fn(cb_arg);
1393 : }
1394 131 : }
1395 :
1396 : static void
1397 54 : accel_sequence_complete_tasks(struct spdk_accel_sequence *seq)
1398 : {
1399 : struct spdk_accel_task *task;
1400 :
1401 72 : while (!TAILQ_EMPTY(&seq->tasks)) {
1402 18 : task = TAILQ_FIRST(&seq->tasks);
1403 18 : accel_sequence_complete_task(seq, task);
1404 : }
1405 54 : }
1406 :
1407 : static void
1408 52 : accel_sequence_complete(struct spdk_accel_sequence *seq)
1409 : {
1410 52 : spdk_accel_completion_cb cb_fn = seq->cb_fn;
1411 52 : void *cb_arg = seq->cb_arg;
1412 52 : int status = seq->status;
1413 :
1414 52 : SPDK_DEBUGLOG(accel, "Completed sequence: %p with status: %d\n", seq, status);
1415 :
1416 52 : accel_update_stats(seq->ch, sequence_executed, 1);
1417 52 : if (spdk_unlikely(status != 0)) {
1418 10 : accel_update_stats(seq->ch, sequence_failed, 1);
1419 : }
1420 :
1421 : /* First notify all users that appended operations to this sequence */
1422 52 : accel_sequence_complete_tasks(seq);
1423 52 : accel_sequence_put(seq);
1424 :
1425 : /* Then notify the user that finished the sequence */
1426 52 : cb_fn(cb_arg, status);
1427 52 : }
1428 :
1429 : static void
1430 28 : accel_update_virt_iov(struct iovec *diov, struct iovec *siov, struct accel_buffer *accel_buf)
1431 : {
1432 : uintptr_t offset;
1433 :
1434 28 : offset = (uintptr_t)siov->iov_base & ACCEL_BUFFER_OFFSET_MASK;
1435 28 : assert(offset < accel_buf->len);
1436 :
1437 28 : diov->iov_base = (char *)accel_buf->buf + offset;
1438 28 : diov->iov_len = siov->iov_len;
1439 28 : }
1440 :
1441 : static void
1442 10 : accel_sequence_set_virtbuf(struct spdk_accel_sequence *seq, struct accel_buffer *buf)
1443 : {
1444 : struct spdk_accel_task *task;
1445 : struct iovec *iov;
1446 :
1447 : /* Now that we've allocated the actual data buffer for this accel_buffer, update all tasks
1448 : * in a sequence that were using it.
1449 : */
1450 38 : TAILQ_FOREACH(task, &seq->tasks, seq_link) {
1451 28 : if (task->src_domain == g_accel_domain && task->src_domain_ctx == buf) {
1452 11 : if (!task->has_aux) {
1453 11 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1454 11 : assert(task->aux && "Can't allocate aux data structure");
1455 11 : task->has_aux = true;
1456 11 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1457 : }
1458 :
1459 11 : iov = &task->aux->iovs[SPDK_ACCEL_AXU_IOV_VIRT_SRC];
1460 11 : assert(task->s.iovcnt == 1);
1461 11 : accel_update_virt_iov(iov, &task->s.iovs[0], buf);
1462 11 : task->src_domain = NULL;
1463 11 : task->s.iovs = iov;
1464 : }
1465 28 : if (task->dst_domain == g_accel_domain && task->dst_domain_ctx == buf) {
1466 17 : if (!task->has_aux) {
1467 2 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1468 2 : assert(task->aux && "Can't allocate aux data structure");
1469 2 : task->has_aux = true;
1470 2 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1471 : }
1472 :
1473 17 : iov = &task->aux->iovs[SPDK_ACCEL_AXU_IOV_VIRT_DST];
1474 17 : assert(task->d.iovcnt == 1);
1475 17 : accel_update_virt_iov(iov, &task->d.iovs[0], buf);
1476 17 : task->dst_domain = NULL;
1477 17 : task->d.iovs = iov;
1478 : }
1479 : }
1480 10 : }
1481 :
1482 : static void accel_process_sequence(struct spdk_accel_sequence *seq);
1483 :
1484 : static void
1485 3 : accel_iobuf_get_virtbuf_cb(struct spdk_iobuf_entry *entry, void *buf)
1486 : {
1487 : struct accel_buffer *accel_buf;
1488 :
1489 3 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
1490 :
1491 3 : assert(accel_buf->seq != NULL);
1492 3 : assert(accel_buf->buf == NULL);
1493 3 : accel_buf->buf = buf;
1494 :
1495 3 : assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF);
1496 3 : accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF);
1497 3 : accel_sequence_set_virtbuf(accel_buf->seq, accel_buf);
1498 3 : accel_process_sequence(accel_buf->seq);
1499 3 : }
1500 :
1501 : static bool
1502 29 : accel_sequence_alloc_buf(struct spdk_accel_sequence *seq, struct accel_buffer *buf,
1503 : spdk_iobuf_get_cb cb_fn)
1504 : {
1505 29 : struct accel_io_channel *ch = seq->ch;
1506 :
1507 29 : assert(buf->seq == NULL);
1508 :
1509 29 : buf->seq = seq;
1510 :
1511 : /* Buffer might be already allocated by memory domain translation. */
1512 29 : if (buf->buf) {
1513 0 : return true;
1514 : }
1515 :
1516 29 : buf->buf = spdk_iobuf_get(&ch->iobuf, buf->len, &buf->iobuf, cb_fn);
1517 29 : if (spdk_unlikely(buf->buf == NULL)) {
1518 5 : accel_update_stats(ch, retry.iobuf, 1);
1519 5 : return false;
1520 : }
1521 :
1522 24 : return true;
1523 : }
1524 :
1525 : static bool
1526 89 : accel_sequence_check_virtbuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
1527 : {
1528 : /* If a task doesn't have dst/src (e.g. fill, crc32), its dst/src domain should be set to
1529 : * NULL */
1530 89 : if (task->src_domain == g_accel_domain) {
1531 0 : if (!accel_sequence_alloc_buf(seq, task->src_domain_ctx,
1532 : accel_iobuf_get_virtbuf_cb)) {
1533 0 : return false;
1534 : }
1535 :
1536 0 : accel_sequence_set_virtbuf(seq, task->src_domain_ctx);
1537 : }
1538 :
1539 89 : if (task->dst_domain == g_accel_domain) {
1540 10 : if (!accel_sequence_alloc_buf(seq, task->dst_domain_ctx,
1541 : accel_iobuf_get_virtbuf_cb)) {
1542 3 : return false;
1543 : }
1544 :
1545 7 : accel_sequence_set_virtbuf(seq, task->dst_domain_ctx);
1546 : }
1547 :
1548 86 : return true;
1549 : }
1550 :
1551 : static void
1552 0 : accel_sequence_get_buf_cb(struct spdk_iobuf_entry *entry, void *buf)
1553 : {
1554 : struct accel_buffer *accel_buf;
1555 :
1556 0 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
1557 :
1558 0 : assert(accel_buf->seq != NULL);
1559 0 : assert(accel_buf->buf == NULL);
1560 0 : accel_buf->buf = buf;
1561 :
1562 0 : accel_sequence_set_virtbuf(accel_buf->seq, accel_buf);
1563 0 : accel_buf->cb_fn(accel_buf->seq, accel_buf->cb_ctx);
1564 0 : }
1565 :
1566 : bool
1567 0 : spdk_accel_alloc_sequence_buf(struct spdk_accel_sequence *seq, void *buf,
1568 : struct spdk_memory_domain *domain, void *domain_ctx,
1569 : spdk_accel_sequence_get_buf_cb cb_fn, void *cb_ctx)
1570 : {
1571 0 : struct accel_buffer *accel_buf = domain_ctx;
1572 :
1573 0 : assert(domain == g_accel_domain);
1574 0 : accel_buf->cb_fn = cb_fn;
1575 0 : accel_buf->cb_ctx = cb_ctx;
1576 :
1577 0 : if (!accel_sequence_alloc_buf(seq, accel_buf, accel_sequence_get_buf_cb)) {
1578 0 : return false;
1579 : }
1580 :
1581 0 : accel_sequence_set_virtbuf(seq, accel_buf);
1582 :
1583 0 : return true;
1584 : }
1585 :
1586 : struct spdk_accel_task *
1587 24 : spdk_accel_sequence_first_task(struct spdk_accel_sequence *seq)
1588 : {
1589 24 : return TAILQ_FIRST(&seq->tasks);
1590 : }
1591 :
1592 : struct spdk_accel_task *
1593 0 : spdk_accel_sequence_next_task(struct spdk_accel_task *task)
1594 : {
1595 0 : return TAILQ_NEXT(task, seq_link);
1596 : }
1597 :
1598 : static inline void
1599 19 : accel_set_bounce_buffer(struct spdk_accel_bounce_buffer *bounce, struct iovec **iovs,
1600 : uint32_t *iovcnt, struct spdk_memory_domain **domain, void **domain_ctx,
1601 : struct accel_buffer *buf)
1602 : {
1603 19 : bounce->orig_iovs = *iovs;
1604 19 : bounce->orig_iovcnt = *iovcnt;
1605 19 : bounce->orig_domain = *domain;
1606 19 : bounce->orig_domain_ctx = *domain_ctx;
1607 19 : bounce->iov.iov_base = buf->buf;
1608 19 : bounce->iov.iov_len = buf->len;
1609 :
1610 19 : *iovs = &bounce->iov;
1611 19 : *iovcnt = 1;
1612 19 : *domain = NULL;
1613 19 : }
1614 :
1615 : static void
1616 1 : accel_iobuf_get_src_bounce_cb(struct spdk_iobuf_entry *entry, void *buf)
1617 : {
1618 : struct spdk_accel_task *task;
1619 : struct accel_buffer *accel_buf;
1620 :
1621 1 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
1622 1 : assert(accel_buf->buf == NULL);
1623 1 : accel_buf->buf = buf;
1624 :
1625 1 : task = TAILQ_FIRST(&accel_buf->seq->tasks);
1626 1 : assert(task != NULL);
1627 :
1628 1 : assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
1629 1 : accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
1630 1 : assert(task->aux);
1631 1 : assert(task->has_aux);
1632 1 : accel_set_bounce_buffer(&task->aux->bounce.s, &task->s.iovs, &task->s.iovcnt, &task->src_domain,
1633 : &task->src_domain_ctx, accel_buf);
1634 1 : accel_process_sequence(accel_buf->seq);
1635 1 : }
1636 :
1637 : static void
1638 1 : accel_iobuf_get_dst_bounce_cb(struct spdk_iobuf_entry *entry, void *buf)
1639 : {
1640 : struct spdk_accel_task *task;
1641 : struct accel_buffer *accel_buf;
1642 :
1643 1 : accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);
1644 1 : assert(accel_buf->buf == NULL);
1645 1 : accel_buf->buf = buf;
1646 :
1647 1 : task = TAILQ_FIRST(&accel_buf->seq->tasks);
1648 1 : assert(task != NULL);
1649 :
1650 1 : assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
1651 1 : accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
1652 1 : assert(task->aux);
1653 1 : assert(task->has_aux);
1654 1 : accel_set_bounce_buffer(&task->aux->bounce.d, &task->d.iovs, &task->d.iovcnt, &task->dst_domain,
1655 : &task->dst_domain_ctx, accel_buf);
1656 1 : accel_process_sequence(accel_buf->seq);
1657 1 : }
1658 :
1659 : static int
1660 73 : accel_sequence_check_bouncebuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
1661 : {
1662 : struct accel_buffer *buf;
1663 :
1664 73 : if (task->src_domain != NULL) {
1665 : /* By the time we're here, accel buffers should have been allocated */
1666 9 : assert(task->src_domain != g_accel_domain);
1667 :
1668 9 : if (!task->has_aux) {
1669 8 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1670 8 : if (spdk_unlikely(!task->aux)) {
1671 0 : SPDK_ERRLOG("Can't allocate aux data structure\n");
1672 0 : assert(0);
1673 : return -EAGAIN;
1674 : }
1675 8 : task->has_aux = true;
1676 8 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1677 : }
1678 9 : buf = accel_get_buf(seq->ch, accel_get_iovlen(task->s.iovs, task->s.iovcnt));
1679 9 : if (buf == NULL) {
1680 0 : SPDK_ERRLOG("Couldn't allocate buffer descriptor\n");
1681 0 : return -ENOMEM;
1682 : }
1683 :
1684 9 : SLIST_INSERT_HEAD(&seq->bounce_bufs, buf, link);
1685 9 : if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_src_bounce_cb)) {
1686 1 : return -EAGAIN;
1687 : }
1688 :
1689 8 : accel_set_bounce_buffer(&task->aux->bounce.s, &task->s.iovs, &task->s.iovcnt,
1690 : &task->src_domain, &task->src_domain_ctx, buf);
1691 : }
1692 :
1693 72 : if (task->dst_domain != NULL) {
1694 : /* By the time we're here, accel buffers should have been allocated */
1695 10 : assert(task->dst_domain != g_accel_domain);
1696 :
1697 10 : if (!task->has_aux) {
1698 0 : task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool);
1699 0 : if (spdk_unlikely(!task->aux)) {
1700 0 : SPDK_ERRLOG("Can't allocate aux data structure\n");
1701 0 : assert(0);
1702 : return -EAGAIN;
1703 : }
1704 0 : task->has_aux = true;
1705 0 : SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link);
1706 : }
1707 10 : buf = accel_get_buf(seq->ch, accel_get_iovlen(task->d.iovs, task->d.iovcnt));
1708 10 : if (buf == NULL) {
1709 : /* The src buffer will be released when a sequence is completed */
1710 0 : SPDK_ERRLOG("Couldn't allocate buffer descriptor\n");
1711 0 : return -ENOMEM;
1712 : }
1713 :
1714 10 : SLIST_INSERT_HEAD(&seq->bounce_bufs, buf, link);
1715 10 : if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_dst_bounce_cb)) {
1716 1 : return -EAGAIN;
1717 : }
1718 :
1719 9 : accel_set_bounce_buffer(&task->aux->bounce.d, &task->d.iovs, &task->d.iovcnt,
1720 : &task->dst_domain, &task->dst_domain_ctx, buf);
1721 : }
1722 :
1723 71 : return 0;
1724 : }
1725 :
1726 : static void
1727 8 : accel_task_pull_data_cb(void *ctx, int status)
1728 : {
1729 8 : struct spdk_accel_sequence *seq = ctx;
1730 :
1731 8 : assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA);
1732 8 : if (spdk_likely(status == 0)) {
1733 7 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
1734 : } else {
1735 1 : accel_sequence_set_fail(seq, status);
1736 : }
1737 :
1738 8 : accel_process_sequence(seq);
1739 8 : }
1740 :
1741 : static void
1742 9 : accel_task_pull_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
1743 : {
1744 : int rc;
1745 :
1746 9 : assert(task->has_aux);
1747 9 : assert(task->aux);
1748 9 : assert(task->aux->bounce.s.orig_iovs != NULL);
1749 9 : assert(task->aux->bounce.s.orig_domain != NULL);
1750 9 : assert(task->aux->bounce.s.orig_domain != g_accel_domain);
1751 9 : assert(!g_modules_opc[task->op_code].supports_memory_domains);
1752 :
1753 9 : rc = spdk_memory_domain_pull_data(task->aux->bounce.s.orig_domain,
1754 9 : task->aux->bounce.s.orig_domain_ctx,
1755 9 : task->aux->bounce.s.orig_iovs, task->aux->bounce.s.orig_iovcnt,
1756 : task->s.iovs, task->s.iovcnt,
1757 : accel_task_pull_data_cb, seq);
1758 9 : if (spdk_unlikely(rc != 0)) {
1759 1 : SPDK_ERRLOG("Failed to pull data from memory domain: %s, rc: %d\n",
1760 : spdk_memory_domain_get_dma_device_id(task->aux->bounce.s.orig_domain), rc);
1761 1 : accel_sequence_set_fail(seq, rc);
1762 : }
1763 9 : }
1764 :
1765 : static void
1766 7 : accel_task_push_data_cb(void *ctx, int status)
1767 : {
1768 7 : struct spdk_accel_sequence *seq = ctx;
1769 :
1770 7 : assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA);
1771 7 : if (spdk_likely(status == 0)) {
1772 6 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK);
1773 : } else {
1774 1 : accel_sequence_set_fail(seq, status);
1775 : }
1776 :
1777 7 : accel_process_sequence(seq);
1778 7 : }
1779 :
1780 : static void
1781 8 : accel_task_push_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)
1782 : {
1783 : int rc;
1784 :
1785 8 : assert(task->has_aux);
1786 8 : assert(task->aux);
1787 8 : assert(task->aux->bounce.d.orig_iovs != NULL);
1788 8 : assert(task->aux->bounce.d.orig_domain != NULL);
1789 8 : assert(task->aux->bounce.d.orig_domain != g_accel_domain);
1790 8 : assert(!g_modules_opc[task->op_code].supports_memory_domains);
1791 :
1792 8 : rc = spdk_memory_domain_push_data(task->aux->bounce.d.orig_domain,
1793 8 : task->aux->bounce.d.orig_domain_ctx,
1794 8 : task->aux->bounce.d.orig_iovs, task->aux->bounce.d.orig_iovcnt,
1795 : task->d.iovs, task->d.iovcnt,
1796 : accel_task_push_data_cb, seq);
1797 8 : if (spdk_unlikely(rc != 0)) {
1798 1 : SPDK_ERRLOG("Failed to push data to memory domain: %s, rc: %d\n",
1799 : spdk_memory_domain_get_dma_device_id(task->aux->bounce.s.orig_domain), rc);
1800 1 : accel_sequence_set_fail(seq, rc);
1801 : }
1802 8 : }
1803 :
1804 : static void
1805 166 : accel_process_sequence(struct spdk_accel_sequence *seq)
1806 : {
1807 166 : struct accel_io_channel *accel_ch = seq->ch;
1808 : struct spdk_accel_task *task;
1809 : enum accel_sequence_state state;
1810 : int rc;
1811 :
1812 : /* Prevent recursive calls to this function */
1813 166 : if (spdk_unlikely(seq->in_process_sequence)) {
1814 74 : return;
1815 : }
1816 92 : seq->in_process_sequence = true;
1817 :
1818 92 : task = TAILQ_FIRST(&seq->tasks);
1819 : do {
1820 375 : state = seq->state;
1821 375 : switch (state) {
1822 91 : case ACCEL_SEQUENCE_STATE_INIT:
1823 91 : if (g_accel_driver != NULL) {
1824 13 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS);
1825 13 : break;
1826 : }
1827 : /* Fall through */
1828 : case ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF:
1829 89 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF);
1830 89 : if (!accel_sequence_check_virtbuf(seq, task)) {
1831 : /* We couldn't allocate a buffer, wait until one is available */
1832 3 : break;
1833 : }
1834 86 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF);
1835 : /* Fall through */
1836 88 : case ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF:
1837 : /* If a module supports memory domains, we don't need to allocate bounce
1838 : * buffers */
1839 88 : if (g_modules_opc[task->op_code].supports_memory_domains) {
1840 15 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
1841 15 : break;
1842 : }
1843 73 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF);
1844 73 : rc = accel_sequence_check_bouncebuf(seq, task);
1845 73 : if (spdk_unlikely(rc != 0)) {
1846 : /* We couldn't allocate a buffer, wait until one is available */
1847 2 : if (rc == -EAGAIN) {
1848 2 : break;
1849 : }
1850 0 : accel_sequence_set_fail(seq, rc);
1851 0 : break;
1852 : }
1853 71 : if (task->has_aux && task->s.iovs == &task->aux->bounce.s.iov) {
1854 9 : assert(task->aux->bounce.s.orig_iovs);
1855 9 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PULL_DATA);
1856 9 : break;
1857 : }
1858 62 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK);
1859 : /* Fall through */
1860 84 : case ACCEL_SEQUENCE_STATE_EXEC_TASK:
1861 84 : SPDK_DEBUGLOG(accel, "Executing %s operation, sequence: %p\n",
1862 : g_opcode_strings[task->op_code], seq);
1863 :
1864 84 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_TASK);
1865 84 : rc = accel_submit_task(accel_ch, task);
1866 84 : if (spdk_unlikely(rc != 0)) {
1867 2 : SPDK_ERRLOG("Failed to submit %s operation, sequence: %p\n",
1868 : g_opcode_strings[task->op_code], seq);
1869 2 : accel_sequence_set_fail(seq, rc);
1870 : }
1871 84 : break;
1872 9 : case ACCEL_SEQUENCE_STATE_PULL_DATA:
1873 9 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA);
1874 9 : accel_task_pull_data(seq, task);
1875 9 : break;
1876 80 : case ACCEL_SEQUENCE_STATE_COMPLETE_TASK:
1877 80 : if (task->has_aux && task->d.iovs == &task->aux->bounce.d.iov) {
1878 8 : assert(task->aux->bounce.d.orig_iovs);
1879 8 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PUSH_DATA);
1880 8 : break;
1881 : }
1882 72 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK);
1883 72 : break;
1884 8 : case ACCEL_SEQUENCE_STATE_PUSH_DATA:
1885 8 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA);
1886 8 : accel_task_push_data(seq, task);
1887 8 : break;
1888 78 : case ACCEL_SEQUENCE_STATE_NEXT_TASK:
1889 78 : accel_sequence_complete_task(seq, task);
1890 : /* Check if there are any remaining tasks */
1891 78 : task = TAILQ_FIRST(&seq->tasks);
1892 78 : if (task == NULL) {
1893 : /* Immediately return here to make sure we don't touch the sequence
1894 : * after it's completed */
1895 39 : accel_sequence_complete(seq);
1896 39 : return;
1897 : }
1898 39 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_INIT);
1899 39 : break;
1900 13 : case ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS:
1901 13 : assert(!TAILQ_EMPTY(&seq->tasks));
1902 :
1903 13 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS);
1904 13 : rc = g_accel_driver->execute_sequence(accel_ch->driver_channel, seq);
1905 13 : if (spdk_unlikely(rc != 0)) {
1906 1 : SPDK_ERRLOG("Failed to execute sequence: %p using driver: %s\n",
1907 : seq, g_accel_driver->name);
1908 1 : accel_sequence_set_fail(seq, rc);
1909 : }
1910 13 : break;
1911 11 : case ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS:
1912 : /* Get the task again, as the driver might have completed some tasks
1913 : * synchronously */
1914 11 : task = TAILQ_FIRST(&seq->tasks);
1915 11 : if (task == NULL) {
1916 : /* Immediately return here to make sure we don't touch the sequence
1917 : * after it's completed */
1918 3 : accel_sequence_complete(seq);
1919 3 : return;
1920 : }
1921 : /* We don't want to execute the next task through the driver, so we
1922 : * explicitly omit the INIT state here */
1923 8 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF);
1924 8 : break;
1925 10 : case ACCEL_SEQUENCE_STATE_ERROR:
1926 : /* Immediately return here to make sure we don't touch the sequence
1927 : * after it's completed */
1928 10 : assert(seq->status != 0);
1929 10 : accel_sequence_complete(seq);
1930 10 : return;
1931 40 : case ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF:
1932 : case ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF:
1933 : case ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA:
1934 : case ACCEL_SEQUENCE_STATE_AWAIT_TASK:
1935 : case ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA:
1936 : case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS:
1937 40 : break;
1938 0 : default:
1939 0 : assert(0 && "bad state");
1940 : break;
1941 : }
1942 323 : } while (seq->state != state);
1943 :
1944 40 : seq->in_process_sequence = false;
1945 : }
1946 :
1947 : static void
1948 94 : accel_sequence_task_cb(void *cb_arg, int status)
1949 : {
1950 94 : struct spdk_accel_sequence *seq = cb_arg;
1951 94 : struct spdk_accel_task *task = TAILQ_FIRST(&seq->tasks);
1952 :
1953 94 : switch (seq->state) {
1954 82 : case ACCEL_SEQUENCE_STATE_AWAIT_TASK:
1955 82 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_COMPLETE_TASK);
1956 82 : if (spdk_unlikely(status != 0)) {
1957 2 : SPDK_ERRLOG("Failed to execute %s operation, sequence: %p\n",
1958 : g_opcode_strings[task->op_code], seq);
1959 2 : accel_sequence_set_fail(seq, status);
1960 : }
1961 :
1962 82 : accel_process_sequence(seq);
1963 82 : break;
1964 12 : case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS:
1965 12 : assert(g_accel_driver != NULL);
1966 : /* Immediately remove the task from the outstanding list to make sure the next call
1967 : * to spdk_accel_sequence_first_task() doesn't return it */
1968 12 : accel_sequence_complete_task(seq, task);
1969 12 : if (spdk_unlikely(status != 0)) {
1970 1 : SPDK_ERRLOG("Failed to execute %s operation, sequence: %p through "
1971 : "driver: %s\n", g_opcode_strings[task->op_code], seq,
1972 : g_accel_driver->name);
1973 : /* Update status without using accel_sequence_set_fail() to avoid changing
1974 : * seq's state to ERROR until driver calls spdk_accel_sequence_continue() */
1975 1 : seq->status = status;
1976 : }
1977 12 : break;
1978 0 : default:
1979 0 : assert(0 && "bad state");
1980 : break;
1981 : }
1982 94 : }
1983 :
1984 : void
1985 12 : spdk_accel_sequence_continue(struct spdk_accel_sequence *seq)
1986 : {
1987 12 : assert(g_accel_driver != NULL);
1988 12 : assert(seq->state == ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS);
1989 :
1990 12 : if (spdk_likely(seq->status == 0)) {
1991 11 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS);
1992 : } else {
1993 1 : accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR);
1994 : }
1995 :
1996 12 : accel_process_sequence(seq);
1997 12 : }
1998 :
1999 : static bool
2000 28 : accel_compare_iovs(struct iovec *iova, uint32_t iovacnt, struct iovec *iovb, uint32_t iovbcnt)
2001 : {
2002 : /* For now, just do a dumb check that the iovecs arrays are exactly the same */
2003 28 : if (iovacnt != iovbcnt) {
2004 0 : return false;
2005 : }
2006 :
2007 28 : return memcmp(iova, iovb, sizeof(*iova) * iovacnt) == 0;
2008 : }
2009 :
2010 : static bool
2011 22 : accel_task_set_dstbuf(struct spdk_accel_task *task, struct spdk_accel_task *next)
2012 : {
2013 : struct spdk_accel_task *prev;
2014 :
2015 22 : switch (task->op_code) {
2016 18 : case SPDK_ACCEL_OPC_DECOMPRESS:
2017 : case SPDK_ACCEL_OPC_FILL:
2018 : case SPDK_ACCEL_OPC_ENCRYPT:
2019 : case SPDK_ACCEL_OPC_DECRYPT:
2020 18 : if (task->dst_domain != next->src_domain) {
2021 0 : return false;
2022 : }
2023 18 : if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt,
2024 : next->s.iovs, next->s.iovcnt)) {
2025 1 : return false;
2026 : }
2027 17 : task->d.iovs = next->d.iovs;
2028 17 : task->d.iovcnt = next->d.iovcnt;
2029 17 : task->dst_domain = next->dst_domain;
2030 17 : task->dst_domain_ctx = next->dst_domain_ctx;
2031 17 : break;
2032 4 : case SPDK_ACCEL_OPC_CRC32C:
2033 : /* crc32 is special, because it doesn't have a dst buffer */
2034 4 : if (task->src_domain != next->src_domain) {
2035 0 : return false;
2036 : }
2037 4 : if (!accel_compare_iovs(task->s.iovs, task->s.iovcnt,
2038 : next->s.iovs, next->s.iovcnt)) {
2039 1 : return false;
2040 : }
2041 : /* We can only change crc32's buffer if we can change previous task's buffer */
2042 3 : prev = TAILQ_PREV(task, accel_sequence_tasks, seq_link);
2043 3 : if (prev == NULL) {
2044 1 : return false;
2045 : }
2046 2 : if (!accel_task_set_dstbuf(prev, next)) {
2047 0 : return false;
2048 : }
2049 2 : task->s.iovs = next->d.iovs;
2050 2 : task->s.iovcnt = next->d.iovcnt;
2051 2 : task->src_domain = next->dst_domain;
2052 2 : task->src_domain_ctx = next->dst_domain_ctx;
2053 2 : break;
2054 0 : default:
2055 0 : return false;
2056 : }
2057 :
2058 19 : return true;
2059 : }
2060 :
2061 : static void
2062 72 : accel_sequence_merge_tasks(struct spdk_accel_sequence *seq, struct spdk_accel_task *task,
2063 : struct spdk_accel_task **next_task)
2064 : {
2065 72 : struct spdk_accel_task *next = *next_task;
2066 :
2067 72 : switch (task->op_code) {
2068 8 : case SPDK_ACCEL_OPC_COPY:
2069 : /* We only allow changing src of operations that actually have a src, e.g. we never
2070 : * do it for fill. Theoretically, it is possible, but we'd have to be careful to
2071 : * change the src of the operation after fill (which in turn could also be a fill).
2072 : * So, for the sake of simplicity, skip this type of operations for now.
2073 : */
2074 8 : if (next->op_code != SPDK_ACCEL_OPC_DECOMPRESS &&
2075 6 : next->op_code != SPDK_ACCEL_OPC_COPY &&
2076 5 : next->op_code != SPDK_ACCEL_OPC_ENCRYPT &&
2077 4 : next->op_code != SPDK_ACCEL_OPC_DECRYPT &&
2078 2 : next->op_code != SPDK_ACCEL_OPC_COPY_CRC32C) {
2079 2 : break;
2080 : }
2081 6 : if (task->dst_domain != next->src_domain) {
2082 0 : break;
2083 : }
2084 6 : if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt,
2085 : next->s.iovs, next->s.iovcnt)) {
2086 0 : break;
2087 : }
2088 6 : next->s.iovs = task->s.iovs;
2089 6 : next->s.iovcnt = task->s.iovcnt;
2090 6 : next->src_domain = task->src_domain;
2091 6 : next->src_domain_ctx = task->src_domain_ctx;
2092 6 : accel_sequence_complete_task(seq, task);
2093 6 : break;
2094 64 : case SPDK_ACCEL_OPC_DECOMPRESS:
2095 : case SPDK_ACCEL_OPC_FILL:
2096 : case SPDK_ACCEL_OPC_ENCRYPT:
2097 : case SPDK_ACCEL_OPC_DECRYPT:
2098 : case SPDK_ACCEL_OPC_CRC32C:
2099 : /* We can only merge tasks when one of them is a copy */
2100 64 : if (next->op_code != SPDK_ACCEL_OPC_COPY) {
2101 44 : break;
2102 : }
2103 20 : if (!accel_task_set_dstbuf(task, next)) {
2104 3 : break;
2105 : }
2106 : /* We're removing next_task from the tasks queue, so we need to update its pointer,
2107 : * so that the TAILQ_FOREACH_SAFE() loop below works correctly */
2108 17 : *next_task = TAILQ_NEXT(next, seq_link);
2109 17 : accel_sequence_complete_task(seq, next);
2110 17 : break;
2111 0 : default:
2112 0 : assert(0 && "bad opcode");
2113 : break;
2114 : }
2115 72 : }
2116 :
2117 : void
2118 52 : spdk_accel_sequence_finish(struct spdk_accel_sequence *seq,
2119 : spdk_accel_completion_cb cb_fn, void *cb_arg)
2120 : {
2121 52 : struct spdk_accel_task *task, *next;
2122 :
2123 : /* Try to remove any copy operations if possible */
2124 124 : TAILQ_FOREACH_SAFE(task, &seq->tasks, seq_link, next) {
2125 109 : if (next == NULL) {
2126 37 : break;
2127 : }
2128 72 : accel_sequence_merge_tasks(seq, task, &next);
2129 : }
2130 :
2131 52 : seq->cb_fn = cb_fn;
2132 52 : seq->cb_arg = cb_arg;
2133 :
2134 52 : accel_process_sequence(seq);
2135 52 : }
2136 :
2137 : void
2138 0 : spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq)
2139 : {
2140 0 : struct accel_sequence_tasks tasks = TAILQ_HEAD_INITIALIZER(tasks);
2141 : struct spdk_accel_task *task;
2142 :
2143 0 : TAILQ_SWAP(&tasks, &seq->tasks, spdk_accel_task, seq_link);
2144 :
2145 0 : while (!TAILQ_EMPTY(&tasks)) {
2146 0 : task = TAILQ_FIRST(&tasks);
2147 0 : TAILQ_REMOVE(&tasks, task, seq_link);
2148 0 : TAILQ_INSERT_HEAD(&seq->tasks, task, seq_link);
2149 : }
2150 0 : }
2151 :
2152 : void
2153 3 : spdk_accel_sequence_abort(struct spdk_accel_sequence *seq)
2154 : {
2155 3 : if (seq == NULL) {
2156 1 : return;
2157 : }
2158 :
2159 2 : accel_sequence_complete_tasks(seq);
2160 2 : accel_sequence_put(seq);
2161 : }
2162 :
2163 : struct spdk_memory_domain *
2164 0 : spdk_accel_get_memory_domain(void)
2165 : {
2166 0 : return g_accel_domain;
2167 : }
2168 :
2169 : static struct spdk_accel_module_if *
2170 7 : _module_find_by_name(const char *name)
2171 : {
2172 7 : struct spdk_accel_module_if *accel_module = NULL;
2173 :
2174 16 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
2175 10 : if (strcmp(name, accel_module->name) == 0) {
2176 1 : break;
2177 : }
2178 : }
2179 :
2180 7 : return accel_module;
2181 : }
2182 :
2183 : static inline struct spdk_accel_crypto_key *
2184 0 : _accel_crypto_key_get(const char *name)
2185 : {
2186 : struct spdk_accel_crypto_key *key;
2187 :
2188 0 : assert(spdk_spin_held(&g_keyring_spin));
2189 :
2190 0 : TAILQ_FOREACH(key, &g_keyring, link) {
2191 0 : if (strcmp(name, key->param.key_name) == 0) {
2192 0 : return key;
2193 : }
2194 : }
2195 :
2196 0 : return NULL;
2197 : }
2198 :
2199 : static void
2200 0 : accel_crypto_key_free_mem(struct spdk_accel_crypto_key *key)
2201 : {
2202 0 : if (key->param.hex_key) {
2203 0 : spdk_memset_s(key->param.hex_key, key->key_size * 2, 0, key->key_size * 2);
2204 0 : free(key->param.hex_key);
2205 : }
2206 0 : if (key->param.hex_key2) {
2207 0 : spdk_memset_s(key->param.hex_key2, key->key2_size * 2, 0, key->key2_size * 2);
2208 0 : free(key->param.hex_key2);
2209 : }
2210 0 : free(key->param.tweak_mode);
2211 0 : free(key->param.key_name);
2212 0 : free(key->param.cipher);
2213 0 : if (key->key) {
2214 0 : spdk_memset_s(key->key, key->key_size, 0, key->key_size);
2215 0 : free(key->key);
2216 : }
2217 0 : if (key->key2) {
2218 0 : spdk_memset_s(key->key2, key->key2_size, 0, key->key2_size);
2219 0 : free(key->key2);
2220 : }
2221 0 : free(key);
2222 0 : }
2223 :
2224 : static void
2225 0 : accel_crypto_key_destroy_unsafe(struct spdk_accel_crypto_key *key)
2226 : {
2227 0 : assert(key->module_if);
2228 0 : assert(key->module_if->crypto_key_deinit);
2229 :
2230 0 : key->module_if->crypto_key_deinit(key);
2231 0 : accel_crypto_key_free_mem(key);
2232 0 : }
2233 :
2234 : /*
2235 : * This function mitigates a timing side channel which could be caused by using strcmp()
2236 : * Please refer to chapter "Mitigating Information Leakage Based on Variable Timing" in
2237 : * the article [1] for more details
2238 : * [1] https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/secure-coding/mitigate-timing-side-channel-crypto-implementation.html
2239 : */
2240 : static bool
2241 0 : accel_aes_xts_keys_equal(const char *k1, size_t k1_len, const char *k2, size_t k2_len)
2242 : {
2243 : size_t i;
2244 0 : volatile size_t x = k1_len ^ k2_len;
2245 :
2246 0 : for (i = 0; ((i < k1_len) & (i < k2_len)); i++) {
2247 0 : x |= k1[i] ^ k2[i];
2248 : }
2249 :
2250 0 : return x == 0;
2251 : }
2252 :
2253 : static const char *g_tweak_modes[] = {
2254 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA] = "SIMPLE_LBA",
2255 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_JOIN_NEG_LBA_WITH_LBA] = "JOIN_NEG_LBA_WITH_LBA",
2256 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_FULL_LBA] = "INCR_512_FULL_LBA",
2257 : [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_UPPER_LBA] = "INCR_512_UPPER_LBA",
2258 : };
2259 :
2260 : static const char *g_ciphers[] = {
2261 : [SPDK_ACCEL_CIPHER_AES_CBC] = "AES_CBC",
2262 : [SPDK_ACCEL_CIPHER_AES_XTS] = "AES_XTS",
2263 : };
2264 :
2265 : int
2266 0 : spdk_accel_crypto_key_create(const struct spdk_accel_crypto_key_create_param *param)
2267 : {
2268 : struct spdk_accel_module_if *module;
2269 : struct spdk_accel_crypto_key *key;
2270 : size_t hex_key_size, hex_key2_size;
2271 0 : bool found = false;
2272 : size_t i;
2273 : int rc;
2274 :
2275 0 : if (!param || !param->hex_key || !param->cipher || !param->key_name) {
2276 0 : return -EINVAL;
2277 : }
2278 :
2279 0 : if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) {
2280 : /* hardly ever possible, but let's check and warn the user */
2281 0 : SPDK_ERRLOG("Different accel modules are used for encryption and decryption\n");
2282 : }
2283 0 : module = g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module;
2284 :
2285 0 : if (!module) {
2286 0 : SPDK_ERRLOG("No accel module found assigned for crypto operation\n");
2287 0 : return -ENOENT;
2288 : }
2289 :
2290 0 : if (!module->crypto_key_init || !module->crypto_supports_cipher) {
2291 0 : SPDK_ERRLOG("Module %s doesn't support crypto operations\n", module->name);
2292 0 : return -ENOTSUP;
2293 : }
2294 :
2295 0 : key = calloc(1, sizeof(*key));
2296 0 : if (!key) {
2297 0 : return -ENOMEM;
2298 : }
2299 :
2300 0 : key->param.key_name = strdup(param->key_name);
2301 0 : if (!key->param.key_name) {
2302 0 : rc = -ENOMEM;
2303 0 : goto error;
2304 : }
2305 :
2306 0 : for (i = 0; i < SPDK_COUNTOF(g_ciphers); ++i) {
2307 0 : assert(g_ciphers[i]);
2308 :
2309 0 : if (strncmp(param->cipher, g_ciphers[i], strlen(g_ciphers[i])) == 0) {
2310 0 : key->cipher = i;
2311 0 : found = true;
2312 0 : break;
2313 : }
2314 : }
2315 :
2316 0 : if (!found) {
2317 0 : SPDK_ERRLOG("Failed to parse cipher\n");
2318 0 : rc = -EINVAL;
2319 0 : goto error;
2320 : }
2321 :
2322 0 : key->param.cipher = strdup(param->cipher);
2323 0 : if (!key->param.cipher) {
2324 0 : rc = -ENOMEM;
2325 0 : goto error;
2326 : }
2327 :
2328 0 : hex_key_size = strnlen(param->hex_key, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2329 0 : if (hex_key_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) {
2330 0 : SPDK_ERRLOG("key1 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2331 0 : rc = -EINVAL;
2332 0 : goto error;
2333 : }
2334 :
2335 0 : if (hex_key_size == 0) {
2336 0 : SPDK_ERRLOG("key1 size cannot be 0\n");
2337 0 : rc = -EINVAL;
2338 0 : goto error;
2339 : }
2340 :
2341 0 : key->param.hex_key = strdup(param->hex_key);
2342 0 : if (!key->param.hex_key) {
2343 0 : rc = -ENOMEM;
2344 0 : goto error;
2345 : }
2346 :
2347 0 : key->key_size = hex_key_size / 2;
2348 0 : key->key = spdk_unhexlify(key->param.hex_key);
2349 0 : if (!key->key) {
2350 0 : SPDK_ERRLOG("Failed to unhexlify key1\n");
2351 0 : rc = -EINVAL;
2352 0 : goto error;
2353 : }
2354 :
2355 0 : if (param->hex_key2) {
2356 0 : hex_key2_size = strnlen(param->hex_key2, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2357 0 : if (hex_key2_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) {
2358 0 : SPDK_ERRLOG("key2 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH);
2359 0 : rc = -EINVAL;
2360 0 : goto error;
2361 : }
2362 :
2363 0 : if (hex_key2_size == 0) {
2364 0 : SPDK_ERRLOG("key2 size cannot be 0\n");
2365 0 : rc = -EINVAL;
2366 0 : goto error;
2367 : }
2368 :
2369 0 : key->param.hex_key2 = strdup(param->hex_key2);
2370 0 : if (!key->param.hex_key2) {
2371 0 : rc = -ENOMEM;
2372 0 : goto error;
2373 : }
2374 :
2375 0 : key->key2_size = hex_key2_size / 2;
2376 0 : key->key2 = spdk_unhexlify(key->param.hex_key2);
2377 0 : if (!key->key2) {
2378 0 : SPDK_ERRLOG("Failed to unhexlify key2\n");
2379 0 : rc = -EINVAL;
2380 0 : goto error;
2381 : }
2382 : }
2383 :
2384 0 : key->tweak_mode = ACCEL_CRYPTO_TWEAK_MODE_DEFAULT;
2385 0 : if (param->tweak_mode) {
2386 0 : found = false;
2387 :
2388 0 : key->param.tweak_mode = strdup(param->tweak_mode);
2389 0 : if (!key->param.tweak_mode) {
2390 0 : rc = -ENOMEM;
2391 0 : goto error;
2392 : }
2393 :
2394 0 : for (i = 0; i < SPDK_COUNTOF(g_tweak_modes); ++i) {
2395 0 : assert(g_tweak_modes[i]);
2396 :
2397 0 : if (strncmp(param->tweak_mode, g_tweak_modes[i], strlen(g_tweak_modes[i])) == 0) {
2398 0 : key->tweak_mode = i;
2399 0 : found = true;
2400 0 : break;
2401 : }
2402 : }
2403 :
2404 0 : if (!found) {
2405 0 : SPDK_ERRLOG("Failed to parse tweak mode\n");
2406 0 : rc = -EINVAL;
2407 0 : goto error;
2408 : }
2409 : }
2410 :
2411 0 : if ((!module->crypto_supports_tweak_mode && key->tweak_mode != ACCEL_CRYPTO_TWEAK_MODE_DEFAULT) ||
2412 0 : (module->crypto_supports_tweak_mode && !module->crypto_supports_tweak_mode(key->tweak_mode))) {
2413 0 : SPDK_ERRLOG("Module %s doesn't support %s tweak mode\n", module->name,
2414 : g_tweak_modes[key->tweak_mode]);
2415 0 : rc = -EINVAL;
2416 0 : goto error;
2417 : }
2418 :
2419 0 : if (!module->crypto_supports_cipher(key->cipher, key->key_size)) {
2420 0 : SPDK_ERRLOG("Module %s doesn't support %s cipher with %zu key size\n", module->name,
2421 : g_ciphers[key->cipher], key->key_size);
2422 0 : rc = -EINVAL;
2423 0 : goto error;
2424 : }
2425 :
2426 0 : if (key->cipher == SPDK_ACCEL_CIPHER_AES_XTS) {
2427 0 : if (!key->key2) {
2428 0 : SPDK_ERRLOG("%s key2 is missing\n", g_ciphers[key->cipher]);
2429 0 : rc = -EINVAL;
2430 0 : goto error;
2431 : }
2432 :
2433 0 : if (key->key_size != key->key2_size) {
2434 0 : SPDK_ERRLOG("%s key size %zu is not equal to key2 size %zu\n", g_ciphers[key->cipher],
2435 : key->key_size,
2436 : key->key2_size);
2437 0 : rc = -EINVAL;
2438 0 : goto error;
2439 : }
2440 :
2441 0 : if (accel_aes_xts_keys_equal(key->key, key->key_size, key->key2, key->key2_size)) {
2442 0 : SPDK_ERRLOG("%s identical keys are not secure\n", g_ciphers[key->cipher]);
2443 0 : rc = -EINVAL;
2444 0 : goto error;
2445 : }
2446 : }
2447 :
2448 0 : if (key->cipher == SPDK_ACCEL_CIPHER_AES_CBC) {
2449 0 : if (key->key2_size) {
2450 0 : SPDK_ERRLOG("%s doesn't use key2\n", g_ciphers[key->cipher]);
2451 0 : rc = -EINVAL;
2452 0 : goto error;
2453 : }
2454 : }
2455 :
2456 0 : key->module_if = module;
2457 :
2458 0 : spdk_spin_lock(&g_keyring_spin);
2459 0 : if (_accel_crypto_key_get(param->key_name)) {
2460 0 : rc = -EEXIST;
2461 : } else {
2462 0 : rc = module->crypto_key_init(key);
2463 0 : if (rc) {
2464 0 : SPDK_ERRLOG("Module %s failed to initialize crypto key\n", module->name);
2465 : } else {
2466 0 : TAILQ_INSERT_TAIL(&g_keyring, key, link);
2467 : }
2468 : }
2469 0 : spdk_spin_unlock(&g_keyring_spin);
2470 :
2471 0 : if (rc) {
2472 0 : goto error;
2473 : }
2474 :
2475 0 : return 0;
2476 :
2477 0 : error:
2478 0 : accel_crypto_key_free_mem(key);
2479 0 : return rc;
2480 : }
2481 :
2482 : int
2483 0 : spdk_accel_crypto_key_destroy(struct spdk_accel_crypto_key *key)
2484 : {
2485 0 : if (!key || !key->module_if) {
2486 0 : return -EINVAL;
2487 : }
2488 :
2489 0 : spdk_spin_lock(&g_keyring_spin);
2490 0 : if (!_accel_crypto_key_get(key->param.key_name)) {
2491 0 : spdk_spin_unlock(&g_keyring_spin);
2492 0 : return -ENOENT;
2493 : }
2494 0 : TAILQ_REMOVE(&g_keyring, key, link);
2495 0 : spdk_spin_unlock(&g_keyring_spin);
2496 :
2497 0 : accel_crypto_key_destroy_unsafe(key);
2498 :
2499 0 : return 0;
2500 : }
2501 :
2502 : struct spdk_accel_crypto_key *
2503 0 : spdk_accel_crypto_key_get(const char *name)
2504 : {
2505 : struct spdk_accel_crypto_key *key;
2506 :
2507 0 : spdk_spin_lock(&g_keyring_spin);
2508 0 : key = _accel_crypto_key_get(name);
2509 0 : spdk_spin_unlock(&g_keyring_spin);
2510 :
2511 0 : return key;
2512 : }
2513 :
2514 : /* Helper function when accel modules register with the framework. */
2515 : void
2516 5 : spdk_accel_module_list_add(struct spdk_accel_module_if *accel_module)
2517 : {
2518 : struct spdk_accel_module_if *tmp;
2519 :
2520 5 : if (_module_find_by_name(accel_module->name)) {
2521 0 : SPDK_NOTICELOG("Module %s already registered\n", accel_module->name);
2522 0 : assert(false);
2523 : return;
2524 : }
2525 :
2526 8 : TAILQ_FOREACH(tmp, &spdk_accel_module_list, tailq) {
2527 5 : if (accel_module->priority < tmp->priority) {
2528 2 : break;
2529 : }
2530 : }
2531 :
2532 5 : if (tmp != NULL) {
2533 2 : TAILQ_INSERT_BEFORE(tmp, accel_module, tailq);
2534 : } else {
2535 3 : TAILQ_INSERT_TAIL(&spdk_accel_module_list, accel_module, tailq);
2536 : }
2537 : }
2538 :
2539 : /* Framework level channel create callback. */
2540 : static int
2541 11 : accel_create_channel(void *io_device, void *ctx_buf)
2542 : {
2543 11 : struct accel_io_channel *accel_ch = ctx_buf;
2544 : struct spdk_accel_task *accel_task;
2545 : struct spdk_accel_task_aux_data *accel_task_aux;
2546 : struct spdk_accel_sequence *seq;
2547 : struct accel_buffer *buf;
2548 : size_t task_size_aligned;
2549 : uint8_t *task_mem;
2550 11 : uint32_t i = 0, j;
2551 : int rc;
2552 :
2553 11 : task_size_aligned = SPDK_ALIGN_CEIL(g_max_accel_module_size, SPDK_CACHE_LINE_SIZE);
2554 11 : accel_ch->task_pool_base = aligned_alloc(SPDK_CACHE_LINE_SIZE,
2555 11 : g_opts.task_count * task_size_aligned);
2556 11 : if (!accel_ch->task_pool_base) {
2557 0 : return -ENOMEM;
2558 : }
2559 11 : memset(accel_ch->task_pool_base, 0, g_opts.task_count * task_size_aligned);
2560 :
2561 11 : accel_ch->seq_pool_base = aligned_alloc(SPDK_CACHE_LINE_SIZE,
2562 11 : g_opts.sequence_count * sizeof(struct spdk_accel_sequence));
2563 11 : if (accel_ch->seq_pool_base == NULL) {
2564 0 : goto err;
2565 : }
2566 11 : memset(accel_ch->seq_pool_base, 0, g_opts.sequence_count * sizeof(struct spdk_accel_sequence));
2567 :
2568 11 : accel_ch->task_aux_data_base = calloc(g_opts.task_count, sizeof(struct spdk_accel_task_aux_data));
2569 11 : if (accel_ch->task_aux_data_base == NULL) {
2570 0 : goto err;
2571 : }
2572 :
2573 11 : accel_ch->buf_pool_base = calloc(g_opts.buf_count, sizeof(struct accel_buffer));
2574 11 : if (accel_ch->buf_pool_base == NULL) {
2575 0 : goto err;
2576 : }
2577 :
2578 11 : STAILQ_INIT(&accel_ch->task_pool);
2579 11 : SLIST_INIT(&accel_ch->task_aux_data_pool);
2580 11 : SLIST_INIT(&accel_ch->seq_pool);
2581 11 : SLIST_INIT(&accel_ch->buf_pool);
2582 :
2583 11 : task_mem = accel_ch->task_pool_base;
2584 22539 : for (i = 0; i < g_opts.task_count; i++) {
2585 22528 : accel_task = (struct spdk_accel_task *)task_mem;
2586 22528 : accel_task->aux = NULL;
2587 22528 : STAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link);
2588 22528 : task_mem += task_size_aligned;
2589 22528 : accel_task_aux = &accel_ch->task_aux_data_base[i];
2590 22528 : SLIST_INSERT_HEAD(&accel_ch->task_aux_data_pool, accel_task_aux, link);
2591 : }
2592 22539 : for (i = 0; i < g_opts.sequence_count; i++) {
2593 22528 : seq = &accel_ch->seq_pool_base[i];
2594 22528 : SLIST_INSERT_HEAD(&accel_ch->seq_pool, seq, link);
2595 : }
2596 22539 : for (i = 0; i < g_opts.buf_count; i++) {
2597 22528 : buf = &accel_ch->buf_pool_base[i];
2598 22528 : SLIST_INSERT_HEAD(&accel_ch->buf_pool, buf, link);
2599 : }
2600 :
2601 : /* Assign modules and get IO channels for each */
2602 176 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
2603 165 : accel_ch->module_ch[i] = g_modules_opc[i].module->get_io_channel();
2604 : /* This can happen if idxd runs out of channels. */
2605 165 : if (accel_ch->module_ch[i] == NULL) {
2606 0 : SPDK_ERRLOG("Module %s failed to get io channel\n", g_modules_opc[i].module->name);
2607 0 : goto err;
2608 : }
2609 : }
2610 :
2611 11 : if (g_accel_driver != NULL) {
2612 0 : accel_ch->driver_channel = g_accel_driver->get_io_channel();
2613 0 : if (accel_ch->driver_channel == NULL) {
2614 0 : SPDK_ERRLOG("Failed to get driver's IO channel\n");
2615 0 : goto err;
2616 : }
2617 : }
2618 :
2619 11 : rc = spdk_iobuf_channel_init(&accel_ch->iobuf, "accel", g_opts.small_cache_size,
2620 : g_opts.large_cache_size);
2621 11 : if (rc != 0) {
2622 0 : SPDK_ERRLOG("Failed to initialize iobuf accel channel\n");
2623 0 : goto err;
2624 : }
2625 :
2626 11 : return 0;
2627 0 : err:
2628 0 : if (accel_ch->driver_channel != NULL) {
2629 0 : spdk_put_io_channel(accel_ch->driver_channel);
2630 : }
2631 0 : for (j = 0; j < i; j++) {
2632 0 : spdk_put_io_channel(accel_ch->module_ch[j]);
2633 : }
2634 0 : free(accel_ch->task_pool_base);
2635 0 : free(accel_ch->task_aux_data_base);
2636 0 : free(accel_ch->seq_pool_base);
2637 0 : free(accel_ch->buf_pool_base);
2638 :
2639 0 : return -ENOMEM;
2640 : }
2641 :
2642 : static void
2643 11 : accel_add_stats(struct accel_stats *total, struct accel_stats *stats)
2644 : {
2645 : int i;
2646 :
2647 11 : total->sequence_executed += stats->sequence_executed;
2648 11 : total->sequence_failed += stats->sequence_failed;
2649 11 : total->sequence_outstanding += stats->sequence_outstanding;
2650 11 : total->task_outstanding += stats->task_outstanding;
2651 11 : total->retry.task += stats->retry.task;
2652 11 : total->retry.sequence += stats->retry.sequence;
2653 11 : total->retry.iobuf += stats->retry.iobuf;
2654 11 : total->retry.bufdesc += stats->retry.bufdesc;
2655 176 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) {
2656 165 : total->operations[i].executed += stats->operations[i].executed;
2657 165 : total->operations[i].failed += stats->operations[i].failed;
2658 165 : total->operations[i].num_bytes += stats->operations[i].num_bytes;
2659 : }
2660 11 : }
2661 :
2662 : /* Framework level channel destroy callback. */
2663 : static void
2664 11 : accel_destroy_channel(void *io_device, void *ctx_buf)
2665 : {
2666 11 : struct accel_io_channel *accel_ch = ctx_buf;
2667 : int i;
2668 :
2669 11 : spdk_iobuf_channel_fini(&accel_ch->iobuf);
2670 :
2671 11 : if (accel_ch->driver_channel != NULL) {
2672 0 : spdk_put_io_channel(accel_ch->driver_channel);
2673 : }
2674 :
2675 176 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
2676 165 : assert(accel_ch->module_ch[i] != NULL);
2677 165 : spdk_put_io_channel(accel_ch->module_ch[i]);
2678 165 : accel_ch->module_ch[i] = NULL;
2679 : }
2680 :
2681 : /* Update global stats to make sure channel's stats aren't lost after a channel is gone */
2682 11 : spdk_spin_lock(&g_stats_lock);
2683 11 : accel_add_stats(&g_stats, &accel_ch->stats);
2684 11 : spdk_spin_unlock(&g_stats_lock);
2685 :
2686 11 : free(accel_ch->task_pool_base);
2687 11 : free(accel_ch->task_aux_data_base);
2688 11 : free(accel_ch->seq_pool_base);
2689 11 : free(accel_ch->buf_pool_base);
2690 11 : }
2691 :
2692 : struct spdk_io_channel *
2693 11 : spdk_accel_get_io_channel(void)
2694 : {
2695 11 : return spdk_get_io_channel(&spdk_accel_module_list);
2696 : }
2697 :
2698 : static int
2699 2 : accel_module_initialize(void)
2700 : {
2701 : struct spdk_accel_module_if *accel_module, *tmp_module;
2702 2 : int rc = 0, module_rc;
2703 :
2704 7 : TAILQ_FOREACH_SAFE(accel_module, &spdk_accel_module_list, tailq, tmp_module) {
2705 5 : module_rc = accel_module->module_init();
2706 5 : if (module_rc) {
2707 0 : TAILQ_REMOVE(&spdk_accel_module_list, accel_module, tailq);
2708 0 : if (module_rc == -ENODEV) {
2709 0 : SPDK_NOTICELOG("No devices for module %s, skipping\n", accel_module->name);
2710 0 : } else if (!rc) {
2711 0 : SPDK_ERRLOG("Module %s initialization failed with %d\n", accel_module->name, module_rc);
2712 0 : rc = module_rc;
2713 : }
2714 0 : continue;
2715 : }
2716 :
2717 5 : SPDK_DEBUGLOG(accel, "Module %s initialized.\n", accel_module->name);
2718 : }
2719 :
2720 2 : return rc;
2721 : }
2722 :
2723 : static void
2724 30 : accel_module_init_opcode(enum spdk_accel_opcode opcode)
2725 : {
2726 30 : struct accel_module *module = &g_modules_opc[opcode];
2727 30 : struct spdk_accel_module_if *module_if = module->module;
2728 :
2729 30 : if (module_if->get_memory_domains != NULL) {
2730 0 : module->supports_memory_domains = module_if->get_memory_domains(NULL, 0) > 0;
2731 : }
2732 30 : }
2733 :
2734 : static int
2735 0 : accel_memory_domain_translate(struct spdk_memory_domain *src_domain, void *src_domain_ctx,
2736 : struct spdk_memory_domain *dst_domain, struct spdk_memory_domain_translation_ctx *dst_domain_ctx,
2737 : void *addr, size_t len, struct spdk_memory_domain_translation_result *result)
2738 : {
2739 0 : struct accel_buffer *buf = src_domain_ctx;
2740 :
2741 0 : SPDK_DEBUGLOG(accel, "translate addr %p, len %zu\n", addr, len);
2742 :
2743 0 : assert(g_accel_domain == src_domain);
2744 0 : assert(spdk_memory_domain_get_system_domain() == dst_domain);
2745 0 : assert(buf->buf == NULL);
2746 0 : assert(addr == ACCEL_BUFFER_BASE);
2747 0 : assert(len == buf->len);
2748 :
2749 0 : buf->buf = spdk_iobuf_get(&buf->ch->iobuf, buf->len, NULL, NULL);
2750 0 : if (spdk_unlikely(buf->buf == NULL)) {
2751 0 : return -ENOMEM;
2752 : }
2753 :
2754 0 : result->iov_count = 1;
2755 0 : result->iov.iov_base = buf->buf;
2756 0 : result->iov.iov_len = buf->len;
2757 0 : SPDK_DEBUGLOG(accel, "translated addr %p\n", result->iov.iov_base);
2758 0 : return 0;
2759 : }
2760 :
2761 : static void
2762 0 : accel_memory_domain_invalidate(struct spdk_memory_domain *domain, void *domain_ctx,
2763 : struct iovec *iov, uint32_t iovcnt)
2764 : {
2765 0 : struct accel_buffer *buf = domain_ctx;
2766 :
2767 0 : SPDK_DEBUGLOG(accel, "invalidate addr %p, len %zu\n", iov[0].iov_base, iov[0].iov_len);
2768 :
2769 0 : assert(g_accel_domain == domain);
2770 0 : assert(iovcnt == 1);
2771 0 : assert(buf->buf != NULL);
2772 0 : assert(iov[0].iov_base == buf->buf);
2773 0 : assert(iov[0].iov_len == buf->len);
2774 :
2775 0 : spdk_iobuf_put(&buf->ch->iobuf, buf->buf, buf->len);
2776 0 : buf->buf = NULL;
2777 0 : }
2778 :
2779 : int
2780 2 : spdk_accel_initialize(void)
2781 : {
2782 : enum spdk_accel_opcode op;
2783 2 : struct spdk_accel_module_if *accel_module = NULL;
2784 : int rc;
2785 :
2786 : /*
2787 : * We need a unique identifier for the accel framework, so use the
2788 : * spdk_accel_module_list address for this purpose.
2789 : */
2790 2 : spdk_io_device_register(&spdk_accel_module_list, accel_create_channel, accel_destroy_channel,
2791 : sizeof(struct accel_io_channel), "accel");
2792 :
2793 2 : spdk_spin_init(&g_keyring_spin);
2794 2 : spdk_spin_init(&g_stats_lock);
2795 :
2796 2 : rc = spdk_memory_domain_create(&g_accel_domain, SPDK_DMA_DEVICE_TYPE_ACCEL, NULL,
2797 : "SPDK_ACCEL_DMA_DEVICE");
2798 2 : if (rc != 0) {
2799 0 : SPDK_ERRLOG("Failed to create accel memory domain\n");
2800 0 : return rc;
2801 : }
2802 :
2803 2 : spdk_memory_domain_set_translation(g_accel_domain, accel_memory_domain_translate);
2804 2 : spdk_memory_domain_set_invalidate(g_accel_domain, accel_memory_domain_invalidate);
2805 :
2806 2 : g_modules_started = true;
2807 2 : rc = accel_module_initialize();
2808 2 : if (rc) {
2809 0 : return rc;
2810 : }
2811 :
2812 2 : if (g_accel_driver != NULL && g_accel_driver->init != NULL) {
2813 0 : rc = g_accel_driver->init();
2814 0 : if (rc != 0) {
2815 0 : SPDK_ERRLOG("Failed to initialize driver %s: %s\n", g_accel_driver->name,
2816 : spdk_strerror(-rc));
2817 0 : return rc;
2818 : }
2819 : }
2820 :
2821 : /* The module list is order by priority, with the highest priority modules being at the end
2822 : * of the list. The software module should be somewhere at the beginning of the list,
2823 : * before all HW modules.
2824 : * NOTE: all opcodes must be supported by software in the event that no HW modules are
2825 : * initialized to support the operation.
2826 : */
2827 7 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
2828 80 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
2829 75 : if (accel_module->supports_opcode(op)) {
2830 75 : g_modules_opc[op].module = accel_module;
2831 75 : SPDK_DEBUGLOG(accel, "OPC 0x%x now assigned to %s\n", op, accel_module->name);
2832 : }
2833 : }
2834 :
2835 5 : if (accel_module->get_ctx_size != NULL) {
2836 1 : g_max_accel_module_size = spdk_max(g_max_accel_module_size,
2837 : accel_module->get_ctx_size());
2838 : }
2839 : }
2840 :
2841 : /* Now lets check for overrides and apply all that exist */
2842 32 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
2843 30 : if (g_modules_opc_override[op] != NULL) {
2844 0 : accel_module = _module_find_by_name(g_modules_opc_override[op]);
2845 0 : if (accel_module == NULL) {
2846 0 : SPDK_ERRLOG("Invalid module name of %s\n", g_modules_opc_override[op]);
2847 0 : return -EINVAL;
2848 : }
2849 0 : if (accel_module->supports_opcode(op) == false) {
2850 0 : SPDK_ERRLOG("Module %s does not support op code %d\n", accel_module->name, op);
2851 0 : return -EINVAL;
2852 : }
2853 0 : g_modules_opc[op].module = accel_module;
2854 : }
2855 : }
2856 :
2857 2 : if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) {
2858 0 : SPDK_ERRLOG("Different accel modules are assigned to encrypt and decrypt operations");
2859 0 : return -EINVAL;
2860 : }
2861 :
2862 32 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
2863 30 : assert(g_modules_opc[op].module != NULL);
2864 30 : accel_module_init_opcode(op);
2865 : }
2866 :
2867 2 : rc = spdk_iobuf_register_module("accel");
2868 2 : if (rc != 0) {
2869 0 : SPDK_ERRLOG("Failed to register accel iobuf module\n");
2870 0 : return rc;
2871 : }
2872 :
2873 2 : return 0;
2874 : }
2875 :
2876 : static void
2877 2 : accel_module_finish_cb(void)
2878 : {
2879 2 : spdk_accel_fini_cb cb_fn = g_fini_cb_fn;
2880 :
2881 2 : cb_fn(g_fini_cb_arg);
2882 2 : g_fini_cb_fn = NULL;
2883 2 : g_fini_cb_arg = NULL;
2884 2 : }
2885 :
2886 : static void
2887 0 : accel_write_overridden_opc(struct spdk_json_write_ctx *w, const char *opc_str,
2888 : const char *module_str)
2889 : {
2890 0 : spdk_json_write_object_begin(w);
2891 0 : spdk_json_write_named_string(w, "method", "accel_assign_opc");
2892 0 : spdk_json_write_named_object_begin(w, "params");
2893 0 : spdk_json_write_named_string(w, "opname", opc_str);
2894 0 : spdk_json_write_named_string(w, "module", module_str);
2895 0 : spdk_json_write_object_end(w);
2896 0 : spdk_json_write_object_end(w);
2897 0 : }
2898 :
2899 : static void
2900 0 : __accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key)
2901 : {
2902 0 : spdk_json_write_named_string(w, "name", key->param.key_name);
2903 0 : spdk_json_write_named_string(w, "cipher", key->param.cipher);
2904 0 : spdk_json_write_named_string(w, "key", key->param.hex_key);
2905 0 : if (key->param.hex_key2) {
2906 0 : spdk_json_write_named_string(w, "key2", key->param.hex_key2);
2907 : }
2908 :
2909 0 : if (key->param.tweak_mode) {
2910 0 : spdk_json_write_named_string(w, "tweak_mode", key->param.tweak_mode);
2911 : }
2912 0 : }
2913 :
2914 : void
2915 0 : _accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key)
2916 : {
2917 0 : spdk_json_write_object_begin(w);
2918 0 : __accel_crypto_key_dump_param(w, key);
2919 0 : spdk_json_write_object_end(w);
2920 0 : }
2921 :
2922 : static void
2923 0 : _accel_crypto_key_write_config_json(struct spdk_json_write_ctx *w,
2924 : struct spdk_accel_crypto_key *key)
2925 : {
2926 0 : spdk_json_write_object_begin(w);
2927 0 : spdk_json_write_named_string(w, "method", "accel_crypto_key_create");
2928 0 : spdk_json_write_named_object_begin(w, "params");
2929 0 : __accel_crypto_key_dump_param(w, key);
2930 0 : spdk_json_write_object_end(w);
2931 0 : spdk_json_write_object_end(w);
2932 0 : }
2933 :
2934 : static void
2935 0 : accel_write_options(struct spdk_json_write_ctx *w)
2936 : {
2937 0 : spdk_json_write_object_begin(w);
2938 0 : spdk_json_write_named_string(w, "method", "accel_set_options");
2939 0 : spdk_json_write_named_object_begin(w, "params");
2940 0 : spdk_json_write_named_uint32(w, "small_cache_size", g_opts.small_cache_size);
2941 0 : spdk_json_write_named_uint32(w, "large_cache_size", g_opts.large_cache_size);
2942 0 : spdk_json_write_named_uint32(w, "task_count", g_opts.task_count);
2943 0 : spdk_json_write_named_uint32(w, "sequence_count", g_opts.sequence_count);
2944 0 : spdk_json_write_named_uint32(w, "buf_count", g_opts.buf_count);
2945 0 : spdk_json_write_object_end(w);
2946 0 : spdk_json_write_object_end(w);
2947 0 : }
2948 :
2949 : static void
2950 0 : _accel_crypto_keys_write_config_json(struct spdk_json_write_ctx *w, bool full_dump)
2951 : {
2952 : struct spdk_accel_crypto_key *key;
2953 :
2954 0 : spdk_spin_lock(&g_keyring_spin);
2955 0 : TAILQ_FOREACH(key, &g_keyring, link) {
2956 0 : if (full_dump) {
2957 0 : _accel_crypto_key_write_config_json(w, key);
2958 : } else {
2959 0 : _accel_crypto_key_dump_param(w, key);
2960 : }
2961 : }
2962 0 : spdk_spin_unlock(&g_keyring_spin);
2963 0 : }
2964 :
2965 : void
2966 0 : _accel_crypto_keys_dump_param(struct spdk_json_write_ctx *w)
2967 : {
2968 0 : _accel_crypto_keys_write_config_json(w, false);
2969 0 : }
2970 :
2971 : void
2972 0 : spdk_accel_write_config_json(struct spdk_json_write_ctx *w)
2973 : {
2974 : struct spdk_accel_module_if *accel_module;
2975 : int i;
2976 :
2977 0 : spdk_json_write_array_begin(w);
2978 0 : accel_write_options(w);
2979 :
2980 0 : TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
2981 0 : if (accel_module->write_config_json) {
2982 0 : accel_module->write_config_json(w);
2983 : }
2984 : }
2985 0 : for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) {
2986 0 : if (g_modules_opc_override[i]) {
2987 0 : accel_write_overridden_opc(w, g_opcode_strings[i], g_modules_opc_override[i]);
2988 : }
2989 : }
2990 :
2991 0 : _accel_crypto_keys_write_config_json(w, true);
2992 :
2993 0 : spdk_json_write_array_end(w);
2994 0 : }
2995 :
2996 : void
2997 7 : spdk_accel_module_finish(void)
2998 : {
2999 7 : if (!g_accel_module) {
3000 2 : g_accel_module = TAILQ_FIRST(&spdk_accel_module_list);
3001 : } else {
3002 5 : g_accel_module = TAILQ_NEXT(g_accel_module, tailq);
3003 : }
3004 :
3005 7 : if (!g_accel_module) {
3006 2 : if (g_accel_driver != NULL && g_accel_driver->fini != NULL) {
3007 0 : g_accel_driver->fini();
3008 : }
3009 :
3010 2 : spdk_spin_destroy(&g_keyring_spin);
3011 2 : spdk_spin_destroy(&g_stats_lock);
3012 2 : if (g_accel_domain) {
3013 2 : spdk_memory_domain_destroy(g_accel_domain);
3014 2 : g_accel_domain = NULL;
3015 : }
3016 2 : accel_module_finish_cb();
3017 2 : return;
3018 : }
3019 :
3020 5 : if (g_accel_module->module_fini) {
3021 1 : spdk_thread_send_msg(spdk_get_thread(), g_accel_module->module_fini, NULL);
3022 : } else {
3023 4 : spdk_accel_module_finish();
3024 : }
3025 : }
3026 :
3027 : static void
3028 2 : accel_io_device_unregister_cb(void *io_device)
3029 : {
3030 : struct spdk_accel_crypto_key *key, *key_tmp;
3031 : enum spdk_accel_opcode op;
3032 :
3033 2 : spdk_spin_lock(&g_keyring_spin);
3034 2 : TAILQ_FOREACH_SAFE(key, &g_keyring, link, key_tmp) {
3035 0 : accel_crypto_key_destroy_unsafe(key);
3036 : }
3037 2 : spdk_spin_unlock(&g_keyring_spin);
3038 :
3039 32 : for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) {
3040 30 : if (g_modules_opc_override[op] != NULL) {
3041 0 : free(g_modules_opc_override[op]);
3042 0 : g_modules_opc_override[op] = NULL;
3043 : }
3044 30 : g_modules_opc[op].module = NULL;
3045 : }
3046 :
3047 2 : spdk_accel_module_finish();
3048 2 : }
3049 :
3050 : void
3051 2 : spdk_accel_finish(spdk_accel_fini_cb cb_fn, void *cb_arg)
3052 : {
3053 2 : assert(cb_fn != NULL);
3054 :
3055 2 : g_fini_cb_fn = cb_fn;
3056 2 : g_fini_cb_arg = cb_arg;
3057 :
3058 2 : spdk_io_device_unregister(&spdk_accel_module_list, accel_io_device_unregister_cb);
3059 2 : }
3060 :
3061 : static struct spdk_accel_driver *
3062 2 : accel_find_driver(const char *name)
3063 : {
3064 : struct spdk_accel_driver *driver;
3065 :
3066 2 : TAILQ_FOREACH(driver, &g_accel_drivers, tailq) {
3067 1 : if (strcmp(driver->name, name) == 0) {
3068 1 : return driver;
3069 : }
3070 : }
3071 :
3072 1 : return NULL;
3073 : }
3074 :
3075 : int
3076 1 : spdk_accel_set_driver(const char *name)
3077 : {
3078 : struct spdk_accel_driver *driver;
3079 :
3080 1 : driver = accel_find_driver(name);
3081 1 : if (driver == NULL) {
3082 0 : SPDK_ERRLOG("Couldn't find driver named '%s'\n", name);
3083 0 : return -ENODEV;
3084 : }
3085 :
3086 1 : g_accel_driver = driver;
3087 :
3088 1 : return 0;
3089 : }
3090 :
3091 : const char *
3092 0 : spdk_accel_get_driver_name(void)
3093 : {
3094 0 : if (!g_accel_driver) {
3095 0 : return NULL;
3096 : }
3097 :
3098 0 : return g_accel_driver->name;
3099 : }
3100 :
3101 : void
3102 1 : spdk_accel_driver_register(struct spdk_accel_driver *driver)
3103 : {
3104 1 : if (accel_find_driver(driver->name)) {
3105 0 : SPDK_ERRLOG("Driver named '%s' has already been registered\n", driver->name);
3106 0 : assert(0);
3107 : return;
3108 : }
3109 :
3110 1 : TAILQ_INSERT_TAIL(&g_accel_drivers, driver, tailq);
3111 : }
3112 :
3113 : int
3114 0 : spdk_accel_set_opts(const struct spdk_accel_opts *opts)
3115 : {
3116 0 : if (!opts) {
3117 0 : SPDK_ERRLOG("opts cannot be NULL\n");
3118 0 : return -1;
3119 : }
3120 :
3121 0 : if (!opts->opts_size) {
3122 0 : SPDK_ERRLOG("opts_size inside opts cannot be zero value\n");
3123 0 : return -1;
3124 : }
3125 :
3126 : #define SET_FIELD(field) \
3127 : if (offsetof(struct spdk_accel_opts, field) + sizeof(opts->field) <= opts->opts_size) { \
3128 : g_opts.field = opts->field; \
3129 : } \
3130 :
3131 0 : SET_FIELD(small_cache_size);
3132 0 : SET_FIELD(large_cache_size);
3133 0 : SET_FIELD(task_count);
3134 0 : SET_FIELD(sequence_count);
3135 0 : SET_FIELD(buf_count);
3136 :
3137 0 : g_opts.opts_size = opts->opts_size;
3138 :
3139 : #undef SET_FIELD
3140 :
3141 0 : return 0;
3142 : }
3143 :
3144 : void
3145 0 : spdk_accel_get_opts(struct spdk_accel_opts *opts, size_t opts_size)
3146 : {
3147 0 : if (!opts) {
3148 0 : SPDK_ERRLOG("opts should not be NULL\n");
3149 0 : return;
3150 : }
3151 :
3152 0 : if (!opts_size) {
3153 0 : SPDK_ERRLOG("opts_size should not be zero value\n");
3154 0 : return;
3155 : }
3156 :
3157 0 : opts->opts_size = opts_size;
3158 :
3159 : #define SET_FIELD(field) \
3160 : if (offsetof(struct spdk_accel_opts, field) + sizeof(opts->field) <= opts_size) { \
3161 : opts->field = g_opts.field; \
3162 : } \
3163 :
3164 0 : SET_FIELD(small_cache_size);
3165 0 : SET_FIELD(large_cache_size);
3166 0 : SET_FIELD(task_count);
3167 0 : SET_FIELD(sequence_count);
3168 0 : SET_FIELD(buf_count);
3169 :
3170 : #undef SET_FIELD
3171 :
3172 : /* Do not remove this statement, you should always update this statement when you adding a new field,
3173 : * and do not forget to add the SET_FIELD statement for your added field. */
3174 : SPDK_STATIC_ASSERT(sizeof(struct spdk_accel_opts) == 28, "Incorrect size");
3175 : }
3176 :
3177 : struct accel_get_stats_ctx {
3178 : struct accel_stats stats;
3179 : accel_get_stats_cb cb_fn;
3180 : void *cb_arg;
3181 : };
3182 :
3183 : static void
3184 0 : accel_get_channel_stats_done(struct spdk_io_channel_iter *iter, int status)
3185 : {
3186 0 : struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter);
3187 :
3188 0 : ctx->cb_fn(&ctx->stats, ctx->cb_arg);
3189 0 : free(ctx);
3190 0 : }
3191 :
3192 : static void
3193 0 : accel_get_channel_stats(struct spdk_io_channel_iter *iter)
3194 : {
3195 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(iter);
3196 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
3197 0 : struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter);
3198 :
3199 0 : accel_add_stats(&ctx->stats, &accel_ch->stats);
3200 0 : spdk_for_each_channel_continue(iter, 0);
3201 0 : }
3202 :
3203 : int
3204 0 : accel_get_stats(accel_get_stats_cb cb_fn, void *cb_arg)
3205 : {
3206 : struct accel_get_stats_ctx *ctx;
3207 :
3208 0 : ctx = calloc(1, sizeof(*ctx));
3209 0 : if (ctx == NULL) {
3210 0 : return -ENOMEM;
3211 : }
3212 :
3213 0 : spdk_spin_lock(&g_stats_lock);
3214 0 : accel_add_stats(&ctx->stats, &g_stats);
3215 0 : spdk_spin_unlock(&g_stats_lock);
3216 :
3217 0 : ctx->cb_fn = cb_fn;
3218 0 : ctx->cb_arg = cb_arg;
3219 :
3220 0 : spdk_for_each_channel(&spdk_accel_module_list, accel_get_channel_stats, ctx,
3221 : accel_get_channel_stats_done);
3222 :
3223 0 : return 0;
3224 : }
3225 :
3226 : void
3227 0 : spdk_accel_get_opcode_stats(struct spdk_io_channel *ch, enum spdk_accel_opcode opcode,
3228 : struct spdk_accel_opcode_stats *stats, size_t size)
3229 : {
3230 0 : struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
3231 :
3232 : #define FIELD_OK(field) \
3233 : offsetof(struct spdk_accel_opcode_stats, field) + sizeof(stats->field) <= size
3234 :
3235 : #define SET_FIELD(field, value) \
3236 : if (FIELD_OK(field)) { \
3237 : stats->field = value; \
3238 : }
3239 :
3240 0 : SET_FIELD(executed, accel_ch->stats.operations[opcode].executed);
3241 0 : SET_FIELD(failed, accel_ch->stats.operations[opcode].failed);
3242 0 : SET_FIELD(num_bytes, accel_ch->stats.operations[opcode].num_bytes);
3243 :
3244 : #undef FIELD_OK
3245 : #undef SET_FIELD
3246 0 : }
3247 :
3248 : uint8_t
3249 0 : spdk_accel_get_buf_align(enum spdk_accel_opcode opcode,
3250 : const struct spdk_accel_operation_exec_ctx *ctx)
3251 : {
3252 0 : struct spdk_accel_module_if *module = g_modules_opc[opcode].module;
3253 0 : struct spdk_accel_opcode_info modinfo = {}, drvinfo = {};
3254 :
3255 0 : if (g_accel_driver != NULL && g_accel_driver->get_operation_info != NULL) {
3256 0 : g_accel_driver->get_operation_info(opcode, ctx, &drvinfo);
3257 : }
3258 :
3259 0 : if (module->get_operation_info != NULL) {
3260 0 : module->get_operation_info(opcode, ctx, &modinfo);
3261 : }
3262 :
3263 : /* If a driver is set, it'll execute most of the operations, while the rest will usually
3264 : * fall back to accel_sw, which doesn't have any alignment requiremenets. However, to be
3265 : * extra safe, return the max(driver, module) if a driver delegates some operations to a
3266 : * hardware module. */
3267 0 : return spdk_max(modinfo.required_alignment, drvinfo.required_alignment);
3268 : }
3269 :
3270 : struct spdk_accel_module_if *
3271 0 : spdk_accel_get_module(const char *name)
3272 : {
3273 : struct spdk_accel_module_if *module;
3274 :
3275 0 : TAILQ_FOREACH(module, &spdk_accel_module_list, tailq) {
3276 0 : if (strcmp(module->name, name) == 0) {
3277 0 : return module;
3278 : }
3279 : }
3280 :
3281 0 : return NULL;
3282 : }
3283 :
3284 : int
3285 0 : spdk_accel_get_opc_memory_domains(enum spdk_accel_opcode opcode,
3286 : struct spdk_memory_domain **domains,
3287 : int array_size)
3288 : {
3289 0 : assert(opcode < SPDK_ACCEL_OPC_LAST);
3290 :
3291 0 : if (g_modules_opc[opcode].module->get_memory_domains) {
3292 0 : return g_modules_opc[opcode].module->get_memory_domains(domains, array_size);
3293 : }
3294 :
3295 0 : return 0;
3296 : }
3297 :
3298 1 : SPDK_LOG_REGISTER_COMPONENT(accel)
|