Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2022 Intel Corporation.
3 : * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES
4 : * All rights reserved.
5 : */
6 :
7 : #include "spdk/stdinc.h"
8 :
9 : #include "spdk/accel_module.h"
10 : #include "accel_internal.h"
11 :
12 : #include "spdk/env.h"
13 : #include "spdk/likely.h"
14 : #include "spdk/log.h"
15 : #include "spdk/thread.h"
16 : #include "spdk/json.h"
17 : #include "spdk/crc32.h"
18 : #include "spdk/util.h"
19 : #include "spdk/xor.h"
20 : #include "spdk/dif.h"
21 :
22 : #ifdef SPDK_CONFIG_HAVE_LZ4
23 : #include <lz4.h>
24 : #endif
25 :
26 : #ifdef SPDK_CONFIG_ISAL
27 : #include "../isa-l/include/igzip_lib.h"
28 : #ifdef SPDK_CONFIG_ISAL_CRYPTO
29 : #include "../isa-l-crypto/include/aes_xts.h"
30 : #include "../isa-l-crypto/include/isal_crypto_api.h"
31 : #endif
32 : #endif
33 :
34 : /* Per the AES-XTS spec, the size of data unit cannot be bigger than 2^20 blocks, 128b each block */
35 : #define ACCEL_AES_XTS_MAX_BLOCK_SIZE (1 << 24)
36 :
37 : #ifdef SPDK_CONFIG_ISAL
38 : #define COMP_DEFLATE_MIN_LEVEL ISAL_DEF_MIN_LEVEL
39 : #define COMP_DEFLATE_MAX_LEVEL ISAL_DEF_MAX_LEVEL
40 : #else
41 : #define COMP_DEFLATE_MIN_LEVEL 0
42 : #define COMP_DEFLATE_MAX_LEVEL 0
43 : #endif
44 :
45 : #define COMP_DEFLATE_LEVEL_NUM (COMP_DEFLATE_MAX_LEVEL + 1)
46 :
47 : struct comp_deflate_level_buf {
48 : uint32_t size;
49 : uint8_t *buf;
50 : };
51 :
52 : struct sw_accel_io_channel {
53 : /* for ISAL */
54 : #ifdef SPDK_CONFIG_ISAL
55 : struct isal_zstream stream;
56 : struct inflate_state state;
57 : /* The array index corresponds to the algorithm level */
58 : struct comp_deflate_level_buf deflate_level_bufs[COMP_DEFLATE_LEVEL_NUM];
59 : uint8_t level_buf_mem[ISAL_DEF_LVL0_DEFAULT + ISAL_DEF_LVL1_DEFAULT +
60 : ISAL_DEF_LVL2_DEFAULT + ISAL_DEF_LVL3_DEFAULT];
61 : #endif
62 : #ifdef SPDK_CONFIG_HAVE_LZ4
63 : /* for lz4 */
64 : LZ4_stream_t *lz4_stream;
65 : LZ4_streamDecode_t *lz4_stream_decode;
66 : #endif
67 : struct spdk_poller *completion_poller;
68 : STAILQ_HEAD(, spdk_accel_task) tasks_to_complete;
69 : };
70 :
71 : typedef int (*sw_accel_crypto_op)(const uint8_t *k2, const uint8_t *k1,
72 : const uint8_t *initial_tweak, const uint64_t len_bytes,
73 : const void *in, void *out);
74 :
75 : struct sw_accel_crypto_key_data {
76 : sw_accel_crypto_op encrypt;
77 : sw_accel_crypto_op decrypt;
78 : };
79 :
80 : static struct spdk_accel_module_if g_sw_module;
81 :
82 : static void sw_accel_crypto_key_deinit(struct spdk_accel_crypto_key *_key);
83 : static int sw_accel_crypto_key_init(struct spdk_accel_crypto_key *key);
84 : static bool sw_accel_crypto_supports_tweak_mode(enum spdk_accel_crypto_tweak_mode tweak_mode);
85 : static bool sw_accel_crypto_supports_cipher(enum spdk_accel_cipher cipher, size_t key_size);
86 :
87 : /* Post SW completions to a list; processed by ->completion_poller. */
88 : inline static void
89 65 : _add_to_comp_list(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task, int status)
90 : {
91 65 : accel_task->status = status;
92 65 : STAILQ_INSERT_TAIL(&sw_ch->tasks_to_complete, accel_task, link);
93 65 : }
94 :
95 : static bool
96 17 : sw_accel_supports_opcode(enum spdk_accel_opcode opc)
97 : {
98 17 : switch (opc) {
99 : case SPDK_ACCEL_OPC_COPY:
100 : case SPDK_ACCEL_OPC_FILL:
101 : case SPDK_ACCEL_OPC_DUALCAST:
102 : case SPDK_ACCEL_OPC_COMPARE:
103 : case SPDK_ACCEL_OPC_CRC32C:
104 : case SPDK_ACCEL_OPC_COPY_CRC32C:
105 : case SPDK_ACCEL_OPC_COMPRESS:
106 : case SPDK_ACCEL_OPC_DECOMPRESS:
107 : case SPDK_ACCEL_OPC_ENCRYPT:
108 : case SPDK_ACCEL_OPC_DECRYPT:
109 : case SPDK_ACCEL_OPC_XOR:
110 : case SPDK_ACCEL_OPC_DIF_VERIFY:
111 : case SPDK_ACCEL_OPC_DIF_GENERATE:
112 : case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:
113 : case SPDK_ACCEL_OPC_DIF_VERIFY_COPY:
114 : case SPDK_ACCEL_OPC_DIX_GENERATE:
115 : case SPDK_ACCEL_OPC_DIX_VERIFY:
116 17 : return true;
117 : default:
118 0 : return false;
119 : }
120 17 : }
121 :
122 : static int
123 1 : _sw_accel_dualcast_iovs(struct iovec *dst_iovs, uint32_t dst_iovcnt,
124 : struct iovec *dst2_iovs, uint32_t dst2_iovcnt,
125 : struct iovec *src_iovs, uint32_t src_iovcnt)
126 : {
127 1 : if (spdk_unlikely(dst_iovcnt != 1 || dst2_iovcnt != 1 || src_iovcnt != 1)) {
128 0 : return -EINVAL;
129 : }
130 :
131 1 : if (spdk_unlikely(dst_iovs[0].iov_len != src_iovs[0].iov_len ||
132 : dst_iovs[0].iov_len != dst2_iovs[0].iov_len)) {
133 0 : return -EINVAL;
134 : }
135 :
136 1 : memcpy(dst_iovs[0].iov_base, src_iovs[0].iov_base, dst_iovs[0].iov_len);
137 1 : memcpy(dst2_iovs[0].iov_base, src_iovs[0].iov_base, dst_iovs[0].iov_len);
138 :
139 1 : return 0;
140 1 : }
141 :
142 : static void
143 9 : _sw_accel_copy_iovs(struct iovec *dst_iovs, uint32_t dst_iovcnt,
144 : struct iovec *src_iovs, uint32_t src_iovcnt)
145 : {
146 : struct spdk_ioviter iter;
147 : void *src, *dst;
148 : size_t len;
149 :
150 27 : for (len = spdk_ioviter_first(&iter, src_iovs, src_iovcnt,
151 9 : dst_iovs, dst_iovcnt, &src, &dst);
152 18 : len != 0;
153 9 : len = spdk_ioviter_next(&iter, &src, &dst)) {
154 9 : memcpy(dst, src, len);
155 9 : }
156 9 : }
157 :
158 : static int
159 1 : _sw_accel_compare(struct iovec *src_iovs, uint32_t src_iovcnt,
160 : struct iovec *src2_iovs, uint32_t src2_iovcnt)
161 : {
162 1 : if (spdk_unlikely(src_iovcnt != 1 || src2_iovcnt != 1)) {
163 0 : return -EINVAL;
164 : }
165 :
166 1 : if (spdk_unlikely(src_iovs[0].iov_len != src2_iovs[0].iov_len)) {
167 0 : return -EINVAL;
168 : }
169 :
170 1 : return memcmp(src_iovs[0].iov_base, src2_iovs[0].iov_base, src_iovs[0].iov_len);
171 1 : }
172 :
173 : static int
174 25 : _sw_accel_fill(struct iovec *iovs, uint32_t iovcnt, uint8_t fill)
175 : {
176 : void *dst;
177 : size_t nbytes;
178 :
179 25 : if (spdk_unlikely(iovcnt != 1)) {
180 0 : return -EINVAL;
181 : }
182 :
183 25 : dst = iovs[0].iov_base;
184 25 : nbytes = iovs[0].iov_len;
185 :
186 25 : memset(dst, fill, nbytes);
187 :
188 25 : return 0;
189 25 : }
190 :
191 : static void
192 9 : _sw_accel_crc32cv(uint32_t *crc_dst, struct iovec *iov, uint32_t iovcnt, uint32_t seed)
193 : {
194 9 : *crc_dst = spdk_crc32c_iov_update(iov, iovcnt, ~seed);
195 9 : }
196 :
197 : static int
198 0 : _sw_accel_compress_lz4(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
199 : {
200 : #ifdef SPDK_CONFIG_HAVE_LZ4
201 0 : LZ4_stream_t *stream = sw_ch->lz4_stream;
202 0 : struct iovec *siov = accel_task->s.iovs;
203 0 : struct iovec *diov = accel_task->d.iovs;
204 0 : size_t dst_segoffset = 0;
205 0 : int32_t comp_size = 0;
206 0 : uint32_t output_size = 0;
207 0 : uint32_t i, d = 0;
208 0 : int rc = 0;
209 :
210 0 : LZ4_resetStream(stream);
211 0 : for (i = 0; i < accel_task->s.iovcnt; i++) {
212 0 : if ((diov[d].iov_len - dst_segoffset) == 0) {
213 0 : if (++d < accel_task->d.iovcnt) {
214 0 : dst_segoffset = 0;
215 0 : } else {
216 0 : SPDK_ERRLOG("Not enough destination buffer provided.\n");
217 0 : rc = -ENOMEM;
218 0 : break;
219 : }
220 0 : }
221 :
222 0 : comp_size = LZ4_compress_fast_continue(stream, siov[i].iov_base, diov[d].iov_base + dst_segoffset,
223 0 : siov[i].iov_len, diov[d].iov_len - dst_segoffset,
224 0 : accel_task->comp.level);
225 0 : if (comp_size <= 0) {
226 0 : SPDK_ERRLOG("LZ4_compress_fast_continue was incorrectly executed.\n");
227 0 : rc = -EIO;
228 0 : break;
229 : }
230 :
231 0 : dst_segoffset += comp_size;
232 0 : output_size += comp_size;
233 0 : }
234 :
235 : /* Get our total output size */
236 0 : if (accel_task->output_size != NULL) {
237 0 : *accel_task->output_size = output_size;
238 0 : }
239 :
240 0 : return rc;
241 : #else
242 : SPDK_ERRLOG("LZ4 library is required to use software compression.\n");
243 : return -EINVAL;
244 : #endif
245 : }
246 :
247 : static int
248 0 : _sw_accel_decompress_lz4(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
249 : {
250 : #ifdef SPDK_CONFIG_HAVE_LZ4
251 0 : LZ4_streamDecode_t *stream = sw_ch->lz4_stream_decode;
252 0 : struct iovec *siov = accel_task->s.iovs;
253 0 : struct iovec *diov = accel_task->d.iovs;
254 0 : size_t dst_segoffset = 0;
255 0 : int32_t decomp_size = 0;
256 0 : uint32_t output_size = 0;
257 0 : uint32_t i, d = 0;
258 0 : int rc = 0;
259 :
260 0 : LZ4_setStreamDecode(stream, NULL, 0);
261 0 : for (i = 0; i < accel_task->s.iovcnt; ++i) {
262 0 : if ((diov[d].iov_len - dst_segoffset) == 0) {
263 0 : if (++d < accel_task->d.iovcnt) {
264 0 : dst_segoffset = 0;
265 0 : } else {
266 0 : SPDK_ERRLOG("Not enough destination buffer provided.\n");
267 0 : rc = -ENOMEM;
268 0 : break;
269 : }
270 0 : }
271 0 : decomp_size = LZ4_decompress_safe_continue(stream, siov[i].iov_base,
272 0 : diov[d].iov_base + dst_segoffset, siov[i].iov_len,
273 0 : diov[d].iov_len - dst_segoffset);
274 0 : if (decomp_size < 0) {
275 0 : SPDK_ERRLOG("LZ4_compress_fast_continue was incorrectly executed.\n");
276 0 : rc = -EIO;
277 0 : break;
278 : }
279 0 : dst_segoffset += decomp_size;
280 0 : output_size += decomp_size;
281 0 : }
282 :
283 : /* Get our total output size */
284 0 : if (accel_task->output_size != NULL) {
285 0 : *accel_task->output_size = output_size;
286 0 : }
287 :
288 0 : return rc;
289 : #else
290 : SPDK_ERRLOG("LZ4 library is required to use software compression.\n");
291 : return -EINVAL;
292 : #endif
293 : }
294 :
295 : static int
296 2 : _sw_accel_compress_deflate(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
297 : {
298 : #ifdef SPDK_CONFIG_ISAL
299 2 : size_t last_seglen = accel_task->s.iovs[accel_task->s.iovcnt - 1].iov_len;
300 2 : struct iovec *siov = accel_task->s.iovs;
301 2 : struct iovec *diov = accel_task->d.iovs;
302 : size_t remaining;
303 2 : uint32_t i, s = 0, d = 0;
304 2 : int rc = 0;
305 :
306 2 : if (accel_task->comp.level > COMP_DEFLATE_MAX_LEVEL) {
307 0 : SPDK_ERRLOG("isal_deflate doesn't support this algorithm level(%u)\n", accel_task->comp.level);
308 0 : return -EINVAL;
309 : }
310 :
311 2 : remaining = 0;
312 4 : for (i = 0; i < accel_task->s.iovcnt; ++i) {
313 2 : remaining += accel_task->s.iovs[i].iov_len;
314 2 : }
315 :
316 2 : isal_deflate_reset(&sw_ch->stream);
317 2 : sw_ch->stream.end_of_stream = 0;
318 2 : sw_ch->stream.next_out = diov[d].iov_base;
319 2 : sw_ch->stream.avail_out = diov[d].iov_len;
320 2 : sw_ch->stream.next_in = siov[s].iov_base;
321 2 : sw_ch->stream.avail_in = siov[s].iov_len;
322 2 : sw_ch->stream.level = accel_task->comp.level;
323 2 : sw_ch->stream.level_buf = sw_ch->deflate_level_bufs[accel_task->comp.level].buf;
324 2 : sw_ch->stream.level_buf_size = sw_ch->deflate_level_bufs[accel_task->comp.level].size;
325 :
326 2 : do {
327 : /* if isal has exhausted the current dst iovec, move to the next
328 : * one if there is one */
329 2 : if (sw_ch->stream.avail_out == 0) {
330 0 : if (++d < accel_task->d.iovcnt) {
331 0 : sw_ch->stream.next_out = diov[d].iov_base;
332 0 : sw_ch->stream.avail_out = diov[d].iov_len;
333 0 : assert(sw_ch->stream.avail_out > 0);
334 0 : } else {
335 : /* we have no avail_out but also no more iovecs left so this is
336 : * the case where either the output buffer was a perfect fit
337 : * or not enough was provided. Check the ISAL state to determine
338 : * which. */
339 0 : if (sw_ch->stream.internal_state.state != ZSTATE_END) {
340 0 : SPDK_ERRLOG("Not enough destination buffer provided.\n");
341 0 : rc = -ENOMEM;
342 0 : }
343 0 : break;
344 : }
345 0 : }
346 :
347 : /* if isal has exhausted the current src iovec, move to the next
348 : * one if there is one */
349 2 : if (sw_ch->stream.avail_in == 0 && ((s + 1) < accel_task->s.iovcnt)) {
350 0 : s++;
351 0 : sw_ch->stream.next_in = siov[s].iov_base;
352 0 : sw_ch->stream.avail_in = siov[s].iov_len;
353 0 : assert(sw_ch->stream.avail_in > 0);
354 0 : }
355 :
356 2 : if (remaining <= last_seglen) {
357 : /* Need to set end of stream on last block */
358 2 : sw_ch->stream.end_of_stream = 1;
359 2 : }
360 :
361 2 : rc = isal_deflate(&sw_ch->stream);
362 2 : if (rc) {
363 0 : SPDK_ERRLOG("isal_deflate returned error %d.\n", rc);
364 0 : }
365 :
366 2 : if (remaining > 0) {
367 2 : assert(siov[s].iov_len > sw_ch->stream.avail_in);
368 2 : remaining -= (siov[s].iov_len - sw_ch->stream.avail_in);
369 2 : }
370 :
371 2 : } while (remaining > 0 || sw_ch->stream.avail_out == 0);
372 2 : assert(sw_ch->stream.avail_in == 0);
373 :
374 : /* Get our total output size */
375 2 : if (accel_task->output_size != NULL) {
376 2 : assert(sw_ch->stream.total_out > 0);
377 2 : *accel_task->output_size = sw_ch->stream.total_out;
378 2 : }
379 :
380 2 : return rc;
381 : #else
382 : SPDK_ERRLOG("ISAL option is required to use software compression.\n");
383 : return -EINVAL;
384 : #endif
385 2 : }
386 :
387 : static int
388 7 : _sw_accel_decompress_deflate(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
389 : {
390 : #ifdef SPDK_CONFIG_ISAL
391 7 : struct iovec *siov = accel_task->s.iovs;
392 7 : struct iovec *diov = accel_task->d.iovs;
393 7 : uint32_t s = 0, d = 0;
394 7 : int rc = 0;
395 :
396 7 : isal_inflate_reset(&sw_ch->state);
397 7 : sw_ch->state.next_out = diov[d].iov_base;
398 7 : sw_ch->state.avail_out = diov[d].iov_len;
399 7 : sw_ch->state.next_in = siov[s].iov_base;
400 7 : sw_ch->state.avail_in = siov[s].iov_len;
401 :
402 7 : do {
403 : /* if isal has exhausted the current dst iovec, move to the next
404 : * one if there is one */
405 7 : if (sw_ch->state.avail_out == 0 && ((d + 1) < accel_task->d.iovcnt)) {
406 0 : d++;
407 0 : sw_ch->state.next_out = diov[d].iov_base;
408 0 : sw_ch->state.avail_out = diov[d].iov_len;
409 0 : assert(sw_ch->state.avail_out > 0);
410 0 : }
411 :
412 : /* if isal has exhausted the current src iovec, move to the next
413 : * one if there is one */
414 7 : if (sw_ch->state.avail_in == 0 && ((s + 1) < accel_task->s.iovcnt)) {
415 0 : s++;
416 0 : sw_ch->state.next_in = siov[s].iov_base;
417 0 : sw_ch->state.avail_in = siov[s].iov_len;
418 0 : assert(sw_ch->state.avail_in > 0);
419 0 : }
420 :
421 7 : rc = isal_inflate(&sw_ch->state);
422 7 : if (rc) {
423 0 : SPDK_ERRLOG("isal_inflate returned error %d.\n", rc);
424 0 : }
425 :
426 7 : } while (sw_ch->state.block_state < ISAL_BLOCK_FINISH);
427 7 : assert(sw_ch->state.avail_in == 0);
428 :
429 : /* Get our total output size */
430 7 : if (accel_task->output_size != NULL) {
431 0 : assert(sw_ch->state.total_out > 0);
432 0 : *accel_task->output_size = sw_ch->state.total_out;
433 0 : }
434 :
435 7 : return rc;
436 : #else
437 : SPDK_ERRLOG("ISAL option is required to use software decompression.\n");
438 : return -EINVAL;
439 : #endif
440 : }
441 :
442 : static int
443 2 : _sw_accel_compress(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
444 : {
445 2 : switch (accel_task->comp.algo) {
446 : case SPDK_ACCEL_COMP_ALGO_DEFLATE:
447 2 : return _sw_accel_compress_deflate(sw_ch, accel_task);
448 : case SPDK_ACCEL_COMP_ALGO_LZ4:
449 0 : return _sw_accel_compress_lz4(sw_ch, accel_task);
450 : default:
451 0 : assert(0);
452 : return -EINVAL;
453 : }
454 2 : }
455 :
456 : static int
457 7 : _sw_accel_decompress(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
458 : {
459 7 : switch (accel_task->comp.algo) {
460 : case SPDK_ACCEL_COMP_ALGO_DEFLATE:
461 7 : return _sw_accel_decompress_deflate(sw_ch, accel_task);
462 : case SPDK_ACCEL_COMP_ALGO_LZ4:
463 0 : return _sw_accel_decompress_lz4(sw_ch, accel_task);
464 : default:
465 0 : assert(0);
466 : return -EINVAL;
467 : }
468 7 : }
469 :
470 : static int
471 5 : _sw_accel_crypto_operation(struct spdk_accel_task *accel_task, struct spdk_accel_crypto_key *key,
472 : sw_accel_crypto_op op)
473 : {
474 : #ifdef SPDK_CONFIG_ISAL_CRYPTO
475 : uint64_t iv[2];
476 : size_t remaining_len, dst_len;
477 5 : uint64_t src_offset = 0, dst_offset = 0;
478 5 : uint32_t src_iovpos = 0, dst_iovpos = 0, src_iovcnt, dst_iovcnt;
479 5 : uint32_t i, block_size, crypto_len, crypto_accum_len = 0;
480 : struct iovec *src_iov, *dst_iov;
481 : uint8_t *src, *dst;
482 : int rc;
483 :
484 : /* iv is 128 bits, since we are using logical block address (64 bits) as iv, fill first 8 bytes with zeroes */
485 5 : iv[0] = 0;
486 5 : iv[1] = accel_task->iv;
487 5 : src_iov = accel_task->s.iovs;
488 5 : src_iovcnt = accel_task->s.iovcnt;
489 5 : if (accel_task->d.iovcnt) {
490 5 : dst_iov = accel_task->d.iovs;
491 5 : dst_iovcnt = accel_task->d.iovcnt;
492 5 : } else {
493 : /* inplace operation */
494 0 : dst_iov = accel_task->s.iovs;
495 0 : dst_iovcnt = accel_task->s.iovcnt;
496 : }
497 5 : block_size = accel_task->block_size;
498 :
499 5 : if (!src_iovcnt || !dst_iovcnt || !block_size || !op) {
500 0 : SPDK_ERRLOG("src_iovcnt %d, dst_iovcnt %d, block_size %d, op %p\n", src_iovcnt, dst_iovcnt,
501 : block_size, op);
502 0 : return -EINVAL;
503 : }
504 :
505 5 : remaining_len = 0;
506 10 : for (i = 0; i < src_iovcnt; i++) {
507 5 : remaining_len += src_iov[i].iov_len;
508 5 : }
509 5 : dst_len = 0;
510 10 : for (i = 0; i < dst_iovcnt; i++) {
511 5 : dst_len += dst_iov[i].iov_len;
512 5 : }
513 :
514 5 : if (spdk_unlikely(remaining_len != dst_len || !remaining_len)) {
515 0 : return -ERANGE;
516 : }
517 5 : if (spdk_unlikely(remaining_len % accel_task->block_size != 0)) {
518 0 : return -EINVAL;
519 : }
520 :
521 5 : while (remaining_len) {
522 5 : crypto_len = spdk_min(block_size - crypto_accum_len, src_iov->iov_len - src_offset);
523 5 : crypto_len = spdk_min(crypto_len, dst_iov->iov_len - dst_offset);
524 5 : src = (uint8_t *)src_iov->iov_base + src_offset;
525 5 : dst = (uint8_t *)dst_iov->iov_base + dst_offset;
526 :
527 5 : rc = op((uint8_t *)key->key2, (uint8_t *)key->key, (uint8_t *)iv, crypto_len, src, dst);
528 5 : if (rc != ISAL_CRYPTO_ERR_NONE) {
529 0 : break;
530 : }
531 :
532 5 : src_offset += crypto_len;
533 5 : dst_offset += crypto_len;
534 5 : crypto_accum_len += crypto_len;
535 5 : remaining_len -= crypto_len;
536 :
537 5 : if (crypto_accum_len == block_size) {
538 : /* we can process part of logical block. Once the whole block is processed, increment iv */
539 5 : crypto_accum_len = 0;
540 5 : iv[1]++;
541 5 : }
542 5 : if (src_offset == src_iov->iov_len) {
543 5 : src_iov++;
544 5 : src_iovpos++;
545 5 : src_offset = 0;
546 5 : }
547 5 : if (src_iovpos == src_iovcnt) {
548 5 : break;
549 : }
550 0 : if (dst_offset == dst_iov->iov_len) {
551 0 : dst_iov++;
552 0 : dst_iovpos++;
553 0 : dst_offset = 0;
554 0 : }
555 0 : if (dst_iovpos == dst_iovcnt) {
556 0 : break;
557 : }
558 : }
559 :
560 5 : if (remaining_len) {
561 0 : SPDK_ERRLOG("remaining len %zu\n", remaining_len);
562 0 : return -EINVAL;
563 : }
564 :
565 5 : return 0;
566 : #else
567 : return -ENOTSUP;
568 : #endif
569 5 : }
570 :
571 : static int
572 3 : _sw_accel_encrypt(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
573 : {
574 : struct spdk_accel_crypto_key *key;
575 : struct sw_accel_crypto_key_data *key_data;
576 :
577 3 : key = accel_task->crypto_key;
578 3 : if (spdk_unlikely(key->module_if != &g_sw_module || !key->priv)) {
579 0 : return -EINVAL;
580 : }
581 3 : if (spdk_unlikely(accel_task->block_size > ACCEL_AES_XTS_MAX_BLOCK_SIZE)) {
582 0 : SPDK_WARNLOG("Max block size for AES_XTS is limited to %u, current size %u\n",
583 : ACCEL_AES_XTS_MAX_BLOCK_SIZE, accel_task->block_size);
584 0 : return -ERANGE;
585 : }
586 3 : key_data = key->priv;
587 3 : return _sw_accel_crypto_operation(accel_task, key, key_data->encrypt);
588 3 : }
589 :
590 : static int
591 2 : _sw_accel_decrypt(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
592 : {
593 : struct spdk_accel_crypto_key *key;
594 : struct sw_accel_crypto_key_data *key_data;
595 :
596 2 : key = accel_task->crypto_key;
597 2 : if (spdk_unlikely(key->module_if != &g_sw_module || !key->priv)) {
598 0 : return -EINVAL;
599 : }
600 2 : if (spdk_unlikely(accel_task->block_size > ACCEL_AES_XTS_MAX_BLOCK_SIZE)) {
601 0 : SPDK_WARNLOG("Max block size for AES_XTS is limited to %u, current size %u\n",
602 : ACCEL_AES_XTS_MAX_BLOCK_SIZE, accel_task->block_size);
603 0 : return -ERANGE;
604 : }
605 2 : key_data = key->priv;
606 2 : return _sw_accel_crypto_operation(accel_task, key, key_data->decrypt);
607 2 : }
608 :
609 : static int
610 1 : _sw_accel_xor(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
611 : {
612 2 : return spdk_xor_gen(accel_task->d.iovs[0].iov_base,
613 1 : accel_task->nsrcs.srcs,
614 1 : accel_task->nsrcs.cnt,
615 1 : accel_task->d.iovs[0].iov_len);
616 : }
617 :
618 : static int
619 0 : _sw_accel_dif_verify(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
620 : {
621 0 : return spdk_dif_verify(accel_task->s.iovs,
622 0 : accel_task->s.iovcnt,
623 0 : accel_task->dif.num_blocks,
624 0 : accel_task->dif.ctx,
625 0 : accel_task->dif.err);
626 : }
627 :
628 : static int
629 0 : _sw_accel_dif_verify_copy(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
630 : {
631 0 : return spdk_dif_verify_copy(accel_task->d.iovs,
632 0 : accel_task->d.iovcnt,
633 0 : accel_task->s.iovs,
634 0 : accel_task->s.iovcnt,
635 0 : accel_task->dif.num_blocks,
636 0 : accel_task->dif.ctx,
637 0 : accel_task->dif.err);
638 : }
639 :
640 : static int
641 0 : _sw_accel_dif_generate(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
642 : {
643 0 : return spdk_dif_generate(accel_task->s.iovs,
644 0 : accel_task->s.iovcnt,
645 0 : accel_task->dif.num_blocks,
646 0 : accel_task->dif.ctx);
647 : }
648 :
649 : static int
650 0 : _sw_accel_dif_generate_copy(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
651 : {
652 0 : return spdk_dif_generate_copy(accel_task->s.iovs,
653 0 : accel_task->s.iovcnt,
654 0 : accel_task->d.iovs,
655 0 : accel_task->d.iovcnt,
656 0 : accel_task->dif.num_blocks,
657 0 : accel_task->dif.ctx);
658 : }
659 :
660 : static int
661 3 : _sw_accel_dix_generate(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
662 : {
663 6 : return spdk_dix_generate(accel_task->s.iovs,
664 3 : accel_task->s.iovcnt,
665 3 : accel_task->d.iovs,
666 3 : accel_task->dif.num_blocks,
667 3 : accel_task->dif.ctx);
668 : }
669 :
670 : static int
671 3 : _sw_accel_dix_verify(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task)
672 : {
673 6 : return spdk_dix_verify(accel_task->s.iovs,
674 3 : accel_task->s.iovcnt,
675 3 : accel_task->d.iovs,
676 3 : accel_task->dif.num_blocks,
677 3 : accel_task->dif.ctx,
678 3 : accel_task->dif.err);
679 : }
680 :
681 : static int
682 140 : accel_comp_poll(void *arg)
683 : {
684 140 : struct sw_accel_io_channel *sw_ch = arg;
685 : STAILQ_HEAD(, spdk_accel_task) tasks_to_complete;
686 : struct spdk_accel_task *accel_task;
687 :
688 140 : if (STAILQ_EMPTY(&sw_ch->tasks_to_complete)) {
689 83 : return SPDK_POLLER_IDLE;
690 : }
691 :
692 57 : STAILQ_INIT(&tasks_to_complete);
693 57 : STAILQ_SWAP(&tasks_to_complete, &sw_ch->tasks_to_complete, spdk_accel_task);
694 :
695 114 : while ((accel_task = STAILQ_FIRST(&tasks_to_complete))) {
696 57 : STAILQ_REMOVE_HEAD(&tasks_to_complete, link);
697 57 : spdk_accel_task_complete(accel_task, accel_task->status);
698 : }
699 :
700 57 : return SPDK_POLLER_BUSY;
701 140 : }
702 :
703 : static int
704 65 : sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task)
705 : {
706 65 : struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch);
707 : struct spdk_accel_task *tmp;
708 65 : int rc = 0;
709 :
710 : /*
711 : * Lazily initialize our completion poller. We don't want to complete
712 : * them inline as they'll likely submit another.
713 : */
714 65 : if (spdk_unlikely(sw_ch->completion_poller == NULL)) {
715 9 : sw_ch->completion_poller = SPDK_POLLER_REGISTER(accel_comp_poll, sw_ch, 0);
716 9 : }
717 :
718 65 : do {
719 65 : switch (accel_task->op_code) {
720 : case SPDK_ACCEL_OPC_COPY:
721 16 : _sw_accel_copy_iovs(accel_task->d.iovs, accel_task->d.iovcnt,
722 8 : accel_task->s.iovs, accel_task->s.iovcnt);
723 8 : break;
724 : case SPDK_ACCEL_OPC_FILL:
725 50 : rc = _sw_accel_fill(accel_task->d.iovs, accel_task->d.iovcnt,
726 25 : accel_task->fill_pattern);
727 25 : break;
728 : case SPDK_ACCEL_OPC_DUALCAST:
729 2 : rc = _sw_accel_dualcast_iovs(accel_task->d.iovs, accel_task->d.iovcnt,
730 1 : accel_task->d2.iovs, accel_task->d2.iovcnt,
731 1 : accel_task->s.iovs, accel_task->s.iovcnt);
732 1 : break;
733 : case SPDK_ACCEL_OPC_COMPARE:
734 2 : rc = _sw_accel_compare(accel_task->s.iovs, accel_task->s.iovcnt,
735 1 : accel_task->s2.iovs, accel_task->s2.iovcnt);
736 1 : break;
737 : case SPDK_ACCEL_OPC_CRC32C:
738 8 : _sw_accel_crc32cv(accel_task->crc_dst, accel_task->s.iovs, accel_task->s.iovcnt, accel_task->seed);
739 8 : break;
740 : case SPDK_ACCEL_OPC_COPY_CRC32C:
741 2 : _sw_accel_copy_iovs(accel_task->d.iovs, accel_task->d.iovcnt,
742 1 : accel_task->s.iovs, accel_task->s.iovcnt);
743 2 : _sw_accel_crc32cv(accel_task->crc_dst, accel_task->s.iovs,
744 1 : accel_task->s.iovcnt, accel_task->seed);
745 1 : break;
746 : case SPDK_ACCEL_OPC_COMPRESS:
747 2 : rc = _sw_accel_compress(sw_ch, accel_task);
748 2 : break;
749 : case SPDK_ACCEL_OPC_DECOMPRESS:
750 7 : rc = _sw_accel_decompress(sw_ch, accel_task);
751 7 : break;
752 : case SPDK_ACCEL_OPC_XOR:
753 1 : rc = _sw_accel_xor(sw_ch, accel_task);
754 1 : break;
755 : case SPDK_ACCEL_OPC_ENCRYPT:
756 3 : rc = _sw_accel_encrypt(sw_ch, accel_task);
757 3 : break;
758 : case SPDK_ACCEL_OPC_DECRYPT:
759 2 : rc = _sw_accel_decrypt(sw_ch, accel_task);
760 2 : break;
761 : case SPDK_ACCEL_OPC_DIF_VERIFY:
762 0 : rc = _sw_accel_dif_verify(sw_ch, accel_task);
763 0 : break;
764 : case SPDK_ACCEL_OPC_DIF_VERIFY_COPY:
765 0 : rc = _sw_accel_dif_verify_copy(sw_ch, accel_task);
766 0 : break;
767 : case SPDK_ACCEL_OPC_DIF_GENERATE:
768 0 : rc = _sw_accel_dif_generate(sw_ch, accel_task);
769 0 : break;
770 : case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:
771 0 : rc = _sw_accel_dif_generate_copy(sw_ch, accel_task);
772 0 : break;
773 : case SPDK_ACCEL_OPC_DIX_GENERATE:
774 3 : rc = _sw_accel_dix_generate(sw_ch, accel_task);
775 3 : break;
776 : case SPDK_ACCEL_OPC_DIX_VERIFY:
777 3 : rc = _sw_accel_dix_verify(sw_ch, accel_task);
778 3 : break;
779 : default:
780 0 : assert(false);
781 : break;
782 : }
783 :
784 65 : tmp = STAILQ_NEXT(accel_task, link);
785 :
786 65 : _add_to_comp_list(sw_ch, accel_task, rc);
787 :
788 65 : accel_task = tmp;
789 65 : } while (accel_task);
790 :
791 65 : return 0;
792 : }
793 :
794 : static int
795 16 : sw_accel_create_cb(void *io_device, void *ctx_buf)
796 : {
797 16 : struct sw_accel_io_channel *sw_ch = ctx_buf;
798 : #ifdef SPDK_CONFIG_ISAL
799 : struct comp_deflate_level_buf *deflate_level_bufs;
800 : int i;
801 : #endif
802 :
803 16 : STAILQ_INIT(&sw_ch->tasks_to_complete);
804 16 : sw_ch->completion_poller = NULL;
805 :
806 : #ifdef SPDK_CONFIG_HAVE_LZ4
807 16 : sw_ch->lz4_stream = LZ4_createStream();
808 16 : if (sw_ch->lz4_stream == NULL) {
809 0 : SPDK_ERRLOG("Failed to create the lz4 stream for compression\n");
810 0 : return -ENOMEM;
811 : }
812 16 : sw_ch->lz4_stream_decode = LZ4_createStreamDecode();
813 16 : if (sw_ch->lz4_stream_decode == NULL) {
814 0 : SPDK_ERRLOG("Failed to create the lz4 stream for decompression\n");
815 0 : LZ4_freeStream(sw_ch->lz4_stream);
816 0 : return -ENOMEM;
817 : }
818 : #endif
819 : #ifdef SPDK_CONFIG_ISAL
820 16 : sw_ch->deflate_level_bufs[0].buf = sw_ch->level_buf_mem;
821 16 : deflate_level_bufs = sw_ch->deflate_level_bufs;
822 16 : deflate_level_bufs[0].size = ISAL_DEF_LVL0_DEFAULT;
823 64 : for (i = 1; i < COMP_DEFLATE_LEVEL_NUM; i++) {
824 96 : deflate_level_bufs[i].buf = deflate_level_bufs[i - 1].buf +
825 48 : deflate_level_bufs[i - 1].size;
826 48 : switch (i) {
827 : case 1:
828 16 : deflate_level_bufs[i].size = ISAL_DEF_LVL1_DEFAULT;
829 16 : break;
830 : case 2:
831 16 : deflate_level_bufs[i].size = ISAL_DEF_LVL2_DEFAULT;
832 16 : break;
833 : case 3:
834 16 : deflate_level_bufs[i].size = ISAL_DEF_LVL3_DEFAULT;
835 16 : break;
836 : default:
837 0 : assert(false);
838 : }
839 48 : }
840 :
841 16 : isal_deflate_init(&sw_ch->stream);
842 16 : sw_ch->stream.flush = NO_FLUSH;
843 16 : isal_inflate_init(&sw_ch->state);
844 : #endif
845 :
846 16 : return 0;
847 16 : }
848 :
849 : static void
850 16 : sw_accel_destroy_cb(void *io_device, void *ctx_buf)
851 : {
852 16 : struct sw_accel_io_channel *sw_ch = ctx_buf;
853 :
854 : #ifdef SPDK_CONFIG_HAVE_LZ4
855 16 : LZ4_freeStream(sw_ch->lz4_stream);
856 16 : LZ4_freeStreamDecode(sw_ch->lz4_stream_decode);
857 : #endif
858 16 : spdk_poller_unregister(&sw_ch->completion_poller);
859 16 : }
860 :
861 : static struct spdk_io_channel *
862 272 : sw_accel_get_io_channel(void)
863 : {
864 272 : return spdk_get_io_channel(&g_sw_module);
865 : }
866 :
867 : static size_t
868 2 : sw_accel_module_get_ctx_size(void)
869 : {
870 2 : return sizeof(struct spdk_accel_task);
871 : }
872 :
873 : static int
874 1 : sw_accel_module_init(void)
875 : {
876 1 : spdk_io_device_register(&g_sw_module, sw_accel_create_cb, sw_accel_destroy_cb,
877 : sizeof(struct sw_accel_io_channel), "sw_accel_module");
878 :
879 1 : return 0;
880 : }
881 :
882 : static void
883 1 : sw_accel_module_fini(void *ctxt)
884 : {
885 1 : spdk_io_device_unregister(&g_sw_module, NULL);
886 1 : spdk_accel_module_finish();
887 1 : }
888 :
889 : static int
890 2 : sw_accel_create_aes_xts(struct spdk_accel_crypto_key *key)
891 : {
892 : #ifdef SPDK_CONFIG_ISAL_CRYPTO
893 : struct sw_accel_crypto_key_data *key_data;
894 :
895 2 : key_data = calloc(1, sizeof(*key_data));
896 2 : if (!key_data) {
897 0 : return -ENOMEM;
898 : }
899 :
900 2 : switch (key->key_size) {
901 : case SPDK_ACCEL_AES_XTS_128_KEY_SIZE:
902 2 : key_data->encrypt = isal_aes_xts_enc_128;
903 2 : key_data->decrypt = isal_aes_xts_dec_128;
904 2 : break;
905 : case SPDK_ACCEL_AES_XTS_256_KEY_SIZE:
906 0 : key_data->encrypt = isal_aes_xts_enc_256;
907 0 : key_data->decrypt = isal_aes_xts_dec_256;
908 0 : break;
909 : default:
910 0 : assert(0);
911 : free(key_data);
912 : return -EINVAL;
913 : }
914 :
915 2 : key->priv = key_data;
916 :
917 2 : return 0;
918 : #else
919 : return -ENOTSUP;
920 : #endif
921 2 : }
922 :
923 : static int
924 2 : sw_accel_crypto_key_init(struct spdk_accel_crypto_key *key)
925 : {
926 2 : return sw_accel_create_aes_xts(key);
927 : }
928 :
929 : static void
930 2 : sw_accel_crypto_key_deinit(struct spdk_accel_crypto_key *key)
931 : {
932 2 : if (!key || key->module_if != &g_sw_module || !key->priv) {
933 0 : return;
934 : }
935 :
936 2 : free(key->priv);
937 2 : }
938 :
939 : static bool
940 2 : sw_accel_crypto_supports_tweak_mode(enum spdk_accel_crypto_tweak_mode tweak_mode)
941 : {
942 2 : return tweak_mode == SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA;
943 : }
944 :
945 : static bool
946 2 : sw_accel_crypto_supports_cipher(enum spdk_accel_cipher cipher, size_t key_size)
947 : {
948 2 : switch (cipher) {
949 : case SPDK_ACCEL_CIPHER_AES_XTS:
950 2 : return key_size == SPDK_ACCEL_AES_XTS_128_KEY_SIZE || key_size == SPDK_ACCEL_AES_XTS_256_KEY_SIZE;
951 : default:
952 0 : return false;
953 : }
954 2 : }
955 :
956 : static bool
957 11 : sw_accel_compress_supports_algo(enum spdk_accel_comp_algo algo)
958 : {
959 11 : switch (algo) {
960 : case SPDK_ACCEL_COMP_ALGO_DEFLATE:
961 : #ifdef SPDK_CONFIG_HAVE_LZ4
962 : case SPDK_ACCEL_COMP_ALGO_LZ4:
963 : #endif
964 11 : return true;
965 : default:
966 0 : return false;
967 : }
968 11 : }
969 :
970 : static int
971 0 : sw_accel_get_compress_level_range(enum spdk_accel_comp_algo algo,
972 : uint32_t *min_level, uint32_t *max_level)
973 : {
974 0 : switch (algo) {
975 : case SPDK_ACCEL_COMP_ALGO_DEFLATE:
976 : #ifdef SPDK_CONFIG_ISAL
977 0 : *min_level = COMP_DEFLATE_MIN_LEVEL;
978 0 : *max_level = COMP_DEFLATE_MAX_LEVEL;
979 0 : return 0;
980 : #else
981 : SPDK_ERRLOG("ISAL option is required to use software compression.\n");
982 : return -EINVAL;
983 : #endif
984 : case SPDK_ACCEL_COMP_ALGO_LZ4:
985 : #ifdef SPDK_CONFIG_HAVE_LZ4
986 0 : *min_level = 1;
987 0 : *max_level = 65537;
988 0 : return 0;
989 : #else
990 : SPDK_ERRLOG("LZ4 library is required to use software compression.\n");
991 : return -EINVAL;
992 : #endif
993 : default:
994 0 : return -EINVAL;
995 : }
996 0 : }
997 :
998 : static int
999 0 : sw_accel_get_operation_info(enum spdk_accel_opcode opcode,
1000 : const struct spdk_accel_operation_exec_ctx *ctx,
1001 : struct spdk_accel_opcode_info *info)
1002 : {
1003 0 : info->required_alignment = 0;
1004 :
1005 0 : return 0;
1006 : }
1007 :
1008 : static struct spdk_accel_module_if g_sw_module = {
1009 : .module_init = sw_accel_module_init,
1010 : .module_fini = sw_accel_module_fini,
1011 : .write_config_json = NULL,
1012 : .get_ctx_size = sw_accel_module_get_ctx_size,
1013 : .name = "software",
1014 : .priority = SPDK_ACCEL_SW_PRIORITY,
1015 : .supports_opcode = sw_accel_supports_opcode,
1016 : .get_io_channel = sw_accel_get_io_channel,
1017 : .submit_tasks = sw_accel_submit_tasks,
1018 : .crypto_key_init = sw_accel_crypto_key_init,
1019 : .crypto_key_deinit = sw_accel_crypto_key_deinit,
1020 : .crypto_supports_tweak_mode = sw_accel_crypto_supports_tweak_mode,
1021 : .crypto_supports_cipher = sw_accel_crypto_supports_cipher,
1022 : .compress_supports_algo = sw_accel_compress_supports_algo,
1023 : .get_compress_level_range = sw_accel_get_compress_level_range,
1024 : .get_operation_info = sw_accel_get_operation_info,
1025 : };
1026 :
1027 1 : SPDK_ACCEL_MODULE_REGISTER(sw, &g_sw_module)
|
