Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2020 Intel Corporation.
3 : * All rights reserved.
4 : */
5 :
6 : #include "spdk/stdinc.h"
7 :
8 : #include "spdk/env.h"
9 : #include "spdk/util.h"
10 : #include "spdk/memory.h"
11 : #include "spdk/likely.h"
12 :
13 : #include "spdk/log.h"
14 : #include "spdk_internal/idxd.h"
15 :
16 : #include "idxd_internal.h"
17 :
18 : #define ALIGN_4K 0x1000
19 : #define USERSPACE_DRIVER_NAME "user"
20 : #define KERNEL_DRIVER_NAME "kernel"
21 :
22 : /* The max number of completions processed per poll */
23 : #define IDXD_MAX_COMPLETIONS 128
24 :
25 : /* The minimum number of entries in batch per flush */
26 : #define IDXD_MIN_BATCH_FLUSH 32
27 :
28 : #define DATA_BLOCK_SIZE_512 512
29 : #define DATA_BLOCK_SIZE_520 520
30 : #define DATA_BLOCK_SIZE_4096 4096
31 : #define DATA_BLOCK_SIZE_4104 4104
32 :
33 : #define METADATA_SIZE_8 8
34 : #define METADATA_SIZE_16 16
35 :
36 : static STAILQ_HEAD(, spdk_idxd_impl) g_idxd_impls = STAILQ_HEAD_INITIALIZER(g_idxd_impls);
37 : static struct spdk_idxd_impl *g_idxd_impl;
38 :
39 : uint32_t
40 0 : spdk_idxd_get_socket(struct spdk_idxd_device *idxd)
41 : {
42 0 : return idxd->socket_id;
43 : }
44 :
45 : static inline void
46 0 : _submit_to_hw(struct spdk_idxd_io_channel *chan, struct idxd_ops *op)
47 : {
48 0 : STAILQ_INSERT_TAIL(&chan->ops_outstanding, op, link);
49 : /*
50 : * We must barrier before writing the descriptor to ensure that data
51 : * has been correctly flushed from the associated data buffers before DMA
52 : * operations begin.
53 : */
54 0 : _spdk_wmb();
55 0 : movdir64b(chan->portal + chan->portal_offset, op->desc);
56 0 : chan->portal_offset = (chan->portal_offset + chan->idxd->chan_per_device * PORTAL_STRIDE) &
57 : PORTAL_MASK;
58 0 : }
59 :
60 : inline static int
61 0 : _vtophys(struct spdk_idxd_io_channel *chan, const void *buf, uint64_t *buf_addr, uint64_t size)
62 : {
63 0 : uint64_t updated_size = size;
64 :
65 0 : if (chan->pasid_enabled) {
66 : /* We can just use virtual addresses */
67 0 : *buf_addr = (uint64_t)buf;
68 0 : return 0;
69 : }
70 :
71 0 : *buf_addr = spdk_vtophys(buf, &updated_size);
72 :
73 0 : if (*buf_addr == SPDK_VTOPHYS_ERROR) {
74 0 : SPDK_ERRLOG("Error translating address\n");
75 0 : return -EINVAL;
76 : }
77 :
78 0 : if (updated_size < size) {
79 0 : SPDK_ERRLOG("Error translating size (0x%lx), return size (0x%lx)\n", size, updated_size);
80 0 : return -EINVAL;
81 : }
82 :
83 0 : return 0;
84 0 : }
85 :
86 : struct idxd_vtophys_iter {
87 : const void *src;
88 : void *dst;
89 : uint64_t len;
90 :
91 : uint64_t offset;
92 :
93 : bool pasid_enabled;
94 : };
95 :
96 : static void
97 0 : idxd_vtophys_iter_init(struct spdk_idxd_io_channel *chan,
98 : struct idxd_vtophys_iter *iter,
99 : const void *src, void *dst, uint64_t len)
100 : {
101 0 : iter->src = src;
102 0 : iter->dst = dst;
103 0 : iter->len = len;
104 0 : iter->offset = 0;
105 0 : iter->pasid_enabled = chan->pasid_enabled;
106 0 : }
107 :
108 : static uint64_t
109 0 : idxd_vtophys_iter_next(struct idxd_vtophys_iter *iter,
110 : uint64_t *src_phys, uint64_t *dst_phys)
111 : {
112 : uint64_t src_off, dst_off, len;
113 : const void *src;
114 : void *dst;
115 :
116 0 : src = iter->src + iter->offset;
117 0 : dst = iter->dst + iter->offset;
118 :
119 0 : if (iter->offset == iter->len) {
120 0 : return 0;
121 : }
122 :
123 0 : if (iter->pasid_enabled) {
124 0 : *src_phys = (uint64_t)src;
125 0 : *dst_phys = (uint64_t)dst;
126 0 : return iter->len;
127 : }
128 :
129 0 : len = iter->len - iter->offset;
130 :
131 0 : src_off = len;
132 0 : *src_phys = spdk_vtophys(src, &src_off);
133 0 : if (*src_phys == SPDK_VTOPHYS_ERROR) {
134 0 : SPDK_ERRLOG("Error translating address\n");
135 0 : return SPDK_VTOPHYS_ERROR;
136 : }
137 :
138 0 : dst_off = len;
139 0 : *dst_phys = spdk_vtophys(dst, &dst_off);
140 0 : if (*dst_phys == SPDK_VTOPHYS_ERROR) {
141 0 : SPDK_ERRLOG("Error translating address\n");
142 0 : return SPDK_VTOPHYS_ERROR;
143 : }
144 :
145 0 : len = spdk_min(src_off, dst_off);
146 0 : iter->offset += len;
147 :
148 0 : return len;
149 0 : }
150 :
151 : /* helper function for DSA specific spdk_idxd_get_channel() stuff */
152 : static int
153 0 : _dsa_alloc_batches(struct spdk_idxd_io_channel *chan, int num_descriptors)
154 : {
155 : struct idxd_batch *batch;
156 : struct idxd_hw_desc *desc;
157 : struct idxd_ops *op;
158 0 : int i, j, num_batches, rc = -1;
159 :
160 : /* Allocate batches */
161 0 : num_batches = num_descriptors;
162 0 : chan->batch_base = calloc(num_batches, sizeof(struct idxd_batch));
163 0 : if (chan->batch_base == NULL) {
164 0 : SPDK_ERRLOG("Failed to allocate batch pool\n");
165 0 : return -ENOMEM;
166 : }
167 0 : batch = chan->batch_base;
168 0 : for (i = 0 ; i < num_batches ; i++) {
169 0 : batch->size = chan->idxd->batch_size;
170 0 : batch->user_desc = desc = spdk_zmalloc(batch->size * sizeof(struct idxd_hw_desc),
171 : 0x40, NULL,
172 : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
173 0 : if (batch->user_desc == NULL) {
174 0 : SPDK_ERRLOG("Failed to allocate batch descriptor memory\n");
175 0 : goto error_user;
176 : }
177 :
178 0 : rc = _vtophys(chan, batch->user_desc, &batch->user_desc_addr,
179 0 : batch->size * sizeof(struct idxd_hw_desc));
180 0 : if (rc) {
181 0 : SPDK_ERRLOG("Failed to translate batch descriptor memory\n");
182 0 : goto error_user;
183 : }
184 :
185 0 : batch->user_ops = op = spdk_zmalloc(batch->size * sizeof(struct idxd_ops),
186 : 0x40, NULL,
187 : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
188 0 : if (batch->user_ops == NULL) {
189 0 : SPDK_ERRLOG("Failed to allocate user completion memory\n");
190 0 : goto error_user;
191 : }
192 :
193 0 : for (j = 0; j < batch->size; j++) {
194 0 : rc = _vtophys(chan, &op->hw, &desc->completion_addr, sizeof(struct dsa_hw_comp_record));
195 0 : if (rc) {
196 0 : SPDK_ERRLOG("Failed to translate batch entry completion memory\n");
197 0 : goto error_user;
198 : }
199 0 : op++;
200 0 : desc++;
201 0 : }
202 0 : TAILQ_INSERT_TAIL(&chan->batch_pool, batch, link);
203 0 : batch++;
204 0 : }
205 0 : return 0;
206 :
207 : error_user:
208 0 : TAILQ_FOREACH(batch, &chan->batch_pool, link) {
209 0 : spdk_free(batch->user_ops);
210 0 : batch->user_ops = NULL;
211 0 : spdk_free(batch->user_desc);
212 0 : batch->user_desc = NULL;
213 0 : }
214 0 : return rc;
215 0 : }
216 :
217 : struct spdk_idxd_io_channel *
218 0 : spdk_idxd_get_channel(struct spdk_idxd_device *idxd)
219 : {
220 : struct spdk_idxd_io_channel *chan;
221 : struct idxd_hw_desc *desc;
222 : struct idxd_ops *op;
223 0 : int i, num_descriptors, rc = -1;
224 : uint32_t comp_rec_size;
225 :
226 0 : assert(idxd != NULL);
227 :
228 0 : chan = calloc(1, sizeof(struct spdk_idxd_io_channel));
229 0 : if (chan == NULL) {
230 0 : SPDK_ERRLOG("Failed to allocate idxd chan\n");
231 0 : return NULL;
232 : }
233 :
234 0 : chan->idxd = idxd;
235 0 : chan->pasid_enabled = idxd->pasid_enabled;
236 0 : STAILQ_INIT(&chan->ops_pool);
237 0 : TAILQ_INIT(&chan->batch_pool);
238 0 : STAILQ_INIT(&chan->ops_outstanding);
239 :
240 : /* Assign WQ, portal */
241 0 : pthread_mutex_lock(&idxd->num_channels_lock);
242 0 : if (idxd->num_channels == idxd->chan_per_device) {
243 : /* too many channels sharing this device */
244 0 : pthread_mutex_unlock(&idxd->num_channels_lock);
245 0 : SPDK_ERRLOG("Too many channels sharing this device\n");
246 0 : goto error;
247 : }
248 :
249 : /* Have each channel start at a different offset. */
250 0 : chan->portal = idxd->impl->portal_get_addr(idxd);
251 0 : chan->portal_offset = (idxd->num_channels * PORTAL_STRIDE) & PORTAL_MASK;
252 0 : idxd->num_channels++;
253 :
254 0 : pthread_mutex_unlock(&idxd->num_channels_lock);
255 :
256 : /* Allocate descriptors and completions */
257 0 : num_descriptors = idxd->total_wq_size / idxd->chan_per_device;
258 0 : chan->desc_base = desc = spdk_zmalloc(num_descriptors * sizeof(struct idxd_hw_desc),
259 : 0x40, NULL,
260 : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
261 0 : if (chan->desc_base == NULL) {
262 0 : SPDK_ERRLOG("Failed to allocate DSA descriptor memory\n");
263 0 : goto error;
264 : }
265 :
266 0 : chan->ops_base = op = spdk_zmalloc(num_descriptors * sizeof(struct idxd_ops),
267 : 0x40, NULL,
268 : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
269 0 : if (chan->ops_base == NULL) {
270 0 : SPDK_ERRLOG("Failed to allocate idxd_ops memory\n");
271 0 : goto error;
272 : }
273 :
274 0 : if (idxd->type == IDXD_DEV_TYPE_DSA) {
275 0 : comp_rec_size = sizeof(struct dsa_hw_comp_record);
276 0 : if (_dsa_alloc_batches(chan, num_descriptors)) {
277 0 : goto error;
278 : }
279 0 : } else {
280 0 : comp_rec_size = sizeof(struct iaa_hw_comp_record);
281 : }
282 :
283 0 : for (i = 0; i < num_descriptors; i++) {
284 0 : STAILQ_INSERT_TAIL(&chan->ops_pool, op, link);
285 0 : op->desc = desc;
286 0 : rc = _vtophys(chan, &op->hw, &desc->completion_addr, comp_rec_size);
287 0 : if (rc) {
288 0 : SPDK_ERRLOG("Failed to translate completion memory\n");
289 0 : goto error;
290 : }
291 0 : op++;
292 0 : desc++;
293 0 : }
294 :
295 0 : return chan;
296 :
297 : error:
298 0 : spdk_free(chan->ops_base);
299 0 : chan->ops_base = NULL;
300 0 : spdk_free(chan->desc_base);
301 0 : chan->desc_base = NULL;
302 0 : free(chan);
303 0 : return NULL;
304 0 : }
305 :
306 : static int idxd_batch_cancel(struct spdk_idxd_io_channel *chan, int status);
307 :
308 : void
309 0 : spdk_idxd_put_channel(struct spdk_idxd_io_channel *chan)
310 : {
311 : struct idxd_batch *batch;
312 :
313 0 : assert(chan != NULL);
314 0 : assert(chan->idxd != NULL);
315 :
316 0 : if (chan->batch) {
317 0 : idxd_batch_cancel(chan, -ECANCELED);
318 0 : }
319 :
320 0 : pthread_mutex_lock(&chan->idxd->num_channels_lock);
321 0 : assert(chan->idxd->num_channels > 0);
322 0 : chan->idxd->num_channels--;
323 0 : pthread_mutex_unlock(&chan->idxd->num_channels_lock);
324 :
325 0 : spdk_free(chan->ops_base);
326 0 : spdk_free(chan->desc_base);
327 0 : while ((batch = TAILQ_FIRST(&chan->batch_pool))) {
328 0 : TAILQ_REMOVE(&chan->batch_pool, batch, link);
329 0 : spdk_free(batch->user_ops);
330 0 : spdk_free(batch->user_desc);
331 : }
332 0 : free(chan->batch_base);
333 0 : free(chan);
334 0 : }
335 :
336 : static inline struct spdk_idxd_impl *
337 0 : idxd_get_impl_by_name(const char *impl_name)
338 : {
339 : struct spdk_idxd_impl *impl;
340 :
341 0 : assert(impl_name != NULL);
342 0 : STAILQ_FOREACH(impl, &g_idxd_impls, link) {
343 0 : if (0 == strcmp(impl_name, impl->name)) {
344 0 : return impl;
345 : }
346 0 : }
347 :
348 0 : return NULL;
349 0 : }
350 :
351 : int
352 0 : spdk_idxd_set_config(bool kernel_mode)
353 : {
354 : struct spdk_idxd_impl *tmp;
355 :
356 0 : if (kernel_mode) {
357 0 : tmp = idxd_get_impl_by_name(KERNEL_DRIVER_NAME);
358 0 : } else {
359 0 : tmp = idxd_get_impl_by_name(USERSPACE_DRIVER_NAME);
360 : }
361 :
362 0 : if (g_idxd_impl != NULL && g_idxd_impl != tmp) {
363 0 : SPDK_ERRLOG("Cannot change idxd implementation after devices are initialized\n");
364 0 : assert(false);
365 : return -EALREADY;
366 : }
367 0 : g_idxd_impl = tmp;
368 :
369 0 : if (g_idxd_impl == NULL) {
370 0 : SPDK_ERRLOG("Cannot set the idxd implementation with %s mode\n",
371 : kernel_mode ? KERNEL_DRIVER_NAME : USERSPACE_DRIVER_NAME);
372 0 : return -EINVAL;
373 : }
374 :
375 0 : return 0;
376 0 : }
377 :
378 : static void
379 0 : idxd_device_destruct(struct spdk_idxd_device *idxd)
380 : {
381 0 : assert(idxd->impl != NULL);
382 :
383 0 : idxd->impl->destruct(idxd);
384 0 : }
385 :
386 : int
387 0 : spdk_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb,
388 : spdk_idxd_probe_cb probe_cb)
389 : {
390 0 : if (g_idxd_impl == NULL) {
391 0 : SPDK_ERRLOG("No idxd impl is selected\n");
392 0 : return -1;
393 : }
394 :
395 0 : return g_idxd_impl->probe(cb_ctx, attach_cb, probe_cb);
396 0 : }
397 :
398 : void
399 0 : spdk_idxd_detach(struct spdk_idxd_device *idxd)
400 : {
401 0 : assert(idxd != NULL);
402 0 : idxd_device_destruct(idxd);
403 0 : }
404 :
405 : static int
406 0 : _idxd_prep_command(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn, void *cb_arg,
407 : int flags, struct idxd_hw_desc **_desc, struct idxd_ops **_op)
408 : {
409 : struct idxd_hw_desc *desc;
410 : struct idxd_ops *op;
411 : uint64_t comp_addr;
412 :
413 0 : if (!STAILQ_EMPTY(&chan->ops_pool)) {
414 0 : op = *_op = STAILQ_FIRST(&chan->ops_pool);
415 0 : desc = *_desc = op->desc;
416 0 : comp_addr = desc->completion_addr;
417 0 : memset(desc, 0, sizeof(*desc));
418 0 : desc->completion_addr = comp_addr;
419 0 : STAILQ_REMOVE_HEAD(&chan->ops_pool, link);
420 0 : } else {
421 : /* The application needs to handle this, violation of flow control */
422 0 : return -EBUSY;
423 : }
424 :
425 0 : flags |= IDXD_FLAG_COMPLETION_ADDR_VALID;
426 0 : flags |= IDXD_FLAG_REQUEST_COMPLETION;
427 :
428 0 : desc->flags = flags;
429 0 : op->cb_arg = cb_arg;
430 0 : op->cb_fn = cb_fn;
431 0 : op->batch = NULL;
432 0 : op->parent = NULL;
433 0 : op->count = 1;
434 :
435 0 : return 0;
436 0 : }
437 :
438 : static int
439 0 : _idxd_prep_batch_cmd(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn,
440 : void *cb_arg, int flags,
441 : struct idxd_hw_desc **_desc, struct idxd_ops **_op)
442 : {
443 : struct idxd_hw_desc *desc;
444 : struct idxd_ops *op;
445 : uint64_t comp_addr;
446 : struct idxd_batch *batch;
447 :
448 0 : batch = chan->batch;
449 :
450 0 : assert(batch != NULL);
451 0 : if (batch->index == batch->size) {
452 0 : return -EBUSY;
453 : }
454 :
455 0 : desc = *_desc = &batch->user_desc[batch->index];
456 0 : op = *_op = &batch->user_ops[batch->index];
457 :
458 0 : op->desc = desc;
459 0 : SPDK_DEBUGLOG(idxd, "Prep batch %p index %u\n", batch, batch->index);
460 :
461 0 : batch->index++;
462 :
463 0 : comp_addr = desc->completion_addr;
464 0 : memset(desc, 0, sizeof(*desc));
465 0 : desc->completion_addr = comp_addr;
466 0 : flags |= IDXD_FLAG_COMPLETION_ADDR_VALID;
467 0 : flags |= IDXD_FLAG_REQUEST_COMPLETION;
468 0 : desc->flags = flags;
469 0 : op->cb_arg = cb_arg;
470 0 : op->cb_fn = cb_fn;
471 0 : op->batch = batch;
472 0 : op->parent = NULL;
473 0 : op->count = 1;
474 0 : op->crc_dst = NULL;
475 :
476 0 : return 0;
477 0 : }
478 :
479 : static struct idxd_batch *
480 0 : idxd_batch_create(struct spdk_idxd_io_channel *chan)
481 : {
482 : struct idxd_batch *batch;
483 :
484 0 : assert(chan != NULL);
485 0 : assert(chan->batch == NULL);
486 :
487 0 : if (!TAILQ_EMPTY(&chan->batch_pool)) {
488 0 : batch = TAILQ_FIRST(&chan->batch_pool);
489 0 : batch->index = 0;
490 0 : batch->chan = chan;
491 0 : chan->batch = batch;
492 0 : TAILQ_REMOVE(&chan->batch_pool, batch, link);
493 0 : } else {
494 : /* The application needs to handle this. */
495 0 : return NULL;
496 : }
497 :
498 0 : return batch;
499 0 : }
500 :
501 : static void
502 0 : _free_batch(struct idxd_batch *batch, struct spdk_idxd_io_channel *chan)
503 : {
504 0 : SPDK_DEBUGLOG(idxd, "Free batch %p\n", batch);
505 0 : assert(batch->refcnt == 0);
506 0 : batch->index = 0;
507 0 : batch->chan = NULL;
508 0 : TAILQ_INSERT_TAIL(&chan->batch_pool, batch, link);
509 0 : }
510 :
511 : static int
512 0 : idxd_batch_cancel(struct spdk_idxd_io_channel *chan, int status)
513 : {
514 : struct idxd_ops *op;
515 : struct idxd_batch *batch;
516 : int i;
517 :
518 0 : assert(chan != NULL);
519 :
520 0 : batch = chan->batch;
521 0 : assert(batch != NULL);
522 :
523 0 : if (batch->index == UINT16_MAX) {
524 0 : SPDK_ERRLOG("Cannot cancel batch, already submitted to HW.\n");
525 0 : return -EINVAL;
526 : }
527 :
528 0 : chan->batch = NULL;
529 :
530 0 : for (i = 0; i < batch->index; i++) {
531 0 : op = &batch->user_ops[i];
532 0 : if (op->cb_fn) {
533 0 : op->cb_fn(op->cb_arg, status);
534 0 : }
535 0 : }
536 :
537 0 : _free_batch(batch, chan);
538 :
539 0 : return 0;
540 0 : }
541 :
542 : static int
543 0 : idxd_batch_submit(struct spdk_idxd_io_channel *chan,
544 : spdk_idxd_req_cb cb_fn, void *cb_arg)
545 : {
546 : struct idxd_hw_desc *desc;
547 : struct idxd_batch *batch;
548 : struct idxd_ops *op;
549 0 : int i, rc, flags = 0;
550 :
551 0 : assert(chan != NULL);
552 :
553 0 : batch = chan->batch;
554 0 : assert(batch != NULL);
555 :
556 0 : if (batch->index == 0) {
557 0 : return idxd_batch_cancel(chan, 0);
558 : }
559 :
560 : /* Common prep. */
561 0 : rc = _idxd_prep_command(chan, cb_fn, cb_arg, flags, &desc, &op);
562 0 : if (rc) {
563 0 : return rc;
564 : }
565 :
566 0 : if (batch->index == 1) {
567 : uint64_t completion_addr;
568 :
569 : /* If there's only one command, convert it away from a batch. */
570 0 : completion_addr = desc->completion_addr;
571 0 : memcpy(desc, &batch->user_desc[0], sizeof(*desc));
572 0 : desc->completion_addr = completion_addr;
573 0 : op->cb_fn = batch->user_ops[0].cb_fn;
574 0 : op->cb_arg = batch->user_ops[0].cb_arg;
575 0 : op->crc_dst = batch->user_ops[0].crc_dst;
576 0 : _free_batch(batch, chan);
577 0 : } else {
578 : /* Command specific. */
579 0 : desc->opcode = IDXD_OPCODE_BATCH;
580 0 : desc->desc_list_addr = batch->user_desc_addr;
581 0 : desc->desc_count = batch->index;
582 0 : assert(batch->index <= batch->size);
583 :
584 : /* Add the batch elements completion contexts to the outstanding list to be polled. */
585 0 : for (i = 0 ; i < batch->index; i++) {
586 0 : batch->refcnt++;
587 0 : STAILQ_INSERT_TAIL(&chan->ops_outstanding, (struct idxd_ops *)&batch->user_ops[i],
588 : link);
589 0 : }
590 0 : batch->index = UINT16_MAX;
591 : }
592 :
593 0 : chan->batch = NULL;
594 :
595 : /* Submit operation. */
596 0 : _submit_to_hw(chan, op);
597 0 : SPDK_DEBUGLOG(idxd, "Submitted batch %p\n", batch);
598 :
599 0 : return 0;
600 0 : }
601 :
602 : static int
603 0 : _idxd_setup_batch(struct spdk_idxd_io_channel *chan)
604 : {
605 : struct idxd_batch *batch;
606 :
607 0 : if (chan->batch == NULL) {
608 0 : batch = idxd_batch_create(chan);
609 0 : if (batch == NULL) {
610 0 : return -EBUSY;
611 : }
612 0 : }
613 :
614 0 : return 0;
615 0 : }
616 :
617 : static int
618 0 : _idxd_flush_batch(struct spdk_idxd_io_channel *chan)
619 : {
620 0 : struct idxd_batch *batch = chan->batch;
621 : int rc;
622 :
623 0 : if (batch != NULL && batch->index >= IDXD_MIN_BATCH_FLUSH) {
624 : /* Close out the full batch */
625 0 : rc = idxd_batch_submit(chan, NULL, NULL);
626 0 : if (rc) {
627 0 : assert(rc == -EBUSY);
628 : /*
629 : * Return 0. This will get re-submitted within idxd_process_events where
630 : * if it fails, it will get correctly aborted.
631 : */
632 0 : return 0;
633 : }
634 0 : }
635 :
636 0 : return 0;
637 0 : }
638 :
639 : static inline void
640 0 : _update_write_flags(struct spdk_idxd_io_channel *chan, struct idxd_hw_desc *desc)
641 : {
642 0 : desc->flags ^= IDXD_FLAG_CACHE_CONTROL;
643 0 : }
644 :
645 : int
646 0 : spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan,
647 : struct iovec *diov, uint32_t diovcnt,
648 : struct iovec *siov, uint32_t siovcnt,
649 : int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)
650 : {
651 : struct idxd_hw_desc *desc;
652 : struct idxd_ops *first_op, *op;
653 : void *src, *dst;
654 : uint64_t src_addr, dst_addr;
655 : int rc, count;
656 : uint64_t len, seg_len;
657 : struct spdk_ioviter iter;
658 : struct idxd_vtophys_iter vtophys_iter;
659 :
660 0 : assert(chan != NULL);
661 0 : assert(diov != NULL);
662 0 : assert(siov != NULL);
663 :
664 0 : rc = _idxd_setup_batch(chan);
665 0 : if (rc) {
666 0 : return rc;
667 : }
668 :
669 0 : count = 0;
670 0 : first_op = NULL;
671 0 : for (len = spdk_ioviter_first(&iter, siov, siovcnt, diov, diovcnt, &src, &dst);
672 0 : len > 0;
673 0 : len = spdk_ioviter_next(&iter, &src, &dst)) {
674 :
675 0 : idxd_vtophys_iter_init(chan, &vtophys_iter, src, dst, len);
676 :
677 0 : while (len > 0) {
678 0 : if (first_op == NULL) {
679 0 : rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, flags, &desc, &op);
680 0 : if (rc) {
681 0 : goto error;
682 : }
683 :
684 0 : first_op = op;
685 0 : } else {
686 0 : rc = _idxd_prep_batch_cmd(chan, NULL, NULL, flags, &desc, &op);
687 0 : if (rc) {
688 0 : goto error;
689 : }
690 :
691 0 : first_op->count++;
692 0 : op->parent = first_op;
693 : }
694 :
695 0 : count++;
696 :
697 0 : src_addr = 0;
698 0 : dst_addr = 0;
699 0 : seg_len = idxd_vtophys_iter_next(&vtophys_iter, &src_addr, &dst_addr);
700 0 : if (seg_len == SPDK_VTOPHYS_ERROR) {
701 0 : rc = -EFAULT;
702 0 : goto error;
703 : }
704 :
705 0 : desc->opcode = IDXD_OPCODE_MEMMOVE;
706 0 : desc->src_addr = src_addr;
707 0 : desc->dst_addr = dst_addr;
708 0 : desc->xfer_size = seg_len;
709 0 : _update_write_flags(chan, desc);
710 :
711 0 : len -= seg_len;
712 : }
713 0 : }
714 :
715 0 : return _idxd_flush_batch(chan);
716 :
717 : error:
718 0 : chan->batch->index -= count;
719 0 : return rc;
720 0 : }
721 :
722 : /* Dual-cast copies the same source to two separate destination buffers. */
723 : int
724 0 : spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, void *dst1, void *dst2,
725 : const void *src, uint64_t nbytes, int flags,
726 : spdk_idxd_req_cb cb_fn, void *cb_arg)
727 : {
728 : struct idxd_hw_desc *desc;
729 : struct idxd_ops *first_op, *op;
730 : uint64_t src_addr, dst1_addr, dst2_addr;
731 : int rc, count;
732 : uint64_t len;
733 : uint64_t outer_seg_len, inner_seg_len;
734 : struct idxd_vtophys_iter iter_outer, iter_inner;
735 :
736 0 : assert(chan != NULL);
737 0 : assert(dst1 != NULL);
738 0 : assert(dst2 != NULL);
739 0 : assert(src != NULL);
740 :
741 0 : if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {
742 0 : SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
743 0 : return -EINVAL;
744 : }
745 :
746 0 : rc = _idxd_setup_batch(chan);
747 0 : if (rc) {
748 0 : return rc;
749 : }
750 :
751 0 : idxd_vtophys_iter_init(chan, &iter_outer, src, dst1, nbytes);
752 :
753 0 : first_op = NULL;
754 0 : count = 0;
755 0 : while (nbytes > 0) {
756 0 : src_addr = 0;
757 0 : dst1_addr = 0;
758 0 : outer_seg_len = idxd_vtophys_iter_next(&iter_outer, &src_addr, &dst1_addr);
759 0 : if (outer_seg_len == SPDK_VTOPHYS_ERROR) {
760 0 : goto error;
761 : }
762 :
763 0 : idxd_vtophys_iter_init(chan, &iter_inner, src, dst2, nbytes);
764 :
765 0 : src += outer_seg_len;
766 0 : nbytes -= outer_seg_len;
767 :
768 0 : while (outer_seg_len > 0) {
769 0 : if (first_op == NULL) {
770 0 : rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, flags, &desc, &op);
771 0 : if (rc) {
772 0 : goto error;
773 : }
774 :
775 0 : first_op = op;
776 0 : } else {
777 0 : rc = _idxd_prep_batch_cmd(chan, NULL, NULL, flags, &desc, &op);
778 0 : if (rc) {
779 0 : goto error;
780 : }
781 :
782 0 : first_op->count++;
783 0 : op->parent = first_op;
784 : }
785 :
786 0 : count++;
787 :
788 0 : src_addr = 0;
789 0 : dst2_addr = 0;
790 0 : inner_seg_len = idxd_vtophys_iter_next(&iter_inner, &src_addr, &dst2_addr);
791 0 : if (inner_seg_len == SPDK_VTOPHYS_ERROR) {
792 0 : rc = -EFAULT;
793 0 : goto error;
794 : }
795 :
796 0 : len = spdk_min(outer_seg_len, inner_seg_len);
797 :
798 : /* Command specific. */
799 0 : desc->opcode = IDXD_OPCODE_DUALCAST;
800 0 : desc->src_addr = src_addr;
801 0 : desc->dst_addr = dst1_addr;
802 0 : desc->dest2 = dst2_addr;
803 0 : desc->xfer_size = len;
804 0 : _update_write_flags(chan, desc);
805 :
806 0 : dst1_addr += len;
807 0 : outer_seg_len -= len;
808 : }
809 : }
810 :
811 0 : return _idxd_flush_batch(chan);
812 :
813 : error:
814 0 : chan->batch->index -= count;
815 0 : return rc;
816 0 : }
817 :
818 : int
819 0 : spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan,
820 : struct iovec *siov1, size_t siov1cnt,
821 : struct iovec *siov2, size_t siov2cnt,
822 : int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)
823 : {
824 :
825 : struct idxd_hw_desc *desc;
826 : struct idxd_ops *first_op, *op;
827 : void *src1, *src2;
828 : uint64_t src1_addr, src2_addr;
829 : int rc, count;
830 : uint64_t len, seg_len;
831 : struct spdk_ioviter iter;
832 : struct idxd_vtophys_iter vtophys_iter;
833 :
834 0 : assert(chan != NULL);
835 0 : assert(siov1 != NULL);
836 0 : assert(siov2 != NULL);
837 :
838 0 : rc = _idxd_setup_batch(chan);
839 0 : if (rc) {
840 0 : return rc;
841 : }
842 :
843 0 : count = 0;
844 0 : first_op = NULL;
845 0 : for (len = spdk_ioviter_first(&iter, siov1, siov1cnt, siov2, siov2cnt, &src1, &src2);
846 0 : len > 0;
847 0 : len = spdk_ioviter_next(&iter, &src1, &src2)) {
848 :
849 0 : idxd_vtophys_iter_init(chan, &vtophys_iter, src1, src2, len);
850 :
851 0 : while (len > 0) {
852 0 : if (first_op == NULL) {
853 0 : rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, flags, &desc, &op);
854 0 : if (rc) {
855 0 : goto error;
856 : }
857 :
858 0 : first_op = op;
859 0 : } else {
860 0 : rc = _idxd_prep_batch_cmd(chan, NULL, NULL, flags, &desc, &op);
861 0 : if (rc) {
862 0 : goto error;
863 : }
864 :
865 0 : first_op->count++;
866 0 : op->parent = first_op;
867 : }
868 :
869 0 : count++;
870 :
871 0 : src1_addr = 0;
872 0 : src2_addr = 0;
873 0 : seg_len = idxd_vtophys_iter_next(&vtophys_iter, &src1_addr, &src2_addr);
874 0 : if (seg_len == SPDK_VTOPHYS_ERROR) {
875 0 : rc = -EFAULT;
876 0 : goto error;
877 : }
878 :
879 0 : desc->opcode = IDXD_OPCODE_COMPARE;
880 0 : desc->src_addr = src1_addr;
881 0 : desc->src2_addr = src2_addr;
882 0 : desc->xfer_size = seg_len;
883 :
884 0 : len -= seg_len;
885 : }
886 0 : }
887 :
888 0 : return _idxd_flush_batch(chan);
889 :
890 : error:
891 0 : chan->batch->index -= count;
892 0 : return rc;
893 0 : }
894 :
895 : int
896 0 : spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan,
897 : struct iovec *diov, size_t diovcnt,
898 : uint64_t fill_pattern, int flags,
899 : spdk_idxd_req_cb cb_fn, void *cb_arg)
900 : {
901 : struct idxd_hw_desc *desc;
902 : struct idxd_ops *first_op, *op;
903 : uint64_t dst_addr;
904 : int rc, count;
905 : uint64_t len, seg_len;
906 : void *dst;
907 : size_t i;
908 :
909 0 : assert(chan != NULL);
910 0 : assert(diov != NULL);
911 :
912 0 : rc = _idxd_setup_batch(chan);
913 0 : if (rc) {
914 0 : return rc;
915 : }
916 :
917 0 : count = 0;
918 0 : first_op = NULL;
919 0 : for (i = 0; i < diovcnt; i++) {
920 0 : len = diov[i].iov_len;
921 0 : dst = diov[i].iov_base;
922 :
923 0 : while (len > 0) {
924 0 : if (first_op == NULL) {
925 0 : rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, flags, &desc, &op);
926 0 : if (rc) {
927 0 : goto error;
928 : }
929 :
930 0 : first_op = op;
931 0 : } else {
932 0 : rc = _idxd_prep_batch_cmd(chan, NULL, NULL, flags, &desc, &op);
933 0 : if (rc) {
934 0 : goto error;
935 : }
936 :
937 0 : first_op->count++;
938 0 : op->parent = first_op;
939 : }
940 :
941 0 : count++;
942 :
943 0 : seg_len = len;
944 0 : if (chan->pasid_enabled) {
945 0 : dst_addr = (uint64_t)dst;
946 0 : } else {
947 0 : dst_addr = spdk_vtophys(dst, &seg_len);
948 0 : if (dst_addr == SPDK_VTOPHYS_ERROR) {
949 0 : SPDK_ERRLOG("Error translating address\n");
950 0 : rc = -EFAULT;
951 0 : goto error;
952 : }
953 : }
954 :
955 0 : seg_len = spdk_min(seg_len, len);
956 :
957 0 : desc->opcode = IDXD_OPCODE_MEMFILL;
958 0 : desc->pattern = fill_pattern;
959 0 : desc->dst_addr = dst_addr;
960 0 : desc->xfer_size = seg_len;
961 0 : _update_write_flags(chan, desc);
962 :
963 0 : len -= seg_len;
964 0 : dst += seg_len;
965 : }
966 0 : }
967 :
968 0 : return _idxd_flush_batch(chan);
969 :
970 : error:
971 0 : chan->batch->index -= count;
972 0 : return rc;
973 0 : }
974 :
975 : int
976 0 : spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan,
977 : struct iovec *siov, size_t siovcnt,
978 : uint32_t seed, uint32_t *crc_dst, int flags,
979 : spdk_idxd_req_cb cb_fn, void *cb_arg)
980 : {
981 : struct idxd_hw_desc *desc;
982 : struct idxd_ops *first_op, *op;
983 : uint64_t src_addr;
984 : int rc, count;
985 : uint64_t len, seg_len;
986 : void *src;
987 : size_t i;
988 0 : uint64_t prev_crc = 0;
989 :
990 0 : assert(chan != NULL);
991 0 : assert(siov != NULL);
992 :
993 0 : rc = _idxd_setup_batch(chan);
994 0 : if (rc) {
995 0 : return rc;
996 : }
997 :
998 0 : count = 0;
999 0 : op = NULL;
1000 0 : first_op = NULL;
1001 0 : for (i = 0; i < siovcnt; i++) {
1002 0 : len = siov[i].iov_len;
1003 0 : src = siov[i].iov_base;
1004 :
1005 0 : while (len > 0) {
1006 0 : if (first_op == NULL) {
1007 0 : rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, flags, &desc, &op);
1008 0 : if (rc) {
1009 0 : goto error;
1010 : }
1011 :
1012 0 : first_op = op;
1013 0 : } else {
1014 0 : rc = _idxd_prep_batch_cmd(chan, NULL, NULL, flags, &desc, &op);
1015 0 : if (rc) {
1016 0 : goto error;
1017 : }
1018 :
1019 0 : first_op->count++;
1020 0 : op->parent = first_op;
1021 : }
1022 :
1023 0 : count++;
1024 :
1025 0 : seg_len = len;
1026 0 : if (chan->pasid_enabled) {
1027 0 : src_addr = (uint64_t)src;
1028 0 : } else {
1029 0 : src_addr = spdk_vtophys(src, &seg_len);
1030 0 : if (src_addr == SPDK_VTOPHYS_ERROR) {
1031 0 : SPDK_ERRLOG("Error translating address\n");
1032 0 : rc = -EFAULT;
1033 0 : goto error;
1034 : }
1035 : }
1036 :
1037 0 : seg_len = spdk_min(seg_len, len);
1038 :
1039 0 : desc->opcode = IDXD_OPCODE_CRC32C_GEN;
1040 0 : desc->src_addr = src_addr;
1041 0 : if (op == first_op) {
1042 0 : desc->crc32c.seed = seed;
1043 0 : } else {
1044 0 : desc->flags |= IDXD_FLAG_FENCE | IDXD_FLAG_CRC_READ_CRC_SEED;
1045 0 : desc->crc32c.addr = prev_crc;
1046 : }
1047 :
1048 0 : desc->xfer_size = seg_len;
1049 0 : prev_crc = desc->completion_addr + offsetof(struct dsa_hw_comp_record, crc32c_val);
1050 :
1051 0 : len -= seg_len;
1052 0 : src += seg_len;
1053 : }
1054 0 : }
1055 :
1056 : /* Only the last op copies the crc to the destination */
1057 0 : if (op) {
1058 0 : op->crc_dst = crc_dst;
1059 0 : }
1060 :
1061 0 : return _idxd_flush_batch(chan);
1062 :
1063 : error:
1064 0 : chan->batch->index -= count;
1065 0 : return rc;
1066 0 : }
1067 :
1068 : int
1069 0 : spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan,
1070 : struct iovec *diov, size_t diovcnt,
1071 : struct iovec *siov, size_t siovcnt,
1072 : uint32_t seed, uint32_t *crc_dst, int flags,
1073 : spdk_idxd_req_cb cb_fn, void *cb_arg)
1074 : {
1075 : struct idxd_hw_desc *desc;
1076 : struct idxd_ops *first_op, *op;
1077 : void *src, *dst;
1078 : uint64_t src_addr, dst_addr;
1079 : int rc, count;
1080 : uint64_t len, seg_len;
1081 : struct spdk_ioviter iter;
1082 : struct idxd_vtophys_iter vtophys_iter;
1083 0 : uint64_t prev_crc = 0;
1084 :
1085 0 : assert(chan != NULL);
1086 0 : assert(diov != NULL);
1087 0 : assert(siov != NULL);
1088 :
1089 0 : rc = _idxd_setup_batch(chan);
1090 0 : if (rc) {
1091 0 : return rc;
1092 : }
1093 :
1094 0 : count = 0;
1095 0 : op = NULL;
1096 0 : first_op = NULL;
1097 0 : for (len = spdk_ioviter_first(&iter, siov, siovcnt, diov, diovcnt, &src, &dst);
1098 0 : len > 0;
1099 0 : len = spdk_ioviter_next(&iter, &src, &dst)) {
1100 :
1101 :
1102 0 : idxd_vtophys_iter_init(chan, &vtophys_iter, src, dst, len);
1103 :
1104 0 : while (len > 0) {
1105 0 : if (first_op == NULL) {
1106 0 : rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, flags, &desc, &op);
1107 0 : if (rc) {
1108 0 : goto error;
1109 : }
1110 :
1111 0 : first_op = op;
1112 0 : } else {
1113 0 : rc = _idxd_prep_batch_cmd(chan, NULL, NULL, flags, &desc, &op);
1114 0 : if (rc) {
1115 0 : goto error;
1116 : }
1117 :
1118 0 : first_op->count++;
1119 0 : op->parent = first_op;
1120 : }
1121 :
1122 0 : count++;
1123 :
1124 0 : src_addr = 0;
1125 0 : dst_addr = 0;
1126 0 : seg_len = idxd_vtophys_iter_next(&vtophys_iter, &src_addr, &dst_addr);
1127 0 : if (seg_len == SPDK_VTOPHYS_ERROR) {
1128 0 : rc = -EFAULT;
1129 0 : goto error;
1130 : }
1131 :
1132 0 : desc->opcode = IDXD_OPCODE_COPY_CRC;
1133 0 : desc->dst_addr = dst_addr;
1134 0 : desc->src_addr = src_addr;
1135 0 : _update_write_flags(chan, desc);
1136 0 : if (op == first_op) {
1137 0 : desc->crc32c.seed = seed;
1138 0 : } else {
1139 0 : desc->flags |= IDXD_FLAG_FENCE | IDXD_FLAG_CRC_READ_CRC_SEED;
1140 0 : desc->crc32c.addr = prev_crc;
1141 : }
1142 :
1143 0 : desc->xfer_size = seg_len;
1144 0 : prev_crc = desc->completion_addr + offsetof(struct dsa_hw_comp_record, crc32c_val);
1145 :
1146 0 : len -= seg_len;
1147 : }
1148 0 : }
1149 :
1150 : /* Only the last op copies the crc to the destination */
1151 0 : if (op) {
1152 0 : op->crc_dst = crc_dst;
1153 0 : }
1154 :
1155 0 : return _idxd_flush_batch(chan);
1156 :
1157 : error:
1158 0 : chan->batch->index -= count;
1159 0 : return rc;
1160 0 : }
1161 :
1162 : static inline int
1163 0 : _idxd_submit_compress_single(struct spdk_idxd_io_channel *chan, void *dst, const void *src,
1164 : uint64_t nbytes_dst, uint64_t nbytes_src, uint32_t *output_size,
1165 : int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)
1166 : {
1167 : struct idxd_hw_desc *desc;
1168 : struct idxd_ops *op;
1169 : uint64_t src_addr, dst_addr;
1170 : int rc;
1171 :
1172 : /* Common prep. */
1173 0 : rc = _idxd_prep_command(chan, cb_fn, cb_arg, flags, &desc, &op);
1174 0 : if (rc) {
1175 0 : return rc;
1176 : }
1177 :
1178 0 : rc = _vtophys(chan, src, &src_addr, nbytes_src);
1179 0 : if (rc) {
1180 0 : goto error;
1181 : }
1182 :
1183 0 : rc = _vtophys(chan, dst, &dst_addr, nbytes_dst);
1184 0 : if (rc) {
1185 0 : goto error;
1186 : }
1187 :
1188 : /* Command specific. */
1189 0 : desc->opcode = IDXD_OPCODE_COMPRESS;
1190 0 : desc->src1_addr = src_addr;
1191 0 : desc->dst_addr = dst_addr;
1192 0 : desc->src1_size = nbytes_src;
1193 0 : desc->iaa.max_dst_size = nbytes_dst;
1194 0 : desc->iaa.src2_size = sizeof(struct iaa_aecs);
1195 0 : desc->iaa.src2_addr = chan->idxd->aecs_addr;
1196 0 : desc->flags |= IAA_FLAG_RD_SRC2_AECS;
1197 0 : desc->compr_flags = IAA_COMP_FLAGS;
1198 0 : op->output_size = output_size;
1199 :
1200 0 : _submit_to_hw(chan, op);
1201 0 : return 0;
1202 : error:
1203 0 : STAILQ_INSERT_TAIL(&chan->ops_pool, op, link);
1204 0 : return rc;
1205 0 : }
1206 :
1207 : int
1208 0 : spdk_idxd_submit_compress(struct spdk_idxd_io_channel *chan,
1209 : void *dst, uint64_t nbytes,
1210 : struct iovec *siov, uint32_t siovcnt, uint32_t *output_size,
1211 : int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)
1212 : {
1213 0 : assert(chan != NULL);
1214 0 : assert(dst != NULL);
1215 0 : assert(siov != NULL);
1216 :
1217 0 : if (siovcnt == 1) {
1218 : /* Simple case - copying one buffer to another */
1219 0 : if (nbytes < siov[0].iov_len) {
1220 0 : return -EINVAL;
1221 : }
1222 :
1223 0 : return _idxd_submit_compress_single(chan, dst, siov[0].iov_base,
1224 0 : nbytes, siov[0].iov_len,
1225 0 : output_size, flags, cb_fn, cb_arg);
1226 : }
1227 : /* TODO: vectored support */
1228 0 : return -EINVAL;
1229 0 : }
1230 :
1231 : static inline int
1232 0 : _idxd_submit_decompress_single(struct spdk_idxd_io_channel *chan, void *dst, const void *src,
1233 : uint64_t nbytes_dst, uint64_t nbytes, int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)
1234 : {
1235 : struct idxd_hw_desc *desc;
1236 : struct idxd_ops *op;
1237 : uint64_t src_addr, dst_addr;
1238 : int rc;
1239 :
1240 : /* Common prep. */
1241 0 : rc = _idxd_prep_command(chan, cb_fn, cb_arg, flags, &desc, &op);
1242 0 : if (rc) {
1243 0 : return rc;
1244 : }
1245 :
1246 0 : rc = _vtophys(chan, src, &src_addr, nbytes);
1247 0 : if (rc) {
1248 0 : goto error;
1249 : }
1250 :
1251 0 : rc = _vtophys(chan, dst, &dst_addr, nbytes_dst);
1252 0 : if (rc) {
1253 0 : goto error;
1254 : }
1255 :
1256 : /* Command specific. */
1257 0 : desc->opcode = IDXD_OPCODE_DECOMPRESS;
1258 0 : desc->src1_addr = src_addr;
1259 0 : desc->dst_addr = dst_addr;
1260 0 : desc->src1_size = nbytes;
1261 0 : desc->iaa.max_dst_size = nbytes_dst;
1262 0 : desc->decompr_flags = IAA_DECOMP_FLAGS;
1263 :
1264 0 : _submit_to_hw(chan, op);
1265 0 : return 0;
1266 : error:
1267 0 : STAILQ_INSERT_TAIL(&chan->ops_pool, op, link);
1268 0 : return rc;
1269 0 : }
1270 :
1271 : int
1272 0 : spdk_idxd_submit_decompress(struct spdk_idxd_io_channel *chan,
1273 : struct iovec *diov, uint32_t diovcnt,
1274 : struct iovec *siov, uint32_t siovcnt,
1275 : int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)
1276 : {
1277 0 : assert(chan != NULL);
1278 0 : assert(diov != NULL);
1279 0 : assert(siov != NULL);
1280 :
1281 0 : if (diovcnt == 1 && siovcnt == 1) {
1282 : /* Simple case - copying one buffer to another */
1283 0 : if (diov[0].iov_len < siov[0].iov_len) {
1284 0 : return -EINVAL;
1285 : }
1286 :
1287 0 : return _idxd_submit_decompress_single(chan, diov[0].iov_base, siov[0].iov_base,
1288 0 : diov[0].iov_len, siov[0].iov_len,
1289 0 : flags, cb_fn, cb_arg);
1290 : }
1291 : /* TODO: vectored support */
1292 0 : return -EINVAL;
1293 0 : }
1294 :
1295 : static inline int
1296 0 : idxd_get_dif_flags(const struct spdk_dif_ctx *ctx, uint8_t *flags)
1297 : {
1298 0 : uint32_t data_block_size = ctx->block_size - ctx->md_size;
1299 :
1300 0 : if (flags == NULL) {
1301 0 : SPDK_ERRLOG("Flag should be non-null");
1302 0 : return -EINVAL;
1303 : }
1304 :
1305 0 : assert(ctx->md_interleave);
1306 :
1307 0 : switch (ctx->guard_interval) {
1308 : case DATA_BLOCK_SIZE_512:
1309 0 : *flags = IDXD_DIF_FLAG_DIF_BLOCK_SIZE_512;
1310 0 : break;
1311 : case DATA_BLOCK_SIZE_520:
1312 0 : *flags = IDXD_DIF_FLAG_DIF_BLOCK_SIZE_520;
1313 0 : break;
1314 : case DATA_BLOCK_SIZE_4096:
1315 0 : *flags = IDXD_DIF_FLAG_DIF_BLOCK_SIZE_4096;
1316 0 : break;
1317 : case DATA_BLOCK_SIZE_4104:
1318 0 : *flags = IDXD_DIF_FLAG_DIF_BLOCK_SIZE_4104;
1319 0 : break;
1320 : default:
1321 0 : SPDK_ERRLOG("Invalid DIF block size %d\n", data_block_size);
1322 0 : return -EINVAL;
1323 : }
1324 :
1325 0 : return 0;
1326 0 : }
1327 :
1328 : static inline int
1329 0 : idxd_get_source_dif_flags(const struct spdk_dif_ctx *ctx, uint8_t *flags)
1330 : {
1331 0 : if (flags == NULL) {
1332 0 : SPDK_ERRLOG("Flag should be non-null");
1333 0 : return -EINVAL;
1334 : }
1335 :
1336 0 : *flags = 0;
1337 :
1338 0 : if (!(ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK)) {
1339 0 : *flags |= IDXD_DIF_SOURCE_FLAG_GUARD_CHECK_DISABLE;
1340 0 : }
1341 :
1342 0 : if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) {
1343 0 : *flags |= IDXD_DIF_SOURCE_FLAG_REF_TAG_CHECK_DISABLE;
1344 0 : }
1345 :
1346 0 : switch (ctx->dif_type) {
1347 : case SPDK_DIF_TYPE1:
1348 : case SPDK_DIF_TYPE2:
1349 : /* If Type 1 or 2 is used, then all DIF checks are disabled when
1350 : * the Application Tag is 0xFFFF.
1351 : */
1352 0 : *flags |= IDXD_DIF_SOURCE_FLAG_APP_TAG_F_DETECT;
1353 0 : break;
1354 : case SPDK_DIF_TYPE3:
1355 : /* If Type 3 is used, then all DIF checks are disabled when the
1356 : * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF
1357 : * (for PI 8 bytes format).
1358 : */
1359 0 : *flags |= IDXD_DIF_SOURCE_FLAG_APP_AND_REF_TAG_F_DETECT;
1360 0 : break;
1361 : default:
1362 0 : SPDK_ERRLOG("Invalid DIF type %d\n", ctx->dif_type);
1363 0 : return -EINVAL;
1364 : }
1365 :
1366 0 : return 0;
1367 0 : }
1368 :
1369 : static inline int
1370 0 : idxd_get_app_tag_mask(const struct spdk_dif_ctx *ctx, uint16_t *app_tag_mask)
1371 : {
1372 0 : if (!(ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK)) {
1373 : /* The Source Application Tag Mask may be set to 0xffff
1374 : * to disable application tag checking */
1375 0 : *app_tag_mask = 0xFFFF;
1376 0 : } else {
1377 0 : *app_tag_mask = ~ctx->apptag_mask;
1378 : }
1379 :
1380 0 : return 0;
1381 : }
1382 :
1383 : static inline int
1384 0 : idxd_validate_dif_common_params(const struct spdk_dif_ctx *ctx)
1385 : {
1386 0 : uint32_t data_block_size = ctx->block_size - ctx->md_size;
1387 :
1388 : /* Check byte offset from the start of the whole data buffer */
1389 0 : if (ctx->data_offset != 0) {
1390 0 : SPDK_ERRLOG("Byte offset from the start of the whole data buffer must be set to 0.");
1391 0 : return -EINVAL;
1392 : }
1393 :
1394 : /* Check seed value for guard computation */
1395 0 : if (ctx->guard_seed != 0) {
1396 0 : SPDK_ERRLOG("Seed value for guard computation must be set to 0.");
1397 0 : return -EINVAL;
1398 : }
1399 :
1400 : /* Check for supported metadata sizes */
1401 0 : if (ctx->md_size != METADATA_SIZE_8 && ctx->md_size != METADATA_SIZE_16) {
1402 0 : SPDK_ERRLOG("Metadata size %d is not supported.\n", ctx->md_size);
1403 0 : return -EINVAL;
1404 : }
1405 :
1406 : /* Check for supported DIF PI formats */
1407 0 : if (ctx->dif_pi_format != SPDK_DIF_PI_FORMAT_16) {
1408 0 : SPDK_ERRLOG("DIF PI format %d is not supported.\n", ctx->dif_pi_format);
1409 0 : return -EINVAL;
1410 : }
1411 :
1412 : /* Check for supported metadata locations */
1413 0 : if (ctx->md_interleave == false) {
1414 0 : SPDK_ERRLOG("Separated metadata location is not supported.\n");
1415 0 : return -EINVAL;
1416 : }
1417 :
1418 : /* Check for supported DIF alignments */
1419 0 : if (ctx->md_size == METADATA_SIZE_16 &&
1420 0 : (ctx->guard_interval == DATA_BLOCK_SIZE_512 ||
1421 0 : ctx->guard_interval == DATA_BLOCK_SIZE_4096)) {
1422 0 : SPDK_ERRLOG("DIF left alignment in metadata is not supported.\n");
1423 0 : return -EINVAL;
1424 : }
1425 :
1426 : /* Check for supported DIF block sizes */
1427 0 : if (data_block_size != DATA_BLOCK_SIZE_512 &&
1428 0 : data_block_size != DATA_BLOCK_SIZE_4096) {
1429 0 : SPDK_ERRLOG("DIF block size %d is not supported.\n", data_block_size);
1430 0 : return -EINVAL;
1431 : }
1432 :
1433 0 : return 0;
1434 0 : }
1435 :
1436 : static inline int
1437 0 : idxd_validate_dif_check_params(const struct spdk_dif_ctx *ctx)
1438 : {
1439 0 : int rc = idxd_validate_dif_common_params(ctx);
1440 0 : if (rc) {
1441 0 : return rc;
1442 : }
1443 :
1444 0 : return 0;
1445 0 : }
1446 :
1447 : static inline int
1448 0 : idxd_validate_dif_check_buf_align(const struct spdk_dif_ctx *ctx, const uint64_t len)
1449 : {
1450 : /* DSA can only process contiguous memory buffers, multiple of the block size */
1451 0 : if (len % ctx->block_size != 0) {
1452 0 : SPDK_ERRLOG("The memory buffer length (%ld) is not a multiple of block size with metadata (%d).\n",
1453 : len, ctx->block_size);
1454 0 : return -EINVAL;
1455 : }
1456 :
1457 0 : return 0;
1458 0 : }
1459 :
1460 : int
1461 0 : spdk_idxd_submit_dif_check(struct spdk_idxd_io_channel *chan,
1462 : struct iovec *siov, size_t siovcnt,
1463 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx, int flags,
1464 : spdk_idxd_req_cb cb_fn, void *cb_arg)
1465 : {
1466 : struct idxd_hw_desc *desc;
1467 0 : struct idxd_ops *first_op = NULL, *op = NULL;
1468 : uint64_t src_seg_addr, src_seg_len;
1469 0 : uint32_t num_blocks_done = 0;
1470 0 : uint8_t dif_flags = 0, src_dif_flags = 0;
1471 0 : uint16_t app_tag_mask = 0;
1472 0 : int rc, count = 0;
1473 : size_t i;
1474 :
1475 0 : assert(ctx != NULL);
1476 0 : assert(chan != NULL);
1477 0 : assert(siov != NULL);
1478 :
1479 0 : rc = idxd_validate_dif_check_params(ctx);
1480 0 : if (rc) {
1481 0 : return rc;
1482 : }
1483 :
1484 0 : rc = idxd_get_dif_flags(ctx, &dif_flags);
1485 0 : if (rc) {
1486 0 : return rc;
1487 : }
1488 :
1489 0 : rc = idxd_get_source_dif_flags(ctx, &src_dif_flags);
1490 0 : if (rc) {
1491 0 : return rc;
1492 : }
1493 :
1494 0 : rc = idxd_get_app_tag_mask(ctx, &app_tag_mask);
1495 0 : if (rc) {
1496 0 : return rc;
1497 : }
1498 :
1499 0 : rc = _idxd_setup_batch(chan);
1500 0 : if (rc) {
1501 0 : return rc;
1502 : }
1503 :
1504 0 : for (i = 0; i < siovcnt; i++) {
1505 0 : src_seg_addr = (uint64_t)siov[i].iov_base;
1506 0 : src_seg_len = siov[i].iov_len;
1507 :
1508 : /* DSA processes the iovec buffers independently, so the buffers cannot
1509 : * be split (must be multiple of the block size) */
1510 :
1511 : /* Validate the memory buffer alignment */
1512 0 : rc = idxd_validate_dif_check_buf_align(ctx, src_seg_len);
1513 0 : if (rc) {
1514 0 : goto error;
1515 : }
1516 :
1517 0 : if (first_op == NULL) {
1518 0 : rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, flags, &desc, &op);
1519 0 : if (rc) {
1520 0 : goto error;
1521 : }
1522 :
1523 0 : first_op = op;
1524 0 : } else {
1525 0 : rc = _idxd_prep_batch_cmd(chan, NULL, NULL, flags, &desc, &op);
1526 0 : if (rc) {
1527 0 : goto error;
1528 : }
1529 :
1530 0 : first_op->count++;
1531 0 : op->parent = first_op;
1532 : }
1533 :
1534 0 : count++;
1535 :
1536 0 : desc->opcode = IDXD_OPCODE_DIF_CHECK;
1537 0 : desc->src_addr = src_seg_addr;
1538 0 : desc->xfer_size = src_seg_len;
1539 0 : desc->dif_chk.flags = dif_flags;
1540 0 : desc->dif_chk.src_flags = src_dif_flags;
1541 0 : desc->dif_chk.app_tag_seed = ctx->app_tag;
1542 0 : desc->dif_chk.app_tag_mask = app_tag_mask;
1543 0 : desc->dif_chk.ref_tag_seed = (uint32_t)ctx->init_ref_tag + num_blocks_done;
1544 :
1545 0 : num_blocks_done += (src_seg_len / ctx->block_size);
1546 0 : }
1547 :
1548 0 : return _idxd_flush_batch(chan);
1549 :
1550 : error:
1551 0 : chan->batch->index -= count;
1552 0 : return rc;
1553 0 : }
1554 :
1555 : static inline int
1556 0 : idxd_validate_dif_insert_params(const struct spdk_dif_ctx *ctx)
1557 : {
1558 0 : int rc = idxd_validate_dif_common_params(ctx);
1559 0 : if (rc) {
1560 0 : return rc;
1561 : }
1562 :
1563 : /* Check for required DIF flags */
1564 0 : if (!(ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK)) {
1565 0 : SPDK_ERRLOG("Guard check flag must be set.\n");
1566 0 : return -EINVAL;
1567 : }
1568 :
1569 0 : if (!(ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK)) {
1570 0 : SPDK_ERRLOG("Application Tag check flag must be set.\n");
1571 0 : return -EINVAL;
1572 : }
1573 :
1574 0 : if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) {
1575 0 : SPDK_ERRLOG("Reference Tag check flag must be set.\n");
1576 0 : return -EINVAL;
1577 : }
1578 :
1579 0 : return 0;
1580 0 : }
1581 :
1582 : static inline int
1583 0 : idxd_validate_dif_insert_iovecs(const struct spdk_dif_ctx *ctx,
1584 : const struct iovec *diov, const size_t diovcnt,
1585 : const struct iovec *siov, const size_t siovcnt)
1586 : {
1587 0 : uint32_t data_block_size = ctx->block_size - ctx->md_size;
1588 : size_t src_len, dst_len;
1589 : uint32_t num_blocks;
1590 : size_t i;
1591 :
1592 0 : if (diovcnt != siovcnt) {
1593 0 : SPDK_ERRLOG("Invalid number of elements in src (%ld) and dst (%ld) iovecs.\n",
1594 : siovcnt, diovcnt);
1595 0 : return -EINVAL;
1596 : }
1597 :
1598 0 : for (i = 0; i < siovcnt; i++) {
1599 0 : src_len = siov[i].iov_len;
1600 0 : dst_len = diov[i].iov_len;
1601 0 : num_blocks = src_len / data_block_size;
1602 0 : if (src_len != dst_len - num_blocks * ctx->md_size) {
1603 0 : SPDK_ERRLOG("Invalid length of data in src (%ld) and dst (%ld) in iovecs[%ld].\n",
1604 : src_len, dst_len, i);
1605 0 : return -EINVAL;
1606 : }
1607 0 : }
1608 :
1609 0 : return 0;
1610 0 : }
1611 :
1612 : static inline int
1613 0 : idxd_validate_dif_insert_buf_align(const struct spdk_dif_ctx *ctx,
1614 : const uint64_t src_len, const uint64_t dst_len)
1615 : {
1616 0 : uint32_t data_block_size = ctx->block_size - ctx->md_size;
1617 :
1618 : /* DSA can only process contiguous memory buffers, multiple of the block size */
1619 0 : if (src_len % data_block_size != 0) {
1620 0 : SPDK_ERRLOG("The memory source buffer length (%ld) is not a multiple of block size without metadata (%d).\n",
1621 : src_len, data_block_size);
1622 0 : return -EINVAL;
1623 : }
1624 :
1625 0 : if (dst_len % ctx->block_size != 0) {
1626 0 : SPDK_ERRLOG("The memory destination buffer length (%ld) is not a multiple of block size with metadata (%d).\n",
1627 : dst_len, ctx->block_size);
1628 0 : return -EINVAL;
1629 : }
1630 :
1631 : /* The memory source and destination must hold the same number of blocks. */
1632 0 : if (src_len / data_block_size != (dst_len / ctx->block_size)) {
1633 0 : SPDK_ERRLOG("The memory source (%ld) and destination (%ld) must hold the same number of blocks.\n",
1634 : src_len / data_block_size, dst_len / ctx->block_size);
1635 0 : return -EINVAL;
1636 : }
1637 :
1638 0 : return 0;
1639 0 : }
1640 :
1641 : int
1642 0 : spdk_idxd_submit_dif_insert(struct spdk_idxd_io_channel *chan,
1643 : struct iovec *diov, size_t diovcnt,
1644 : struct iovec *siov, size_t siovcnt,
1645 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx, int flags,
1646 : spdk_idxd_req_cb cb_fn, void *cb_arg)
1647 : {
1648 : struct idxd_hw_desc *desc;
1649 0 : struct idxd_ops *first_op = NULL, *op = NULL;
1650 0 : uint32_t data_block_size = ctx->block_size - ctx->md_size;
1651 : uint64_t src_seg_addr, src_seg_len;
1652 : uint64_t dst_seg_addr, dst_seg_len;
1653 0 : uint32_t num_blocks_done = 0;
1654 0 : uint8_t dif_flags = 0;
1655 0 : int rc, count = 0;
1656 : size_t i;
1657 :
1658 0 : assert(ctx != NULL);
1659 0 : assert(chan != NULL);
1660 0 : assert(siov != NULL);
1661 :
1662 0 : rc = idxd_validate_dif_insert_params(ctx);
1663 0 : if (rc) {
1664 0 : return rc;
1665 : }
1666 :
1667 0 : rc = idxd_validate_dif_insert_iovecs(ctx, diov, diovcnt, siov, siovcnt);
1668 0 : if (rc) {
1669 0 : return rc;
1670 : }
1671 :
1672 0 : rc = idxd_get_dif_flags(ctx, &dif_flags);
1673 0 : if (rc) {
1674 0 : return rc;
1675 : }
1676 :
1677 0 : rc = _idxd_setup_batch(chan);
1678 0 : if (rc) {
1679 0 : return rc;
1680 : }
1681 :
1682 0 : for (i = 0; i < siovcnt; i++) {
1683 0 : src_seg_addr = (uint64_t)siov[i].iov_base;
1684 0 : src_seg_len = siov[i].iov_len;
1685 0 : dst_seg_addr = (uint64_t)diov[i].iov_base;
1686 0 : dst_seg_len = diov[i].iov_len;
1687 :
1688 : /* DSA processes the iovec buffers independently, so the buffers cannot
1689 : * be split (must be multiple of the block size). The destination memory
1690 : * size needs to be same as the source memory size + metadata size */
1691 :
1692 0 : rc = idxd_validate_dif_insert_buf_align(ctx, src_seg_len, dst_seg_len);
1693 0 : if (rc) {
1694 0 : goto error;
1695 : }
1696 :
1697 0 : if (first_op == NULL) {
1698 0 : rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, flags, &desc, &op);
1699 0 : if (rc) {
1700 0 : goto error;
1701 : }
1702 :
1703 0 : first_op = op;
1704 0 : } else {
1705 0 : rc = _idxd_prep_batch_cmd(chan, NULL, NULL, flags, &desc, &op);
1706 0 : if (rc) {
1707 0 : goto error;
1708 : }
1709 :
1710 0 : first_op->count++;
1711 0 : op->parent = first_op;
1712 : }
1713 :
1714 0 : count++;
1715 :
1716 0 : desc->opcode = IDXD_OPCODE_DIF_INS;
1717 0 : desc->src_addr = src_seg_addr;
1718 0 : desc->dst_addr = dst_seg_addr;
1719 0 : desc->xfer_size = src_seg_len;
1720 0 : desc->dif_ins.flags = dif_flags;
1721 0 : desc->dif_ins.app_tag_seed = ctx->app_tag;
1722 0 : desc->dif_ins.app_tag_mask = ~ctx->apptag_mask;
1723 0 : desc->dif_ins.ref_tag_seed = (uint32_t)ctx->init_ref_tag + num_blocks_done;
1724 :
1725 0 : num_blocks_done += src_seg_len / data_block_size;
1726 0 : }
1727 :
1728 0 : return _idxd_flush_batch(chan);
1729 :
1730 : error:
1731 0 : chan->batch->index -= count;
1732 0 : return rc;
1733 0 : }
1734 :
1735 : static inline int
1736 0 : idxd_validate_dif_strip_buf_align(const struct spdk_dif_ctx *ctx,
1737 : const uint64_t src_len, const uint64_t dst_len)
1738 : {
1739 0 : uint32_t data_block_size = ctx->block_size - ctx->md_size;
1740 :
1741 : /* DSA can only process contiguous memory buffers, multiple of the block size. */
1742 0 : if (src_len % ctx->block_size != 0) {
1743 0 : SPDK_ERRLOG("The src buffer length (%ld) is not a multiple of block size (%d).\n",
1744 : src_len, ctx->block_size);
1745 0 : return -EINVAL;
1746 : }
1747 0 : if (dst_len % data_block_size != 0) {
1748 0 : SPDK_ERRLOG("The dst buffer length (%ld) is not a multiple of block size without metadata (%d).\n",
1749 : dst_len, data_block_size);
1750 0 : return -EINVAL;
1751 : }
1752 : /* The memory source and destination must hold the same number of blocks. */
1753 0 : if (src_len / ctx->block_size != dst_len / data_block_size) {
1754 0 : SPDK_ERRLOG("The memory source (%ld) and destination (%ld) must hold the same number of blocks.\n",
1755 : src_len / data_block_size, dst_len / ctx->block_size);
1756 0 : return -EINVAL;
1757 : }
1758 0 : return 0;
1759 0 : }
1760 :
1761 : int
1762 0 : spdk_idxd_submit_dif_strip(struct spdk_idxd_io_channel *chan,
1763 : struct iovec *diov, size_t diovcnt,
1764 : struct iovec *siov, size_t siovcnt,
1765 : uint32_t num_blocks, const struct spdk_dif_ctx *ctx, int flags,
1766 : spdk_idxd_req_cb cb_fn, void *cb_arg)
1767 : {
1768 : struct idxd_hw_desc *desc;
1769 0 : struct idxd_ops *first_op = NULL, *op = NULL;
1770 : uint64_t src_seg_addr, src_seg_len;
1771 : uint64_t dst_seg_addr, dst_seg_len;
1772 0 : uint8_t dif_flags = 0, src_dif_flags = 0;
1773 0 : uint16_t app_tag_mask = 0;
1774 0 : int rc, count = 0;
1775 : size_t i;
1776 :
1777 0 : rc = idxd_validate_dif_common_params(ctx);
1778 0 : if (rc) {
1779 0 : return rc;
1780 : }
1781 :
1782 0 : rc = idxd_get_dif_flags(ctx, &dif_flags);
1783 0 : if (rc) {
1784 0 : return rc;
1785 : }
1786 :
1787 0 : rc = idxd_get_source_dif_flags(ctx, &src_dif_flags);
1788 0 : if (rc) {
1789 0 : return rc;
1790 : }
1791 :
1792 0 : rc = idxd_get_app_tag_mask(ctx, &app_tag_mask);
1793 0 : if (rc) {
1794 0 : return rc;
1795 : }
1796 :
1797 0 : rc = _idxd_setup_batch(chan);
1798 0 : if (rc) {
1799 0 : return rc;
1800 : }
1801 :
1802 0 : if (diovcnt != siovcnt) {
1803 0 : SPDK_ERRLOG("Mismatched iovcnts: src=%ld, dst=%ld\n",
1804 : siovcnt, diovcnt);
1805 0 : return -EINVAL;
1806 : }
1807 :
1808 0 : for (i = 0; i < siovcnt; i++) {
1809 0 : src_seg_addr = (uint64_t)siov[i].iov_base;
1810 0 : src_seg_len = siov[i].iov_len;
1811 0 : dst_seg_addr = (uint64_t)diov[i].iov_base;
1812 0 : dst_seg_len = diov[i].iov_len;
1813 :
1814 : /* DSA processes the iovec buffers independently, so the buffers cannot
1815 : * be split (must be multiple of the block size). The source memory
1816 : * size needs to be same as the destination memory size + metadata size */
1817 :
1818 0 : rc = idxd_validate_dif_strip_buf_align(ctx, src_seg_len, dst_seg_len);
1819 0 : if (rc) {
1820 0 : goto error;
1821 : }
1822 :
1823 0 : if (first_op == NULL) {
1824 0 : rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, flags, &desc, &op);
1825 0 : if (rc) {
1826 0 : goto error;
1827 : }
1828 :
1829 0 : first_op = op;
1830 0 : } else {
1831 0 : rc = _idxd_prep_batch_cmd(chan, NULL, NULL, flags, &desc, &op);
1832 0 : if (rc) {
1833 0 : goto error;
1834 : }
1835 :
1836 0 : first_op->count++;
1837 0 : op->parent = first_op;
1838 : }
1839 :
1840 0 : count++;
1841 :
1842 0 : desc->opcode = IDXD_OPCODE_DIF_STRP;
1843 0 : desc->src_addr = src_seg_addr;
1844 0 : desc->dst_addr = dst_seg_addr;
1845 0 : desc->xfer_size = src_seg_len;
1846 0 : desc->dif_strip.flags = dif_flags;
1847 0 : desc->dif_strip.src_flags = src_dif_flags;
1848 0 : desc->dif_strip.app_tag_seed = ctx->app_tag;
1849 0 : desc->dif_strip.app_tag_mask = app_tag_mask;
1850 0 : desc->dif_strip.ref_tag_seed = (uint32_t)ctx->init_ref_tag;
1851 0 : }
1852 :
1853 0 : return _idxd_flush_batch(chan);
1854 :
1855 : error:
1856 0 : chan->batch->index -= count;
1857 0 : return rc;
1858 0 : }
1859 :
1860 : static inline int
1861 0 : idxd_get_dix_flags(const struct spdk_dif_ctx *ctx, uint8_t *flags)
1862 : {
1863 0 : uint32_t data_block_size = ctx->block_size;
1864 :
1865 0 : assert(!ctx->md_interleave);
1866 :
1867 0 : if (flags == NULL) {
1868 0 : SPDK_ERRLOG("Flag should be non-null");
1869 0 : return -EINVAL;
1870 : }
1871 :
1872 0 : switch (data_block_size) {
1873 : case DATA_BLOCK_SIZE_512:
1874 0 : *flags = IDXD_DIF_FLAG_DIF_BLOCK_SIZE_512;
1875 0 : break;
1876 : case DATA_BLOCK_SIZE_4096:
1877 0 : *flags = IDXD_DIF_FLAG_DIF_BLOCK_SIZE_4096;
1878 0 : break;
1879 : default:
1880 0 : SPDK_ERRLOG("Invalid DIX block size %d\n", data_block_size);
1881 0 : return -EINVAL;
1882 : }
1883 :
1884 0 : return 0;
1885 0 : }
1886 :
1887 : static inline int
1888 0 : idxd_validate_dix_generate_params(const struct spdk_dif_ctx *ctx)
1889 : {
1890 : /* Check for required DIF flags. Intel DSA is able to only generate all DIF fields. */
1891 0 : if (!(ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK)) {
1892 0 : SPDK_ERRLOG("Guard check flag must be set.\n");
1893 0 : return -EINVAL;
1894 : }
1895 :
1896 0 : if (!(ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK)) {
1897 0 : SPDK_ERRLOG("Application Tag check flag must be set.\n");
1898 0 : return -EINVAL;
1899 : }
1900 :
1901 0 : if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) {
1902 0 : SPDK_ERRLOG("Reference Tag check flag must be set.\n");
1903 0 : return -EINVAL;
1904 : }
1905 :
1906 : /* Check byte offset from the start of the whole data buffer */
1907 0 : if (ctx->data_offset != 0) {
1908 0 : SPDK_ERRLOG("Byte offset from the start of the whole data buffer must be set to 0.");
1909 0 : return -EINVAL;
1910 : }
1911 :
1912 : /* Check seed value for guard computation */
1913 0 : if (ctx->guard_seed != 0) {
1914 0 : SPDK_ERRLOG("Seed value for guard computation must be set to 0.");
1915 0 : return -EINVAL;
1916 : }
1917 :
1918 : /* Check for supported metadata sizes */
1919 0 : if (ctx->md_size != METADATA_SIZE_8) {
1920 0 : SPDK_ERRLOG("Metadata size %d is not supported.\n", ctx->md_size);
1921 0 : return -EINVAL;
1922 : }
1923 :
1924 : /* Check for supported DIF PI formats */
1925 0 : if (ctx->dif_pi_format != SPDK_DIF_PI_FORMAT_16) {
1926 0 : SPDK_ERRLOG("DIF PI format %d is not supported.\n", ctx->dif_pi_format);
1927 0 : return -EINVAL;
1928 : }
1929 :
1930 : /* Check for supported DIF block sizes */
1931 0 : if (ctx->block_size != DATA_BLOCK_SIZE_512 &&
1932 0 : ctx->block_size != DATA_BLOCK_SIZE_4096) {
1933 0 : SPDK_ERRLOG("DIF block size %d is not supported.\n", ctx->block_size);
1934 0 : return -EINVAL;
1935 : }
1936 :
1937 0 : return 0;
1938 0 : }
1939 :
1940 : int
1941 0 : spdk_idxd_submit_dix_generate(struct spdk_idxd_io_channel *chan, struct iovec *siov,
1942 : size_t siovcnt, struct iovec *mdiov, uint32_t num_blocks,
1943 : const struct spdk_dif_ctx *ctx, int flags,
1944 : spdk_idxd_req_cb cb_fn, void *cb_arg)
1945 : {
1946 : struct idxd_hw_desc *desc;
1947 0 : struct idxd_ops *first_op = NULL, *op = NULL;
1948 : uint64_t src_seg_addr, src_seg_len;
1949 : uint64_t md_seg_addr, md_seg_len;
1950 0 : uint32_t num_blocks_done = 0;
1951 0 : uint8_t dif_flags = 0;
1952 0 : uint16_t app_tag_mask = 0;
1953 0 : int rc, count = 0;
1954 : size_t i;
1955 :
1956 0 : rc = idxd_validate_dix_generate_params(ctx);
1957 0 : if (rc) {
1958 0 : return rc;
1959 : }
1960 :
1961 0 : rc = idxd_get_dix_flags(ctx, &dif_flags);
1962 0 : if (rc) {
1963 0 : return rc;
1964 : }
1965 :
1966 0 : rc = idxd_get_app_tag_mask(ctx, &app_tag_mask);
1967 0 : if (rc) {
1968 0 : return rc;
1969 : }
1970 :
1971 0 : rc = _idxd_setup_batch(chan);
1972 0 : if (rc) {
1973 0 : return rc;
1974 : }
1975 :
1976 0 : md_seg_len = mdiov->iov_len;
1977 0 : md_seg_addr = (uint64_t)mdiov->iov_base;
1978 :
1979 0 : if (md_seg_len % ctx->md_size != 0) {
1980 0 : SPDK_ERRLOG("The metadata buffer length (%ld) is not a multiple of metadata size.\n",
1981 : md_seg_len);
1982 0 : return -EINVAL;
1983 : }
1984 :
1985 0 : for (i = 0; i < siovcnt; i++) {
1986 0 : src_seg_addr = (uint64_t)siov[i].iov_base;
1987 0 : src_seg_len = siov[i].iov_len;
1988 :
1989 0 : if (src_seg_len % ctx->block_size != 0) {
1990 0 : SPDK_ERRLOG("The source buffer length (%ld) is not a multiple of block size (%d).\n",
1991 : src_seg_len, ctx->block_size);
1992 0 : goto error;
1993 : }
1994 :
1995 0 : if (first_op == NULL) {
1996 0 : rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, flags, &desc, &op);
1997 0 : if (rc) {
1998 0 : goto error;
1999 : }
2000 :
2001 0 : first_op = op;
2002 0 : } else {
2003 0 : rc = _idxd_prep_batch_cmd(chan, NULL, NULL, flags, &desc, &op);
2004 0 : if (rc) {
2005 0 : goto error;
2006 : }
2007 :
2008 0 : first_op->count++;
2009 0 : op->parent = first_op;
2010 : }
2011 :
2012 0 : count++;
2013 :
2014 0 : desc->opcode = IDXD_OPCODE_DIX_GEN;
2015 0 : desc->src_addr = src_seg_addr;
2016 0 : desc->dst_addr = md_seg_addr;
2017 0 : desc->xfer_size = src_seg_len;
2018 0 : desc->dix_gen.flags = dif_flags;
2019 0 : desc->dix_gen.app_tag_seed = ctx->app_tag;
2020 0 : desc->dix_gen.app_tag_mask = ~ctx->apptag_mask;
2021 0 : desc->dix_gen.ref_tag_seed = (uint32_t)ctx->init_ref_tag + num_blocks_done;
2022 :
2023 0 : num_blocks_done += src_seg_len / ctx->block_size;
2024 :
2025 0 : md_seg_addr = (uint64_t)mdiov->iov_base + (num_blocks_done * ctx->md_size);
2026 0 : }
2027 :
2028 0 : return _idxd_flush_batch(chan);
2029 :
2030 : error:
2031 0 : chan->batch->index -= count;
2032 0 : return rc;
2033 0 : }
2034 :
2035 : int
2036 0 : spdk_idxd_submit_raw_desc(struct spdk_idxd_io_channel *chan,
2037 : struct idxd_hw_desc *_desc,
2038 : spdk_idxd_req_cb cb_fn, void *cb_arg)
2039 : {
2040 : struct idxd_hw_desc *desc;
2041 : struct idxd_ops *op;
2042 0 : int rc, flags = 0;
2043 : uint64_t comp_addr;
2044 :
2045 0 : assert(chan != NULL);
2046 0 : assert(_desc != NULL);
2047 :
2048 : /* Common prep. */
2049 0 : rc = _idxd_prep_command(chan, cb_fn, cb_arg, flags, &desc, &op);
2050 0 : if (rc) {
2051 0 : return rc;
2052 : }
2053 :
2054 : /* Command specific. */
2055 0 : flags = desc->flags;
2056 0 : comp_addr = desc->completion_addr;
2057 0 : memcpy(desc, _desc, sizeof(*desc));
2058 0 : desc->flags |= flags;
2059 0 : desc->completion_addr = comp_addr;
2060 :
2061 : /* Submit operation. */
2062 0 : _submit_to_hw(chan, op);
2063 :
2064 0 : return 0;
2065 0 : }
2066 :
2067 : static inline void
2068 0 : _dump_sw_error_reg(struct spdk_idxd_io_channel *chan)
2069 : {
2070 0 : struct spdk_idxd_device *idxd = chan->idxd;
2071 :
2072 0 : assert(idxd != NULL);
2073 0 : idxd->impl->dump_sw_error(idxd, chan->portal);
2074 0 : }
2075 :
2076 : /* TODO: more performance experiments. */
2077 : #define IDXD_COMPLETION(x) ((x) > (0) ? (1) : (0))
2078 : #define IDXD_FAILURE(x) ((x) > (1) ? (1) : (0))
2079 : #define IDXD_SW_ERROR(x) ((x) &= (0x1) ? (1) : (0))
2080 : int
2081 0 : spdk_idxd_process_events(struct spdk_idxd_io_channel *chan)
2082 : {
2083 : struct idxd_ops *op, *tmp, *parent_op;
2084 0 : int status = 0;
2085 0 : int rc2, rc = 0;
2086 : void *cb_arg;
2087 : spdk_idxd_req_cb cb_fn;
2088 :
2089 0 : assert(chan != NULL);
2090 :
2091 0 : STAILQ_FOREACH_SAFE(op, &chan->ops_outstanding, link, tmp) {
2092 0 : if (!IDXD_COMPLETION(op->hw.status)) {
2093 : /*
2094 : * oldest locations are at the head of the list so if
2095 : * we've polled a location that hasn't completed, bail
2096 : * now as there are unlikely to be any more completions.
2097 : */
2098 0 : break;
2099 : }
2100 :
2101 0 : STAILQ_REMOVE_HEAD(&chan->ops_outstanding, link);
2102 0 : rc++;
2103 :
2104 : /* Status is in the same location for both IAA and DSA completion records. */
2105 0 : if (spdk_unlikely(IDXD_FAILURE(op->hw.status))) {
2106 0 : SPDK_ERRLOG("Completion status 0x%x\n", op->hw.status);
2107 0 : status = -EINVAL;
2108 0 : _dump_sw_error_reg(chan);
2109 0 : }
2110 :
2111 0 : switch (op->desc->opcode) {
2112 : case IDXD_OPCODE_BATCH:
2113 0 : SPDK_DEBUGLOG(idxd, "Complete batch %p\n", op->batch);
2114 0 : break;
2115 : case IDXD_OPCODE_CRC32C_GEN:
2116 : case IDXD_OPCODE_COPY_CRC:
2117 0 : if (spdk_likely(status == 0 && op->crc_dst != NULL)) {
2118 0 : *op->crc_dst = op->hw.crc32c_val;
2119 0 : *op->crc_dst ^= ~0;
2120 0 : }
2121 0 : break;
2122 : case IDXD_OPCODE_COMPARE:
2123 0 : if (spdk_likely(status == 0)) {
2124 0 : status = op->hw.result;
2125 0 : }
2126 0 : break;
2127 : case IDXD_OPCODE_COMPRESS:
2128 0 : if (spdk_likely(status == 0 && op->output_size != NULL)) {
2129 0 : *op->output_size = op->iaa_hw.output_size;
2130 0 : }
2131 0 : break;
2132 : case IDXD_OPCODE_DIF_CHECK:
2133 : case IDXD_OPCODE_DIF_STRP:
2134 0 : if (spdk_unlikely(op->hw.status == IDXD_DSA_STATUS_DIF_ERROR)) {
2135 0 : status = -EIO;
2136 0 : }
2137 0 : break;
2138 : }
2139 :
2140 : /* TODO: WHAT IF THIS FAILED!? */
2141 0 : op->hw.status = 0;
2142 :
2143 0 : assert(op->count > 0);
2144 0 : op->count--;
2145 :
2146 0 : parent_op = op->parent;
2147 0 : if (parent_op != NULL) {
2148 0 : assert(parent_op->count > 0);
2149 0 : parent_op->count--;
2150 :
2151 0 : if (parent_op->count == 0) {
2152 0 : cb_fn = parent_op->cb_fn;
2153 0 : cb_arg = parent_op->cb_arg;
2154 :
2155 0 : assert(parent_op->batch != NULL);
2156 :
2157 : /*
2158 : * Now that parent_op count is 0, we can release its ref
2159 : * to its batch. We have not released the ref to the batch
2160 : * that the op is pointing to yet, which will be done below.
2161 : */
2162 0 : parent_op->batch->refcnt--;
2163 0 : if (parent_op->batch->refcnt == 0) {
2164 0 : _free_batch(parent_op->batch, chan);
2165 0 : }
2166 :
2167 0 : if (cb_fn) {
2168 0 : cb_fn(cb_arg, status);
2169 0 : }
2170 0 : }
2171 0 : }
2172 :
2173 0 : if (op->count == 0) {
2174 0 : cb_fn = op->cb_fn;
2175 0 : cb_arg = op->cb_arg;
2176 :
2177 0 : if (op->batch != NULL) {
2178 0 : assert(op->batch->refcnt > 0);
2179 0 : op->batch->refcnt--;
2180 :
2181 0 : if (op->batch->refcnt == 0) {
2182 0 : _free_batch(op->batch, chan);
2183 0 : }
2184 0 : } else {
2185 0 : STAILQ_INSERT_HEAD(&chan->ops_pool, op, link);
2186 : }
2187 :
2188 0 : if (cb_fn) {
2189 0 : cb_fn(cb_arg, status);
2190 0 : }
2191 0 : }
2192 :
2193 : /* reset the status */
2194 0 : status = 0;
2195 : /* break the processing loop to prevent from starving the rest of the system */
2196 0 : if (rc > IDXD_MAX_COMPLETIONS) {
2197 0 : break;
2198 : }
2199 0 : }
2200 :
2201 : /* Submit any built-up batch */
2202 0 : if (chan->batch) {
2203 0 : rc2 = idxd_batch_submit(chan, NULL, NULL);
2204 0 : if (rc2) {
2205 0 : assert(rc2 == -EBUSY);
2206 0 : }
2207 0 : }
2208 :
2209 0 : return rc;
2210 : }
2211 :
2212 : void
2213 0 : idxd_impl_register(struct spdk_idxd_impl *impl)
2214 : {
2215 0 : STAILQ_INSERT_HEAD(&g_idxd_impls, impl, link);
2216 0 : }
2217 :
2218 0 : SPDK_LOG_REGISTER_COMPONENT(idxd)
|
