Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2017 Intel Corporation.
3 : * All rights reserved.
4 : * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : */
6 :
7 : #include "spdk/stdinc.h"
8 :
9 : #include "bdev_malloc.h"
10 : #include "spdk/endian.h"
11 : #include "spdk/env.h"
12 : #include "spdk/accel.h"
13 : #include "spdk/dma.h"
14 : #include "spdk/likely.h"
15 : #include "spdk/string.h"
16 :
17 : #include "spdk/log.h"
18 :
19 : struct malloc_disk {
20 : struct spdk_bdev disk;
21 : void *malloc_buf;
22 : void *malloc_md_buf;
23 : TAILQ_ENTRY(malloc_disk) link;
24 : };
25 :
26 : struct malloc_task {
27 : struct iovec iov;
28 : int num_outstanding;
29 : enum spdk_bdev_io_status status;
30 : TAILQ_ENTRY(malloc_task) tailq;
31 : };
32 :
33 : struct malloc_channel {
34 : struct spdk_io_channel *accel_channel;
35 : struct spdk_poller *completion_poller;
36 : TAILQ_HEAD(, malloc_task) completed_tasks;
37 : };
38 :
39 : static int
40 0 : malloc_verify_pi(struct spdk_bdev_io *bdev_io)
41 : {
42 0 : struct spdk_bdev *bdev = bdev_io->bdev;
43 0 : struct spdk_dif_ctx dif_ctx;
44 0 : struct spdk_dif_error err_blk;
45 : int rc;
46 0 : struct spdk_dif_ctx_init_ext_opts dif_opts;
47 :
48 0 : assert(bdev_io->u.bdev.memory_domain == NULL);
49 0 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
50 0 : dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;
51 0 : rc = spdk_dif_ctx_init(&dif_ctx,
52 : bdev->blocklen,
53 : bdev->md_len,
54 0 : bdev->md_interleave,
55 0 : bdev->dif_is_head_of_md,
56 : bdev->dif_type,
57 : bdev->dif_check_flags,
58 0 : bdev_io->u.bdev.offset_blocks & 0xFFFFFFFF,
59 : 0xFFFF, 0, 0, 0, &dif_opts);
60 0 : if (rc != 0) {
61 0 : SPDK_ERRLOG("Failed to initialize DIF/DIX context\n");
62 0 : return rc;
63 : }
64 :
65 0 : if (spdk_bdev_is_md_interleaved(bdev)) {
66 0 : rc = spdk_dif_verify(bdev_io->u.bdev.iovs,
67 : bdev_io->u.bdev.iovcnt,
68 0 : bdev_io->u.bdev.num_blocks,
69 : &dif_ctx,
70 : &err_blk);
71 : } else {
72 0 : struct iovec md_iov = {
73 0 : .iov_base = bdev_io->u.bdev.md_buf,
74 0 : .iov_len = bdev_io->u.bdev.num_blocks * bdev->md_len,
75 : };
76 :
77 0 : if (bdev_io->u.bdev.md_buf == NULL) {
78 0 : return 0;
79 : }
80 :
81 0 : rc = spdk_dix_verify(bdev_io->u.bdev.iovs,
82 : bdev_io->u.bdev.iovcnt,
83 : &md_iov,
84 0 : bdev_io->u.bdev.num_blocks,
85 : &dif_ctx,
86 : &err_blk);
87 : }
88 :
89 0 : if (rc != 0) {
90 0 : SPDK_ERRLOG("DIF/DIX verify failed: lba %" PRIu64 ", num_blocks %" PRIu64 ", "
91 : "err_type %u, expected %lu, actual %lu, err_offset %u\n",
92 : bdev_io->u.bdev.offset_blocks,
93 : bdev_io->u.bdev.num_blocks,
94 : err_blk.err_type,
95 : err_blk.expected,
96 : err_blk.actual,
97 : err_blk.err_offset);
98 : }
99 :
100 0 : return rc;
101 : }
102 :
103 : static int
104 0 : malloc_unmap_write_zeroes_generate_pi(struct spdk_bdev_io *bdev_io)
105 : {
106 0 : struct spdk_bdev *bdev = bdev_io->bdev;
107 0 : struct malloc_disk *mdisk = bdev_io->bdev->ctxt;
108 0 : uint32_t block_size = bdev_io->bdev->blocklen;
109 0 : struct spdk_dif_ctx dif_ctx;
110 0 : struct spdk_dif_ctx_init_ext_opts dif_opts;
111 : int rc;
112 :
113 0 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
114 0 : dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;
115 0 : rc = spdk_dif_ctx_init(&dif_ctx,
116 : bdev->blocklen,
117 : bdev->md_len,
118 0 : bdev->md_interleave,
119 0 : bdev->dif_is_head_of_md,
120 : bdev->dif_type,
121 : bdev->dif_check_flags,
122 : SPDK_DIF_REFTAG_IGNORE,
123 : 0xFFFF, SPDK_DIF_APPTAG_IGNORE,
124 : 0, 0, &dif_opts);
125 0 : if (rc != 0) {
126 0 : SPDK_ERRLOG("Initialization of DIF/DIX context failed\n");
127 0 : return rc;
128 : }
129 :
130 0 : if (bdev->md_interleave) {
131 0 : struct iovec iov = {
132 0 : .iov_base = mdisk->malloc_buf + bdev_io->u.bdev.offset_blocks * block_size,
133 0 : .iov_len = bdev_io->u.bdev.num_blocks * block_size,
134 : };
135 :
136 0 : rc = spdk_dif_generate(&iov, 1, bdev_io->u.bdev.num_blocks, &dif_ctx);
137 : } else {
138 0 : struct iovec iov = {
139 0 : .iov_base = mdisk->malloc_buf + bdev_io->u.bdev.offset_blocks * block_size,
140 0 : .iov_len = bdev_io->u.bdev.num_blocks * block_size,
141 : };
142 :
143 0 : struct iovec md_iov = {
144 0 : .iov_base = mdisk->malloc_md_buf + bdev_io->u.bdev.offset_blocks * bdev->md_len,
145 0 : .iov_len = bdev_io->u.bdev.num_blocks * bdev->md_len,
146 : };
147 :
148 0 : rc = spdk_dix_generate(&iov, 1, &md_iov, bdev_io->u.bdev.num_blocks, &dif_ctx);
149 : }
150 :
151 0 : if (rc != 0) {
152 0 : SPDK_ERRLOG("Formatting by DIF/DIX failed\n");
153 : }
154 :
155 :
156 0 : return rc;
157 : }
158 :
159 : static void
160 0 : malloc_done(void *ref, int status)
161 : {
162 0 : struct malloc_task *task = (struct malloc_task *)ref;
163 0 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(task);
164 : int rc;
165 :
166 0 : if (status != 0) {
167 0 : if (status == -ENOMEM) {
168 0 : if (task->status == SPDK_BDEV_IO_STATUS_SUCCESS) {
169 0 : task->status = SPDK_BDEV_IO_STATUS_NOMEM;
170 : }
171 : } else {
172 0 : task->status = SPDK_BDEV_IO_STATUS_FAILED;
173 : }
174 : }
175 :
176 0 : if (--task->num_outstanding != 0) {
177 0 : return;
178 : }
179 :
180 0 : if (bdev_io->bdev->dif_type != SPDK_DIF_DISABLE &&
181 0 : bdev_io->type == SPDK_BDEV_IO_TYPE_READ &&
182 0 : task->status == SPDK_BDEV_IO_STATUS_SUCCESS) {
183 0 : rc = malloc_verify_pi(bdev_io);
184 0 : if (rc != 0) {
185 0 : task->status = SPDK_BDEV_IO_STATUS_FAILED;
186 : }
187 : }
188 :
189 0 : if (bdev_io->bdev->dif_type != SPDK_DIF_DISABLE &&
190 0 : (bdev_io->type == SPDK_BDEV_IO_TYPE_UNMAP || bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE_ZEROES) &&
191 0 : task->status == SPDK_BDEV_IO_STATUS_SUCCESS) {
192 0 : rc = malloc_unmap_write_zeroes_generate_pi(bdev_io);
193 0 : if (rc != 0) {
194 0 : task->status = SPDK_BDEV_IO_STATUS_FAILED;
195 : }
196 : }
197 :
198 0 : assert(!bdev_io->u.bdev.accel_sequence || task->status == SPDK_BDEV_IO_STATUS_NOMEM);
199 0 : spdk_bdev_io_complete(spdk_bdev_io_from_ctx(task), task->status);
200 : }
201 :
202 : static void
203 0 : malloc_complete_task(struct malloc_task *task, struct malloc_channel *mch,
204 : enum spdk_bdev_io_status status)
205 : {
206 0 : task->status = status;
207 0 : TAILQ_INSERT_TAIL(&mch->completed_tasks, task, tailq);
208 0 : }
209 :
210 : static TAILQ_HEAD(, malloc_disk) g_malloc_disks = TAILQ_HEAD_INITIALIZER(g_malloc_disks);
211 :
212 : int malloc_disk_count = 0;
213 :
214 : static int bdev_malloc_initialize(void);
215 : static void bdev_malloc_deinitialize(void);
216 :
217 : static int
218 0 : bdev_malloc_get_ctx_size(void)
219 : {
220 0 : return sizeof(struct malloc_task);
221 : }
222 :
223 : static struct spdk_bdev_module malloc_if = {
224 : .name = "malloc",
225 : .module_init = bdev_malloc_initialize,
226 : .module_fini = bdev_malloc_deinitialize,
227 : .get_ctx_size = bdev_malloc_get_ctx_size,
228 :
229 : };
230 :
231 0 : SPDK_BDEV_MODULE_REGISTER(malloc, &malloc_if)
232 :
233 : static void
234 0 : malloc_disk_free(struct malloc_disk *malloc_disk)
235 : {
236 0 : if (!malloc_disk) {
237 0 : return;
238 : }
239 :
240 0 : free(malloc_disk->disk.name);
241 0 : spdk_free(malloc_disk->malloc_buf);
242 0 : spdk_free(malloc_disk->malloc_md_buf);
243 0 : free(malloc_disk);
244 : }
245 :
246 : static int
247 0 : bdev_malloc_destruct(void *ctx)
248 : {
249 0 : struct malloc_disk *malloc_disk = ctx;
250 :
251 0 : TAILQ_REMOVE(&g_malloc_disks, malloc_disk, link);
252 0 : malloc_disk_free(malloc_disk);
253 0 : return 0;
254 : }
255 :
256 : static int
257 0 : bdev_malloc_check_iov_len(struct iovec *iovs, int iovcnt, size_t nbytes)
258 : {
259 : int i;
260 :
261 0 : for (i = 0; i < iovcnt; i++) {
262 0 : if (nbytes < iovs[i].iov_len) {
263 0 : return 0;
264 : }
265 :
266 0 : nbytes -= iovs[i].iov_len;
267 : }
268 :
269 0 : return nbytes != 0;
270 : }
271 :
272 : static void
273 0 : malloc_sequence_fail(struct malloc_task *task, int status)
274 : {
275 0 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(task);
276 :
277 : /* For ENOMEM, the IO will be retried by the bdev layer, so we don't abort the sequence */
278 0 : if (status != -ENOMEM) {
279 0 : spdk_accel_sequence_abort(bdev_io->u.bdev.accel_sequence);
280 0 : bdev_io->u.bdev.accel_sequence = NULL;
281 : }
282 :
283 0 : malloc_done(task, status);
284 0 : }
285 :
286 : static void
287 0 : malloc_sequence_done(void *ctx, int status)
288 : {
289 0 : struct malloc_task *task = ctx;
290 0 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(task);
291 :
292 0 : bdev_io->u.bdev.accel_sequence = NULL;
293 : /* Prevent bdev layer from retrying the request if the sequence failed with ENOMEM */
294 0 : malloc_done(task, status != -ENOMEM ? status : -EFAULT);
295 0 : }
296 :
297 : static void
298 0 : bdev_malloc_readv(struct malloc_disk *mdisk, struct spdk_io_channel *ch,
299 : struct malloc_task *task, struct spdk_bdev_io *bdev_io)
300 : {
301 : uint64_t len, offset, md_offset;
302 0 : int res = 0;
303 : size_t md_len;
304 :
305 0 : len = bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;
306 0 : offset = bdev_io->u.bdev.offset_blocks * bdev_io->bdev->blocklen;
307 :
308 0 : if (bdev_malloc_check_iov_len(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, len)) {
309 0 : spdk_bdev_io_complete(spdk_bdev_io_from_ctx(task),
310 : SPDK_BDEV_IO_STATUS_FAILED);
311 0 : return;
312 : }
313 :
314 0 : task->status = SPDK_BDEV_IO_STATUS_SUCCESS;
315 0 : task->num_outstanding = 0;
316 0 : task->iov.iov_base = mdisk->malloc_buf + offset;
317 0 : task->iov.iov_len = len;
318 :
319 0 : SPDK_DEBUGLOG(bdev_malloc, "read %zu bytes from offset %#" PRIx64 ", iovcnt=%d\n",
320 : len, offset, bdev_io->u.bdev.iovcnt);
321 :
322 0 : task->num_outstanding++;
323 0 : res = spdk_accel_append_copy(&bdev_io->u.bdev.accel_sequence, ch,
324 0 : bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
325 : bdev_io->u.bdev.memory_domain,
326 : bdev_io->u.bdev.memory_domain_ctx,
327 : &task->iov, 1, NULL, NULL, 0, NULL, NULL);
328 0 : if (spdk_unlikely(res != 0)) {
329 0 : malloc_sequence_fail(task, res);
330 0 : return;
331 : }
332 :
333 0 : spdk_accel_sequence_reverse(bdev_io->u.bdev.accel_sequence);
334 0 : spdk_accel_sequence_finish(bdev_io->u.bdev.accel_sequence, malloc_sequence_done, task);
335 :
336 0 : if (bdev_io->u.bdev.md_buf == NULL) {
337 0 : return;
338 : }
339 :
340 0 : md_len = bdev_io->u.bdev.num_blocks * bdev_io->bdev->md_len;
341 0 : md_offset = bdev_io->u.bdev.offset_blocks * bdev_io->bdev->md_len;
342 :
343 0 : SPDK_DEBUGLOG(bdev_malloc, "read metadata %zu bytes from offset%#" PRIx64 "\n",
344 : md_len, md_offset);
345 :
346 0 : task->num_outstanding++;
347 0 : res = spdk_accel_submit_copy(ch, bdev_io->u.bdev.md_buf, mdisk->malloc_md_buf + md_offset,
348 : md_len, 0, malloc_done, task);
349 0 : if (res != 0) {
350 0 : malloc_done(task, res);
351 : }
352 : }
353 :
354 : static void
355 0 : bdev_malloc_writev(struct malloc_disk *mdisk, struct spdk_io_channel *ch,
356 : struct malloc_task *task, struct spdk_bdev_io *bdev_io)
357 : {
358 : uint64_t len, offset, md_offset;
359 0 : int res = 0;
360 : size_t md_len;
361 :
362 0 : len = bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;
363 0 : offset = bdev_io->u.bdev.offset_blocks * bdev_io->bdev->blocklen;
364 :
365 0 : if (bdev_malloc_check_iov_len(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, len)) {
366 0 : spdk_bdev_io_complete(spdk_bdev_io_from_ctx(task),
367 : SPDK_BDEV_IO_STATUS_FAILED);
368 0 : return;
369 : }
370 :
371 0 : task->status = SPDK_BDEV_IO_STATUS_SUCCESS;
372 0 : task->num_outstanding = 0;
373 0 : task->iov.iov_base = mdisk->malloc_buf + offset;
374 0 : task->iov.iov_len = len;
375 :
376 0 : SPDK_DEBUGLOG(bdev_malloc, "wrote %zu bytes to offset %#" PRIx64 ", iovcnt=%d\n",
377 : len, offset, bdev_io->u.bdev.iovcnt);
378 :
379 0 : task->num_outstanding++;
380 0 : res = spdk_accel_append_copy(&bdev_io->u.bdev.accel_sequence, ch, &task->iov, 1, NULL, NULL,
381 0 : bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
382 : bdev_io->u.bdev.memory_domain,
383 : bdev_io->u.bdev.memory_domain_ctx, 0, NULL, NULL);
384 0 : if (spdk_unlikely(res != 0)) {
385 0 : malloc_sequence_fail(task, res);
386 0 : return;
387 : }
388 :
389 0 : spdk_accel_sequence_finish(bdev_io->u.bdev.accel_sequence, malloc_sequence_done, task);
390 :
391 0 : if (bdev_io->u.bdev.md_buf == NULL) {
392 0 : return;
393 : }
394 :
395 0 : md_len = bdev_io->u.bdev.num_blocks * bdev_io->bdev->md_len;
396 0 : md_offset = bdev_io->u.bdev.offset_blocks * bdev_io->bdev->md_len;
397 :
398 0 : SPDK_DEBUGLOG(bdev_malloc, "wrote metadata %zu bytes to offset %#" PRIx64 "\n",
399 : md_len, md_offset);
400 :
401 0 : task->num_outstanding++;
402 0 : res = spdk_accel_submit_copy(ch, mdisk->malloc_md_buf + md_offset, bdev_io->u.bdev.md_buf,
403 : md_len, 0, malloc_done, task);
404 0 : if (res != 0) {
405 0 : malloc_done(task, res);
406 : }
407 : }
408 :
409 : static int
410 0 : bdev_malloc_unmap(struct malloc_disk *mdisk,
411 : struct spdk_io_channel *ch,
412 : struct malloc_task *task,
413 : uint64_t offset,
414 : uint64_t byte_count)
415 : {
416 0 : task->status = SPDK_BDEV_IO_STATUS_SUCCESS;
417 0 : task->num_outstanding = 1;
418 :
419 0 : return spdk_accel_submit_fill(ch, mdisk->malloc_buf + offset, 0,
420 : byte_count, 0, malloc_done, task);
421 : }
422 :
423 : static void
424 0 : bdev_malloc_copy(struct malloc_disk *mdisk, struct spdk_io_channel *ch,
425 : struct malloc_task *task,
426 : uint64_t dst_offset, uint64_t src_offset, size_t len)
427 : {
428 0 : int64_t res = 0;
429 0 : void *dst = mdisk->malloc_buf + dst_offset;
430 0 : void *src = mdisk->malloc_buf + src_offset;
431 :
432 0 : SPDK_DEBUGLOG(bdev_malloc, "Copy %zu bytes from offset %#" PRIx64 " to offset %#" PRIx64 "\n",
433 : len, src_offset, dst_offset);
434 :
435 0 : task->status = SPDK_BDEV_IO_STATUS_SUCCESS;
436 0 : task->num_outstanding = 1;
437 :
438 0 : res = spdk_accel_submit_copy(ch, dst, src, len, 0, malloc_done, task);
439 0 : if (res != 0) {
440 0 : malloc_done(task, res);
441 : }
442 0 : }
443 :
444 : static int
445 0 : _bdev_malloc_submit_request(struct malloc_channel *mch, struct spdk_bdev_io *bdev_io)
446 : {
447 0 : struct malloc_task *task = (struct malloc_task *)bdev_io->driver_ctx;
448 0 : struct malloc_disk *disk = bdev_io->bdev->ctxt;
449 0 : uint32_t block_size = bdev_io->bdev->blocklen;
450 : int rc;
451 :
452 0 : switch (bdev_io->type) {
453 0 : case SPDK_BDEV_IO_TYPE_READ:
454 0 : if (bdev_io->u.bdev.iovs[0].iov_base == NULL) {
455 0 : assert(bdev_io->u.bdev.iovcnt == 1);
456 0 : assert(bdev_io->u.bdev.memory_domain == NULL);
457 0 : bdev_io->u.bdev.iovs[0].iov_base =
458 0 : disk->malloc_buf + bdev_io->u.bdev.offset_blocks * block_size;
459 0 : bdev_io->u.bdev.iovs[0].iov_len = bdev_io->u.bdev.num_blocks * block_size;
460 0 : malloc_complete_task(task, mch, SPDK_BDEV_IO_STATUS_SUCCESS);
461 0 : return 0;
462 : }
463 :
464 0 : bdev_malloc_readv(disk, mch->accel_channel, task, bdev_io);
465 0 : return 0;
466 :
467 0 : case SPDK_BDEV_IO_TYPE_WRITE:
468 0 : if (bdev_io->bdev->dif_type != SPDK_DIF_DISABLE) {
469 0 : rc = malloc_verify_pi(bdev_io);
470 0 : if (rc != 0) {
471 0 : malloc_complete_task(task, mch, SPDK_BDEV_IO_STATUS_FAILED);
472 0 : return 0;
473 : }
474 : }
475 :
476 0 : bdev_malloc_writev(disk, mch->accel_channel, task, bdev_io);
477 0 : return 0;
478 :
479 0 : case SPDK_BDEV_IO_TYPE_RESET:
480 0 : malloc_complete_task(task, mch, SPDK_BDEV_IO_STATUS_SUCCESS);
481 0 : return 0;
482 :
483 0 : case SPDK_BDEV_IO_TYPE_FLUSH:
484 0 : malloc_complete_task(task, mch, SPDK_BDEV_IO_STATUS_SUCCESS);
485 0 : return 0;
486 :
487 0 : case SPDK_BDEV_IO_TYPE_UNMAP:
488 0 : return bdev_malloc_unmap(disk, mch->accel_channel, task,
489 0 : bdev_io->u.bdev.offset_blocks * block_size,
490 0 : bdev_io->u.bdev.num_blocks * block_size);
491 :
492 0 : case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
493 : /* bdev_malloc_unmap is implemented with a call to mem_cpy_fill which zeroes out all of the requested bytes. */
494 0 : return bdev_malloc_unmap(disk, mch->accel_channel, task,
495 0 : bdev_io->u.bdev.offset_blocks * block_size,
496 0 : bdev_io->u.bdev.num_blocks * block_size);
497 :
498 0 : case SPDK_BDEV_IO_TYPE_ZCOPY:
499 0 : if (bdev_io->u.bdev.zcopy.start) {
500 : void *buf;
501 : size_t len;
502 :
503 0 : buf = disk->malloc_buf + bdev_io->u.bdev.offset_blocks * block_size;
504 0 : len = bdev_io->u.bdev.num_blocks * block_size;
505 0 : spdk_bdev_io_set_buf(bdev_io, buf, len);
506 :
507 : }
508 0 : malloc_complete_task(task, mch, SPDK_BDEV_IO_STATUS_SUCCESS);
509 0 : return 0;
510 0 : case SPDK_BDEV_IO_TYPE_ABORT:
511 0 : malloc_complete_task(task, mch, SPDK_BDEV_IO_STATUS_FAILED);
512 0 : return 0;
513 0 : case SPDK_BDEV_IO_TYPE_COPY:
514 0 : bdev_malloc_copy(disk, mch->accel_channel, task,
515 0 : bdev_io->u.bdev.offset_blocks * block_size,
516 0 : bdev_io->u.bdev.copy.src_offset_blocks * block_size,
517 0 : bdev_io->u.bdev.num_blocks * block_size);
518 0 : return 0;
519 :
520 0 : default:
521 0 : return -1;
522 : }
523 : return 0;
524 : }
525 :
526 : static void
527 0 : bdev_malloc_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
528 : {
529 0 : struct malloc_channel *mch = spdk_io_channel_get_ctx(ch);
530 :
531 0 : if (_bdev_malloc_submit_request(mch, bdev_io) != 0) {
532 0 : malloc_complete_task((struct malloc_task *)bdev_io->driver_ctx, mch,
533 : SPDK_BDEV_IO_STATUS_FAILED);
534 : }
535 0 : }
536 :
537 : static bool
538 0 : bdev_malloc_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type)
539 : {
540 0 : switch (io_type) {
541 0 : case SPDK_BDEV_IO_TYPE_READ:
542 : case SPDK_BDEV_IO_TYPE_WRITE:
543 : case SPDK_BDEV_IO_TYPE_FLUSH:
544 : case SPDK_BDEV_IO_TYPE_RESET:
545 : case SPDK_BDEV_IO_TYPE_UNMAP:
546 : case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
547 : case SPDK_BDEV_IO_TYPE_ZCOPY:
548 : case SPDK_BDEV_IO_TYPE_ABORT:
549 : case SPDK_BDEV_IO_TYPE_COPY:
550 0 : return true;
551 :
552 0 : default:
553 0 : return false;
554 : }
555 : }
556 :
557 : static struct spdk_io_channel *
558 0 : bdev_malloc_get_io_channel(void *ctx)
559 : {
560 0 : return spdk_get_io_channel(&g_malloc_disks);
561 : }
562 :
563 : static void
564 0 : bdev_malloc_write_json_config(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
565 : {
566 0 : spdk_json_write_object_begin(w);
567 :
568 0 : spdk_json_write_named_string(w, "method", "bdev_malloc_create");
569 :
570 0 : spdk_json_write_named_object_begin(w, "params");
571 0 : spdk_json_write_named_string(w, "name", bdev->name);
572 0 : spdk_json_write_named_uint64(w, "num_blocks", bdev->blockcnt);
573 0 : spdk_json_write_named_uint32(w, "block_size", bdev->blocklen);
574 0 : spdk_json_write_named_uint32(w, "physical_block_size", bdev->phys_blocklen);
575 0 : spdk_json_write_named_uuid(w, "uuid", &bdev->uuid);
576 0 : spdk_json_write_named_uint32(w, "optimal_io_boundary", bdev->optimal_io_boundary);
577 :
578 0 : spdk_json_write_object_end(w);
579 :
580 0 : spdk_json_write_object_end(w);
581 0 : }
582 :
583 : static int
584 0 : bdev_malloc_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size)
585 : {
586 0 : struct malloc_disk *malloc_disk = ctx;
587 : struct spdk_memory_domain *domain;
588 0 : int num_domains = 0;
589 :
590 0 : if (malloc_disk->disk.dif_type != SPDK_DIF_DISABLE) {
591 0 : return 0;
592 : }
593 :
594 : /* Report support for every memory domain */
595 0 : for (domain = spdk_memory_domain_get_first(NULL); domain != NULL;
596 0 : domain = spdk_memory_domain_get_next(domain, NULL)) {
597 0 : if (domains != NULL && num_domains < array_size) {
598 0 : domains[num_domains] = domain;
599 : }
600 0 : num_domains++;
601 : }
602 :
603 0 : return num_domains;
604 : }
605 :
606 : static bool
607 0 : bdev_malloc_accel_sequence_supported(void *ctx, enum spdk_bdev_io_type type)
608 : {
609 0 : struct malloc_disk *malloc_disk = ctx;
610 :
611 0 : if (malloc_disk->disk.dif_type != SPDK_DIF_DISABLE) {
612 0 : return false;
613 : }
614 :
615 0 : switch (type) {
616 0 : case SPDK_BDEV_IO_TYPE_READ:
617 : case SPDK_BDEV_IO_TYPE_WRITE:
618 0 : return true;
619 0 : default:
620 0 : return false;
621 : }
622 : }
623 :
624 : static const struct spdk_bdev_fn_table malloc_fn_table = {
625 : .destruct = bdev_malloc_destruct,
626 : .submit_request = bdev_malloc_submit_request,
627 : .io_type_supported = bdev_malloc_io_type_supported,
628 : .get_io_channel = bdev_malloc_get_io_channel,
629 : .write_config_json = bdev_malloc_write_json_config,
630 : .get_memory_domains = bdev_malloc_get_memory_domains,
631 : .accel_sequence_supported = bdev_malloc_accel_sequence_supported,
632 : };
633 :
634 : static int
635 0 : malloc_disk_setup_pi(struct malloc_disk *mdisk)
636 : {
637 0 : struct spdk_bdev *bdev = &mdisk->disk;
638 0 : struct spdk_dif_ctx dif_ctx;
639 0 : struct iovec iov, md_iov;
640 : int rc;
641 0 : struct spdk_dif_ctx_init_ext_opts dif_opts;
642 :
643 0 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
644 0 : dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;
645 : /* Set APPTAG|REFTAG_IGNORE to PI fields after creation of malloc bdev */
646 0 : rc = spdk_dif_ctx_init(&dif_ctx,
647 : bdev->blocklen,
648 : bdev->md_len,
649 0 : bdev->md_interleave,
650 0 : bdev->dif_is_head_of_md,
651 : bdev->dif_type,
652 : bdev->dif_check_flags,
653 : SPDK_DIF_REFTAG_IGNORE,
654 : 0xFFFF, SPDK_DIF_APPTAG_IGNORE,
655 : 0, 0, &dif_opts);
656 0 : if (rc != 0) {
657 0 : SPDK_ERRLOG("Initialization of DIF/DIX context failed\n");
658 0 : return rc;
659 : }
660 :
661 0 : iov.iov_base = mdisk->malloc_buf;
662 0 : iov.iov_len = bdev->blockcnt * bdev->blocklen;
663 :
664 0 : if (mdisk->disk.md_interleave) {
665 0 : rc = spdk_dif_generate(&iov, 1, bdev->blockcnt, &dif_ctx);
666 : } else {
667 0 : md_iov.iov_base = mdisk->malloc_md_buf;
668 0 : md_iov.iov_len = bdev->blockcnt * bdev->md_len;
669 :
670 0 : rc = spdk_dix_generate(&iov, 1, &md_iov, bdev->blockcnt, &dif_ctx);
671 : }
672 :
673 0 : if (rc != 0) {
674 0 : SPDK_ERRLOG("Formatting by DIF/DIX failed\n");
675 : }
676 :
677 0 : return rc;
678 : }
679 :
680 : int
681 0 : create_malloc_disk(struct spdk_bdev **bdev, const struct malloc_bdev_opts *opts)
682 : {
683 : struct malloc_disk *mdisk;
684 : uint32_t block_size;
685 : int rc;
686 :
687 0 : assert(opts != NULL);
688 :
689 0 : if (opts->num_blocks == 0) {
690 0 : SPDK_ERRLOG("Disk num_blocks must be greater than 0");
691 0 : return -EINVAL;
692 : }
693 :
694 0 : if (opts->block_size % 512) {
695 0 : SPDK_ERRLOG("Data block size must be 512 bytes aligned\n");
696 0 : return -EINVAL;
697 : }
698 :
699 0 : if (opts->physical_block_size % 512) {
700 0 : SPDK_ERRLOG("Physical block must be 512 bytes aligned\n");
701 0 : return -EINVAL;
702 : }
703 :
704 0 : switch (opts->md_size) {
705 0 : case 0:
706 : case 8:
707 : case 16:
708 : case 32:
709 : case 64:
710 : case 128:
711 0 : break;
712 0 : default:
713 0 : SPDK_ERRLOG("metadata size %u is not supported\n", opts->md_size);
714 0 : return -EINVAL;
715 : }
716 :
717 0 : if (opts->md_interleave) {
718 0 : block_size = opts->block_size + opts->md_size;
719 : } else {
720 0 : block_size = opts->block_size;
721 : }
722 :
723 0 : if (opts->dif_type < SPDK_DIF_DISABLE || opts->dif_type > SPDK_DIF_TYPE3) {
724 0 : SPDK_ERRLOG("DIF type is invalid\n");
725 0 : return -EINVAL;
726 : }
727 :
728 0 : if (opts->dif_type != SPDK_DIF_DISABLE && opts->md_size == 0) {
729 0 : SPDK_ERRLOG("Metadata size should not be zero if DIF is enabled\n");
730 0 : return -EINVAL;
731 : }
732 :
733 0 : mdisk = calloc(1, sizeof(*mdisk));
734 0 : if (!mdisk) {
735 0 : SPDK_ERRLOG("mdisk calloc() failed\n");
736 0 : return -ENOMEM;
737 : }
738 :
739 : /*
740 : * Allocate the large backend memory buffer from pinned memory.
741 : *
742 : * TODO: need to pass a hint so we know which socket to allocate
743 : * from on multi-socket systems.
744 : */
745 0 : mdisk->malloc_buf = spdk_zmalloc(opts->num_blocks * block_size, 2 * 1024 * 1024, NULL,
746 : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
747 0 : if (!mdisk->malloc_buf) {
748 0 : SPDK_ERRLOG("malloc_buf spdk_zmalloc() failed\n");
749 0 : malloc_disk_free(mdisk);
750 0 : return -ENOMEM;
751 : }
752 :
753 0 : if (!opts->md_interleave && opts->md_size != 0) {
754 0 : mdisk->malloc_md_buf = spdk_zmalloc(opts->num_blocks * opts->md_size, 2 * 1024 * 1024, NULL,
755 : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
756 0 : if (!mdisk->malloc_md_buf) {
757 0 : SPDK_ERRLOG("malloc_md_buf spdk_zmalloc() failed\n");
758 0 : malloc_disk_free(mdisk);
759 0 : return -ENOMEM;
760 : }
761 : }
762 :
763 0 : if (opts->name) {
764 0 : mdisk->disk.name = strdup(opts->name);
765 : } else {
766 : /* Auto-generate a name */
767 0 : mdisk->disk.name = spdk_sprintf_alloc("Malloc%d", malloc_disk_count);
768 0 : malloc_disk_count++;
769 : }
770 0 : if (!mdisk->disk.name) {
771 0 : malloc_disk_free(mdisk);
772 0 : return -ENOMEM;
773 : }
774 0 : mdisk->disk.product_name = "Malloc disk";
775 :
776 0 : mdisk->disk.write_cache = 1;
777 0 : mdisk->disk.blocklen = block_size;
778 0 : mdisk->disk.phys_blocklen = opts->physical_block_size;
779 0 : mdisk->disk.blockcnt = opts->num_blocks;
780 0 : mdisk->disk.md_len = opts->md_size;
781 0 : mdisk->disk.md_interleave = opts->md_interleave;
782 0 : mdisk->disk.dif_type = opts->dif_type;
783 0 : mdisk->disk.dif_is_head_of_md = opts->dif_is_head_of_md;
784 : /* Current block device layer API does not propagate
785 : * any DIF related information from user. So, we can
786 : * not generate or verify Application Tag.
787 : */
788 0 : switch (opts->dif_type) {
789 0 : case SPDK_DIF_TYPE1:
790 : case SPDK_DIF_TYPE2:
791 0 : mdisk->disk.dif_check_flags = SPDK_DIF_FLAGS_GUARD_CHECK |
792 : SPDK_DIF_FLAGS_REFTAG_CHECK;
793 0 : break;
794 0 : case SPDK_DIF_TYPE3:
795 0 : mdisk->disk.dif_check_flags = SPDK_DIF_FLAGS_GUARD_CHECK;
796 0 : break;
797 0 : case SPDK_DIF_DISABLE:
798 0 : break;
799 : }
800 :
801 0 : if (opts->dif_type != SPDK_DIF_DISABLE) {
802 0 : rc = malloc_disk_setup_pi(mdisk);
803 0 : if (rc) {
804 0 : SPDK_ERRLOG("Failed to set up protection information.\n");
805 0 : malloc_disk_free(mdisk);
806 0 : return rc;
807 : }
808 : }
809 :
810 0 : if (opts->optimal_io_boundary) {
811 0 : mdisk->disk.optimal_io_boundary = opts->optimal_io_boundary;
812 0 : mdisk->disk.split_on_optimal_io_boundary = true;
813 : }
814 0 : if (!spdk_uuid_is_null(&opts->uuid)) {
815 0 : spdk_uuid_copy(&mdisk->disk.uuid, &opts->uuid);
816 : }
817 :
818 0 : mdisk->disk.max_copy = 0;
819 0 : mdisk->disk.ctxt = mdisk;
820 0 : mdisk->disk.fn_table = &malloc_fn_table;
821 0 : mdisk->disk.module = &malloc_if;
822 :
823 0 : rc = spdk_bdev_register(&mdisk->disk);
824 0 : if (rc) {
825 0 : malloc_disk_free(mdisk);
826 0 : return rc;
827 : }
828 :
829 0 : *bdev = &(mdisk->disk);
830 :
831 0 : TAILQ_INSERT_TAIL(&g_malloc_disks, mdisk, link);
832 :
833 0 : return rc;
834 : }
835 :
836 : void
837 0 : delete_malloc_disk(const char *name, spdk_delete_malloc_complete cb_fn, void *cb_arg)
838 : {
839 : int rc;
840 :
841 0 : rc = spdk_bdev_unregister_by_name(name, &malloc_if, cb_fn, cb_arg);
842 0 : if (rc != 0) {
843 0 : cb_fn(cb_arg, rc);
844 : }
845 0 : }
846 :
847 : static int
848 0 : malloc_completion_poller(void *ctx)
849 : {
850 0 : struct malloc_channel *ch = ctx;
851 : struct malloc_task *task;
852 0 : TAILQ_HEAD(, malloc_task) completed_tasks;
853 0 : uint32_t num_completions = 0;
854 :
855 0 : TAILQ_INIT(&completed_tasks);
856 0 : TAILQ_SWAP(&completed_tasks, &ch->completed_tasks, malloc_task, tailq);
857 :
858 0 : while (!TAILQ_EMPTY(&completed_tasks)) {
859 0 : task = TAILQ_FIRST(&completed_tasks);
860 0 : TAILQ_REMOVE(&completed_tasks, task, tailq);
861 0 : spdk_bdev_io_complete(spdk_bdev_io_from_ctx(task), task->status);
862 0 : num_completions++;
863 : }
864 :
865 0 : return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;
866 : }
867 :
868 : static int
869 0 : malloc_create_channel_cb(void *io_device, void *ctx)
870 : {
871 0 : struct malloc_channel *ch = ctx;
872 :
873 0 : ch->accel_channel = spdk_accel_get_io_channel();
874 0 : if (!ch->accel_channel) {
875 0 : SPDK_ERRLOG("Failed to get accel framework's IO channel\n");
876 0 : return -ENOMEM;
877 : }
878 :
879 0 : ch->completion_poller = SPDK_POLLER_REGISTER(malloc_completion_poller, ch, 0);
880 0 : if (!ch->completion_poller) {
881 0 : SPDK_ERRLOG("Failed to register malloc completion poller\n");
882 0 : spdk_put_io_channel(ch->accel_channel);
883 0 : return -ENOMEM;
884 : }
885 :
886 0 : TAILQ_INIT(&ch->completed_tasks);
887 :
888 0 : return 0;
889 : }
890 :
891 : static void
892 0 : malloc_destroy_channel_cb(void *io_device, void *ctx)
893 : {
894 0 : struct malloc_channel *ch = ctx;
895 :
896 0 : assert(TAILQ_EMPTY(&ch->completed_tasks));
897 :
898 0 : spdk_put_io_channel(ch->accel_channel);
899 0 : spdk_poller_unregister(&ch->completion_poller);
900 0 : }
901 :
902 : static int
903 0 : bdev_malloc_initialize(void)
904 : {
905 : /* This needs to be reset for each reinitialization of submodules.
906 : * Otherwise after enough devices or reinitializations the value gets too high.
907 : * TODO: Make malloc bdev name mandatory and remove this counter. */
908 0 : malloc_disk_count = 0;
909 :
910 0 : spdk_io_device_register(&g_malloc_disks, malloc_create_channel_cb,
911 : malloc_destroy_channel_cb, sizeof(struct malloc_channel),
912 : "bdev_malloc");
913 :
914 0 : return 0;
915 : }
916 :
917 : static void
918 0 : bdev_malloc_deinitialize(void)
919 : {
920 0 : spdk_io_device_unregister(&g_malloc_disks, NULL);
921 0 : }
922 :
923 0 : SPDK_LOG_REGISTER_COMPONENT(bdev_malloc)
|