Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright 2023 Solidigm All Rights Reserved
3 : */
4 :
5 : #include "ftl_core.h"
6 : #include "ftl_io.h"
7 : #include "ftl_layout.h"
8 : #include "utils/ftl_defs.h"
9 :
10 : struct ftl_pl2_log_item {
11 : uint64_t lba;
12 : uint64_t num_blocks;
13 : uint64_t seq_id;
14 : ftl_addr addr;
15 : };
16 : #define FTL_P2L_LOG_ITEMS_IN_PAGE ((FTL_BLOCK_SIZE - sizeof(union ftl_md_vss)) / sizeof(struct ftl_pl2_log_item))
17 : #define FTL_P2L_LOG_PAGE_COUNT_DEFAULT 128
18 :
19 : struct ftl_p2l_log_page {
20 : union ftl_md_vss hdr;
21 : struct ftl_pl2_log_item items[FTL_P2L_LOG_ITEMS_IN_PAGE];
22 : };
23 : SPDK_STATIC_ASSERT(sizeof(struct ftl_p2l_log_page) == FTL_BLOCK_SIZE, "Invalid size of P2L page");
24 :
25 : struct ftl_p2l_log_page_ctrl {
26 : struct ftl_p2l_log_page page;
27 : struct ftl_p2l_log *p2l;
28 : uint64_t entry_idx;
29 : TAILQ_HEAD(, ftl_io) ios;
30 : struct ftl_md_io_entry_ctx md_ctx;
31 : };
32 :
33 : struct ftl_p2l_log {
34 : struct spdk_ftl_dev *dev;
35 : TAILQ_ENTRY(ftl_p2l_log) link;
36 : TAILQ_HEAD(, ftl_io) ios;
37 : struct ftl_md *md;
38 : uint64_t seq_id;
39 : struct ftl_mempool *page_pool;
40 : uint64_t entry_idx;
41 : uint64_t entry_max;
42 : ftl_p2l_log_cb cb_fn;
43 : uint32_t ref_cnt;
44 : bool in_use;
45 :
46 : struct {
47 : spdk_ftl_fn cb_fn;
48 : void *cb_arg;
49 : ftl_p2l_log_rd_cb cb_rd;
50 : uint64_t qd;
51 : uint64_t idx;
52 : uint64_t seq_id;
53 : int result;
54 : } read_ctx;
55 : };
56 :
57 : static void p2l_log_page_io(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl);
58 : static void ftl_p2l_log_read_process(struct ftl_p2l_log *p2l);
59 :
60 : static struct ftl_p2l_log *
61 0 : p2l_log_create(struct spdk_ftl_dev *dev, uint32_t region_type)
62 : {
63 0 : struct ftl_p2l_log *p2l;
64 :
65 0 : p2l = calloc(1, sizeof(struct ftl_p2l_log));
66 0 : if (!p2l) {
67 0 : return NULL;
68 : }
69 :
70 0 : TAILQ_INIT(&p2l->ios);
71 0 : p2l->dev = dev;
72 0 : p2l->md = dev->layout.md[region_type];
73 0 : p2l->entry_max = ftl_md_get_buffer_size(p2l->md) / FTL_BLOCK_SIZE;
74 0 : p2l->page_pool = ftl_mempool_create(FTL_P2L_LOG_PAGE_COUNT_DEFAULT,
75 : sizeof(struct ftl_p2l_log_page_ctrl),
76 : FTL_BLOCK_SIZE, SPDK_ENV_SOCKET_ID_ANY);
77 0 : if (!p2l->page_pool) {
78 0 : goto ERROR;
79 : }
80 :
81 0 : return p2l;
82 : ERROR:
83 0 : free(p2l);
84 0 : return NULL;
85 0 : }
86 :
87 : static void
88 0 : p2l_log_destroy(struct ftl_p2l_log *p2l)
89 : {
90 0 : if (!p2l) {
91 0 : return;
92 : }
93 :
94 0 : ftl_mempool_destroy(p2l->page_pool);
95 0 : free(p2l);
96 0 : }
97 :
98 : static struct ftl_p2l_log_page_ctrl *
99 0 : p2l_log_get_page(struct ftl_p2l_log *p2l)
100 : {
101 0 : struct ftl_p2l_log_page_ctrl *ctrl;
102 :
103 0 : ctrl = ftl_mempool_get(p2l->page_pool);
104 0 : if (!ctrl) {
105 0 : return NULL;
106 : }
107 :
108 : /* Initialize P2L header */
109 0 : ctrl->page.hdr.p2l_ckpt.seq_id = p2l->seq_id;
110 0 : ctrl->page.hdr.p2l_ckpt.count = 0;
111 0 : ctrl->page.hdr.p2l_ckpt.p2l_checksum = 0;
112 0 : ctrl->page.hdr.p2l_ckpt.idx = ctrl->entry_idx = p2l->entry_idx;
113 :
114 : /* Initialize the page control structure */
115 0 : ctrl->p2l = p2l;
116 0 : TAILQ_INIT(&ctrl->ios);
117 :
118 : /* Increase P2L page index */
119 0 : p2l->entry_idx++;
120 :
121 : /* Check if the index exceeding the buffer size */
122 0 : ftl_bug(p2l->entry_idx > p2l->entry_max);
123 :
124 0 : return ctrl;
125 0 : }
126 :
127 : static bool
128 0 : l2p_log_page_is_full(struct ftl_p2l_log_page_ctrl *ctrl)
129 : {
130 0 : return ctrl->page.hdr.p2l_ckpt.count == FTL_P2L_LOG_ITEMS_IN_PAGE;
131 : }
132 :
133 : static void
134 0 : p2l_log_page_free(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl)
135 : {
136 0 : ftl_mempool_put(p2l->page_pool, ctrl);
137 0 : }
138 :
139 : static void
140 0 : p2l_log_handle_io_error(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl)
141 : {
142 : #ifdef SPDK_FTL_RETRY_ON_ERROR
143 : p2l_log_page_io(p2l, ctrl);
144 : #else
145 0 : ftl_abort();
146 : #endif
147 0 : }
148 :
149 : static uint32_t
150 0 : p2l_log_page_crc(struct ftl_p2l_log_page *page)
151 : {
152 0 : uint32_t crc = 0;
153 0 : void *buffer = page;
154 0 : size_t size = sizeof(*page);
155 0 : size_t offset = offsetof(struct ftl_p2l_log_page, hdr.p2l_ckpt.p2l_checksum);
156 :
157 0 : crc = spdk_crc32c_update(buffer, offset, crc);
158 0 : buffer += offset + sizeof(page->hdr.p2l_ckpt.p2l_checksum);
159 0 : size -= offset + sizeof(page->hdr.p2l_ckpt.p2l_checksum);
160 :
161 0 : return spdk_crc32c_update(buffer, size, crc);
162 0 : }
163 :
164 : static void
165 0 : p2l_log_page_io_cb(int status, void *arg)
166 : {
167 0 : struct ftl_p2l_log_page_ctrl *ctrl = arg;
168 0 : struct ftl_p2l_log *p2l = ctrl->p2l;
169 0 : struct ftl_io *io;
170 :
171 0 : if (status) {
172 0 : p2l_log_handle_io_error(p2l, ctrl);
173 0 : return;
174 : }
175 :
176 0 : while ((io = TAILQ_FIRST(&ctrl->ios))) {
177 0 : TAILQ_REMOVE(&ctrl->ios, io, queue_entry);
178 0 : p2l->cb_fn(io);
179 : }
180 :
181 0 : p2l_log_page_free(p2l, ctrl);
182 0 : }
183 :
184 : static void
185 0 : p2l_log_page_io(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl)
186 : {
187 0 : ctrl->page.hdr.p2l_ckpt.p2l_checksum = p2l_log_page_crc(&ctrl->page);
188 :
189 0 : ftl_md_persist_entries(p2l->md, ctrl->page.hdr.p2l_ckpt.idx, 1, &ctrl->page, NULL,
190 : p2l_log_page_io_cb,
191 0 : ctrl, &ctrl->md_ctx);
192 0 : }
193 :
194 : static void
195 0 : p2l_log_add_io(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl, struct ftl_io *io)
196 : {
197 0 : uint64_t i = ctrl->page.hdr.p2l_ckpt.count++;
198 :
199 0 : assert(i < FTL_P2L_LOG_ITEMS_IN_PAGE);
200 0 : ctrl->page.items[i].lba = io->lba;
201 0 : ctrl->page.items[i].num_blocks = io->num_blocks;
202 0 : ctrl->page.items[i].seq_id = io->nv_cache_chunk->md->seq_id;
203 0 : ctrl->page.items[i].addr = io->addr;
204 :
205 : /* TODO Make sure P2L map is updated respectively */
206 :
207 0 : TAILQ_REMOVE(&p2l->ios, io, queue_entry);
208 0 : TAILQ_INSERT_TAIL(&ctrl->ios, io, queue_entry);
209 0 : }
210 :
211 : void
212 0 : ftl_p2l_log_io(struct ftl_p2l_log *p2l, struct ftl_io *io)
213 : {
214 0 : TAILQ_INSERT_TAIL(&p2l->ios, io, queue_entry);
215 0 : }
216 :
217 : static void
218 0 : p2l_log_flush(struct ftl_p2l_log *p2l)
219 : {
220 0 : struct ftl_p2l_log_page_ctrl *ctrl = NULL;
221 0 : struct ftl_io *io;
222 :
223 0 : while ((io = TAILQ_FIRST(&p2l->ios))) {
224 0 : if (!ctrl) {
225 0 : ctrl = p2l_log_get_page(p2l);
226 0 : if (!ctrl) {
227 : /* No page at the moment, try next time */
228 0 : break;
229 : }
230 0 : }
231 :
232 0 : p2l_log_add_io(p2l, ctrl, io);
233 :
234 0 : if (l2p_log_page_is_full(ctrl)) {
235 0 : p2l_log_page_io(p2l, ctrl);
236 0 : ctrl = NULL;
237 0 : }
238 : }
239 :
240 0 : if (ctrl) {
241 0 : p2l_log_page_io(p2l, ctrl);
242 0 : }
243 0 : }
244 :
245 : void
246 0 : ftl_p2l_log_flush(struct spdk_ftl_dev *dev)
247 : {
248 0 : struct ftl_p2l_log *p2l;
249 :
250 0 : TAILQ_FOREACH(p2l, &dev->p2l_ckpt.log.inuse, link) {
251 0 : p2l_log_flush(p2l);
252 0 : }
253 0 : }
254 :
255 : uint64_t
256 0 : ftl_p2l_log_get_md_blocks_required(struct spdk_ftl_dev *dev, uint64_t write_unit_blocks,
257 : uint64_t max_user_data_blocks)
258 : {
259 0 : return spdk_divide_round_up(max_user_data_blocks, write_unit_blocks);
260 : }
261 :
262 : int
263 0 : ftl_p2l_log_init(struct spdk_ftl_dev *dev)
264 : {
265 0 : struct ftl_p2l_log *p2l;
266 0 : uint32_t region_type;
267 :
268 0 : TAILQ_INIT(&dev->p2l_ckpt.log.free);
269 0 : TAILQ_INIT(&dev->p2l_ckpt.log.inuse);
270 :
271 0 : for (region_type = FTL_LAYOUT_REGION_TYPE_P2L_LOG_IO_MIN;
272 0 : region_type <= FTL_LAYOUT_REGION_TYPE_P2L_LOG_IO_MAX;
273 0 : region_type++) {
274 0 : p2l = p2l_log_create(dev, region_type);
275 0 : if (!p2l) {
276 0 : return -ENOMEM;
277 : }
278 :
279 0 : TAILQ_INSERT_TAIL(&dev->p2l_ckpt.log.free, p2l, link);
280 0 : }
281 :
282 0 : return 0;
283 0 : }
284 :
285 : void
286 0 : ftl_p2l_log_deinit(struct spdk_ftl_dev *dev)
287 : {
288 0 : struct ftl_p2l_log *p2l, *p2l_next;
289 :
290 0 : TAILQ_FOREACH_SAFE(p2l, &dev->p2l_ckpt.log.free, link, p2l_next) {
291 0 : TAILQ_REMOVE(&dev->p2l_ckpt.log.free, p2l, link);
292 0 : p2l_log_destroy(p2l);
293 0 : }
294 :
295 0 : TAILQ_FOREACH_SAFE(p2l, &dev->p2l_ckpt.log.inuse, link, p2l_next) {
296 0 : TAILQ_REMOVE(&dev->p2l_ckpt.log.inuse, p2l, link);
297 0 : p2l_log_destroy(p2l);
298 0 : }
299 0 : }
300 :
301 : enum ftl_layout_region_type
302 0 : ftl_p2l_log_type(struct ftl_p2l_log *p2l) {
303 0 : return p2l->md->region->type;
304 : }
305 :
306 : struct ftl_p2l_log *
307 0 : ftl_p2l_log_acquire(struct spdk_ftl_dev *dev, uint64_t seq_id, ftl_p2l_log_cb cb)
308 : {
309 0 : struct ftl_p2l_log *p2l;
310 :
311 0 : p2l = TAILQ_FIRST(&dev->p2l_ckpt.log.free);
312 0 : assert(p2l);
313 0 : TAILQ_REMOVE(&dev->p2l_ckpt.log.free, p2l, link);
314 0 : TAILQ_INSERT_TAIL(&dev->p2l_ckpt.log.inuse, p2l, link);
315 :
316 0 : p2l->entry_idx = 0;
317 0 : p2l->seq_id = seq_id;
318 0 : p2l->cb_fn = cb;
319 :
320 0 : return p2l;
321 0 : }
322 :
323 : void
324 0 : ftl_p2l_log_release(struct spdk_ftl_dev *dev, struct ftl_p2l_log *p2l)
325 : {
326 0 : assert(p2l);
327 :
328 : /* TODO: Add assert if no ongoing IOs on the P2L log */
329 : /* TODO: Add assert if the P2L log already open */
330 :
331 0 : TAILQ_REMOVE(&dev->p2l_ckpt.log.inuse, p2l, link);
332 0 : TAILQ_INSERT_TAIL(&dev->p2l_ckpt.log.free, p2l, link);
333 0 : }
334 :
335 : static struct ftl_p2l_log *
336 0 : p2l_log_get(struct spdk_ftl_dev *dev, enum ftl_layout_region_type type)
337 : {
338 0 : struct ftl_p2l_log *p2l_Log;
339 :
340 0 : TAILQ_FOREACH(p2l_Log, &dev->p2l_ckpt.log.free, link) {
341 0 : if (type == p2l_Log->md->region->type) {
342 0 : return p2l_Log;
343 : }
344 0 : }
345 :
346 0 : TAILQ_FOREACH(p2l_Log, &dev->p2l_ckpt.log.inuse, link) {
347 0 : if (type == p2l_Log->md->region->type) {
348 0 : return p2l_Log;
349 : }
350 0 : }
351 :
352 0 : return NULL;
353 0 : }
354 :
355 : static bool
356 0 : p2l_log_read_in_progress(struct ftl_p2l_log *p2l)
357 : {
358 0 : return p2l->read_ctx.cb_fn ? true : false;
359 : }
360 :
361 : static bool
362 0 : ftl_p2l_log_read_is_next(struct ftl_p2l_log *p2l)
363 : {
364 0 : if (p2l->read_ctx.result) {
365 0 : return false;
366 : }
367 :
368 0 : return p2l->read_ctx.idx < p2l->entry_max;
369 0 : }
370 :
371 : static bool
372 0 : ftl_p2l_log_read_is_qd(struct ftl_p2l_log *p2l)
373 : {
374 0 : return p2l->read_ctx.qd > 0;
375 : }
376 :
377 : static bool
378 0 : ftl_p2l_log_read_is_finished(struct ftl_p2l_log *p2l)
379 : {
380 0 : if (ftl_p2l_log_read_is_next(p2l) || ftl_p2l_log_read_is_qd(p2l)) {
381 0 : return false;
382 : }
383 :
384 0 : return true;
385 0 : }
386 :
387 : static void
388 0 : ftl_p2l_log_read_finish(struct ftl_p2l_log *p2l)
389 : {
390 0 : spdk_ftl_fn cb_fn = p2l->read_ctx.cb_fn;
391 0 : void *cb_arg = p2l->read_ctx.cb_arg;
392 0 : int result = p2l->read_ctx.result;
393 :
394 0 : memset(&p2l->read_ctx, 0, sizeof(p2l->read_ctx));
395 0 : cb_fn(cb_arg, result);
396 0 : }
397 :
398 : static void
399 0 : ftl_p2l_log_read_visit(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl)
400 : {
401 0 : struct ftl_p2l_log_page *page = &ctrl->page;
402 0 : struct spdk_ftl_dev *dev = p2l->dev;
403 0 : ftl_p2l_log_rd_cb cb_rd = p2l->read_ctx.cb_rd;
404 0 : void *cb_arg = p2l->read_ctx.cb_arg;
405 0 : uint64_t crc = p2l_log_page_crc(&ctrl->page);
406 0 : uint64_t i, j;
407 0 : int rc = 0;
408 :
409 0 : ftl_bug(ctrl->entry_idx > p2l->entry_max);
410 :
411 0 : if (p2l->read_ctx.seq_id != page->hdr.p2l_ckpt.seq_id) {
412 : /* This page contains entires older than the owner's sequence ID */
413 0 : return;
414 : }
415 :
416 0 : if (ctrl->entry_idx != page->hdr.p2l_ckpt.idx) {
417 0 : FTL_ERRLOG(p2l->dev, "Read P2L IO Logs ERROR, invalid index, type %d\n",
418 : p2l->md->region->type);
419 0 : p2l->read_ctx.result = -EINVAL;
420 0 : return;
421 : }
422 :
423 0 : if (crc != page->hdr.p2l_ckpt.p2l_checksum) {
424 0 : FTL_ERRLOG(p2l->dev, "Read P2L IO Log ERROR, CRC problem, type %d\n",
425 : p2l->md->region->type);
426 0 : p2l->read_ctx.result = -EINVAL;
427 0 : return;
428 : }
429 :
430 0 : if (page->hdr.p2l_ckpt.count > SPDK_COUNTOF(page->items)) {
431 0 : FTL_ERRLOG(p2l->dev, "Read P2L IO Log ERROR, inconsistent format, type %d\n",
432 : p2l->md->region->type);
433 0 : p2l->read_ctx.result = -EINVAL;
434 0 : return;
435 : }
436 :
437 0 : for (i = 0; i < page->hdr.p2l_ckpt.count; i++) {
438 0 : struct ftl_pl2_log_item *item = &page->items[i];
439 :
440 0 : for (j = 0; j < item->num_blocks; j++) {
441 0 : rc = cb_rd(dev, cb_arg, item->lba + j, item->addr + j, item->seq_id);
442 0 : if (rc) {
443 0 : p2l->read_ctx.result = rc;
444 0 : break;
445 : }
446 0 : }
447 :
448 0 : if (rc) {
449 0 : break;
450 : }
451 0 : }
452 0 : }
453 :
454 : static void
455 0 : ftl_p2l_log_read_cb(int status, void *arg)
456 : {
457 0 : struct ftl_p2l_log_page_ctrl *ctrl = arg;
458 0 : struct ftl_p2l_log *p2l = ctrl->p2l;
459 :
460 0 : assert(p2l->read_ctx.qd > 0);
461 0 : p2l->read_ctx.qd--;
462 :
463 0 : if (status) {
464 0 : p2l->read_ctx.result = status;
465 0 : } else {
466 0 : ftl_p2l_log_read_visit(p2l, ctrl);
467 : }
468 :
469 : /* Release page control */
470 0 : ftl_mempool_put(p2l->page_pool, ctrl);
471 0 : ftl_p2l_log_read_process(p2l);
472 0 : }
473 :
474 : static void
475 0 : ftl_p2l_log_read_process(struct ftl_p2l_log *p2l)
476 : {
477 0 : struct ftl_p2l_log_page_ctrl *ctrl;
478 :
479 0 : while (ftl_p2l_log_read_is_next(p2l)) {
480 0 : ctrl = ftl_mempool_get(p2l->page_pool);
481 0 : if (!ctrl) {
482 0 : break;
483 : }
484 :
485 0 : ctrl->p2l = p2l;
486 0 : ctrl->entry_idx = p2l->read_ctx.idx++;
487 :
488 : /* Check if the index exceeding the buffer size */
489 0 : ftl_bug(p2l->read_ctx.idx > p2l->entry_max);
490 :
491 0 : p2l->read_ctx.qd++;
492 0 : ftl_md_read_entry(p2l->md, ctrl->entry_idx, &ctrl->page, NULL,
493 0 : ftl_p2l_log_read_cb, ctrl, &ctrl->md_ctx);
494 : }
495 :
496 0 : if (ftl_p2l_log_read_is_finished(p2l)) {
497 0 : ftl_p2l_log_read_finish(p2l);
498 0 : }
499 0 : }
500 :
501 : int
502 0 : ftl_p2l_log_read(struct spdk_ftl_dev *dev, enum ftl_layout_region_type type, uint64_t seq_id,
503 : spdk_ftl_fn cb_fn, void *cb_arg, ftl_p2l_log_rd_cb cb_rd)
504 : {
505 0 : struct ftl_p2l_log *p2l_log = p2l_log_get(dev, type);
506 :
507 0 : if (!p2l_log) {
508 0 : FTL_ERRLOG(dev, "Read P2L IO Log ERROR, no such log, type %d\n", type);
509 0 : return -ENODEV;
510 : }
511 0 : if (p2l_log_read_in_progress(p2l_log)) {
512 0 : FTL_ERRLOG(dev, "Read P2L IO Log ERROR, read busy, type %d\n", type);
513 0 : return -EBUSY;
514 : }
515 :
516 0 : memset(&p2l_log->read_ctx, 0, sizeof(p2l_log->read_ctx));
517 0 : p2l_log->read_ctx.cb_fn = cb_fn;
518 0 : p2l_log->read_ctx.cb_arg = cb_arg;
519 0 : p2l_log->read_ctx.cb_rd = cb_rd;
520 0 : p2l_log->read_ctx.seq_id = seq_id;
521 :
522 0 : ftl_p2l_log_read_process(p2l_log);
523 0 : if (ftl_p2l_log_read_is_qd(p2l_log)) {
524 : /* Read in progress */
525 0 : return 0;
526 : } else {
527 0 : FTL_ERRLOG(dev, "Read P2L IO Log ERROR, operation not started, type %d\n", type);
528 0 : return -EINVAL;
529 : }
530 0 : }
|
