Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2022 Intel Corporation.
3 : * Copyright 2023 Solidigm All Rights Reserved
4 : * All rights reserved.
5 : */
6 :
7 : #include "spdk/stdinc.h"
8 : #include "spdk/queue.h"
9 : #include "spdk/bdev_module.h"
10 :
11 : #include "ftl_core.h"
12 : #include "ftl_band.h"
13 : #include "ftl_internal.h"
14 :
15 : static void
16 0 : write_rq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg)
17 : {
18 0 : struct ftl_rq *rq = arg;
19 0 : struct spdk_ftl_dev *dev = rq->dev;
20 :
21 0 : ftl_stats_bdev_io_completed(dev, rq->owner.compaction ? FTL_STATS_TYPE_CMP : FTL_STATS_TYPE_GC,
22 : bdev_io);
23 0 : spdk_bdev_free_io(bdev_io);
24 :
25 0 : rq->success = success;
26 0 : if (spdk_likely(rq->success)) {
27 0 : ftl_p2l_ckpt_issue(rq);
28 : } else {
29 : #ifdef SPDK_FTL_RETRY_ON_ERROR
30 : assert(rq->io.band->queue_depth > 0);
31 : rq->io.band->queue_depth--;
32 : rq->owner.cb(rq);
33 :
34 : #else
35 0 : ftl_abort();
36 : #endif
37 : }
38 0 : }
39 :
40 : static void
41 0 : ftl_band_rq_bdev_write(void *_rq)
42 : {
43 0 : struct ftl_rq *rq = _rq;
44 0 : struct ftl_band *band = rq->io.band;
45 0 : struct spdk_ftl_dev *dev = band->dev;
46 : int rc;
47 :
48 0 : rc = spdk_bdev_write_blocks(dev->base_bdev_desc, dev->base_ioch,
49 : rq->io_payload, rq->io.addr, rq->num_blocks,
50 : write_rq_end, rq);
51 :
52 0 : if (spdk_unlikely(rc)) {
53 0 : if (rc == -ENOMEM) {
54 0 : struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc);
55 0 : rq->io.bdev_io_wait.bdev = bdev;
56 0 : rq->io.bdev_io_wait.cb_fn = ftl_band_rq_bdev_write;
57 0 : rq->io.bdev_io_wait.cb_arg = rq;
58 0 : spdk_bdev_queue_io_wait(bdev, dev->base_ioch, &rq->io.bdev_io_wait);
59 : } else {
60 0 : ftl_abort();
61 : }
62 : }
63 0 : }
64 :
65 : void
66 0 : ftl_band_rq_write(struct ftl_band *band, struct ftl_rq *rq)
67 : {
68 0 : struct spdk_ftl_dev *dev = band->dev;
69 :
70 0 : rq->success = false;
71 0 : rq->io.band = band;
72 0 : rq->io.addr = band->md->iter.addr;
73 :
74 0 : ftl_band_rq_bdev_write(rq);
75 :
76 0 : band->queue_depth++;
77 0 : dev->stats.io_activity_total += rq->num_blocks;
78 :
79 0 : ftl_band_iter_advance(band, rq->num_blocks);
80 0 : if (ftl_band_filled(band, band->md->iter.offset)) {
81 0 : ftl_band_set_state(band, FTL_BAND_STATE_FULL);
82 0 : band->owner.state_change_fn(band);
83 : }
84 0 : }
85 :
86 : static void ftl_band_rq_bdev_read(void *_entry);
87 :
88 : static void
89 0 : read_rq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg)
90 : {
91 0 : struct ftl_rq_entry *entry = arg;
92 0 : struct ftl_band *band = entry->io.band;
93 0 : struct ftl_rq *rq = ftl_rq_from_entry(entry);
94 :
95 0 : ftl_stats_bdev_io_completed(band->dev, FTL_STATS_TYPE_GC, bdev_io);
96 :
97 0 : rq->success = success;
98 0 : if (spdk_unlikely(!success)) {
99 0 : ftl_band_rq_bdev_read(entry);
100 0 : spdk_bdev_free_io(bdev_io);
101 0 : return;
102 : }
103 :
104 0 : assert(band->queue_depth > 0);
105 0 : band->queue_depth--;
106 :
107 0 : rq->owner.cb(rq);
108 0 : spdk_bdev_free_io(bdev_io);
109 : }
110 :
111 : static void
112 0 : ftl_band_rq_bdev_read(void *_entry)
113 : {
114 0 : struct ftl_rq_entry *entry = _entry;
115 0 : struct ftl_rq *rq = ftl_rq_from_entry(entry);
116 0 : struct spdk_ftl_dev *dev = rq->dev;
117 : int rc;
118 :
119 0 : rc = spdk_bdev_read_blocks(dev->base_bdev_desc, dev->base_ioch, entry->io_payload,
120 : entry->bdev_io.offset_blocks, entry->bdev_io.num_blocks,
121 : read_rq_end, entry);
122 0 : if (spdk_unlikely(rc)) {
123 0 : if (rc == -ENOMEM) {
124 0 : struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc);
125 0 : entry->bdev_io.wait_entry.bdev = bdev;
126 0 : entry->bdev_io.wait_entry.cb_fn = ftl_band_rq_bdev_read;
127 0 : entry->bdev_io.wait_entry.cb_arg = entry;
128 0 : spdk_bdev_queue_io_wait(bdev, dev->base_ioch, &entry->bdev_io.wait_entry);
129 : } else {
130 0 : ftl_abort();
131 : }
132 : }
133 0 : }
134 :
135 : void
136 0 : ftl_band_rq_read(struct ftl_band *band, struct ftl_rq *rq)
137 : {
138 0 : struct spdk_ftl_dev *dev = band->dev;
139 0 : struct ftl_rq_entry *entry = &rq->entries[rq->iter.idx];
140 :
141 0 : assert(rq->iter.idx + rq->iter.count <= rq->num_blocks);
142 :
143 0 : rq->success = false;
144 0 : rq->io.band = band;
145 0 : rq->io.addr = band->md->iter.addr;
146 0 : entry->io.band = band;
147 0 : entry->bdev_io.offset_blocks = rq->io.addr;
148 0 : entry->bdev_io.num_blocks = rq->iter.count;
149 :
150 0 : ftl_band_rq_bdev_read(entry);
151 :
152 0 : dev->stats.io_activity_total += rq->num_blocks;
153 0 : band->queue_depth++;
154 0 : }
155 :
156 : static void
157 0 : write_brq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg)
158 : {
159 0 : struct ftl_basic_rq *brq = arg;
160 0 : struct ftl_band *band = brq->io.band;
161 :
162 0 : ftl_stats_bdev_io_completed(band->dev, FTL_STATS_TYPE_MD_BASE, bdev_io);
163 :
164 0 : brq->success = success;
165 :
166 0 : assert(band->queue_depth > 0);
167 0 : band->queue_depth--;
168 :
169 0 : brq->owner.cb(brq);
170 0 : spdk_bdev_free_io(bdev_io);
171 0 : }
172 :
173 : static void
174 0 : ftl_band_brq_bdev_write(void *_brq)
175 : {
176 0 : struct ftl_basic_rq *brq = _brq;
177 0 : struct spdk_ftl_dev *dev = brq->dev;
178 : int rc;
179 :
180 0 : rc = spdk_bdev_write_blocks(dev->base_bdev_desc, dev->base_ioch,
181 : brq->io_payload, brq->io.addr,
182 : brq->num_blocks, write_brq_end, brq);
183 :
184 0 : if (spdk_unlikely(rc)) {
185 0 : if (rc == -ENOMEM) {
186 0 : struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc);
187 0 : brq->io.bdev_io_wait.bdev = bdev;
188 0 : brq->io.bdev_io_wait.cb_fn = ftl_band_brq_bdev_write;
189 0 : brq->io.bdev_io_wait.cb_arg = brq;
190 0 : spdk_bdev_queue_io_wait(bdev, dev->base_ioch, &brq->io.bdev_io_wait);
191 : } else {
192 0 : ftl_abort();
193 : }
194 : }
195 0 : }
196 :
197 : void
198 0 : ftl_band_basic_rq_write(struct ftl_band *band, struct ftl_basic_rq *brq)
199 : {
200 0 : struct spdk_ftl_dev *dev = band->dev;
201 :
202 0 : brq->io.addr = band->md->iter.addr;
203 0 : brq->io.band = band;
204 0 : brq->success = false;
205 :
206 0 : ftl_band_brq_bdev_write(brq);
207 :
208 0 : dev->stats.io_activity_total += brq->num_blocks;
209 0 : band->queue_depth++;
210 0 : ftl_band_iter_advance(band, brq->num_blocks);
211 0 : if (ftl_band_filled(band, band->md->iter.offset)) {
212 0 : ftl_band_set_state(band, FTL_BAND_STATE_FULL);
213 0 : band->owner.state_change_fn(band);
214 : }
215 0 : }
216 :
217 : static void
218 0 : read_brq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg)
219 : {
220 0 : struct ftl_basic_rq *brq = arg;
221 0 : struct ftl_band *band = brq->io.band;
222 :
223 0 : ftl_stats_bdev_io_completed(band->dev, FTL_STATS_TYPE_MD_BASE, bdev_io);
224 :
225 0 : brq->success = success;
226 :
227 0 : assert(band->queue_depth > 0);
228 0 : band->queue_depth--;
229 :
230 0 : brq->owner.cb(brq);
231 0 : spdk_bdev_free_io(bdev_io);
232 0 : }
233 :
234 : static void
235 0 : ftl_band_brq_bdev_read(void *_brq)
236 : {
237 0 : struct ftl_basic_rq *brq = _brq;
238 0 : struct spdk_ftl_dev *dev = brq->dev;
239 : int rc;
240 :
241 0 : rc = spdk_bdev_read_blocks(dev->base_bdev_desc, dev->base_ioch,
242 : brq->io_payload, brq->io.addr,
243 : brq->num_blocks, read_brq_end, brq);
244 0 : if (spdk_unlikely(rc)) {
245 0 : if (rc == -ENOMEM) {
246 0 : struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc);
247 0 : brq->io.bdev_io_wait.bdev = bdev;
248 0 : brq->io.bdev_io_wait.cb_fn = ftl_band_brq_bdev_read;
249 0 : brq->io.bdev_io_wait.cb_arg = brq;
250 0 : spdk_bdev_queue_io_wait(bdev, dev->base_ioch, &brq->io.bdev_io_wait);
251 : } else {
252 0 : ftl_abort();
253 : }
254 : }
255 0 : }
256 :
257 : void
258 0 : ftl_band_basic_rq_read(struct ftl_band *band, struct ftl_basic_rq *brq)
259 : {
260 0 : struct spdk_ftl_dev *dev = brq->dev;
261 :
262 0 : brq->io.band = band;
263 :
264 0 : ftl_band_brq_bdev_read(brq);
265 :
266 0 : brq->io.band->queue_depth++;
267 0 : dev->stats.io_activity_total += brq->num_blocks;
268 0 : }
269 :
270 : static void
271 0 : band_open_cb(int status, void *cb_arg)
272 : {
273 0 : struct ftl_band *band = cb_arg;
274 :
275 0 : if (spdk_unlikely(status)) {
276 : #ifdef SPDK_FTL_RETRY_ON_ERROR
277 : ftl_md_persist_entry_retry(&band->md_persist_entry_ctx);
278 : return;
279 : #else
280 0 : ftl_abort();
281 : #endif
282 : }
283 :
284 0 : ftl_band_set_state(band, FTL_BAND_STATE_OPEN);
285 0 : }
286 :
287 : void
288 0 : ftl_band_open(struct ftl_band *band, enum ftl_band_type type)
289 : {
290 0 : struct spdk_ftl_dev *dev = band->dev;
291 0 : struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD];
292 0 : struct ftl_layout_region *region = ftl_layout_region_get(dev, FTL_LAYOUT_REGION_TYPE_BAND_MD);
293 0 : struct ftl_p2l_map *p2l_map = &band->p2l_map;
294 :
295 0 : ftl_band_set_type(band, type);
296 0 : ftl_band_set_state(band, FTL_BAND_STATE_OPENING);
297 :
298 0 : memcpy(p2l_map->band_dma_md, band->md, region->entry_size * FTL_BLOCK_SIZE);
299 0 : p2l_map->band_dma_md->state = FTL_BAND_STATE_OPEN;
300 0 : p2l_map->band_dma_md->p2l_map_checksum = 0;
301 :
302 0 : if (spdk_unlikely(0 != band->p2l_map.num_valid)) {
303 : /*
304 : * This is inconsistent state, a band with valid block,
305 : * it could be moved on the free list
306 : */
307 0 : assert(false && 0 == band->p2l_map.num_valid);
308 : ftl_abort();
309 : }
310 :
311 0 : ftl_md_persist_entries(md, band->id, 1, p2l_map->band_dma_md, NULL,
312 : band_open_cb, band, &band->md_persist_entry_ctx);
313 0 : }
314 :
315 : static void
316 0 : band_close_cb(int status, void *cb_arg)
317 : {
318 0 : struct ftl_band *band = cb_arg;
319 :
320 0 : if (spdk_unlikely(status)) {
321 : #ifdef SPDK_FTL_RETRY_ON_ERROR
322 : ftl_md_persist_entry_retry(&band->md_persist_entry_ctx);
323 : return;
324 : #else
325 0 : ftl_abort();
326 : #endif
327 : }
328 :
329 0 : band->md->p2l_map_checksum = band->p2l_map.band_dma_md->p2l_map_checksum;
330 0 : ftl_band_set_state(band, FTL_BAND_STATE_CLOSED);
331 0 : }
332 :
333 : static void
334 0 : band_map_write_cb(struct ftl_basic_rq *brq)
335 : {
336 0 : struct ftl_band *band = brq->io.band;
337 0 : struct ftl_p2l_map *p2l_map = &band->p2l_map;
338 0 : struct spdk_ftl_dev *dev = band->dev;
339 0 : struct ftl_layout_region *region = ftl_layout_region_get(dev, FTL_LAYOUT_REGION_TYPE_BAND_MD);
340 0 : struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD];
341 : uint32_t band_map_crc;
342 :
343 0 : if (spdk_likely(brq->success)) {
344 :
345 0 : band_map_crc = spdk_crc32c_update(p2l_map->band_map,
346 0 : ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE, 0);
347 0 : memcpy(p2l_map->band_dma_md, band->md, region->entry_size * FTL_BLOCK_SIZE);
348 0 : p2l_map->band_dma_md->state = FTL_BAND_STATE_CLOSED;
349 0 : p2l_map->band_dma_md->p2l_map_checksum = band_map_crc;
350 :
351 0 : ftl_md_persist_entries(md, band->id, 1, p2l_map->band_dma_md, NULL,
352 : band_close_cb, band, &band->md_persist_entry_ctx);
353 : } else {
354 : #ifdef SPDK_FTL_RETRY_ON_ERROR
355 : /* Try to retry in case of failure */
356 : ftl_band_brq_bdev_write(brq);
357 : band->queue_depth++;
358 : #else
359 0 : ftl_abort();
360 : #endif
361 : }
362 0 : }
363 :
364 : void
365 0 : ftl_band_close(struct ftl_band *band)
366 : {
367 0 : struct spdk_ftl_dev *dev = band->dev;
368 0 : void *metadata = band->p2l_map.band_map;
369 0 : uint64_t num_blocks = ftl_tail_md_num_blocks(dev);
370 :
371 : /* Write P2L map first, after completion, set the state to close on nvcache, then internally */
372 0 : band->md->close_seq_id = ftl_get_next_seq_id(dev);
373 0 : ftl_band_set_state(band, FTL_BAND_STATE_CLOSING);
374 0 : ftl_basic_rq_init(dev, &band->metadata_rq, metadata, num_blocks);
375 0 : ftl_basic_rq_set_owner(&band->metadata_rq, band_map_write_cb, band);
376 :
377 0 : ftl_band_basic_rq_write(band, &band->metadata_rq);
378 0 : }
379 :
380 : static void
381 0 : band_free_cb(int status, void *ctx)
382 : {
383 0 : struct ftl_band *band = (struct ftl_band *)ctx;
384 :
385 0 : if (spdk_unlikely(status)) {
386 : #ifdef SPDK_FTL_RETRY_ON_ERROR
387 : ftl_md_persist_entry_retry(&band->md_persist_entry_ctx);
388 : return;
389 : #else
390 0 : ftl_abort();
391 : #endif
392 : }
393 :
394 0 : ftl_band_release_p2l_map(band);
395 0 : FTL_DEBUGLOG(band->dev, "Band is going to free state. Band id: %u\n", band->id);
396 0 : ftl_band_set_state(band, FTL_BAND_STATE_FREE);
397 0 : assert(0 == band->p2l_map.ref_cnt);
398 0 : }
399 :
400 : void
401 0 : ftl_band_free(struct ftl_band *band)
402 : {
403 0 : struct spdk_ftl_dev *dev = band->dev;
404 0 : struct ftl_p2l_map *p2l_map = &band->p2l_map;
405 0 : struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD];
406 0 : struct ftl_layout_region *region = ftl_layout_region_get(dev, FTL_LAYOUT_REGION_TYPE_BAND_MD);
407 :
408 0 : memcpy(p2l_map->band_dma_md, band->md, region->entry_size * FTL_BLOCK_SIZE);
409 0 : p2l_map->band_dma_md->state = FTL_BAND_STATE_FREE;
410 0 : p2l_map->band_dma_md->close_seq_id = 0;
411 0 : p2l_map->band_dma_md->p2l_map_checksum = 0;
412 :
413 0 : ftl_md_persist_entries(md, band->id, 1, p2l_map->band_dma_md, NULL,
414 : band_free_cb, band, &band->md_persist_entry_ctx);
415 :
416 : /* TODO: The whole band erase code should probably be done here instead */
417 0 : }
418 :
419 : static void
420 0 : read_md_cb(struct ftl_basic_rq *brq)
421 : {
422 0 : struct ftl_band *band = brq->owner.priv;
423 0 : struct spdk_ftl_dev *dev = band->dev;
424 : ftl_band_ops_cb cb;
425 : uint32_t band_map_crc;
426 0 : bool success = true;
427 : void *priv;
428 :
429 0 : cb = band->owner.ops_fn;
430 0 : priv = band->owner.priv;
431 :
432 0 : if (!brq->success) {
433 0 : ftl_band_basic_rq_read(band, &band->metadata_rq);
434 0 : return;
435 : }
436 :
437 0 : band_map_crc = spdk_crc32c_update(band->p2l_map.band_map,
438 0 : ftl_tail_md_num_blocks(band->dev) * FTL_BLOCK_SIZE, 0);
439 0 : if (band->md->p2l_map_checksum && band->md->p2l_map_checksum != band_map_crc) {
440 0 : FTL_ERRLOG(dev, "GC error, inconsistent P2L map CRC\n");
441 0 : success = false;
442 :
443 0 : ftl_stats_crc_error(band->dev, FTL_STATS_TYPE_GC);
444 : }
445 0 : band->owner.ops_fn = NULL;
446 0 : band->owner.priv = NULL;
447 0 : cb(band, priv, success);
448 : }
449 :
450 : static int
451 0 : _read_md(struct ftl_band *band)
452 : {
453 0 : struct spdk_ftl_dev *dev = band->dev;
454 0 : struct ftl_basic_rq *rq = &band->metadata_rq;
455 :
456 0 : if (ftl_band_alloc_p2l_map(band)) {
457 0 : return -ENOMEM;
458 : }
459 :
460 : /* Read P2L map */
461 0 : ftl_basic_rq_init(dev, rq, band->p2l_map.band_map, ftl_p2l_map_num_blocks(dev));
462 0 : ftl_basic_rq_set_owner(rq, read_md_cb, band);
463 :
464 0 : rq->io.band = band;
465 0 : rq->io.addr = ftl_band_p2l_map_addr(band);
466 :
467 0 : ftl_band_basic_rq_read(band, &band->metadata_rq);
468 :
469 0 : return 0;
470 : }
471 :
472 : static void
473 0 : read_md(void *band)
474 : {
475 : int rc;
476 :
477 0 : rc = _read_md(band);
478 0 : if (spdk_unlikely(rc)) {
479 0 : spdk_thread_send_msg(spdk_get_thread(), read_md, band);
480 : }
481 0 : }
482 :
483 : static void
484 0 : read_tail_md_cb(struct ftl_basic_rq *brq)
485 : {
486 0 : struct ftl_band *band = brq->owner.priv;
487 0 : enum ftl_md_status status = FTL_MD_IO_FAILURE;
488 : ftl_band_md_cb cb;
489 : void *priv;
490 :
491 0 : if (spdk_unlikely(!brq->success)) {
492 : /* Retries the read in case of error */
493 0 : ftl_band_basic_rq_read(band, &band->metadata_rq);
494 0 : return;
495 : }
496 :
497 0 : cb = band->owner.md_fn;
498 0 : band->owner.md_fn = NULL;
499 :
500 0 : priv = band->owner.priv;
501 0 : band->owner.priv = NULL;
502 :
503 0 : status = FTL_MD_SUCCESS;
504 :
505 0 : cb(band, priv, status);
506 : }
507 :
508 : void
509 0 : ftl_band_read_tail_brq_md(struct ftl_band *band, ftl_band_md_cb cb, void *cntx)
510 : {
511 0 : struct spdk_ftl_dev *dev = band->dev;
512 0 : struct ftl_basic_rq *rq = &band->metadata_rq;
513 :
514 0 : ftl_basic_rq_init(dev, rq, band->p2l_map.band_map, ftl_tail_md_num_blocks(dev));
515 0 : ftl_basic_rq_set_owner(rq, read_tail_md_cb, band);
516 :
517 0 : assert(!band->owner.md_fn);
518 0 : assert(!band->owner.priv);
519 0 : band->owner.md_fn = cb;
520 0 : band->owner.priv = cntx;
521 :
522 0 : rq->io.band = band;
523 0 : rq->io.addr = band->tail_md_addr;
524 :
525 0 : ftl_band_basic_rq_read(band, &band->metadata_rq);
526 0 : }
527 :
528 : void
529 0 : ftl_band_get_next_gc(struct spdk_ftl_dev *dev, ftl_band_ops_cb cb, void *cntx)
530 : {
531 0 : struct ftl_band *band = ftl_band_search_next_to_reloc(dev);
532 :
533 : /* if disk is very small, GC start very early that no band is ready for it */
534 0 : if (spdk_unlikely(!band)) {
535 0 : cb(NULL, cntx, false);
536 0 : return;
537 : }
538 :
539 : /* Only one owner is allowed */
540 0 : assert(!band->queue_depth);
541 0 : assert(!band->owner.ops_fn);
542 0 : assert(!band->owner.priv);
543 0 : band->owner.ops_fn = cb;
544 0 : band->owner.priv = cntx;
545 :
546 0 : read_md(band);
547 : }
|
