LCOV - code coverage report
Current view: top level - spdk/lib/ftl - ftl_p2l.c (source / functions) Hit Total Coverage
Test: Combined Lines: 255 312 81.7 %
Date: 2024-12-15 06:38:34 Functions: 20 20 100.0 %
Legend: Lines: hit not hit | Branches: + taken - not taken # not executed Branches: 130 1818 7.2 %

           Branch data     Line data    Source code
       1                 :            : /*   SPDX-License-Identifier: BSD-3-Clause
       2                 :            :  *   Copyright (C) 2022 Intel Corporation.
       3                 :            :  *   Copyright 2023 Solidigm All Rights Reserved
       4                 :            :  *   All rights reserved.
       5                 :            :  */
       6                 :            : 
       7                 :            : #include "spdk/bdev_module.h"
       8                 :            : #include "spdk/crc32.h"
       9                 :            : 
      10                 :            : #include "ftl_internal.h"
      11                 :            : #include "ftl_band.h"
      12                 :            : #include "ftl_core.h"
      13                 :            : #include "ftl_layout.h"
      14                 :            : #include "ftl_nv_cache_io.h"
      15                 :            : #include "ftl_writer.h"
      16                 :            : #include "mngt/ftl_mngt.h"
      17                 :            : 
      18                 :            : struct ftl_p2l_ckpt {
      19                 :            :         TAILQ_ENTRY(ftl_p2l_ckpt)       link;
      20                 :            :         union ftl_md_vss                *vss_md_page;
      21                 :            :         struct ftl_md                   *md;
      22                 :            :         struct ftl_layout_region        *layout_region;
      23                 :            :         uint64_t                        num_pages;
      24                 :            :         uint64_t                        pages_per_xfer;
      25                 :            : 
      26                 :            : #if defined(DEBUG)
      27                 :            :         uint64_t                        dbg_bmp_sz;
      28                 :            :         void                            *dbg_bmp;
      29                 :            :         struct ftl_bitmap               *bmp;
      30                 :            : #endif
      31                 :            : };
      32                 :            : 
      33                 :            : static struct ftl_p2l_ckpt *
      34                 :        165 : ftl_p2l_ckpt_new(struct spdk_ftl_dev *dev, int region_type)
      35                 :            : {
      36                 :            :         struct ftl_p2l_ckpt *ckpt;
      37                 :        165 :         struct ftl_layout_region *region = ftl_layout_region_get(dev, region_type);
      38                 :            : 
      39                 :        165 :         ckpt = calloc(1, sizeof(struct ftl_p2l_ckpt));
      40         [ -  + ]:        165 :         if (!ckpt) {
      41                 :          0 :                 return NULL;
      42                 :            :         }
      43                 :            : 
      44   [ #  #  #  # ]:        165 :         ckpt->layout_region = region;
      45   [ #  #  #  #  :        165 :         ckpt->md = dev->layout.md[region_type];
          #  #  #  #  #  
                #  #  # ]
      46   [ #  #  #  #  :        165 :         ckpt->pages_per_xfer = dev->layout.p2l.pages_per_xfer;
          #  #  #  #  #  
                #  #  # ]
      47   [ #  #  #  #  :        165 :         ckpt->num_pages = dev->layout.p2l.ckpt_pages;
          #  #  #  #  #  
                #  #  # ]
      48   [ +  +  #  #  :        165 :         if (dev->nv_cache.md_size) {
             #  #  #  # ]
      49   [ #  #  #  #  :        108 :                 ckpt->vss_md_page = ftl_md_vss_buf_alloc(region, region->num_entries);
             #  #  #  # ]
      50   [ -  +  #  #  :        108 :                 if (!ckpt->vss_md_page) {
                   #  # ]
      51                 :          0 :                         free(ckpt);
      52                 :          0 :                         return NULL;
      53                 :            :                 }
      54                 :          0 :         }
      55                 :            : 
      56                 :            : #if defined(DEBUG)
      57                 :            :         /* The bitmap size must be a multiple of word size (8b) - round up */
      58   [ #  #  #  #  :        165 :         ckpt->dbg_bmp_sz = spdk_divide_round_up(ckpt->num_pages, 8);
             #  #  #  # ]
      59                 :            : 
      60   [ #  #  #  #  :        165 :         ckpt->dbg_bmp = calloc(1, ckpt->dbg_bmp_sz);
             #  #  #  # ]
      61   [ -  +  #  #  :        165 :         assert(ckpt->dbg_bmp);
             #  #  #  # ]
      62   [ #  #  #  #  :        165 :         ckpt->bmp = ftl_bitmap_create(ckpt->dbg_bmp, ckpt->dbg_bmp_sz);
          #  #  #  #  #  
                #  #  # ]
      63   [ -  +  #  #  :        165 :         assert(ckpt->bmp);
             #  #  #  # ]
      64                 :            : #endif
      65                 :            : 
      66                 :        165 :         return ckpt;
      67                 :          0 : }
      68                 :            : 
      69                 :            : static void
      70                 :        165 : ftl_p2l_ckpt_destroy(struct ftl_p2l_ckpt *ckpt)
      71                 :            : {
      72                 :            : #if defined(DEBUG)
      73   [ #  #  #  # ]:        165 :         ftl_bitmap_destroy(ckpt->bmp);
      74   [ #  #  #  # ]:        165 :         free(ckpt->dbg_bmp);
      75                 :            : #endif
      76   [ #  #  #  # ]:        165 :         spdk_dma_free(ckpt->vss_md_page);
      77                 :        165 :         free(ckpt);
      78                 :        165 : }
      79                 :            : 
      80                 :            : int
      81                 :         27 : ftl_p2l_ckpt_init(struct spdk_ftl_dev *dev)
      82                 :            : {
      83                 :            :         int region_type;
      84                 :            :         struct ftl_p2l_ckpt *ckpt;
      85                 :            : 
      86   [ #  #  #  #  :         27 :         TAILQ_INIT(&dev->p2l_ckpt.free);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
      87   [ #  #  #  #  :         27 :         TAILQ_INIT(&dev->p2l_ckpt.inuse);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
      88         [ #  # ]:         27 :         for (region_type = FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MIN;
      89         [ +  + ]:        135 :              region_type <= FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MAX;
      90         [ #  # ]:        108 :              region_type++) {
      91                 :        108 :                 ckpt = ftl_p2l_ckpt_new(dev, region_type);
      92         [ -  + ]:        108 :                 if (!ckpt) {
      93                 :          0 :                         return -1;
      94                 :            :                 }
      95   [ #  #  #  #  :        108 :                 TAILQ_INSERT_TAIL(&dev->p2l_ckpt.free, ckpt, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
      96                 :          0 :         }
      97                 :         27 :         return 0;
      98                 :          0 : }
      99                 :            : 
     100                 :            : void
     101                 :         27 : ftl_p2l_ckpt_deinit(struct spdk_ftl_dev *dev)
     102                 :            : {
     103                 :            :         struct ftl_p2l_ckpt *ckpt, *ckpt_next;
     104                 :            : 
     105   [ +  +  #  #  :        128 :         TAILQ_FOREACH_SAFE(ckpt, &dev->p2l_ckpt.free, link, ckpt_next) {
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  # ]
     106   [ +  +  #  #  :        101 :                 TAILQ_REMOVE(&dev->p2l_ckpt.free, ckpt, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     107                 :        101 :                 ftl_p2l_ckpt_destroy(ckpt);
     108                 :          0 :         }
     109                 :            : 
     110   [ +  +  #  #  :         34 :         TAILQ_FOREACH_SAFE(ckpt, &dev->p2l_ckpt.inuse, link, ckpt_next) {
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  # ]
     111   [ -  +  #  #  :          7 :                 TAILQ_REMOVE(&dev->p2l_ckpt.inuse, ckpt, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     112                 :          7 :                 ftl_p2l_ckpt_destroy(ckpt);
     113                 :          0 :         }
     114                 :         27 : }
     115                 :            : 
     116                 :            : struct ftl_p2l_ckpt *
     117                 :          7 : ftl_p2l_ckpt_acquire(struct spdk_ftl_dev *dev)
     118                 :            : {
     119                 :            :         struct ftl_p2l_ckpt *ckpt;
     120                 :            : 
     121   [ #  #  #  #  :          7 :         ckpt = TAILQ_FIRST(&dev->p2l_ckpt.free);
             #  #  #  # ]
     122   [ -  +  #  # ]:          7 :         assert(ckpt);
     123   [ +  -  #  #  :          7 :         TAILQ_REMOVE(&dev->p2l_ckpt.free, ckpt, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     124   [ #  #  #  #  :          7 :         TAILQ_INSERT_TAIL(&dev->p2l_ckpt.inuse, ckpt, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
     125                 :          7 :         return ckpt;
     126                 :            : }
     127                 :            : 
     128                 :            : void
     129                 :          4 : ftl_p2l_ckpt_release(struct spdk_ftl_dev *dev, struct ftl_p2l_ckpt *ckpt)
     130                 :            : {
     131   [ -  +  #  # ]:          4 :         assert(ckpt);
     132                 :            : #if defined(DEBUG)
     133   [ -  +  #  #  :          4 :         memset(ckpt->dbg_bmp, 0, ckpt->dbg_bmp_sz);
          #  #  #  #  #  
                      # ]
     134                 :            : #endif
     135   [ +  +  #  #  :          4 :         TAILQ_REMOVE(&dev->p2l_ckpt.inuse, ckpt, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     136   [ #  #  #  #  :          4 :         TAILQ_INSERT_TAIL(&dev->p2l_ckpt.free, ckpt, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
     137                 :          4 : }
     138                 :            : 
     139                 :            : static void
     140                 :       5110 : ftl_p2l_ckpt_issue_end(int status, void *arg)
     141                 :            : {
     142                 :       5110 :         struct ftl_rq *rq = arg;
     143   [ -  +  #  # ]:       5110 :         assert(rq);
     144                 :            : 
     145         [ -  + ]:       5110 :         if (status) {
     146                 :            : #ifdef SPDK_FTL_RETRY_ON_ERROR
     147                 :            :                 /* retry */
     148                 :            :                 ftl_md_persist_entry_retry(&rq->md_persist_entry_ctx);
     149                 :            :                 return;
     150                 :            : #else
     151         [ #  # ]:          0 :                 ftl_abort();
     152                 :            : #endif
     153                 :          0 :         }
     154                 :            : 
     155   [ -  +  #  #  :       5110 :         assert(rq->io.band->queue_depth > 0);
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     156   [ #  #  #  #  :       5110 :         rq->io.band->queue_depth--;
             #  #  #  # ]
     157                 :            : 
     158   [ #  #  #  #  :       5110 :         rq->owner.cb(rq);
          #  #  #  #  #  
                      # ]
     159                 :       5110 : }
     160                 :            : 
     161                 :            : void
     162                 :       5122 : ftl_p2l_ckpt_issue(struct ftl_rq *rq)
     163                 :            : {
     164         [ #  # ]:       5122 :         struct ftl_rq_entry *iter = rq->entries;
     165   [ #  #  #  # ]:       5122 :         struct spdk_ftl_dev *dev = rq->dev;
     166   [ #  #  #  #  :       5122 :         ftl_addr addr = rq->io.addr;
                   #  # ]
     167                 :       5122 :         struct ftl_p2l_ckpt *ckpt = NULL;
     168                 :            :         struct ftl_p2l_ckpt_page_no_vss *map_page;
     169                 :            :         struct ftl_band *band;
     170                 :            :         uint64_t band_offs, p2l_map_page_no, cur_page, i, j;
     171                 :            : 
     172   [ -  +  #  # ]:       5122 :         assert(rq);
     173   [ #  #  #  #  :       5122 :         band = rq->io.band;
                   #  # ]
     174   [ #  #  #  #  :       5122 :         ckpt = band->p2l_map.p2l_ckpt;
                   #  # ]
     175   [ -  +  #  # ]:       5122 :         assert(ckpt);
     176   [ -  +  #  #  :       5122 :         assert(rq->num_blocks == dev->xfer_size);
          #  #  #  #  #  
                #  #  # ]
     177                 :            : 
     178                 :            :         /* Derive the P2L map page no */
     179   [ #  #  #  #  :       5122 :         band_offs = ftl_band_block_offset_from_addr(band, rq->io.addr);
                   #  # ]
     180   [ -  +  #  #  :       5122 :         p2l_map_page_no = band_offs / dev->xfer_size * ckpt->pages_per_xfer;
          #  #  #  #  #  
                      # ]
     181   [ -  +  #  #  :       5122 :         assert(p2l_map_page_no < ckpt->num_pages);
             #  #  #  # ]
     182                 :            : 
     183                 :            :         /* Get the corresponding P2L map page - the underlying stored data has the same entries as in the end metadata of band P2L (ftl_p2l_map_entry),
     184                 :            :          * however we're interested in a whole page (4KiB) worth of content and submit it in two requests with additional metadata
     185                 :            :          */
     186   [ #  #  #  # ]:       5122 :         map_page = ftl_md_get_buffer(ckpt->md);
     187   [ -  +  #  # ]:       5122 :         assert(map_page);
     188         [ #  # ]:       5122 :         map_page += p2l_map_page_no;
     189                 :       5122 :         i = 0;
     190   [ +  +  #  #  :      15360 :         for (cur_page = 0; cur_page < ckpt->pages_per_xfer; cur_page++) {
                   #  # ]
     191         [ #  # ]:      10238 :                 struct ftl_p2l_ckpt_page_no_vss *page = map_page + cur_page;
     192                 :            :                 /* Update the band P2L map */
     193   [ +  +  +  +  :    1320702 :                 for (j = 0; i < rq->num_blocks && j < FTL_NUM_P2L_ENTRIES_NO_VSS; i++, iter++, j++) {
             #  #  #  # ]
     194   [ +  +  #  # ]:    1310464 :                         if (iter->lba != FTL_LBA_INVALID) {
     195                 :            :                                 /* This is compaction or reloc */
     196   [ -  +  #  #  :    1051392 :                                 assert(!ftl_addr_in_nvc(rq->dev, addr));
             #  #  #  # ]
     197   [ #  #  #  #  :    1051392 :                                 ftl_band_set_p2l(band, iter->lba, addr, iter->seq_id);
             #  #  #  # ]
     198                 :          0 :                         }
     199   [ #  #  #  #  :    1310464 :                         page->map[j].lba = iter->lba;
          #  #  #  #  #  
                #  #  # ]
     200   [ #  #  #  #  :    1310464 :                         page->map[j].seq_id = iter->seq_id;
          #  #  #  #  #  
                #  #  # ]
     201                 :    1310464 :                         addr = ftl_band_next_addr(band, addr, 1);
     202                 :          0 :                 }
     203                 :            : 
     204                 :            :                 /* Set up the md */
     205   [ #  #  #  #  :      10238 :                 page->metadata.p2l_ckpt.seq_id = band->md->seq;
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     206   [ #  #  #  #  :      10238 :                 page->metadata.p2l_ckpt.count = j;
             #  #  #  # ]
     207                 :            : 
     208                 :            : #if defined(DEBUG)
     209   [ #  #  #  # ]:      10238 :                 ftl_bitmap_set(ckpt->bmp, p2l_map_page_no + cur_page);
     210                 :            : #endif
     211   [ #  #  #  #  :      10238 :                 page->metadata.p2l_ckpt.p2l_checksum = spdk_crc32c_update(page->map,
          #  #  #  #  #  
                      # ]
     212                 :            :                                                        FTL_NUM_P2L_ENTRIES_NO_VSS * sizeof(struct ftl_p2l_map_entry), 0);
     213                 :          0 :         }
     214                 :            :         /* Save the P2L map entry */
     215   [ #  #  #  #  :       5122 :         ftl_md_persist_entries(ckpt->md, p2l_map_page_no, ckpt->pages_per_xfer, map_page, NULL,
             #  #  #  # ]
     216         [ #  # ]:          0 :                                ftl_p2l_ckpt_issue_end, rq, &rq->md_persist_entry_ctx);
     217                 :       5122 : }
     218                 :            : 
     219                 :            : #if defined(DEBUG)
     220                 :            : static void
     221                 :          8 : ftl_p2l_validate_pages(struct ftl_band *band, struct ftl_p2l_ckpt *ckpt,
     222                 :            :                        uint64_t page_begin, uint64_t page_end, bool val)
     223                 :            : {
     224                 :            :         uint64_t page_no;
     225                 :            : 
     226         [ +  + ]:       8200 :         for (page_no = page_begin; page_no < page_end; page_no++) {
     227   [ -  +  #  #  :       8192 :                 assert(ftl_bitmap_get(ckpt->bmp, page_no) == val);
          #  #  #  #  #  
                      # ]
     228                 :          0 :         }
     229                 :          8 : }
     230                 :            : 
     231                 :            : void
     232                 :          4 : ftl_p2l_validate_ckpt(struct ftl_band *band)
     233                 :            : {
     234   [ #  #  #  #  :          4 :         struct ftl_p2l_ckpt *ckpt = band->p2l_map.p2l_ckpt;
                   #  # ]
     235   [ #  #  #  # ]:          4 :         uint64_t num_blks_tail_md = ftl_tail_md_num_blocks(band->dev);
     236                 :            :         uint64_t num_pages_tail_md;
     237                 :            : 
     238         [ -  + ]:          4 :         if (!ckpt) {
     239                 :          0 :                 return;
     240                 :            :         }
     241                 :            : 
     242   [ -  +  #  #  :          4 :         num_pages_tail_md = num_blks_tail_md / band->dev->xfer_size * ckpt->pages_per_xfer;
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     243                 :            : 
     244   [ -  +  -  +  :          4 :         assert(num_blks_tail_md % band->dev->xfer_size == 0);
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     245                 :            : 
     246                 :            :         /* all data pages written */
     247                 :          4 :         ftl_p2l_validate_pages(band, ckpt,
     248   [ #  #  #  # ]:          4 :                                0, ckpt->num_pages - num_pages_tail_md, true);
     249                 :            : 
     250                 :            :         /* tail md pages not written */
     251   [ #  #  #  # ]:          4 :         ftl_p2l_validate_pages(band, ckpt, ckpt->num_pages - num_pages_tail_md,
     252   [ #  #  #  # ]:          0 :                                ckpt->num_pages, false);
     253                 :          0 : }
     254                 :            : #endif
     255                 :            : 
     256                 :            : static struct ftl_band *
     257                 :         92 : ftl_get_band_from_region(struct spdk_ftl_dev *dev, enum ftl_layout_region_type type)
     258                 :            : {
     259                 :         92 :         struct ftl_band *band = NULL;
     260                 :            :         uint64_t i;
     261                 :            : 
     262   [ -  +  #  # ]:         92 :         assert(type >= FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MIN);
     263   [ -  +  #  # ]:         92 :         assert(type <= FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MAX);
     264                 :            : 
     265         [ +  + ]:       8138 :         for (i = 0; i < ftl_get_num_bands(dev); i++) {
     266   [ #  #  #  #  :       8051 :                 band = &dev->bands[i];
                   #  # ]
     267   [ +  +  #  #  :       8051 :                 if ((band->md->state == FTL_BAND_STATE_OPEN ||
          #  #  #  #  #  
                #  #  # ]
     268   [ -  +  #  #  :       8031 :                      band->md->state == FTL_BAND_STATE_FULL) &&
             #  #  #  # ]
     269   [ +  +  #  #  :         20 :                     band->md->p2l_md_region == type) {
             #  #  #  # ]
     270                 :          5 :                         return band;
     271                 :            :                 }
     272                 :          0 :         }
     273                 :            : 
     274                 :         87 :         return NULL;
     275                 :          0 : }
     276                 :            : 
     277                 :            : static void ftl_mngt_persist_band_p2l(struct ftl_mngt_process *mngt, struct ftl_p2l_sync_ctx *ctx);
     278                 :            : 
     279                 :            : static void
     280                 :       1318 : ftl_p2l_ckpt_persist_end(int status, void *arg)
     281                 :            : {
     282                 :       1318 :         struct ftl_mngt_process *mngt = arg;
     283                 :            :         struct ftl_p2l_sync_ctx *ctx;
     284                 :            : 
     285   [ -  +  #  # ]:       1318 :         assert(mngt);
     286                 :            : 
     287         [ -  + ]:       1318 :         if (status) {
     288                 :          0 :                 ftl_mngt_fail_step(mngt);
     289                 :          0 :                 return;
     290                 :            :         }
     291                 :            : 
     292                 :       1318 :         ctx = ftl_mngt_get_step_ctx(mngt);
     293         [ #  # ]:       1318 :         ctx->xfer_start++;
     294                 :            : 
     295   [ +  +  #  #  :       1318 :         if (ctx->xfer_start == ctx->xfer_end) {
          #  #  #  #  #  
                      # ]
     296   [ #  #  #  # ]:          5 :                 ctx->md_region++;
     297                 :          5 :                 ftl_mngt_continue_step(mngt);
     298                 :          0 :         } else {
     299                 :       1313 :                 ftl_mngt_persist_band_p2l(mngt, ctx);
     300                 :            :         }
     301                 :          0 : }
     302                 :            : 
     303                 :            : static void
     304                 :       1330 : ftl_mngt_persist_band_p2l(struct ftl_mngt_process *mngt, struct ftl_p2l_sync_ctx *ctx)
     305                 :            : {
     306   [ #  #  #  # ]:       1330 :         struct ftl_band *band = ctx->band;
     307                 :            :         struct ftl_p2l_ckpt_page_no_vss *map_page;
     308                 :            :         struct ftl_p2l_map_entry *band_entries;
     309                 :            :         struct ftl_p2l_ckpt *ckpt;
     310   [ #  #  #  # ]:       1330 :         struct spdk_ftl_dev *dev = band->dev;
     311                 :            :         uint64_t cur_page;
     312                 :       1330 :         uint64_t lbas_synced = 0;
     313                 :            : 
     314   [ #  #  #  #  :       1330 :         ckpt = band->p2l_map.p2l_ckpt;
                   #  # ]
     315                 :            : 
     316   [ #  #  #  # ]:       1330 :         map_page = ftl_md_get_buffer(ckpt->md);
     317   [ -  +  #  # ]:       1330 :         assert(map_page);
     318                 :            : 
     319   [ #  #  #  #  :       1330 :         map_page += ctx->xfer_start * ckpt->pages_per_xfer;
          #  #  #  #  #  
                      # ]
     320                 :            : 
     321   [ +  +  #  #  :       3984 :         for (cur_page = 0; cur_page < ckpt->pages_per_xfer; cur_page++) {
                   #  # ]
     322         [ #  # ]:       2654 :                 struct ftl_p2l_ckpt_page_no_vss *page = map_page + cur_page;
     323   [ #  #  #  #  :       2654 :                 uint64_t lbas_to_copy = spdk_min(FTL_NUM_P2L_ENTRIES_NO_VSS, dev->xfer_size - lbas_synced);
          #  #  #  #  #  
                      # ]
     324                 :            : 
     325   [ #  #  #  #  :       2654 :                 band_entries = band->p2l_map.band_map + ctx->xfer_start * dev->xfer_size + lbas_synced;
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
     326   [ -  +  -  +  :       2654 :                 memcpy(page->map, band_entries, lbas_to_copy * sizeof(struct ftl_p2l_map_entry));
                   #  # ]
     327                 :            : 
     328   [ #  #  #  #  :       2654 :                 page->metadata.p2l_ckpt.seq_id = band->md->seq;
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     329   [ #  #  #  #  :       2654 :                 page->metadata.p2l_ckpt.p2l_checksum = spdk_crc32c_update(page->map,
          #  #  #  #  #  
                      # ]
     330                 :            :                                                        FTL_NUM_P2L_ENTRIES_NO_VSS * sizeof(struct ftl_p2l_map_entry), 0);
     331   [ #  #  #  #  :       2654 :                 page->metadata.p2l_ckpt.count = lbas_to_copy;
             #  #  #  # ]
     332                 :       2654 :                 lbas_synced += lbas_to_copy;
     333                 :          0 :         }
     334                 :            : 
     335   [ -  +  #  #  :       1330 :         assert(lbas_synced == dev->xfer_size);
             #  #  #  # ]
     336                 :            :         /* Save the P2L map entry */
     337   [ #  #  #  #  :       1330 :         ftl_md_persist_entries(ckpt->md, ctx->xfer_start * ckpt->pages_per_xfer, ckpt->pages_per_xfer,
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     338                 :          0 :                                map_page, NULL,
     339         [ #  # ]:          0 :                                ftl_p2l_ckpt_persist_end, mngt, &band->md_persist_entry_ctx);
     340                 :       1330 : }
     341                 :            : 
     342                 :            : void
     343                 :        115 : ftl_mngt_persist_bands_p2l(struct ftl_mngt_process *mngt)
     344                 :            : {
     345                 :        115 :         struct ftl_p2l_sync_ctx *ctx = ftl_mngt_get_step_ctx(mngt);
     346                 :            :         struct ftl_band *band;
     347                 :            :         uint64_t band_offs, num_xfers;
     348                 :            : 
     349   [ +  +  #  #  :        115 :         if (ctx->md_region > FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MAX) {
                   #  # ]
     350                 :         23 :                 ftl_mngt_next_step(mngt);
     351                 :         23 :                 return;
     352                 :            :         }
     353                 :            : 
     354   [ #  #  #  # ]:         92 :         band = ftl_get_band_from_region(ftl_mngt_get_dev(mngt), ctx->md_region);
     355                 :            : 
     356                 :            :         /* No band has the md region assigned (shutdown happened before next_band was assigned) */
     357         [ +  + ]:         92 :         if (!band) {
     358   [ #  #  #  # ]:         87 :                 ctx->xfer_start = 0;
     359   [ #  #  #  # ]:         87 :                 ctx->xfer_end = 0;
     360   [ #  #  #  # ]:         87 :                 ctx->md_region++;
     361                 :         87 :                 ftl_mngt_continue_step(mngt);
     362                 :         87 :                 return;
     363                 :            :         }
     364                 :            : 
     365   [ #  #  #  #  :          5 :         band_offs = ftl_band_block_offset_from_addr(band, band->md->iter.addr);
          #  #  #  #  #  
                      # ]
     366   [ -  +  #  #  :          5 :         num_xfers = band_offs / band->dev->xfer_size;
          #  #  #  #  #  
                      # ]
     367                 :            : 
     368   [ #  #  #  # ]:          5 :         ctx->xfer_start = 0;
     369   [ #  #  #  # ]:          5 :         ctx->xfer_end = num_xfers;
     370   [ #  #  #  # ]:          5 :         ctx->band = band;
     371                 :            : 
     372                 :            :         /* Band wasn't written to - no need to sync its P2L */
     373   [ -  +  #  #  :          5 :         if (ctx->xfer_end == 0) {
                   #  # ]
     374   [ #  #  #  # ]:          0 :                 ctx->md_region++;
     375                 :          0 :                 ftl_mngt_continue_step(mngt);
     376                 :          0 :                 return;
     377                 :            :         }
     378                 :            : 
     379                 :          5 :         ftl_mngt_persist_band_p2l(mngt, ctx);
     380                 :          0 : }
     381                 :            : 
     382                 :            : uint64_t
     383                 :          4 : ftl_mngt_p2l_ckpt_get_seq_id(struct spdk_ftl_dev *dev, int md_region)
     384                 :            : {
     385         [ #  # ]:          4 :         struct ftl_layout *layout = &dev->layout;
     386   [ #  #  #  #  :          4 :         struct ftl_md *md = layout->md[md_region];
                   #  # ]
     387                 :          4 :         struct ftl_p2l_ckpt_page_no_vss *page = ftl_md_get_buffer(md);
     388                 :          4 :         uint64_t page_no, seq_id = 0;
     389                 :            : 
     390   [ +  +  #  #  :       8196 :         for (page_no = 0; page_no < layout->p2l.ckpt_pages; page_no++, page++) {
          #  #  #  #  #  
                      # ]
     391   [ +  +  #  #  :       8192 :                 if (seq_id < page->metadata.p2l_ckpt.seq_id) {
          #  #  #  #  #  
                      # ]
     392   [ #  #  #  #  :          3 :                         seq_id = page->metadata.p2l_ckpt.seq_id;
             #  #  #  # ]
     393                 :          0 :                 }
     394                 :          0 :         }
     395                 :          4 :         return seq_id;
     396                 :            : }
     397                 :            : 
     398                 :            : int
     399                 :         15 : ftl_mngt_p2l_ckpt_restore(struct ftl_band *band, uint32_t md_region, uint64_t seq_id)
     400                 :            : {
     401   [ #  #  #  #  :         15 :         struct ftl_layout *layout = &band->dev->layout;
                   #  # ]
     402   [ #  #  #  #  :         15 :         struct ftl_md *md = layout->md[md_region];
                   #  # ]
     403                 :         15 :         struct ftl_p2l_ckpt_page_no_vss *page = ftl_md_get_buffer(md);
     404                 :            :         struct ftl_p2l_map_entry *band_entries;
     405   [ #  #  #  # ]:         15 :         struct spdk_ftl_dev *dev = band->dev;
     406                 :         15 :         uint64_t page_no, page_max = 0, xfer_count, lbas_synced;
     407   [ #  #  #  # ]:         15 :         uint64_t pages_per_xfer = spdk_divide_round_up(dev->xfer_size, FTL_NUM_P2L_ENTRIES_NO_VSS);
     408                 :         15 :         bool page_found = false;
     409                 :            : 
     410   [ -  +  #  #  :         15 :         assert(band->md->p2l_md_region == md_region);
          #  #  #  #  #  
                #  #  # ]
     411   [ -  +  #  #  :         15 :         if (band->md->p2l_md_region != md_region) {
          #  #  #  #  #  
                      # ]
     412                 :          0 :                 return -EINVAL;
     413                 :            :         }
     414                 :            : 
     415   [ -  +  #  #  :         15 :         assert(band->md->seq == seq_id);
          #  #  #  #  #  
                #  #  # ]
     416   [ -  +  #  #  :         15 :         if (band->md->seq != seq_id) {
          #  #  #  #  #  
                      # ]
     417                 :          0 :                 return -EINVAL;
     418                 :            :         }
     419                 :            : 
     420   [ +  +  #  #  :      30735 :         for (page_no = 0; page_no < layout->p2l.ckpt_pages; page_no++, page++) {
          #  #  #  #  #  
                      # ]
     421   [ +  +  #  #  :      30720 :                 if (page->metadata.p2l_ckpt.seq_id != seq_id) {
          #  #  #  #  #  
                      # ]
     422                 :      30690 :                         continue;
     423                 :            :                 }
     424                 :            : 
     425                 :         30 :                 page_max = page_no;
     426                 :         30 :                 page_found = true;
     427                 :            : 
     428   [ -  +  -  -  :         30 :                 if (page->metadata.p2l_ckpt.p2l_checksum &&
          #  #  #  #  #  
                #  #  # ]
     429   [ #  #  #  #  :          0 :                     page->metadata.p2l_ckpt.p2l_checksum != spdk_crc32c_update(page->map,
          #  #  #  #  #  
                      # ]
     430                 :            :                                     FTL_NUM_P2L_ENTRIES_NO_VSS * sizeof(struct ftl_p2l_map_entry), 0)) {
     431   [ #  #  #  # ]:          0 :                         ftl_stats_crc_error(band->dev, FTL_STATS_TYPE_MD_NV_CACHE);
     432                 :          0 :                         return -EINVAL;
     433                 :            :                 }
     434                 :            : 
     435         [ -  + ]:         30 :                 xfer_count = page_no / pages_per_xfer;
     436         [ -  + ]:         30 :                 lbas_synced = (page_no % pages_per_xfer) * FTL_NUM_P2L_ENTRIES_NO_VSS;
     437                 :            : 
     438                 :            :                 /* Restore the page from P2L checkpoint */
     439   [ #  #  #  #  :         30 :                 band_entries = band->p2l_map.band_map + xfer_count * dev->xfer_size + lbas_synced;
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     440                 :            : 
     441   [ -  +  -  +  :         30 :                 memcpy(band_entries, page->map, page->metadata.p2l_ckpt.count * sizeof(struct ftl_p2l_map_entry));
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     442                 :          0 :         }
     443                 :            : 
     444   [ -  +  #  #  :         15 :         assert(page_found);
                   #  # ]
     445   [ -  +  #  # ]:         15 :         if (!page_found) {
     446                 :          0 :                 return -EINVAL;
     447                 :            :         }
     448                 :            : 
     449                 :            :         /* Restore check point in band P2L map */
     450   [ #  #  #  #  :         15 :         band->p2l_map.p2l_ckpt = ftl_p2l_ckpt_acquire_region_type(
                   #  # ]
     451   [ #  #  #  # ]:          0 :                                          band->dev, md_region);
     452                 :            : 
     453                 :            :         /* Align page_max to xfer_size aligned pages */
     454   [ +  +  +  + ]:         15 :         if ((page_max + 1) % pages_per_xfer != 0) {
     455         [ -  + ]:          3 :                 page_max += (pages_per_xfer - page_max % pages_per_xfer - 1);
     456                 :          0 :         }
     457                 :            : #ifdef DEBUG
     458                 :            :         /* Set check point valid map for validation */
     459   [ #  #  #  #  :         15 :         struct ftl_p2l_ckpt *ckpt = band->p2l_map.p2l_ckpt;
                   #  # ]
     460         [ +  + ]:         69 :         for (uint64_t i = 0; i <= page_max; i++) {
     461   [ #  #  #  # ]:         54 :                 ftl_bitmap_set(ckpt->bmp, i);
     462                 :          0 :         }
     463                 :            : #endif
     464                 :            : 
     465                 :         15 :         ftl_band_iter_init(band);
     466                 :            :         /* Align page max to xfer size and set iter */
     467   [ -  +  #  #  :         15 :         ftl_band_iter_set(band, (page_max / band->p2l_map.p2l_ckpt->pages_per_xfer + 1) * dev->xfer_size);
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     468                 :            : 
     469                 :         15 :         return 0;
     470                 :          0 : }
     471                 :            : 
     472                 :            : enum ftl_layout_region_type
     473                 :          7 : ftl_p2l_ckpt_region_type(const struct ftl_p2l_ckpt *ckpt) {
     474   [ #  #  #  #  :          7 :         return ckpt->layout_region->type;
             #  #  #  # ]
     475                 :            : }
     476                 :            : 
     477                 :            : struct ftl_p2l_ckpt *
     478                 :         19 : ftl_p2l_ckpt_acquire_region_type(struct spdk_ftl_dev *dev, uint32_t region_type)
     479                 :            : {
     480                 :         19 :         struct ftl_p2l_ckpt *ckpt = NULL;
     481                 :            : 
     482   [ +  -  #  #  :         20 :         TAILQ_FOREACH(ckpt, &dev->p2l_ckpt.free, link) {
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     483   [ +  +  #  #  :         20 :                 if (ckpt->layout_region->type == region_type) {
          #  #  #  #  #  
                      # ]
     484                 :         19 :                         break;
     485                 :            :                 }
     486                 :          0 :         }
     487                 :            : 
     488   [ -  +  #  # ]:         19 :         assert(ckpt);
     489                 :            : 
     490   [ +  +  #  #  :         19 :         TAILQ_REMOVE(&dev->p2l_ckpt.free, ckpt, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     491   [ #  #  #  #  :         19 :         TAILQ_INSERT_TAIL(&dev->p2l_ckpt.inuse, ckpt, link);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                #  #  # ]
     492                 :            : 
     493                 :         19 :         return ckpt;
     494                 :            : }
     495                 :            : 
     496                 :            : int
     497                 :          9 : ftl_mngt_p2l_ckpt_restore_clean(struct ftl_band *band)
     498                 :            : {
     499   [ #  #  #  # ]:          9 :         struct spdk_ftl_dev *dev = band->dev;
     500         [ #  # ]:          9 :         struct ftl_layout *layout = &dev->layout;
     501                 :            :         struct ftl_p2l_ckpt_page_no_vss *page;
     502   [ #  #  #  #  :          9 :         enum ftl_layout_region_type md_region = band->md->p2l_md_region;
             #  #  #  # ]
     503                 :            :         struct ftl_p2l_ckpt *ckpt;
     504                 :            :         uint64_t page_no;
     505                 :            :         uint64_t num_written_pages, lbas_synced;
     506                 :            : 
     507   [ +  -  -  + ]:          9 :         if (md_region < FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MIN ||
     508                 :          0 :             md_region > FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MAX) {
     509                 :          0 :                 return -EINVAL;
     510                 :            :         }
     511                 :            : 
     512   [ -  +  -  +  :          9 :         assert(band->md->iter.offset % dev->xfer_size == 0);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
     513                 :            : 
     514                 :            :         /* Associate band with md region before shutdown */
     515   [ +  +  #  #  :          9 :         if (!band->p2l_map.p2l_ckpt) {
             #  #  #  # ]
     516   [ #  #  #  #  :          3 :                 band->p2l_map.p2l_ckpt = ftl_p2l_ckpt_acquire_region_type(dev, md_region);
                   #  # ]
     517                 :          0 :         }
     518                 :            : 
     519                 :            :         /* Band was opened but no data was written */
     520   [ -  +  #  #  :          9 :         if (band->md->iter.offset == 0) {
          #  #  #  #  #  
                #  #  # ]
     521                 :          0 :                 return 0;
     522                 :            :         }
     523                 :            : 
     524   [ #  #  #  #  :          9 :         ckpt = band->p2l_map.p2l_ckpt;
                   #  # ]
     525   [ -  +  #  #  :          9 :         num_written_pages = band->md->iter.offset / dev->xfer_size * ckpt->pages_per_xfer;
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
     526                 :            : 
     527                 :          9 :         page_no = 0;
     528                 :          9 :         lbas_synced = 0;
     529                 :            : 
     530                 :            :         /* Restore P2L map up to last written page */
     531   [ #  #  #  #  :          9 :         page = ftl_md_get_buffer(layout->md[md_region]);
                   #  # ]
     532                 :            : 
     533                 :            : 
     534   [ +  +  #  # ]:       1627 :         for (; page_no < num_written_pages; page_no++, page++) {
     535   [ -  +  #  #  :       1618 :                 assert(page->metadata.p2l_ckpt.seq_id == band->md->seq);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
     536                 :            :                 /* Restore the page from P2L checkpoint */
     537   [ -  +  -  +  :       1618 :                 memcpy(band->p2l_map.band_map + lbas_synced, page->map,
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     538   [ #  #  #  #  :       1618 :                        page->metadata.p2l_ckpt.count * sizeof(struct ftl_p2l_map_entry));
             #  #  #  # ]
     539                 :            : 
     540   [ #  #  #  #  :       1618 :                 lbas_synced += page->metadata.p2l_ckpt.count;
             #  #  #  # ]
     541                 :            : 
     542                 :            : #if defined(DEBUG)
     543   [ -  +  #  #  :       1618 :                 assert(ftl_bitmap_get(band->p2l_map.p2l_ckpt->bmp, page_no) == false);
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     544   [ #  #  #  #  :       1618 :                 ftl_bitmap_set(band->p2l_map.p2l_ckpt->bmp, page_no);
          #  #  #  #  #  
                      # ]
     545                 :            : #endif
     546                 :          0 :         }
     547                 :            : 
     548   [ -  +  -  +  :          9 :         assert(lbas_synced % dev->xfer_size == 0);
          #  #  #  #  #  
                      # ]
     549                 :            : 
     550   [ -  +  #  #  :          9 :         assert(page->metadata.p2l_ckpt.seq_id < band->md->seq);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
     551                 :            : 
     552                 :          9 :         return 0;
     553                 :          0 : }
     554                 :            : 
     555                 :            : void
     556                 :          1 : ftl_mngt_p2l_ckpt_restore_shm_clean(struct ftl_band *band)
     557                 :            : {
     558   [ #  #  #  # ]:          1 :         struct spdk_ftl_dev *dev = band->dev;
     559   [ #  #  #  #  :          1 :         enum ftl_layout_region_type md_region = band->md->p2l_md_region;
             #  #  #  # ]
     560                 :            : 
     561                 :            :         /* Associate band with md region before shutdown */
     562   [ +  -  #  #  :          1 :         if (!band->p2l_map.p2l_ckpt) {
             #  #  #  # ]
     563   [ #  #  #  #  :          1 :                 band->p2l_map.p2l_ckpt = ftl_p2l_ckpt_acquire_region_type(dev, md_region);
                   #  # ]
     564                 :          0 :         }
     565                 :            : 
     566                 :            : #if defined(DEBUG)
     567                 :            :         uint64_t page_no;
     568                 :            :         uint64_t num_written_pages;
     569                 :            : 
     570   [ -  +  -  +  :          1 :         assert(band->md->iter.offset % dev->xfer_size == 0);
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
     571   [ -  +  #  #  :          1 :         num_written_pages = band->md->iter.offset / dev->xfer_size * band->p2l_map.p2l_ckpt->pages_per_xfer;
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     572                 :            : 
     573                 :            :         /* Band was opened but no data was written */
     574   [ -  +  #  #  :          1 :         if (band->md->iter.offset == 0) {
          #  #  #  #  #  
                #  #  # ]
     575                 :          0 :                 return;
     576                 :            :         }
     577                 :            : 
     578                 :            :         /* Set page number to first data page - skip head md */
     579                 :          1 :         page_no = 0;
     580                 :            : 
     581         [ +  + ]:        997 :         for (; page_no < num_written_pages; page_no++) {
     582   [ -  +  #  #  :        996 :                 assert(ftl_bitmap_get(band->p2l_map.p2l_ckpt->bmp, page_no) == false);
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     583   [ #  #  #  #  :        996 :                 ftl_bitmap_set(band->p2l_map.p2l_ckpt->bmp, page_no);
          #  #  #  #  #  
                      # ]
     584                 :          0 :         }
     585                 :            : #endif
     586                 :          0 : }

Generated by: LCOV version 1.15