Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright (C) 2018 Intel Corporation.
3 : : * All rights reserved.
4 : : * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : : */
6 : :
7 : : #include "spdk/stdinc.h"
8 : :
9 : : #include "queue_internal.h"
10 : :
11 : : #include "spdk/reduce.h"
12 : : #include "spdk/env.h"
13 : : #include "spdk/string.h"
14 : : #include "spdk/bit_array.h"
15 : : #include "spdk/util.h"
16 : : #include "spdk/log.h"
17 : : #include "spdk/memory.h"
18 : : #include "spdk/tree.h"
19 : : #include "spdk/queue.h"
20 : :
21 : : #include "libpmem.h"
22 : :
23 : : /* Always round up the size of the PM region to the nearest cacheline. */
24 : : #define REDUCE_PM_SIZE_ALIGNMENT 64
25 : :
26 : : /* Offset into the backing device where the persistent memory file's path is stored. */
27 : : #define REDUCE_BACKING_DEV_PATH_OFFSET 4096
28 : :
29 : : #define REDUCE_EMPTY_MAP_ENTRY -1ULL
30 : :
31 : : #define REDUCE_NUM_VOL_REQUESTS 256
32 : :
33 : : /* Structure written to offset 0 of both the pm file and the backing device. */
34 : : struct spdk_reduce_vol_superblock {
35 : : uint8_t signature[8];
36 : : struct spdk_reduce_vol_params params;
37 : : uint8_t reserved[3983];
38 : : };
39 : : SPDK_STATIC_ASSERT(sizeof(struct spdk_reduce_vol_superblock) == 4096, "size incorrect");
40 : :
41 : : #define SPDK_REDUCE_SIGNATURE "SPDKREDU"
42 : : /* null terminator counts one */
43 : : SPDK_STATIC_ASSERT(sizeof(SPDK_REDUCE_SIGNATURE) - 1 ==
44 : : SPDK_SIZEOF_MEMBER(struct spdk_reduce_vol_superblock, signature), "size incorrect");
45 : :
46 : : #define REDUCE_PATH_MAX 4096
47 : :
48 : : #define REDUCE_ZERO_BUF_SIZE 0x100000
49 : :
50 : : /* REDUCE_META_CACHE_BLOCK_NUM should not less than 2*REDUCE_NUM_VOL_REQUESTS */
51 : : #define REDUCE_META_CACHE_BLOCK_NUM (4 * REDUCE_NUM_VOL_REQUESTS)
52 : : #define REDUCE_META_INIT_BUF_SIZE (256 << 10) /* 256KB */
53 : : #define REDUCE_META_CACHE_RECLAIM_RATIO 20
54 : :
55 : : /**
56 : : * Describes a persistent memory file used to hold metadata associated with a
57 : : * compressed volume.
58 : : */
59 : : struct spdk_reduce_pm_file {
60 : : char path[REDUCE_PATH_MAX];
61 : : void *pm_buf;
62 : : int pm_is_pmem;
63 : : uint64_t size;
64 : : };
65 : :
66 : : #define REDUCE_IO_READV 1
67 : : #define REDUCE_IO_WRITEV 2
68 : : #define REDUCE_IO_UNMAP 3
69 : :
70 : : struct spdk_reduce_chunk_map {
71 : : uint32_t compressed_size;
72 : : uint32_t reserved;
73 : : uint64_t io_unit_index[0];
74 : : };
75 : :
76 : : struct spdk_reduce_vol_request {
77 : : /**
78 : : * Scratch buffer used for uncompressed chunk. This is used for:
79 : : * 1) source buffer for compression operations
80 : : * 2) destination buffer for decompression operations
81 : : * 3) data buffer when writing uncompressed chunk to disk
82 : : * 4) data buffer when reading uncompressed chunk from disk
83 : : */
84 : : uint8_t *decomp_buf;
85 : : struct iovec *decomp_buf_iov;
86 : :
87 : : /**
88 : : * These are used to construct the iovecs that are sent to
89 : : * the decomp engine, they point to a mix of the scratch buffer
90 : : * and user buffer
91 : : */
92 : : struct iovec decomp_iov[REDUCE_MAX_IOVECS + 2];
93 : : int decomp_iovcnt;
94 : :
95 : : /**
96 : : * Scratch buffer used for compressed chunk. This is used for:
97 : : * 1) destination buffer for compression operations
98 : : * 2) source buffer for decompression operations
99 : : * 3) data buffer when writing compressed chunk to disk
100 : : * 4) data buffer when reading compressed chunk from disk
101 : : */
102 : : uint8_t *comp_buf;
103 : : struct iovec *comp_buf_iov;
104 : : struct iovec *iov;
105 : : bool rmw;
106 : : struct spdk_reduce_vol *vol;
107 : : int type;
108 : : int reduce_errno;
109 : : int iovcnt;
110 : : int num_backing_ops;
111 : : uint32_t num_io_units;
112 : : struct spdk_reduce_backing_io *backing_io;
113 : : bool chunk_is_compressed;
114 : : bool copy_after_decompress;
115 : : uint64_t offset;
116 : : uint64_t logical_map_index;
117 : : uint64_t length;
118 : : uint64_t chunk_map_index;
119 : : /* record the old chunkmap idx when rmw */
120 : : uint64_t read_chunk_map_index;
121 : : struct spdk_reduce_chunk_map *chunk;
122 : : /* save the new chunkinfo to the pre-allocated chunk mem,
123 : : * then memcpy from here to the metablock which used to submit backend
124 : : */
125 : : struct spdk_reduce_chunk_map *prealloc_chunk;
126 : : spdk_reduce_vol_op_complete cb_fn;
127 : : void *cb_arg;
128 : : TAILQ_ENTRY(spdk_reduce_vol_request) tailq;
129 : : RB_ENTRY(spdk_reduce_vol_request) rbnode;
130 : : struct spdk_reduce_vol_cb_args backing_cb_args;
131 : : };
132 : :
133 : : struct spdk_reduce_vol {
134 : : struct spdk_reduce_vol_params params;
135 : : struct spdk_reduce_vol_info info;
136 : : uint32_t backing_io_units_per_chunk;
137 : : uint32_t backing_lba_per_io_unit;
138 : : uint32_t logical_blocks_per_chunk;
139 : : struct spdk_reduce_pm_file pm_file;
140 : : struct spdk_reduce_backing_dev *backing_dev;
141 : : struct spdk_reduce_vol_superblock *backing_super;
142 : : struct spdk_reduce_vol_superblock *pm_super;
143 : : uint64_t *pm_logical_map;
144 : : uint64_t *pm_chunk_maps;
145 : :
146 : : struct spdk_bit_array *allocated_chunk_maps;
147 : : /* The starting position when looking for a block from allocated_chunk_maps */
148 : : uint64_t find_chunk_offset;
149 : : /* Cache free chunks to speed up lookup of free chunk. */
150 : : struct reduce_queue free_chunks_queue;
151 : : struct spdk_bit_array *allocated_backing_io_units;
152 : : /* The starting position when looking for a block from allocated_backing_io_units */
153 : : uint64_t find_block_offset;
154 : : /* Cache free blocks for backing bdev to speed up lookup of free backing blocks. */
155 : : struct reduce_queue free_backing_blocks_queue;
156 : :
157 : : struct spdk_reduce_vol_request *request_mem;
158 : : TAILQ_HEAD(, spdk_reduce_vol_request) free_requests;
159 : : RB_HEAD(executing_req_tree, spdk_reduce_vol_request) executing_requests;
160 : : TAILQ_HEAD(, spdk_reduce_vol_request) queued_requests;
161 : :
162 : : /* Single contiguous buffer used for all request buffers for this volume. */
163 : : uint8_t *buf_mem;
164 : : struct iovec *buf_iov_mem;
165 : : /* Single contiguous buffer used for backing io buffers for this volume. */
166 : : uint8_t *buf_backing_io_mem;
167 : :
168 : : /* the count of meta cache block we prepare */
169 : : uint32_t metablocks_num;
170 : : /* A pointer of many 4k DMA mem from huge page, It has metablocks_num * 4K Bytes */
171 : : uint8_t *metablock_buf;
172 : : /* A pointer of many mblk describtion struct mem. The count of these struct is equal to metablocks num */
173 : : struct reduce_metablock_extent *metablock_extent_mem;
174 : : TAILQ_HEAD(, reduce_metablock_extent) metablocks_free;
175 : : RB_HEAD(metablock_cache_tree, reduce_metablock_extent) metablocks_caching;
176 : : TAILQ_HEAD(, reduce_metablock_extent) metablocks_lru;
177 : :
178 : : /* use independent mem for meta opetation backingio */
179 : : struct reduce_meta_request *meta_request_buf;
180 : : TAILQ_HEAD(, reduce_meta_request) free_meta_request;
181 : : };
182 : :
183 : : struct reduce_metablock_extent {
184 : : uint32_t mblk_sn;
185 : : /* pointer to 4K mem */
186 : : struct iovec iov;
187 : : /* the node to attach free list or lru list */
188 : : TAILQ_ENTRY(reduce_metablock_extent) tailq;
189 : : /* the node to attach read/write tree */
190 : : RB_ENTRY(reduce_metablock_extent) rbnode;
191 : :
192 : : /* the write_meta_req cnt that this one metadata write operation catching */
193 : : uint32_t batch_write_cnt;
194 : : /* the refcnt should be both pending_to_supply_read and pending_to_write elem count */
195 : : uint32_t refcnt;
196 : : /* when multi io to access metablock, callback them by the tailq */
197 : : TAILQ_HEAD(, reduce_meta_request) pending_to_supply_read;
198 : : TAILQ_HEAD(, reduce_meta_request) pending_to_write;
199 : : };
200 : :
201 : : struct meta_builtin_readv_ctx {
202 : : struct spdk_reduce_vol *vol;
203 : : struct iovec *iov;
204 : : int iovcnt;
205 : : uint64_t offset;
206 : : uint64_t length;
207 : :
208 : : spdk_reduce_vol_op_complete cb_fn;
209 : : void *cb_arg;
210 : : };
211 : :
212 : : typedef void (*reduce_request_fn)(void *_req, int reduce_errno);
213 : :
214 : : struct meta_builtin_read_chunk_ctx {
215 : : struct spdk_reduce_vol *vol;
216 : : struct spdk_reduce_vol_request *req;
217 : : reduce_request_fn next_fn;
218 : : };
219 : :
220 : : struct meta_builtin_request_ctx {
221 : : struct spdk_reduce_vol_request *req;
222 : : void *logicalmap_meta_req;
223 : : };
224 : :
225 : : struct meta_builtin_load_ctx {
226 : : struct reduce_init_load_ctx *load_ctx;
227 : : };
228 : :
229 : : /* meta read/write operations are async, should save the context before meta opetations */
230 : : union reduce_meta_restore_ctx_container {
231 : : struct meta_builtin_readv_ctx readv_ctx;
232 : : struct meta_builtin_read_chunk_ctx read_chunk_ctx;
233 : : struct meta_builtin_request_ctx req_ctx;
234 : : struct meta_builtin_load_ctx load_ctx;
235 : : };
236 : :
237 : : struct reduce_meta_request {
238 : : /* the node to attach free_meta_backingio or pending list of struct reduce_metablock_extent */
239 : : TAILQ_ENTRY(reduce_meta_request) tailq;
240 : :
241 : : struct spdk_reduce_vol *vol;
242 : : /* if need or may need to update meta, set the is_write flag */
243 : : bool is_write;
244 : : /* when we want to update meta, should do read-modify-write about the metablock.
245 : : * 1. set the supply_read flag, and do read.
246 : : * 2. read callback, modify the elem on metablock, and reset the supply_read flag. send do write.
247 : : * 3. write callback, we can judge the meta stage according the supply_read flag.
248 : : */
249 : : bool supply_read;
250 : : enum spdk_reduce_meta_type mtype;
251 : : uint32_t mblk_sn;
252 : : uint32_t elem_sn_on_mblk;
253 : : /* A pointer to caching meta block */
254 : : struct reduce_metablock_extent *mblk_extent;
255 : : union reduce_meta_restore_ctx_container restore_ctx;
256 : :
257 : : struct spdk_reduce_vol_cb_args cb_args;
258 : : struct spdk_reduce_backing_io backing_io;
259 : : };
260 : :
261 : : static void _start_readv_request(struct spdk_reduce_vol_request *req);
262 : : static void _start_writev_request(struct spdk_reduce_vol_request *req);
263 : : static uint8_t *g_zero_buf;
264 : : static int g_vol_count = 0;
265 : :
266 : : /*
267 : : * Allocate extra metadata chunks and corresponding backing io units to account for
268 : : * outstanding IO in worst case scenario where logical map is completely allocated
269 : : * and no data can be compressed. We need extra chunks in this case to handle
270 : : * in-flight writes since reduce never writes data in place.
271 : : */
272 : : #define REDUCE_NUM_EXTRA_CHUNKS 128
273 : :
274 : : static void
275 : 611370 : _reduce_persist(struct spdk_reduce_vol *vol, const void *addr, size_t len)
276 : : {
277 [ - + ]: 611370 : if (vol->pm_file.pm_is_pmem) {
278 : 0 : pmem_persist(addr, len);
279 : : } else {
280 : 611370 : pmem_msync(addr, len);
281 : : }
282 : 611370 : }
283 : :
284 : : static uint64_t
285 : 24 : _get_pm_logical_map_size(uint64_t vol_size, uint64_t chunk_size)
286 : : {
287 : : uint64_t chunks_in_logical_map, logical_map_size;
288 : :
289 [ - + ]: 24 : chunks_in_logical_map = vol_size / chunk_size;
290 : 24 : logical_map_size = chunks_in_logical_map * sizeof(uint64_t);
291 : :
292 : : /* Round up to next cacheline. */
293 : 24 : return spdk_divide_round_up(logical_map_size, REDUCE_PM_SIZE_ALIGNMENT) *
294 : : REDUCE_PM_SIZE_ALIGNMENT;
295 : : }
296 : :
297 : : static uint64_t
298 : 973607 : _get_total_chunks(uint64_t vol_size, uint64_t chunk_size)
299 : : {
300 : : uint64_t num_chunks;
301 : :
302 [ - + ]: 973607 : num_chunks = vol_size / chunk_size;
303 : 973607 : num_chunks += REDUCE_NUM_EXTRA_CHUNKS;
304 : :
305 : 973607 : return num_chunks;
306 : : }
307 : :
308 : : static inline uint32_t
309 : 1679472 : _reduce_vol_get_chunk_struct_size(uint64_t backing_io_units_per_chunk)
310 : : {
311 : 1679472 : return sizeof(struct spdk_reduce_chunk_map) + sizeof(uint64_t) * backing_io_units_per_chunk;
312 : : }
313 : :
314 : : static uint64_t
315 : 12 : _get_pm_total_chunks_size(uint64_t vol_size, uint64_t chunk_size, uint64_t backing_io_unit_size)
316 : : {
317 : : uint64_t io_units_per_chunk, num_chunks, total_chunks_size;
318 : :
319 : 12 : num_chunks = _get_total_chunks(vol_size, chunk_size);
320 [ - + ]: 12 : io_units_per_chunk = chunk_size / backing_io_unit_size;
321 : :
322 : 12 : total_chunks_size = num_chunks * _reduce_vol_get_chunk_struct_size(io_units_per_chunk);
323 : :
324 : 12 : return spdk_divide_round_up(total_chunks_size, REDUCE_PM_SIZE_ALIGNMENT) *
325 : : REDUCE_PM_SIZE_ALIGNMENT;
326 : : }
327 : :
328 : : static struct spdk_reduce_chunk_map *
329 : 973573 : _reduce_vol_get_chunk_map(struct spdk_reduce_vol *vol, uint64_t chunk_map_index)
330 : : {
331 : : uintptr_t chunk_map_addr;
332 : :
333 [ - + ]: 973573 : assert(chunk_map_index < _get_total_chunks(vol->params.vol_size, vol->params.chunk_size));
334 : :
335 : 973573 : chunk_map_addr = (uintptr_t)vol->pm_chunk_maps;
336 : 973573 : chunk_map_addr += chunk_map_index *
337 : 973573 : _reduce_vol_get_chunk_struct_size(vol->backing_io_units_per_chunk);
338 : :
339 : 973573 : return (struct spdk_reduce_chunk_map *)chunk_map_addr;
340 : : }
341 : :
342 : : static int
343 : 22 : _validate_vol_params(struct spdk_reduce_vol_params *params)
344 : : {
345 [ - + ]: 22 : if (params->vol_size > 0) {
346 : : /**
347 : : * User does not pass in the vol size - it gets calculated by libreduce from
348 : : * values in this structure plus the size of the backing device.
349 : : */
350 : 0 : return -EINVAL;
351 : : }
352 : :
353 [ + - + - ]: 22 : if (params->chunk_size == 0 || params->backing_io_unit_size == 0 ||
354 [ - + ]: 22 : params->logical_block_size == 0) {
355 : 0 : return -EINVAL;
356 : : }
357 : :
358 : : /* Chunk size must be an even multiple of the backing io unit size. */
359 [ - + - + ]: 22 : if ((params->chunk_size % params->backing_io_unit_size) != 0) {
360 : 0 : return -EINVAL;
361 : : }
362 : :
363 : : /* Chunk size must be an even multiple of the logical block size. */
364 [ - + - + ]: 22 : if ((params->chunk_size % params->logical_block_size) != 0) {
365 : 0 : return -1;
366 : : }
367 : :
368 : 22 : return 0;
369 : : }
370 : :
371 : : static uint64_t
372 : 22 : _get_vol_size(uint64_t chunk_size, uint64_t backing_dev_size)
373 : : {
374 : : uint64_t num_chunks;
375 : :
376 [ - + ]: 22 : num_chunks = backing_dev_size / chunk_size;
377 [ - + ]: 22 : if (num_chunks <= REDUCE_NUM_EXTRA_CHUNKS) {
378 : 0 : return 0;
379 : : }
380 : :
381 : 22 : num_chunks -= REDUCE_NUM_EXTRA_CHUNKS;
382 : 22 : return num_chunks * chunk_size;
383 : : }
384 : :
385 : : static uint64_t
386 : 12 : _get_pm_file_size(struct spdk_reduce_vol_params *params)
387 : : {
388 : : uint64_t total_pm_size;
389 : :
390 : 12 : total_pm_size = sizeof(struct spdk_reduce_vol_superblock);
391 : 12 : total_pm_size += _get_pm_logical_map_size(params->vol_size, params->chunk_size);
392 : 12 : total_pm_size += _get_pm_total_chunks_size(params->vol_size, params->chunk_size,
393 : 12 : params->backing_io_unit_size);
394 : 12 : return total_pm_size;
395 : : }
396 : :
397 : : const struct spdk_uuid *
398 : 0 : spdk_reduce_vol_get_uuid(struct spdk_reduce_vol *vol)
399 : : {
400 : 0 : return &vol->params.uuid;
401 : : }
402 : :
403 : : static uint32_t
404 : 1184144 : _meta_get_mblksn(struct spdk_reduce_vol_params *params, enum spdk_reduce_meta_type mtype,
405 : : uint64_t elem_sn)
406 : : {
407 : : uint32_t mblksn;
408 : :
409 : 1184144 : mblksn = params->meta_region_desc[mtype].offset + elem_sn /
410 [ - + ]: 1184144 : params->meta_region_desc[mtype].elems_per_mblk;
411 [ - + ]: 1184144 : assert(mblksn < params->meta_region_desc[mtype].offset + params->meta_region_desc[mtype].length);
412 : :
413 : 1184144 : return mblksn;
414 : : }
415 : :
416 : : static inline uint32_t
417 : 1184144 : _meta_get_elem_sn_on_mblk(struct spdk_reduce_vol_params *params, enum spdk_reduce_meta_type mtype,
418 : : uint64_t elem_sn)
419 : : {
420 [ - + ]: 1184144 : return elem_sn % params->meta_region_desc[mtype].elems_per_mblk;
421 : : }
422 : :
423 : : static inline void *
424 : 1184144 : _reduce_get_meta_elem_addr(struct reduce_meta_request *meta_req)
425 : : {
426 : 2368288 : return meta_req->mblk_extent->iov.iov_base + meta_req->elem_sn_on_mblk *
427 : 1184144 : meta_req->vol->params.meta_region_desc[meta_req->mtype].size_per_elem;
428 : : }
429 : :
430 : : static void
431 : 12 : _initialize_vol_pm_pointers(struct spdk_reduce_vol *vol)
432 : : {
433 : : uint64_t logical_map_size;
434 : :
435 : : /* Superblock is at the beginning of the pm file. */
436 : 12 : vol->pm_super = (struct spdk_reduce_vol_superblock *)vol->pm_file.pm_buf;
437 : :
438 : : /* Logical map immediately follows the super block. */
439 : 12 : vol->pm_logical_map = (uint64_t *)(vol->pm_super + 1);
440 : :
441 : : /* Chunks maps follow the logical map. */
442 : 12 : logical_map_size = _get_pm_logical_map_size(vol->params.vol_size, vol->params.chunk_size);
443 : 12 : vol->pm_chunk_maps = (uint64_t *)((uint8_t *)vol->pm_logical_map + logical_map_size);
444 : 12 : }
445 : :
446 : : struct reduce_mblk_init_ctx {
447 : : /* unit: bytes */
448 : : void *meta_init_buf;
449 : : size_t buf_len;
450 : : /* unit: backing blocklen */
451 : : uint64_t off_need_init;
452 : : uint64_t len_need_init;
453 : : uint64_t off_need_init_next;
454 : : uint32_t maxlen_per_io;
455 : : };
456 : :
457 : : /* We need 2 iovs during load - one for the superblock, another for the path */
458 : : #define LOAD_IOV_COUNT 2
459 : :
460 : : struct reduce_init_load_ctx {
461 : : struct spdk_reduce_vol *vol;
462 : : struct spdk_reduce_vol_cb_args backing_cb_args;
463 : : spdk_reduce_vol_op_with_handle_complete cb_fn;
464 : : void *cb_arg;
465 : : struct iovec iov[LOAD_IOV_COUNT];
466 : : void *path;
467 : : struct spdk_reduce_backing_io *backing_io;
468 : : struct reduce_mblk_init_ctx mblk_init_ctx;
469 : : };
470 : :
471 : : static inline bool
472 : 15272 : _addr_crosses_huge_page(const void *addr, size_t *size)
473 : : {
474 : : size_t _size;
475 : : uint64_t rc;
476 : :
477 [ - + ]: 15272 : assert(size);
478 : :
479 : 15272 : _size = *size;
480 : 15272 : rc = spdk_vtophys(addr, size);
481 : :
482 [ + - - + ]: 15272 : return rc == SPDK_VTOPHYS_ERROR || _size != *size;
483 : : }
484 : :
485 : : static inline int
486 : 11264 : _set_buffer(uint8_t **vol_buffer, uint8_t **_addr, uint8_t *addr_range, size_t buffer_size)
487 : : {
488 : : uint8_t *addr;
489 : 11264 : size_t size_tmp = buffer_size;
490 : :
491 : 11264 : addr = *_addr;
492 : :
493 : : /* Verify that addr + buffer_size doesn't cross huge page boundary */
494 [ - + ]: 11264 : if (_addr_crosses_huge_page(addr, &size_tmp)) {
495 : : /* Memory start is aligned on 2MiB, so buffer should be located at the end of the page.
496 : : * Skip remaining bytes and continue from the beginning of the next page */
497 : 0 : addr += size_tmp;
498 : : }
499 : :
500 [ - + ]: 11264 : if (addr + buffer_size > addr_range) {
501 : 0 : SPDK_ERRLOG("Vol buffer %p out of range %p\n", addr, addr_range);
502 : 0 : return -ERANGE;
503 : : }
504 : :
505 : 11264 : *vol_buffer = addr;
506 : 11264 : *_addr = addr + buffer_size;
507 : :
508 : 11264 : return 0;
509 : : }
510 : :
511 : : static inline uint64_t
512 : 348668 : _reduce_meta_backingio_get_lba(struct reduce_meta_request *meta_req)
513 : : {
514 [ - + ]: 348668 : return meta_req->mblk_sn * REDUCE_META_BLKSZ / meta_req->vol->backing_dev->blocklen;
515 : : }
516 : :
517 : : static inline uint64_t
518 : 348668 : _reduce_meta_backingio_get_lba_count(struct reduce_meta_request *meta_req)
519 : : {
520 [ - + ]: 348668 : return REDUCE_META_BLKSZ / meta_req->vol->backing_dev->blocklen;
521 : : }
522 : :
523 : : static void
524 : 348668 : _reduce_meta_block_rw_backing(struct reduce_meta_request *meta_req,
525 : : enum spdk_reduce_backing_io_type io_type)
526 : : {
527 : 348668 : struct spdk_reduce_vol *vol = meta_req->vol;
528 : 348668 : struct spdk_reduce_backing_io *backing_io = &meta_req->backing_io;
529 : :
530 : 348668 : backing_io->dev = vol->backing_dev;
531 : 348668 : backing_io->iov = &meta_req->mblk_extent->iov;
532 : 348668 : backing_io->iovcnt = 1;
533 : 348668 : backing_io->lba = _reduce_meta_backingio_get_lba(meta_req);
534 : 348668 : backing_io->lba_count = _reduce_meta_backingio_get_lba_count(meta_req);
535 : 348668 : backing_io->backing_io_type = io_type;
536 : 348668 : vol->backing_dev->submit_backing_io(backing_io);
537 : 348668 : }
538 : :
539 : : static int
540 : 5016645 : metablock_cmp(struct reduce_metablock_extent *mblk1, struct reduce_metablock_extent *mblk2)
541 : : {
542 [ + + ]: 8383856 : return (mblk1->mblk_sn < mblk2->mblk_sn ? -1 : mblk1->mblk_sn >
543 : 3367211 : mblk2->mblk_sn);
544 : : }
545 [ + + + + : 5018209 : RB_GENERATE_STATIC(metablock_cache_tree, reduce_metablock_extent, rbnode, metablock_cmp);
+ + + + +
+ + + + -
+ + - + +
+ - + + +
+ + + + -
+ - - - -
- - - - -
- + + + +
+ + + + #
# # # # #
# # # # #
# # # ]
546 : :
547 : : static inline void
548 : 1184144 : _metablock_lru_update(struct spdk_reduce_vol *vol, struct reduce_metablock_extent *mblkext,
549 : : bool new_add)
550 : : {
551 [ + + ]: 1184144 : if (!new_add) {
552 [ + + ]: 1183813 : TAILQ_REMOVE(&vol->metablocks_lru, mblkext, tailq);
553 : : }
554 : 1184144 : TAILQ_INSERT_TAIL(&vol->metablocks_lru, mblkext, tailq);
555 : 1184144 : }
556 : :
557 : : static inline void
558 : 0 : _metablock_lru_delete(struct spdk_reduce_vol *vol, struct reduce_metablock_extent *mblkext)
559 : : {
560 : 0 : RB_REMOVE(metablock_cache_tree, &vol->metablocks_caching, mblkext);
561 [ # # ]: 0 : TAILQ_REMOVE(&vol->metablocks_lru, mblkext, tailq);
562 : 0 : TAILQ_INSERT_TAIL(&vol->metablocks_free, mblkext, tailq);
563 : 0 : }
564 : :
565 : : /* When the read process is complete, call this function to trigger
566 : : * the callback function registered in the metadata request on pending_to_supply_read.
567 : : * There will be metadata write requests and metadata read requests in the queue.
568 : : *
569 : : * When reading requests on pending_to_supply_read are callbacked, meta_req is released after the callback is completed.
570 : : * When the write request on pending_to_supply_read is callbacked, the metadata cache block data will be updated
571 : : * and the write request will be moved to the pending_to_write queue.
572 : : *
573 : : * After pending_to_supply_read is empty, if pending_to_write is not empty, one metadata write is triggered,
574 : : * and the metadata in multiple write requests is modified and solidify at one time.
575 : : */
576 : : static void
577 : 1184144 : _reduce_metablock_cache_read_done_update(struct reduce_meta_request *meta_req, int32_t error)
578 : : {
579 : 1184144 : struct reduce_metablock_extent *mblkext = meta_req->mblk_extent;
580 : : struct reduce_meta_request *meta_req_next, *meta_req_write;
581 : 1184144 : struct spdk_reduce_vol *vol = meta_req->vol;
582 : :
583 [ + + ]: 1184144 : TAILQ_REMOVE(&mblkext->pending_to_supply_read, meta_req, tailq);
584 [ - + + + : 1184144 : if (meta_req->is_write && !meta_req->supply_read) {
- + + + ]
585 : 504388 : TAILQ_INSERT_TAIL(&mblkext->pending_to_write, meta_req, tailq);
586 : 504388 : mblkext->batch_write_cnt++;
587 : : } else {
588 : 679756 : mblkext->refcnt--;
589 : 679756 : TAILQ_INSERT_TAIL(&vol->free_meta_request, meta_req, tailq);
590 : : }
591 : :
592 : : /* check whether collected all modify region */
593 : 1184144 : meta_req_next = TAILQ_FIRST(&mblkext->pending_to_supply_read);
594 [ + + ]: 1184144 : if (meta_req_next) {
595 [ - + - + ]: 476067 : SPDK_INFOLOG(reduce,
596 : : "metareq:%p mblkext %p mblksn %u ref %u. triger next readreq cpcb %p, error %d\n",
597 : : meta_req, mblkext,
598 : : mblkext->mblk_sn, mblkext->refcnt, meta_req_next, error);
599 [ - + ]: 476067 : assert(mblkext->refcnt);
600 : 476067 : meta_req_next->backing_io.backing_cb_args->cb_fn(meta_req_next->backing_io.backing_cb_args->cb_arg,
601 : : error);
602 : : } else {
603 : : /* we have callback all supply read metareq, start write to backend */
604 : 708077 : meta_req_write = TAILQ_FIRST(&mblkext->pending_to_write);
605 [ + + ]: 708077 : if (meta_req_write) {
606 [ - + - + ]: 348337 : SPDK_INFOLOG(reduce,
607 : : "metareq:%p mblkext %p mblksn %u ref %u. collect write req cnt %u, triger writereq do write %p\n",
608 : : meta_req, mblkext, mblkext->mblk_sn, mblkext->refcnt, mblkext->batch_write_cnt, meta_req_write);
609 [ - + ]: 348337 : assert(mblkext->refcnt);
610 [ - + ]: 348337 : assert(mblkext->batch_write_cnt);
611 [ - + ]: 348337 : assert(error == 0);
612 : 348337 : _reduce_meta_block_rw_backing(meta_req_write, SPDK_REDUCE_BACKING_IO_WRITE);
613 : : } else {
614 [ - + ]: 359740 : if (error != 0) {
615 : : /* This cache block has experienced abnormal reads or writes. Release it */
616 [ # # ]: 0 : assert(mblkext->refcnt == 0);
617 [ # # ]: 0 : assert(mblkext->batch_write_cnt == 0);
618 : 0 : _metablock_lru_delete(vol, mblkext);
619 : : }
620 : : }
621 : : }
622 : 1184144 : }
623 : :
624 : : /* After meta block has written done, callback meta_req in pending_to_write one by one.
625 : : * Of course, during the process of metadata asynchronous disks, there will appear metadata read and write requests,
626 : : * but they will be in pending_to_supply_read.
627 : : * Because the metadata has just been written to backend, the metadata cache block in memory is the same as that on backend,
628 : : * and the requests on the queue can be called back one by one.
629 : : */
630 : : static void
631 : 504388 : _reduce_metablock_cache_write_done_update(struct reduce_meta_request *meta_req, int32_t error)
632 : : {
633 : 504388 : struct reduce_metablock_extent *mblkext = meta_req->mblk_extent;
634 : : struct reduce_meta_request *meta_req_next;
635 : 504388 : struct spdk_reduce_vol *vol = meta_req->vol;
636 : :
637 : 504388 : mblkext->refcnt--;
638 : 504388 : mblkext->batch_write_cnt--;
639 : :
640 [ + + ]: 504388 : TAILQ_REMOVE(&mblkext->pending_to_write, meta_req, tailq);
641 : 504388 : TAILQ_INSERT_TAIL(&vol->free_meta_request, meta_req, tailq);
642 : :
643 : : /* trigger next */
644 : 504388 : meta_req_next = TAILQ_FIRST(&mblkext->pending_to_write);
645 [ + + ]: 504388 : if (meta_req_next) {
646 [ - + - + ]: 156051 : SPDK_INFOLOG(reduce,
647 : : "metareq:%p mblkext %p mblksn %u ref %u. write done to triger next writereq cpcb %p, error %d\n",
648 : : meta_req, mblkext,
649 : : mblkext->mblk_sn, mblkext->refcnt, meta_req_next, error);
650 [ - + ]: 156051 : assert(mblkext->refcnt);
651 [ - + ]: 156051 : assert(mblkext->batch_write_cnt);
652 : 156051 : meta_req_next->backing_io.backing_cb_args->cb_fn(meta_req_next->backing_io.backing_cb_args->cb_arg,
653 : : error);
654 : : } else {
655 : : /* when we async write to backend, there maybe new read come in, callback them */
656 [ - + ]: 348337 : assert(mblkext->batch_write_cnt == 0);
657 : 348337 : meta_req_next = TAILQ_FIRST(&mblkext->pending_to_supply_read);
658 [ + + ]: 348337 : if (meta_req_next) {
659 [ - + - + ]: 254005 : SPDK_INFOLOG(reduce,
660 : : "metareq:%p mblkext %p mblksn %u ref %u. triger next readreq cpcb %p, error %d\n",
661 : : meta_req, mblkext,
662 : : mblkext->mblk_sn, mblkext->refcnt, meta_req_next, error);
663 [ - + ]: 254005 : assert(mblkext->refcnt);
664 : 254005 : meta_req_next->backing_io.backing_cb_args->cb_fn(meta_req_next->backing_io.backing_cb_args->cb_arg,
665 : : error);
666 : : } else {
667 [ - + ]: 94332 : if (error != 0) {
668 : : /* This cache block has experienced abnormal reads or writes. Release it */
669 [ # # ]: 0 : assert(mblkext->refcnt == 0);
670 [ # # ]: 0 : assert(mblkext->batch_write_cnt == 0);
671 : 0 : _metablock_lru_delete(vol, mblkext);
672 : : }
673 : : }
674 : : }
675 : 504388 : }
676 : :
677 : : static uint32_t
678 : 0 : _metablock_lru_reclaim_batch(struct spdk_reduce_vol *vol)
679 : : {
680 : : struct reduce_metablock_extent *mblkext, *mblkext_next;
681 : 0 : uint32_t reclaim_num_expected = vol->metablocks_num * REDUCE_META_CACHE_RECLAIM_RATIO / 100;
682 : 0 : uint32_t reclaim_count = 0;
683 : :
684 [ # # ]: 0 : TAILQ_FOREACH_SAFE(mblkext, &vol->metablocks_lru, tailq, mblkext_next) {
685 [ # # # # ]: 0 : if (mblkext->refcnt == 0 && reclaim_count < reclaim_num_expected) {
686 [ # # ]: 0 : assert(TAILQ_EMPTY(&mblkext->pending_to_supply_read));
687 [ # # ]: 0 : assert(TAILQ_EMPTY(&mblkext->pending_to_write));
688 : 0 : _metablock_lru_delete(vol, mblkext);
689 : 0 : reclaim_count++;
690 : : } else {
691 : : break;
692 : : }
693 : : }
694 [ # # # # ]: 0 : SPDK_INFOLOG(reduce, "reclaim count %u\n", reclaim_count);
695 : :
696 : 0 : return reclaim_count;
697 : : }
698 : :
699 : : /* The entry of meta block operation */
700 : : static void
701 : 1184144 : _reduce_metablock_cache_read(struct reduce_meta_request *meta_req)
702 : : {
703 : 1184144 : struct reduce_metablock_extent *mblkext_res = NULL;
704 : : struct reduce_metablock_extent mblkext_find;
705 : 1184144 : struct spdk_reduce_vol *vol = meta_req->vol;
706 : :
707 : 1184144 : mblkext_find.mblk_sn = meta_req->mblk_sn;
708 : 1184144 : mblkext_res = RB_FIND(metablock_cache_tree, &vol->metablocks_caching, &mblkext_find);
709 : :
710 [ + + ]: 1184144 : if (mblkext_res) {
711 : 1183813 : meta_req->mblk_extent = mblkext_res;
712 : 1183813 : TAILQ_INSERT_TAIL(&mblkext_res->pending_to_supply_read, meta_req, tailq);
713 : 1183813 : mblkext_res->refcnt++;
714 : 1183813 : _metablock_lru_update(vol, mblkext_res, false);
715 [ + + ]: 1183813 : if (mblkext_res->refcnt == 1) {
716 : : /* metablock in cache, just sync cb */
717 [ - + - + ]: 453741 : SPDK_INFOLOG(reduce, "metareq %p mblkext %p mblksn %u ref %u. direct get from cache.\n", meta_req,
718 : : mblkext_res,
719 : : mblkext_res->mblk_sn, mblkext_res->refcnt);
720 : 453741 : meta_req->backing_io.backing_cb_args->cb_fn(meta_req->backing_io.backing_cb_args->cb_arg, 0);
721 : : } else {
722 : : /* pending wait cb */
723 [ - + - + ]: 730072 : SPDK_INFOLOG(reduce, "metareq %p mblkext %p mblksn %u ref %u. wait callback.\n", meta_req,
724 : : mblkext_res,
725 : : mblkext_res->mblk_sn, mblkext_res->refcnt);
726 : : }
727 : : } else {
728 : 331 : mblkext_res = TAILQ_FIRST(&vol->metablocks_free);
729 [ - + ]: 331 : if (!mblkext_res) {
730 : 0 : _metablock_lru_reclaim_batch(vol);
731 : 0 : mblkext_res = TAILQ_FIRST(&vol->metablocks_free);
732 [ # # ]: 0 : assert(mblkext_res);
733 : : }
734 [ + - ]: 331 : TAILQ_REMOVE(&vol->metablocks_free, mblkext_res, tailq);
735 [ - + ]: 331 : assert(mblkext_res->refcnt == 0);
736 [ - + ]: 331 : assert(mblkext_res->batch_write_cnt == 0);
737 : :
738 : 331 : mblkext_res->mblk_sn = meta_req->mblk_sn;
739 : 331 : RB_INSERT(metablock_cache_tree, &vol->metablocks_caching, mblkext_res);
740 : 331 : _metablock_lru_update(vol, mblkext_res, true);
741 : :
742 : 331 : TAILQ_INSERT_TAIL(&mblkext_res->pending_to_supply_read, meta_req, tailq);
743 : 331 : mblkext_res->refcnt++;
744 : 331 : meta_req->mblk_extent = mblkext_res;
745 : :
746 [ - + - + ]: 331 : SPDK_INFOLOG(reduce, "metareq %p mblkext %p mtype %u mblksn %u ref %u. add to cache\n", meta_req,
747 : : mblkext_res, meta_req->mtype,
748 : : mblkext_res->mblk_sn, mblkext_res->refcnt);
749 : 331 : _reduce_meta_block_rw_backing(meta_req, SPDK_REDUCE_BACKING_IO_READ);
750 : : }
751 : 1184144 : }
752 : :
753 : : static int
754 : 22 : _allocate_vol_requests(struct spdk_reduce_vol *vol)
755 : : {
756 : : struct spdk_reduce_vol_request *req;
757 : 22 : struct spdk_reduce_backing_dev *backing_dev = vol->backing_dev;
758 : : uint32_t reqs_in_2mb_page, huge_pages_needed, request_memlen;
759 : : uint8_t *buffer, *buffer_end;
760 : 22 : int i = 0;
761 : 22 : int rc = 0;
762 : :
763 : : /* It is needed to allocate comp and decomp buffers so that they do not cross physical
764 : : * page boundaries. Assume that the system uses default 2MiB pages and chunk_size is not
765 : : * necessarily power of 2
766 : : * Allocate 2x since we need buffers for both read/write and compress/decompress
767 : : * intermediate buffers. */
768 [ - + ]: 22 : reqs_in_2mb_page = VALUE_2MB / (vol->params.chunk_size * 2);
769 [ - + ]: 22 : if (!reqs_in_2mb_page) {
770 : 0 : return -EINVAL;
771 : : }
772 [ - + ]: 22 : huge_pages_needed = SPDK_CEIL_DIV(REDUCE_NUM_VOL_REQUESTS, reqs_in_2mb_page);
773 : :
774 : 22 : vol->buf_mem = spdk_dma_malloc(VALUE_2MB * huge_pages_needed, VALUE_2MB, NULL);
775 [ - + ]: 22 : if (vol->buf_mem == NULL) {
776 : 0 : return -ENOMEM;
777 : : }
778 : :
779 : 22 : request_memlen = sizeof(*req);
780 [ + + ]: 22 : if (vol->params.meta_builtin) {
781 : 10 : request_memlen = sizeof(*req) + _reduce_vol_get_chunk_struct_size(vol->backing_io_units_per_chunk);
782 : : }
783 : 22 : vol->request_mem = calloc(REDUCE_NUM_VOL_REQUESTS, request_memlen);
784 [ - + ]: 22 : if (vol->request_mem == NULL) {
785 : 0 : spdk_free(vol->buf_mem);
786 : 0 : vol->buf_mem = NULL;
787 : 0 : return -ENOMEM;
788 : : }
789 : :
790 : : /* Allocate 2x since we need iovs for both read/write and compress/decompress intermediate
791 : : * buffers.
792 : : */
793 : 22 : vol->buf_iov_mem = calloc(REDUCE_NUM_VOL_REQUESTS,
794 : 22 : 2 * sizeof(struct iovec) * vol->backing_io_units_per_chunk);
795 [ - + ]: 22 : if (vol->buf_iov_mem == NULL) {
796 : 0 : free(vol->request_mem);
797 : 0 : spdk_free(vol->buf_mem);
798 : 0 : vol->request_mem = NULL;
799 : 0 : vol->buf_mem = NULL;
800 : 0 : return -ENOMEM;
801 : : }
802 : :
803 : 22 : vol->buf_backing_io_mem = calloc(REDUCE_NUM_VOL_REQUESTS, (sizeof(struct spdk_reduce_backing_io) +
804 : 22 : backing_dev->user_ctx_size) * vol->backing_io_units_per_chunk);
805 [ - + ]: 22 : if (vol->buf_backing_io_mem == NULL) {
806 : 0 : free(vol->request_mem);
807 : 0 : free(vol->buf_iov_mem);
808 : 0 : spdk_free(vol->buf_mem);
809 : 0 : vol->request_mem = NULL;
810 : 0 : vol->buf_iov_mem = NULL;
811 : 0 : vol->buf_mem = NULL;
812 : 0 : return -ENOMEM;
813 : : }
814 : :
815 : 22 : buffer = vol->buf_mem;
816 : 22 : buffer_end = buffer + VALUE_2MB * huge_pages_needed;
817 : :
818 [ + + ]: 5654 : for (i = 0; i < REDUCE_NUM_VOL_REQUESTS; i++) {
819 : 5632 : req = (void *)vol->request_mem + i * request_memlen;
820 [ + + ]: 5632 : TAILQ_INSERT_HEAD(&vol->free_requests, req, tailq);
821 [ + + ]: 5632 : if (vol->params.meta_builtin) {
822 : 2560 : req->prealloc_chunk = (void *)req + sizeof(*req);
823 : : }
824 : 11264 : req->backing_io = (struct spdk_reduce_backing_io *)(vol->buf_backing_io_mem + i *
825 : 5632 : (sizeof(struct spdk_reduce_backing_io) + backing_dev->user_ctx_size) *
826 : 5632 : vol->backing_io_units_per_chunk);
827 : :
828 : 5632 : req->decomp_buf_iov = &vol->buf_iov_mem[(2 * i) * vol->backing_io_units_per_chunk];
829 : 5632 : req->comp_buf_iov = &vol->buf_iov_mem[(2 * i + 1) * vol->backing_io_units_per_chunk];
830 : :
831 : 5632 : rc = _set_buffer(&req->comp_buf, &buffer, buffer_end, vol->params.chunk_size);
832 [ - + ]: 5632 : if (rc) {
833 : 0 : SPDK_ERRLOG("Failed to set comp buffer for req idx %u, addr %p, start %p, end %p\n", i, buffer,
834 : : vol->buf_mem, buffer_end);
835 : 0 : break;
836 : : }
837 : 5632 : rc = _set_buffer(&req->decomp_buf, &buffer, buffer_end, vol->params.chunk_size);
838 [ - + ]: 5632 : if (rc) {
839 : 0 : SPDK_ERRLOG("Failed to set decomp buffer for req idx %u, addr %p, start %p, end %p\n", i, buffer,
840 : : vol->buf_mem, buffer_end);
841 : 0 : break;
842 : : }
843 : : }
844 : :
845 [ - + ]: 22 : if (rc) {
846 : 0 : free(vol->buf_backing_io_mem);
847 : 0 : free(vol->buf_iov_mem);
848 : 0 : free(vol->request_mem);
849 : 0 : spdk_free(vol->buf_mem);
850 : 0 : vol->buf_mem = NULL;
851 : 0 : vol->buf_backing_io_mem = NULL;
852 : 0 : vol->buf_iov_mem = NULL;
853 : 0 : vol->request_mem = NULL;
854 : : }
855 : :
856 : 22 : return rc;
857 : : }
858 : :
859 : : const struct spdk_reduce_vol_info *
860 : 22 : spdk_reduce_vol_get_info(const struct spdk_reduce_vol *vol)
861 : : {
862 : 22 : return &vol->info;
863 : : }
864 : :
865 : : static void
866 : 516 : _init_load_cleanup(struct spdk_reduce_vol *vol, struct reduce_init_load_ctx *ctx)
867 : : {
868 [ + + ]: 516 : if (ctx != NULL) {
869 : 472 : spdk_free(ctx->path);
870 : 472 : spdk_free(ctx->mblk_init_ctx.meta_init_buf);
871 : 472 : free(ctx->backing_io);
872 : 472 : free(ctx);
873 : : }
874 : :
875 [ + + ]: 516 : if (vol != NULL) {
876 [ + + ]: 472 : if (vol->pm_file.pm_buf != NULL) {
877 : 12 : pmem_unmap(vol->pm_file.pm_buf, vol->pm_file.size);
878 : : }
879 : :
880 : 472 : spdk_free(vol->backing_super);
881 : 472 : spdk_bit_array_free(&vol->allocated_chunk_maps);
882 : 472 : spdk_bit_array_free(&vol->allocated_backing_io_units);
883 : 472 : free(vol->request_mem);
884 : 472 : free(vol->buf_backing_io_mem);
885 : 472 : free(vol->buf_iov_mem);
886 : 472 : spdk_free(vol->buf_mem);
887 : 472 : spdk_free(vol->metablock_buf);
888 : 472 : free(vol->metablock_extent_mem);
889 : 472 : free(vol->meta_request_buf);
890 : 472 : free(vol);
891 : : }
892 : 516 : }
893 : :
894 : : static int
895 : 472 : _alloc_zero_buff(void)
896 : : {
897 : 472 : int rc = 0;
898 : :
899 : : /* The zero buffer is shared between all volumes and just used
900 : : * for reads so allocate one global instance here if not already
901 : : * allocated when another vol init'd or loaded.
902 : : */
903 [ + + ]: 472 : if (g_vol_count++ == 0) {
904 : 78 : g_zero_buf = spdk_zmalloc(REDUCE_ZERO_BUF_SIZE,
905 : : 64, NULL, SPDK_ENV_LCORE_ID_ANY,
906 : : SPDK_MALLOC_DMA);
907 [ - + ]: 78 : if (g_zero_buf == NULL) {
908 : 0 : g_vol_count--;
909 : 0 : rc = -ENOMEM;
910 : : }
911 : : }
912 : 472 : return rc;
913 : : }
914 : :
915 : : static int
916 : 10 : _allocate_vol_metablock_and_metaio(struct spdk_reduce_vol *vol)
917 : : {
918 : : uint32_t mblk_in_2mb_page, total_mblk, total_metareq, huge_pages_needed, backing_io_struct_size, i;
919 : : struct reduce_metablock_extent *mblk_extent;
920 : : struct reduce_meta_request *meta_request;
921 : 10 : int rc = 0;
922 : :
923 : 10 : mblk_in_2mb_page = VALUE_2MB / REDUCE_META_BLKSZ;
924 [ - + ]: 10 : huge_pages_needed = SPDK_CEIL_DIV(REDUCE_META_CACHE_BLOCK_NUM, mblk_in_2mb_page);
925 : 10 : total_mblk = huge_pages_needed * mblk_in_2mb_page;
926 : 10 : vol->metablocks_num = total_mblk;
927 : :
928 : : /* alloc metablock mem */
929 : 10 : vol->metablock_buf = spdk_dma_malloc(VALUE_2MB * huge_pages_needed, VALUE_2MB, NULL);
930 [ - + ]: 10 : if (vol->metablock_buf == NULL) {
931 : 0 : rc = -ENOMEM;
932 : 0 : goto err;
933 : : }
934 : :
935 : : /* alloc metablock control struct */
936 : 10 : vol->metablock_extent_mem = calloc(total_mblk, sizeof(*vol->metablock_extent_mem));
937 [ - + ]: 10 : if (vol->request_mem == NULL) {
938 : 0 : rc = -ENOMEM;
939 : 0 : goto err;
940 : : }
941 [ + + ]: 10250 : for (i = 0; i < total_mblk; i++) {
942 : 10240 : mblk_extent = &vol->metablock_extent_mem[i];
943 : 10240 : mblk_extent->iov.iov_base = vol->metablock_buf + i * REDUCE_META_BLKSZ;
944 : 10240 : mblk_extent->iov.iov_len = REDUCE_META_BLKSZ;
945 : 10240 : TAILQ_INIT(&mblk_extent->pending_to_supply_read);
946 : 10240 : TAILQ_INIT(&mblk_extent->pending_to_write);
947 [ + + ]: 10240 : TAILQ_INSERT_HEAD(&vol->metablocks_free, mblk_extent, tailq);
948 : : }
949 : :
950 : : /* alloc reduce_meta_request struct mem
951 : : * Use these struct memory to record the ctx before meta read/write operetion, wait complete callback to restore them.
952 : : */
953 : 10 : backing_io_struct_size = sizeof(struct reduce_meta_request) + vol->backing_dev->user_ctx_size;
954 : : /* every vol date request may need 2 meta req for logical_map and chunk_map */
955 : 10 : total_metareq = 2 * REDUCE_NUM_VOL_REQUESTS;
956 : 10 : vol->meta_request_buf = (struct reduce_meta_request *)calloc(total_metareq, backing_io_struct_size);
957 [ - + ]: 10 : if (vol->meta_request_buf == NULL) {
958 : 0 : rc = -ENOMEM;
959 : 0 : goto err;
960 : : }
961 [ + + ]: 5130 : for (i = 0; i < total_metareq; i++) {
962 : 5120 : meta_request = (struct reduce_meta_request *)((uint8_t *)vol->meta_request_buf + i *
963 : : backing_io_struct_size);
964 : 5120 : meta_request->backing_io.backing_cb_args = &meta_request->cb_args;
965 [ + + ]: 5120 : TAILQ_INSERT_HEAD(&vol->free_meta_request, meta_request, tailq);
966 : : }
967 : :
968 : 10 : return 0;
969 : 0 : err:
970 : 0 : spdk_free(vol->metablock_buf);
971 : 0 : free(vol->metablock_extent_mem);
972 : 0 : free(vol->meta_request_buf);
973 : 0 : vol->metablock_buf = NULL;
974 : 0 : vol->metablock_extent_mem = NULL;
975 : 0 : vol->meta_request_buf = NULL;
976 : :
977 : 0 : return rc;
978 : : }
979 : :
980 : : static void
981 : 22 : _init_write_super_cpl(void *cb_arg, int reduce_errno)
982 : : {
983 : 22 : struct reduce_init_load_ctx *init_ctx = cb_arg;
984 : 22 : int rc = 0;
985 : :
986 [ - + ]: 22 : if (reduce_errno != 0) {
987 : 0 : rc = reduce_errno;
988 : 0 : goto err;
989 : : }
990 : :
991 : 22 : rc = _allocate_vol_requests(init_ctx->vol);
992 [ - + ]: 22 : if (rc != 0) {
993 : 0 : goto err;
994 : : }
995 : :
996 [ + + ]: 22 : if (init_ctx->vol->params.meta_builtin) {
997 : 10 : rc = _allocate_vol_metablock_and_metaio(init_ctx->vol);
998 [ - + ]: 10 : if (rc != 0) {
999 : 0 : goto err;
1000 : : }
1001 : : }
1002 : :
1003 : 22 : rc = _alloc_zero_buff();
1004 [ - + ]: 22 : if (rc != 0) {
1005 : 0 : goto err;
1006 : : }
1007 : :
1008 : 22 : init_ctx->cb_fn(init_ctx->cb_arg, init_ctx->vol, rc);
1009 : : /* Only clean up the ctx - the vol has been passed to the application
1010 : : * for use now that initialization was successful.
1011 : : */
1012 : 22 : _init_load_cleanup(NULL, init_ctx);
1013 : :
1014 : 22 : return;
1015 : 0 : err:
1016 [ # # # # ]: 0 : if (unlink(init_ctx->path)) {
1017 : 0 : SPDK_ERRLOG("%s could not be unlinked: %s\n",
1018 : : (char *)init_ctx->path, spdk_strerror(errno));
1019 : : }
1020 : :
1021 : 0 : init_ctx->cb_fn(init_ctx->cb_arg, NULL, rc);
1022 : 0 : _init_load_cleanup(init_ctx->vol, init_ctx);
1023 : : }
1024 : :
1025 : : static void
1026 : 22 : _init_write_path_cpl(void *cb_arg, int reduce_errno)
1027 : : {
1028 : 22 : struct reduce_init_load_ctx *init_ctx = cb_arg;
1029 : 22 : struct spdk_reduce_vol *vol = init_ctx->vol;
1030 : 22 : struct spdk_reduce_backing_io *backing_io = init_ctx->backing_io;
1031 : :
1032 [ - + ]: 22 : if (reduce_errno != 0) {
1033 : 0 : _init_write_super_cpl(cb_arg, reduce_errno);
1034 : 0 : return;
1035 : : }
1036 : :
1037 : 22 : init_ctx->iov[0].iov_base = vol->backing_super;
1038 : 22 : init_ctx->iov[0].iov_len = sizeof(*vol->backing_super);
1039 : 22 : init_ctx->backing_cb_args.cb_fn = _init_write_super_cpl;
1040 : 22 : init_ctx->backing_cb_args.cb_arg = init_ctx;
1041 : :
1042 : 22 : backing_io->dev = vol->backing_dev;
1043 : 22 : backing_io->iov = init_ctx->iov;
1044 : 22 : backing_io->iovcnt = 1;
1045 : 22 : backing_io->lba = 0;
1046 [ - + ]: 22 : backing_io->lba_count = sizeof(*vol->backing_super) / vol->backing_dev->blocklen;
1047 : 22 : backing_io->backing_cb_args = &init_ctx->backing_cb_args;
1048 : 22 : backing_io->backing_io_type = SPDK_REDUCE_BACKING_IO_WRITE;
1049 : :
1050 : 22 : vol->backing_dev->submit_backing_io(backing_io);
1051 : : }
1052 : :
1053 : : static void
1054 : 22 : _init_builtin_meta_region_cpl(void *cb_arg, int reduce_errno)
1055 : : {
1056 : 22 : struct reduce_init_load_ctx *init_ctx = cb_arg;
1057 : 22 : struct spdk_reduce_vol *vol = init_ctx->vol;
1058 : 22 : struct spdk_reduce_backing_io *backing_io = init_ctx->backing_io;
1059 : :
1060 [ - + ]: 22 : if (reduce_errno != 0) {
1061 : 0 : _init_write_path_cpl(cb_arg, reduce_errno);
1062 : 0 : return;
1063 : : }
1064 : :
1065 : 22 : init_ctx->iov[0].iov_base = init_ctx->path;
1066 : 22 : init_ctx->iov[0].iov_len = REDUCE_PATH_MAX;
1067 : 22 : init_ctx->backing_cb_args.cb_fn = _init_write_path_cpl;
1068 : 22 : init_ctx->backing_cb_args.cb_arg = init_ctx;
1069 : : /* Write path to offset 4K on backing device - just after where the super
1070 : : * block will be written. We wait until this is committed before writing the
1071 : : * super block to guarantee we don't get the super block written without the
1072 : : * the path if the system crashed in the middle of a write operation.
1073 : : */
1074 : 22 : backing_io->dev = vol->backing_dev;
1075 : 22 : backing_io->iov = init_ctx->iov;
1076 : 22 : backing_io->iovcnt = 1;
1077 [ - + ]: 22 : backing_io->lba = REDUCE_BACKING_DEV_PATH_OFFSET / vol->backing_dev->blocklen;
1078 [ - + ]: 22 : backing_io->lba_count = REDUCE_PATH_MAX / vol->backing_dev->blocklen;
1079 : 22 : backing_io->backing_cb_args = &init_ctx->backing_cb_args;
1080 : 22 : backing_io->backing_io_type = SPDK_REDUCE_BACKING_IO_WRITE;
1081 : :
1082 : 22 : vol->backing_dev->submit_backing_io(backing_io);
1083 : : }
1084 : :
1085 : : static void
1086 : 30 : _init_builtin_meta_region(void *cb_arg, int reduce_errno)
1087 : : {
1088 : 30 : struct reduce_init_load_ctx *init_ctx = cb_arg;
1089 : 30 : struct spdk_reduce_vol *vol = init_ctx->vol;
1090 : 30 : struct reduce_mblk_init_ctx *ctx = &init_ctx->mblk_init_ctx;
1091 : 30 : struct spdk_reduce_backing_io *backing_io = init_ctx->backing_io;
1092 : : uint64_t offset, len;
1093 : :
1094 [ + - + + ]: 30 : if (reduce_errno || ctx->off_need_init_next == ctx->off_need_init + ctx->len_need_init) {
1095 : 10 : _init_builtin_meta_region_cpl(cb_arg, reduce_errno);
1096 : 10 : return;
1097 : : }
1098 : :
1099 : 20 : offset = ctx->off_need_init_next;
1100 : 20 : len = spdk_min(ctx->maxlen_per_io,
1101 : : ctx->off_need_init + ctx->len_need_init - ctx->off_need_init_next);
1102 : 20 : ctx->off_need_init_next = offset + len;
1103 : :
1104 : 20 : init_ctx->iov[0].iov_base = ctx->meta_init_buf;
1105 : 20 : init_ctx->iov[0].iov_len = len * vol->backing_dev->blocklen;
1106 : 20 : init_ctx->backing_cb_args.cb_fn = _init_builtin_meta_region;
1107 : 20 : init_ctx->backing_cb_args.cb_arg = init_ctx;
1108 : :
1109 : 20 : backing_io->dev = vol->backing_dev;
1110 : 20 : backing_io->iov = init_ctx->iov;
1111 : 20 : backing_io->iovcnt = 1;
1112 : 20 : backing_io->lba = offset;
1113 : 20 : backing_io->lba_count = len;
1114 : 20 : backing_io->backing_cb_args = &init_ctx->backing_cb_args;
1115 : 20 : backing_io->backing_io_type = SPDK_REDUCE_BACKING_IO_WRITE;
1116 : :
1117 : 20 : vol->backing_dev->submit_backing_io(backing_io);
1118 : : }
1119 : :
1120 : : static void
1121 : 10 : _init_builtin_meta_region_ctx_init(struct reduce_init_load_ctx *init_ctx)
1122 : : {
1123 : 10 : struct reduce_mblk_init_ctx *mblk_init_ctx = &init_ctx->mblk_init_ctx;
1124 : 10 : struct spdk_reduce_vol *vol = init_ctx->vol;
1125 : :
1126 : 20 : mblk_init_ctx->off_need_init = vol->params.meta_region_desc[REDUCE_MTYPE_SB].offset /
1127 : 10 : REDUCE_META_BLKSZ *
1128 : 10 : vol->backing_dev->blocklen;
1129 [ - + ]: 20 : mblk_init_ctx->len_need_init = vol->params.data_region_off / vol->backing_dev->blocklen -
1130 : 10 : mblk_init_ctx->off_need_init;
1131 : 10 : mblk_init_ctx->off_need_init_next = mblk_init_ctx->off_need_init;
1132 [ - + ]: 10 : mblk_init_ctx->maxlen_per_io = mblk_init_ctx->buf_len / vol->backing_dev->blocklen;
1133 : 10 : }
1134 : :
1135 : : static int
1136 : 22 : _allocate_bit_arrays(struct spdk_reduce_vol *vol)
1137 : : {
1138 : : uint64_t total_chunks, total_backing_io_units;
1139 : : uint32_t i, num_metadata_io_units;
1140 : :
1141 : 22 : total_chunks = _get_total_chunks(vol->params.vol_size, vol->params.chunk_size);
1142 [ + + ]: 22 : if (vol->params.meta_builtin) {
1143 [ - + ]: 10 : total_chunks += vol->params.data_region_off / vol->params.chunk_size;
1144 : : }
1145 : 22 : vol->allocated_chunk_maps = spdk_bit_array_create(total_chunks);
1146 : 22 : vol->find_chunk_offset = 0;
1147 [ - + ]: 22 : total_backing_io_units = total_chunks * (vol->params.chunk_size / vol->params.backing_io_unit_size);
1148 : 22 : vol->allocated_backing_io_units = spdk_bit_array_create(total_backing_io_units);
1149 : 22 : vol->find_block_offset = 0;
1150 : :
1151 [ + - - + ]: 22 : if (vol->allocated_chunk_maps == NULL || vol->allocated_backing_io_units == NULL) {
1152 : 0 : return -ENOMEM;
1153 : : }
1154 : :
1155 : : /* Set backing io unit bits associated with metadata. */
1156 [ + + ]: 22 : if (vol->params.meta_builtin) {
1157 [ - + ]: 10 : num_metadata_io_units = vol->params.data_region_off / vol->params.backing_io_unit_size;
1158 : : } else {
1159 : 12 : num_metadata_io_units = (sizeof(*vol->backing_super) + REDUCE_PATH_MAX) /
1160 [ - + ]: 12 : vol->params.backing_io_unit_size;
1161 : : }
1162 [ + + ]: 826 : for (i = 0; i < num_metadata_io_units; i++) {
1163 : 804 : spdk_bit_array_set(vol->allocated_backing_io_units, i);
1164 : 804 : vol->info.allocated_io_units++;
1165 : : }
1166 : :
1167 : 22 : return 0;
1168 : : }
1169 : :
1170 : : static int
1171 : 9692139 : overlap_cmp(struct spdk_reduce_vol_request *req1, struct spdk_reduce_vol_request *req2)
1172 : : {
1173 [ + + ]: 17210965 : return (req1->logical_map_index < req2->logical_map_index ? -1 : req1->logical_map_index >
1174 : 7518826 : req2->logical_map_index);
1175 : : }
1176 [ + + + + : 16565254 : RB_GENERATE_STATIC(executing_req_tree, spdk_reduce_vol_request, rbnode, overlap_cmp);
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
- + + + +
+ + + + +
+ + + ]
1177 : :
1178 : : static inline int
1179 : 44 : _check_meta_builtin(const char *path)
1180 : : {
1181 [ - + ]: 44 : return strlen(path) == strlen(REDUCE_META_BUILTIN) &&
1182 [ + + - + : 44 : memcmp(path, REDUCE_META_BUILTIN, sizeof(REDUCE_META_BUILTIN)) == 0 ? 1 : 0;
+ - ]
1183 : : }
1184 : :
1185 : : static inline uint32_t
1186 : 40 : _meta_get_elem_struct_size(struct spdk_reduce_vol_params *params, enum spdk_reduce_meta_type mtype)
1187 : : {
1188 : 40 : uint32_t size = 0;
1189 : :
1190 [ + + + + : 40 : switch (mtype) {
- ]
1191 : 10 : case REDUCE_MTYPE_SB:
1192 : 10 : size = sizeof(struct spdk_reduce_vol_superblock);
1193 : 10 : break;
1194 : 10 : case REDUCE_MTYPE_PATH:
1195 : 10 : size = REDUCE_PATH_MAX;
1196 : 10 : break;
1197 : 10 : case REDUCE_MTYPE_LOGICAL_MAP:
1198 : 10 : size = sizeof(uint64_t);
1199 : 10 : break;
1200 : 10 : case REDUCE_MTYPE_CHUNK_MAP:
1201 [ - + ]: 10 : size = _reduce_vol_get_chunk_struct_size(params->chunk_size / params->backing_io_unit_size);
1202 : 10 : break;
1203 : 0 : default:
1204 : 0 : break;
1205 : : }
1206 : :
1207 : 40 : return size;
1208 : : }
1209 : :
1210 : : static inline uint64_t
1211 : 40 : _meta_get_elem_count(struct spdk_reduce_vol_params *params, enum spdk_reduce_meta_type mtype,
1212 : : uint64_t backing_dev_size)
1213 : : {
1214 : 40 : uint64_t count = 0;
1215 : :
1216 [ + + - ]: 40 : switch (mtype) {
1217 : 20 : case REDUCE_MTYPE_SB:
1218 : : case REDUCE_MTYPE_PATH:
1219 : 20 : count = 1;
1220 : 20 : break;
1221 : 20 : case REDUCE_MTYPE_LOGICAL_MAP:
1222 : : case REDUCE_MTYPE_CHUNK_MAP:
1223 [ - + ]: 20 : count = backing_dev_size / params->chunk_size;
1224 : 20 : break;
1225 : 0 : default:
1226 : 0 : assert(0);
1227 : : }
1228 : :
1229 : 40 : return count;
1230 : : }
1231 : :
1232 : :
1233 : : static uint64_t
1234 : 10 : _meta_builtin_get_vol_size(uint64_t backing_dev_size, struct spdk_reduce_vol_params *params)
1235 : : {
1236 : : uint64_t num_chunks;
1237 : :
1238 [ - + ]: 10 : num_chunks = (backing_dev_size - params->data_region_off) / params->chunk_size;
1239 : 10 : num_chunks -= REDUCE_NUM_EXTRA_CHUNKS;
1240 : :
1241 : 10 : return num_chunks * params->chunk_size;
1242 : : }
1243 : :
1244 : : /* Calc the metaregion offset and length according backing dev size,
1245 : : * then datasize = backing dev size - metasize.
1246 : : * The data region that 'logical_map' and 'chunkmap' can represent is a little more than the true data region.
1247 : : * It is neccessary to set the bits on the allocated bitarray of allocated_backing_io_units which represent meta region.
1248 : : */
1249 : : static void
1250 : 10 : _init_builtin_meta_region_params(struct spdk_reduce_vol_params *params, uint64_t backing_dev_size)
1251 : : {
1252 : : /* tmpoff unit is REDUCE_META_BLKSZ */
1253 : 10 : uint64_t tmpoff = 0;
1254 : : uint32_t mtype;
1255 : 10 : struct spdk_reduce_meta_desc *meta_region_desc = params->meta_region_desc;
1256 : :
1257 [ + + ]: 50 : for (mtype = REDUCE_MTYPE_SB; mtype < REDUCE_MTYPE_NR; mtype++) {
1258 : : /* Although size_per_elem and elems_per_mblk could calc at runtime,
1259 : : * calc it at init time could avoid calc it every time when edit a new mblk,
1260 : : * especially at I/O path.
1261 : : */
1262 : 40 : meta_region_desc[mtype].size_per_elem = _meta_get_elem_struct_size(params, mtype);
1263 [ - + ]: 40 : meta_region_desc[mtype].elems_per_mblk = REDUCE_META_BLKSZ / meta_region_desc[mtype].size_per_elem;
1264 : 40 : meta_region_desc[mtype].offset = tmpoff;
1265 : 40 : meta_region_desc[mtype].length = spdk_divide_round_up(_meta_get_elem_count(params, mtype,
1266 : 40 : backing_dev_size), meta_region_desc[mtype].elems_per_mblk);
1267 : 40 : tmpoff += meta_region_desc[mtype].length;
1268 : : }
1269 : :
1270 : : /* data */
1271 : 10 : params->data_region_off = tmpoff * REDUCE_META_BLKSZ;
1272 : 10 : params->vol_size = _meta_builtin_get_vol_size(backing_dev_size, params);
1273 : 10 : }
1274 : :
1275 : : static int
1276 : 10 : _alloc_meta_buf(struct spdk_reduce_vol *vol, struct reduce_init_load_ctx *init_ctx)
1277 : : {
1278 : 10 : struct reduce_mblk_init_ctx *mblk_init_ctx = &init_ctx->mblk_init_ctx;
1279 : :
1280 : 10 : mblk_init_ctx->buf_len = SPDK_ALIGN_FLOOR(REDUCE_META_INIT_BUF_SIZE, REDUCE_META_BLKSZ);
1281 : 10 : mblk_init_ctx->meta_init_buf = spdk_malloc(mblk_init_ctx->buf_len,
1282 : : 64, NULL, SPDK_ENV_LCORE_ID_ANY,
1283 : : SPDK_MALLOC_DMA);
1284 [ - + ]: 10 : if (mblk_init_ctx->meta_init_buf == NULL) {
1285 : 0 : return -ENOMEM;
1286 : : }
1287 [ - + ]: 10 : memset(mblk_init_ctx->meta_init_buf, 0xff, mblk_init_ctx->buf_len);
1288 : :
1289 : 10 : return 0;
1290 : : }
1291 : :
1292 : : void
1293 : 22 : spdk_reduce_vol_init(struct spdk_reduce_vol_params *params,
1294 : : struct spdk_reduce_backing_dev *backing_dev,
1295 : : const char *pm_file_dir,
1296 : : spdk_reduce_vol_op_with_handle_complete cb_fn, void *cb_arg)
1297 : : {
1298 : : struct spdk_reduce_vol *vol;
1299 : : struct reduce_init_load_ctx *init_ctx;
1300 : : struct spdk_reduce_backing_io *backing_io;
1301 : : uint64_t backing_dev_size;
1302 : : size_t mapped_len;
1303 : 22 : int dir_len = 0, max_dir_len, rc;
1304 : :
1305 : 22 : rc = _validate_vol_params(params);
1306 [ - + ]: 22 : if (rc != 0) {
1307 : 0 : SPDK_ERRLOG("invalid vol params\n");
1308 : 0 : cb_fn(cb_arg, NULL, rc);
1309 : 0 : return;
1310 : : }
1311 : :
1312 : 22 : backing_dev_size = backing_dev->blockcnt * backing_dev->blocklen;
1313 : 22 : params->vol_size = _get_vol_size(params->chunk_size, backing_dev_size);
1314 [ - + ]: 22 : if (params->vol_size == 0) {
1315 : 0 : SPDK_ERRLOG("backing device is too small\n");
1316 : 0 : cb_fn(cb_arg, NULL, -EINVAL);
1317 : 0 : return;
1318 : : }
1319 : :
1320 [ - + ]: 22 : if (backing_dev->submit_backing_io == NULL) {
1321 : 0 : SPDK_ERRLOG("backing_dev function pointer not specified\n");
1322 : 0 : cb_fn(cb_arg, NULL, -EINVAL);
1323 : 0 : return;
1324 : : }
1325 : :
1326 [ + - ]: 22 : if (spdk_uuid_is_null(¶ms->uuid)) {
1327 : 22 : spdk_uuid_generate(¶ms->uuid);
1328 : : }
1329 : :
1330 [ + + ]: 22 : if (!_check_meta_builtin(pm_file_dir)) {
1331 : : /* Use pmem to host metadata */
1332 : 12 : params->meta_builtin = false;
1333 : : /* We need to append a path separator and the UUID to the supplied
1334 : : * path.
1335 : : */
1336 : 12 : max_dir_len = REDUCE_PATH_MAX - SPDK_UUID_STRING_LEN - 1;
1337 [ - + ]: 12 : dir_len = strnlen(pm_file_dir, max_dir_len);
1338 : : /* Strip trailing slash if the user provided one - we will add it back
1339 : : * later when appending the filename.
1340 : : */
1341 [ - + ]: 12 : if (pm_file_dir[dir_len - 1] == '/') {
1342 : 0 : dir_len--;
1343 : : }
1344 [ - + ]: 12 : if (dir_len == max_dir_len) {
1345 : 0 : SPDK_ERRLOG("pm_file_dir (%s) too long\n", pm_file_dir);
1346 : 0 : cb_fn(cb_arg, NULL, -EINVAL);
1347 : 0 : return;
1348 : : }
1349 : : } else {
1350 : : /* Use backing dev space to hold metadata */
1351 : 10 : params->meta_builtin = true;
1352 : : }
1353 : :
1354 : 22 : vol = calloc(1, sizeof(*vol));
1355 [ - + ]: 22 : if (vol == NULL) {
1356 : 0 : cb_fn(cb_arg, NULL, -ENOMEM);
1357 : 0 : return;
1358 : : }
1359 : :
1360 : 22 : TAILQ_INIT(&vol->free_requests);
1361 : 22 : RB_INIT(&vol->executing_requests);
1362 : 22 : TAILQ_INIT(&vol->queued_requests);
1363 : 22 : queue_init(&vol->free_chunks_queue);
1364 : 22 : queue_init(&vol->free_backing_blocks_queue);
1365 : 22 : TAILQ_INIT(&vol->metablocks_free);
1366 : 22 : TAILQ_INIT(&vol->metablocks_lru);
1367 : 22 : RB_INIT(&vol->metablocks_caching);
1368 : 22 : TAILQ_INIT(&vol->free_meta_request);
1369 : :
1370 : 22 : vol->backing_super = spdk_zmalloc(sizeof(*vol->backing_super), 0, NULL,
1371 : : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
1372 [ - + ]: 22 : if (vol->backing_super == NULL) {
1373 : 0 : cb_fn(cb_arg, NULL, -ENOMEM);
1374 : 0 : _init_load_cleanup(vol, NULL);
1375 : 0 : return;
1376 : : }
1377 : :
1378 : 22 : init_ctx = calloc(1, sizeof(*init_ctx));
1379 [ - + ]: 22 : if (init_ctx == NULL) {
1380 : 0 : cb_fn(cb_arg, NULL, -ENOMEM);
1381 : 0 : _init_load_cleanup(vol, NULL);
1382 : 0 : return;
1383 : : }
1384 : :
1385 : 22 : backing_io = calloc(1, sizeof(*backing_io) + backing_dev->user_ctx_size);
1386 [ - + ]: 22 : if (backing_io == NULL) {
1387 : 0 : cb_fn(cb_arg, NULL, -ENOMEM);
1388 : 0 : _init_load_cleanup(vol, init_ctx);
1389 : 0 : return;
1390 : : }
1391 : 22 : init_ctx->backing_io = backing_io;
1392 : :
1393 : 22 : init_ctx->path = spdk_zmalloc(REDUCE_PATH_MAX, 0, NULL,
1394 : : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
1395 [ - + ]: 22 : if (init_ctx->path == NULL) {
1396 : 0 : cb_fn(cb_arg, NULL, -ENOMEM);
1397 : 0 : _init_load_cleanup(vol, init_ctx);
1398 : 0 : return;
1399 : : }
1400 : :
1401 [ - + ]: 22 : vol->backing_io_units_per_chunk = params->chunk_size / params->backing_io_unit_size;
1402 [ - + ]: 22 : vol->logical_blocks_per_chunk = params->chunk_size / params->logical_block_size;
1403 [ - + ]: 22 : vol->backing_lba_per_io_unit = params->backing_io_unit_size / backing_dev->blocklen;
1404 : 22 : vol->backing_dev = backing_dev;
1405 : :
1406 : : /* meta in pmem: init metadata in pmemdir.
1407 : : meta in backing_bdev: split total region to meta region and data region.
1408 : : */
1409 [ + + ]: 22 : if (params->meta_builtin == false) {
1410 [ - + ]: 12 : assert(dir_len != 0);
1411 [ - + - + ]: 12 : memcpy(vol->pm_file.path, pm_file_dir, dir_len);
1412 : 12 : vol->pm_file.path[dir_len] = '/';
1413 : 12 : spdk_uuid_fmt_lower(&vol->pm_file.path[dir_len + 1], SPDK_UUID_STRING_LEN,
1414 : 12 : ¶ms->uuid);
1415 : 12 : vol->pm_file.size = _get_pm_file_size(params);
1416 : 12 : vol->pm_file.pm_buf = pmem_map_file(vol->pm_file.path, vol->pm_file.size,
1417 : : PMEM_FILE_CREATE | PMEM_FILE_EXCL, 0600,
1418 : : &mapped_len, &vol->pm_file.pm_is_pmem);
1419 [ - + ]: 12 : if (vol->pm_file.pm_buf == NULL) {
1420 : 0 : SPDK_ERRLOG("could not pmem_map_file(%s): %s\n",
1421 : : vol->pm_file.path, strerror(errno));
1422 : 0 : cb_fn(cb_arg, NULL, -errno);
1423 : 0 : _init_load_cleanup(vol, init_ctx);
1424 : 0 : return;
1425 : : }
1426 : :
1427 [ - + ]: 12 : if (vol->pm_file.size != mapped_len) {
1428 : 0 : SPDK_ERRLOG("could not map entire pmem file (size=%" PRIu64 " mapped=%" PRIu64 ")\n",
1429 : : vol->pm_file.size, mapped_len);
1430 : 0 : cb_fn(cb_arg, NULL, -ENOMEM);
1431 : 0 : _init_load_cleanup(vol, init_ctx);
1432 : 0 : return;
1433 : : }
1434 : :
1435 [ - + - + ]: 12 : memcpy(&vol->params, params, sizeof(*params));
1436 : 12 : _initialize_vol_pm_pointers(vol);
1437 : : } else {
1438 : : /* metadata region on backingbdev
1439 : : |0 |4096 |8192... |
1440 : : |sb |mdpath |logical_map... |chunk_map...
1441 : : the order of initailization: pmpath, normalmeta(logical_map, chunk_map), sb
1442 : : */
1443 : 10 : _init_builtin_meta_region_params(params, backing_dev_size);
1444 [ - + - + ]: 10 : memcpy(&vol->params, params, sizeof(*params));
1445 : : }
1446 : :
1447 [ - + ]: 22 : memcpy(vol->backing_super->signature, SPDK_REDUCE_SIGNATURE,
1448 : : sizeof(vol->backing_super->signature));
1449 [ - + - + ]: 22 : memcpy(&vol->backing_super->params, params, sizeof(*params));
1450 : :
1451 [ + + ]: 22 : if (params->meta_builtin == false) {
1452 [ - + - + ]: 12 : memcpy(vol->pm_super, vol->backing_super, sizeof(*vol->backing_super));
1453 : : /* Writing 0xFF's is equivalent of filling it all with SPDK_EMPTY_MAP_ENTRY.
1454 : : * Note that this writes 0xFF to not just the logical map but the chunk maps as well.
1455 : : */
1456 [ - + ]: 12 : memset(vol->pm_logical_map, 0xFF, vol->pm_file.size - sizeof(*vol->backing_super));
1457 : 12 : _reduce_persist(vol, vol->pm_file.pm_buf, vol->pm_file.size);
1458 : : }
1459 : :
1460 : 22 : rc = _allocate_bit_arrays(vol);
1461 [ - + ]: 22 : if (rc != 0) {
1462 : 0 : cb_fn(cb_arg, NULL, rc);
1463 : 0 : _init_load_cleanup(vol, init_ctx);
1464 : 0 : return;
1465 : : }
1466 : :
1467 : 22 : init_ctx->vol = vol;
1468 : 22 : init_ctx->cb_fn = cb_fn;
1469 : 22 : init_ctx->cb_arg = cb_arg;
1470 : :
1471 [ + + ]: 22 : if (params->meta_builtin) {
1472 [ - + ]: 10 : memcpy(init_ctx->path, REDUCE_META_BUILTIN, sizeof(REDUCE_META_BUILTIN));
1473 : 10 : rc = _alloc_meta_buf(vol, init_ctx);
1474 [ - + ]: 10 : if (rc != 0) {
1475 : 0 : cb_fn(cb_arg, NULL, rc);
1476 : 0 : _init_load_cleanup(vol, init_ctx);
1477 : 0 : return;
1478 : : }
1479 : 10 : _init_builtin_meta_region_ctx_init(init_ctx);
1480 : 10 : _init_builtin_meta_region(init_ctx, 0);
1481 : : } else {
1482 [ - + - + ]: 12 : memcpy(init_ctx->path, vol->pm_file.path, REDUCE_PATH_MAX);
1483 : 12 : _init_builtin_meta_region_cpl(init_ctx, 0);
1484 : : }
1485 : : }
1486 : :
1487 : : static struct reduce_meta_request *
1488 : 1184144 : _get_init_meta_req(struct spdk_reduce_vol *vol, bool is_write,
1489 : : enum spdk_reduce_meta_type mtype, uint32_t mblk_sn, uint32_t elemsn_on_mblk,
1490 : : spdk_reduce_dev_cpl cb_fn)
1491 : : {
1492 : : struct reduce_meta_request *meta_req;
1493 : 1184144 : meta_req = TAILQ_FIRST(&vol->free_meta_request);
1494 [ - + ]: 1184144 : if (meta_req == NULL) {
1495 : : /* should not happend, because we have allocated enough metateq. 2 metareq for each vol req */
1496 : 0 : assert(0);
1497 : : return meta_req;
1498 : : }
1499 [ + - ]: 1184144 : TAILQ_REMOVE(&vol->free_meta_request, meta_req, tailq);
1500 : 1184144 : meta_req->vol = vol;
1501 : 1184144 : meta_req->is_write = is_write;
1502 : 1184144 : meta_req->supply_read = is_write ? 1 : 0;
1503 : 1184144 : meta_req->mtype = mtype;
1504 : 1184144 : meta_req->mblk_sn = mblk_sn;
1505 : 1184144 : meta_req->elem_sn_on_mblk = elemsn_on_mblk;
1506 : 1184144 : meta_req->cb_args.cb_fn = cb_fn;
1507 : 1184144 : meta_req->cb_args.cb_arg = meta_req;
1508 : :
1509 : 1184144 : return meta_req;
1510 : : }
1511 : :
1512 : : static void _load_allocated_backing_io_unit(struct reduce_init_load_ctx *load_ctx, uint32_t mblksn);
1513 : :
1514 : : /* restore the vol->allocated_backing_io_units by meta blocks */
1515 : : static void
1516 : 0 : _load_allocated_backing_io_unit_cb(void *cb_arg, int reduce_errno)
1517 : : {
1518 : 0 : struct reduce_meta_request *meta_req = cb_arg;
1519 : 0 : struct reduce_init_load_ctx *load_ctx = meta_req->restore_ctx.load_ctx.load_ctx;
1520 : 0 : struct spdk_reduce_vol *vol = meta_req->vol;
1521 : 0 : struct spdk_reduce_meta_desc *mdesc = &vol->params.meta_region_desc[REDUCE_MTYPE_CHUNK_MAP];
1522 : : void *chunk_array;
1523 : 0 : struct spdk_reduce_chunk_map *chunk = NULL;
1524 : : /* chunk_off chunk_count describes the location of the chunkmap contained in this metadata block
1525 : : * among all chunkmaps in the reduce volume. Just like offset and length.
1526 : : */
1527 : : uint32_t chunk_off, chunk_count, chunk_size, chunk_idx, next_mblksn, chunkmap_idx, i, io_unit;
1528 : : uint64_t io_unit_index;
1529 : :
1530 [ # # ]: 0 : if (reduce_errno != 0) {
1531 : 0 : goto error;
1532 : : }
1533 : :
1534 : 0 : chunk_size = mdesc->size_per_elem;
1535 : 0 : chunk_off = (meta_req->mblk_sn - mdesc->offset) * mdesc->elems_per_mblk;
1536 : 0 : chunk_count = mdesc->elems_per_mblk;
1537 : :
1538 : 0 : chunk_array = _reduce_get_meta_elem_addr(meta_req);
1539 [ # # ]: 0 : for (i = 0; i < chunk_count; i++) {
1540 : 0 : chunk_idx = chunk_off + i;
1541 : : /* one chunkmap metablock contains lots of chunkmap, skip unused chunks */
1542 [ # # ]: 0 : if (0 == spdk_bit_array_get(vol->allocated_chunk_maps, chunk_idx)) {
1543 : 0 : continue;
1544 : : }
1545 : 0 : chunk = (struct spdk_reduce_chunk_map *)(chunk_array + i * chunk_size);
1546 [ # # ]: 0 : for (io_unit = 0; io_unit < vol->backing_io_units_per_chunk; io_unit++) {
1547 : 0 : io_unit_index = chunk->io_unit_index[io_unit];
1548 [ # # ]: 0 : if (io_unit_index == REDUCE_EMPTY_MAP_ENTRY) {
1549 : 0 : continue;
1550 : : }
1551 : 0 : spdk_bit_array_set(vol->allocated_backing_io_units, io_unit_index);
1552 : : }
1553 : : }
1554 : : /* we only read the chunkmap mblks which contain used chunks */
1555 [ # # ]: 0 : assert(chunk);
1556 : 0 : _reduce_metablock_cache_read_done_update(meta_req, reduce_errno);
1557 : :
1558 : 0 : chunkmap_idx = spdk_bit_array_find_first_set(vol->allocated_chunk_maps, chunk_off + chunk_count);
1559 [ # # ]: 0 : if (chunkmap_idx == UINT32_MAX) {
1560 : 0 : SPDK_NOTICELOG("vol:%p io unit bitarray load finish\n", vol);
1561 : 0 : load_ctx->cb_fn(load_ctx->cb_arg, vol, 0);
1562 : 0 : _init_load_cleanup(NULL, load_ctx);
1563 : 0 : return;
1564 : : }
1565 : :
1566 : : /* load next chunkmap mblks */
1567 : 0 : next_mblksn = _meta_get_mblksn(&vol->params, REDUCE_MTYPE_CHUNK_MAP, chunkmap_idx);
1568 : 0 : _load_allocated_backing_io_unit(load_ctx, next_mblksn);
1569 : 0 : return;
1570 : :
1571 : 0 : error:
1572 : 0 : load_ctx->cb_fn(load_ctx->cb_arg, NULL, reduce_errno);
1573 : 0 : _init_load_cleanup(vol, load_ctx);
1574 : : }
1575 : :
1576 : : static void
1577 : 0 : _load_allocated_backing_io_unit(struct reduce_init_load_ctx *load_ctx, uint32_t mblksn)
1578 : : {
1579 : 0 : struct spdk_reduce_vol *vol = load_ctx->vol;
1580 : : struct reduce_meta_request *meta_req;
1581 : : uint32_t next_mblksn, chunkmap_idx;
1582 : :
1583 [ # # ]: 0 : if (mblksn == UINT32_MAX) {
1584 : 0 : chunkmap_idx = spdk_bit_array_find_first_set(vol->allocated_chunk_maps, 0);
1585 [ # # ]: 0 : if (chunkmap_idx == UINT32_MAX) {
1586 : : /* means the compress volume is empty */
1587 : 0 : SPDK_NOTICELOG("vol:%p backing io units don't need load\n", vol);
1588 : 0 : load_ctx->cb_fn(load_ctx->cb_arg, vol, 0);
1589 : 0 : _init_load_cleanup(NULL, load_ctx);
1590 : 0 : return;
1591 : : }
1592 : 0 : next_mblksn = _meta_get_mblksn(&vol->params, REDUCE_MTYPE_CHUNK_MAP, chunkmap_idx);
1593 : : } else {
1594 : 0 : next_mblksn = mblksn;
1595 : : }
1596 : :
1597 : 0 : meta_req = _get_init_meta_req(vol, false, REDUCE_MTYPE_CHUNK_MAP,
1598 : : next_mblksn, 0,
1599 : : _load_allocated_backing_io_unit_cb);
1600 : 0 : meta_req->restore_ctx.load_ctx.load_ctx = load_ctx;
1601 : 0 : _reduce_metablock_cache_read(meta_req);
1602 : : }
1603 : :
1604 : : static void _load_allocated_chunk_map_by_md(struct reduce_init_load_ctx *load_ctx, uint32_t mblksn);
1605 : :
1606 : : /* restore the vol->allocated_chunk_maps by meta blocks */
1607 : : static void
1608 : 0 : _load_allocated_chunk_map_cb(void *cb_arg, int reduce_errno)
1609 : : {
1610 : 0 : struct reduce_meta_request *meta_req = cb_arg;
1611 : 0 : struct reduce_init_load_ctx *load_ctx = meta_req->restore_ctx.load_ctx.load_ctx;
1612 : 0 : struct spdk_reduce_vol *vol = meta_req->vol;
1613 : 0 : struct spdk_reduce_meta_desc *mdesc = &vol->params.meta_region_desc[REDUCE_MTYPE_LOGICAL_MAP];
1614 : : uint64_t *logical_map_array, logical_map;
1615 : : uint32_t next_mblksn;
1616 : :
1617 [ # # ]: 0 : if (reduce_errno != 0) {
1618 : 0 : goto error;
1619 : : }
1620 : :
1621 : 0 : logical_map_array = _reduce_get_meta_elem_addr(meta_req);
1622 [ # # ]: 0 : for (int i = 0; i < mdesc->elems_per_mblk; i++) {
1623 : 0 : logical_map = logical_map_array[i];
1624 [ # # ]: 0 : if (logical_map == REDUCE_EMPTY_MAP_ENTRY) {
1625 : 0 : continue;
1626 : : }
1627 : 0 : spdk_bit_array_set(vol->allocated_chunk_maps, logical_map);
1628 : : }
1629 : :
1630 : 0 : next_mblksn = meta_req->mblk_sn + 1;
1631 : 0 : _reduce_metablock_cache_read_done_update(meta_req, reduce_errno);
1632 : :
1633 [ # # ]: 0 : if (next_mblksn == mdesc->offset + mdesc->length) {
1634 : 0 : SPDK_NOTICELOG("vol:%p logical map bitarray load finish\n", vol);
1635 : 0 : _load_allocated_backing_io_unit(load_ctx, UINT32_MAX);
1636 : 0 : return;
1637 : : }
1638 : :
1639 : 0 : _load_allocated_chunk_map_by_md(load_ctx, next_mblksn);
1640 : 0 : return;
1641 : :
1642 : 0 : error:
1643 : 0 : load_ctx->cb_fn(load_ctx->cb_arg, NULL, reduce_errno);
1644 : 0 : _init_load_cleanup(vol, load_ctx);
1645 : : }
1646 : :
1647 : : static void
1648 : 0 : _load_allocated_chunk_map_by_md(struct reduce_init_load_ctx *load_ctx, uint32_t mblksn)
1649 : : {
1650 : 0 : struct spdk_reduce_vol *vol = load_ctx->vol;
1651 : 0 : struct spdk_reduce_meta_desc *md_desc = &vol->params.meta_region_desc[REDUCE_MTYPE_LOGICAL_MAP];
1652 : : struct reduce_meta_request *meta_req;
1653 [ # # ]: 0 : uint32_t next_mblksn = mblksn == UINT32_MAX ? md_desc->offset : mblksn;
1654 : :
1655 : 0 : meta_req = _get_init_meta_req(vol, false, REDUCE_MTYPE_LOGICAL_MAP,
1656 : : next_mblksn, 0,
1657 : : _load_allocated_chunk_map_cb);
1658 : 0 : meta_req->restore_ctx.load_ctx.load_ctx = load_ctx;
1659 : 0 : _reduce_metablock_cache_read(meta_req);
1660 : 0 : }
1661 : :
1662 : : static void destroy_load_cb(void *cb_arg, struct spdk_reduce_vol *vol, int reduce_errno);
1663 : :
1664 : : static void
1665 : 450 : _load_read_super_and_path_cpl(void *cb_arg, int reduce_errno)
1666 : : {
1667 : 450 : struct reduce_init_load_ctx *load_ctx = cb_arg;
1668 : 450 : struct spdk_reduce_vol *vol = load_ctx->vol;
1669 : : uint64_t backing_dev_size;
1670 : : uint64_t i, num_chunks, logical_map_index;
1671 : : struct spdk_reduce_chunk_map *chunk;
1672 : : size_t mapped_len;
1673 : : uint32_t j;
1674 : : int rc;
1675 : :
1676 [ - + ]: 450 : if (reduce_errno != 0) {
1677 : 0 : rc = reduce_errno;
1678 : 0 : goto error;
1679 : : }
1680 : :
1681 : 450 : rc = _alloc_zero_buff();
1682 [ - + ]: 450 : if (rc) {
1683 : 0 : goto error;
1684 : : }
1685 : :
1686 [ - + + + ]: 450 : if (memcmp(vol->backing_super->signature,
1687 : : SPDK_REDUCE_SIGNATURE,
1688 : : sizeof(vol->backing_super->signature)) != 0) {
1689 : : /* This backing device isn't a libreduce backing device. */
1690 : 428 : rc = -EILSEQ;
1691 : 428 : goto error;
1692 : : }
1693 : :
1694 : : /* If the cb_fn is destroy_load_cb, it means we are wanting to destroy this compress bdev.
1695 : : * So don't bother getting the volume ready to use - invoke the callback immediately
1696 : : * so destroy_load_cb can delete the metadata off of the block device and delete the
1697 : : * persistent memory file if it exists.
1698 : : */
1699 [ - + - + ]: 22 : memcpy(vol->pm_file.path, load_ctx->path, sizeof(vol->pm_file.path));
1700 [ + - ]: 22 : if (load_ctx->cb_fn == (*destroy_load_cb)) {
1701 : 22 : load_ctx->cb_fn(load_ctx->cb_arg, vol, 0);
1702 : 22 : _init_load_cleanup(NULL, load_ctx);
1703 : 22 : return;
1704 : : }
1705 : :
1706 [ # # ]: 0 : assert(vol->backing_super->params.meta_builtin == _check_meta_builtin(vol->pm_file.path));
1707 : :
1708 [ # # # # ]: 0 : memcpy(&vol->params, &vol->backing_super->params, sizeof(vol->params));
1709 [ # # ]: 0 : vol->backing_io_units_per_chunk = vol->params.chunk_size / vol->params.backing_io_unit_size;
1710 [ # # ]: 0 : vol->logical_blocks_per_chunk = vol->params.chunk_size / vol->params.logical_block_size;
1711 [ # # ]: 0 : vol->backing_lba_per_io_unit = vol->params.backing_io_unit_size / vol->backing_dev->blocklen;
1712 : :
1713 : 0 : rc = _allocate_bit_arrays(vol);
1714 [ # # ]: 0 : if (rc != 0) {
1715 : 0 : goto error;
1716 : : }
1717 : :
1718 : 0 : backing_dev_size = vol->backing_dev->blockcnt * vol->backing_dev->blocklen;
1719 [ # # ]: 0 : if (_get_vol_size(vol->params.chunk_size, backing_dev_size) < vol->params.vol_size) {
1720 : 0 : SPDK_ERRLOG("backing device size %" PRIi64 " smaller than expected\n",
1721 : : backing_dev_size);
1722 : 0 : rc = -EILSEQ;
1723 : 0 : goto error;
1724 : : }
1725 : :
1726 : 0 : rc = _allocate_vol_requests(vol);
1727 [ # # ]: 0 : if (rc != 0) {
1728 : 0 : goto error;
1729 : : }
1730 : :
1731 [ # # ]: 0 : if (vol->params.meta_builtin) {
1732 : 0 : rc = _allocate_vol_metablock_and_metaio(load_ctx->vol);
1733 [ # # ]: 0 : if (rc != 0) {
1734 : 0 : goto error;
1735 : : }
1736 : 0 : _load_allocated_chunk_map_by_md(load_ctx, UINT32_MAX);
1737 : 0 : return;
1738 : : }
1739 : :
1740 : 0 : vol->pm_file.size = _get_pm_file_size(&vol->params);
1741 : 0 : vol->pm_file.pm_buf = pmem_map_file(vol->pm_file.path, 0, 0, 0, &mapped_len,
1742 : : &vol->pm_file.pm_is_pmem);
1743 [ # # ]: 0 : if (vol->pm_file.pm_buf == NULL) {
1744 : 0 : SPDK_ERRLOG("could not pmem_map_file(%s): %s\n", vol->pm_file.path, strerror(errno));
1745 : 0 : rc = -errno;
1746 : 0 : goto error;
1747 : : }
1748 : :
1749 [ # # ]: 0 : if (vol->pm_file.size != mapped_len) {
1750 : 0 : SPDK_ERRLOG("could not map entire pmem file (size=%" PRIu64 " mapped=%" PRIu64 ")\n",
1751 : : vol->pm_file.size, mapped_len);
1752 : 0 : rc = -ENOMEM;
1753 : 0 : goto error;
1754 : : }
1755 : :
1756 : 0 : _initialize_vol_pm_pointers(vol);
1757 : :
1758 [ # # ]: 0 : num_chunks = vol->params.vol_size / vol->params.chunk_size;
1759 [ # # ]: 0 : for (i = 0; i < num_chunks; i++) {
1760 : 0 : logical_map_index = vol->pm_logical_map[i];
1761 [ # # ]: 0 : if (logical_map_index == REDUCE_EMPTY_MAP_ENTRY) {
1762 : 0 : continue;
1763 : : }
1764 : 0 : spdk_bit_array_set(vol->allocated_chunk_maps, logical_map_index);
1765 : 0 : chunk = _reduce_vol_get_chunk_map(vol, logical_map_index);
1766 [ # # ]: 0 : for (j = 0; j < vol->backing_io_units_per_chunk; j++) {
1767 [ # # ]: 0 : if (chunk->io_unit_index[j] != REDUCE_EMPTY_MAP_ENTRY) {
1768 : 0 : spdk_bit_array_set(vol->allocated_backing_io_units, chunk->io_unit_index[j]);
1769 : 0 : vol->info.allocated_io_units++;
1770 : : }
1771 : : }
1772 : : }
1773 : :
1774 : 0 : load_ctx->cb_fn(load_ctx->cb_arg, vol, 0);
1775 : : /* Only clean up the ctx - the vol has been passed to the application
1776 : : * for use now that volume load was successful.
1777 : : */
1778 : 0 : _init_load_cleanup(NULL, load_ctx);
1779 : 0 : return;
1780 : :
1781 : 428 : error:
1782 : 428 : load_ctx->cb_fn(load_ctx->cb_arg, NULL, rc);
1783 : 428 : _init_load_cleanup(vol, load_ctx);
1784 : : }
1785 : :
1786 : : void
1787 : 450 : spdk_reduce_vol_load(struct spdk_reduce_backing_dev *backing_dev,
1788 : : spdk_reduce_vol_op_with_handle_complete cb_fn, void *cb_arg)
1789 : : {
1790 : : struct spdk_reduce_vol *vol;
1791 : : struct reduce_init_load_ctx *load_ctx;
1792 : : struct spdk_reduce_backing_io *backing_io;
1793 : :
1794 [ - + ]: 450 : if (backing_dev->submit_backing_io == NULL) {
1795 : 0 : SPDK_ERRLOG("backing_dev function pointer not specified\n");
1796 : 0 : cb_fn(cb_arg, NULL, -EINVAL);
1797 : 0 : return;
1798 : : }
1799 : :
1800 : 450 : vol = calloc(1, sizeof(*vol));
1801 [ - + ]: 450 : if (vol == NULL) {
1802 : 0 : cb_fn(cb_arg, NULL, -ENOMEM);
1803 : 0 : return;
1804 : : }
1805 : :
1806 : 450 : TAILQ_INIT(&vol->free_requests);
1807 : 450 : RB_INIT(&vol->executing_requests);
1808 : 450 : TAILQ_INIT(&vol->queued_requests);
1809 : 450 : queue_init(&vol->free_chunks_queue);
1810 : 450 : queue_init(&vol->free_backing_blocks_queue);
1811 : 450 : TAILQ_INIT(&vol->metablocks_free);
1812 : 450 : TAILQ_INIT(&vol->metablocks_lru);
1813 : 450 : RB_INIT(&vol->metablocks_caching);
1814 : 450 : TAILQ_INIT(&vol->free_meta_request);
1815 : :
1816 : 450 : vol->backing_super = spdk_zmalloc(sizeof(*vol->backing_super), 64, NULL,
1817 : : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
1818 [ - + ]: 450 : if (vol->backing_super == NULL) {
1819 : 0 : _init_load_cleanup(vol, NULL);
1820 : 0 : cb_fn(cb_arg, NULL, -ENOMEM);
1821 : 0 : return;
1822 : : }
1823 : :
1824 : 450 : vol->backing_dev = backing_dev;
1825 : :
1826 : 450 : load_ctx = calloc(1, sizeof(*load_ctx));
1827 [ - + ]: 450 : if (load_ctx == NULL) {
1828 : 0 : _init_load_cleanup(vol, NULL);
1829 : 0 : cb_fn(cb_arg, NULL, -ENOMEM);
1830 : 0 : return;
1831 : : }
1832 : :
1833 : 450 : backing_io = calloc(1, sizeof(*backing_io) + backing_dev->user_ctx_size);
1834 [ - + ]: 450 : if (backing_io == NULL) {
1835 : 0 : _init_load_cleanup(vol, load_ctx);
1836 : 0 : cb_fn(cb_arg, NULL, -ENOMEM);
1837 : 0 : return;
1838 : : }
1839 : :
1840 : 450 : load_ctx->backing_io = backing_io;
1841 : :
1842 : 450 : load_ctx->path = spdk_zmalloc(REDUCE_PATH_MAX, 64, NULL,
1843 : : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
1844 [ - + ]: 450 : if (load_ctx->path == NULL) {
1845 : 0 : _init_load_cleanup(vol, load_ctx);
1846 : 0 : cb_fn(cb_arg, NULL, -ENOMEM);
1847 : 0 : return;
1848 : : }
1849 : :
1850 : 450 : load_ctx->vol = vol;
1851 : 450 : load_ctx->cb_fn = cb_fn;
1852 : 450 : load_ctx->cb_arg = cb_arg;
1853 : :
1854 : 450 : load_ctx->iov[0].iov_base = vol->backing_super;
1855 : 450 : load_ctx->iov[0].iov_len = sizeof(*vol->backing_super);
1856 : 450 : load_ctx->iov[1].iov_base = load_ctx->path;
1857 : 450 : load_ctx->iov[1].iov_len = REDUCE_PATH_MAX;
1858 : 450 : backing_io->dev = vol->backing_dev;
1859 : 450 : backing_io->iov = load_ctx->iov;
1860 : 450 : backing_io->iovcnt = LOAD_IOV_COUNT;
1861 : 450 : backing_io->lba = 0;
1862 : 450 : backing_io->lba_count = (sizeof(*vol->backing_super) + REDUCE_PATH_MAX) /
1863 [ - + ]: 450 : vol->backing_dev->blocklen;
1864 : 450 : backing_io->backing_cb_args = &load_ctx->backing_cb_args;
1865 : 450 : backing_io->backing_io_type = SPDK_REDUCE_BACKING_IO_READ;
1866 : :
1867 : 450 : load_ctx->backing_cb_args.cb_fn = _load_read_super_and_path_cpl;
1868 : 450 : load_ctx->backing_cb_args.cb_arg = load_ctx;
1869 : 450 : vol->backing_dev->submit_backing_io(backing_io);
1870 : : }
1871 : :
1872 : : void
1873 : 44 : spdk_reduce_vol_unload(struct spdk_reduce_vol *vol,
1874 : : spdk_reduce_vol_op_complete cb_fn, void *cb_arg)
1875 : : {
1876 [ - + ]: 44 : if (vol == NULL) {
1877 : : /* This indicates a programming error. */
1878 : 0 : assert(false);
1879 : : cb_fn(cb_arg, -EINVAL);
1880 : : return;
1881 : : }
1882 : :
1883 [ - + ]: 44 : if (--g_vol_count == 0) {
1884 : 0 : spdk_free(g_zero_buf);
1885 : : }
1886 [ - + ]: 44 : assert(g_vol_count >= 0);
1887 : 44 : _init_load_cleanup(vol, NULL);
1888 : 44 : cb_fn(cb_arg, 0);
1889 : : }
1890 : :
1891 : : struct reduce_destroy_ctx {
1892 : : spdk_reduce_vol_op_complete cb_fn;
1893 : : void *cb_arg;
1894 : : struct spdk_reduce_vol *vol;
1895 : : struct spdk_reduce_vol_superblock *super;
1896 : : struct iovec iov;
1897 : : struct spdk_reduce_vol_cb_args backing_cb_args;
1898 : : int reduce_errno;
1899 : : char pm_path[REDUCE_PATH_MAX];
1900 : : struct spdk_reduce_backing_io *backing_io;
1901 : : };
1902 : :
1903 : : static void
1904 : 22 : destroy_unload_cpl(void *cb_arg, int reduce_errno)
1905 : : {
1906 : 22 : struct reduce_destroy_ctx *destroy_ctx = cb_arg;
1907 : :
1908 [ + - ]: 22 : if (destroy_ctx->reduce_errno == 0) {
1909 [ + + - + : 22 : if (!_check_meta_builtin(destroy_ctx->pm_path) && unlink(destroy_ctx->pm_path)) {
- + ]
1910 : 0 : SPDK_ERRLOG("%s could not be unlinked: %s\n",
1911 : : destroy_ctx->pm_path, strerror(errno));
1912 : : }
1913 : : }
1914 : :
1915 : : /* Even if the unload somehow failed, we still pass the destroy_ctx
1916 : : * reduce_errno since that indicates whether or not the volume was
1917 : : * actually destroyed.
1918 : : */
1919 : 22 : destroy_ctx->cb_fn(destroy_ctx->cb_arg, destroy_ctx->reduce_errno);
1920 : 22 : spdk_free(destroy_ctx->super);
1921 : 22 : free(destroy_ctx->backing_io);
1922 : 22 : free(destroy_ctx);
1923 : 22 : }
1924 : :
1925 : : static void
1926 : 22 : _destroy_zero_super_cpl(void *cb_arg, int reduce_errno)
1927 : : {
1928 : 22 : struct reduce_destroy_ctx *destroy_ctx = cb_arg;
1929 : 22 : struct spdk_reduce_vol *vol = destroy_ctx->vol;
1930 : :
1931 : 22 : destroy_ctx->reduce_errno = reduce_errno;
1932 : 22 : spdk_reduce_vol_unload(vol, destroy_unload_cpl, destroy_ctx);
1933 : 22 : }
1934 : :
1935 : : static void
1936 : 22 : destroy_load_cb(void *cb_arg, struct spdk_reduce_vol *vol, int reduce_errno)
1937 : : {
1938 : 22 : struct reduce_destroy_ctx *destroy_ctx = cb_arg;
1939 : 22 : struct spdk_reduce_backing_io *backing_io = destroy_ctx->backing_io;
1940 : :
1941 [ - + ]: 22 : if (reduce_errno != 0) {
1942 : 0 : destroy_ctx->cb_fn(destroy_ctx->cb_arg, reduce_errno);
1943 : 0 : spdk_free(destroy_ctx->super);
1944 : 0 : free(destroy_ctx);
1945 : 0 : return;
1946 : : }
1947 : :
1948 : 22 : destroy_ctx->vol = vol;
1949 [ - + - + ]: 22 : memcpy(destroy_ctx->pm_path, vol->pm_file.path, sizeof(destroy_ctx->pm_path));
1950 : 22 : destroy_ctx->iov.iov_base = destroy_ctx->super;
1951 : 22 : destroy_ctx->iov.iov_len = sizeof(*destroy_ctx->super);
1952 : 22 : destroy_ctx->backing_cb_args.cb_fn = _destroy_zero_super_cpl;
1953 : 22 : destroy_ctx->backing_cb_args.cb_arg = destroy_ctx;
1954 : :
1955 : 22 : backing_io->dev = vol->backing_dev;
1956 : 22 : backing_io->iov = &destroy_ctx->iov;
1957 : 22 : backing_io->iovcnt = 1;
1958 : 22 : backing_io->lba = 0;
1959 [ - + ]: 22 : backing_io->lba_count = sizeof(*destroy_ctx->super) / vol->backing_dev->blocklen;
1960 : 22 : backing_io->backing_cb_args = &destroy_ctx->backing_cb_args;
1961 : 22 : backing_io->backing_io_type = SPDK_REDUCE_BACKING_IO_WRITE;
1962 : :
1963 : 22 : vol->backing_dev->submit_backing_io(backing_io);
1964 : : }
1965 : :
1966 : : void
1967 : 22 : spdk_reduce_vol_destroy(struct spdk_reduce_backing_dev *backing_dev,
1968 : : spdk_reduce_vol_op_complete cb_fn, void *cb_arg)
1969 : : {
1970 : : struct reduce_destroy_ctx *destroy_ctx;
1971 : : struct spdk_reduce_backing_io *backing_io;
1972 : :
1973 : 22 : destroy_ctx = calloc(1, sizeof(*destroy_ctx));
1974 [ - + ]: 22 : if (destroy_ctx == NULL) {
1975 : 0 : cb_fn(cb_arg, -ENOMEM);
1976 : 0 : return;
1977 : : }
1978 : :
1979 : 22 : backing_io = calloc(1, sizeof(*backing_io) + backing_dev->user_ctx_size);
1980 [ - + ]: 22 : if (backing_io == NULL) {
1981 : 0 : free(destroy_ctx);
1982 : 0 : cb_fn(cb_arg, -ENOMEM);
1983 : 0 : return;
1984 : : }
1985 : :
1986 : 22 : destroy_ctx->backing_io = backing_io;
1987 : :
1988 : 22 : destroy_ctx->super = spdk_zmalloc(sizeof(*destroy_ctx->super), 64, NULL,
1989 : : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
1990 [ - + ]: 22 : if (destroy_ctx->super == NULL) {
1991 : 0 : free(destroy_ctx);
1992 : 0 : free(backing_io);
1993 : 0 : cb_fn(cb_arg, -ENOMEM);
1994 : 0 : return;
1995 : : }
1996 : 22 : destroy_ctx->cb_fn = cb_fn;
1997 : 22 : destroy_ctx->cb_arg = cb_arg;
1998 : 22 : spdk_reduce_vol_load(backing_dev, destroy_load_cb, destroy_ctx);
1999 : : }
2000 : :
2001 : : static bool
2002 : 1237561 : _request_spans_chunk_boundary(struct spdk_reduce_vol *vol, uint64_t offset, uint64_t length)
2003 : : {
2004 : : uint64_t start_chunk, end_chunk;
2005 : :
2006 [ - + ]: 1237561 : start_chunk = offset / vol->logical_blocks_per_chunk;
2007 [ - + ]: 1237561 : end_chunk = (offset + length - 1) / vol->logical_blocks_per_chunk;
2008 : :
2009 : 1237561 : return (start_chunk != end_chunk);
2010 : : }
2011 : :
2012 : : static void _start_unmap_request_full_chunk(void *ctx);
2013 : :
2014 : : static void
2015 : 896103 : _reduce_vol_complete_req(struct spdk_reduce_vol_request *req, int reduce_errno)
2016 : : {
2017 : : struct spdk_reduce_vol_request *next_req;
2018 : 896103 : struct spdk_reduce_vol *vol = req->vol;
2019 : :
2020 : 896103 : req->cb_fn(req->cb_arg, reduce_errno);
2021 : 896103 : RB_REMOVE(executing_req_tree, &vol->executing_requests, req);
2022 : :
2023 [ + + ]: 11506602 : TAILQ_FOREACH(next_req, &vol->queued_requests, tailq) {
2024 [ + + ]: 11099456 : if (next_req->logical_map_index == req->logical_map_index) {
2025 [ + + ]: 488957 : TAILQ_REMOVE(&vol->queued_requests, next_req, tailq);
2026 [ + + ]: 488957 : if (next_req->type == REDUCE_IO_READV) {
2027 : 202238 : _start_readv_request(next_req);
2028 [ + - ]: 286719 : } else if (next_req->type == REDUCE_IO_WRITEV) {
2029 : 286719 : _start_writev_request(next_req);
2030 : : } else {
2031 [ # # ]: 0 : assert(next_req->type == REDUCE_IO_UNMAP);
2032 : 0 : _start_unmap_request_full_chunk(next_req);
2033 : : }
2034 : 488957 : break;
2035 : : }
2036 : : }
2037 : :
2038 [ + - ]: 896103 : TAILQ_INSERT_HEAD(&vol->free_requests, req, tailq);
2039 : 896103 : }
2040 : :
2041 : : static void
2042 : 419365 : _reduce_vol_reset_chunk(struct spdk_reduce_vol *vol, uint64_t chunk_map_index,
2043 : : struct spdk_reduce_chunk_map *chunkaddr)
2044 : : {
2045 : : struct spdk_reduce_chunk_map *chunk;
2046 : : uint64_t index;
2047 : : bool success;
2048 : : uint32_t i;
2049 : :
2050 [ - + ]: 419365 : assert(chunk_map_index != REDUCE_EMPTY_MAP_ENTRY);
2051 [ + + ]: 419365 : if (chunkaddr == NULL) {
2052 : 266737 : chunk = _reduce_vol_get_chunk_map(vol, chunk_map_index);
2053 : : } else {
2054 : 152628 : chunk = chunkaddr;
2055 : : }
2056 [ - + - + ]: 419365 : SPDK_INFOLOG(reduce, "release %lu. %lu %lu %lu %lu\n",
2057 : : chunk_map_index, chunk->io_unit_index[0], chunk->io_unit_index[1], chunk->io_unit_index[2],
2058 : : chunk->io_unit_index[3]);
2059 [ + - ]: 838730 : for (i = 0; i < vol->backing_io_units_per_chunk; i++) {
2060 : 838730 : index = chunk->io_unit_index[i];
2061 [ + + ]: 838730 : if (index == REDUCE_EMPTY_MAP_ENTRY) {
2062 : 419365 : break;
2063 : : }
2064 [ - + ]: 419365 : assert(spdk_bit_array_get(vol->allocated_backing_io_units,
2065 : : index) == true);
2066 : 419365 : spdk_bit_array_clear(vol->allocated_backing_io_units, index);
2067 : 419365 : vol->info.allocated_io_units--;
2068 : 419365 : success = queue_enqueue(&vol->free_backing_blocks_queue, index);
2069 [ + + + + ]: 419365 : if (!success && index < vol->find_block_offset) {
2070 : 22 : vol->find_block_offset = index;
2071 : : }
2072 : 419365 : chunk->io_unit_index[i] = REDUCE_EMPTY_MAP_ENTRY;
2073 : : }
2074 : 419365 : success = queue_enqueue(&vol->free_chunks_queue, chunk_map_index);
2075 [ + + + + ]: 419365 : if (!success && chunk_map_index < vol->find_chunk_offset) {
2076 : 22 : vol->find_chunk_offset = chunk_map_index;
2077 : : }
2078 : 419365 : spdk_bit_array_clear(vol->allocated_chunk_maps, chunk_map_index);
2079 : 419365 : }
2080 : :
2081 : : static inline void
2082 : 504388 : convert_supply_read_to_write(struct reduce_meta_request *meta_req)
2083 : : {
2084 [ - + + - : 504388 : assert(meta_req->is_write && meta_req->supply_read);
- + + - ]
2085 : 504388 : meta_req->supply_read = false;
2086 : 504388 : }
2087 : :
2088 : : static void
2089 : 0 : _write_done_mapinfo_update_error_process(struct reduce_meta_request *meta_req, int error)
2090 : : {
2091 : 0 : struct spdk_reduce_vol_request *req = meta_req->restore_ctx.req_ctx.req;
2092 : : char uuid_str[SPDK_UUID_STRING_LEN];
2093 : :
2094 : : /* update chunk map or logical fail, we need to release the bit from
2095 : : * allocated_chunk_maps and allocated_backing_io_units where we want to write.
2096 : : */
2097 : 0 : _reduce_vol_reset_chunk(req->vol, req->chunk_map_index, req->chunk);
2098 : 0 : spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), spdk_reduce_vol_get_uuid(req->vol));
2099 : 0 : SPDK_ERRLOG("%s write update new mapping error %d, release chunkmap idx: %lu, backing io unit: %lu %lu %lu %lu\n",
2100 : : uuid_str, error, req->chunk_map_index,
2101 : : req->chunk->io_unit_index[0], req->chunk->io_unit_index[1], req->chunk->io_unit_index[2],
2102 : : req->chunk->io_unit_index[3]);
2103 : 0 : req->reduce_errno = error;
2104 : 0 : _reduce_vol_complete_req(req, req->reduce_errno);
2105 [ # # # # : 0 : if (meta_req->supply_read || !meta_req->is_write) {
# # # # ]
2106 : 0 : _reduce_metablock_cache_read_done_update(meta_req, error);
2107 : : } else {
2108 : 0 : _reduce_metablock_cache_write_done_update(meta_req, error);
2109 : : }
2110 : 0 : }
2111 : :
2112 : : static void
2113 : 351760 : _write_write_done_update_logical_map(void *_meta_req, int error)
2114 : : {
2115 : 351760 : struct reduce_meta_request *meta_req = _meta_req;
2116 : 351760 : struct spdk_reduce_vol_request *req = meta_req->restore_ctx.req_ctx.req;
2117 : : uint64_t *logical_map;
2118 : :
2119 [ - + ]: 351760 : if (error != 0) {
2120 : 0 : _write_done_mapinfo_update_error_process(meta_req, error);
2121 : 0 : return;
2122 : : }
2123 : :
2124 [ - + + + ]: 351760 : if (meta_req->supply_read) {
2125 : 175880 : logical_map = (uint64_t *)_reduce_get_meta_elem_addr(meta_req);
2126 : 175880 : *logical_map = req->chunk_map_index;
2127 : 175880 : convert_supply_read_to_write(meta_req);
2128 : 175880 : _reduce_metablock_cache_read_done_update(meta_req, error);
2129 : : } else {
2130 : 175880 : _reduce_metablock_cache_write_done_update(meta_req, error);
2131 : :
2132 : 175880 : _reduce_vol_complete_req(req, error);
2133 : : }
2134 : : }
2135 : :
2136 : : static void
2137 : 351760 : _write_write_done_update_chunk_map(void *_meta_req, int error)
2138 : : {
2139 : 351760 : struct reduce_meta_request *meta_req = _meta_req;
2140 : 351760 : struct spdk_reduce_vol_request *req = meta_req->restore_ctx.req_ctx.req;
2141 : 351760 : struct spdk_reduce_vol *vol = req->vol;
2142 : : struct spdk_reduce_chunk_map *chunk;
2143 : :
2144 [ - + ]: 351760 : if (error != 0) {
2145 : 0 : _write_done_mapinfo_update_error_process(meta_req, error);
2146 : 0 : return;
2147 : : }
2148 : :
2149 [ - + + + ]: 351760 : if (meta_req->supply_read) {
2150 : 175880 : chunk = (struct spdk_reduce_chunk_map *)_reduce_get_meta_elem_addr(meta_req);
2151 [ - + - + ]: 175880 : memcpy(chunk, req->chunk, _reduce_vol_get_chunk_struct_size(vol->backing_io_units_per_chunk));
2152 : 175880 : convert_supply_read_to_write(meta_req);
2153 : 175880 : _reduce_metablock_cache_read_done_update(meta_req, error);
2154 : : } else {
2155 : 175880 : _reduce_metablock_cache_write_done_update(meta_req, error);
2156 : 175880 : meta_req = _get_init_meta_req(req->vol, true, REDUCE_MTYPE_LOGICAL_MAP,
2157 : : _meta_get_mblksn(&vol->params, REDUCE_MTYPE_LOGICAL_MAP, req->logical_map_index),
2158 : : _meta_get_elem_sn_on_mblk(&vol->params, REDUCE_MTYPE_LOGICAL_MAP, req->logical_map_index),
2159 : : _write_write_done_update_logical_map);
2160 : 175880 : meta_req->restore_ctx.req_ctx.req = req;
2161 : 175880 : _reduce_metablock_cache_read(meta_req);
2162 : : }
2163 : : }
2164 : :
2165 : : static void
2166 : 305256 : _write_write_done_clear_old_chunkmap(void *_meta_req, int error)
2167 : : {
2168 : 305256 : struct reduce_meta_request *meta_req = _meta_req;
2169 : 305256 : struct spdk_reduce_vol_request *req = meta_req->restore_ctx.req_ctx.req;
2170 : 305256 : struct spdk_reduce_vol *vol = req->vol;
2171 : : struct spdk_reduce_chunk_map *chunk_addr;
2172 : :
2173 [ - + ]: 305256 : if (error != 0) {
2174 : : /* If clear the old chunkmap meta fail, we don't consider it will success when update new chunk map.
2175 : : * There is no necessary to continue the rest of process.
2176 : : * We also don't release old chunkmap and old logical map to protect the data when we read chunkmap failed.
2177 : : * So just return error.
2178 : : */
2179 : 0 : _write_done_mapinfo_update_error_process(meta_req, error);
2180 : 0 : return;
2181 : : }
2182 : :
2183 : : /* read the meta elem */
2184 [ - + + + ]: 305256 : if (meta_req->supply_read) {
2185 : 152628 : chunk_addr = (struct spdk_reduce_chunk_map *)_reduce_get_meta_elem_addr(meta_req);
2186 : 152628 : _reduce_vol_reset_chunk(req->vol, req->read_chunk_map_index, chunk_addr);
2187 : :
2188 : : /* meta data write backing */
2189 : 152628 : convert_supply_read_to_write(meta_req);
2190 : 152628 : _reduce_metablock_cache_read_done_update(meta_req, error);
2191 : : } else {
2192 : 152628 : _reduce_metablock_cache_write_done_update(meta_req, error);
2193 : :
2194 : : /* set the new chunk map */
2195 : 152628 : meta_req = _get_init_meta_req(req->vol, true, REDUCE_MTYPE_CHUNK_MAP,
2196 : : _meta_get_mblksn(&vol->params, REDUCE_MTYPE_CHUNK_MAP, req->chunk_map_index),
2197 : : _meta_get_elem_sn_on_mblk(&vol->params, REDUCE_MTYPE_CHUNK_MAP, req->chunk_map_index),
2198 : : _write_write_done_update_chunk_map);
2199 : 152628 : meta_req->restore_ctx.req_ctx.req = req;
2200 : 152628 : _reduce_metablock_cache_read(meta_req);
2201 : :
2202 : 152628 : return;
2203 : : }
2204 : : }
2205 : :
2206 : : static void
2207 : 481559 : _write_write_done(void *_req, int reduce_errno)
2208 : : {
2209 : 481559 : struct spdk_reduce_vol_request *req = _req;
2210 : 481559 : struct spdk_reduce_vol *vol = req->vol;
2211 : : uint64_t old_chunk_map_index;
2212 : : struct reduce_meta_request *meta_req;
2213 : :
2214 [ - + ]: 481559 : if (reduce_errno != 0) {
2215 : 0 : req->reduce_errno = reduce_errno;
2216 : : }
2217 : :
2218 [ - + ]: 481559 : assert(req->num_backing_ops > 0);
2219 [ - + ]: 481559 : if (--req->num_backing_ops > 0) {
2220 : 0 : return;
2221 : : }
2222 : :
2223 [ - + ]: 481559 : if (req->reduce_errno != 0) {
2224 [ # # ]: 0 : if (!vol->params.meta_builtin) {
2225 : 0 : _reduce_vol_reset_chunk(vol, req->chunk_map_index, NULL);
2226 : : }
2227 : : /* for meta_builtin == true, we don't need handle the case, because we haven't change the metadata yet */
2228 : 0 : _reduce_vol_complete_req(req, req->reduce_errno);
2229 : 0 : return;
2230 : : }
2231 : :
2232 [ + + ]: 481559 : if (vol->params.meta_builtin) {
2233 [ + + ]: 175880 : if (req->read_chunk_map_index != REDUCE_EMPTY_MAP_ENTRY) {
2234 : 152628 : meta_req = _get_init_meta_req(req->vol, true, REDUCE_MTYPE_CHUNK_MAP,
2235 : : _meta_get_mblksn(&vol->params, REDUCE_MTYPE_CHUNK_MAP, req->read_chunk_map_index),
2236 : : _meta_get_elem_sn_on_mblk(&vol->params, REDUCE_MTYPE_CHUNK_MAP, req->read_chunk_map_index),
2237 : : _write_write_done_clear_old_chunkmap);
2238 : 152628 : meta_req->restore_ctx.req_ctx.req = req;
2239 : 152628 : _reduce_metablock_cache_read(meta_req);
2240 : 152628 : return;
2241 : : } else {
2242 : : /* set the new chunk map */
2243 : 23252 : meta_req = _get_init_meta_req(req->vol, true, REDUCE_MTYPE_CHUNK_MAP,
2244 : : _meta_get_mblksn(&vol->params, REDUCE_MTYPE_CHUNK_MAP, req->chunk_map_index),
2245 : : _meta_get_elem_sn_on_mblk(&vol->params, REDUCE_MTYPE_CHUNK_MAP, req->chunk_map_index),
2246 : : _write_write_done_update_chunk_map);
2247 : 23252 : meta_req->restore_ctx.req_ctx.req = req;
2248 : 23252 : _reduce_metablock_cache_read(meta_req);
2249 : 23252 : return;
2250 : : }
2251 : : }
2252 : :
2253 : 305679 : old_chunk_map_index = vol->pm_logical_map[req->logical_map_index];
2254 [ + + ]: 305679 : if (old_chunk_map_index != REDUCE_EMPTY_MAP_ENTRY) {
2255 : 266737 : _reduce_vol_reset_chunk(vol, old_chunk_map_index, NULL);
2256 : : }
2257 : :
2258 : : /*
2259 : : * We don't need to persist the clearing of the old chunk map here. The old chunk map
2260 : : * becomes invalid after we update the logical map, since the old chunk map will no
2261 : : * longer have a reference to it in the logical map.
2262 : : */
2263 : :
2264 : : /* Persist the new chunk map. This must be persisted before we update the logical map. */
2265 : 305679 : _reduce_persist(vol, req->chunk,
2266 : 305679 : _reduce_vol_get_chunk_struct_size(vol->backing_io_units_per_chunk));
2267 : :
2268 : 305679 : vol->pm_logical_map[req->logical_map_index] = req->chunk_map_index;
2269 : :
2270 : 305679 : _reduce_persist(vol, &vol->pm_logical_map[req->logical_map_index], sizeof(uint64_t));
2271 : :
2272 : 305679 : _reduce_vol_complete_req(req, 0);
2273 : : }
2274 : :
2275 : : static struct spdk_reduce_backing_io *
2276 : 1107024 : _reduce_vol_req_get_backing_io(struct spdk_reduce_vol_request *req, uint32_t index)
2277 : : {
2278 : 1107024 : struct spdk_reduce_backing_dev *backing_dev = req->vol->backing_dev;
2279 : : struct spdk_reduce_backing_io *backing_io;
2280 : :
2281 : 1107024 : backing_io = (struct spdk_reduce_backing_io *)((uint8_t *)req->backing_io +
2282 : 1107024 : (sizeof(*backing_io) + backing_dev->user_ctx_size) * index);
2283 : :
2284 : 1107024 : return backing_io;
2285 : :
2286 : : }
2287 : :
2288 : : struct reduce_merged_io_desc {
2289 : : uint64_t io_unit_index;
2290 : : uint32_t num_io_units;
2291 : : };
2292 : :
2293 : : static void
2294 : 0 : _issue_backing_ops_without_merge(struct spdk_reduce_vol_request *req, struct spdk_reduce_vol *vol,
2295 : : reduce_request_fn next_fn, bool is_write)
2296 : : {
2297 : : struct iovec *iov;
2298 : : struct spdk_reduce_backing_io *backing_io;
2299 : : uint8_t *buf;
2300 : : uint32_t i;
2301 : :
2302 [ # # # # ]: 0 : if (req->chunk_is_compressed) {
2303 : 0 : iov = req->comp_buf_iov;
2304 : 0 : buf = req->comp_buf;
2305 : : } else {
2306 : 0 : iov = req->decomp_buf_iov;
2307 : 0 : buf = req->decomp_buf;
2308 : : }
2309 : :
2310 : 0 : req->num_backing_ops = req->num_io_units;
2311 : 0 : req->backing_cb_args.cb_fn = next_fn;
2312 : 0 : req->backing_cb_args.cb_arg = req;
2313 [ # # ]: 0 : for (i = 0; i < req->num_io_units; i++) {
2314 : 0 : backing_io = _reduce_vol_req_get_backing_io(req, i);
2315 : 0 : iov[i].iov_base = buf + i * vol->params.backing_io_unit_size;
2316 : 0 : iov[i].iov_len = vol->params.backing_io_unit_size;
2317 : 0 : backing_io->dev = vol->backing_dev;
2318 : 0 : backing_io->iov = &iov[i];
2319 : 0 : backing_io->iovcnt = 1;
2320 : 0 : backing_io->lba = req->chunk->io_unit_index[i] * vol->backing_lba_per_io_unit;
2321 : 0 : backing_io->lba_count = vol->backing_lba_per_io_unit;
2322 : 0 : backing_io->backing_cb_args = &req->backing_cb_args;
2323 [ # # ]: 0 : if (is_write) {
2324 : 0 : backing_io->backing_io_type = SPDK_REDUCE_BACKING_IO_WRITE;
2325 : : } else {
2326 : 0 : backing_io->backing_io_type = SPDK_REDUCE_BACKING_IO_READ;
2327 : : }
2328 : 0 : vol->backing_dev->submit_backing_io(backing_io);
2329 : : }
2330 : 0 : }
2331 : :
2332 : : static void
2333 : 1107024 : _issue_backing_ops(struct spdk_reduce_vol_request *req, struct spdk_reduce_vol *vol,
2334 : : reduce_request_fn next_fn, bool is_write)
2335 : : {
2336 : : struct iovec *iov;
2337 : : struct spdk_reduce_backing_io *backing_io;
2338 : : struct reduce_merged_io_desc merged_io_desc[4];
2339 : : uint8_t *buf;
2340 : 1107024 : bool merge = false;
2341 : 1107024 : uint32_t num_io = 0;
2342 : 1107024 : uint32_t io_unit_counts = 0;
2343 : 1107024 : uint32_t merged_io_idx = 0;
2344 : : uint32_t i;
2345 : :
2346 : : /* The merged_io_desc value is defined here to contain four elements,
2347 : : * and the chunk size must be four times the maximum of the io unit.
2348 : : * if chunk size is too big, don't merge IO.
2349 : : */
2350 [ - + ]: 1107024 : if (vol->backing_io_units_per_chunk > 4) {
2351 : 0 : _issue_backing_ops_without_merge(req, vol, next_fn, is_write);
2352 : 0 : return;
2353 : : }
2354 : :
2355 [ - + + - ]: 1107024 : if (req->chunk_is_compressed) {
2356 : 1107024 : iov = req->comp_buf_iov;
2357 : 1107024 : buf = req->comp_buf;
2358 : : } else {
2359 : 0 : iov = req->decomp_buf_iov;
2360 : 0 : buf = req->decomp_buf;
2361 : : }
2362 : :
2363 [ + - ]: 1107024 : for (i = 0; i < req->num_io_units; i++) {
2364 [ + - ]: 1107024 : if (!merge) {
2365 : 1107024 : merged_io_desc[merged_io_idx].io_unit_index = req->chunk->io_unit_index[i];
2366 : 1107024 : merged_io_desc[merged_io_idx].num_io_units = 1;
2367 : 1107024 : num_io++;
2368 : : }
2369 : :
2370 [ + - ]: 1107024 : if (i + 1 == req->num_io_units) {
2371 : 1107024 : break;
2372 : : }
2373 : :
2374 [ # # ]: 0 : if (req->chunk->io_unit_index[i] + 1 == req->chunk->io_unit_index[i + 1]) {
2375 : 0 : merged_io_desc[merged_io_idx].num_io_units += 1;
2376 : 0 : merge = true;
2377 : 0 : continue;
2378 : : }
2379 : 0 : merge = false;
2380 : 0 : merged_io_idx++;
2381 : : }
2382 : :
2383 : 1107024 : req->num_backing_ops = num_io;
2384 : 1107024 : req->backing_cb_args.cb_fn = next_fn;
2385 : 1107024 : req->backing_cb_args.cb_arg = req;
2386 [ + + ]: 2214048 : for (i = 0; i < num_io; i++) {
2387 : 1107024 : backing_io = _reduce_vol_req_get_backing_io(req, i);
2388 : 1107024 : iov[i].iov_base = buf + io_unit_counts * vol->params.backing_io_unit_size;
2389 : 1107024 : iov[i].iov_len = vol->params.backing_io_unit_size * merged_io_desc[i].num_io_units;
2390 : 1107024 : backing_io->dev = vol->backing_dev;
2391 : 1107024 : backing_io->iov = &iov[i];
2392 : 1107024 : backing_io->iovcnt = 1;
2393 : 1107024 : backing_io->lba = merged_io_desc[i].io_unit_index * vol->backing_lba_per_io_unit;
2394 : 1107024 : backing_io->lba_count = vol->backing_lba_per_io_unit * merged_io_desc[i].num_io_units;
2395 : 1107024 : backing_io->backing_cb_args = &req->backing_cb_args;
2396 [ + + ]: 1107024 : if (is_write) {
2397 : 481559 : backing_io->backing_io_type = SPDK_REDUCE_BACKING_IO_WRITE;
2398 : : } else {
2399 : 625465 : backing_io->backing_io_type = SPDK_REDUCE_BACKING_IO_READ;
2400 : : }
2401 : 1107024 : vol->backing_dev->submit_backing_io(backing_io);
2402 : :
2403 : : /* Collects the number of processed I/O. */
2404 : 1107024 : io_unit_counts += merged_io_desc[i].num_io_units;
2405 : : }
2406 : : }
2407 : :
2408 : : static void
2409 : 481559 : _reduce_vol_write_chunk(struct spdk_reduce_vol_request *req, reduce_request_fn next_fn,
2410 : : uint32_t compressed_size)
2411 : : {
2412 : 481559 : struct spdk_reduce_vol *vol = req->vol;
2413 : : uint32_t i;
2414 : 481559 : uint64_t chunk_offset, remainder, free_index, total_len = 0;
2415 : : uint8_t *buf;
2416 : : bool success;
2417 : : int j;
2418 : :
2419 : 481559 : success = queue_dequeue(&vol->free_chunks_queue, &free_index);
2420 [ + + ]: 481559 : if (success) {
2421 : 418479 : req->chunk_map_index = free_index;
2422 : : } else {
2423 : 63080 : req->chunk_map_index = spdk_bit_array_find_first_clear(vol->allocated_chunk_maps,
2424 : 63080 : vol->find_chunk_offset);
2425 : 63080 : vol->find_chunk_offset = req->chunk_map_index + 1;
2426 : : }
2427 : :
2428 : : /* TODO: fail if no chunk map found - but really this should not happen if we
2429 : : * size the number of requests similarly to number of extra chunk maps
2430 : : */
2431 [ - + ]: 481559 : assert(req->chunk_map_index != REDUCE_EMPTY_MAP_ENTRY);
2432 : 481559 : spdk_bit_array_set(vol->allocated_chunk_maps, req->chunk_map_index);
2433 : :
2434 [ + + ]: 481559 : if (vol->params.meta_builtin) {
2435 : : /* when use builtin meta, need the chunk memory from heap */
2436 : 175880 : req->chunk = req->prealloc_chunk;
2437 [ - + ]: 175880 : memset(req->chunk->io_unit_index, 0XFF, sizeof(uint64_t) * vol->backing_io_units_per_chunk);
2438 : : } else {
2439 : 305679 : req->chunk = _reduce_vol_get_chunk_map(vol, req->chunk_map_index);
2440 : : }
2441 : 481559 : req->num_io_units = spdk_divide_round_up(compressed_size,
2442 : 481559 : vol->params.backing_io_unit_size);
2443 : 481559 : req->chunk_is_compressed = (req->num_io_units != vol->backing_io_units_per_chunk);
2444 : 481559 : req->chunk->compressed_size =
2445 [ - + + - ]: 481559 : req->chunk_is_compressed ? compressed_size : vol->params.chunk_size;
2446 : :
2447 : : /* if the chunk is uncompressed we need to copy the data from the host buffers. */
2448 [ - + - + ]: 481559 : if (req->chunk_is_compressed == false) {
2449 [ # # ]: 0 : chunk_offset = req->offset % vol->logical_blocks_per_chunk;
2450 : 0 : buf = req->decomp_buf;
2451 : 0 : total_len = chunk_offset * vol->params.logical_block_size;
2452 : :
2453 : : /* zero any offset into chunk */
2454 [ # # # # : 0 : if (req->rmw == false && chunk_offset) {
# # ]
2455 [ # # ]: 0 : memset(buf, 0, total_len);
2456 : : }
2457 : 0 : buf += total_len;
2458 : :
2459 : : /* copy the data */
2460 [ # # ]: 0 : for (j = 0; j < req->iovcnt; j++) {
2461 [ # # # # ]: 0 : memcpy(buf, req->iov[j].iov_base, req->iov[j].iov_len);
2462 : 0 : buf += req->iov[j].iov_len;
2463 : 0 : total_len += req->iov[j].iov_len;
2464 : : }
2465 : :
2466 : : /* zero any remainder */
2467 : 0 : remainder = vol->params.chunk_size - total_len;
2468 : 0 : total_len += remainder;
2469 [ # # # # : 0 : if (req->rmw == false && remainder) {
# # ]
2470 [ # # ]: 0 : memset(buf, 0, remainder);
2471 : : }
2472 [ # # ]: 0 : assert(total_len == vol->params.chunk_size);
2473 : : }
2474 : :
2475 [ + + ]: 963118 : for (i = 0; i < req->num_io_units; i++) {
2476 : 481559 : success = queue_dequeue(&vol->free_backing_blocks_queue, &free_index);
2477 [ + + ]: 481559 : if (success) {
2478 : 418479 : req->chunk->io_unit_index[i] = free_index;
2479 : : } else {
2480 : 63080 : req->chunk->io_unit_index[i] = spdk_bit_array_find_first_clear(vol->allocated_backing_io_units,
2481 : 63080 : vol->find_block_offset);
2482 : 63080 : vol->find_block_offset = req->chunk->io_unit_index[i] + 1;
2483 : : }
2484 : : /* TODO: fail if no backing block found - but really this should also not
2485 : : * happen (see comment above).
2486 : : */
2487 [ - + ]: 481559 : assert(req->chunk->io_unit_index[i] != REDUCE_EMPTY_MAP_ENTRY);
2488 : 481559 : spdk_bit_array_set(vol->allocated_backing_io_units, req->chunk->io_unit_index[i]);
2489 : 481559 : vol->info.allocated_io_units++;
2490 : : }
2491 [ - + - + ]: 481559 : SPDK_INFOLOG(reduce, "datareq %p, use %lu. %lu %lu %lu %lu\n", req,
2492 : : req->chunk_map_index, req->chunk->io_unit_index[0], req->chunk->io_unit_index[1],
2493 : : req->chunk->io_unit_index[2], req->chunk->io_unit_index[3]);
2494 : 481559 : _issue_backing_ops(req, vol, next_fn, true /* write */);
2495 : 481559 : }
2496 : :
2497 : : static void
2498 : 481559 : _write_compress_done(void *_req, int reduce_errno)
2499 : : {
2500 : 481559 : struct spdk_reduce_vol_request *req = _req;
2501 : :
2502 : : /* Negative reduce_errno indicates failure for compression operations.
2503 : : * Just write the uncompressed data instead. Force this to happen
2504 : : * by just passing the full chunk size to _reduce_vol_write_chunk.
2505 : : * When it sees the data couldn't be compressed, it will just write
2506 : : * the uncompressed buffer to disk.
2507 : : */
2508 [ - + ]: 481559 : if (reduce_errno < 0) {
2509 : 0 : req->backing_cb_args.output_size = req->vol->params.chunk_size;
2510 : : }
2511 : :
2512 : 481559 : _reduce_vol_write_chunk(req, _write_write_done, req->backing_cb_args.output_size);
2513 : 481559 : }
2514 : :
2515 : : static void
2516 : 481559 : _reduce_vol_compress_chunk(struct spdk_reduce_vol_request *req, reduce_request_fn next_fn)
2517 : : {
2518 : 481559 : struct spdk_reduce_vol *vol = req->vol;
2519 : :
2520 : 481559 : req->backing_cb_args.cb_fn = next_fn;
2521 : 481559 : req->backing_cb_args.cb_arg = req;
2522 : 481559 : req->comp_buf_iov[0].iov_base = req->comp_buf;
2523 : 481559 : req->comp_buf_iov[0].iov_len = vol->params.chunk_size;
2524 : 963118 : vol->backing_dev->compress(vol->backing_dev,
2525 : 481559 : req->decomp_iov, req->decomp_iovcnt, req->comp_buf_iov, 1,
2526 : : &req->backing_cb_args);
2527 : 481559 : }
2528 : :
2529 : : static void
2530 : 418253 : _reduce_vol_decompress_chunk_scratch(struct spdk_reduce_vol_request *req, reduce_request_fn next_fn)
2531 : : {
2532 : 418253 : struct spdk_reduce_vol *vol = req->vol;
2533 : :
2534 : 418253 : req->backing_cb_args.cb_fn = next_fn;
2535 : 418253 : req->backing_cb_args.cb_arg = req;
2536 : 418253 : req->comp_buf_iov[0].iov_base = req->comp_buf;
2537 : 418253 : req->comp_buf_iov[0].iov_len = req->chunk->compressed_size;
2538 : 418253 : req->decomp_buf_iov[0].iov_base = req->decomp_buf;
2539 : 418253 : req->decomp_buf_iov[0].iov_len = vol->params.chunk_size;
2540 : 418253 : vol->backing_dev->decompress(vol->backing_dev,
2541 : : req->comp_buf_iov, 1, req->decomp_buf_iov, 1,
2542 : : &req->backing_cb_args);
2543 : 418253 : }
2544 : :
2545 : : static void
2546 : 207212 : _reduce_vol_decompress_chunk(struct spdk_reduce_vol_request *req, reduce_request_fn next_fn)
2547 : : {
2548 : 207212 : struct spdk_reduce_vol *vol = req->vol;
2549 : 207212 : uint64_t chunk_offset, remainder = 0;
2550 : 207212 : uint64_t ttl_len = 0;
2551 : : size_t iov_len;
2552 : : int i;
2553 : :
2554 : 207212 : req->decomp_iovcnt = 0;
2555 [ - + ]: 207212 : chunk_offset = req->offset % vol->logical_blocks_per_chunk;
2556 : :
2557 : : /* If backing device doesn't support SGL output then we should copy the result of decompression to user's buffer
2558 : : * if at least one of the conditions below is true:
2559 : : * 1. User's buffer is fragmented
2560 : : * 2. Length of the user's buffer is less than the chunk
2561 : : * 3. User's buffer is contig, equals chunk_size but crosses huge page boundary */
2562 : 207212 : iov_len = req->iov[0].iov_len;
2563 [ - + + - : 414416 : req->copy_after_decompress = !vol->backing_dev->sgl_out && (req->iovcnt > 1 ||
+ + ]
2564 [ + + - + ]: 209208 : req->iov[0].iov_len < vol->params.chunk_size ||
2565 : 2004 : _addr_crosses_huge_page(req->iov[0].iov_base, &iov_len));
2566 [ - + + + ]: 207212 : if (req->copy_after_decompress) {
2567 : 205208 : req->decomp_iov[0].iov_base = req->decomp_buf;
2568 : 205208 : req->decomp_iov[0].iov_len = vol->params.chunk_size;
2569 : 205208 : req->decomp_iovcnt = 1;
2570 : 205208 : goto decompress;
2571 : : }
2572 : :
2573 [ - + ]: 2004 : if (chunk_offset) {
2574 : : /* first iov point to our scratch buffer for any offset into the chunk */
2575 : 0 : req->decomp_iov[0].iov_base = req->decomp_buf;
2576 : 0 : req->decomp_iov[0].iov_len = chunk_offset * vol->params.logical_block_size;
2577 : 0 : ttl_len += req->decomp_iov[0].iov_len;
2578 : 0 : req->decomp_iovcnt = 1;
2579 : : }
2580 : :
2581 : : /* now the user data iov, direct to the user buffer */
2582 [ + + ]: 4008 : for (i = 0; i < req->iovcnt; i++) {
2583 : 2004 : req->decomp_iov[i + req->decomp_iovcnt].iov_base = req->iov[i].iov_base;
2584 : 2004 : req->decomp_iov[i + req->decomp_iovcnt].iov_len = req->iov[i].iov_len;
2585 : 2004 : ttl_len += req->decomp_iov[i + req->decomp_iovcnt].iov_len;
2586 : : }
2587 : 2004 : req->decomp_iovcnt += req->iovcnt;
2588 : :
2589 : : /* send the rest of the chunk to our scratch buffer */
2590 : 2004 : remainder = vol->params.chunk_size - ttl_len;
2591 [ - + ]: 2004 : if (remainder) {
2592 : 0 : req->decomp_iov[req->decomp_iovcnt].iov_base = req->decomp_buf + ttl_len;
2593 : 0 : req->decomp_iov[req->decomp_iovcnt].iov_len = remainder;
2594 : 0 : ttl_len += req->decomp_iov[req->decomp_iovcnt].iov_len;
2595 : 0 : req->decomp_iovcnt++;
2596 : : }
2597 [ - + ]: 2004 : assert(ttl_len == vol->params.chunk_size);
2598 : :
2599 : 207212 : decompress:
2600 [ - + + + : 207212 : assert(!req->copy_after_decompress || (req->copy_after_decompress && req->decomp_iovcnt == 1));
- + + - +
- ]
2601 : 207212 : req->backing_cb_args.cb_fn = next_fn;
2602 : 207212 : req->backing_cb_args.cb_arg = req;
2603 : 207212 : req->comp_buf_iov[0].iov_base = req->comp_buf;
2604 : 207212 : req->comp_buf_iov[0].iov_len = req->chunk->compressed_size;
2605 : 414424 : vol->backing_dev->decompress(vol->backing_dev,
2606 : 207212 : req->comp_buf_iov, 1, req->decomp_iov, req->decomp_iovcnt,
2607 : : &req->backing_cb_args);
2608 : 207212 : }
2609 : :
2610 : : static inline void
2611 : 479555 : _prepare_compress_chunk_copy_user_buffers(struct spdk_reduce_vol_request *req, bool zero_paddings)
2612 : : {
2613 : 479555 : struct spdk_reduce_vol *vol = req->vol;
2614 : 479555 : uint64_t chunk_offset, ttl_len = 0;
2615 : 479555 : uint64_t remainder = 0;
2616 : 479555 : char *copy_offset = NULL;
2617 : 479555 : uint32_t lbsize = vol->params.logical_block_size;
2618 : : int i;
2619 : :
2620 : 479555 : req->decomp_iov[0].iov_base = req->decomp_buf;
2621 : 479555 : req->decomp_iov[0].iov_len = vol->params.chunk_size;
2622 : 479555 : req->decomp_iovcnt = 1;
2623 : 479555 : copy_offset = req->decomp_iov[0].iov_base;
2624 [ - + ]: 479555 : chunk_offset = req->offset % vol->logical_blocks_per_chunk;
2625 : :
2626 [ + + ]: 479555 : if (chunk_offset) {
2627 : 291044 : ttl_len += chunk_offset * lbsize;
2628 : : /* copy_offset already points to the correct buffer if zero_paddings=false */
2629 [ + + ]: 291044 : if (zero_paddings) {
2630 [ - + ]: 1146 : memset(copy_offset, 0, ttl_len);
2631 : : }
2632 : 291044 : copy_offset += ttl_len;
2633 : : }
2634 : :
2635 : : /* now the user data iov, direct from the user buffer */
2636 [ + + ]: 959230 : for (i = 0; i < req->iovcnt; i++) {
2637 [ - + - + ]: 479675 : memcpy(copy_offset, req->iov[i].iov_base, req->iov[i].iov_len);
2638 : 479675 : copy_offset += req->iov[i].iov_len;
2639 : 479675 : ttl_len += req->iov[i].iov_len;
2640 : : }
2641 : :
2642 : 479555 : remainder = vol->params.chunk_size - ttl_len;
2643 [ + + ]: 479555 : if (remainder) {
2644 : : /* copy_offset already points to the correct buffer if zero_paddings=false */
2645 [ + + ]: 291092 : if (zero_paddings) {
2646 [ - + ]: 60156 : memset(copy_offset, 0, remainder);
2647 : : }
2648 : 291092 : ttl_len += remainder;
2649 : : }
2650 : :
2651 [ - + ]: 479555 : assert(ttl_len == req->vol->params.chunk_size);
2652 : 479555 : }
2653 : :
2654 : : /* This function can be called when we are compressing a new data or in case of read-modify-write
2655 : : * In the first case possible paddings should be filled with zeroes, in the second case the paddings
2656 : : * should point to already read and decompressed buffer */
2657 : : static inline void
2658 : 481559 : _prepare_compress_chunk(struct spdk_reduce_vol_request *req, bool zero_paddings)
2659 : : {
2660 : 481559 : struct spdk_reduce_vol *vol = req->vol;
2661 [ + + ]: 481559 : char *padding_buffer = zero_paddings ? g_zero_buf : req->decomp_buf;
2662 : 481559 : uint64_t chunk_offset, ttl_len = 0;
2663 : 481559 : uint64_t remainder = 0;
2664 : 481559 : uint32_t lbsize = vol->params.logical_block_size;
2665 : : size_t iov_len;
2666 : : int i;
2667 : :
2668 : : /* If backing device doesn't support SGL input then we should copy user's buffer into decomp_buf
2669 : : * if at least one of the conditions below is true:
2670 : : * 1. User's buffer is fragmented
2671 : : * 2. Length of the user's buffer is less than the chunk
2672 : : * 3. User's buffer is contig, equals chunk_size but crosses huge page boundary */
2673 : 481559 : iov_len = req->iov[0].iov_len;
2674 [ - + + - : 481559 : if (!vol->backing_dev->sgl_in && (req->iovcnt > 1 ||
+ + ]
2675 [ + + - + ]: 483555 : req->iov[0].iov_len < vol->params.chunk_size ||
2676 : 2004 : _addr_crosses_huge_page(req->iov[0].iov_base, &iov_len))) {
2677 : 479555 : _prepare_compress_chunk_copy_user_buffers(req, zero_paddings);
2678 : 479555 : return;
2679 : : }
2680 : :
2681 : 2004 : req->decomp_iovcnt = 0;
2682 [ - + ]: 2004 : chunk_offset = req->offset % vol->logical_blocks_per_chunk;
2683 : :
2684 [ - + ]: 2004 : if (chunk_offset != 0) {
2685 : 0 : ttl_len += chunk_offset * lbsize;
2686 : 0 : req->decomp_iov[0].iov_base = padding_buffer;
2687 : 0 : req->decomp_iov[0].iov_len = ttl_len;
2688 : 0 : req->decomp_iovcnt = 1;
2689 : : }
2690 : :
2691 : : /* now the user data iov, direct from the user buffer */
2692 [ + + ]: 4008 : for (i = 0; i < req->iovcnt; i++) {
2693 : 2004 : req->decomp_iov[i + req->decomp_iovcnt].iov_base = req->iov[i].iov_base;
2694 : 2004 : req->decomp_iov[i + req->decomp_iovcnt].iov_len = req->iov[i].iov_len;
2695 : 2004 : ttl_len += req->iov[i].iov_len;
2696 : : }
2697 : 2004 : req->decomp_iovcnt += req->iovcnt;
2698 : :
2699 : 2004 : remainder = vol->params.chunk_size - ttl_len;
2700 [ - + ]: 2004 : if (remainder) {
2701 : 0 : req->decomp_iov[req->decomp_iovcnt].iov_base = padding_buffer + ttl_len;
2702 : 0 : req->decomp_iov[req->decomp_iovcnt].iov_len = remainder;
2703 : 0 : req->decomp_iovcnt++;
2704 : 0 : ttl_len += remainder;
2705 : : }
2706 [ - + ]: 2004 : assert(ttl_len == req->vol->params.chunk_size);
2707 : : }
2708 : :
2709 : : static void
2710 : 418253 : _write_decompress_done(void *_req, int reduce_errno)
2711 : : {
2712 : 418253 : struct spdk_reduce_vol_request *req = _req;
2713 : :
2714 : : /* Negative reduce_errno indicates failure for compression operations. */
2715 [ - + ]: 418253 : if (reduce_errno < 0) {
2716 : 0 : _reduce_vol_complete_req(req, reduce_errno);
2717 : 0 : return;
2718 : : }
2719 : :
2720 : : /* Positive reduce_errno indicates that the output size field in the backing_cb_args
2721 : : * represents the output_size.
2722 : : */
2723 [ - + ]: 418253 : if (req->backing_cb_args.output_size != req->vol->params.chunk_size) {
2724 : 0 : _reduce_vol_complete_req(req, -EIO);
2725 : 0 : return;
2726 : : }
2727 : :
2728 : 418253 : _prepare_compress_chunk(req, false);
2729 : 418253 : _reduce_vol_compress_chunk(req, _write_compress_done);
2730 : : }
2731 : :
2732 : : static void
2733 : 418253 : _write_read_done(void *_req, int reduce_errno)
2734 : : {
2735 : 418253 : struct spdk_reduce_vol_request *req = _req;
2736 : :
2737 [ - + ]: 418253 : if (reduce_errno != 0) {
2738 : 0 : req->reduce_errno = reduce_errno;
2739 : : }
2740 : :
2741 [ - + ]: 418253 : assert(req->num_backing_ops > 0);
2742 [ - + ]: 418253 : if (--req->num_backing_ops > 0) {
2743 : 0 : return;
2744 : : }
2745 : :
2746 [ - + ]: 418253 : if (req->reduce_errno != 0) {
2747 : 0 : _reduce_vol_complete_req(req, req->reduce_errno);
2748 : 0 : return;
2749 : : }
2750 : :
2751 [ - + + - ]: 418253 : if (req->chunk_is_compressed) {
2752 : 418253 : _reduce_vol_decompress_chunk_scratch(req, _write_decompress_done);
2753 : : } else {
2754 : 0 : req->backing_cb_args.output_size = req->chunk->compressed_size;
2755 : :
2756 : 0 : _write_decompress_done(req, 0);
2757 : : }
2758 : : }
2759 : :
2760 : : static void
2761 : 207212 : _read_decompress_done(void *_req, int reduce_errno)
2762 : : {
2763 : 207212 : struct spdk_reduce_vol_request *req = _req;
2764 : 207212 : struct spdk_reduce_vol *vol = req->vol;
2765 : :
2766 : : /* Negative reduce_errno indicates failure for compression operations. */
2767 [ - + ]: 207212 : if (reduce_errno < 0) {
2768 : 0 : _reduce_vol_complete_req(req, reduce_errno);
2769 : 0 : return;
2770 : : }
2771 : :
2772 : : /* Positive reduce_errno indicates that the output size field in the backing_cb_args
2773 : : * represents the output_size.
2774 : : */
2775 [ - + ]: 207212 : if (req->backing_cb_args.output_size != vol->params.chunk_size) {
2776 : 0 : _reduce_vol_complete_req(req, -EIO);
2777 : 0 : return;
2778 : : }
2779 : :
2780 [ - + + + ]: 207212 : if (req->copy_after_decompress) {
2781 [ - + ]: 205208 : uint64_t chunk_offset = req->offset % vol->logical_blocks_per_chunk;
2782 : 205208 : char *decomp_buffer = (char *)req->decomp_buf + chunk_offset * vol->params.logical_block_size;
2783 : : int i;
2784 : :
2785 [ + + ]: 410536 : for (i = 0; i < req->iovcnt; i++) {
2786 [ - + - + ]: 205328 : memcpy(req->iov[i].iov_base, decomp_buffer, req->iov[i].iov_len);
2787 : 205328 : decomp_buffer += req->iov[i].iov_len;
2788 [ - + ]: 205328 : assert(decomp_buffer <= (char *)req->decomp_buf + vol->params.chunk_size);
2789 : : }
2790 : : }
2791 : :
2792 : 207212 : _reduce_vol_complete_req(req, 0);
2793 : : }
2794 : :
2795 : : static void
2796 : 207212 : _read_read_done(void *_req, int reduce_errno)
2797 : : {
2798 : 207212 : struct spdk_reduce_vol_request *req = _req;
2799 : :
2800 [ - + ]: 207212 : if (reduce_errno != 0) {
2801 : 0 : req->reduce_errno = reduce_errno;
2802 : : }
2803 : :
2804 [ - + ]: 207212 : assert(req->num_backing_ops > 0);
2805 [ - + ]: 207212 : if (--req->num_backing_ops > 0) {
2806 : 0 : return;
2807 : : }
2808 : :
2809 [ - + ]: 207212 : if (req->reduce_errno != 0) {
2810 : 0 : _reduce_vol_complete_req(req, req->reduce_errno);
2811 : 0 : return;
2812 : : }
2813 : :
2814 [ - + + - ]: 207212 : if (req->chunk_is_compressed) {
2815 : 207212 : _reduce_vol_decompress_chunk(req, _read_decompress_done);
2816 : : } else {
2817 : :
2818 : : /* If the chunk was compressed, the data would have been sent to the
2819 : : * host buffers by the decompression operation, if not we need to memcpy
2820 : : * from req->decomp_buf.
2821 : : */
2822 : 0 : req->copy_after_decompress = true;
2823 : 0 : req->backing_cb_args.output_size = req->chunk->compressed_size;
2824 : :
2825 : 0 : _read_decompress_done(req, 0);
2826 : : }
2827 : : }
2828 : :
2829 : : static void
2830 : 224308 : _reduce_meta_read_chunk_map_for_read_chunk(void *_meta_req, int error)
2831 : : {
2832 : 224308 : struct reduce_meta_request *meta_req = _meta_req;
2833 : 224308 : struct spdk_reduce_vol *vol = meta_req->restore_ctx.read_chunk_ctx.vol;
2834 : 224308 : struct spdk_reduce_vol_request *req = meta_req->restore_ctx.read_chunk_ctx.req;
2835 : 224308 : reduce_request_fn next_fn = meta_req->restore_ctx.read_chunk_ctx.next_fn;
2836 : :
2837 [ - + ]: 224308 : if (error != 0) {
2838 : 0 : next_fn(req, error);
2839 : 0 : _reduce_metablock_cache_read_done_update(meta_req, error);
2840 : 0 : return;
2841 : : }
2842 : :
2843 : : /* copy the elem memory from metablock */
2844 : 224308 : req->chunk = req->prealloc_chunk;
2845 [ - + - + ]: 224308 : memcpy(req->chunk, _reduce_get_meta_elem_addr(meta_req),
2846 : 224308 : _reduce_vol_get_chunk_struct_size(vol->backing_io_units_per_chunk));
2847 : 224308 : _reduce_metablock_cache_read_done_update(meta_req, error);
2848 : :
2849 [ - + ]: 224308 : assert(req->chunk->compressed_size != UINT32_MAX);
2850 : 224308 : req->num_io_units = spdk_divide_round_up(req->chunk->compressed_size,
2851 : 224308 : vol->params.backing_io_unit_size);
2852 : 224308 : req->chunk_is_compressed = (req->num_io_units != vol->backing_io_units_per_chunk);
2853 : :
2854 : 224308 : _issue_backing_ops(req, vol, next_fn, false /* read */);
2855 : : }
2856 : :
2857 : : static void
2858 : 72276 : _reduce_meta_read_logical_map_for_read_chunk(void *_meta_req, int error)
2859 : : {
2860 : 72276 : struct reduce_meta_request *meta_req = _meta_req, *chunkmap_meta_req;
2861 : 72276 : struct spdk_reduce_vol *vol = meta_req->restore_ctx.read_chunk_ctx.vol;
2862 : 72276 : struct spdk_reduce_vol_request *req = meta_req->restore_ctx.read_chunk_ctx.req;
2863 : 72276 : reduce_request_fn next_fn = meta_req->restore_ctx.read_chunk_ctx.next_fn;
2864 : :
2865 [ - + ]: 72276 : if (error != 0) {
2866 : 0 : next_fn(req, error);
2867 : 0 : _reduce_metablock_cache_read_done_update(meta_req, error);
2868 : 0 : return;
2869 : : }
2870 : :
2871 : : /* read the meta elem */
2872 : 72276 : req->chunk_map_index = *(uint64_t *)_reduce_get_meta_elem_addr(meta_req);
2873 : 72276 : _reduce_metablock_cache_read_done_update(meta_req, error);
2874 : :
2875 : : /* check chunk allocated */
2876 [ + + ]: 72276 : if (req->chunk_map_index == REDUCE_EMPTY_MAP_ENTRY) {
2877 [ + + ]: 80 : for (int i = 0; i < req->iovcnt; i++) {
2878 [ - + ]: 40 : memset(req->iov[i].iov_base, 0, req->iov[i].iov_len);
2879 : : }
2880 : 40 : _reduce_vol_complete_req(req, 0);
2881 : 40 : return;
2882 : : }
2883 : :
2884 : : /* read chunkmap */
2885 : 72236 : chunkmap_meta_req = _get_init_meta_req(vol, false, REDUCE_MTYPE_CHUNK_MAP,
2886 : : _meta_get_mblksn(&vol->params, REDUCE_MTYPE_CHUNK_MAP, req->chunk_map_index),
2887 : : _meta_get_elem_sn_on_mblk(&vol->params, REDUCE_MTYPE_CHUNK_MAP, req->chunk_map_index),
2888 : : _reduce_meta_read_chunk_map_for_read_chunk);
2889 : 72236 : chunkmap_meta_req->restore_ctx.read_chunk_ctx.vol = vol;
2890 : 72236 : chunkmap_meta_req->restore_ctx.read_chunk_ctx.req = req;
2891 : 72236 : chunkmap_meta_req->restore_ctx.read_chunk_ctx.next_fn = next_fn;
2892 : 72236 : _reduce_metablock_cache_read(chunkmap_meta_req);
2893 : :
2894 : 72236 : return;
2895 : : }
2896 : :
2897 : : static void
2898 : 625505 : _reduce_vol_read_chunk(struct spdk_reduce_vol_request *req, reduce_request_fn next_fn)
2899 : : {
2900 : 625505 : struct spdk_reduce_vol *vol = req->vol;
2901 : 625505 : struct reduce_meta_request *meta_req = NULL;
2902 : :
2903 [ + + ]: 625505 : if (vol->params.meta_builtin) {
2904 [ + + ]: 224348 : if (next_fn == _write_read_done) {
2905 : 152072 : meta_req = _get_init_meta_req(vol, false, REDUCE_MTYPE_CHUNK_MAP,
2906 : : _meta_get_mblksn(&vol->params, REDUCE_MTYPE_CHUNK_MAP, req->chunk_map_index),
2907 : : _meta_get_elem_sn_on_mblk(&vol->params, REDUCE_MTYPE_CHUNK_MAP, req->chunk_map_index),
2908 : : _reduce_meta_read_chunk_map_for_read_chunk);
2909 [ + - ]: 72276 : } else if (next_fn == _read_read_done) {
2910 : 72276 : meta_req = _get_init_meta_req(vol, false, REDUCE_MTYPE_LOGICAL_MAP,
2911 : : _meta_get_mblksn(&vol->params, REDUCE_MTYPE_LOGICAL_MAP, req->logical_map_index),
2912 : : _meta_get_elem_sn_on_mblk(&vol->params, REDUCE_MTYPE_LOGICAL_MAP, req->logical_map_index),
2913 : : _reduce_meta_read_logical_map_for_read_chunk);
2914 : : } else {
2915 : 0 : assert(0);
2916 : : }
2917 : 224348 : meta_req->restore_ctx.read_chunk_ctx.vol = vol;
2918 : 224348 : meta_req->restore_ctx.read_chunk_ctx.req = req;
2919 : 224348 : meta_req->restore_ctx.read_chunk_ctx.next_fn = next_fn;
2920 : 224348 : _reduce_metablock_cache_read(meta_req);
2921 : :
2922 : 224348 : return;
2923 : : }
2924 : :
2925 : 401157 : req->chunk_map_index = vol->pm_logical_map[req->logical_map_index];
2926 [ - + ]: 401157 : assert(req->chunk_map_index != REDUCE_EMPTY_MAP_ENTRY);
2927 : :
2928 : 401157 : req->chunk = _reduce_vol_get_chunk_map(vol, req->chunk_map_index);
2929 : 401157 : req->num_io_units = spdk_divide_round_up(req->chunk->compressed_size,
2930 : 401157 : vol->params.backing_io_unit_size);
2931 : 401157 : req->chunk_is_compressed = (req->num_io_units != vol->backing_io_units_per_chunk);
2932 : :
2933 : 401157 : _issue_backing_ops(req, vol, next_fn, false /* read */);
2934 : : }
2935 : :
2936 : : static bool
2937 : 688859 : _iov_array_is_valid(struct spdk_reduce_vol *vol, struct iovec *iov, int iovcnt,
2938 : : uint64_t length)
2939 : : {
2940 : 688859 : uint64_t size = 0;
2941 : : int i;
2942 : :
2943 [ - + ]: 688859 : if (iovcnt > REDUCE_MAX_IOVECS) {
2944 : 0 : return false;
2945 : : }
2946 : :
2947 [ + + ]: 1377958 : for (i = 0; i < iovcnt; i++) {
2948 : 689099 : size += iov[i].iov_len;
2949 : : }
2950 : :
2951 : 688859 : return size == (length * vol->params.logical_block_size);
2952 : : }
2953 : :
2954 : : static bool
2955 : 1237561 : _check_overlap(struct spdk_reduce_vol *vol, uint64_t logical_map_index)
2956 : : {
2957 : : struct spdk_reduce_vol_request req;
2958 : :
2959 : 1237561 : req.logical_map_index = logical_map_index;
2960 : :
2961 : 1237561 : return (NULL != RB_FIND(executing_req_tree, &vol->executing_requests, &req));
2962 : : }
2963 : :
2964 : : static void
2965 : 207252 : _start_readv_request(struct spdk_reduce_vol_request *req)
2966 : : {
2967 : 207252 : RB_INSERT(executing_req_tree, &req->vol->executing_requests, req);
2968 : 207252 : _reduce_vol_read_chunk(req, _read_read_done);
2969 : 207252 : }
2970 : :
2971 : : void
2972 : 207300 : spdk_reduce_vol_readv(struct spdk_reduce_vol *vol,
2973 : : struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length,
2974 : : spdk_reduce_vol_op_complete cb_fn, void *cb_arg)
2975 : : {
2976 : : struct spdk_reduce_vol_request *req;
2977 : : uint64_t logical_map_index;
2978 : : bool overlapped;
2979 : : int i;
2980 : :
2981 [ - + ]: 207300 : if (length == 0) {
2982 : 0 : cb_fn(cb_arg, 0);
2983 : 0 : return;
2984 : : }
2985 : :
2986 [ - + ]: 207300 : if (_request_spans_chunk_boundary(vol, offset, length)) {
2987 : 0 : cb_fn(cb_arg, -EINVAL);
2988 : 0 : return;
2989 : : }
2990 : :
2991 [ - + ]: 207300 : if (!_iov_array_is_valid(vol, iov, iovcnt, length)) {
2992 : 0 : cb_fn(cb_arg, -EINVAL);
2993 : 0 : return;
2994 : : }
2995 : :
2996 [ - + ]: 207300 : logical_map_index = offset / vol->logical_blocks_per_chunk;
2997 : 207300 : overlapped = _check_overlap(vol, logical_map_index);
2998 : :
2999 : : /* If we use builtin metadata, chunk allocated detection needs async read backend.
3000 : : * and get the metadata async and check the chunk allocated status, maybe late,
3001 : : * there may be another write about the chunk.
3002 : : * So we need insert the overlap tree, then check the chunk allocated status.
3003 : : */
3004 [ + + ]: 207300 : if (!vol->params.meta_builtin) {
3005 [ + + + + ]: 135024 : if (!overlapped && vol->pm_logical_map[logical_map_index] == REDUCE_EMPTY_MAP_ENTRY) {
3006 : : /*
3007 : : * This chunk hasn't been allocated. So treat the data as all
3008 : : * zeroes for this chunk - do the memset and immediately complete
3009 : : * the operation.
3010 : : */
3011 [ + + ]: 96 : for (i = 0; i < iovcnt; i++) {
3012 [ - + ]: 48 : memset(iov[i].iov_base, 0, iov[i].iov_len);
3013 : : }
3014 : 48 : cb_fn(cb_arg, 0);
3015 : 48 : return;
3016 : : }
3017 : : }
3018 : :
3019 : 207252 : req = TAILQ_FIRST(&vol->free_requests);
3020 [ - + ]: 207252 : if (req == NULL) {
3021 : 0 : cb_fn(cb_arg, -ENOMEM);
3022 : 0 : return;
3023 : : }
3024 : :
3025 [ + - ]: 207252 : TAILQ_REMOVE(&vol->free_requests, req, tailq);
3026 : 207252 : req->type = REDUCE_IO_READV;
3027 : 207252 : req->vol = vol;
3028 : 207252 : req->iov = iov;
3029 : 207252 : req->iovcnt = iovcnt;
3030 : 207252 : req->offset = offset;
3031 : 207252 : req->logical_map_index = logical_map_index;
3032 : 207252 : req->length = length;
3033 : 207252 : req->copy_after_decompress = false;
3034 : 207252 : req->cb_fn = cb_fn;
3035 : 207252 : req->cb_arg = cb_arg;
3036 : 207252 : req->reduce_errno = 0;
3037 : :
3038 [ + + ]: 207252 : if (!overlapped) {
3039 : 5014 : _start_readv_request(req);
3040 : : } else {
3041 : 202238 : TAILQ_INSERT_TAIL(&vol->queued_requests, req, tailq);
3042 : : }
3043 : : }
3044 : :
3045 : : static void
3046 : 175880 : _reduce_meta_read_logical_map_for_writev(void *_meta_req, int error)
3047 : : {
3048 : 175880 : struct reduce_meta_request *meta_req = _meta_req;
3049 : 175880 : struct spdk_reduce_vol_request *req = meta_req->restore_ctx.req_ctx.req;
3050 : 175880 : struct spdk_reduce_vol *vol = req->vol;
3051 : :
3052 [ - + ]: 175880 : if (error != 0) {
3053 : 0 : req->reduce_errno = error;
3054 : 0 : _reduce_vol_complete_req(req, req->reduce_errno);
3055 : 0 : return;
3056 : : }
3057 : :
3058 : : /* read the meta elem */
3059 : 175880 : req->chunk_map_index = *(uint64_t *)_reduce_get_meta_elem_addr(meta_req);
3060 : 175880 : req->read_chunk_map_index = req->chunk_map_index;
3061 : 175880 : _reduce_metablock_cache_read_done_update(meta_req, error);
3062 : :
3063 : : /* restore */
3064 [ + + ]: 175880 : if (req->chunk_map_index != REDUCE_EMPTY_MAP_ENTRY) {
3065 [ + + ]: 152628 : if ((req->length * vol->params.logical_block_size) < vol->params.chunk_size) {
3066 : : /* Read old chunk, then overwrite with data from this write
3067 : : * operation.
3068 : : */
3069 : 152072 : req->rmw = true;
3070 : 152072 : _reduce_vol_read_chunk(req, _write_read_done);
3071 : 152072 : return;
3072 : : }
3073 : : }
3074 : :
3075 : 23808 : req->rmw = false;
3076 : :
3077 : 23808 : _prepare_compress_chunk(req, true);
3078 : 23808 : _reduce_vol_compress_chunk(req, _write_compress_done);
3079 : : }
3080 : :
3081 : : static void
3082 : 481559 : _start_writev_request(struct spdk_reduce_vol_request *req)
3083 : : {
3084 : 481559 : struct spdk_reduce_vol *vol = req->vol;
3085 : : struct reduce_meta_request *meta_req;
3086 : :
3087 : 481559 : RB_INSERT(executing_req_tree, &req->vol->executing_requests, req);
3088 : :
3089 [ + + ]: 481559 : if (vol->params.meta_builtin) {
3090 : 175880 : meta_req = _get_init_meta_req(vol, false, REDUCE_MTYPE_LOGICAL_MAP,
3091 : : _meta_get_mblksn(&vol->params, REDUCE_MTYPE_LOGICAL_MAP, req->logical_map_index),
3092 : : _meta_get_elem_sn_on_mblk(&vol->params, REDUCE_MTYPE_LOGICAL_MAP, req->logical_map_index),
3093 : : _reduce_meta_read_logical_map_for_writev);
3094 : 175880 : meta_req->restore_ctx.req_ctx.req = req;
3095 : :
3096 : 175880 : _reduce_metablock_cache_read(meta_req);
3097 : 175880 : return;
3098 : : }
3099 : :
3100 [ + + ]: 305679 : if (vol->pm_logical_map[req->logical_map_index] != REDUCE_EMPTY_MAP_ENTRY) {
3101 [ + + ]: 266737 : if ((req->length * vol->params.logical_block_size) < vol->params.chunk_size) {
3102 : : /* Read old chunk, then overwrite with data from this write
3103 : : * operation.
3104 : : */
3105 : 266181 : req->rmw = true;
3106 : 266181 : _reduce_vol_read_chunk(req, _write_read_done);
3107 : 266181 : return;
3108 : : }
3109 : : }
3110 : :
3111 : 39498 : req->rmw = false;
3112 : :
3113 : 39498 : _prepare_compress_chunk(req, true);
3114 : 39498 : _reduce_vol_compress_chunk(req, _write_compress_done);
3115 : : }
3116 : :
3117 : : void
3118 : 481559 : spdk_reduce_vol_writev(struct spdk_reduce_vol *vol,
3119 : : struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length,
3120 : : spdk_reduce_vol_op_complete cb_fn, void *cb_arg)
3121 : : {
3122 : : struct spdk_reduce_vol_request *req;
3123 : : uint64_t logical_map_index;
3124 : : bool overlapped;
3125 : :
3126 [ - + ]: 481559 : if (length == 0) {
3127 : 0 : cb_fn(cb_arg, 0);
3128 : 0 : return;
3129 : : }
3130 : :
3131 [ - + ]: 481559 : if (_request_spans_chunk_boundary(vol, offset, length)) {
3132 : 0 : cb_fn(cb_arg, -EINVAL);
3133 : 0 : return;
3134 : : }
3135 : :
3136 [ - + ]: 481559 : if (!_iov_array_is_valid(vol, iov, iovcnt, length)) {
3137 : 0 : cb_fn(cb_arg, -EINVAL);
3138 : 0 : return;
3139 : : }
3140 : :
3141 [ - + ]: 481559 : logical_map_index = offset / vol->logical_blocks_per_chunk;
3142 : 481559 : overlapped = _check_overlap(vol, logical_map_index);
3143 : :
3144 : 481559 : req = TAILQ_FIRST(&vol->free_requests);
3145 [ - + ]: 481559 : if (req == NULL) {
3146 : 0 : cb_fn(cb_arg, -ENOMEM);
3147 : 0 : return;
3148 : : }
3149 : :
3150 [ + - ]: 481559 : TAILQ_REMOVE(&vol->free_requests, req, tailq);
3151 : 481559 : req->type = REDUCE_IO_WRITEV;
3152 : 481559 : req->vol = vol;
3153 : 481559 : req->iov = iov;
3154 : 481559 : req->iovcnt = iovcnt;
3155 : 481559 : req->offset = offset;
3156 : 481559 : req->logical_map_index = logical_map_index;
3157 : 481559 : req->length = length;
3158 : 481559 : req->copy_after_decompress = false;
3159 : 481559 : req->cb_fn = cb_fn;
3160 : 481559 : req->cb_arg = cb_arg;
3161 : 481559 : req->reduce_errno = 0;
3162 : :
3163 [ + + ]: 481559 : if (!overlapped) {
3164 : 194840 : _start_writev_request(req);
3165 : : } else {
3166 : 286719 : TAILQ_INSERT_TAIL(&vol->queued_requests, req, tailq);
3167 : : }
3168 : : }
3169 : :
3170 : : static void
3171 : 0 : _unmap_chunkmap_update_error_process(struct reduce_meta_request *chunk_map_meta_req, int error)
3172 : : {
3173 : 0 : struct reduce_meta_request *logical_map_meta_req =
3174 : : chunk_map_meta_req->restore_ctx.req_ctx.logicalmap_meta_req;
3175 : 0 : struct spdk_reduce_vol_request *req = chunk_map_meta_req->restore_ctx.req_ctx.req;
3176 : :
3177 [ # # # # : 0 : if (chunk_map_meta_req->supply_read || !chunk_map_meta_req->is_write) {
# # # # ]
3178 : 0 : _reduce_metablock_cache_read_done_update(chunk_map_meta_req, error);
3179 : : } else {
3180 : 0 : _reduce_metablock_cache_write_done_update(chunk_map_meta_req, error);
3181 : : }
3182 : 0 : _reduce_metablock_cache_read_done_update(logical_map_meta_req, error);
3183 : 0 : _reduce_vol_complete_req(req, error);
3184 : 0 : }
3185 : :
3186 : : static void
3187 : 0 : _reduce_meta_read_chunk_map_for_unmap_full_chunk(void *cb_arg, int reduce_errno)
3188 : : {
3189 : 0 : struct reduce_meta_request *chunk_map_meta_req = cb_arg;
3190 : 0 : struct spdk_reduce_vol_request *req = chunk_map_meta_req->restore_ctx.req_ctx.req;
3191 : 0 : struct spdk_reduce_vol *vol = req->vol;
3192 : 0 : struct reduce_meta_request *logical_map_meta_req =
3193 : : chunk_map_meta_req->restore_ctx.req_ctx.logicalmap_meta_req;
3194 : : struct spdk_reduce_chunk_map *chunk_map;
3195 : : uint64_t *logical_map;
3196 : :
3197 [ # # # # ]: 0 : if (chunk_map_meta_req->supply_read) {
3198 [ # # ]: 0 : if (reduce_errno != 0) {
3199 : 0 : _unmap_chunkmap_update_error_process(chunk_map_meta_req, reduce_errno);
3200 : 0 : return;
3201 : : }
3202 : :
3203 : 0 : chunk_map = (struct spdk_reduce_chunk_map *)_reduce_get_meta_elem_addr(chunk_map_meta_req);
3204 : 0 : _reduce_vol_reset_chunk(vol, req->chunk_map_index, chunk_map);
3205 : 0 : convert_supply_read_to_write(chunk_map_meta_req);
3206 : 0 : _reduce_metablock_cache_read_done_update(chunk_map_meta_req, reduce_errno);
3207 : : } else {
3208 [ # # ]: 0 : if (reduce_errno != 0) {
3209 : 0 : _unmap_chunkmap_update_error_process(chunk_map_meta_req, reduce_errno);
3210 : 0 : return;
3211 : : }
3212 : 0 : _reduce_metablock_cache_write_done_update(chunk_map_meta_req, reduce_errno);
3213 : :
3214 : : /* clear logical map */
3215 : 0 : logical_map = (uint64_t *)_reduce_get_meta_elem_addr(logical_map_meta_req);
3216 : 0 : *logical_map = REDUCE_EMPTY_MAP_ENTRY;
3217 : 0 : convert_supply_read_to_write(logical_map_meta_req);
3218 : 0 : _reduce_metablock_cache_read_done_update(logical_map_meta_req, reduce_errno);
3219 : : }
3220 : : }
3221 : :
3222 : : static void
3223 : 207292 : _reduce_meta_read_logical_map_for_unmap_full_chunk(void *cb_arg, int reduce_errno)
3224 : : {
3225 : 207292 : struct reduce_meta_request *logical_map_meta_req = cb_arg, *chunk_map_meta_req;
3226 : 207292 : struct spdk_reduce_vol_request *req = logical_map_meta_req->restore_ctx.req_ctx.req;
3227 : :
3228 [ - + + - ]: 207292 : if (logical_map_meta_req->supply_read) {
3229 : : /* the entry process of unmap process */
3230 [ - + ]: 207292 : if (reduce_errno != 0) {
3231 : 0 : _reduce_metablock_cache_read_done_update(logical_map_meta_req, reduce_errno);
3232 : 0 : _reduce_vol_complete_req(req, reduce_errno);
3233 : 0 : return;
3234 : : }
3235 : :
3236 : 207292 : req->chunk_map_index = *(uint64_t *)_reduce_get_meta_elem_addr(logical_map_meta_req);
3237 [ + - ]: 207292 : if (req->chunk_map_index == REDUCE_EMPTY_MAP_ENTRY) {
3238 : 207292 : _reduce_metablock_cache_read_done_update(logical_map_meta_req, reduce_errno);
3239 : 207292 : _reduce_vol_complete_req(req, reduce_errno);
3240 : 207292 : return;
3241 : : }
3242 : :
3243 : : /* get_chunk_map */
3244 : 0 : chunk_map_meta_req = _get_init_meta_req(req->vol, true, REDUCE_MTYPE_CHUNK_MAP,
3245 : 0 : _meta_get_mblksn(&req->vol->params, REDUCE_MTYPE_CHUNK_MAP, req->chunk_map_index),
3246 : 0 : _meta_get_elem_sn_on_mblk(&req->vol->params, REDUCE_MTYPE_CHUNK_MAP, req->chunk_map_index),
3247 : : _reduce_meta_read_chunk_map_for_unmap_full_chunk);
3248 : 0 : chunk_map_meta_req->restore_ctx.req_ctx.req = req;
3249 : 0 : chunk_map_meta_req->restore_ctx.req_ctx.logicalmap_meta_req = logical_map_meta_req;
3250 : 0 : _reduce_metablock_cache_read(chunk_map_meta_req);
3251 : : } else {
3252 : : /* the final process of unmap process */
3253 : 0 : _reduce_metablock_cache_write_done_update(logical_map_meta_req, reduce_errno);
3254 : 0 : _reduce_vol_complete_req(req, reduce_errno);
3255 : : }
3256 : : }
3257 : :
3258 : : static void
3259 : 207292 : _reduce_meta_unmap_full_chunk(void *ctx)
3260 : : {
3261 : 207292 : struct spdk_reduce_vol_request *req = ctx;
3262 : : struct reduce_meta_request *logical_map_meta_req;
3263 : :
3264 : : /* alloc write meta_req, if we find the logical map is EMPTY, don't need write */
3265 : 621876 : logical_map_meta_req = _get_init_meta_req(req->vol, true, REDUCE_MTYPE_LOGICAL_MAP,
3266 : 207292 : _meta_get_mblksn(&req->vol->params, REDUCE_MTYPE_LOGICAL_MAP, req->logical_map_index),
3267 : 207292 : _meta_get_elem_sn_on_mblk(&req->vol->params, REDUCE_MTYPE_LOGICAL_MAP, req->logical_map_index),
3268 : : _reduce_meta_read_logical_map_for_unmap_full_chunk);
3269 : 207292 : logical_map_meta_req->restore_ctx.req_ctx.req = req;
3270 : 207292 : _reduce_metablock_cache_read(logical_map_meta_req);
3271 : 207292 : }
3272 : :
3273 : : static void
3274 : 207292 : _start_unmap_request_full_chunk(void *ctx)
3275 : : {
3276 : 207292 : struct spdk_reduce_vol_request *req = ctx;
3277 : 207292 : struct spdk_reduce_vol *vol = req->vol;
3278 : : uint64_t chunk_map_index;
3279 : :
3280 : 207292 : RB_INSERT(executing_req_tree, &req->vol->executing_requests, req);
3281 : :
3282 [ + - ]: 207292 : if (vol->params.meta_builtin) {
3283 : 207292 : _reduce_meta_unmap_full_chunk(ctx);
3284 : 207292 : return;
3285 : : }
3286 : :
3287 : 0 : chunk_map_index = vol->pm_logical_map[req->logical_map_index];
3288 [ # # ]: 0 : if (chunk_map_index != REDUCE_EMPTY_MAP_ENTRY) {
3289 : 0 : _reduce_vol_reset_chunk(vol, chunk_map_index, NULL);
3290 : 0 : vol->pm_logical_map[req->logical_map_index] = REDUCE_EMPTY_MAP_ENTRY;
3291 : 0 : _reduce_persist(vol, &vol->pm_logical_map[req->logical_map_index], sizeof(uint64_t));
3292 : : }
3293 : 0 : _reduce_vol_complete_req(req, 0);
3294 : : }
3295 : :
3296 : : static void
3297 : 548702 : _reduce_vol_unmap_full_chunk(struct spdk_reduce_vol *vol,
3298 : : uint64_t offset, uint64_t length,
3299 : : spdk_reduce_vol_op_complete cb_fn, void *cb_arg)
3300 : : {
3301 : : struct spdk_reduce_vol_request *req;
3302 : : uint64_t logical_map_index;
3303 : : bool overlapped;
3304 : :
3305 [ - + ]: 548702 : if (_request_spans_chunk_boundary(vol, offset, length)) {
3306 : 0 : cb_fn(cb_arg, -EINVAL);
3307 : 0 : return;
3308 : : }
3309 : :
3310 [ - + ]: 548702 : logical_map_index = offset / vol->logical_blocks_per_chunk;
3311 : 548702 : overlapped = _check_overlap(vol, logical_map_index);
3312 : :
3313 [ + + ]: 548702 : if (!vol->params.meta_builtin) {
3314 [ + - + - ]: 341410 : if (!overlapped && vol->pm_logical_map[logical_map_index] == REDUCE_EMPTY_MAP_ENTRY) {
3315 : : /*
3316 : : * This chunk hasn't been allocated. Nothing needs to be done.
3317 : : */
3318 : 341410 : cb_fn(cb_arg, 0);
3319 : 341410 : return;
3320 : : }
3321 : : }
3322 : :
3323 : 207292 : req = TAILQ_FIRST(&vol->free_requests);
3324 [ - + ]: 207292 : if (req == NULL) {
3325 : 0 : cb_fn(cb_arg, -ENOMEM);
3326 : 0 : return;
3327 : : }
3328 : :
3329 [ + - ]: 207292 : TAILQ_REMOVE(&vol->free_requests, req, tailq);
3330 : 207292 : req->type = REDUCE_IO_UNMAP;
3331 : 207292 : req->vol = vol;
3332 : 207292 : req->iov = NULL;
3333 : 207292 : req->iovcnt = 0;
3334 : 207292 : req->offset = offset;
3335 : 207292 : req->logical_map_index = logical_map_index;
3336 : 207292 : req->length = length;
3337 : 207292 : req->copy_after_decompress = false;
3338 : 207292 : req->cb_fn = cb_fn;
3339 : 207292 : req->cb_arg = cb_arg;
3340 : 207292 : req->reduce_errno = 0;
3341 : :
3342 [ + - ]: 207292 : if (!overlapped) {
3343 : 207292 : _start_unmap_request_full_chunk(req);
3344 : : } else {
3345 : 0 : TAILQ_INSERT_TAIL(&vol->queued_requests, req, tailq);
3346 : : }
3347 : : }
3348 : :
3349 : : struct unmap_partial_chunk_ctx {
3350 : : struct spdk_reduce_vol *vol;
3351 : : struct iovec iov;
3352 : : spdk_reduce_vol_op_complete cb_fn;
3353 : : void *cb_arg;
3354 : : };
3355 : :
3356 : : static void
3357 : 274351 : _reduce_unmap_partial_chunk_complete(void *_ctx, int reduce_errno)
3358 : : {
3359 : 274351 : struct unmap_partial_chunk_ctx *ctx = _ctx;
3360 : :
3361 : 274351 : ctx->cb_fn(ctx->cb_arg, reduce_errno);
3362 : 274351 : free(ctx);
3363 : 274351 : }
3364 : :
3365 : : static void
3366 : 274351 : _reduce_vol_unmap_partial_chunk(struct spdk_reduce_vol *vol, uint64_t offset, uint64_t length,
3367 : : spdk_reduce_vol_op_complete cb_fn, void *cb_arg)
3368 : : {
3369 : : struct unmap_partial_chunk_ctx *ctx;
3370 : :
3371 : 274351 : ctx = calloc(1, sizeof(struct unmap_partial_chunk_ctx));
3372 [ - + ]: 274351 : if (ctx == NULL) {
3373 : 0 : cb_fn(cb_arg, -ENOMEM);
3374 : 0 : return;
3375 : : }
3376 : :
3377 : 274351 : ctx->vol = vol;
3378 : 274351 : ctx->iov.iov_base = g_zero_buf;
3379 : 274351 : ctx->iov.iov_len = length * vol->params.logical_block_size;
3380 : 274351 : ctx->cb_fn = cb_fn;
3381 : 274351 : ctx->cb_arg = cb_arg;
3382 : :
3383 : 274351 : spdk_reduce_vol_writev(vol, &ctx->iov, 1, offset, length, _reduce_unmap_partial_chunk_complete,
3384 : : ctx);
3385 : : }
3386 : :
3387 : : void
3388 : 823053 : spdk_reduce_vol_unmap(struct spdk_reduce_vol *vol,
3389 : : uint64_t offset, uint64_t length,
3390 : : spdk_reduce_vol_op_complete cb_fn, void *cb_arg)
3391 : : {
3392 [ + + ]: 823053 : if (length < vol->logical_blocks_per_chunk) {
3393 : 274351 : _reduce_vol_unmap_partial_chunk(vol, offset, length, cb_fn, cb_arg);
3394 [ + - ]: 548702 : } else if (length == vol->logical_blocks_per_chunk) {
3395 : 548702 : _reduce_vol_unmap_full_chunk(vol, offset, length, cb_fn, cb_arg);
3396 : : } else {
3397 : 0 : cb_fn(cb_arg, -EINVAL);
3398 : : }
3399 : 823053 : }
3400 : :
3401 : : const struct spdk_reduce_vol_params *
3402 : 274351 : spdk_reduce_vol_get_params(struct spdk_reduce_vol *vol)
3403 : : {
3404 : 274351 : return &vol->params;
3405 : : }
3406 : :
3407 : : const char *
3408 : 22 : spdk_reduce_vol_get_pm_path(const struct spdk_reduce_vol *vol)
3409 : : {
3410 : 22 : return vol->pm_file.path;
3411 : : }
3412 : :
3413 : : void
3414 : 0 : spdk_reduce_vol_print_info(struct spdk_reduce_vol *vol)
3415 : : {
3416 : : uint64_t logical_map_size, num_chunks, ttl_chunk_sz;
3417 : : uint32_t struct_size;
3418 : : uint64_t chunk_map_size;
3419 : :
3420 : 0 : SPDK_NOTICELOG("vol info:\n");
3421 : 0 : SPDK_NOTICELOG("\tvol->params.backing_io_unit_size = 0x%x\n", vol->params.backing_io_unit_size);
3422 : 0 : SPDK_NOTICELOG("\tvol->params.logical_block_size = 0x%x\n", vol->params.logical_block_size);
3423 : 0 : SPDK_NOTICELOG("\tvol->params.chunk_size = 0x%x\n", vol->params.chunk_size);
3424 : 0 : SPDK_NOTICELOG("\tvol->params.vol_size = 0x%" PRIx64 "\n", vol->params.vol_size);
3425 : 0 : num_chunks = _get_total_chunks(vol->params.vol_size, vol->params.chunk_size);
3426 : 0 : SPDK_NOTICELOG("\ttotal chunks (including extra) = 0x%" PRIx64 "\n", num_chunks);
3427 [ # # ]: 0 : SPDK_NOTICELOG("\ttotal chunks (excluding extra) = 0x%" PRIx64 "\n",
3428 : : vol->params.vol_size / vol->params.chunk_size);
3429 : 0 : ttl_chunk_sz = _get_pm_total_chunks_size(vol->params.vol_size, vol->params.chunk_size,
3430 : 0 : vol->params.backing_io_unit_size);
3431 : 0 : SPDK_NOTICELOG("\ttotal_chunks_size = 0x%" PRIx64 "\n", ttl_chunk_sz);
3432 : 0 : struct_size = _reduce_vol_get_chunk_struct_size(vol->backing_io_units_per_chunk);
3433 : 0 : SPDK_NOTICELOG("\tchunk_struct_size = 0x%x\n", struct_size);
3434 : :
3435 : 0 : SPDK_NOTICELOG("pmem info:\n");
3436 : 0 : SPDK_NOTICELOG("\tvol->pm_file.size = 0x%" PRIx64 "\n", vol->pm_file.size);
3437 : 0 : SPDK_NOTICELOG("\tvol->pm_file.pm_buf = %p\n", (void *)vol->pm_file.pm_buf);
3438 : 0 : SPDK_NOTICELOG("\tvol->pm_super = %p\n", (void *)vol->pm_super);
3439 : 0 : SPDK_NOTICELOG("\tvol->pm_logical_map = %p\n", (void *)vol->pm_logical_map);
3440 : 0 : logical_map_size = _get_pm_logical_map_size(vol->params.vol_size,
3441 : 0 : vol->params.chunk_size);
3442 : 0 : SPDK_NOTICELOG("\tlogical_map_size = 0x%" PRIx64 "\n", logical_map_size);
3443 : 0 : SPDK_NOTICELOG("\tvol->pm_chunk_maps = %p\n", (void *)vol->pm_chunk_maps);
3444 : 0 : chunk_map_size = _get_pm_total_chunks_size(vol->params.vol_size, vol->params.chunk_size,
3445 : 0 : vol->params.backing_io_unit_size);
3446 : 0 : SPDK_NOTICELOG("\tchunk_map_size = 0x%" PRIx64 "\n", chunk_map_size);
3447 : 0 : }
3448 : :
3449 : 108 : SPDK_LOG_REGISTER_COMPONENT(reduce)
|