LCOV - code coverage report
Current view: top level - lib/blobfs - blobfs.c (source / functions) Hit Total Coverage
Test: ut_cov_unit.info Lines: 1095 1476 74.2 %
Date: 2024-07-10 13:30:00 Functions: 113 131 86.3 %

          Line data    Source code
       1             : /*   SPDX-License-Identifier: BSD-3-Clause
       2             :  *   Copyright (C) 2017 Intel Corporation.
       3             :  *   All rights reserved.
       4             :  */
       5             : 
       6             : #include "spdk/stdinc.h"
       7             : 
       8             : #include "spdk/blobfs.h"
       9             : #include "cache_tree.h"
      10             : 
      11             : #include "spdk/queue.h"
      12             : #include "spdk/thread.h"
      13             : #include "spdk/assert.h"
      14             : #include "spdk/env.h"
      15             : #include "spdk/util.h"
      16             : #include "spdk/log.h"
      17             : #include "spdk/trace.h"
      18             : 
      19             : #include "spdk_internal/trace_defs.h"
      20             : 
      21             : #define BLOBFS_TRACE(file, str, args...) \
      22             :         SPDK_DEBUGLOG(blobfs, "file=%s " str, file->name, ##args)
      23             : 
      24             : #define BLOBFS_TRACE_RW(file, str, args...) \
      25             :         SPDK_DEBUGLOG(blobfs_rw, "file=%s " str, file->name, ##args)
      26             : 
      27             : #define BLOBFS_DEFAULT_CACHE_SIZE (4ULL * 1024 * 1024 * 1024)
      28             : #define SPDK_BLOBFS_DEFAULT_OPTS_CLUSTER_SZ (1024 * 1024)
      29             : 
      30             : #define SPDK_BLOBFS_SIGNATURE   "BLOBFS"
      31             : 
      32             : static uint64_t g_fs_cache_size = BLOBFS_DEFAULT_CACHE_SIZE;
      33             : static struct spdk_mempool *g_cache_pool;
      34             : static TAILQ_HEAD(, spdk_file) g_caches = TAILQ_HEAD_INITIALIZER(g_caches);
      35             : static struct spdk_poller *g_cache_pool_mgmt_poller;
      36             : static struct spdk_thread *g_cache_pool_thread;
      37             : #define BLOBFS_CACHE_POOL_POLL_PERIOD_IN_US 1000ULL
      38             : static int g_fs_count = 0;
      39             : static pthread_mutex_t g_cache_init_lock = PTHREAD_MUTEX_INITIALIZER;
      40             : 
      41           2 : SPDK_TRACE_REGISTER_FN(blobfs_trace, "blobfs", TRACE_GROUP_BLOBFS)
      42             : {
      43           0 :         struct spdk_trace_tpoint_opts opts[] = {
      44             :                 {
      45             :                         "BLOBFS_XATTR_START", TRACE_BLOBFS_XATTR_START,
      46             :                         OWNER_NONE, OBJECT_NONE, 0,
      47             :                         {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
      48             :                 },
      49             :                 {
      50             :                         "BLOBFS_XATTR_END", TRACE_BLOBFS_XATTR_END,
      51             :                         OWNER_NONE, OBJECT_NONE, 0,
      52             :                         {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
      53             :                 },
      54             :                 {
      55             :                         "BLOBFS_OPEN", TRACE_BLOBFS_OPEN,
      56             :                         OWNER_NONE, OBJECT_NONE, 0,
      57             :                         {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
      58             :                 },
      59             :                 {
      60             :                         "BLOBFS_CLOSE", TRACE_BLOBFS_CLOSE,
      61             :                         OWNER_NONE, OBJECT_NONE, 0,
      62             :                         {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
      63             :                 },
      64             :                 {
      65             :                         "BLOBFS_DELETE_START", TRACE_BLOBFS_DELETE_START,
      66             :                         OWNER_NONE, OBJECT_NONE, 0,
      67             :                         {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
      68             :                 },
      69             :                 {
      70             :                         "BLOBFS_DELETE_DONE", TRACE_BLOBFS_DELETE_DONE,
      71             :                         OWNER_NONE, OBJECT_NONE, 0,
      72             :                         {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
      73             :                 }
      74             :         };
      75             : 
      76           0 :         spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts));
      77           0 : }
      78             : 
      79             : void
      80          13 : cache_buffer_free(struct cache_buffer *cache_buffer)
      81             : {
      82          13 :         spdk_mempool_put(g_cache_pool, cache_buffer->buf);
      83          13 :         free(cache_buffer);
      84          13 : }
      85             : 
      86             : #define CACHE_READAHEAD_THRESHOLD       (128 * 1024)
      87             : 
      88             : struct spdk_file {
      89             :         struct spdk_filesystem  *fs;
      90             :         struct spdk_blob        *blob;
      91             :         char                    *name;
      92             :         uint64_t                length;
      93             :         bool                    is_deleted;
      94             :         bool                    open_for_writing;
      95             :         uint64_t                length_flushed;
      96             :         uint64_t                length_xattr;
      97             :         uint64_t                append_pos;
      98             :         uint64_t                seq_byte_count;
      99             :         uint64_t                next_seq_offset;
     100             :         uint32_t                priority;
     101             :         TAILQ_ENTRY(spdk_file)  tailq;
     102             :         spdk_blob_id            blobid;
     103             :         uint32_t                ref_count;
     104             :         pthread_spinlock_t      lock;
     105             :         struct cache_buffer     *last;
     106             :         struct cache_tree       *tree;
     107             :         TAILQ_HEAD(open_requests_head, spdk_fs_request) open_requests;
     108             :         TAILQ_HEAD(sync_requests_head, spdk_fs_request) sync_requests;
     109             :         TAILQ_ENTRY(spdk_file)  cache_tailq;
     110             : };
     111             : 
     112             : struct spdk_deleted_file {
     113             :         spdk_blob_id    id;
     114             :         TAILQ_ENTRY(spdk_deleted_file)  tailq;
     115             : };
     116             : 
     117             : struct spdk_filesystem {
     118             :         struct spdk_blob_store  *bs;
     119             :         TAILQ_HEAD(, spdk_file) files;
     120             :         struct spdk_bs_opts     bs_opts;
     121             :         struct spdk_bs_dev      *bdev;
     122             :         fs_send_request_fn      send_request;
     123             : 
     124             :         struct {
     125             :                 uint32_t                max_ops;
     126             :                 struct spdk_io_channel  *sync_io_channel;
     127             :                 struct spdk_fs_channel  *sync_fs_channel;
     128             :         } sync_target;
     129             : 
     130             :         struct {
     131             :                 uint32_t                max_ops;
     132             :                 struct spdk_io_channel  *md_io_channel;
     133             :                 struct spdk_fs_channel  *md_fs_channel;
     134             :         } md_target;
     135             : 
     136             :         struct {
     137             :                 uint32_t                max_ops;
     138             :         } io_target;
     139             : };
     140             : 
     141             : struct spdk_fs_cb_args {
     142             :         union {
     143             :                 spdk_fs_op_with_handle_complete         fs_op_with_handle;
     144             :                 spdk_fs_op_complete                     fs_op;
     145             :                 spdk_file_op_with_handle_complete       file_op_with_handle;
     146             :                 spdk_file_op_complete                   file_op;
     147             :                 spdk_file_stat_op_complete              stat_op;
     148             :         } fn;
     149             :         void *arg;
     150             :         sem_t *sem;
     151             :         struct spdk_filesystem *fs;
     152             :         struct spdk_file *file;
     153             :         int rc;
     154             :         int *rwerrno;
     155             :         struct iovec *iovs;
     156             :         uint32_t iovcnt;
     157             :         struct iovec iov;
     158             :         union {
     159             :                 struct {
     160             :                         TAILQ_HEAD(, spdk_deleted_file) deleted_files;
     161             :                 } fs_load;
     162             :                 struct {
     163             :                         uint64_t        length;
     164             :                 } truncate;
     165             :                 struct {
     166             :                         struct spdk_io_channel  *channel;
     167             :                         void            *pin_buf;
     168             :                         int             is_read;
     169             :                         off_t           offset;
     170             :                         size_t          length;
     171             :                         uint64_t        start_lba;
     172             :                         uint64_t        num_lba;
     173             :                         uint32_t        blocklen;
     174             :                 } rw;
     175             :                 struct {
     176             :                         const char      *old_name;
     177             :                         const char      *new_name;
     178             :                 } rename;
     179             :                 struct {
     180             :                         struct cache_buffer     *cache_buffer;
     181             :                         uint64_t                length;
     182             :                 } flush;
     183             :                 struct {
     184             :                         struct cache_buffer     *cache_buffer;
     185             :                         uint64_t                length;
     186             :                         uint64_t                offset;
     187             :                 } readahead;
     188             :                 struct {
     189             :                         /* offset of the file when the sync request was made */
     190             :                         uint64_t                        offset;
     191             :                         TAILQ_ENTRY(spdk_fs_request)    tailq;
     192             :                         bool                            xattr_in_progress;
     193             :                         /* length written to the xattr for this file - this should
     194             :                          * always be the same as the offset if only one thread is
     195             :                          * writing to the file, but could differ if multiple threads
     196             :                          * are appending
     197             :                          */
     198             :                         uint64_t                        length;
     199             :                 } sync;
     200             :                 struct {
     201             :                         uint32_t                        num_clusters;
     202             :                 } resize;
     203             :                 struct {
     204             :                         const char      *name;
     205             :                         uint32_t        flags;
     206             :                         TAILQ_ENTRY(spdk_fs_request)    tailq;
     207             :                 } open;
     208             :                 struct {
     209             :                         const char              *name;
     210             :                         struct spdk_blob        *blob;
     211             :                 } create;
     212             :                 struct {
     213             :                         const char      *name;
     214             :                 } delete;
     215             :                 struct {
     216             :                         const char      *name;
     217             :                 } stat;
     218             :         } op;
     219             : };
     220             : 
     221             : static void file_free(struct spdk_file *file);
     222             : static void fs_io_device_unregister(struct spdk_filesystem *fs);
     223             : static void fs_free_io_channels(struct spdk_filesystem *fs);
     224             : 
     225             : void
     226           0 : spdk_fs_opts_init(struct spdk_blobfs_opts *opts)
     227             : {
     228           0 :         opts->cluster_sz = SPDK_BLOBFS_DEFAULT_OPTS_CLUSTER_SZ;
     229           0 : }
     230             : 
     231             : static int _blobfs_cache_pool_reclaim(void *arg);
     232             : 
     233             : static bool
     234           0 : blobfs_cache_pool_need_reclaim(void)
     235             : {
     236             :         size_t count;
     237             : 
     238           0 :         count = spdk_mempool_count(g_cache_pool);
     239             :         /* We define a aggressive policy here as the requirements from db_bench are batched, so start the poller
     240             :          *  when the number of available cache buffer is less than 1/5 of total buffers.
     241             :          */
     242           0 :         if (count > (size_t)g_fs_cache_size / CACHE_BUFFER_SIZE / 5) {
     243           0 :                 return false;
     244             :         }
     245             : 
     246           0 :         return true;
     247             : }
     248             : 
     249             : static void
     250          20 : __start_cache_pool_mgmt(void *ctx)
     251             : {
     252          20 :         assert(g_cache_pool == NULL);
     253             : 
     254          20 :         g_cache_pool = spdk_mempool_create("spdk_fs_cache",
     255             :                                            g_fs_cache_size / CACHE_BUFFER_SIZE,
     256             :                                            CACHE_BUFFER_SIZE,
     257             :                                            SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
     258             :                                            SPDK_ENV_SOCKET_ID_ANY);
     259          20 :         if (!g_cache_pool) {
     260           0 :                 if (spdk_mempool_lookup("spdk_fs_cache") != NULL) {
     261           0 :                         SPDK_ERRLOG("Unable to allocate mempool: already exists\n");
     262           0 :                         SPDK_ERRLOG("Probably running in multiprocess environment, which is "
     263             :                                     "unsupported by the blobfs library\n");
     264             :                 } else {
     265           0 :                         SPDK_ERRLOG("Create mempool failed, you may "
     266             :                                     "increase the memory and try again\n");
     267             :                 }
     268           0 :                 assert(false);
     269             :         }
     270             : 
     271          20 :         assert(g_cache_pool_mgmt_poller == NULL);
     272          20 :         g_cache_pool_mgmt_poller = SPDK_POLLER_REGISTER(_blobfs_cache_pool_reclaim, NULL,
     273             :                                    BLOBFS_CACHE_POOL_POLL_PERIOD_IN_US);
     274          20 : }
     275             : 
     276             : static void
     277          20 : __stop_cache_pool_mgmt(void *ctx)
     278             : {
     279          20 :         spdk_poller_unregister(&g_cache_pool_mgmt_poller);
     280             : 
     281          20 :         assert(g_cache_pool != NULL);
     282          20 :         assert(spdk_mempool_count(g_cache_pool) == g_fs_cache_size / CACHE_BUFFER_SIZE);
     283          20 :         spdk_mempool_free(g_cache_pool);
     284          20 :         g_cache_pool = NULL;
     285             : 
     286          20 :         spdk_thread_exit(g_cache_pool_thread);
     287          20 : }
     288             : 
     289             : static void
     290          20 : initialize_global_cache(void)
     291             : {
     292          20 :         pthread_mutex_lock(&g_cache_init_lock);
     293          20 :         if (g_fs_count == 0) {
     294          20 :                 g_cache_pool_thread = spdk_thread_create("cache_pool_mgmt", NULL);
     295          20 :                 assert(g_cache_pool_thread != NULL);
     296          20 :                 spdk_thread_send_msg(g_cache_pool_thread, __start_cache_pool_mgmt, NULL);
     297             :         }
     298          20 :         g_fs_count++;
     299          20 :         pthread_mutex_unlock(&g_cache_init_lock);
     300          20 : }
     301             : 
     302             : static void
     303          20 : free_global_cache(void)
     304             : {
     305          20 :         pthread_mutex_lock(&g_cache_init_lock);
     306          20 :         g_fs_count--;
     307          20 :         if (g_fs_count == 0) {
     308          20 :                 spdk_thread_send_msg(g_cache_pool_thread, __stop_cache_pool_mgmt, NULL);
     309             :         }
     310          20 :         pthread_mutex_unlock(&g_cache_init_lock);
     311          20 : }
     312             : 
     313             : static uint64_t
     314          21 : __file_get_blob_size(struct spdk_file *file)
     315             : {
     316             :         uint64_t cluster_sz;
     317             : 
     318          21 :         cluster_sz = file->fs->bs_opts.cluster_sz;
     319          21 :         return cluster_sz * spdk_blob_get_num_clusters(file->blob);
     320             : }
     321             : 
     322             : struct spdk_fs_request {
     323             :         struct spdk_fs_cb_args          args;
     324             :         TAILQ_ENTRY(spdk_fs_request)    link;
     325             :         struct spdk_fs_channel          *channel;
     326             : };
     327             : 
     328             : struct spdk_fs_channel {
     329             :         struct spdk_fs_request          *req_mem;
     330             :         TAILQ_HEAD(, spdk_fs_request)   reqs;
     331             :         sem_t                           sem;
     332             :         struct spdk_filesystem          *fs;
     333             :         struct spdk_io_channel          *bs_channel;
     334             :         fs_send_request_fn              send_request;
     335             :         bool                            sync;
     336             :         uint32_t                        outstanding_reqs;
     337             :         pthread_spinlock_t              lock;
     338             : };
     339             : 
     340             : /* For now, this is effectively an alias. But eventually we'll shift
     341             :  * some data members over. */
     342             : struct spdk_fs_thread_ctx {
     343             :         struct spdk_fs_channel  ch;
     344             : };
     345             : 
     346             : static struct spdk_fs_request *
     347         156 : alloc_fs_request_with_iov(struct spdk_fs_channel *channel, uint32_t iovcnt)
     348             : {
     349             :         struct spdk_fs_request *req;
     350         156 :         struct iovec *iovs = NULL;
     351             : 
     352         156 :         if (iovcnt > 1) {
     353           4 :                 iovs = calloc(iovcnt, sizeof(struct iovec));
     354           4 :                 if (!iovs) {
     355           0 :                         return NULL;
     356             :                 }
     357             :         }
     358             : 
     359         156 :         if (channel->sync) {
     360          66 :                 pthread_spin_lock(&channel->lock);
     361             :         }
     362             : 
     363         156 :         req = TAILQ_FIRST(&channel->reqs);
     364         156 :         if (req) {
     365         156 :                 channel->outstanding_reqs++;
     366         156 :                 TAILQ_REMOVE(&channel->reqs, req, link);
     367             :         }
     368             : 
     369         156 :         if (channel->sync) {
     370          66 :                 pthread_spin_unlock(&channel->lock);
     371             :         }
     372             : 
     373         156 :         if (req == NULL) {
     374           0 :                 SPDK_ERRLOG("Cannot allocate req on spdk_fs_channel =%p\n", channel);
     375           0 :                 free(iovs);
     376           0 :                 return NULL;
     377             :         }
     378         156 :         memset(req, 0, sizeof(*req));
     379         156 :         req->channel = channel;
     380         156 :         if (iovcnt > 1) {
     381           4 :                 req->args.iovs = iovs;
     382             :         } else {
     383         152 :                 req->args.iovs = &req->args.iov;
     384             :         }
     385         156 :         req->args.iovcnt = iovcnt;
     386             : 
     387         156 :         return req;
     388             : }
     389             : 
     390             : static struct spdk_fs_request *
     391         148 : alloc_fs_request(struct spdk_fs_channel *channel)
     392             : {
     393         148 :         return alloc_fs_request_with_iov(channel, 0);
     394             : }
     395             : 
     396             : static void
     397         156 : free_fs_request(struct spdk_fs_request *req)
     398             : {
     399         156 :         struct spdk_fs_channel *channel = req->channel;
     400             : 
     401         156 :         if (req->args.iovcnt > 1) {
     402           4 :                 free(req->args.iovs);
     403             :         }
     404             : 
     405         156 :         if (channel->sync) {
     406          66 :                 pthread_spin_lock(&channel->lock);
     407             :         }
     408             : 
     409         156 :         TAILQ_INSERT_HEAD(&req->channel->reqs, req, link);
     410         156 :         channel->outstanding_reqs--;
     411             : 
     412         156 :         if (channel->sync) {
     413          66 :                 pthread_spin_unlock(&channel->lock);
     414             :         }
     415         156 : }
     416             : 
     417             : static int
     418          53 : fs_channel_create(struct spdk_filesystem *fs, struct spdk_fs_channel *channel,
     419             :                   uint32_t max_ops)
     420             : {
     421             :         uint32_t i;
     422             : 
     423          53 :         channel->req_mem = calloc(max_ops, sizeof(struct spdk_fs_request));
     424          53 :         if (!channel->req_mem) {
     425           0 :                 return -1;
     426             :         }
     427             : 
     428          53 :         channel->outstanding_reqs = 0;
     429          53 :         TAILQ_INIT(&channel->reqs);
     430          53 :         sem_init(&channel->sem, 0, 0);
     431             : 
     432       27189 :         for (i = 0; i < max_ops; i++) {
     433       27136 :                 TAILQ_INSERT_TAIL(&channel->reqs, &channel->req_mem[i], link);
     434             :         }
     435             : 
     436          53 :         channel->fs = fs;
     437             : 
     438          53 :         return 0;
     439             : }
     440             : 
     441             : static int
     442          20 : fs_md_channel_create(void *io_device, void *ctx_buf)
     443             : {
     444             :         struct spdk_filesystem          *fs;
     445          20 :         struct spdk_fs_channel          *channel = ctx_buf;
     446             : 
     447          20 :         fs = SPDK_CONTAINEROF(io_device, struct spdk_filesystem, md_target);
     448             : 
     449          20 :         return fs_channel_create(fs, channel, fs->md_target.max_ops);
     450             : }
     451             : 
     452             : static int
     453          20 : fs_sync_channel_create(void *io_device, void *ctx_buf)
     454             : {
     455             :         struct spdk_filesystem          *fs;
     456          20 :         struct spdk_fs_channel          *channel = ctx_buf;
     457             : 
     458          20 :         fs = SPDK_CONTAINEROF(io_device, struct spdk_filesystem, sync_target);
     459             : 
     460          20 :         return fs_channel_create(fs, channel, fs->sync_target.max_ops);
     461             : }
     462             : 
     463             : static int
     464           1 : fs_io_channel_create(void *io_device, void *ctx_buf)
     465             : {
     466             :         struct spdk_filesystem          *fs;
     467           1 :         struct spdk_fs_channel          *channel = ctx_buf;
     468             : 
     469           1 :         fs = SPDK_CONTAINEROF(io_device, struct spdk_filesystem, io_target);
     470             : 
     471           1 :         return fs_channel_create(fs, channel, fs->io_target.max_ops);
     472             : }
     473             : 
     474             : static void
     475          53 : fs_channel_destroy(void *io_device, void *ctx_buf)
     476             : {
     477          53 :         struct spdk_fs_channel *channel = ctx_buf;
     478             : 
     479          53 :         if (channel->outstanding_reqs > 0) {
     480           0 :                 SPDK_ERRLOG("channel freed with %" PRIu32 " outstanding requests!\n",
     481             :                             channel->outstanding_reqs);
     482             :         }
     483             : 
     484          53 :         free(channel->req_mem);
     485          53 :         if (channel->bs_channel != NULL) {
     486          41 :                 spdk_bs_free_io_channel(channel->bs_channel);
     487             :         }
     488          53 : }
     489             : 
     490             : static void
     491           0 : __send_request_direct(fs_request_fn fn, void *arg)
     492             : {
     493           0 :         fn(arg);
     494           0 : }
     495             : 
     496             : static void
     497          20 : common_fs_bs_init(struct spdk_filesystem *fs, struct spdk_blob_store *bs)
     498             : {
     499          20 :         fs->bs = bs;
     500          20 :         fs->bs_opts.cluster_sz = spdk_bs_get_cluster_size(bs);
     501          20 :         fs->md_target.md_fs_channel->bs_channel = spdk_bs_alloc_io_channel(fs->bs);
     502          20 :         fs->md_target.md_fs_channel->send_request = __send_request_direct;
     503          20 :         fs->sync_target.sync_fs_channel->bs_channel = spdk_bs_alloc_io_channel(fs->bs);
     504          20 :         fs->sync_target.sync_fs_channel->send_request = __send_request_direct;
     505             : 
     506          20 :         initialize_global_cache();
     507          20 : }
     508             : 
     509             : static void
     510          18 : init_cb(void *ctx, struct spdk_blob_store *bs, int bserrno)
     511             : {
     512          18 :         struct spdk_fs_request *req = ctx;
     513          18 :         struct spdk_fs_cb_args *args = &req->args;
     514          18 :         struct spdk_filesystem *fs = args->fs;
     515             : 
     516          18 :         if (bserrno == 0) {
     517          18 :                 common_fs_bs_init(fs, bs);
     518             :         } else {
     519           0 :                 free(fs);
     520           0 :                 fs = NULL;
     521             :         }
     522             : 
     523          18 :         args->fn.fs_op_with_handle(args->arg, fs, bserrno);
     524          18 :         free_fs_request(req);
     525          18 : }
     526             : 
     527             : static struct spdk_filesystem *
     528          20 : fs_alloc(struct spdk_bs_dev *dev, fs_send_request_fn send_request_fn)
     529             : {
     530             :         struct spdk_filesystem *fs;
     531             : 
     532          20 :         fs = calloc(1, sizeof(*fs));
     533          20 :         if (fs == NULL) {
     534           0 :                 return NULL;
     535             :         }
     536             : 
     537          20 :         fs->bdev = dev;
     538          20 :         fs->send_request = send_request_fn;
     539          20 :         TAILQ_INIT(&fs->files);
     540             : 
     541          20 :         fs->md_target.max_ops = 512;
     542          20 :         spdk_io_device_register(&fs->md_target, fs_md_channel_create, fs_channel_destroy,
     543             :                                 sizeof(struct spdk_fs_channel), "blobfs_md");
     544          20 :         fs->md_target.md_io_channel = spdk_get_io_channel(&fs->md_target);
     545          20 :         fs->md_target.md_fs_channel = spdk_io_channel_get_ctx(fs->md_target.md_io_channel);
     546             : 
     547          20 :         fs->sync_target.max_ops = 512;
     548          20 :         spdk_io_device_register(&fs->sync_target, fs_sync_channel_create, fs_channel_destroy,
     549             :                                 sizeof(struct spdk_fs_channel), "blobfs_sync");
     550          20 :         fs->sync_target.sync_io_channel = spdk_get_io_channel(&fs->sync_target);
     551          20 :         fs->sync_target.sync_fs_channel = spdk_io_channel_get_ctx(fs->sync_target.sync_io_channel);
     552             : 
     553          20 :         fs->io_target.max_ops = 512;
     554          20 :         spdk_io_device_register(&fs->io_target, fs_io_channel_create, fs_channel_destroy,
     555             :                                 sizeof(struct spdk_fs_channel), "blobfs_io");
     556             : 
     557          20 :         return fs;
     558             : }
     559             : 
     560             : static void
     561          51 : __wake_caller(void *arg, int fserrno)
     562             : {
     563          51 :         struct spdk_fs_cb_args *args = arg;
     564             : 
     565          51 :         if ((args->rwerrno != NULL) && (*(args->rwerrno) == 0) && fserrno) {
     566           0 :                 *(args->rwerrno) = fserrno;
     567             :         }
     568          51 :         args->rc = fserrno;
     569          51 :         sem_post(args->sem);
     570          51 : }
     571             : 
     572             : void
     573          18 : spdk_fs_init(struct spdk_bs_dev *dev, struct spdk_blobfs_opts *opt,
     574             :              fs_send_request_fn send_request_fn,
     575             :              spdk_fs_op_with_handle_complete cb_fn, void *cb_arg)
     576             : {
     577             :         struct spdk_filesystem *fs;
     578             :         struct spdk_fs_request *req;
     579             :         struct spdk_fs_cb_args *args;
     580          18 :         struct spdk_bs_opts opts = {};
     581             : 
     582          18 :         fs = fs_alloc(dev, send_request_fn);
     583          18 :         if (fs == NULL) {
     584           0 :                 cb_fn(cb_arg, NULL, -ENOMEM);
     585           0 :                 return;
     586             :         }
     587             : 
     588          18 :         req = alloc_fs_request(fs->md_target.md_fs_channel);
     589          18 :         if (req == NULL) {
     590           0 :                 fs_free_io_channels(fs);
     591           0 :                 fs_io_device_unregister(fs);
     592           0 :                 cb_fn(cb_arg, NULL, -ENOMEM);
     593           0 :                 return;
     594             :         }
     595             : 
     596          18 :         args = &req->args;
     597          18 :         args->fn.fs_op_with_handle = cb_fn;
     598          18 :         args->arg = cb_arg;
     599          18 :         args->fs = fs;
     600             : 
     601          18 :         spdk_bs_opts_init(&opts, sizeof(opts));
     602          18 :         snprintf(opts.bstype.bstype, sizeof(opts.bstype.bstype), SPDK_BLOBFS_SIGNATURE);
     603          18 :         if (opt) {
     604           0 :                 opts.cluster_sz = opt->cluster_sz;
     605             :         }
     606          18 :         spdk_bs_init(dev, &opts, init_cb, req);
     607             : }
     608             : 
     609             : static struct spdk_file *
     610          18 : file_alloc(struct spdk_filesystem *fs)
     611             : {
     612             :         struct spdk_file *file;
     613             : 
     614          18 :         file = calloc(1, sizeof(*file));
     615          18 :         if (file == NULL) {
     616           0 :                 return NULL;
     617             :         }
     618             : 
     619          18 :         file->tree = calloc(1, sizeof(*file->tree));
     620          18 :         if (file->tree == NULL) {
     621           0 :                 free(file);
     622           0 :                 return NULL;
     623             :         }
     624             : 
     625          18 :         if (pthread_spin_init(&file->lock, 0)) {
     626           0 :                 free(file->tree);
     627           0 :                 free(file);
     628           0 :                 return NULL;
     629             :         }
     630             : 
     631          18 :         file->fs = fs;
     632          18 :         TAILQ_INIT(&file->open_requests);
     633          18 :         TAILQ_INIT(&file->sync_requests);
     634          18 :         TAILQ_INSERT_TAIL(&fs->files, file, tailq);
     635          18 :         file->priority = SPDK_FILE_PRIORITY_LOW;
     636          18 :         return file;
     637             : }
     638             : 
     639             : static void fs_load_done(void *ctx, int bserrno);
     640             : 
     641             : static int
     642           2 : _handle_deleted_files(struct spdk_fs_request *req)
     643             : {
     644           2 :         struct spdk_fs_cb_args *args = &req->args;
     645           2 :         struct spdk_filesystem *fs = args->fs;
     646             : 
     647           2 :         if (!TAILQ_EMPTY(&args->op.fs_load.deleted_files)) {
     648             :                 struct spdk_deleted_file *deleted_file;
     649             : 
     650           0 :                 deleted_file = TAILQ_FIRST(&args->op.fs_load.deleted_files);
     651           0 :                 TAILQ_REMOVE(&args->op.fs_load.deleted_files, deleted_file, tailq);
     652           0 :                 spdk_bs_delete_blob(fs->bs, deleted_file->id, fs_load_done, req);
     653           0 :                 free(deleted_file);
     654           0 :                 return 0;
     655             :         }
     656             : 
     657           2 :         return 1;
     658             : }
     659             : 
     660             : static void
     661           2 : fs_load_done(void *ctx, int bserrno)
     662             : {
     663           2 :         struct spdk_fs_request *req = ctx;
     664           2 :         struct spdk_fs_cb_args *args = &req->args;
     665           2 :         struct spdk_filesystem *fs = args->fs;
     666             : 
     667             :         /* The filesystem has been loaded.  Now check if there are any files that
     668             :          *  were marked for deletion before last unload.  Do not complete the
     669             :          *  fs_load callback until all of them have been deleted on disk.
     670             :          */
     671           2 :         if (_handle_deleted_files(req) == 0) {
     672             :                 /* We found a file that's been marked for deleting but not actually
     673             :                  *  deleted yet.  This function will get called again once the delete
     674             :                  *  operation is completed.
     675             :                  */
     676           0 :                 return;
     677             :         }
     678             : 
     679           2 :         args->fn.fs_op_with_handle(args->arg, fs, 0);
     680           2 :         free_fs_request(req);
     681             : 
     682             : }
     683             : 
     684             : static void
     685           2 : iter_cb(void *ctx, struct spdk_blob *blob, int rc)
     686             : {
     687           2 :         struct spdk_fs_request *req = ctx;
     688           2 :         struct spdk_fs_cb_args *args = &req->args;
     689           2 :         struct spdk_filesystem *fs = args->fs;
     690           2 :         uint64_t *length;
     691           2 :         const char *name;
     692           2 :         uint32_t *is_deleted;
     693           2 :         size_t value_len;
     694             : 
     695           2 :         if (rc < 0) {
     696           0 :                 args->fn.fs_op_with_handle(args->arg, fs, rc);
     697           0 :                 free_fs_request(req);
     698           0 :                 return;
     699             :         }
     700             : 
     701           2 :         rc = spdk_blob_get_xattr_value(blob, "name", (const void **)&name, &value_len);
     702           2 :         if (rc < 0) {
     703           0 :                 args->fn.fs_op_with_handle(args->arg, fs, rc);
     704           0 :                 free_fs_request(req);
     705           0 :                 return;
     706             :         }
     707             : 
     708           2 :         rc = spdk_blob_get_xattr_value(blob, "length", (const void **)&length, &value_len);
     709           2 :         if (rc < 0) {
     710           0 :                 args->fn.fs_op_with_handle(args->arg, fs, rc);
     711           0 :                 free_fs_request(req);
     712           0 :                 return;
     713             :         }
     714             : 
     715           2 :         assert(value_len == 8);
     716             : 
     717             :         /* This file could be deleted last time without close it, then app crashed, so we delete it now */
     718           2 :         rc = spdk_blob_get_xattr_value(blob, "is_deleted", (const void **)&is_deleted, &value_len);
     719           2 :         if (rc < 0) {
     720             :                 struct spdk_file *f;
     721             : 
     722           2 :                 f = file_alloc(fs);
     723           2 :                 if (f == NULL) {
     724           0 :                         SPDK_ERRLOG("Cannot allocate file to handle deleted file on disk\n");
     725           0 :                         args->fn.fs_op_with_handle(args->arg, fs, -ENOMEM);
     726           0 :                         free_fs_request(req);
     727           0 :                         return;
     728             :                 }
     729             : 
     730           2 :                 f->name = strdup(name);
     731           2 :                 if (!f->name) {
     732           0 :                         SPDK_ERRLOG("Cannot allocate memory for file name\n");
     733           0 :                         args->fn.fs_op_with_handle(args->arg, fs, -ENOMEM);
     734           0 :                         free_fs_request(req);
     735           0 :                         file_free(f);
     736           0 :                         return;
     737             :                 }
     738             : 
     739           2 :                 f->blobid = spdk_blob_get_id(blob);
     740           2 :                 f->length = *length;
     741           2 :                 f->length_flushed = *length;
     742           2 :                 f->length_xattr = *length;
     743           2 :                 f->append_pos = *length;
     744           2 :                 SPDK_DEBUGLOG(blobfs, "added file %s length=%ju\n", f->name, f->length);
     745             :         } else {
     746             :                 struct spdk_deleted_file *deleted_file;
     747             : 
     748           0 :                 deleted_file = calloc(1, sizeof(*deleted_file));
     749           0 :                 if (deleted_file == NULL) {
     750           0 :                         args->fn.fs_op_with_handle(args->arg, fs, -ENOMEM);
     751           0 :                         free_fs_request(req);
     752           0 :                         return;
     753             :                 }
     754           0 :                 deleted_file->id = spdk_blob_get_id(blob);
     755           0 :                 TAILQ_INSERT_TAIL(&args->op.fs_load.deleted_files, deleted_file, tailq);
     756             :         }
     757             : }
     758             : 
     759             : static void
     760           2 : load_cb(void *ctx, struct spdk_blob_store *bs, int bserrno)
     761             : {
     762           2 :         struct spdk_fs_request *req = ctx;
     763           2 :         struct spdk_fs_cb_args *args = &req->args;
     764           2 :         struct spdk_filesystem *fs = args->fs;
     765           2 :         struct spdk_bs_type bstype;
     766             :         static const struct spdk_bs_type blobfs_type = {SPDK_BLOBFS_SIGNATURE};
     767             :         static const struct spdk_bs_type zeros;
     768             : 
     769           2 :         if (bserrno != 0) {
     770           0 :                 args->fn.fs_op_with_handle(args->arg, NULL, bserrno);
     771           0 :                 free_fs_request(req);
     772           0 :                 fs_free_io_channels(fs);
     773           0 :                 fs_io_device_unregister(fs);
     774           0 :                 return;
     775             :         }
     776             : 
     777           2 :         bstype = spdk_bs_get_bstype(bs);
     778             : 
     779           2 :         if (!memcmp(&bstype, &zeros, sizeof(bstype))) {
     780           0 :                 SPDK_DEBUGLOG(blobfs, "assigning bstype\n");
     781           0 :                 spdk_bs_set_bstype(bs, blobfs_type);
     782           2 :         } else if (memcmp(&bstype, &blobfs_type, sizeof(bstype))) {
     783           0 :                 SPDK_ERRLOG("not blobfs\n");
     784           0 :                 SPDK_LOGDUMP(blobfs, "bstype", &bstype, sizeof(bstype));
     785           0 :                 args->fn.fs_op_with_handle(args->arg, NULL, -EINVAL);
     786           0 :                 free_fs_request(req);
     787           0 :                 fs_free_io_channels(fs);
     788           0 :                 fs_io_device_unregister(fs);
     789           0 :                 return;
     790             :         }
     791             : 
     792           2 :         common_fs_bs_init(fs, bs);
     793           2 :         fs_load_done(req, 0);
     794             : }
     795             : 
     796             : static void
     797          20 : fs_io_device_unregister(struct spdk_filesystem *fs)
     798             : {
     799          20 :         assert(fs != NULL);
     800          20 :         spdk_io_device_unregister(&fs->md_target, NULL);
     801          20 :         spdk_io_device_unregister(&fs->sync_target, NULL);
     802          20 :         spdk_io_device_unregister(&fs->io_target, NULL);
     803          20 :         free(fs);
     804          20 : }
     805             : 
     806             : static void
     807          20 : fs_free_io_channels(struct spdk_filesystem *fs)
     808             : {
     809          20 :         assert(fs != NULL);
     810          20 :         spdk_fs_free_io_channel(fs->md_target.md_io_channel);
     811          20 :         spdk_fs_free_io_channel(fs->sync_target.sync_io_channel);
     812          20 : }
     813             : 
     814             : void
     815           2 : spdk_fs_load(struct spdk_bs_dev *dev, fs_send_request_fn send_request_fn,
     816             :              spdk_fs_op_with_handle_complete cb_fn, void *cb_arg)
     817             : {
     818             :         struct spdk_filesystem *fs;
     819             :         struct spdk_fs_cb_args *args;
     820             :         struct spdk_fs_request *req;
     821           2 :         struct spdk_bs_opts     bs_opts;
     822             : 
     823           2 :         fs = fs_alloc(dev, send_request_fn);
     824           2 :         if (fs == NULL) {
     825           0 :                 cb_fn(cb_arg, NULL, -ENOMEM);
     826           0 :                 return;
     827             :         }
     828             : 
     829           2 :         req = alloc_fs_request(fs->md_target.md_fs_channel);
     830           2 :         if (req == NULL) {
     831           0 :                 fs_free_io_channels(fs);
     832           0 :                 fs_io_device_unregister(fs);
     833           0 :                 cb_fn(cb_arg, NULL, -ENOMEM);
     834           0 :                 return;
     835             :         }
     836             : 
     837           2 :         args = &req->args;
     838           2 :         args->fn.fs_op_with_handle = cb_fn;
     839           2 :         args->arg = cb_arg;
     840           2 :         args->fs = fs;
     841           2 :         TAILQ_INIT(&args->op.fs_load.deleted_files);
     842           2 :         spdk_bs_opts_init(&bs_opts, sizeof(bs_opts));
     843           2 :         bs_opts.iter_cb_fn = iter_cb;
     844           2 :         bs_opts.iter_cb_arg = req;
     845           2 :         spdk_bs_load(dev, &bs_opts, load_cb, req);
     846             : }
     847             : 
     848             : static void
     849          20 : unload_cb(void *ctx, int bserrno)
     850             : {
     851          20 :         struct spdk_fs_request *req = ctx;
     852          20 :         struct spdk_fs_cb_args *args = &req->args;
     853          20 :         struct spdk_filesystem *fs = args->fs;
     854             :         struct spdk_file *file, *tmp;
     855             : 
     856          26 :         TAILQ_FOREACH_SAFE(file, &fs->files, tailq, tmp) {
     857           6 :                 TAILQ_REMOVE(&fs->files, file, tailq);
     858           6 :                 file_free(file);
     859             :         }
     860             : 
     861          20 :         free_global_cache();
     862             : 
     863          20 :         args->fn.fs_op(args->arg, bserrno);
     864          20 :         free(req);
     865             : 
     866          20 :         fs_io_device_unregister(fs);
     867          20 : }
     868             : 
     869             : void
     870          20 : spdk_fs_unload(struct spdk_filesystem *fs, spdk_fs_op_complete cb_fn, void *cb_arg)
     871             : {
     872             :         struct spdk_fs_request *req;
     873             :         struct spdk_fs_cb_args *args;
     874             : 
     875             :         /*
     876             :          * We must free the md_channel before unloading the blobstore, so just
     877             :          *  allocate this request from the general heap.
     878             :          */
     879          20 :         req = calloc(1, sizeof(*req));
     880          20 :         if (req == NULL) {
     881           0 :                 cb_fn(cb_arg, -ENOMEM);
     882           0 :                 return;
     883             :         }
     884             : 
     885          20 :         args = &req->args;
     886          20 :         args->fn.fs_op = cb_fn;
     887          20 :         args->arg = cb_arg;
     888          20 :         args->fs = fs;
     889             : 
     890          20 :         fs_free_io_channels(fs);
     891          20 :         spdk_bs_unload(fs->bs, unload_cb, req);
     892             : }
     893             : 
     894             : static struct spdk_file *
     895          75 : fs_find_file(struct spdk_filesystem *fs, const char *name)
     896             : {
     897             :         struct spdk_file *file;
     898             : 
     899          82 :         TAILQ_FOREACH(file, &fs->files, tailq) {
     900          48 :                 if (!strncmp(name, file->name, SPDK_FILE_NAME_MAX)) {
     901          41 :                         return file;
     902             :                 }
     903             :         }
     904             : 
     905          34 :         return NULL;
     906             : }
     907             : 
     908             : void
     909           4 : spdk_fs_file_stat_async(struct spdk_filesystem *fs, const char *name,
     910             :                         spdk_file_stat_op_complete cb_fn, void *cb_arg)
     911             : {
     912           4 :         struct spdk_file_stat stat;
     913           4 :         struct spdk_file *f = NULL;
     914             : 
     915           4 :         if (strnlen(name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
     916           0 :                 cb_fn(cb_arg, NULL, -ENAMETOOLONG);
     917           0 :                 return;
     918             :         }
     919             : 
     920           4 :         f = fs_find_file(fs, name);
     921           4 :         if (f != NULL) {
     922           4 :                 stat.blobid = f->blobid;
     923           4 :                 stat.size = f->append_pos >= f->length ? f->append_pos : f->length;
     924           4 :                 cb_fn(cb_arg, &stat, 0);
     925           4 :                 return;
     926             :         }
     927             : 
     928           0 :         cb_fn(cb_arg, NULL, -ENOENT);
     929             : }
     930             : 
     931             : static void
     932           4 : __copy_stat(void *arg, struct spdk_file_stat *stat, int fserrno)
     933             : {
     934           4 :         struct spdk_fs_request *req = arg;
     935           4 :         struct spdk_fs_cb_args *args = &req->args;
     936             : 
     937           4 :         args->rc = fserrno;
     938           4 :         if (fserrno == 0) {
     939           4 :                 memcpy(args->arg, stat, sizeof(*stat));
     940             :         }
     941           4 :         sem_post(args->sem);
     942           4 : }
     943             : 
     944             : static void
     945           4 : __file_stat(void *arg)
     946             : {
     947           4 :         struct spdk_fs_request *req = arg;
     948           4 :         struct spdk_fs_cb_args *args = &req->args;
     949             : 
     950           4 :         spdk_fs_file_stat_async(args->fs, args->op.stat.name,
     951             :                                 args->fn.stat_op, req);
     952           4 : }
     953             : 
     954             : int
     955           4 : spdk_fs_file_stat(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx,
     956             :                   const char *name, struct spdk_file_stat *stat)
     957             : {
     958           4 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
     959             :         struct spdk_fs_request *req;
     960             :         int rc;
     961             : 
     962           4 :         req = alloc_fs_request(channel);
     963           4 :         if (req == NULL) {
     964           0 :                 SPDK_ERRLOG("Cannot allocate stat req on file=%s\n", name);
     965           0 :                 return -ENOMEM;
     966             :         }
     967             : 
     968           4 :         req->args.fs = fs;
     969           4 :         req->args.op.stat.name = name;
     970           4 :         req->args.fn.stat_op = __copy_stat;
     971           4 :         req->args.arg = stat;
     972           4 :         req->args.sem = &channel->sem;
     973           4 :         channel->send_request(__file_stat, req);
     974           4 :         sem_wait(&channel->sem);
     975             : 
     976           4 :         rc = req->args.rc;
     977           4 :         free_fs_request(req);
     978             : 
     979           4 :         return rc;
     980             : }
     981             : 
     982             : static void
     983          16 : fs_create_blob_close_cb(void *ctx, int bserrno)
     984             : {
     985             :         int rc;
     986          16 :         struct spdk_fs_request *req = ctx;
     987          16 :         struct spdk_fs_cb_args *args = &req->args;
     988             : 
     989          16 :         rc = args->rc ? args->rc : bserrno;
     990          16 :         args->fn.file_op(args->arg, rc);
     991          16 :         free_fs_request(req);
     992          16 : }
     993             : 
     994             : static void
     995          16 : fs_create_blob_resize_cb(void *ctx, int bserrno)
     996             : {
     997          16 :         struct spdk_fs_request *req = ctx;
     998          16 :         struct spdk_fs_cb_args *args = &req->args;
     999          16 :         struct spdk_file *f = args->file;
    1000          16 :         struct spdk_blob *blob = args->op.create.blob;
    1001          16 :         uint64_t length = 0;
    1002             : 
    1003          16 :         args->rc = bserrno;
    1004          16 :         if (bserrno) {
    1005           0 :                 spdk_blob_close(blob, fs_create_blob_close_cb, args);
    1006           0 :                 return;
    1007             :         }
    1008             : 
    1009          16 :         spdk_blob_set_xattr(blob, "name", f->name, strlen(f->name) + 1);
    1010          16 :         spdk_blob_set_xattr(blob, "length", &length, sizeof(length));
    1011             : 
    1012          16 :         spdk_blob_close(blob, fs_create_blob_close_cb, args);
    1013             : }
    1014             : 
    1015             : static void
    1016          16 : fs_create_blob_open_cb(void *ctx, struct spdk_blob *blob, int bserrno)
    1017             : {
    1018          16 :         struct spdk_fs_request *req = ctx;
    1019          16 :         struct spdk_fs_cb_args *args = &req->args;
    1020             : 
    1021          16 :         if (bserrno) {
    1022           0 :                 args->fn.file_op(args->arg, bserrno);
    1023           0 :                 free_fs_request(req);
    1024           0 :                 return;
    1025             :         }
    1026             : 
    1027          16 :         args->op.create.blob = blob;
    1028          16 :         spdk_blob_resize(blob, 1, fs_create_blob_resize_cb, req);
    1029             : }
    1030             : 
    1031             : static void
    1032          16 : fs_create_blob_create_cb(void *ctx, spdk_blob_id blobid, int bserrno)
    1033             : {
    1034          16 :         struct spdk_fs_request *req = ctx;
    1035          16 :         struct spdk_fs_cb_args *args = &req->args;
    1036          16 :         struct spdk_file *f = args->file;
    1037             : 
    1038          16 :         if (bserrno) {
    1039           0 :                 args->fn.file_op(args->arg, bserrno);
    1040           0 :                 free_fs_request(req);
    1041           0 :                 return;
    1042             :         }
    1043             : 
    1044          16 :         f->blobid = blobid;
    1045          16 :         spdk_bs_open_blob(f->fs->bs, blobid, fs_create_blob_open_cb, req);
    1046             : }
    1047             : 
    1048             : void
    1049          19 : spdk_fs_create_file_async(struct spdk_filesystem *fs, const char *name,
    1050             :                           spdk_file_op_complete cb_fn, void *cb_arg)
    1051             : {
    1052             :         struct spdk_file *file;
    1053             :         struct spdk_fs_request *req;
    1054             :         struct spdk_fs_cb_args *args;
    1055             : 
    1056          19 :         if (strnlen(name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
    1057           1 :                 cb_fn(cb_arg, -ENAMETOOLONG);
    1058           1 :                 return;
    1059             :         }
    1060             : 
    1061          18 :         file = fs_find_file(fs, name);
    1062          18 :         if (file != NULL) {
    1063           2 :                 cb_fn(cb_arg, -EEXIST);
    1064           2 :                 return;
    1065             :         }
    1066             : 
    1067          16 :         file = file_alloc(fs);
    1068          16 :         if (file == NULL) {
    1069           0 :                 SPDK_ERRLOG("Cannot allocate new file for creation\n");
    1070           0 :                 cb_fn(cb_arg, -ENOMEM);
    1071           0 :                 return;
    1072             :         }
    1073             : 
    1074          16 :         req = alloc_fs_request(fs->md_target.md_fs_channel);
    1075          16 :         if (req == NULL) {
    1076           0 :                 SPDK_ERRLOG("Cannot allocate create async req for file=%s\n", name);
    1077           0 :                 TAILQ_REMOVE(&fs->files, file, tailq);
    1078           0 :                 file_free(file);
    1079           0 :                 cb_fn(cb_arg, -ENOMEM);
    1080           0 :                 return;
    1081             :         }
    1082             : 
    1083          16 :         args = &req->args;
    1084          16 :         args->file = file;
    1085          16 :         args->fn.file_op = cb_fn;
    1086          16 :         args->arg = cb_arg;
    1087             : 
    1088          16 :         file->name = strdup(name);
    1089          16 :         if (!file->name) {
    1090           0 :                 SPDK_ERRLOG("Cannot allocate file->name for file=%s\n", name);
    1091           0 :                 free_fs_request(req);
    1092           0 :                 TAILQ_REMOVE(&fs->files, file, tailq);
    1093           0 :                 file_free(file);
    1094           0 :                 cb_fn(cb_arg, -ENOMEM);
    1095           0 :                 return;
    1096             :         }
    1097          16 :         spdk_bs_create_blob(fs->bs, fs_create_blob_create_cb, args);
    1098             : }
    1099             : 
    1100             : static void
    1101           2 : __fs_create_file_done(void *arg, int fserrno)
    1102             : {
    1103           2 :         struct spdk_fs_request *req = arg;
    1104           2 :         struct spdk_fs_cb_args *args = &req->args;
    1105             : 
    1106           2 :         SPDK_DEBUGLOG(blobfs, "file=%s\n", args->op.create.name);
    1107           2 :         __wake_caller(args, fserrno);
    1108           2 : }
    1109             : 
    1110             : static void
    1111           2 : __fs_create_file(void *arg)
    1112             : {
    1113           2 :         struct spdk_fs_request *req = arg;
    1114           2 :         struct spdk_fs_cb_args *args = &req->args;
    1115             : 
    1116           2 :         SPDK_DEBUGLOG(blobfs, "file=%s\n", args->op.create.name);
    1117           2 :         spdk_fs_create_file_async(args->fs, args->op.create.name, __fs_create_file_done, req);
    1118           2 : }
    1119             : 
    1120             : int
    1121           2 : spdk_fs_create_file(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx, const char *name)
    1122             : {
    1123           2 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    1124             :         struct spdk_fs_request *req;
    1125             :         struct spdk_fs_cb_args *args;
    1126             :         int rc;
    1127             : 
    1128           2 :         SPDK_DEBUGLOG(blobfs, "file=%s\n", name);
    1129             : 
    1130           2 :         req = alloc_fs_request(channel);
    1131           2 :         if (req == NULL) {
    1132           0 :                 SPDK_ERRLOG("Cannot allocate req to create file=%s\n", name);
    1133           0 :                 return -ENOMEM;
    1134             :         }
    1135             : 
    1136           2 :         args = &req->args;
    1137           2 :         args->fs = fs;
    1138           2 :         args->op.create.name = name;
    1139           2 :         args->sem = &channel->sem;
    1140           2 :         fs->send_request(__fs_create_file, req);
    1141           2 :         sem_wait(&channel->sem);
    1142           2 :         rc = args->rc;
    1143           2 :         free_fs_request(req);
    1144             : 
    1145           2 :         return rc;
    1146             : }
    1147             : 
    1148             : static void
    1149          17 : fs_open_blob_done(void *ctx, struct spdk_blob *blob, int bserrno)
    1150             : {
    1151          17 :         struct spdk_fs_request *req = ctx;
    1152          17 :         struct spdk_fs_cb_args *args = &req->args;
    1153          17 :         struct spdk_file *f = args->file;
    1154             : 
    1155          17 :         f->blob = blob;
    1156          34 :         while (!TAILQ_EMPTY(&f->open_requests)) {
    1157          17 :                 req = TAILQ_FIRST(&f->open_requests);
    1158          17 :                 args = &req->args;
    1159          17 :                 TAILQ_REMOVE(&f->open_requests, req, args.op.open.tailq);
    1160          17 :                 spdk_trace_record(TRACE_BLOBFS_OPEN, 0, 0, 0, f->name);
    1161          17 :                 args->fn.file_op_with_handle(args->arg, f, bserrno);
    1162          17 :                 free_fs_request(req);
    1163             :         }
    1164          17 : }
    1165             : 
    1166             : static void
    1167          17 : fs_open_blob_create_cb(void *ctx, int bserrno)
    1168             : {
    1169          17 :         struct spdk_fs_request *req = ctx;
    1170          17 :         struct spdk_fs_cb_args *args = &req->args;
    1171          17 :         struct spdk_file *file = args->file;
    1172          17 :         struct spdk_filesystem *fs = args->fs;
    1173             : 
    1174          17 :         if (file == NULL) {
    1175             :                 /*
    1176             :                  * This is from an open with CREATE flag - the file
    1177             :                  *  is now created so look it up in the file list for this
    1178             :                  *  filesystem.
    1179             :                  */
    1180          13 :                 file = fs_find_file(fs, args->op.open.name);
    1181          13 :                 assert(file != NULL);
    1182          13 :                 args->file = file;
    1183             :         }
    1184             : 
    1185          17 :         file->ref_count++;
    1186          17 :         TAILQ_INSERT_TAIL(&file->open_requests, req, args.op.open.tailq);
    1187          17 :         if (file->ref_count == 1) {
    1188          17 :                 assert(file->blob == NULL);
    1189          17 :                 spdk_bs_open_blob(fs->bs, file->blobid, fs_open_blob_done, req);
    1190           0 :         } else if (file->blob != NULL) {
    1191           0 :                 fs_open_blob_done(req, file->blob, 0);
    1192             :         } else {
    1193             :                 /*
    1194             :                  * The blob open for this file is in progress due to a previous
    1195             :                  *  open request.  When that open completes, it will invoke the
    1196             :                  *  open callback for this request.
    1197             :                  */
    1198             :         }
    1199          17 : }
    1200             : 
    1201             : void
    1202          21 : spdk_fs_open_file_async(struct spdk_filesystem *fs, const char *name, uint32_t flags,
    1203             :                         spdk_file_op_with_handle_complete cb_fn, void *cb_arg)
    1204             : {
    1205          21 :         struct spdk_file *f = NULL;
    1206             :         struct spdk_fs_request *req;
    1207             :         struct spdk_fs_cb_args *args;
    1208             : 
    1209          21 :         if (strnlen(name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
    1210           1 :                 cb_fn(cb_arg, NULL, -ENAMETOOLONG);
    1211           1 :                 return;
    1212             :         }
    1213             : 
    1214          20 :         f = fs_find_file(fs, name);
    1215          20 :         if (f == NULL && !(flags & SPDK_BLOBFS_OPEN_CREATE)) {
    1216           2 :                 cb_fn(cb_arg, NULL, -ENOENT);
    1217           2 :                 return;
    1218             :         }
    1219             : 
    1220          18 :         if (f != NULL && f->is_deleted == true) {
    1221           1 :                 cb_fn(cb_arg, NULL, -ENOENT);
    1222           1 :                 return;
    1223             :         }
    1224             : 
    1225          17 :         req = alloc_fs_request(fs->md_target.md_fs_channel);
    1226          17 :         if (req == NULL) {
    1227           0 :                 SPDK_ERRLOG("Cannot allocate async open req for file=%s\n", name);
    1228           0 :                 cb_fn(cb_arg, NULL, -ENOMEM);
    1229           0 :                 return;
    1230             :         }
    1231             : 
    1232          17 :         args = &req->args;
    1233          17 :         args->fn.file_op_with_handle = cb_fn;
    1234          17 :         args->arg = cb_arg;
    1235          17 :         args->file = f;
    1236          17 :         args->fs = fs;
    1237          17 :         args->op.open.name = name;
    1238             : 
    1239          17 :         if (f == NULL) {
    1240          13 :                 spdk_fs_create_file_async(fs, name, fs_open_blob_create_cb, req);
    1241             :         } else {
    1242           4 :                 fs_open_blob_create_cb(req, 0);
    1243             :         }
    1244             : }
    1245             : 
    1246             : static void
    1247          13 : __fs_open_file_done(void *arg, struct spdk_file *file, int bserrno)
    1248             : {
    1249          13 :         struct spdk_fs_request *req = arg;
    1250          13 :         struct spdk_fs_cb_args *args = &req->args;
    1251             : 
    1252          13 :         args->file = file;
    1253          13 :         SPDK_DEBUGLOG(blobfs, "file=%s\n", args->op.open.name);
    1254          13 :         __wake_caller(args, bserrno);
    1255          13 : }
    1256             : 
    1257             : static void
    1258          13 : __fs_open_file(void *arg)
    1259             : {
    1260          13 :         struct spdk_fs_request *req = arg;
    1261          13 :         struct spdk_fs_cb_args *args = &req->args;
    1262             : 
    1263          13 :         SPDK_DEBUGLOG(blobfs, "file=%s\n", args->op.open.name);
    1264          13 :         spdk_fs_open_file_async(args->fs, args->op.open.name, args->op.open.flags,
    1265             :                                 __fs_open_file_done, req);
    1266          13 : }
    1267             : 
    1268             : int
    1269          13 : spdk_fs_open_file(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx,
    1270             :                   const char *name, uint32_t flags, struct spdk_file **file)
    1271             : {
    1272          13 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    1273             :         struct spdk_fs_request *req;
    1274             :         struct spdk_fs_cb_args *args;
    1275             :         int rc;
    1276             : 
    1277          13 :         SPDK_DEBUGLOG(blobfs, "file=%s\n", name);
    1278             : 
    1279          13 :         req = alloc_fs_request(channel);
    1280          13 :         if (req == NULL) {
    1281           0 :                 SPDK_ERRLOG("Cannot allocate req for opening file=%s\n", name);
    1282           0 :                 return -ENOMEM;
    1283             :         }
    1284             : 
    1285          13 :         args = &req->args;
    1286          13 :         args->fs = fs;
    1287          13 :         args->op.open.name = name;
    1288          13 :         args->op.open.flags = flags;
    1289          13 :         args->sem = &channel->sem;
    1290          13 :         fs->send_request(__fs_open_file, req);
    1291          13 :         sem_wait(&channel->sem);
    1292          13 :         rc = args->rc;
    1293          13 :         if (rc == 0) {
    1294          11 :                 *file = args->file;
    1295             :         } else {
    1296           2 :                 *file = NULL;
    1297             :         }
    1298          13 :         free_fs_request(req);
    1299             : 
    1300          13 :         return rc;
    1301             : }
    1302             : 
    1303             : static void
    1304           2 : fs_rename_blob_close_cb(void *ctx, int bserrno)
    1305             : {
    1306           2 :         struct spdk_fs_request *req = ctx;
    1307           2 :         struct spdk_fs_cb_args *args = &req->args;
    1308             : 
    1309           2 :         args->fn.fs_op(args->arg, bserrno);
    1310           2 :         free_fs_request(req);
    1311           2 : }
    1312             : 
    1313             : static void
    1314           2 : fs_rename_blob_open_cb(void *ctx, struct spdk_blob *blob, int bserrno)
    1315             : {
    1316           2 :         struct spdk_fs_request *req = ctx;
    1317           2 :         struct spdk_fs_cb_args *args = &req->args;
    1318           2 :         const char *new_name = args->op.rename.new_name;
    1319             : 
    1320           2 :         spdk_blob_set_xattr(blob, "name", new_name, strlen(new_name) + 1);
    1321           2 :         spdk_blob_close(blob, fs_rename_blob_close_cb, req);
    1322           2 : }
    1323             : 
    1324             : static void
    1325           2 : _fs_md_rename_file(struct spdk_fs_request *req)
    1326             : {
    1327           2 :         struct spdk_fs_cb_args *args = &req->args;
    1328             :         struct spdk_file *f;
    1329             : 
    1330           2 :         f = fs_find_file(args->fs, args->op.rename.old_name);
    1331           2 :         if (f == NULL) {
    1332           0 :                 args->fn.fs_op(args->arg, -ENOENT);
    1333           0 :                 free_fs_request(req);
    1334           0 :                 return;
    1335             :         }
    1336             : 
    1337           2 :         free(f->name);
    1338           2 :         f->name = strdup(args->op.rename.new_name);
    1339           2 :         if (!f->name) {
    1340           0 :                 SPDK_ERRLOG("Cannot allocate memory for file name\n");
    1341           0 :                 args->fn.fs_op(args->arg, -ENOMEM);
    1342           0 :                 free_fs_request(req);
    1343           0 :                 return;
    1344             :         }
    1345             : 
    1346           2 :         args->file = f;
    1347           2 :         spdk_bs_open_blob(args->fs->bs, f->blobid, fs_rename_blob_open_cb, req);
    1348             : }
    1349             : 
    1350             : static void
    1351           1 : fs_rename_delete_done(void *arg, int fserrno)
    1352             : {
    1353           1 :         _fs_md_rename_file(arg);
    1354           1 : }
    1355             : 
    1356             : void
    1357           2 : spdk_fs_rename_file_async(struct spdk_filesystem *fs,
    1358             :                           const char *old_name, const char *new_name,
    1359             :                           spdk_file_op_complete cb_fn, void *cb_arg)
    1360             : {
    1361             :         struct spdk_file *f;
    1362             :         struct spdk_fs_request *req;
    1363             :         struct spdk_fs_cb_args *args;
    1364             : 
    1365           2 :         SPDK_DEBUGLOG(blobfs, "old=%s new=%s\n", old_name, new_name);
    1366           2 :         if (strnlen(new_name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
    1367           0 :                 cb_fn(cb_arg, -ENAMETOOLONG);
    1368           0 :                 return;
    1369             :         }
    1370             : 
    1371           2 :         req = alloc_fs_request(fs->md_target.md_fs_channel);
    1372           2 :         if (req == NULL) {
    1373           0 :                 SPDK_ERRLOG("Cannot allocate rename async req for renaming file from %s to %s\n", old_name,
    1374             :                             new_name);
    1375           0 :                 cb_fn(cb_arg, -ENOMEM);
    1376           0 :                 return;
    1377             :         }
    1378             : 
    1379           2 :         args = &req->args;
    1380           2 :         args->fn.fs_op = cb_fn;
    1381           2 :         args->fs = fs;
    1382           2 :         args->arg = cb_arg;
    1383           2 :         args->op.rename.old_name = old_name;
    1384           2 :         args->op.rename.new_name = new_name;
    1385             : 
    1386           2 :         f = fs_find_file(fs, new_name);
    1387           2 :         if (f == NULL) {
    1388           1 :                 _fs_md_rename_file(req);
    1389           1 :                 return;
    1390             :         }
    1391             : 
    1392             :         /*
    1393             :          * The rename overwrites an existing file.  So delete the existing file, then
    1394             :          *  do the actual rename.
    1395             :          */
    1396           1 :         spdk_fs_delete_file_async(fs, new_name, fs_rename_delete_done, req);
    1397             : }
    1398             : 
    1399             : static void
    1400           1 : __fs_rename_file_done(void *arg, int fserrno)
    1401             : {
    1402           1 :         struct spdk_fs_request *req = arg;
    1403           1 :         struct spdk_fs_cb_args *args = &req->args;
    1404             : 
    1405           1 :         __wake_caller(args, fserrno);
    1406           1 : }
    1407             : 
    1408             : static void
    1409           1 : __fs_rename_file(void *arg)
    1410             : {
    1411           1 :         struct spdk_fs_request *req = arg;
    1412           1 :         struct spdk_fs_cb_args *args = &req->args;
    1413             : 
    1414           1 :         spdk_fs_rename_file_async(args->fs, args->op.rename.old_name, args->op.rename.new_name,
    1415             :                                   __fs_rename_file_done, req);
    1416           1 : }
    1417             : 
    1418             : int
    1419           1 : spdk_fs_rename_file(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx,
    1420             :                     const char *old_name, const char *new_name)
    1421             : {
    1422           1 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    1423             :         struct spdk_fs_request *req;
    1424             :         struct spdk_fs_cb_args *args;
    1425             :         int rc;
    1426             : 
    1427           1 :         req = alloc_fs_request(channel);
    1428           1 :         if (req == NULL) {
    1429           0 :                 SPDK_ERRLOG("Cannot allocate rename req for file=%s\n", old_name);
    1430           0 :                 return -ENOMEM;
    1431             :         }
    1432             : 
    1433           1 :         args = &req->args;
    1434             : 
    1435           1 :         args->fs = fs;
    1436           1 :         args->op.rename.old_name = old_name;
    1437           1 :         args->op.rename.new_name = new_name;
    1438           1 :         args->sem = &channel->sem;
    1439           1 :         fs->send_request(__fs_rename_file, req);
    1440           1 :         sem_wait(&channel->sem);
    1441           1 :         rc = args->rc;
    1442           1 :         free_fs_request(req);
    1443           1 :         return rc;
    1444             : }
    1445             : 
    1446             : static void
    1447          16 : blob_delete_cb(void *ctx, int bserrno)
    1448             : {
    1449          16 :         struct spdk_fs_request *req = ctx;
    1450          16 :         struct spdk_fs_cb_args *args = &req->args;
    1451             : 
    1452          16 :         args->fn.file_op(args->arg, bserrno);
    1453          16 :         free_fs_request(req);
    1454          16 : }
    1455             : 
    1456             : void
    1457          16 : spdk_fs_delete_file_async(struct spdk_filesystem *fs, const char *name,
    1458             :                           spdk_file_op_complete cb_fn, void *cb_arg)
    1459             : {
    1460             :         struct spdk_file *f;
    1461             :         spdk_blob_id blobid;
    1462             :         struct spdk_fs_request *req;
    1463             :         struct spdk_fs_cb_args *args;
    1464             : 
    1465          16 :         SPDK_DEBUGLOG(blobfs, "file=%s\n", name);
    1466             : 
    1467          16 :         if (strnlen(name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
    1468           0 :                 cb_fn(cb_arg, -ENAMETOOLONG);
    1469           0 :                 return;
    1470             :         }
    1471             : 
    1472          16 :         f = fs_find_file(fs, name);
    1473          16 :         if (f == NULL) {
    1474           2 :                 SPDK_ERRLOG("Cannot find the file=%s to deleted\n", name);
    1475           2 :                 cb_fn(cb_arg, -ENOENT);
    1476           2 :                 return;
    1477             :         }
    1478             : 
    1479          14 :         req = alloc_fs_request(fs->md_target.md_fs_channel);
    1480          14 :         if (req == NULL) {
    1481           0 :                 SPDK_ERRLOG("Cannot allocate the req for the file=%s to deleted\n", name);
    1482           0 :                 cb_fn(cb_arg, -ENOMEM);
    1483           0 :                 return;
    1484             :         }
    1485             : 
    1486          14 :         args = &req->args;
    1487          14 :         args->fn.file_op = cb_fn;
    1488          14 :         args->arg = cb_arg;
    1489             : 
    1490          14 :         if (f->ref_count > 0) {
    1491             :                 /* If the ref > 0, we mark the file as deleted and delete it when we close it. */
    1492           2 :                 f->is_deleted = true;
    1493           2 :                 spdk_blob_set_xattr(f->blob, "is_deleted", &f->is_deleted, sizeof(bool));
    1494           2 :                 spdk_blob_sync_md(f->blob, blob_delete_cb, req);
    1495           2 :                 return;
    1496             :         }
    1497             : 
    1498          12 :         blobid = f->blobid;
    1499          12 :         TAILQ_REMOVE(&fs->files, f, tailq);
    1500             : 
    1501          12 :         file_free(f);
    1502             : 
    1503          12 :         spdk_bs_delete_blob(fs->bs, blobid, blob_delete_cb, req);
    1504             : }
    1505             : 
    1506             : static void
    1507           8 : __fs_delete_file_done(void *arg, int fserrno)
    1508             : {
    1509           8 :         struct spdk_fs_request *req = arg;
    1510           8 :         struct spdk_fs_cb_args *args = &req->args;
    1511             : 
    1512           8 :         spdk_trace_record(TRACE_BLOBFS_DELETE_DONE, 0, 0, 0, args->op.delete.name);
    1513           8 :         __wake_caller(args, fserrno);
    1514           8 : }
    1515             : 
    1516             : static void
    1517           8 : __fs_delete_file(void *arg)
    1518             : {
    1519           8 :         struct spdk_fs_request *req = arg;
    1520           8 :         struct spdk_fs_cb_args *args = &req->args;
    1521             : 
    1522           8 :         spdk_trace_record(TRACE_BLOBFS_DELETE_START, 0, 0, 0, args->op.delete.name);
    1523           8 :         spdk_fs_delete_file_async(args->fs, args->op.delete.name, __fs_delete_file_done, req);
    1524           8 : }
    1525             : 
    1526             : int
    1527           8 : spdk_fs_delete_file(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx,
    1528             :                     const char *name)
    1529             : {
    1530           8 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    1531             :         struct spdk_fs_request *req;
    1532             :         struct spdk_fs_cb_args *args;
    1533             :         int rc;
    1534             : 
    1535           8 :         req = alloc_fs_request(channel);
    1536           8 :         if (req == NULL) {
    1537           0 :                 SPDK_DEBUGLOG(blobfs, "Cannot allocate req to delete file=%s\n", name);
    1538           0 :                 return -ENOMEM;
    1539             :         }
    1540             : 
    1541           8 :         args = &req->args;
    1542           8 :         args->fs = fs;
    1543           8 :         args->op.delete.name = name;
    1544           8 :         args->sem = &channel->sem;
    1545           8 :         fs->send_request(__fs_delete_file, req);
    1546           8 :         sem_wait(&channel->sem);
    1547           8 :         rc = args->rc;
    1548           8 :         free_fs_request(req);
    1549             : 
    1550           8 :         return rc;
    1551             : }
    1552             : 
    1553             : spdk_fs_iter
    1554           1 : spdk_fs_iter_first(struct spdk_filesystem *fs)
    1555             : {
    1556             :         struct spdk_file *f;
    1557             : 
    1558           1 :         f = TAILQ_FIRST(&fs->files);
    1559           1 :         return f;
    1560             : }
    1561             : 
    1562             : spdk_fs_iter
    1563           1 : spdk_fs_iter_next(spdk_fs_iter iter)
    1564             : {
    1565           1 :         struct spdk_file *f = iter;
    1566             : 
    1567           1 :         if (f == NULL) {
    1568           0 :                 return NULL;
    1569             :         }
    1570             : 
    1571           1 :         f = TAILQ_NEXT(f, tailq);
    1572           1 :         return f;
    1573             : }
    1574             : 
    1575             : const char *
    1576           2 : spdk_file_get_name(struct spdk_file *file)
    1577             : {
    1578           2 :         return file->name;
    1579             : }
    1580             : 
    1581             : uint64_t
    1582           6 : spdk_file_get_length(struct spdk_file *file)
    1583             : {
    1584             :         uint64_t length;
    1585             : 
    1586           6 :         assert(file != NULL);
    1587             : 
    1588           6 :         length = file->append_pos >= file->length ? file->append_pos : file->length;
    1589           6 :         SPDK_DEBUGLOG(blobfs, "file=%s length=0x%jx\n", file->name, length);
    1590           6 :         return length;
    1591             : }
    1592             : 
    1593             : static void
    1594           8 : fs_truncate_complete_cb(void *ctx, int bserrno)
    1595             : {
    1596           8 :         struct spdk_fs_request *req = ctx;
    1597           8 :         struct spdk_fs_cb_args *args = &req->args;
    1598             : 
    1599           8 :         args->fn.file_op(args->arg, bserrno);
    1600           8 :         free_fs_request(req);
    1601           8 : }
    1602             : 
    1603             : static void
    1604           8 : fs_truncate_resize_cb(void *ctx, int bserrno)
    1605             : {
    1606           8 :         struct spdk_fs_request *req = ctx;
    1607           8 :         struct spdk_fs_cb_args *args = &req->args;
    1608           8 :         struct spdk_file *file = args->file;
    1609           8 :         uint64_t *length = &args->op.truncate.length;
    1610             : 
    1611           8 :         if (bserrno) {
    1612           0 :                 args->fn.file_op(args->arg, bserrno);
    1613           0 :                 free_fs_request(req);
    1614           0 :                 return;
    1615             :         }
    1616             : 
    1617           8 :         spdk_blob_set_xattr(file->blob, "length", length, sizeof(*length));
    1618             : 
    1619           8 :         file->length = *length;
    1620           8 :         if (file->append_pos > file->length) {
    1621           0 :                 file->append_pos = file->length;
    1622             :         }
    1623             : 
    1624           8 :         spdk_blob_sync_md(file->blob, fs_truncate_complete_cb, req);
    1625             : }
    1626             : 
    1627             : static uint64_t
    1628           8 : __bytes_to_clusters(uint64_t length, uint64_t cluster_sz)
    1629             : {
    1630           8 :         return (length + cluster_sz - 1) / cluster_sz;
    1631             : }
    1632             : 
    1633             : void
    1634           9 : spdk_file_truncate_async(struct spdk_file *file, uint64_t length,
    1635             :                          spdk_file_op_complete cb_fn, void *cb_arg)
    1636             : {
    1637             :         struct spdk_filesystem *fs;
    1638             :         size_t num_clusters;
    1639             :         struct spdk_fs_request *req;
    1640             :         struct spdk_fs_cb_args *args;
    1641             : 
    1642           9 :         SPDK_DEBUGLOG(blobfs, "file=%s old=0x%jx new=0x%jx\n", file->name, file->length, length);
    1643           9 :         if (length == file->length) {
    1644           1 :                 cb_fn(cb_arg, 0);
    1645           1 :                 return;
    1646             :         }
    1647             : 
    1648           8 :         req = alloc_fs_request(file->fs->md_target.md_fs_channel);
    1649           8 :         if (req == NULL) {
    1650           0 :                 cb_fn(cb_arg, -ENOMEM);
    1651           0 :                 return;
    1652             :         }
    1653             : 
    1654           8 :         args = &req->args;
    1655           8 :         args->fn.file_op = cb_fn;
    1656           8 :         args->arg = cb_arg;
    1657           8 :         args->file = file;
    1658           8 :         args->op.truncate.length = length;
    1659           8 :         fs = file->fs;
    1660             : 
    1661           8 :         num_clusters = __bytes_to_clusters(length, fs->bs_opts.cluster_sz);
    1662             : 
    1663           8 :         spdk_blob_resize(file->blob, num_clusters, fs_truncate_resize_cb, req);
    1664             : }
    1665             : 
    1666             : static void
    1667           3 : __truncate(void *arg)
    1668             : {
    1669           3 :         struct spdk_fs_request *req = arg;
    1670           3 :         struct spdk_fs_cb_args *args = &req->args;
    1671             : 
    1672           3 :         spdk_file_truncate_async(args->file, args->op.truncate.length,
    1673             :                                  args->fn.file_op, args);
    1674           3 : }
    1675             : 
    1676             : int
    1677           3 : spdk_file_truncate(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx,
    1678             :                    uint64_t length)
    1679             : {
    1680           3 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    1681             :         struct spdk_fs_request *req;
    1682             :         struct spdk_fs_cb_args *args;
    1683             :         int rc;
    1684             : 
    1685           3 :         req = alloc_fs_request(channel);
    1686           3 :         if (req == NULL) {
    1687           0 :                 return -ENOMEM;
    1688             :         }
    1689             : 
    1690           3 :         args = &req->args;
    1691             : 
    1692           3 :         args->file = file;
    1693           3 :         args->op.truncate.length = length;
    1694           3 :         args->fn.file_op = __wake_caller;
    1695           3 :         args->sem = &channel->sem;
    1696             : 
    1697           3 :         channel->send_request(__truncate, req);
    1698           3 :         sem_wait(&channel->sem);
    1699           3 :         rc = args->rc;
    1700           3 :         free_fs_request(req);
    1701             : 
    1702           3 :         return rc;
    1703             : }
    1704             : 
    1705             : static void
    1706           7 : __rw_done(void *ctx, int bserrno)
    1707             : {
    1708           7 :         struct spdk_fs_request *req = ctx;
    1709           7 :         struct spdk_fs_cb_args *args = &req->args;
    1710             : 
    1711           7 :         spdk_free(args->op.rw.pin_buf);
    1712           7 :         args->fn.file_op(args->arg, bserrno);
    1713           7 :         free_fs_request(req);
    1714           7 : }
    1715             : 
    1716             : static void
    1717           6 : __read_done(void *ctx, int bserrno)
    1718             : {
    1719           6 :         struct spdk_fs_request *req = ctx;
    1720           6 :         struct spdk_fs_cb_args *args = &req->args;
    1721             :         void *buf;
    1722             : 
    1723           6 :         if (bserrno) {
    1724           0 :                 __rw_done(req, bserrno);
    1725           0 :                 return;
    1726             :         }
    1727             : 
    1728           6 :         assert(req != NULL);
    1729           6 :         buf = (void *)((uintptr_t)args->op.rw.pin_buf + (args->op.rw.offset & (args->op.rw.blocklen - 1)));
    1730           6 :         if (args->op.rw.is_read) {
    1731           3 :                 spdk_copy_buf_to_iovs(args->iovs, args->iovcnt, buf, args->op.rw.length);
    1732           3 :                 __rw_done(req, 0);
    1733             :         } else {
    1734           3 :                 spdk_copy_iovs_to_buf(buf, args->op.rw.length, args->iovs, args->iovcnt);
    1735           3 :                 spdk_blob_io_write(args->file->blob, args->op.rw.channel,
    1736             :                                    args->op.rw.pin_buf,
    1737             :                                    args->op.rw.start_lba, args->op.rw.num_lba,
    1738             :                                    __rw_done, req);
    1739             :         }
    1740             : }
    1741             : 
    1742             : static void
    1743           6 : __do_blob_read(void *ctx, int fserrno)
    1744             : {
    1745           6 :         struct spdk_fs_request *req = ctx;
    1746           6 :         struct spdk_fs_cb_args *args = &req->args;
    1747             : 
    1748           6 :         if (fserrno) {
    1749           0 :                 __rw_done(req, fserrno);
    1750           0 :                 return;
    1751             :         }
    1752           6 :         spdk_blob_io_read(args->file->blob, args->op.rw.channel,
    1753             :                           args->op.rw.pin_buf,
    1754             :                           args->op.rw.start_lba, args->op.rw.num_lba,
    1755             :                           __read_done, req);
    1756             : }
    1757             : 
    1758             : static void
    1759          17 : __get_page_parameters(struct spdk_file *file, uint64_t offset, uint64_t length,
    1760             :                       uint64_t *start_lba, uint32_t *lba_size, uint64_t *num_lba)
    1761             : {
    1762             :         uint64_t end_lba;
    1763             : 
    1764          17 :         *lba_size = spdk_bs_get_io_unit_size(file->fs->bs);
    1765          17 :         *start_lba = offset / *lba_size;
    1766          17 :         end_lba = (offset + length - 1) / *lba_size;
    1767          17 :         *num_lba = (end_lba - *start_lba + 1);
    1768          17 : }
    1769             : 
    1770             : static bool
    1771           1 : __is_lba_aligned(struct spdk_file *file, uint64_t offset, uint64_t length)
    1772             : {
    1773           1 :         uint32_t lba_size = spdk_bs_get_io_unit_size(file->fs->bs);
    1774             : 
    1775           1 :         if ((offset % lba_size == 0) && (length % lba_size == 0)) {
    1776           1 :                 return true;
    1777             :         }
    1778             : 
    1779           0 :         return false;
    1780             : }
    1781             : 
    1782             : static void
    1783           7 : _fs_request_setup_iovs(struct spdk_fs_request *req, struct iovec *iovs, uint32_t iovcnt)
    1784             : {
    1785             :         uint32_t i;
    1786             : 
    1787          18 :         for (i = 0; i < iovcnt; i++) {
    1788          11 :                 req->args.iovs[i].iov_base = iovs[i].iov_base;
    1789          11 :                 req->args.iovs[i].iov_len = iovs[i].iov_len;
    1790             :         }
    1791           7 : }
    1792             : 
    1793             : static void
    1794           7 : __readvwritev(struct spdk_file *file, struct spdk_io_channel *_channel,
    1795             :               struct iovec *iovs, uint32_t iovcnt, uint64_t offset, uint64_t length,
    1796             :               spdk_file_op_complete cb_fn, void *cb_arg, int is_read)
    1797             : {
    1798             :         struct spdk_fs_request *req;
    1799             :         struct spdk_fs_cb_args *args;
    1800           7 :         struct spdk_fs_channel *channel = spdk_io_channel_get_ctx(_channel);
    1801           7 :         uint64_t start_lba, num_lba, pin_buf_length;
    1802           7 :         uint32_t lba_size;
    1803             : 
    1804           7 :         if (is_read && offset + length > file->length) {
    1805           0 :                 cb_fn(cb_arg, -EINVAL);
    1806           0 :                 return;
    1807             :         }
    1808             : 
    1809           7 :         req = alloc_fs_request_with_iov(channel, iovcnt);
    1810           7 :         if (req == NULL) {
    1811           0 :                 cb_fn(cb_arg, -ENOMEM);
    1812           0 :                 return;
    1813             :         }
    1814             : 
    1815           7 :         __get_page_parameters(file, offset, length, &start_lba, &lba_size, &num_lba);
    1816             : 
    1817           7 :         args = &req->args;
    1818           7 :         args->fn.file_op = cb_fn;
    1819           7 :         args->arg = cb_arg;
    1820           7 :         args->file = file;
    1821           7 :         args->op.rw.channel = channel->bs_channel;
    1822           7 :         _fs_request_setup_iovs(req, iovs, iovcnt);
    1823           7 :         args->op.rw.is_read = is_read;
    1824           7 :         args->op.rw.offset = offset;
    1825           7 :         args->op.rw.blocklen = lba_size;
    1826             : 
    1827           7 :         pin_buf_length = num_lba * lba_size;
    1828           7 :         args->op.rw.length = pin_buf_length;
    1829           7 :         args->op.rw.pin_buf = spdk_malloc(pin_buf_length, lba_size, NULL,
    1830             :                                           SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
    1831           7 :         if (args->op.rw.pin_buf == NULL) {
    1832           0 :                 SPDK_DEBUGLOG(blobfs, "Failed to allocate buf for: file=%s offset=%jx length=%jx\n",
    1833             :                               file->name, offset, length);
    1834           0 :                 free_fs_request(req);
    1835           0 :                 cb_fn(cb_arg, -ENOMEM);
    1836           0 :                 return;
    1837             :         }
    1838             : 
    1839           7 :         args->op.rw.start_lba = start_lba;
    1840           7 :         args->op.rw.num_lba = num_lba;
    1841             : 
    1842           7 :         if (!is_read && file->length < offset + length) {
    1843           3 :                 spdk_file_truncate_async(file, offset + length, __do_blob_read, req);
    1844           4 :         } else if (!is_read && __is_lba_aligned(file, offset, length)) {
    1845           1 :                 spdk_copy_iovs_to_buf(args->op.rw.pin_buf, args->op.rw.length, args->iovs, args->iovcnt);
    1846           1 :                 spdk_blob_io_write(args->file->blob, args->op.rw.channel,
    1847             :                                    args->op.rw.pin_buf,
    1848             :                                    args->op.rw.start_lba, args->op.rw.num_lba,
    1849             :                                    __rw_done, req);
    1850             :         } else {
    1851           3 :                 __do_blob_read(req, 0);
    1852             :         }
    1853             : }
    1854             : 
    1855             : static void
    1856           3 : __readwrite(struct spdk_file *file, struct spdk_io_channel *channel,
    1857             :             void *payload, uint64_t offset, uint64_t length,
    1858             :             spdk_file_op_complete cb_fn, void *cb_arg, int is_read)
    1859             : {
    1860           3 :         struct iovec iov;
    1861             : 
    1862           3 :         iov.iov_base = payload;
    1863           3 :         iov.iov_len = (size_t)length;
    1864             : 
    1865           3 :         __readvwritev(file, channel, &iov, 1, offset, length, cb_fn, cb_arg, is_read);
    1866           3 : }
    1867             : 
    1868             : void
    1869           2 : spdk_file_write_async(struct spdk_file *file, struct spdk_io_channel *channel,
    1870             :                       void *payload, uint64_t offset, uint64_t length,
    1871             :                       spdk_file_op_complete cb_fn, void *cb_arg)
    1872             : {
    1873           2 :         __readwrite(file, channel, payload, offset, length, cb_fn, cb_arg, 0);
    1874           2 : }
    1875             : 
    1876             : void
    1877           2 : spdk_file_writev_async(struct spdk_file *file, struct spdk_io_channel *channel,
    1878             :                        struct iovec *iovs, uint32_t iovcnt, uint64_t offset, uint64_t length,
    1879             :                        spdk_file_op_complete cb_fn, void *cb_arg)
    1880             : {
    1881           2 :         SPDK_DEBUGLOG(blobfs, "file=%s offset=%jx length=%jx\n",
    1882             :                       file->name, offset, length);
    1883             : 
    1884           2 :         __readvwritev(file, channel, iovs, iovcnt, offset, length, cb_fn, cb_arg, 0);
    1885           2 : }
    1886             : 
    1887             : void
    1888           1 : spdk_file_read_async(struct spdk_file *file, struct spdk_io_channel *channel,
    1889             :                      void *payload, uint64_t offset, uint64_t length,
    1890             :                      spdk_file_op_complete cb_fn, void *cb_arg)
    1891             : {
    1892           1 :         SPDK_DEBUGLOG(blobfs, "file=%s offset=%jx length=%jx\n",
    1893             :                       file->name, offset, length);
    1894             : 
    1895           1 :         __readwrite(file, channel, payload, offset, length, cb_fn, cb_arg, 1);
    1896           1 : }
    1897             : 
    1898             : void
    1899           2 : spdk_file_readv_async(struct spdk_file *file, struct spdk_io_channel *channel,
    1900             :                       struct iovec *iovs, uint32_t iovcnt, uint64_t offset, uint64_t length,
    1901             :                       spdk_file_op_complete cb_fn, void *cb_arg)
    1902             : {
    1903           2 :         SPDK_DEBUGLOG(blobfs, "file=%s offset=%jx length=%jx\n",
    1904             :                       file->name, offset, length);
    1905             : 
    1906           2 :         __readvwritev(file, channel, iovs, iovcnt, offset, length, cb_fn, cb_arg, 1);
    1907           2 : }
    1908             : 
    1909             : struct spdk_io_channel *
    1910           1 : spdk_fs_alloc_io_channel(struct spdk_filesystem *fs)
    1911             : {
    1912             :         struct spdk_io_channel *io_channel;
    1913             :         struct spdk_fs_channel *fs_channel;
    1914             : 
    1915           1 :         io_channel = spdk_get_io_channel(&fs->io_target);
    1916           1 :         fs_channel = spdk_io_channel_get_ctx(io_channel);
    1917           1 :         fs_channel->bs_channel = spdk_bs_alloc_io_channel(fs->bs);
    1918           1 :         fs_channel->send_request = __send_request_direct;
    1919             : 
    1920           1 :         return io_channel;
    1921             : }
    1922             : 
    1923             : void
    1924          41 : spdk_fs_free_io_channel(struct spdk_io_channel *channel)
    1925             : {
    1926          41 :         spdk_put_io_channel(channel);
    1927          41 : }
    1928             : 
    1929             : struct spdk_fs_thread_ctx *
    1930          12 : spdk_fs_alloc_thread_ctx(struct spdk_filesystem *fs)
    1931             : {
    1932             :         struct spdk_fs_thread_ctx *ctx;
    1933             : 
    1934          12 :         ctx = calloc(1, sizeof(*ctx));
    1935          12 :         if (!ctx) {
    1936           0 :                 return NULL;
    1937             :         }
    1938             : 
    1939          12 :         if (pthread_spin_init(&ctx->ch.lock, 0)) {
    1940           0 :                 free(ctx);
    1941           0 :                 return NULL;
    1942             :         }
    1943             : 
    1944          12 :         fs_channel_create(fs, &ctx->ch, 512);
    1945             : 
    1946          12 :         ctx->ch.send_request = fs->send_request;
    1947          12 :         ctx->ch.sync = 1;
    1948             : 
    1949          12 :         return ctx;
    1950             : }
    1951             : 
    1952             : 
    1953             : void
    1954          12 : spdk_fs_free_thread_ctx(struct spdk_fs_thread_ctx *ctx)
    1955             : {
    1956          12 :         assert(ctx->ch.sync == 1);
    1957             : 
    1958             :         while (true) {
    1959          12 :                 pthread_spin_lock(&ctx->ch.lock);
    1960          12 :                 if (ctx->ch.outstanding_reqs == 0) {
    1961          12 :                         pthread_spin_unlock(&ctx->ch.lock);
    1962          12 :                         break;
    1963             :                 }
    1964           0 :                 pthread_spin_unlock(&ctx->ch.lock);
    1965           0 :                 usleep(1000);
    1966             :         }
    1967             : 
    1968          12 :         fs_channel_destroy(NULL, &ctx->ch);
    1969          12 :         free(ctx);
    1970          12 : }
    1971             : 
    1972             : int
    1973           0 : spdk_fs_set_cache_size(uint64_t size_in_mb)
    1974             : {
    1975             :         /* setting g_fs_cache_size is only permitted if cache pool
    1976             :          * is already freed or hasn't been initialized
    1977             :          */
    1978           0 :         if (g_cache_pool != NULL) {
    1979           0 :                 return -EPERM;
    1980             :         }
    1981             : 
    1982           0 :         g_fs_cache_size = size_in_mb * 1024 * 1024;
    1983             : 
    1984           0 :         return 0;
    1985             : }
    1986             : 
    1987             : uint64_t
    1988           0 : spdk_fs_get_cache_size(void)
    1989             : {
    1990           0 :         return g_fs_cache_size / (1024 * 1024);
    1991             : }
    1992             : 
    1993             : static void __file_flush(void *ctx);
    1994             : 
    1995             : /* Try to free some cache buffers from this file.
    1996             :  */
    1997             : static int
    1998           0 : reclaim_cache_buffers(struct spdk_file *file)
    1999             : {
    2000             :         int rc;
    2001             : 
    2002           0 :         BLOBFS_TRACE(file, "free=%s\n", file->name);
    2003             : 
    2004             :         /* The function is safe to be called with any threads, while the file
    2005             :          * lock maybe locked by other thread for now, so try to get the file
    2006             :          * lock here.
    2007             :          */
    2008           0 :         rc = pthread_spin_trylock(&file->lock);
    2009           0 :         if (rc != 0) {
    2010           0 :                 return -1;
    2011             :         }
    2012             : 
    2013           0 :         if (file->tree->present_mask == 0) {
    2014           0 :                 pthread_spin_unlock(&file->lock);
    2015           0 :                 return -1;
    2016             :         }
    2017           0 :         tree_free_buffers(file->tree);
    2018             : 
    2019           0 :         TAILQ_REMOVE(&g_caches, file, cache_tailq);
    2020             :         /* If not freed, put it in the end of the queue */
    2021           0 :         if (file->tree->present_mask != 0) {
    2022           0 :                 TAILQ_INSERT_TAIL(&g_caches, file, cache_tailq);
    2023             :         } else {
    2024           0 :                 file->last = NULL;
    2025             :         }
    2026           0 :         pthread_spin_unlock(&file->lock);
    2027             : 
    2028           0 :         return 0;
    2029             : }
    2030             : 
    2031             : static int
    2032           0 : _blobfs_cache_pool_reclaim(void *arg)
    2033             : {
    2034             :         struct spdk_file *file, *tmp;
    2035             :         int rc;
    2036             : 
    2037           0 :         if (!blobfs_cache_pool_need_reclaim()) {
    2038           0 :                 return SPDK_POLLER_IDLE;
    2039             :         }
    2040             : 
    2041           0 :         TAILQ_FOREACH_SAFE(file, &g_caches, cache_tailq, tmp) {
    2042           0 :                 if (!file->open_for_writing &&
    2043           0 :                     file->priority == SPDK_FILE_PRIORITY_LOW) {
    2044           0 :                         rc = reclaim_cache_buffers(file);
    2045           0 :                         if (rc < 0) {
    2046           0 :                                 continue;
    2047             :                         }
    2048           0 :                         if (!blobfs_cache_pool_need_reclaim()) {
    2049           0 :                                 return SPDK_POLLER_BUSY;
    2050             :                         }
    2051           0 :                         break;
    2052             :                 }
    2053             :         }
    2054             : 
    2055           0 :         TAILQ_FOREACH_SAFE(file, &g_caches, cache_tailq, tmp) {
    2056           0 :                 if (!file->open_for_writing) {
    2057           0 :                         rc = reclaim_cache_buffers(file);
    2058           0 :                         if (rc < 0) {
    2059           0 :                                 continue;
    2060             :                         }
    2061           0 :                         if (!blobfs_cache_pool_need_reclaim()) {
    2062           0 :                                 return SPDK_POLLER_BUSY;
    2063             :                         }
    2064           0 :                         break;
    2065             :                 }
    2066             :         }
    2067             : 
    2068           0 :         TAILQ_FOREACH_SAFE(file, &g_caches, cache_tailq, tmp) {
    2069           0 :                 rc = reclaim_cache_buffers(file);
    2070           0 :                 if (rc < 0) {
    2071           0 :                         continue;
    2072             :                 }
    2073           0 :                 break;
    2074             :         }
    2075             : 
    2076           0 :         return SPDK_POLLER_BUSY;
    2077             : }
    2078             : 
    2079             : static void
    2080           5 : _add_file_to_cache_pool(void *ctx)
    2081             : {
    2082           5 :         struct spdk_file *file = ctx;
    2083             : 
    2084           5 :         TAILQ_INSERT_TAIL(&g_caches, file, cache_tailq);
    2085           5 : }
    2086             : 
    2087             : static void
    2088           0 : _remove_file_from_cache_pool(void *ctx)
    2089             : {
    2090           0 :         struct spdk_file *file = ctx;
    2091             : 
    2092           0 :         TAILQ_REMOVE(&g_caches, file, cache_tailq);
    2093           0 : }
    2094             : 
    2095             : static struct cache_buffer *
    2096          10 : cache_insert_buffer(struct spdk_file *file, uint64_t offset)
    2097             : {
    2098             :         struct cache_buffer *buf;
    2099          10 :         int count = 0;
    2100          10 :         bool need_update = false;
    2101             : 
    2102          10 :         buf = calloc(1, sizeof(*buf));
    2103          10 :         if (buf == NULL) {
    2104           0 :                 SPDK_DEBUGLOG(blobfs, "calloc failed\n");
    2105           0 :                 return NULL;
    2106             :         }
    2107             : 
    2108             :         do {
    2109          10 :                 buf->buf = spdk_mempool_get(g_cache_pool);
    2110          10 :                 if (buf->buf) {
    2111          10 :                         break;
    2112             :                 }
    2113           0 :                 if (count++ == 100) {
    2114           0 :                         SPDK_ERRLOG("Could not allocate cache buffer for file=%p on offset=%jx\n",
    2115             :                                     file, offset);
    2116           0 :                         free(buf);
    2117           0 :                         return NULL;
    2118             :                 }
    2119           0 :                 usleep(BLOBFS_CACHE_POOL_POLL_PERIOD_IN_US);
    2120             :         } while (true);
    2121             : 
    2122          10 :         buf->buf_size = CACHE_BUFFER_SIZE;
    2123          10 :         buf->offset = offset;
    2124             : 
    2125          10 :         if (file->tree->present_mask == 0) {
    2126           5 :                 need_update = true;
    2127             :         }
    2128          10 :         file->tree = tree_insert_buffer(file->tree, buf);
    2129             : 
    2130          10 :         if (need_update) {
    2131           5 :                 spdk_thread_send_msg(g_cache_pool_thread, _add_file_to_cache_pool, file);
    2132             :         }
    2133             : 
    2134          10 :         return buf;
    2135             : }
    2136             : 
    2137             : static struct cache_buffer *
    2138          10 : cache_append_buffer(struct spdk_file *file)
    2139             : {
    2140             :         struct cache_buffer *last;
    2141             : 
    2142          10 :         assert(file->last == NULL || file->last->bytes_filled == file->last->buf_size);
    2143          10 :         assert((file->append_pos % CACHE_BUFFER_SIZE) == 0);
    2144             : 
    2145          10 :         last = cache_insert_buffer(file, file->append_pos);
    2146          10 :         if (last == NULL) {
    2147           0 :                 SPDK_DEBUGLOG(blobfs, "cache_insert_buffer failed\n");
    2148           0 :                 return NULL;
    2149             :         }
    2150             : 
    2151          10 :         file->last = last;
    2152             : 
    2153          10 :         return last;
    2154             : }
    2155             : 
    2156             : static void __check_sync_reqs(struct spdk_file *file);
    2157             : 
    2158             : static void
    2159           7 : __file_cache_finish_sync(void *ctx, int bserrno)
    2160             : {
    2161             :         struct spdk_file *file;
    2162           7 :         struct spdk_fs_request *sync_req = ctx;
    2163             :         struct spdk_fs_cb_args *sync_args;
    2164             : 
    2165           7 :         sync_args = &sync_req->args;
    2166           7 :         file = sync_args->file;
    2167           7 :         pthread_spin_lock(&file->lock);
    2168           7 :         file->length_xattr = sync_args->op.sync.length;
    2169           7 :         assert(sync_args->op.sync.offset <= file->length_flushed);
    2170           7 :         spdk_trace_record(TRACE_BLOBFS_XATTR_END, 0, sync_args->op.sync.offset,
    2171             :                           0, file->name);
    2172           7 :         BLOBFS_TRACE(file, "sync done offset=%jx\n", sync_args->op.sync.offset);
    2173           7 :         TAILQ_REMOVE(&file->sync_requests, sync_req, args.op.sync.tailq);
    2174           7 :         pthread_spin_unlock(&file->lock);
    2175             : 
    2176           7 :         sync_args->fn.file_op(sync_args->arg, bserrno);
    2177             : 
    2178           7 :         free_fs_request(sync_req);
    2179           7 :         __check_sync_reqs(file);
    2180           7 : }
    2181             : 
    2182             : static void
    2183          24 : __check_sync_reqs(struct spdk_file *file)
    2184             : {
    2185             :         struct spdk_fs_request *sync_req;
    2186             : 
    2187          24 :         pthread_spin_lock(&file->lock);
    2188             : 
    2189          26 :         TAILQ_FOREACH(sync_req, &file->sync_requests, args.op.sync.tailq) {
    2190          14 :                 if (sync_req->args.op.sync.offset <= file->length_flushed) {
    2191          12 :                         break;
    2192             :                 }
    2193             :         }
    2194             : 
    2195          24 :         if (sync_req != NULL && !sync_req->args.op.sync.xattr_in_progress) {
    2196           7 :                 BLOBFS_TRACE(file, "set xattr length 0x%jx\n", file->length_flushed);
    2197           7 :                 sync_req->args.op.sync.xattr_in_progress = true;
    2198           7 :                 sync_req->args.op.sync.length = file->length_flushed;
    2199           7 :                 spdk_blob_set_xattr(file->blob, "length", &file->length_flushed,
    2200             :                                     sizeof(file->length_flushed));
    2201             : 
    2202           7 :                 pthread_spin_unlock(&file->lock);
    2203           7 :                 spdk_trace_record(TRACE_BLOBFS_XATTR_START, 0, file->length_flushed,
    2204             :                                   0, file->name);
    2205           7 :                 spdk_blob_sync_md(file->blob, __file_cache_finish_sync, sync_req);
    2206             :         } else {
    2207          17 :                 pthread_spin_unlock(&file->lock);
    2208             :         }
    2209          24 : }
    2210             : 
    2211             : static void
    2212          10 : __file_flush_done(void *ctx, int bserrno)
    2213             : {
    2214          10 :         struct spdk_fs_request *req = ctx;
    2215          10 :         struct spdk_fs_cb_args *args = &req->args;
    2216          10 :         struct spdk_file *file = args->file;
    2217          10 :         struct cache_buffer *next = args->op.flush.cache_buffer;
    2218             : 
    2219          10 :         BLOBFS_TRACE(file, "length=%jx\n", args->op.flush.length);
    2220             : 
    2221          10 :         pthread_spin_lock(&file->lock);
    2222          10 :         next->in_progress = false;
    2223          10 :         next->bytes_flushed += args->op.flush.length;
    2224          10 :         file->length_flushed += args->op.flush.length;
    2225          10 :         if (file->length_flushed > file->length) {
    2226           0 :                 file->length = file->length_flushed;
    2227             :         }
    2228          10 :         if (next->bytes_flushed == next->buf_size) {
    2229           5 :                 BLOBFS_TRACE(file, "write buffer fully flushed 0x%jx\n", file->length_flushed);
    2230           5 :                 next = tree_find_buffer(file->tree, file->length_flushed);
    2231             :         }
    2232             : 
    2233             :         /*
    2234             :          * Assert that there is no cached data that extends past the end of the underlying
    2235             :          *  blob.
    2236             :          */
    2237          10 :         assert(next == NULL || next->offset < __file_get_blob_size(file) ||
    2238             :                next->bytes_filled == 0);
    2239             : 
    2240          10 :         pthread_spin_unlock(&file->lock);
    2241             : 
    2242          10 :         __check_sync_reqs(file);
    2243             : 
    2244          10 :         __file_flush(req);
    2245          10 : }
    2246             : 
    2247             : static void
    2248          20 : __file_flush(void *ctx)
    2249             : {
    2250          20 :         struct spdk_fs_request *req = ctx;
    2251          20 :         struct spdk_fs_cb_args *args = &req->args;
    2252          20 :         struct spdk_file *file = args->file;
    2253             :         struct cache_buffer *next;
    2254          20 :         uint64_t offset, length, start_lba, num_lba;
    2255          20 :         uint32_t lba_size;
    2256             : 
    2257          20 :         pthread_spin_lock(&file->lock);
    2258          20 :         next = tree_find_buffer(file->tree, file->length_flushed);
    2259          20 :         if (next == NULL || next->in_progress ||
    2260          18 :             ((next->bytes_filled < next->buf_size) && TAILQ_EMPTY(&file->sync_requests))) {
    2261             :                 /*
    2262             :                  * There is either no data to flush, a flush I/O is already in
    2263             :                  *  progress, or the next buffer is partially filled but there's no
    2264             :                  *  outstanding request to sync it.
    2265             :                  * So return immediately - if a flush I/O is in progress we will flush
    2266             :                  *  more data after that is completed, or a partial buffer will get flushed
    2267             :                  *  when it is either filled or the file is synced.
    2268             :                  */
    2269           4 :                 free_fs_request(req);
    2270           4 :                 if (next == NULL) {
    2271             :                         /*
    2272             :                          * For cases where a file's cache was evicted, and then the
    2273             :                          *  file was later appended, we will write the data directly
    2274             :                          *  to disk and bypass cache.  So just update length_flushed
    2275             :                          *  here to reflect that all data was already written to disk.
    2276             :                          */
    2277           1 :                         file->length_flushed = file->append_pos;
    2278             :                 }
    2279           4 :                 pthread_spin_unlock(&file->lock);
    2280           4 :                 if (next == NULL) {
    2281             :                         /*
    2282             :                          * There is no data to flush, but we still need to check for any
    2283             :                          *  outstanding sync requests to make sure metadata gets updated.
    2284             :                          */
    2285           1 :                         __check_sync_reqs(file);
    2286             :                 }
    2287           4 :                 return;
    2288             :         }
    2289             : 
    2290          16 :         offset = next->offset + next->bytes_flushed;
    2291          16 :         length = next->bytes_filled - next->bytes_flushed;
    2292          16 :         if (length == 0) {
    2293           6 :                 free_fs_request(req);
    2294           6 :                 pthread_spin_unlock(&file->lock);
    2295             :                 /*
    2296             :                  * There is no data to flush, but we still need to check for any
    2297             :                  *  outstanding sync requests to make sure metadata gets updated.
    2298             :                  */
    2299           6 :                 __check_sync_reqs(file);
    2300           6 :                 return;
    2301             :         }
    2302          10 :         args->op.flush.length = length;
    2303          10 :         args->op.flush.cache_buffer = next;
    2304             : 
    2305          10 :         __get_page_parameters(file, offset, length, &start_lba, &lba_size, &num_lba);
    2306             : 
    2307          10 :         next->in_progress = true;
    2308          10 :         BLOBFS_TRACE(file, "offset=0x%jx length=0x%jx page start=0x%jx num=0x%jx\n",
    2309             :                      offset, length, start_lba, num_lba);
    2310          10 :         pthread_spin_unlock(&file->lock);
    2311          10 :         spdk_blob_io_write(file->blob, file->fs->sync_target.sync_fs_channel->bs_channel,
    2312          10 :                            next->buf + (start_lba * lba_size) - next->offset,
    2313             :                            start_lba, num_lba, __file_flush_done, req);
    2314             : }
    2315             : 
    2316             : static void
    2317           0 : __file_extend_done(void *arg, int bserrno)
    2318             : {
    2319           0 :         struct spdk_fs_cb_args *args = arg;
    2320             : 
    2321           0 :         __wake_caller(args, bserrno);
    2322           0 : }
    2323             : 
    2324             : static void
    2325           0 : __file_extend_resize_cb(void *_args, int bserrno)
    2326             : {
    2327           0 :         struct spdk_fs_cb_args *args = _args;
    2328           0 :         struct spdk_file *file = args->file;
    2329             : 
    2330           0 :         if (bserrno) {
    2331           0 :                 __wake_caller(args, bserrno);
    2332           0 :                 return;
    2333             :         }
    2334             : 
    2335           0 :         spdk_blob_sync_md(file->blob, __file_extend_done, args);
    2336             : }
    2337             : 
    2338             : static void
    2339           0 : __file_extend_blob(void *_args)
    2340             : {
    2341           0 :         struct spdk_fs_cb_args *args = _args;
    2342           0 :         struct spdk_file *file = args->file;
    2343             : 
    2344           0 :         spdk_blob_resize(file->blob, args->op.resize.num_clusters, __file_extend_resize_cb, args);
    2345           0 : }
    2346             : 
    2347             : static void
    2348           1 : __rw_from_file_done(void *ctx, int bserrno)
    2349             : {
    2350           1 :         struct spdk_fs_request *req = ctx;
    2351             : 
    2352           1 :         __wake_caller(&req->args, bserrno);
    2353           1 :         free_fs_request(req);
    2354           1 : }
    2355             : 
    2356             : static void
    2357           1 : __rw_from_file(void *ctx)
    2358             : {
    2359           1 :         struct spdk_fs_request *req = ctx;
    2360           1 :         struct spdk_fs_cb_args *args = &req->args;
    2361           1 :         struct spdk_file *file = args->file;
    2362             : 
    2363           1 :         if (args->op.rw.is_read) {
    2364           0 :                 spdk_file_read_async(file, file->fs->sync_target.sync_io_channel, args->iovs[0].iov_base,
    2365           0 :                                      args->op.rw.offset, (uint64_t)args->iovs[0].iov_len,
    2366             :                                      __rw_from_file_done, req);
    2367             :         } else {
    2368           1 :                 spdk_file_write_async(file, file->fs->sync_target.sync_io_channel, args->iovs[0].iov_base,
    2369           1 :                                       args->op.rw.offset, (uint64_t)args->iovs[0].iov_len,
    2370             :                                       __rw_from_file_done, req);
    2371             :         }
    2372           1 : }
    2373             : 
    2374             : struct rw_from_file_arg {
    2375             :         struct spdk_fs_channel *channel;
    2376             :         int rwerrno;
    2377             : };
    2378             : 
    2379             : static int
    2380           1 : __send_rw_from_file(struct spdk_file *file, void *payload,
    2381             :                     uint64_t offset, uint64_t length, bool is_read,
    2382             :                     struct rw_from_file_arg *arg)
    2383             : {
    2384             :         struct spdk_fs_request *req;
    2385             :         struct spdk_fs_cb_args *args;
    2386             : 
    2387           1 :         req = alloc_fs_request_with_iov(arg->channel, 1);
    2388           1 :         if (req == NULL) {
    2389           0 :                 sem_post(&arg->channel->sem);
    2390           0 :                 return -ENOMEM;
    2391             :         }
    2392             : 
    2393           1 :         args = &req->args;
    2394           1 :         args->file = file;
    2395           1 :         args->sem = &arg->channel->sem;
    2396           1 :         args->iovs[0].iov_base = payload;
    2397           1 :         args->iovs[0].iov_len = (size_t)length;
    2398           1 :         args->op.rw.offset = offset;
    2399           1 :         args->op.rw.is_read = is_read;
    2400           1 :         args->rwerrno = &arg->rwerrno;
    2401           1 :         file->fs->send_request(__rw_from_file, req);
    2402           1 :         return 0;
    2403             : }
    2404             : 
    2405             : int
    2406          11 : spdk_file_write(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx,
    2407             :                 void *payload, uint64_t offset, uint64_t length)
    2408             : {
    2409          11 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    2410             :         struct spdk_fs_request *flush_req;
    2411             :         uint64_t rem_length, copy, blob_size, cluster_sz;
    2412          11 :         uint32_t cache_buffers_filled = 0;
    2413             :         uint8_t *cur_payload;
    2414             :         struct cache_buffer *last;
    2415             : 
    2416          11 :         BLOBFS_TRACE_RW(file, "offset=%jx length=%jx\n", offset, length);
    2417             : 
    2418          11 :         if (length == 0) {
    2419           1 :                 return 0;
    2420             :         }
    2421             : 
    2422          10 :         if (offset != file->append_pos) {
    2423           0 :                 BLOBFS_TRACE(file, " error offset=%jx append_pos=%jx\n", offset, file->append_pos);
    2424           0 :                 return -EINVAL;
    2425             :         }
    2426             : 
    2427          10 :         pthread_spin_lock(&file->lock);
    2428          10 :         file->open_for_writing = true;
    2429             : 
    2430          10 :         if ((file->last == NULL) && (file->append_pos % CACHE_BUFFER_SIZE == 0)) {
    2431           5 :                 cache_append_buffer(file);
    2432             :         }
    2433             : 
    2434          10 :         if (file->last == NULL) {
    2435           1 :                 struct rw_from_file_arg arg = {};
    2436             :                 int rc;
    2437             : 
    2438           1 :                 arg.channel = channel;
    2439           1 :                 arg.rwerrno = 0;
    2440           1 :                 file->append_pos += length;
    2441           1 :                 pthread_spin_unlock(&file->lock);
    2442           1 :                 rc = __send_rw_from_file(file, payload, offset, length, false, &arg);
    2443           1 :                 if (rc != 0) {
    2444           0 :                         return rc;
    2445             :                 }
    2446           1 :                 sem_wait(&channel->sem);
    2447           1 :                 return arg.rwerrno;
    2448             :         }
    2449             : 
    2450           9 :         blob_size = __file_get_blob_size(file);
    2451             : 
    2452           9 :         if ((offset + length) > blob_size) {
    2453           0 :                 struct spdk_fs_cb_args extend_args = {};
    2454             : 
    2455           0 :                 cluster_sz = file->fs->bs_opts.cluster_sz;
    2456           0 :                 extend_args.sem = &channel->sem;
    2457           0 :                 extend_args.op.resize.num_clusters = __bytes_to_clusters((offset + length), cluster_sz);
    2458           0 :                 extend_args.file = file;
    2459           0 :                 BLOBFS_TRACE(file, "start resize to %u clusters\n", extend_args.op.resize.num_clusters);
    2460           0 :                 pthread_spin_unlock(&file->lock);
    2461           0 :                 file->fs->send_request(__file_extend_blob, &extend_args);
    2462           0 :                 sem_wait(&channel->sem);
    2463           0 :                 if (extend_args.rc) {
    2464           0 :                         return extend_args.rc;
    2465             :                 }
    2466           0 :                 pthread_spin_lock(&file->lock);
    2467             :         }
    2468             : 
    2469           9 :         flush_req = alloc_fs_request(channel);
    2470           9 :         if (flush_req == NULL) {
    2471           0 :                 pthread_spin_unlock(&file->lock);
    2472           0 :                 return -ENOMEM;
    2473             :         }
    2474             : 
    2475           9 :         last = file->last;
    2476           9 :         rem_length = length;
    2477           9 :         cur_payload = payload;
    2478          22 :         while (rem_length > 0) {
    2479          13 :                 copy = last->buf_size - last->bytes_filled;
    2480          13 :                 if (copy > rem_length) {
    2481           8 :                         copy = rem_length;
    2482             :                 }
    2483          13 :                 BLOBFS_TRACE_RW(file, "  fill offset=%jx length=%jx\n", file->append_pos, copy);
    2484          13 :                 memcpy(&last->buf[last->bytes_filled], cur_payload, copy);
    2485          13 :                 file->append_pos += copy;
    2486          13 :                 if (file->length < file->append_pos) {
    2487          12 :                         file->length = file->append_pos;
    2488             :                 }
    2489          13 :                 cur_payload += copy;
    2490          13 :                 last->bytes_filled += copy;
    2491          13 :                 rem_length -= copy;
    2492          13 :                 if (last->bytes_filled == last->buf_size) {
    2493           5 :                         cache_buffers_filled++;
    2494           5 :                         last = cache_append_buffer(file);
    2495           5 :                         if (last == NULL) {
    2496           0 :                                 BLOBFS_TRACE(file, "nomem\n");
    2497           0 :                                 free_fs_request(flush_req);
    2498           0 :                                 pthread_spin_unlock(&file->lock);
    2499           0 :                                 return -ENOMEM;
    2500             :                         }
    2501             :                 }
    2502             :         }
    2503             : 
    2504           9 :         pthread_spin_unlock(&file->lock);
    2505             : 
    2506           9 :         if (cache_buffers_filled == 0) {
    2507           6 :                 free_fs_request(flush_req);
    2508           6 :                 return 0;
    2509             :         }
    2510             : 
    2511           3 :         flush_req->args.file = file;
    2512           3 :         file->fs->send_request(__file_flush, flush_req);
    2513           3 :         return 0;
    2514             : }
    2515             : 
    2516             : static void
    2517           0 : __readahead_done(void *ctx, int bserrno)
    2518             : {
    2519           0 :         struct spdk_fs_request *req = ctx;
    2520           0 :         struct spdk_fs_cb_args *args = &req->args;
    2521           0 :         struct cache_buffer *cache_buffer = args->op.readahead.cache_buffer;
    2522           0 :         struct spdk_file *file = args->file;
    2523             : 
    2524           0 :         BLOBFS_TRACE(file, "offset=%jx\n", cache_buffer->offset);
    2525             : 
    2526           0 :         pthread_spin_lock(&file->lock);
    2527           0 :         cache_buffer->bytes_filled = args->op.readahead.length;
    2528           0 :         cache_buffer->bytes_flushed = args->op.readahead.length;
    2529           0 :         cache_buffer->in_progress = false;
    2530           0 :         pthread_spin_unlock(&file->lock);
    2531             : 
    2532           0 :         free_fs_request(req);
    2533           0 : }
    2534             : 
    2535             : static void
    2536           0 : __readahead(void *ctx)
    2537             : {
    2538           0 :         struct spdk_fs_request *req = ctx;
    2539           0 :         struct spdk_fs_cb_args *args = &req->args;
    2540           0 :         struct spdk_file *file = args->file;
    2541           0 :         uint64_t offset, length, start_lba, num_lba;
    2542           0 :         uint32_t lba_size;
    2543             : 
    2544           0 :         offset = args->op.readahead.offset;
    2545           0 :         length = args->op.readahead.length;
    2546           0 :         assert(length > 0);
    2547             : 
    2548           0 :         __get_page_parameters(file, offset, length, &start_lba, &lba_size, &num_lba);
    2549             : 
    2550           0 :         BLOBFS_TRACE(file, "offset=%jx length=%jx page start=%jx num=%jx\n",
    2551             :                      offset, length, start_lba, num_lba);
    2552           0 :         spdk_blob_io_read(file->blob, file->fs->sync_target.sync_fs_channel->bs_channel,
    2553           0 :                           args->op.readahead.cache_buffer->buf,
    2554             :                           start_lba, num_lba, __readahead_done, req);
    2555           0 : }
    2556             : 
    2557             : static uint64_t
    2558           0 : __next_cache_buffer_offset(uint64_t offset)
    2559             : {
    2560           0 :         return (offset + CACHE_BUFFER_SIZE) & ~(CACHE_TREE_LEVEL_MASK(0));
    2561             : }
    2562             : 
    2563             : static void
    2564           0 : check_readahead(struct spdk_file *file, uint64_t offset,
    2565             :                 struct spdk_fs_channel *channel)
    2566             : {
    2567             :         struct spdk_fs_request *req;
    2568             :         struct spdk_fs_cb_args *args;
    2569             : 
    2570           0 :         offset = __next_cache_buffer_offset(offset);
    2571           0 :         if (tree_find_buffer(file->tree, offset) != NULL || file->length <= offset) {
    2572           0 :                 return;
    2573             :         }
    2574             : 
    2575           0 :         req = alloc_fs_request(channel);
    2576           0 :         if (req == NULL) {
    2577           0 :                 return;
    2578             :         }
    2579           0 :         args = &req->args;
    2580             : 
    2581           0 :         BLOBFS_TRACE(file, "offset=%jx\n", offset);
    2582             : 
    2583           0 :         args->file = file;
    2584           0 :         args->op.readahead.offset = offset;
    2585           0 :         args->op.readahead.cache_buffer = cache_insert_buffer(file, offset);
    2586           0 :         if (!args->op.readahead.cache_buffer) {
    2587           0 :                 BLOBFS_TRACE(file, "Cannot allocate buf for offset=%jx\n", offset);
    2588           0 :                 free_fs_request(req);
    2589           0 :                 return;
    2590             :         }
    2591             : 
    2592           0 :         args->op.readahead.cache_buffer->in_progress = true;
    2593           0 :         if (file->length < (offset + CACHE_BUFFER_SIZE)) {
    2594           0 :                 args->op.readahead.length = file->length & (CACHE_BUFFER_SIZE - 1);
    2595             :         } else {
    2596           0 :                 args->op.readahead.length = CACHE_BUFFER_SIZE;
    2597             :         }
    2598           0 :         file->fs->send_request(__readahead, req);
    2599             : }
    2600             : 
    2601             : int64_t
    2602           1 : spdk_file_read(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx,
    2603             :                void *payload, uint64_t offset, uint64_t length)
    2604             : {
    2605           1 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    2606             :         uint64_t final_offset, final_length;
    2607           1 :         uint32_t sub_reads = 0;
    2608             :         struct cache_buffer *buf;
    2609             :         uint64_t read_len;
    2610           1 :         struct rw_from_file_arg arg = {};
    2611             : 
    2612           1 :         pthread_spin_lock(&file->lock);
    2613             : 
    2614           1 :         BLOBFS_TRACE_RW(file, "offset=%ju length=%ju\n", offset, length);
    2615             : 
    2616           1 :         file->open_for_writing = false;
    2617             : 
    2618           1 :         if (length == 0 || offset >= file->append_pos) {
    2619           0 :                 pthread_spin_unlock(&file->lock);
    2620           0 :                 return 0;
    2621             :         }
    2622             : 
    2623           1 :         if (offset + length > file->append_pos) {
    2624           0 :                 length = file->append_pos - offset;
    2625             :         }
    2626             : 
    2627           1 :         if (offset != file->next_seq_offset) {
    2628           0 :                 file->seq_byte_count = 0;
    2629             :         }
    2630           1 :         file->seq_byte_count += length;
    2631           1 :         file->next_seq_offset = offset + length;
    2632           1 :         if (file->seq_byte_count >= CACHE_READAHEAD_THRESHOLD) {
    2633           0 :                 check_readahead(file, offset, channel);
    2634           0 :                 check_readahead(file, offset + CACHE_BUFFER_SIZE, channel);
    2635             :         }
    2636             : 
    2637           1 :         arg.channel = channel;
    2638           1 :         arg.rwerrno = 0;
    2639           1 :         final_length = 0;
    2640           1 :         final_offset = offset + length;
    2641           2 :         while (offset < final_offset) {
    2642           1 :                 int ret = 0;
    2643           1 :                 length = NEXT_CACHE_BUFFER_OFFSET(offset) - offset;
    2644           1 :                 if (length > (final_offset - offset)) {
    2645           1 :                         length = final_offset - offset;
    2646             :                 }
    2647             : 
    2648           1 :                 buf = tree_find_filled_buffer(file->tree, offset);
    2649           1 :                 if (buf == NULL) {
    2650           0 :                         pthread_spin_unlock(&file->lock);
    2651           0 :                         ret = __send_rw_from_file(file, payload, offset, length, true, &arg);
    2652           0 :                         pthread_spin_lock(&file->lock);
    2653           0 :                         if (ret == 0) {
    2654           0 :                                 sub_reads++;
    2655             :                         }
    2656             :                 } else {
    2657           1 :                         read_len = length;
    2658           1 :                         if ((offset + length) > (buf->offset + buf->bytes_filled)) {
    2659           0 :                                 read_len = buf->offset + buf->bytes_filled - offset;
    2660             :                         }
    2661           1 :                         BLOBFS_TRACE(file, "read %p offset=%ju length=%ju\n", payload, offset, read_len);
    2662           1 :                         memcpy(payload, &buf->buf[offset - buf->offset], read_len);
    2663           1 :                         if ((offset + read_len) % CACHE_BUFFER_SIZE == 0) {
    2664           0 :                                 tree_remove_buffer(file->tree, buf);
    2665           0 :                                 if (file->tree->present_mask == 0) {
    2666           0 :                                         spdk_thread_send_msg(g_cache_pool_thread, _remove_file_from_cache_pool, file);
    2667             :                                 }
    2668             :                         }
    2669             :                 }
    2670             : 
    2671           1 :                 if (ret == 0) {
    2672           1 :                         final_length += length;
    2673             :                 } else {
    2674           0 :                         arg.rwerrno = ret;
    2675           0 :                         break;
    2676             :                 }
    2677           1 :                 payload += length;
    2678           1 :                 offset += length;
    2679             :         }
    2680           1 :         pthread_spin_unlock(&file->lock);
    2681           1 :         while (sub_reads > 0) {
    2682           0 :                 sem_wait(&channel->sem);
    2683           0 :                 sub_reads--;
    2684             :         }
    2685           1 :         if (arg.rwerrno == 0) {
    2686           1 :                 return final_length;
    2687             :         } else {
    2688           0 :                 return arg.rwerrno;
    2689             :         }
    2690             : }
    2691             : 
    2692             : static void
    2693          18 : _file_sync(struct spdk_file *file, struct spdk_fs_channel *channel,
    2694             :            spdk_file_op_complete cb_fn, void *cb_arg)
    2695             : {
    2696             :         struct spdk_fs_request *sync_req;
    2697             :         struct spdk_fs_request *flush_req;
    2698             :         struct spdk_fs_cb_args *sync_args;
    2699             :         struct spdk_fs_cb_args *flush_args;
    2700             : 
    2701          18 :         BLOBFS_TRACE(file, "offset=%jx\n", file->append_pos);
    2702             : 
    2703          18 :         pthread_spin_lock(&file->lock);
    2704          18 :         if (file->append_pos <= file->length_xattr) {
    2705          11 :                 BLOBFS_TRACE(file, "done - file already synced\n");
    2706          11 :                 pthread_spin_unlock(&file->lock);
    2707          11 :                 cb_fn(cb_arg, 0);
    2708          11 :                 return;
    2709             :         }
    2710             : 
    2711           7 :         sync_req = alloc_fs_request(channel);
    2712           7 :         if (!sync_req) {
    2713           0 :                 SPDK_ERRLOG("Cannot allocate sync req for file=%s\n", file->name);
    2714           0 :                 pthread_spin_unlock(&file->lock);
    2715           0 :                 cb_fn(cb_arg, -ENOMEM);
    2716           0 :                 return;
    2717             :         }
    2718           7 :         sync_args = &sync_req->args;
    2719             : 
    2720           7 :         flush_req = alloc_fs_request(channel);
    2721           7 :         if (!flush_req) {
    2722           0 :                 SPDK_ERRLOG("Cannot allocate flush req for file=%s\n", file->name);
    2723           0 :                 free_fs_request(sync_req);
    2724           0 :                 pthread_spin_unlock(&file->lock);
    2725           0 :                 cb_fn(cb_arg, -ENOMEM);
    2726           0 :                 return;
    2727             :         }
    2728           7 :         flush_args = &flush_req->args;
    2729             : 
    2730           7 :         sync_args->file = file;
    2731           7 :         sync_args->fn.file_op = cb_fn;
    2732           7 :         sync_args->arg = cb_arg;
    2733           7 :         sync_args->op.sync.offset = file->append_pos;
    2734           7 :         sync_args->op.sync.xattr_in_progress = false;
    2735           7 :         TAILQ_INSERT_TAIL(&file->sync_requests, sync_req, args.op.sync.tailq);
    2736           7 :         pthread_spin_unlock(&file->lock);
    2737             : 
    2738           7 :         flush_args->file = file;
    2739           7 :         channel->send_request(__file_flush, flush_req);
    2740             : }
    2741             : 
    2742             : int
    2743          12 : spdk_file_sync(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx)
    2744             : {
    2745          12 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    2746          12 :         struct spdk_fs_cb_args args = {};
    2747             : 
    2748          12 :         args.sem = &channel->sem;
    2749          12 :         _file_sync(file, channel, __wake_caller, &args);
    2750          12 :         sem_wait(&channel->sem);
    2751             : 
    2752          12 :         return args.rc;
    2753             : }
    2754             : 
    2755             : void
    2756           6 : spdk_file_sync_async(struct spdk_file *file, struct spdk_io_channel *_channel,
    2757             :                      spdk_file_op_complete cb_fn, void *cb_arg)
    2758             : {
    2759           6 :         struct spdk_fs_channel *channel = spdk_io_channel_get_ctx(_channel);
    2760             : 
    2761           6 :         _file_sync(file, channel, cb_fn, cb_arg);
    2762           6 : }
    2763             : 
    2764             : void
    2765           0 : spdk_file_set_priority(struct spdk_file *file, uint32_t priority)
    2766             : {
    2767           0 :         BLOBFS_TRACE(file, "priority=%u\n", priority);
    2768           0 :         file->priority = priority;
    2769             : 
    2770           0 : }
    2771             : 
    2772             : /*
    2773             :  * Close routines
    2774             :  */
    2775             : 
    2776             : static void
    2777          17 : __file_close_async_done(void *ctx, int bserrno)
    2778             : {
    2779          17 :         struct spdk_fs_request *req = ctx;
    2780          17 :         struct spdk_fs_cb_args *args = &req->args;
    2781          17 :         struct spdk_file *file = args->file;
    2782             : 
    2783          17 :         spdk_trace_record(TRACE_BLOBFS_CLOSE, 0, 0, 0, file->name);
    2784             : 
    2785          17 :         if (file->is_deleted) {
    2786           2 :                 spdk_fs_delete_file_async(file->fs, file->name, blob_delete_cb, ctx);
    2787           2 :                 return;
    2788             :         }
    2789             : 
    2790          15 :         args->fn.file_op(args->arg, bserrno);
    2791          15 :         free_fs_request(req);
    2792             : }
    2793             : 
    2794             : static void
    2795          17 : __file_close_async(struct spdk_file *file, struct spdk_fs_request *req)
    2796             : {
    2797             :         struct spdk_blob *blob;
    2798             : 
    2799          17 :         pthread_spin_lock(&file->lock);
    2800          17 :         if (file->ref_count == 0) {
    2801           0 :                 pthread_spin_unlock(&file->lock);
    2802           0 :                 __file_close_async_done(req, -EBADF);
    2803           0 :                 return;
    2804             :         }
    2805             : 
    2806          17 :         file->ref_count--;
    2807          17 :         if (file->ref_count > 0) {
    2808           0 :                 pthread_spin_unlock(&file->lock);
    2809           0 :                 req->args.fn.file_op(req->args.arg, 0);
    2810           0 :                 free_fs_request(req);
    2811           0 :                 return;
    2812             :         }
    2813             : 
    2814          17 :         pthread_spin_unlock(&file->lock);
    2815             : 
    2816          17 :         blob = file->blob;
    2817          17 :         file->blob = NULL;
    2818          17 :         spdk_blob_close(blob, __file_close_async_done, req);
    2819             : }
    2820             : 
    2821             : static void
    2822           6 : __file_close_async__sync_done(void *arg, int fserrno)
    2823             : {
    2824           6 :         struct spdk_fs_request *req = arg;
    2825           6 :         struct spdk_fs_cb_args *args = &req->args;
    2826             : 
    2827           6 :         __file_close_async(args->file, req);
    2828           6 : }
    2829             : 
    2830             : void
    2831           6 : spdk_file_close_async(struct spdk_file *file, spdk_file_op_complete cb_fn, void *cb_arg)
    2832             : {
    2833             :         struct spdk_fs_request *req;
    2834             :         struct spdk_fs_cb_args *args;
    2835             : 
    2836           6 :         req = alloc_fs_request(file->fs->md_target.md_fs_channel);
    2837           6 :         if (req == NULL) {
    2838           0 :                 SPDK_ERRLOG("Cannot allocate close async req for file=%s\n", file->name);
    2839           0 :                 cb_fn(cb_arg, -ENOMEM);
    2840           0 :                 return;
    2841             :         }
    2842             : 
    2843           6 :         args = &req->args;
    2844           6 :         args->file = file;
    2845           6 :         args->fn.file_op = cb_fn;
    2846           6 :         args->arg = cb_arg;
    2847             : 
    2848           6 :         spdk_file_sync_async(file, file->fs->md_target.md_io_channel, __file_close_async__sync_done, req);
    2849             : }
    2850             : 
    2851             : static void
    2852          11 : __file_close(void *arg)
    2853             : {
    2854          11 :         struct spdk_fs_request *req = arg;
    2855          11 :         struct spdk_fs_cb_args *args = &req->args;
    2856          11 :         struct spdk_file *file = args->file;
    2857             : 
    2858          11 :         __file_close_async(file, req);
    2859          11 : }
    2860             : 
    2861             : int
    2862          11 : spdk_file_close(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx)
    2863             : {
    2864          11 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    2865             :         struct spdk_fs_request *req;
    2866             :         struct spdk_fs_cb_args *args;
    2867             : 
    2868          11 :         req = alloc_fs_request(channel);
    2869          11 :         if (req == NULL) {
    2870           0 :                 SPDK_ERRLOG("Cannot allocate close req for file=%s\n", file->name);
    2871           0 :                 return -ENOMEM;
    2872             :         }
    2873             : 
    2874          11 :         args = &req->args;
    2875             : 
    2876          11 :         spdk_file_sync(file, ctx);
    2877          11 :         BLOBFS_TRACE(file, "name=%s\n", file->name);
    2878          11 :         args->file = file;
    2879          11 :         args->sem = &channel->sem;
    2880          11 :         args->fn.file_op = __wake_caller;
    2881          11 :         args->arg = args;
    2882          11 :         channel->send_request(__file_close, req);
    2883          11 :         sem_wait(&channel->sem);
    2884             : 
    2885          11 :         return args->rc;
    2886             : }
    2887             : 
    2888             : int
    2889           0 : spdk_file_get_id(struct spdk_file *file, void *id, size_t size)
    2890             : {
    2891           0 :         if (size < sizeof(spdk_blob_id)) {
    2892           0 :                 return -EINVAL;
    2893             :         }
    2894             : 
    2895           0 :         memcpy(id, &file->blobid, sizeof(spdk_blob_id));
    2896             : 
    2897           0 :         return sizeof(spdk_blob_id);
    2898             : }
    2899             : 
    2900             : static void
    2901           5 : _file_free(void *ctx)
    2902             : {
    2903           5 :         struct spdk_file *file = ctx;
    2904             : 
    2905           5 :         TAILQ_REMOVE(&g_caches, file, cache_tailq);
    2906             : 
    2907           5 :         free(file->name);
    2908           5 :         free(file->tree);
    2909           5 :         free(file);
    2910           5 : }
    2911             : 
    2912             : static void
    2913          18 : file_free(struct spdk_file *file)
    2914             : {
    2915          18 :         BLOBFS_TRACE(file, "free=%s\n", file->name);
    2916          18 :         pthread_spin_lock(&file->lock);
    2917          18 :         if (file->tree->present_mask == 0) {
    2918          13 :                 pthread_spin_unlock(&file->lock);
    2919          13 :                 free(file->name);
    2920          13 :                 free(file->tree);
    2921          13 :                 free(file);
    2922          13 :                 return;
    2923             :         }
    2924             : 
    2925           5 :         tree_free_buffers(file->tree);
    2926           5 :         assert(file->tree->present_mask == 0);
    2927           5 :         spdk_thread_send_msg(g_cache_pool_thread, _file_free, file);
    2928           5 :         pthread_spin_unlock(&file->lock);
    2929             : }
    2930             : 
    2931           2 : SPDK_LOG_REGISTER_COMPONENT(blobfs)
    2932           2 : SPDK_LOG_REGISTER_COMPONENT(blobfs_rw)

Generated by: LCOV version 1.15