LCOV - code coverage report
Current view: top level - lib/nvme - nvme.c (source / functions) Hit Total Coverage
Test: ut_cov_unit.info Lines: 621 748 83.0 %
Date: 2024-07-14 18:22:36 Functions: 47 50 94.0 %

          Line data    Source code
       1             : /*   SPDX-License-Identifier: BSD-3-Clause
       2             :  *   Copyright (C) 2015 Intel Corporation. All rights reserved.
       3             :  *   Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved.
       4             :  */
       5             : 
       6             : #include "spdk/config.h"
       7             : #include "spdk/nvmf_spec.h"
       8             : #include "spdk/string.h"
       9             : #include "spdk/env.h"
      10             : #include "nvme_internal.h"
      11             : #include "nvme_io_msg.h"
      12             : 
      13             : #define SPDK_NVME_DRIVER_NAME "spdk_nvme_driver"
      14             : 
      15             : struct nvme_driver      *g_spdk_nvme_driver;
      16             : pid_t                   g_spdk_nvme_pid;
      17             : 
      18             : /* gross timeout of 180 seconds in milliseconds */
      19             : static int g_nvme_driver_timeout_ms = 3 * 60 * 1000;
      20             : 
      21             : /* Per-process attached controller list */
      22             : static TAILQ_HEAD(, spdk_nvme_ctrlr) g_nvme_attached_ctrlrs =
      23             :         TAILQ_HEAD_INITIALIZER(g_nvme_attached_ctrlrs);
      24             : 
      25             : /* Returns true if ctrlr should be stored on the multi-process shared_attached_ctrlrs list */
      26             : static bool
      27          12 : nvme_ctrlr_shared(const struct spdk_nvme_ctrlr *ctrlr)
      28             : {
      29          12 :         return ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE;
      30             : }
      31             : 
      32             : void
      33           0 : nvme_ctrlr_connected(struct spdk_nvme_probe_ctx *probe_ctx,
      34             :                      struct spdk_nvme_ctrlr *ctrlr)
      35             : {
      36           0 :         TAILQ_INSERT_TAIL(&probe_ctx->init_ctrlrs, ctrlr, tailq);
      37           0 : }
      38             : 
      39             : static void
      40          10 : nvme_ctrlr_detach_async_finish(struct spdk_nvme_ctrlr *ctrlr)
      41             : {
      42          10 :         nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
      43          10 :         if (nvme_ctrlr_shared(ctrlr)) {
      44           8 :                 TAILQ_REMOVE(&g_spdk_nvme_driver->shared_attached_ctrlrs, ctrlr, tailq);
      45             :         } else {
      46           2 :                 TAILQ_REMOVE(&g_nvme_attached_ctrlrs, ctrlr, tailq);
      47             :         }
      48          10 :         nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
      49          10 : }
      50             : 
      51             : static int
      52          11 : nvme_ctrlr_detach_async(struct spdk_nvme_ctrlr *ctrlr,
      53             :                         struct nvme_ctrlr_detach_ctx **_ctx)
      54             : {
      55             :         struct nvme_ctrlr_detach_ctx *ctx;
      56             :         int ref_count;
      57             : 
      58          11 :         nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
      59             : 
      60          11 :         ref_count = nvme_ctrlr_get_ref_count(ctrlr);
      61          11 :         assert(ref_count > 0);
      62             : 
      63          11 :         if (ref_count == 1) {
      64             :                 /* This is the last reference to the controller, so we need to
      65             :                  * allocate a context to destruct it.
      66             :                  */
      67          10 :                 ctx = calloc(1, sizeof(*ctx));
      68          10 :                 if (ctx == NULL) {
      69           0 :                         nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
      70             : 
      71           0 :                         return -ENOMEM;
      72             :                 }
      73          10 :                 ctx->ctrlr = ctrlr;
      74          10 :                 ctx->cb_fn = nvme_ctrlr_detach_async_finish;
      75             : 
      76          10 :                 nvme_ctrlr_proc_put_ref(ctrlr);
      77             : 
      78          10 :                 nvme_io_msg_ctrlr_detach(ctrlr);
      79             : 
      80          10 :                 nvme_ctrlr_destruct_async(ctrlr, ctx);
      81             : 
      82          10 :                 *_ctx = ctx;
      83             :         } else {
      84           1 :                 nvme_ctrlr_proc_put_ref(ctrlr);
      85             :         }
      86             : 
      87          11 :         nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
      88             : 
      89          11 :         return 0;
      90             : }
      91             : 
      92             : static int
      93          11 : nvme_ctrlr_detach_poll_async(struct nvme_ctrlr_detach_ctx *ctx)
      94             : {
      95             :         int rc;
      96             : 
      97          11 :         rc = nvme_ctrlr_destruct_poll_async(ctx->ctrlr, ctx);
      98          11 :         if (rc == -EAGAIN) {
      99           1 :                 return -EAGAIN;
     100             :         }
     101             : 
     102          10 :         free(ctx);
     103             : 
     104          10 :         return rc;
     105             : }
     106             : 
     107             : int
     108           5 : spdk_nvme_detach(struct spdk_nvme_ctrlr *ctrlr)
     109             : {
     110           5 :         struct nvme_ctrlr_detach_ctx *ctx = NULL;
     111             :         int rc;
     112             : 
     113           5 :         rc = nvme_ctrlr_detach_async(ctrlr, &ctx);
     114           5 :         if (rc != 0) {
     115           0 :                 return rc;
     116           5 :         } else if (ctx == NULL) {
     117             :                 /* ctrlr was detached from the caller process but any other process
     118             :                  * still attaches it.
     119             :                  */
     120           1 :                 return 0;
     121             :         }
     122             : 
     123             :         while (1) {
     124           4 :                 rc = nvme_ctrlr_detach_poll_async(ctx);
     125           4 :                 if (rc != -EAGAIN) {
     126           4 :                         break;
     127             :                 }
     128           0 :                 nvme_delay(1000);
     129             :         }
     130             : 
     131           4 :         return 0;
     132             : }
     133             : 
     134             : int
     135           6 : spdk_nvme_detach_async(struct spdk_nvme_ctrlr *ctrlr,
     136             :                        struct spdk_nvme_detach_ctx **_detach_ctx)
     137             : {
     138             :         struct spdk_nvme_detach_ctx *detach_ctx;
     139           6 :         struct nvme_ctrlr_detach_ctx *ctx = NULL;
     140             :         int rc;
     141             : 
     142           6 :         if (ctrlr == NULL || _detach_ctx == NULL) {
     143           0 :                 return -EINVAL;
     144             :         }
     145             : 
     146             :         /* Use a context header to poll detachment for multiple controllers.
     147             :          * Allocate an new one if not allocated yet, or use the passed one otherwise.
     148             :          */
     149           6 :         detach_ctx = *_detach_ctx;
     150           6 :         if (detach_ctx == NULL) {
     151           3 :                 detach_ctx = calloc(1, sizeof(*detach_ctx));
     152           3 :                 if (detach_ctx == NULL) {
     153           0 :                         return -ENOMEM;
     154             :                 }
     155           3 :                 TAILQ_INIT(&detach_ctx->head);
     156             :         }
     157             : 
     158           6 :         rc = nvme_ctrlr_detach_async(ctrlr, &ctx);
     159           6 :         if (rc != 0 || ctx == NULL) {
     160             :                 /* If this detach failed and the context header is empty, it means we just
     161             :                  * allocated the header and need to free it before returning.
     162             :                  */
     163           0 :                 if (TAILQ_EMPTY(&detach_ctx->head)) {
     164           0 :                         free(detach_ctx);
     165             :                 }
     166           0 :                 return rc;
     167             :         }
     168             : 
     169             :         /* Append a context for this detachment to the context header. */
     170           6 :         TAILQ_INSERT_TAIL(&detach_ctx->head, ctx, link);
     171             : 
     172           6 :         *_detach_ctx = detach_ctx;
     173             : 
     174           6 :         return 0;
     175             : }
     176             : 
     177             : int
     178           4 : spdk_nvme_detach_poll_async(struct spdk_nvme_detach_ctx *detach_ctx)
     179             : {
     180             :         struct nvme_ctrlr_detach_ctx *ctx, *tmp_ctx;
     181             :         int rc;
     182             : 
     183           4 :         if (detach_ctx == NULL) {
     184           0 :                 return -EINVAL;
     185             :         }
     186             : 
     187          11 :         TAILQ_FOREACH_SAFE(ctx, &detach_ctx->head, link, tmp_ctx) {
     188           7 :                 TAILQ_REMOVE(&detach_ctx->head, ctx, link);
     189             : 
     190           7 :                 rc = nvme_ctrlr_detach_poll_async(ctx);
     191           7 :                 if (rc == -EAGAIN) {
     192             :                         /* If not -EAGAIN, ctx was freed by nvme_ctrlr_detach_poll_async(). */
     193           1 :                         TAILQ_INSERT_HEAD(&detach_ctx->head, ctx, link);
     194             :                 }
     195             :         }
     196             : 
     197           4 :         if (!TAILQ_EMPTY(&detach_ctx->head)) {
     198           1 :                 return -EAGAIN;
     199             :         }
     200             : 
     201           3 :         free(detach_ctx);
     202           3 :         return 0;
     203             : }
     204             : 
     205             : void
     206           0 : spdk_nvme_detach_poll(struct spdk_nvme_detach_ctx *detach_ctx)
     207             : {
     208           0 :         while (detach_ctx && spdk_nvme_detach_poll_async(detach_ctx) == -EAGAIN) {
     209             :                 ;
     210             :         }
     211           0 : }
     212             : 
     213             : void
     214           1 : nvme_completion_poll_cb(void *arg, const struct spdk_nvme_cpl *cpl)
     215             : {
     216           1 :         struct nvme_completion_poll_status      *status = arg;
     217             : 
     218           1 :         if (status->timed_out) {
     219             :                 /* There is no routine waiting for the completion of this request, free allocated memory */
     220           0 :                 spdk_free(status->dma_data);
     221           0 :                 free(status);
     222           0 :                 return;
     223             :         }
     224             : 
     225             :         /*
     226             :          * Copy status into the argument passed by the caller, so that
     227             :          *  the caller can check the status to determine if the
     228             :          *  the request passed or failed.
     229             :          */
     230           1 :         memcpy(&status->cpl, cpl, sizeof(*cpl));
     231           1 :         status->done = true;
     232             : }
     233             : 
     234             : static void
     235           0 : dummy_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx)
     236             : {
     237           0 : }
     238             : 
     239             : int
     240          10 : nvme_wait_for_completion_robust_lock_timeout_poll(struct spdk_nvme_qpair *qpair,
     241             :                 struct nvme_completion_poll_status *status,
     242             :                 pthread_mutex_t *robust_mutex)
     243             : {
     244             :         int rc;
     245             : 
     246          10 :         if (robust_mutex) {
     247           5 :                 nvme_robust_mutex_lock(robust_mutex);
     248             :         }
     249             : 
     250          10 :         if (qpair->poll_group) {
     251           0 :                 rc = (int)spdk_nvme_poll_group_process_completions(qpair->poll_group->group, 0,
     252             :                                 dummy_disconnected_qpair_cb);
     253             :         } else {
     254          10 :                 rc = spdk_nvme_qpair_process_completions(qpair, 0);
     255             :         }
     256             : 
     257          10 :         if (robust_mutex) {
     258           5 :                 nvme_robust_mutex_unlock(robust_mutex);
     259             :         }
     260             : 
     261          10 :         if (rc < 0) {
     262           4 :                 status->cpl.status.sct = SPDK_NVME_SCT_GENERIC;
     263           4 :                 status->cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION;
     264           4 :                 goto error;
     265             :         }
     266             : 
     267           6 :         if (!status->done && status->timeout_tsc && spdk_get_ticks() > status->timeout_tsc) {
     268           2 :                 goto error;
     269             :         }
     270             : 
     271           4 :         if (qpair->ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
     272           4 :                 union spdk_nvme_csts_register csts = spdk_nvme_ctrlr_get_regs_csts(qpair->ctrlr);
     273           4 :                 if (csts.raw == SPDK_NVME_INVALID_REGISTER_VALUE) {
     274           0 :                         status->cpl.status.sct = SPDK_NVME_SCT_GENERIC;
     275           0 :                         status->cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
     276           0 :                         goto error;
     277             :                 }
     278             :         }
     279             : 
     280           4 :         if (!status->done) {
     281           0 :                 return -EAGAIN;
     282           4 :         } else if (spdk_nvme_cpl_is_error(&status->cpl)) {
     283           0 :                 return -EIO;
     284             :         } else {
     285           4 :                 return 0;
     286             :         }
     287           6 : error:
     288             :         /* Either transport error occurred or we've timed out.  Either way, if the response hasn't
     289             :          * been received yet, mark the command as timed out, so the status gets freed when the
     290             :          * command is completed or aborted.
     291             :          */
     292           6 :         if (!status->done) {
     293           6 :                 status->timed_out = true;
     294             :         }
     295             : 
     296           6 :         return -ECANCELED;
     297             : }
     298             : 
     299             : /**
     300             :  * Poll qpair for completions until a command completes.
     301             :  *
     302             :  * \param qpair queue to poll
     303             :  * \param status completion status. The user must fill this structure with zeroes before calling
     304             :  * this function
     305             :  * \param robust_mutex optional robust mutex to lock while polling qpair
     306             :  * \param timeout_in_usecs optional timeout
     307             :  *
     308             :  * \return 0 if command completed without error,
     309             :  * -EIO if command completed with error,
     310             :  * -ECANCELED if command is not completed due to transport/device error or time expired
     311             :  *
     312             :  *  The command to wait upon must be submitted with nvme_completion_poll_cb as the callback
     313             :  *  and status as the callback argument.
     314             :  */
     315             : int
     316          10 : nvme_wait_for_completion_robust_lock_timeout(
     317             :         struct spdk_nvme_qpair *qpair,
     318             :         struct nvme_completion_poll_status *status,
     319             :         pthread_mutex_t *robust_mutex,
     320             :         uint64_t timeout_in_usecs)
     321             : {
     322             :         int rc;
     323             : 
     324          10 :         if (timeout_in_usecs) {
     325           6 :                 status->timeout_tsc = spdk_get_ticks() + timeout_in_usecs *
     326           6 :                                       spdk_get_ticks_hz() / SPDK_SEC_TO_USEC;
     327             :         } else {
     328           4 :                 status->timeout_tsc = 0;
     329             :         }
     330             : 
     331          10 :         status->cpl.status_raw = 0;
     332             :         do {
     333          10 :                 rc = nvme_wait_for_completion_robust_lock_timeout_poll(qpair, status, robust_mutex);
     334          10 :         } while (rc == -EAGAIN);
     335             : 
     336          10 :         return rc;
     337             : }
     338             : 
     339             : /**
     340             :  * Poll qpair for completions until a command completes.
     341             :  *
     342             :  * \param qpair queue to poll
     343             :  * \param status completion status. The user must fill this structure with zeroes before calling
     344             :  * this function
     345             :  * \param robust_mutex optional robust mutex to lock while polling qpair
     346             :  *
     347             :  * \return 0 if command completed without error,
     348             :  * -EIO if command completed with error,
     349             :  * -ECANCELED if command is not completed due to transport/device error
     350             :  *
     351             :  * The command to wait upon must be submitted with nvme_completion_poll_cb as the callback
     352             :  * and status as the callback argument.
     353             :  */
     354             : int
     355           2 : nvme_wait_for_completion_robust_lock(
     356             :         struct spdk_nvme_qpair *qpair,
     357             :         struct nvme_completion_poll_status *status,
     358             :         pthread_mutex_t *robust_mutex)
     359             : {
     360           2 :         return nvme_wait_for_completion_robust_lock_timeout(qpair, status, robust_mutex, 0);
     361             : }
     362             : 
     363             : int
     364           2 : nvme_wait_for_completion(struct spdk_nvme_qpair *qpair,
     365             :                          struct nvme_completion_poll_status *status)
     366             : {
     367           2 :         return nvme_wait_for_completion_robust_lock_timeout(qpair, status, NULL, 0);
     368             : }
     369             : 
     370             : /**
     371             :  * Poll qpair for completions until a command completes.
     372             :  *
     373             :  * \param qpair queue to poll
     374             :  * \param status completion status. The user must fill this structure with zeroes before calling
     375             :  * this function
     376             :  * \param timeout_in_usecs optional timeout
     377             :  *
     378             :  * \return 0 if command completed without error,
     379             :  * -EIO if command completed with error,
     380             :  * -ECANCELED if command is not completed due to transport/device error or time expired
     381             :  *
     382             :  * The command to wait upon must be submitted with nvme_completion_poll_cb as the callback
     383             :  * and status as the callback argument.
     384             :  */
     385             : int
     386           3 : nvme_wait_for_completion_timeout(struct spdk_nvme_qpair *qpair,
     387             :                                  struct nvme_completion_poll_status *status,
     388             :                                  uint64_t timeout_in_usecs)
     389             : {
     390           3 :         return nvme_wait_for_completion_robust_lock_timeout(qpair, status, NULL, timeout_in_usecs);
     391             : }
     392             : 
     393             : static void
     394           3 : nvme_user_copy_cmd_complete(void *arg, const struct spdk_nvme_cpl *cpl)
     395             : {
     396           3 :         struct nvme_request *req = arg;
     397             :         spdk_nvme_cmd_cb user_cb_fn;
     398             :         void *user_cb_arg;
     399             :         enum spdk_nvme_data_transfer xfer;
     400             : 
     401           3 :         if (req->user_buffer && req->payload_size) {
     402             :                 /* Copy back to the user buffer */
     403           2 :                 assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_CONTIG);
     404           2 :                 xfer = spdk_nvme_opc_get_data_transfer(req->cmd.opc);
     405           2 :                 if (xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST ||
     406             :                     xfer == SPDK_NVME_DATA_BIDIRECTIONAL) {
     407           1 :                         assert(req->pid == getpid());
     408           1 :                         memcpy(req->user_buffer, req->payload.contig_or_cb_arg, req->payload_size);
     409             :                 }
     410             :         }
     411             : 
     412           3 :         user_cb_fn = req->user_cb_fn;
     413           3 :         user_cb_arg = req->user_cb_arg;
     414           3 :         nvme_cleanup_user_req(req);
     415             : 
     416             :         /* Call the user's original callback now that the buffer has been copied */
     417           3 :         user_cb_fn(user_cb_arg, cpl);
     418             : 
     419           3 : }
     420             : 
     421             : /**
     422             :  * Allocate a request as well as a DMA-capable buffer to copy to/from the user's buffer.
     423             :  *
     424             :  * This is intended for use in non-fast-path functions (admin commands, reservations, etc.)
     425             :  * where the overhead of a copy is not a problem.
     426             :  */
     427             : struct nvme_request *
     428          14 : nvme_allocate_request_user_copy(struct spdk_nvme_qpair *qpair,
     429             :                                 void *buffer, uint32_t payload_size, spdk_nvme_cmd_cb cb_fn,
     430             :                                 void *cb_arg, bool host_to_controller)
     431             : {
     432             :         struct nvme_request *req;
     433          14 :         void *dma_buffer = NULL;
     434             : 
     435          14 :         if (buffer && payload_size) {
     436          13 :                 dma_buffer = spdk_zmalloc(payload_size, 4096, NULL,
     437             :                                           SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
     438          13 :                 if (!dma_buffer) {
     439           1 :                         return NULL;
     440             :                 }
     441             : 
     442          12 :                 if (host_to_controller) {
     443           8 :                         memcpy(dma_buffer, buffer, payload_size);
     444             :                 }
     445             :         }
     446             : 
     447          13 :         req = nvme_allocate_request_contig(qpair, dma_buffer, payload_size, nvme_user_copy_cmd_complete,
     448             :                                            NULL);
     449          13 :         if (!req) {
     450           1 :                 spdk_free(dma_buffer);
     451           1 :                 return NULL;
     452             :         }
     453             : 
     454          12 :         req->user_cb_fn = cb_fn;
     455          12 :         req->user_cb_arg = cb_arg;
     456          12 :         req->user_buffer = buffer;
     457          12 :         req->cb_arg = req;
     458             : 
     459          12 :         return req;
     460             : }
     461             : 
     462             : /**
     463             :  * Check if a request has exceeded the controller timeout.
     464             :  *
     465             :  * \param req request to check for timeout.
     466             :  * \param cid command ID for command submitted by req (will be passed to timeout_cb_fn)
     467             :  * \param active_proc per-process data for the controller associated with req
     468             :  * \param now_tick current time from spdk_get_ticks()
     469             :  * \return 0 if requests submitted more recently than req should still be checked for timeouts, or
     470             :  * 1 if requests newer than req need not be checked.
     471             :  *
     472             :  * The request's timeout callback will be called if needed; the caller is only responsible for
     473             :  * calling this function on each outstanding request.
     474             :  */
     475             : int
     476           6 : nvme_request_check_timeout(struct nvme_request *req, uint16_t cid,
     477             :                            struct spdk_nvme_ctrlr_process *active_proc,
     478             :                            uint64_t now_tick)
     479             : {
     480           6 :         struct spdk_nvme_qpair *qpair = req->qpair;
     481           6 :         struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
     482           6 :         uint64_t timeout_ticks = nvme_qpair_is_admin_queue(qpair) ?
     483           6 :                                  active_proc->timeout_admin_ticks : active_proc->timeout_io_ticks;
     484             : 
     485           6 :         assert(active_proc->timeout_cb_fn != NULL);
     486             : 
     487           6 :         if (req->timed_out || req->submit_tick == 0) {
     488           2 :                 return 0;
     489             :         }
     490             : 
     491           4 :         if (req->pid != g_spdk_nvme_pid) {
     492           1 :                 return 0;
     493             :         }
     494             : 
     495           3 :         if (nvme_qpair_is_admin_queue(qpair) &&
     496           1 :             req->cmd.opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST) {
     497           1 :                 return 0;
     498             :         }
     499             : 
     500           2 :         if (req->submit_tick + timeout_ticks > now_tick) {
     501           1 :                 return 1;
     502             :         }
     503             : 
     504           1 :         req->timed_out = true;
     505             : 
     506             :         /*
     507             :          * We don't want to expose the admin queue to the user,
     508             :          * so when we're timing out admin commands set the
     509             :          * qpair to NULL.
     510             :          */
     511           1 :         active_proc->timeout_cb_fn(active_proc->timeout_cb_arg, ctrlr,
     512           1 :                                    nvme_qpair_is_admin_queue(qpair) ? NULL : qpair,
     513             :                                    cid);
     514           1 :         return 0;
     515             : }
     516             : 
     517             : int
     518           7 : nvme_robust_mutex_init_shared(pthread_mutex_t *mtx)
     519             : {
     520           7 :         int rc = 0;
     521             : 
     522             : #ifdef __FreeBSD__
     523             :         pthread_mutex_init(mtx, NULL);
     524             : #else
     525           7 :         pthread_mutexattr_t attr;
     526             : 
     527           7 :         if (pthread_mutexattr_init(&attr)) {
     528           2 :                 return -1;
     529             :         }
     530          10 :         if (pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) ||
     531          10 :             pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST) ||
     532           5 :             pthread_mutex_init(mtx, &attr)) {
     533           1 :                 rc = -1;
     534             :         }
     535           5 :         pthread_mutexattr_destroy(&attr);
     536             : #endif
     537             : 
     538           5 :         return rc;
     539             : }
     540             : 
     541             : int
     542          23 : nvme_driver_init(void)
     543             : {
     544             :         static pthread_mutex_t g_init_mutex = PTHREAD_MUTEX_INITIALIZER;
     545          23 :         int ret = 0;
     546             :         /* Any socket ID */
     547          23 :         int socket_id = -1;
     548             : 
     549             :         /* Use a special process-private mutex to ensure the global
     550             :          * nvme driver object (g_spdk_nvme_driver) gets initialized by
     551             :          * only one thread.  Once that object is established and its
     552             :          * mutex is initialized, we can unlock this mutex and use that
     553             :          * one instead.
     554             :          */
     555          23 :         pthread_mutex_lock(&g_init_mutex);
     556             : 
     557             :         /* Each process needs its own pid. */
     558          23 :         g_spdk_nvme_pid = getpid();
     559             : 
     560             :         /*
     561             :          * Only one thread from one process will do this driver init work.
     562             :          * The primary process will reserve the shared memory and do the
     563             :          *  initialization.
     564             :          * The secondary process will lookup the existing reserved memory.
     565             :          */
     566          23 :         if (spdk_process_is_primary()) {
     567             :                 /* The unique named memzone already reserved. */
     568          11 :                 if (g_spdk_nvme_driver != NULL) {
     569           6 :                         pthread_mutex_unlock(&g_init_mutex);
     570           6 :                         return 0;
     571             :                 } else {
     572           5 :                         g_spdk_nvme_driver = spdk_memzone_reserve(SPDK_NVME_DRIVER_NAME,
     573             :                                              sizeof(struct nvme_driver), socket_id,
     574             :                                              SPDK_MEMZONE_NO_IOVA_CONTIG);
     575             :                 }
     576             : 
     577           5 :                 if (g_spdk_nvme_driver == NULL) {
     578           1 :                         SPDK_ERRLOG("primary process failed to reserve memory\n");
     579           1 :                         pthread_mutex_unlock(&g_init_mutex);
     580           1 :                         return -1;
     581             :                 }
     582             :         } else {
     583          12 :                 g_spdk_nvme_driver = spdk_memzone_lookup(SPDK_NVME_DRIVER_NAME);
     584             : 
     585             :                 /* The unique named memzone already reserved by the primary process. */
     586          12 :                 if (g_spdk_nvme_driver != NULL) {
     587           9 :                         int ms_waited = 0;
     588             : 
     589             :                         /* Wait the nvme driver to get initialized. */
     590         109 :                         while ((g_spdk_nvme_driver->initialized == false) &&
     591         101 :                                (ms_waited < g_nvme_driver_timeout_ms)) {
     592         100 :                                 ms_waited++;
     593         100 :                                 nvme_delay(1000); /* delay 1ms */
     594             :                         }
     595           9 :                         if (g_spdk_nvme_driver->initialized == false) {
     596           1 :                                 SPDK_ERRLOG("timeout waiting for primary process to init\n");
     597           1 :                                 pthread_mutex_unlock(&g_init_mutex);
     598           1 :                                 return -1;
     599             :                         }
     600             :                 } else {
     601           3 :                         SPDK_ERRLOG("primary process is not started yet\n");
     602           3 :                         pthread_mutex_unlock(&g_init_mutex);
     603           3 :                         return -1;
     604             :                 }
     605             : 
     606           8 :                 pthread_mutex_unlock(&g_init_mutex);
     607           8 :                 return 0;
     608             :         }
     609             : 
     610             :         /*
     611             :          * At this moment, only one thread from the primary process will do
     612             :          * the g_spdk_nvme_driver initialization
     613             :          */
     614           4 :         assert(spdk_process_is_primary());
     615             : 
     616           4 :         ret = nvme_robust_mutex_init_shared(&g_spdk_nvme_driver->lock);
     617           4 :         if (ret != 0) {
     618           1 :                 SPDK_ERRLOG("failed to initialize mutex\n");
     619           1 :                 spdk_memzone_free(SPDK_NVME_DRIVER_NAME);
     620           1 :                 pthread_mutex_unlock(&g_init_mutex);
     621           1 :                 return ret;
     622             :         }
     623             : 
     624             :         /* The lock in the shared g_spdk_nvme_driver object is now ready to
     625             :          * be used - so we can unlock the g_init_mutex here.
     626             :          */
     627           3 :         pthread_mutex_unlock(&g_init_mutex);
     628           3 :         nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
     629             : 
     630           3 :         g_spdk_nvme_driver->initialized = false;
     631           3 :         g_spdk_nvme_driver->hotplug_fd = spdk_pci_event_listen();
     632           3 :         if (g_spdk_nvme_driver->hotplug_fd < 0) {
     633           0 :                 SPDK_DEBUGLOG(nvme, "Failed to open uevent netlink socket\n");
     634             :         }
     635             : 
     636           3 :         TAILQ_INIT(&g_spdk_nvme_driver->shared_attached_ctrlrs);
     637             : 
     638           3 :         spdk_uuid_generate(&g_spdk_nvme_driver->default_extended_host_id);
     639             : 
     640           3 :         nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
     641             : 
     642           3 :         return ret;
     643             : }
     644             : 
     645             : /* This function must only be called while holding g_spdk_nvme_driver->lock */
     646             : int
     647           5 : nvme_ctrlr_probe(const struct spdk_nvme_transport_id *trid,
     648             :                  struct spdk_nvme_probe_ctx *probe_ctx, void *devhandle)
     649             : {
     650             :         struct spdk_nvme_ctrlr *ctrlr;
     651           5 :         struct spdk_nvme_ctrlr_opts opts;
     652             : 
     653           5 :         assert(trid != NULL);
     654             : 
     655           5 :         spdk_nvme_ctrlr_get_default_ctrlr_opts(&opts, sizeof(opts));
     656             : 
     657           5 :         if (!probe_ctx->probe_cb || probe_ctx->probe_cb(probe_ctx->cb_ctx, trid, &opts)) {
     658           4 :                 ctrlr = nvme_get_ctrlr_by_trid_unsafe(trid);
     659           4 :                 if (ctrlr) {
     660             :                         /* This ctrlr already exists. */
     661             : 
     662           0 :                         if (ctrlr->is_destructed) {
     663             :                                 /* This ctrlr is being destructed asynchronously. */
     664           0 :                                 SPDK_ERRLOG("NVMe controller for SSD: %s is being destructed\n",
     665             :                                             trid->traddr);
     666           0 :                                 return -EBUSY;
     667             :                         }
     668             : 
     669             :                         /* Increase the ref count before calling attach_cb() as the user may
     670             :                         * call nvme_detach() immediately. */
     671           0 :                         nvme_ctrlr_proc_get_ref(ctrlr);
     672             : 
     673           0 :                         if (probe_ctx->attach_cb) {
     674           0 :                                 nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
     675           0 :                                 probe_ctx->attach_cb(probe_ctx->cb_ctx, &ctrlr->trid, ctrlr, &ctrlr->opts);
     676           0 :                                 nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
     677             :                         }
     678           0 :                         return 0;
     679             :                 }
     680             : 
     681           4 :                 ctrlr = nvme_transport_ctrlr_construct(trid, &opts, devhandle);
     682           4 :                 if (ctrlr == NULL) {
     683           2 :                         SPDK_ERRLOG("Failed to construct NVMe controller for SSD: %s\n", trid->traddr);
     684           2 :                         return -1;
     685             :                 }
     686           2 :                 ctrlr->remove_cb = probe_ctx->remove_cb;
     687           2 :                 ctrlr->cb_ctx = probe_ctx->cb_ctx;
     688             : 
     689           2 :                 nvme_qpair_set_state(ctrlr->adminq, NVME_QPAIR_ENABLED);
     690           2 :                 TAILQ_INSERT_TAIL(&probe_ctx->init_ctrlrs, ctrlr, tailq);
     691           2 :                 return 0;
     692             :         }
     693             : 
     694           1 :         return 1;
     695             : }
     696             : 
     697             : static void
     698           3 : nvme_ctrlr_poll_internal(struct spdk_nvme_ctrlr *ctrlr,
     699             :                          struct spdk_nvme_probe_ctx *probe_ctx)
     700             : {
     701           3 :         int rc = 0;
     702             : 
     703           3 :         rc = nvme_ctrlr_process_init(ctrlr);
     704             : 
     705           3 :         if (rc) {
     706             :                 /* Controller failed to initialize. */
     707           1 :                 TAILQ_REMOVE(&probe_ctx->init_ctrlrs, ctrlr, tailq);
     708           1 :                 SPDK_ERRLOG("Failed to initialize SSD: %s\n", ctrlr->trid.traddr);
     709           1 :                 nvme_ctrlr_lock(ctrlr);
     710           1 :                 nvme_ctrlr_fail(ctrlr, false);
     711           1 :                 nvme_ctrlr_unlock(ctrlr);
     712           1 :                 nvme_ctrlr_destruct(ctrlr);
     713           1 :                 return;
     714             :         }
     715             : 
     716           2 :         if (ctrlr->state != NVME_CTRLR_STATE_READY) {
     717           0 :                 return;
     718             :         }
     719             : 
     720           2 :         STAILQ_INIT(&ctrlr->io_producers);
     721             : 
     722             :         /*
     723             :          * Controller has been initialized.
     724             :          *  Move it to the attached_ctrlrs list.
     725             :          */
     726           2 :         TAILQ_REMOVE(&probe_ctx->init_ctrlrs, ctrlr, tailq);
     727             : 
     728           2 :         nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
     729           2 :         if (nvme_ctrlr_shared(ctrlr)) {
     730           1 :                 TAILQ_INSERT_TAIL(&g_spdk_nvme_driver->shared_attached_ctrlrs, ctrlr, tailq);
     731             :         } else {
     732           1 :                 TAILQ_INSERT_TAIL(&g_nvme_attached_ctrlrs, ctrlr, tailq);
     733             :         }
     734             : 
     735             :         /*
     736             :          * Increase the ref count before calling attach_cb() as the user may
     737             :          * call nvme_detach() immediately.
     738             :          */
     739           2 :         nvme_ctrlr_proc_get_ref(ctrlr);
     740           2 :         nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
     741             : 
     742           2 :         if (probe_ctx->attach_cb) {
     743           0 :                 probe_ctx->attach_cb(probe_ctx->cb_ctx, &ctrlr->trid, ctrlr, &ctrlr->opts);
     744             :         }
     745             : }
     746             : 
     747             : static int
     748          14 : nvme_init_controllers(struct spdk_nvme_probe_ctx *probe_ctx)
     749             : {
     750          14 :         int rc = 0;
     751             : 
     752             :         while (true) {
     753          14 :                 rc = spdk_nvme_probe_poll_async(probe_ctx);
     754          14 :                 if (rc != -EAGAIN) {
     755          14 :                         return rc;
     756             :                 }
     757             :         }
     758             : 
     759             :         return rc;
     760             : }
     761             : 
     762             : /* This function must not be called while holding g_spdk_nvme_driver->lock */
     763             : static struct spdk_nvme_ctrlr *
     764          13 : nvme_get_ctrlr_by_trid(const struct spdk_nvme_transport_id *trid)
     765             : {
     766             :         struct spdk_nvme_ctrlr *ctrlr;
     767             : 
     768          13 :         nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
     769          13 :         ctrlr = nvme_get_ctrlr_by_trid_unsafe(trid);
     770          13 :         nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
     771             : 
     772          13 :         return ctrlr;
     773             : }
     774             : 
     775             : /* This function must be called while holding g_spdk_nvme_driver->lock */
     776             : struct spdk_nvme_ctrlr *
     777          17 : nvme_get_ctrlr_by_trid_unsafe(const struct spdk_nvme_transport_id *trid)
     778             : {
     779             :         struct spdk_nvme_ctrlr *ctrlr;
     780             : 
     781             :         /* Search per-process list */
     782          17 :         TAILQ_FOREACH(ctrlr, &g_nvme_attached_ctrlrs, tailq) {
     783           0 :                 if (spdk_nvme_transport_id_compare(&ctrlr->trid, trid) == 0) {
     784           0 :                         return ctrlr;
     785             :                 }
     786             :         }
     787             : 
     788             :         /* Search multi-process shared list */
     789          17 :         TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->shared_attached_ctrlrs, tailq) {
     790          12 :                 if (spdk_nvme_transport_id_compare(&ctrlr->trid, trid) == 0) {
     791          12 :                         return ctrlr;
     792             :                 }
     793             :         }
     794             : 
     795           5 :         return NULL;
     796             : }
     797             : 
     798             : /* This function must only be called while holding g_spdk_nvme_driver->lock */
     799             : static int
     800          13 : nvme_probe_internal(struct spdk_nvme_probe_ctx *probe_ctx,
     801             :                     bool direct_connect)
     802             : {
     803             :         int rc;
     804             :         struct spdk_nvme_ctrlr *ctrlr, *ctrlr_tmp;
     805             : 
     806          13 :         if (strlen(probe_ctx->trid.trstring) == 0) {
     807             :                 /* If user didn't provide trstring, derive it from trtype */
     808          10 :                 spdk_nvme_trid_populate_transport(&probe_ctx->trid, probe_ctx->trid.trtype);
     809             :         }
     810             : 
     811          13 :         if (!spdk_nvme_transport_available_by_name(probe_ctx->trid.trstring)) {
     812           1 :                 SPDK_ERRLOG("NVMe trtype %u (%s) not available\n",
     813             :                             probe_ctx->trid.trtype, probe_ctx->trid.trstring);
     814           1 :                 return -1;
     815             :         }
     816             : 
     817          12 :         nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
     818             : 
     819          12 :         rc = nvme_transport_ctrlr_scan(probe_ctx, direct_connect);
     820          12 :         if (rc != 0) {
     821           1 :                 SPDK_ERRLOG("NVMe ctrlr scan failed\n");
     822           2 :                 TAILQ_FOREACH_SAFE(ctrlr, &probe_ctx->init_ctrlrs, tailq, ctrlr_tmp) {
     823           1 :                         TAILQ_REMOVE(&probe_ctx->init_ctrlrs, ctrlr, tailq);
     824           1 :                         nvme_transport_ctrlr_destruct(ctrlr);
     825             :                 }
     826           1 :                 nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
     827           1 :                 return -1;
     828             :         }
     829             : 
     830             :         /*
     831             :          * Probe controllers on the shared_attached_ctrlrs list
     832             :          */
     833          11 :         if (!spdk_process_is_primary() && (probe_ctx->trid.trtype == SPDK_NVME_TRANSPORT_PCIE)) {
     834          13 :                 TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->shared_attached_ctrlrs, tailq) {
     835             :                         /* Do not attach other ctrlrs if user specify a valid trid */
     836          11 :                         if ((strlen(probe_ctx->trid.traddr) != 0) &&
     837           5 :                             (spdk_nvme_transport_id_compare(&probe_ctx->trid, &ctrlr->trid))) {
     838           0 :                                 continue;
     839             :                         }
     840             : 
     841             :                         /* Do not attach if we failed to initialize it in this process */
     842           6 :                         if (nvme_ctrlr_get_current_process(ctrlr) == NULL) {
     843           0 :                                 continue;
     844             :                         }
     845             : 
     846           6 :                         nvme_ctrlr_proc_get_ref(ctrlr);
     847             : 
     848             :                         /*
     849             :                          * Unlock while calling attach_cb() so the user can call other functions
     850             :                          *  that may take the driver lock, like nvme_detach().
     851             :                          */
     852           6 :                         if (probe_ctx->attach_cb) {
     853           1 :                                 nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
     854           1 :                                 probe_ctx->attach_cb(probe_ctx->cb_ctx, &ctrlr->trid, ctrlr, &ctrlr->opts);
     855           1 :                                 nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
     856             :                         }
     857             :                 }
     858             :         }
     859             : 
     860          11 :         nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
     861             : 
     862          11 :         return 0;
     863             : }
     864             : 
     865             : static void
     866          16 : nvme_probe_ctx_init(struct spdk_nvme_probe_ctx *probe_ctx,
     867             :                     const struct spdk_nvme_transport_id *trid,
     868             :                     void *cb_ctx,
     869             :                     spdk_nvme_probe_cb probe_cb,
     870             :                     spdk_nvme_attach_cb attach_cb,
     871             :                     spdk_nvme_remove_cb remove_cb)
     872             : {
     873          16 :         probe_ctx->trid = *trid;
     874          16 :         probe_ctx->cb_ctx = cb_ctx;
     875          16 :         probe_ctx->probe_cb = probe_cb;
     876          16 :         probe_ctx->attach_cb = attach_cb;
     877          16 :         probe_ctx->remove_cb = remove_cb;
     878          16 :         TAILQ_INIT(&probe_ctx->init_ctrlrs);
     879          16 : }
     880             : 
     881             : int
     882           4 : spdk_nvme_probe(const struct spdk_nvme_transport_id *trid, void *cb_ctx,
     883             :                 spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb,
     884             :                 spdk_nvme_remove_cb remove_cb)
     885             : {
     886           4 :         struct spdk_nvme_transport_id trid_pcie;
     887             :         struct spdk_nvme_probe_ctx *probe_ctx;
     888             : 
     889           4 :         if (trid == NULL) {
     890           4 :                 memset(&trid_pcie, 0, sizeof(trid_pcie));
     891           4 :                 spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE);
     892           4 :                 trid = &trid_pcie;
     893             :         }
     894             : 
     895           4 :         probe_ctx = spdk_nvme_probe_async(trid, cb_ctx, probe_cb,
     896             :                                           attach_cb, remove_cb);
     897           4 :         if (!probe_ctx) {
     898           2 :                 SPDK_ERRLOG("Create probe context failed\n");
     899           2 :                 return -1;
     900             :         }
     901             : 
     902             :         /*
     903             :          * Keep going even if one or more nvme_attach() calls failed,
     904             :          *  but maintain the value of rc to signal errors when we return.
     905             :          */
     906           2 :         return nvme_init_controllers(probe_ctx);
     907             : }
     908             : 
     909             : static bool
     910           4 : nvme_connect_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
     911             :                       struct spdk_nvme_ctrlr_opts *opts)
     912             : {
     913           4 :         struct spdk_nvme_ctrlr_opts *requested_opts = cb_ctx;
     914             : 
     915           4 :         assert(requested_opts);
     916           4 :         memcpy(opts, requested_opts, sizeof(*opts));
     917             : 
     918           4 :         return true;
     919             : }
     920             : 
     921             : static void
     922           4 : nvme_ctrlr_opts_init(struct spdk_nvme_ctrlr_opts *opts,
     923             :                      const struct spdk_nvme_ctrlr_opts *opts_user,
     924             :                      size_t opts_size_user)
     925             : {
     926           4 :         assert(opts);
     927           4 :         assert(opts_user);
     928             : 
     929           4 :         spdk_nvme_ctrlr_get_default_ctrlr_opts(opts, opts_size_user);
     930             : 
     931             : #define FIELD_OK(field) \
     932             :         offsetof(struct spdk_nvme_ctrlr_opts, field) + sizeof(opts->field) <= (opts->opts_size)
     933             : 
     934             : #define SET_FIELD(field) \
     935             :         if (FIELD_OK(field)) { \
     936             :                         opts->field = opts_user->field; \
     937             :         }
     938             : 
     939             : #define SET_FIELD_ARRAY(field) \
     940             :         if (FIELD_OK(field)) { \
     941             :                 memcpy(opts->field, opts_user->field, sizeof(opts_user->field)); \
     942             :         }
     943             : 
     944           4 :         SET_FIELD(num_io_queues);
     945           4 :         SET_FIELD(use_cmb_sqs);
     946           4 :         SET_FIELD(no_shn_notification);
     947           4 :         SET_FIELD(arb_mechanism);
     948           4 :         SET_FIELD(arbitration_burst);
     949           4 :         SET_FIELD(low_priority_weight);
     950           4 :         SET_FIELD(medium_priority_weight);
     951           4 :         SET_FIELD(high_priority_weight);
     952           4 :         SET_FIELD(keep_alive_timeout_ms);
     953           4 :         SET_FIELD(transport_retry_count);
     954           4 :         SET_FIELD(io_queue_size);
     955           4 :         SET_FIELD_ARRAY(hostnqn);
     956           4 :         SET_FIELD(io_queue_requests);
     957           4 :         SET_FIELD_ARRAY(src_addr);
     958           4 :         SET_FIELD_ARRAY(src_svcid);
     959           4 :         SET_FIELD_ARRAY(host_id);
     960           4 :         SET_FIELD_ARRAY(extended_host_id);
     961           4 :         SET_FIELD(command_set);
     962           4 :         SET_FIELD(admin_timeout_ms);
     963           4 :         SET_FIELD(header_digest);
     964           4 :         SET_FIELD(data_digest);
     965           4 :         SET_FIELD(disable_error_logging);
     966           4 :         SET_FIELD(transport_ack_timeout);
     967           4 :         SET_FIELD(admin_queue_size);
     968           4 :         SET_FIELD(fabrics_connect_timeout_us);
     969           4 :         SET_FIELD(disable_read_ana_log_page);
     970           4 :         SET_FIELD(disable_read_changed_ns_list_log_page);
     971           4 :         SET_FIELD_ARRAY(psk);
     972             : 
     973             : #undef FIELD_OK
     974             : #undef SET_FIELD
     975             : #undef SET_FIELD_ARRAY
     976           4 : }
     977             : 
     978             : struct spdk_nvme_ctrlr *
     979          11 : spdk_nvme_connect(const struct spdk_nvme_transport_id *trid,
     980             :                   const struct spdk_nvme_ctrlr_opts *opts, size_t opts_size)
     981             : {
     982             :         int rc;
     983          11 :         struct spdk_nvme_ctrlr *ctrlr = NULL;
     984             :         struct spdk_nvme_probe_ctx *probe_ctx;
     985          11 :         struct spdk_nvme_ctrlr_opts *opts_local_p = NULL;
     986          11 :         struct spdk_nvme_ctrlr_opts opts_local;
     987             : 
     988          11 :         if (trid == NULL) {
     989           1 :                 SPDK_ERRLOG("No transport ID specified\n");
     990           1 :                 return NULL;
     991             :         }
     992             : 
     993          10 :         if (opts) {
     994           4 :                 opts_local_p = &opts_local;
     995           4 :                 nvme_ctrlr_opts_init(opts_local_p, opts, opts_size);
     996             :         }
     997             : 
     998          10 :         probe_ctx = spdk_nvme_connect_async(trid, opts_local_p, NULL);
     999          10 :         if (!probe_ctx) {
    1000           1 :                 SPDK_ERRLOG("Create probe context failed\n");
    1001           1 :                 return NULL;
    1002             :         }
    1003             : 
    1004           9 :         rc = nvme_init_controllers(probe_ctx);
    1005           9 :         if (rc != 0) {
    1006           0 :                 return NULL;
    1007             :         }
    1008             : 
    1009           9 :         ctrlr = nvme_get_ctrlr_by_trid(trid);
    1010             : 
    1011           9 :         return ctrlr;
    1012             : }
    1013             : 
    1014             : void
    1015          14 : spdk_nvme_trid_populate_transport(struct spdk_nvme_transport_id *trid,
    1016             :                                   enum spdk_nvme_transport_type trtype)
    1017             : {
    1018             :         const char *trstring;
    1019             : 
    1020          14 :         trid->trtype = trtype;
    1021          14 :         switch (trtype) {
    1022           0 :         case SPDK_NVME_TRANSPORT_FC:
    1023           0 :                 trstring = SPDK_NVME_TRANSPORT_NAME_FC;
    1024           0 :                 break;
    1025          14 :         case SPDK_NVME_TRANSPORT_PCIE:
    1026          14 :                 trstring = SPDK_NVME_TRANSPORT_NAME_PCIE;
    1027          14 :                 break;
    1028           0 :         case SPDK_NVME_TRANSPORT_RDMA:
    1029           0 :                 trstring = SPDK_NVME_TRANSPORT_NAME_RDMA;
    1030           0 :                 break;
    1031           0 :         case SPDK_NVME_TRANSPORT_TCP:
    1032           0 :                 trstring = SPDK_NVME_TRANSPORT_NAME_TCP;
    1033           0 :                 break;
    1034           0 :         case SPDK_NVME_TRANSPORT_VFIOUSER:
    1035           0 :                 trstring = SPDK_NVME_TRANSPORT_NAME_VFIOUSER;
    1036           0 :                 break;
    1037           0 :         case SPDK_NVME_TRANSPORT_CUSTOM:
    1038           0 :                 trstring = SPDK_NVME_TRANSPORT_NAME_CUSTOM;
    1039           0 :                 break;
    1040           0 :         default:
    1041           0 :                 SPDK_ERRLOG("no available transports\n");
    1042           0 :                 assert(0);
    1043             :                 return;
    1044             :         }
    1045          14 :         snprintf(trid->trstring, SPDK_NVMF_TRSTRING_MAX_LEN, "%s", trstring);
    1046          14 : }
    1047             : 
    1048             : int
    1049          11 : spdk_nvme_transport_id_populate_trstring(struct spdk_nvme_transport_id *trid, const char *trstring)
    1050             : {
    1051          11 :         int i = 0;
    1052             : 
    1053             :         /* Note: gcc-11 has some false positive -Wstringop-overread warnings with LTO builds if we
    1054             :          * use strnlen here.  So do the trstring copy manually instead.  See GitHub issue #2391.
    1055             :          */
    1056             : 
    1057             :         /* cast official trstring to uppercase version of input. */
    1058          54 :         while (i < SPDK_NVMF_TRSTRING_MAX_LEN && trstring[i] != 0) {
    1059          43 :                 trid->trstring[i] = toupper(trstring[i]);
    1060          43 :                 i++;
    1061             :         }
    1062             : 
    1063          11 :         if (trstring[i] != 0) {
    1064           0 :                 return -EINVAL;
    1065             :         } else {
    1066          11 :                 trid->trstring[i] = 0;
    1067          11 :                 return 0;
    1068             :         }
    1069             : }
    1070             : 
    1071             : int
    1072          22 : spdk_nvme_transport_id_parse_trtype(enum spdk_nvme_transport_type *trtype, const char *str)
    1073             : {
    1074          22 :         if (trtype == NULL || str == NULL) {
    1075           2 :                 return -EINVAL;
    1076             :         }
    1077             : 
    1078          20 :         if (strcasecmp(str, "PCIe") == 0) {
    1079          11 :                 *trtype = SPDK_NVME_TRANSPORT_PCIE;
    1080           9 :         } else if (strcasecmp(str, "RDMA") == 0) {
    1081           3 :                 *trtype = SPDK_NVME_TRANSPORT_RDMA;
    1082           6 :         } else if (strcasecmp(str, "FC") == 0) {
    1083           2 :                 *trtype = SPDK_NVME_TRANSPORT_FC;
    1084           4 :         } else if (strcasecmp(str, "TCP") == 0) {
    1085           3 :                 *trtype = SPDK_NVME_TRANSPORT_TCP;
    1086           1 :         } else if (strcasecmp(str, "VFIOUSER") == 0) {
    1087           0 :                 *trtype = SPDK_NVME_TRANSPORT_VFIOUSER;
    1088             :         } else {
    1089           1 :                 *trtype = SPDK_NVME_TRANSPORT_CUSTOM;
    1090             :         }
    1091          20 :         return 0;
    1092             : }
    1093             : 
    1094             : const char *
    1095           5 : spdk_nvme_transport_id_trtype_str(enum spdk_nvme_transport_type trtype)
    1096             : {
    1097           5 :         switch (trtype) {
    1098           1 :         case SPDK_NVME_TRANSPORT_PCIE:
    1099           1 :                 return "PCIe";
    1100           1 :         case SPDK_NVME_TRANSPORT_RDMA:
    1101           1 :                 return "RDMA";
    1102           1 :         case SPDK_NVME_TRANSPORT_FC:
    1103           1 :                 return "FC";
    1104           1 :         case SPDK_NVME_TRANSPORT_TCP:
    1105           1 :                 return "TCP";
    1106           0 :         case SPDK_NVME_TRANSPORT_VFIOUSER:
    1107           0 :                 return "VFIOUSER";
    1108           0 :         case SPDK_NVME_TRANSPORT_CUSTOM:
    1109           0 :                 return "CUSTOM";
    1110           1 :         default:
    1111           1 :                 return NULL;
    1112             :         }
    1113             : }
    1114             : 
    1115             : int
    1116          13 : spdk_nvme_transport_id_parse_adrfam(enum spdk_nvmf_adrfam *adrfam, const char *str)
    1117             : {
    1118          13 :         if (adrfam == NULL || str == NULL) {
    1119           2 :                 return -EINVAL;
    1120             :         }
    1121             : 
    1122          11 :         if (strcasecmp(str, "IPv4") == 0) {
    1123           4 :                 *adrfam = SPDK_NVMF_ADRFAM_IPV4;
    1124           7 :         } else if (strcasecmp(str, "IPv6") == 0) {
    1125           2 :                 *adrfam = SPDK_NVMF_ADRFAM_IPV6;
    1126           5 :         } else if (strcasecmp(str, "IB") == 0) {
    1127           2 :                 *adrfam = SPDK_NVMF_ADRFAM_IB;
    1128           3 :         } else if (strcasecmp(str, "FC") == 0) {
    1129           2 :                 *adrfam = SPDK_NVMF_ADRFAM_FC;
    1130             :         } else {
    1131           1 :                 return -ENOENT;
    1132             :         }
    1133          10 :         return 0;
    1134             : }
    1135             : 
    1136             : const char *
    1137           5 : spdk_nvme_transport_id_adrfam_str(enum spdk_nvmf_adrfam adrfam)
    1138             : {
    1139           5 :         switch (adrfam) {
    1140           1 :         case SPDK_NVMF_ADRFAM_IPV4:
    1141           1 :                 return "IPv4";
    1142           1 :         case SPDK_NVMF_ADRFAM_IPV6:
    1143           1 :                 return "IPv6";
    1144           1 :         case SPDK_NVMF_ADRFAM_IB:
    1145           1 :                 return "IB";
    1146           1 :         case SPDK_NVMF_ADRFAM_FC:
    1147           1 :                 return "FC";
    1148           1 :         default:
    1149           1 :                 return NULL;
    1150             :         }
    1151             : }
    1152             : 
    1153             : static size_t
    1154          38 : parse_next_key(const char **str, char *key, char *val, size_t key_buf_size, size_t val_buf_size)
    1155             : {
    1156             : 
    1157             :         const char *sep, *sep1;
    1158          38 :         const char *whitespace = " \t\n";
    1159             :         size_t key_len, val_len;
    1160             : 
    1161          38 :         *str += strspn(*str, whitespace);
    1162             : 
    1163          38 :         sep = strchr(*str, ':');
    1164          38 :         if (!sep) {
    1165           1 :                 sep = strchr(*str, '=');
    1166           1 :                 if (!sep) {
    1167           1 :                         SPDK_ERRLOG("Key without ':' or '=' separator\n");
    1168           1 :                         return 0;
    1169             :                 }
    1170             :         } else {
    1171          37 :                 sep1 = strchr(*str, '=');
    1172          37 :                 if ((sep1 != NULL) && (sep1 < sep)) {
    1173           4 :                         sep = sep1;
    1174             :                 }
    1175             :         }
    1176             : 
    1177          37 :         key_len = sep - *str;
    1178          37 :         if (key_len >= key_buf_size) {
    1179           1 :                 SPDK_ERRLOG("Key length %zu greater than maximum allowed %zu\n",
    1180             :                             key_len, key_buf_size - 1);
    1181           1 :                 return 0;
    1182             :         }
    1183             : 
    1184          36 :         memcpy(key, *str, key_len);
    1185          36 :         key[key_len] = '\0';
    1186             : 
    1187          36 :         *str += key_len + 1; /* Skip key: */
    1188          36 :         val_len = strcspn(*str, whitespace);
    1189          36 :         if (val_len == 0) {
    1190           1 :                 SPDK_ERRLOG("Key without value\n");
    1191           1 :                 return 0;
    1192             :         }
    1193             : 
    1194          35 :         if (val_len >= val_buf_size) {
    1195           0 :                 SPDK_ERRLOG("Value length %zu greater than maximum allowed %zu\n",
    1196             :                             val_len, val_buf_size - 1);
    1197           0 :                 return 0;
    1198             :         }
    1199             : 
    1200          35 :         memcpy(val, *str, val_len);
    1201          35 :         val[val_len] = '\0';
    1202             : 
    1203          35 :         *str += val_len;
    1204             : 
    1205          35 :         return val_len;
    1206             : }
    1207             : 
    1208             : int
    1209          17 : spdk_nvme_transport_id_parse(struct spdk_nvme_transport_id *trid, const char *str)
    1210             : {
    1211             :         size_t val_len;
    1212          17 :         char key[32];
    1213          17 :         char val[1024];
    1214             : 
    1215          17 :         if (trid == NULL || str == NULL) {
    1216           3 :                 return -EINVAL;
    1217             :         }
    1218             : 
    1219          43 :         while (*str != '\0') {
    1220             : 
    1221          32 :                 val_len = parse_next_key(&str, key, val, sizeof(key), sizeof(val));
    1222             : 
    1223          32 :                 if (val_len == 0) {
    1224           3 :                         SPDK_ERRLOG("Failed to parse transport ID\n");
    1225           3 :                         return -EINVAL;
    1226             :                 }
    1227             : 
    1228          29 :                 if (strcasecmp(key, "trtype") == 0) {
    1229          11 :                         if (spdk_nvme_transport_id_populate_trstring(trid, val) != 0) {
    1230           0 :                                 SPDK_ERRLOG("invalid transport '%s'\n", val);
    1231           0 :                                 return -EINVAL;
    1232             :                         }
    1233          11 :                         if (spdk_nvme_transport_id_parse_trtype(&trid->trtype, val) != 0) {
    1234           0 :                                 SPDK_ERRLOG("Unknown trtype '%s'\n", val);
    1235           0 :                                 return -EINVAL;
    1236             :                         }
    1237          18 :                 } else if (strcasecmp(key, "adrfam") == 0) {
    1238           2 :                         if (spdk_nvme_transport_id_parse_adrfam(&trid->adrfam, val) != 0) {
    1239           0 :                                 SPDK_ERRLOG("Unknown adrfam '%s'\n", val);
    1240           0 :                                 return -EINVAL;
    1241             :                         }
    1242          16 :                 } else if (strcasecmp(key, "traddr") == 0) {
    1243          11 :                         if (val_len > SPDK_NVMF_TRADDR_MAX_LEN) {
    1244           0 :                                 SPDK_ERRLOG("traddr length %zu greater than maximum allowed %u\n",
    1245             :                                             val_len, SPDK_NVMF_TRADDR_MAX_LEN);
    1246           0 :                                 return -EINVAL;
    1247             :                         }
    1248          11 :                         memcpy(trid->traddr, val, val_len + 1);
    1249           5 :                 } else if (strcasecmp(key, "trsvcid") == 0) {
    1250           2 :                         if (val_len > SPDK_NVMF_TRSVCID_MAX_LEN) {
    1251           0 :                                 SPDK_ERRLOG("trsvcid length %zu greater than maximum allowed %u\n",
    1252             :                                             val_len, SPDK_NVMF_TRSVCID_MAX_LEN);
    1253           0 :                                 return -EINVAL;
    1254             :                         }
    1255           2 :                         memcpy(trid->trsvcid, val, val_len + 1);
    1256           3 :                 } else if (strcasecmp(key, "priority") == 0) {
    1257           1 :                         if (val_len > SPDK_NVMF_PRIORITY_MAX_LEN) {
    1258           0 :                                 SPDK_ERRLOG("priority length %zu greater than maximum allowed %u\n",
    1259             :                                             val_len, SPDK_NVMF_PRIORITY_MAX_LEN);
    1260           0 :                                 return -EINVAL;
    1261             :                         }
    1262           1 :                         trid->priority = spdk_strtol(val, 10);
    1263           2 :                 } else if (strcasecmp(key, "subnqn") == 0) {
    1264           2 :                         if (val_len > SPDK_NVMF_NQN_MAX_LEN) {
    1265           0 :                                 SPDK_ERRLOG("subnqn length %zu greater than maximum allowed %u\n",
    1266             :                                             val_len, SPDK_NVMF_NQN_MAX_LEN);
    1267           0 :                                 return -EINVAL;
    1268             :                         }
    1269           2 :                         memcpy(trid->subnqn, val, val_len + 1);
    1270           0 :                 } else if (strcasecmp(key, "hostaddr") == 0) {
    1271           0 :                         continue;
    1272           0 :                 } else if (strcasecmp(key, "hostsvcid") == 0) {
    1273           0 :                         continue;
    1274           0 :                 } else if (strcasecmp(key, "hostnqn") == 0) {
    1275           0 :                         continue;
    1276           0 :                 } else if (strcasecmp(key, "ns") == 0) {
    1277             :                         /*
    1278             :                          * Special case.  The namespace id parameter may
    1279             :                          * optionally be passed in the transport id string
    1280             :                          * for an SPDK application (e.g. spdk_nvme_perf)
    1281             :                          * and additionally parsed therein to limit
    1282             :                          * targeting a specific namespace.  For this
    1283             :                          * scenario, just silently ignore this key
    1284             :                          * rather than letting it default to logging
    1285             :                          * it as an invalid key.
    1286             :                          */
    1287           0 :                         continue;
    1288           0 :                 } else if (strcasecmp(key, "alt_traddr") == 0) {
    1289             :                         /*
    1290             :                          * Used by applications for enabling transport ID failover.
    1291             :                          * Please see the case above for more information on custom parameters.
    1292             :                          */
    1293           0 :                         continue;
    1294             :                 } else {
    1295           0 :                         SPDK_ERRLOG("Unknown transport ID key '%s'\n", key);
    1296             :                 }
    1297             :         }
    1298             : 
    1299          11 :         return 0;
    1300             : }
    1301             : 
    1302             : int
    1303           3 : spdk_nvme_host_id_parse(struct spdk_nvme_host_id *hostid, const char *str)
    1304           3 : {
    1305             : 
    1306           3 :         size_t key_size = 32;
    1307           3 :         size_t val_size = 1024;
    1308             :         size_t val_len;
    1309           3 :         char key[key_size];
    1310           3 :         char val[val_size];
    1311             : 
    1312           3 :         if (hostid == NULL || str == NULL) {
    1313           0 :                 return -EINVAL;
    1314             :         }
    1315             : 
    1316           6 :         while (*str != '\0') {
    1317             : 
    1318           3 :                 val_len = parse_next_key(&str, key, val, key_size, val_size);
    1319             : 
    1320           3 :                 if (val_len == 0) {
    1321           0 :                         SPDK_ERRLOG("Failed to parse host ID\n");
    1322           0 :                         return val_len;
    1323             :                 }
    1324             : 
    1325             :                 /* Ignore the rest of the options from the transport ID. */
    1326           3 :                 if (strcasecmp(key, "trtype") == 0) {
    1327           1 :                         continue;
    1328           2 :                 } else if (strcasecmp(key, "adrfam") == 0) {
    1329           0 :                         continue;
    1330           2 :                 } else if (strcasecmp(key, "traddr") == 0) {
    1331           0 :                         continue;
    1332           2 :                 } else if (strcasecmp(key, "trsvcid") == 0) {
    1333           0 :                         continue;
    1334           2 :                 } else if (strcasecmp(key, "subnqn") == 0) {
    1335           0 :                         continue;
    1336           2 :                 } else if (strcasecmp(key, "priority") == 0) {
    1337           0 :                         continue;
    1338           2 :                 } else if (strcasecmp(key, "ns") == 0) {
    1339           0 :                         continue;
    1340           2 :                 } else if (strcasecmp(key, "hostaddr") == 0) {
    1341           1 :                         if (val_len > SPDK_NVMF_TRADDR_MAX_LEN) {
    1342           0 :                                 SPDK_ERRLOG("hostaddr length %zu greater than maximum allowed %u\n",
    1343             :                                             val_len, SPDK_NVMF_TRADDR_MAX_LEN);
    1344           0 :                                 return -EINVAL;
    1345             :                         }
    1346           1 :                         memcpy(hostid->hostaddr, val, val_len + 1);
    1347             : 
    1348           1 :                 } else if (strcasecmp(key, "hostsvcid") == 0) {
    1349           1 :                         if (val_len > SPDK_NVMF_TRSVCID_MAX_LEN) {
    1350           0 :                                 SPDK_ERRLOG("trsvcid length %zu greater than maximum allowed %u\n",
    1351             :                                             val_len, SPDK_NVMF_TRSVCID_MAX_LEN);
    1352           0 :                                 return -EINVAL;
    1353             :                         }
    1354           1 :                         memcpy(hostid->hostsvcid, val, val_len + 1);
    1355             :                 } else {
    1356           0 :                         SPDK_ERRLOG("Unknown transport ID key '%s'\n", key);
    1357             :                 }
    1358             :         }
    1359             : 
    1360           3 :         return 0;
    1361             : }
    1362             : 
    1363             : static int
    1364          35 : cmp_int(int a, int b)
    1365             : {
    1366          35 :         return a - b;
    1367             : }
    1368             : 
    1369             : int
    1370          29 : spdk_nvme_transport_id_compare(const struct spdk_nvme_transport_id *trid1,
    1371             :                                const struct spdk_nvme_transport_id *trid2)
    1372             : {
    1373             :         int cmp;
    1374             : 
    1375          29 :         if (trid1->trtype == SPDK_NVME_TRANSPORT_CUSTOM) {
    1376           0 :                 cmp = strcasecmp(trid1->trstring, trid2->trstring);
    1377             :         } else {
    1378          29 :                 cmp = cmp_int(trid1->trtype, trid2->trtype);
    1379             :         }
    1380             : 
    1381          29 :         if (cmp) {
    1382           1 :                 return cmp;
    1383             :         }
    1384             : 
    1385          28 :         if (trid1->trtype == SPDK_NVME_TRANSPORT_PCIE) {
    1386          21 :                 struct spdk_pci_addr pci_addr1 = {};
    1387          21 :                 struct spdk_pci_addr pci_addr2 = {};
    1388             : 
    1389             :                 /* Normalize PCI addresses before comparing */
    1390          42 :                 if (spdk_pci_addr_parse(&pci_addr1, trid1->traddr) < 0 ||
    1391          21 :                     spdk_pci_addr_parse(&pci_addr2, trid2->traddr) < 0) {
    1392           0 :                         return -1;
    1393             :                 }
    1394             : 
    1395             :                 /* PCIe transport ID only uses trtype and traddr */
    1396          21 :                 return spdk_pci_addr_compare(&pci_addr1, &pci_addr2);
    1397             :         }
    1398             : 
    1399           7 :         cmp = strcasecmp(trid1->traddr, trid2->traddr);
    1400           7 :         if (cmp) {
    1401           1 :                 return cmp;
    1402             :         }
    1403             : 
    1404           6 :         cmp = cmp_int(trid1->adrfam, trid2->adrfam);
    1405           6 :         if (cmp) {
    1406           1 :                 return cmp;
    1407             :         }
    1408             : 
    1409           5 :         cmp = strcasecmp(trid1->trsvcid, trid2->trsvcid);
    1410           5 :         if (cmp) {
    1411           1 :                 return cmp;
    1412             :         }
    1413             : 
    1414           4 :         cmp = strcmp(trid1->subnqn, trid2->subnqn);
    1415           4 :         if (cmp) {
    1416           2 :                 return cmp;
    1417             :         }
    1418             : 
    1419           2 :         return 0;
    1420             : }
    1421             : 
    1422             : int
    1423           4 : spdk_nvme_prchk_flags_parse(uint32_t *prchk_flags, const char *str)
    1424             : {
    1425             :         size_t val_len;
    1426           4 :         char key[32];
    1427           4 :         char val[1024];
    1428             : 
    1429           4 :         if (prchk_flags == NULL || str == NULL) {
    1430           1 :                 return -EINVAL;
    1431             :         }
    1432             : 
    1433           6 :         while (*str != '\0') {
    1434           3 :                 val_len = parse_next_key(&str, key, val, sizeof(key), sizeof(val));
    1435             : 
    1436           3 :                 if (val_len == 0) {
    1437           0 :                         SPDK_ERRLOG("Failed to parse prchk\n");
    1438           0 :                         return -EINVAL;
    1439             :                 }
    1440             : 
    1441           3 :                 if (strcasecmp(key, "prchk") == 0) {
    1442           3 :                         if (strcasestr(val, "reftag") != NULL) {
    1443           2 :                                 *prchk_flags |= SPDK_NVME_IO_FLAGS_PRCHK_REFTAG;
    1444             :                         }
    1445           3 :                         if (strcasestr(val, "guard") != NULL) {
    1446           2 :                                 *prchk_flags |= SPDK_NVME_IO_FLAGS_PRCHK_GUARD;
    1447             :                         }
    1448             :                 } else {
    1449           0 :                         SPDK_ERRLOG("Unknown key '%s'\n", key);
    1450           0 :                         return -EINVAL;
    1451             :                 }
    1452             :         }
    1453             : 
    1454           3 :         return 0;
    1455             : }
    1456             : 
    1457             : const char *
    1458           3 : spdk_nvme_prchk_flags_str(uint32_t prchk_flags)
    1459             : {
    1460           3 :         if (prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) {
    1461           2 :                 if (prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) {
    1462           1 :                         return "prchk:reftag|guard";
    1463             :                 } else {
    1464           1 :                         return "prchk:reftag";
    1465             :                 }
    1466             :         } else {
    1467           1 :                 if (prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) {
    1468           1 :                         return "prchk:guard";
    1469             :                 } else {
    1470           0 :                         return NULL;
    1471             :                 }
    1472             :         }
    1473             : }
    1474             : 
    1475             : struct spdk_nvme_probe_ctx *
    1476           4 : spdk_nvme_probe_async(const struct spdk_nvme_transport_id *trid,
    1477             :                       void *cb_ctx,
    1478             :                       spdk_nvme_probe_cb probe_cb,
    1479             :                       spdk_nvme_attach_cb attach_cb,
    1480             :                       spdk_nvme_remove_cb remove_cb)
    1481             : {
    1482             :         int rc;
    1483             :         struct spdk_nvme_probe_ctx *probe_ctx;
    1484             : 
    1485           4 :         rc = nvme_driver_init();
    1486           4 :         if (rc != 0) {
    1487           1 :                 return NULL;
    1488             :         }
    1489             : 
    1490           3 :         probe_ctx = calloc(1, sizeof(*probe_ctx));
    1491           3 :         if (!probe_ctx) {
    1492           0 :                 return NULL;
    1493             :         }
    1494             : 
    1495           3 :         nvme_probe_ctx_init(probe_ctx, trid, cb_ctx, probe_cb, attach_cb, remove_cb);
    1496           3 :         rc = nvme_probe_internal(probe_ctx, false);
    1497           3 :         if (rc != 0) {
    1498           1 :                 free(probe_ctx);
    1499           1 :                 return NULL;
    1500             :         }
    1501             : 
    1502           2 :         return probe_ctx;
    1503             : }
    1504             : 
    1505             : int
    1506          14 : spdk_nvme_probe_poll_async(struct spdk_nvme_probe_ctx *probe_ctx)
    1507             : {
    1508             :         struct spdk_nvme_ctrlr *ctrlr, *ctrlr_tmp;
    1509             : 
    1510          14 :         if (!spdk_process_is_primary() && probe_ctx->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
    1511           7 :                 free(probe_ctx);
    1512           7 :                 return 0;
    1513             :         }
    1514             : 
    1515          10 :         TAILQ_FOREACH_SAFE(ctrlr, &probe_ctx->init_ctrlrs, tailq, ctrlr_tmp) {
    1516           3 :                 nvme_ctrlr_poll_internal(ctrlr, probe_ctx);
    1517             :         }
    1518             : 
    1519           7 :         if (TAILQ_EMPTY(&probe_ctx->init_ctrlrs)) {
    1520           7 :                 nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
    1521           7 :                 g_spdk_nvme_driver->initialized = true;
    1522           7 :                 nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
    1523           7 :                 free(probe_ctx);
    1524           7 :                 return 0;
    1525             :         }
    1526             : 
    1527           0 :         return -EAGAIN;
    1528             : }
    1529             : 
    1530             : struct spdk_nvme_probe_ctx *
    1531          10 : spdk_nvme_connect_async(const struct spdk_nvme_transport_id *trid,
    1532             :                         const struct spdk_nvme_ctrlr_opts *opts,
    1533             :                         spdk_nvme_attach_cb attach_cb)
    1534             : {
    1535             :         int rc;
    1536          10 :         spdk_nvme_probe_cb probe_cb = NULL;
    1537             :         struct spdk_nvme_probe_ctx *probe_ctx;
    1538             : 
    1539          10 :         rc = nvme_driver_init();
    1540          10 :         if (rc != 0) {
    1541           1 :                 return NULL;
    1542             :         }
    1543             : 
    1544           9 :         probe_ctx = calloc(1, sizeof(*probe_ctx));
    1545           9 :         if (!probe_ctx) {
    1546           0 :                 return NULL;
    1547             :         }
    1548             : 
    1549           9 :         if (opts) {
    1550           4 :                 probe_cb = nvme_connect_probe_cb;
    1551             :         }
    1552             : 
    1553           9 :         nvme_probe_ctx_init(probe_ctx, trid, (void *)opts, probe_cb, attach_cb, NULL);
    1554           9 :         rc = nvme_probe_internal(probe_ctx, true);
    1555           9 :         if (rc != 0) {
    1556           0 :                 free(probe_ctx);
    1557           0 :                 return NULL;
    1558             :         }
    1559             : 
    1560           9 :         return probe_ctx;
    1561             : }
    1562             : 
    1563             : int
    1564           2 : nvme_parse_addr(struct sockaddr_storage *sa, int family, const char *addr, const char *service,
    1565             :                 long int *port)
    1566             : {
    1567           2 :         struct addrinfo *res;
    1568           2 :         struct addrinfo hints;
    1569             :         int ret;
    1570             : 
    1571           2 :         memset(&hints, 0, sizeof(hints));
    1572           2 :         hints.ai_family = family;
    1573           2 :         hints.ai_socktype = SOCK_STREAM;
    1574           2 :         hints.ai_protocol = 0;
    1575             : 
    1576           2 :         if (addr == NULL || service == NULL) {
    1577           1 :                 SPDK_ERRLOG("addr and service must both be non-NULL\n");
    1578           1 :                 return -EINVAL;
    1579             :         }
    1580             : 
    1581           1 :         *port = spdk_strtol(service, 10);
    1582           1 :         if (*port <= 0 || *port >= 65536) {
    1583           0 :                 SPDK_ERRLOG("Invalid port: %s\n", service);
    1584           0 :                 return -EINVAL;
    1585             :         }
    1586             : 
    1587           1 :         ret = getaddrinfo(addr, service, &hints, &res);
    1588           1 :         if (ret) {
    1589           0 :                 SPDK_ERRLOG("getaddrinfo failed: %s (%d)\n", gai_strerror(ret), ret);
    1590           0 :                 return -(abs(ret));
    1591             :         }
    1592             : 
    1593           1 :         if (res->ai_addrlen > sizeof(*sa)) {
    1594           0 :                 SPDK_ERRLOG("getaddrinfo() ai_addrlen %zu too large\n", (size_t)res->ai_addrlen);
    1595           0 :                 ret = -EINVAL;
    1596             :         } else {
    1597           1 :                 memcpy(sa, res->ai_addr, res->ai_addrlen);
    1598             :         }
    1599             : 
    1600           1 :         freeaddrinfo(res);
    1601           1 :         return ret;
    1602             : }
    1603             : 
    1604             : #ifndef SPDK_CONFIG_RDMA
    1605             : void
    1606             : spdk_nvme_rdma_init_hooks(struct spdk_nvme_rdma_hooks *hooks)
    1607             : {
    1608             :         SPDK_ERRLOG("spdk_nvme_rdma_init_hooks() is unsupported: RDMA transport is not available\n");
    1609             :         abort();
    1610             : }
    1611             : #endif
    1612             : 
    1613           3 : SPDK_LOG_REGISTER_COMPONENT(nvme)

Generated by: LCOV version 1.15