LCOV - code coverage report
Current view: top level - lib/nvme - nvme_transport.c (source / functions) Hit Total Coverage
Test: ut_cov_unit.info Lines: 57 379 15.0 %
Date: 2024-07-15 09:35:21 Functions: 6 56 10.7 %

          Line data    Source code
       1             : /*   SPDX-License-Identifier: BSD-3-Clause
       2             :  *   Copyright (C) 2016 Intel Corporation.
       3             :  *   All rights reserved.
       4             :  *   Copyright (c) 2021 Mellanox Technologies LTD. All rights reserved.
       5             :  *   Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
       6             :  */
       7             : 
       8             : /*
       9             :  * NVMe transport abstraction
      10             :  */
      11             : 
      12             : #include "nvme_internal.h"
      13             : #include "spdk/queue.h"
      14             : 
      15             : #define SPDK_MAX_NUM_OF_TRANSPORTS 16
      16             : 
      17             : struct spdk_nvme_transport {
      18             :         struct spdk_nvme_transport_ops  ops;
      19             :         TAILQ_ENTRY(spdk_nvme_transport)        link;
      20             : };
      21             : 
      22             : TAILQ_HEAD(nvme_transport_list, spdk_nvme_transport) g_spdk_nvme_transports =
      23             :         TAILQ_HEAD_INITIALIZER(g_spdk_nvme_transports);
      24             : 
      25             : struct spdk_nvme_transport g_spdk_transports[SPDK_MAX_NUM_OF_TRANSPORTS] = {};
      26             : int g_current_transport_index = 0;
      27             : 
      28             : struct spdk_nvme_transport_opts g_spdk_nvme_transport_opts = {
      29             :         .rdma_srq_size = 0,
      30             :         .rdma_max_cq_size = 0,
      31             :         .rdma_cm_event_timeout_ms = 1000
      32             : };
      33             : 
      34             : const struct spdk_nvme_transport *
      35           0 : nvme_get_first_transport(void)
      36             : {
      37           0 :         return TAILQ_FIRST(&g_spdk_nvme_transports);
      38             : }
      39             : 
      40             : const struct spdk_nvme_transport *
      41           0 : nvme_get_next_transport(const struct spdk_nvme_transport *transport)
      42             : {
      43           0 :         return TAILQ_NEXT(transport, link);
      44             : }
      45             : 
      46             : /*
      47             :  * Unfortunately, due to NVMe PCIe multiprocess support, we cannot store the
      48             :  * transport object in either the controller struct or the admin qpair. This means
      49             :  * that a lot of admin related transport calls will have to call nvme_get_transport
      50             :  * in order to know which functions to call.
      51             :  * In the I/O path, we have the ability to store the transport struct in the I/O
      52             :  * qpairs to avoid taking a performance hit.
      53             :  */
      54             : const struct spdk_nvme_transport *
      55           4 : nvme_get_transport(const char *transport_name)
      56             : {
      57             :         struct spdk_nvme_transport *registered_transport;
      58             : 
      59           4 :         TAILQ_FOREACH(registered_transport, &g_spdk_nvme_transports, link) {
      60           3 :                 if (strcasecmp(transport_name, registered_transport->ops.name) == 0) {
      61           3 :                         return registered_transport;
      62             :                 }
      63             :         }
      64             : 
      65           1 :         return NULL;
      66             : }
      67             : 
      68             : bool
      69           0 : spdk_nvme_transport_available(enum spdk_nvme_transport_type trtype)
      70             : {
      71           0 :         return nvme_get_transport(spdk_nvme_transport_id_trtype_str(trtype)) == NULL ? false : true;
      72             : }
      73             : 
      74             : bool
      75           0 : spdk_nvme_transport_available_by_name(const char *transport_name)
      76             : {
      77           0 :         return nvme_get_transport(transport_name) == NULL ? false : true;
      78             : }
      79             : 
      80             : void
      81           0 : spdk_nvme_transport_register(const struct spdk_nvme_transport_ops *ops)
      82             : {
      83             :         struct spdk_nvme_transport *new_transport;
      84             : 
      85           0 :         if (nvme_get_transport(ops->name)) {
      86           0 :                 SPDK_ERRLOG("Double registering NVMe transport %s is prohibited.\n", ops->name);
      87           0 :                 assert(false);
      88             :         }
      89             : 
      90           0 :         if (g_current_transport_index == SPDK_MAX_NUM_OF_TRANSPORTS) {
      91           0 :                 SPDK_ERRLOG("Unable to register new NVMe transport.\n");
      92           0 :                 assert(false);
      93             :                 return;
      94             :         }
      95           0 :         new_transport = &g_spdk_transports[g_current_transport_index++];
      96             : 
      97           0 :         new_transport->ops = *ops;
      98           0 :         TAILQ_INSERT_TAIL(&g_spdk_nvme_transports, new_transport, link);
      99             : }
     100             : 
     101           0 : struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
     102             :                 const struct spdk_nvme_ctrlr_opts *opts,
     103             :                 void *devhandle)
     104             : {
     105           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(trid->trstring);
     106             :         struct spdk_nvme_ctrlr *ctrlr;
     107             : 
     108           0 :         if (transport == NULL) {
     109           0 :                 SPDK_ERRLOG("Transport %s doesn't exist.", trid->trstring);
     110           0 :                 return NULL;
     111             :         }
     112             : 
     113           0 :         ctrlr = transport->ops.ctrlr_construct(trid, opts, devhandle);
     114             : 
     115           0 :         return ctrlr;
     116             : }
     117             : 
     118             : int
     119           0 : nvme_transport_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,
     120             :                           bool direct_connect)
     121             : {
     122           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(probe_ctx->trid.trstring);
     123             : 
     124           0 :         if (transport == NULL) {
     125           0 :                 SPDK_ERRLOG("Transport %s doesn't exist.", probe_ctx->trid.trstring);
     126           0 :                 return -ENOENT;
     127             :         }
     128             : 
     129           0 :         return transport->ops.ctrlr_scan(probe_ctx, direct_connect);
     130             : }
     131             : 
     132             : int
     133           0 : nvme_transport_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
     134             : {
     135           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     136             : 
     137           0 :         assert(transport != NULL);
     138           0 :         return transport->ops.ctrlr_destruct(ctrlr);
     139             : }
     140             : 
     141             : int
     142           0 : nvme_transport_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr)
     143             : {
     144           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     145             : 
     146           0 :         assert(transport != NULL);
     147           0 :         return transport->ops.ctrlr_enable(ctrlr);
     148             : }
     149             : 
     150             : int
     151           0 : nvme_transport_ctrlr_ready(struct spdk_nvme_ctrlr *ctrlr)
     152             : {
     153           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     154             : 
     155           0 :         assert(transport != NULL);
     156           0 :         if (transport->ops.ctrlr_ready) {
     157           0 :                 return transport->ops.ctrlr_ready(ctrlr);
     158             :         }
     159             : 
     160           0 :         return 0;
     161             : }
     162             : 
     163             : int
     164           0 : nvme_transport_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value)
     165             : {
     166           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     167             : 
     168           0 :         assert(transport != NULL);
     169           0 :         return transport->ops.ctrlr_set_reg_4(ctrlr, offset, value);
     170             : }
     171             : 
     172             : int
     173           0 : nvme_transport_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value)
     174             : {
     175           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     176             : 
     177           0 :         assert(transport != NULL);
     178           0 :         return transport->ops.ctrlr_set_reg_8(ctrlr, offset, value);
     179             : }
     180             : 
     181             : int
     182           0 : nvme_transport_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value)
     183             : {
     184           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     185             : 
     186           0 :         assert(transport != NULL);
     187           0 :         return transport->ops.ctrlr_get_reg_4(ctrlr, offset, value);
     188             : }
     189             : 
     190             : int
     191           0 : nvme_transport_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value)
     192             : {
     193           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     194             : 
     195           0 :         assert(transport != NULL);
     196           0 :         return transport->ops.ctrlr_get_reg_8(ctrlr, offset, value);
     197             : }
     198             : 
     199             : static int
     200           0 : nvme_queue_register_operation_completion(struct spdk_nvme_ctrlr *ctrlr, uint64_t value,
     201             :                 spdk_nvme_reg_cb cb_fn, void *cb_ctx)
     202             : {
     203             :         struct nvme_register_completion *ctx;
     204             : 
     205           0 :         ctx = spdk_zmalloc(sizeof(*ctx), 0, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE);
     206           0 :         if (ctx == NULL) {
     207           0 :                 return -ENOMEM;
     208             :         }
     209             : 
     210           0 :         ctx->cpl.status.sct = SPDK_NVME_SCT_GENERIC;
     211           0 :         ctx->cpl.status.sc = SPDK_NVME_SC_SUCCESS;
     212           0 :         ctx->cb_fn = cb_fn;
     213           0 :         ctx->cb_ctx = cb_ctx;
     214           0 :         ctx->value = value;
     215           0 :         ctx->pid = getpid();
     216             : 
     217           0 :         nvme_ctrlr_lock(ctrlr);
     218           0 :         STAILQ_INSERT_TAIL(&ctrlr->register_operations, ctx, stailq);
     219           0 :         nvme_ctrlr_unlock(ctrlr);
     220             : 
     221           0 :         return 0;
     222             : }
     223             : 
     224             : int
     225           0 : nvme_transport_ctrlr_set_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value,
     226             :                                      spdk_nvme_reg_cb cb_fn, void *cb_arg)
     227             : {
     228           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     229             :         int rc;
     230             : 
     231           0 :         assert(transport != NULL);
     232           0 :         if (transport->ops.ctrlr_set_reg_4_async == NULL) {
     233           0 :                 rc = transport->ops.ctrlr_set_reg_4(ctrlr, offset, value);
     234           0 :                 if (rc != 0) {
     235           0 :                         return rc;
     236             :                 }
     237             : 
     238           0 :                 return nvme_queue_register_operation_completion(ctrlr, value, cb_fn, cb_arg);
     239             :         }
     240             : 
     241           0 :         return transport->ops.ctrlr_set_reg_4_async(ctrlr, offset, value, cb_fn, cb_arg);
     242             : }
     243             : 
     244             : int
     245           0 : nvme_transport_ctrlr_set_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value,
     246             :                                      spdk_nvme_reg_cb cb_fn, void *cb_arg)
     247             : 
     248             : {
     249           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     250             :         int rc;
     251             : 
     252           0 :         assert(transport != NULL);
     253           0 :         if (transport->ops.ctrlr_set_reg_8_async == NULL) {
     254           0 :                 rc = transport->ops.ctrlr_set_reg_8(ctrlr, offset, value);
     255           0 :                 if (rc != 0) {
     256           0 :                         return rc;
     257             :                 }
     258             : 
     259           0 :                 return nvme_queue_register_operation_completion(ctrlr, value, cb_fn, cb_arg);
     260             :         }
     261             : 
     262           0 :         return transport->ops.ctrlr_set_reg_8_async(ctrlr, offset, value, cb_fn, cb_arg);
     263             : }
     264             : 
     265             : int
     266           0 : nvme_transport_ctrlr_get_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
     267             :                                      spdk_nvme_reg_cb cb_fn, void *cb_arg)
     268             : {
     269           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     270           0 :         uint32_t value;
     271             :         int rc;
     272             : 
     273           0 :         assert(transport != NULL);
     274           0 :         if (transport->ops.ctrlr_get_reg_4_async == NULL) {
     275           0 :                 rc = transport->ops.ctrlr_get_reg_4(ctrlr, offset, &value);
     276           0 :                 if (rc != 0) {
     277           0 :                         return rc;
     278             :                 }
     279             : 
     280           0 :                 return nvme_queue_register_operation_completion(ctrlr, value, cb_fn, cb_arg);
     281             :         }
     282             : 
     283           0 :         return transport->ops.ctrlr_get_reg_4_async(ctrlr, offset, cb_fn, cb_arg);
     284             : }
     285             : 
     286             : int
     287           0 : nvme_transport_ctrlr_get_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset,
     288             :                                      spdk_nvme_reg_cb cb_fn, void *cb_arg)
     289             : {
     290           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     291           0 :         uint64_t value;
     292             :         int rc;
     293             : 
     294           0 :         assert(transport != NULL);
     295           0 :         if (transport->ops.ctrlr_get_reg_8_async == NULL) {
     296           0 :                 rc = transport->ops.ctrlr_get_reg_8(ctrlr, offset, &value);
     297           0 :                 if (rc != 0) {
     298           0 :                         return rc;
     299             :                 }
     300             : 
     301           0 :                 return nvme_queue_register_operation_completion(ctrlr, value, cb_fn, cb_arg);
     302             :         }
     303             : 
     304           0 :         return transport->ops.ctrlr_get_reg_8_async(ctrlr, offset, cb_fn, cb_arg);
     305             : }
     306             : 
     307             : uint32_t
     308           0 : nvme_transport_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr)
     309             : {
     310           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     311             : 
     312           0 :         assert(transport != NULL);
     313           0 :         return transport->ops.ctrlr_get_max_xfer_size(ctrlr);
     314             : }
     315             : 
     316             : uint16_t
     317           0 : nvme_transport_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)
     318             : {
     319           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     320             : 
     321           0 :         assert(transport != NULL);
     322           0 :         return transport->ops.ctrlr_get_max_sges(ctrlr);
     323             : }
     324             : 
     325             : int
     326           0 : nvme_transport_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr)
     327             : {
     328           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     329             : 
     330           0 :         assert(transport != NULL);
     331           0 :         if (transport->ops.ctrlr_reserve_cmb != NULL) {
     332           0 :                 return transport->ops.ctrlr_reserve_cmb(ctrlr);
     333             :         }
     334             : 
     335           0 :         return -ENOTSUP;
     336             : }
     337             : 
     338             : void *
     339           0 : nvme_transport_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size)
     340             : {
     341           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     342             : 
     343           0 :         assert(transport != NULL);
     344           0 :         if (transport->ops.ctrlr_map_cmb != NULL) {
     345           0 :                 return transport->ops.ctrlr_map_cmb(ctrlr, size);
     346             :         }
     347             : 
     348           0 :         return NULL;
     349             : }
     350             : 
     351             : int
     352           0 : nvme_transport_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr)
     353             : {
     354           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     355             : 
     356           0 :         assert(transport != NULL);
     357           0 :         if (transport->ops.ctrlr_unmap_cmb != NULL) {
     358           0 :                 return transport->ops.ctrlr_unmap_cmb(ctrlr);
     359             :         }
     360             : 
     361           0 :         return 0;
     362             : }
     363             : 
     364             : int
     365           0 : nvme_transport_ctrlr_enable_pmr(struct spdk_nvme_ctrlr *ctrlr)
     366             : {
     367           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     368             : 
     369           0 :         assert(transport != NULL);
     370           0 :         if (transport->ops.ctrlr_enable_pmr != NULL) {
     371           0 :                 return transport->ops.ctrlr_enable_pmr(ctrlr);
     372             :         }
     373             : 
     374           0 :         return -ENOSYS;
     375             : }
     376             : 
     377             : int
     378           0 : nvme_transport_ctrlr_disable_pmr(struct spdk_nvme_ctrlr *ctrlr)
     379             : {
     380           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     381             : 
     382           0 :         assert(transport != NULL);
     383           0 :         if (transport->ops.ctrlr_disable_pmr != NULL) {
     384           0 :                 return transport->ops.ctrlr_disable_pmr(ctrlr);
     385             :         }
     386             : 
     387           0 :         return -ENOSYS;
     388             : }
     389             : 
     390             : void *
     391           0 : nvme_transport_ctrlr_map_pmr(struct spdk_nvme_ctrlr *ctrlr, size_t *size)
     392             : {
     393           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     394             : 
     395           0 :         assert(transport != NULL);
     396           0 :         if (transport->ops.ctrlr_map_pmr != NULL) {
     397           0 :                 return transport->ops.ctrlr_map_pmr(ctrlr, size);
     398             :         }
     399             : 
     400           0 :         return NULL;
     401             : }
     402             : 
     403             : int
     404           0 : nvme_transport_ctrlr_unmap_pmr(struct spdk_nvme_ctrlr *ctrlr)
     405             : {
     406           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     407             : 
     408           0 :         assert(transport != NULL);
     409           0 :         if (transport->ops.ctrlr_unmap_pmr != NULL) {
     410           0 :                 return transport->ops.ctrlr_unmap_pmr(ctrlr);
     411             :         }
     412             : 
     413           0 :         return -ENOSYS;
     414             : }
     415             : 
     416             : struct spdk_nvme_qpair *
     417           0 : nvme_transport_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid,
     418             :                                      const struct spdk_nvme_io_qpair_opts *opts)
     419             : {
     420             :         struct spdk_nvme_qpair *qpair;
     421           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     422             : 
     423           0 :         assert(transport != NULL);
     424           0 :         qpair = transport->ops.ctrlr_create_io_qpair(ctrlr, qid, opts);
     425           0 :         if (qpair != NULL && !nvme_qpair_is_admin_queue(qpair)) {
     426           0 :                 qpair->transport = transport;
     427             :         }
     428             : 
     429           0 :         return qpair;
     430             : }
     431             : 
     432             : void
     433           0 : nvme_transport_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
     434             : {
     435           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     436             :         int rc;
     437             : 
     438           0 :         assert(transport != NULL);
     439             : 
     440             :         /* Do not rely on qpair->transport.  For multi-process cases, a foreign process may delete
     441             :          * the IO qpair, in which case the transport object would be invalid (each process has their
     442             :          * own unique transport objects since they contain function pointers).  So we look up the
     443             :          * transport object in the delete_io_qpair case.
     444             :          */
     445           0 :         rc = transport->ops.ctrlr_delete_io_qpair(ctrlr, qpair);
     446           0 :         if (rc != 0) {
     447           0 :                 SPDK_ERRLOG("transport %s returned non-zero for ctrlr_delete_io_qpair op\n",
     448             :                             transport->ops.name);
     449           0 :                 assert(false);
     450             :         }
     451           0 : }
     452             : 
     453             : static void
     454           0 : nvme_transport_connect_qpair_fail(struct spdk_nvme_qpair *qpair, void *unused)
     455             : {
     456           0 :         struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
     457             : 
     458             :         /* If the qpair was unable to reconnect, restore the original failure reason */
     459           0 :         qpair->transport_failure_reason = qpair->last_transport_failure_reason;
     460           0 :         nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair);
     461           0 : }
     462             : 
     463             : int
     464           0 : nvme_transport_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
     465             : {
     466           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     467             :         int rc;
     468             : 
     469           0 :         assert(transport != NULL);
     470           0 :         if (!nvme_qpair_is_admin_queue(qpair) && qpair->transport == NULL) {
     471           0 :                 qpair->transport = transport;
     472             :         }
     473             : 
     474           0 :         qpair->last_transport_failure_reason = qpair->transport_failure_reason;
     475           0 :         qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_NONE;
     476             : 
     477           0 :         nvme_qpair_set_state(qpair, NVME_QPAIR_CONNECTING);
     478           0 :         rc = transport->ops.ctrlr_connect_qpair(ctrlr, qpair);
     479           0 :         if (rc != 0) {
     480           0 :                 goto err;
     481             :         }
     482             : 
     483           0 :         if (qpair->poll_group) {
     484           0 :                 rc = nvme_poll_group_connect_qpair(qpair);
     485           0 :                 if (rc) {
     486           0 :                         goto err;
     487             :                 }
     488             :         }
     489             : 
     490           0 :         if (!qpair->async) {
     491             :                 /* Busy wait until the qpair exits the connecting state */
     492           0 :                 while (nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING) {
     493           0 :                         if (qpair->poll_group && spdk_nvme_ctrlr_is_fabrics(ctrlr)) {
     494           0 :                                 rc = spdk_nvme_poll_group_process_completions(
     495           0 :                                              qpair->poll_group->group, 0,
     496             :                                              nvme_transport_connect_qpair_fail);
     497             :                         } else {
     498           0 :                                 rc = spdk_nvme_qpair_process_completions(qpair, 0);
     499             :                         }
     500             : 
     501           0 :                         if (rc < 0) {
     502           0 :                                 goto err;
     503             :                         }
     504             :                 }
     505             :         }
     506             : 
     507           0 :         return 0;
     508           0 : err:
     509           0 :         nvme_transport_connect_qpair_fail(qpair, NULL);
     510           0 :         if (nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTING) {
     511           0 :                 assert(qpair->async == true);
     512             :                 /* Let the caller to poll the qpair until it is actually disconnected. */
     513           0 :                 return 0;
     514             :         }
     515             : 
     516           0 :         return rc;
     517             : }
     518             : 
     519             : void
     520           0 : nvme_transport_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
     521             : {
     522           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     523             : 
     524           0 :         if (nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTING ||
     525           0 :             nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTED) {
     526           0 :                 return;
     527             :         }
     528             : 
     529           0 :         nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTING);
     530           0 :         assert(transport != NULL);
     531             : 
     532           0 :         if (qpair->poll_group && (qpair->active_proc == nvme_ctrlr_get_current_process(ctrlr))) {
     533           0 :                 nvme_poll_group_disconnect_qpair(qpair);
     534             :         }
     535             : 
     536           0 :         transport->ops.ctrlr_disconnect_qpair(ctrlr, qpair);
     537             : }
     538             : 
     539             : void
     540           0 : nvme_transport_ctrlr_disconnect_qpair_done(struct spdk_nvme_qpair *qpair)
     541             : {
     542           0 :         if (qpair->active_proc == nvme_ctrlr_get_current_process(qpair->ctrlr) ||
     543           0 :             nvme_qpair_is_admin_queue(qpair)) {
     544           0 :                 nvme_qpair_abort_all_queued_reqs(qpair);
     545             :         }
     546           0 :         nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTED);
     547           0 : }
     548             : 
     549             : int
     550           2 : nvme_transport_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
     551             :                                         struct spdk_memory_domain **domains, int array_size)
     552             : {
     553           2 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     554             : 
     555           2 :         assert(transport != NULL);
     556           2 :         if (transport->ops.ctrlr_get_memory_domains) {
     557           1 :                 return transport->ops.ctrlr_get_memory_domains(ctrlr, domains, array_size);
     558             :         }
     559             : 
     560           1 :         return 0;
     561             : }
     562             : 
     563             : void
     564           0 : nvme_transport_qpair_abort_reqs(struct spdk_nvme_qpair *qpair)
     565             : {
     566             :         const struct spdk_nvme_transport *transport;
     567             : 
     568           0 :         if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
     569           0 :                 qpair->transport->ops.qpair_abort_reqs(qpair, qpair->abort_dnr);
     570             :         } else {
     571           0 :                 transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
     572           0 :                 assert(transport != NULL);
     573           0 :                 transport->ops.qpair_abort_reqs(qpair, qpair->abort_dnr);
     574             :         }
     575           0 : }
     576             : 
     577             : int
     578           0 : nvme_transport_qpair_reset(struct spdk_nvme_qpair *qpair)
     579             : {
     580             :         const struct spdk_nvme_transport *transport;
     581             : 
     582           0 :         if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
     583           0 :                 return qpair->transport->ops.qpair_reset(qpair);
     584             :         }
     585             : 
     586           0 :         transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
     587           0 :         assert(transport != NULL);
     588           0 :         return transport->ops.qpair_reset(qpair);
     589             : }
     590             : 
     591             : int
     592           0 : nvme_transport_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
     593             : {
     594             :         const struct spdk_nvme_transport *transport;
     595             : 
     596           0 :         if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
     597           0 :                 return qpair->transport->ops.qpair_submit_request(qpair, req);
     598             :         }
     599             : 
     600           0 :         transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
     601           0 :         assert(transport != NULL);
     602           0 :         return transport->ops.qpair_submit_request(qpair, req);
     603             : }
     604             : 
     605             : int32_t
     606           0 : nvme_transport_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
     607             : {
     608             :         const struct spdk_nvme_transport *transport;
     609             : 
     610           0 :         if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
     611           0 :                 return qpair->transport->ops.qpair_process_completions(qpair, max_completions);
     612             :         }
     613             : 
     614           0 :         transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
     615           0 :         assert(transport != NULL);
     616           0 :         return transport->ops.qpair_process_completions(qpair, max_completions);
     617             : }
     618             : 
     619             : int
     620           0 : nvme_transport_qpair_iterate_requests(struct spdk_nvme_qpair *qpair,
     621             :                                       int (*iter_fn)(struct nvme_request *req, void *arg),
     622             :                                       void *arg)
     623             : {
     624             :         const struct spdk_nvme_transport *transport;
     625             : 
     626           0 :         if (spdk_likely(!nvme_qpair_is_admin_queue(qpair))) {
     627           0 :                 return qpair->transport->ops.qpair_iterate_requests(qpair, iter_fn, arg);
     628             :         }
     629             : 
     630           0 :         transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
     631           0 :         assert(transport != NULL);
     632           0 :         return transport->ops.qpair_iterate_requests(qpair, iter_fn, arg);
     633             : }
     634             : 
     635             : void
     636           0 : nvme_transport_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair)
     637             : {
     638           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(qpair->ctrlr->trid.trstring);
     639             : 
     640           0 :         assert(transport != NULL);
     641           0 :         transport->ops.admin_qpair_abort_aers(qpair);
     642           0 : }
     643             : 
     644             : struct spdk_nvme_transport_poll_group *
     645           0 : nvme_transport_poll_group_create(const struct spdk_nvme_transport *transport)
     646             : {
     647           0 :         struct spdk_nvme_transport_poll_group *group = NULL;
     648             : 
     649           0 :         group = transport->ops.poll_group_create();
     650           0 :         if (group) {
     651           0 :                 group->transport = transport;
     652           0 :                 STAILQ_INIT(&group->connected_qpairs);
     653           0 :                 STAILQ_INIT(&group->disconnected_qpairs);
     654           0 :                 group->num_connected_qpairs = 0;
     655             :         }
     656             : 
     657           0 :         return group;
     658             : }
     659             : 
     660             : struct spdk_nvme_transport_poll_group *
     661           0 : nvme_transport_qpair_get_optimal_poll_group(const struct spdk_nvme_transport *transport,
     662             :                 struct spdk_nvme_qpair *qpair)
     663             : {
     664           0 :         if (transport->ops.qpair_get_optimal_poll_group) {
     665           0 :                 return transport->ops.qpair_get_optimal_poll_group(qpair);
     666             :         } else {
     667           0 :                 return NULL;
     668             :         }
     669             : }
     670             : 
     671             : int
     672           1 : nvme_transport_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup,
     673             :                               struct spdk_nvme_qpair *qpair)
     674             : {
     675             :         int rc;
     676             : 
     677           1 :         rc = tgroup->transport->ops.poll_group_add(tgroup, qpair);
     678           1 :         if (rc == 0) {
     679           1 :                 qpair->poll_group = tgroup;
     680           1 :                 assert(nvme_qpair_get_state(qpair) < NVME_QPAIR_CONNECTED);
     681           1 :                 qpair->poll_group_tailq_head = &tgroup->disconnected_qpairs;
     682           1 :                 STAILQ_INSERT_TAIL(&tgroup->disconnected_qpairs, qpair, poll_group_stailq);
     683             :         }
     684             : 
     685           1 :         return rc;
     686             : }
     687             : 
     688             : int
     689           3 : nvme_transport_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,
     690             :                                  struct spdk_nvme_qpair *qpair)
     691             : {
     692             :         int rc __attribute__((unused));
     693             : 
     694           3 :         if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
     695           1 :                 return -EINVAL;
     696           2 :         } else if (qpair->poll_group_tailq_head != &tgroup->disconnected_qpairs) {
     697           1 :                 return -ENOENT;
     698             :         }
     699             : 
     700           1 :         rc = tgroup->transport->ops.poll_group_remove(tgroup, qpair);
     701           1 :         assert(rc == 0);
     702             : 
     703           1 :         STAILQ_REMOVE(&tgroup->disconnected_qpairs, qpair, spdk_nvme_qpair, poll_group_stailq);
     704             : 
     705           1 :         qpair->poll_group = NULL;
     706           1 :         qpair->poll_group_tailq_head = NULL;
     707             : 
     708           1 :         return 0;
     709             : }
     710             : 
     711             : int64_t
     712           0 : nvme_transport_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,
     713             :                 uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
     714             : {
     715           0 :         return tgroup->transport->ops.poll_group_process_completions(tgroup, completions_per_qpair,
     716             :                         disconnected_qpair_cb);
     717             : }
     718             : 
     719             : int
     720           0 : nvme_transport_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup)
     721             : {
     722           0 :         return tgroup->transport->ops.poll_group_destroy(tgroup);
     723             : }
     724             : 
     725             : int
     726           3 : nvme_transport_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)
     727             : {
     728             :         struct spdk_nvme_transport_poll_group *tgroup;
     729             :         int rc __attribute__((unused));
     730             : 
     731           3 :         tgroup = qpair->poll_group;
     732             : 
     733           3 :         if (qpair->poll_group_tailq_head == &tgroup->disconnected_qpairs) {
     734           1 :                 return 0;
     735             :         }
     736             : 
     737           2 :         if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
     738           1 :                 rc = tgroup->transport->ops.poll_group_disconnect_qpair(qpair);
     739           1 :                 assert(rc == 0);
     740             : 
     741           1 :                 qpair->poll_group_tailq_head = &tgroup->disconnected_qpairs;
     742           1 :                 STAILQ_REMOVE(&tgroup->connected_qpairs, qpair, spdk_nvme_qpair, poll_group_stailq);
     743           1 :                 assert(tgroup->num_connected_qpairs > 0);
     744           1 :                 tgroup->num_connected_qpairs--;
     745           1 :                 STAILQ_INSERT_TAIL(&tgroup->disconnected_qpairs, qpair, poll_group_stailq);
     746             : 
     747           1 :                 return 0;
     748             :         }
     749             : 
     750           1 :         return -EINVAL;
     751             : }
     752             : 
     753             : int
     754           3 : nvme_transport_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
     755             : {
     756             :         struct spdk_nvme_transport_poll_group *tgroup;
     757             :         int rc;
     758             : 
     759           3 :         tgroup = qpair->poll_group;
     760             : 
     761           3 :         if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
     762           1 :                 return 0;
     763             :         }
     764             : 
     765           2 :         if (qpair->poll_group_tailq_head == &tgroup->disconnected_qpairs) {
     766           1 :                 rc = tgroup->transport->ops.poll_group_connect_qpair(qpair);
     767           1 :                 if (rc == 0) {
     768           1 :                         qpair->poll_group_tailq_head = &tgroup->connected_qpairs;
     769           1 :                         STAILQ_REMOVE(&tgroup->disconnected_qpairs, qpair, spdk_nvme_qpair, poll_group_stailq);
     770           1 :                         STAILQ_INSERT_TAIL(&tgroup->connected_qpairs, qpair, poll_group_stailq);
     771           1 :                         tgroup->num_connected_qpairs++;
     772             :                 }
     773             : 
     774           1 :                 return rc == -EINPROGRESS ? 0 : rc;
     775             :         }
     776             : 
     777             : 
     778           1 :         return -EINVAL;
     779             : }
     780             : 
     781             : int
     782           0 : nvme_transport_poll_group_get_stats(struct spdk_nvme_transport_poll_group *tgroup,
     783             :                                     struct spdk_nvme_transport_poll_group_stat **stats)
     784             : {
     785           0 :         if (tgroup->transport->ops.poll_group_get_stats) {
     786           0 :                 return tgroup->transport->ops.poll_group_get_stats(tgroup, stats);
     787             :         }
     788           0 :         return -ENOTSUP;
     789             : }
     790             : 
     791             : void
     792           0 : nvme_transport_poll_group_free_stats(struct spdk_nvme_transport_poll_group *tgroup,
     793             :                                      struct spdk_nvme_transport_poll_group_stat *stats)
     794             : {
     795           0 :         if (tgroup->transport->ops.poll_group_free_stats) {
     796           0 :                 tgroup->transport->ops.poll_group_free_stats(tgroup, stats);
     797             :         }
     798           0 : }
     799             : 
     800             : spdk_nvme_transport_type_t
     801           0 : nvme_transport_get_trtype(const struct spdk_nvme_transport *transport)
     802             : {
     803           0 :         return transport->ops.type;
     804             : }
     805             : 
     806             : void
     807           0 : spdk_nvme_transport_get_opts(struct spdk_nvme_transport_opts *opts, size_t opts_size)
     808             : {
     809           0 :         if (opts == NULL) {
     810           0 :                 SPDK_ERRLOG("opts should not be NULL.\n");
     811           0 :                 return;
     812             :         }
     813             : 
     814           0 :         if (opts_size == 0) {
     815           0 :                 SPDK_ERRLOG("opts_size should not be zero.\n");
     816           0 :                 return;
     817             :         }
     818             : 
     819           0 :         opts->opts_size = opts_size;
     820             : 
     821             : #define SET_FIELD(field) \
     822             :         if (offsetof(struct spdk_nvme_transport_opts, field) + sizeof(opts->field) <= opts_size) { \
     823             :                 opts->field = g_spdk_nvme_transport_opts.field; \
     824             :         } \
     825             : 
     826           0 :         SET_FIELD(rdma_srq_size);
     827           0 :         SET_FIELD(rdma_max_cq_size);
     828           0 :         SET_FIELD(rdma_cm_event_timeout_ms);
     829             : 
     830             :         /* Do not remove this statement, you should always update this statement when you adding a new field,
     831             :          * and do not forget to add the SET_FIELD statement for your added field. */
     832             :         SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_transport_opts) == 24, "Incorrect size");
     833             : 
     834             : #undef SET_FIELD
     835             : }
     836             : 
     837             : int
     838           0 : spdk_nvme_transport_set_opts(const struct spdk_nvme_transport_opts *opts, size_t opts_size)
     839             : {
     840           0 :         if (opts == NULL) {
     841           0 :                 SPDK_ERRLOG("opts should not be NULL.\n");
     842           0 :                 return -EINVAL;
     843             :         }
     844             : 
     845           0 :         if (opts_size == 0) {
     846           0 :                 SPDK_ERRLOG("opts_size should not be zero.\n");
     847           0 :                 return -EINVAL;
     848             :         }
     849             : 
     850             : #define SET_FIELD(field) \
     851             :         if (offsetof(struct spdk_nvme_transport_opts, field) + sizeof(opts->field) <= opts->opts_size) { \
     852             :                 g_spdk_nvme_transport_opts.field = opts->field; \
     853             :         } \
     854             : 
     855           0 :         SET_FIELD(rdma_srq_size);
     856           0 :         SET_FIELD(rdma_max_cq_size);
     857           0 :         SET_FIELD(rdma_cm_event_timeout_ms);
     858             : 
     859           0 :         g_spdk_nvme_transport_opts.opts_size = opts->opts_size;
     860             : 
     861             : #undef SET_FIELD
     862             : 
     863           0 :         return 0;
     864             : }
     865             : 
     866             : volatile struct spdk_nvme_registers *
     867           0 : spdk_nvme_ctrlr_get_registers(struct spdk_nvme_ctrlr *ctrlr)
     868             : {
     869           0 :         const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
     870             : 
     871           0 :         if (transport == NULL) {
     872             :                 /* Transport does not exist. */
     873           0 :                 return NULL;
     874             :         }
     875             : 
     876           0 :         if (transport->ops.ctrlr_get_registers) {
     877           0 :                 return transport->ops.ctrlr_get_registers(ctrlr);
     878             :         }
     879             : 
     880           0 :         return NULL;
     881             : }

Generated by: LCOV version 1.15