Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause 2 : * Copyright (c) Mellanox Technologies LTD. All rights reserved. 3 : * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 : */ 5 : 6 : #ifndef SPDK_RDMA_H 7 : #define SPDK_RDMA_H 8 : 9 : #include <infiniband/verbs.h> 10 : #include <rdma/rdma_cma.h> 11 : #include <rdma/rdma_verbs.h> 12 : 13 : /* Contains hooks definition */ 14 : #include "spdk/nvme.h" 15 : 16 : /* rxe driver vendor_id has been changed from 0 to 0XFFFFFF in 0184afd15a141d7ce24c32c0d86a1e3ba6bc0eb3 */ 17 : #define SPDK_RDMA_RXE_VENDOR_ID_OLD 0 18 : #define SPDK_RDMA_RXE_VENDOR_ID_NEW 0XFFFFFF 19 : 20 : struct spdk_rdma_wr_stats { 21 : /* Total number of submitted requests */ 22 : uint64_t num_submitted_wrs; 23 : /* Total number of doorbell updates */ 24 : uint64_t doorbell_updates; 25 : }; 26 : 27 : struct spdk_rdma_qp_stats { 28 : struct spdk_rdma_wr_stats send; 29 : struct spdk_rdma_wr_stats recv; 30 : }; 31 : 32 : struct spdk_rdma_qp_init_attr { 33 : void *qp_context; 34 : struct ibv_cq *send_cq; 35 : struct ibv_cq *recv_cq; 36 : struct ibv_srq *srq; 37 : struct ibv_qp_cap cap; 38 : struct ibv_pd *pd; 39 : struct spdk_rdma_qp_stats *stats; 40 : }; 41 : 42 : struct spdk_rdma_send_wr_list { 43 : struct ibv_send_wr *first; 44 : struct ibv_send_wr *last; 45 : }; 46 : 47 : struct spdk_rdma_recv_wr_list { 48 : struct ibv_recv_wr *first; 49 : struct ibv_recv_wr *last; 50 : }; 51 : 52 : struct spdk_rdma_qp { 53 : struct ibv_qp *qp; 54 : struct rdma_cm_id *cm_id; 55 : struct spdk_rdma_send_wr_list send_wrs; 56 : struct spdk_rdma_recv_wr_list recv_wrs; 57 : struct spdk_rdma_qp_stats *stats; 58 : bool shared_stats; 59 : }; 60 : 61 : struct spdk_rdma_mem_map; 62 : 63 : union spdk_rdma_mr { 64 : struct ibv_mr *mr; 65 : uint64_t key; 66 : }; 67 : 68 : enum SPDK_RDMA_TRANSLATION_TYPE { 69 : SPDK_RDMA_TRANSLATION_MR = 0, 70 : SPDK_RDMA_TRANSLATION_KEY 71 : }; 72 : 73 : struct spdk_rdma_memory_translation { 74 : union spdk_rdma_mr mr_or_key; 75 : uint8_t translation_type; 76 : }; 77 : struct spdk_rdma_srq_init_attr { 78 : struct ibv_pd *pd; 79 : struct spdk_rdma_wr_stats *stats; 80 : struct ibv_srq_init_attr srq_init_attr; 81 : }; 82 : 83 : struct spdk_rdma_srq { 84 : struct ibv_srq *srq; 85 : struct spdk_rdma_recv_wr_list recv_wrs; 86 : struct spdk_rdma_wr_stats *stats; 87 : bool shared_stats; 88 : }; 89 : 90 : enum spdk_rdma_memory_map_role { 91 : SPDK_RDMA_MEMORY_MAP_ROLE_TARGET, 92 : SPDK_RDMA_MEMORY_MAP_ROLE_INITIATOR 93 : }; 94 : 95 : /** 96 : * Create RDMA SRQ 97 : * 98 : * \param init_attr Pointer to SRQ init attr 99 : * \return pointer to srq on success or NULL on failure. errno is updated in failure case. 100 : */ 101 : struct spdk_rdma_srq *spdk_rdma_srq_create(struct spdk_rdma_srq_init_attr *init_attr); 102 : 103 : /** 104 : * Destroy RDMA SRQ 105 : * 106 : * \param rdma_srq Pointer to SRQ 107 : * \return 0 on succes, errno on failure 108 : */ 109 : int spdk_rdma_srq_destroy(struct spdk_rdma_srq *rdma_srq); 110 : 111 : /** 112 : * Append the given recv wr structure to the SRQ's outstanding recv list. 113 : * This function accepts either a single Work Request or the first WR in a linked list. 114 : * 115 : * \param rdma_srq Pointer to SRQ 116 : * \param first pointer to the first Work Request 117 : * \return true if there were no outstanding WRs before, false otherwise 118 : */ 119 : bool spdk_rdma_srq_queue_recv_wrs(struct spdk_rdma_srq *rdma_srq, struct ibv_recv_wr *first); 120 : 121 : /** 122 : * Submit all queued receive Work Request 123 : * 124 : * \param rdma_srq Pointer to SRQ 125 : * \param bad_wr Stores a pointer to the first failed WR if this function return nonzero value 126 : * \return 0 on succes, errno on failure 127 : */ 128 : int spdk_rdma_srq_flush_recv_wrs(struct spdk_rdma_srq *rdma_srq, struct ibv_recv_wr **bad_wr); 129 : 130 : /** 131 : * Create RDMA provider specific qpair 132 : * 133 : * \param cm_id Pointer to RDMA_CM cm_id 134 : * \param qp_attr Pointer to qpair init attributes 135 : * \return Pointer to a newly created qpair on success or NULL on failure 136 : */ 137 : struct spdk_rdma_qp *spdk_rdma_qp_create(struct rdma_cm_id *cm_id, 138 : struct spdk_rdma_qp_init_attr *qp_attr); 139 : 140 : /** 141 : * Accept a connection request. Called by the passive side (NVMEoF target) 142 : * 143 : * \param spdk_rdma_qp Pointer to SPDK RDMA qpair 144 : * \param conn_param Optional information needed to establish the connection 145 : * \return 0 on success, errno on failure 146 : */ 147 : int spdk_rdma_qp_accept(struct spdk_rdma_qp *spdk_rdma_qp, struct rdma_conn_param *conn_param); 148 : 149 : /** 150 : * Complete the connection process, must be called by the active 151 : * side (NVMEoF initiator) upon receipt RDMA_CM_EVENT_CONNECT_RESPONSE 152 : * 153 : * \param spdk_rdma_qp Pointer to SPDK RDMA qpair 154 : * \return 0 on success, errno on failure 155 : */ 156 : int spdk_rdma_qp_complete_connect(struct spdk_rdma_qp *spdk_rdma_qp); 157 : 158 : /** 159 : * Destroy RDMA provider specific qpair 160 : * 161 : * \param spdk_rdma_qp Pointer to SPDK RDMA qpair to be destroyed 162 : */ 163 : void spdk_rdma_qp_destroy(struct spdk_rdma_qp *spdk_rdma_qp); 164 : 165 : /** 166 : * Disconnect a connection and transition associated qpair to error state. 167 : * Generates RDMA_CM_EVENT_DISCONNECTED on both connection sides 168 : * 169 : * \param spdk_rdma_qp Pointer to qpair to be disconnected 170 : */ 171 : int spdk_rdma_qp_disconnect(struct spdk_rdma_qp *spdk_rdma_qp); 172 : 173 : /** 174 : * Append the given send wr structure to the qpair's outstanding sends list. 175 : * This function accepts either a single Work Request or the first WR in a linked list. 176 : * 177 : * \param spdk_rdma_qp Pointer to SPDK RDMA qpair 178 : * \param first Pointer to the first Work Request 179 : * \return true if there were no outstanding WRs before, false otherwise 180 : */ 181 : bool spdk_rdma_qp_queue_send_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_send_wr *first); 182 : 183 : /** 184 : * Submit all queued send Work Request 185 : * 186 : * \param spdk_rdma_qp Pointer to SPDK RDMA qpair 187 : * \param bad_wr Stores a pointer to the first failed WR if this function return nonzero value 188 : * \return 0 on succes, errno on failure 189 : */ 190 : int spdk_rdma_qp_flush_send_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_send_wr **bad_wr); 191 : 192 : /** 193 : * Append the given recv wr structure to the qpair's outstanding recv list. 194 : * This function accepts either a single Work Request or the first WR in a linked list. 195 : * 196 : * \param spdk_rdma_qp Pointer to SPDK RDMA qpair 197 : * \param first Pointer to the first Work Request 198 : * \return true if there were no outstanding WRs before, false otherwise 199 : */ 200 : bool spdk_rdma_qp_queue_recv_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_recv_wr *first); 201 : 202 : /** 203 : * Submit all queued recv Work Request 204 : * \param spdk_rdma_qp Pointer to SPDK RDMA qpair 205 : * \param bad_wr Stores a pointer to the first failed WR if this function return nonzero value 206 : * \return 0 on succes, errno on failure 207 : */ 208 : int spdk_rdma_qp_flush_recv_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_recv_wr **bad_wr); 209 : 210 : /** 211 : * Create a memory map which is used to register Memory Regions and perform address -> memory 212 : * key translations 213 : * 214 : * \param pd Protection Domain which will be used to create Memory Regions 215 : * \param hooks Optional hooks which are used to create Protection Domain or ger RKey 216 : * \param role Specifies whether this map is used by RDMA target or initiator, determines access flags of registered MRs 217 : * \return Pointer to memory map or NULL on failure 218 : */ 219 : struct spdk_rdma_mem_map * 220 : spdk_rdma_create_mem_map(struct ibv_pd *pd, struct spdk_nvme_rdma_hooks *hooks, 221 : enum spdk_rdma_memory_map_role role); 222 : 223 : /** 224 : * Free previously allocated memory map 225 : * 226 : * \param map Pointer to memory map to free 227 : */ 228 : void spdk_rdma_free_mem_map(struct spdk_rdma_mem_map **map); 229 : 230 : /** 231 : * Get a translation for the given address and length. 232 : * 233 : * Note: the user of this function should use address returned in \b translation structure 234 : * 235 : * \param map Pointer to translation map 236 : * \param address Memory address for translation 237 : * \param length Length of the memory address 238 : * \param[in,out] translation Pointer to translation result to be filled by this function 239 : * \retval -EINVAL if translation is not found 240 : * \retval 0 translation succeed 241 : */ 242 : int spdk_rdma_get_translation(struct spdk_rdma_mem_map *map, void *address, 243 : size_t length, struct spdk_rdma_memory_translation *translation); 244 : 245 : /** 246 : * Helper function for retrieving Local Memory Key. Should be applied to a translation 247 : * returned by \b spdk_rdma_get_translation 248 : * 249 : * \param translation Memory translation 250 : * \return Local Memory Key 251 : */ 252 : static inline uint32_t 253 466 : spdk_rdma_memory_translation_get_lkey(struct spdk_rdma_memory_translation 254 : *translation) 255 : { 256 466 : return translation->translation_type == SPDK_RDMA_TRANSLATION_MR ? 257 466 : translation->mr_or_key.mr->lkey : (uint32_t)translation->mr_or_key.key; 258 : } 259 : 260 : /** 261 : * Helper function for retrieving Remote Memory Key. Should be applied to a translation 262 : * returned by \b spdk_rdma_get_translation 263 : * 264 : * \param translation Memory translation 265 : * \return Remote Memory Key 266 : */ 267 : static inline uint32_t 268 : spdk_rdma_memory_translation_get_rkey(struct spdk_rdma_memory_translation 269 : *translation) 270 : { 271 : return translation->translation_type == SPDK_RDMA_TRANSLATION_MR ? 272 : translation->mr_or_key.mr->rkey : (uint32_t)translation->mr_or_key.key; 273 : } 274 : 275 : /** 276 : * Get a Protection Domain for an RDMA device context. 277 : * 278 : * \param context RDMA device context 279 : * \return Pointer to the allocated Protection Domain 280 : */ 281 : struct ibv_pd * 282 : spdk_rdma_get_pd(struct ibv_context *context); 283 : 284 : /** 285 : * Return a Protection Domain. 286 : * 287 : * \param pd Pointer to the Protection Domain 288 : */ 289 : void spdk_rdma_put_pd(struct ibv_pd *pd); 290 : 291 : #endif /* SPDK_RDMA_H */