Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright (C) 2016 Intel Corporation. All rights reserved.
3 : : * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
4 : : * Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : : * Copyright (c) 2022 Dell Inc, or its subsidiaries. All rights reserved.
6 : : */
7 : :
8 : : #include "spdk/stdinc.h"
9 : :
10 : : #include "bdev_nvme.h"
11 : :
12 : : #include "spdk/accel.h"
13 : : #include "spdk/config.h"
14 : : #include "spdk/endian.h"
15 : : #include "spdk/bdev.h"
16 : : #include "spdk/json.h"
17 : : #include "spdk/keyring.h"
18 : : #include "spdk/likely.h"
19 : : #include "spdk/nvme.h"
20 : : #include "spdk/nvme_ocssd.h"
21 : : #include "spdk/nvme_zns.h"
22 : : #include "spdk/opal.h"
23 : : #include "spdk/thread.h"
24 : : #include "spdk/trace.h"
25 : : #include "spdk/string.h"
26 : : #include "spdk/util.h"
27 : : #include "spdk/uuid.h"
28 : :
29 : : #include "spdk/bdev_module.h"
30 : : #include "spdk/log.h"
31 : :
32 : : #include "spdk_internal/usdt.h"
33 : : #include "spdk_internal/trace_defs.h"
34 : :
35 : : #define CTRLR_STRING(nvme_ctrlr) \
36 : : (spdk_nvme_trtype_is_fabrics(nvme_ctrlr->active_path_id->trid.trtype) ? \
37 : : nvme_ctrlr->active_path_id->trid.subnqn : nvme_ctrlr->active_path_id->trid.traddr)
38 : :
39 : : #define CTRLR_ID(nvme_ctrlr) (spdk_nvme_ctrlr_get_id(nvme_ctrlr->ctrlr))
40 : :
41 : : #define NVME_CTRLR_ERRLOG(ctrlr, format, ...) \
42 : : SPDK_ERRLOG("[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
43 : :
44 : : #define NVME_CTRLR_WARNLOG(ctrlr, format, ...) \
45 : : SPDK_WARNLOG("[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
46 : :
47 : : #define NVME_CTRLR_NOTICELOG(ctrlr, format, ...) \
48 : : SPDK_NOTICELOG("[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
49 : :
50 : : #define NVME_CTRLR_INFOLOG(ctrlr, format, ...) \
51 : : SPDK_INFOLOG(bdev_nvme, "[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
52 : :
53 : : #ifdef DEBUG
54 : : #define NVME_CTRLR_DEBUGLOG(ctrlr, format, ...) \
55 : : SPDK_DEBUGLOG(bdev_nvme, "[%s, %u] " format, CTRLR_STRING(ctrlr), CTRLR_ID(ctrlr), ##__VA_ARGS__);
56 : : #else
57 : : #define NVME_CTRLR_DEBUGLOG(ctrlr, ...) do { } while (0)
58 : : #endif
59 : :
60 : : #define BDEV_STRING(nbdev) (nbdev->disk.name)
61 : :
62 : : #define NVME_BDEV_ERRLOG(nbdev, format, ...) \
63 : : SPDK_ERRLOG("[%s] " format, BDEV_STRING(nbdev), ##__VA_ARGS__);
64 : :
65 : : #define NVME_BDEV_WARNLOG(nbdev, format, ...) \
66 : : SPDK_WARNLOG("[%s] " format, BDEV_STRING(nbdev), ##__VA_ARGS__);
67 : :
68 : : #define NVME_BDEV_NOTICELOG(nbdev, format, ...) \
69 : : SPDK_NOTICELOG("[%s] " format, BDEV_STRING(nbdev), ##__VA_ARGS__);
70 : :
71 : : #define NVME_BDEV_INFOLOG(nbdev, format, ...) \
72 : : SPDK_INFOLOG(bdev_nvme, "[%s] " format, BDEV_STRING(nbdev), ##__VA_ARGS__);
73 : :
74 : : #define SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT true
75 : : #define SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS (10000)
76 : :
77 : : #define NSID_STR_LEN 10
78 : :
79 : : #define SPDK_CONTROLLER_NAME_MAX 512
80 : :
81 : : static int bdev_nvme_config_json(struct spdk_json_write_ctx *w);
82 : :
83 : : struct nvme_bdev_io {
84 : : /** array of iovecs to transfer. */
85 : : struct iovec *iovs;
86 : :
87 : : /** Number of iovecs in iovs array. */
88 : : int iovcnt;
89 : :
90 : : /** Current iovec position. */
91 : : int iovpos;
92 : :
93 : : /** Offset in current iovec. */
94 : : uint32_t iov_offset;
95 : :
96 : : /** Offset in current iovec. */
97 : : uint32_t fused_iov_offset;
98 : :
99 : : /** array of iovecs to transfer. */
100 : : struct iovec *fused_iovs;
101 : :
102 : : /** Number of iovecs in iovs array. */
103 : : int fused_iovcnt;
104 : :
105 : : /** Current iovec position. */
106 : : int fused_iovpos;
107 : :
108 : : /** I/O path the current I/O or admin passthrough is submitted on, or the I/O path
109 : : * being reset in a reset I/O.
110 : : */
111 : : struct nvme_io_path *io_path;
112 : :
113 : : /** Saved status for admin passthru completion event, PI error verification, or intermediate compare-and-write status */
114 : : struct spdk_nvme_cpl cpl;
115 : :
116 : : /** Extended IO opts passed by the user to bdev layer and mapped to NVME format */
117 : : struct spdk_nvme_ns_cmd_ext_io_opts ext_opts;
118 : :
119 : : /** Keeps track if first of fused commands was submitted */
120 : : bool first_fused_submitted;
121 : :
122 : : /** Keeps track if first of fused commands was completed */
123 : : bool first_fused_completed;
124 : :
125 : : /* How many times the current I/O was retried. */
126 : : int32_t retry_count;
127 : :
128 : : /** Expiration value in ticks to retry the current I/O. */
129 : : uint64_t retry_ticks;
130 : :
131 : : /** Temporary pointer to zone report buffer */
132 : : struct spdk_nvme_zns_zone_report *zone_report_buf;
133 : :
134 : : /** Keep track of how many zones that have been copied to the spdk_bdev_zone_info struct */
135 : : uint64_t handled_zones;
136 : :
137 : : /* Current tsc at submit time. */
138 : : uint64_t submit_tsc;
139 : :
140 : : /* Used to put nvme_bdev_io into the list */
141 : : TAILQ_ENTRY(nvme_bdev_io) retry_link;
142 : : };
143 : :
144 : : struct nvme_probe_skip_entry {
145 : : struct spdk_nvme_transport_id trid;
146 : : TAILQ_ENTRY(nvme_probe_skip_entry) tailq;
147 : : };
148 : : /* All the controllers deleted by users via RPC are skipped by hotplug monitor */
149 : : static TAILQ_HEAD(, nvme_probe_skip_entry) g_skipped_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER(
150 : : g_skipped_nvme_ctrlrs);
151 : :
152 : : #define BDEV_NVME_DEFAULT_DIGESTS (SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA256) | \
153 : : SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA384) | \
154 : : SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA512))
155 : :
156 : : #define BDEV_NVME_DEFAULT_DHGROUPS (SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_NULL) | \
157 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_2048) | \
158 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_3072) | \
159 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_4096) | \
160 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_6144) | \
161 : : SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_8192))
162 : :
163 : : static struct spdk_bdev_nvme_opts g_opts = {
164 : : .action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE,
165 : : .keep_alive_timeout_ms = SPDK_BDEV_NVME_DEFAULT_KEEP_ALIVE_TIMEOUT_IN_MS,
166 : : .timeout_us = 0,
167 : : .timeout_admin_us = 0,
168 : : .transport_retry_count = 4,
169 : : .arbitration_burst = 0,
170 : : .low_priority_weight = 0,
171 : : .medium_priority_weight = 0,
172 : : .high_priority_weight = 0,
173 : : .io_queue_requests = 0,
174 : : .nvme_adminq_poll_period_us = 10000ULL,
175 : : .nvme_ioq_poll_period_us = 0,
176 : : .delay_cmd_submit = SPDK_BDEV_NVME_DEFAULT_DELAY_CMD_SUBMIT,
177 : : .bdev_retry_count = 3,
178 : : .ctrlr_loss_timeout_sec = 0,
179 : : .reconnect_delay_sec = 0,
180 : : .fast_io_fail_timeout_sec = 0,
181 : : .transport_ack_timeout = 0,
182 : : .disable_auto_failback = false,
183 : : .generate_uuids = false,
184 : : .transport_tos = 0,
185 : : .nvme_error_stat = false,
186 : : .io_path_stat = false,
187 : : .allow_accel_sequence = false,
188 : : .dhchap_digests = BDEV_NVME_DEFAULT_DIGESTS,
189 : : .dhchap_dhgroups = BDEV_NVME_DEFAULT_DHGROUPS,
190 : : .rdma_umr_per_io = false,
191 : : };
192 : :
193 : : #define NVME_HOTPLUG_POLL_PERIOD_MAX 10000000ULL
194 : : #define NVME_HOTPLUG_POLL_PERIOD_DEFAULT 100000ULL
195 : :
196 : : static int g_hot_insert_nvme_controller_index = 0;
197 : : static uint64_t g_nvme_hotplug_poll_period_us = NVME_HOTPLUG_POLL_PERIOD_DEFAULT;
198 : : static bool g_nvme_hotplug_enabled = false;
199 : : struct spdk_thread *g_bdev_nvme_init_thread;
200 : : static struct spdk_poller *g_hotplug_poller;
201 : : static struct spdk_poller *g_hotplug_probe_poller;
202 : : static struct spdk_nvme_probe_ctx *g_hotplug_probe_ctx;
203 : :
204 : : static void nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr,
205 : : struct nvme_async_probe_ctx *ctx);
206 : : static void nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr,
207 : : struct nvme_async_probe_ctx *ctx);
208 : : static int bdev_nvme_library_init(void);
209 : : static void bdev_nvme_library_fini(void);
210 : : static void _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch,
211 : : struct spdk_bdev_io *bdev_io);
212 : : static void bdev_nvme_submit_request(struct spdk_io_channel *ch,
213 : : struct spdk_bdev_io *bdev_io);
214 : : static int bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
215 : : void *md, uint64_t lba_count, uint64_t lba,
216 : : uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx,
217 : : struct spdk_accel_sequence *seq);
218 : : static int bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
219 : : void *md, uint64_t lba_count, uint64_t lba);
220 : : static int bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
221 : : void *md, uint64_t lba_count, uint64_t lba,
222 : : uint32_t flags, struct spdk_memory_domain *domain, void *domain_ctx,
223 : : struct spdk_accel_sequence *seq,
224 : : union spdk_bdev_nvme_cdw12 cdw12, union spdk_bdev_nvme_cdw13 cdw13);
225 : : static int bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
226 : : void *md, uint64_t lba_count,
227 : : uint64_t zslba, uint32_t flags);
228 : : static int bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
229 : : void *md, uint64_t lba_count, uint64_t lba,
230 : : uint32_t flags);
231 : : static int bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio,
232 : : struct iovec *cmp_iov, int cmp_iovcnt, struct iovec *write_iov,
233 : : int write_iovcnt, void *md, uint64_t lba_count, uint64_t lba,
234 : : uint32_t flags);
235 : : static int bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id,
236 : : uint32_t num_zones, struct spdk_bdev_zone_info *info);
237 : : static int bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id,
238 : : enum spdk_bdev_zone_action action);
239 : : static void bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch,
240 : : struct nvme_bdev_io *bio,
241 : : struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes);
242 : : static int bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
243 : : void *buf, size_t nbytes);
244 : : static int bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
245 : : void *buf, size_t nbytes, void *md_buf, size_t md_len);
246 : : static int bdev_nvme_iov_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
247 : : struct iovec *iov, int iovcnt, size_t nbytes,
248 : : void *md_buf, size_t md_len);
249 : : static void bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch,
250 : : struct nvme_bdev_io *bio, struct nvme_bdev_io *bio_to_abort);
251 : : static void bdev_nvme_reset_io(struct nvme_bdev *nbdev, struct nvme_bdev_io *bio);
252 : : static int bdev_nvme_reset_ctrlr(struct nvme_ctrlr *nvme_ctrlr);
253 : : static int bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr);
254 : : static void remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr);
255 : : static int nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr);
256 : :
257 : : static struct nvme_ns *nvme_ns_alloc(void);
258 : : static void nvme_ns_free(struct nvme_ns *ns);
259 : :
260 : : static int
261 : 2045 : nvme_ns_cmp(struct nvme_ns *ns1, struct nvme_ns *ns2)
262 : : {
263 [ + + + - : 2045 : return ns1->id < ns2->id ? -1 : ns1->id > ns2->id;
+ - + - -
+ + - + -
+ - + - ]
264 : : }
265 : :
266 [ + + + + : 22816 : RB_GENERATE_STATIC(nvme_ns_tree, nvme_ns, node, nvme_ns_cmp);
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + - +
- + + - -
- - + # #
# # # # -
+ + + + -
+ + - + #
# # # # #
# # # # #
# # # # #
# # # # +
- + + + +
+ + + - +
+ + + - +
+ + + + +
+ + + + -
# # + - +
- + - + -
+ - + + +
- + - + +
+ - + - +
- - + # #
# # - + +
+ + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - + -
- + + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # -
+ # # # #
# # # # #
# # # - +
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
+ - + - -
+ # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
- + + - +
- # # # #
# # # # #
# # # # #
- + + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
267 : :
268 : : struct spdk_nvme_qpair *
269 : 4 : bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch)
270 : : {
271 : : struct nvme_ctrlr_channel *ctrlr_ch;
272 : :
273 [ + + # # ]: 4 : assert(ctrlr_io_ch != NULL);
274 : :
275 : 4 : ctrlr_ch = spdk_io_channel_get_ctx(ctrlr_io_ch);
276 : :
277 [ # # # # : 4 : return ctrlr_ch->qpair->qpair;
# # # # ]
278 : : }
279 : :
280 : : static int
281 : 1960 : bdev_nvme_get_ctx_size(void)
282 : : {
283 : 1960 : return sizeof(struct nvme_bdev_io);
284 : : }
285 : :
286 : : static struct spdk_bdev_module nvme_if = {
287 : : .name = "nvme",
288 : : .async_fini = true,
289 : : .module_init = bdev_nvme_library_init,
290 : : .module_fini = bdev_nvme_library_fini,
291 : : .config_json = bdev_nvme_config_json,
292 : : .get_ctx_size = bdev_nvme_get_ctx_size,
293 : :
294 : : };
295 : 2149 : SPDK_BDEV_MODULE_REGISTER(nvme, &nvme_if)
296 : :
297 : : struct nvme_bdev_ctrlrs g_nvme_bdev_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_bdev_ctrlrs);
298 : : pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER;
299 : : bool g_bdev_nvme_module_finish;
300 : :
301 : : struct nvme_bdev_ctrlr *
302 : 39028 : nvme_bdev_ctrlr_get_by_name(const char *name)
303 : : {
304 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
305 : :
306 [ + + # # : 40941 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
# # # # ]
307 [ + + + + : 33597 : if (strcmp(name, nbdev_ctrlr->name) == 0) {
+ + + - -
+ ]
308 : 31684 : break;
309 : : }
310 : 0 : }
311 : :
312 : 39028 : return nbdev_ctrlr;
313 : : }
314 : :
315 : : static struct nvme_ctrlr *
316 : 731 : nvme_bdev_ctrlr_get_ctrlr(struct nvme_bdev_ctrlr *nbdev_ctrlr,
317 : : const struct spdk_nvme_transport_id *trid, const char *hostnqn)
318 : : {
319 : : const struct spdk_nvme_ctrlr_opts *opts;
320 : : struct nvme_ctrlr *nvme_ctrlr;
321 : :
322 [ + + # # : 1390 : TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
323 [ # # # # ]: 795 : opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr);
324 [ + + + - : 795 : if (spdk_nvme_transport_id_compare(trid, &nvme_ctrlr->active_path_id->trid) == 0 &&
# # # # #
# ]
325 [ + + - + : 139 : strcmp(hostnqn, opts->hostnqn) == 0) {
+ + ]
326 : 136 : break;
327 : : }
328 : 41 : }
329 : :
330 : 731 : return nvme_ctrlr;
331 : : }
332 : :
333 : : struct nvme_ctrlr *
334 : 0 : nvme_bdev_ctrlr_get_ctrlr_by_id(struct nvme_bdev_ctrlr *nbdev_ctrlr,
335 : : uint16_t cntlid)
336 : : {
337 : : struct nvme_ctrlr *nvme_ctrlr;
338 : : const struct spdk_nvme_ctrlr_data *cdata;
339 : :
340 [ # # # # : 0 : TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
341 [ # # # # ]: 0 : cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
342 [ # # # # : 0 : if (cdata->cntlid == cntlid) {
# # ]
343 : 0 : break;
344 : : }
345 : 0 : }
346 : :
347 : 0 : return nvme_ctrlr;
348 : : }
349 : :
350 : : static struct nvme_bdev *
351 : 1643 : nvme_bdev_ctrlr_get_bdev(struct nvme_bdev_ctrlr *nbdev_ctrlr, uint32_t nsid)
352 : : {
353 : : struct nvme_bdev *nbdev;
354 : :
355 [ + + ]: 1643 : pthread_mutex_lock(&g_bdev_nvme_mutex);
356 [ + + + - : 1941 : TAILQ_FOREACH(nbdev, &nbdev_ctrlr->bdevs, tailq) {
+ - + - #
# # # #
# ]
357 [ + + # # : 456 : if (nbdev->nsid == nsid) {
# # ]
358 : 158 : break;
359 : : }
360 : 34 : }
361 [ + + ]: 1643 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
362 : :
363 : 1643 : return nbdev;
364 : : }
365 : :
366 : : struct nvme_ns *
367 : 3365 : nvme_ctrlr_get_ns(struct nvme_ctrlr *nvme_ctrlr, uint32_t nsid)
368 : : {
369 : 1142 : struct nvme_ns ns;
370 : :
371 [ + + # # ]: 3365 : assert(nsid > 0);
372 : :
373 : 3365 : ns.id = nsid;
374 [ + - ]: 3365 : return RB_FIND(nvme_ns_tree, &nvme_ctrlr->namespaces, &ns);
375 : : }
376 : :
377 : : struct nvme_ns *
378 : 3728 : nvme_ctrlr_get_first_active_ns(struct nvme_ctrlr *nvme_ctrlr)
379 : : {
380 [ + - ]: 3728 : return RB_MIN(nvme_ns_tree, &nvme_ctrlr->namespaces);
381 : : }
382 : :
383 : : struct nvme_ns *
384 : 1645 : nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns)
385 : : {
386 [ - + ]: 1645 : if (ns == NULL) {
387 : 0 : return NULL;
388 : : }
389 : :
390 : 1645 : return RB_NEXT(nvme_ns_tree, &nvme_ctrlr->namespaces, ns);
391 : 87 : }
392 : :
393 : : static struct nvme_ctrlr *
394 : 1756 : nvme_ctrlr_get(const struct spdk_nvme_transport_id *trid, const char *hostnqn)
395 : : {
396 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
397 : 1756 : struct nvme_ctrlr *nvme_ctrlr = NULL;
398 : :
399 [ + + ]: 1756 : pthread_mutex_lock(&g_bdev_nvme_mutex);
400 [ + + # # : 2327 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
# # # # ]
401 : 571 : nvme_ctrlr = nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, trid, hostnqn);
402 [ - + ]: 571 : if (nvme_ctrlr != NULL) {
403 : 0 : break;
404 : : }
405 : 19 : }
406 [ + + ]: 1756 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
407 : :
408 : 1756 : return nvme_ctrlr;
409 : : }
410 : :
411 : : struct nvme_ctrlr *
412 : 3603 : nvme_ctrlr_get_by_name(const char *name)
413 : : {
414 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
415 : 3603 : struct nvme_ctrlr *nvme_ctrlr = NULL;
416 : :
417 [ + + ]: 3603 : if (name == NULL) {
418 : 0 : return NULL;
419 : : }
420 : :
421 [ + + ]: 3603 : pthread_mutex_lock(&g_bdev_nvme_mutex);
422 : 3603 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
423 [ + + ]: 3603 : if (nbdev_ctrlr != NULL) {
424 [ + - + - : 347 : nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs);
+ - ]
425 : 63 : }
426 [ + + ]: 3603 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
427 : :
428 : 3603 : return nvme_ctrlr;
429 : 151 : }
430 : :
431 : : void
432 : 764 : nvme_bdev_ctrlr_for_each(nvme_bdev_ctrlr_for_each_fn fn, void *ctx)
433 : : {
434 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
435 : :
436 [ - + ]: 764 : pthread_mutex_lock(&g_bdev_nvme_mutex);
437 [ + + # # : 1483 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
# # # # ]
438 [ # # # # ]: 719 : fn(nbdev_ctrlr, ctx);
439 : 1 : }
440 [ - + ]: 764 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
441 : 764 : }
442 : :
443 : : struct nvme_ctrlr_channel_iter {
444 : : nvme_ctrlr_for_each_channel_msg fn;
445 : : nvme_ctrlr_for_each_channel_done cpl;
446 : : struct spdk_io_channel_iter *i;
447 : : void *ctx;
448 : : };
449 : :
450 : : void
451 : 2583 : nvme_ctrlr_for_each_channel_continue(struct nvme_ctrlr_channel_iter *iter, int status)
452 : : {
453 [ # # # # ]: 2583 : spdk_for_each_channel_continue(iter->i, status);
454 : 2583 : }
455 : :
456 : : static void
457 : 2583 : nvme_ctrlr_each_channel_msg(struct spdk_io_channel_iter *i)
458 : : {
459 : 2583 : struct nvme_ctrlr_channel_iter *iter = spdk_io_channel_iter_get_ctx(i);
460 : 2583 : struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
461 : 2583 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
462 : 2583 : struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch);
463 : :
464 [ # # # # ]: 2583 : iter->i = i;
465 [ # # # # : 2583 : iter->fn(iter, nvme_ctrlr, ctrlr_ch, iter->ctx);
# # # # #
# # # ]
466 : 2583 : }
467 : :
468 : : static void
469 : 2440 : nvme_ctrlr_each_channel_cpl(struct spdk_io_channel_iter *i, int status)
470 : : {
471 : 2440 : struct nvme_ctrlr_channel_iter *iter = spdk_io_channel_iter_get_ctx(i);
472 : 2440 : struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
473 : :
474 [ # # # # ]: 2440 : iter->i = i;
475 [ # # # # : 2440 : iter->cpl(nvme_ctrlr, iter->ctx, status);
# # # # #
# # # ]
476 : :
477 : 2440 : free(iter);
478 : 2440 : }
479 : :
480 : : void
481 : 2440 : nvme_ctrlr_for_each_channel(struct nvme_ctrlr *nvme_ctrlr,
482 : : nvme_ctrlr_for_each_channel_msg fn, void *ctx,
483 : : nvme_ctrlr_for_each_channel_done cpl)
484 : : {
485 : : struct nvme_ctrlr_channel_iter *iter;
486 : :
487 [ + - + - ]: 2440 : assert(nvme_ctrlr != NULL && fn != NULL);
488 : :
489 : 2440 : iter = calloc(1, sizeof(struct nvme_ctrlr_channel_iter));
490 [ - + ]: 2440 : if (iter == NULL) {
491 : 0 : SPDK_ERRLOG("Unable to allocate iterator\n");
492 [ # # ]: 0 : assert(false);
493 : : return;
494 : : }
495 : :
496 [ # # # # ]: 2440 : iter->fn = fn;
497 [ # # # # ]: 2440 : iter->cpl = cpl;
498 [ # # # # ]: 2440 : iter->ctx = ctx;
499 : :
500 : 2541 : spdk_for_each_channel(nvme_ctrlr, nvme_ctrlr_each_channel_msg,
501 : 101 : iter, nvme_ctrlr_each_channel_cpl);
502 : 101 : }
503 : :
504 : : struct nvme_bdev_channel_iter {
505 : : nvme_bdev_for_each_channel_msg fn;
506 : : nvme_bdev_for_each_channel_done cpl;
507 : : struct spdk_io_channel_iter *i;
508 : : void *ctx;
509 : : };
510 : :
511 : : void
512 : 360 : nvme_bdev_for_each_channel_continue(struct nvme_bdev_channel_iter *iter, int status)
513 : : {
514 [ # # # # ]: 360 : spdk_for_each_channel_continue(iter->i, status);
515 : 360 : }
516 : :
517 : : static void
518 : 360 : nvme_bdev_each_channel_msg(struct spdk_io_channel_iter *i)
519 : : {
520 : 360 : struct nvme_bdev_channel_iter *iter = spdk_io_channel_iter_get_ctx(i);
521 : 360 : struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i);
522 : 360 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
523 : 360 : struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
524 : :
525 [ # # # # ]: 360 : iter->i = i;
526 [ # # # # : 360 : iter->fn(iter, nbdev, nbdev_ch, iter->ctx);
# # # # #
# # # ]
527 : 360 : }
528 : :
529 : : static void
530 : 354 : nvme_bdev_each_channel_cpl(struct spdk_io_channel_iter *i, int status)
531 : : {
532 : 354 : struct nvme_bdev_channel_iter *iter = spdk_io_channel_iter_get_ctx(i);
533 : 354 : struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i);
534 : :
535 [ # # # # ]: 354 : iter->i = i;
536 [ # # # # : 354 : iter->cpl(nbdev, iter->ctx, status);
# # # # #
# # # ]
537 : :
538 : 354 : free(iter);
539 : 354 : }
540 : :
541 : : void
542 : 354 : nvme_bdev_for_each_channel(struct nvme_bdev *nbdev,
543 : : nvme_bdev_for_each_channel_msg fn, void *ctx,
544 : : nvme_bdev_for_each_channel_done cpl)
545 : : {
546 : : struct nvme_bdev_channel_iter *iter;
547 : :
548 [ + - + - ]: 354 : assert(nbdev != NULL && fn != NULL);
549 : :
550 : 354 : iter = calloc(1, sizeof(struct nvme_bdev_channel_iter));
551 [ - + ]: 354 : if (iter == NULL) {
552 : 0 : SPDK_ERRLOG("Unable to allocate iterator\n");
553 [ # # ]: 0 : assert(false);
554 : : return;
555 : : }
556 : :
557 [ # # # # ]: 354 : iter->fn = fn;
558 [ # # # # ]: 354 : iter->cpl = cpl;
559 [ # # # # ]: 354 : iter->ctx = ctx;
560 : :
561 : 354 : spdk_for_each_channel(nbdev, nvme_bdev_each_channel_msg, iter,
562 : : nvme_bdev_each_channel_cpl);
563 : 62 : }
564 : :
565 : : void
566 : 2175 : nvme_bdev_dump_trid_json(const struct spdk_nvme_transport_id *trid, struct spdk_json_write_ctx *w)
567 : : {
568 : : const char *trtype_str;
569 : : const char *adrfam_str;
570 : :
571 [ # # # # ]: 2175 : trtype_str = spdk_nvme_transport_id_trtype_str(trid->trtype);
572 [ + - ]: 2175 : if (trtype_str) {
573 : 2175 : spdk_json_write_named_string(w, "trtype", trtype_str);
574 : 8 : }
575 : :
576 [ # # # # ]: 2175 : adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam);
577 [ + + ]: 2175 : if (adrfam_str) {
578 : 1149 : spdk_json_write_named_string(w, "adrfam", adrfam_str);
579 : 0 : }
580 : :
581 [ + - # # : 2175 : if (trid->traddr[0] != '\0') {
# # # # #
# ]
582 [ # # ]: 2175 : spdk_json_write_named_string(w, "traddr", trid->traddr);
583 : 8 : }
584 : :
585 [ + + # # : 2175 : if (trid->trsvcid[0] != '\0') {
# # # # #
# ]
586 [ # # ]: 1149 : spdk_json_write_named_string(w, "trsvcid", trid->trsvcid);
587 : 0 : }
588 : :
589 [ + + # # : 2175 : if (trid->subnqn[0] != '\0') {
# # # # #
# ]
590 [ # # ]: 1149 : spdk_json_write_named_string(w, "subnqn", trid->subnqn);
591 : 0 : }
592 : 2175 : }
593 : :
594 : : static void
595 : 1764 : nvme_bdev_ctrlr_delete(struct nvme_bdev_ctrlr *nbdev_ctrlr,
596 : : struct nvme_ctrlr *nvme_ctrlr)
597 : : {
598 : 514 : SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_delete, nvme_ctrlr->nbdev_ctrlr->name);
599 [ + + ]: 1764 : pthread_mutex_lock(&g_bdev_nvme_mutex);
600 : :
601 [ + + + - : 1764 : TAILQ_REMOVE(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - ]
602 [ + + + - : 1764 : if (!TAILQ_EMPTY(&nbdev_ctrlr->ctrlrs)) {
+ - - + ]
603 [ - + ]: 73 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
604 : :
605 : 73 : return;
606 : : }
607 [ + + + - : 1691 : TAILQ_REMOVE(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - ]
608 : :
609 [ + + ]: 1691 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
610 : :
611 [ + + + - : 1691 : assert(TAILQ_EMPTY(&nbdev_ctrlr->bdevs));
+ - + - #
# ]
612 : :
613 [ + - + - ]: 1691 : free(nbdev_ctrlr->name);
614 : 1691 : free(nbdev_ctrlr);
615 : 72 : }
616 : :
617 : : static void
618 : 1768 : _nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr)
619 : : {
620 : : struct nvme_path_id *path_id, *tmp_path;
621 : : struct nvme_ns *ns, *tmp_ns;
622 : :
623 [ + - + - ]: 1768 : free(nvme_ctrlr->copied_ana_desc);
624 [ + - + - ]: 1768 : spdk_free(nvme_ctrlr->ana_log_page);
625 : :
626 [ + + + - : 1768 : if (nvme_ctrlr->opal_dev) {
+ - ]
627 [ # # # # ]: 21 : spdk_opal_dev_destruct(nvme_ctrlr->opal_dev);
628 [ # # # # ]: 21 : nvme_ctrlr->opal_dev = NULL;
629 : 0 : }
630 : :
631 [ + + + - : 1768 : if (nvme_ctrlr->nbdev_ctrlr) {
- + ]
632 [ + - + - ]: 1764 : nvme_bdev_ctrlr_delete(nvme_ctrlr->nbdev_ctrlr, nvme_ctrlr);
633 : 72 : }
634 : :
635 [ + + + + : 1768 : RB_FOREACH_SAFE(ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp_ns) {
- + ]
636 [ # # ]: 0 : RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, ns);
637 : 0 : nvme_ns_free(ns);
638 : 0 : }
639 : :
640 [ + + + + : 3544 : TAILQ_FOREACH_SAFE(path_id, &nvme_ctrlr->trids, link, tmp_path) {
+ - + + +
- + - + -
+ + ]
641 [ + + + - : 1776 : TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - ]
642 : 1776 : free(path_id);
643 : 73 : }
644 : :
645 [ + + + - ]: 1768 : pthread_mutex_destroy(&nvme_ctrlr->mutex);
646 [ + - + - ]: 1768 : spdk_keyring_put_key(nvme_ctrlr->psk);
647 [ + - + - ]: 1768 : spdk_keyring_put_key(nvme_ctrlr->dhchap_key);
648 [ + - + - ]: 1768 : spdk_keyring_put_key(nvme_ctrlr->dhchap_ctrlr_key);
649 : 1768 : free(nvme_ctrlr);
650 : :
651 [ + + ]: 1768 : pthread_mutex_lock(&g_bdev_nvme_mutex);
652 [ + + + + : 1768 : if (g_bdev_nvme_module_finish && TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
+ + ]
653 [ + + ]: 536 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
654 : 536 : spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL);
655 : 536 : spdk_bdev_module_fini_done();
656 : 536 : return;
657 : : }
658 [ - + ]: 1232 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
659 : 73 : }
660 : :
661 : : static int
662 : 191765 : nvme_detach_poller(void *arg)
663 : : {
664 : 191765 : struct nvme_ctrlr *nvme_ctrlr = arg;
665 : : int rc;
666 : :
667 [ + - + - ]: 191765 : rc = spdk_nvme_detach_poll_async(nvme_ctrlr->detach_ctx);
668 [ + + ]: 191765 : if (rc != -EAGAIN) {
669 [ + - ]: 1768 : spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
670 : 1768 : _nvme_ctrlr_delete(nvme_ctrlr);
671 : 73 : }
672 : :
673 : 191765 : return SPDK_POLLER_BUSY;
674 : : }
675 : :
676 : : static void
677 : 1768 : nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr)
678 : : {
679 : : int rc;
680 : :
681 [ + - ]: 1768 : spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
682 : :
683 [ + + ]: 1768 : if (spdk_interrupt_mode_is_enabled()) {
684 [ # # ]: 1 : spdk_interrupt_unregister(&nvme_ctrlr->intr);
685 : 0 : }
686 : :
687 : : /* First, unregister the adminq poller, as the driver will poll adminq if necessary */
688 [ + - ]: 1768 : spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller);
689 : :
690 : : /* If we got here, the reset/detach poller cannot be active */
691 [ + + + - : 1768 : assert(nvme_ctrlr->reset_detach_poller == NULL);
+ - # # ]
692 [ + - + - ]: 1768 : nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(nvme_detach_poller,
693 : : nvme_ctrlr, 1000);
694 [ + + + - : 1768 : if (nvme_ctrlr->reset_detach_poller == NULL) {
+ - ]
695 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to register detach poller\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
696 : 0 : goto error;
697 : : }
698 : :
699 [ + - + - : 1768 : rc = spdk_nvme_detach_async(nvme_ctrlr->ctrlr, &nvme_ctrlr->detach_ctx);
+ - ]
700 [ - + ]: 1768 : if (rc != 0) {
701 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to detach the NVMe controller\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
702 : 0 : goto error;
703 : : }
704 : :
705 : 1768 : return;
706 : 0 : error:
707 : : /* We don't have a good way to handle errors here, so just do what we can and delete the
708 : : * controller without detaching the underlying NVMe device.
709 : : */
710 [ # # ]: 0 : spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
711 : 0 : _nvme_ctrlr_delete(nvme_ctrlr);
712 : 73 : }
713 : :
714 : : static void
715 : 1764 : nvme_ctrlr_unregister_cb(void *io_device)
716 : : {
717 : 1764 : struct nvme_ctrlr *nvme_ctrlr = io_device;
718 : :
719 : 1764 : nvme_ctrlr_delete(nvme_ctrlr);
720 : 1764 : }
721 : :
722 : : static void
723 : 1764 : nvme_ctrlr_unregister(void *ctx)
724 : : {
725 : 1764 : struct nvme_ctrlr *nvme_ctrlr = ctx;
726 : :
727 : 1764 : spdk_io_device_unregister(nvme_ctrlr, nvme_ctrlr_unregister_cb);
728 : 1764 : }
729 : :
730 : : static bool
731 : 8162 : nvme_ctrlr_can_be_unregistered(struct nvme_ctrlr *nvme_ctrlr)
732 : : {
733 [ + + + + ]: 8162 : if (!nvme_ctrlr->destruct) {
734 : 4774 : return false;
735 : : }
736 : :
737 [ + + + - : 3388 : if (nvme_ctrlr->ref > 0) {
+ + ]
738 : 1609 : return false;
739 : : }
740 : :
741 [ + + - + ]: 1779 : if (nvme_ctrlr->resetting) {
742 : 15 : return false;
743 : : }
744 : :
745 [ + + - + ]: 1764 : if (nvme_ctrlr->ana_log_page_updating) {
746 : 0 : return false;
747 : : }
748 : :
749 [ + + - + ]: 1764 : if (nvme_ctrlr->io_path_cache_clearing) {
750 : 0 : return false;
751 : : }
752 : :
753 : 1764 : return true;
754 : 294 : }
755 : :
756 : : static void
757 : 5730 : nvme_ctrlr_put_ref(struct nvme_ctrlr *nvme_ctrlr)
758 : : {
759 [ + + + - ]: 5730 : pthread_mutex_lock(&nvme_ctrlr->mutex);
760 : 1519 : SPDK_DTRACE_PROBE2(bdev_nvme_ctrlr_release, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ref);
761 : :
762 [ + + + - : 5730 : assert(nvme_ctrlr->ref > 0);
+ - # # ]
763 [ + - + - ]: 5730 : nvme_ctrlr->ref--;
764 : :
765 [ + + ]: 5730 : if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
766 [ + + + - ]: 3981 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
767 : 3981 : return;
768 : : }
769 : :
770 [ + + + - ]: 1749 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
771 : :
772 [ + - + - ]: 1749 : spdk_thread_exec_msg(nvme_ctrlr->thread, nvme_ctrlr_unregister, nvme_ctrlr);
773 : 215 : }
774 : :
775 : : static void
776 : 3938 : nvme_ctrlr_get_ref(struct nvme_ctrlr *nvme_ctrlr)
777 : : {
778 [ + + + - ]: 3938 : pthread_mutex_lock(&nvme_ctrlr->mutex);
779 [ + - + - ]: 3938 : nvme_ctrlr->ref++;
780 [ + + + - ]: 3938 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
781 : 3938 : }
782 : :
783 : : static void
784 : 496999 : bdev_nvme_clear_current_io_path(struct nvme_bdev_channel *nbdev_ch)
785 : : {
786 [ + - + - ]: 496999 : nbdev_ch->current_io_path = NULL;
787 [ + - + - ]: 496999 : nbdev_ch->rr_counter = 0;
788 : 496999 : }
789 : :
790 : : static struct nvme_io_path *
791 : 32 : _bdev_nvme_get_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns)
792 : : {
793 : : struct nvme_io_path *io_path;
794 : :
795 [ + + # # : 64 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
796 [ + + # # : 60 : if (io_path->nvme_ns == nvme_ns) {
# # ]
797 : 28 : break;
798 : : }
799 : 8 : }
800 : :
801 : 32 : return io_path;
802 : : }
803 : :
804 : : static struct nvme_io_path *
805 : 2431 : nvme_io_path_alloc(void)
806 : : {
807 : : struct nvme_io_path *io_path;
808 : :
809 : 2431 : io_path = calloc(1, sizeof(*io_path));
810 [ + + ]: 2431 : if (io_path == NULL) {
811 : 0 : SPDK_ERRLOG("Failed to alloc io_path.\n");
812 : 0 : return NULL;
813 : : }
814 : :
815 [ + + + + ]: 2431 : if (g_opts.io_path_stat) {
816 [ # # # # ]: 0 : io_path->stat = calloc(1, sizeof(struct spdk_bdev_io_stat));
817 [ # # # # : 0 : if (io_path->stat == NULL) {
# # ]
818 : 0 : free(io_path);
819 : 0 : SPDK_ERRLOG("Failed to alloc io_path stat.\n");
820 : 0 : return NULL;
821 : : }
822 [ # # # # ]: 0 : spdk_bdev_reset_io_stat(io_path->stat, SPDK_BDEV_RESET_STAT_MAXMIN);
823 : 0 : }
824 : :
825 : 2431 : return io_path;
826 : 60 : }
827 : :
828 : : static void
829 : 2431 : nvme_io_path_free(struct nvme_io_path *io_path)
830 : : {
831 [ + - + - ]: 2431 : free(io_path->stat);
832 : 2431 : free(io_path);
833 : 2431 : }
834 : :
835 : : static int
836 : 2431 : _bdev_nvme_add_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ns)
837 : : {
838 : : struct nvme_io_path *io_path;
839 : : struct spdk_io_channel *ch;
840 : : struct nvme_ctrlr_channel *ctrlr_ch;
841 : : struct nvme_qpair *nvme_qpair;
842 : :
843 : 2431 : io_path = nvme_io_path_alloc();
844 [ + + ]: 2431 : if (io_path == NULL) {
845 : 0 : return -ENOMEM;
846 : : }
847 : :
848 [ + - + - ]: 2431 : io_path->nvme_ns = nvme_ns;
849 : :
850 [ + - + - ]: 2431 : ch = spdk_get_io_channel(nvme_ns->ctrlr);
851 [ + + ]: 2431 : if (ch == NULL) {
852 : 0 : nvme_io_path_free(io_path);
853 : 0 : SPDK_ERRLOG("Failed to alloc io_channel.\n");
854 : 0 : return -ENOMEM;
855 : : }
856 : :
857 : 2431 : ctrlr_ch = spdk_io_channel_get_ctx(ch);
858 : :
859 [ + - + - ]: 2431 : nvme_qpair = ctrlr_ch->qpair;
860 [ + + # # ]: 2431 : assert(nvme_qpair != NULL);
861 : :
862 [ + - + - ]: 2431 : io_path->qpair = nvme_qpair;
863 [ + - + - : 2431 : TAILQ_INSERT_TAIL(&nvme_qpair->io_path_list, io_path, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
864 : :
865 [ + - + - ]: 2431 : io_path->nbdev_ch = nbdev_ch;
866 [ + - + - : 2431 : STAILQ_INSERT_TAIL(&nbdev_ch->io_path_list, io_path, stailq);
+ - + - +
- + - + -
+ - + - +
- + - +
- ]
867 : :
868 : 2431 : bdev_nvme_clear_current_io_path(nbdev_ch);
869 : :
870 : 2431 : return 0;
871 : 60 : }
872 : :
873 : : static void
874 : 2431 : bdev_nvme_clear_retry_io_path(struct nvme_bdev_channel *nbdev_ch,
875 : : struct nvme_io_path *io_path)
876 : : {
877 : : struct nvme_bdev_io *bio;
878 : :
879 [ + + + - : 2435 : TAILQ_FOREACH(bio, &nbdev_ch->retry_io_list, retry_link) {
+ - + - #
# # # #
# ]
880 [ + + # # : 4 : if (bio->io_path == io_path) {
# # ]
881 [ # # # # ]: 4 : bio->io_path = NULL;
882 : 1 : }
883 : 1 : }
884 : 2431 : }
885 : :
886 : : static void
887 : 2431 : _bdev_nvme_delete_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *io_path)
888 : : {
889 : : struct spdk_io_channel *ch;
890 : : struct nvme_qpair *nvme_qpair;
891 : : struct nvme_ctrlr_channel *ctrlr_ch;
892 : : struct nvme_bdev *nbdev;
893 : :
894 : 2431 : nbdev = spdk_io_channel_get_io_device(spdk_io_channel_from_ctx(nbdev_ch));
895 : :
896 : : /* Add the statistics to nvme_ns before this path is destroyed. */
897 [ + + + - ]: 2431 : pthread_mutex_lock(&nbdev->mutex);
898 [ + + + + : 2431 : if (nbdev->ref != 0 && io_path->nvme_ns->stat != NULL && io_path->stat != NULL) {
+ - + - +
- + - + -
- + # # #
# # # ]
899 [ # # # # : 0 : spdk_bdev_add_io_stat(io_path->nvme_ns->stat, io_path->stat);
# # # # #
# # # ]
900 : 0 : }
901 [ + + + - ]: 2431 : pthread_mutex_unlock(&nbdev->mutex);
902 : :
903 : 2431 : bdev_nvme_clear_current_io_path(nbdev_ch);
904 : 2431 : bdev_nvme_clear_retry_io_path(nbdev_ch, io_path);
905 : :
906 [ + + + + : 2433 : STAILQ_REMOVE(&nbdev_ch->io_path_list, io_path, nvme_io_path, stailq);
+ + + + +
- + - + -
+ - + - +
- + - + -
+ - - + +
- + - + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
907 [ + - + - ]: 2431 : io_path->nbdev_ch = NULL;
908 : :
909 [ + - + - ]: 2431 : nvme_qpair = io_path->qpair;
910 [ + + # # ]: 2431 : assert(nvme_qpair != NULL);
911 : :
912 [ + - + - ]: 2431 : ctrlr_ch = nvme_qpair->ctrlr_ch;
913 [ + + # # ]: 2431 : assert(ctrlr_ch != NULL);
914 : :
915 : 2431 : ch = spdk_io_channel_from_ctx(ctrlr_ch);
916 : 2431 : spdk_put_io_channel(ch);
917 : :
918 : : /* After an io_path is removed, I/Os submitted to it may complete and update statistics
919 : : * of the io_path. To avoid heap-use-after-free error from this case, do not free the
920 : : * io_path here but free the io_path when the associated qpair is freed. It is ensured
921 : : * that all I/Os submitted to the io_path are completed when the associated qpair is freed.
922 : : */
923 : 2431 : }
924 : :
925 : : static void
926 : 2373 : _bdev_nvme_delete_io_paths(struct nvme_bdev_channel *nbdev_ch)
927 : : {
928 : : struct nvme_io_path *io_path, *tmp_io_path;
929 : :
930 [ + + + + : 4796 : STAILQ_FOREACH_SAFE(io_path, &nbdev_ch->io_path_list, stailq, tmp_io_path) {
+ - + + +
- + - + -
+ + ]
931 : 2423 : _bdev_nvme_delete_io_path(nbdev_ch, io_path);
932 : 58 : }
933 : 2373 : }
934 : :
935 : : static int
936 : 2373 : bdev_nvme_create_bdev_channel_cb(void *io_device, void *ctx_buf)
937 : : {
938 : 2373 : struct nvme_bdev_channel *nbdev_ch = ctx_buf;
939 : 2373 : struct nvme_bdev *nbdev = io_device;
940 : : struct nvme_ns *nvme_ns;
941 : : int rc;
942 : :
943 [ + - + - : 2373 : STAILQ_INIT(&nbdev_ch->io_path_list);
+ - + - +
- + - + -
+ - ]
944 [ + - + - : 2373 : TAILQ_INIT(&nbdev_ch->retry_io_list);
+ - + - +
- + - + -
+ - ]
945 : :
946 [ + + + - ]: 2373 : pthread_mutex_lock(&nbdev->mutex);
947 : :
948 [ + - + - : 2373 : nbdev_ch->mp_policy = nbdev->mp_policy;
+ - + - ]
949 [ + - + - : 2373 : nbdev_ch->mp_selector = nbdev->mp_selector;
+ - + - ]
950 [ + - + - : 2373 : nbdev_ch->rr_min_io = nbdev->rr_min_io;
+ - + - ]
951 : :
952 [ + + + - : 4796 : TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
+ - + + +
- + - +
- ]
953 : 2423 : rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns);
954 [ + + ]: 2423 : if (rc != 0) {
955 [ # # # # ]: 0 : pthread_mutex_unlock(&nbdev->mutex);
956 : :
957 : 0 : _bdev_nvme_delete_io_paths(nbdev_ch);
958 : 0 : return rc;
959 : : }
960 : 58 : }
961 [ + + + - ]: 2373 : pthread_mutex_unlock(&nbdev->mutex);
962 : :
963 : 2373 : return 0;
964 : 47 : }
965 : :
966 : : /* If cpl != NULL, complete the bdev_io with nvme status based on 'cpl'.
967 : : * If cpl == NULL, complete the bdev_io with bdev status based on 'status'.
968 : : */
969 : : static inline void
970 : 29452623 : __bdev_nvme_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status,
971 : : const struct spdk_nvme_cpl *cpl)
972 : : {
973 [ + + + + : 29452623 : spdk_trace_record(TRACE_BDEV_NVME_IO_DONE, 0, 0, (uintptr_t)bdev_io->driver_ctx,
+ - + - +
- + - + -
- + # # ]
974 : : (uintptr_t)bdev_io);
975 [ + + ]: 29452623 : if (cpl) {
976 [ + - + - : 27519613 : spdk_bdev_io_complete_nvme_status(bdev_io, cpl->cdw0, cpl->status.sct, cpl->status.sc);
+ - + - +
- + - + -
+ - ]
977 : 606066 : } else {
978 : 1933010 : spdk_bdev_io_complete(bdev_io, status);
979 : : }
980 : 29452623 : }
981 : :
982 : : static void bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch);
983 : :
984 : : static void
985 : 2373 : bdev_nvme_destroy_bdev_channel_cb(void *io_device, void *ctx_buf)
986 : : {
987 : 2373 : struct nvme_bdev_channel *nbdev_ch = ctx_buf;
988 : :
989 : 2373 : bdev_nvme_abort_retry_ios(nbdev_ch);
990 : 2373 : _bdev_nvme_delete_io_paths(nbdev_ch);
991 : 2373 : }
992 : :
993 : : static inline bool
994 : 29947771 : bdev_nvme_io_type_is_admin(enum spdk_bdev_io_type io_type)
995 : : {
996 [ + + ]: 29947771 : switch (io_type) {
997 : 136 : case SPDK_BDEV_IO_TYPE_RESET:
998 : : case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
999 : : case SPDK_BDEV_IO_TYPE_ABORT:
1000 : 142 : return true;
1001 : 29341523 : default:
1002 : 29947629 : break;
1003 : : }
1004 : :
1005 : 29947629 : return false;
1006 : 606112 : }
1007 : :
1008 : : static inline bool
1009 : 10505631 : nvme_ns_is_active(struct nvme_ns *nvme_ns)
1010 : : {
1011 [ + + + + : 10505631 : if (spdk_unlikely(nvme_ns->ana_state_updating)) {
+ - - + ]
1012 : 4051 : return false;
1013 : : }
1014 : :
1015 [ + + + - : 10501580 : if (spdk_unlikely(nvme_ns->ns == NULL)) {
- + ]
1016 : 0 : return false;
1017 : : }
1018 : :
1019 : 10501580 : return true;
1020 : 265672 : }
1021 : :
1022 : : static inline bool
1023 : 10505574 : nvme_ns_is_accessible(struct nvme_ns *nvme_ns)
1024 : : {
1025 [ + + ]: 10505574 : if (spdk_unlikely(!nvme_ns_is_active(nvme_ns))) {
1026 : 4051 : return false;
1027 : : }
1028 : :
1029 [ + + + - : 10501523 : switch (nvme_ns->ana_state) {
+ - ]
1030 : 9465706 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
1031 : : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
1032 : 9731347 : return true;
1033 : 770167 : default:
1034 : 770176 : break;
1035 : : }
1036 : :
1037 : 770176 : return false;
1038 : 265651 : }
1039 : :
1040 : : static inline bool
1041 : 11830482 : nvme_qpair_is_connected(struct nvme_qpair *nvme_qpair)
1042 : : {
1043 [ + + + - : 11830482 : if (spdk_unlikely(nvme_qpair->qpair == NULL)) {
- + ]
1044 : 1303154 : return false;
1045 : : }
1046 : :
1047 [ + + + - : 10527328 : if (spdk_unlikely(spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) !=
- + ]
1048 : : SPDK_NVME_QPAIR_FAILURE_NONE)) {
1049 : 2696 : return false;
1050 : : }
1051 : :
1052 [ + + + - : 10524632 : if (spdk_unlikely(nvme_qpair->ctrlr_ch->reset_iter != NULL)) {
+ - + - -
+ ]
1053 : 9907 : return false;
1054 : : }
1055 : :
1056 : 10514725 : return true;
1057 : 265690 : }
1058 : :
1059 : : static inline bool
1060 : 11340610 : nvme_io_path_is_available(struct nvme_io_path *io_path)
1061 : : {
1062 [ + + + - : 11340610 : if (spdk_unlikely(!nvme_qpair_is_connected(io_path->qpair))) {
- + ]
1063 : 835526 : return false;
1064 : : }
1065 : :
1066 [ + + + - : 10505084 : if (spdk_unlikely(!nvme_ns_is_accessible(io_path->nvme_ns))) {
- + ]
1067 : 774107 : return false;
1068 : : }
1069 : :
1070 : 9730977 : return true;
1071 : 265657 : }
1072 : :
1073 : : static inline bool
1074 : 480231 : nvme_ctrlr_is_failed(struct nvme_ctrlr *nvme_ctrlr)
1075 : : {
1076 [ + + # # ]: 480231 : if (nvme_ctrlr->destruct) {
1077 : 512 : return true;
1078 : : }
1079 : :
1080 [ + + # # ]: 479719 : if (nvme_ctrlr->fast_io_fail_timedout) {
1081 : 74300 : return true;
1082 : : }
1083 : :
1084 [ + + # # ]: 405419 : if (nvme_ctrlr->resetting) {
1085 [ + + # # : 276022 : if (nvme_ctrlr->opts.reconnect_delay_sec != 0) {
# # # # ]
1086 : 1044 : return false;
1087 : : } else {
1088 : 274978 : return true;
1089 : : }
1090 : : }
1091 : :
1092 [ + + # # ]: 129397 : if (nvme_ctrlr->reconnect_is_delayed) {
1093 : 4104 : return false;
1094 : : }
1095 : :
1096 [ - + # # ]: 125293 : if (nvme_ctrlr->disabled) {
1097 : 0 : return true;
1098 : : }
1099 : :
1100 [ + + # # : 125293 : if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
# # ]
1101 : 123245 : return true;
1102 : : } else {
1103 : 2048 : return false;
1104 : : }
1105 : 9 : }
1106 : :
1107 : : static bool
1108 : 13244 : nvme_ctrlr_is_available(struct nvme_ctrlr *nvme_ctrlr)
1109 : : {
1110 [ + + # # ]: 13244 : if (nvme_ctrlr->destruct) {
1111 : 987 : return false;
1112 : : }
1113 : :
1114 [ + + # # : 12257 : if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
# # ]
1115 : 12 : return false;
1116 : : }
1117 : :
1118 [ + + - + : 12245 : if (nvme_ctrlr->resetting || nvme_ctrlr->reconnect_is_delayed) {
# # # # ]
1119 : 4023 : return false;
1120 : : }
1121 : :
1122 [ + + # # ]: 8222 : if (nvme_ctrlr->disabled) {
1123 : 0 : return false;
1124 : : }
1125 : :
1126 : 8222 : return true;
1127 : 33 : }
1128 : :
1129 : : /* Simulate circular linked list. */
1130 : : static inline struct nvme_io_path *
1131 : 6985478 : nvme_io_path_get_next(struct nvme_bdev_channel *nbdev_ch, struct nvme_io_path *prev_path)
1132 : : {
1133 : : struct nvme_io_path *next_path;
1134 : :
1135 [ + + ]: 6985478 : if (prev_path != NULL) {
1136 [ # # # # : 5351019 : next_path = STAILQ_NEXT(prev_path, stailq);
# # ]
1137 [ + + ]: 5351019 : if (next_path != NULL) {
1138 : 2554231 : return next_path;
1139 : : }
1140 : 25 : }
1141 : :
1142 [ + - + - : 4431247 : return STAILQ_FIRST(&nbdev_ch->io_path_list);
+ - ]
1143 : 120 : }
1144 : :
1145 : : static struct nvme_io_path *
1146 : 2804961 : _bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch)
1147 : : {
1148 : 2804961 : struct nvme_io_path *io_path, *start, *non_optimized = NULL;
1149 : :
1150 [ + - + - ]: 2804961 : start = nvme_io_path_get_next(nbdev_ch, nbdev_ch->current_io_path);
1151 : :
1152 : 2804961 : io_path = start;
1153 : 86 : do {
1154 [ + + ]: 4818833 : if (spdk_likely(nvme_io_path_is_available(io_path))) {
1155 [ + + + - : 3209336 : switch (io_path->nvme_ns->ana_state) {
+ - + - +
- - ]
1156 : 638252 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
1157 [ + - + - ]: 638318 : nbdev_ch->current_io_path = io_path;
1158 : 638318 : return io_path;
1159 : 2571008 : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
1160 [ + + ]: 2571018 : if (non_optimized == NULL) {
1161 : 1983971 : non_optimized = io_path;
1162 : 7 : }
1163 : 2571018 : break;
1164 : 0 : default:
1165 [ # # ]: 0 : assert(false);
1166 : : break;
1167 : : }
1168 : 10 : }
1169 : 4180515 : io_path = nvme_io_path_get_next(nbdev_ch, io_path);
1170 [ + + ]: 4180515 : } while (io_path != start);
1171 : :
1172 [ + + # # : 2166643 : if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) {
# # ]
1173 : : /* We come here only if there is no optimized path. Cache even non_optimized
1174 : : * path for load balance across multiple non_optimized paths.
1175 : : */
1176 [ # # # # ]: 534749 : nbdev_ch->current_io_path = non_optimized;
1177 : 1 : }
1178 : :
1179 : 2166643 : return non_optimized;
1180 : 86 : }
1181 : :
1182 : : static struct nvme_io_path *
1183 : 16 : _bdev_nvme_find_io_path_min_qd(struct nvme_bdev_channel *nbdev_ch)
1184 : : {
1185 : : struct nvme_io_path *io_path;
1186 : 16 : struct nvme_io_path *optimized = NULL, *non_optimized = NULL;
1187 : 16 : uint32_t opt_min_qd = UINT32_MAX, non_opt_min_qd = UINT32_MAX;
1188 : : uint32_t num_outstanding_reqs;
1189 : :
1190 [ + + # # : 64 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
1191 [ - + # # : 48 : if (spdk_unlikely(!nvme_qpair_is_connected(io_path->qpair))) {
# # ]
1192 : : /* The device is currently resetting. */
1193 : 0 : continue;
1194 : : }
1195 : :
1196 [ - + # # : 48 : if (spdk_unlikely(!nvme_ns_is_active(io_path->nvme_ns))) {
# # ]
1197 : 0 : continue;
1198 : : }
1199 : :
1200 [ # # # # : 48 : num_outstanding_reqs = spdk_nvme_qpair_get_num_outstanding_reqs(io_path->qpair->qpair);
# # # # ]
1201 [ + + + # : 48 : switch (io_path->nvme_ns->ana_state) {
# # # # #
# # ]
1202 : 18 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
1203 [ + + ]: 24 : if (num_outstanding_reqs < opt_min_qd) {
1204 : 20 : opt_min_qd = num_outstanding_reqs;
1205 : 20 : optimized = io_path;
1206 : 5 : }
1207 : 24 : break;
1208 : 9 : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
1209 [ + + ]: 12 : if (num_outstanding_reqs < non_opt_min_qd) {
1210 : 12 : non_opt_min_qd = num_outstanding_reqs;
1211 : 12 : non_optimized = io_path;
1212 : 3 : }
1213 : 12 : break;
1214 : 9 : default:
1215 : 12 : break;
1216 : : }
1217 : 12 : }
1218 : :
1219 : : /* don't cache io path for BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH selector */
1220 [ + + ]: 16 : if (optimized != NULL) {
1221 : 12 : return optimized;
1222 : : }
1223 : :
1224 : 4 : return non_optimized;
1225 : 4 : }
1226 : :
1227 : : static inline struct nvme_io_path *
1228 : 29468543 : bdev_nvme_find_io_path(struct nvme_bdev_channel *nbdev_ch)
1229 : : {
1230 [ + + + - : 29468543 : if (spdk_likely(nbdev_ch->current_io_path != NULL)) {
+ + ]
1231 [ + + + - : 27834068 : if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE) {
- + ]
1232 [ + - + - ]: 26663552 : return nbdev_ch->current_io_path;
1233 [ + + # # : 1170516 : } else if (nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) {
# # ]
1234 [ + + # # : 1170516 : if (++nbdev_ch->rr_counter < nbdev_ch->rr_min_io) {
# # # # ]
1235 [ # # # # ]: 12 : return nbdev_ch->current_io_path;
1236 : : }
1237 [ # # # # ]: 1170504 : nbdev_ch->rr_counter = 0;
1238 : 7 : }
1239 : 7 : }
1240 : :
1241 [ + + + + : 2804979 : if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE ||
- + # # ]
1242 [ + + # # ]: 1170593 : nbdev_ch->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) {
1243 : 2804963 : return _bdev_nvme_find_io_path(nbdev_ch);
1244 : : } else {
1245 : 16 : return _bdev_nvme_find_io_path_min_qd(nbdev_ch);
1246 : : }
1247 : 605621 : }
1248 : :
1249 : : /* Return true if there is any io_path whose qpair is active or ctrlr is not failed,
1250 : : * or false otherwise.
1251 : : *
1252 : : * If any io_path has an active qpair but find_io_path() returned NULL, its namespace
1253 : : * is likely to be non-accessible now but may become accessible.
1254 : : *
1255 : : * If any io_path has an unfailed ctrlr but find_io_path() returned NULL, the ctrlr
1256 : : * is likely to be resetting now but the reset may succeed. A ctrlr is set to unfailed
1257 : : * when starting to reset it but it is set to failed when the reset failed. Hence, if
1258 : : * a ctrlr is unfailed, it is likely that it works fine or is resetting.
1259 : : */
1260 : : static bool
1261 : 489341 : any_io_path_may_become_available(struct nvme_bdev_channel *nbdev_ch)
1262 : : {
1263 : : struct nvme_io_path *io_path;
1264 : :
1265 [ + + + + : 489341 : if (nbdev_ch->resetting) {
# # # # ]
1266 : 4 : return false;
1267 : : }
1268 : :
1269 [ + + # # : 963396 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
1270 [ - + + + : 490361 : if (io_path->nvme_ns->ana_transition_timedout) {
# # # # #
# # # ]
1271 : 1024 : continue;
1272 : : }
1273 : :
1274 [ + + + + : 489337 : if (nvme_qpair_is_connected(io_path->qpair) ||
# # # # ]
1275 [ + + # # : 480231 : !nvme_ctrlr_is_failed(io_path->qpair->ctrlr)) {
# # # # ]
1276 : 16302 : return true;
1277 : : }
1278 : 2 : }
1279 : :
1280 : 473035 : return false;
1281 : 15 : }
1282 : :
1283 : : static void
1284 : 19941 : bdev_nvme_retry_io(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io)
1285 : : {
1286 [ # # ]: 19941 : struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx;
1287 : : struct spdk_io_channel *ch;
1288 : :
1289 [ + + + - : 19941 : if (nbdev_io->io_path != NULL && nvme_io_path_is_available(nbdev_io->io_path)) {
# # # # #
# # # ]
1290 : 3643 : _bdev_nvme_submit_request(nbdev_ch, bdev_io);
1291 : 3 : } else {
1292 : 16298 : ch = spdk_io_channel_from_ctx(nbdev_ch);
1293 : 16298 : bdev_nvme_submit_request(ch, bdev_io);
1294 : : }
1295 : 19941 : }
1296 : :
1297 : : static int
1298 : 4454 : bdev_nvme_retry_ios(void *arg)
1299 : : {
1300 : 4454 : struct nvme_bdev_channel *nbdev_ch = arg;
1301 : : struct nvme_bdev_io *bio, *tmp_bio;
1302 : : uint64_t now, delay_us;
1303 : :
1304 : 4454 : now = spdk_get_ticks();
1305 : :
1306 [ + + + + : 24395 : TAILQ_FOREACH_SAFE(bio, &nbdev_ch->retry_io_list, retry_link, tmp_bio) {
# # # # #
# # # # #
# # ]
1307 [ + + # # : 20614 : if (bio->retry_ticks > now) {
# # ]
1308 : 673 : break;
1309 : : }
1310 : :
1311 [ + + # # : 19941 : TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1312 : :
1313 : 19941 : bdev_nvme_retry_io(nbdev_ch, spdk_bdev_io_from_ctx(bio));
1314 : 14 : }
1315 : :
1316 [ # # ]: 4454 : spdk_poller_unregister(&nbdev_ch->retry_io_poller);
1317 : :
1318 [ # # # # : 4454 : bio = TAILQ_FIRST(&nbdev_ch->retry_io_list);
# # ]
1319 [ + + ]: 4454 : if (bio != NULL) {
1320 [ - + # # : 685 : delay_us = (bio->retry_ticks - now) * SPDK_SEC_TO_USEC / spdk_get_ticks_hz();
# # ]
1321 : :
1322 [ # # # # ]: 685 : nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch,
1323 : : delay_us);
1324 : 4 : }
1325 : :
1326 : 4454 : return SPDK_POLLER_BUSY;
1327 : : }
1328 : :
1329 : : static void
1330 : 19949 : bdev_nvme_queue_retry_io(struct nvme_bdev_channel *nbdev_ch,
1331 : : struct nvme_bdev_io *bio, uint64_t delay_ms)
1332 : : {
1333 : : struct nvme_bdev_io *tmp_bio;
1334 : :
1335 [ # # # # : 19949 : bio->retry_ticks = spdk_get_ticks() + delay_ms * spdk_get_ticks_hz() / 1000ULL;
# # ]
1336 : :
1337 [ + + # # : 52378 : TAILQ_FOREACH_REVERSE(tmp_bio, &nbdev_ch->retry_io_list, retry_io_head, retry_link) {
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1338 [ + + # # : 48571 : if (tmp_bio->retry_ticks <= bio->retry_ticks) {
# # # # #
# ]
1339 [ + + # # : 16142 : TAILQ_INSERT_AFTER(&nbdev_ch->retry_io_list, tmp_bio, bio,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1340 : : retry_link);
1341 : 16142 : return;
1342 : : }
1343 : 0 : }
1344 : :
1345 : : /* No earlier I/Os were found. This I/O must be the new head. */
1346 [ + + # # : 3807 : TAILQ_INSERT_HEAD(&nbdev_ch->retry_io_list, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1347 : :
1348 [ # # ]: 3807 : spdk_poller_unregister(&nbdev_ch->retry_io_poller);
1349 : :
1350 [ # # # # ]: 3807 : nbdev_ch->retry_io_poller = SPDK_POLLER_REGISTER(bdev_nvme_retry_ios, nbdev_ch,
1351 : : delay_ms * 1000ULL);
1352 : 16 : }
1353 : :
1354 : : static void
1355 : 2541 : bdev_nvme_abort_retry_ios(struct nvme_bdev_channel *nbdev_ch)
1356 : : {
1357 : : struct nvme_bdev_io *bio, *tmp_bio;
1358 : :
1359 [ + + + + : 2545 : TAILQ_FOREACH_SAFE(bio, &nbdev_ch->retry_io_list, retry_link, tmp_bio) {
+ - + - #
# # # # #
+ - ]
1360 [ - + # # : 4 : TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1361 : 4 : __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED, NULL);
1362 : 1 : }
1363 : :
1364 [ + - ]: 2541 : spdk_poller_unregister(&nbdev_ch->retry_io_poller);
1365 : 2541 : }
1366 : :
1367 : : static int
1368 : 7249 : bdev_nvme_abort_retry_io(struct nvme_bdev_channel *nbdev_ch,
1369 : : struct nvme_bdev_io *bio_to_abort)
1370 : : {
1371 : : struct nvme_bdev_io *bio;
1372 : :
1373 [ + + # # : 7249 : TAILQ_FOREACH(bio, &nbdev_ch->retry_io_list, retry_link) {
# # # # #
# # # #
# ]
1374 [ + + ]: 4 : if (bio == bio_to_abort) {
1375 [ - + # # : 4 : TAILQ_REMOVE(&nbdev_ch->retry_io_list, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1376 : 4 : __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_ABORTED, NULL);
1377 : 4 : return 0;
1378 : : }
1379 : 0 : }
1380 : :
1381 : 7245 : return -ENOENT;
1382 : 6 : }
1383 : :
1384 : : static void
1385 : 10524 : bdev_nvme_update_nvme_error_stat(struct spdk_bdev_io *bdev_io, const struct spdk_nvme_cpl *cpl)
1386 : : {
1387 : : struct nvme_bdev *nbdev;
1388 : : uint16_t sct, sc;
1389 : :
1390 [ + + + + : 10524 : assert(spdk_nvme_cpl_is_error(cpl));
# # # # #
# # # # #
# # # # ]
1391 : :
1392 [ # # # # : 10524 : nbdev = bdev_io->bdev->ctxt;
# # # # ]
1393 : :
1394 [ + + # # : 10524 : if (nbdev->err_stat == NULL) {
# # ]
1395 : 6947 : return;
1396 : : }
1397 : :
1398 [ # # # # : 3577 : sct = cpl->status.sct;
# # ]
1399 [ # # # # : 3577 : sc = cpl->status.sc;
# # ]
1400 : :
1401 [ - + # # ]: 3577 : pthread_mutex_lock(&nbdev->mutex);
1402 : :
1403 [ # # # # : 3577 : nbdev->err_stat->status_type[sct]++;
# # # # #
# ]
1404 [ + - ]: 3577 : switch (sct) {
1405 : 3577 : case SPDK_NVME_SCT_GENERIC:
1406 : : case SPDK_NVME_SCT_COMMAND_SPECIFIC:
1407 : : case SPDK_NVME_SCT_MEDIA_ERROR:
1408 : : case SPDK_NVME_SCT_PATH:
1409 [ # # # # : 3577 : nbdev->err_stat->status[sct][sc]++;
# # # # #
# # # ]
1410 : 3577 : break;
1411 : 0 : default:
1412 : 0 : break;
1413 : : }
1414 : :
1415 [ - + # # ]: 3577 : pthread_mutex_unlock(&nbdev->mutex);
1416 : 14 : }
1417 : :
1418 : : static inline void
1419 : 27519244 : bdev_nvme_update_io_path_stat(struct nvme_bdev_io *bio)
1420 : : {
1421 : 27519244 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1422 [ + - + - : 27519244 : uint64_t num_blocks = bdev_io->u.bdev.num_blocks;
+ - + - ]
1423 [ + - + - : 27519244 : uint32_t blocklen = bdev_io->bdev->blocklen;
+ - + - ]
1424 : : struct spdk_bdev_io_stat *stat;
1425 : : uint64_t tsc_diff;
1426 : :
1427 [ + + + - : 27519244 : if (bio->io_path->stat == NULL) {
+ - + - +
- ]
1428 : 27519244 : return;
1429 : : }
1430 : :
1431 [ # # # # ]: 0 : tsc_diff = spdk_get_ticks() - bio->submit_tsc;
1432 [ # # # # : 0 : stat = bio->io_path->stat;
# # # # ]
1433 : :
1434 [ # # # # : 0 : switch (bdev_io->type) {
# # # # #
# ]
1435 : 0 : case SPDK_BDEV_IO_TYPE_READ:
1436 [ # # # # ]: 0 : stat->bytes_read += num_blocks * blocklen;
1437 [ # # ]: 0 : stat->num_read_ops++;
1438 [ # # # # ]: 0 : stat->read_latency_ticks += tsc_diff;
1439 [ # # # # : 0 : if (stat->max_read_latency_ticks < tsc_diff) {
# # ]
1440 [ # # # # ]: 0 : stat->max_read_latency_ticks = tsc_diff;
1441 : 0 : }
1442 [ # # # # : 0 : if (stat->min_read_latency_ticks > tsc_diff) {
# # ]
1443 [ # # # # ]: 0 : stat->min_read_latency_ticks = tsc_diff;
1444 : 0 : }
1445 : 0 : break;
1446 : 0 : case SPDK_BDEV_IO_TYPE_WRITE:
1447 [ # # # # ]: 0 : stat->bytes_written += num_blocks * blocklen;
1448 [ # # ]: 0 : stat->num_write_ops++;
1449 [ # # # # ]: 0 : stat->write_latency_ticks += tsc_diff;
1450 [ # # # # : 0 : if (stat->max_write_latency_ticks < tsc_diff) {
# # ]
1451 [ # # # # ]: 0 : stat->max_write_latency_ticks = tsc_diff;
1452 : 0 : }
1453 [ # # # # : 0 : if (stat->min_write_latency_ticks > tsc_diff) {
# # ]
1454 [ # # # # ]: 0 : stat->min_write_latency_ticks = tsc_diff;
1455 : 0 : }
1456 : 0 : break;
1457 : 0 : case SPDK_BDEV_IO_TYPE_UNMAP:
1458 [ # # # # ]: 0 : stat->bytes_unmapped += num_blocks * blocklen;
1459 [ # # ]: 0 : stat->num_unmap_ops++;
1460 [ # # # # ]: 0 : stat->unmap_latency_ticks += tsc_diff;
1461 [ # # # # : 0 : if (stat->max_unmap_latency_ticks < tsc_diff) {
# # ]
1462 [ # # # # ]: 0 : stat->max_unmap_latency_ticks = tsc_diff;
1463 : 0 : }
1464 [ # # # # : 0 : if (stat->min_unmap_latency_ticks > tsc_diff) {
# # ]
1465 [ # # # # ]: 0 : stat->min_unmap_latency_ticks = tsc_diff;
1466 : 0 : }
1467 : 0 : break;
1468 : 0 : case SPDK_BDEV_IO_TYPE_ZCOPY:
1469 : : /* Track the data in the start phase only */
1470 [ # # # # : 0 : if (!bdev_io->u.bdev.zcopy.start) {
# # # # #
# ]
1471 : 0 : break;
1472 : : }
1473 [ # # # # : 0 : if (bdev_io->u.bdev.zcopy.populate) {
# # # # #
# ]
1474 [ # # # # ]: 0 : stat->bytes_read += num_blocks * blocklen;
1475 [ # # ]: 0 : stat->num_read_ops++;
1476 [ # # # # ]: 0 : stat->read_latency_ticks += tsc_diff;
1477 [ # # # # : 0 : if (stat->max_read_latency_ticks < tsc_diff) {
# # ]
1478 [ # # # # ]: 0 : stat->max_read_latency_ticks = tsc_diff;
1479 : 0 : }
1480 [ # # # # : 0 : if (stat->min_read_latency_ticks > tsc_diff) {
# # ]
1481 [ # # # # ]: 0 : stat->min_read_latency_ticks = tsc_diff;
1482 : 0 : }
1483 : 0 : } else {
1484 [ # # # # ]: 0 : stat->bytes_written += num_blocks * blocklen;
1485 [ # # ]: 0 : stat->num_write_ops++;
1486 [ # # # # ]: 0 : stat->write_latency_ticks += tsc_diff;
1487 [ # # # # : 0 : if (stat->max_write_latency_ticks < tsc_diff) {
# # ]
1488 [ # # # # ]: 0 : stat->max_write_latency_ticks = tsc_diff;
1489 : 0 : }
1490 [ # # # # : 0 : if (stat->min_write_latency_ticks > tsc_diff) {
# # ]
1491 [ # # # # ]: 0 : stat->min_write_latency_ticks = tsc_diff;
1492 : 0 : }
1493 : : }
1494 : 0 : break;
1495 : 0 : case SPDK_BDEV_IO_TYPE_COPY:
1496 [ # # # # ]: 0 : stat->bytes_copied += num_blocks * blocklen;
1497 [ # # ]: 0 : stat->num_copy_ops++;
1498 [ # # # # ]: 0 : stat->copy_latency_ticks += tsc_diff;
1499 [ # # # # : 0 : if (stat->max_copy_latency_ticks < tsc_diff) {
# # ]
1500 [ # # # # ]: 0 : stat->max_copy_latency_ticks = tsc_diff;
1501 : 0 : }
1502 [ # # # # : 0 : if (stat->min_copy_latency_ticks > tsc_diff) {
# # ]
1503 [ # # # # ]: 0 : stat->min_copy_latency_ticks = tsc_diff;
1504 : 0 : }
1505 : 0 : break;
1506 : 0 : default:
1507 : 0 : break;
1508 : : }
1509 : 605970 : }
1510 : :
1511 : : static bool
1512 : 10445 : bdev_nvme_check_retry_io(struct nvme_bdev_io *bio,
1513 : : const struct spdk_nvme_cpl *cpl,
1514 : : struct nvme_bdev_channel *nbdev_ch,
1515 : : uint64_t *_delay_ms)
1516 : : {
1517 [ # # # # ]: 10445 : struct nvme_io_path *io_path = bio->io_path;
1518 [ # # # # : 10445 : struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;
# # # # ]
1519 : : const struct spdk_nvme_ctrlr_data *cdata;
1520 : :
1521 [ + + - + : 10449 : if (spdk_nvme_cpl_is_path_error(cpl) ||
# # # # #
# ]
1522 [ + + + + : 7675 : spdk_nvme_cpl_is_aborted_sq_deletion(cpl) ||
# # # # #
# # # #
# ]
1523 [ + + ]: 3643 : !nvme_io_path_is_available(io_path) ||
1524 [ - + ]: 3647 : !nvme_ctrlr_is_available(nvme_ctrlr)) {
1525 : 6806 : bdev_nvme_clear_current_io_path(nbdev_ch);
1526 [ # # # # ]: 6806 : bio->io_path = NULL;
1527 [ + + + - : 6806 : if (spdk_nvme_cpl_is_ana_error(cpl)) {
+ + - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1528 [ + + ]: 2762 : if (nvme_ctrlr_read_ana_log_page(nvme_ctrlr) == 0) {
1529 [ # # # # : 19 : io_path->nvme_ns->ana_state_updating = true;
# # # # ]
1530 : 1 : }
1531 : 1 : }
1532 [ + + ]: 6794 : if (!any_io_path_may_become_available(nbdev_ch)) {
1533 : 64 : return false;
1534 : : }
1535 [ # # ]: 6730 : *_delay_ms = 0;
1536 : 3 : } else {
1537 [ # # # # ]: 3647 : bio->retry_count++;
1538 : :
1539 [ # # # # ]: 3647 : cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
1540 : :
1541 [ + + # # : 3647 : if (cpl->status.crd != 0) {
# # # # ]
1542 [ # # # # : 4 : *_delay_ms = cdata->crdt[cpl->status.crd] * 100;
# # # # #
# # # # #
# # # # ]
1543 : 1 : } else {
1544 [ # # ]: 3643 : *_delay_ms = 0;
1545 : : }
1546 : : }
1547 : :
1548 : 10377 : return true;
1549 : 7 : }
1550 : :
1551 : : static inline void
1552 : 27529893 : bdev_nvme_io_complete_nvme_status(struct nvme_bdev_io *bio,
1553 : : const struct spdk_nvme_cpl *cpl)
1554 : : {
1555 : 27529893 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1556 : : struct nvme_bdev_channel *nbdev_ch;
1557 : 6532826 : uint64_t delay_ms;
1558 : :
1559 [ + + + - : 27529893 : assert(!bdev_nvme_io_type_is_admin(bdev_io->type));
+ - # # ]
1560 : :
1561 [ + + + + : 27529893 : if (spdk_likely(spdk_nvme_cpl_is_success(cpl))) {
+ - - + +
- + - + -
+ - ]
1562 : 27519361 : bdev_nvme_update_io_path_stat(bio);
1563 : 27519361 : goto complete;
1564 : : }
1565 : :
1566 : : /* Update error counts before deciding if retry is needed.
1567 : : * Hence, error counts may be more than the number of I/O errors.
1568 : : */
1569 : 10532 : bdev_nvme_update_nvme_error_stat(bdev_io, cpl);
1570 : :
1571 [ + + + + : 10539 : if (cpl->status.dnr != 0 || spdk_nvme_cpl_is_aborted_by_request(cpl) ||
+ + # # #
# # # # #
# # # # #
# # # #
# ]
1572 [ + + + + : 10457 : (g_opts.bdev_retry_count != -1 && bio->retry_count >= g_opts.bdev_retry_count)) {
# # # # #
# ]
1573 : 101 : goto complete;
1574 : : }
1575 : :
1576 : : /* At this point we don't know whether the sequence was successfully executed or not, so we
1577 : : * cannot retry the IO */
1578 [ - + # # : 10441 : if (bdev_io->u.bdev.accel_sequence != NULL) {
# # # # #
# ]
1579 : 0 : goto complete;
1580 : : }
1581 : :
1582 : 10441 : nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
1583 : :
1584 [ + + ]: 10441 : if (bdev_nvme_check_retry_io(bio, cpl, nbdev_ch, &delay_ms)) {
1585 : 10377 : bdev_nvme_queue_retry_io(nbdev_ch, bio, delay_ms);
1586 : 10377 : return;
1587 : : }
1588 : :
1589 : 6532869 : complete:
1590 [ + - + - ]: 27519508 : bio->retry_count = 0;
1591 [ + - + - ]: 27519508 : bio->submit_tsc = 0;
1592 [ + - + - : 27519508 : bdev_io->u.bdev.accel_sequence = NULL;
+ - + - ]
1593 : 27519508 : __bdev_nvme_io_complete(bdev_io, 0, cpl);
1594 : 606101 : }
1595 : :
1596 : : static inline void
1597 : 1935217 : bdev_nvme_io_complete(struct nvme_bdev_io *bio, int rc)
1598 : : {
1599 : 1935217 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1600 : : struct nvme_bdev_channel *nbdev_ch;
1601 : : enum spdk_bdev_io_status io_status;
1602 : :
1603 [ - + # # : 1935217 : assert(!bdev_nvme_io_type_is_admin(bdev_io->type));
# # # # ]
1604 : :
1605 [ + + + + ]: 1935217 : switch (rc) {
1606 : 1281460 : case 0:
1607 : 1281461 : io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
1608 : 1281461 : break;
1609 : 171209 : case -ENOMEM:
1610 : 171209 : io_status = SPDK_BDEV_IO_STATUS_NOMEM;
1611 : 171209 : break;
1612 : 482535 : case -ENXIO:
1613 [ + + + - : 482550 : if (g_opts.bdev_retry_count == -1 || bio->retry_count < g_opts.bdev_retry_count) {
# # # # #
# # # ]
1614 : 482547 : nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
1615 : :
1616 : 482547 : bdev_nvme_clear_current_io_path(nbdev_ch);
1617 [ # # # # ]: 482547 : bio->io_path = NULL;
1618 : :
1619 [ + + ]: 482547 : if (any_io_path_may_become_available(nbdev_ch)) {
1620 : 9572 : bdev_nvme_queue_retry_io(nbdev_ch, bio, 1000ULL);
1621 : 9572 : return;
1622 : : }
1623 : 3 : }
1624 : :
1625 : : /* fallthrough */
1626 : : default:
1627 [ # # # # : 472975 : spdk_accel_sequence_abort(bdev_io->u.bdev.accel_sequence);
# # # # ]
1628 [ # # # # : 472975 : bdev_io->u.bdev.accel_sequence = NULL;
# # # # ]
1629 : 472975 : io_status = SPDK_BDEV_IO_STATUS_FAILED;
1630 : 472975 : break;
1631 : : }
1632 : :
1633 [ # # # # ]: 1925645 : bio->retry_count = 0;
1634 [ # # # # ]: 1925645 : bio->submit_tsc = 0;
1635 : 1925645 : __bdev_nvme_io_complete(bdev_io, io_status, NULL);
1636 : 13 : }
1637 : :
1638 : : static inline void
1639 : 16 : bdev_nvme_admin_complete(struct nvme_bdev_io *bio, int rc)
1640 : : {
1641 : 16 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
1642 : : enum spdk_bdev_io_status io_status;
1643 : :
1644 [ + + + + ]: 16 : switch (rc) {
1645 : 3 : case 0:
1646 : 4 : io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
1647 : 4 : break;
1648 : 0 : case -ENOMEM:
1649 : 0 : io_status = SPDK_BDEV_IO_STATUS_NOMEM;
1650 : 0 : break;
1651 : 10 : case -ENXIO:
1652 : : /* fallthrough */
1653 : : default:
1654 : 12 : io_status = SPDK_BDEV_IO_STATUS_FAILED;
1655 : 12 : break;
1656 : : }
1657 : :
1658 : 16 : __bdev_nvme_io_complete(bdev_io, io_status, NULL);
1659 : 16 : }
1660 : :
1661 : : static void
1662 : 1557 : bdev_nvme_clear_io_path_caches_done(struct nvme_ctrlr *nvme_ctrlr,
1663 : : void *ctx, int status)
1664 : : {
1665 [ - + # # ]: 1557 : pthread_mutex_lock(&nvme_ctrlr->mutex);
1666 : :
1667 [ + + # # : 1557 : assert(nvme_ctrlr->io_path_cache_clearing == true);
# # ]
1668 [ # # ]: 1557 : nvme_ctrlr->io_path_cache_clearing = false;
1669 : :
1670 [ + + ]: 1557 : if (!nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
1671 [ - + # # ]: 1557 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
1672 : 1557 : return;
1673 : : }
1674 : :
1675 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
1676 : :
1677 : 0 : nvme_ctrlr_unregister(nvme_ctrlr);
1678 : 3 : }
1679 : :
1680 : : static void
1681 : 10176 : _bdev_nvme_clear_io_path_cache(struct nvme_qpair *nvme_qpair)
1682 : : {
1683 : : struct nvme_io_path *io_path;
1684 : :
1685 [ + + + - : 17767 : TAILQ_FOREACH(io_path, &nvme_qpair->io_path_list, tailq) {
+ - + + +
- + - +
- ]
1686 [ + + + - : 7591 : if (io_path->nbdev_ch == NULL) {
- + ]
1687 : 4811 : continue;
1688 : : }
1689 [ # # # # ]: 2780 : bdev_nvme_clear_current_io_path(io_path->nbdev_ch);
1690 : 167 : }
1691 : 10176 : }
1692 : :
1693 : : static void
1694 : 1509 : bdev_nvme_clear_io_path_cache(struct nvme_ctrlr_channel_iter *i,
1695 : : struct nvme_ctrlr *nvme_ctrlr,
1696 : : struct nvme_ctrlr_channel *ctrlr_ch,
1697 : : void *ctx)
1698 : : {
1699 [ + + # # : 1509 : assert(ctrlr_ch->qpair != NULL);
# # # # ]
1700 : :
1701 [ # # # # ]: 1509 : _bdev_nvme_clear_io_path_cache(ctrlr_ch->qpair);
1702 : :
1703 : 1509 : nvme_ctrlr_for_each_channel_continue(i, 0);
1704 : 1509 : }
1705 : :
1706 : : static void
1707 : 6565 : bdev_nvme_clear_io_path_caches(struct nvme_ctrlr *nvme_ctrlr)
1708 : : {
1709 [ - + # # ]: 6565 : pthread_mutex_lock(&nvme_ctrlr->mutex);
1710 [ + + + + ]: 6565 : if (!nvme_ctrlr_is_available(nvme_ctrlr) ||
1711 [ # # ]: 3 : nvme_ctrlr->io_path_cache_clearing) {
1712 [ - + # # ]: 5008 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
1713 : 5008 : return;
1714 : : }
1715 : :
1716 [ # # ]: 1557 : nvme_ctrlr->io_path_cache_clearing = true;
1717 [ - + # # ]: 1557 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
1718 : :
1719 : 1557 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
1720 : : bdev_nvme_clear_io_path_cache,
1721 : : NULL,
1722 : : bdev_nvme_clear_io_path_caches_done);
1723 : 15 : }
1724 : :
1725 : : static struct nvme_qpair *
1726 : 2697 : nvme_poll_group_get_qpair(struct nvme_poll_group *group, struct spdk_nvme_qpair *qpair)
1727 : : {
1728 : : struct nvme_qpair *nvme_qpair;
1729 : :
1730 [ + + + - : 2857 : TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) {
+ - - + #
# # # #
# ]
1731 [ + + + - : 2857 : if (nvme_qpair->qpair == qpair) {
- + ]
1732 : 2697 : break;
1733 : : }
1734 : 17 : }
1735 : :
1736 : 2697 : return nvme_qpair;
1737 : : }
1738 : :
1739 : : static void nvme_qpair_delete(struct nvme_qpair *nvme_qpair);
1740 : :
1741 : : static void
1742 : 2697 : bdev_nvme_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx)
1743 : : {
1744 : 2697 : struct nvme_poll_group *group = poll_group_ctx;
1745 : : struct nvme_qpair *nvme_qpair;
1746 : : struct nvme_ctrlr *nvme_ctrlr;
1747 : : struct nvme_ctrlr_channel *ctrlr_ch;
1748 : : int status;
1749 : :
1750 : 2697 : nvme_qpair = nvme_poll_group_get_qpair(group, qpair);
1751 [ + + ]: 2697 : if (nvme_qpair == NULL) {
1752 : 0 : return;
1753 : : }
1754 : :
1755 [ + + + - : 2697 : if (nvme_qpair->qpair != NULL) {
- + ]
1756 [ + - + - ]: 2697 : spdk_nvme_ctrlr_free_io_qpair(nvme_qpair->qpair);
1757 [ + - + - ]: 2697 : nvme_qpair->qpair = NULL;
1758 : 143 : }
1759 : :
1760 : 2697 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
1761 : :
1762 [ + - + - ]: 2697 : nvme_ctrlr = nvme_qpair->ctrlr;
1763 [ + - + - ]: 2697 : ctrlr_ch = nvme_qpair->ctrlr_ch;
1764 : :
1765 [ + + ]: 2697 : if (ctrlr_ch != NULL) {
1766 [ + + # # : 385 : if (ctrlr_ch->reset_iter != NULL) {
# # ]
1767 : : /* We are in a full reset sequence. */
1768 [ - + # # : 321 : if (ctrlr_ch->connect_poller != NULL) {
# # ]
1769 : : /* qpair was failed to connect. Abort the reset sequence. */
1770 [ # # # # : 0 : NVME_CTRLR_INFOLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1771 : : "qpair %p was failed to connect. abort the reset ctrlr sequence.\n",
1772 : : qpair);
1773 [ # # ]: 0 : spdk_poller_unregister(&ctrlr_ch->connect_poller);
1774 : 0 : status = -1;
1775 : 0 : } else {
1776 : : /* qpair was completed to disconnect. Just move to the next ctrlr_channel. */
1777 [ + + - + : 321 : NVME_CTRLR_INFOLOG(nvme_ctrlr,
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1778 : : "qpair %p was disconnected and freed in a reset ctrlr sequence.\n",
1779 : : qpair);
1780 : 321 : status = 0;
1781 : : }
1782 [ # # # # ]: 321 : nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, status);
1783 [ # # # # ]: 321 : ctrlr_ch->reset_iter = NULL;
1784 : 70 : } else {
1785 : : /* qpair was disconnected unexpectedly. Reset controller for recovery. */
1786 [ + + - + : 64 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpair %p was disconnected and freed. reset controller.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1787 : : qpair);
1788 : 64 : bdev_nvme_failover_ctrlr(nvme_ctrlr);
1789 : : }
1790 : 75 : } else {
1791 : : /* In this case, ctrlr_channel is already deleted. */
1792 [ + + + + : 2312 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpair %p was disconnected and freed. delete nvme_qpair.\n",
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1793 : : qpair);
1794 : 2312 : nvme_qpair_delete(nvme_qpair);
1795 : : }
1796 : 143 : }
1797 : :
1798 : : static void
1799 : 15 : bdev_nvme_check_io_qpairs(struct nvme_poll_group *group)
1800 : : {
1801 : : struct nvme_qpair *nvme_qpair;
1802 : :
1803 [ + + # # : 30 : TAILQ_FOREACH(nvme_qpair, &group->qpair_list, tailq) {
# # # # #
# # # #
# ]
1804 [ + + - + : 15 : if (nvme_qpair->qpair == NULL || nvme_qpair->ctrlr_ch == NULL) {
# # # # #
# # # ]
1805 : 1 : continue;
1806 : : }
1807 : :
1808 [ + - # # : 14 : if (spdk_nvme_qpair_get_failure_reason(nvme_qpair->qpair) !=
# # ]
1809 : : SPDK_NVME_QPAIR_FAILURE_NONE) {
1810 : 14 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
1811 : 0 : }
1812 : 0 : }
1813 : 15 : }
1814 : :
1815 : : static int
1816 : 582022501 : bdev_nvme_poll(void *arg)
1817 : : {
1818 : 582022501 : struct nvme_poll_group *group = arg;
1819 : : int64_t num_completions;
1820 : :
1821 [ + + + + : 582022501 : if (group->collect_spin_stat && group->start_ticks == 0) {
+ - - + #
# # # #
# ]
1822 [ # # # # ]: 0 : group->start_ticks = spdk_get_ticks();
1823 : 0 : }
1824 : :
1825 [ + - + - ]: 582022501 : num_completions = spdk_nvme_poll_group_process_completions(group->group, 0,
1826 : : bdev_nvme_disconnected_qpair_cb);
1827 [ + + + + : 582022501 : if (group->collect_spin_stat) {
+ - + - ]
1828 [ # # ]: 0 : if (num_completions > 0) {
1829 [ # # # # : 0 : if (group->end_ticks != 0) {
# # ]
1830 [ # # # # : 0 : group->spin_ticks += (group->end_ticks - group->start_ticks);
# # # # #
# # # ]
1831 [ # # # # ]: 0 : group->end_ticks = 0;
1832 : 0 : }
1833 [ # # # # ]: 0 : group->start_ticks = 0;
1834 : 0 : } else {
1835 [ # # # # ]: 0 : group->end_ticks = spdk_get_ticks();
1836 : : }
1837 : 0 : }
1838 : :
1839 [ + + ]: 582022501 : if (spdk_unlikely(num_completions < 0)) {
1840 : 15 : bdev_nvme_check_io_qpairs(group);
1841 : 0 : }
1842 : :
1843 : 582022501 : return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;
1844 : : }
1845 : :
1846 : : static int bdev_nvme_poll_adminq(void *arg);
1847 : :
1848 : : static void
1849 : 1516 : bdev_nvme_change_adminq_poll_period(struct nvme_ctrlr *nvme_ctrlr, uint64_t new_period_us)
1850 : : {
1851 [ - + ]: 1516 : if (spdk_interrupt_mode_is_enabled()) {
1852 : 0 : return;
1853 : : }
1854 : :
1855 [ # # ]: 1516 : spdk_poller_unregister(&nvme_ctrlr->adminq_timer_poller);
1856 : :
1857 [ # # # # ]: 1516 : nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq,
1858 : : nvme_ctrlr, new_period_us);
1859 : 146 : }
1860 : :
1861 : : static int
1862 : 529596 : bdev_nvme_poll_adminq(void *arg)
1863 : : {
1864 : : int32_t rc;
1865 : 529596 : struct nvme_ctrlr *nvme_ctrlr = arg;
1866 : : nvme_ctrlr_disconnected_cb disconnected_cb;
1867 : :
1868 [ + + # # ]: 529596 : assert(nvme_ctrlr != NULL);
1869 : :
1870 [ + - + - ]: 529596 : rc = spdk_nvme_ctrlr_process_admin_completions(nvme_ctrlr->ctrlr);
1871 [ + + ]: 529596 : if (rc < 0) {
1872 [ # # # # ]: 1595 : disconnected_cb = nvme_ctrlr->disconnected_cb;
1873 [ # # # # ]: 1595 : nvme_ctrlr->disconnected_cb = NULL;
1874 : :
1875 [ + + ]: 1595 : if (disconnected_cb != NULL) {
1876 : 831 : bdev_nvme_change_adminq_poll_period(nvme_ctrlr,
1877 [ # # ]: 73 : g_opts.nvme_adminq_poll_period_us);
1878 [ # # # # ]: 758 : disconnected_cb(nvme_ctrlr);
1879 : 73 : } else {
1880 : 837 : bdev_nvme_failover_ctrlr(nvme_ctrlr);
1881 : : }
1882 [ + + + - : 528089 : } else if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(nvme_ctrlr->ctrlr) !=
+ - ]
1883 : : SPDK_NVME_QPAIR_FAILURE_NONE) {
1884 : 6448 : bdev_nvme_clear_io_path_caches(nvme_ctrlr);
1885 : 12 : }
1886 : :
1887 : 529596 : return rc == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY;
1888 : : }
1889 : :
1890 : : static void
1891 : 1481 : nvme_bdev_free(void *io_device)
1892 : : {
1893 : 1481 : struct nvme_bdev *nbdev = io_device;
1894 : :
1895 [ + + + - ]: 1481 : pthread_mutex_destroy(&nbdev->mutex);
1896 [ + - + - : 1481 : free(nbdev->disk.name);
+ - ]
1897 [ + - + - ]: 1481 : free(nbdev->err_stat);
1898 : 1481 : free(nbdev);
1899 : 1481 : }
1900 : :
1901 : : static int
1902 : 1475 : bdev_nvme_destruct(void *ctx)
1903 : : {
1904 : 1475 : struct nvme_bdev *nbdev = ctx;
1905 : : struct nvme_ns *nvme_ns, *tmp_nvme_ns;
1906 : :
1907 : 394 : SPDK_DTRACE_PROBE2(bdev_nvme_destruct, nbdev->nbdev_ctrlr->name, nbdev->nsid);
1908 : :
1909 [ + + + - ]: 1475 : pthread_mutex_lock(&nbdev->mutex);
1910 : :
1911 [ + + + + : 2960 : TAILQ_FOREACH_SAFE(nvme_ns, &nbdev->nvme_ns_list, tailq, tmp_nvme_ns) {
+ - + + +
- + - + -
+ + ]
1912 [ + + + - : 1485 : pthread_mutex_lock(&nvme_ns->ctrlr->mutex);
+ - + - ]
1913 : :
1914 [ + - + - ]: 1485 : nvme_ns->bdev = NULL;
1915 : :
1916 [ + + + - : 1485 : assert(nvme_ns->id > 0);
+ - # # ]
1917 : :
1918 [ + + + - : 1485 : if (nvme_ctrlr_get_ns(nvme_ns->ctrlr, nvme_ns->id) == NULL) {
+ - + - +
- ]
1919 [ - + # # : 515 : pthread_mutex_unlock(&nvme_ns->ctrlr->mutex);
# # # # ]
1920 : :
1921 [ # # # # ]: 515 : nvme_ctrlr_put_ref(nvme_ns->ctrlr);
1922 : 515 : nvme_ns_free(nvme_ns);
1923 : 3 : } else {
1924 [ + + + - : 970 : pthread_mutex_unlock(&nvme_ns->ctrlr->mutex);
+ - + - ]
1925 : : }
1926 : 50 : }
1927 : :
1928 [ + + + - ]: 1475 : pthread_mutex_unlock(&nbdev->mutex);
1929 : :
1930 [ + + ]: 1475 : pthread_mutex_lock(&g_bdev_nvme_mutex);
1931 [ + + + - : 1475 : TAILQ_REMOVE(&nbdev->nbdev_ctrlr->bdevs, nbdev, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
1932 [ + + ]: 1475 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
1933 : :
1934 : 1475 : spdk_io_device_unregister(nbdev, nvme_bdev_free);
1935 : :
1936 : 1475 : return 0;
1937 : : }
1938 : :
1939 : : static int
1940 : 2701 : bdev_nvme_create_qpair(struct nvme_qpair *nvme_qpair)
1941 : : {
1942 : : struct nvme_ctrlr *nvme_ctrlr;
1943 : 1021 : struct spdk_nvme_io_qpair_opts opts;
1944 : : struct spdk_nvme_qpair *qpair;
1945 : : int rc;
1946 : :
1947 [ + - + - ]: 2701 : nvme_ctrlr = nvme_qpair->ctrlr;
1948 : :
1949 [ + - + - ]: 2701 : spdk_nvme_ctrlr_get_default_io_qpair_opts(nvme_ctrlr->ctrlr, &opts, sizeof(opts));
1950 [ + - ]: 2701 : opts.create_only = true;
1951 : : /* In interrupt mode qpairs must be created in sync mode, else it will never be connected.
1952 : : * delay_cmd_submit must be false as in interrupt mode requests cannot be submitted in
1953 : : * completion context.
1954 : : */
1955 [ + + ]: 2701 : if (!spdk_interrupt_mode_is_enabled()) {
1956 : 2699 : opts.async_mode = true;
1957 [ + + + - : 2699 : opts.delay_cmd_submit = g_opts.delay_cmd_submit;
+ - + - ]
1958 : 144 : }
1959 [ + + + - : 2701 : opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests);
- + # # +
- + - ]
1960 [ + - + - ]: 2701 : g_opts.io_queue_requests = opts.io_queue_requests;
1961 : :
1962 [ + - + - ]: 2701 : qpair = spdk_nvme_ctrlr_alloc_io_qpair(nvme_ctrlr->ctrlr, &opts, sizeof(opts));
1963 [ + + ]: 2701 : if (qpair == NULL) {
1964 : 0 : return -1;
1965 : : }
1966 : :
1967 : 601 : SPDK_DTRACE_PROBE3(bdev_nvme_create_qpair, nvme_ctrlr->nbdev_ctrlr->name,
1968 : : spdk_nvme_qpair_get_id(qpair), spdk_thread_get_id(nvme_ctrlr->thread));
1969 : :
1970 [ + + + - : 2701 : assert(nvme_qpair->group != NULL);
+ - # # ]
1971 : :
1972 [ + - + - : 2701 : rc = spdk_nvme_poll_group_add(nvme_qpair->group->group, qpair);
+ - + - ]
1973 [ - + ]: 2701 : if (rc != 0) {
1974 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Unable to begin polling on NVMe Channel.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1975 : 0 : goto err;
1976 : : }
1977 : :
1978 [ + - + - ]: 2701 : rc = spdk_nvme_ctrlr_connect_io_qpair(nvme_ctrlr->ctrlr, qpair);
1979 [ - + ]: 2701 : if (rc != 0) {
1980 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Unable to connect I/O qpair.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1981 : 0 : goto err;
1982 : : }
1983 : :
1984 [ + - + - ]: 2701 : nvme_qpair->qpair = qpair;
1985 : :
1986 [ + + + + ]: 2701 : if (!g_opts.disable_auto_failback) {
1987 : 2553 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
1988 : 107 : }
1989 : :
1990 [ + + + + : 2701 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Connecting qpair %p:%u started.\n",
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1991 : : qpair, spdk_nvme_qpair_get_id(qpair));
1992 : :
1993 : 2701 : return 0;
1994 : :
1995 : 0 : err:
1996 : 0 : spdk_nvme_ctrlr_free_io_qpair(qpair);
1997 : :
1998 : 0 : return rc;
1999 : 144 : }
2000 : :
2001 : : static void bdev_nvme_reset_io_continue(void *cb_arg, int rc);
2002 : :
2003 : : static void
2004 : 758 : bdev_nvme_complete_pending_resets(struct nvme_ctrlr *nvme_ctrlr, bool success)
2005 : : {
2006 : 758 : int rc = 0;
2007 : : struct nvme_bdev_io *bio;
2008 : :
2009 [ + + # # ]: 758 : if (!success) {
2010 : 528 : rc = -1;
2011 : 33 : }
2012 : :
2013 [ + + # # : 806 : while (!TAILQ_EMPTY(&nvme_ctrlr->pending_resets)) {
# # # # ]
2014 [ # # # # : 48 : bio = TAILQ_FIRST(&nvme_ctrlr->pending_resets);
# # ]
2015 [ - + # # : 48 : TAILQ_REMOVE(&nvme_ctrlr->pending_resets, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2016 : :
2017 : 48 : bdev_nvme_reset_io_continue(bio, rc);
2018 : : }
2019 : 758 : }
2020 : :
2021 : : /* This function marks the current trid as failed by storing the current ticks
2022 : : * and then sets the next trid to the active trid within a controller if exists.
2023 : : *
2024 : : * The purpose of the boolean return value is to request the caller to disconnect
2025 : : * the current trid now to try connecting the next trid.
2026 : : */
2027 : : static bool
2028 : 1023 : bdev_nvme_failover_trid(struct nvme_ctrlr *nvme_ctrlr, bool remove, bool start)
2029 : : {
2030 : : struct nvme_path_id *path_id, *next_path;
2031 : : int rc __attribute__((unused));
2032 : :
2033 [ # # # # : 1023 : path_id = TAILQ_FIRST(&nvme_ctrlr->trids);
# # ]
2034 [ + + # # ]: 1023 : assert(path_id);
2035 [ + + # # : 1023 : assert(path_id == nvme_ctrlr->active_path_id);
# # # # ]
2036 [ # # # # : 1023 : next_path = TAILQ_NEXT(path_id, link);
# # ]
2037 : :
2038 : : /* Update the last failed time. It means the trid is failed if its last
2039 : : * failed time is non-zero.
2040 : : */
2041 [ # # # # ]: 1023 : path_id->last_failed_tsc = spdk_get_ticks();
2042 : :
2043 [ + + ]: 1023 : if (next_path == NULL) {
2044 : : /* There is no alternate trid within a controller. */
2045 : 963 : return false;
2046 : : }
2047 : :
2048 [ + + + + : 60 : if (!start && nvme_ctrlr->opts.reconnect_delay_sec == 0) {
# # # # #
# # # ]
2049 : : /* Connect is not retried in a controller reset sequence. Connecting
2050 : : * the next trid will be done by the next bdev_nvme_failover_ctrlr() call.
2051 : : */
2052 : 12 : return false;
2053 : : }
2054 : :
2055 [ + + # # : 48 : assert(path_id->trid.trtype != SPDK_NVME_TRANSPORT_PCIE);
# # # # #
# ]
2056 : :
2057 [ + - # # : 48 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Start failover from %s:%s to %s:%s\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2058 : : path_id->trid.traddr, path_id->trid.trsvcid,
2059 : : next_path->trid.traddr, next_path->trid.trsvcid);
2060 : :
2061 [ # # # # ]: 48 : spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr);
2062 [ # # # # ]: 48 : nvme_ctrlr->active_path_id = next_path;
2063 [ # # # # : 48 : rc = spdk_nvme_ctrlr_set_trid(nvme_ctrlr->ctrlr, &next_path->trid);
# # ]
2064 [ - + # # ]: 48 : assert(rc == 0);
2065 [ + - # # : 48 : TAILQ_REMOVE(&nvme_ctrlr->trids, path_id, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2066 [ + + # # ]: 48 : if (!remove) {
2067 : : /** Shuffle the old trid to the end of the list and use the new one.
2068 : : * Allows for round robin through multiple connections.
2069 : : */
2070 [ # # # # : 36 : TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, path_id, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
2071 : 6 : } else {
2072 : 12 : free(path_id);
2073 : : }
2074 : :
2075 [ + + + + : 48 : if (start || next_path->last_failed_tsc == 0) {
# # # # #
# ]
2076 : : /* bdev_nvme_failover_ctrlr() is just called or the next trid is not failed
2077 : : * or used yet. Try the next trid now.
2078 : : */
2079 : 44 : return true;
2080 : : }
2081 : :
2082 [ - + - + : 5 : if (spdk_get_ticks() > next_path->last_failed_tsc + spdk_get_ticks_hz() *
# # ]
2083 [ - + # # : 4 : nvme_ctrlr->opts.reconnect_delay_sec) {
# # ]
2084 : : /* Enough backoff passed since the next trid failed. Try the next trid now. */
2085 : 0 : return true;
2086 : : }
2087 : :
2088 : : /* The next trid will be tried after reconnect_delay_sec seconds. */
2089 : 4 : return false;
2090 : 62 : }
2091 : :
2092 : : static bool
2093 : 676709 : bdev_nvme_check_ctrlr_loss_timeout(struct nvme_ctrlr *nvme_ctrlr)
2094 : : {
2095 : : int32_t elapsed;
2096 : :
2097 [ + + + + : 676709 : if (nvme_ctrlr->opts.ctrlr_loss_timeout_sec == 0 ||
# # # # #
# ]
2098 [ + + # # : 208369 : nvme_ctrlr->opts.ctrlr_loss_timeout_sec == -1) {
# # ]
2099 : 472989 : return false;
2100 : : }
2101 : :
2102 [ - + # # : 203720 : elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz();
# # ]
2103 [ + + # # : 203720 : if (elapsed >= nvme_ctrlr->opts.ctrlr_loss_timeout_sec) {
# # # # ]
2104 : 70 : return true;
2105 : : } else {
2106 : 203650 : return false;
2107 : : }
2108 : 248 : }
2109 : :
2110 : : static bool
2111 : 93 : bdev_nvme_check_fast_io_fail_timeout(struct nvme_ctrlr *nvme_ctrlr)
2112 : : {
2113 : : uint32_t elapsed;
2114 : :
2115 [ + + # # : 93 : if (nvme_ctrlr->opts.fast_io_fail_timeout_sec == 0) {
# # # # ]
2116 : 62 : return false;
2117 : : }
2118 : :
2119 [ - + # # : 31 : elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz();
# # ]
2120 [ + + # # : 31 : if (elapsed >= nvme_ctrlr->opts.fast_io_fail_timeout_sec) {
# # # # ]
2121 : 15 : return true;
2122 : : } else {
2123 : 16 : return false;
2124 : : }
2125 : 12 : }
2126 : :
2127 : : static void bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, bool success);
2128 : :
2129 : : static void
2130 : 762 : nvme_ctrlr_disconnect(struct nvme_ctrlr *nvme_ctrlr, nvme_ctrlr_disconnected_cb cb_fn)
2131 : : {
2132 : : int rc;
2133 : :
2134 [ + + + + : 762 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Start disconnecting ctrlr.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2135 : :
2136 [ # # # # ]: 762 : rc = spdk_nvme_ctrlr_disconnect(nvme_ctrlr->ctrlr);
2137 [ + + ]: 762 : if (rc != 0) {
2138 [ + - # # : 4 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "disconnecting ctrlr failed.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2139 : :
2140 : : /* Disconnect fails if ctrlr is already resetting or removed. In this case,
2141 : : * fail the reset sequence immediately.
2142 : : */
2143 : 4 : bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false);
2144 : 4 : return;
2145 : : }
2146 : :
2147 : : /* spdk_nvme_ctrlr_disconnect() may complete asynchronously later by polling adminq.
2148 : : * Set callback here to execute the specified operation after ctrlr is really disconnected.
2149 : : */
2150 [ + + # # : 758 : assert(nvme_ctrlr->disconnected_cb == NULL);
# # # # ]
2151 [ # # # # ]: 758 : nvme_ctrlr->disconnected_cb = cb_fn;
2152 : :
2153 : : /* During disconnection, reduce the period to poll adminq more often. */
2154 : 758 : bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 0);
2155 : 74 : }
2156 : :
2157 : : enum bdev_nvme_op_after_reset {
2158 : : OP_NONE,
2159 : : OP_COMPLETE_PENDING_DESTRUCT,
2160 : : OP_DESTRUCT,
2161 : : OP_DELAYED_RECONNECT,
2162 : : OP_FAILOVER,
2163 : : };
2164 : :
2165 : : typedef enum bdev_nvme_op_after_reset _bdev_nvme_op_after_reset;
2166 : :
2167 : : static _bdev_nvme_op_after_reset
2168 : 758 : bdev_nvme_check_op_after_reset(struct nvme_ctrlr *nvme_ctrlr, bool success)
2169 : : {
2170 [ + + ]: 758 : if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
2171 : : /* Complete pending destruct after reset completes. */
2172 : 15 : return OP_COMPLETE_PENDING_DESTRUCT;
2173 [ + + # # ]: 743 : } else if (nvme_ctrlr->pending_failover) {
2174 [ # # ]: 12 : nvme_ctrlr->pending_failover = false;
2175 [ # # # # ]: 12 : nvme_ctrlr->reset_start_tsc = 0;
2176 : 12 : return OP_FAILOVER;
2177 [ + + + + : 731 : } else if (success || nvme_ctrlr->opts.reconnect_delay_sec == 0) {
# # # # #
# # # ]
2178 [ # # # # ]: 607 : nvme_ctrlr->reset_start_tsc = 0;
2179 : 607 : return OP_NONE;
2180 [ + + ]: 124 : } else if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) {
2181 : 31 : return OP_DESTRUCT;
2182 : : } else {
2183 [ + + ]: 93 : if (bdev_nvme_check_fast_io_fail_timeout(nvme_ctrlr)) {
2184 [ # # ]: 15 : nvme_ctrlr->fast_io_fail_timedout = true;
2185 : 2 : }
2186 : 93 : return OP_DELAYED_RECONNECT;
2187 : : }
2188 : 73 : }
2189 : :
2190 : : static int bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug);
2191 : : static void bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr);
2192 : :
2193 : : static int
2194 : 81 : bdev_nvme_reconnect_delay_timer_expired(void *ctx)
2195 : : {
2196 : 81 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2197 : :
2198 : 36 : SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect_delay, nvme_ctrlr->nbdev_ctrlr->name);
2199 [ - + # # ]: 81 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2200 : :
2201 [ # # ]: 81 : spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
2202 : :
2203 [ + + # # ]: 81 : if (!nvme_ctrlr->reconnect_is_delayed) {
2204 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2205 : 0 : return SPDK_POLLER_BUSY;
2206 : : }
2207 : :
2208 [ # # ]: 81 : nvme_ctrlr->reconnect_is_delayed = false;
2209 : :
2210 [ - + # # ]: 81 : if (nvme_ctrlr->destruct) {
2211 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2212 : 0 : return SPDK_POLLER_BUSY;
2213 : : }
2214 : :
2215 [ - + # # : 81 : assert(nvme_ctrlr->resetting == false);
# # ]
2216 [ # # ]: 81 : nvme_ctrlr->resetting = true;
2217 : :
2218 [ - + # # ]: 81 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2219 : :
2220 [ # # # # ]: 81 : spdk_poller_resume(nvme_ctrlr->adminq_timer_poller);
2221 : :
2222 : 81 : bdev_nvme_reconnect_ctrlr(nvme_ctrlr);
2223 : 81 : return SPDK_POLLER_BUSY;
2224 : 9 : }
2225 : :
2226 : : static void
2227 : 93 : bdev_nvme_start_reconnect_delay_timer(struct nvme_ctrlr *nvme_ctrlr)
2228 : : {
2229 [ # # # # ]: 93 : spdk_poller_pause(nvme_ctrlr->adminq_timer_poller);
2230 : :
2231 [ + + # # : 93 : assert(nvme_ctrlr->reconnect_is_delayed == false);
# # ]
2232 [ # # ]: 93 : nvme_ctrlr->reconnect_is_delayed = true;
2233 : :
2234 [ + + # # : 93 : assert(nvme_ctrlr->reconnect_delay_timer == NULL);
# # # # ]
2235 [ # # # # : 93 : nvme_ctrlr->reconnect_delay_timer = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_delay_timer_expired,
# # # # #
# ]
2236 : : nvme_ctrlr,
2237 : : nvme_ctrlr->opts.reconnect_delay_sec * SPDK_SEC_TO_USEC);
2238 : 93 : }
2239 : :
2240 : : static void remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr);
2241 : :
2242 : : static void
2243 : 758 : bdev_nvme_reset_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr, bool success)
2244 : : {
2245 [ # # # # ]: 758 : bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn = nvme_ctrlr->ctrlr_op_cb_fn;
2246 [ # # # # ]: 758 : void *ctrlr_op_cb_arg = nvme_ctrlr->ctrlr_op_cb_arg;
2247 : : enum bdev_nvme_op_after_reset op_after_reset;
2248 : :
2249 [ + + # # : 758 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2250 : :
2251 [ - + # # ]: 758 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2252 [ + + # # ]: 758 : if (!success) {
2253 : : /* Connecting the active trid failed. Set the next alternate trid to the
2254 : : * active trid if it exists.
2255 : : */
2256 [ + + ]: 536 : if (bdev_nvme_failover_trid(nvme_ctrlr, false, false)) {
2257 : : /* The next alternate trid exists and is ready to try. Try it now. */
2258 [ - + # # ]: 8 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2259 : :
2260 [ + + - + : 8 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Try the next alternate trid %s:%s now.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
2261 : : nvme_ctrlr->active_path_id->trid.traddr,
2262 : : nvme_ctrlr->active_path_id->trid.trsvcid);
2263 : :
2264 : 8 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr);
2265 : 8 : return;
2266 : : }
2267 : :
2268 : : /* We came here if there is no alternate trid or if the next trid exists but
2269 : : * is not ready to try. We will try the active trid after reconnect_delay_sec
2270 : : * seconds if it is non-zero or at the next reset call otherwise.
2271 : : */
2272 : 33 : } else {
2273 : : /* Connecting the active trid succeeded. Clear the last failed time because it
2274 : : * means the trid is failed if its last failed time is non-zero.
2275 : : */
2276 [ # # # # : 222 : nvme_ctrlr->active_path_id->last_failed_tsc = 0;
# # # # ]
2277 : : }
2278 : :
2279 [ + + + + : 750 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Clear pending resets.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2280 : :
2281 : : /* Make sure we clear any pending resets before returning. */
2282 [ # # ]: 750 : bdev_nvme_complete_pending_resets(nvme_ctrlr, success);
2283 : :
2284 [ + + # # ]: 750 : if (!success) {
2285 [ + - # # : 528 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Resetting controller failed.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2286 : 33 : } else {
2287 [ + + # # : 222 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Resetting controller successful.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2288 : : }
2289 : :
2290 [ # # ]: 750 : nvme_ctrlr->resetting = false;
2291 [ # # ]: 750 : nvme_ctrlr->dont_retry = false;
2292 [ # # ]: 750 : nvme_ctrlr->in_failover = false;
2293 : :
2294 [ # # # # ]: 750 : nvme_ctrlr->ctrlr_op_cb_fn = NULL;
2295 [ # # # # ]: 750 : nvme_ctrlr->ctrlr_op_cb_arg = NULL;
2296 : :
2297 [ # # ]: 750 : op_after_reset = bdev_nvme_check_op_after_reset(nvme_ctrlr, success);
2298 [ - + # # ]: 750 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2299 : :
2300 : : /* Delay callbacks when the next operation is a failover. */
2301 [ + + + + ]: 750 : if (ctrlr_op_cb_fn && op_after_reset != OP_FAILOVER) {
2302 [ + + # # : 118 : ctrlr_op_cb_fn(ctrlr_op_cb_arg, success ? 0 : -1);
# # ]
2303 : 19 : }
2304 : :
2305 [ + + + + : 750 : switch (op_after_reset) {
+ ]
2306 : 15 : case OP_COMPLETE_PENDING_DESTRUCT:
2307 : 15 : nvme_ctrlr_unregister(nvme_ctrlr);
2308 : 15 : break;
2309 : 29 : case OP_DESTRUCT:
2310 : 31 : bdev_nvme_delete_ctrlr(nvme_ctrlr, false);
2311 : 31 : remove_discovery_entry(nvme_ctrlr);
2312 : 31 : break;
2313 : 81 : case OP_DELAYED_RECONNECT:
2314 : 93 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_start_reconnect_delay_timer);
2315 : 93 : break;
2316 : 9 : case OP_FAILOVER:
2317 [ # # # # ]: 12 : nvme_ctrlr->ctrlr_op_cb_fn = ctrlr_op_cb_fn;
2318 [ # # # # ]: 12 : nvme_ctrlr->ctrlr_op_cb_arg = ctrlr_op_cb_arg;
2319 : 12 : bdev_nvme_failover_ctrlr(nvme_ctrlr);
2320 : 12 : break;
2321 : 545 : default:
2322 : 599 : break;
2323 : : }
2324 : 73 : }
2325 : :
2326 : : static void
2327 : 0 : bdev_nvme_reset_create_qpairs_failed(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status)
2328 : : {
2329 : 0 : bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false);
2330 : 0 : }
2331 : :
2332 : : static void
2333 : 770 : bdev_nvme_reset_destroy_qpair(struct nvme_ctrlr_channel_iter *i,
2334 : : struct nvme_ctrlr *nvme_ctrlr,
2335 : : struct nvme_ctrlr_channel *ctrlr_ch, void *ctx)
2336 : : {
2337 : : struct nvme_qpair *nvme_qpair;
2338 : : struct spdk_nvme_qpair *qpair;
2339 : :
2340 [ # # # # ]: 770 : nvme_qpair = ctrlr_ch->qpair;
2341 [ + + # # ]: 770 : assert(nvme_qpair != NULL);
2342 : :
2343 : 770 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
2344 : :
2345 [ # # # # ]: 770 : qpair = nvme_qpair->qpair;
2346 [ + + ]: 770 : if (qpair != NULL) {
2347 [ + + - + : 321 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Start disconnecting qpair %p:%u.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2348 : : qpair, spdk_nvme_qpair_get_id(qpair));
2349 : :
2350 [ + + # # : 321 : if (nvme_qpair->ctrlr->dont_retry) {
# # # # ]
2351 : 252 : spdk_nvme_qpair_set_abort_dnr(qpair, true);
2352 : 54 : }
2353 : 321 : spdk_nvme_ctrlr_disconnect_io_qpair(qpair);
2354 : :
2355 : : /* The current full reset sequence will move to the next
2356 : : * ctrlr_channel after the qpair is actually disconnected.
2357 : : */
2358 [ - + # # : 321 : assert(ctrlr_ch->reset_iter == NULL);
# # # # ]
2359 [ # # # # ]: 321 : ctrlr_ch->reset_iter = i;
2360 : 70 : } else {
2361 : 449 : nvme_ctrlr_for_each_channel_continue(i, 0);
2362 : : }
2363 : 770 : }
2364 : :
2365 : : static void
2366 : 222 : bdev_nvme_reset_create_qpairs_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status)
2367 : : {
2368 [ + + ]: 222 : if (status == 0) {
2369 [ + + - + : 222 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpairs were created after ctrlr reset.\n");
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2370 : :
2371 : 222 : bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, true);
2372 : 38 : } else {
2373 [ # # # # : 0 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpairs were failed to create after ctrlr reset.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2374 : :
2375 : : /* Delete the added qpairs and quiesce ctrlr to make the states clean. */
2376 : 0 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
2377 : : bdev_nvme_reset_destroy_qpair,
2378 : : NULL,
2379 : : bdev_nvme_reset_create_qpairs_failed);
2380 : : }
2381 : 222 : }
2382 : :
2383 : : static int
2384 : 13872 : bdev_nvme_reset_check_qpair_connected(void *ctx)
2385 : : {
2386 : 13872 : struct nvme_ctrlr_channel *ctrlr_ch = ctx;
2387 [ # # # # ]: 13872 : struct nvme_qpair *nvme_qpair = ctrlr_ch->qpair;
2388 : : struct spdk_nvme_qpair *qpair;
2389 : :
2390 [ + + # # : 13872 : if (ctrlr_ch->reset_iter == NULL) {
# # ]
2391 : : /* qpair was already failed to connect and the reset sequence is being aborted. */
2392 [ # # # # : 0 : assert(ctrlr_ch->connect_poller == NULL);
# # # # ]
2393 [ # # # # : 0 : assert(nvme_qpair->qpair == NULL);
# # # # ]
2394 : :
2395 [ # # # # : 0 : NVME_CTRLR_INFOLOG(nvme_qpair->ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
2396 : : "qpair was already failed to connect. reset is being aborted.\n");
2397 : 0 : return SPDK_POLLER_BUSY;
2398 : : }
2399 : :
2400 [ # # # # ]: 13872 : qpair = nvme_qpair->qpair;
2401 [ + + # # ]: 13872 : assert(qpair != NULL);
2402 : :
2403 [ + + ]: 13872 : if (!spdk_nvme_qpair_is_connected(qpair)) {
2404 : 13568 : return SPDK_POLLER_BUSY;
2405 : : }
2406 : :
2407 [ + + - + : 304 : NVME_CTRLR_INFOLOG(nvme_qpair->ctrlr, "qpair %p:%u was connected.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
2408 : : qpair, spdk_nvme_qpair_get_id(qpair));
2409 : :
2410 [ # # ]: 304 : spdk_poller_unregister(&ctrlr_ch->connect_poller);
2411 : :
2412 : : /* qpair was completed to connect. Move to the next ctrlr_channel */
2413 [ # # # # ]: 304 : nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, 0);
2414 [ # # # # ]: 304 : ctrlr_ch->reset_iter = NULL;
2415 : :
2416 [ + + + + ]: 304 : if (!g_opts.disable_auto_failback) {
2417 : 236 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
2418 : 45 : }
2419 : :
2420 : 304 : return SPDK_POLLER_BUSY;
2421 : 65 : }
2422 : :
2423 : : static void
2424 : 304 : bdev_nvme_reset_create_qpair(struct nvme_ctrlr_channel_iter *i,
2425 : : struct nvme_ctrlr *nvme_ctrlr,
2426 : : struct nvme_ctrlr_channel *ctrlr_ch,
2427 : : void *ctx)
2428 : : {
2429 [ # # # # ]: 304 : struct nvme_qpair *nvme_qpair = ctrlr_ch->qpair;
2430 : : struct spdk_nvme_qpair *qpair;
2431 : 304 : int rc = 0;
2432 : :
2433 [ + + # # : 304 : if (nvme_qpair->qpair == NULL) {
# # ]
2434 : 304 : rc = bdev_nvme_create_qpair(nvme_qpair);
2435 : 62 : }
2436 [ + + ]: 304 : if (rc == 0) {
2437 [ # # # # ]: 304 : ctrlr_ch->connect_poller = SPDK_POLLER_REGISTER(bdev_nvme_reset_check_qpair_connected,
2438 : : ctrlr_ch, 0);
2439 : :
2440 [ # # # # ]: 304 : qpair = nvme_qpair->qpair;
2441 : :
2442 [ + + - + : 304 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Start checking qpair %p:%u to be connected.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2443 : : qpair, spdk_nvme_qpair_get_id(qpair));
2444 : :
2445 : : /* The current full reset sequence will move to the next
2446 : : * ctrlr_channel after the qpair is actually connected.
2447 : : */
2448 [ - + # # : 304 : assert(ctrlr_ch->reset_iter == NULL);
# # # # ]
2449 [ # # # # ]: 304 : ctrlr_ch->reset_iter = i;
2450 : 62 : } else {
2451 : 0 : nvme_ctrlr_for_each_channel_continue(i, rc);
2452 : : }
2453 : 304 : }
2454 : :
2455 : : static void
2456 : 222 : nvme_ctrlr_check_namespaces(struct nvme_ctrlr *nvme_ctrlr)
2457 : : {
2458 [ # # # # ]: 222 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
2459 : : struct nvme_ns *nvme_ns;
2460 : :
2461 [ + + ]: 245 : for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
2462 [ + + ]: 392 : nvme_ns != NULL;
2463 : 170 : nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) {
2464 [ + + # # : 170 : if (!spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) {
# # ]
2465 [ + + - + : 4 : SPDK_DEBUGLOG(bdev_nvme, "NSID %u was removed during reset.\n", nvme_ns->id);
# # # # #
# ]
2466 : : /* NS can be added again. Just nullify nvme_ns->ns. */
2467 [ # # # # ]: 4 : nvme_ns->ns = NULL;
2468 : 1 : }
2469 : 23 : }
2470 : 222 : }
2471 : :
2472 : :
2473 : : static int
2474 : 676565 : bdev_nvme_reconnect_ctrlr_poll(void *arg)
2475 : : {
2476 : 676565 : struct nvme_ctrlr *nvme_ctrlr = arg;
2477 : : struct spdk_nvme_transport_id *trid;
2478 : 676565 : int rc = -ETIMEDOUT;
2479 : :
2480 [ + + ]: 676565 : if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) {
2481 : : /* Mark the ctrlr as failed. The next call to
2482 : : * spdk_nvme_ctrlr_reconnect_poll_async() will then
2483 : : * do the necessary cleanup and return failure.
2484 : : */
2485 [ # # # # ]: 31 : spdk_nvme_ctrlr_fail(nvme_ctrlr->ctrlr);
2486 : 2 : }
2487 : :
2488 [ # # # # ]: 676565 : rc = spdk_nvme_ctrlr_reconnect_poll_async(nvme_ctrlr->ctrlr);
2489 [ + + ]: 676565 : if (rc == -EAGAIN) {
2490 : 675811 : return SPDK_POLLER_BUSY;
2491 : : }
2492 : :
2493 [ # # ]: 754 : spdk_poller_unregister(&nvme_ctrlr->reset_detach_poller);
2494 [ + + ]: 754 : if (rc == 0) {
2495 [ # # # # : 222 : trid = &nvme_ctrlr->active_path_id->trid;
# # ]
2496 : :
2497 [ + + # # : 222 : if (spdk_nvme_trtype_is_fabrics(trid->trtype)) {
# # ]
2498 [ + + - + : 184 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr was connected to %s:%s. Create qpairs.\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2499 : : trid->traddr, trid->trsvcid);
2500 : 36 : } else {
2501 [ + + - + : 38 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr was connected. Create qpairs.\n");
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2502 : : }
2503 : :
2504 : 222 : nvme_ctrlr_check_namespaces(nvme_ctrlr);
2505 : :
2506 : : /* Recreate all of the I/O queue pairs */
2507 : 222 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
2508 : : bdev_nvme_reset_create_qpair,
2509 : : NULL,
2510 : : bdev_nvme_reset_create_qpairs_done);
2511 : 38 : } else {
2512 [ + + + + : 532 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr could not be connected.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2513 : :
2514 : 532 : bdev_nvme_reset_ctrlr_complete(nvme_ctrlr, false);
2515 : : }
2516 : 754 : return SPDK_POLLER_BUSY;
2517 : 229 : }
2518 : :
2519 : : static void
2520 : 754 : bdev_nvme_reconnect_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
2521 : : {
2522 [ + + + + : 754 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Start reconnecting ctrlr.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2523 : :
2524 [ # # # # ]: 754 : spdk_nvme_ctrlr_reconnect_async(nvme_ctrlr->ctrlr);
2525 : :
2526 : 103 : SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reconnect, nvme_ctrlr->nbdev_ctrlr->name);
2527 [ + + # # : 754 : assert(nvme_ctrlr->reset_detach_poller == NULL);
# # # # ]
2528 [ # # # # ]: 754 : nvme_ctrlr->reset_detach_poller = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_ctrlr_poll,
2529 : : nvme_ctrlr, 0);
2530 : 754 : }
2531 : :
2532 : : static void
2533 : 657 : bdev_nvme_reset_destroy_qpair_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status)
2534 : : {
2535 : 67 : SPDK_DTRACE_PROBE1(bdev_nvme_ctrlr_reset, nvme_ctrlr->nbdev_ctrlr->name);
2536 [ + + # # ]: 657 : assert(status == 0);
2537 : :
2538 [ + + + + : 657 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "qpairs were deleted.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2539 : :
2540 [ + + # # : 657 : if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) {
# # ]
2541 : 38 : bdev_nvme_reconnect_ctrlr(nvme_ctrlr);
2542 : 2 : } else {
2543 : 619 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr);
2544 : : }
2545 : 657 : }
2546 : :
2547 : : static void
2548 : 657 : bdev_nvme_reset_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr)
2549 : : {
2550 [ + + + + : 657 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Delete qpairs for reset.\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2551 : :
2552 : 657 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
2553 : : bdev_nvme_reset_destroy_qpair,
2554 : : NULL,
2555 : : bdev_nvme_reset_destroy_qpair_done);
2556 : 657 : }
2557 : :
2558 : : static void
2559 : 12 : bdev_nvme_reconnect_ctrlr_now(void *ctx)
2560 : : {
2561 : 12 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2562 : :
2563 [ + + # # : 12 : assert(nvme_ctrlr->resetting == true);
# # ]
2564 [ + + # # : 12 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2565 : :
2566 [ # # ]: 12 : spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
2567 : :
2568 [ # # # # ]: 12 : spdk_poller_resume(nvme_ctrlr->adminq_timer_poller);
2569 : :
2570 : 12 : bdev_nvme_reconnect_ctrlr(nvme_ctrlr);
2571 : 12 : }
2572 : :
2573 : : static void
2574 : 657 : _bdev_nvme_reset_ctrlr(void *ctx)
2575 : : {
2576 : 657 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2577 : :
2578 [ + + # # : 657 : assert(nvme_ctrlr->resetting == true);
# # ]
2579 [ - + # # : 657 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2580 : :
2581 [ + + # # : 657 : if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) {
# # ]
2582 : 38 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reset_destroy_qpairs);
2583 : 2 : } else {
2584 : 619 : bdev_nvme_reset_destroy_qpairs(nvme_ctrlr);
2585 : : }
2586 : 657 : }
2587 : :
2588 : : static int
2589 : 250 : bdev_nvme_reset_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, spdk_msg_fn *msg_fn)
2590 : : {
2591 [ + + # # ]: 250 : if (nvme_ctrlr->destruct) {
2592 : 12 : return -ENXIO;
2593 : : }
2594 : :
2595 [ + + # # ]: 238 : if (nvme_ctrlr->resetting) {
2596 [ + - # # : 56 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Unable to perform reset, already in progress.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2597 : 56 : return -EBUSY;
2598 : : }
2599 : :
2600 [ + + # # ]: 182 : if (nvme_ctrlr->disabled) {
2601 [ + - # # : 4 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Unable to perform reset. Controller is disabled.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2602 : 4 : return -EALREADY;
2603 : : }
2604 : :
2605 [ # # ]: 178 : nvme_ctrlr->resetting = true;
2606 [ # # ]: 178 : nvme_ctrlr->dont_retry = true;
2607 : :
2608 [ + + # # ]: 178 : if (nvme_ctrlr->reconnect_is_delayed) {
2609 [ + + - + : 4 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "Reconnect is already scheduled.\n");
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
2610 [ # # ]: 4 : *msg_fn = bdev_nvme_reconnect_ctrlr_now;
2611 [ # # ]: 4 : nvme_ctrlr->reconnect_is_delayed = false;
2612 : 1 : } else {
2613 [ # # ]: 174 : *msg_fn = _bdev_nvme_reset_ctrlr;
2614 [ + + # # : 174 : assert(nvme_ctrlr->reset_start_tsc == 0);
# # # # ]
2615 : : }
2616 : :
2617 [ # # # # ]: 178 : nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
2618 : :
2619 : 178 : return 0;
2620 : 52 : }
2621 : :
2622 : : static int
2623 : 106 : bdev_nvme_reset_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
2624 : : {
2625 : 75 : spdk_msg_fn msg_fn;
2626 : : int rc;
2627 : :
2628 [ - + # # ]: 106 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2629 : 106 : rc = bdev_nvme_reset_ctrlr_unsafe(nvme_ctrlr, &msg_fn);
2630 [ - + # # ]: 106 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2631 : :
2632 [ + + ]: 106 : if (rc == 0) {
2633 [ # # # # ]: 86 : spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr);
2634 : 20 : }
2635 : :
2636 : 106 : return rc;
2637 : : }
2638 : :
2639 : : static int
2640 : 12 : bdev_nvme_enable_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
2641 : : {
2642 [ - + # # ]: 12 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2643 [ - + # # ]: 12 : if (nvme_ctrlr->destruct) {
2644 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2645 : 0 : return -ENXIO;
2646 : : }
2647 : :
2648 [ - + # # ]: 12 : if (nvme_ctrlr->resetting) {
2649 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2650 : 0 : return -EBUSY;
2651 : : }
2652 : :
2653 [ + + # # ]: 12 : if (!nvme_ctrlr->disabled) {
2654 [ - + # # ]: 4 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2655 : 4 : return -EALREADY;
2656 : : }
2657 : :
2658 [ # # ]: 8 : nvme_ctrlr->disabled = false;
2659 [ # # ]: 8 : nvme_ctrlr->resetting = true;
2660 : :
2661 [ # # # # ]: 8 : nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
2662 : :
2663 [ - + # # ]: 8 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2664 : :
2665 [ # # # # ]: 8 : spdk_thread_send_msg(nvme_ctrlr->thread, bdev_nvme_reconnect_ctrlr_now, nvme_ctrlr);
2666 : 8 : return 0;
2667 : 3 : }
2668 : :
2669 : : static void
2670 : 8 : bdev_nvme_disable_ctrlr_complete(struct nvme_ctrlr *nvme_ctrlr)
2671 : : {
2672 [ # # # # ]: 8 : bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn = nvme_ctrlr->ctrlr_op_cb_fn;
2673 [ # # # # ]: 8 : void *ctrlr_op_cb_arg = nvme_ctrlr->ctrlr_op_cb_arg;
2674 : : enum bdev_nvme_op_after_reset op_after_disable;
2675 : :
2676 [ + + # # : 8 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2677 : :
2678 [ # # # # ]: 8 : nvme_ctrlr->ctrlr_op_cb_fn = NULL;
2679 [ # # # # ]: 8 : nvme_ctrlr->ctrlr_op_cb_arg = NULL;
2680 : :
2681 [ - + # # ]: 8 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2682 : :
2683 [ # # ]: 8 : nvme_ctrlr->resetting = false;
2684 [ # # ]: 8 : nvme_ctrlr->dont_retry = false;
2685 : :
2686 : 8 : op_after_disable = bdev_nvme_check_op_after_reset(nvme_ctrlr, true);
2687 : :
2688 [ # # ]: 8 : nvme_ctrlr->disabled = true;
2689 [ # # # # ]: 8 : spdk_poller_pause(nvme_ctrlr->adminq_timer_poller);
2690 : :
2691 : : /* Make sure we clear any pending resets before returning. */
2692 : 8 : bdev_nvme_complete_pending_resets(nvme_ctrlr, true);
2693 : :
2694 [ - + # # ]: 8 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2695 : :
2696 [ + + ]: 8 : if (ctrlr_op_cb_fn) {
2697 [ # # # # ]: 0 : ctrlr_op_cb_fn(ctrlr_op_cb_arg, 0);
2698 : 0 : }
2699 : :
2700 [ + + ]: 8 : switch (op_after_disable) {
2701 : 0 : case OP_COMPLETE_PENDING_DESTRUCT:
2702 : 0 : nvme_ctrlr_unregister(nvme_ctrlr);
2703 : 0 : break;
2704 : 6 : default:
2705 : 8 : break;
2706 : : }
2707 : 8 : }
2708 : :
2709 : : static void
2710 : 4 : bdev_nvme_disable_destroy_qpairs_done(struct nvme_ctrlr *nvme_ctrlr, void *ctx, int status)
2711 : : {
2712 [ + + # # ]: 4 : assert(status == 0);
2713 : :
2714 [ + + # # : 4 : if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) {
# # ]
2715 : 0 : bdev_nvme_disable_ctrlr_complete(nvme_ctrlr);
2716 : 0 : } else {
2717 : 4 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_disable_ctrlr_complete);
2718 : : }
2719 : 4 : }
2720 : :
2721 : : static void
2722 : 4 : bdev_nvme_disable_destroy_qpairs(struct nvme_ctrlr *nvme_ctrlr)
2723 : : {
2724 : 4 : nvme_ctrlr_for_each_channel(nvme_ctrlr,
2725 : : bdev_nvme_reset_destroy_qpair,
2726 : : NULL,
2727 : : bdev_nvme_disable_destroy_qpairs_done);
2728 : 4 : }
2729 : :
2730 : : static void
2731 : 4 : _bdev_nvme_cancel_reconnect_and_disable_ctrlr(void *ctx)
2732 : : {
2733 : 4 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2734 : :
2735 [ + + # # : 4 : assert(nvme_ctrlr->resetting == true);
# # ]
2736 [ + + # # : 4 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2737 : :
2738 [ # # ]: 4 : spdk_poller_unregister(&nvme_ctrlr->reconnect_delay_timer);
2739 : :
2740 : 4 : bdev_nvme_disable_ctrlr_complete(nvme_ctrlr);
2741 : 4 : }
2742 : :
2743 : : static void
2744 : 4 : _bdev_nvme_disconnect_and_disable_ctrlr(void *ctx)
2745 : : {
2746 : 4 : struct nvme_ctrlr *nvme_ctrlr = ctx;
2747 : :
2748 [ + + # # : 4 : assert(nvme_ctrlr->resetting == true);
# # ]
2749 [ - + # # : 4 : assert(nvme_ctrlr->thread == spdk_get_thread());
# # # # ]
2750 : :
2751 [ + + # # : 4 : if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) {
# # ]
2752 : 0 : nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_disable_destroy_qpairs);
2753 : 0 : } else {
2754 : 4 : bdev_nvme_disable_destroy_qpairs(nvme_ctrlr);
2755 : : }
2756 : 4 : }
2757 : :
2758 : : static int
2759 : 20 : bdev_nvme_disable_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
2760 : : {
2761 : : spdk_msg_fn msg_fn;
2762 : :
2763 [ - + # # ]: 20 : pthread_mutex_lock(&nvme_ctrlr->mutex);
2764 [ + + # # ]: 20 : if (nvme_ctrlr->destruct) {
2765 [ - + # # ]: 4 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2766 : 4 : return -ENXIO;
2767 : : }
2768 : :
2769 [ + + # # ]: 16 : if (nvme_ctrlr->resetting) {
2770 [ - + # # ]: 4 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2771 : 4 : return -EBUSY;
2772 : : }
2773 : :
2774 [ + + # # ]: 12 : if (nvme_ctrlr->disabled) {
2775 [ - + # # ]: 4 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2776 : 4 : return -EALREADY;
2777 : : }
2778 : :
2779 [ # # ]: 8 : nvme_ctrlr->resetting = true;
2780 [ # # ]: 8 : nvme_ctrlr->dont_retry = true;
2781 : :
2782 [ + + # # ]: 8 : if (nvme_ctrlr->reconnect_is_delayed) {
2783 : 4 : msg_fn = _bdev_nvme_cancel_reconnect_and_disable_ctrlr;
2784 [ # # ]: 4 : nvme_ctrlr->reconnect_is_delayed = false;
2785 : 1 : } else {
2786 : 4 : msg_fn = _bdev_nvme_disconnect_and_disable_ctrlr;
2787 : : }
2788 : :
2789 [ # # # # ]: 8 : nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
2790 : :
2791 [ - + # # ]: 8 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
2792 : :
2793 [ # # # # ]: 8 : spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr);
2794 : 8 : return 0;
2795 : 5 : }
2796 : :
2797 : : static int
2798 : 34 : nvme_ctrlr_op(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op,
2799 : : bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg)
2800 : : {
2801 : : int rc;
2802 : :
2803 [ + + - + ]: 34 : switch (op) {
2804 : 24 : case NVME_CTRLR_OP_RESET:
2805 : 30 : rc = bdev_nvme_reset_ctrlr(nvme_ctrlr);
2806 : 30 : break;
2807 : 0 : case NVME_CTRLR_OP_ENABLE:
2808 : 0 : rc = bdev_nvme_enable_ctrlr(nvme_ctrlr);
2809 : 0 : break;
2810 : 0 : case NVME_CTRLR_OP_DISABLE:
2811 : 0 : rc = bdev_nvme_disable_ctrlr(nvme_ctrlr);
2812 : 0 : break;
2813 : 3 : default:
2814 : 4 : rc = -EINVAL;
2815 : 4 : break;
2816 : : }
2817 : :
2818 [ + + ]: 34 : if (rc == 0) {
2819 [ + + # # : 22 : assert(nvme_ctrlr->ctrlr_op_cb_fn == NULL);
# # # # ]
2820 [ - + # # : 22 : assert(nvme_ctrlr->ctrlr_op_cb_arg == NULL);
# # # # ]
2821 [ # # # # ]: 22 : nvme_ctrlr->ctrlr_op_cb_fn = cb_fn;
2822 [ # # # # ]: 22 : nvme_ctrlr->ctrlr_op_cb_arg = cb_arg;
2823 : 4 : }
2824 : 34 : return rc;
2825 : : }
2826 : :
2827 : : struct nvme_ctrlr_op_rpc_ctx {
2828 : : struct nvme_ctrlr *nvme_ctrlr;
2829 : : struct spdk_thread *orig_thread;
2830 : : enum nvme_ctrlr_op op;
2831 : : int rc;
2832 : : bdev_nvme_ctrlr_op_cb cb_fn;
2833 : : void *cb_arg;
2834 : : };
2835 : :
2836 : : static void
2837 : 16 : _nvme_ctrlr_op_rpc_complete(void *_ctx)
2838 : : {
2839 : 16 : struct nvme_ctrlr_op_rpc_ctx *ctx = _ctx;
2840 : :
2841 [ + + # # ]: 16 : assert(ctx != NULL);
2842 [ + + # # : 16 : assert(ctx->cb_fn != NULL);
# # # # ]
2843 : :
2844 [ # # # # : 16 : ctx->cb_fn(ctx->cb_arg, ctx->rc);
# # # # #
# # # # #
# # ]
2845 : :
2846 : 16 : free(ctx);
2847 : 16 : }
2848 : :
2849 : : static void
2850 : 16 : nvme_ctrlr_op_rpc_complete(void *cb_arg, int rc)
2851 : : {
2852 : 16 : struct nvme_ctrlr_op_rpc_ctx *ctx = cb_arg;
2853 : :
2854 [ # # # # ]: 16 : ctx->rc = rc;
2855 : :
2856 [ # # # # ]: 16 : spdk_thread_send_msg(ctx->orig_thread, _nvme_ctrlr_op_rpc_complete, ctx);
2857 : 16 : }
2858 : :
2859 : : void
2860 : 16 : nvme_ctrlr_op_rpc(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op,
2861 : : bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg)
2862 : : {
2863 : : struct nvme_ctrlr_op_rpc_ctx *ctx;
2864 : : int rc;
2865 : :
2866 [ + + # # ]: 16 : assert(cb_fn != NULL);
2867 : :
2868 : 16 : ctx = calloc(1, sizeof(*ctx));
2869 [ + + ]: 16 : if (ctx == NULL) {
2870 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to allocate nvme_ctrlr_op_rpc_ctx.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
2871 [ # # # # ]: 0 : cb_fn(cb_arg, -ENOMEM);
2872 : 0 : return;
2873 : : }
2874 : :
2875 [ # # # # ]: 16 : ctx->orig_thread = spdk_get_thread();
2876 [ # # # # ]: 16 : ctx->cb_fn = cb_fn;
2877 [ # # # # ]: 16 : ctx->cb_arg = cb_arg;
2878 : :
2879 : 16 : rc = nvme_ctrlr_op(nvme_ctrlr, op, nvme_ctrlr_op_rpc_complete, ctx);
2880 [ + + ]: 16 : if (rc == 0) {
2881 : 4 : return;
2882 [ + + ]: 12 : } else if (rc == -EALREADY) {
2883 : 0 : rc = 0;
2884 : 0 : }
2885 : :
2886 : 12 : nvme_ctrlr_op_rpc_complete(ctx, rc);
2887 : 4 : }
2888 : :
2889 : : static void nvme_bdev_ctrlr_op_rpc_continue(void *cb_arg, int rc);
2890 : :
2891 : : static void
2892 : 18 : _nvme_bdev_ctrlr_op_rpc_continue(void *_ctx)
2893 : : {
2894 : 18 : struct nvme_ctrlr_op_rpc_ctx *ctx = _ctx;
2895 : : struct nvme_ctrlr *prev_nvme_ctrlr, *next_nvme_ctrlr;
2896 : : int rc;
2897 : :
2898 [ # # # # ]: 18 : prev_nvme_ctrlr = ctx->nvme_ctrlr;
2899 [ # # # # ]: 18 : ctx->nvme_ctrlr = NULL;
2900 : :
2901 [ - + # # : 18 : if (ctx->rc != 0) {
# # ]
2902 : 0 : goto complete;
2903 : : }
2904 : :
2905 [ # # # # : 18 : next_nvme_ctrlr = TAILQ_NEXT(prev_nvme_ctrlr, tailq);
# # ]
2906 [ + + ]: 18 : if (next_nvme_ctrlr == NULL) {
2907 : 14 : goto complete;
2908 : : }
2909 : :
2910 [ # # # # ]: 4 : rc = nvme_ctrlr_op(next_nvme_ctrlr, ctx->op, nvme_bdev_ctrlr_op_rpc_continue, ctx);
2911 [ + + ]: 4 : if (rc == 0) {
2912 [ # # # # ]: 4 : ctx->nvme_ctrlr = next_nvme_ctrlr;
2913 : 4 : return;
2914 [ # # ]: 0 : } else if (rc == -EALREADY) {
2915 [ # # # # ]: 0 : ctx->nvme_ctrlr = next_nvme_ctrlr;
2916 : 0 : rc = 0;
2917 : 0 : }
2918 : :
2919 [ # # # # ]: 0 : ctx->rc = rc;
2920 : :
2921 : 12 : complete:
2922 [ # # # # : 14 : ctx->cb_fn(ctx->cb_arg, ctx->rc);
# # # # #
# # # # #
# # ]
2923 : 14 : free(ctx);
2924 : 3 : }
2925 : :
2926 : : static void
2927 : 18 : nvme_bdev_ctrlr_op_rpc_continue(void *cb_arg, int rc)
2928 : : {
2929 : 18 : struct nvme_ctrlr_op_rpc_ctx *ctx = cb_arg;
2930 : :
2931 [ # # # # ]: 18 : ctx->rc = rc;
2932 : :
2933 [ # # # # ]: 18 : spdk_thread_send_msg(ctx->orig_thread, _nvme_bdev_ctrlr_op_rpc_continue, ctx);
2934 : 18 : }
2935 : :
2936 : : void
2937 : 14 : nvme_bdev_ctrlr_op_rpc(struct nvme_bdev_ctrlr *nbdev_ctrlr, enum nvme_ctrlr_op op,
2938 : : bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg)
2939 : : {
2940 : : struct nvme_ctrlr_op_rpc_ctx *ctx;
2941 : : struct nvme_ctrlr *nvme_ctrlr;
2942 : : int rc;
2943 : :
2944 [ + + # # ]: 14 : assert(cb_fn != NULL);
2945 : :
2946 : 14 : ctx = calloc(1, sizeof(*ctx));
2947 [ + + ]: 14 : if (ctx == NULL) {
2948 : 0 : SPDK_ERRLOG("Failed to allocate nvme_ctrlr_op_rpc_ctx.\n");
2949 [ # # # # ]: 0 : cb_fn(cb_arg, -ENOMEM);
2950 : 0 : return;
2951 : : }
2952 : :
2953 [ # # # # ]: 14 : ctx->orig_thread = spdk_get_thread();
2954 [ # # # # ]: 14 : ctx->op = op;
2955 [ # # # # ]: 14 : ctx->cb_fn = cb_fn;
2956 [ # # # # ]: 14 : ctx->cb_arg = cb_arg;
2957 : :
2958 [ # # # # : 14 : nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs);
# # ]
2959 [ - + # # ]: 14 : assert(nvme_ctrlr != NULL);
2960 : :
2961 : 14 : rc = nvme_ctrlr_op(nvme_ctrlr, op, nvme_bdev_ctrlr_op_rpc_continue, ctx);
2962 [ + + ]: 14 : if (rc == 0) {
2963 [ # # # # ]: 14 : ctx->nvme_ctrlr = nvme_ctrlr;
2964 : 14 : return;
2965 [ # # ]: 0 : } else if (rc == -EALREADY) {
2966 [ # # # # ]: 0 : ctx->nvme_ctrlr = nvme_ctrlr;
2967 : 0 : rc = 0;
2968 : 0 : }
2969 : :
2970 : 0 : nvme_bdev_ctrlr_op_rpc_continue(ctx, rc);
2971 : 2 : }
2972 : :
2973 : : static int _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio);
2974 : :
2975 : : static void
2976 : 104 : bdev_nvme_unfreeze_bdev_channel_done(struct nvme_bdev *nbdev, void *ctx, int status)
2977 : : {
2978 : 104 : struct nvme_bdev_io *bio = ctx;
2979 : : enum spdk_bdev_io_status io_status;
2980 : :
2981 [ + + # # : 104 : if (bio->cpl.cdw0 == 0) {
# # # # ]
2982 : 88 : io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
2983 : 13 : } else {
2984 : 16 : io_status = SPDK_BDEV_IO_STATUS_FAILED;
2985 : : }
2986 : :
2987 [ + + - + : 104 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p completed, status:%d\n", bio, io_status);
# # # # #
# # # ]
2988 : :
2989 : 104 : __bdev_nvme_io_complete(spdk_bdev_io_from_ctx(bio), io_status, NULL);
2990 : 104 : }
2991 : :
2992 : : static void
2993 : 168 : bdev_nvme_unfreeze_bdev_channel(struct nvme_bdev_channel_iter *i,
2994 : : struct nvme_bdev *nbdev,
2995 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
2996 : : {
2997 : 168 : bdev_nvme_abort_retry_ios(nbdev_ch);
2998 [ # # # # ]: 168 : nbdev_ch->resetting = false;
2999 : :
3000 : 168 : nvme_bdev_for_each_channel_continue(i, 0);
3001 : 168 : }
3002 : :
3003 : : static void
3004 : 104 : bdev_nvme_reset_io_complete(struct nvme_bdev_io *bio)
3005 : : {
3006 : 104 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
3007 [ # # # # : 104 : struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt;
# # # # ]
3008 : :
3009 : : /* Abort all queued I/Os for retry. */
3010 : 121 : nvme_bdev_for_each_channel(nbdev,
3011 : : bdev_nvme_unfreeze_bdev_channel,
3012 : 17 : bio,
3013 : : bdev_nvme_unfreeze_bdev_channel_done);
3014 : 104 : }
3015 : :
3016 : : static void
3017 : 144 : _bdev_nvme_reset_io_continue(void *ctx)
3018 : : {
3019 : 144 : struct nvme_bdev_io *bio = ctx;
3020 : : struct nvme_io_path *prev_io_path, *next_io_path;
3021 : : int rc;
3022 : :
3023 [ # # # # ]: 144 : prev_io_path = bio->io_path;
3024 [ # # # # ]: 144 : bio->io_path = NULL;
3025 : :
3026 [ # # # # : 144 : next_io_path = STAILQ_NEXT(prev_io_path, stailq);
# # ]
3027 [ + + ]: 144 : if (next_io_path == NULL) {
3028 : 104 : goto complete;
3029 : : }
3030 : :
3031 : 40 : rc = _bdev_nvme_reset_io(next_io_path, bio);
3032 [ + - ]: 40 : if (rc == 0) {
3033 : 40 : return;
3034 : : }
3035 : :
3036 : 0 : complete:
3037 : 104 : bdev_nvme_reset_io_complete(bio);
3038 : 27 : }
3039 : :
3040 : : static void
3041 : 144 : bdev_nvme_reset_io_continue(void *cb_arg, int rc)
3042 : : {
3043 : 144 : struct nvme_bdev_io *bio = cb_arg;
3044 : 144 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
3045 [ # # # # : 144 : struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt;
# # # # ]
3046 : :
3047 [ + + - + : 144 : NVME_BDEV_INFOLOG(nbdev, "continue reset_io %p, rc:%d\n", bio, rc);
# # # # #
# # # ]
3048 : :
3049 : : /* Reset status is initialized as "failed". Set to "success" once we have at least one
3050 : : * successfully reset nvme_ctrlr.
3051 : : */
3052 [ + + ]: 144 : if (rc == 0) {
3053 [ # # # # : 104 : bio->cpl.cdw0 = 0;
# # ]
3054 : 17 : }
3055 : :
3056 : 144 : spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), _bdev_nvme_reset_io_continue, bio);
3057 : 144 : }
3058 : :
3059 : : static int
3060 : 144 : _bdev_nvme_reset_io(struct nvme_io_path *io_path, struct nvme_bdev_io *bio)
3061 : : {
3062 : 144 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
3063 [ # # # # : 144 : struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev_io->bdev->ctxt;
# # # # ]
3064 [ # # # # : 144 : struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;
# # # # ]
3065 : 97 : spdk_msg_fn msg_fn;
3066 : : int rc;
3067 : :
3068 [ + + # # : 144 : assert(bio->io_path == NULL);
# # # # ]
3069 [ # # # # ]: 144 : bio->io_path = io_path;
3070 : :
3071 [ - + # # ]: 144 : pthread_mutex_lock(&nvme_ctrlr->mutex);
3072 : 144 : rc = bdev_nvme_reset_ctrlr_unsafe(nvme_ctrlr, &msg_fn);
3073 [ + + ]: 144 : if (rc == -EBUSY) {
3074 : : /*
3075 : : * Reset call is queued only if it is from the app framework. This is on purpose so that
3076 : : * we don't interfere with the app framework reset strategy. i.e. we are deferring to the
3077 : : * upper level. If they are in the middle of a reset, we won't try to schedule another one.
3078 : : */
3079 [ # # # # : 48 : TAILQ_INSERT_TAIL(&nvme_ctrlr->pending_resets, bio, retry_link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
3080 : 12 : }
3081 [ - + # # ]: 144 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
3082 : :
3083 [ + + ]: 144 : if (rc == 0) {
3084 [ + + # # : 92 : assert(nvme_ctrlr->ctrlr_op_cb_fn == NULL);
# # # # ]
3085 [ + + # # : 92 : assert(nvme_ctrlr->ctrlr_op_cb_arg == NULL);
# # # # ]
3086 [ # # # # ]: 92 : nvme_ctrlr->ctrlr_op_cb_fn = bdev_nvme_reset_io_continue;
3087 [ # # # # ]: 92 : nvme_ctrlr->ctrlr_op_cb_arg = bio;
3088 : :
3089 [ # # # # ]: 92 : spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr);
3090 : :
3091 [ + + - + : 92 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p started resetting ctrlr [%s, %u].\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3092 : : bio, CTRLR_STRING(nvme_ctrlr), CTRLR_ID(nvme_ctrlr));
3093 [ + + ]: 66 : } else if (rc == -EBUSY) {
3094 : 48 : rc = 0;
3095 : :
3096 [ + + - + : 48 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p was queued to ctrlr [%s, %u].\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3097 : : bio, CTRLR_STRING(nvme_ctrlr), CTRLR_ID(nvme_ctrlr));
3098 : 12 : } else {
3099 [ + + - + : 4 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p could not reset ctrlr [%s, %u], rc:%d\n",
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
3100 : : bio, CTRLR_STRING(nvme_ctrlr), CTRLR_ID(nvme_ctrlr), rc);
3101 : : }
3102 : :
3103 : 144 : return rc;
3104 : : }
3105 : :
3106 : : static void
3107 : 104 : bdev_nvme_freeze_bdev_channel_done(struct nvme_bdev *nbdev, void *ctx, int status)
3108 : : {
3109 : 104 : struct nvme_bdev_io *bio = ctx;
3110 : 104 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
3111 : : struct nvme_bdev_channel *nbdev_ch;
3112 : : struct nvme_io_path *io_path;
3113 : : int rc;
3114 : :
3115 : 104 : nbdev_ch = spdk_io_channel_get_ctx(spdk_bdev_io_get_io_channel(bdev_io));
3116 : :
3117 : : /* Initialize with failed status. With multipath it is enough to have at least one successful
3118 : : * nvme_ctrlr reset. If there is none, reset status will remain failed.
3119 : : */
3120 [ # # # # : 104 : bio->cpl.cdw0 = 1;
# # ]
3121 : :
3122 : : /* Reset all nvme_ctrlrs of a bdev controller sequentially. */
3123 [ # # # # : 104 : io_path = STAILQ_FIRST(&nbdev_ch->io_path_list);
# # ]
3124 [ + + # # ]: 104 : assert(io_path != NULL);
3125 : :
3126 : 104 : rc = _bdev_nvme_reset_io(io_path, bio);
3127 [ + + ]: 104 : if (rc != 0) {
3128 : : /* If the current nvme_ctrlr is disabled, skip it and move to the next nvme_ctrlr. */
3129 [ + + ]: 4 : rc = (rc == -EALREADY) ? 0 : rc;
3130 : :
3131 : 4 : bdev_nvme_reset_io_continue(bio, rc);
3132 : 1 : }
3133 : 104 : }
3134 : :
3135 : : static void
3136 : 160 : bdev_nvme_freeze_bdev_channel(struct nvme_bdev_channel_iter *i,
3137 : : struct nvme_bdev *nbdev,
3138 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
3139 : : {
3140 [ # # # # ]: 160 : nbdev_ch->resetting = true;
3141 : :
3142 : 160 : nvme_bdev_for_each_channel_continue(i, 0);
3143 : 160 : }
3144 : :
3145 : : static void
3146 : 100 : bdev_nvme_reset_io(struct nvme_bdev *nbdev, struct nvme_bdev_io *bio)
3147 : : {
3148 [ + + - + : 100 : NVME_BDEV_INFOLOG(nbdev, "reset_io %p started.\n", bio);
# # # # #
# # # ]
3149 : :
3150 : 116 : nvme_bdev_for_each_channel(nbdev,
3151 : : bdev_nvme_freeze_bdev_channel,
3152 : 16 : bio,
3153 : : bdev_nvme_freeze_bdev_channel_done);
3154 : 100 : }
3155 : :
3156 : : static int
3157 : 953 : bdev_nvme_failover_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool remove)
3158 : : {
3159 [ + + # # ]: 953 : if (nvme_ctrlr->destruct) {
3160 : : /* Don't bother resetting if the controller is in the process of being destructed. */
3161 : 418 : return -ENXIO;
3162 : : }
3163 : :
3164 [ + + # # ]: 535 : if (nvme_ctrlr->resetting) {
3165 [ + + # # ]: 48 : if (!nvme_ctrlr->in_failover) {
3166 [ + - # # : 12 : NVME_CTRLR_NOTICELOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3167 : : "Reset is already in progress. Defer failover until reset completes.\n");
3168 : :
3169 : : /* Defer failover until reset completes. */
3170 [ # # ]: 12 : nvme_ctrlr->pending_failover = true;
3171 : 12 : return -EINPROGRESS;
3172 : : } else {
3173 [ + - # # : 36 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Unable to perform failover, already in progress.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3174 : 36 : return -EBUSY;
3175 : : }
3176 : : }
3177 : :
3178 [ # # ]: 487 : bdev_nvme_failover_trid(nvme_ctrlr, remove, true);
3179 : :
3180 [ + + # # ]: 487 : if (nvme_ctrlr->reconnect_is_delayed) {
3181 [ + - # # : 4 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Reconnect is already scheduled.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3182 : :
3183 : : /* We rely on the next reconnect for the failover. */
3184 : 4 : return -EALREADY;
3185 : : }
3186 : :
3187 [ - + # # ]: 483 : if (nvme_ctrlr->disabled) {
3188 [ # # # # : 0 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Controller is disabled.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3189 : :
3190 : : /* We rely on the enablement for the failover. */
3191 : 0 : return -EALREADY;
3192 : : }
3193 : :
3194 [ # # ]: 483 : nvme_ctrlr->resetting = true;
3195 [ # # ]: 483 : nvme_ctrlr->in_failover = true;
3196 : :
3197 [ + + # # : 483 : assert(nvme_ctrlr->reset_start_tsc == 0);
# # # # ]
3198 [ # # # # ]: 483 : nvme_ctrlr->reset_start_tsc = spdk_get_ticks();
3199 : :
3200 : 483 : return 0;
3201 : 32 : }
3202 : :
3203 : : static int
3204 : 941 : bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr)
3205 : : {
3206 : : int rc;
3207 : :
3208 [ - + # # ]: 941 : pthread_mutex_lock(&nvme_ctrlr->mutex);
3209 : 941 : rc = bdev_nvme_failover_ctrlr_unsafe(nvme_ctrlr, false);
3210 [ - + # # ]: 941 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
3211 : :
3212 [ + + ]: 941 : if (rc == 0) {
3213 [ # # # # ]: 475 : spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset_ctrlr, nvme_ctrlr);
3214 [ + + ]: 491 : } else if (rc == -EALREADY) {
3215 : 0 : rc = 0;
3216 : 0 : }
3217 : :
3218 : 941 : return rc;
3219 : : }
3220 : :
3221 : : static int bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks,
3222 : : uint64_t num_blocks);
3223 : :
3224 : : static int bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks,
3225 : : uint64_t num_blocks);
3226 : :
3227 : : static int bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks,
3228 : : uint64_t src_offset_blocks,
3229 : : uint64_t num_blocks);
3230 : :
3231 : : static void
3232 : 6513868 : bdev_nvme_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,
3233 : : bool success)
3234 : : {
3235 [ # # ]: 6513868 : struct nvme_bdev_io *bio = (struct nvme_bdev_io *)bdev_io->driver_ctx;
3236 : : int ret;
3237 : :
3238 [ + + # # ]: 6513868 : if (!success) {
3239 : 0 : ret = -EINVAL;
3240 : 0 : goto exit;
3241 : : }
3242 : :
3243 [ + + # # : 6513868 : if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) {
# # ]
3244 : 0 : ret = -ENXIO;
3245 : 0 : goto exit;
3246 : : }
3247 : :
3248 : 6779399 : ret = bdev_nvme_readv(bio,
3249 [ # # # # : 265531 : bdev_io->u.bdev.iovs,
# # # # ]
3250 [ # # # # : 265531 : bdev_io->u.bdev.iovcnt,
# # # # ]
3251 [ # # # # : 265531 : bdev_io->u.bdev.md_buf,
# # # # ]
3252 [ # # # # : 265531 : bdev_io->u.bdev.num_blocks,
# # # # ]
3253 [ # # # # : 265531 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3254 [ # # # # : 265531 : bdev_io->u.bdev.dif_check_flags,
# # # # ]
3255 [ # # # # : 265531 : bdev_io->u.bdev.memory_domain,
# # # # ]
3256 [ # # # # : 265531 : bdev_io->u.bdev.memory_domain_ctx,
# # # # ]
3257 [ # # # # : 265531 : bdev_io->u.bdev.accel_sequence);
# # # # ]
3258 : :
3259 : 6248337 : exit:
3260 [ + + ]: 6513868 : if (spdk_unlikely(ret != 0)) {
3261 : 85691 : bdev_nvme_io_complete(bio, ret);
3262 : 0 : }
3263 : 6513868 : }
3264 : :
3265 : : static inline void
3266 : 28990118 : _bdev_nvme_submit_request(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io)
3267 : : {
3268 [ + - ]: 28990118 : struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx;
3269 [ + - + - ]: 28990118 : struct spdk_bdev *bdev = bdev_io->bdev;
3270 : : struct nvme_bdev_io *nbdev_io_to_abort;
3271 : 28990118 : int rc = 0;
3272 : :
3273 [ + + + + : 28990118 : switch (bdev_io->type) {
+ + + + +
+ + + + +
- + + + -
- - - ]
3274 : 12648018 : case SPDK_BDEV_IO_TYPE_READ:
3275 [ + + + + : 12913629 : if (bdev_io->u.bdev.iovs && bdev_io->u.bdev.iovs[0].iov_base) {
+ - + - +
- + - + -
+ - + - +
- + - + -
- + ]
3276 : :
3277 : 6399939 : rc = bdev_nvme_readv(nbdev_io,
3278 [ - + - + : 129 : bdev_io->u.bdev.iovs,
- + - + ]
3279 [ - + - + : 129 : bdev_io->u.bdev.iovcnt,
- + - + ]
3280 [ - + - + : 129 : bdev_io->u.bdev.md_buf,
- + - + ]
3281 [ - + - + : 129 : bdev_io->u.bdev.num_blocks,
- + - + ]
3282 [ - + - + : 129 : bdev_io->u.bdev.offset_blocks,
- + - + ]
3283 [ - + - + : 129 : bdev_io->u.bdev.dif_check_flags,
- + - + ]
3284 [ - + - + : 129 : bdev_io->u.bdev.memory_domain,
- + - + ]
3285 [ - + - + : 129 : bdev_io->u.bdev.memory_domain_ctx,
- + - + ]
3286 [ - + - + : 129 : bdev_io->u.bdev.accel_sequence);
- + - + ]
3287 : 129 : } else {
3288 : 6779301 : spdk_bdev_io_get_buf(bdev_io, bdev_nvme_get_buf_cb,
3289 [ # # # # : 6513819 : bdev_io->u.bdev.num_blocks * bdev->blocklen);
# # # # #
# # # ]
3290 : 6513819 : rc = 0;
3291 : : }
3292 : 12913629 : break;
3293 : 13482918 : case SPDK_BDEV_IO_TYPE_WRITE:
3294 : 14014290 : rc = bdev_nvme_writev(nbdev_io,
3295 [ # # # # : 265686 : bdev_io->u.bdev.iovs,
# # # # ]
3296 [ # # # # : 265686 : bdev_io->u.bdev.iovcnt,
# # # # ]
3297 [ # # # # : 265686 : bdev_io->u.bdev.md_buf,
# # # # ]
3298 [ # # # # : 265686 : bdev_io->u.bdev.num_blocks,
# # # # ]
3299 [ # # # # : 265686 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3300 [ # # # # : 265686 : bdev_io->u.bdev.dif_check_flags,
# # # # ]
3301 [ # # # # : 265686 : bdev_io->u.bdev.memory_domain,
# # # # ]
3302 [ # # # # : 265686 : bdev_io->u.bdev.memory_domain_ctx,
# # # # ]
3303 [ # # # # : 265686 : bdev_io->u.bdev.accel_sequence,
# # # # ]
3304 [ # # # # : 265686 : bdev_io->u.bdev.nvme_cdw12,
# # ]
3305 [ # # # # : 265686 : bdev_io->u.bdev.nvme_cdw13);
# # ]
3306 : 13748604 : break;
3307 : 53 : case SPDK_BDEV_IO_TYPE_COMPARE:
3308 : 59 : rc = bdev_nvme_comparev(nbdev_io,
3309 [ # # # # : 3 : bdev_io->u.bdev.iovs,
# # # # ]
3310 [ # # # # : 3 : bdev_io->u.bdev.iovcnt,
# # # # ]
3311 [ # # # # : 3 : bdev_io->u.bdev.md_buf,
# # # # ]
3312 [ # # # # : 3 : bdev_io->u.bdev.num_blocks,
# # # # ]
3313 [ # # # # : 3 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3314 [ # # # # : 3 : bdev_io->u.bdev.dif_check_flags);
# # # # ]
3315 : 56 : break;
3316 : 51 : case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE:
3317 : 55 : rc = bdev_nvme_comparev_and_writev(nbdev_io,
3318 [ # # # # : 2 : bdev_io->u.bdev.iovs,
# # # # ]
3319 [ # # # # : 2 : bdev_io->u.bdev.iovcnt,
# # # # ]
3320 [ # # # # : 2 : bdev_io->u.bdev.fused_iovs,
# # # # ]
3321 [ # # # # : 2 : bdev_io->u.bdev.fused_iovcnt,
# # # # ]
3322 [ # # # # : 2 : bdev_io->u.bdev.md_buf,
# # # # ]
3323 [ # # # # : 2 : bdev_io->u.bdev.num_blocks,
# # # # ]
3324 [ # # # # : 2 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3325 [ # # # # : 2 : bdev_io->u.bdev.dif_check_flags);
# # # # ]
3326 : 53 : break;
3327 : 69078 : case SPDK_BDEV_IO_TYPE_UNMAP:
3328 : 69084 : rc = bdev_nvme_unmap(nbdev_io,
3329 [ # # # # : 3 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3330 [ # # # # : 3 : bdev_io->u.bdev.num_blocks);
# # # # ]
3331 : 69081 : break;
3332 : 637807 : case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
3333 : 787535 : rc = bdev_nvme_write_zeroes(nbdev_io,
3334 [ # # # # : 74864 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3335 [ # # # # : 74864 : bdev_io->u.bdev.num_blocks);
# # # # ]
3336 : 712671 : break;
3337 : 84 : case SPDK_BDEV_IO_TYPE_RESET:
3338 [ # # # # ]: 100 : nbdev_io->io_path = NULL;
3339 [ # # # # ]: 100 : bdev_nvme_reset_io(bdev->ctxt, nbdev_io);
3340 : 100 : return;
3341 : :
3342 : 1281460 : case SPDK_BDEV_IO_TYPE_FLUSH:
3343 : 1281461 : bdev_nvme_io_complete(nbdev_io, 0);
3344 : 1281461 : return;
3345 : :
3346 : 256879 : case SPDK_BDEV_IO_TYPE_ZONE_APPEND:
3347 : 256879 : rc = bdev_nvme_zone_appendv(nbdev_io,
3348 [ # # # # : 0 : bdev_io->u.bdev.iovs,
# # # # ]
3349 [ # # # # : 0 : bdev_io->u.bdev.iovcnt,
# # # # ]
3350 [ # # # # : 0 : bdev_io->u.bdev.md_buf,
# # # # ]
3351 [ # # # # : 0 : bdev_io->u.bdev.num_blocks,
# # # # ]
3352 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3353 [ # # # # : 0 : bdev_io->u.bdev.dif_check_flags);
# # # # ]
3354 : 256879 : break;
3355 : 1 : case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO:
3356 : 1 : rc = bdev_nvme_get_zone_info(nbdev_io,
3357 [ # # # # : 0 : bdev_io->u.zone_mgmt.zone_id,
# # # # ]
3358 [ # # # # : 0 : bdev_io->u.zone_mgmt.num_zones,
# # # # ]
3359 [ # # # # : 1 : bdev_io->u.zone_mgmt.buf);
# # # # ]
3360 : 1 : break;
3361 : 44 : case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT:
3362 : 44 : rc = bdev_nvme_zone_management(nbdev_io,
3363 [ # # # # : 0 : bdev_io->u.zone_mgmt.zone_id,
# # # # ]
3364 [ # # # # : 0 : bdev_io->u.zone_mgmt.zone_action);
# # # # ]
3365 : 44 : break;
3366 : 136 : case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
3367 [ # # # # ]: 142 : nbdev_io->io_path = NULL;
3368 : 148 : bdev_nvme_admin_passthru(nbdev_ch,
3369 : 6 : nbdev_io,
3370 [ # # # # : 6 : &bdev_io->u.nvme_passthru.cmd,
# # ]
3371 [ # # # # : 6 : bdev_io->u.nvme_passthru.buf,
# # # # ]
3372 [ # # # # : 6 : bdev_io->u.nvme_passthru.nbytes);
# # # # ]
3373 : 142 : return;
3374 : :
3375 : 108 : case SPDK_BDEV_IO_TYPE_NVME_IO:
3376 : 114 : rc = bdev_nvme_io_passthru(nbdev_io,
3377 [ # # # # : 3 : &bdev_io->u.nvme_passthru.cmd,
# # ]
3378 [ # # # # : 3 : bdev_io->u.nvme_passthru.buf,
# # # # ]
3379 [ # # # # : 3 : bdev_io->u.nvme_passthru.nbytes);
# # # # ]
3380 : 111 : break;
3381 : 0 : case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
3382 : 0 : rc = bdev_nvme_io_passthru_md(nbdev_io,
3383 [ # # # # : 0 : &bdev_io->u.nvme_passthru.cmd,
# # ]
3384 [ # # # # : 0 : bdev_io->u.nvme_passthru.buf,
# # # # ]
3385 [ # # # # : 0 : bdev_io->u.nvme_passthru.nbytes,
# # # # ]
3386 [ # # # # : 0 : bdev_io->u.nvme_passthru.md_buf,
# # # # ]
3387 [ # # # # : 0 : bdev_io->u.nvme_passthru.md_len);
# # # # ]
3388 : 0 : break;
3389 : 0 : case SPDK_BDEV_IO_TYPE_NVME_IOV_MD:
3390 : 0 : rc = bdev_nvme_iov_passthru_md(nbdev_io,
3391 [ # # # # : 0 : &bdev_io->u.nvme_passthru.cmd,
# # ]
3392 [ # # # # : 0 : bdev_io->u.nvme_passthru.iovs,
# # # # ]
3393 [ # # # # : 0 : bdev_io->u.nvme_passthru.iovcnt,
# # # # ]
3394 [ # # # # : 0 : bdev_io->u.nvme_passthru.nbytes,
# # # # ]
3395 [ # # # # : 0 : bdev_io->u.nvme_passthru.md_buf,
# # # # ]
3396 [ # # # # : 0 : bdev_io->u.nvme_passthru.md_len);
# # # # ]
3397 : 0 : break;
3398 : 7243 : case SPDK_BDEV_IO_TYPE_ABORT:
3399 [ # # # # ]: 7249 : nbdev_io->io_path = NULL;
3400 [ # # # # : 7249 : nbdev_io_to_abort = (struct nvme_bdev_io *)bdev_io->u.abort.bio_to_abort->driver_ctx;
# # # # #
# ]
3401 : 7255 : bdev_nvme_abort(nbdev_ch,
3402 : 6 : nbdev_io,
3403 : 6 : nbdev_io_to_abort);
3404 : 7249 : return;
3405 : :
3406 : 36 : case SPDK_BDEV_IO_TYPE_COPY:
3407 : 38 : rc = bdev_nvme_copy(nbdev_io,
3408 [ # # # # : 1 : bdev_io->u.bdev.offset_blocks,
# # # # ]
3409 [ # # # # : 1 : bdev_io->u.bdev.copy.src_offset_blocks,
# # # # #
# ]
3410 [ # # # # : 1 : bdev_io->u.bdev.num_blocks);
# # # # ]
3411 : 37 : break;
3412 : 0 : default:
3413 : 0 : rc = -EINVAL;
3414 : 0 : break;
3415 : : }
3416 : :
3417 [ + + ]: 27701166 : if (spdk_unlikely(rc != 0)) {
3418 : 85518 : bdev_nvme_io_complete(nbdev_io, rc);
3419 : 0 : }
3420 : 606202 : }
3421 : :
3422 : : static void
3423 : 29468397 : bdev_nvme_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
3424 : : {
3425 : 29468397 : struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
3426 [ + - ]: 29468397 : struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx;
3427 : :
3428 [ + + + - : 29468397 : if (spdk_likely(nbdev_io->submit_tsc == 0)) {
- + ]
3429 [ + - + - ]: 29432774 : nbdev_io->submit_tsc = spdk_bdev_io_get_submit_tsc(bdev_io);
3430 : 605584 : } else {
3431 : : /* There are cases where submit_tsc != 0, i.e. retry I/O.
3432 : : * We need to update submit_tsc here.
3433 : : */
3434 [ # # # # ]: 35623 : nbdev_io->submit_tsc = spdk_get_ticks();
3435 : : }
3436 : :
3437 [ + + + + : 29468397 : spdk_trace_record(TRACE_BDEV_NVME_IO_START, 0, 0, (uintptr_t)nbdev_io, (uintptr_t)bdev_io);
+ - + - +
- + - + -
+ - ]
3438 [ + - + - ]: 29468397 : nbdev_io->io_path = bdev_nvme_find_io_path(nbdev_ch);
3439 [ + + + - : 29468397 : if (spdk_unlikely(!nbdev_io->io_path)) {
+ - ]
3440 [ + + # # : 482551 : if (!bdev_nvme_io_type_is_admin(bdev_io->type)) {
# # ]
3441 : 482547 : bdev_nvme_io_complete(nbdev_io, -ENXIO);
3442 : 482547 : return;
3443 : : }
3444 : :
3445 : : /* Admin commands do not use the optimal I/O path.
3446 : : * Simply fall through even if it is not found.
3447 : : */
3448 : 1 : }
3449 : :
3450 : 28985850 : _bdev_nvme_submit_request(nbdev_ch, bdev_io);
3451 : 605586 : }
3452 : :
3453 : : static bool
3454 : 3781353 : bdev_nvme_is_supported_csi(enum spdk_nvme_csi csi)
3455 : : {
3456 [ + + + ]: 3781353 : switch (csi) {
3457 : 3631430 : case SPDK_NVME_CSI_NVM:
3458 : 3781324 : return true;
3459 : 29 : case SPDK_NVME_CSI_ZNS:
3460 : 29 : return true;
3461 : 0 : default:
3462 : 0 : return false;
3463 : : }
3464 : 149894 : }
3465 : :
3466 : : static bool
3467 : 3781353 : bdev_nvme_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type)
3468 : : {
3469 : 3781353 : struct nvme_bdev *nbdev = ctx;
3470 : : struct nvme_ns *nvme_ns;
3471 : : struct spdk_nvme_ns *ns;
3472 : : struct spdk_nvme_ctrlr *ctrlr;
3473 : : const struct spdk_nvme_ctrlr_data *cdata;
3474 : :
3475 [ + - + - : 3781353 : nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
+ - ]
3476 [ + + # # ]: 3781353 : assert(nvme_ns != NULL);
3477 [ + - + - ]: 3781353 : ns = nvme_ns->ns;
3478 [ + + ]: 3781353 : if (ns == NULL) {
3479 : 0 : return false;
3480 : : }
3481 : :
3482 [ + + ]: 3781353 : if (!bdev_nvme_is_supported_csi(spdk_nvme_ns_get_csi(ns))) {
3483 [ # # # ]: 0 : switch (io_type) {
3484 : 0 : case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
3485 : : case SPDK_BDEV_IO_TYPE_NVME_IO:
3486 : 0 : return true;
3487 : :
3488 : 0 : case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
3489 : 0 : return spdk_nvme_ns_get_md_size(ns) ? true : false;
3490 : :
3491 : 0 : default:
3492 : 0 : return false;
3493 : : }
3494 : : }
3495 : :
3496 : 3781353 : ctrlr = spdk_nvme_ns_get_ctrlr(ns);
3497 : :
3498 [ + + + + : 3781353 : switch (io_type) {
+ + + + +
+ ]
3499 : 1499903 : case SPDK_BDEV_IO_TYPE_READ:
3500 : : case SPDK_BDEV_IO_TYPE_WRITE:
3501 : : case SPDK_BDEV_IO_TYPE_RESET:
3502 : : case SPDK_BDEV_IO_TYPE_FLUSH:
3503 : : case SPDK_BDEV_IO_TYPE_NVME_ADMIN:
3504 : : case SPDK_BDEV_IO_TYPE_NVME_IO:
3505 : : case SPDK_BDEV_IO_TYPE_ABORT:
3506 : 1499950 : return true;
3507 : :
3508 : 3800 : case SPDK_BDEV_IO_TYPE_COMPARE:
3509 : 3806 : return spdk_nvme_ns_supports_compare(ns);
3510 : :
3511 : 1108 : case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
3512 : 1110 : return spdk_nvme_ns_get_md_size(ns) ? true : false;
3513 : :
3514 : 12712 : case SPDK_BDEV_IO_TYPE_UNMAP:
3515 : 12718 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3516 [ # # # # ]: 12718 : return cdata->oncs.dsm;
3517 : :
3518 : 2077307 : case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
3519 : 2227070 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3520 [ + - + - ]: 2227070 : return cdata->oncs.write_zeroes;
3521 : :
3522 : 3798 : case SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE:
3523 [ + + ]: 3804 : if (spdk_nvme_ctrlr_get_flags(ctrlr) &
3524 : : SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED) {
3525 : 236 : return true;
3526 : : }
3527 : 3568 : return false;
3528 : :
3529 : 7448 : case SPDK_BDEV_IO_TYPE_GET_ZONE_INFO:
3530 : : case SPDK_BDEV_IO_TYPE_ZONE_MANAGEMENT:
3531 : 7456 : return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS;
3532 : :
3533 : 3725 : case SPDK_BDEV_IO_TYPE_ZONE_APPEND:
3534 [ + + ]: 3731 : return spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS &&
3535 [ + - ]: 2 : spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED;
3536 : :
3537 : 6728 : case SPDK_BDEV_IO_TYPE_COPY:
3538 : 6764 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
3539 [ + - + - ]: 6764 : return cdata->oncs.copy;
3540 : :
3541 : 14930 : default:
3542 : 14946 : return false;
3543 : : }
3544 : 149894 : }
3545 : :
3546 : : static int
3547 : 2397 : nvme_qpair_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ctrlr_channel *ctrlr_ch)
3548 : : {
3549 : : struct nvme_qpair *nvme_qpair;
3550 : : struct spdk_io_channel *pg_ch;
3551 : : int rc;
3552 : :
3553 : 2397 : nvme_qpair = calloc(1, sizeof(*nvme_qpair));
3554 [ + + ]: 2397 : if (!nvme_qpair) {
3555 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to alloc nvme_qpair.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
3556 : 0 : return -1;
3557 : : }
3558 : :
3559 [ + - + - : 2397 : TAILQ_INIT(&nvme_qpair->io_path_list);
+ - + - +
- + - + -
+ - ]
3560 : :
3561 [ + - + - ]: 2397 : nvme_qpair->ctrlr = nvme_ctrlr;
3562 [ + - + - ]: 2397 : nvme_qpair->ctrlr_ch = ctrlr_ch;
3563 : :
3564 : 2397 : pg_ch = spdk_get_io_channel(&g_nvme_bdev_ctrlrs);
3565 [ + + ]: 2397 : if (!pg_ch) {
3566 : 0 : free(nvme_qpair);
3567 : 0 : return -1;
3568 : : }
3569 : :
3570 [ + - + - ]: 2397 : nvme_qpair->group = spdk_io_channel_get_ctx(pg_ch);
3571 : :
3572 : : #ifdef SPDK_CONFIG_VTUNE
3573 : : nvme_qpair->group->collect_spin_stat = true;
3574 : : #else
3575 [ + - + - : 2397 : nvme_qpair->group->collect_spin_stat = false;
+ - + - ]
3576 : : #endif
3577 : :
3578 [ + + - + ]: 2397 : if (!nvme_ctrlr->disabled) {
3579 : : /* If a nvme_ctrlr is disabled, don't try to create qpair for it. Qpair will
3580 : : * be created when it's enabled.
3581 : : */
3582 : 2397 : rc = bdev_nvme_create_qpair(nvme_qpair);
3583 [ + + ]: 2397 : if (rc != 0) {
3584 : : /* nvme_ctrlr can't create IO qpair if connection is down.
3585 : : * If reconnect_delay_sec is non-zero, creating IO qpair is retried
3586 : : * after reconnect_delay_sec seconds. If bdev_retry_count is non-zero,
3587 : : * submitted IO will be queued until IO qpair is successfully created.
3588 : : *
3589 : : * Hence, if both are satisfied, ignore the failure.
3590 : : */
3591 [ # # # # : 0 : if (nvme_ctrlr->opts.reconnect_delay_sec == 0 || g_opts.bdev_retry_count == 0) {
# # # # #
# # # ]
3592 : 0 : spdk_put_io_channel(pg_ch);
3593 : 0 : free(nvme_qpair);
3594 : 0 : return rc;
3595 : : }
3596 : 0 : }
3597 : 82 : }
3598 : :
3599 [ + - + - : 2397 : TAILQ_INSERT_TAIL(&nvme_qpair->group->qpair_list, nvme_qpair, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - ]
3600 : :
3601 [ + - + - ]: 2397 : ctrlr_ch->qpair = nvme_qpair;
3602 : :
3603 : 2397 : nvme_ctrlr_get_ref(nvme_ctrlr);
3604 : :
3605 : 2397 : return 0;
3606 : 82 : }
3607 : :
3608 : : static int
3609 : 2397 : bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf)
3610 : : {
3611 : 2397 : struct nvme_ctrlr *nvme_ctrlr = io_device;
3612 : 2397 : struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf;
3613 : :
3614 : 2397 : return nvme_qpair_create(nvme_ctrlr, ctrlr_ch);
3615 : : }
3616 : :
3617 : : static void
3618 : 2397 : nvme_qpair_delete(struct nvme_qpair *nvme_qpair)
3619 : : {
3620 : : struct nvme_io_path *io_path, *next;
3621 : :
3622 [ + + + - : 2397 : assert(nvme_qpair->group != NULL);
+ - # # ]
3623 : :
3624 [ + + + + : 4828 : TAILQ_FOREACH_SAFE(io_path, &nvme_qpair->io_path_list, tailq, next) {
+ - + + +
- + - + -
+ + ]
3625 [ + + + - : 2431 : TAILQ_REMOVE(&nvme_qpair->io_path_list, io_path, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - ]
3626 : 2431 : nvme_io_path_free(io_path);
3627 : 60 : }
3628 : :
3629 [ + + + - : 2397 : TAILQ_REMOVE(&nvme_qpair->group->qpair_list, nvme_qpair, tailq);
+ - - + #
# # # # #
# # # # #
# # # # #
# # + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
3630 : :
3631 [ + - + - ]: 2397 : spdk_put_io_channel(spdk_io_channel_from_ctx(nvme_qpair->group));
3632 : :
3633 [ + - + - ]: 2397 : nvme_ctrlr_put_ref(nvme_qpair->ctrlr);
3634 : :
3635 : 2397 : free(nvme_qpair);
3636 : 2397 : }
3637 : :
3638 : : static void
3639 : 2397 : bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf)
3640 : : {
3641 : 2397 : struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf;
3642 : : struct nvme_qpair *nvme_qpair;
3643 : :
3644 [ + - + - ]: 2397 : nvme_qpair = ctrlr_ch->qpair;
3645 [ + + # # ]: 2397 : assert(nvme_qpair != NULL);
3646 : :
3647 : 2397 : _bdev_nvme_clear_io_path_cache(nvme_qpair);
3648 : :
3649 [ + + + - : 2397 : if (nvme_qpair->qpair != NULL) {
+ - ]
3650 : : /* Always try to disconnect the qpair, even if a reset is in progress.
3651 : : * The qpair may have been created after the reset process started.
3652 : : */
3653 [ + - + - ]: 2312 : spdk_nvme_ctrlr_disconnect_io_qpair(nvme_qpair->qpair);
3654 [ + + + - : 2312 : if (ctrlr_ch->reset_iter) {
- + ]
3655 : : /* Skip current ctrlr_channel in a full reset sequence because
3656 : : * it is being deleted now.
3657 : : */
3658 [ # # # # ]: 0 : nvme_ctrlr_for_each_channel_continue(ctrlr_ch->reset_iter, 0);
3659 : 0 : }
3660 : :
3661 : : /* We cannot release a reference to the poll group now.
3662 : : * The qpair may be disconnected asynchronously later.
3663 : : * We need to poll it until it is actually disconnected.
3664 : : * Just detach the qpair from the deleting ctrlr_channel.
3665 : : */
3666 [ + - + - ]: 2312 : nvme_qpair->ctrlr_ch = NULL;
3667 : 68 : } else {
3668 [ - + # # : 85 : assert(ctrlr_ch->reset_iter == NULL);
# # # # ]
3669 : :
3670 : 85 : nvme_qpair_delete(nvme_qpair);
3671 : : }
3672 : 2397 : }
3673 : :
3674 : : static inline struct spdk_io_channel *
3675 : 843111 : bdev_nvme_get_accel_channel(struct nvme_poll_group *group)
3676 : : {
3677 [ + + # # : 843111 : if (spdk_unlikely(!group->accel_channel)) {
# # ]
3678 [ # # # # ]: 53 : group->accel_channel = spdk_accel_get_io_channel();
3679 [ - + # # : 53 : if (!group->accel_channel) {
# # ]
3680 : 0 : SPDK_ERRLOG("Cannot get the accel_channel for bdev nvme polling group=%p\n",
3681 : : group);
3682 : 0 : return NULL;
3683 : : }
3684 : 0 : }
3685 : :
3686 [ # # # # ]: 843111 : return group->accel_channel;
3687 : 0 : }
3688 : :
3689 : : static void
3690 : 843111 : bdev_nvme_finish_sequence(void *seq, spdk_nvme_accel_completion_cb cb_fn, void *cb_arg)
3691 : : {
3692 : 843111 : spdk_accel_sequence_finish(seq, cb_fn, cb_arg);
3693 : 843111 : }
3694 : :
3695 : : static void
3696 : 0 : bdev_nvme_abort_sequence(void *seq)
3697 : : {
3698 : 0 : spdk_accel_sequence_abort(seq);
3699 : 0 : }
3700 : :
3701 : : static void
3702 : 407063 : bdev_nvme_reverse_sequence(void *seq)
3703 : : {
3704 : 407063 : spdk_accel_sequence_reverse(seq);
3705 : 407063 : }
3706 : :
3707 : : static int
3708 : 843111 : bdev_nvme_append_crc32c(void *ctx, void **seq, uint32_t *dst, struct iovec *iovs, uint32_t iovcnt,
3709 : : struct spdk_memory_domain *domain, void *domain_ctx, uint32_t seed,
3710 : : spdk_nvme_accel_step_cb cb_fn, void *cb_arg)
3711 : : {
3712 : : struct spdk_io_channel *ch;
3713 : 843111 : struct nvme_poll_group *group = ctx;
3714 : :
3715 : 843111 : ch = bdev_nvme_get_accel_channel(group);
3716 [ - + ]: 843111 : if (spdk_unlikely(ch == NULL)) {
3717 : 0 : return -ENOMEM;
3718 : : }
3719 : :
3720 : 843111 : return spdk_accel_append_crc32c((struct spdk_accel_sequence **)seq, ch, dst, iovs, iovcnt,
3721 : 0 : domain, domain_ctx, seed, cb_fn, cb_arg);
3722 : 0 : }
3723 : :
3724 : : static int
3725 : 0 : bdev_nvme_append_copy(void *ctx, void **seq, struct iovec *dst_iovs, uint32_t dst_iovcnt,
3726 : : struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
3727 : : struct iovec *src_iovs, uint32_t src_iovcnt,
3728 : : struct spdk_memory_domain *src_domain, void *src_domain_ctx,
3729 : : spdk_nvme_accel_step_cb cb_fn, void *cb_arg)
3730 : : {
3731 : : struct spdk_io_channel *ch;
3732 : 0 : struct nvme_poll_group *group = ctx;
3733 : :
3734 : 0 : ch = bdev_nvme_get_accel_channel(group);
3735 [ # # ]: 0 : if (spdk_unlikely(ch == NULL)) {
3736 : 0 : return -ENOMEM;
3737 : : }
3738 : :
3739 : 0 : return spdk_accel_append_copy((struct spdk_accel_sequence **)seq, ch,
3740 : 0 : dst_iovs, dst_iovcnt, dst_domain, dst_domain_ctx,
3741 : 0 : src_iovs, src_iovcnt, src_domain, src_domain_ctx,
3742 : 0 : cb_fn, cb_arg);
3743 : 0 : }
3744 : :
3745 : : static struct spdk_nvme_accel_fn_table g_bdev_nvme_accel_fn_table = {
3746 : : .table_size = sizeof(struct spdk_nvme_accel_fn_table),
3747 : : .append_crc32c = bdev_nvme_append_crc32c,
3748 : : .append_copy = bdev_nvme_append_copy,
3749 : : .finish_sequence = bdev_nvme_finish_sequence,
3750 : : .reverse_sequence = bdev_nvme_reverse_sequence,
3751 : : .abort_sequence = bdev_nvme_abort_sequence,
3752 : : };
3753 : :
3754 : : static void
3755 : 2 : bdev_nvme_poll_group_interrupt_cb(struct spdk_nvme_poll_group *group, void *ctx)
3756 : : {
3757 : 2 : bdev_nvme_poll(ctx);
3758 : 2 : }
3759 : :
3760 : : static int
3761 : 2237 : bdev_nvme_create_poll_group_cb(void *io_device, void *ctx_buf)
3762 : : {
3763 : 2237 : struct nvme_poll_group *group = ctx_buf;
3764 : : struct spdk_fd_group *fgrp;
3765 : : uint64_t period;
3766 : : int rc;
3767 : :
3768 [ + - + - : 2237 : TAILQ_INIT(&group->qpair_list);
+ - + - +
- + - + -
+ - ]
3769 : :
3770 [ + - + - ]: 2237 : group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table);
3771 [ + + + - : 2237 : if (group->group == NULL) {
+ - ]
3772 : 0 : return -1;
3773 : : }
3774 : :
3775 [ + + + - ]: 2237 : period = spdk_interrupt_mode_is_enabled() ? 0 : g_opts.nvme_ioq_poll_period_us;
3776 [ + - + - ]: 2237 : group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, period);
3777 : :
3778 [ + + + - : 2237 : if (group->poller == NULL) {
+ - ]
3779 [ # # # # ]: 0 : spdk_nvme_poll_group_destroy(group->group);
3780 : 0 : return -1;
3781 : : }
3782 : :
3783 [ + + ]: 2237 : if (spdk_interrupt_mode_is_enabled()) {
3784 [ # # # # ]: 2 : spdk_poller_register_interrupt(group->poller, NULL, NULL);
3785 : :
3786 [ # # # # ]: 2 : fgrp = spdk_nvme_poll_group_get_fd_group(group->group);
3787 [ - + ]: 2 : if (fgrp == NULL) {
3788 [ # # # # ]: 0 : spdk_nvme_poll_group_destroy(group->group);
3789 : 0 : return -1;
3790 : : }
3791 : :
3792 [ # # # # ]: 2 : rc = spdk_nvme_poll_group_set_interrupt_callback(group->group,
3793 : 0 : bdev_nvme_poll_group_interrupt_cb, group);
3794 [ - + ]: 2 : if (rc != 0) {
3795 [ # # # # ]: 0 : spdk_nvme_poll_group_destroy(group->group);
3796 : 0 : return -1;
3797 : : }
3798 : :
3799 [ # # # # ]: 2 : group->intr = spdk_interrupt_register_fd_group(fgrp, "bdev_nvme_interrupt");
3800 [ - + # # : 2 : if (!group->intr) {
# # ]
3801 [ # # # # ]: 0 : spdk_nvme_poll_group_destroy(group->group);
3802 : 0 : return -1;
3803 : : }
3804 : 0 : }
3805 : :
3806 : 2237 : return 0;
3807 : 67 : }
3808 : :
3809 : : static void
3810 : 2237 : bdev_nvme_destroy_poll_group_cb(void *io_device, void *ctx_buf)
3811 : : {
3812 : 2237 : struct nvme_poll_group *group = ctx_buf;
3813 : :
3814 [ + + + - : 2237 : assert(TAILQ_EMPTY(&group->qpair_list));
+ - + - #
# ]
3815 : :
3816 [ + + + - : 2237 : if (group->accel_channel) {
+ - ]
3817 [ # # # # ]: 53 : spdk_put_io_channel(group->accel_channel);
3818 : 0 : }
3819 : :
3820 [ + + ]: 2237 : if (spdk_interrupt_mode_is_enabled()) {
3821 [ # # ]: 2 : spdk_interrupt_unregister(&group->intr);
3822 : 0 : }
3823 : :
3824 [ + - ]: 2237 : spdk_poller_unregister(&group->poller);
3825 [ + + + - : 2237 : if (spdk_nvme_poll_group_destroy(group->group)) {
+ - ]
3826 : 0 : SPDK_ERRLOG("Unable to destroy a poll group for the NVMe bdev module.\n");
3827 [ # # ]: 0 : assert(false);
3828 : : }
3829 : 2237 : }
3830 : :
3831 : : static struct spdk_io_channel *
3832 : 2279 : bdev_nvme_get_io_channel(void *ctx)
3833 : : {
3834 : 2279 : struct nvme_bdev *nbdev = ctx;
3835 : :
3836 : 2279 : return spdk_get_io_channel(nbdev);
3837 : : }
3838 : :
3839 : : static void *
3840 : 0 : bdev_nvme_get_module_ctx(void *ctx)
3841 : : {
3842 : 0 : struct nvme_bdev *nbdev = ctx;
3843 : : struct nvme_ns *nvme_ns;
3844 : :
3845 [ # # # # : 0 : if (!nbdev || nbdev->disk.module != &nvme_if) {
# # # # #
# ]
3846 : 0 : return NULL;
3847 : : }
3848 : :
3849 [ # # # # : 0 : nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
# # ]
3850 [ # # ]: 0 : if (!nvme_ns) {
3851 : 0 : return NULL;
3852 : : }
3853 : :
3854 [ # # # # ]: 0 : return nvme_ns->ns;
3855 : 0 : }
3856 : :
3857 : : static const char *
3858 : 0 : _nvme_ana_state_str(enum spdk_nvme_ana_state ana_state)
3859 : : {
3860 [ # # # # : 0 : switch (ana_state) {
# # ]
3861 : 0 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
3862 : 0 : return "optimized";
3863 : 0 : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
3864 : 0 : return "non_optimized";
3865 : 0 : case SPDK_NVME_ANA_INACCESSIBLE_STATE:
3866 : 0 : return "inaccessible";
3867 : 0 : case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE:
3868 : 0 : return "persistent_loss";
3869 : 0 : case SPDK_NVME_ANA_CHANGE_STATE:
3870 : 0 : return "change";
3871 : 0 : default:
3872 : 0 : return NULL;
3873 : : }
3874 : 0 : }
3875 : :
3876 : : static int
3877 : 10379 : bdev_nvme_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size)
3878 : : {
3879 : 10379 : struct spdk_memory_domain **_domains = NULL;
3880 : 10379 : struct nvme_bdev *nbdev = ctx;
3881 : : struct nvme_ns *nvme_ns;
3882 : 10379 : int i = 0, _array_size = array_size;
3883 : 10379 : int rc = 0;
3884 : :
3885 [ + + + - : 20828 : TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
+ - + + +
- + - +
- ]
3886 [ + + + + ]: 10449 : if (domains && array_size >= i) {
3887 [ # # ]: 273 : _domains = &domains[i];
3888 : 11 : } else {
3889 : 10176 : _domains = NULL;
3890 : : }
3891 [ + - + - : 10449 : rc = spdk_nvme_ctrlr_get_memory_domains(nvme_ns->ctrlr->ctrlr, _domains, _array_size);
+ - + - ]
3892 [ + + ]: 10449 : if (rc > 0) {
3893 [ # # ]: 4306 : i += rc;
3894 [ + + ]: 4306 : if (_array_size >= rc) {
3895 [ # # ]: 265 : _array_size -= rc;
3896 : 9 : } else {
3897 : 4041 : _array_size = 0;
3898 : : }
3899 [ + + ]: 6156 : } else if (rc < 0) {
3900 : 0 : return rc;
3901 : : }
3902 : 77 : }
3903 : :
3904 : 10379 : return i;
3905 : 71 : }
3906 : :
3907 : : static const char *
3908 : 745 : nvme_ctrlr_get_state_str(struct nvme_ctrlr *nvme_ctrlr)
3909 : : {
3910 [ - + # # ]: 745 : if (nvme_ctrlr->destruct) {
3911 : 0 : return "deleting";
3912 [ - + # # : 745 : } else if (spdk_nvme_ctrlr_is_failed(nvme_ctrlr->ctrlr)) {
# # ]
3913 : 0 : return "failed";
3914 [ + + # # ]: 745 : } else if (nvme_ctrlr->resetting) {
3915 : 12 : return "resetting";
3916 [ + + # # ]: 733 : } else if (nvme_ctrlr->reconnect_is_delayed > 0) {
3917 : 5 : return "reconnect_is_delayed";
3918 [ + + # # ]: 728 : } else if (nvme_ctrlr->disabled) {
3919 : 0 : return "disabled";
3920 : : } else {
3921 : 728 : return "enabled";
3922 : : }
3923 : 1 : }
3924 : :
3925 : : void
3926 : 745 : nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrlr)
3927 : 744 : {
3928 : : struct spdk_nvme_transport_id *trid;
3929 : : const struct spdk_nvme_ctrlr_opts *opts;
3930 : : const struct spdk_nvme_ctrlr_data *cdata;
3931 : : struct nvme_path_id *path_id;
3932 : : int32_t numa_id;
3933 : :
3934 : 745 : spdk_json_write_object_begin(w);
3935 : :
3936 : 745 : spdk_json_write_named_string(w, "state", nvme_ctrlr_get_state_str(nvme_ctrlr));
3937 : :
3938 : : #ifdef SPDK_CONFIG_NVME_CUSE
3939 : 744 : size_t cuse_name_size = 128;
3940 [ - + ]: 744 : char cuse_name[cuse_name_size];
3941 : :
3942 [ # # # # ]: 744 : int rc = spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr, cuse_name, &cuse_name_size);
3943 [ + + ]: 744 : if (rc == 0) {
3944 : 3 : spdk_json_write_named_string(w, "cuse_device", cuse_name);
3945 : 0 : }
3946 : : #endif
3947 [ # # # # : 745 : trid = &nvme_ctrlr->active_path_id->trid;
# # ]
3948 : 745 : spdk_json_write_named_object_begin(w, "trid");
3949 : 745 : nvme_bdev_dump_trid_json(trid, w);
3950 : 745 : spdk_json_write_object_end(w);
3951 : :
3952 [ # # # # : 745 : path_id = TAILQ_NEXT(nvme_ctrlr->active_path_id, link);
# # # # #
# ]
3953 [ + + ]: 745 : if (path_id != NULL) {
3954 : 12 : spdk_json_write_named_array_begin(w, "alternate_trids");
3955 : 0 : do {
3956 [ # # ]: 16 : trid = &path_id->trid;
3957 : 16 : spdk_json_write_object_begin(w);
3958 : 16 : nvme_bdev_dump_trid_json(trid, w);
3959 : 16 : spdk_json_write_object_end(w);
3960 : :
3961 [ # # # # : 16 : path_id = TAILQ_NEXT(path_id, link);
# # ]
3962 [ + + ]: 16 : } while (path_id != NULL);
3963 : 12 : spdk_json_write_array_end(w);
3964 : 0 : }
3965 : :
3966 [ # # # # ]: 745 : cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
3967 [ # # # # ]: 745 : spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid);
3968 : :
3969 [ # # # # ]: 745 : opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr);
3970 : 745 : spdk_json_write_named_object_begin(w, "host");
3971 [ # # ]: 745 : spdk_json_write_named_string(w, "nqn", opts->hostnqn);
3972 [ # # ]: 745 : spdk_json_write_named_string(w, "addr", opts->src_addr);
3973 [ # # ]: 745 : spdk_json_write_named_string(w, "svcid", opts->src_svcid);
3974 : 745 : spdk_json_write_object_end(w);
3975 : :
3976 [ # # # # ]: 745 : numa_id = spdk_nvme_ctrlr_get_numa_id(nvme_ctrlr->ctrlr);
3977 [ + + ]: 745 : if (numa_id != SPDK_ENV_NUMA_ID_ANY) {
3978 : 358 : spdk_json_write_named_uint32(w, "numa_id", numa_id);
3979 : 0 : }
3980 : 745 : spdk_json_write_object_end(w);
3981 : 745 : }
3982 : :
3983 : : static void
3984 : 1118 : nvme_namespace_info_json(struct spdk_json_write_ctx *w,
3985 : : struct nvme_ns *nvme_ns)
3986 : 1116 : {
3987 : : struct spdk_nvme_ns *ns;
3988 : : struct spdk_nvme_ctrlr *ctrlr;
3989 : : const struct spdk_nvme_ctrlr_data *cdata;
3990 : : const struct spdk_nvme_transport_id *trid;
3991 : : union spdk_nvme_vs_register vs;
3992 : : const struct spdk_nvme_ns_data *nsdata;
3993 : 894 : char buf[128];
3994 : :
3995 [ # # # # ]: 1118 : ns = nvme_ns->ns;
3996 [ + + ]: 1118 : if (ns == NULL) {
3997 : 0 : return;
3998 : : }
3999 : :
4000 : 1118 : ctrlr = spdk_nvme_ns_get_ctrlr(ns);
4001 : :
4002 : 1118 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
4003 : 1118 : trid = spdk_nvme_ctrlr_get_transport_id(ctrlr);
4004 : 1118 : vs = spdk_nvme_ctrlr_get_regs_vs(ctrlr);
4005 : :
4006 : 1118 : spdk_json_write_object_begin(w);
4007 : :
4008 [ + + # # : 1118 : if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
# # ]
4009 [ # # ]: 892 : spdk_json_write_named_string(w, "pci_address", trid->traddr);
4010 : 2 : }
4011 : :
4012 : 1118 : spdk_json_write_named_object_begin(w, "trid");
4013 : :
4014 : 1118 : nvme_bdev_dump_trid_json(trid, w);
4015 : :
4016 : 1118 : spdk_json_write_object_end(w);
4017 : :
4018 : : #ifdef SPDK_CONFIG_NVME_CUSE
4019 : 1116 : size_t cuse_name_size = 128;
4020 [ - + ]: 1116 : char cuse_name[cuse_name_size];
4021 : :
4022 : 1116 : int rc = spdk_nvme_cuse_get_ns_name(ctrlr, spdk_nvme_ns_get_id(ns),
4023 : : cuse_name, &cuse_name_size);
4024 [ + + ]: 1116 : if (rc == 0) {
4025 : 4 : spdk_json_write_named_string(w, "cuse_device", cuse_name);
4026 : 0 : }
4027 : : #endif
4028 : :
4029 : 1118 : spdk_json_write_named_object_begin(w, "ctrlr_data");
4030 : :
4031 [ # # # # ]: 1118 : spdk_json_write_named_uint16(w, "cntlid", cdata->cntlid);
4032 : :
4033 [ # # # # ]: 1118 : spdk_json_write_named_string_fmt(w, "vendor_id", "0x%04x", cdata->vid);
4034 : :
4035 [ - + ]: 1118 : snprintf(buf, sizeof(cdata->mn) + 1, "%s", cdata->mn);
4036 : 1118 : spdk_str_trim(buf);
4037 : 1118 : spdk_json_write_named_string(w, "model_number", buf);
4038 : :
4039 [ - + ]: 1118 : snprintf(buf, sizeof(cdata->sn) + 1, "%s", cdata->sn);
4040 : 1118 : spdk_str_trim(buf);
4041 : 1118 : spdk_json_write_named_string(w, "serial_number", buf);
4042 : :
4043 [ - + ]: 1118 : snprintf(buf, sizeof(cdata->fr) + 1, "%s", cdata->fr);
4044 : 1118 : spdk_str_trim(buf);
4045 : 1118 : spdk_json_write_named_string(w, "firmware_revision", buf);
4046 : :
4047 [ + + # # : 1118 : if (cdata->subnqn[0] != '\0') {
# # # # #
# ]
4048 [ # # ]: 1083 : spdk_json_write_named_string(w, "subnqn", cdata->subnqn);
4049 : 2 : }
4050 : :
4051 : 1118 : spdk_json_write_named_object_begin(w, "oacs");
4052 : :
4053 [ # # # # ]: 1118 : spdk_json_write_named_uint32(w, "security", cdata->oacs.security);
4054 [ # # # # ]: 1118 : spdk_json_write_named_uint32(w, "format", cdata->oacs.format);
4055 [ # # # # ]: 1118 : spdk_json_write_named_uint32(w, "firmware", cdata->oacs.firmware);
4056 [ # # # # ]: 1118 : spdk_json_write_named_uint32(w, "ns_manage", cdata->oacs.ns_manage);
4057 : :
4058 : 1118 : spdk_json_write_object_end(w);
4059 : :
4060 [ # # # # ]: 1118 : spdk_json_write_named_bool(w, "multi_ctrlr", cdata->cmic.multi_ctrlr);
4061 [ # # # # ]: 1118 : spdk_json_write_named_bool(w, "ana_reporting", cdata->cmic.ana_reporting);
4062 : :
4063 : 1118 : spdk_json_write_object_end(w);
4064 : :
4065 : 1118 : spdk_json_write_named_object_begin(w, "vs");
4066 : :
4067 : 1118 : spdk_json_write_name(w, "nvme_version");
4068 [ + + ]: 1118 : if (vs.bits.ter) {
4069 : 0 : spdk_json_write_string_fmt(w, "%u.%u.%u", vs.bits.mjr, vs.bits.mnr, vs.bits.ter);
4070 : 0 : } else {
4071 : 1118 : spdk_json_write_string_fmt(w, "%u.%u", vs.bits.mjr, vs.bits.mnr);
4072 : : }
4073 : :
4074 : 1118 : spdk_json_write_object_end(w);
4075 : :
4076 : 1118 : nsdata = spdk_nvme_ns_get_data(ns);
4077 : :
4078 : 1118 : spdk_json_write_named_object_begin(w, "ns_data");
4079 : :
4080 : 1118 : spdk_json_write_named_uint32(w, "id", spdk_nvme_ns_get_id(ns));
4081 : :
4082 [ + + # # : 1118 : if (cdata->cmic.ana_reporting) {
# # ]
4083 : 0 : spdk_json_write_named_string(w, "ana_state",
4084 [ # # # # ]: 0 : _nvme_ana_state_str(nvme_ns->ana_state));
4085 : 0 : }
4086 : :
4087 [ # # # # ]: 1118 : spdk_json_write_named_bool(w, "can_share", nsdata->nmic.can_share);
4088 : :
4089 : 1118 : spdk_json_write_object_end(w);
4090 : :
4091 [ + + # # : 1118 : if (cdata->oacs.security) {
# # ]
4092 : 709 : spdk_json_write_named_object_begin(w, "security");
4093 : :
4094 [ - + # # : 709 : spdk_json_write_named_bool(w, "opal", nvme_ns->bdev->opal);
# # # # #
# ]
4095 : :
4096 : 709 : spdk_json_write_object_end(w);
4097 : 0 : }
4098 : :
4099 : 1118 : spdk_json_write_object_end(w);
4100 : 2 : }
4101 : :
4102 : : static const char *
4103 : 1110 : nvme_bdev_get_mp_policy_str(struct nvme_bdev *nbdev)
4104 : : {
4105 [ + - + # : 1110 : switch (nbdev->mp_policy) {
# # # ]
4106 : 1108 : case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE:
4107 : 1110 : return "active_passive";
4108 : 0 : case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE:
4109 : 0 : return "active_active";
4110 : 0 : default:
4111 [ # # ]: 0 : assert(false);
4112 : : return "invalid";
4113 : : }
4114 : 2 : }
4115 : :
4116 : : static const char *
4117 : 0 : nvme_bdev_get_mp_selector_str(struct nvme_bdev *nbdev)
4118 : : {
4119 [ # # # # : 0 : switch (nbdev->mp_selector) {
# # # ]
4120 : 0 : case BDEV_NVME_MP_SELECTOR_ROUND_ROBIN:
4121 : 0 : return "round_robin";
4122 : 0 : case BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH:
4123 : 0 : return "queue_depth";
4124 : 0 : default:
4125 [ # # ]: 0 : assert(false);
4126 : : return "invalid";
4127 : : }
4128 : 0 : }
4129 : :
4130 : : static int
4131 : 1110 : bdev_nvme_dump_info_json(void *ctx, struct spdk_json_write_ctx *w)
4132 : : {
4133 : 1110 : struct nvme_bdev *nbdev = ctx;
4134 : : struct nvme_ns *nvme_ns;
4135 : :
4136 [ - + # # ]: 1110 : pthread_mutex_lock(&nbdev->mutex);
4137 : 1110 : spdk_json_write_named_array_begin(w, "nvme");
4138 [ + + # # : 2228 : TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
# # # # #
# # # #
# ]
4139 : 1118 : nvme_namespace_info_json(w, nvme_ns);
4140 : 2 : }
4141 : 1110 : spdk_json_write_array_end(w);
4142 : 1110 : spdk_json_write_named_string(w, "mp_policy", nvme_bdev_get_mp_policy_str(nbdev));
4143 [ + + # # : 1110 : if (nbdev->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) {
# # ]
4144 : 0 : spdk_json_write_named_string(w, "selector", nvme_bdev_get_mp_selector_str(nbdev));
4145 [ # # # # : 0 : if (nbdev->mp_selector == BDEV_NVME_MP_SELECTOR_ROUND_ROBIN) {
# # ]
4146 [ # # # # ]: 0 : spdk_json_write_named_uint32(w, "rr_min_io", nbdev->rr_min_io);
4147 : 0 : }
4148 : 0 : }
4149 [ - + # # ]: 1110 : pthread_mutex_unlock(&nbdev->mutex);
4150 : :
4151 : 1110 : return 0;
4152 : : }
4153 : :
4154 : : static void
4155 : 169 : bdev_nvme_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
4156 : : {
4157 : : /* No config per bdev needed */
4158 : 169 : }
4159 : :
4160 : : static uint64_t
4161 : 0 : bdev_nvme_get_spin_time(struct spdk_io_channel *ch)
4162 : : {
4163 : 0 : struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
4164 : : struct nvme_io_path *io_path;
4165 : : struct nvme_poll_group *group;
4166 : 0 : uint64_t spin_time = 0;
4167 : :
4168 [ # # # # : 0 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
4169 [ # # # # : 0 : group = io_path->qpair->group;
# # # # ]
4170 : :
4171 [ # # # # : 0 : if (!group || !group->collect_spin_stat) {
# # # # #
# ]
4172 : 0 : continue;
4173 : : }
4174 : :
4175 [ # # # # : 0 : if (group->end_ticks != 0) {
# # ]
4176 [ # # # # : 0 : group->spin_ticks += (group->end_ticks - group->start_ticks);
# # # # #
# # # ]
4177 [ # # # # ]: 0 : group->end_ticks = 0;
4178 : 0 : }
4179 : :
4180 [ # # # # ]: 0 : spin_time += group->spin_ticks;
4181 [ # # # # ]: 0 : group->start_ticks = 0;
4182 [ # # # # ]: 0 : group->spin_ticks = 0;
4183 : 0 : }
4184 : :
4185 [ # # ]: 0 : return (spin_time * 1000000ULL) / spdk_get_ticks_hz();
4186 : : }
4187 : :
4188 : : static void
4189 : 0 : bdev_nvme_reset_device_stat(void *ctx)
4190 : : {
4191 : 0 : struct nvme_bdev *nbdev = ctx;
4192 : :
4193 [ # # # # : 0 : if (nbdev->err_stat != NULL) {
# # ]
4194 [ # # # # : 0 : memset(nbdev->err_stat, 0, sizeof(struct nvme_error_stat));
# # ]
4195 : 0 : }
4196 : 0 : }
4197 : :
4198 : : /* JSON string should be lowercases and underscore delimited string. */
4199 : : static void
4200 : 24 : bdev_nvme_format_nvme_status(char *dst, const char *src)
4201 : : {
4202 : 0 : char tmp[256];
4203 : :
4204 : 24 : spdk_strcpy_replace(dst, 256, src, " - ", "_");
4205 : 24 : spdk_strcpy_replace(tmp, 256, dst, "-", "_");
4206 : 24 : spdk_strcpy_replace(dst, 256, tmp, " ", "_");
4207 : 24 : spdk_strlwr(dst);
4208 : 24 : }
4209 : :
4210 : : static void
4211 : 31 : bdev_nvme_dump_device_stat_json(void *ctx, struct spdk_json_write_ctx *w)
4212 : : {
4213 : 31 : struct nvme_bdev *nbdev = ctx;
4214 : 31 : struct spdk_nvme_status status = {};
4215 : : uint16_t sct, sc;
4216 : 0 : char status_json[256];
4217 : : const char *status_str;
4218 : :
4219 [ + + # # : 31 : if (nbdev->err_stat == NULL) {
# # ]
4220 : 19 : return;
4221 : : }
4222 : :
4223 : 12 : spdk_json_write_named_object_begin(w, "nvme_error");
4224 : :
4225 : 12 : spdk_json_write_named_object_begin(w, "status_type");
4226 [ + + ]: 108 : for (sct = 0; sct < 8; sct++) {
4227 [ + + # # : 96 : if (nbdev->err_stat->status_type[sct] == 0) {
# # # # #
# # # #
# ]
4228 : 84 : continue;
4229 : : }
4230 : 12 : status.sct = sct;
4231 : :
4232 : 12 : status_str = spdk_nvme_cpl_get_status_type_string(&status);
4233 [ - + # # ]: 12 : assert(status_str != NULL);
4234 : 12 : bdev_nvme_format_nvme_status(status_json, status_str);
4235 : :
4236 [ # # # # : 12 : spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status_type[sct]);
# # # # #
# # # ]
4237 : 0 : }
4238 : 12 : spdk_json_write_object_end(w);
4239 : :
4240 : 12 : spdk_json_write_named_object_begin(w, "status_code");
4241 [ + + ]: 60 : for (sct = 0; sct < 4; sct++) {
4242 : 48 : status.sct = sct;
4243 [ + + ]: 12336 : for (sc = 0; sc < 256; sc++) {
4244 [ + + # # : 12288 : if (nbdev->err_stat->status[sct][sc] == 0) {
# # # # #
# # # # #
# # ]
4245 : 12276 : continue;
4246 : : }
4247 : 12 : status.sc = sc;
4248 : :
4249 : 12 : status_str = spdk_nvme_cpl_get_status_string(&status);
4250 [ - + # # ]: 12 : assert(status_str != NULL);
4251 : 12 : bdev_nvme_format_nvme_status(status_json, status_str);
4252 : :
4253 [ # # # # : 12 : spdk_json_write_named_uint32(w, status_json, nbdev->err_stat->status[sct][sc]);
# # # # #
# # # #
# ]
4254 : 0 : }
4255 : 0 : }
4256 : 12 : spdk_json_write_object_end(w);
4257 : :
4258 : 12 : spdk_json_write_object_end(w);
4259 : 0 : }
4260 : :
4261 : : static bool
4262 : 161595 : bdev_nvme_accel_sequence_supported(void *ctx, enum spdk_bdev_io_type type)
4263 : : {
4264 : 161595 : struct nvme_bdev *nbdev = ctx;
4265 : : struct nvme_ns *nvme_ns;
4266 : : struct spdk_nvme_ctrlr *ctrlr;
4267 : :
4268 [ + + + + : 161595 : if (!g_opts.allow_accel_sequence) {
- + ]
4269 : 161511 : return false;
4270 : : }
4271 : :
4272 [ + + ]: 84 : switch (type) {
4273 : 8 : case SPDK_BDEV_IO_TYPE_WRITE:
4274 : : case SPDK_BDEV_IO_TYPE_READ:
4275 : 8 : break;
4276 : 76 : default:
4277 : 76 : return false;
4278 : : }
4279 : :
4280 [ # # # # : 8 : nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
# # ]
4281 [ - + # # ]: 8 : assert(nvme_ns != NULL);
4282 : :
4283 [ # # # # : 8 : ctrlr = nvme_ns->ctrlr->ctrlr;
# # # # ]
4284 [ - + # # ]: 8 : assert(ctrlr != NULL);
4285 : :
4286 : 8 : return spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ACCEL_SEQUENCE_SUPPORTED;
4287 : 1281 : }
4288 : :
4289 : : static const struct spdk_bdev_fn_table nvmelib_fn_table = {
4290 : : .destruct = bdev_nvme_destruct,
4291 : : .submit_request = bdev_nvme_submit_request,
4292 : : .io_type_supported = bdev_nvme_io_type_supported,
4293 : : .get_io_channel = bdev_nvme_get_io_channel,
4294 : : .dump_info_json = bdev_nvme_dump_info_json,
4295 : : .write_config_json = bdev_nvme_write_config_json,
4296 : : .get_spin_time = bdev_nvme_get_spin_time,
4297 : : .get_module_ctx = bdev_nvme_get_module_ctx,
4298 : : .get_memory_domains = bdev_nvme_get_memory_domains,
4299 : : .accel_sequence_supported = bdev_nvme_accel_sequence_supported,
4300 : : .reset_device_stat = bdev_nvme_reset_device_stat,
4301 : : .dump_device_stat_json = bdev_nvme_dump_device_stat_json,
4302 : : };
4303 : :
4304 : : typedef int (*bdev_nvme_parse_ana_log_page_cb)(
4305 : : const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg);
4306 : :
4307 : : static int
4308 : 596 : bdev_nvme_parse_ana_log_page(struct nvme_ctrlr *nvme_ctrlr,
4309 : : bdev_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg)
4310 : : {
4311 : : struct spdk_nvme_ana_group_descriptor *copied_desc;
4312 : : uint8_t *orig_desc;
4313 : : uint32_t i, desc_size, copy_len;
4314 : 596 : int rc = 0;
4315 : :
4316 [ + + # # : 596 : if (nvme_ctrlr->ana_log_page == NULL) {
# # ]
4317 : 0 : return -EINVAL;
4318 : : }
4319 : :
4320 [ # # # # ]: 596 : copied_desc = nvme_ctrlr->copied_ana_desc;
4321 : :
4322 [ # # # # : 596 : orig_desc = (uint8_t *)nvme_ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page);
# # ]
4323 [ # # # # ]: 596 : copy_len = nvme_ctrlr->max_ana_log_page_size - sizeof(struct spdk_nvme_ana_page);
4324 : :
4325 [ + + # # : 820 : for (i = 0; i < nvme_ctrlr->ana_log_page->num_ana_group_desc; i++) {
# # # # #
# ]
4326 [ - + - + ]: 696 : memcpy(copied_desc, orig_desc, copy_len);
4327 : :
4328 [ # # # # ]: 696 : rc = cb_fn(copied_desc, cb_arg);
4329 [ + + ]: 696 : if (rc != 0) {
4330 : 472 : break;
4331 : : }
4332 : :
4333 : 224 : desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) +
4334 [ # # # # ]: 224 : copied_desc->num_of_nsid * sizeof(uint32_t);
4335 [ # # ]: 224 : orig_desc += desc_size;
4336 : 224 : copy_len -= desc_size;
4337 : 30 : }
4338 : :
4339 : 596 : return rc;
4340 : 42 : }
4341 : :
4342 : : static int
4343 : 22 : nvme_ns_ana_transition_timedout(void *ctx)
4344 : : {
4345 : 22 : struct nvme_ns *nvme_ns = ctx;
4346 : :
4347 [ # # ]: 22 : spdk_poller_unregister(&nvme_ns->anatt_timer);
4348 [ # # # # ]: 22 : nvme_ns->ana_transition_timedout = true;
4349 : :
4350 : 22 : return SPDK_POLLER_BUSY;
4351 : : }
4352 : :
4353 : : static void
4354 : 612 : _nvme_ns_set_ana_state(struct nvme_ns *nvme_ns,
4355 : : const struct spdk_nvme_ana_group_descriptor *desc)
4356 : : {
4357 : : const struct spdk_nvme_ctrlr_data *cdata;
4358 : :
4359 [ # # # # : 612 : nvme_ns->ana_group_id = desc->ana_group_id;
# # # # ]
4360 [ # # # # : 612 : nvme_ns->ana_state = desc->ana_state;
# # ]
4361 [ # # # # ]: 612 : nvme_ns->ana_state_updating = false;
4362 : :
4363 [ + + + # : 612 : switch (nvme_ns->ana_state) {
# # # ]
4364 : 513 : case SPDK_NVME_ANA_OPTIMIZED_STATE:
4365 : : case SPDK_NVME_ANA_NON_OPTIMIZED_STATE:
4366 [ # # # # ]: 552 : nvme_ns->ana_transition_timedout = false;
4367 [ # # ]: 552 : spdk_poller_unregister(&nvme_ns->anatt_timer);
4368 : 552 : break;
4369 : :
4370 : 50 : case SPDK_NVME_ANA_INACCESSIBLE_STATE:
4371 : : case SPDK_NVME_ANA_CHANGE_STATE:
4372 [ + + # # : 56 : if (nvme_ns->anatt_timer != NULL) {
# # ]
4373 : 18 : break;
4374 : : }
4375 : :
4376 [ # # # # : 38 : cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr);
# # # # ]
4377 [ # # # # : 38 : nvme_ns->anatt_timer = SPDK_POLLER_REGISTER(nvme_ns_ana_transition_timedout,
# # # # ]
4378 : : nvme_ns,
4379 : : cdata->anatt * SPDK_SEC_TO_USEC);
4380 : 38 : break;
4381 : 3 : default:
4382 : 4 : break;
4383 : : }
4384 : 612 : }
4385 : :
4386 : : static int
4387 : 564 : nvme_ns_set_ana_state(const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg)
4388 : : {
4389 : 564 : struct nvme_ns *nvme_ns = cb_arg;
4390 : : uint32_t i;
4391 : :
4392 [ + + # # : 564 : assert(nvme_ns->ns != NULL);
# # # # ]
4393 : :
4394 [ + + # # : 652 : for (i = 0; i < desc->num_of_nsid; i++) {
# # ]
4395 [ + + # # : 560 : if (desc->nsid[i] != spdk_nvme_ns_get_id(nvme_ns->ns)) {
# # # # #
# # # ]
4396 : 88 : continue;
4397 : : }
4398 : :
4399 : 472 : _nvme_ns_set_ana_state(nvme_ns, desc);
4400 : 472 : return 1;
4401 : : }
4402 : :
4403 : 92 : return 0;
4404 : 60 : }
4405 : :
4406 : : static int
4407 : 20 : nvme_generate_uuid(const char *sn, uint32_t nsid, struct spdk_uuid *uuid)
4408 : : {
4409 : 20 : int rc = 0;
4410 : 15 : struct spdk_uuid new_uuid, namespace_uuid;
4411 : 20 : char merged_str[SPDK_NVME_CTRLR_SN_LEN + NSID_STR_LEN + 1] = {'\0'};
4412 : : /* This namespace UUID was generated using uuid_generate() method. */
4413 : 20 : const char *namespace_str = {"edaed2de-24bc-4b07-b559-f47ecbe730fd"};
4414 : : int size;
4415 : :
4416 [ + + - + : 20 : assert(strlen(sn) <= SPDK_NVME_CTRLR_SN_LEN);
# # ]
4417 : :
4418 : 20 : spdk_uuid_set_null(&new_uuid);
4419 : 20 : spdk_uuid_set_null(&namespace_uuid);
4420 : :
4421 : 20 : size = snprintf(merged_str, sizeof(merged_str), "%s%"PRIu32, sn, nsid);
4422 [ + - - + ]: 20 : if (size <= 0 || (unsigned long)size >= sizeof(merged_str)) {
4423 : 0 : return -EINVAL;
4424 : : }
4425 : :
4426 : 20 : spdk_uuid_parse(&namespace_uuid, namespace_str);
4427 : :
4428 : 20 : rc = spdk_uuid_generate_sha1(&new_uuid, &namespace_uuid, merged_str, size);
4429 [ + + ]: 20 : if (rc == 0) {
4430 [ # # # # ]: 20 : memcpy(uuid, &new_uuid, sizeof(struct spdk_uuid));
4431 : 5 : }
4432 : :
4433 : 20 : return rc;
4434 : 5 : }
4435 : :
4436 : : static int
4437 : 1481 : nbdev_create(struct spdk_bdev *disk, const char *base_name,
4438 : : struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns,
4439 : : struct spdk_bdev_nvme_ctrlr_opts *bdev_opts, void *ctx)
4440 : : {
4441 : : const struct spdk_uuid *uuid;
4442 : : const uint8_t *nguid;
4443 : : const struct spdk_nvme_ctrlr_data *cdata;
4444 : : const struct spdk_nvme_ns_data *nsdata;
4445 : : const struct spdk_nvme_ctrlr_opts *opts;
4446 : : enum spdk_nvme_csi csi;
4447 : : uint32_t atomic_bs, phys_bs, bs;
4448 : 1481 : char sn_tmp[SPDK_NVME_CTRLR_SN_LEN + 1] = {'\0'};
4449 : : int rc;
4450 : :
4451 : 1481 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
4452 : 1481 : csi = spdk_nvme_ns_get_csi(ns);
4453 : 1481 : opts = spdk_nvme_ctrlr_get_opts(ctrlr);
4454 : :
4455 [ + + - ]: 1481 : switch (csi) {
4456 : 1429 : case SPDK_NVME_CSI_NVM:
4457 [ + - + - ]: 1479 : disk->product_name = "NVMe disk";
4458 : 1479 : break;
4459 : 2 : case SPDK_NVME_CSI_ZNS:
4460 [ # # # # ]: 2 : disk->product_name = "NVMe ZNS disk";
4461 [ # # # # ]: 2 : disk->zoned = true;
4462 [ # # # # ]: 2 : disk->zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
4463 [ # # # # : 2 : disk->max_zone_append_size = spdk_nvme_zns_ctrlr_get_max_zone_append_size(ctrlr) /
# # # # ]
4464 : 2 : spdk_nvme_ns_get_extended_sector_size(ns);
4465 [ # # # # ]: 2 : disk->max_open_zones = spdk_nvme_zns_ns_get_max_open_zones(ns);
4466 [ # # # # ]: 2 : disk->max_active_zones = spdk_nvme_zns_ns_get_max_active_zones(ns);
4467 : 2 : break;
4468 : 0 : default:
4469 [ # # # # : 0 : if (bdev_opts->allow_unrecognized_csi) {
# # # # ]
4470 [ # # # # ]: 0 : disk->product_name = "NVMe Passthrough disk";
4471 : 0 : break;
4472 : : }
4473 : 0 : SPDK_ERRLOG("unsupported CSI: %u\n", csi);
4474 : 0 : return -ENOTSUP;
4475 : : }
4476 : :
4477 : 1481 : nguid = spdk_nvme_ns_get_nguid(ns);
4478 [ + + ]: 1481 : if (!nguid) {
4479 : 1097 : uuid = spdk_nvme_ns_get_uuid(ns);
4480 [ + + ]: 1097 : if (uuid) {
4481 [ # # ]: 360 : disk->uuid = *uuid;
4482 [ + + + + : 749 : } else if (g_opts.generate_uuids) {
+ - ]
4483 [ # # ]: 0 : spdk_strcpy_pad(sn_tmp, cdata->sn, SPDK_NVME_CTRLR_SN_LEN, '\0');
4484 [ # # ]: 0 : rc = nvme_generate_uuid(sn_tmp, spdk_nvme_ns_get_id(ns), &disk->uuid);
4485 [ # # ]: 0 : if (rc < 0) {
4486 [ # # ]: 0 : SPDK_ERRLOG("UUID generation failed (%s)\n", spdk_strerror(-rc));
4487 : 0 : return rc;
4488 : : }
4489 : 0 : }
4490 : 50 : } else {
4491 [ # # # # : 384 : memcpy(&disk->uuid, nguid, sizeof(disk->uuid));
# # ]
4492 : : }
4493 : :
4494 [ + - + - ]: 1481 : disk->name = spdk_sprintf_alloc("%sn%d", base_name, spdk_nvme_ns_get_id(ns));
4495 [ + + + - : 1481 : if (!disk->name) {
- + ]
4496 : 0 : return -ENOMEM;
4497 : : }
4498 : :
4499 [ + - + - ]: 1481 : disk->write_cache = 0;
4500 [ + + + - : 1481 : if (cdata->vwc.present) {
+ - ]
4501 : : /* Enable if the Volatile Write Cache exists */
4502 [ # # # # ]: 1235 : disk->write_cache = 1;
4503 : 10 : }
4504 [ + + + - : 1481 : if (cdata->oncs.write_zeroes) {
+ - ]
4505 [ # # # # ]: 1245 : disk->max_write_zeroes = UINT16_MAX + 1;
4506 : 10 : }
4507 [ + - + - ]: 1481 : disk->blocklen = spdk_nvme_ns_get_extended_sector_size(ns);
4508 [ + - + - ]: 1481 : disk->blockcnt = spdk_nvme_ns_get_num_sectors(ns);
4509 [ + - + - ]: 1481 : disk->max_segment_size = spdk_nvme_ctrlr_get_max_xfer_size(ctrlr);
4510 [ + - + - : 1481 : disk->ctratt.raw = cdata->ctratt.raw;
+ - + - +
- + - ]
4511 [ + - + - ]: 1481 : disk->nsid = spdk_nvme_ns_get_id(ns);
4512 : : /* NVMe driver will split one request into multiple requests
4513 : : * based on MDTS and stripe boundary, the bdev layer will use
4514 : : * max_segment_size and max_num_segments to split one big IO
4515 : : * into multiple requests, then small request can't run out
4516 : : * of NVMe internal requests data structure.
4517 : : */
4518 [ + - + + : 1481 : if (opts && opts->io_queue_requests) {
+ - - + ]
4519 [ + - + - : 1325 : disk->max_num_segments = opts->io_queue_requests / 2;
+ - + - +
- ]
4520 : 11 : }
4521 [ + + ]: 1481 : if (spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_SGL_SUPPORTED) {
4522 : : /* The nvme driver will try to split I/O that have too many
4523 : : * SGEs, but it doesn't work if that last SGE doesn't end on
4524 : : * an aggregate total that is block aligned. The bdev layer has
4525 : : * a more robust splitting framework, so use that instead for
4526 : : * this case. (See issue #3269.)
4527 : : */
4528 : 1235 : uint16_t max_sges = spdk_nvme_ctrlr_get_max_sges(ctrlr);
4529 : :
4530 [ + + # # : 1235 : if (disk->max_num_segments == 0) {
# # ]
4531 [ # # # # ]: 0 : disk->max_num_segments = max_sges;
4532 : 0 : } else {
4533 [ - + # # : 1235 : disk->max_num_segments = spdk_min(disk->max_num_segments, max_sges);
# # # # #
# # # #
# ]
4534 : : }
4535 : 10 : }
4536 [ + - + - ]: 1481 : disk->optimal_io_boundary = spdk_nvme_ns_get_optimal_io_boundary(ns);
4537 : :
4538 : 1481 : nsdata = spdk_nvme_ns_get_data(ns);
4539 : 1481 : bs = spdk_nvme_ns_get_sector_size(ns);
4540 : 1481 : atomic_bs = bs;
4541 : 1481 : phys_bs = bs;
4542 [ + + + - : 1481 : if (nsdata->nabo == 0) {
- + ]
4543 [ + + + + : 1481 : if (nsdata->nsfeat.ns_atomic_write_unit && nsdata->nawupf) {
- + # # #
# # # ]
4544 [ # # # # : 10 : atomic_bs = bs * (1 + nsdata->nawupf);
# # ]
4545 : 0 : } else {
4546 [ + - + - : 1471 : atomic_bs = bs * (1 + cdata->awupf);
+ - ]
4547 : : }
4548 : 50 : }
4549 [ + + + - : 1481 : if (nsdata->nsfeat.optperf) {
+ - ]
4550 [ # # # # : 1245 : phys_bs = bs * (1 + nsdata->npwg);
# # ]
4551 : 10 : }
4552 [ - + + - : 1481 : disk->phys_blocklen = spdk_min(phys_bs, atomic_bs);
+ - ]
4553 : :
4554 [ + - + - ]: 1481 : disk->md_len = spdk_nvme_ns_get_md_size(ns);
4555 [ + + + - : 1481 : if (disk->md_len != 0) {
- + ]
4556 [ # # # # : 47 : disk->md_interleave = nsdata->flbas.extended;
# # # # ]
4557 [ # # # # ]: 47 : disk->dif_type = (enum spdk_dif_type)spdk_nvme_ns_get_pi_type(ns);
4558 [ - + # # : 47 : if (disk->dif_type != SPDK_DIF_DISABLE) {
# # ]
4559 [ # # # # : 0 : disk->dif_is_head_of_md = nsdata->dps.md_start;
# # # # ]
4560 [ # # # # : 0 : disk->dif_check_flags = bdev_opts->prchk_flags;
# # # # ]
4561 [ # # # # ]: 0 : disk->dif_pi_format = (enum spdk_dif_pi_format)spdk_nvme_ns_get_pi_format(ns);
4562 : 0 : }
4563 : 0 : }
4564 : :
4565 [ + + ]: 1481 : if (!(spdk_nvme_ctrlr_get_flags(ctrlr) &
4566 : : SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED)) {
4567 [ + - + - ]: 1107 : disk->acwu = 0;
4568 [ + - # # : 424 : } else if (nsdata->nsfeat.ns_atomic_write_unit) {
# # ]
4569 [ # # # # : 374 : disk->acwu = nsdata->nacwu + 1; /* 0-based */
# # # # #
# ]
4570 : 0 : } else {
4571 [ # # # # : 0 : disk->acwu = cdata->acwu + 1; /* 0-based */
# # # # #
# ]
4572 : : }
4573 : :
4574 [ + + + - : 1481 : if (cdata->oncs.copy) {
+ - ]
4575 : : /* For now bdev interface allows only single segment copy */
4576 [ # # # # : 923 : disk->max_copy = nsdata->mssrl;
# # # # ]
4577 : 10 : }
4578 : :
4579 [ + - + - ]: 1481 : disk->ctxt = ctx;
4580 [ + - + - ]: 1481 : disk->fn_table = &nvmelib_fn_table;
4581 [ + - + - ]: 1481 : disk->module = &nvme_if;
4582 : :
4583 [ + - + - ]: 1481 : disk->numa.id_valid = 1;
4584 [ + - + - ]: 1481 : disk->numa.id = spdk_nvme_ctrlr_get_numa_id(ctrlr);
4585 : :
4586 : 1481 : return 0;
4587 : 50 : }
4588 : :
4589 : : static struct nvme_bdev *
4590 : 1481 : nvme_bdev_alloc(void)
4591 : : {
4592 : : struct nvme_bdev *nbdev;
4593 : : int rc;
4594 : :
4595 : 1481 : nbdev = calloc(1, sizeof(*nbdev));
4596 [ + + ]: 1481 : if (!nbdev) {
4597 : 0 : SPDK_ERRLOG("nbdev calloc() failed\n");
4598 : 0 : return NULL;
4599 : : }
4600 : :
4601 [ + + + + : 1481 : if (g_opts.nvme_error_stat) {
+ - ]
4602 [ # # # # ]: 12 : nbdev->err_stat = calloc(1, sizeof(struct nvme_error_stat));
4603 [ - + # # : 12 : if (!nbdev->err_stat) {
# # ]
4604 : 0 : SPDK_ERRLOG("err_stat calloc() failed\n");
4605 : 0 : free(nbdev);
4606 : 0 : return NULL;
4607 : : }
4608 : 0 : }
4609 : :
4610 [ + + + - ]: 1481 : rc = pthread_mutex_init(&nbdev->mutex, NULL);
4611 [ - + ]: 1481 : if (rc != 0) {
4612 [ # # # # ]: 0 : free(nbdev->err_stat);
4613 : 0 : free(nbdev);
4614 : 0 : return NULL;
4615 : : }
4616 : :
4617 [ - + - + ]: 1481 : nbdev->ref = 1;
4618 [ - + - + ]: 1481 : nbdev->mp_policy = BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE;
4619 [ - + - + ]: 1481 : nbdev->mp_selector = BDEV_NVME_MP_SELECTOR_ROUND_ROBIN;
4620 [ - + - + ]: 1481 : nbdev->rr_min_io = UINT32_MAX;
4621 [ - + - + : 1481 : TAILQ_INIT(&nbdev->nvme_ns_list);
- + - + -
+ - + - +
- + ]
4622 : :
4623 : 1481 : return nbdev;
4624 : 50 : }
4625 : :
4626 : : static int
4627 : 1481 : nvme_bdev_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
4628 : : {
4629 : : struct nvme_bdev *nbdev;
4630 [ + - + - ]: 1481 : struct nvme_bdev_ctrlr *nbdev_ctrlr = nvme_ctrlr->nbdev_ctrlr;
4631 : : int rc;
4632 : :
4633 : 1481 : nbdev = nvme_bdev_alloc();
4634 [ + + ]: 1481 : if (nbdev == NULL) {
4635 : 0 : SPDK_ERRLOG("Failed to allocate NVMe bdev\n");
4636 : 0 : return -ENOMEM;
4637 : : }
4638 : :
4639 [ + - + - : 1481 : nbdev->opal = nvme_ctrlr->opal_dev != NULL;
+ - + - ]
4640 : :
4641 [ + - + - : 1531 : rc = nbdev_create(&nbdev->disk, nbdev_ctrlr->name, nvme_ctrlr->ctrlr,
+ - + - +
- ]
4642 [ + - + - : 50 : nvme_ns->ns, &nvme_ctrlr->opts, nbdev);
+ - ]
4643 [ - + ]: 1481 : if (rc != 0) {
4644 : 0 : SPDK_ERRLOG("Failed to create NVMe disk\n");
4645 : 0 : nvme_bdev_free(nbdev);
4646 : 0 : return rc;
4647 : : }
4648 : :
4649 : 1531 : spdk_io_device_register(nbdev,
4650 : : bdev_nvme_create_bdev_channel_cb,
4651 : : bdev_nvme_destroy_bdev_channel_cb,
4652 : : sizeof(struct nvme_bdev_channel),
4653 [ + - + - : 1481 : nbdev->disk.name);
+ - ]
4654 : :
4655 [ + - + - ]: 1481 : nvme_ns->bdev = nbdev;
4656 [ + - + - : 1481 : nbdev->nsid = nvme_ns->id;
+ - + - ]
4657 [ + - + - : 1481 : TAILQ_INSERT_TAIL(&nbdev->nvme_ns_list, nvme_ns, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
4658 : :
4659 [ + + ]: 1481 : pthread_mutex_lock(&g_bdev_nvme_mutex);
4660 : :
4661 [ + - + - ]: 1481 : nbdev->nbdev_ctrlr = nbdev_ctrlr;
4662 [ + - + - : 1481 : TAILQ_INSERT_TAIL(&nbdev_ctrlr->bdevs, nbdev, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
4663 : :
4664 [ + - ]: 1481 : rc = spdk_bdev_register(&nbdev->disk);
4665 [ + + ]: 1481 : if (rc != 0) {
4666 : 6 : SPDK_ERRLOG("spdk_bdev_register() failed\n");
4667 : 6 : spdk_io_device_unregister(nbdev, NULL);
4668 [ # # # # ]: 6 : nvme_ns->bdev = NULL;
4669 : :
4670 [ + + # # : 6 : TAILQ_REMOVE(&nbdev_ctrlr->bdevs, nbdev, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4671 : :
4672 [ - + ]: 6 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
4673 : :
4674 : 6 : nvme_bdev_free(nbdev);
4675 : 6 : return rc;
4676 : : }
4677 : :
4678 [ + + ]: 1475 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
4679 : :
4680 : 1475 : return 0;
4681 : 50 : }
4682 : :
4683 : : static bool
4684 : 126 : bdev_nvme_compare_ns(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2)
4685 : : {
4686 : : const struct spdk_nvme_ns_data *nsdata1, *nsdata2;
4687 : : const struct spdk_uuid *uuid1, *uuid2;
4688 : :
4689 : 126 : nsdata1 = spdk_nvme_ns_get_data(ns1);
4690 : 126 : nsdata2 = spdk_nvme_ns_get_data(ns2);
4691 : 126 : uuid1 = spdk_nvme_ns_get_uuid(ns1);
4692 : 126 : uuid2 = spdk_nvme_ns_get_uuid(ns2);
4693 : :
4694 [ + + - + : 196 : return memcmp(nsdata1->nguid, nsdata2->nguid, sizeof(nsdata1->nguid)) == 0 &&
# # # # #
# ]
4695 [ + + + + : 122 : nsdata1->eui64 == nsdata2->eui64 &&
# # # # #
# ]
4696 [ + + + + ]: 118 : ((uuid1 == NULL && uuid2 == NULL) ||
4697 [ + + + - : 308 : (uuid1 != NULL && uuid2 != NULL && spdk_uuid_compare(uuid1, uuid2) == 0)) &&
+ + + + ]
4698 : 106 : spdk_nvme_ns_get_csi(ns1) == spdk_nvme_ns_get_csi(ns2);
4699 : : }
4700 : :
4701 : : static bool
4702 : 56 : hotplug_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
4703 : : struct spdk_nvme_ctrlr_opts *opts)
4704 : : {
4705 : : struct nvme_probe_skip_entry *entry;
4706 : :
4707 [ - + # # : 56 : TAILQ_FOREACH(entry, &g_skipped_nvme_ctrlrs, tailq) {
# # # # ]
4708 [ # # # # ]: 0 : if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) {
4709 : 0 : return false;
4710 : : }
4711 : 0 : }
4712 : :
4713 [ # # # # : 56 : opts->arbitration_burst = (uint8_t)g_opts.arbitration_burst;
# # ]
4714 [ # # # # : 56 : opts->low_priority_weight = (uint8_t)g_opts.low_priority_weight;
# # ]
4715 [ # # # # : 56 : opts->medium_priority_weight = (uint8_t)g_opts.medium_priority_weight;
# # ]
4716 [ # # # # : 56 : opts->high_priority_weight = (uint8_t)g_opts.high_priority_weight;
# # ]
4717 [ # # # # ]: 56 : opts->disable_read_ana_log_page = true;
4718 : :
4719 [ - + - + : 56 : SPDK_DEBUGLOG(bdev_nvme, "Attaching to %s\n", trid->traddr);
# # # # ]
4720 : :
4721 : 56 : return true;
4722 : 0 : }
4723 : :
4724 : : static void
4725 : 0 : nvme_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl)
4726 : : {
4727 : 0 : struct nvme_ctrlr *nvme_ctrlr = ctx;
4728 : :
4729 [ # # # # : 0 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
4730 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "Abort failed. Resetting controller. sc is %u, sct is %u.\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
4731 : : cpl->status.sc, cpl->status.sct);
4732 : 0 : bdev_nvme_reset_ctrlr(nvme_ctrlr);
4733 [ # # # # : 0 : } else if (cpl->cdw0 & 0x1) {
# # ]
4734 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "Specified command could not be aborted.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4735 : 0 : bdev_nvme_reset_ctrlr(nvme_ctrlr);
4736 : 0 : }
4737 : 0 : }
4738 : :
4739 : : static void
4740 : 0 : timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr,
4741 : : struct spdk_nvme_qpair *qpair, uint16_t cid)
4742 : : {
4743 : 0 : struct nvme_ctrlr *nvme_ctrlr = cb_arg;
4744 : : union spdk_nvme_csts_register csts;
4745 : : int rc;
4746 : :
4747 [ # # # # : 0 : assert(nvme_ctrlr->ctrlr == ctrlr);
# # # # ]
4748 : :
4749 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "Warning: Detected a timeout. ctrlr=%p qpair=%p cid=%u\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4750 : : ctrlr, qpair, cid);
4751 : :
4752 : : /* Only try to read CSTS if it's a PCIe controller or we have a timeout on an I/O
4753 : : * queue. (Note: qpair == NULL when there's an admin cmd timeout.) Otherwise we
4754 : : * would submit another fabrics cmd on the admin queue to read CSTS and check for its
4755 : : * completion recursively.
4756 : : */
4757 [ # # # # : 0 : if (nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE || qpair != NULL) {
# # # # #
# # # #
# ]
4758 : 0 : csts = spdk_nvme_ctrlr_get_regs_csts(ctrlr);
4759 [ # # ]: 0 : if (csts.bits.cfs) {
4760 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Controller Fatal Status, reset required\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4761 : 0 : bdev_nvme_reset_ctrlr(nvme_ctrlr);
4762 : 0 : return;
4763 : : }
4764 : 0 : }
4765 : :
4766 [ # # # # : 0 : switch (g_opts.action_on_timeout) {
# # ]
4767 : 0 : case SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT:
4768 [ # # ]: 0 : if (qpair) {
4769 : : /* Don't send abort to ctrlr when ctrlr is not available. */
4770 [ # # # # ]: 0 : pthread_mutex_lock(&nvme_ctrlr->mutex);
4771 [ # # ]: 0 : if (!nvme_ctrlr_is_available(nvme_ctrlr)) {
4772 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
4773 [ # # # # : 0 : NVME_CTRLR_NOTICELOG(nvme_ctrlr, "Quit abort. Ctrlr is not available.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4774 : 0 : return;
4775 : : }
4776 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
4777 : :
4778 : 0 : rc = spdk_nvme_ctrlr_cmd_abort(ctrlr, qpair, cid,
4779 : 0 : nvme_abort_cpl, nvme_ctrlr);
4780 [ # # ]: 0 : if (rc == 0) {
4781 : 0 : return;
4782 : : }
4783 : :
4784 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Unable to send abort. Resetting, rc is %d.\n", rc);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4785 : 0 : }
4786 : :
4787 : : /* FALLTHROUGH */
4788 : : case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET:
4789 : 0 : bdev_nvme_reset_ctrlr(nvme_ctrlr);
4790 : 0 : break;
4791 : 0 : case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE:
4792 [ # # # # : 0 : NVME_CTRLR_DEBUGLOG(nvme_ctrlr, "No action for nvme controller timeout.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
4793 : 0 : break;
4794 : 0 : default:
4795 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "An invalid timeout action value is found.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4796 : 0 : break;
4797 : : }
4798 : 0 : }
4799 : :
4800 : : static struct nvme_ns *
4801 : 1551 : nvme_ns_alloc(void)
4802 : : {
4803 : : struct nvme_ns *nvme_ns;
4804 : :
4805 : 1551 : nvme_ns = calloc(1, sizeof(struct nvme_ns));
4806 [ + + ]: 1551 : if (nvme_ns == NULL) {
4807 : 0 : return NULL;
4808 : : }
4809 : :
4810 [ + + + + ]: 1551 : if (g_opts.io_path_stat) {
4811 [ # # # # ]: 0 : nvme_ns->stat = calloc(1, sizeof(struct spdk_bdev_io_stat));
4812 [ # # # # : 0 : if (nvme_ns->stat == NULL) {
# # ]
4813 : 0 : free(nvme_ns);
4814 : 0 : return NULL;
4815 : : }
4816 [ # # # # ]: 0 : spdk_bdev_reset_io_stat(nvme_ns->stat, SPDK_BDEV_RESET_STAT_MAXMIN);
4817 : 0 : }
4818 : :
4819 : 1551 : return nvme_ns;
4820 : 63 : }
4821 : :
4822 : : static void
4823 : 1551 : nvme_ns_free(struct nvme_ns *nvme_ns)
4824 : : {
4825 [ + - + - ]: 1551 : free(nvme_ns->stat);
4826 : 1551 : free(nvme_ns);
4827 : 1551 : }
4828 : :
4829 : : static void
4830 : 1551 : nvme_ctrlr_populate_namespace_done(struct nvme_ns *nvme_ns, int rc)
4831 : : {
4832 [ + - + - ]: 1551 : struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr;
4833 [ + - + - ]: 1551 : struct nvme_async_probe_ctx *ctx = nvme_ns->probe_ctx;
4834 : :
4835 [ + + ]: 1551 : if (rc == 0) {
4836 [ - + - + ]: 1541 : nvme_ns->probe_ctx = NULL;
4837 : 1541 : nvme_ctrlr_get_ref(nvme_ctrlr);
4838 : 61 : } else {
4839 [ - + # # ]: 10 : pthread_mutex_lock(&nvme_ctrlr->mutex);
4840 [ # # ]: 10 : RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
4841 [ - + # # ]: 10 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
4842 : :
4843 : 10 : nvme_ns_free(nvme_ns);
4844 : : }
4845 : :
4846 [ + + ]: 1551 : if (ctx) {
4847 [ + - ]: 1486 : ctx->populates_in_progress--;
4848 [ + + + - : 1486 : if (ctx->populates_in_progress == 0) {
+ - ]
4849 : 61 : nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx);
4850 : 12 : }
4851 : 62 : }
4852 : 1551 : }
4853 : :
4854 : : static void
4855 : 8 : bdev_nvme_add_io_path(struct nvme_bdev_channel_iter *i,
4856 : : struct nvme_bdev *nbdev,
4857 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
4858 : : {
4859 : 8 : struct nvme_ns *nvme_ns = ctx;
4860 : : int rc;
4861 : :
4862 : 8 : rc = _bdev_nvme_add_io_path(nbdev_ch, nvme_ns);
4863 [ - + ]: 8 : if (rc != 0) {
4864 : 0 : SPDK_ERRLOG("Failed to add I/O path to bdev_channel dynamically.\n");
4865 : 0 : }
4866 : :
4867 : 8 : nvme_bdev_for_each_channel_continue(i, rc);
4868 : 8 : }
4869 : :
4870 : : static void
4871 : 8 : bdev_nvme_delete_io_path(struct nvme_bdev_channel_iter *i,
4872 : : struct nvme_bdev *nbdev,
4873 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
4874 : : {
4875 : 8 : struct nvme_ns *nvme_ns = ctx;
4876 : : struct nvme_io_path *io_path;
4877 : :
4878 : 8 : io_path = _bdev_nvme_get_io_path(nbdev_ch, nvme_ns);
4879 [ + - ]: 8 : if (io_path != NULL) {
4880 : 8 : _bdev_nvme_delete_io_path(nbdev_ch, io_path);
4881 : 2 : }
4882 : :
4883 : 8 : nvme_bdev_for_each_channel_continue(i, 0);
4884 : 8 : }
4885 : :
4886 : : static void
4887 : 0 : bdev_nvme_add_io_path_failed(struct nvme_bdev *nbdev, void *ctx, int status)
4888 : : {
4889 : 0 : struct nvme_ns *nvme_ns = ctx;
4890 : :
4891 : 0 : nvme_ctrlr_populate_namespace_done(nvme_ns, -1);
4892 : 0 : }
4893 : :
4894 : : static void
4895 : 66 : bdev_nvme_add_io_path_done(struct nvme_bdev *nbdev, void *ctx, int status)
4896 : : {
4897 : 66 : struct nvme_ns *nvme_ns = ctx;
4898 : :
4899 [ + - ]: 66 : if (status == 0) {
4900 : 66 : nvme_ctrlr_populate_namespace_done(nvme_ns, 0);
4901 : 12 : } else {
4902 : : /* Delete the added io_paths and fail populating the namespace. */
4903 : 0 : nvme_bdev_for_each_channel(nbdev,
4904 : : bdev_nvme_delete_io_path,
4905 : 0 : nvme_ns,
4906 : : bdev_nvme_add_io_path_failed);
4907 : : }
4908 : 66 : }
4909 : :
4910 : : static int
4911 : 70 : nvme_bdev_add_ns(struct nvme_bdev *nbdev, struct nvme_ns *nvme_ns)
4912 : : {
4913 : : struct nvme_ns *tmp_ns;
4914 : : const struct spdk_nvme_ns_data *nsdata;
4915 : :
4916 [ # # # # ]: 70 : nsdata = spdk_nvme_ns_get_data(nvme_ns->ns);
4917 [ - + # # : 70 : if (!nsdata->nmic.can_share) {
# # ]
4918 : 0 : SPDK_ERRLOG("Namespace cannot be shared.\n");
4919 : 0 : return -EINVAL;
4920 : : }
4921 : :
4922 [ - + # # ]: 70 : pthread_mutex_lock(&nbdev->mutex);
4923 : :
4924 [ # # # # : 70 : tmp_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
# # ]
4925 [ - + # # ]: 70 : assert(tmp_ns != NULL);
4926 : :
4927 [ + - + + : 70 : if (tmp_ns->ns != NULL && !bdev_nvme_compare_ns(nvme_ns->ns, tmp_ns->ns)) {
# # # # #
# # # # #
# # ]
4928 [ - + # # ]: 4 : pthread_mutex_unlock(&nbdev->mutex);
4929 : 4 : SPDK_ERRLOG("Namespaces are not identical.\n");
4930 : 4 : return -EINVAL;
4931 : : }
4932 : :
4933 [ # # # # ]: 66 : nbdev->ref++;
4934 [ # # # # : 66 : TAILQ_INSERT_TAIL(&nbdev->nvme_ns_list, nvme_ns, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
4935 [ # # # # ]: 66 : nvme_ns->bdev = nbdev;
4936 : :
4937 [ - + # # ]: 66 : pthread_mutex_unlock(&nbdev->mutex);
4938 : :
4939 : : /* Add nvme_io_path to nvme_bdev_channels dynamically. */
4940 : 78 : nvme_bdev_for_each_channel(nbdev,
4941 : : bdev_nvme_add_io_path,
4942 : 12 : nvme_ns,
4943 : : bdev_nvme_add_io_path_done);
4944 : :
4945 : 66 : return 0;
4946 : 13 : }
4947 : :
4948 : : static void
4949 : 1551 : nvme_ctrlr_populate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
4950 : : {
4951 : : struct spdk_nvme_ns *ns;
4952 : : struct nvme_bdev *bdev;
4953 : 1551 : int rc = 0;
4954 : :
4955 [ + - + - : 1551 : ns = spdk_nvme_ctrlr_get_ns(nvme_ctrlr->ctrlr, nvme_ns->id);
+ - + - ]
4956 [ - + ]: 1551 : if (!ns) {
4957 [ # # # # : 0 : NVME_CTRLR_DEBUGLOG(nvme_ctrlr, "Invalid NS %d\n", nvme_ns->id);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
4958 : 0 : rc = -EINVAL;
4959 : 0 : goto done;
4960 : : }
4961 : :
4962 [ + - + - ]: 1551 : nvme_ns->ns = ns;
4963 [ + - + - ]: 1551 : nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
4964 : :
4965 [ + + + - : 1551 : if (nvme_ctrlr->ana_log_page != NULL) {
+ - ]
4966 : 476 : bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ns_set_ana_state, nvme_ns);
4967 : 38 : }
4968 : :
4969 [ + - + - : 1551 : bdev = nvme_bdev_ctrlr_get_bdev(nvme_ctrlr->nbdev_ctrlr, nvme_ns->id);
+ - + - ]
4970 [ + + ]: 1602 : if (bdev == NULL) {
4971 : 1481 : rc = nvme_bdev_create(nvme_ctrlr, nvme_ns);
4972 : 50 : } else {
4973 : 70 : rc = nvme_bdev_add_ns(bdev, nvme_ns);
4974 [ + + ]: 70 : if (rc == 0) {
4975 : 66 : return;
4976 : : }
4977 : : }
4978 : 3 : done:
4979 : 1485 : nvme_ctrlr_populate_namespace_done(nvme_ns, rc);
4980 : 63 : }
4981 : :
4982 : : static void
4983 : 1541 : nvme_ctrlr_depopulate_namespace_done(struct nvme_ns *nvme_ns)
4984 : : {
4985 [ + - + - ]: 1541 : struct nvme_ctrlr *nvme_ctrlr = nvme_ns->ctrlr;
4986 : :
4987 [ + + # # ]: 1541 : assert(nvme_ctrlr != NULL);
4988 : :
4989 [ + + + - ]: 1541 : pthread_mutex_lock(&nvme_ctrlr->mutex);
4990 : :
4991 [ + - ]: 1541 : RB_REMOVE(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
4992 : :
4993 [ + + + - : 1541 : if (nvme_ns->bdev != NULL) {
- + ]
4994 [ - + # # ]: 515 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
4995 : 515 : return;
4996 : : }
4997 : :
4998 : 1026 : nvme_ns_free(nvme_ns);
4999 [ + + + - ]: 1026 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5000 : :
5001 : 1026 : nvme_ctrlr_put_ref(nvme_ctrlr);
5002 : 61 : }
5003 : :
5004 : : static void
5005 : 56 : bdev_nvme_delete_io_path_done(struct nvme_bdev *nbdev, void *ctx, int status)
5006 : : {
5007 : 56 : struct nvme_ns *nvme_ns = ctx;
5008 : :
5009 : 56 : nvme_ctrlr_depopulate_namespace_done(nvme_ns);
5010 : 56 : }
5011 : :
5012 : : static void
5013 : 1541 : nvme_ctrlr_depopulate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
5014 : : {
5015 : : struct nvme_bdev *nbdev;
5016 : :
5017 [ + + + + : 1541 : if (nvme_ns->depopulating) {
+ - - + ]
5018 : : /* Maybe we received 2 AENs in a row */
5019 : 0 : return;
5020 : : }
5021 [ + - + - ]: 1541 : nvme_ns->depopulating = true;
5022 : :
5023 [ + - ]: 1541 : spdk_poller_unregister(&nvme_ns->anatt_timer);
5024 : :
5025 [ + - + - ]: 1541 : nbdev = nvme_ns->bdev;
5026 [ + + ]: 1541 : if (nbdev != NULL) {
5027 [ - + # # ]: 711 : pthread_mutex_lock(&nbdev->mutex);
5028 : :
5029 [ + + # # : 711 : assert(nbdev->ref > 0);
# # # # ]
5030 [ # # # # ]: 711 : nbdev->ref--;
5031 [ + + # # : 711 : if (nbdev->ref == 0) {
# # ]
5032 [ - + # # ]: 655 : pthread_mutex_unlock(&nbdev->mutex);
5033 : :
5034 [ # # ]: 655 : spdk_bdev_unregister(&nbdev->disk, NULL, NULL);
5035 : 38 : } else {
5036 : : /* spdk_bdev_unregister() is not called until the last nvme_ns is
5037 : : * depopulated. Hence we need to remove nvme_ns from bdev->nvme_ns_list
5038 : : * and clear nvme_ns->bdev here.
5039 : : */
5040 [ + + # # : 56 : TAILQ_REMOVE(&nbdev->nvme_ns_list, nvme_ns, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5041 : :
5042 [ - + # # : 56 : pthread_mutex_lock(&nvme_ns->ctrlr->mutex);
# # # # ]
5043 [ # # # # ]: 56 : nvme_ns->bdev = NULL;
5044 [ - + # # : 56 : pthread_mutex_unlock(&nvme_ns->ctrlr->mutex);
# # # # ]
5045 : :
5046 [ - + # # ]: 56 : pthread_mutex_unlock(&nbdev->mutex);
5047 : :
5048 : : /* Delete nvme_io_paths from nvme_bdev_channels dynamically. After that,
5049 : : * we call depopulate_namespace_done() to avoid use-after-free.
5050 : : */
5051 : 67 : nvme_bdev_for_each_channel(nbdev,
5052 : : bdev_nvme_delete_io_path,
5053 : 11 : nvme_ns,
5054 : : bdev_nvme_delete_io_path_done);
5055 : 56 : return;
5056 : : }
5057 : 38 : }
5058 : :
5059 : 1485 : nvme_ctrlr_depopulate_namespace_done(nvme_ns);
5060 : 61 : }
5061 : :
5062 : : static void
5063 : 1798 : nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr,
5064 : : struct nvme_async_probe_ctx *ctx)
5065 : : {
5066 [ + - + - ]: 1798 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
5067 : : struct nvme_ns *nvme_ns, *next;
5068 : : struct spdk_nvme_ns *ns;
5069 : : struct nvme_bdev *nbdev;
5070 : : uint32_t nsid;
5071 : : int rc;
5072 : : uint64_t num_sectors;
5073 : :
5074 [ + + ]: 1798 : if (ctx) {
5075 : : /* Initialize this count to 1 to handle the populate functions
5076 : : * calling nvme_ctrlr_populate_namespace_done() immediately.
5077 : : */
5078 [ + - + - ]: 1652 : ctx->populates_in_progress = 1;
5079 : 58 : }
5080 : :
5081 : : /* First loop over our existing namespaces and see if they have been
5082 : : * removed. */
5083 : 1798 : nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
5084 [ + + ]: 1832 : while (nvme_ns != NULL) {
5085 : 34 : next = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
5086 : :
5087 [ + + # # : 34 : if (spdk_nvme_ctrlr_is_active_ns(ctrlr, nvme_ns->id)) {
# # ]
5088 : : /* NS is still there or added again. Its attributes may have changed. */
5089 [ # # # # ]: 21 : ns = spdk_nvme_ctrlr_get_ns(ctrlr, nvme_ns->id);
5090 [ + + # # : 21 : if (nvme_ns->ns != ns) {
# # ]
5091 [ + + # # : 4 : assert(nvme_ns->ns == NULL);
# # # # ]
5092 [ # # # # ]: 4 : nvme_ns->ns = ns;
5093 [ + + - + : 4 : NVME_CTRLR_DEBUGLOG(nvme_ctrlr, "NSID %u was added\n", nvme_ns->id);
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5094 : 1 : }
5095 : :
5096 : 21 : num_sectors = spdk_nvme_ns_get_num_sectors(ns);
5097 [ # # # # ]: 21 : nbdev = nvme_ns->bdev;
5098 [ + + # # ]: 21 : assert(nbdev != NULL);
5099 [ + + # # : 21 : if (nbdev->disk.blockcnt != num_sectors) {
# # # # ]
5100 [ + - # # : 4 : NVME_CTRLR_NOTICELOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5101 : : "NSID %u is resized: bdev name %s, old size %" PRIu64 ", new size %" PRIu64 "\n",
5102 : : nvme_ns->id,
5103 : : nbdev->disk.name,
5104 : : nbdev->disk.blockcnt,
5105 : : num_sectors);
5106 [ # # ]: 4 : rc = spdk_bdev_notify_blockcnt_change(&nbdev->disk, num_sectors);
5107 [ + + ]: 4 : if (rc != 0) {
5108 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5109 : : "Could not change num blocks for nvme bdev: name %s, errno: %d.\n",
5110 : : nbdev->disk.name, rc);
5111 : 0 : }
5112 : 1 : }
5113 : 3 : } else {
5114 : : /* Namespace was removed */
5115 : 13 : nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns);
5116 : : }
5117 : :
5118 : 34 : nvme_ns = next;
5119 : : }
5120 : :
5121 : : /* Loop through all of the namespaces at the nvme level and see if any of them are new */
5122 : 1798 : nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
5123 [ + + ]: 3370 : while (nsid != 0) {
5124 : 1572 : nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid);
5125 : :
5126 [ + + ]: 1572 : if (nvme_ns == NULL) {
5127 : : /* Found a new one */
5128 : 1551 : nvme_ns = nvme_ns_alloc();
5129 [ + + ]: 1551 : if (nvme_ns == NULL) {
5130 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to allocate namespace\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5131 : : /* This just fails to attach the namespace. It may work on a future attempt. */
5132 : 0 : continue;
5133 : : }
5134 : :
5135 [ + - + - ]: 1551 : nvme_ns->id = nsid;
5136 [ + - + - ]: 1551 : nvme_ns->ctrlr = nvme_ctrlr;
5137 : :
5138 [ + - + - ]: 1551 : nvme_ns->bdev = NULL;
5139 : :
5140 [ + + ]: 1551 : if (ctx) {
5141 [ + - ]: 1486 : ctx->populates_in_progress++;
5142 : 62 : }
5143 [ + - + - ]: 1551 : nvme_ns->probe_ctx = ctx;
5144 : :
5145 [ + + + - ]: 1551 : pthread_mutex_lock(&nvme_ctrlr->mutex);
5146 [ + - ]: 1551 : RB_INSERT(nvme_ns_tree, &nvme_ctrlr->namespaces, nvme_ns);
5147 [ + + + - ]: 1551 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5148 : :
5149 : 1551 : nvme_ctrlr_populate_namespace(nvme_ctrlr, nvme_ns);
5150 : 63 : }
5151 : :
5152 : 1572 : nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid);
5153 : : }
5154 : :
5155 [ + + ]: 1798 : if (ctx) {
5156 : : /* Decrement this count now that the loop is over to account
5157 : : * for the one we started with. If the count is then 0, we
5158 : : * know any populate_namespace functions completed immediately,
5159 : : * so we'll kick the callback here.
5160 : : */
5161 [ + - ]: 1652 : ctx->populates_in_progress--;
5162 [ + + + - : 1652 : if (ctx->populates_in_progress == 0) {
- + ]
5163 : 1591 : nvme_ctrlr_populate_namespaces_done(nvme_ctrlr, ctx);
5164 : 46 : }
5165 : 58 : }
5166 : :
5167 : 1798 : }
5168 : :
5169 : : static void
5170 : 1768 : nvme_ctrlr_depopulate_namespaces(struct nvme_ctrlr *nvme_ctrlr)
5171 : : {
5172 : : struct nvme_ns *nvme_ns, *tmp;
5173 : :
5174 [ + + + + : 3296 : RB_FOREACH_SAFE(nvme_ns, nvme_ns_tree, &nvme_ctrlr->namespaces, tmp) {
+ + ]
5175 : 1528 : nvme_ctrlr_depopulate_namespace(nvme_ctrlr, nvme_ns);
5176 : 60 : }
5177 : 1768 : }
5178 : :
5179 : : static uint32_t
5180 : 3330 : nvme_ctrlr_get_ana_log_page_size(struct nvme_ctrlr *nvme_ctrlr)
5181 : : {
5182 [ # # # # ]: 3330 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
5183 : : const struct spdk_nvme_ctrlr_data *cdata;
5184 : 3330 : uint32_t nsid, ns_count = 0;
5185 : :
5186 : 3330 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
5187 : :
5188 [ + + ]: 3375 : for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
5189 [ + + ]: 6692 : nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
5190 : 3362 : ns_count++;
5191 : 45 : }
5192 : :
5193 [ # # # # ]: 3367 : return sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid *
5194 : 3330 : sizeof(struct spdk_nvme_ana_group_descriptor) + ns_count *
5195 : : sizeof(uint32_t);
5196 : : }
5197 : :
5198 : : static int
5199 : 132 : nvme_ctrlr_set_ana_states(const struct spdk_nvme_ana_group_descriptor *desc,
5200 : : void *cb_arg)
5201 : : {
5202 : 132 : struct nvme_ctrlr *nvme_ctrlr = cb_arg;
5203 : : struct nvme_ns *nvme_ns;
5204 : : uint32_t i, nsid;
5205 : :
5206 [ + + # # : 260 : for (i = 0; i < desc->num_of_nsid; i++) {
# # ]
5207 [ # # # # : 128 : nsid = desc->nsid[i];
# # ]
5208 [ + + ]: 128 : if (nsid == 0) {
5209 : 0 : continue;
5210 : : }
5211 : :
5212 : 128 : nvme_ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid);
5213 : :
5214 [ + + ]: 128 : if (nvme_ns == NULL) {
5215 : : /* Target told us that an inactive namespace had an ANA change */
5216 : 4 : continue;
5217 : : }
5218 : :
5219 : 124 : _nvme_ns_set_ana_state(nvme_ns, desc);
5220 : 5 : }
5221 : :
5222 : 132 : return 0;
5223 : : }
5224 : :
5225 : : static void
5226 : 1 : bdev_nvme_disable_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr)
5227 : : {
5228 : : struct nvme_ns *nvme_ns;
5229 : :
5230 [ # # # # ]: 1 : spdk_free(nvme_ctrlr->ana_log_page);
5231 [ # # # # ]: 1 : nvme_ctrlr->ana_log_page = NULL;
5232 : :
5233 [ # # ]: 1 : for (nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
5234 [ + + ]: 2 : nvme_ns != NULL;
5235 : 1 : nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns)) {
5236 [ # # # # ]: 1 : nvme_ns->ana_state_updating = false;
5237 [ # # # # ]: 1 : nvme_ns->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
5238 : 0 : }
5239 : 1 : }
5240 : :
5241 : : static void
5242 : 117 : nvme_ctrlr_read_ana_log_page_done(void *ctx, const struct spdk_nvme_cpl *cpl)
5243 : : {
5244 : 117 : struct nvme_ctrlr *nvme_ctrlr = ctx;
5245 : :
5246 [ + - + + : 117 : if (cpl != NULL && spdk_nvme_cpl_is_success(cpl)) {
+ + # # #
# # # # #
# # # # ]
5247 : 119 : bdev_nvme_parse_ana_log_page(nvme_ctrlr, nvme_ctrlr_set_ana_states,
5248 : 3 : nvme_ctrlr);
5249 : 3 : } else {
5250 : 1 : bdev_nvme_disable_read_ana_log_page(nvme_ctrlr);
5251 : : }
5252 : :
5253 [ - + # # ]: 117 : pthread_mutex_lock(&nvme_ctrlr->mutex);
5254 : :
5255 [ + + # # : 117 : assert(nvme_ctrlr->ana_log_page_updating == true);
# # ]
5256 [ # # ]: 117 : nvme_ctrlr->ana_log_page_updating = false;
5257 : :
5258 [ - + ]: 117 : if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {
5259 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5260 : :
5261 : 0 : nvme_ctrlr_unregister(nvme_ctrlr);
5262 : 0 : } else {
5263 [ - + # # ]: 117 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5264 : :
5265 : 117 : bdev_nvme_clear_io_path_caches(nvme_ctrlr);
5266 : : }
5267 : 117 : }
5268 : :
5269 : : static int
5270 : 2882 : nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr)
5271 : : {
5272 : : uint32_t ana_log_page_size;
5273 : : int rc;
5274 : :
5275 [ + + # # : 2882 : if (nvme_ctrlr->ana_log_page == NULL) {
# # ]
5276 : 0 : return -EINVAL;
5277 : : }
5278 : :
5279 : 2882 : ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr);
5280 : :
5281 [ - + # # : 2882 : if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) {
# # ]
5282 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5283 : : "ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n",
5284 : : ana_log_page_size, nvme_ctrlr->max_ana_log_page_size);
5285 : 0 : return -EINVAL;
5286 : : }
5287 : :
5288 [ - + # # ]: 2882 : pthread_mutex_lock(&nvme_ctrlr->mutex);
5289 [ + + + + ]: 2882 : if (!nvme_ctrlr_is_available(nvme_ctrlr) ||
5290 [ # # ]: 5 : nvme_ctrlr->ana_log_page_updating) {
5291 [ - + # # ]: 2765 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5292 : 2765 : return -EBUSY;
5293 : : }
5294 : :
5295 [ # # ]: 117 : nvme_ctrlr->ana_log_page_updating = true;
5296 [ - + # # ]: 117 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
5297 : :
5298 [ # # # # ]: 120 : rc = spdk_nvme_ctrlr_cmd_get_log_page(nvme_ctrlr->ctrlr,
5299 : : SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS,
5300 : : SPDK_NVME_GLOBAL_NS_TAG,
5301 [ # # # # ]: 117 : nvme_ctrlr->ana_log_page,
5302 : 3 : ana_log_page_size, 0,
5303 : : nvme_ctrlr_read_ana_log_page_done,
5304 : 3 : nvme_ctrlr);
5305 [ - + ]: 117 : if (rc != 0) {
5306 : 0 : nvme_ctrlr_read_ana_log_page_done(nvme_ctrlr, NULL);
5307 : 0 : }
5308 : :
5309 : 117 : return rc;
5310 : 6 : }
5311 : :
5312 : : static void
5313 : 0 : dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx)
5314 : : {
5315 : 0 : }
5316 : :
5317 : : struct bdev_nvme_set_preferred_path_ctx {
5318 : : struct spdk_bdev_desc *desc;
5319 : : struct nvme_ns *nvme_ns;
5320 : : bdev_nvme_set_preferred_path_cb cb_fn;
5321 : : void *cb_arg;
5322 : : };
5323 : :
5324 : : static void
5325 : 12 : bdev_nvme_set_preferred_path_done(struct nvme_bdev *nbdev, void *_ctx, int status)
5326 : : {
5327 : 12 : struct bdev_nvme_set_preferred_path_ctx *ctx = _ctx;
5328 : :
5329 [ + + # # ]: 12 : assert(ctx != NULL);
5330 [ + + # # : 12 : assert(ctx->desc != NULL);
# # # # ]
5331 [ + + # # : 12 : assert(ctx->cb_fn != NULL);
# # # # ]
5332 : :
5333 [ # # # # ]: 12 : spdk_bdev_close(ctx->desc);
5334 : :
5335 [ # # # # : 12 : ctx->cb_fn(ctx->cb_arg, status);
# # # # #
# # # ]
5336 : :
5337 : 12 : free(ctx);
5338 : 12 : }
5339 : :
5340 : : static void
5341 : 8 : _bdev_nvme_set_preferred_path(struct nvme_bdev_channel_iter *i,
5342 : : struct nvme_bdev *nbdev,
5343 : : struct nvme_bdev_channel *nbdev_ch, void *_ctx)
5344 : : {
5345 : 8 : struct bdev_nvme_set_preferred_path_ctx *ctx = _ctx;
5346 : : struct nvme_io_path *io_path, *prev;
5347 : :
5348 : 8 : prev = NULL;
5349 [ + + # # : 12 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
5350 [ + + # # : 12 : if (io_path->nvme_ns == ctx->nvme_ns) {
# # # # #
# ]
5351 : 8 : break;
5352 : : }
5353 : 4 : prev = io_path;
5354 : 1 : }
5355 : :
5356 [ + - ]: 8 : if (io_path != NULL) {
5357 [ + + ]: 8 : if (prev != NULL) {
5358 [ - + # # : 4 : STAILQ_REMOVE_AFTER(&nbdev_ch->io_path_list, prev, stailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5359 [ - + # # : 4 : STAILQ_INSERT_HEAD(&nbdev_ch->io_path_list, io_path, stailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
5360 : 1 : }
5361 : :
5362 : : /* We can set io_path to nbdev_ch->current_io_path directly here.
5363 : : * However, it needs to be conditional. To simplify the code,
5364 : : * just clear nbdev_ch->current_io_path and let find_io_path()
5365 : : * fill it.
5366 : : *
5367 : : * Automatic failback may be disabled. Hence even if the io_path is
5368 : : * already at the head, clear nbdev_ch->current_io_path.
5369 : : */
5370 : 8 : bdev_nvme_clear_current_io_path(nbdev_ch);
5371 : 2 : }
5372 : :
5373 : 8 : nvme_bdev_for_each_channel_continue(i, 0);
5374 : 8 : }
5375 : :
5376 : : static struct nvme_ns *
5377 : 12 : bdev_nvme_set_preferred_ns(struct nvme_bdev *nbdev, uint16_t cntlid)
5378 : : {
5379 : : struct nvme_ns *nvme_ns, *prev;
5380 : : const struct spdk_nvme_ctrlr_data *cdata;
5381 : :
5382 : 12 : prev = NULL;
5383 [ + + # # : 24 : TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
# # # # #
# # # #
# ]
5384 [ # # # # : 24 : cdata = spdk_nvme_ctrlr_get_data(nvme_ns->ctrlr->ctrlr);
# # # # ]
5385 : :
5386 [ + + # # : 24 : if (cdata->cntlid == cntlid) {
# # ]
5387 : 12 : break;
5388 : : }
5389 : 12 : prev = nvme_ns;
5390 : 3 : }
5391 : :
5392 [ + - + + ]: 12 : if (nvme_ns != NULL && prev != NULL) {
5393 [ + + # # : 8 : TAILQ_REMOVE(&nbdev->nvme_ns_list, nvme_ns, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5394 [ + + # # : 8 : TAILQ_INSERT_HEAD(&nbdev->nvme_ns_list, nvme_ns, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5395 : 2 : }
5396 : :
5397 : 12 : return nvme_ns;
5398 : : }
5399 : :
5400 : : /* This function supports only multipath mode. There is only a single I/O path
5401 : : * for each NVMe-oF controller. Hence, just move the matched I/O path to the
5402 : : * head of the I/O path list for each NVMe bdev channel.
5403 : : *
5404 : : * NVMe bdev channel may be acquired after completing this function. move the
5405 : : * matched namespace to the head of the namespace list for the NVMe bdev too.
5406 : : */
5407 : : void
5408 : 12 : bdev_nvme_set_preferred_path(const char *name, uint16_t cntlid,
5409 : : bdev_nvme_set_preferred_path_cb cb_fn, void *cb_arg)
5410 : : {
5411 : : struct bdev_nvme_set_preferred_path_ctx *ctx;
5412 : : struct spdk_bdev *bdev;
5413 : : struct nvme_bdev *nbdev;
5414 : 12 : int rc = 0;
5415 : :
5416 [ + + # # ]: 12 : assert(cb_fn != NULL);
5417 : :
5418 : 12 : ctx = calloc(1, sizeof(*ctx));
5419 [ + + ]: 12 : if (ctx == NULL) {
5420 : 0 : SPDK_ERRLOG("Failed to alloc context.\n");
5421 : 0 : rc = -ENOMEM;
5422 : 0 : goto err_alloc;
5423 : : }
5424 : :
5425 [ # # # # ]: 12 : ctx->cb_fn = cb_fn;
5426 [ # # # # ]: 12 : ctx->cb_arg = cb_arg;
5427 : :
5428 [ # # ]: 12 : rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc);
5429 [ - + ]: 12 : if (rc != 0) {
5430 : 0 : SPDK_ERRLOG("Failed to open bdev %s.\n", name);
5431 : 0 : goto err_open;
5432 : : }
5433 : :
5434 [ # # # # ]: 12 : bdev = spdk_bdev_desc_get_bdev(ctx->desc);
5435 : :
5436 [ - + # # : 12 : if (bdev->module != &nvme_if) {
# # ]
5437 : 0 : SPDK_ERRLOG("bdev %s is not registered in this module.\n", name);
5438 : 0 : rc = -ENODEV;
5439 : 0 : goto err_bdev;
5440 : : }
5441 : :
5442 : 12 : nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk);
5443 : :
5444 [ - + # # ]: 12 : pthread_mutex_lock(&nbdev->mutex);
5445 : :
5446 [ # # # # ]: 12 : ctx->nvme_ns = bdev_nvme_set_preferred_ns(nbdev, cntlid);
5447 [ + + # # : 12 : if (ctx->nvme_ns == NULL) {
# # ]
5448 [ # # # # ]: 0 : pthread_mutex_unlock(&nbdev->mutex);
5449 : :
5450 : 0 : SPDK_ERRLOG("bdev %s does not have namespace to controller %u.\n", name, cntlid);
5451 : 0 : rc = -ENODEV;
5452 : 0 : goto err_bdev;
5453 : : }
5454 : :
5455 [ - + # # ]: 12 : pthread_mutex_unlock(&nbdev->mutex);
5456 : :
5457 : 15 : nvme_bdev_for_each_channel(nbdev,
5458 : : _bdev_nvme_set_preferred_path,
5459 : 3 : ctx,
5460 : : bdev_nvme_set_preferred_path_done);
5461 : 12 : return;
5462 : :
5463 : 0 : err_bdev:
5464 [ # # # # ]: 0 : spdk_bdev_close(ctx->desc);
5465 : 0 : err_open:
5466 : 0 : free(ctx);
5467 : 0 : err_alloc:
5468 [ # # # # ]: 0 : cb_fn(cb_arg, rc);
5469 : 3 : }
5470 : :
5471 : : struct bdev_nvme_set_multipath_policy_ctx {
5472 : : struct spdk_bdev_desc *desc;
5473 : : spdk_bdev_nvme_set_multipath_policy_cb cb_fn;
5474 : : void *cb_arg;
5475 : : };
5476 : :
5477 : : static void
5478 : 16 : bdev_nvme_set_multipath_policy_done(struct nvme_bdev *nbdev, void *_ctx, int status)
5479 : : {
5480 : 16 : struct bdev_nvme_set_multipath_policy_ctx *ctx = _ctx;
5481 : :
5482 [ + + # # ]: 16 : assert(ctx != NULL);
5483 [ + + # # : 16 : assert(ctx->desc != NULL);
# # # # ]
5484 [ + + # # : 16 : assert(ctx->cb_fn != NULL);
# # # # ]
5485 : :
5486 [ # # # # ]: 16 : spdk_bdev_close(ctx->desc);
5487 : :
5488 [ # # # # : 16 : ctx->cb_fn(ctx->cb_arg, status);
# # # # #
# # # ]
5489 : :
5490 : 16 : free(ctx);
5491 : 16 : }
5492 : :
5493 : : static void
5494 : 8 : _bdev_nvme_set_multipath_policy(struct nvme_bdev_channel_iter *i,
5495 : : struct nvme_bdev *nbdev,
5496 : : struct nvme_bdev_channel *nbdev_ch, void *ctx)
5497 : : {
5498 [ # # # # : 8 : nbdev_ch->mp_policy = nbdev->mp_policy;
# # # # ]
5499 [ # # # # : 8 : nbdev_ch->mp_selector = nbdev->mp_selector;
# # # # ]
5500 [ # # # # : 8 : nbdev_ch->rr_min_io = nbdev->rr_min_io;
# # # # ]
5501 : 8 : bdev_nvme_clear_current_io_path(nbdev_ch);
5502 : :
5503 : 8 : nvme_bdev_for_each_channel_continue(i, 0);
5504 : 8 : }
5505 : :
5506 : : void
5507 : 16 : spdk_bdev_nvme_set_multipath_policy(const char *name, enum spdk_bdev_nvme_multipath_policy policy,
5508 : : enum spdk_bdev_nvme_multipath_selector selector, uint32_t rr_min_io,
5509 : : spdk_bdev_nvme_set_multipath_policy_cb cb_fn, void *cb_arg)
5510 : : {
5511 : : struct bdev_nvme_set_multipath_policy_ctx *ctx;
5512 : : struct spdk_bdev *bdev;
5513 : : struct nvme_bdev *nbdev;
5514 : : int rc;
5515 : :
5516 [ + + # # ]: 16 : assert(cb_fn != NULL);
5517 : :
5518 [ + + + ]: 16 : switch (policy) {
5519 : 3 : case BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE:
5520 : 4 : break;
5521 [ + + - ]: 10 : case BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE:
5522 [ + + + ]: 6 : switch (selector) {
5523 : 7 : case BDEV_NVME_MP_SELECTOR_ROUND_ROBIN:
5524 [ + + ]: 8 : if (rr_min_io == UINT32_MAX) {
5525 : 4 : rr_min_io = 1;
5526 [ - + ]: 4 : } else if (rr_min_io == 0) {
5527 : 0 : rc = -EINVAL;
5528 : 0 : goto exit;
5529 : : }
5530 : 8 : break;
5531 : 3 : case BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH:
5532 : 4 : break;
5533 : 0 : default:
5534 : 0 : rc = -EINVAL;
5535 : 0 : goto exit;
5536 : : }
5537 : 12 : break;
5538 : 0 : default:
5539 : 0 : rc = -EINVAL;
5540 : 0 : goto exit;
5541 : : }
5542 : :
5543 : 16 : ctx = calloc(1, sizeof(*ctx));
5544 [ + + ]: 16 : if (ctx == NULL) {
5545 : 0 : SPDK_ERRLOG("Failed to alloc context.\n");
5546 : 0 : rc = -ENOMEM;
5547 : 0 : goto exit;
5548 : : }
5549 : :
5550 [ # # # # ]: 16 : ctx->cb_fn = cb_fn;
5551 [ # # # # ]: 16 : ctx->cb_arg = cb_arg;
5552 : :
5553 [ # # ]: 16 : rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &ctx->desc);
5554 [ - + ]: 16 : if (rc != 0) {
5555 : 0 : SPDK_ERRLOG("Failed to open bdev %s.\n", name);
5556 : 0 : rc = -ENODEV;
5557 : 0 : goto err_open;
5558 : : }
5559 : :
5560 [ # # # # ]: 16 : bdev = spdk_bdev_desc_get_bdev(ctx->desc);
5561 [ - + # # : 16 : if (bdev->module != &nvme_if) {
# # ]
5562 : 0 : SPDK_ERRLOG("bdev %s is not registered in this module.\n", name);
5563 : 0 : rc = -ENODEV;
5564 : 0 : goto err_module;
5565 : : }
5566 : 16 : nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk);
5567 : :
5568 [ - + # # ]: 16 : pthread_mutex_lock(&nbdev->mutex);
5569 [ # # # # ]: 16 : nbdev->mp_policy = policy;
5570 [ # # # # ]: 16 : nbdev->mp_selector = selector;
5571 [ # # # # ]: 16 : nbdev->rr_min_io = rr_min_io;
5572 [ - + # # ]: 16 : pthread_mutex_unlock(&nbdev->mutex);
5573 : :
5574 : 19 : nvme_bdev_for_each_channel(nbdev,
5575 : : _bdev_nvme_set_multipath_policy,
5576 : 3 : ctx,
5577 : : bdev_nvme_set_multipath_policy_done);
5578 : 16 : return;
5579 : :
5580 : 0 : err_module:
5581 [ # # # # ]: 0 : spdk_bdev_close(ctx->desc);
5582 : 0 : err_open:
5583 : 0 : free(ctx);
5584 : 0 : exit:
5585 [ # # # # ]: 0 : cb_fn(cb_arg, rc);
5586 : 3 : }
5587 : :
5588 : : static void
5589 : 142 : aer_cb(void *arg, const struct spdk_nvme_cpl *cpl)
5590 : : {
5591 : 142 : struct nvme_ctrlr *nvme_ctrlr = arg;
5592 : : union spdk_nvme_async_event_completion event;
5593 : :
5594 [ + + - + : 142 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
5595 : 4 : SPDK_WARNLOG("AER request execute failed\n");
5596 : 4 : return;
5597 : : }
5598 : :
5599 [ # # # # ]: 138 : event.raw = cpl->cdw0;
5600 [ + - + + ]: 138 : if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) &&
5601 [ + + ]: 138 : (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) {
5602 : 34 : nvme_ctrlr_populate_namespaces(nvme_ctrlr, NULL);
5603 [ + - - + ]: 106 : } else if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) &&
5604 [ + - ]: 104 : (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) {
5605 : 104 : nvme_ctrlr_read_ana_log_page(nvme_ctrlr);
5606 : 1 : }
5607 : 3 : }
5608 : :
5609 : : static void
5610 : 1756 : free_nvme_async_probe_ctx(struct nvme_async_probe_ctx *ctx)
5611 : : {
5612 [ + - + - : 1756 : spdk_keyring_put_key(ctx->drv_opts.tls_psk);
+ - ]
5613 [ + - + - : 1756 : spdk_keyring_put_key(ctx->drv_opts.dhchap_key);
+ - ]
5614 [ + - + - : 1756 : spdk_keyring_put_key(ctx->drv_opts.dhchap_ctrlr_key);
+ - ]
5615 [ + - + - ]: 1756 : free(ctx->base_name);
5616 : 1756 : free(ctx);
5617 : 1756 : }
5618 : :
5619 : : static void
5620 : 1747 : populate_namespaces_cb(struct nvme_async_probe_ctx *ctx, int rc)
5621 : : {
5622 [ + + + - : 1747 : if (ctx->cb_fn) {
+ - ]
5623 [ + - + - : 1747 : ctx->cb_fn(ctx->cb_ctx, ctx->reported_bdevs, rc);
- + + - +
- + - + -
+ - ]
5624 : 64 : }
5625 : :
5626 [ + - + - ]: 1747 : ctx->namespaces_populated = true;
5627 [ + + + + : 1747 : if (ctx->probe_done) {
+ - + - ]
5628 : : /* The probe was already completed, so we need to free the context
5629 : : * here. This can happen for cases like OCSSD, where we need to
5630 : : * send additional commands to the SSD after attach.
5631 : : */
5632 : 514 : free_nvme_async_probe_ctx(ctx);
5633 : 32 : }
5634 : 1747 : }
5635 : :
5636 : : static int
5637 : 37771 : bdev_nvme_remove_poller(void *ctx)
5638 : : {
5639 : 15884 : struct spdk_nvme_transport_id trid_pcie;
5640 : :
5641 [ + + ]: 37771 : if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
5642 : 869 : spdk_poller_unregister(&g_hotplug_poller);
5643 : 869 : return SPDK_POLLER_IDLE;
5644 : : }
5645 : :
5646 [ + + ]: 36902 : memset(&trid_pcie, 0, sizeof(trid_pcie));
5647 : 36902 : spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE);
5648 : :
5649 [ + + ]: 36902 : if (spdk_nvme_scan_attached(&trid_pcie)) {
5650 [ # # # # ]: 0 : SPDK_ERRLOG_RATELIMIT("spdk_nvme_scan_attached() failed\n");
5651 : 0 : }
5652 : :
5653 : 36902 : return SPDK_POLLER_BUSY;
5654 : 273 : }
5655 : :
5656 : : static void set_nvme_hotplug_period_cb(void *_ctx);
5657 : :
5658 : : static void
5659 : 1764 : nvme_ctrlr_create_done(struct nvme_ctrlr *nvme_ctrlr,
5660 : : struct nvme_async_probe_ctx *ctx)
5661 : : {
5662 [ + - + - : 1764 : struct spdk_nvme_transport_id *trid = &nvme_ctrlr->active_path_id->trid;
+ - ]
5663 : :
5664 [ + + + - : 1764 : if (spdk_nvme_trtype_is_fabrics(trid->trtype)) {
- + ]
5665 [ + + + + : 1216 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr was created to %s:%s\n",
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5666 : : trid->traddr, trid->trsvcid);
5667 : 61 : } else {
5668 [ + + + + : 548 : NVME_CTRLR_INFOLOG(nvme_ctrlr, "ctrlr was created\n");
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
5669 : : }
5670 : :
5671 : 1836 : spdk_io_device_register(nvme_ctrlr,
5672 : : bdev_nvme_create_ctrlr_channel_cb,
5673 : : bdev_nvme_destroy_ctrlr_channel_cb,
5674 : : sizeof(struct nvme_ctrlr_channel),
5675 [ + - + - : 1764 : nvme_ctrlr->nbdev_ctrlr->name);
+ - + - ]
5676 : :
5677 : 1764 : nvme_ctrlr_populate_namespaces(nvme_ctrlr, ctx);
5678 : :
5679 [ + + ]: 1764 : if (g_hotplug_poller == NULL) {
5680 : 1261 : spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, NULL);
5681 : 13 : }
5682 : 1764 : }
5683 : :
5684 : : static void
5685 : 448 : nvme_ctrlr_init_ana_log_page_done(void *_ctx, const struct spdk_nvme_cpl *cpl)
5686 : : {
5687 : 448 : struct nvme_ctrlr *nvme_ctrlr = _ctx;
5688 [ # # # # ]: 448 : struct nvme_async_probe_ctx *ctx = nvme_ctrlr->probe_ctx;
5689 : :
5690 [ # # # # ]: 448 : nvme_ctrlr->probe_ctx = NULL;
5691 : :
5692 [ + - + + : 448 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
5693 : 0 : nvme_ctrlr_delete(nvme_ctrlr);
5694 : :
5695 [ # # ]: 0 : if (ctx != NULL) {
5696 [ # # # # ]: 0 : ctx->reported_bdevs = 0;
5697 : 0 : populate_namespaces_cb(ctx, -1);
5698 : 0 : }
5699 : 0 : return;
5700 : : }
5701 : :
5702 : 448 : nvme_ctrlr_create_done(nvme_ctrlr, ctx);
5703 : 31 : }
5704 : :
5705 : : static int
5706 : 448 : nvme_ctrlr_init_ana_log_page(struct nvme_ctrlr *nvme_ctrlr,
5707 : : struct nvme_async_probe_ctx *ctx)
5708 : : {
5709 [ # # # # ]: 448 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
5710 : : const struct spdk_nvme_ctrlr_data *cdata;
5711 : : uint32_t ana_log_page_size;
5712 : :
5713 : 448 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
5714 : :
5715 : : /* Set buffer size enough to include maximum number of allowed namespaces. */
5716 [ # # # # ]: 479 : ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + cdata->nanagrpid *
5717 [ # # # # ]: 448 : sizeof(struct spdk_nvme_ana_group_descriptor) + cdata->mnan *
5718 : : sizeof(uint32_t);
5719 : :
5720 [ # # # # ]: 448 : nvme_ctrlr->ana_log_page = spdk_zmalloc(ana_log_page_size, 64, NULL,
5721 : : SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_DMA);
5722 [ + + # # : 448 : if (nvme_ctrlr->ana_log_page == NULL) {
# # ]
5723 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "could not allocate ANA log page buffer\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5724 : 0 : return -ENXIO;
5725 : : }
5726 : :
5727 : : /* Each descriptor in a ANA log page is not ensured to be 8-bytes aligned.
5728 : : * Hence copy each descriptor to a temporary area when parsing it.
5729 : : *
5730 : : * Allocate a buffer whose size is as large as ANA log page buffer because
5731 : : * we do not know the size of a descriptor until actually reading it.
5732 : : */
5733 [ # # # # ]: 448 : nvme_ctrlr->copied_ana_desc = calloc(1, ana_log_page_size);
5734 [ + + # # : 448 : if (nvme_ctrlr->copied_ana_desc == NULL) {
# # ]
5735 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "could not allocate a buffer to parse ANA descriptor\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5736 : 0 : return -ENOMEM;
5737 : : }
5738 : :
5739 [ # # # # ]: 448 : nvme_ctrlr->max_ana_log_page_size = ana_log_page_size;
5740 : :
5741 [ # # # # ]: 448 : nvme_ctrlr->probe_ctx = ctx;
5742 : :
5743 : : /* Then, set the read size only to include the current active namespaces. */
5744 : 448 : ana_log_page_size = nvme_ctrlr_get_ana_log_page_size(nvme_ctrlr);
5745 : :
5746 [ - + # # : 448 : if (ana_log_page_size > nvme_ctrlr->max_ana_log_page_size) {
# # ]
5747 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "ANA log page size %" PRIu32 " is larger than allowed %" PRIu32 "\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5748 : : ana_log_page_size, nvme_ctrlr->max_ana_log_page_size);
5749 : 0 : return -EINVAL;
5750 : : }
5751 : :
5752 : 479 : return spdk_nvme_ctrlr_cmd_get_log_page(ctrlr,
5753 : : SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS,
5754 : : SPDK_NVME_GLOBAL_NS_TAG,
5755 [ # # # # ]: 448 : nvme_ctrlr->ana_log_page,
5756 : 31 : ana_log_page_size, 0,
5757 : : nvme_ctrlr_init_ana_log_page_done,
5758 : 31 : nvme_ctrlr);
5759 : 31 : }
5760 : :
5761 : : /* hostnqn and subnqn were already verified before attaching a controller.
5762 : : * Hence check only the multipath capability and cntlid here.
5763 : : */
5764 : : static bool
5765 : 77 : bdev_nvme_check_multipath(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct spdk_nvme_ctrlr *ctrlr)
5766 : : {
5767 : : struct nvme_ctrlr *tmp;
5768 : : const struct spdk_nvme_ctrlr_data *cdata, *tmp_cdata;
5769 : :
5770 : 77 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
5771 : :
5772 [ - + # # : 77 : if (!cdata->cmic.multi_ctrlr) {
# # ]
5773 [ # # # # ]: 0 : SPDK_ERRLOG("Ctrlr%u does not support multipath.\n", cdata->cntlid);
5774 : 0 : return false;
5775 : : }
5776 : :
5777 [ + + # # : 158 : TAILQ_FOREACH(tmp, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
5778 [ # # # # ]: 85 : tmp_cdata = spdk_nvme_ctrlr_get_data(tmp->ctrlr);
5779 : :
5780 [ + + # # : 85 : if (!tmp_cdata->cmic.multi_ctrlr) {
# # ]
5781 [ # # # # : 0 : NVME_CTRLR_ERRLOG(tmp, "Ctrlr%u does not support multipath.\n", cdata->cntlid);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5782 : 0 : return false;
5783 : : }
5784 [ + + # # : 85 : if (cdata->cntlid == tmp_cdata->cntlid) {
# # # # #
# ]
5785 [ + - # # : 4 : NVME_CTRLR_ERRLOG(tmp, "cntlid %u are duplicated.\n", tmp_cdata->cntlid);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
5786 : 4 : return false;
5787 : : }
5788 : 17 : }
5789 : :
5790 : 73 : return true;
5791 : 16 : }
5792 : :
5793 : :
5794 : : static int
5795 : 1768 : nvme_bdev_ctrlr_create(const char *name, struct nvme_ctrlr *nvme_ctrlr)
5796 : : {
5797 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
5798 [ + - + - ]: 1768 : struct spdk_nvme_ctrlr *ctrlr = nvme_ctrlr->ctrlr;
5799 : : struct nvme_ctrlr *nctrlr;
5800 : 1768 : int rc = 0;
5801 : :
5802 [ + + ]: 1768 : pthread_mutex_lock(&g_bdev_nvme_mutex);
5803 : :
5804 : 1768 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
5805 [ + + ]: 1768 : if (nbdev_ctrlr != NULL) {
5806 [ + + ]: 77 : if (!bdev_nvme_check_multipath(nbdev_ctrlr, ctrlr)) {
5807 : 4 : rc = -EINVAL;
5808 : 4 : goto exit;
5809 : : }
5810 [ + + # # : 154 : TAILQ_FOREACH(nctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
5811 [ + + - + : 81 : if (nctrlr->opts.multipath != nvme_ctrlr->opts.multipath) {
- + # # #
# # # # #
# # # # ]
5812 : : /* All controllers with the same name must be configured the same
5813 : : * way, either for multipath or failover. If the configuration doesn't
5814 : : * match - report error.
5815 : : */
5816 : 0 : rc = -EINVAL;
5817 : 0 : goto exit;
5818 : : }
5819 : 17 : }
5820 : 15 : } else {
5821 : 1691 : nbdev_ctrlr = calloc(1, sizeof(*nbdev_ctrlr));
5822 [ + + ]: 1691 : if (nbdev_ctrlr == NULL) {
5823 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to allocate nvme_bdev_ctrlr.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5824 : 0 : rc = -ENOMEM;
5825 : 0 : goto exit;
5826 : : }
5827 [ + + + - : 1691 : nbdev_ctrlr->name = strdup(name);
+ - ]
5828 [ + + + - : 1691 : if (nbdev_ctrlr->name == NULL) {
+ - ]
5829 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "Failed to allocate name of nvme_bdev_ctrlr.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
5830 : 0 : free(nbdev_ctrlr);
5831 : 0 : goto exit;
5832 : : }
5833 [ + - + - : 1691 : TAILQ_INIT(&nbdev_ctrlr->ctrlrs);
+ - + - +
- + - + -
+ - ]
5834 [ + - + - : 1691 : TAILQ_INIT(&nbdev_ctrlr->bdevs);
+ - + - +
- + - + -
+ - ]
5835 [ + - + - : 1691 : TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nbdev_ctrlr, tailq);
+ - + - +
- + - + -
+ - + - +
- + - +
- ]
5836 : : }
5837 [ + - + - ]: 1764 : nvme_ctrlr->nbdev_ctrlr = nbdev_ctrlr;
5838 [ + - + - : 1764 : TAILQ_INSERT_TAIL(&nbdev_ctrlr->ctrlrs, nvme_ctrlr, tailq);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
5839 : 1695 : exit:
5840 [ + + ]: 1768 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
5841 : 1768 : return rc;
5842 : : }
5843 : :
5844 : : static int
5845 : 1768 : nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr,
5846 : : const char *name,
5847 : : const struct spdk_nvme_transport_id *trid,
5848 : : struct nvme_async_probe_ctx *ctx)
5849 : : {
5850 : : struct nvme_ctrlr *nvme_ctrlr;
5851 : : struct nvme_path_id *path_id;
5852 : : const struct spdk_nvme_ctrlr_data *cdata;
5853 : 1768 : struct spdk_event_handler_opts opts = {
5854 : : .opts_size = SPDK_SIZEOF(&opts, fd_type),
5855 : : };
5856 : : uint64_t period;
5857 : : int fd, rc;
5858 : :
5859 : 1768 : nvme_ctrlr = calloc(1, sizeof(*nvme_ctrlr));
5860 [ + + ]: 1768 : if (nvme_ctrlr == NULL) {
5861 : 0 : SPDK_ERRLOG("Failed to allocate device struct\n");
5862 : 0 : return -ENOMEM;
5863 : : }
5864 : :
5865 [ + + + - ]: 1768 : rc = pthread_mutex_init(&nvme_ctrlr->mutex, NULL);
5866 [ - + ]: 1768 : if (rc != 0) {
5867 : 0 : free(nvme_ctrlr);
5868 : 0 : return rc;
5869 : : }
5870 : :
5871 [ + - + - : 1768 : TAILQ_INIT(&nvme_ctrlr->trids);
+ - + - +
- + - + -
+ - ]
5872 [ + - + - : 1768 : TAILQ_INIT(&nvme_ctrlr->pending_resets);
+ - + - +
- + - + -
+ - ]
5873 [ + - + - : 1768 : RB_INIT(&nvme_ctrlr->namespaces);
+ - ]
5874 : :
5875 : : /* Get another reference to the key, so the first one can be released from probe_ctx */
5876 [ + + ]: 1768 : if (ctx != NULL) {
5877 [ + + + - : 1656 : if (ctx->drv_opts.tls_psk != NULL) {
+ - + - ]
5878 [ # # # # ]: 42 : nvme_ctrlr->psk = spdk_keyring_get_key(
5879 [ # # # # : 0 : spdk_key_get_name(ctx->drv_opts.tls_psk));
# # ]
5880 [ - + # # : 42 : if (nvme_ctrlr->psk == NULL) {
# # ]
5881 : : /* Could only happen if the key was removed in the meantime */
5882 [ # # # # : 0 : SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n",
# # ]
5883 : : spdk_key_get_name(ctx->drv_opts.tls_psk));
5884 : 0 : rc = -ENOKEY;
5885 : 0 : goto err;
5886 : : }
5887 : 0 : }
5888 : :
5889 [ + + + - : 1656 : if (ctx->drv_opts.dhchap_key != NULL) {
+ - + - ]
5890 [ # # # # ]: 628 : nvme_ctrlr->dhchap_key = spdk_keyring_get_key(
5891 [ # # # # : 0 : spdk_key_get_name(ctx->drv_opts.dhchap_key));
# # ]
5892 [ - + # # : 628 : if (nvme_ctrlr->dhchap_key == NULL) {
# # ]
5893 [ # # # # : 0 : SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n",
# # ]
5894 : : spdk_key_get_name(ctx->drv_opts.dhchap_key));
5895 : 0 : rc = -ENOKEY;
5896 : 0 : goto err;
5897 : : }
5898 : 0 : }
5899 : :
5900 [ + + + - : 1656 : if (ctx->drv_opts.dhchap_ctrlr_key != NULL) {
+ - + - ]
5901 [ # # # # ]: 484 : nvme_ctrlr->dhchap_ctrlr_key =
5902 : 484 : spdk_keyring_get_key(
5903 [ # # # # : 0 : spdk_key_get_name(ctx->drv_opts.dhchap_ctrlr_key));
# # ]
5904 [ - + # # : 484 : if (nvme_ctrlr->dhchap_ctrlr_key == NULL) {
# # ]
5905 [ # # # # : 0 : SPDK_ERRLOG("Couldn't get a reference to the key '%s'\n",
# # ]
5906 : : spdk_key_get_name(ctx->drv_opts.dhchap_ctrlr_key));
5907 : 0 : rc = -ENOKEY;
5908 : 0 : goto err;
5909 : : }
5910 : 0 : }
5911 : 59 : }
5912 : :
5913 : : /* Check if we manage to enable interrupts on the controller. */
5914 [ + + + - : 1768 : if (spdk_interrupt_mode_is_enabled() && ctx != NULL && !ctx->drv_opts.enable_interrupts) {
- + - + #
# # # #
# ]
5915 : 0 : SPDK_ERRLOG("Failed to enable interrupts on the controller\n");
5916 : 0 : rc = -ENOTSUP;
5917 : 0 : goto err;
5918 : : }
5919 : :
5920 : 1768 : path_id = calloc(1, sizeof(*path_id));
5921 [ + + ]: 1768 : if (path_id == NULL) {
5922 : 0 : SPDK_ERRLOG("Failed to allocate trid entry pointer\n");
5923 : 0 : rc = -ENOMEM;
5924 : 0 : goto err;
5925 : : }
5926 : :
5927 [ + - ]: 1768 : path_id->trid = *trid;
5928 [ + + ]: 1768 : if (ctx != NULL) {
5929 [ + + + + : 1656 : memcpy(path_id->hostid.hostaddr, ctx->drv_opts.src_addr, sizeof(path_id->hostid.hostaddr));
+ - + - +
- + - ]
5930 [ + + + + : 1656 : memcpy(path_id->hostid.hostsvcid, ctx->drv_opts.src_svcid, sizeof(path_id->hostid.hostsvcid));
+ - + - +
- + - ]
5931 : 59 : }
5932 [ + - + - ]: 1768 : nvme_ctrlr->active_path_id = path_id;
5933 [ + + + - : 1768 : TAILQ_INSERT_HEAD(&nvme_ctrlr->trids, path_id, link);
+ - + - +
- + - - +
# # # # #
# # # # #
# # # # #
# + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - ]
5934 : :
5935 [ + - + - ]: 1768 : nvme_ctrlr->thread = spdk_get_thread();
5936 [ + - + - ]: 1768 : nvme_ctrlr->ctrlr = ctrlr;
5937 [ + - + - ]: 1768 : nvme_ctrlr->ref = 1;
5938 : :
5939 [ - + ]: 1768 : if (spdk_nvme_ctrlr_is_ocssd_supported(ctrlr)) {
5940 : 0 : SPDK_ERRLOG("OCSSDs are not supported");
5941 : 0 : rc = -ENOTSUP;
5942 : 0 : goto err;
5943 : : }
5944 : :
5945 [ + + ]: 1768 : if (ctx != NULL) {
5946 [ + + + + : 1656 : memcpy(&nvme_ctrlr->opts, &ctx->bdev_opts, sizeof(ctx->bdev_opts));
+ - + - ]
5947 : 59 : } else {
5948 [ # # ]: 112 : spdk_bdev_nvme_get_default_ctrlr_opts(&nvme_ctrlr->opts);
5949 : : }
5950 : :
5951 [ + + + - ]: 1768 : period = spdk_interrupt_mode_is_enabled() ? 0 : g_opts.nvme_adminq_poll_period_us;
5952 : :
5953 [ + - + - ]: 1768 : nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, nvme_ctrlr,
5954 : : period);
5955 : :
5956 [ + + ]: 1768 : if (spdk_interrupt_mode_is_enabled()) {
5957 [ # # # # ]: 1 : spdk_poller_register_interrupt(nvme_ctrlr->adminq_timer_poller, NULL, NULL);
5958 : :
5959 [ # # # # ]: 1 : fd = spdk_nvme_ctrlr_get_admin_qp_fd(nvme_ctrlr->ctrlr, &opts);
5960 [ - + ]: 1 : if (fd < 0) {
5961 : 0 : rc = fd;
5962 : 0 : goto err;
5963 : : }
5964 : :
5965 [ # # # # ]: 1 : nvme_ctrlr->intr = SPDK_INTERRUPT_REGISTER_EXT(fd, bdev_nvme_poll_adminq,
5966 : : nvme_ctrlr, &opts);
5967 [ - + # # : 1 : if (!nvme_ctrlr->intr) {
# # ]
5968 : 0 : rc = -EINVAL;
5969 : 0 : goto err;
5970 : : }
5971 : 0 : }
5972 : :
5973 [ + + + - ]: 1768 : if (g_opts.timeout_us > 0) {
5974 : : /* Register timeout callback. Timeout values for IO vs. admin reqs can be different. */
5975 : : /* If timeout_admin_us is 0 (not specified), admin uses same timeout as IO. */
5976 [ # # # # ]: 0 : uint64_t adm_timeout_us = (g_opts.timeout_admin_us == 0) ?
5977 [ # # # # ]: 0 : g_opts.timeout_us : g_opts.timeout_admin_us;
5978 [ # # ]: 0 : spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_opts.timeout_us,
5979 : 0 : adm_timeout_us, timeout_cb, nvme_ctrlr);
5980 : 0 : }
5981 : :
5982 : 1768 : spdk_nvme_ctrlr_register_aer_callback(ctrlr, aer_cb, nvme_ctrlr);
5983 : 1768 : spdk_nvme_ctrlr_set_remove_cb(ctrlr, remove_cb, nvme_ctrlr);
5984 : :
5985 [ + + ]: 1768 : if (spdk_nvme_ctrlr_get_flags(ctrlr) &
5986 : : SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED) {
5987 [ # # # # ]: 21 : nvme_ctrlr->opal_dev = spdk_opal_dev_construct(ctrlr);
5988 : 0 : }
5989 : :
5990 : 1768 : rc = nvme_bdev_ctrlr_create(name, nvme_ctrlr);
5991 [ + + ]: 1768 : if (rc != 0) {
5992 : 4 : goto err;
5993 : : }
5994 : :
5995 : 1764 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
5996 : :
5997 [ + + - + : 1764 : if (cdata->cmic.ana_reporting) {
- + ]
5998 : 448 : rc = nvme_ctrlr_init_ana_log_page(nvme_ctrlr, ctx);
5999 [ + - ]: 448 : if (rc == 0) {
6000 : 448 : return 0;
6001 : : }
6002 : 0 : } else {
6003 : 1316 : nvme_ctrlr_create_done(nvme_ctrlr, ctx);
6004 : 1316 : return 0;
6005 : : }
6006 : :
6007 : 3 : err:
6008 : 4 : nvme_ctrlr_delete(nvme_ctrlr);
6009 : 4 : return rc;
6010 : 73 : }
6011 : :
6012 : : void
6013 : 1707 : spdk_bdev_nvme_get_default_ctrlr_opts(struct spdk_bdev_nvme_ctrlr_opts *opts)
6014 : : {
6015 [ + - + - ]: 1707 : opts->prchk_flags = 0;
6016 [ + - + - : 1707 : opts->ctrlr_loss_timeout_sec = g_opts.ctrlr_loss_timeout_sec;
+ - ]
6017 [ + - + - : 1707 : opts->reconnect_delay_sec = g_opts.reconnect_delay_sec;
+ - ]
6018 [ + - + - : 1707 : opts->fast_io_fail_timeout_sec = g_opts.fast_io_fail_timeout_sec;
+ - ]
6019 [ + - + - ]: 1707 : opts->multipath = true;
6020 : 1707 : }
6021 : :
6022 : : static void
6023 : 56 : attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
6024 : : struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *drv_opts)
6025 : : {
6026 : : char *name;
6027 : :
6028 [ # # ]: 56 : name = spdk_sprintf_alloc("HotInNvme%d", g_hot_insert_nvme_controller_index++);
6029 [ - + ]: 56 : if (!name) {
6030 : 0 : SPDK_ERRLOG("Failed to assign name to NVMe device\n");
6031 : 0 : return;
6032 : : }
6033 : :
6034 [ + - ]: 56 : if (nvme_ctrlr_create(ctrlr, name, trid, NULL) == 0) {
6035 [ - + - + : 56 : SPDK_DEBUGLOG(bdev_nvme, "Attached to %s (%s)\n", trid->traddr, name);
# # # # ]
6036 : 0 : } else {
6037 [ # # ]: 0 : SPDK_ERRLOG("Failed to attach to %s (%s)\n", trid->traddr, name);
6038 : : }
6039 : :
6040 : 56 : free(name);
6041 : 0 : }
6042 : :
6043 : : static void
6044 : 1764 : _nvme_ctrlr_destruct(void *ctx)
6045 : : {
6046 : 1764 : struct nvme_ctrlr *nvme_ctrlr = ctx;
6047 : :
6048 : 1764 : nvme_ctrlr_depopulate_namespaces(nvme_ctrlr);
6049 : 1764 : nvme_ctrlr_put_ref(nvme_ctrlr);
6050 : 1764 : }
6051 : :
6052 : : static int
6053 : 1027 : bdev_nvme_delete_ctrlr_unsafe(struct nvme_ctrlr *nvme_ctrlr, bool hotplug)
6054 : : {
6055 : : struct nvme_probe_skip_entry *entry;
6056 : :
6057 : : /* The controller's destruction was already started */
6058 [ - + # # ]: 1027 : if (nvme_ctrlr->destruct) {
6059 : 0 : return -EALREADY;
6060 : : }
6061 : :
6062 [ + + + + : 1027 : if (!hotplug &&
# # ]
6063 [ + + # # : 979 : nvme_ctrlr->active_path_id->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
# # # # #
# ]
6064 : 50 : entry = calloc(1, sizeof(*entry));
6065 [ - + ]: 50 : if (!entry) {
6066 : 0 : return -ENOMEM;
6067 : : }
6068 [ # # # # : 50 : entry->trid = nvme_ctrlr->active_path_id->trid;
# # # # ]
6069 [ # # # # : 50 : TAILQ_INSERT_TAIL(&g_skipped_nvme_ctrlrs, entry, tailq);
# # # # #
# # # # #
# # # # #
# # # #
# ]
6070 : 3 : }
6071 : :
6072 [ # # ]: 1027 : nvme_ctrlr->destruct = true;
6073 : 1027 : return 0;
6074 : 61 : }
6075 : :
6076 : : static int
6077 : 79 : bdev_nvme_delete_ctrlr(struct nvme_ctrlr *nvme_ctrlr, bool hotplug)
6078 : : {
6079 : : int rc;
6080 : :
6081 [ - + # # ]: 79 : pthread_mutex_lock(&nvme_ctrlr->mutex);
6082 [ # # ]: 79 : rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, hotplug);
6083 [ - + # # ]: 79 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
6084 : :
6085 [ + + ]: 79 : if (rc == 0) {
6086 : 79 : _nvme_ctrlr_destruct(nvme_ctrlr);
6087 [ # # ]: 2 : } else if (rc == -EALREADY) {
6088 : 0 : rc = 0;
6089 : 0 : }
6090 : :
6091 : 79 : return rc;
6092 : : }
6093 : :
6094 : : static void
6095 : 48 : remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr)
6096 : : {
6097 : 48 : struct nvme_ctrlr *nvme_ctrlr = cb_ctx;
6098 : :
6099 : 48 : bdev_nvme_delete_ctrlr(nvme_ctrlr, true);
6100 : 48 : }
6101 : :
6102 : : static int
6103 : 20066 : bdev_nvme_hotplug_probe(void *arg)
6104 : : {
6105 [ - + ]: 20066 : if (g_hotplug_probe_ctx == NULL) {
6106 : 0 : spdk_poller_unregister(&g_hotplug_probe_poller);
6107 : 0 : return SPDK_POLLER_IDLE;
6108 : : }
6109 : :
6110 [ + + ]: 20066 : if (spdk_nvme_probe_poll_async(g_hotplug_probe_ctx) != -EAGAIN) {
6111 : 3881 : g_hotplug_probe_ctx = NULL;
6112 : 3881 : spdk_poller_unregister(&g_hotplug_probe_poller);
6113 : 0 : }
6114 : :
6115 : 20066 : return SPDK_POLLER_BUSY;
6116 : 0 : }
6117 : :
6118 : : static int
6119 : 4028 : bdev_nvme_hotplug(void *arg)
6120 : : {
6121 : 1952 : struct spdk_nvme_transport_id trid_pcie;
6122 : :
6123 [ + + ]: 4028 : if (g_hotplug_probe_ctx) {
6124 : 147 : return SPDK_POLLER_BUSY;
6125 : : }
6126 : :
6127 [ - + ]: 3881 : memset(&trid_pcie, 0, sizeof(trid_pcie));
6128 : 3881 : spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE);
6129 : :
6130 : 3881 : g_hotplug_probe_ctx = spdk_nvme_probe_async(&trid_pcie, NULL,
6131 : : hotplug_probe_cb, attach_cb, NULL);
6132 : :
6133 [ + - ]: 3881 : if (g_hotplug_probe_ctx) {
6134 [ - + # # ]: 3881 : assert(g_hotplug_probe_poller == NULL);
6135 : 3881 : g_hotplug_probe_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug_probe, NULL, 1000);
6136 : 0 : }
6137 : :
6138 : 3881 : return SPDK_POLLER_BUSY;
6139 : 0 : }
6140 : :
6141 : : void
6142 : 1032 : spdk_bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts, size_t opts_size)
6143 : : {
6144 [ + + ]: 1032 : if (!opts) {
6145 : 0 : SPDK_ERRLOG("opts should not be NULL\n");
6146 : 0 : return;
6147 : : }
6148 : :
6149 [ + + ]: 1032 : if (!opts_size) {
6150 : 0 : SPDK_ERRLOG("opts_size should not be zero value\n");
6151 : 0 : return;
6152 : : }
6153 : :
6154 [ + - + - ]: 1032 : opts->opts_size = opts_size;
6155 : :
6156 : : #define SET_FIELD(field, defval) \
6157 : : opts->field = SPDK_GET_FIELD(&g_opts, field, defval, opts_size); \
6158 : :
6159 [ + + + - : 1032 : SET_FIELD(action_on_timeout, 0);
+ - + - ]
6160 [ + + + - : 1032 : SET_FIELD(keep_alive_timeout_ms, 0);
+ - + - ]
6161 [ + + + - : 1032 : SET_FIELD(timeout_us, 0);
+ - + - ]
6162 [ + + + - : 1032 : SET_FIELD(timeout_admin_us, 0);
+ - + - ]
6163 [ + + + - : 1032 : SET_FIELD(transport_retry_count, 0);
+ - + - ]
6164 [ + + + - : 1032 : SET_FIELD(arbitration_burst, 0);
+ - + - ]
6165 [ + + + - : 1032 : SET_FIELD(low_priority_weight, 0);
+ - + - ]
6166 [ + + + - : 1032 : SET_FIELD(medium_priority_weight, 0);
+ - + - ]
6167 [ + + + - : 1032 : SET_FIELD(high_priority_weight, 0);
+ - + - ]
6168 [ + + + - : 1032 : SET_FIELD(io_queue_requests, 0);
+ - + - ]
6169 [ + + + - : 1032 : SET_FIELD(nvme_adminq_poll_period_us, 0);
+ - + - ]
6170 [ + + + - : 1032 : SET_FIELD(nvme_ioq_poll_period_us, 0);
+ - + - ]
6171 [ + + + + : 1032 : SET_FIELD(delay_cmd_submit, 0);
+ - + - +
- ]
6172 [ + + + - : 1032 : SET_FIELD(bdev_retry_count, 0);
+ - + - ]
6173 [ + + + - : 1032 : SET_FIELD(ctrlr_loss_timeout_sec, 0);
+ - + - ]
6174 [ + + + - : 1032 : SET_FIELD(reconnect_delay_sec, 0);
+ - + - ]
6175 [ + + + - : 1032 : SET_FIELD(fast_io_fail_timeout_sec, 0);
+ - + - ]
6176 [ + + + - : 1032 : SET_FIELD(transport_ack_timeout, 0);
+ - + - ]
6177 [ + + + + : 1032 : SET_FIELD(disable_auto_failback, false);
+ + + - ]
6178 [ + + + + : 1032 : SET_FIELD(generate_uuids, false);
+ + + - +
- ]
6179 [ + + + - : 1032 : SET_FIELD(transport_tos, 0);
+ - ]
6180 [ + + + + : 1032 : SET_FIELD(nvme_error_stat, false);
+ + + - +
- ]
6181 [ + + + + : 1032 : SET_FIELD(io_path_stat, false);
+ + + - ]
6182 [ + + + + : 1032 : SET_FIELD(allow_accel_sequence, false);
+ + + - +
- ]
6183 [ + + + - : 1032 : SET_FIELD(rdma_srq_size, 0);
+ - + - ]
6184 [ + + + - : 1032 : SET_FIELD(rdma_max_cq_size, 0);
+ - + - ]
6185 [ + + + - : 1032 : SET_FIELD(rdma_cm_event_timeout_ms, 0);
+ - + - ]
6186 [ + + + - : 1032 : SET_FIELD(dhchap_digests, 0);
+ - + - ]
6187 [ + + + - : 1032 : SET_FIELD(dhchap_dhgroups, 0);
+ - + - ]
6188 [ + - + + : 1032 : SET_FIELD(rdma_umr_per_io, false);
+ + + - +
- ]
6189 : :
6190 : : #undef SET_FIELD
6191 : :
6192 : : /* Do not remove this statement, you should always update this statement when you adding a new field,
6193 : : * and do not forget to add the SET_FIELD statement for your added field. */
6194 : : SPDK_STATIC_ASSERT(sizeof(struct spdk_bdev_nvme_opts) == 128, "Incorrect size");
6195 : 43 : }
6196 : :
6197 : : static bool bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec,
6198 : : uint32_t reconnect_delay_sec,
6199 : : uint32_t fast_io_fail_timeout_sec);
6200 : :
6201 : : static int
6202 : 1032 : bdev_nvme_validate_opts(const struct spdk_bdev_nvme_opts *opts)
6203 : : {
6204 [ + + + + : 1032 : if ((opts->timeout_us == 0) && (opts->timeout_admin_us != 0)) {
+ - + - +
- + - ]
6205 : : /* Can't set timeout_admin_us without also setting timeout_us */
6206 : 0 : SPDK_WARNLOG("Invalid options: Can't have (timeout_us == 0) with (timeout_admin_us > 0)\n");
6207 : 0 : return -EINVAL;
6208 : : }
6209 : :
6210 [ + + + - : 1032 : if (opts->bdev_retry_count < -1) {
- + ]
6211 : 0 : SPDK_WARNLOG("Invalid option: bdev_retry_count can't be less than -1.\n");
6212 : 0 : return -EINVAL;
6213 : : }
6214 : :
6215 [ + + + - : 1075 : if (!bdev_nvme_check_io_error_resiliency_params(opts->ctrlr_loss_timeout_sec,
+ - ]
6216 [ + - + - ]: 1032 : opts->reconnect_delay_sec,
6217 [ + - + - ]: 1032 : opts->fast_io_fail_timeout_sec)) {
6218 : 0 : return -EINVAL;
6219 : : }
6220 : :
6221 : 1032 : return 0;
6222 : 43 : }
6223 : :
6224 : : int
6225 : 1032 : spdk_bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts)
6226 : : {
6227 : 133 : struct spdk_nvme_transport_opts drv_opts;
6228 : : int ret;
6229 : :
6230 [ + + ]: 1032 : if (!opts) {
6231 : 0 : SPDK_ERRLOG("opts cannot be NULL\n");
6232 : 0 : return -1;
6233 : : }
6234 : :
6235 [ + + + - : 1032 : if (!opts->opts_size) {
+ - ]
6236 : 0 : SPDK_ERRLOG("opts_size inside opts cannot be zero value\n");
6237 : 0 : return -1;
6238 : : }
6239 : :
6240 : 1032 : ret = bdev_nvme_validate_opts(opts);
6241 [ - + ]: 1032 : if (ret) {
6242 : 0 : SPDK_WARNLOG("Failed to set nvme opts.\n");
6243 : 0 : return ret;
6244 : : }
6245 : :
6246 [ + + ]: 1032 : if (g_bdev_nvme_init_thread != NULL) {
6247 [ - + ]: 653 : if (!TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
6248 : 0 : return -EPERM;
6249 : : }
6250 : 1 : }
6251 : :
6252 : 1032 : spdk_nvme_transport_get_opts(&drv_opts, sizeof(drv_opts));
6253 [ + + + - : 1032 : if (opts->rdma_srq_size != 0) {
+ - ]
6254 [ # # # # ]: 0 : drv_opts.rdma_srq_size = opts->rdma_srq_size;
6255 : 0 : }
6256 [ + + + - : 1032 : if (opts->rdma_max_cq_size != 0) {
+ - ]
6257 [ # # # # : 0 : drv_opts.rdma_max_cq_size = opts->rdma_max_cq_size;
# # ]
6258 : 0 : }
6259 [ + + + - : 1032 : if (opts->rdma_cm_event_timeout_ms != 0) {
+ - ]
6260 [ # # # # : 0 : drv_opts.rdma_cm_event_timeout_ms = opts->rdma_cm_event_timeout_ms;
# # ]
6261 : 0 : }
6262 [ + + + + : 1032 : if (drv_opts.rdma_umr_per_io != opts->rdma_umr_per_io) {
+ + + - +
- + - ]
6263 [ # # # # : 0 : drv_opts.rdma_umr_per_io = opts->rdma_umr_per_io;
# # # # ]
6264 : 0 : }
6265 : 1032 : ret = spdk_nvme_transport_set_opts(&drv_opts, sizeof(drv_opts));
6266 [ - + ]: 1032 : if (ret) {
6267 : 0 : SPDK_ERRLOG("Failed to set NVMe transport opts.\n");
6268 : 0 : return ret;
6269 : : }
6270 : :
6271 : : #define SET_FIELD(field, defval) \
6272 : : g_opts.field = SPDK_GET_FIELD(opts, field, defval, opts->opts_size); \
6273 : :
6274 [ + + + - : 1032 : SET_FIELD(action_on_timeout, 0);
+ - + - +
- + - ]
6275 [ + + + - : 1032 : SET_FIELD(keep_alive_timeout_ms, 0);
+ - + - +
- + - ]
6276 [ + + + - : 1032 : SET_FIELD(timeout_us, 0);
+ - + - +
- + - ]
6277 [ + + + - : 1032 : SET_FIELD(timeout_admin_us, 0);
+ - + - +
- + - ]
6278 [ + + + - : 1032 : SET_FIELD(transport_retry_count, 0);
+ - + - +
- + - ]
6279 [ + + + - : 1032 : SET_FIELD(arbitration_burst, 0);
+ - + - +
- + - ]
6280 [ + + + - : 1032 : SET_FIELD(low_priority_weight, 0);
+ - + - +
- + - ]
6281 [ + + + - : 1032 : SET_FIELD(medium_priority_weight, 0);
+ - + - +
- + - ]
6282 [ + + + - : 1032 : SET_FIELD(high_priority_weight, 0);
+ - + - +
- + - ]
6283 [ + + + - : 1032 : SET_FIELD(io_queue_requests, 0);
+ - + - +
- + - ]
6284 [ + + + - : 1032 : SET_FIELD(nvme_adminq_poll_period_us, 0);
+ - + - +
- + - ]
6285 [ + + + - : 1032 : SET_FIELD(nvme_ioq_poll_period_us, 0);
+ - + - +
- + - ]
6286 [ + + + + : 1032 : SET_FIELD(delay_cmd_submit, 0);
+ - + - +
- + - +
- ]
6287 [ + + + - : 1032 : SET_FIELD(bdev_retry_count, 0);
+ - + - +
- + - ]
6288 [ + + + - : 1032 : SET_FIELD(ctrlr_loss_timeout_sec, 0);
+ - + - +
- + - ]
6289 [ + + + - : 1032 : SET_FIELD(reconnect_delay_sec, 0);
+ - + - +
- + - ]
6290 [ + + + - : 1032 : SET_FIELD(fast_io_fail_timeout_sec, 0);
+ - + - +
- + - ]
6291 [ + + + - : 1032 : SET_FIELD(transport_ack_timeout, 0);
+ - + - +
- + - ]
6292 [ + + + + : 1032 : SET_FIELD(disable_auto_failback, false);
+ + + - +
- + - ]
6293 [ + + + + : 1032 : SET_FIELD(generate_uuids, false);
+ + + - +
- + - +
- ]
6294 [ + + + - : 1032 : SET_FIELD(transport_tos, 0);
+ - + - +
- ]
6295 [ + + + + : 1032 : SET_FIELD(nvme_error_stat, false);
+ + + - +
- + - +
- ]
6296 [ + + + + : 1032 : SET_FIELD(io_path_stat, false);
+ + + - +
- + - ]
6297 [ + + + + : 1032 : SET_FIELD(allow_accel_sequence, false);
+ + + - +
- + - +
- ]
6298 [ + + + - : 1032 : SET_FIELD(rdma_srq_size, 0);
+ - + - +
- + - ]
6299 [ + + + - : 1032 : SET_FIELD(rdma_max_cq_size, 0);
+ - + - +
- + - ]
6300 [ + + + - : 1032 : SET_FIELD(rdma_cm_event_timeout_ms, 0);
+ - + - +
- + - ]
6301 [ + + + - : 1032 : SET_FIELD(dhchap_digests, 0);
+ - + - +
- + - ]
6302 [ + - + - : 1032 : SET_FIELD(dhchap_dhgroups, 0);
+ - + - +
- + - ]
6303 : :
6304 [ + - + - ]: 1032 : g_opts.opts_size = opts->opts_size;
6305 : :
6306 : : #undef SET_FIELD
6307 : :
6308 : 1032 : return 0;
6309 : 43 : }
6310 : :
6311 : : struct set_nvme_hotplug_ctx {
6312 : : uint64_t period_us;
6313 : : bool enabled;
6314 : : spdk_msg_fn fn;
6315 : : void *fn_ctx;
6316 : : };
6317 : :
6318 : : static void
6319 : 1572 : set_nvme_hotplug_period_cb(void *_ctx)
6320 : : {
6321 : 1572 : struct set_nvme_hotplug_ctx *ctx = _ctx;
6322 : :
6323 : 1572 : spdk_poller_unregister(&g_hotplug_poller);
6324 [ + + + + : 1572 : if (ctx && ctx->enabled) {
+ + + - +
- ]
6325 [ # # # # ]: 12 : g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_hotplug, NULL, ctx->period_us);
6326 : 0 : } else {
6327 : 1560 : g_hotplug_poller = SPDK_POLLER_REGISTER(bdev_nvme_remove_poller, NULL,
6328 : : NVME_HOTPLUG_POLL_PERIOD_DEFAULT);
6329 : : }
6330 : :
6331 [ + + ]: 1572 : if (!ctx) {
6332 : 1261 : return;
6333 : : }
6334 : :
6335 [ + - + - ]: 311 : g_nvme_hotplug_poll_period_us = ctx->period_us;
6336 [ + + + - : 311 : g_nvme_hotplug_enabled = ctx->enabled;
+ - ]
6337 [ + + + - : 311 : if (ctx->fn) {
- + ]
6338 [ + - + - : 311 : ctx->fn(ctx->fn_ctx);
- + + - +
- + - ]
6339 : 41 : }
6340 : :
6341 : 311 : free(ctx);
6342 : 54 : }
6343 : :
6344 : : int
6345 : 311 : bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx)
6346 : : {
6347 : : struct set_nvme_hotplug_ctx *ctx;
6348 : :
6349 [ + + - + : 311 : if (enabled == true && !spdk_process_is_primary()) {
# # ]
6350 : 0 : return -EPERM;
6351 : : }
6352 : :
6353 : 311 : ctx = calloc(1, sizeof(*ctx));
6354 [ + + ]: 311 : if (ctx == NULL) {
6355 : 0 : return -ENOMEM;
6356 : : }
6357 : :
6358 [ + + ]: 311 : period_us = period_us == 0 ? NVME_HOTPLUG_POLL_PERIOD_DEFAULT : period_us;
6359 [ + - - + : 311 : ctx->period_us = spdk_min(period_us, NVME_HOTPLUG_POLL_PERIOD_MAX);
- + ]
6360 [ - + - + : 311 : ctx->enabled = enabled;
- + ]
6361 [ - + - + ]: 311 : ctx->fn = cb;
6362 [ - + - + ]: 311 : ctx->fn_ctx = cb_ctx;
6363 : :
6364 : 311 : spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, ctx);
6365 : 311 : return 0;
6366 : 41 : }
6367 : :
6368 : : static void
6369 : 1652 : nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr,
6370 : : struct nvme_async_probe_ctx *ctx)
6371 : : {
6372 : : struct nvme_ns *nvme_ns;
6373 : : struct nvme_bdev *nvme_bdev;
6374 : : size_t j;
6375 : :
6376 [ + + # # ]: 1652 : assert(nvme_ctrlr != NULL);
6377 : :
6378 [ + + + - : 1652 : if (ctx->names == NULL) {
+ - ]
6379 [ # # # # ]: 37 : ctx->reported_bdevs = 0;
6380 : 37 : populate_namespaces_cb(ctx, 0);
6381 : 37 : return;
6382 : : }
6383 : :
6384 : : /*
6385 : : * Report the new bdevs that were created in this call.
6386 : : * There can be more than one bdev per NVMe controller.
6387 : : */
6388 : 1615 : j = 0;
6389 : :
6390 [ + + + - ]: 1615 : pthread_mutex_lock(&nvme_ctrlr->mutex);
6391 : :
6392 : 1615 : nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
6393 [ + + ]: 3039 : while (nvme_ns != NULL) {
6394 [ + - + - ]: 1424 : nvme_bdev = nvme_ns->bdev;
6395 [ + - + - : 1424 : if (j < ctx->max_bdevs) {
+ - ]
6396 [ + - + - : 1424 : ctx->names[j] = nvme_bdev->disk.name;
+ - + - +
- + - +
- ]
6397 : 1424 : j++;
6398 : 60 : } else {
6399 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
6400 : :
6401 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6402 : : "Maximum number of namespaces supported per NVMe controller is %du. "
6403 : : "Unable to return all names of created bdevs\n",
6404 : : ctx->max_bdevs);
6405 [ # # # # ]: 0 : ctx->reported_bdevs = 0;
6406 : 0 : populate_namespaces_cb(ctx, -ERANGE);
6407 : 0 : return;
6408 : : }
6409 : :
6410 : 1424 : nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
6411 : : }
6412 : :
6413 [ + + + - ]: 1615 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
6414 : :
6415 [ + - + - ]: 1615 : ctx->reported_bdevs = j;
6416 : 1615 : populate_namespaces_cb(ctx, 0);
6417 : 58 : }
6418 : :
6419 : : static int
6420 : 52 : bdev_nvme_check_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,
6421 : : struct spdk_nvme_ctrlr *new_ctrlr,
6422 : : struct spdk_nvme_transport_id *trid)
6423 : : {
6424 : : struct nvme_path_id *tmp_trid;
6425 : :
6426 [ - + # # : 52 : if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
# # ]
6427 [ # # # # : 0 : NVME_CTRLR_ERRLOG(nvme_ctrlr, "PCIe failover is not supported.\n");
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6428 : 0 : return -ENOTSUP;
6429 : : }
6430 : :
6431 : : /* Currently we only support failover to the same transport type. */
6432 [ - + # # : 52 : if (nvme_ctrlr->active_path_id->trid.trtype != trid->trtype) {
# # # # #
# # # # #
# # ]
6433 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6434 : : "Failover from trtype: %s to a different trtype: %s is not supported currently\n",
6435 : : spdk_nvme_transport_id_trtype_str(nvme_ctrlr->active_path_id->trid.trtype),
6436 : : spdk_nvme_transport_id_trtype_str(trid->trtype));
6437 : 0 : return -EINVAL;
6438 : : }
6439 : :
6440 : :
6441 : : /* Currently we only support failover to the same NQN. */
6442 [ - + - + : 52 : if (strncmp(trid->subnqn, nvme_ctrlr->active_path_id->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) {
- + # # #
# # # # #
# # ]
6443 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6444 : : "Failover from subnqn: %s to a different subnqn: %s is not supported currently\n",
6445 : : nvme_ctrlr->active_path_id->trid.subnqn, trid->subnqn);
6446 : 0 : return -EINVAL;
6447 : : }
6448 : :
6449 : : /* Skip all the other checks if we've already registered this path. */
6450 [ + + # # : 124 : TAILQ_FOREACH(tmp_trid, &nvme_ctrlr->trids, link) {
# # # # #
# # # #
# ]
6451 [ + + # # ]: 72 : if (!spdk_nvme_transport_id_compare(&tmp_trid->trid, trid)) {
6452 [ # # # # : 0 : NVME_CTRLR_WARNLOG(nvme_ctrlr, "This path (traddr: %s subnqn: %s) is already registered\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6453 : : trid->traddr, trid->subnqn);
6454 : 0 : return -EALREADY;
6455 : : }
6456 : 12 : }
6457 : :
6458 : 52 : return 0;
6459 : 9 : }
6460 : :
6461 : : static int
6462 : 52 : bdev_nvme_check_secondary_namespace(struct nvme_ctrlr *nvme_ctrlr,
6463 : : struct spdk_nvme_ctrlr *new_ctrlr)
6464 : : {
6465 : : struct nvme_ns *nvme_ns;
6466 : : struct spdk_nvme_ns *new_ns;
6467 : :
6468 : 52 : nvme_ns = nvme_ctrlr_get_first_active_ns(nvme_ctrlr);
6469 [ + + ]: 68 : while (nvme_ns != NULL) {
6470 [ # # # # ]: 16 : new_ns = spdk_nvme_ctrlr_get_ns(new_ctrlr, nvme_ns->id);
6471 [ - + # # ]: 16 : assert(new_ns != NULL);
6472 : :
6473 [ - + # # : 16 : if (!bdev_nvme_compare_ns(nvme_ns->ns, new_ns)) {
# # ]
6474 : 0 : return -EINVAL;
6475 : : }
6476 : :
6477 : 16 : nvme_ns = nvme_ctrlr_get_next_active_ns(nvme_ctrlr, nvme_ns);
6478 : : }
6479 : :
6480 : 52 : return 0;
6481 : 9 : }
6482 : :
6483 : : static int
6484 : 52 : _bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,
6485 : : struct spdk_nvme_transport_id *trid)
6486 : : {
6487 : : struct nvme_path_id *active_id, *new_trid, *tmp_trid;
6488 : :
6489 : 52 : new_trid = calloc(1, sizeof(*new_trid));
6490 [ + + ]: 52 : if (new_trid == NULL) {
6491 : 0 : return -ENOMEM;
6492 : : }
6493 [ # # ]: 52 : new_trid->trid = *trid;
6494 : :
6495 [ # # # # ]: 52 : active_id = nvme_ctrlr->active_path_id;
6496 [ + + # # ]: 52 : assert(active_id != NULL);
6497 [ - + # # : 52 : assert(active_id == TAILQ_FIRST(&nvme_ctrlr->trids));
# # # # #
# ]
6498 : :
6499 : : /* Skip the active trid not to replace it until it is failed. */
6500 [ # # # # : 52 : tmp_trid = TAILQ_NEXT(active_id, link);
# # ]
6501 [ + + ]: 52 : if (tmp_trid == NULL) {
6502 : 32 : goto add_tail;
6503 : : }
6504 : :
6505 : : /* It means the trid is faled if its last failed time is non-zero.
6506 : : * Insert the new alternate trid before any failed trid.
6507 : : */
6508 [ - + + + : 32 : TAILQ_FOREACH_FROM(tmp_trid, &nvme_ctrlr->trids, link) {
# # # # #
# # # # #
# # ]
6509 [ + + # # : 20 : if (tmp_trid->last_failed_tsc != 0) {
# # ]
6510 [ # # # # : 8 : TAILQ_INSERT_BEFORE(tmp_trid, new_trid, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6511 : 8 : return 0;
6512 : : }
6513 : 4 : }
6514 : :
6515 : 10 : add_tail:
6516 [ # # # # : 44 : TAILQ_INSERT_TAIL(&nvme_ctrlr->trids, new_trid, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
6517 : 44 : return 0;
6518 : 9 : }
6519 : :
6520 : : /* This is the case that a secondary path is added to an existing
6521 : : * nvme_ctrlr for failover. After checking if it can access the same
6522 : : * namespaces as the primary path, it is disconnected until failover occurs.
6523 : : */
6524 : : static int
6525 : 52 : bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,
6526 : : struct spdk_nvme_ctrlr *new_ctrlr,
6527 : : struct spdk_nvme_transport_id *trid)
6528 : : {
6529 : : int rc;
6530 : :
6531 [ + + # # ]: 52 : assert(nvme_ctrlr != NULL);
6532 : :
6533 [ - + # # ]: 52 : pthread_mutex_lock(&nvme_ctrlr->mutex);
6534 : :
6535 : 52 : rc = bdev_nvme_check_secondary_trid(nvme_ctrlr, new_ctrlr, trid);
6536 [ - + ]: 52 : if (rc != 0) {
6537 : 0 : goto exit;
6538 : : }
6539 : :
6540 : 52 : rc = bdev_nvme_check_secondary_namespace(nvme_ctrlr, new_ctrlr);
6541 [ - + ]: 52 : if (rc != 0) {
6542 : 0 : goto exit;
6543 : : }
6544 : :
6545 : 52 : rc = _bdev_nvme_add_secondary_trid(nvme_ctrlr, trid);
6546 : :
6547 : 43 : exit:
6548 [ - + # # ]: 52 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
6549 : :
6550 : 52 : spdk_nvme_detach(new_ctrlr);
6551 : :
6552 : 52 : return rc;
6553 : : }
6554 : :
6555 : : static void
6556 : 1656 : connect_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
6557 : : struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
6558 : : {
6559 : 1656 : struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
6560 : : struct nvme_async_probe_ctx *ctx;
6561 : : int rc;
6562 : :
6563 : 1656 : ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts);
6564 [ + - + - ]: 1656 : ctx->ctrlr_attached = true;
6565 : :
6566 [ + - + - : 1656 : rc = nvme_ctrlr_create(ctrlr, ctx->base_name, &ctx->trid, ctx);
+ - ]
6567 [ + + ]: 1656 : if (rc != 0) {
6568 [ # # # # ]: 4 : ctx->reported_bdevs = 0;
6569 : 4 : populate_namespaces_cb(ctx, rc);
6570 : 1 : }
6571 : 1656 : }
6572 : :
6573 : :
6574 : : static void
6575 : 32 : connect_set_failover_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
6576 : : struct spdk_nvme_ctrlr *ctrlr,
6577 : : const struct spdk_nvme_ctrlr_opts *opts)
6578 : : {
6579 : 32 : struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
6580 : : struct nvme_ctrlr *nvme_ctrlr;
6581 : : struct nvme_async_probe_ctx *ctx;
6582 : : int rc;
6583 : :
6584 : 32 : ctx = SPDK_CONTAINEROF(user_opts, struct nvme_async_probe_ctx, drv_opts);
6585 [ # # # # ]: 32 : ctx->ctrlr_attached = true;
6586 : :
6587 [ # # # # ]: 32 : nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name);
6588 [ + - ]: 32 : if (nvme_ctrlr) {
6589 [ # # ]: 32 : rc = bdev_nvme_add_secondary_trid(nvme_ctrlr, ctrlr, &ctx->trid);
6590 : 4 : } else {
6591 : 0 : rc = -ENODEV;
6592 : : }
6593 : :
6594 [ # # # # ]: 32 : ctx->reported_bdevs = 0;
6595 : 32 : populate_namespaces_cb(ctx, rc);
6596 : 32 : }
6597 : :
6598 : : static int
6599 : 265334 : bdev_nvme_async_poll(void *arg)
6600 : : {
6601 : 265334 : struct nvme_async_probe_ctx *ctx = arg;
6602 : : int rc;
6603 : :
6604 [ + - + - ]: 265334 : rc = spdk_nvme_probe_poll_async(ctx->probe_ctx);
6605 [ + + ]: 265334 : if (spdk_unlikely(rc != -EAGAIN)) {
6606 [ + - + - ]: 1747 : ctx->probe_done = true;
6607 [ + - ]: 1747 : spdk_poller_unregister(&ctx->poller);
6608 [ + + + + : 1747 : if (!ctx->ctrlr_attached) {
+ - + - ]
6609 : : /* The probe is done, but no controller was attached.
6610 : : * That means we had a failure, so report -EIO back to
6611 : : * the caller (usually the RPC). populate_namespaces_cb()
6612 : : * will take care of freeing the nvme_async_probe_ctx.
6613 : : */
6614 [ # # # # ]: 59 : ctx->reported_bdevs = 0;
6615 : 59 : populate_namespaces_cb(ctx, -EIO);
6616 [ + + + + : 1689 : } else if (ctx->namespaces_populated) {
+ - - + ]
6617 : : /* The namespaces for the attached controller were all
6618 : : * populated and the response was already sent to the
6619 : : * caller (usually the RPC). So free the context here.
6620 : : */
6621 : 1233 : free_nvme_async_probe_ctx(ctx);
6622 : 32 : }
6623 : 64 : }
6624 : :
6625 : 265334 : return SPDK_POLLER_BUSY;
6626 : : }
6627 : :
6628 : : static bool
6629 : 2864 : bdev_nvme_check_io_error_resiliency_params(int32_t ctrlr_loss_timeout_sec,
6630 : : uint32_t reconnect_delay_sec,
6631 : : uint32_t fast_io_fail_timeout_sec)
6632 : : {
6633 [ + + ]: 2864 : if (ctrlr_loss_timeout_sec < -1) {
6634 : 4 : SPDK_ERRLOG("ctrlr_loss_timeout_sec can't be less than -1.\n");
6635 : 4 : return false;
6636 [ + + ]: 2860 : } else if (ctrlr_loss_timeout_sec == -1) {
6637 [ + + ]: 68 : if (reconnect_delay_sec == 0) {
6638 : 4 : SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n");
6639 : 4 : return false;
6640 [ + + + + ]: 64 : } else if (fast_io_fail_timeout_sec != 0 &&
6641 : 3 : fast_io_fail_timeout_sec < reconnect_delay_sec) {
6642 : 4 : SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io-fail_timeout_sec.\n");
6643 : 4 : return false;
6644 : : }
6645 [ + + ]: 2804 : } else if (ctrlr_loss_timeout_sec != 0) {
6646 [ + + ]: 72 : if (reconnect_delay_sec == 0) {
6647 : 4 : SPDK_ERRLOG("reconnect_delay_sec can't be 0 if ctrlr_loss_timeout_sec is not 0.\n");
6648 : 4 : return false;
6649 [ + + ]: 68 : } else if (reconnect_delay_sec > (uint32_t)ctrlr_loss_timeout_sec) {
6650 : 4 : SPDK_ERRLOG("reconnect_delay_sec can't be more than ctrlr_loss_timeout_sec.\n");
6651 : 4 : return false;
6652 [ + + ]: 64 : } else if (fast_io_fail_timeout_sec != 0) {
6653 [ + + ]: 32 : if (fast_io_fail_timeout_sec < reconnect_delay_sec) {
6654 : 4 : SPDK_ERRLOG("reconnect_delay_sec can't be more than fast_io_fail_timeout_sec.\n");
6655 : 4 : return false;
6656 [ + + ]: 28 : } else if (fast_io_fail_timeout_sec > (uint32_t)ctrlr_loss_timeout_sec) {
6657 : 4 : SPDK_ERRLOG("fast_io_fail_timeout_sec can't be more than ctrlr_loss_timeout_sec.\n");
6658 : 4 : return false;
6659 : : }
6660 : 4 : }
6661 [ + + + + ]: 2727 : } else if (reconnect_delay_sec != 0 || fast_io_fail_timeout_sec != 0) {
6662 : 8 : SPDK_ERRLOG("Both reconnect_delay_sec and fast_io_fail_timeout_sec must be 0 if ctrlr_loss_timeout_sec is 0.\n");
6663 : 8 : return false;
6664 : : }
6665 : :
6666 : 2828 : return true;
6667 : 126 : }
6668 : :
6669 : : int
6670 : 1756 : spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid,
6671 : : const char *base_name,
6672 : : const char **names,
6673 : : uint32_t count,
6674 : : spdk_bdev_nvme_create_cb cb_fn,
6675 : : void *cb_ctx,
6676 : : struct spdk_nvme_ctrlr_opts *drv_opts,
6677 : : struct spdk_bdev_nvme_ctrlr_opts *bdev_opts)
6678 : : {
6679 : : struct nvme_probe_skip_entry *entry, *tmp;
6680 : : struct nvme_async_probe_ctx *ctx;
6681 : : spdk_nvme_attach_cb attach_cb;
6682 : : struct nvme_ctrlr *nvme_ctrlr;
6683 : : int len;
6684 : :
6685 : : /* TODO expand this check to include both the host and target TRIDs.
6686 : : * Only if both are the same should we fail.
6687 : : */
6688 [ + + - + ]: 1756 : if (nvme_ctrlr_get(trid, drv_opts->hostnqn) != NULL) {
6689 [ # # # # ]: 0 : SPDK_ERRLOG("A controller with the provided trid (traddr: %s, hostnqn: %s) "
6690 : : "already exists.\n", trid->traddr, drv_opts->hostnqn);
6691 : 0 : return -EEXIST;
6692 : : }
6693 : :
6694 [ + + ]: 1756 : len = strnlen(base_name, SPDK_CONTROLLER_NAME_MAX);
6695 : :
6696 [ + - - + ]: 1756 : if (len == 0 || len == SPDK_CONTROLLER_NAME_MAX) {
6697 : 0 : SPDK_ERRLOG("controller name must be between 1 and %d characters\n", SPDK_CONTROLLER_NAME_MAX - 1);
6698 : 0 : return -EINVAL;
6699 : : }
6700 : :
6701 [ + - + - ]: 1757 : if (bdev_opts != NULL &&
6702 [ + + + - ]: 1820 : !bdev_nvme_check_io_error_resiliency_params(bdev_opts->ctrlr_loss_timeout_sec,
6703 [ + - + - ]: 64 : bdev_opts->reconnect_delay_sec,
6704 [ + - + - ]: 64 : bdev_opts->fast_io_fail_timeout_sec)) {
6705 : 0 : return -EINVAL;
6706 : : }
6707 : :
6708 : 1756 : ctx = calloc(1, sizeof(*ctx));
6709 [ + + ]: 1756 : if (!ctx) {
6710 : 0 : return -ENOMEM;
6711 : : }
6712 [ + + + - : 1756 : ctx->base_name = strdup(base_name);
+ - ]
6713 [ + + + - : 1756 : if (!ctx->base_name) {
+ - ]
6714 : 0 : free(ctx);
6715 : 0 : return -ENOMEM;
6716 : : }
6717 [ + - + - ]: 1756 : ctx->names = names;
6718 [ + - + - ]: 1756 : ctx->max_bdevs = count;
6719 [ + - + - ]: 1756 : ctx->cb_fn = cb_fn;
6720 [ + - + - ]: 1756 : ctx->cb_ctx = cb_ctx;
6721 [ + - ]: 1756 : ctx->trid = *trid;
6722 : :
6723 [ + + ]: 1756 : if (bdev_opts) {
6724 [ + + + + : 1756 : memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts));
+ - ]
6725 : 64 : } else {
6726 [ # # ]: 0 : spdk_bdev_nvme_get_default_ctrlr_opts(&ctx->bdev_opts);
6727 : : }
6728 : :
6729 [ + + + - : 1756 : if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
- + ]
6730 [ + + + - : 492 : TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, tmp) {
# # # # +
- ]
6731 [ + - # # ]: 2 : if (spdk_nvme_transport_id_compare(trid, &entry->trid) == 0) {
6732 [ - + # # : 2 : TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6733 : 2 : free(entry);
6734 : 2 : break;
6735 : : }
6736 : 0 : }
6737 : 11 : }
6738 : :
6739 [ + + + + : 1756 : memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts));
+ - ]
6740 [ + - + - : 1756 : ctx->drv_opts.transport_retry_count = g_opts.transport_retry_count;
+ - + - ]
6741 [ + - + - : 1756 : ctx->drv_opts.transport_ack_timeout = g_opts.transport_ack_timeout;
+ - + - ]
6742 [ + - + - : 1756 : ctx->drv_opts.keep_alive_timeout_ms = g_opts.keep_alive_timeout_ms;
+ - + - ]
6743 [ + - + - : 1756 : ctx->drv_opts.disable_read_ana_log_page = true;
+ - ]
6744 [ + - + - : 1756 : ctx->drv_opts.transport_tos = g_opts.transport_tos;
+ - ]
6745 : :
6746 [ + + ]: 1756 : if (spdk_interrupt_mode_is_enabled()) {
6747 [ + - # # : 1 : if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
# # ]
6748 [ # # # # : 1 : ctx->drv_opts.enable_interrupts = true;
# # ]
6749 : 0 : } else {
6750 : 0 : SPDK_ERRLOG("Interrupt mode is only supported with PCIe transport\n");
6751 : 0 : free_nvme_async_probe_ctx(ctx);
6752 : 0 : return -ENOTSUP;
6753 : : }
6754 : 0 : }
6755 : :
6756 [ + + + - : 1756 : if (ctx->bdev_opts.psk != NULL) {
+ - + - ]
6757 [ # # # # : 66 : ctx->drv_opts.tls_psk = spdk_keyring_get_key(ctx->bdev_opts.psk);
# # # # #
# # # ]
6758 [ + + # # : 66 : if (ctx->drv_opts.tls_psk == NULL) {
# # # # ]
6759 [ # # # # : 6 : SPDK_ERRLOG("Could not load PSK: %s\n", ctx->bdev_opts.psk);
# # ]
6760 : 6 : free_nvme_async_probe_ctx(ctx);
6761 : 6 : return -ENOKEY;
6762 : : }
6763 : 0 : }
6764 : :
6765 [ + + + - : 1750 : if (ctx->bdev_opts.dhchap_key != NULL) {
+ - + - ]
6766 [ # # # # : 664 : ctx->drv_opts.dhchap_key = spdk_keyring_get_key(ctx->bdev_opts.dhchap_key);
# # # # #
# # # ]
6767 [ - + # # : 664 : if (ctx->drv_opts.dhchap_key == NULL) {
# # # # ]
6768 [ # # # # : 0 : SPDK_ERRLOG("Could not load DH-HMAC-CHAP key: %s\n",
# # ]
6769 : : ctx->bdev_opts.dhchap_key);
6770 : 0 : free_nvme_async_probe_ctx(ctx);
6771 : 0 : return -ENOKEY;
6772 : : }
6773 : :
6774 [ # # # # : 664 : ctx->drv_opts.dhchap_digests = g_opts.dhchap_digests;
# # # # ]
6775 [ # # # # : 664 : ctx->drv_opts.dhchap_dhgroups = g_opts.dhchap_dhgroups;
# # # # ]
6776 : 0 : }
6777 [ + + + - : 1750 : if (ctx->bdev_opts.dhchap_ctrlr_key != NULL) {
+ - + - ]
6778 [ # # # # : 500 : ctx->drv_opts.dhchap_ctrlr_key =
# # ]
6779 [ # # # # : 500 : spdk_keyring_get_key(ctx->bdev_opts.dhchap_ctrlr_key);
# # ]
6780 [ - + # # : 500 : if (ctx->drv_opts.dhchap_ctrlr_key == NULL) {
# # # # ]
6781 [ # # # # : 0 : SPDK_ERRLOG("Could not load DH-HMAC-CHAP controller key: %s\n",
# # ]
6782 : : ctx->bdev_opts.dhchap_ctrlr_key);
6783 : 0 : free_nvme_async_probe_ctx(ctx);
6784 : 0 : return -ENOKEY;
6785 : : }
6786 : 0 : }
6787 : :
6788 [ + + + + : 1750 : if (nvme_bdev_ctrlr_get_by_name(base_name) == NULL || ctx->bdev_opts.multipath) {
+ + # # #
# # # ]
6789 : 1718 : attach_cb = connect_attach_cb;
6790 : 60 : } else {
6791 : 32 : attach_cb = connect_set_failover_cb;
6792 : : }
6793 : :
6794 [ + - + - ]: 1750 : nvme_ctrlr = nvme_ctrlr_get_by_name(ctx->base_name);
6795 [ + + + + : 1750 : if (nvme_ctrlr && nvme_ctrlr->opts.multipath != ctx->bdev_opts.multipath) {
- + - + #
# # # # #
# # # # #
# ]
6796 : : /* All controllers with the same name must be configured the same
6797 : : * way, either for multipath or failover. If the configuration doesn't
6798 : : * match - report error.
6799 : : */
6800 : 0 : free_nvme_async_probe_ctx(ctx);
6801 : 0 : return -EINVAL;
6802 : : }
6803 : :
6804 [ + - + - : 1750 : ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, attach_cb);
+ - ]
6805 [ + + + - : 1750 : if (ctx->probe_ctx == NULL) {
+ - ]
6806 [ # # ]: 3 : SPDK_ERRLOG("No controller was found with provided trid (traddr: %s)\n", trid->traddr);
6807 : 3 : free_nvme_async_probe_ctx(ctx);
6808 : 3 : return -ENODEV;
6809 : : }
6810 [ + - + - ]: 1747 : ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_async_poll, ctx, 1000);
6811 : :
6812 : 1747 : return 0;
6813 : 64 : }
6814 : :
6815 : : struct bdev_nvme_delete_ctx {
6816 : : char *name;
6817 : : struct nvme_path_id path_id;
6818 : : bdev_nvme_delete_done_fn delete_done;
6819 : : void *delete_done_ctx;
6820 : : uint64_t timeout_ticks;
6821 : : struct spdk_poller *poller;
6822 : : };
6823 : :
6824 : : static void
6825 : 706 : free_bdev_nvme_delete_ctx(struct bdev_nvme_delete_ctx *ctx)
6826 : : {
6827 [ + + ]: 706 : if (ctx != NULL) {
6828 [ # # # # ]: 702 : free(ctx->name);
6829 : 702 : free(ctx);
6830 : 4 : }
6831 : 706 : }
6832 : :
6833 : : static bool
6834 : 31139 : nvme_path_id_compare(struct nvme_path_id *p, const struct nvme_path_id *path_id)
6835 : : {
6836 [ + + # # : 31139 : if (path_id->trid.trtype != 0) {
# # # # ]
6837 [ - + # # : 226 : if (path_id->trid.trtype == SPDK_NVME_TRANSPORT_CUSTOM) {
# # # # ]
6838 [ # # # # : 0 : if (strcasecmp(path_id->trid.trstring, p->trid.trstring) != 0) {
# # # # #
# # # #
# ]
6839 : 0 : return false;
6840 : : }
6841 : 0 : } else {
6842 [ - + # # : 226 : if (path_id->trid.trtype != p->trid.trtype) {
# # # # #
# # # #
# ]
6843 : 0 : return false;
6844 : : }
6845 : : }
6846 : 21 : }
6847 : :
6848 [ + + # # : 31139 : if (!spdk_mem_all_zero(path_id->trid.traddr, sizeof(path_id->trid.traddr))) {
# # ]
6849 [ + + - + : 226 : if (strcasecmp(path_id->trid.traddr, p->trid.traddr) != 0) {
+ + # # #
# # # #
# ]
6850 : 44 : return false;
6851 : : }
6852 : 10 : }
6853 : :
6854 [ + + # # : 31095 : if (path_id->trid.adrfam != 0) {
# # # # ]
6855 [ - + # # : 142 : if (path_id->trid.adrfam != p->trid.adrfam) {
# # # # #
# # # #
# ]
6856 : 0 : return false;
6857 : : }
6858 : 0 : }
6859 : :
6860 [ + + # # : 31095 : if (!spdk_mem_all_zero(path_id->trid.trsvcid, sizeof(path_id->trid.trsvcid))) {
# # ]
6861 [ - + - + : 182 : if (strcasecmp(path_id->trid.trsvcid, p->trid.trsvcid) != 0) {
+ + # # #
# # # #
# ]
6862 : 49 : return false;
6863 : : }
6864 : 10 : }
6865 : :
6866 [ + + # # : 31046 : if (!spdk_mem_all_zero(path_id->trid.subnqn, sizeof(path_id->trid.subnqn))) {
# # ]
6867 [ - + - + : 133 : if (strcmp(path_id->trid.subnqn, p->trid.subnqn) != 0) {
- + # # #
# # # #
# ]
6868 : 0 : return false;
6869 : : }
6870 : 10 : }
6871 : :
6872 [ + + # # : 31046 : if (!spdk_mem_all_zero(path_id->hostid.hostaddr, sizeof(path_id->hostid.hostaddr))) {
# # ]
6873 [ # # # # : 0 : if (strcmp(path_id->hostid.hostaddr, p->hostid.hostaddr) != 0) {
# # # # #
# # # #
# ]
6874 : 0 : return false;
6875 : : }
6876 : 0 : }
6877 : :
6878 [ + + # # : 31046 : if (!spdk_mem_all_zero(path_id->hostid.hostsvcid, sizeof(path_id->hostid.hostsvcid))) {
# # ]
6879 [ # # # # : 0 : if (strcmp(path_id->hostid.hostsvcid, p->hostid.hostsvcid) != 0) {
# # # # #
# # # #
# ]
6880 : 0 : return false;
6881 : : }
6882 : 0 : }
6883 : :
6884 : 31046 : return true;
6885 : 94 : }
6886 : :
6887 : : static bool
6888 : 30760 : nvme_path_id_exists(const char *name, const struct nvme_path_id *path_id)
6889 : : {
6890 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
6891 : : struct nvme_ctrlr *ctrlr;
6892 : : struct nvme_path_id *p;
6893 : :
6894 [ - + ]: 30760 : pthread_mutex_lock(&g_bdev_nvme_mutex);
6895 : 30760 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
6896 [ + + ]: 30760 : if (!nbdev_ctrlr) {
6897 [ - + ]: 692 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6898 : 692 : return false;
6899 : : }
6900 : :
6901 [ + + # # : 30094 : TAILQ_FOREACH(ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
6902 [ - + # # ]: 30084 : pthread_mutex_lock(&ctrlr->mutex);
6903 [ + + # # : 30114 : TAILQ_FOREACH(p, &ctrlr->trids, link) {
# # # # #
# # # #
# ]
6904 [ + + ]: 30088 : if (nvme_path_id_compare(p, path_id)) {
6905 [ - + # # ]: 30058 : pthread_mutex_unlock(&ctrlr->mutex);
6906 [ - + ]: 30058 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6907 : 30058 : return true;
6908 : : }
6909 : 0 : }
6910 [ - + # # ]: 26 : pthread_mutex_unlock(&ctrlr->mutex);
6911 : 0 : }
6912 [ - + ]: 10 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
6913 : :
6914 : 10 : return false;
6915 : 20 : }
6916 : :
6917 : : static int
6918 : 30760 : bdev_nvme_delete_complete_poll(void *arg)
6919 : : {
6920 : 30760 : struct bdev_nvme_delete_ctx *ctx = arg;
6921 : 30760 : int rc = 0;
6922 : :
6923 [ + + # # : 30760 : if (nvme_path_id_exists(ctx->name, &ctx->path_id)) {
# # # # ]
6924 [ + - # # : 30058 : if (ctx->timeout_ticks > spdk_get_ticks()) {
# # ]
6925 : 30058 : return SPDK_POLLER_BUSY;
6926 : : }
6927 : :
6928 [ # # # # ]: 0 : SPDK_ERRLOG("NVMe path '%s' still exists after delete\n", ctx->name);
6929 : 0 : rc = -ETIMEDOUT;
6930 : 0 : }
6931 : :
6932 [ # # ]: 702 : spdk_poller_unregister(&ctx->poller);
6933 : :
6934 [ # # # # : 702 : ctx->delete_done(ctx->delete_done_ctx, rc);
# # # # #
# # # ]
6935 : 702 : free_bdev_nvme_delete_ctx(ctx);
6936 : :
6937 : 702 : return SPDK_POLLER_BUSY;
6938 : 20 : }
6939 : :
6940 : : static int
6941 : 999 : _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path_id)
6942 : : {
6943 : : struct nvme_path_id *p, *t;
6944 : : spdk_msg_fn msg_fn;
6945 : 999 : int rc = -ENXIO;
6946 : :
6947 [ - + # # ]: 999 : pthread_mutex_lock(&nvme_ctrlr->mutex);
6948 : :
6949 [ + + - + : 1051 : TAILQ_FOREACH_REVERSE_SAFE(p, &nvme_ctrlr->trids, nvme_paths, link, t) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
6950 [ + + # # : 1051 : if (p == TAILQ_FIRST(&nvme_ctrlr->trids)) {
# # # # ]
6951 : 999 : break;
6952 : : }
6953 : :
6954 [ + + ]: 52 : if (!nvme_path_id_compare(p, path_id)) {
6955 : 20 : continue;
6956 : : }
6957 : :
6958 : : /* We are not using the specified path. */
6959 [ + + # # : 32 : TAILQ_REMOVE(&nvme_ctrlr->trids, p, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
6960 : 32 : free(p);
6961 : 32 : rc = 0;
6962 : 7 : }
6963 : :
6964 [ + - + + ]: 999 : if (p == NULL || !nvme_path_id_compare(p, path_id)) {
6965 [ - + # # ]: 43 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
6966 : 43 : return rc;
6967 : : }
6968 : :
6969 : : /* If we made it here, then this path is a match! Now we need to remove it. */
6970 : :
6971 : : /* This is the active path in use right now. The active path is always the first in the list. */
6972 [ + + # # : 956 : assert(p == nvme_ctrlr->active_path_id);
# # # # ]
6973 : :
6974 [ + + # # : 956 : if (!TAILQ_NEXT(p, link)) {
# # # # ]
6975 : : /* The current path is the only path. */
6976 : 948 : msg_fn = _nvme_ctrlr_destruct;
6977 : 948 : rc = bdev_nvme_delete_ctrlr_unsafe(nvme_ctrlr, false);
6978 : 59 : } else {
6979 : : /* There is an alternative path. */
6980 : 8 : msg_fn = _bdev_nvme_reset_ctrlr;
6981 : 8 : rc = bdev_nvme_failover_ctrlr_unsafe(nvme_ctrlr, true);
6982 : : }
6983 : :
6984 [ - + # # ]: 956 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
6985 : :
6986 [ + + ]: 956 : if (rc == 0) {
6987 [ # # # # ]: 956 : spdk_thread_send_msg(nvme_ctrlr->thread, msg_fn, nvme_ctrlr);
6988 [ # # ]: 60 : } else if (rc == -EALREADY) {
6989 : 0 : rc = 0;
6990 : 0 : }
6991 : :
6992 : 956 : return rc;
6993 : 68 : }
6994 : :
6995 : : int
6996 : 932 : bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id,
6997 : : bdev_nvme_delete_done_fn delete_done, void *delete_done_ctx)
6998 : : {
6999 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
7000 : : struct nvme_ctrlr *nvme_ctrlr, *tmp_nvme_ctrlr;
7001 : 932 : struct bdev_nvme_delete_ctx *ctx = NULL;
7002 : 932 : int rc = -ENXIO, _rc;
7003 : :
7004 [ + - - + ]: 932 : if (name == NULL || path_id == NULL) {
7005 : 0 : rc = -EINVAL;
7006 : 0 : goto exit;
7007 : : }
7008 : :
7009 [ - + ]: 932 : pthread_mutex_lock(&g_bdev_nvme_mutex);
7010 : :
7011 : 932 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
7012 [ + + ]: 932 : if (nbdev_ctrlr == NULL) {
7013 [ # # ]: 0 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
7014 : :
7015 : 0 : SPDK_ERRLOG("Failed to find NVMe bdev controller\n");
7016 : 0 : rc = -ENODEV;
7017 : 0 : goto exit;
7018 : : }
7019 : :
7020 [ + + + + : 1931 : TAILQ_FOREACH_SAFE(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq, tmp_nvme_ctrlr) {
# # # # #
# # # # #
# # ]
7021 : 999 : _rc = _bdev_nvme_delete(nvme_ctrlr, path_id);
7022 [ + + + + ]: 999 : if (_rc < 0 && _rc != -ENXIO) {
7023 [ # # ]: 0 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
7024 : 0 : rc = _rc;
7025 : 0 : goto exit;
7026 [ + + ]: 999 : } else if (_rc == 0) {
7027 : : /* We traverse all remaining nvme_ctrlrs even if one nvme_ctrlr
7028 : : * was deleted successfully. To remember the successful deletion,
7029 : : * overwrite rc only if _rc is zero.
7030 : : */
7031 : 968 : rc = 0;
7032 : 62 : }
7033 : 68 : }
7034 : :
7035 [ - + ]: 932 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
7036 : :
7037 [ + + + + ]: 932 : if (rc != 0 || delete_done == NULL) {
7038 : 230 : goto exit;
7039 : : }
7040 : :
7041 : 702 : ctx = calloc(1, sizeof(*ctx));
7042 [ + + ]: 702 : if (ctx == NULL) {
7043 : 0 : SPDK_ERRLOG("Failed to allocate context for bdev_nvme_delete\n");
7044 : 0 : rc = -ENOMEM;
7045 : 0 : goto exit;
7046 : : }
7047 : :
7048 [ - + # # : 702 : ctx->name = strdup(name);
# # ]
7049 [ + + # # : 702 : if (ctx->name == NULL) {
# # ]
7050 : 0 : SPDK_ERRLOG("Failed to copy controller name for deletion\n");
7051 : 0 : rc = -ENOMEM;
7052 : 0 : goto exit;
7053 : : }
7054 : :
7055 [ # # # # ]: 702 : ctx->delete_done = delete_done;
7056 [ # # # # ]: 702 : ctx->delete_done_ctx = delete_done_ctx;
7057 [ # # ]: 702 : ctx->path_id = *path_id;
7058 [ # # # # ]: 702 : ctx->timeout_ticks = spdk_get_ticks() + 10 * spdk_get_ticks_hz();
7059 [ # # # # ]: 702 : ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_delete_complete_poll, ctx, 1000);
7060 [ + + # # : 702 : if (ctx->poller == NULL) {
# # ]
7061 : 0 : SPDK_ERRLOG("Failed to register bdev_nvme_delete poller\n");
7062 : 0 : rc = -ENOMEM;
7063 : 0 : goto exit;
7064 : : }
7065 : :
7066 : 698 : exit:
7067 [ + + ]: 932 : if (rc != 0) {
7068 : 4 : free_bdev_nvme_delete_ctx(ctx);
7069 : 1 : }
7070 : :
7071 : 932 : return rc;
7072 : : }
7073 : :
7074 : : #define DISCOVERY_INFOLOG(ctx, format, ...) \
7075 : : SPDK_INFOLOG(bdev_nvme, "Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__);
7076 : :
7077 : : #define DISCOVERY_ERRLOG(ctx, format, ...) \
7078 : : SPDK_ERRLOG("Discovery[%s:%s] " format, ctx->trid.traddr, ctx->trid.trsvcid, ##__VA_ARGS__);
7079 : :
7080 : : struct discovery_entry_ctx {
7081 : : char name[128];
7082 : : struct spdk_nvme_transport_id trid;
7083 : : struct spdk_nvme_ctrlr_opts drv_opts;
7084 : : struct spdk_nvmf_discovery_log_page_entry entry;
7085 : : TAILQ_ENTRY(discovery_entry_ctx) tailq;
7086 : : struct discovery_ctx *ctx;
7087 : : };
7088 : :
7089 : : struct discovery_ctx {
7090 : : char *name;
7091 : : spdk_bdev_nvme_start_discovery_fn start_cb_fn;
7092 : : spdk_bdev_nvme_stop_discovery_fn stop_cb_fn;
7093 : : void *cb_ctx;
7094 : : struct spdk_nvme_probe_ctx *probe_ctx;
7095 : : struct spdk_nvme_detach_ctx *detach_ctx;
7096 : : struct spdk_nvme_ctrlr *ctrlr;
7097 : : struct spdk_nvme_transport_id trid;
7098 : : struct discovery_entry_ctx *entry_ctx_in_use;
7099 : : struct spdk_poller *poller;
7100 : : struct spdk_nvme_ctrlr_opts drv_opts;
7101 : : struct spdk_bdev_nvme_ctrlr_opts bdev_opts;
7102 : : struct spdk_nvmf_discovery_log_page *log_page;
7103 : : TAILQ_ENTRY(discovery_ctx) tailq;
7104 : : TAILQ_HEAD(, discovery_entry_ctx) nvm_entry_ctxs;
7105 : : TAILQ_HEAD(, discovery_entry_ctx) discovery_entry_ctxs;
7106 : : int rc;
7107 : : bool wait_for_attach;
7108 : : uint64_t timeout_ticks;
7109 : : /* Denotes that the discovery service is being started. We're waiting
7110 : : * for the initial connection to the discovery controller to be
7111 : : * established and attach discovered NVM ctrlrs.
7112 : : */
7113 : : bool initializing;
7114 : : /* Denotes if a discovery is currently in progress for this context.
7115 : : * That includes connecting to newly discovered subsystems. Used to
7116 : : * ensure we do not start a new discovery until an existing one is
7117 : : * complete.
7118 : : */
7119 : : bool in_progress;
7120 : :
7121 : : /* Denotes if another discovery is needed after the one in progress
7122 : : * completes. Set when we receive an AER completion while a discovery
7123 : : * is already in progress.
7124 : : */
7125 : : bool pending;
7126 : :
7127 : : /* Signal to the discovery context poller that it should stop the
7128 : : * discovery service, including detaching from the current discovery
7129 : : * controller.
7130 : : */
7131 : : bool stop;
7132 : :
7133 : : struct spdk_thread *calling_thread;
7134 : : uint32_t index;
7135 : : uint32_t attach_in_progress;
7136 : : char *hostnqn;
7137 : :
7138 : : /* Denotes if the discovery service was started by the mdns discovery.
7139 : : */
7140 : : bool from_mdns_discovery_service;
7141 : : };
7142 : :
7143 : : TAILQ_HEAD(discovery_ctxs, discovery_ctx);
7144 : : static struct discovery_ctxs g_discovery_ctxs = TAILQ_HEAD_INITIALIZER(g_discovery_ctxs);
7145 : :
7146 : : static void get_discovery_log_page(struct discovery_ctx *ctx);
7147 : :
7148 : : static void
7149 : 34 : free_discovery_ctx(struct discovery_ctx *ctx)
7150 : : {
7151 [ # # # # ]: 34 : free(ctx->log_page);
7152 [ # # # # ]: 34 : free(ctx->hostnqn);
7153 [ # # # # ]: 34 : free(ctx->name);
7154 : 34 : free(ctx);
7155 : 34 : }
7156 : :
7157 : : static void
7158 : 49 : discovery_complete(struct discovery_ctx *ctx)
7159 : : {
7160 [ # # # # ]: 49 : ctx->initializing = false;
7161 [ # # # # ]: 49 : ctx->in_progress = false;
7162 [ - + + + : 49 : if (ctx->pending) {
# # # # ]
7163 [ # # # # ]: 5 : ctx->pending = false;
7164 : 5 : get_discovery_log_page(ctx);
7165 : 0 : }
7166 : 49 : }
7167 : :
7168 : : static void
7169 : 161 : build_trid_from_log_page_entry(struct spdk_nvme_transport_id *trid,
7170 : : struct spdk_nvmf_discovery_log_page_entry *entry)
7171 : : {
7172 : : char *space;
7173 : :
7174 [ # # # # : 161 : trid->trtype = entry->trtype;
# # # # ]
7175 [ # # # # : 161 : trid->adrfam = entry->adrfam;
# # # # ]
7176 [ - + - + : 161 : memcpy(trid->traddr, entry->traddr, sizeof(entry->traddr));
# # # # ]
7177 [ # # # # : 161 : memcpy(trid->trsvcid, entry->trsvcid, sizeof(entry->trsvcid));
# # # # ]
7178 : : /* Because the source buffer (entry->subnqn) is longer than trid->subnqn, and
7179 : : * before call to this function trid->subnqn is zeroed out, we need
7180 : : * to copy sizeof(trid->subnqn) minus one byte to make sure the last character
7181 : : * remains 0. Then we can shorten the string (replace ' ' with 0) if required
7182 : : */
7183 [ - + - + : 161 : memcpy(trid->subnqn, entry->subnqn, sizeof(trid->subnqn) - 1);
# # # # ]
7184 : :
7185 : : /* We want the traddr, trsvcid and subnqn fields to be NULL-terminated.
7186 : : * But the log page entries typically pad them with spaces, not zeroes.
7187 : : * So add a NULL terminator to each of these fields at the appropriate
7188 : : * location.
7189 : : */
7190 [ - + # # ]: 161 : space = strchr(trid->traddr, ' ');
7191 [ + - ]: 161 : if (space) {
7192 [ # # ]: 161 : *space = 0;
7193 : 0 : }
7194 [ - + # # ]: 161 : space = strchr(trid->trsvcid, ' ');
7195 [ + - ]: 161 : if (space) {
7196 [ # # ]: 161 : *space = 0;
7197 : 0 : }
7198 [ - + # # ]: 161 : space = strchr(trid->subnqn, ' ');
7199 [ - + ]: 161 : if (space) {
7200 [ # # ]: 0 : *space = 0;
7201 : 0 : }
7202 : 161 : }
7203 : :
7204 : : static void
7205 : 34 : _stop_discovery(void *_ctx)
7206 : : {
7207 : 34 : struct discovery_ctx *ctx = _ctx;
7208 : :
7209 [ - + # # : 34 : if (ctx->attach_in_progress > 0) {
# # ]
7210 : 0 : spdk_thread_send_msg(spdk_get_thread(), _stop_discovery, ctx);
7211 : 0 : return;
7212 : : }
7213 : :
7214 [ # # # # ]: 34 : ctx->stop = true;
7215 : :
7216 [ + + # # : 63 : while (!TAILQ_EMPTY(&ctx->nvm_entry_ctxs)) {
# # # # ]
7217 : : struct discovery_entry_ctx *entry_ctx;
7218 : 29 : struct nvme_path_id path = {};
7219 : :
7220 [ # # # # : 29 : entry_ctx = TAILQ_FIRST(&ctx->nvm_entry_ctxs);
# # ]
7221 [ # # ]: 29 : path.trid = entry_ctx->trid;
7222 [ # # ]: 29 : bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL);
7223 [ - + # # : 29 : TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7224 : 29 : free(entry_ctx);
7225 : : }
7226 : :
7227 [ + + # # : 77 : while (!TAILQ_EMPTY(&ctx->discovery_entry_ctxs)) {
# # # # ]
7228 : : struct discovery_entry_ctx *entry_ctx;
7229 : :
7230 [ # # # # : 43 : entry_ctx = TAILQ_FIRST(&ctx->discovery_entry_ctxs);
# # ]
7231 [ + + # # : 43 : TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7232 : 43 : free(entry_ctx);
7233 : : }
7234 : :
7235 [ # # # # ]: 34 : free(ctx->entry_ctx_in_use);
7236 [ # # # # ]: 34 : ctx->entry_ctx_in_use = NULL;
7237 : 0 : }
7238 : :
7239 : : static void
7240 : 34 : stop_discovery(struct discovery_ctx *ctx, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx)
7241 : : {
7242 [ # # # # ]: 34 : ctx->stop_cb_fn = cb_fn;
7243 [ # # # # ]: 34 : ctx->cb_ctx = cb_ctx;
7244 : :
7245 [ - + # # : 34 : if (ctx->attach_in_progress > 0) {
# # ]
7246 [ # # # # : 0 : DISCOVERY_INFOLOG(ctx, "stopping discovery with attach_in_progress: %"PRIu32"\n",
# # # # #
# # # # #
# # # # ]
7247 : : ctx->attach_in_progress);
7248 : 0 : }
7249 : :
7250 : 34 : _stop_discovery(ctx);
7251 : 34 : }
7252 : :
7253 : : static void
7254 : 31 : remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr)
7255 : : {
7256 : : struct discovery_ctx *d_ctx;
7257 : : struct nvme_path_id *path_id;
7258 : 31 : struct spdk_nvme_transport_id trid = {};
7259 : : struct discovery_entry_ctx *entry_ctx, *tmp;
7260 : :
7261 [ # # # # : 31 : path_id = TAILQ_FIRST(&nvme_ctrlr->trids);
# # ]
7262 : :
7263 [ + + # # : 34 : TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7264 [ + + # # : 6 : TAILQ_FOREACH_SAFE(entry_ctx, &d_ctx->nvm_entry_ctxs, tailq, tmp) {
# # # # #
# # # # #
# # ]
7265 [ # # ]: 3 : build_trid_from_log_page_entry(&trid, &entry_ctx->entry);
7266 [ - + # # ]: 3 : if (spdk_nvme_transport_id_compare(&trid, &path_id->trid) != 0) {
7267 : 0 : continue;
7268 : : }
7269 : :
7270 [ - + # # : 3 : TAILQ_REMOVE(&d_ctx->nvm_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7271 : 3 : free(entry_ctx);
7272 [ - + + - : 3 : DISCOVERY_INFOLOG(d_ctx, "Remove discovery entry: %s:%s:%s\n",
# # # # #
# # # #
# ]
7273 : : trid.subnqn, trid.traddr, trid.trsvcid);
7274 : :
7275 : : /* Fail discovery ctrlr to force reattach attempt */
7276 [ # # # # ]: 3 : spdk_nvme_ctrlr_fail(d_ctx->ctrlr);
7277 : 0 : }
7278 : 0 : }
7279 : 31 : }
7280 : :
7281 : : static void
7282 : 49 : discovery_remove_controllers(struct discovery_ctx *ctx)
7283 : : {
7284 [ # # # # ]: 49 : struct spdk_nvmf_discovery_log_page *log_page = ctx->log_page;
7285 : : struct discovery_entry_ctx *entry_ctx, *tmp;
7286 : : struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry;
7287 : 49 : struct spdk_nvme_transport_id old_trid = {};
7288 : : uint64_t numrec, i;
7289 : : bool found;
7290 : :
7291 [ # # ]: 49 : numrec = from_le64(&log_page->numrec);
7292 [ + + # # : 109 : TAILQ_FOREACH_SAFE(entry_ctx, &ctx->nvm_entry_ctxs, tailq, tmp) {
# # # # #
# # # # #
# # ]
7293 : 60 : found = false;
7294 [ # # ]: 60 : old_entry = &entry_ctx->entry;
7295 : 60 : build_trid_from_log_page_entry(&old_trid, old_entry);
7296 [ + + ]: 143 : for (i = 0; i < numrec; i++) {
7297 [ # # # # ]: 138 : new_entry = &log_page->entries[i];
7298 [ - + - + : 138 : if (!memcmp(old_entry, new_entry, sizeof(*old_entry))) {
+ + ]
7299 [ - + + + : 55 : DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s found again\n",
# # # # #
# # # #
# ]
7300 : : old_trid.subnqn, old_trid.traddr, old_trid.trsvcid);
7301 : 55 : found = true;
7302 : 55 : break;
7303 : : }
7304 : 0 : }
7305 [ + + # # ]: 60 : if (!found) {
7306 : 5 : struct nvme_path_id path = {};
7307 : :
7308 [ - + + - : 5 : DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s not found\n",
# # # # #
# # # #
# ]
7309 : : old_trid.subnqn, old_trid.traddr, old_trid.trsvcid);
7310 : :
7311 [ # # ]: 5 : path.trid = entry_ctx->trid;
7312 [ # # ]: 5 : bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL);
7313 [ + - # # : 5 : TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7314 : 5 : free(entry_ctx);
7315 : 0 : }
7316 : 0 : }
7317 : 49 : free(log_page);
7318 [ # # # # ]: 49 : ctx->log_page = NULL;
7319 : 49 : discovery_complete(ctx);
7320 : 49 : }
7321 : :
7322 : : static void
7323 : 42 : complete_discovery_start(struct discovery_ctx *ctx, int status)
7324 : : {
7325 [ # # # # ]: 42 : ctx->timeout_ticks = 0;
7326 [ # # # # ]: 42 : ctx->rc = status;
7327 [ + + # # : 42 : if (ctx->start_cb_fn) {
# # ]
7328 [ # # # # : 27 : ctx->start_cb_fn(ctx->cb_ctx, status);
# # # # #
# # # ]
7329 [ # # # # ]: 27 : ctx->start_cb_fn = NULL;
7330 [ # # # # ]: 27 : ctx->cb_ctx = NULL;
7331 : 0 : }
7332 : 42 : }
7333 : :
7334 : : static void
7335 : 37 : discovery_attach_controller_done(void *cb_ctx, size_t bdev_count, int rc)
7336 : : {
7337 : 37 : struct discovery_entry_ctx *entry_ctx = cb_ctx;
7338 [ # # # # ]: 37 : struct discovery_ctx *ctx = entry_ctx->ctx;
7339 : :
7340 [ - + + + : 37 : DISCOVERY_INFOLOG(ctx, "attach %s done\n", entry_ctx->name);
# # # # #
# # # # #
# # ]
7341 [ # # ]: 37 : ctx->attach_in_progress--;
7342 [ + - # # : 37 : if (ctx->attach_in_progress == 0) {
# # ]
7343 [ # # # # ]: 37 : complete_discovery_start(ctx, ctx->rc);
7344 [ - + + + : 37 : if (ctx->initializing && ctx->rc != 0) {
- + # # #
# # # #
# ]
7345 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "stopping discovery due to errors: %d\n", ctx->rc);
# # # # #
# # # ]
7346 [ # # # # ]: 0 : stop_discovery(ctx, NULL, ctx->cb_ctx);
7347 : 0 : } else {
7348 : 37 : discovery_remove_controllers(ctx);
7349 : : }
7350 : 0 : }
7351 : 37 : }
7352 : :
7353 : : static struct discovery_entry_ctx *
7354 : 95 : create_discovery_entry_ctx(struct discovery_ctx *ctx, struct spdk_nvme_transport_id *trid)
7355 : : {
7356 : : struct discovery_entry_ctx *new_ctx;
7357 : :
7358 : 95 : new_ctx = calloc(1, sizeof(*new_ctx));
7359 [ - + ]: 95 : if (new_ctx == NULL) {
7360 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
# # # # ]
7361 : 0 : return NULL;
7362 : : }
7363 : :
7364 [ # # # # ]: 95 : new_ctx->ctx = ctx;
7365 [ - + - + : 95 : memcpy(&new_ctx->trid, trid, sizeof(*trid));
# # ]
7366 [ # # ]: 95 : spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts));
7367 [ - + # # : 95 : snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn);
# # # # ]
7368 : 95 : return new_ctx;
7369 : 0 : }
7370 : :
7371 : : static void
7372 : 49 : discovery_log_page_cb(void *cb_arg, int rc, const struct spdk_nvme_cpl *cpl,
7373 : : struct spdk_nvmf_discovery_log_page *log_page)
7374 : : {
7375 : 49 : struct discovery_ctx *ctx = cb_arg;
7376 : : struct discovery_entry_ctx *entry_ctx, *tmp;
7377 : : struct spdk_nvmf_discovery_log_page_entry *new_entry, *old_entry;
7378 : : uint64_t numrec, i;
7379 : : bool found;
7380 : :
7381 [ + - + - : 49 : if (rc || spdk_nvme_cpl_is_error(cpl)) {
- + # # #
# # # # #
# # # # ]
7382 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n");
# # # # ]
7383 : 0 : return;
7384 : : }
7385 : :
7386 [ # # # # ]: 49 : ctx->log_page = log_page;
7387 [ - + # # : 49 : assert(ctx->attach_in_progress == 0);
# # # # ]
7388 [ # # ]: 49 : numrec = from_le64(&log_page->numrec);
7389 [ + + # # : 71 : TAILQ_FOREACH_SAFE(entry_ctx, &ctx->discovery_entry_ctxs, tailq, tmp) {
# # # # #
# # # # #
# # ]
7390 [ + + # # : 22 : TAILQ_REMOVE(&ctx->discovery_entry_ctxs, entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7391 : 22 : free(entry_ctx);
7392 : 0 : }
7393 [ + + ]: 165 : for (i = 0; i < numrec; i++) {
7394 : 116 : found = false;
7395 [ # # # # ]: 116 : new_entry = &log_page->entries[i];
7396 [ + + # # : 116 : if (new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY_CURRENT ||
# # # # ]
7397 [ - + # # ]: 55 : new_entry->subtype == SPDK_NVMF_SUBTYPE_DISCOVERY) {
7398 : : struct discovery_entry_ctx *new_ctx;
7399 : 61 : struct spdk_nvme_transport_id trid = {};
7400 : :
7401 : 61 : build_trid_from_log_page_entry(&trid, new_entry);
7402 : 61 : new_ctx = create_discovery_entry_ctx(ctx, &trid);
7403 [ - + ]: 61 : if (new_ctx == NULL) {
7404 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
# # # # ]
7405 : 0 : break;
7406 : : }
7407 : :
7408 [ # # # # : 61 : TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, new_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
7409 : 61 : continue;
7410 : : }
7411 [ + + # # : 67 : TAILQ_FOREACH(entry_ctx, &ctx->nvm_entry_ctxs, tailq) {
# # # # #
# # # #
# ]
7412 [ # # ]: 30 : old_entry = &entry_ctx->entry;
7413 [ - + - + : 30 : if (!memcmp(new_entry, old_entry, sizeof(*new_entry))) {
+ + ]
7414 : 18 : found = true;
7415 : 18 : break;
7416 : : }
7417 : 0 : }
7418 [ + + # # ]: 55 : if (!found) {
7419 : 37 : struct discovery_entry_ctx *subnqn_ctx = NULL, *new_ctx;
7420 : : struct discovery_ctx *d_ctx;
7421 : :
7422 [ + + # # : 77 : TAILQ_FOREACH(d_ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7423 [ + + # # : 51 : TAILQ_FOREACH(subnqn_ctx, &d_ctx->nvm_entry_ctxs, tailq) {
# # # # #
# # # #
# ]
7424 [ - + - + : 11 : if (!memcmp(subnqn_ctx->entry.subnqn, new_entry->subnqn,
+ + # # #
# # # ]
7425 : : sizeof(new_entry->subnqn))) {
7426 : 5 : break;
7427 : : }
7428 : 0 : }
7429 [ + + ]: 45 : if (subnqn_ctx) {
7430 : 5 : break;
7431 : : }
7432 : 0 : }
7433 : :
7434 : 37 : new_ctx = calloc(1, sizeof(*new_ctx));
7435 [ - + ]: 37 : if (new_ctx == NULL) {
7436 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
# # # # ]
7437 : 0 : break;
7438 : : }
7439 : :
7440 [ # # # # ]: 37 : new_ctx->ctx = ctx;
7441 [ - + - + : 37 : memcpy(&new_ctx->entry, new_entry, sizeof(*new_entry));
# # ]
7442 [ # # ]: 37 : build_trid_from_log_page_entry(&new_ctx->trid, new_entry);
7443 [ + + ]: 37 : if (subnqn_ctx) {
7444 [ # # # # ]: 5 : snprintf(new_ctx->name, sizeof(new_ctx->name), "%s", subnqn_ctx->name);
7445 [ - + + - : 5 : DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new path for %s\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7446 : : new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid,
7447 : : new_ctx->name);
7448 : 0 : } else {
7449 [ # # # # : 32 : snprintf(new_ctx->name, sizeof(new_ctx->name), "%s%d", ctx->name, ctx->index++);
# # # # ]
7450 [ - + + + : 32 : DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s new subsystem %s\n",
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7451 : : new_ctx->trid.subnqn, new_ctx->trid.traddr, new_ctx->trid.trsvcid,
7452 : : new_ctx->name);
7453 : : }
7454 [ # # ]: 37 : spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts));
7455 [ # # # # : 37 : snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn);
# # # # ]
7456 [ # # # # ]: 37 : rc = spdk_bdev_nvme_create(&new_ctx->trid, new_ctx->name, NULL, 0,
7457 : 0 : discovery_attach_controller_done, new_ctx,
7458 [ # # # # ]: 0 : &new_ctx->drv_opts, &ctx->bdev_opts);
7459 [ + - ]: 37 : if (rc == 0) {
7460 [ # # # # : 37 : TAILQ_INSERT_TAIL(&ctx->nvm_entry_ctxs, new_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
7461 [ # # ]: 37 : ctx->attach_in_progress++;
7462 : 0 : } else {
7463 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "spdk_bdev_nvme_create failed (%s)\n", spdk_strerror(-rc));
# # # # #
# ]
7464 : : }
7465 : 0 : }
7466 : 0 : }
7467 : :
7468 [ + + # # : 49 : if (ctx->attach_in_progress == 0) {
# # ]
7469 : 12 : discovery_remove_controllers(ctx);
7470 : 0 : }
7471 : 0 : }
7472 : :
7473 : : static void
7474 : 49 : get_discovery_log_page(struct discovery_ctx *ctx)
7475 : : {
7476 : : int rc;
7477 : :
7478 [ - + - + : 49 : assert(ctx->in_progress == false);
# # # # #
# ]
7479 [ # # # # ]: 49 : ctx->in_progress = true;
7480 [ # # # # ]: 49 : rc = spdk_nvme_ctrlr_get_discovery_log_page(ctx->ctrlr, discovery_log_page_cb, ctx);
7481 [ - + ]: 49 : if (rc != 0) {
7482 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not get discovery log page\n");
# # # # ]
7483 : 0 : }
7484 [ - + + + : 49 : DISCOVERY_INFOLOG(ctx, "sent discovery log page command\n");
# # # # #
# # # #
# ]
7485 : 49 : }
7486 : :
7487 : : static void
7488 : 16 : discovery_aer_cb(void *arg, const struct spdk_nvme_cpl *cpl)
7489 : : {
7490 : 16 : struct discovery_ctx *ctx = arg;
7491 [ # # # # : 16 : uint32_t log_page_id = (cpl->cdw0 & 0xFF0000) >> 16;
# # ]
7492 : :
7493 [ + - - + : 16 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
7494 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "aer failed\n");
# # # # ]
7495 : 0 : return;
7496 : : }
7497 : :
7498 [ - + ]: 16 : if (log_page_id != SPDK_NVME_LOG_DISCOVERY) {
7499 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "unexpected log page 0x%x\n", log_page_id);
# # # # ]
7500 : 0 : return;
7501 : : }
7502 : :
7503 [ - + + - : 16 : DISCOVERY_INFOLOG(ctx, "got aer\n");
# # # # #
# # # #
# ]
7504 [ - + + + : 16 : if (ctx->in_progress) {
# # # # ]
7505 [ # # # # ]: 5 : ctx->pending = true;
7506 : 5 : return;
7507 : : }
7508 : :
7509 : 11 : get_discovery_log_page(ctx);
7510 : 0 : }
7511 : :
7512 : : static void
7513 : 33 : discovery_attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
7514 : : struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
7515 : : {
7516 : 33 : struct spdk_nvme_ctrlr_opts *user_opts = cb_ctx;
7517 : : struct discovery_ctx *ctx;
7518 : :
7519 : 33 : ctx = SPDK_CONTAINEROF(user_opts, struct discovery_ctx, drv_opts);
7520 : :
7521 [ - + + + : 33 : DISCOVERY_INFOLOG(ctx, "discovery ctrlr attached\n");
# # # # #
# # # #
# ]
7522 [ # # # # ]: 33 : ctx->probe_ctx = NULL;
7523 [ # # # # ]: 33 : ctx->ctrlr = ctrlr;
7524 : :
7525 [ - + # # : 33 : if (ctx->rc != 0) {
# # ]
7526 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "encountered error while attaching discovery ctrlr: %d\n",
# # # # #
# # # ]
7527 : : ctx->rc);
7528 : 0 : return;
7529 : : }
7530 : :
7531 [ # # # # ]: 33 : spdk_nvme_ctrlr_register_aer_callback(ctx->ctrlr, discovery_aer_cb, ctx);
7532 : 0 : }
7533 : :
7534 : : static int
7535 : 109777 : discovery_poller(void *arg)
7536 : : {
7537 : 109777 : struct discovery_ctx *ctx = arg;
7538 : : struct spdk_nvme_transport_id *trid;
7539 : : int rc;
7540 : :
7541 [ + + # # : 109777 : if (ctx->detach_ctx) {
# # ]
7542 [ # # # # ]: 243 : rc = spdk_nvme_detach_poll_async(ctx->detach_ctx);
7543 [ + + ]: 243 : if (rc != -EAGAIN) {
7544 [ # # # # ]: 33 : ctx->detach_ctx = NULL;
7545 [ # # # # ]: 33 : ctx->ctrlr = NULL;
7546 : 0 : }
7547 [ - + + + : 109534 : } else if (ctx->stop) {
# # # # ]
7548 [ + + # # : 60 : if (ctx->ctrlr != NULL) {
# # ]
7549 [ # # # # : 30 : rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx);
# # ]
7550 [ + - ]: 30 : if (rc == 0) {
7551 : 30 : return SPDK_POLLER_BUSY;
7552 : : }
7553 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n");
# # # # ]
7554 : 0 : }
7555 [ # # ]: 30 : spdk_poller_unregister(&ctx->poller);
7556 [ + + # # : 30 : TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7557 [ - + # # : 30 : assert(ctx->start_cb_fn == NULL);
# # # # ]
7558 [ + + # # : 30 : if (ctx->stop_cb_fn != NULL) {
# # ]
7559 [ # # # # : 25 : ctx->stop_cb_fn(ctx->cb_ctx);
# # # # #
# # # ]
7560 : 0 : }
7561 : 30 : free_discovery_ctx(ctx);
7562 [ + + + + : 109474 : } else if (ctx->probe_ctx == NULL && ctx->ctrlr == NULL) {
# # # # #
# # # ]
7563 [ + + + + : 50 : if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) {
# # # # #
# # # ]
7564 [ # # # # : 4 : DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n");
# # # # ]
7565 [ - + - + : 4 : assert(ctx->initializing);
# # # # #
# ]
7566 [ # # ]: 4 : spdk_poller_unregister(&ctx->poller);
7567 [ - + # # : 4 : TAILQ_REMOVE(&g_discovery_ctxs, ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7568 : 4 : complete_discovery_start(ctx, -ETIMEDOUT);
7569 : 4 : stop_discovery(ctx, NULL, NULL);
7570 : 4 : free_discovery_ctx(ctx);
7571 : 4 : return SPDK_POLLER_BUSY;
7572 : : }
7573 : :
7574 [ - + # # : 46 : assert(ctx->entry_ctx_in_use == NULL);
# # # # ]
7575 [ # # # # : 46 : ctx->entry_ctx_in_use = TAILQ_FIRST(&ctx->discovery_entry_ctxs);
# # # # #
# ]
7576 [ + + # # : 46 : TAILQ_REMOVE(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
7577 [ # # # # : 46 : trid = &ctx->entry_ctx_in_use->trid;
# # ]
7578 : :
7579 : : /* All controllers must be configured explicitely either for multipath or failover.
7580 : : * While discovery use multipath mode, we need to set this in bdev options as well.
7581 : : */
7582 [ # # # # : 46 : ctx->bdev_opts.multipath = true;
# # ]
7583 : :
7584 [ # # # # : 46 : ctx->probe_ctx = spdk_nvme_connect_async(trid, &ctx->drv_opts, discovery_attach_cb);
# # ]
7585 [ + + # # : 46 : if (ctx->probe_ctx) {
# # ]
7586 [ # # ]: 33 : spdk_poller_unregister(&ctx->poller);
7587 [ # # # # ]: 33 : ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000);
7588 : 0 : } else {
7589 [ # # # # : 13 : DISCOVERY_ERRLOG(ctx, "could not start discovery connect\n");
# # # # ]
7590 [ # # # # : 13 : TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7591 [ # # # # ]: 13 : ctx->entry_ctx_in_use = NULL;
7592 : : }
7593 [ + + # # : 109424 : } else if (ctx->probe_ctx) {
# # ]
7594 [ + + - + : 33 : if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) {
# # # # #
# # # ]
7595 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "timed out while attaching discovery ctrlr\n");
# # # # ]
7596 : 0 : complete_discovery_start(ctx, -ETIMEDOUT);
7597 : 0 : return SPDK_POLLER_BUSY;
7598 : : }
7599 : :
7600 [ # # # # ]: 33 : rc = spdk_nvme_probe_poll_async(ctx->probe_ctx);
7601 [ + - ]: 33 : if (rc != -EAGAIN) {
7602 [ - + # # : 33 : if (ctx->rc != 0) {
# # ]
7603 [ # # # # : 0 : assert(ctx->initializing);
# # # # #
# ]
7604 [ # # # # ]: 0 : stop_discovery(ctx, NULL, ctx->cb_ctx);
7605 : 0 : } else {
7606 [ - + # # ]: 33 : assert(rc == 0);
7607 [ - + + + : 33 : DISCOVERY_INFOLOG(ctx, "discovery ctrlr connected\n");
# # # # #
# # # #
# ]
7608 [ # # # # ]: 33 : ctx->rc = rc;
7609 : 33 : get_discovery_log_page(ctx);
7610 : : }
7611 : 0 : }
7612 : 0 : } else {
7613 [ + + + + : 109391 : if (ctx->timeout_ticks != 0 && ctx->timeout_ticks < spdk_get_ticks()) {
# # # # #
# # # ]
7614 [ # # # # : 1 : DISCOVERY_ERRLOG(ctx, "timed out while attaching NVM ctrlrs\n");
# # # # ]
7615 : 1 : complete_discovery_start(ctx, -ETIMEDOUT);
7616 : : /* We need to wait until all NVM ctrlrs are attached before we stop the
7617 : : * discovery service to make sure we don't detach a ctrlr that is still
7618 : : * being attached.
7619 : : */
7620 [ + - # # : 1 : if (ctx->attach_in_progress == 0) {
# # ]
7621 [ # # # # ]: 1 : stop_discovery(ctx, NULL, ctx->cb_ctx);
7622 : 1 : return SPDK_POLLER_BUSY;
7623 : : }
7624 : 0 : }
7625 : :
7626 [ # # # # ]: 109390 : rc = spdk_nvme_ctrlr_process_admin_completions(ctx->ctrlr);
7627 [ + + ]: 109390 : if (rc < 0) {
7628 [ # # ]: 3 : spdk_poller_unregister(&ctx->poller);
7629 [ # # # # ]: 3 : ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000);
7630 [ # # # # : 3 : TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, ctx->entry_ctx_in_use, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7631 [ # # # # ]: 3 : ctx->entry_ctx_in_use = NULL;
7632 : :
7633 [ # # # # : 3 : rc = spdk_nvme_detach_async(ctx->ctrlr, &ctx->detach_ctx);
# # ]
7634 [ - + ]: 3 : if (rc != 0) {
7635 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not detach discovery ctrlr\n");
# # # # ]
7636 [ # # # # ]: 0 : ctx->ctrlr = NULL;
7637 : 0 : }
7638 : 0 : }
7639 : : }
7640 : :
7641 : 109742 : return SPDK_POLLER_BUSY;
7642 : 0 : }
7643 : :
7644 : : static void
7645 : 34 : start_discovery_poller(void *arg)
7646 : : {
7647 : 34 : struct discovery_ctx *ctx = arg;
7648 : :
7649 [ # # # # : 34 : TAILQ_INSERT_TAIL(&g_discovery_ctxs, ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # #
# ]
7650 [ # # # # ]: 34 : ctx->poller = SPDK_POLLER_REGISTER(discovery_poller, ctx, 1000 * 1000);
7651 : 34 : }
7652 : :
7653 : : int
7654 : 40 : bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid,
7655 : : const char *base_name,
7656 : : struct spdk_nvme_ctrlr_opts *drv_opts,
7657 : : struct spdk_bdev_nvme_ctrlr_opts *bdev_opts,
7658 : : uint64_t attach_timeout,
7659 : : bool from_mdns,
7660 : : spdk_bdev_nvme_start_discovery_fn cb_fn, void *cb_ctx)
7661 : : {
7662 : : struct discovery_ctx *ctx;
7663 : : struct discovery_entry_ctx *discovery_entry_ctx;
7664 : :
7665 [ # # ]: 40 : snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
7666 [ + + # # : 50 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7667 [ - + - + : 16 : if (strcmp(ctx->name, base_name) == 0) {
+ + # # #
# ]
7668 : 3 : return -EEXIST;
7669 : : }
7670 : :
7671 [ + + # # : 13 : if (ctx->entry_ctx_in_use != NULL) {
# # ]
7672 [ + + # # : 11 : if (!spdk_nvme_transport_id_compare(trid, &ctx->entry_ctx_in_use->trid)) {
# # # # ]
7673 : 3 : return -EEXIST;
7674 : : }
7675 : 0 : }
7676 : :
7677 [ + + # # : 20 : TAILQ_FOREACH(discovery_entry_ctx, &ctx->discovery_entry_ctxs, tailq) {
# # # # #
# # # #
# ]
7678 [ - + # # ]: 10 : if (!spdk_nvme_transport_id_compare(trid, &discovery_entry_ctx->trid)) {
7679 : 0 : return -EEXIST;
7680 : : }
7681 : 0 : }
7682 : 0 : }
7683 : :
7684 : 34 : ctx = calloc(1, sizeof(*ctx));
7685 [ - + ]: 34 : if (ctx == NULL) {
7686 : 0 : return -ENOMEM;
7687 : : }
7688 : :
7689 [ - + # # : 34 : ctx->name = strdup(base_name);
# # ]
7690 [ - + # # : 34 : if (ctx->name == NULL) {
# # ]
7691 : 0 : free_discovery_ctx(ctx);
7692 : 0 : return -ENOMEM;
7693 : : }
7694 [ - + - + : 34 : memcpy(&ctx->drv_opts, drv_opts, sizeof(*drv_opts));
# # ]
7695 [ - + - + : 34 : memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts));
# # ]
7696 [ # # # # : 34 : ctx->from_mdns_discovery_service = from_mdns;
# # ]
7697 [ # # # # : 34 : ctx->bdev_opts.from_discovery_service = true;
# # ]
7698 [ # # # # ]: 34 : ctx->calling_thread = spdk_get_thread();
7699 [ # # # # ]: 34 : ctx->start_cb_fn = cb_fn;
7700 [ # # # # ]: 34 : ctx->cb_ctx = cb_ctx;
7701 [ # # # # ]: 34 : ctx->initializing = true;
7702 [ + + # # : 34 : if (ctx->start_cb_fn) {
# # ]
7703 : : /* We can use this when dumping json to denote if this RPC parameter
7704 : : * was specified or not.
7705 : : */
7706 [ # # # # ]: 27 : ctx->wait_for_attach = true;
7707 : 0 : }
7708 [ + + ]: 34 : if (attach_timeout != 0) {
7709 [ # # # # ]: 42 : ctx->timeout_ticks = spdk_get_ticks() + attach_timeout *
7710 [ # # # # ]: 21 : spdk_get_ticks_hz() / 1000ull;
7711 : 0 : }
7712 [ # # # # : 34 : TAILQ_INIT(&ctx->nvm_entry_ctxs);
# # # # #
# # # # #
# # ]
7713 [ # # # # : 34 : TAILQ_INIT(&ctx->discovery_entry_ctxs);
# # # # #
# # # # #
# # ]
7714 [ - + - + : 34 : memcpy(&ctx->trid, trid, sizeof(*trid));
# # ]
7715 : : /* Even if user did not specify hostnqn, we can still strdup("\0"); */
7716 [ - + # # : 34 : ctx->hostnqn = strdup(ctx->drv_opts.hostnqn);
# # # # #
# ]
7717 [ - + # # : 34 : if (ctx->hostnqn == NULL) {
# # ]
7718 : 0 : free_discovery_ctx(ctx);
7719 : 0 : return -ENOMEM;
7720 : : }
7721 : 34 : discovery_entry_ctx = create_discovery_entry_ctx(ctx, trid);
7722 [ - + ]: 34 : if (discovery_entry_ctx == NULL) {
7723 [ # # # # : 0 : DISCOVERY_ERRLOG(ctx, "could not allocate new entry_ctx\n");
# # # # ]
7724 : 0 : free_discovery_ctx(ctx);
7725 : 0 : return -ENOMEM;
7726 : : }
7727 : :
7728 [ # # # # : 34 : TAILQ_INSERT_TAIL(&ctx->discovery_entry_ctxs, discovery_entry_ctx, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
7729 : 34 : spdk_thread_send_msg(g_bdev_nvme_init_thread, start_discovery_poller, ctx);
7730 : 34 : return 0;
7731 : 0 : }
7732 : :
7733 : : int
7734 : 23 : bdev_nvme_stop_discovery(const char *name, spdk_bdev_nvme_stop_discovery_fn cb_fn, void *cb_ctx)
7735 : : {
7736 : : struct discovery_ctx *ctx;
7737 : :
7738 [ + - # # : 26 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7739 [ - + - + : 26 : if (strcmp(name, ctx->name) == 0) {
+ + # # #
# ]
7740 [ - + - + : 23 : if (ctx->stop) {
# # # # ]
7741 : 0 : return -EALREADY;
7742 : : }
7743 : : /* If we're still starting the discovery service and ->rc is non-zero, we're
7744 : : * going to stop it as soon as we can
7745 : : */
7746 [ - + - + : 23 : if (ctx->initializing && ctx->rc != 0) {
- - # # #
# # # #
# ]
7747 : 0 : return -EALREADY;
7748 : : }
7749 : 23 : stop_discovery(ctx, cb_fn, cb_ctx);
7750 : 23 : return 0;
7751 : : }
7752 : 0 : }
7753 : :
7754 : 0 : return -ENOENT;
7755 : 0 : }
7756 : :
7757 : : static int
7758 : 1964 : bdev_nvme_library_init(void)
7759 : : {
7760 : 1964 : g_bdev_nvme_init_thread = spdk_get_thread();
7761 : :
7762 : 1964 : spdk_io_device_register(&g_nvme_bdev_ctrlrs, bdev_nvme_create_poll_group_cb,
7763 : : bdev_nvme_destroy_poll_group_cb,
7764 : : sizeof(struct nvme_poll_group), "nvme_poll_groups");
7765 : :
7766 : 1964 : return 0;
7767 : : }
7768 : :
7769 : : static void
7770 : 1964 : bdev_nvme_fini_destruct_ctrlrs(void)
7771 : : {
7772 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
7773 : : struct nvme_ctrlr *nvme_ctrlr;
7774 : :
7775 [ + + ]: 1964 : pthread_mutex_lock(&g_bdev_nvme_mutex);
7776 [ + + + - : 2683 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
+ - + - ]
7777 [ + + + - : 1444 : TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
+ - + + +
- + - +
- ]
7778 [ + + + - ]: 725 : pthread_mutex_lock(&nvme_ctrlr->mutex);
7779 [ + + - + ]: 725 : if (nvme_ctrlr->destruct) {
7780 : : /* This controller's destruction was already started
7781 : : * before the application started shutting down
7782 : : */
7783 [ # # # # ]: 0 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
7784 : 0 : continue;
7785 : : }
7786 [ - + ]: 725 : nvme_ctrlr->destruct = true;
7787 [ - + - + ]: 725 : pthread_mutex_unlock(&nvme_ctrlr->mutex);
7788 : :
7789 [ - + - + ]: 732 : spdk_thread_send_msg(nvme_ctrlr->thread, _nvme_ctrlr_destruct,
7790 : 8 : nvme_ctrlr);
7791 : 8 : }
7792 : 8 : }
7793 : :
7794 : 1964 : g_bdev_nvme_module_finish = true;
7795 [ + + ]: 1964 : if (TAILQ_EMPTY(&g_nvme_bdev_ctrlrs)) {
7796 [ - + ]: 1428 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
7797 : 1428 : spdk_io_device_unregister(&g_nvme_bdev_ctrlrs, NULL);
7798 : 1428 : spdk_bdev_module_fini_done();
7799 : 1428 : return;
7800 : : }
7801 : :
7802 [ + + ]: 536 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
7803 : 93 : }
7804 : :
7805 : : static void
7806 : 6 : check_discovery_fini(void *arg)
7807 : : {
7808 [ + - ]: 6 : if (TAILQ_EMPTY(&g_discovery_ctxs)) {
7809 : 6 : bdev_nvme_fini_destruct_ctrlrs();
7810 : 0 : }
7811 : 6 : }
7812 : :
7813 : : static void
7814 : 1964 : bdev_nvme_library_fini(void)
7815 : : {
7816 : : struct nvme_probe_skip_entry *entry, *entry_tmp;
7817 : : struct discovery_ctx *ctx;
7818 : :
7819 : 1964 : spdk_poller_unregister(&g_hotplug_poller);
7820 : 1964 : free(g_hotplug_probe_ctx);
7821 : 1964 : g_hotplug_probe_ctx = NULL;
7822 : :
7823 [ + + + + : 2012 : TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, entry_tmp) {
# # # # -
+ ]
7824 [ + + # # : 48 : TAILQ_REMOVE(&g_skipped_nvme_ctrlrs, entry, tailq);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
7825 : 48 : free(entry);
7826 : 3 : }
7827 : :
7828 [ + + # # ]: 1964 : assert(spdk_get_thread() == g_bdev_nvme_init_thread);
7829 [ + + ]: 1964 : if (TAILQ_EMPTY(&g_discovery_ctxs)) {
7830 : 1958 : bdev_nvme_fini_destruct_ctrlrs();
7831 : 93 : } else {
7832 [ + + # # : 12 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
7833 : 6 : stop_discovery(ctx, check_discovery_fini, NULL);
7834 : 0 : }
7835 : : }
7836 : 1964 : }
7837 : :
7838 : : static void
7839 : 0 : bdev_nvme_verify_pi_error(struct nvme_bdev_io *bio)
7840 : : {
7841 : 0 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
7842 [ # # # # ]: 0 : struct spdk_bdev *bdev = bdev_io->bdev;
7843 : 0 : struct spdk_dif_ctx dif_ctx;
7844 : 0 : struct spdk_dif_error err_blk = {};
7845 : : int rc;
7846 : 0 : struct spdk_dif_ctx_init_ext_opts dif_opts;
7847 : :
7848 : 0 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
7849 [ # # # # : 0 : dif_opts.dif_pi_format = bdev->dif_pi_format;
# # ]
7850 : 0 : rc = spdk_dif_ctx_init(&dif_ctx,
7851 [ # # # # : 0 : bdev->blocklen, bdev->md_len, bdev->md_interleave,
# # # # #
# # # #
# ]
7852 [ # # # # : 0 : bdev->dif_is_head_of_md, bdev->dif_type,
# # # # #
# ]
7853 [ # # # # : 0 : bdev_io->u.bdev.dif_check_flags,
# # # # ]
7854 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks, 0, 0, 0, 0, &dif_opts);
# # # # ]
7855 [ # # ]: 0 : if (rc != 0) {
7856 : 0 : SPDK_ERRLOG("Initialization of DIF context failed\n");
7857 : 0 : return;
7858 : : }
7859 : :
7860 [ # # # # : 0 : if (bdev->md_interleave) {
# # # # ]
7861 [ # # # # : 0 : rc = spdk_dif_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
# # # # #
# # # # #
# # ]
7862 [ # # # # : 0 : bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk);
# # # # ]
7863 : 0 : } else {
7864 : 0 : struct iovec md_iov = {
7865 [ # # # # : 0 : .iov_base = bdev_io->u.bdev.md_buf,
# # # # ]
7866 [ # # # # : 0 : .iov_len = bdev_io->u.bdev.num_blocks * bdev->md_len,
# # # # #
# # # ]
7867 : : };
7868 : :
7869 [ # # # # : 0 : rc = spdk_dix_verify(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
# # # # #
# # # # #
# # ]
7870 [ # # # # : 0 : &md_iov, bdev_io->u.bdev.num_blocks, &dif_ctx, &err_blk);
# # # # ]
7871 : : }
7872 : :
7873 [ # # ]: 0 : if (rc != 0) {
7874 [ # # ]: 0 : SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n",
7875 : : err_blk.err_type, err_blk.err_offset);
7876 : 0 : } else {
7877 : 0 : SPDK_ERRLOG("Hardware reported PI error but SPDK could not find any.\n");
7878 : : }
7879 : 0 : }
7880 : :
7881 : : static void
7882 : 0 : bdev_nvme_no_pi_readv_done(void *ref, const struct spdk_nvme_cpl *cpl)
7883 : : {
7884 : 0 : struct nvme_bdev_io *bio = ref;
7885 : :
7886 [ # # # # : 0 : if (spdk_nvme_cpl_is_success(cpl)) {
# # # # #
# # # # #
# # ]
7887 : : /* Run PI verification for read data buffer. */
7888 : 0 : bdev_nvme_verify_pi_error(bio);
7889 : 0 : }
7890 : :
7891 : : /* Return original completion status */
7892 [ # # ]: 0 : bdev_nvme_io_complete_nvme_status(bio, &bio->cpl);
7893 : 0 : }
7894 : :
7895 : : static void
7896 : 12827881 : bdev_nvme_readv_done(void *ref, const struct spdk_nvme_cpl *cpl)
7897 : : {
7898 : 12827881 : struct nvme_bdev_io *bio = ref;
7899 : 12827881 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
7900 : : int ret;
7901 : :
7902 [ + + + - : 12827881 : if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) {
+ - + - -
- - - # #
# # # # #
# # # # #
# # # # #
# + - ]
7903 [ # # # # : 0 : SPDK_ERRLOG("readv completed with PI error (sct=%d, sc=%d)\n",
# # # # #
# # # ]
7904 : : cpl->status.sct, cpl->status.sc);
7905 : :
7906 : : /* Save completion status to use after verifying PI error. */
7907 [ # # ]: 0 : bio->cpl = *cpl;
7908 : :
7909 [ # # # # : 0 : if (spdk_likely(nvme_io_path_is_available(bio->io_path))) {
# # ]
7910 : : /* Read without PI checking to verify PI error. */
7911 : 0 : ret = bdev_nvme_no_pi_readv(bio,
7912 [ # # # # : 0 : bdev_io->u.bdev.iovs,
# # # # ]
7913 [ # # # # : 0 : bdev_io->u.bdev.iovcnt,
# # # # ]
7914 [ # # # # : 0 : bdev_io->u.bdev.md_buf,
# # # # ]
7915 [ # # # # : 0 : bdev_io->u.bdev.num_blocks,
# # # # ]
7916 [ # # # # : 0 : bdev_io->u.bdev.offset_blocks);
# # # # ]
7917 [ # # ]: 0 : if (ret == 0) {
7918 : 0 : return;
7919 : : }
7920 : 0 : }
7921 : 0 : }
7922 : :
7923 : 12827881 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7924 : 265740 : }
7925 : :
7926 : : static void
7927 : 13663343 : bdev_nvme_writev_done(void *ref, const struct spdk_nvme_cpl *cpl)
7928 : : {
7929 : 13663343 : struct nvme_bdev_io *bio = ref;
7930 : :
7931 [ + + - - : 13663343 : if (spdk_unlikely(spdk_nvme_cpl_is_pi_error(cpl))) {
- - + - -
- - - # #
# # # # #
# # # # #
# # # # #
# # # ]
7932 [ # # # # : 0 : SPDK_ERRLOG("writev completed with PI error (sct=%d, sc=%d)\n",
# # # # #
# # # ]
7933 : : cpl->status.sct, cpl->status.sc);
7934 : : /* Run PI verification for write data buffer if PI error is detected. */
7935 : 0 : bdev_nvme_verify_pi_error(bio);
7936 : 0 : }
7937 : :
7938 : 13663343 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7939 : 13663343 : }
7940 : :
7941 : : static void
7942 : 256879 : bdev_nvme_zone_appendv_done(void *ref, const struct spdk_nvme_cpl *cpl)
7943 : : {
7944 : 256879 : struct nvme_bdev_io *bio = ref;
7945 : 256879 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
7946 : :
7947 : : /* spdk_bdev_io_get_append_location() requires that the ALBA is stored in offset_blocks.
7948 : : * Additionally, offset_blocks has to be set before calling bdev_nvme_verify_pi_error().
7949 : : */
7950 [ # # # # : 256879 : bdev_io->u.bdev.offset_blocks = *(uint64_t *)&cpl->cdw0;
# # # # #
# # # ]
7951 : :
7952 [ - + - - : 256879 : if (spdk_nvme_cpl_is_pi_error(cpl)) {
- - - - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7953 [ # # # # : 0 : SPDK_ERRLOG("zone append completed with PI error (sct=%d, sc=%d)\n",
# # # # #
# # # ]
7954 : : cpl->status.sct, cpl->status.sc);
7955 : : /* Run PI verification for zone append data buffer if PI error is detected. */
7956 : 0 : bdev_nvme_verify_pi_error(bio);
7957 : 0 : }
7958 : :
7959 : 256879 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7960 : 256879 : }
7961 : :
7962 : : static void
7963 : 56 : bdev_nvme_comparev_done(void *ref, const struct spdk_nvme_cpl *cpl)
7964 : : {
7965 : 56 : struct nvme_bdev_io *bio = ref;
7966 : :
7967 [ + + + - : 56 : if (spdk_nvme_cpl_is_pi_error(cpl)) {
+ - - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
7968 [ # # # # : 0 : SPDK_ERRLOG("comparev completed with PI error (sct=%d, sc=%d)\n",
# # # # #
# # # ]
7969 : : cpl->status.sct, cpl->status.sc);
7970 : : /* Run PI verification for compare data buffer if PI error is detected. */
7971 : 0 : bdev_nvme_verify_pi_error(bio);
7972 : 0 : }
7973 : :
7974 : 56 : bdev_nvme_io_complete_nvme_status(bio, cpl);
7975 : 56 : }
7976 : :
7977 : : static void
7978 : 106 : bdev_nvme_comparev_and_writev_done(void *ref, const struct spdk_nvme_cpl *cpl)
7979 : : {
7980 : 106 : struct nvme_bdev_io *bio = ref;
7981 : :
7982 : : /* Compare operation completion */
7983 [ + + + + : 106 : if (!bio->first_fused_completed) {
# # # # ]
7984 : : /* Save compare result for write callback */
7985 [ # # ]: 53 : bio->cpl = *cpl;
7986 [ # # # # ]: 53 : bio->first_fused_completed = true;
7987 : 53 : return;
7988 : : }
7989 : :
7990 : : /* Write operation completion */
7991 [ + + - + : 53 : if (spdk_nvme_cpl_is_error(&bio->cpl)) {
# # # # #
# # # # #
# # # # #
# ]
7992 : : /* If bio->cpl is already an error, it means the compare operation failed. In that case,
7993 : : * complete the IO with the compare operation's status.
7994 : : */
7995 [ + + + - : 40 : if (!spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
7996 : 4 : SPDK_ERRLOG("Unexpected write success after compare failure.\n");
7997 : 1 : }
7998 : :
7999 [ # # ]: 40 : bdev_nvme_io_complete_nvme_status(bio, &bio->cpl);
8000 : 1 : } else {
8001 : 13 : bdev_nvme_io_complete_nvme_status(bio, cpl);
8002 : : }
8003 : 4 : }
8004 : :
8005 : : static void
8006 : 781852 : bdev_nvme_queued_done(void *ref, const struct spdk_nvme_cpl *cpl)
8007 : : {
8008 : 781852 : struct nvme_bdev_io *bio = ref;
8009 : :
8010 : 781852 : bdev_nvme_io_complete_nvme_status(bio, cpl);
8011 : 781852 : }
8012 : :
8013 : : static int
8014 : 40 : fill_zone_from_report(struct spdk_bdev_zone_info *info, struct spdk_nvme_zns_zone_desc *desc)
8015 : : {
8016 [ + - # # ]: 40 : switch (desc->zt) {
8017 : 40 : case SPDK_NVME_ZONE_TYPE_SEQWR:
8018 [ # # # # ]: 40 : info->type = SPDK_BDEV_ZONE_TYPE_SEQWR;
8019 : 40 : break;
8020 : 0 : default:
8021 [ # # ]: 0 : SPDK_ERRLOG("Invalid zone type: %#x in zone report\n", desc->zt);
8022 : 0 : return -EIO;
8023 : : }
8024 : :
8025 [ + - - - : 40 : switch (desc->zs) {
- - - - #
# ]
8026 : 40 : case SPDK_NVME_ZONE_STATE_EMPTY:
8027 [ # # # # ]: 40 : info->state = SPDK_BDEV_ZONE_STATE_EMPTY;
8028 : 40 : break;
8029 : 0 : case SPDK_NVME_ZONE_STATE_IOPEN:
8030 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_IMP_OPEN;
8031 : 0 : break;
8032 : 0 : case SPDK_NVME_ZONE_STATE_EOPEN:
8033 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_EXP_OPEN;
8034 : 0 : break;
8035 : 0 : case SPDK_NVME_ZONE_STATE_CLOSED:
8036 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_CLOSED;
8037 : 0 : break;
8038 : 0 : case SPDK_NVME_ZONE_STATE_RONLY:
8039 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_READ_ONLY;
8040 : 0 : break;
8041 : 0 : case SPDK_NVME_ZONE_STATE_FULL:
8042 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_FULL;
8043 : 0 : break;
8044 : 0 : case SPDK_NVME_ZONE_STATE_OFFLINE:
8045 [ # # # # ]: 0 : info->state = SPDK_BDEV_ZONE_STATE_OFFLINE;
8046 : 0 : break;
8047 : 0 : default:
8048 [ # # ]: 0 : SPDK_ERRLOG("Invalid zone state: %#x in zone report\n", desc->zs);
8049 : 0 : return -EIO;
8050 : : }
8051 : :
8052 [ # # # # : 40 : info->zone_id = desc->zslba;
# # # # ]
8053 [ # # # # : 40 : info->write_pointer = desc->wp;
# # # # ]
8054 [ # # # # : 40 : info->capacity = desc->zcap;
# # # # ]
8055 : :
8056 : 40 : return 0;
8057 : 0 : }
8058 : :
8059 : : static void
8060 : 1 : bdev_nvme_get_zone_info_done(void *ref, const struct spdk_nvme_cpl *cpl)
8061 : : {
8062 : 1 : struct nvme_bdev_io *bio = ref;
8063 : 1 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
8064 [ # # # # : 1 : uint64_t zone_id = bdev_io->u.zone_mgmt.zone_id;
# # # # ]
8065 [ # # # # : 1 : uint32_t zones_to_copy = bdev_io->u.zone_mgmt.num_zones;
# # # # ]
8066 [ # # # # : 1 : struct spdk_bdev_zone_info *info = bdev_io->u.zone_mgmt.buf;
# # # # ]
8067 : : uint64_t max_zones_per_buf, i;
8068 : : uint32_t zone_report_bufsize;
8069 : : struct spdk_nvme_ns *ns;
8070 : : struct spdk_nvme_qpair *qpair;
8071 : : int ret;
8072 : :
8073 [ + - - + : 1 : if (spdk_nvme_cpl_is_error(cpl)) {
# # # # #
# # # # #
# # ]
8074 : 0 : goto out_complete_io_nvme_cpl;
8075 : : }
8076 : :
8077 [ - + # # : 1 : if (spdk_unlikely(!nvme_io_path_is_available(bio->io_path))) {
# # ]
8078 : 0 : ret = -ENXIO;
8079 : 0 : goto out_complete_io_ret;
8080 : : }
8081 : :
8082 [ # # # # : 1 : ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8083 [ # # # # : 1 : qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8084 : :
8085 : 1 : zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns);
8086 [ # # ]: 1 : max_zones_per_buf = (zone_report_bufsize - sizeof(*bio->zone_report_buf)) /
8087 : : sizeof(bio->zone_report_buf->descs[0]);
8088 : :
8089 [ - + # # : 1 : if (bio->zone_report_buf->nr_zones > max_zones_per_buf) {
# # # # #
# ]
8090 : 0 : ret = -EINVAL;
8091 : 0 : goto out_complete_io_ret;
8092 : : }
8093 : :
8094 [ - + # # : 1 : if (!bio->zone_report_buf->nr_zones) {
# # # # #
# ]
8095 : 0 : ret = -EINVAL;
8096 : 0 : goto out_complete_io_ret;
8097 : : }
8098 : :
8099 [ + + + - : 41 : for (i = 0; i < bio->zone_report_buf->nr_zones && bio->handled_zones < zones_to_copy; i++) {
# # # # #
# # # # #
# # ]
8100 [ # # # # : 40 : ret = fill_zone_from_report(&info[bio->handled_zones],
# # ]
8101 [ # # # # : 40 : &bio->zone_report_buf->descs[i]);
# # # # ]
8102 [ - + ]: 40 : if (ret) {
8103 : 0 : goto out_complete_io_ret;
8104 : : }
8105 [ # # ]: 40 : bio->handled_zones++;
8106 : 0 : }
8107 : :
8108 [ - + # # : 1 : if (bio->handled_zones < zones_to_copy) {
# # ]
8109 : 0 : uint64_t zone_size_lba = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
8110 [ # # # # ]: 0 : uint64_t slba = zone_id + (zone_size_lba * bio->handled_zones);
8111 : :
8112 [ # # # # : 0 : memset(bio->zone_report_buf, 0, zone_report_bufsize);
# # ]
8113 : 0 : ret = spdk_nvme_zns_report_zones(ns, qpair,
8114 [ # # # # ]: 0 : bio->zone_report_buf, zone_report_bufsize,
8115 : 0 : slba, SPDK_NVME_ZRA_LIST_ALL, true,
8116 : 0 : bdev_nvme_get_zone_info_done, bio);
8117 [ # # ]: 0 : if (!ret) {
8118 : 0 : return;
8119 : : } else {
8120 : 0 : goto out_complete_io_ret;
8121 : : }
8122 : : }
8123 : :
8124 : 1 : out_complete_io_nvme_cpl:
8125 [ # # # # ]: 1 : free(bio->zone_report_buf);
8126 [ # # # # ]: 1 : bio->zone_report_buf = NULL;
8127 : 1 : bdev_nvme_io_complete_nvme_status(bio, cpl);
8128 : 1 : return;
8129 : :
8130 : 0 : out_complete_io_ret:
8131 [ # # # # ]: 0 : free(bio->zone_report_buf);
8132 [ # # # # ]: 0 : bio->zone_report_buf = NULL;
8133 : 0 : bdev_nvme_io_complete(bio, ret);
8134 : 0 : }
8135 : :
8136 : : static void
8137 : 44 : bdev_nvme_zone_management_done(void *ref, const struct spdk_nvme_cpl *cpl)
8138 : : {
8139 : 44 : struct nvme_bdev_io *bio = ref;
8140 : :
8141 : 44 : bdev_nvme_io_complete_nvme_status(bio, cpl);
8142 : 44 : }
8143 : :
8144 : : static void
8145 : 138 : bdev_nvme_admin_passthru_complete_nvme_status(void *ctx)
8146 : : {
8147 : 138 : struct nvme_bdev_io *bio = ctx;
8148 : 138 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
8149 [ # # ]: 138 : const struct spdk_nvme_cpl *cpl = &bio->cpl;
8150 : :
8151 [ + + # # : 138 : assert(bdev_nvme_io_type_is_admin(bdev_io->type));
# # # # ]
8152 : :
8153 : 138 : __bdev_nvme_io_complete(bdev_io, 0, cpl);
8154 : 138 : }
8155 : :
8156 : : static void
8157 : 7237 : bdev_nvme_abort_complete(void *ctx)
8158 : : {
8159 : 7237 : struct nvme_bdev_io *bio = ctx;
8160 : 7237 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
8161 : :
8162 [ + - + - : 7237 : if (spdk_nvme_cpl_is_abort_success(&bio->cpl)) {
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # ]
8163 : 12 : __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS, NULL);
8164 : 3 : } else {
8165 : 7225 : __bdev_nvme_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED, NULL);
8166 : : }
8167 : 7237 : }
8168 : :
8169 : : static void
8170 : 7237 : bdev_nvme_abort_done(void *ref, const struct spdk_nvme_cpl *cpl)
8171 : : {
8172 : 7237 : struct nvme_bdev_io *bio = ref;
8173 : 7237 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
8174 : :
8175 [ # # ]: 7237 : bio->cpl = *cpl;
8176 : 7237 : spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), bdev_nvme_abort_complete, bio);
8177 : 7237 : }
8178 : :
8179 : : static void
8180 : 138 : bdev_nvme_admin_passthru_done(void *ref, const struct spdk_nvme_cpl *cpl)
8181 : : {
8182 : 138 : struct nvme_bdev_io *bio = ref;
8183 : 138 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
8184 : :
8185 [ # # ]: 138 : bio->cpl = *cpl;
8186 : 143 : spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io),
8187 : 5 : bdev_nvme_admin_passthru_complete_nvme_status, bio);
8188 : 138 : }
8189 : :
8190 : : static void
8191 : 3902166 : bdev_nvme_queued_reset_sgl(void *ref, uint32_t sgl_offset)
8192 : : {
8193 : 3902166 : struct nvme_bdev_io *bio = ref;
8194 : : struct iovec *iov;
8195 : :
8196 [ # # # # ]: 3902166 : bio->iov_offset = sgl_offset;
8197 [ + + # # : 10037278 : for (bio->iovpos = 0; bio->iovpos < bio->iovcnt; bio->iovpos++) {
# # # # #
# # # # #
# # # # ]
8198 [ # # # # : 10037278 : iov = &bio->iovs[bio->iovpos];
# # # # #
# ]
8199 [ + + # # : 10037278 : if (bio->iov_offset < iov->iov_len) {
# # # # #
# ]
8200 : 3902166 : break;
8201 : : }
8202 : :
8203 [ # # # # : 6135112 : bio->iov_offset -= iov->iov_len;
# # # # ]
8204 : 74 : }
8205 : 3902166 : }
8206 : :
8207 : : static int
8208 : 5685648 : bdev_nvme_queued_next_sge(void *ref, void **address, uint32_t *length)
8209 : : {
8210 : 5685648 : struct nvme_bdev_io *bio = ref;
8211 : : struct iovec *iov;
8212 : :
8213 [ + + # # : 5685648 : assert(bio->iovpos < bio->iovcnt);
# # # # #
# # # ]
8214 : :
8215 [ # # # # : 5685648 : iov = &bio->iovs[bio->iovpos];
# # # # #
# ]
8216 : :
8217 [ # # # # : 5685648 : *address = iov->iov_base;
# # ]
8218 [ # # # # : 5685648 : *length = iov->iov_len;
# # ]
8219 : :
8220 [ + + # # : 5685648 : if (bio->iov_offset) {
# # ]
8221 [ - + # # : 112397 : assert(bio->iov_offset <= iov->iov_len);
# # # # #
# # # ]
8222 [ # # # # : 112397 : *address += bio->iov_offset;
# # ]
8223 [ # # # # : 112397 : *length -= bio->iov_offset;
# # ]
8224 : 0 : }
8225 : :
8226 [ # # # # : 5685648 : bio->iov_offset += *length;
# # ]
8227 [ + + # # : 5685648 : if (bio->iov_offset == iov->iov_len) {
# # # # #
# ]
8228 [ # # # # ]: 5685648 : bio->iovpos++;
8229 [ # # # # ]: 5685648 : bio->iov_offset = 0;
8230 : 162 : }
8231 : :
8232 : 5685648 : return 0;
8233 : : }
8234 : :
8235 : : static void
8236 : 90 : bdev_nvme_queued_reset_fused_sgl(void *ref, uint32_t sgl_offset)
8237 : : {
8238 : 90 : struct nvme_bdev_io *bio = ref;
8239 : : struct iovec *iov;
8240 : :
8241 [ # # # # ]: 90 : bio->fused_iov_offset = sgl_offset;
8242 [ + - # # : 90 : for (bio->fused_iovpos = 0; bio->fused_iovpos < bio->fused_iovcnt; bio->fused_iovpos++) {
# # # # #
# # # # #
# # # # ]
8243 [ # # # # : 90 : iov = &bio->fused_iovs[bio->fused_iovpos];
# # # # #
# ]
8244 [ + - # # : 90 : if (bio->fused_iov_offset < iov->iov_len) {
# # # # #
# ]
8245 : 90 : break;
8246 : : }
8247 : :
8248 [ # # # # : 0 : bio->fused_iov_offset -= iov->iov_len;
# # # # ]
8249 : 0 : }
8250 : 90 : }
8251 : :
8252 : : static int
8253 : 90 : bdev_nvme_queued_next_fused_sge(void *ref, void **address, uint32_t *length)
8254 : : {
8255 : 90 : struct nvme_bdev_io *bio = ref;
8256 : : struct iovec *iov;
8257 : :
8258 [ - + # # : 90 : assert(bio->fused_iovpos < bio->fused_iovcnt);
# # # # #
# # # ]
8259 : :
8260 [ # # # # : 90 : iov = &bio->fused_iovs[bio->fused_iovpos];
# # # # #
# ]
8261 : :
8262 [ # # # # : 90 : *address = iov->iov_base;
# # ]
8263 [ # # # # : 90 : *length = iov->iov_len;
# # ]
8264 : :
8265 [ - + # # : 90 : if (bio->fused_iov_offset) {
# # ]
8266 [ # # # # : 0 : assert(bio->fused_iov_offset <= iov->iov_len);
# # # # #
# # # ]
8267 [ # # # # : 0 : *address += bio->fused_iov_offset;
# # ]
8268 [ # # # # : 0 : *length -= bio->fused_iov_offset;
# # ]
8269 : 0 : }
8270 : :
8271 [ # # # # : 90 : bio->fused_iov_offset += *length;
# # ]
8272 [ + - # # : 90 : if (bio->fused_iov_offset == iov->iov_len) {
# # # # #
# ]
8273 [ # # # # ]: 90 : bio->fused_iovpos++;
8274 [ # # # # ]: 90 : bio->fused_iov_offset = 0;
8275 : 0 : }
8276 : :
8277 : 90 : return 0;
8278 : : }
8279 : :
8280 : : static int
8281 : 0 : bdev_nvme_no_pi_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8282 : : void *md, uint64_t lba_count, uint64_t lba)
8283 : : {
8284 : : int rc;
8285 : :
8286 [ # # # # : 0 : SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 " without PI check\n",
# # ]
8287 : : lba_count, lba);
8288 : :
8289 [ # # # # ]: 0 : bio->iovs = iov;
8290 [ # # # # ]: 0 : bio->iovcnt = iovcnt;
8291 [ # # # # ]: 0 : bio->iovpos = 0;
8292 [ # # # # ]: 0 : bio->iov_offset = 0;
8293 : :
8294 [ # # # # : 0 : rc = spdk_nvme_ns_cmd_readv_with_md(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8295 [ # # # # : 0 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8296 : 0 : lba, lba_count,
8297 : 0 : bdev_nvme_no_pi_readv_done, bio, 0,
8298 : : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
8299 : 0 : md, 0, 0);
8300 : :
8301 [ # # # # ]: 0 : if (rc != 0 && rc != -ENOMEM) {
8302 : 0 : SPDK_ERRLOG("no_pi_readv failed: rc = %d\n", rc);
8303 : 0 : }
8304 : 0 : return rc;
8305 : : }
8306 : :
8307 : : static int
8308 : 12913696 : bdev_nvme_readv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8309 : : void *md, uint64_t lba_count, uint64_t lba, uint32_t flags,
8310 : : struct spdk_memory_domain *domain, void *domain_ctx,
8311 : : struct spdk_accel_sequence *seq)
8312 : : {
8313 [ + - + - : 12913696 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
+ - + - +
- + - ]
8314 [ + - + - : 12913696 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
+ - + - +
- + - ]
8315 : : int rc;
8316 : :
8317 [ + + + + : 12913696 : SPDK_DEBUGLOG(bdev_nvme, "read %" PRIu64 " blocks with offset %#" PRIx64 "\n",
+ - ]
8318 : : lba_count, lba);
8319 : :
8320 [ + - + - ]: 12913696 : bio->iovs = iov;
8321 [ + - + - ]: 12913696 : bio->iovcnt = iovcnt;
8322 [ + - + - ]: 12913696 : bio->iovpos = 0;
8323 [ + - + - ]: 12913696 : bio->iov_offset = 0;
8324 : :
8325 [ + + + + ]: 12913696 : if (domain != NULL || seq != NULL) {
8326 [ # # # # : 355842 : bio->ext_opts.size = SPDK_SIZEOF(&bio->ext_opts, accel_sequence);
# # ]
8327 [ # # # # : 355842 : bio->ext_opts.memory_domain = domain;
# # ]
8328 [ # # # # : 355842 : bio->ext_opts.memory_domain_ctx = domain_ctx;
# # ]
8329 [ # # # # : 355842 : bio->ext_opts.io_flags = flags;
# # ]
8330 [ # # # # : 355842 : bio->ext_opts.metadata = md;
# # ]
8331 [ # # # # : 355842 : bio->ext_opts.accel_sequence = seq;
# # ]
8332 : :
8333 [ + - ]: 355842 : if (iovcnt == 1) {
8334 [ # # # # : 355831 : rc = spdk_nvme_ns_cmd_read_ext(ns, qpair, iov[0].iov_base, lba, lba_count, bdev_nvme_readv_done,
# # ]
8335 [ # # ]: 1 : bio, &bio->ext_opts);
8336 : 1 : } else {
8337 : 0 : rc = spdk_nvme_ns_cmd_readv_ext(ns, qpair, lba, lba_count,
8338 : 0 : bdev_nvme_readv_done, bio,
8339 : : bdev_nvme_queued_reset_sgl,
8340 : : bdev_nvme_queued_next_sge,
8341 [ # # ]: 0 : &bio->ext_opts);
8342 : : }
8343 [ + + ]: 12557867 : } else if (iovcnt == 1) {
8344 [ - + - + : 12462284 : rc = spdk_nvme_ns_cmd_read_with_md(ns, qpair, iov[0].iov_base,
- + ]
8345 : 265672 : md, lba, lba_count, bdev_nvme_readv_done,
8346 : 265672 : bio, flags, 0, 0);
8347 : 265672 : } else {
8348 : 361256 : rc = spdk_nvme_ns_cmd_readv_with_md(ns, qpair, lba, lba_count,
8349 : 5 : bdev_nvme_readv_done, bio, flags,
8350 : : bdev_nvme_queued_reset_sgl,
8351 : 5 : bdev_nvme_queued_next_sge, md, 0, 0);
8352 : : }
8353 : :
8354 [ + + + + ]: 12913696 : if (spdk_unlikely(rc != 0 && rc != -ENOMEM)) {
8355 : 0 : SPDK_ERRLOG("readv failed: rc = %d\n", rc);
8356 : 0 : }
8357 : 12913696 : return rc;
8358 : : }
8359 : :
8360 : : static int
8361 : 13748683 : bdev_nvme_writev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8362 : : void *md, uint64_t lba_count, uint64_t lba, uint32_t flags,
8363 : : struct spdk_memory_domain *domain, void *domain_ctx,
8364 : : struct spdk_accel_sequence *seq,
8365 : : union spdk_bdev_nvme_cdw12 cdw12, union spdk_bdev_nvme_cdw13 cdw13)
8366 : : {
8367 [ # # # # : 13748683 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8368 [ # # # # : 13748683 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8369 : : int rc;
8370 : :
8371 [ + + - + : 13748683 : SPDK_DEBUGLOG(bdev_nvme, "write %" PRIu64 " blocks with offset %#" PRIx64 "\n",
# # ]
8372 : : lba_count, lba);
8373 : :
8374 [ # # # # ]: 13748683 : bio->iovs = iov;
8375 [ # # # # ]: 13748683 : bio->iovcnt = iovcnt;
8376 [ # # # # ]: 13748683 : bio->iovpos = 0;
8377 [ # # # # ]: 13748683 : bio->iov_offset = 0;
8378 : :
8379 [ + + + + ]: 13748683 : if (domain != NULL || seq != NULL) {
8380 [ # # # # : 196771 : bio->ext_opts.size = SPDK_SIZEOF(&bio->ext_opts, accel_sequence);
# # ]
8381 [ # # # # : 196771 : bio->ext_opts.memory_domain = domain;
# # ]
8382 [ # # # # : 196771 : bio->ext_opts.memory_domain_ctx = domain_ctx;
# # ]
8383 [ - + # # : 196771 : bio->ext_opts.io_flags = flags | SPDK_NVME_IO_FLAGS_DIRECTIVE(cdw12.write.dtype);
# # # # #
# ]
8384 [ # # # # : 196771 : bio->ext_opts.cdw13 = cdw13.raw;
# # ]
8385 [ # # # # : 196771 : bio->ext_opts.metadata = md;
# # ]
8386 [ # # # # : 196771 : bio->ext_opts.accel_sequence = seq;
# # ]
8387 : :
8388 [ + - ]: 196771 : if (iovcnt == 1) {
8389 [ # # # # : 196769 : rc = spdk_nvme_ns_cmd_write_ext(ns, qpair, iov[0].iov_base, lba, lba_count, bdev_nvme_writev_done,
# # ]
8390 [ # # ]: 0 : bio, &bio->ext_opts);
8391 : 0 : } else {
8392 : 0 : rc = spdk_nvme_ns_cmd_writev_ext(ns, qpair, lba, lba_count,
8393 : 0 : bdev_nvme_writev_done, bio,
8394 : : bdev_nvme_queued_reset_sgl,
8395 : : bdev_nvme_queued_next_sge,
8396 [ # # ]: 0 : &bio->ext_opts);
8397 : : }
8398 [ + + ]: 13551914 : } else if (iovcnt == 1) {
8399 [ # # # # : 13418760 : rc = spdk_nvme_ns_cmd_write_with_md(ns, qpair, iov[0].iov_base,
# # ]
8400 : 265763 : md, lba, lba_count, bdev_nvme_writev_done,
8401 : 265763 : bio, flags, 0, 0);
8402 : 265763 : } else {
8403 : 398919 : rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count,
8404 : 2 : bdev_nvme_writev_done, bio, flags,
8405 : : bdev_nvme_queued_reset_sgl,
8406 : 2 : bdev_nvme_queued_next_sge, md, 0, 0);
8407 : : }
8408 : :
8409 [ + + - + ]: 13748683 : if (spdk_unlikely(rc != 0 && rc != -ENOMEM)) {
8410 : 0 : SPDK_ERRLOG("writev failed: rc = %d\n", rc);
8411 : 0 : }
8412 : 13748683 : return rc;
8413 : : }
8414 : :
8415 : : static int
8416 : 256879 : bdev_nvme_zone_appendv(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8417 : : void *md, uint64_t lba_count, uint64_t zslba,
8418 : : uint32_t flags)
8419 : : {
8420 [ # # # # : 256879 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8421 [ # # # # : 256879 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8422 : : int rc;
8423 : :
8424 [ - + # # : 256879 : SPDK_DEBUGLOG(bdev_nvme, "zone append %" PRIu64 " blocks to zone start lba %#" PRIx64 "\n",
# # ]
8425 : : lba_count, zslba);
8426 : :
8427 [ # # # # ]: 256879 : bio->iovs = iov;
8428 [ # # # # ]: 256879 : bio->iovcnt = iovcnt;
8429 [ # # # # ]: 256879 : bio->iovpos = 0;
8430 [ # # # # ]: 256879 : bio->iov_offset = 0;
8431 : :
8432 [ + - ]: 256879 : if (iovcnt == 1) {
8433 [ # # # # : 256879 : rc = spdk_nvme_zns_zone_append_with_md(ns, qpair, iov[0].iov_base, md, zslba,
# # ]
8434 : 0 : lba_count,
8435 : 0 : bdev_nvme_zone_appendv_done, bio,
8436 : 0 : flags,
8437 : : 0, 0);
8438 : 0 : } else {
8439 : 0 : rc = spdk_nvme_zns_zone_appendv_with_md(ns, qpair, zslba, lba_count,
8440 : 0 : bdev_nvme_zone_appendv_done, bio, flags,
8441 : : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
8442 : 0 : md, 0, 0);
8443 : : }
8444 : :
8445 [ - + - - ]: 256879 : if (rc != 0 && rc != -ENOMEM) {
8446 : 0 : SPDK_ERRLOG("zone append failed: rc = %d\n", rc);
8447 : 0 : }
8448 : 256879 : return rc;
8449 : : }
8450 : :
8451 : : static int
8452 : 56 : bdev_nvme_comparev(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt,
8453 : : void *md, uint64_t lba_count, uint64_t lba,
8454 : : uint32_t flags)
8455 : : {
8456 : : int rc;
8457 : :
8458 [ + + - + : 56 : SPDK_DEBUGLOG(bdev_nvme, "compare %" PRIu64 " blocks with offset %#" PRIx64 "\n",
# # ]
8459 : : lba_count, lba);
8460 : :
8461 [ # # # # ]: 56 : bio->iovs = iov;
8462 [ # # # # ]: 56 : bio->iovcnt = iovcnt;
8463 [ # # # # ]: 56 : bio->iovpos = 0;
8464 [ # # # # ]: 56 : bio->iov_offset = 0;
8465 : :
8466 [ # # # # : 59 : rc = spdk_nvme_ns_cmd_comparev_with_md(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8467 [ # # # # : 56 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8468 : 3 : lba, lba_count,
8469 : 3 : bdev_nvme_comparev_done, bio, flags,
8470 : : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge,
8471 : 3 : md, 0, 0);
8472 : :
8473 [ - + - - ]: 56 : if (rc != 0 && rc != -ENOMEM) {
8474 : 0 : SPDK_ERRLOG("comparev failed: rc = %d\n", rc);
8475 : 0 : }
8476 : 56 : return rc;
8477 : : }
8478 : :
8479 : : static int
8480 : 53 : bdev_nvme_comparev_and_writev(struct nvme_bdev_io *bio, struct iovec *cmp_iov, int cmp_iovcnt,
8481 : : struct iovec *write_iov, int write_iovcnt,
8482 : : void *md, uint64_t lba_count, uint64_t lba, uint32_t flags)
8483 : : {
8484 [ # # # # : 53 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8485 [ # # # # : 53 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8486 : 53 : struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(bio);
8487 : : int rc;
8488 : :
8489 [ + + - + : 53 : SPDK_DEBUGLOG(bdev_nvme, "compare and write %" PRIu64 " blocks with offset %#" PRIx64 "\n",
# # ]
8490 : : lba_count, lba);
8491 : :
8492 [ # # # # ]: 53 : bio->iovs = cmp_iov;
8493 [ # # # # ]: 53 : bio->iovcnt = cmp_iovcnt;
8494 [ # # # # ]: 53 : bio->iovpos = 0;
8495 [ # # # # ]: 53 : bio->iov_offset = 0;
8496 [ # # # # ]: 53 : bio->fused_iovs = write_iov;
8497 [ # # # # ]: 53 : bio->fused_iovcnt = write_iovcnt;
8498 [ # # # # ]: 53 : bio->fused_iovpos = 0;
8499 [ # # # # ]: 53 : bio->fused_iov_offset = 0;
8500 : :
8501 [ + + # # : 53 : if (bdev_io->num_retries == 0) {
# # ]
8502 [ # # # # ]: 53 : bio->first_fused_submitted = false;
8503 [ # # # # ]: 53 : bio->first_fused_completed = false;
8504 : 2 : }
8505 : :
8506 [ + + + - : 53 : if (!bio->first_fused_submitted) {
# # # # ]
8507 [ # # # # ]: 53 : flags |= SPDK_NVME_IO_FLAGS_FUSE_FIRST;
8508 [ - + # # ]: 53 : memset(&bio->cpl, 0, sizeof(bio->cpl));
8509 : :
8510 : 55 : rc = spdk_nvme_ns_cmd_comparev_with_md(ns, qpair, lba, lba_count,
8511 : 2 : bdev_nvme_comparev_and_writev_done, bio, flags,
8512 : 2 : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge, md, 0, 0);
8513 [ + + ]: 53 : if (rc == 0) {
8514 [ # # # # ]: 53 : bio->first_fused_submitted = true;
8515 [ # # # # ]: 53 : flags &= ~SPDK_NVME_IO_FLAGS_FUSE_FIRST;
8516 : 2 : } else {
8517 [ # # ]: 0 : if (rc != -ENOMEM) {
8518 : 0 : SPDK_ERRLOG("compare failed: rc = %d\n", rc);
8519 : 0 : }
8520 : 0 : return rc;
8521 : : }
8522 : 2 : }
8523 : :
8524 [ # # # # ]: 53 : flags |= SPDK_NVME_IO_FLAGS_FUSE_SECOND;
8525 : :
8526 : 55 : rc = spdk_nvme_ns_cmd_writev_with_md(ns, qpair, lba, lba_count,
8527 : 2 : bdev_nvme_comparev_and_writev_done, bio, flags,
8528 : 2 : bdev_nvme_queued_reset_fused_sgl, bdev_nvme_queued_next_fused_sge, md, 0, 0);
8529 [ - + - - ]: 53 : if (rc != 0 && rc != -ENOMEM) {
8530 : 0 : SPDK_ERRLOG("write failed: rc = %d\n", rc);
8531 : 0 : rc = 0;
8532 : 0 : }
8533 : :
8534 : 53 : return rc;
8535 : 2 : }
8536 : :
8537 : : static int
8538 : 69081 : bdev_nvme_unmap(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks)
8539 : : {
8540 : 5521 : struct spdk_nvme_dsm_range dsm_ranges[SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES];
8541 : : struct spdk_nvme_dsm_range *range;
8542 : : uint64_t offset, remaining;
8543 : : uint64_t num_ranges_u64;
8544 : : uint16_t num_ranges;
8545 : : int rc;
8546 : :
8547 [ # # ]: 69081 : num_ranges_u64 = (num_blocks + SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS - 1) /
8548 : : SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
8549 [ - + ]: 69081 : if (num_ranges_u64 > SPDK_COUNTOF(dsm_ranges)) {
8550 : 0 : SPDK_ERRLOG("Unmap request for %" PRIu64 " blocks is too large\n", num_blocks);
8551 : 0 : return -EINVAL;
8552 : : }
8553 : 69081 : num_ranges = (uint16_t)num_ranges_u64;
8554 : :
8555 : 69081 : offset = offset_blocks;
8556 : 69081 : remaining = num_blocks;
8557 [ # # # # ]: 69081 : range = &dsm_ranges[0];
8558 : :
8559 : : /* Fill max-size ranges until the remaining blocks fit into one range */
8560 [ + + ]: 69083 : while (remaining > SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS) {
8561 [ # # # # : 2 : range->attributes.raw = 0;
# # ]
8562 [ # # # # ]: 2 : range->length = SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
8563 [ # # # # ]: 2 : range->starting_lba = offset;
8564 : :
8565 : 2 : offset += SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
8566 : 2 : remaining -= SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS;
8567 [ # # ]: 2 : range++;
8568 : : }
8569 : :
8570 : : /* Final range describes the remaining blocks */
8571 [ # # # # : 69081 : range->attributes.raw = 0;
# # ]
8572 [ # # # # ]: 69081 : range->length = remaining;
8573 [ # # # # ]: 69081 : range->starting_lba = offset;
8574 : :
8575 [ # # # # : 69084 : rc = spdk_nvme_ns_cmd_dataset_management(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8576 [ # # # # : 69081 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8577 : : SPDK_NVME_DSM_ATTR_DEALLOCATE,
8578 : 3 : dsm_ranges, num_ranges,
8579 : 3 : bdev_nvme_queued_done, bio);
8580 : :
8581 : 69081 : return rc;
8582 : 3 : }
8583 : :
8584 : : static int
8585 : 712671 : bdev_nvme_write_zeroes(struct nvme_bdev_io *bio, uint64_t offset_blocks, uint64_t num_blocks)
8586 : : {
8587 [ - + ]: 712671 : if (num_blocks > UINT16_MAX + 1) {
8588 : 0 : SPDK_ERRLOG("NVMe write zeroes is limited to 16-bit block count\n");
8589 : 0 : return -EINVAL;
8590 : : }
8591 : :
8592 [ # # # # : 787535 : return spdk_nvme_ns_cmd_write_zeroes(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8593 [ # # # # : 712671 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8594 : 74864 : offset_blocks, num_blocks,
8595 : 74864 : bdev_nvme_queued_done, bio,
8596 : : 0);
8597 : 74864 : }
8598 : :
8599 : : static int
8600 : 1 : bdev_nvme_get_zone_info(struct nvme_bdev_io *bio, uint64_t zone_id, uint32_t num_zones,
8601 : : struct spdk_bdev_zone_info *info)
8602 : : {
8603 [ # # # # : 1 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8604 [ # # # # : 1 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8605 : 1 : uint32_t zone_report_bufsize = spdk_nvme_ns_get_max_io_xfer_size(ns);
8606 : 1 : uint64_t zone_size = spdk_nvme_zns_ns_get_zone_size_sectors(ns);
8607 : 1 : uint64_t total_zones = spdk_nvme_zns_ns_get_num_zones(ns);
8608 : :
8609 [ - + # # ]: 1 : if (zone_id % zone_size != 0) {
8610 : 0 : return -EINVAL;
8611 : : }
8612 : :
8613 [ + - - + ]: 1 : if (num_zones > total_zones || !num_zones) {
8614 : 0 : return -EINVAL;
8615 : : }
8616 : :
8617 [ - + # # : 1 : assert(!bio->zone_report_buf);
# # # # ]
8618 [ # # # # ]: 1 : bio->zone_report_buf = calloc(1, zone_report_bufsize);
8619 [ - + # # : 1 : if (!bio->zone_report_buf) {
# # ]
8620 : 0 : return -ENOMEM;
8621 : : }
8622 : :
8623 [ # # # # ]: 1 : bio->handled_zones = 0;
8624 : :
8625 [ # # # # ]: 1 : return spdk_nvme_zns_report_zones(ns, qpair, bio->zone_report_buf, zone_report_bufsize,
8626 : 0 : zone_id, SPDK_NVME_ZRA_LIST_ALL, true,
8627 : 0 : bdev_nvme_get_zone_info_done, bio);
8628 : 0 : }
8629 : :
8630 : : static int
8631 : 44 : bdev_nvme_zone_management(struct nvme_bdev_io *bio, uint64_t zone_id,
8632 : : enum spdk_bdev_zone_action action)
8633 : : {
8634 [ # # # # : 44 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8635 [ # # # # : 44 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8636 : :
8637 [ - - - + : 44 : switch (action) {
- - ]
8638 : 0 : case SPDK_BDEV_ZONE_CLOSE:
8639 : 0 : return spdk_nvme_zns_close_zone(ns, qpair, zone_id, false,
8640 : 0 : bdev_nvme_zone_management_done, bio);
8641 : 0 : case SPDK_BDEV_ZONE_FINISH:
8642 : 0 : return spdk_nvme_zns_finish_zone(ns, qpair, zone_id, false,
8643 : 0 : bdev_nvme_zone_management_done, bio);
8644 : 0 : case SPDK_BDEV_ZONE_OPEN:
8645 : 0 : return spdk_nvme_zns_open_zone(ns, qpair, zone_id, false,
8646 : 0 : bdev_nvme_zone_management_done, bio);
8647 : 44 : case SPDK_BDEV_ZONE_RESET:
8648 : 44 : return spdk_nvme_zns_reset_zone(ns, qpair, zone_id, false,
8649 : 0 : bdev_nvme_zone_management_done, bio);
8650 : 0 : case SPDK_BDEV_ZONE_OFFLINE:
8651 : 0 : return spdk_nvme_zns_offline_zone(ns, qpair, zone_id, false,
8652 : 0 : bdev_nvme_zone_management_done, bio);
8653 : 0 : default:
8654 : 0 : return -EINVAL;
8655 : : }
8656 : 0 : }
8657 : :
8658 : : static void
8659 : 142 : bdev_nvme_admin_passthru(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio,
8660 : : struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes)
8661 : : {
8662 : : struct nvme_io_path *io_path;
8663 : : struct nvme_ctrlr *nvme_ctrlr;
8664 : : uint32_t max_xfer_size;
8665 : 142 : int rc = -ENXIO;
8666 : :
8667 : : /* Choose the first ctrlr which is not failed. */
8668 [ + + # # : 154 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
8669 [ # # # # : 150 : nvme_ctrlr = io_path->qpair->ctrlr;
# # # # ]
8670 : :
8671 : : /* We should skip any unavailable nvme_ctrlr rather than checking
8672 : : * if the return value of spdk_nvme_ctrlr_cmd_admin_raw() is -ENXIO.
8673 : : */
8674 [ + + ]: 150 : if (!nvme_ctrlr_is_available(nvme_ctrlr)) {
8675 : 12 : continue;
8676 : : }
8677 : :
8678 [ # # # # ]: 138 : max_xfer_size = spdk_nvme_ctrlr_get_max_xfer_size(nvme_ctrlr->ctrlr);
8679 : :
8680 [ + + ]: 138 : if (nbytes > max_xfer_size) {
8681 : 0 : SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
8682 : 0 : rc = -EINVAL;
8683 : 0 : goto err;
8684 : : }
8685 : :
8686 [ # # # # ]: 143 : rc = spdk_nvme_ctrlr_cmd_admin_raw(nvme_ctrlr->ctrlr, cmd, buf, (uint32_t)nbytes,
8687 : 5 : bdev_nvme_admin_passthru_done, bio);
8688 [ + + ]: 138 : if (rc == 0) {
8689 : 138 : return;
8690 : : }
8691 : 1 : }
8692 : :
8693 : 3 : err:
8694 : 4 : bdev_nvme_admin_complete(bio, rc);
8695 : 6 : }
8696 : :
8697 : : static int
8698 : 111 : bdev_nvme_io_passthru(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
8699 : : void *buf, size_t nbytes)
8700 : : {
8701 [ # # # # : 111 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8702 [ # # # # : 111 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8703 : 111 : uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
8704 : 111 : struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
8705 : :
8706 [ - + ]: 111 : if (nbytes > max_xfer_size) {
8707 : 0 : SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
8708 : 0 : return -EINVAL;
8709 : : }
8710 : :
8711 : : /*
8712 : : * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid,
8713 : : * so fill it out automatically.
8714 : : */
8715 [ # # # # ]: 111 : cmd->nsid = spdk_nvme_ns_get_id(ns);
8716 : :
8717 : 114 : return spdk_nvme_ctrlr_cmd_io_raw(ctrlr, qpair, cmd, buf,
8718 : 3 : (uint32_t)nbytes, bdev_nvme_queued_done, bio);
8719 : 3 : }
8720 : :
8721 : : static int
8722 : 0 : bdev_nvme_io_passthru_md(struct nvme_bdev_io *bio, struct spdk_nvme_cmd *cmd,
8723 : : void *buf, size_t nbytes, void *md_buf, size_t md_len)
8724 : : {
8725 [ # # # # : 0 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8726 [ # # # # : 0 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8727 [ # # ]: 0 : size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns);
8728 : 0 : uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
8729 : 0 : struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
8730 : :
8731 [ # # ]: 0 : if (nbytes > max_xfer_size) {
8732 : 0 : SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
8733 : 0 : return -EINVAL;
8734 : : }
8735 : :
8736 [ # # ]: 0 : if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) {
8737 : 0 : SPDK_ERRLOG("invalid meta data buffer size\n");
8738 : 0 : return -EINVAL;
8739 : : }
8740 : :
8741 : : /*
8742 : : * Each NVMe bdev is a specific namespace, and all NVMe I/O commands require a nsid,
8743 : : * so fill it out automatically.
8744 : : */
8745 [ # # # # ]: 0 : cmd->nsid = spdk_nvme_ns_get_id(ns);
8746 : :
8747 : 0 : return spdk_nvme_ctrlr_cmd_io_raw_with_md(ctrlr, qpair, cmd, buf,
8748 : 0 : (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio);
8749 : 0 : }
8750 : :
8751 : : static int
8752 : 0 : bdev_nvme_iov_passthru_md(struct nvme_bdev_io *bio,
8753 : : struct spdk_nvme_cmd *cmd, struct iovec *iov, int iovcnt,
8754 : : size_t nbytes, void *md_buf, size_t md_len)
8755 : : {
8756 [ # # # # : 0 : struct spdk_nvme_ns *ns = bio->io_path->nvme_ns->ns;
# # # # #
# # # ]
8757 [ # # # # : 0 : struct spdk_nvme_qpair *qpair = bio->io_path->qpair->qpair;
# # # # #
# # # ]
8758 [ # # ]: 0 : size_t nr_sectors = nbytes / spdk_nvme_ns_get_extended_sector_size(ns);
8759 : 0 : uint32_t max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns);
8760 : 0 : struct spdk_nvme_ctrlr *ctrlr = spdk_nvme_ns_get_ctrlr(ns);
8761 : :
8762 [ # # # # ]: 0 : bio->iovs = iov;
8763 [ # # # # ]: 0 : bio->iovcnt = iovcnt;
8764 [ # # # # ]: 0 : bio->iovpos = 0;
8765 [ # # # # ]: 0 : bio->iov_offset = 0;
8766 : :
8767 [ # # ]: 0 : if (nbytes > max_xfer_size) {
8768 : 0 : SPDK_ERRLOG("nbytes is greater than MDTS %" PRIu32 ".\n", max_xfer_size);
8769 : 0 : return -EINVAL;
8770 : : }
8771 : :
8772 [ # # ]: 0 : if (md_len != nr_sectors * spdk_nvme_ns_get_md_size(ns)) {
8773 : 0 : SPDK_ERRLOG("invalid meta data buffer size\n");
8774 : 0 : return -EINVAL;
8775 : : }
8776 : :
8777 : : /*
8778 : : * Each NVMe bdev is a specific namespace, and all NVMe I/O commands
8779 : : * require a nsid, so fill it out automatically.
8780 : : */
8781 [ # # # # ]: 0 : cmd->nsid = spdk_nvme_ns_get_id(ns);
8782 : :
8783 : 0 : return spdk_nvme_ctrlr_cmd_iov_raw_with_md(
8784 : 0 : ctrlr, qpair, cmd, (uint32_t)nbytes, md_buf, bdev_nvme_queued_done, bio,
8785 : : bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge);
8786 : 0 : }
8787 : :
8788 : : static void
8789 : 7249 : bdev_nvme_abort(struct nvme_bdev_channel *nbdev_ch, struct nvme_bdev_io *bio,
8790 : : struct nvme_bdev_io *bio_to_abort)
8791 : : {
8792 : : struct nvme_io_path *io_path;
8793 : 7249 : int rc = 0;
8794 : :
8795 : 7249 : rc = bdev_nvme_abort_retry_io(nbdev_ch, bio_to_abort);
8796 [ + + ]: 7249 : if (rc == 0) {
8797 : 4 : bdev_nvme_admin_complete(bio, 0);
8798 : 4 : return;
8799 : : }
8800 : :
8801 [ # # # # ]: 7245 : io_path = bio_to_abort->io_path;
8802 [ + + ]: 7245 : if (io_path != NULL) {
8803 [ # # # # : 7240 : rc = spdk_nvme_ctrlr_cmd_abort_ext(io_path->qpair->ctrlr->ctrlr,
# # # # #
# # # ]
8804 [ # # # # : 7237 : io_path->qpair->qpair,
# # # # ]
8805 : 3 : bio_to_abort,
8806 : 3 : bdev_nvme_abort_done, bio);
8807 : 3 : } else {
8808 [ + + # # : 12 : STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
8809 [ # # # # : 10 : rc = spdk_nvme_ctrlr_cmd_abort_ext(io_path->qpair->ctrlr->ctrlr,
# # # # #
# # # ]
8810 : : NULL,
8811 : 2 : bio_to_abort,
8812 : 2 : bdev_nvme_abort_done, bio);
8813 : :
8814 [ + + ]: 8 : if (rc != -ENOENT) {
8815 : 4 : break;
8816 : : }
8817 : 1 : }
8818 : : }
8819 : :
8820 [ + + ]: 7245 : if (rc != 0) {
8821 : : /* If no command was found or there was any error, complete the abort
8822 : : * request with failure.
8823 : : */
8824 : 8 : bdev_nvme_admin_complete(bio, rc);
8825 : 2 : }
8826 : 6 : }
8827 : :
8828 : : static int
8829 : 37 : bdev_nvme_copy(struct nvme_bdev_io *bio, uint64_t dst_offset_blocks, uint64_t src_offset_blocks,
8830 : : uint64_t num_blocks)
8831 : : {
8832 : 39 : struct spdk_nvme_scc_source_range range = {
8833 : 1 : .slba = src_offset_blocks,
8834 : 37 : .nlb = num_blocks - 1
8835 : : };
8836 : :
8837 [ # # # # : 46 : return spdk_nvme_ns_cmd_copy(bio->io_path->nvme_ns->ns,
# # # # #
# # # ]
8838 [ # # # # : 37 : bio->io_path->qpair->qpair,
# # # # #
# # # ]
8839 : 1 : &range, 1, dst_offset_blocks,
8840 : 1 : bdev_nvme_queued_done, bio);
8841 : : }
8842 : :
8843 : : static void
8844 : 197 : bdev_nvme_opts_config_json(struct spdk_json_write_ctx *w)
8845 : : {
8846 : : const char *action;
8847 : : uint32_t i;
8848 : :
8849 [ + + - + ]: 197 : if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET) {
8850 : 0 : action = "reset";
8851 [ + + - + ]: 197 : } else if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT) {
8852 : 7 : action = "abort";
8853 : 1 : } else {
8854 : 190 : action = "none";
8855 : : }
8856 : :
8857 : 197 : spdk_json_write_object_begin(w);
8858 : :
8859 : 197 : spdk_json_write_named_string(w, "method", "bdev_nvme_set_options");
8860 : :
8861 : 197 : spdk_json_write_named_object_begin(w, "params");
8862 : 197 : spdk_json_write_named_string(w, "action_on_timeout", action);
8863 [ + - ]: 197 : spdk_json_write_named_uint64(w, "timeout_us", g_opts.timeout_us);
8864 [ + - ]: 197 : spdk_json_write_named_uint64(w, "timeout_admin_us", g_opts.timeout_admin_us);
8865 [ + - ]: 197 : spdk_json_write_named_uint32(w, "keep_alive_timeout_ms", g_opts.keep_alive_timeout_ms);
8866 [ + - ]: 197 : spdk_json_write_named_uint32(w, "arbitration_burst", g_opts.arbitration_burst);
8867 [ + - ]: 197 : spdk_json_write_named_uint32(w, "low_priority_weight", g_opts.low_priority_weight);
8868 [ + - ]: 197 : spdk_json_write_named_uint32(w, "medium_priority_weight", g_opts.medium_priority_weight);
8869 [ + - ]: 197 : spdk_json_write_named_uint32(w, "high_priority_weight", g_opts.high_priority_weight);
8870 [ + - ]: 197 : spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us);
8871 [ + - ]: 197 : spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us);
8872 [ + - ]: 197 : spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests);
8873 [ + + + - ]: 197 : spdk_json_write_named_bool(w, "delay_cmd_submit", g_opts.delay_cmd_submit);
8874 [ + - ]: 197 : spdk_json_write_named_uint32(w, "transport_retry_count", g_opts.transport_retry_count);
8875 [ + - ]: 197 : spdk_json_write_named_int32(w, "bdev_retry_count", g_opts.bdev_retry_count);
8876 [ + - ]: 197 : spdk_json_write_named_uint8(w, "transport_ack_timeout", g_opts.transport_ack_timeout);
8877 [ + - ]: 197 : spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", g_opts.ctrlr_loss_timeout_sec);
8878 [ + - ]: 197 : spdk_json_write_named_uint32(w, "reconnect_delay_sec", g_opts.reconnect_delay_sec);
8879 [ + - ]: 197 : spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", g_opts.fast_io_fail_timeout_sec);
8880 [ + + ]: 197 : spdk_json_write_named_bool(w, "disable_auto_failback", g_opts.disable_auto_failback);
8881 [ + + + - ]: 197 : spdk_json_write_named_bool(w, "generate_uuids", g_opts.generate_uuids);
8882 : 197 : spdk_json_write_named_uint8(w, "transport_tos", g_opts.transport_tos);
8883 [ + + + - ]: 197 : spdk_json_write_named_bool(w, "nvme_error_stat", g_opts.nvme_error_stat);
8884 [ + - ]: 197 : spdk_json_write_named_uint32(w, "rdma_srq_size", g_opts.rdma_srq_size);
8885 [ + + ]: 197 : spdk_json_write_named_bool(w, "io_path_stat", g_opts.io_path_stat);
8886 [ + + + - ]: 197 : spdk_json_write_named_bool(w, "allow_accel_sequence", g_opts.allow_accel_sequence);
8887 [ + - ]: 197 : spdk_json_write_named_uint32(w, "rdma_max_cq_size", g_opts.rdma_max_cq_size);
8888 [ + - ]: 197 : spdk_json_write_named_uint16(w, "rdma_cm_event_timeout_ms", g_opts.rdma_cm_event_timeout_ms);
8889 : 197 : spdk_json_write_named_array_begin(w, "dhchap_digests");
8890 [ + + ]: 6501 : for (i = 0; i < 32; ++i) {
8891 [ + + + + : 6304 : if (g_opts.dhchap_digests & SPDK_BIT(i)) {
+ + ]
8892 : 591 : spdk_json_write_string(w, spdk_nvme_dhchap_get_digest_name(i));
8893 : 33 : }
8894 : 352 : }
8895 : 197 : spdk_json_write_array_end(w);
8896 : 197 : spdk_json_write_named_array_begin(w, "dhchap_dhgroups");
8897 [ + + ]: 6501 : for (i = 0; i < 32; ++i) {
8898 [ + + + + : 6304 : if (g_opts.dhchap_dhgroups & SPDK_BIT(i)) {
+ + ]
8899 : 1182 : spdk_json_write_string(w, spdk_nvme_dhchap_get_dhgroup_name(i));
8900 : 66 : }
8901 : 352 : }
8902 : :
8903 : 197 : spdk_json_write_array_end(w);
8904 [ + + + - ]: 197 : spdk_json_write_named_bool(w, "rdma_umr_per_io", g_opts.rdma_umr_per_io);
8905 : 197 : spdk_json_write_object_end(w);
8906 : :
8907 : 197 : spdk_json_write_object_end(w);
8908 : 197 : }
8909 : :
8910 : : static void
8911 : 0 : bdev_nvme_discovery_config_json(struct spdk_json_write_ctx *w, struct discovery_ctx *ctx)
8912 : : {
8913 : 0 : struct spdk_nvme_transport_id trid;
8914 : :
8915 : 0 : spdk_json_write_object_begin(w);
8916 : :
8917 : 0 : spdk_json_write_named_string(w, "method", "bdev_nvme_start_discovery");
8918 : :
8919 : 0 : spdk_json_write_named_object_begin(w, "params");
8920 [ # # # # ]: 0 : spdk_json_write_named_string(w, "name", ctx->name);
8921 [ # # # # ]: 0 : spdk_json_write_named_string(w, "hostnqn", ctx->hostnqn);
8922 : :
8923 [ # # ]: 0 : trid = ctx->trid;
8924 [ # # ]: 0 : memset(trid.subnqn, 0, sizeof(trid.subnqn));
8925 : 0 : nvme_bdev_dump_trid_json(&trid, w);
8926 : :
8927 [ # # # # : 0 : spdk_json_write_named_bool(w, "wait_for_attach", ctx->wait_for_attach);
# # ]
8928 [ # # # # : 0 : spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", ctx->bdev_opts.ctrlr_loss_timeout_sec);
# # ]
8929 [ # # # # : 0 : spdk_json_write_named_uint32(w, "reconnect_delay_sec", ctx->bdev_opts.reconnect_delay_sec);
# # ]
8930 : 0 : spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec",
8931 [ # # # # : 0 : ctx->bdev_opts.fast_io_fail_timeout_sec);
# # ]
8932 : 0 : spdk_json_write_object_end(w);
8933 : :
8934 : 0 : spdk_json_write_object_end(w);
8935 : 0 : }
8936 : :
8937 : : #ifdef SPDK_CONFIG_NVME_CUSE
8938 : : static void
8939 : 116 : nvme_ctrlr_cuse_config_json(struct spdk_json_write_ctx *w,
8940 : : struct nvme_ctrlr *nvme_ctrlr)
8941 : 116 : {
8942 : 116 : size_t cuse_name_size = 128;
8943 [ - + ]: 116 : char cuse_name[cuse_name_size];
8944 : :
8945 [ + - # # : 116 : if (spdk_nvme_cuse_get_ctrlr_name(nvme_ctrlr->ctrlr,
# # # # ]
8946 : 0 : cuse_name, &cuse_name_size) != 0) {
8947 : 116 : return;
8948 : : }
8949 : :
8950 : 0 : spdk_json_write_object_begin(w);
8951 : :
8952 : 0 : spdk_json_write_named_string(w, "method", "bdev_nvme_cuse_register");
8953 : :
8954 : 0 : spdk_json_write_named_object_begin(w, "params");
8955 [ # # # # : 0 : spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name);
# # # # ]
8956 : 0 : spdk_json_write_object_end(w);
8957 : :
8958 : 0 : spdk_json_write_object_end(w);
8959 [ # # ]: 0 : }
8960 : : #endif
8961 : :
8962 : : static void
8963 : 121 : nvme_ctrlr_config_json(struct spdk_json_write_ctx *w,
8964 : : struct nvme_ctrlr *nvme_ctrlr,
8965 : : struct nvme_path_id *path_id)
8966 : : {
8967 : : struct spdk_nvme_transport_id *trid;
8968 : : const struct spdk_nvme_ctrlr_opts *opts;
8969 : :
8970 [ - + - + : 121 : if (nvme_ctrlr->opts.from_discovery_service) {
# # # # #
# ]
8971 : : /* Do not emit an RPC for this - it will be implicitly
8972 : : * covered by a separate bdev_nvme_start_discovery or
8973 : : * bdev_nvme_start_mdns_discovery RPC.
8974 : : */
8975 : 0 : return;
8976 : : }
8977 : :
8978 [ # # ]: 121 : trid = &path_id->trid;
8979 : :
8980 : 121 : spdk_json_write_object_begin(w);
8981 : :
8982 : 121 : spdk_json_write_named_string(w, "method", "bdev_nvme_attach_controller");
8983 : :
8984 : 121 : spdk_json_write_named_object_begin(w, "params");
8985 [ # # # # : 121 : spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name);
# # # # ]
8986 : 121 : nvme_bdev_dump_trid_json(trid, w);
8987 : 126 : spdk_json_write_named_bool(w, "prchk_reftag",
8988 [ # # # # : 121 : (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) != 0);
# # # # ]
8989 : 126 : spdk_json_write_named_bool(w, "prchk_guard",
8990 [ # # # # : 121 : (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) != 0);
# # # # ]
8991 [ # # # # : 121 : spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", nvme_ctrlr->opts.ctrlr_loss_timeout_sec);
# # ]
8992 [ # # # # : 121 : spdk_json_write_named_uint32(w, "reconnect_delay_sec", nvme_ctrlr->opts.reconnect_delay_sec);
# # ]
8993 : 126 : spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec",
8994 [ # # # # : 5 : nvme_ctrlr->opts.fast_io_fail_timeout_sec);
# # ]
8995 [ + + # # : 121 : if (nvme_ctrlr->psk != NULL) {
# # ]
8996 [ # # # # ]: 9 : spdk_json_write_named_string(w, "psk", spdk_key_get_name(nvme_ctrlr->psk));
8997 : 0 : }
8998 [ + + # # : 121 : if (nvme_ctrlr->dhchap_key != NULL) {
# # ]
8999 : 0 : spdk_json_write_named_string(w, "dhchap_key",
9000 [ # # # # ]: 0 : spdk_key_get_name(nvme_ctrlr->dhchap_key));
9001 : 0 : }
9002 [ + + # # : 121 : if (nvme_ctrlr->dhchap_ctrlr_key != NULL) {
# # ]
9003 : 0 : spdk_json_write_named_string(w, "dhchap_ctrlr_key",
9004 [ # # # # ]: 0 : spdk_key_get_name(nvme_ctrlr->dhchap_ctrlr_key));
9005 : 0 : }
9006 [ # # # # ]: 121 : opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr);
9007 [ # # ]: 121 : spdk_json_write_named_string(w, "hostnqn", opts->hostnqn);
9008 [ - + # # : 121 : spdk_json_write_named_bool(w, "hdgst", opts->header_digest);
# # ]
9009 [ - + # # : 121 : spdk_json_write_named_bool(w, "ddgst", opts->data_digest);
# # ]
9010 [ + + # # : 121 : if (opts->src_addr[0] != '\0') {
# # # # #
# ]
9011 [ # # ]: 0 : spdk_json_write_named_string(w, "hostaddr", opts->src_addr);
9012 : 0 : }
9013 [ + + # # : 121 : if (opts->src_svcid[0] != '\0') {
# # # # #
# ]
9014 [ # # ]: 0 : spdk_json_write_named_string(w, "hostsvcid", opts->src_svcid);
9015 : 0 : }
9016 : :
9017 [ + + + - : 121 : if (nvme_ctrlr->opts.multipath) {
# # # # #
# ]
9018 : 121 : spdk_json_write_named_string(w, "multipath", "multipath");
9019 : 5 : }
9020 : 121 : spdk_json_write_object_end(w);
9021 : :
9022 : 121 : spdk_json_write_object_end(w);
9023 : 5 : }
9024 : :
9025 : : static void
9026 : 197 : bdev_nvme_hotplug_config_json(struct spdk_json_write_ctx *w)
9027 : : {
9028 : 197 : spdk_json_write_object_begin(w);
9029 : 197 : spdk_json_write_named_string(w, "method", "bdev_nvme_set_hotplug");
9030 : :
9031 : 197 : spdk_json_write_named_object_begin(w, "params");
9032 : 197 : spdk_json_write_named_uint64(w, "period_us", g_nvme_hotplug_poll_period_us);
9033 [ + + ]: 197 : spdk_json_write_named_bool(w, "enable", g_nvme_hotplug_enabled);
9034 : 197 : spdk_json_write_object_end(w);
9035 : :
9036 : 197 : spdk_json_write_object_end(w);
9037 : 197 : }
9038 : :
9039 : : static int
9040 : 197 : bdev_nvme_config_json(struct spdk_json_write_ctx *w)
9041 : : {
9042 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
9043 : : struct nvme_ctrlr *nvme_ctrlr;
9044 : : struct discovery_ctx *ctx;
9045 : : struct nvme_path_id *path_id;
9046 : :
9047 : 197 : bdev_nvme_opts_config_json(w);
9048 : :
9049 [ + + ]: 197 : pthread_mutex_lock(&g_bdev_nvme_mutex);
9050 : :
9051 [ + + # # : 318 : TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
# # # # ]
9052 [ + + # # : 242 : TAILQ_FOREACH(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {
# # # # #
# # # #
# ]
9053 [ # # # # ]: 121 : path_id = nvme_ctrlr->active_path_id;
9054 [ + + # # : 121 : assert(path_id == TAILQ_FIRST(&nvme_ctrlr->trids));
# # # # #
# ]
9055 : 121 : nvme_ctrlr_config_json(w, nvme_ctrlr, path_id);
9056 : :
9057 [ # # # # : 121 : path_id = TAILQ_NEXT(path_id, link);
# # ]
9058 [ - + ]: 121 : while (path_id != NULL) {
9059 : 0 : nvme_ctrlr_config_json(w, nvme_ctrlr, path_id);
9060 [ # # # # : 0 : path_id = TAILQ_NEXT(path_id, link);
# # ]
9061 : : }
9062 : :
9063 : : #ifdef SPDK_CONFIG_NVME_CUSE
9064 : 116 : nvme_ctrlr_cuse_config_json(w, nvme_ctrlr);
9065 : : #endif
9066 : 5 : }
9067 : 5 : }
9068 : :
9069 [ - + # # : 197 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
9070 [ # # # # : 0 : if (!ctx->from_mdns_discovery_service) {
# # # # ]
9071 : 0 : bdev_nvme_discovery_config_json(w, ctx);
9072 : 0 : }
9073 : 0 : }
9074 : :
9075 : 197 : bdev_nvme_mdns_discovery_config_json(w);
9076 : :
9077 : : /* Dump as last parameter to give all NVMe bdevs chance to be constructed
9078 : : * before enabling hotplug poller.
9079 : : */
9080 : 197 : bdev_nvme_hotplug_config_json(w);
9081 : :
9082 [ + + ]: 197 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
9083 : 197 : return 0;
9084 : : }
9085 : :
9086 : : struct spdk_nvme_ctrlr *
9087 : 11 : bdev_nvme_get_ctrlr(struct spdk_bdev *bdev)
9088 : : {
9089 : : struct nvme_bdev *nbdev;
9090 : : struct nvme_ns *nvme_ns;
9091 : :
9092 [ + - - + : 11 : if (!bdev || bdev->module != &nvme_if) {
# # # # ]
9093 : 0 : return NULL;
9094 : : }
9095 : :
9096 : 11 : nbdev = SPDK_CONTAINEROF(bdev, struct nvme_bdev, disk);
9097 [ # # # # : 11 : nvme_ns = TAILQ_FIRST(&nbdev->nvme_ns_list);
# # ]
9098 [ + + # # ]: 11 : assert(nvme_ns != NULL);
9099 : :
9100 [ # # # # : 11 : return nvme_ns->ctrlr->ctrlr;
# # # # ]
9101 : 2 : }
9102 : :
9103 : : static bool
9104 : 528 : nvme_io_path_is_current(struct nvme_io_path *io_path)
9105 : : {
9106 : : const struct nvme_bdev_channel *nbdev_ch;
9107 : : bool current;
9108 : :
9109 [ + + ]: 528 : if (!nvme_io_path_is_available(io_path)) {
9110 : 136 : return false;
9111 : : }
9112 : :
9113 [ # # # # ]: 392 : nbdev_ch = io_path->nbdev_ch;
9114 [ + + ]: 392 : if (nbdev_ch == NULL) {
9115 : 4 : current = false;
9116 [ + + # # : 389 : } else if (nbdev_ch->mp_policy == BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE) {
# # ]
9117 : 180 : struct nvme_io_path *optimized_io_path = NULL;
9118 : :
9119 [ + + # # : 384 : STAILQ_FOREACH(optimized_io_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
9120 [ + + # # : 308 : if (optimized_io_path->nvme_ns->ana_state == SPDK_NVME_ANA_OPTIMIZED_STATE) {
# # # # #
# ]
9121 : 104 : break;
9122 : : }
9123 : 3 : }
9124 : :
9125 : : /* A non-optimized path is only current if there are no optimized paths. */
9126 [ + + + + : 180 : current = (io_path->nvme_ns->ana_state == SPDK_NVME_ANA_OPTIMIZED_STATE) ||
# # # # #
# ]
9127 : 2 : (optimized_io_path == NULL);
9128 : 3 : } else {
9129 [ + + # # : 208 : if (nbdev_ch->current_io_path) {
# # ]
9130 [ # # # # ]: 124 : current = (io_path == nbdev_ch->current_io_path);
9131 : 1 : } else {
9132 : : struct nvme_io_path *first_path;
9133 : :
9134 : : /* We arrived here as there are no optimized paths for active-passive
9135 : : * mode. Check if this io_path is the first one available on the list.
9136 : : */
9137 : 84 : current = false;
9138 [ + + # # : 84 : STAILQ_FOREACH(first_path, &nbdev_ch->io_path_list, stailq) {
# # # # #
# # # #
# ]
9139 [ + - ]: 84 : if (nvme_io_path_is_available(first_path)) {
9140 : 84 : current = (io_path == first_path);
9141 : 84 : break;
9142 : : }
9143 : 0 : }
9144 : : }
9145 : : }
9146 : :
9147 [ # # ]: 392 : return current;
9148 : 12 : }
9149 : :
9150 : : static struct nvme_ctrlr *
9151 : 40 : bdev_nvme_next_ctrlr_unsafe(struct nvme_bdev_ctrlr *nbdev_ctrlr, struct nvme_ctrlr *prev)
9152 : : {
9153 : : struct nvme_ctrlr *next;
9154 : :
9155 : : /* Must be called under g_bdev_nvme_mutex */
9156 [ + + # # : 40 : next = prev != NULL ? TAILQ_NEXT(prev, tailq) : TAILQ_FIRST(&nbdev_ctrlr->ctrlrs);
# # # # #
# # # #
# ]
9157 [ + + ]: 40 : while (next != NULL) {
9158 : : /* ref can be 0 when the ctrlr was released, but hasn't been detached yet */
9159 [ - + # # ]: 28 : pthread_mutex_lock(&next->mutex);
9160 [ + - # # : 28 : if (next->ref > 0) {
# # ]
9161 [ # # # # ]: 28 : next->ref++;
9162 [ - + # # ]: 28 : pthread_mutex_unlock(&next->mutex);
9163 : 28 : return next;
9164 : : }
9165 : :
9166 [ # # # # ]: 0 : pthread_mutex_unlock(&next->mutex);
9167 [ # # # # : 0 : next = TAILQ_NEXT(next, tailq);
# # ]
9168 : : }
9169 : :
9170 : 12 : return NULL;
9171 : 0 : }
9172 : :
9173 : : struct bdev_nvme_set_keys_ctx {
9174 : : struct nvme_ctrlr *nctrlr;
9175 : : struct spdk_key *dhchap_key;
9176 : : struct spdk_key *dhchap_ctrlr_key;
9177 : : struct spdk_thread *thread;
9178 : : bdev_nvme_set_keys_cb cb_fn;
9179 : : void *cb_ctx;
9180 : : int status;
9181 : : };
9182 : :
9183 : : static void
9184 : 28 : bdev_nvme_free_set_keys_ctx(struct bdev_nvme_set_keys_ctx *ctx)
9185 : : {
9186 [ - + ]: 28 : if (ctx == NULL) {
9187 : 0 : return;
9188 : : }
9189 : :
9190 [ # # # # ]: 28 : spdk_keyring_put_key(ctx->dhchap_key);
9191 [ # # # # ]: 28 : spdk_keyring_put_key(ctx->dhchap_ctrlr_key);
9192 : 28 : free(ctx);
9193 : 0 : }
9194 : :
9195 : : static void
9196 : 28 : _bdev_nvme_set_keys_done(void *_ctx)
9197 : : {
9198 : 28 : struct bdev_nvme_set_keys_ctx *ctx = _ctx;
9199 : :
9200 [ # # # # : 28 : ctx->cb_fn(ctx->cb_ctx, ctx->status);
# # # # #
# # # # #
# # ]
9201 : :
9202 [ + + # # : 28 : if (ctx->nctrlr != NULL) {
# # ]
9203 [ # # # # ]: 16 : nvme_ctrlr_put_ref(ctx->nctrlr);
9204 : 0 : }
9205 : 28 : bdev_nvme_free_set_keys_ctx(ctx);
9206 : 28 : }
9207 : :
9208 : : static void
9209 : 28 : bdev_nvme_set_keys_done(struct bdev_nvme_set_keys_ctx *ctx, int status)
9210 : : {
9211 [ # # # # ]: 28 : ctx->status = status;
9212 [ # # # # ]: 28 : spdk_thread_exec_msg(ctx->thread, _bdev_nvme_set_keys_done, ctx);
9213 : 28 : }
9214 : :
9215 : : static void bdev_nvme_authenticate_ctrlr(struct bdev_nvme_set_keys_ctx *ctx);
9216 : :
9217 : : static void
9218 : 12 : bdev_nvme_authenticate_ctrlr_continue(struct bdev_nvme_set_keys_ctx *ctx)
9219 : : {
9220 : : struct nvme_ctrlr *next;
9221 : :
9222 [ - + ]: 12 : pthread_mutex_lock(&g_bdev_nvme_mutex);
9223 [ # # # # ]: 12 : next = bdev_nvme_next_ctrlr_unsafe(NULL, ctx->nctrlr);
9224 [ - + ]: 12 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
9225 : :
9226 [ # # # # ]: 12 : nvme_ctrlr_put_ref(ctx->nctrlr);
9227 [ # # # # ]: 12 : ctx->nctrlr = next;
9228 : :
9229 [ + - ]: 12 : if (next == NULL) {
9230 : 12 : bdev_nvme_set_keys_done(ctx, 0);
9231 : 0 : } else {
9232 : 0 : bdev_nvme_authenticate_ctrlr(ctx);
9233 : : }
9234 : 12 : }
9235 : :
9236 : : static void
9237 : 8 : bdev_nvme_authenticate_qpairs_done(struct spdk_io_channel_iter *i, int status)
9238 : : {
9239 : 8 : struct bdev_nvme_set_keys_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
9240 : :
9241 [ - + ]: 8 : if (status != 0) {
9242 : 0 : bdev_nvme_set_keys_done(ctx, status);
9243 : 0 : return;
9244 : : }
9245 : 8 : bdev_nvme_authenticate_ctrlr_continue(ctx);
9246 : 0 : }
9247 : :
9248 : : static void
9249 : 0 : bdev_nvme_authenticate_qpair_done(void *ctx, int status)
9250 : : {
9251 : 0 : spdk_for_each_channel_continue(ctx, status);
9252 : 0 : }
9253 : :
9254 : : static void
9255 : 0 : bdev_nvme_authenticate_qpair(struct spdk_io_channel_iter *i)
9256 : : {
9257 : 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
9258 : 0 : struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch);
9259 [ # # # # ]: 0 : struct nvme_qpair *qpair = ctrlr_ch->qpair;
9260 : : int rc;
9261 : :
9262 [ # # ]: 0 : if (!nvme_qpair_is_connected(qpair)) {
9263 : 0 : spdk_for_each_channel_continue(i, 0);
9264 : 0 : return;
9265 : : }
9266 : :
9267 [ # # # # ]: 0 : rc = spdk_nvme_qpair_authenticate(qpair->qpair, bdev_nvme_authenticate_qpair_done, i);
9268 [ # # ]: 0 : if (rc != 0) {
9269 : 0 : spdk_for_each_channel_continue(i, rc);
9270 : 0 : }
9271 : 0 : }
9272 : :
9273 : : static void
9274 : 20 : bdev_nvme_authenticate_ctrlr_done(void *_ctx, int status)
9275 : : {
9276 : 20 : struct bdev_nvme_set_keys_ctx *ctx = _ctx;
9277 : :
9278 [ + + ]: 20 : if (status != 0) {
9279 : 12 : bdev_nvme_set_keys_done(ctx, status);
9280 : 12 : return;
9281 : : }
9282 : :
9283 [ # # # # ]: 8 : spdk_for_each_channel(ctx->nctrlr, bdev_nvme_authenticate_qpair, ctx,
9284 : : bdev_nvme_authenticate_qpairs_done);
9285 : 0 : }
9286 : :
9287 : : static void
9288 : 28 : bdev_nvme_authenticate_ctrlr(struct bdev_nvme_set_keys_ctx *ctx)
9289 : : {
9290 : 28 : struct spdk_nvme_ctrlr_key_opts opts = {};
9291 [ # # # # ]: 28 : struct nvme_ctrlr *nctrlr = ctx->nctrlr;
9292 : : int rc;
9293 : :
9294 : 28 : opts.size = SPDK_SIZEOF(&opts, dhchap_ctrlr_key);
9295 [ # # # # : 28 : opts.dhchap_key = ctx->dhchap_key;
# # ]
9296 [ # # # # : 28 : opts.dhchap_ctrlr_key = ctx->dhchap_ctrlr_key;
# # ]
9297 [ # # # # ]: 28 : rc = spdk_nvme_ctrlr_set_keys(nctrlr->ctrlr, &opts);
9298 [ - + ]: 28 : if (rc != 0) {
9299 : 0 : bdev_nvme_set_keys_done(ctx, rc);
9300 : 0 : return;
9301 : : }
9302 : :
9303 [ + + # # : 28 : if (ctx->dhchap_key != NULL) {
# # ]
9304 [ # # # # ]: 24 : rc = spdk_nvme_ctrlr_authenticate(nctrlr->ctrlr,
9305 : 0 : bdev_nvme_authenticate_ctrlr_done, ctx);
9306 [ + + ]: 24 : if (rc != 0) {
9307 : 4 : bdev_nvme_set_keys_done(ctx, rc);
9308 : 0 : }
9309 : 0 : } else {
9310 : 4 : bdev_nvme_authenticate_ctrlr_continue(ctx);
9311 : : }
9312 : 0 : }
9313 : :
9314 : : int
9315 : 28 : bdev_nvme_set_keys(const char *name, const char *dhchap_key, const char *dhchap_ctrlr_key,
9316 : : bdev_nvme_set_keys_cb cb_fn, void *cb_ctx)
9317 : : {
9318 : : struct bdev_nvme_set_keys_ctx *ctx;
9319 : : struct nvme_bdev_ctrlr *nbdev_ctrlr;
9320 : : struct nvme_ctrlr *nctrlr;
9321 : :
9322 : 28 : ctx = calloc(1, sizeof(*ctx));
9323 [ - + ]: 28 : if (ctx == NULL) {
9324 : 0 : return -ENOMEM;
9325 : : }
9326 : :
9327 [ + + ]: 28 : if (dhchap_key != NULL) {
9328 [ # # # # ]: 24 : ctx->dhchap_key = spdk_keyring_get_key(dhchap_key);
9329 [ - + # # : 24 : if (ctx->dhchap_key == NULL) {
# # ]
9330 : 0 : SPDK_ERRLOG("Could not find key %s for bdev %s\n", dhchap_key, name);
9331 : 0 : bdev_nvme_free_set_keys_ctx(ctx);
9332 : 0 : return -ENOKEY;
9333 : : }
9334 : 0 : }
9335 [ + + ]: 28 : if (dhchap_ctrlr_key != NULL) {
9336 [ # # # # ]: 24 : ctx->dhchap_ctrlr_key = spdk_keyring_get_key(dhchap_ctrlr_key);
9337 [ - + # # : 24 : if (ctx->dhchap_ctrlr_key == NULL) {
# # ]
9338 : 0 : SPDK_ERRLOG("Could not find key %s for bdev %s\n", dhchap_ctrlr_key, name);
9339 : 0 : bdev_nvme_free_set_keys_ctx(ctx);
9340 : 0 : return -ENOKEY;
9341 : : }
9342 : 0 : }
9343 : :
9344 [ - + ]: 28 : pthread_mutex_lock(&g_bdev_nvme_mutex);
9345 : 28 : nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
9346 [ - + ]: 28 : if (nbdev_ctrlr == NULL) {
9347 : 0 : SPDK_ERRLOG("Could not find bdev_ctrlr %s\n", name);
9348 [ # # ]: 0 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
9349 : 0 : bdev_nvme_free_set_keys_ctx(ctx);
9350 : 0 : return -ENODEV;
9351 : : }
9352 : 28 : nctrlr = bdev_nvme_next_ctrlr_unsafe(nbdev_ctrlr, NULL);
9353 [ - + ]: 28 : if (nctrlr == NULL) {
9354 : 0 : SPDK_ERRLOG("Could not find any nvme_ctrlrs on bdev_ctrlr %s\n", name);
9355 [ # # ]: 0 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
9356 : 0 : bdev_nvme_free_set_keys_ctx(ctx);
9357 : 0 : return -ENODEV;
9358 : : }
9359 [ - + ]: 28 : pthread_mutex_unlock(&g_bdev_nvme_mutex);
9360 : :
9361 [ # # # # ]: 28 : ctx->nctrlr = nctrlr;
9362 [ # # # # ]: 28 : ctx->cb_fn = cb_fn;
9363 [ # # # # ]: 28 : ctx->cb_ctx = cb_ctx;
9364 [ # # # # ]: 28 : ctx->thread = spdk_get_thread();
9365 : :
9366 : 28 : bdev_nvme_authenticate_ctrlr(ctx);
9367 : :
9368 : 28 : return 0;
9369 : 0 : }
9370 : :
9371 : : void
9372 : 480 : nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path)
9373 : : {
9374 [ # # # # ]: 480 : struct nvme_ns *nvme_ns = io_path->nvme_ns;
9375 [ # # # # : 480 : struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;
# # # # ]
9376 : : const struct spdk_nvme_ctrlr_data *cdata;
9377 : : const struct spdk_nvme_transport_id *trid;
9378 : : const char *adrfam_str;
9379 : :
9380 : 480 : spdk_json_write_object_begin(w);
9381 : :
9382 [ # # # # : 480 : spdk_json_write_named_string(w, "bdev_name", nvme_ns->bdev->disk.name);
# # # # #
# ]
9383 : :
9384 [ # # # # ]: 480 : cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
9385 [ # # # # ]: 480 : trid = spdk_nvme_ctrlr_get_transport_id(nvme_ctrlr->ctrlr);
9386 : :
9387 [ # # # # ]: 480 : spdk_json_write_named_uint32(w, "cntlid", cdata->cntlid);
9388 : 480 : spdk_json_write_named_bool(w, "current", nvme_io_path_is_current(io_path));
9389 [ # # # # ]: 480 : spdk_json_write_named_bool(w, "connected", nvme_qpair_is_connected(io_path->qpair));
9390 : 480 : spdk_json_write_named_bool(w, "accessible", nvme_ns_is_accessible(nvme_ns));
9391 : :
9392 : 480 : spdk_json_write_named_object_begin(w, "transport");
9393 [ # # ]: 480 : spdk_json_write_named_string(w, "trtype", trid->trstring);
9394 [ # # ]: 480 : spdk_json_write_named_string(w, "traddr", trid->traddr);
9395 [ + - # # : 480 : if (trid->trsvcid[0] != '\0') {
# # # # #
# ]
9396 [ # # ]: 480 : spdk_json_write_named_string(w, "trsvcid", trid->trsvcid);
9397 : 0 : }
9398 [ # # # # ]: 480 : adrfam_str = spdk_nvme_transport_id_adrfam_str(trid->adrfam);
9399 [ + - ]: 480 : if (adrfam_str) {
9400 : 480 : spdk_json_write_named_string(w, "adrfam", adrfam_str);
9401 : 0 : }
9402 : 480 : spdk_json_write_object_end(w);
9403 : :
9404 : 480 : spdk_json_write_object_end(w);
9405 : 480 : }
9406 : :
9407 : : void
9408 : 77 : bdev_nvme_get_discovery_info(struct spdk_json_write_ctx *w)
9409 : : {
9410 : : struct discovery_ctx *ctx;
9411 : : struct discovery_entry_ctx *entry_ctx;
9412 : :
9413 : 77 : spdk_json_write_array_begin(w);
9414 [ + + # # : 150 : TAILQ_FOREACH(ctx, &g_discovery_ctxs, tailq) {
# # # # ]
9415 : 73 : spdk_json_write_object_begin(w);
9416 [ # # # # ]: 73 : spdk_json_write_named_string(w, "name", ctx->name);
9417 : :
9418 : 73 : spdk_json_write_named_object_begin(w, "trid");
9419 [ # # ]: 73 : nvme_bdev_dump_trid_json(&ctx->trid, w);
9420 : 73 : spdk_json_write_object_end(w);
9421 : :
9422 : 73 : spdk_json_write_named_array_begin(w, "referrals");
9423 [ + + # # : 171 : TAILQ_FOREACH(entry_ctx, &ctx->discovery_entry_ctxs, tailq) {
# # # # #
# # # #
# ]
9424 : 98 : spdk_json_write_object_begin(w);
9425 : 98 : spdk_json_write_named_object_begin(w, "trid");
9426 [ # # ]: 98 : nvme_bdev_dump_trid_json(&entry_ctx->trid, w);
9427 : 98 : spdk_json_write_object_end(w);
9428 : 98 : spdk_json_write_object_end(w);
9429 : 0 : }
9430 : 73 : spdk_json_write_array_end(w);
9431 : :
9432 : 73 : spdk_json_write_object_end(w);
9433 : 0 : }
9434 : 77 : spdk_json_write_array_end(w);
9435 : 77 : }
9436 : :
9437 : 2149 : SPDK_LOG_REGISTER_COMPONENT(bdev_nvme)
9438 : :
9439 : : static void
9440 : 1973 : bdev_nvme_trace(void)
9441 : : {
9442 : 1973 : struct spdk_trace_tpoint_opts opts[] = {
9443 : : {
9444 : : "BDEV_NVME_IO_START", TRACE_BDEV_NVME_IO_START,
9445 : : OWNER_TYPE_NONE, OBJECT_BDEV_NVME_IO, 1,
9446 : : {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }}
9447 : : },
9448 : : {
9449 : : "BDEV_NVME_IO_DONE", TRACE_BDEV_NVME_IO_DONE,
9450 : : OWNER_TYPE_NONE, OBJECT_BDEV_NVME_IO, 0,
9451 : : {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }}
9452 : : }
9453 : : };
9454 : :
9455 : :
9456 : 1973 : spdk_trace_register_object(OBJECT_BDEV_NVME_IO, 'N');
9457 : 1973 : spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts));
9458 : 1973 : spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_SUBMIT, OBJECT_BDEV_NVME_IO, 0);
9459 : 1973 : spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_SUBMIT, OBJECT_BDEV_NVME_IO, 0);
9460 : 1973 : spdk_trace_tpoint_register_relation(TRACE_NVME_PCIE_COMPLETE, OBJECT_BDEV_NVME_IO, 0);
9461 : 1973 : spdk_trace_tpoint_register_relation(TRACE_NVME_TCP_COMPLETE, OBJECT_BDEV_NVME_IO, 0);
9462 : 1973 : }
9463 : 2149 : SPDK_TRACE_REGISTER_FN(bdev_nvme_trace, "bdev_nvme", TRACE_GROUP_BDEV_NVME)
|