Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2016 Intel Corporation. All rights reserved.
3 : * Copyright (c) 2018-2019, 2021 Mellanox Technologies LTD. All rights reserved.
4 : * Copyright (c) 2021, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : */
6 :
7 : #include "spdk/stdinc.h"
8 :
9 : #include "spdk/bdev.h"
10 : #include "spdk/bit_array.h"
11 : #include "spdk/thread.h"
12 : #include "spdk/nvmf.h"
13 : #include "spdk/endian.h"
14 : #include "spdk/string.h"
15 : #include "spdk/log.h"
16 : #include "spdk_internal/usdt.h"
17 :
18 : #include "nvmf_internal.h"
19 : #include "transport.h"
20 :
21 1 : SPDK_LOG_REGISTER_COMPONENT(nvmf)
22 :
23 : #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024
24 :
25 : static TAILQ_HEAD(, spdk_nvmf_tgt) g_nvmf_tgts = TAILQ_HEAD_INITIALIZER(g_nvmf_tgts);
26 :
27 : typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status);
28 :
29 : /* supplied to a single call to nvmf_qpair_disconnect */
30 : struct nvmf_qpair_disconnect_ctx {
31 : struct spdk_nvmf_qpair *qpair;
32 : struct spdk_nvmf_ctrlr *ctrlr;
33 : uint16_t qid;
34 : };
35 :
36 : /*
37 : * There are several times when we need to iterate through the list of all qpairs and selectively delete them.
38 : * In order to do this sequentially without overlap, we must provide a context to recover the next qpair from
39 : * to enable calling nvmf_qpair_disconnect on the next desired qpair.
40 : */
41 : struct nvmf_qpair_disconnect_many_ctx {
42 : struct spdk_nvmf_subsystem *subsystem;
43 : struct spdk_nvmf_poll_group *group;
44 : spdk_nvmf_poll_group_mod_done cpl_fn;
45 : void *cpl_ctx;
46 : };
47 :
48 : static struct spdk_nvmf_referral *
49 0 : nvmf_tgt_find_referral(struct spdk_nvmf_tgt *tgt,
50 : const struct spdk_nvme_transport_id *trid)
51 : {
52 : struct spdk_nvmf_referral *referral;
53 :
54 0 : TAILQ_FOREACH(referral, &tgt->referrals, link) {
55 0 : if (spdk_nvme_transport_id_compare(&referral->trid, trid) == 0) {
56 0 : return referral;
57 : }
58 : }
59 :
60 0 : return NULL;
61 : }
62 :
63 : int
64 0 : spdk_nvmf_tgt_add_referral(struct spdk_nvmf_tgt *tgt,
65 : const struct spdk_nvmf_referral_opts *uopts)
66 : {
67 : struct spdk_nvmf_referral *referral;
68 0 : struct spdk_nvmf_referral_opts opts = {};
69 0 : struct spdk_nvme_transport_id *trid = &opts.trid;
70 :
71 0 : memcpy(&opts, uopts, spdk_min(uopts->size, sizeof(opts)));
72 0 : if (trid->subnqn[0] == '\0') {
73 0 : snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
74 : }
75 :
76 0 : if (!nvmf_nqn_is_valid(trid->subnqn)) {
77 0 : SPDK_ERRLOG("Invalid subsystem NQN\n");
78 0 : return -EINVAL;
79 : }
80 :
81 : /* If the entry already exists, just ignore it. */
82 0 : if (nvmf_tgt_find_referral(tgt, trid)) {
83 0 : return 0;
84 : }
85 :
86 0 : referral = calloc(1, sizeof(*referral));
87 0 : if (!referral) {
88 0 : SPDK_ERRLOG("Failed to allocate memory for a referral\n");
89 0 : return -ENOMEM;
90 : }
91 :
92 0 : referral->entry.subtype = nvmf_nqn_is_discovery(trid->subnqn) ?
93 : SPDK_NVMF_SUBTYPE_DISCOVERY :
94 : SPDK_NVMF_SUBTYPE_NVME;
95 0 : referral->entry.treq.secure_channel = opts.secure_channel ?
96 0 : SPDK_NVMF_TREQ_SECURE_CHANNEL_REQUIRED :
97 : SPDK_NVMF_TREQ_SECURE_CHANNEL_NOT_REQUIRED;
98 0 : referral->entry.cntlid = 0xffff;
99 0 : referral->entry.trtype = trid->trtype;
100 0 : referral->entry.adrfam = trid->adrfam;
101 0 : memcpy(&referral->trid, trid, sizeof(struct spdk_nvme_transport_id));
102 0 : spdk_strcpy_pad(referral->entry.subnqn, trid->subnqn, sizeof(trid->subnqn), '\0');
103 0 : spdk_strcpy_pad(referral->entry.trsvcid, trid->trsvcid, sizeof(referral->entry.trsvcid), ' ');
104 0 : spdk_strcpy_pad(referral->entry.traddr, trid->traddr, sizeof(referral->entry.traddr), ' ');
105 :
106 0 : TAILQ_INSERT_HEAD(&tgt->referrals, referral, link);
107 0 : spdk_nvmf_send_discovery_log_notice(tgt, NULL);
108 :
109 0 : return 0;
110 : }
111 :
112 : int
113 0 : spdk_nvmf_tgt_remove_referral(struct spdk_nvmf_tgt *tgt,
114 : const struct spdk_nvmf_referral_opts *uopts)
115 : {
116 : struct spdk_nvmf_referral *referral;
117 0 : struct spdk_nvmf_referral_opts opts = {};
118 0 : struct spdk_nvme_transport_id *trid = &opts.trid;
119 :
120 0 : memcpy(&opts, uopts, spdk_min(uopts->size, sizeof(opts)));
121 0 : if (trid->subnqn[0] == '\0') {
122 0 : snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
123 : }
124 :
125 0 : referral = nvmf_tgt_find_referral(tgt, &opts.trid);
126 0 : if (referral == NULL) {
127 0 : return -ENOENT;
128 : }
129 :
130 0 : TAILQ_REMOVE(&tgt->referrals, referral, link);
131 0 : spdk_nvmf_send_discovery_log_notice(tgt, NULL);
132 :
133 0 : free(referral);
134 :
135 0 : return 0;
136 : }
137 :
138 : void
139 0 : nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair,
140 : enum spdk_nvmf_qpair_state state)
141 : {
142 0 : assert(qpair != NULL);
143 0 : assert(qpair->group->thread == spdk_get_thread());
144 :
145 0 : qpair->state = state;
146 0 : }
147 :
148 : /*
149 : * Reset and clean up the poll group (I/O channel code will actually free the
150 : * group).
151 : */
152 : static void
153 1 : nvmf_tgt_cleanup_poll_group(struct spdk_nvmf_poll_group *group)
154 : {
155 : struct spdk_nvmf_transport_poll_group *tgroup, *tmp;
156 : struct spdk_nvmf_subsystem_poll_group *sgroup;
157 : uint32_t sid, nsid;
158 :
159 2 : TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) {
160 1 : TAILQ_REMOVE(&group->tgroups, tgroup, link);
161 1 : nvmf_transport_poll_group_destroy(tgroup);
162 : }
163 :
164 2 : for (sid = 0; sid < group->num_sgroups; sid++) {
165 1 : sgroup = &group->sgroups[sid];
166 :
167 1 : assert(sgroup != NULL);
168 :
169 2 : for (nsid = 0; nsid < sgroup->num_ns; nsid++) {
170 1 : if (sgroup->ns_info[nsid].channel) {
171 1 : spdk_put_io_channel(sgroup->ns_info[nsid].channel);
172 1 : sgroup->ns_info[nsid].channel = NULL;
173 : }
174 : }
175 :
176 1 : free(sgroup->ns_info);
177 : }
178 :
179 1 : free(group->sgroups);
180 :
181 1 : if (group->destroy_cb_fn) {
182 0 : group->destroy_cb_fn(group->destroy_cb_arg, 0);
183 : }
184 1 : }
185 :
186 : /*
187 : * Callback to unregister a poll group from the target, and clean up its state.
188 : */
189 : static void
190 1 : nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf)
191 : {
192 1 : struct spdk_nvmf_tgt *tgt = io_device;
193 1 : struct spdk_nvmf_poll_group *group = ctx_buf;
194 :
195 : SPDK_DTRACE_PROBE1_TICKS(nvmf_destroy_poll_group, spdk_thread_get_id(group->thread));
196 :
197 1 : pthread_mutex_lock(&tgt->mutex);
198 1 : TAILQ_REMOVE(&tgt->poll_groups, group, link);
199 1 : tgt->num_poll_groups--;
200 1 : pthread_mutex_unlock(&tgt->mutex);
201 :
202 1 : assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING));
203 1 : nvmf_tgt_cleanup_poll_group(group);
204 1 : }
205 :
206 : static int
207 1 : nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group,
208 : struct spdk_nvmf_transport *transport)
209 : {
210 1 : struct spdk_nvmf_transport_poll_group *tgroup = nvmf_get_transport_poll_group(group, transport);
211 :
212 1 : if (tgroup != NULL) {
213 : /* Transport already in the poll group */
214 0 : return 0;
215 : }
216 :
217 1 : tgroup = nvmf_transport_poll_group_create(transport, group);
218 1 : if (!tgroup) {
219 0 : SPDK_ERRLOG("Unable to create poll group for transport\n");
220 0 : return -1;
221 : }
222 : SPDK_DTRACE_PROBE2_TICKS(nvmf_transport_poll_group_create, transport,
223 : spdk_thread_get_id(group->thread));
224 :
225 1 : tgroup->group = group;
226 1 : TAILQ_INSERT_TAIL(&group->tgroups, tgroup, link);
227 :
228 1 : return 0;
229 : }
230 :
231 : static int
232 1 : nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf)
233 : {
234 1 : struct spdk_nvmf_tgt *tgt = io_device;
235 1 : struct spdk_nvmf_poll_group *group = ctx_buf;
236 : struct spdk_nvmf_transport *transport;
237 : struct spdk_nvmf_subsystem *subsystem;
238 1 : struct spdk_thread *thread = spdk_get_thread();
239 : uint32_t i;
240 : int rc;
241 :
242 1 : group->tgt = tgt;
243 1 : TAILQ_INIT(&group->tgroups);
244 1 : TAILQ_INIT(&group->qpairs);
245 1 : group->thread = thread;
246 1 : pthread_mutex_init(&group->mutex, NULL);
247 :
248 : SPDK_DTRACE_PROBE1_TICKS(nvmf_create_poll_group, spdk_thread_get_id(thread));
249 :
250 2 : TAILQ_FOREACH(transport, &tgt->transports, link) {
251 1 : rc = nvmf_poll_group_add_transport(group, transport);
252 1 : if (rc != 0) {
253 0 : nvmf_tgt_cleanup_poll_group(group);
254 0 : return rc;
255 : }
256 : }
257 :
258 1 : group->num_sgroups = tgt->max_subsystems;
259 1 : group->sgroups = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group));
260 1 : if (!group->sgroups) {
261 0 : nvmf_tgt_cleanup_poll_group(group);
262 0 : return -ENOMEM;
263 : }
264 :
265 2 : for (i = 0; i < tgt->max_subsystems; i++) {
266 1 : TAILQ_INIT(&group->sgroups[i].queued);
267 : }
268 :
269 1 : for (subsystem = spdk_nvmf_subsystem_get_first(tgt);
270 2 : subsystem != NULL;
271 1 : subsystem = spdk_nvmf_subsystem_get_next(subsystem)) {
272 1 : if (nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) {
273 0 : nvmf_tgt_cleanup_poll_group(group);
274 0 : return -1;
275 : }
276 : }
277 :
278 1 : pthread_mutex_lock(&tgt->mutex);
279 1 : tgt->num_poll_groups++;
280 1 : TAILQ_INSERT_TAIL(&tgt->poll_groups, group, link);
281 1 : pthread_mutex_unlock(&tgt->mutex);
282 :
283 1 : return 0;
284 : }
285 :
286 : static void
287 0 : _nvmf_tgt_disconnect_qpairs(void *ctx)
288 : {
289 : struct spdk_nvmf_qpair *qpair, *qpair_tmp;
290 0 : struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx;
291 0 : struct spdk_nvmf_poll_group *group = qpair_ctx->group;
292 : struct spdk_io_channel *ch;
293 : int rc;
294 :
295 0 : TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) {
296 0 : rc = spdk_nvmf_qpair_disconnect(qpair);
297 0 : if (rc && rc != -EINPROGRESS) {
298 0 : break;
299 : }
300 : }
301 :
302 0 : if (TAILQ_EMPTY(&group->qpairs)) {
303 : /* When the refcount from the channels reaches 0, nvmf_tgt_destroy_poll_group will be called. */
304 0 : ch = spdk_io_channel_from_ctx(group);
305 0 : spdk_put_io_channel(ch);
306 0 : free(qpair_ctx);
307 0 : return;
308 : }
309 :
310 : /* Some qpairs are in process of being disconnected. Send a message and try to remove them again */
311 0 : spdk_thread_send_msg(spdk_get_thread(), _nvmf_tgt_disconnect_qpairs, ctx);
312 : }
313 :
314 : static void
315 0 : nvmf_tgt_destroy_poll_group_qpairs(struct spdk_nvmf_poll_group *group)
316 : {
317 : struct nvmf_qpair_disconnect_many_ctx *ctx;
318 :
319 : SPDK_DTRACE_PROBE1_TICKS(nvmf_destroy_poll_group_qpairs, spdk_thread_get_id(group->thread));
320 :
321 0 : ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx));
322 0 : if (!ctx) {
323 0 : SPDK_ERRLOG("Failed to allocate memory for destroy poll group ctx\n");
324 0 : return;
325 : }
326 :
327 0 : ctx->group = group;
328 0 : _nvmf_tgt_disconnect_qpairs(ctx);
329 : }
330 :
331 : struct spdk_nvmf_tgt *
332 0 : spdk_nvmf_tgt_create(struct spdk_nvmf_target_opts *_opts)
333 : {
334 : struct spdk_nvmf_tgt *tgt, *tmp_tgt;
335 0 : struct spdk_nvmf_target_opts opts = {
336 : .max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS,
337 : .discovery_filter = SPDK_NVMF_TGT_DISCOVERY_MATCH_ANY,
338 : };
339 :
340 0 : memcpy(&opts, _opts, _opts->size);
341 0 : if (strnlen(opts.name, NVMF_TGT_NAME_MAX_LENGTH) == NVMF_TGT_NAME_MAX_LENGTH) {
342 0 : SPDK_ERRLOG("Provided target name exceeds the max length of %u.\n", NVMF_TGT_NAME_MAX_LENGTH);
343 0 : return NULL;
344 : }
345 :
346 0 : TAILQ_FOREACH(tmp_tgt, &g_nvmf_tgts, link) {
347 0 : if (!strncmp(opts.name, tmp_tgt->name, NVMF_TGT_NAME_MAX_LENGTH)) {
348 0 : SPDK_ERRLOG("Provided target name must be unique.\n");
349 0 : return NULL;
350 : }
351 : }
352 :
353 0 : tgt = calloc(1, sizeof(*tgt));
354 0 : if (!tgt) {
355 0 : return NULL;
356 : }
357 :
358 0 : snprintf(tgt->name, NVMF_TGT_NAME_MAX_LENGTH, "%s", opts.name);
359 :
360 0 : if (!opts.max_subsystems) {
361 0 : tgt->max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS;
362 : } else {
363 0 : tgt->max_subsystems = opts.max_subsystems;
364 : }
365 :
366 0 : tgt->crdt[0] = opts.crdt[0];
367 0 : tgt->crdt[1] = opts.crdt[1];
368 0 : tgt->crdt[2] = opts.crdt[2];
369 0 : tgt->discovery_filter = opts.discovery_filter;
370 0 : tgt->discovery_genctr = 0;
371 0 : tgt->dhchap_digests = opts.dhchap_digests;
372 0 : tgt->dhchap_dhgroups = opts.dhchap_dhgroups;
373 0 : TAILQ_INIT(&tgt->transports);
374 0 : TAILQ_INIT(&tgt->poll_groups);
375 0 : TAILQ_INIT(&tgt->referrals);
376 0 : tgt->num_poll_groups = 0;
377 :
378 0 : tgt->subsystem_ids = spdk_bit_array_create(tgt->max_subsystems);
379 0 : if (tgt->subsystem_ids == NULL) {
380 0 : free(tgt);
381 0 : return NULL;
382 : }
383 :
384 0 : RB_INIT(&tgt->subsystems);
385 :
386 0 : pthread_mutex_init(&tgt->mutex, NULL);
387 :
388 0 : spdk_io_device_register(tgt,
389 : nvmf_tgt_create_poll_group,
390 : nvmf_tgt_destroy_poll_group,
391 : sizeof(struct spdk_nvmf_poll_group),
392 0 : tgt->name);
393 :
394 0 : tgt->state = NVMF_TGT_RUNNING;
395 :
396 0 : TAILQ_INSERT_HEAD(&g_nvmf_tgts, tgt, link);
397 :
398 0 : return tgt;
399 : }
400 :
401 : static void
402 0 : _nvmf_tgt_destroy_next_transport(void *ctx)
403 : {
404 0 : struct spdk_nvmf_tgt *tgt = ctx;
405 : struct spdk_nvmf_transport *transport;
406 :
407 0 : if (!TAILQ_EMPTY(&tgt->transports)) {
408 0 : transport = TAILQ_FIRST(&tgt->transports);
409 0 : TAILQ_REMOVE(&tgt->transports, transport, link);
410 0 : spdk_nvmf_transport_destroy(transport, _nvmf_tgt_destroy_next_transport, tgt);
411 : } else {
412 0 : spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn = tgt->destroy_cb_fn;
413 0 : void *destroy_cb_arg = tgt->destroy_cb_arg;
414 :
415 0 : pthread_mutex_destroy(&tgt->mutex);
416 0 : free(tgt);
417 :
418 0 : if (destroy_cb_fn) {
419 0 : destroy_cb_fn(destroy_cb_arg, 0);
420 : }
421 : }
422 0 : }
423 :
424 : static void
425 0 : nvmf_tgt_destroy_cb(void *io_device)
426 : {
427 0 : struct spdk_nvmf_tgt *tgt = io_device;
428 : struct spdk_nvmf_subsystem *subsystem, *subsystem_next;
429 : int rc;
430 : struct spdk_nvmf_referral *referral;
431 :
432 0 : while ((referral = TAILQ_FIRST(&tgt->referrals))) {
433 0 : TAILQ_REMOVE(&tgt->referrals, referral, link);
434 0 : free(referral);
435 : }
436 :
437 0 : nvmf_tgt_stop_mdns_prr(tgt);
438 :
439 : /* We will be freeing subsystems in this loop, so we always need to get the next one
440 : * ahead of time, since we can't call get_next() on a subsystem that's been freed.
441 : */
442 0 : for (subsystem = spdk_nvmf_subsystem_get_first(tgt),
443 0 : subsystem_next = spdk_nvmf_subsystem_get_next(subsystem);
444 0 : subsystem != NULL;
445 0 : subsystem = subsystem_next,
446 0 : subsystem_next = spdk_nvmf_subsystem_get_next(subsystem_next)) {
447 0 : nvmf_subsystem_remove_all_listeners(subsystem, true);
448 :
449 0 : rc = spdk_nvmf_subsystem_destroy(subsystem, nvmf_tgt_destroy_cb, tgt);
450 0 : if (rc) {
451 0 : if (rc == -EINPROGRESS) {
452 : /* If rc is -EINPROGRESS, nvmf_tgt_destroy_cb will be called again when subsystem #i
453 : * is destroyed, nvmf_tgt_destroy_cb will continue to destroy other subsystems if any */
454 0 : return;
455 : } else {
456 0 : SPDK_ERRLOG("Failed to destroy subsystem %s, rc %d\n", subsystem->subnqn, rc);
457 : }
458 : }
459 : }
460 0 : spdk_bit_array_free(&tgt->subsystem_ids);
461 0 : _nvmf_tgt_destroy_next_transport(tgt);
462 : }
463 :
464 : void
465 0 : spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt,
466 : spdk_nvmf_tgt_destroy_done_fn cb_fn,
467 : void *cb_arg)
468 : {
469 0 : assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING));
470 :
471 0 : tgt->destroy_cb_fn = cb_fn;
472 0 : tgt->destroy_cb_arg = cb_arg;
473 :
474 0 : TAILQ_REMOVE(&g_nvmf_tgts, tgt, link);
475 :
476 0 : spdk_io_device_unregister(tgt, nvmf_tgt_destroy_cb);
477 0 : }
478 :
479 : const char *
480 0 : spdk_nvmf_tgt_get_name(struct spdk_nvmf_tgt *tgt)
481 : {
482 0 : return tgt->name;
483 : }
484 :
485 : struct spdk_nvmf_tgt *
486 0 : spdk_nvmf_get_tgt(const char *name)
487 : {
488 : struct spdk_nvmf_tgt *tgt;
489 0 : uint32_t num_targets = 0;
490 :
491 0 : TAILQ_FOREACH(tgt, &g_nvmf_tgts, link) {
492 0 : if (name) {
493 0 : if (!strncmp(tgt->name, name, NVMF_TGT_NAME_MAX_LENGTH)) {
494 0 : return tgt;
495 : }
496 : }
497 0 : num_targets++;
498 : }
499 :
500 : /*
501 : * special case. If there is only one target and
502 : * no name was specified, return the only available
503 : * target. If there is more than one target, name must
504 : * be specified.
505 : */
506 0 : if (!name && num_targets == 1) {
507 0 : return TAILQ_FIRST(&g_nvmf_tgts);
508 : }
509 :
510 0 : return NULL;
511 : }
512 :
513 : struct spdk_nvmf_tgt *
514 0 : spdk_nvmf_get_first_tgt(void)
515 : {
516 0 : return TAILQ_FIRST(&g_nvmf_tgts);
517 : }
518 :
519 : struct spdk_nvmf_tgt *
520 0 : spdk_nvmf_get_next_tgt(struct spdk_nvmf_tgt *prev)
521 : {
522 0 : return TAILQ_NEXT(prev, link);
523 : }
524 :
525 : static void
526 0 : nvmf_write_nvme_subsystem_config(struct spdk_json_write_ctx *w,
527 : struct spdk_nvmf_subsystem *subsystem)
528 : {
529 : struct spdk_nvmf_host *host;
530 : struct spdk_nvmf_ns *ns;
531 0 : struct spdk_nvmf_ns_opts ns_opts;
532 : uint32_t max_namespaces;
533 : struct spdk_nvmf_transport *transport;
534 :
535 0 : assert(spdk_nvmf_subsystem_get_type(subsystem) == SPDK_NVMF_SUBTYPE_NVME);
536 :
537 : /* { */
538 0 : spdk_json_write_object_begin(w);
539 0 : spdk_json_write_named_string(w, "method", "nvmf_create_subsystem");
540 :
541 : /* "params" : { */
542 0 : spdk_json_write_named_object_begin(w, "params");
543 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
544 0 : spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_subsystem_get_allow_any_host(subsystem));
545 0 : spdk_json_write_named_string(w, "serial_number", spdk_nvmf_subsystem_get_sn(subsystem));
546 0 : spdk_json_write_named_string(w, "model_number", spdk_nvmf_subsystem_get_mn(subsystem));
547 :
548 0 : max_namespaces = spdk_nvmf_subsystem_get_max_namespaces(subsystem);
549 0 : if (max_namespaces != 0) {
550 0 : spdk_json_write_named_uint32(w, "max_namespaces", max_namespaces);
551 : }
552 :
553 0 : spdk_json_write_named_uint32(w, "min_cntlid", spdk_nvmf_subsystem_get_min_cntlid(subsystem));
554 0 : spdk_json_write_named_uint32(w, "max_cntlid", spdk_nvmf_subsystem_get_max_cntlid(subsystem));
555 0 : spdk_json_write_named_bool(w, "ana_reporting", spdk_nvmf_subsystem_get_ana_reporting(subsystem));
556 :
557 : /* } "params" */
558 0 : spdk_json_write_object_end(w);
559 :
560 : /* } */
561 0 : spdk_json_write_object_end(w);
562 :
563 0 : for (host = spdk_nvmf_subsystem_get_first_host(subsystem); host != NULL;
564 0 : host = spdk_nvmf_subsystem_get_next_host(subsystem, host)) {
565 :
566 0 : spdk_json_write_object_begin(w);
567 0 : spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_host");
568 :
569 : /* "params" : { */
570 0 : spdk_json_write_named_object_begin(w, "params");
571 :
572 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
573 0 : spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host));
574 0 : if (host->dhchap_key != NULL) {
575 0 : spdk_json_write_named_string(w, "dhchap_key",
576 : spdk_key_get_name(host->dhchap_key));
577 : }
578 0 : if (host->dhchap_ctrlr_key != NULL) {
579 0 : spdk_json_write_named_string(w, "dhchap_ctrlr_key",
580 : spdk_key_get_name(host->dhchap_ctrlr_key));
581 : }
582 0 : TAILQ_FOREACH(transport, &subsystem->tgt->transports, link) {
583 0 : if (transport->ops->subsystem_dump_host != NULL) {
584 0 : transport->ops->subsystem_dump_host(transport, subsystem, host->nqn, w);
585 : }
586 : }
587 :
588 : /* } "params" */
589 0 : spdk_json_write_object_end(w);
590 :
591 : /* } */
592 0 : spdk_json_write_object_end(w);
593 : }
594 :
595 0 : for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL;
596 0 : ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) {
597 0 : spdk_nvmf_ns_get_opts(ns, &ns_opts, sizeof(ns_opts));
598 :
599 0 : spdk_json_write_object_begin(w);
600 0 : spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_ns");
601 :
602 : /* "params" : { */
603 0 : spdk_json_write_named_object_begin(w, "params");
604 :
605 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
606 :
607 : /* "namespace" : { */
608 0 : spdk_json_write_named_object_begin(w, "namespace");
609 :
610 0 : spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns));
611 0 : spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns)));
612 :
613 0 : if (ns->ptpl_file != NULL) {
614 0 : spdk_json_write_named_string(w, "ptpl_file", ns->ptpl_file);
615 : }
616 :
617 0 : if (!spdk_mem_all_zero(ns_opts.nguid, sizeof(ns_opts.nguid))) {
618 : SPDK_STATIC_ASSERT(sizeof(ns_opts.nguid) == sizeof(uint64_t) * 2, "size mismatch");
619 0 : spdk_json_write_named_string_fmt(w, "nguid", "%016"PRIX64"%016"PRIX64, from_be64(&ns_opts.nguid[0]),
620 : from_be64(&ns_opts.nguid[8]));
621 : }
622 :
623 0 : if (!spdk_mem_all_zero(ns_opts.eui64, sizeof(ns_opts.eui64))) {
624 : SPDK_STATIC_ASSERT(sizeof(ns_opts.eui64) == sizeof(uint64_t), "size mismatch");
625 0 : spdk_json_write_named_string_fmt(w, "eui64", "%016"PRIX64, from_be64(&ns_opts.eui64));
626 : }
627 :
628 0 : if (!spdk_uuid_is_null(&ns_opts.uuid)) {
629 0 : spdk_json_write_named_uuid(w, "uuid", &ns_opts.uuid);
630 : }
631 :
632 0 : if (spdk_nvmf_subsystem_get_ana_reporting(subsystem)) {
633 0 : spdk_json_write_named_uint32(w, "anagrpid", ns_opts.anagrpid);
634 : }
635 :
636 0 : spdk_json_write_named_bool(w, "no_auto_visible", !ns->always_visible);
637 :
638 : /* "namespace" */
639 0 : spdk_json_write_object_end(w);
640 :
641 : /* } "params" */
642 0 : spdk_json_write_object_end(w);
643 :
644 : /* } */
645 0 : spdk_json_write_object_end(w);
646 :
647 0 : TAILQ_FOREACH(host, &ns->hosts, link) {
648 0 : spdk_json_write_object_begin(w);
649 0 : spdk_json_write_named_string(w, "method", "nvmf_ns_add_host");
650 0 : spdk_json_write_named_object_begin(w, "params");
651 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
652 0 : spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns));
653 0 : spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host));
654 0 : spdk_json_write_object_end(w);
655 0 : spdk_json_write_object_end(w);
656 : }
657 : }
658 0 : }
659 :
660 : static void
661 0 : nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w,
662 : struct spdk_nvmf_subsystem *subsystem)
663 : {
664 : struct spdk_nvmf_subsystem_listener *listener;
665 : struct spdk_nvmf_transport *transport;
666 : const struct spdk_nvme_transport_id *trid;
667 :
668 0 : if (spdk_nvmf_subsystem_get_type(subsystem) == SPDK_NVMF_SUBTYPE_NVME) {
669 0 : nvmf_write_nvme_subsystem_config(w, subsystem);
670 : }
671 :
672 0 : for (listener = spdk_nvmf_subsystem_get_first_listener(subsystem); listener != NULL;
673 0 : listener = spdk_nvmf_subsystem_get_next_listener(subsystem, listener)) {
674 0 : transport = listener->transport;
675 0 : trid = spdk_nvmf_subsystem_listener_get_trid(listener);
676 :
677 0 : spdk_json_write_object_begin(w);
678 0 : spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_listener");
679 :
680 : /* "params" : { */
681 0 : spdk_json_write_named_object_begin(w, "params");
682 :
683 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
684 :
685 0 : spdk_json_write_named_object_begin(w, "listen_address");
686 0 : nvmf_transport_listen_dump_trid(trid, w);
687 0 : spdk_json_write_object_end(w);
688 0 : if (transport->ops->listen_dump_opts) {
689 0 : transport->ops->listen_dump_opts(transport, trid, w);
690 : }
691 :
692 0 : spdk_json_write_named_bool(w, "secure_channel", listener->opts.secure_channel);
693 :
694 0 : if (listener->opts.sock_impl) {
695 0 : spdk_json_write_named_string(w, "sock_impl", listener->opts.sock_impl);
696 : }
697 :
698 : /* } "params" */
699 0 : spdk_json_write_object_end(w);
700 :
701 : /* } */
702 0 : spdk_json_write_object_end(w);
703 : }
704 0 : }
705 :
706 : void
707 0 : spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt)
708 : {
709 : struct spdk_nvmf_subsystem *subsystem;
710 : struct spdk_nvmf_transport *transport;
711 : struct spdk_nvmf_referral *referral;
712 :
713 0 : spdk_json_write_object_begin(w);
714 0 : spdk_json_write_named_string(w, "method", "nvmf_set_max_subsystems");
715 :
716 0 : spdk_json_write_named_object_begin(w, "params");
717 0 : spdk_json_write_named_uint32(w, "max_subsystems", tgt->max_subsystems);
718 0 : spdk_json_write_object_end(w);
719 :
720 0 : spdk_json_write_object_end(w);
721 :
722 0 : spdk_json_write_object_begin(w);
723 0 : spdk_json_write_named_string(w, "method", "nvmf_set_crdt");
724 0 : spdk_json_write_named_object_begin(w, "params");
725 0 : spdk_json_write_named_uint32(w, "crdt1", tgt->crdt[0]);
726 0 : spdk_json_write_named_uint32(w, "crdt2", tgt->crdt[1]);
727 0 : spdk_json_write_named_uint32(w, "crdt3", tgt->crdt[2]);
728 0 : spdk_json_write_object_end(w);
729 0 : spdk_json_write_object_end(w);
730 :
731 : /* write transports */
732 0 : TAILQ_FOREACH(transport, &tgt->transports, link) {
733 0 : spdk_json_write_object_begin(w);
734 0 : spdk_json_write_named_string(w, "method", "nvmf_create_transport");
735 0 : nvmf_transport_dump_opts(transport, w, true);
736 0 : spdk_json_write_object_end(w);
737 : }
738 :
739 0 : TAILQ_FOREACH(referral, &tgt->referrals, link) {
740 0 : spdk_json_write_object_begin(w);
741 0 : spdk_json_write_named_string(w, "method", "nvmf_discovery_add_referral");
742 :
743 0 : spdk_json_write_named_object_begin(w, "params");
744 0 : spdk_json_write_named_object_begin(w, "address");
745 0 : nvmf_transport_listen_dump_trid(&referral->trid, w);
746 0 : spdk_json_write_object_end(w);
747 0 : spdk_json_write_named_bool(w, "secure_channel",
748 0 : referral->entry.treq.secure_channel ==
749 : SPDK_NVMF_TREQ_SECURE_CHANNEL_REQUIRED);
750 0 : spdk_json_write_named_string(w, "subnqn", referral->trid.subnqn);
751 0 : spdk_json_write_object_end(w);
752 :
753 0 : spdk_json_write_object_end(w);
754 : }
755 :
756 0 : subsystem = spdk_nvmf_subsystem_get_first(tgt);
757 0 : while (subsystem) {
758 0 : nvmf_write_subsystem_config_json(w, subsystem);
759 0 : subsystem = spdk_nvmf_subsystem_get_next(subsystem);
760 : }
761 0 : }
762 :
763 : static void
764 0 : nvmf_listen_opts_copy(struct spdk_nvmf_listen_opts *opts,
765 : const struct spdk_nvmf_listen_opts *opts_src, size_t opts_size)
766 : {
767 0 : assert(opts);
768 0 : assert(opts_src);
769 :
770 0 : opts->opts_size = opts_size;
771 :
772 : #define SET_FIELD(field) \
773 : if (offsetof(struct spdk_nvmf_listen_opts, field) + sizeof(opts->field) <= opts_size) { \
774 : opts->field = opts_src->field; \
775 : } \
776 :
777 0 : SET_FIELD(transport_specific);
778 0 : SET_FIELD(secure_channel);
779 0 : SET_FIELD(ana_state);
780 0 : SET_FIELD(sock_impl);
781 : #undef SET_FIELD
782 :
783 : /* Do not remove this statement, you should always update this statement when you adding a new field,
784 : * and do not forget to add the SET_FIELD statement for your added field. */
785 : SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_listen_opts) == 32, "Incorrect size");
786 0 : }
787 :
788 : void
789 0 : spdk_nvmf_listen_opts_init(struct spdk_nvmf_listen_opts *opts, size_t opts_size)
790 : {
791 0 : struct spdk_nvmf_listen_opts opts_local = {};
792 :
793 : /* local version of opts should have defaults set here */
794 0 : opts_local.ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
795 0 : nvmf_listen_opts_copy(opts, &opts_local, opts_size);
796 0 : }
797 :
798 : int
799 0 : spdk_nvmf_tgt_listen_ext(struct spdk_nvmf_tgt *tgt, const struct spdk_nvme_transport_id *trid,
800 : struct spdk_nvmf_listen_opts *opts)
801 : {
802 : struct spdk_nvmf_transport *transport;
803 : int rc;
804 0 : struct spdk_nvmf_listen_opts opts_local = {};
805 :
806 0 : if (!opts) {
807 0 : SPDK_ERRLOG("opts should not be NULL\n");
808 0 : return -EINVAL;
809 : }
810 :
811 0 : if (!opts->opts_size) {
812 0 : SPDK_ERRLOG("The opts_size in opts structure should not be zero\n");
813 0 : return -EINVAL;
814 : }
815 :
816 0 : transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring);
817 0 : if (!transport) {
818 0 : SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n",
819 : trid->trstring);
820 0 : return -EINVAL;
821 : }
822 :
823 0 : nvmf_listen_opts_copy(&opts_local, opts, opts->opts_size);
824 0 : rc = spdk_nvmf_transport_listen(transport, trid, &opts_local);
825 0 : if (rc < 0) {
826 0 : SPDK_ERRLOG("Unable to listen on address '%s'\n", trid->traddr);
827 : }
828 :
829 0 : return rc;
830 : }
831 :
832 : int
833 0 : spdk_nvmf_tgt_stop_listen(struct spdk_nvmf_tgt *tgt,
834 : struct spdk_nvme_transport_id *trid)
835 : {
836 : struct spdk_nvmf_transport *transport;
837 : int rc;
838 :
839 0 : transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring);
840 0 : if (!transport) {
841 0 : SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n",
842 : trid->trstring);
843 0 : return -EINVAL;
844 : }
845 :
846 0 : rc = spdk_nvmf_transport_stop_listen(transport, trid);
847 0 : if (rc < 0) {
848 0 : SPDK_ERRLOG("Failed to stop listening on address '%s'\n", trid->traddr);
849 0 : return rc;
850 : }
851 0 : return 0;
852 : }
853 :
854 : struct spdk_nvmf_tgt_add_transport_ctx {
855 : struct spdk_nvmf_tgt *tgt;
856 : struct spdk_nvmf_transport *transport;
857 : spdk_nvmf_tgt_add_transport_done_fn cb_fn;
858 : void *cb_arg;
859 : int status;
860 : };
861 :
862 : static void
863 0 : _nvmf_tgt_remove_transport_done(struct spdk_io_channel_iter *i, int status)
864 : {
865 0 : struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
866 :
867 0 : ctx->cb_fn(ctx->cb_arg, ctx->status);
868 0 : free(ctx);
869 0 : }
870 :
871 : static void
872 0 : _nvmf_tgt_remove_transport(struct spdk_io_channel_iter *i)
873 : {
874 0 : struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
875 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
876 0 : struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch);
877 : struct spdk_nvmf_transport_poll_group *tgroup, *tmp;
878 :
879 0 : TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) {
880 0 : if (tgroup->transport == ctx->transport) {
881 0 : TAILQ_REMOVE(&group->tgroups, tgroup, link);
882 0 : nvmf_transport_poll_group_destroy(tgroup);
883 : }
884 : }
885 :
886 0 : spdk_for_each_channel_continue(i, 0);
887 0 : }
888 :
889 : static void
890 0 : _nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status)
891 : {
892 0 : struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
893 :
894 0 : if (status) {
895 0 : ctx->status = status;
896 0 : spdk_for_each_channel(ctx->tgt,
897 : _nvmf_tgt_remove_transport,
898 : ctx,
899 : _nvmf_tgt_remove_transport_done);
900 0 : return;
901 : }
902 :
903 0 : ctx->transport->tgt = ctx->tgt;
904 0 : TAILQ_INSERT_TAIL(&ctx->tgt->transports, ctx->transport, link);
905 0 : ctx->cb_fn(ctx->cb_arg, status);
906 0 : free(ctx);
907 : }
908 :
909 : static void
910 0 : _nvmf_tgt_add_transport(struct spdk_io_channel_iter *i)
911 : {
912 0 : struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
913 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
914 0 : struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch);
915 : int rc;
916 :
917 0 : rc = nvmf_poll_group_add_transport(group, ctx->transport);
918 0 : spdk_for_each_channel_continue(i, rc);
919 0 : }
920 :
921 : void
922 0 : spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt,
923 : struct spdk_nvmf_transport *transport,
924 : spdk_nvmf_tgt_add_transport_done_fn cb_fn,
925 : void *cb_arg)
926 : {
927 : struct spdk_nvmf_tgt_add_transport_ctx *ctx;
928 :
929 : SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_add_transport, transport, tgt->name);
930 :
931 0 : if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->name)) {
932 0 : cb_fn(cb_arg, -EEXIST);
933 0 : return; /* transport already created */
934 : }
935 :
936 0 : ctx = calloc(1, sizeof(*ctx));
937 0 : if (!ctx) {
938 0 : cb_fn(cb_arg, -ENOMEM);
939 0 : return;
940 : }
941 :
942 0 : ctx->tgt = tgt;
943 0 : ctx->transport = transport;
944 0 : ctx->cb_fn = cb_fn;
945 0 : ctx->cb_arg = cb_arg;
946 :
947 0 : spdk_for_each_channel(tgt,
948 : _nvmf_tgt_add_transport,
949 : ctx,
950 : _nvmf_tgt_add_transport_done);
951 : }
952 :
953 : struct nvmf_tgt_pause_ctx {
954 : struct spdk_nvmf_tgt *tgt;
955 : spdk_nvmf_tgt_pause_polling_cb_fn cb_fn;
956 : void *cb_arg;
957 : };
958 :
959 : static void
960 0 : _nvmf_tgt_pause_polling_done(struct spdk_io_channel_iter *i, int status)
961 : {
962 0 : struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
963 :
964 0 : ctx->tgt->state = NVMF_TGT_PAUSED;
965 :
966 0 : ctx->cb_fn(ctx->cb_arg, status);
967 0 : free(ctx);
968 0 : }
969 :
970 : static void
971 0 : _nvmf_tgt_pause_polling(struct spdk_io_channel_iter *i)
972 : {
973 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
974 0 : struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch);
975 : struct spdk_nvmf_transport_poll_group *tgroup;
976 :
977 0 : TAILQ_FOREACH(tgroup, &group->tgroups, link) {
978 0 : nvmf_transport_poll_group_pause(tgroup);
979 : }
980 :
981 0 : spdk_for_each_channel_continue(i, 0);
982 0 : }
983 :
984 : int
985 0 : spdk_nvmf_tgt_pause_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_pause_polling_cb_fn cb_fn,
986 : void *cb_arg)
987 : {
988 : struct nvmf_tgt_pause_ctx *ctx;
989 :
990 : SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_pause_polling, tgt, tgt->name);
991 :
992 0 : switch (tgt->state) {
993 0 : case NVMF_TGT_PAUSING:
994 : case NVMF_TGT_RESUMING:
995 0 : return -EBUSY;
996 0 : case NVMF_TGT_RUNNING:
997 0 : break;
998 0 : default:
999 0 : return -EINVAL;
1000 : }
1001 :
1002 0 : ctx = calloc(1, sizeof(*ctx));
1003 0 : if (!ctx) {
1004 0 : return -ENOMEM;
1005 : }
1006 :
1007 :
1008 0 : tgt->state = NVMF_TGT_PAUSING;
1009 :
1010 0 : ctx->tgt = tgt;
1011 0 : ctx->cb_fn = cb_fn;
1012 0 : ctx->cb_arg = cb_arg;
1013 :
1014 0 : spdk_for_each_channel(tgt,
1015 : _nvmf_tgt_pause_polling,
1016 : ctx,
1017 : _nvmf_tgt_pause_polling_done);
1018 0 : return 0;
1019 : }
1020 :
1021 : static void
1022 0 : _nvmf_tgt_resume_polling_done(struct spdk_io_channel_iter *i, int status)
1023 : {
1024 0 : struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
1025 :
1026 0 : ctx->tgt->state = NVMF_TGT_RUNNING;
1027 :
1028 0 : ctx->cb_fn(ctx->cb_arg, status);
1029 0 : free(ctx);
1030 0 : }
1031 :
1032 : static void
1033 0 : _nvmf_tgt_resume_polling(struct spdk_io_channel_iter *i)
1034 : {
1035 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
1036 0 : struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch);
1037 : struct spdk_nvmf_transport_poll_group *tgroup;
1038 :
1039 0 : TAILQ_FOREACH(tgroup, &group->tgroups, link) {
1040 0 : nvmf_transport_poll_group_resume(tgroup);
1041 : }
1042 :
1043 0 : spdk_for_each_channel_continue(i, 0);
1044 0 : }
1045 :
1046 : int
1047 0 : spdk_nvmf_tgt_resume_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_resume_polling_cb_fn cb_fn,
1048 : void *cb_arg)
1049 : {
1050 : struct nvmf_tgt_pause_ctx *ctx;
1051 :
1052 : SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_resume_polling, tgt, tgt->name);
1053 :
1054 0 : switch (tgt->state) {
1055 0 : case NVMF_TGT_PAUSING:
1056 : case NVMF_TGT_RESUMING:
1057 0 : return -EBUSY;
1058 0 : case NVMF_TGT_PAUSED:
1059 0 : break;
1060 0 : default:
1061 0 : return -EINVAL;
1062 : }
1063 :
1064 0 : ctx = calloc(1, sizeof(*ctx));
1065 0 : if (!ctx) {
1066 0 : return -ENOMEM;
1067 : }
1068 :
1069 0 : tgt->state = NVMF_TGT_RESUMING;
1070 :
1071 0 : ctx->tgt = tgt;
1072 0 : ctx->cb_fn = cb_fn;
1073 0 : ctx->cb_arg = cb_arg;
1074 :
1075 0 : spdk_for_each_channel(tgt,
1076 : _nvmf_tgt_resume_polling,
1077 : ctx,
1078 : _nvmf_tgt_resume_polling_done);
1079 0 : return 0;
1080 : }
1081 :
1082 : struct spdk_nvmf_subsystem *
1083 0 : spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn)
1084 : {
1085 0 : struct spdk_nvmf_subsystem subsystem;
1086 :
1087 0 : if (!subnqn) {
1088 0 : return NULL;
1089 : }
1090 :
1091 : /* Ensure that subnqn is null terminated */
1092 0 : if (!memchr(subnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) {
1093 0 : SPDK_ERRLOG("Connect SUBNQN is not null terminated\n");
1094 0 : return NULL;
1095 : }
1096 :
1097 0 : snprintf(subsystem.subnqn, sizeof(subsystem.subnqn), "%s", subnqn);
1098 0 : return RB_FIND(subsystem_tree, &tgt->subsystems, &subsystem);
1099 : }
1100 :
1101 : struct spdk_nvmf_transport *
1102 0 : spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, const char *transport_name)
1103 : {
1104 : struct spdk_nvmf_transport *transport;
1105 :
1106 0 : TAILQ_FOREACH(transport, &tgt->transports, link) {
1107 0 : if (!strncasecmp(transport->ops->name, transport_name, SPDK_NVMF_TRSTRING_MAX_LEN)) {
1108 0 : return transport;
1109 : }
1110 : }
1111 0 : return NULL;
1112 : }
1113 :
1114 : struct nvmf_new_qpair_ctx {
1115 : struct spdk_nvmf_qpair *qpair;
1116 : struct spdk_nvmf_poll_group *group;
1117 : };
1118 :
1119 : static void
1120 0 : _nvmf_poll_group_add(void *_ctx)
1121 : {
1122 0 : struct nvmf_new_qpair_ctx *ctx = _ctx;
1123 0 : struct spdk_nvmf_qpair *qpair = ctx->qpair;
1124 0 : struct spdk_nvmf_poll_group *group = ctx->group;
1125 :
1126 0 : free(_ctx);
1127 :
1128 0 : if (spdk_nvmf_poll_group_add(group, qpair) != 0) {
1129 0 : SPDK_ERRLOG("Unable to add the qpair to a poll group.\n");
1130 0 : spdk_nvmf_qpair_disconnect(qpair);
1131 : }
1132 0 : }
1133 :
1134 : void
1135 0 : spdk_nvmf_tgt_new_qpair(struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair)
1136 : {
1137 : struct spdk_nvmf_poll_group *group;
1138 : struct nvmf_new_qpair_ctx *ctx;
1139 :
1140 0 : group = spdk_nvmf_get_optimal_poll_group(qpair);
1141 0 : if (group == NULL) {
1142 0 : if (tgt->next_poll_group == NULL) {
1143 0 : tgt->next_poll_group = TAILQ_FIRST(&tgt->poll_groups);
1144 0 : if (tgt->next_poll_group == NULL) {
1145 0 : SPDK_ERRLOG("No poll groups exist.\n");
1146 0 : spdk_nvmf_qpair_disconnect(qpair);
1147 0 : return;
1148 : }
1149 : }
1150 0 : group = tgt->next_poll_group;
1151 0 : tgt->next_poll_group = TAILQ_NEXT(group, link);
1152 : }
1153 :
1154 0 : ctx = calloc(1, sizeof(*ctx));
1155 0 : if (!ctx) {
1156 0 : SPDK_ERRLOG("Unable to send message to poll group.\n");
1157 0 : spdk_nvmf_qpair_disconnect(qpair);
1158 0 : return;
1159 : }
1160 :
1161 0 : ctx->qpair = qpair;
1162 0 : ctx->group = group;
1163 :
1164 0 : pthread_mutex_lock(&group->mutex);
1165 0 : group->current_unassociated_qpairs++;
1166 0 : pthread_mutex_unlock(&group->mutex);
1167 :
1168 0 : spdk_thread_send_msg(group->thread, _nvmf_poll_group_add, ctx);
1169 : }
1170 :
1171 : struct spdk_nvmf_poll_group *
1172 0 : spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt)
1173 : {
1174 : struct spdk_io_channel *ch;
1175 :
1176 0 : ch = spdk_get_io_channel(tgt);
1177 0 : if (!ch) {
1178 0 : SPDK_ERRLOG("Unable to get I/O channel for target\n");
1179 0 : return NULL;
1180 : }
1181 :
1182 0 : return spdk_io_channel_get_ctx(ch);
1183 : }
1184 :
1185 : void
1186 0 : spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group,
1187 : spdk_nvmf_poll_group_destroy_done_fn cb_fn,
1188 : void *cb_arg)
1189 : {
1190 0 : assert(group->destroy_cb_fn == NULL);
1191 0 : group->destroy_cb_fn = cb_fn;
1192 0 : group->destroy_cb_arg = cb_arg;
1193 :
1194 : /* This function will put the io_channel associated with this poll group */
1195 0 : nvmf_tgt_destroy_poll_group_qpairs(group);
1196 0 : }
1197 :
1198 : int
1199 0 : spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group,
1200 : struct spdk_nvmf_qpair *qpair)
1201 : {
1202 : int rc;
1203 : struct spdk_nvmf_transport_poll_group *tgroup;
1204 :
1205 0 : TAILQ_INIT(&qpair->outstanding);
1206 0 : qpair->group = group;
1207 0 : qpair->ctrlr = NULL;
1208 0 : qpair->disconnect_started = false;
1209 :
1210 0 : tgroup = nvmf_get_transport_poll_group(group, qpair->transport);
1211 0 : if (tgroup == NULL) {
1212 0 : return -1;
1213 : }
1214 :
1215 0 : rc = nvmf_transport_poll_group_add(tgroup, qpair);
1216 :
1217 : /* We add the qpair to the group only it is successfully added into the tgroup */
1218 0 : if (rc == 0) {
1219 : SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_add_qpair, qpair, spdk_thread_get_id(group->thread));
1220 0 : TAILQ_INSERT_TAIL(&group->qpairs, qpair, link);
1221 0 : nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_CONNECTING);
1222 : }
1223 :
1224 0 : return rc;
1225 : }
1226 :
1227 : static void
1228 0 : _nvmf_ctrlr_destruct(void *ctx)
1229 : {
1230 0 : struct spdk_nvmf_ctrlr *ctrlr = ctx;
1231 :
1232 0 : nvmf_ctrlr_destruct(ctrlr);
1233 0 : }
1234 :
1235 : static void
1236 0 : _nvmf_ctrlr_free_from_qpair(void *ctx)
1237 : {
1238 0 : struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;
1239 0 : struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr;
1240 : uint32_t count;
1241 :
1242 0 : spdk_bit_array_clear(ctrlr->qpair_mask, qpair_ctx->qid);
1243 0 : SPDK_DEBUGLOG(nvmf, "qpair_mask cleared, qid %u\n", qpair_ctx->qid);
1244 0 : count = spdk_bit_array_count_set(ctrlr->qpair_mask);
1245 0 : if (count == 0) {
1246 0 : assert(!ctrlr->in_destruct);
1247 0 : SPDK_DEBUGLOG(nvmf, "Last qpair %u, destroy ctrlr 0x%hx\n", qpair_ctx->qid, ctrlr->cntlid);
1248 0 : ctrlr->in_destruct = true;
1249 0 : spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr);
1250 : }
1251 0 : free(qpair_ctx);
1252 0 : }
1253 :
1254 : static void
1255 0 : _nvmf_transport_qpair_fini_complete(void *cb_ctx)
1256 : {
1257 0 : struct nvmf_qpair_disconnect_ctx *qpair_ctx = cb_ctx;
1258 : struct spdk_nvmf_ctrlr *ctrlr;
1259 :
1260 0 : ctrlr = qpair_ctx->ctrlr;
1261 0 : SPDK_DEBUGLOG(nvmf, "Finish destroying qid %u\n", qpair_ctx->qid);
1262 :
1263 0 : if (ctrlr) {
1264 0 : if (qpair_ctx->qid == 0) {
1265 : /* Admin qpair is removed, so set the pointer to NULL.
1266 : * This operation is safe since we are on ctrlr thread now, admin qpair's thread is the same
1267 : * as controller's thread */
1268 0 : assert(ctrlr->thread == spdk_get_thread());
1269 0 : ctrlr->admin_qpair = NULL;
1270 : }
1271 : /* Free qpair id from controller's bit mask and destroy the controller if it is the last qpair */
1272 0 : if (ctrlr->thread) {
1273 0 : spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_free_from_qpair, qpair_ctx);
1274 : } else {
1275 0 : _nvmf_ctrlr_free_from_qpair(qpair_ctx);
1276 : }
1277 : } else {
1278 0 : free(qpair_ctx);
1279 : }
1280 0 : }
1281 :
1282 : void
1283 0 : spdk_nvmf_poll_group_remove(struct spdk_nvmf_qpair *qpair)
1284 : {
1285 : struct spdk_nvmf_transport_poll_group *tgroup;
1286 : int rc;
1287 :
1288 : SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_remove_qpair, qpair,
1289 : spdk_thread_get_id(qpair->group->thread));
1290 0 : nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR);
1291 :
1292 : /* Find the tgroup and remove the qpair from the tgroup */
1293 0 : tgroup = nvmf_get_transport_poll_group(qpair->group, qpair->transport);
1294 0 : if (tgroup != NULL) {
1295 0 : rc = nvmf_transport_poll_group_remove(tgroup, qpair);
1296 0 : if (rc && (rc != ENOTSUP)) {
1297 0 : SPDK_ERRLOG("Cannot remove qpair=%p from transport group=%p\n",
1298 : qpair, tgroup);
1299 : }
1300 : }
1301 :
1302 0 : TAILQ_REMOVE(&qpair->group->qpairs, qpair, link);
1303 0 : qpair->group = NULL;
1304 0 : }
1305 :
1306 : static void
1307 0 : _nvmf_qpair_sgroup_req_clean(struct spdk_nvmf_subsystem_poll_group *sgroup,
1308 : const struct spdk_nvmf_qpair *qpair)
1309 : {
1310 : struct spdk_nvmf_request *req, *tmp;
1311 0 : TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) {
1312 0 : if (req->qpair == qpair) {
1313 0 : TAILQ_REMOVE(&sgroup->queued, req, link);
1314 0 : if (nvmf_transport_req_free(req)) {
1315 0 : SPDK_ERRLOG("Transport request free error!\n");
1316 : }
1317 : }
1318 : }
1319 0 : }
1320 :
1321 : static void
1322 0 : _nvmf_qpair_destroy(void *ctx, int status)
1323 : {
1324 0 : struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;
1325 0 : struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair;
1326 0 : struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr;
1327 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1328 : uint32_t sid;
1329 :
1330 0 : assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING);
1331 0 : qpair_ctx->qid = qpair->qid;
1332 :
1333 0 : if (qpair->connect_received) {
1334 0 : if (0 == qpair->qid) {
1335 0 : assert(qpair->group->stat.current_admin_qpairs > 0);
1336 0 : qpair->group->stat.current_admin_qpairs--;
1337 : } else {
1338 0 : assert(qpair->group->stat.current_io_qpairs > 0);
1339 0 : qpair->group->stat.current_io_qpairs--;
1340 : }
1341 : } else {
1342 0 : pthread_mutex_lock(&qpair->group->mutex);
1343 0 : qpair->group->current_unassociated_qpairs--;
1344 0 : pthread_mutex_unlock(&qpair->group->mutex);
1345 : }
1346 :
1347 0 : if (ctrlr) {
1348 0 : sgroup = &qpair->group->sgroups[ctrlr->subsys->id];
1349 0 : _nvmf_qpair_sgroup_req_clean(sgroup, qpair);
1350 : } else {
1351 0 : for (sid = 0; sid < qpair->group->num_sgroups; sid++) {
1352 0 : sgroup = &qpair->group->sgroups[sid];
1353 0 : assert(sgroup != NULL);
1354 0 : _nvmf_qpair_sgroup_req_clean(sgroup, qpair);
1355 : }
1356 : }
1357 :
1358 0 : nvmf_qpair_auth_destroy(qpair);
1359 0 : qpair_ctx->ctrlr = ctrlr;
1360 0 : spdk_nvmf_poll_group_remove(qpair);
1361 0 : nvmf_transport_qpair_fini(qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx);
1362 0 : }
1363 :
1364 : static void
1365 0 : _nvmf_qpair_disconnect_msg(void *ctx)
1366 : {
1367 0 : struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;
1368 :
1369 0 : spdk_nvmf_qpair_disconnect(qpair_ctx->qpair);
1370 0 : free(ctx);
1371 0 : }
1372 :
1373 : int
1374 0 : spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair)
1375 : {
1376 0 : struct spdk_nvmf_poll_group *group = qpair->group;
1377 : struct nvmf_qpair_disconnect_ctx *qpair_ctx;
1378 :
1379 0 : if (__atomic_test_and_set(&qpair->disconnect_started, __ATOMIC_RELAXED)) {
1380 0 : return -EINPROGRESS;
1381 : }
1382 :
1383 : /* If we get a qpair in the uninitialized state, we can just destroy it immediately */
1384 0 : if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) {
1385 0 : nvmf_transport_qpair_fini(qpair, NULL, NULL);
1386 0 : return 0;
1387 : }
1388 :
1389 0 : assert(group != NULL);
1390 0 : if (spdk_get_thread() != group->thread) {
1391 : /* clear the atomic so we can set it on the next call on the proper thread. */
1392 0 : __atomic_clear(&qpair->disconnect_started, __ATOMIC_RELAXED);
1393 0 : qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx));
1394 0 : if (!qpair_ctx) {
1395 0 : SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n");
1396 0 : return -ENOMEM;
1397 : }
1398 0 : qpair_ctx->qpair = qpair;
1399 0 : spdk_thread_send_msg(group->thread, _nvmf_qpair_disconnect_msg, qpair_ctx);
1400 0 : return 0;
1401 : }
1402 :
1403 : SPDK_DTRACE_PROBE2_TICKS(nvmf_qpair_disconnect, qpair, spdk_thread_get_id(group->thread));
1404 0 : assert(spdk_nvmf_qpair_is_active(qpair));
1405 0 : nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_DEACTIVATING);
1406 :
1407 0 : qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx));
1408 0 : if (!qpair_ctx) {
1409 0 : SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n");
1410 0 : return -ENOMEM;
1411 : }
1412 :
1413 0 : qpair_ctx->qpair = qpair;
1414 :
1415 : /* Check for outstanding I/O */
1416 0 : if (!TAILQ_EMPTY(&qpair->outstanding)) {
1417 : SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_drain_qpair, qpair, spdk_thread_get_id(group->thread));
1418 0 : qpair->state_cb = _nvmf_qpair_destroy;
1419 0 : qpair->state_cb_arg = qpair_ctx;
1420 0 : nvmf_qpair_abort_pending_zcopy_reqs(qpair);
1421 0 : nvmf_qpair_free_aer(qpair);
1422 0 : return 0;
1423 : }
1424 :
1425 0 : _nvmf_qpair_destroy(qpair_ctx, 0);
1426 :
1427 0 : return 0;
1428 : }
1429 :
1430 : int
1431 0 : spdk_nvmf_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair,
1432 : struct spdk_nvme_transport_id *trid)
1433 : {
1434 0 : memset(trid, 0, sizeof(*trid));
1435 0 : return nvmf_transport_qpair_get_peer_trid(qpair, trid);
1436 : }
1437 :
1438 : int
1439 0 : spdk_nvmf_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair,
1440 : struct spdk_nvme_transport_id *trid)
1441 : {
1442 0 : memset(trid, 0, sizeof(*trid));
1443 0 : return nvmf_transport_qpair_get_local_trid(qpair, trid);
1444 : }
1445 :
1446 : int
1447 0 : spdk_nvmf_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair,
1448 : struct spdk_nvme_transport_id *trid)
1449 : {
1450 0 : memset(trid, 0, sizeof(*trid));
1451 0 : return nvmf_transport_qpair_get_listen_trid(qpair, trid);
1452 : }
1453 :
1454 : static int
1455 1 : poll_group_update_subsystem(struct spdk_nvmf_poll_group *group,
1456 : struct spdk_nvmf_subsystem *subsystem)
1457 : {
1458 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1459 : uint32_t i, j;
1460 : struct spdk_nvmf_ns *ns;
1461 : struct spdk_nvmf_registrant *reg, *tmp;
1462 : struct spdk_io_channel *ch;
1463 : struct spdk_nvmf_subsystem_pg_ns_info *ns_info;
1464 : struct spdk_nvmf_ctrlr *ctrlr;
1465 : bool ns_changed;
1466 :
1467 : /* Make sure our poll group has memory for this subsystem allocated */
1468 1 : if (subsystem->id >= group->num_sgroups) {
1469 0 : return -ENOMEM;
1470 : }
1471 :
1472 1 : sgroup = &group->sgroups[subsystem->id];
1473 :
1474 : /* Make sure the array of namespace information is the correct size */
1475 1 : if (sgroup->num_ns == 0 && subsystem->max_nsid > 0) {
1476 : /* First allocation */
1477 1 : sgroup->ns_info = calloc(subsystem->max_nsid, sizeof(struct spdk_nvmf_subsystem_pg_ns_info));
1478 1 : if (!sgroup->ns_info) {
1479 0 : return -ENOMEM;
1480 : }
1481 1 : sgroup->num_ns = subsystem->max_nsid;
1482 : }
1483 :
1484 1 : ns_changed = false;
1485 :
1486 : /* Detect bdevs that were added or removed */
1487 2 : for (i = 0; i < sgroup->num_ns; i++) {
1488 1 : ns = subsystem->ns[i];
1489 1 : ns_info = &sgroup->ns_info[i];
1490 1 : ch = ns_info->channel;
1491 :
1492 1 : if (ns == NULL && ch == NULL) {
1493 : /* Both NULL. Leave empty */
1494 1 : } else if (ns == NULL && ch != NULL) {
1495 : /* There was a channel here, but the namespace is gone. */
1496 0 : ns_changed = true;
1497 0 : spdk_put_io_channel(ch);
1498 0 : ns_info->channel = NULL;
1499 1 : } else if (ns != NULL && ch == NULL) {
1500 : /* A namespace appeared but there is no channel yet */
1501 1 : ns_changed = true;
1502 1 : ch = spdk_bdev_get_io_channel(ns->desc);
1503 1 : if (ch == NULL) {
1504 0 : SPDK_ERRLOG("Could not allocate I/O channel.\n");
1505 0 : return -ENOMEM;
1506 : }
1507 1 : ns_info->channel = ch;
1508 0 : } else if (spdk_uuid_compare(&ns_info->uuid, spdk_bdev_get_uuid(ns->bdev)) != 0) {
1509 : /* A namespace was here before, but was replaced by a new one. */
1510 0 : ns_changed = true;
1511 0 : spdk_put_io_channel(ns_info->channel);
1512 0 : memset(ns_info, 0, sizeof(*ns_info));
1513 :
1514 0 : ch = spdk_bdev_get_io_channel(ns->desc);
1515 0 : if (ch == NULL) {
1516 0 : SPDK_ERRLOG("Could not allocate I/O channel.\n");
1517 0 : return -ENOMEM;
1518 : }
1519 0 : ns_info->channel = ch;
1520 0 : } else if (ns_info->num_blocks != spdk_bdev_get_num_blocks(ns->bdev)) {
1521 : /* Namespace is still there but size has changed */
1522 0 : SPDK_DEBUGLOG(nvmf, "Namespace resized: subsystem_id %u,"
1523 : " nsid %u, pg %p, old %" PRIu64 ", new %" PRIu64 "\n",
1524 : subsystem->id,
1525 : ns->nsid,
1526 : group,
1527 : ns_info->num_blocks,
1528 : spdk_bdev_get_num_blocks(ns->bdev));
1529 0 : ns_changed = true;
1530 0 : } else if (ns_info->anagrpid != ns->anagrpid) {
1531 : /* Namespace is still there but ANA group ID has changed */
1532 0 : SPDK_DEBUGLOG(nvmf, "ANA group ID changed: subsystem_id %u,"
1533 : "nsid %u, pg %p, old %u, new %u\n",
1534 : subsystem->id,
1535 : ns->nsid,
1536 : group,
1537 : ns_info->anagrpid,
1538 : ns->anagrpid);
1539 0 : ns_changed = true;
1540 : }
1541 :
1542 1 : if (ns == NULL) {
1543 0 : memset(ns_info, 0, sizeof(*ns_info));
1544 : } else {
1545 1 : ns_info->uuid = *spdk_bdev_get_uuid(ns->bdev);
1546 1 : ns_info->num_blocks = spdk_bdev_get_num_blocks(ns->bdev);
1547 1 : ns_info->anagrpid = ns->anagrpid;
1548 1 : ns_info->crkey = ns->crkey;
1549 1 : ns_info->rtype = ns->rtype;
1550 1 : if (ns->holder) {
1551 0 : ns_info->holder_id = ns->holder->hostid;
1552 : }
1553 :
1554 1 : memset(&ns_info->reg_hostid, 0, SPDK_NVMF_MAX_NUM_REGISTRANTS * sizeof(struct spdk_uuid));
1555 1 : j = 0;
1556 1 : TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) {
1557 0 : if (j >= SPDK_NVMF_MAX_NUM_REGISTRANTS) {
1558 0 : SPDK_ERRLOG("Maximum %u registrants can support.\n", SPDK_NVMF_MAX_NUM_REGISTRANTS);
1559 0 : return -EINVAL;
1560 : }
1561 0 : ns_info->reg_hostid[j++] = reg->hostid;
1562 : }
1563 : }
1564 : }
1565 :
1566 1 : if (ns_changed) {
1567 1 : TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) {
1568 0 : if (ctrlr->thread != spdk_get_thread()) {
1569 0 : continue;
1570 : }
1571 : /* It is possible that a ctrlr was added but the admin_qpair hasn't been
1572 : * assigned yet.
1573 : */
1574 0 : if (!ctrlr->admin_qpair) {
1575 0 : continue;
1576 : }
1577 0 : if (ctrlr->admin_qpair->group == group) {
1578 0 : nvmf_ctrlr_async_event_ns_notice(ctrlr);
1579 0 : nvmf_ctrlr_async_event_ana_change_notice(ctrlr);
1580 : }
1581 : }
1582 : }
1583 :
1584 1 : return 0;
1585 : }
1586 :
1587 : int
1588 0 : nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group,
1589 : struct spdk_nvmf_subsystem *subsystem)
1590 : {
1591 0 : return poll_group_update_subsystem(group, subsystem);
1592 : }
1593 :
1594 : int
1595 1 : nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group,
1596 : struct spdk_nvmf_subsystem *subsystem,
1597 : spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)
1598 : {
1599 1 : int rc = 0;
1600 1 : struct spdk_nvmf_subsystem_poll_group *sgroup = &group->sgroups[subsystem->id];
1601 : struct spdk_nvmf_request *req, *tmp;
1602 : uint32_t i;
1603 :
1604 1 : if (!TAILQ_EMPTY(&sgroup->queued)) {
1605 0 : SPDK_ERRLOG("sgroup->queued not empty when adding subsystem\n");
1606 0 : TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) {
1607 0 : TAILQ_REMOVE(&sgroup->queued, req, link);
1608 0 : if (nvmf_transport_req_free(req)) {
1609 0 : SPDK_ERRLOG("Transport request free error!\n");
1610 : }
1611 : }
1612 : }
1613 :
1614 1 : rc = poll_group_update_subsystem(group, subsystem);
1615 1 : if (rc) {
1616 0 : nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL);
1617 0 : goto fini;
1618 : }
1619 :
1620 1 : sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE;
1621 :
1622 2 : for (i = 0; i < sgroup->num_ns; i++) {
1623 1 : sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE;
1624 : }
1625 :
1626 1 : fini:
1627 1 : if (cb_fn) {
1628 0 : cb_fn(cb_arg, rc);
1629 : }
1630 :
1631 : SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_add_subsystem, spdk_thread_get_id(group->thread),
1632 : subsystem->subnqn);
1633 :
1634 1 : return rc;
1635 : }
1636 :
1637 : static void
1638 0 : _nvmf_poll_group_remove_subsystem_cb(void *ctx, int status)
1639 : {
1640 0 : struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx;
1641 : struct spdk_nvmf_subsystem *subsystem;
1642 : struct spdk_nvmf_poll_group *group;
1643 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1644 0 : spdk_nvmf_poll_group_mod_done cpl_fn = NULL;
1645 0 : void *cpl_ctx = NULL;
1646 : uint32_t nsid;
1647 :
1648 0 : group = qpair_ctx->group;
1649 0 : subsystem = qpair_ctx->subsystem;
1650 0 : cpl_fn = qpair_ctx->cpl_fn;
1651 0 : cpl_ctx = qpair_ctx->cpl_ctx;
1652 0 : sgroup = &group->sgroups[subsystem->id];
1653 :
1654 0 : if (status) {
1655 0 : goto fini;
1656 : }
1657 :
1658 0 : for (nsid = 0; nsid < sgroup->num_ns; nsid++) {
1659 0 : if (sgroup->ns_info[nsid].channel) {
1660 0 : spdk_put_io_channel(sgroup->ns_info[nsid].channel);
1661 0 : sgroup->ns_info[nsid].channel = NULL;
1662 : }
1663 : }
1664 :
1665 0 : sgroup->num_ns = 0;
1666 0 : free(sgroup->ns_info);
1667 0 : sgroup->ns_info = NULL;
1668 0 : fini:
1669 0 : free(qpair_ctx);
1670 0 : if (cpl_fn) {
1671 0 : cpl_fn(cpl_ctx, status);
1672 : }
1673 0 : }
1674 :
1675 : static void nvmf_poll_group_remove_subsystem_msg(void *ctx);
1676 :
1677 : static void
1678 0 : nvmf_poll_group_remove_subsystem_msg(void *ctx)
1679 : {
1680 : struct spdk_nvmf_qpair *qpair, *qpair_tmp;
1681 : struct spdk_nvmf_subsystem *subsystem;
1682 : struct spdk_nvmf_poll_group *group;
1683 0 : struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx;
1684 0 : bool qpairs_found = false;
1685 0 : int rc = 0;
1686 :
1687 0 : group = qpair_ctx->group;
1688 0 : subsystem = qpair_ctx->subsystem;
1689 :
1690 0 : TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) {
1691 0 : if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) {
1692 0 : qpairs_found = true;
1693 0 : rc = spdk_nvmf_qpair_disconnect(qpair);
1694 0 : if (rc && rc != -EINPROGRESS) {
1695 0 : break;
1696 : }
1697 : }
1698 : }
1699 :
1700 0 : if (!qpairs_found) {
1701 0 : _nvmf_poll_group_remove_subsystem_cb(ctx, 0);
1702 0 : return;
1703 : }
1704 :
1705 : /* Some qpairs are in process of being disconnected. Send a message and try to remove them again */
1706 0 : spdk_thread_send_msg(spdk_get_thread(), nvmf_poll_group_remove_subsystem_msg, ctx);
1707 : }
1708 :
1709 : void
1710 0 : nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group,
1711 : struct spdk_nvmf_subsystem *subsystem,
1712 : spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)
1713 : {
1714 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1715 : struct nvmf_qpair_disconnect_many_ctx *ctx;
1716 : uint32_t i;
1717 :
1718 : SPDK_DTRACE_PROBE3_TICKS(nvmf_poll_group_remove_subsystem, group, spdk_thread_get_id(group->thread),
1719 : subsystem->subnqn);
1720 :
1721 0 : ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx));
1722 0 : if (!ctx) {
1723 0 : SPDK_ERRLOG("Unable to allocate memory for context to remove poll subsystem\n");
1724 0 : if (cb_fn) {
1725 0 : cb_fn(cb_arg, -1);
1726 : }
1727 0 : return;
1728 : }
1729 :
1730 0 : ctx->group = group;
1731 0 : ctx->subsystem = subsystem;
1732 0 : ctx->cpl_fn = cb_fn;
1733 0 : ctx->cpl_ctx = cb_arg;
1734 :
1735 0 : sgroup = &group->sgroups[subsystem->id];
1736 0 : sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE;
1737 :
1738 0 : for (i = 0; i < sgroup->num_ns; i++) {
1739 0 : sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_INACTIVE;
1740 : }
1741 :
1742 0 : nvmf_poll_group_remove_subsystem_msg(ctx);
1743 : }
1744 :
1745 : void
1746 0 : nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group,
1747 : struct spdk_nvmf_subsystem *subsystem,
1748 : uint32_t nsid,
1749 : spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)
1750 : {
1751 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1752 0 : struct spdk_nvmf_subsystem_pg_ns_info *ns_info = NULL;
1753 0 : int rc = 0;
1754 : uint32_t i;
1755 :
1756 0 : if (subsystem->id >= group->num_sgroups) {
1757 0 : rc = -1;
1758 0 : goto fini;
1759 : }
1760 :
1761 0 : sgroup = &group->sgroups[subsystem->id];
1762 0 : if (sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSED) {
1763 0 : goto fini;
1764 : }
1765 0 : sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSING;
1766 :
1767 0 : if (nsid == SPDK_NVME_GLOBAL_NS_TAG) {
1768 0 : for (i = 0; i < sgroup->num_ns; i++) {
1769 0 : ns_info = &sgroup->ns_info[i];
1770 0 : ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING;
1771 : }
1772 : } else {
1773 : /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */
1774 0 : if (nsid - 1 < sgroup->num_ns) {
1775 0 : ns_info = &sgroup->ns_info[nsid - 1];
1776 0 : ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING;
1777 : }
1778 : }
1779 :
1780 0 : if (sgroup->mgmt_io_outstanding > 0) {
1781 0 : assert(sgroup->cb_fn == NULL);
1782 0 : sgroup->cb_fn = cb_fn;
1783 0 : assert(sgroup->cb_arg == NULL);
1784 0 : sgroup->cb_arg = cb_arg;
1785 0 : return;
1786 : }
1787 :
1788 0 : if (nsid == SPDK_NVME_GLOBAL_NS_TAG) {
1789 0 : for (i = 0; i < sgroup->num_ns; i++) {
1790 0 : ns_info = &sgroup->ns_info[i];
1791 :
1792 0 : if (ns_info->io_outstanding > 0) {
1793 0 : assert(sgroup->cb_fn == NULL);
1794 0 : sgroup->cb_fn = cb_fn;
1795 0 : assert(sgroup->cb_arg == NULL);
1796 0 : sgroup->cb_arg = cb_arg;
1797 0 : return;
1798 : }
1799 : }
1800 : } else {
1801 0 : if (ns_info != NULL && ns_info->io_outstanding > 0) {
1802 0 : assert(sgroup->cb_fn == NULL);
1803 0 : sgroup->cb_fn = cb_fn;
1804 0 : assert(sgroup->cb_arg == NULL);
1805 0 : sgroup->cb_arg = cb_arg;
1806 0 : return;
1807 : }
1808 : }
1809 :
1810 0 : assert(sgroup->mgmt_io_outstanding == 0);
1811 0 : sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED;
1812 0 : fini:
1813 0 : if (cb_fn) {
1814 0 : cb_fn(cb_arg, rc);
1815 : }
1816 : }
1817 :
1818 : void
1819 0 : nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group,
1820 : struct spdk_nvmf_subsystem *subsystem,
1821 : spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)
1822 : {
1823 : struct spdk_nvmf_request *req, *tmp;
1824 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1825 0 : int rc = 0;
1826 : uint32_t i;
1827 :
1828 0 : if (subsystem->id >= group->num_sgroups) {
1829 0 : rc = -1;
1830 0 : goto fini;
1831 : }
1832 :
1833 0 : sgroup = &group->sgroups[subsystem->id];
1834 :
1835 0 : if (sgroup->state == SPDK_NVMF_SUBSYSTEM_ACTIVE) {
1836 0 : goto fini;
1837 : }
1838 :
1839 0 : rc = poll_group_update_subsystem(group, subsystem);
1840 0 : if (rc) {
1841 0 : goto fini;
1842 : }
1843 :
1844 0 : for (i = 0; i < sgroup->num_ns; i++) {
1845 0 : sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE;
1846 : }
1847 :
1848 0 : sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE;
1849 :
1850 : /* Release all queued requests */
1851 0 : TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) {
1852 0 : TAILQ_REMOVE(&sgroup->queued, req, link);
1853 0 : if (spdk_nvmf_request_using_zcopy(req)) {
1854 0 : spdk_nvmf_request_zcopy_start(req);
1855 : } else {
1856 0 : spdk_nvmf_request_exec(req);
1857 : }
1858 :
1859 : }
1860 0 : fini:
1861 0 : if (cb_fn) {
1862 0 : cb_fn(cb_arg, rc);
1863 : }
1864 0 : }
1865 :
1866 :
1867 : struct spdk_nvmf_poll_group *
1868 0 : spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair)
1869 : {
1870 : struct spdk_nvmf_transport_poll_group *tgroup;
1871 :
1872 0 : tgroup = nvmf_transport_get_optimal_poll_group(qpair->transport, qpair);
1873 :
1874 0 : if (tgroup == NULL) {
1875 0 : return NULL;
1876 : }
1877 :
1878 0 : return tgroup->group;
1879 : }
1880 :
1881 : void
1882 0 : spdk_nvmf_poll_group_dump_stat(struct spdk_nvmf_poll_group *group, struct spdk_json_write_ctx *w)
1883 : {
1884 : struct spdk_nvmf_transport_poll_group *tgroup;
1885 :
1886 0 : spdk_json_write_object_begin(w);
1887 :
1888 0 : spdk_json_write_named_string(w, "name", spdk_thread_get_name(spdk_get_thread()));
1889 0 : spdk_json_write_named_uint32(w, "admin_qpairs", group->stat.admin_qpairs);
1890 0 : spdk_json_write_named_uint32(w, "io_qpairs", group->stat.io_qpairs);
1891 0 : spdk_json_write_named_uint32(w, "current_admin_qpairs", group->stat.current_admin_qpairs);
1892 0 : spdk_json_write_named_uint32(w, "current_io_qpairs", group->stat.current_io_qpairs);
1893 0 : spdk_json_write_named_uint64(w, "pending_bdev_io", group->stat.pending_bdev_io);
1894 0 : spdk_json_write_named_uint64(w, "completed_nvme_io", group->stat.completed_nvme_io);
1895 :
1896 0 : spdk_json_write_named_array_begin(w, "transports");
1897 :
1898 0 : TAILQ_FOREACH(tgroup, &group->tgroups, link) {
1899 0 : spdk_json_write_object_begin(w);
1900 : /*
1901 : * The trtype field intentionally contains a transport name as this is more informative.
1902 : * The field has not been renamed for backward compatibility.
1903 : */
1904 0 : spdk_json_write_named_string(w, "trtype", spdk_nvmf_get_transport_name(tgroup->transport));
1905 :
1906 0 : if (tgroup->transport->ops->poll_group_dump_stat) {
1907 0 : tgroup->transport->ops->poll_group_dump_stat(tgroup, w);
1908 : }
1909 :
1910 0 : spdk_json_write_object_end(w);
1911 : }
1912 :
1913 0 : spdk_json_write_array_end(w);
1914 0 : spdk_json_write_object_end(w);
1915 0 : }
|
