Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2016 Intel Corporation. All rights reserved.
3 : * Copyright (c) 2018-2019, 2021 Mellanox Technologies LTD. All rights reserved.
4 : * Copyright (c) 2021, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : */
6 :
7 : #include "spdk/stdinc.h"
8 :
9 : #include "spdk/bdev.h"
10 : #include "spdk/bit_array.h"
11 : #include "spdk/thread.h"
12 : #include "spdk/nvmf.h"
13 : #include "spdk/endian.h"
14 : #include "spdk/string.h"
15 : #include "spdk/log.h"
16 : #include "spdk_internal/usdt.h"
17 :
18 : #include "nvmf_internal.h"
19 : #include "transport.h"
20 :
21 1 : SPDK_LOG_REGISTER_COMPONENT(nvmf)
22 :
23 : #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024
24 :
25 : static TAILQ_HEAD(, spdk_nvmf_tgt) g_nvmf_tgts = TAILQ_HEAD_INITIALIZER(g_nvmf_tgts);
26 :
27 : typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status);
28 :
29 : /* supplied to a single call to nvmf_qpair_disconnect */
30 : struct nvmf_qpair_disconnect_ctx {
31 : struct spdk_nvmf_qpair *qpair;
32 : struct spdk_nvmf_ctrlr *ctrlr;
33 : uint16_t qid;
34 : };
35 :
36 : /*
37 : * There are several times when we need to iterate through the list of all qpairs and selectively delete them.
38 : * In order to do this sequentially without overlap, we must provide a context to recover the next qpair from
39 : * to enable calling nvmf_qpair_disconnect on the next desired qpair.
40 : */
41 : struct nvmf_qpair_disconnect_many_ctx {
42 : struct spdk_nvmf_subsystem *subsystem;
43 : struct spdk_nvmf_poll_group *group;
44 : spdk_nvmf_poll_group_mod_done cpl_fn;
45 : void *cpl_ctx;
46 : };
47 :
48 : static struct spdk_nvmf_referral *
49 0 : nvmf_tgt_find_referral(struct spdk_nvmf_tgt *tgt,
50 : const struct spdk_nvme_transport_id *trid)
51 : {
52 : struct spdk_nvmf_referral *referral;
53 :
54 0 : TAILQ_FOREACH(referral, &tgt->referrals, link) {
55 0 : if (spdk_nvme_transport_id_compare(&referral->trid, trid) == 0) {
56 0 : return referral;
57 : }
58 0 : }
59 :
60 0 : return NULL;
61 0 : }
62 :
63 : int
64 0 : spdk_nvmf_tgt_add_referral(struct spdk_nvmf_tgt *tgt,
65 : const struct spdk_nvmf_referral_opts *uopts)
66 : {
67 : struct spdk_nvmf_referral *referral;
68 0 : struct spdk_nvmf_referral_opts opts = {};
69 0 : struct spdk_nvme_transport_id *trid = &opts.trid;
70 :
71 0 : memcpy(&opts, uopts, spdk_min(uopts->size, sizeof(opts)));
72 0 : if (trid->subnqn[0] == '\0') {
73 0 : snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
74 0 : }
75 :
76 0 : if (!nvmf_nqn_is_valid(trid->subnqn)) {
77 0 : SPDK_ERRLOG("Invalid subsystem NQN\n");
78 0 : return -EINVAL;
79 : }
80 :
81 : /* If the entry already exists, just ignore it. */
82 0 : if (nvmf_tgt_find_referral(tgt, trid)) {
83 0 : return 0;
84 : }
85 :
86 0 : referral = calloc(1, sizeof(*referral));
87 0 : if (!referral) {
88 0 : SPDK_ERRLOG("Failed to allocate memory for a referral\n");
89 0 : return -ENOMEM;
90 : }
91 :
92 0 : referral->entry.subtype = nvmf_nqn_is_discovery(trid->subnqn) ?
93 : SPDK_NVMF_SUBTYPE_DISCOVERY :
94 : SPDK_NVMF_SUBTYPE_NVME;
95 0 : referral->entry.treq.secure_channel = opts.secure_channel ?
96 : SPDK_NVMF_TREQ_SECURE_CHANNEL_REQUIRED :
97 : SPDK_NVMF_TREQ_SECURE_CHANNEL_NOT_REQUIRED;
98 0 : referral->entry.cntlid = 0xffff;
99 0 : referral->entry.trtype = trid->trtype;
100 0 : referral->entry.adrfam = trid->adrfam;
101 0 : memcpy(&referral->trid, trid, sizeof(struct spdk_nvme_transport_id));
102 0 : spdk_strcpy_pad(referral->entry.subnqn, trid->subnqn, sizeof(trid->subnqn), '\0');
103 0 : spdk_strcpy_pad(referral->entry.trsvcid, trid->trsvcid, sizeof(referral->entry.trsvcid), ' ');
104 0 : spdk_strcpy_pad(referral->entry.traddr, trid->traddr, sizeof(referral->entry.traddr), ' ');
105 :
106 0 : TAILQ_INSERT_HEAD(&tgt->referrals, referral, link);
107 0 : spdk_nvmf_send_discovery_log_notice(tgt, NULL);
108 :
109 0 : return 0;
110 0 : }
111 :
112 : int
113 0 : spdk_nvmf_tgt_remove_referral(struct spdk_nvmf_tgt *tgt,
114 : const struct spdk_nvmf_referral_opts *uopts)
115 : {
116 : struct spdk_nvmf_referral *referral;
117 0 : struct spdk_nvmf_referral_opts opts = {};
118 0 : struct spdk_nvme_transport_id *trid = &opts.trid;
119 :
120 0 : memcpy(&opts, uopts, spdk_min(uopts->size, sizeof(opts)));
121 0 : if (trid->subnqn[0] == '\0') {
122 0 : snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
123 0 : }
124 :
125 0 : referral = nvmf_tgt_find_referral(tgt, &opts.trid);
126 0 : if (referral == NULL) {
127 0 : return -ENOENT;
128 : }
129 :
130 0 : TAILQ_REMOVE(&tgt->referrals, referral, link);
131 0 : spdk_nvmf_send_discovery_log_notice(tgt, NULL);
132 :
133 0 : free(referral);
134 :
135 0 : return 0;
136 0 : }
137 :
138 : void
139 0 : nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair,
140 : enum spdk_nvmf_qpair_state state)
141 : {
142 0 : assert(qpair != NULL);
143 0 : assert(qpair->group->thread == spdk_get_thread());
144 :
145 0 : qpair->state = state;
146 0 : }
147 :
148 : static int
149 0 : nvmf_poll_group_poll(void *ctx)
150 : {
151 0 : struct spdk_nvmf_poll_group *group = ctx;
152 : int rc;
153 0 : int count = 0;
154 : struct spdk_nvmf_transport_poll_group *tgroup;
155 :
156 0 : TAILQ_FOREACH(tgroup, &group->tgroups, link) {
157 0 : rc = nvmf_transport_poll_group_poll(tgroup);
158 0 : if (rc < 0) {
159 0 : return SPDK_POLLER_BUSY;
160 : }
161 0 : count += rc;
162 0 : }
163 :
164 0 : return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;
165 0 : }
166 :
167 : /*
168 : * Reset and clean up the poll group (I/O channel code will actually free the
169 : * group).
170 : */
171 : static void
172 1 : nvmf_tgt_cleanup_poll_group(struct spdk_nvmf_poll_group *group)
173 : {
174 : struct spdk_nvmf_transport_poll_group *tgroup, *tmp;
175 : struct spdk_nvmf_subsystem_poll_group *sgroup;
176 : uint32_t sid, nsid;
177 :
178 2 : TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) {
179 1 : TAILQ_REMOVE(&group->tgroups, tgroup, link);
180 1 : nvmf_transport_poll_group_destroy(tgroup);
181 1 : }
182 :
183 2 : for (sid = 0; sid < group->num_sgroups; sid++) {
184 1 : sgroup = &group->sgroups[sid];
185 :
186 1 : assert(sgroup != NULL);
187 :
188 2 : for (nsid = 0; nsid < sgroup->num_ns; nsid++) {
189 1 : if (sgroup->ns_info[nsid].channel) {
190 1 : spdk_put_io_channel(sgroup->ns_info[nsid].channel);
191 1 : sgroup->ns_info[nsid].channel = NULL;
192 1 : }
193 1 : }
194 :
195 1 : free(sgroup->ns_info);
196 1 : }
197 :
198 1 : free(group->sgroups);
199 :
200 1 : spdk_poller_unregister(&group->poller);
201 :
202 1 : if (group->destroy_cb_fn) {
203 0 : group->destroy_cb_fn(group->destroy_cb_arg, 0);
204 0 : }
205 1 : }
206 :
207 : /*
208 : * Callback to unregister a poll group from the target, and clean up its state.
209 : */
210 : static void
211 1 : nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf)
212 : {
213 1 : struct spdk_nvmf_tgt *tgt = io_device;
214 1 : struct spdk_nvmf_poll_group *group = ctx_buf;
215 :
216 : SPDK_DTRACE_PROBE1_TICKS(nvmf_destroy_poll_group, spdk_thread_get_id(group->thread));
217 :
218 1 : pthread_mutex_lock(&tgt->mutex);
219 1 : TAILQ_REMOVE(&tgt->poll_groups, group, link);
220 1 : tgt->num_poll_groups--;
221 1 : pthread_mutex_unlock(&tgt->mutex);
222 :
223 1 : assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING));
224 1 : nvmf_tgt_cleanup_poll_group(group);
225 1 : }
226 :
227 : static int
228 1 : nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group,
229 : struct spdk_nvmf_transport *transport)
230 : {
231 1 : struct spdk_nvmf_transport_poll_group *tgroup = nvmf_get_transport_poll_group(group, transport);
232 :
233 1 : if (tgroup != NULL) {
234 : /* Transport already in the poll group */
235 0 : return 0;
236 : }
237 :
238 1 : tgroup = nvmf_transport_poll_group_create(transport, group);
239 1 : if (!tgroup) {
240 0 : SPDK_ERRLOG("Unable to create poll group for transport\n");
241 0 : return -1;
242 : }
243 : SPDK_DTRACE_PROBE2_TICKS(nvmf_transport_poll_group_create, transport,
244 : spdk_thread_get_id(group->thread));
245 :
246 1 : tgroup->group = group;
247 1 : TAILQ_INSERT_TAIL(&group->tgroups, tgroup, link);
248 :
249 1 : return 0;
250 1 : }
251 :
252 : static int
253 1 : nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf)
254 : {
255 1 : struct spdk_nvmf_tgt *tgt = io_device;
256 1 : struct spdk_nvmf_poll_group *group = ctx_buf;
257 : struct spdk_nvmf_transport *transport;
258 : struct spdk_nvmf_subsystem *subsystem;
259 1 : struct spdk_thread *thread = spdk_get_thread();
260 : uint32_t i;
261 : int rc;
262 :
263 1 : group->tgt = tgt;
264 1 : TAILQ_INIT(&group->tgroups);
265 1 : TAILQ_INIT(&group->qpairs);
266 1 : group->thread = thread;
267 1 : pthread_mutex_init(&group->mutex, NULL);
268 :
269 1 : group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0);
270 1 : spdk_poller_register_interrupt(group->poller, NULL, NULL);
271 :
272 : SPDK_DTRACE_PROBE1_TICKS(nvmf_create_poll_group, spdk_thread_get_id(thread));
273 :
274 2 : TAILQ_FOREACH(transport, &tgt->transports, link) {
275 1 : rc = nvmf_poll_group_add_transport(group, transport);
276 1 : if (rc != 0) {
277 0 : nvmf_tgt_cleanup_poll_group(group);
278 0 : return rc;
279 : }
280 1 : }
281 :
282 1 : group->num_sgroups = tgt->max_subsystems;
283 1 : group->sgroups = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group));
284 1 : if (!group->sgroups) {
285 0 : nvmf_tgt_cleanup_poll_group(group);
286 0 : return -ENOMEM;
287 : }
288 :
289 2 : for (i = 0; i < tgt->max_subsystems; i++) {
290 1 : TAILQ_INIT(&group->sgroups[i].queued);
291 1 : }
292 :
293 2 : for (subsystem = spdk_nvmf_subsystem_get_first(tgt);
294 2 : subsystem != NULL;
295 1 : subsystem = spdk_nvmf_subsystem_get_next(subsystem)) {
296 1 : if (nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) {
297 0 : nvmf_tgt_cleanup_poll_group(group);
298 0 : return -1;
299 : }
300 1 : }
301 :
302 1 : pthread_mutex_lock(&tgt->mutex);
303 1 : tgt->num_poll_groups++;
304 1 : TAILQ_INSERT_TAIL(&tgt->poll_groups, group, link);
305 1 : pthread_mutex_unlock(&tgt->mutex);
306 :
307 1 : return 0;
308 1 : }
309 :
310 : static void
311 0 : _nvmf_tgt_disconnect_qpairs(void *ctx)
312 : {
313 : struct spdk_nvmf_qpair *qpair, *qpair_tmp;
314 0 : struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx;
315 0 : struct spdk_nvmf_poll_group *group = qpair_ctx->group;
316 : struct spdk_io_channel *ch;
317 : int rc;
318 :
319 0 : TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) {
320 0 : rc = spdk_nvmf_qpair_disconnect(qpair);
321 0 : if (rc && rc != -EINPROGRESS) {
322 0 : break;
323 : }
324 0 : }
325 :
326 0 : if (TAILQ_EMPTY(&group->qpairs)) {
327 : /* When the refcount from the channels reaches 0, nvmf_tgt_destroy_poll_group will be called. */
328 0 : ch = spdk_io_channel_from_ctx(group);
329 0 : spdk_put_io_channel(ch);
330 0 : free(qpair_ctx);
331 0 : return;
332 : }
333 :
334 : /* Some qpairs are in process of being disconnected. Send a message and try to remove them again */
335 0 : spdk_thread_send_msg(spdk_get_thread(), _nvmf_tgt_disconnect_qpairs, ctx);
336 0 : }
337 :
338 : static void
339 0 : nvmf_tgt_destroy_poll_group_qpairs(struct spdk_nvmf_poll_group *group)
340 : {
341 : struct nvmf_qpair_disconnect_many_ctx *ctx;
342 :
343 : SPDK_DTRACE_PROBE1_TICKS(nvmf_destroy_poll_group_qpairs, spdk_thread_get_id(group->thread));
344 :
345 0 : ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx));
346 0 : if (!ctx) {
347 0 : SPDK_ERRLOG("Failed to allocate memory for destroy poll group ctx\n");
348 0 : return;
349 : }
350 :
351 0 : ctx->group = group;
352 0 : _nvmf_tgt_disconnect_qpairs(ctx);
353 0 : }
354 :
355 : struct spdk_nvmf_tgt *
356 0 : spdk_nvmf_tgt_create(struct spdk_nvmf_target_opts *_opts)
357 : {
358 : struct spdk_nvmf_tgt *tgt, *tmp_tgt;
359 0 : struct spdk_nvmf_target_opts opts = {
360 : .max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS,
361 : .discovery_filter = SPDK_NVMF_TGT_DISCOVERY_MATCH_ANY,
362 : };
363 :
364 0 : memcpy(&opts, _opts, _opts->size);
365 0 : if (strnlen(opts.name, NVMF_TGT_NAME_MAX_LENGTH) == NVMF_TGT_NAME_MAX_LENGTH) {
366 0 : SPDK_ERRLOG("Provided target name exceeds the max length of %u.\n", NVMF_TGT_NAME_MAX_LENGTH);
367 0 : return NULL;
368 : }
369 :
370 0 : TAILQ_FOREACH(tmp_tgt, &g_nvmf_tgts, link) {
371 0 : if (!strncmp(opts.name, tmp_tgt->name, NVMF_TGT_NAME_MAX_LENGTH)) {
372 0 : SPDK_ERRLOG("Provided target name must be unique.\n");
373 0 : return NULL;
374 : }
375 0 : }
376 :
377 0 : tgt = calloc(1, sizeof(*tgt));
378 0 : if (!tgt) {
379 0 : return NULL;
380 : }
381 :
382 0 : snprintf(tgt->name, NVMF_TGT_NAME_MAX_LENGTH, "%s", opts.name);
383 :
384 0 : if (!opts.max_subsystems) {
385 0 : tgt->max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS;
386 0 : } else {
387 0 : tgt->max_subsystems = opts.max_subsystems;
388 : }
389 :
390 0 : tgt->crdt[0] = opts.crdt[0];
391 0 : tgt->crdt[1] = opts.crdt[1];
392 0 : tgt->crdt[2] = opts.crdt[2];
393 0 : tgt->discovery_filter = opts.discovery_filter;
394 0 : tgt->discovery_genctr = 0;
395 0 : tgt->dhchap_digests = opts.dhchap_digests;
396 0 : tgt->dhchap_dhgroups = opts.dhchap_dhgroups;
397 0 : TAILQ_INIT(&tgt->transports);
398 0 : TAILQ_INIT(&tgt->poll_groups);
399 0 : TAILQ_INIT(&tgt->referrals);
400 0 : tgt->num_poll_groups = 0;
401 :
402 0 : tgt->subsystem_ids = spdk_bit_array_create(tgt->max_subsystems);
403 0 : if (tgt->subsystem_ids == NULL) {
404 0 : free(tgt);
405 0 : return NULL;
406 : }
407 :
408 0 : RB_INIT(&tgt->subsystems);
409 :
410 0 : pthread_mutex_init(&tgt->mutex, NULL);
411 :
412 0 : spdk_io_device_register(tgt,
413 : nvmf_tgt_create_poll_group,
414 : nvmf_tgt_destroy_poll_group,
415 : sizeof(struct spdk_nvmf_poll_group),
416 0 : tgt->name);
417 :
418 0 : tgt->state = NVMF_TGT_RUNNING;
419 :
420 0 : TAILQ_INSERT_HEAD(&g_nvmf_tgts, tgt, link);
421 :
422 0 : return tgt;
423 0 : }
424 :
425 : static void
426 0 : _nvmf_tgt_destroy_next_transport(void *ctx)
427 : {
428 0 : struct spdk_nvmf_tgt *tgt = ctx;
429 : struct spdk_nvmf_transport *transport;
430 :
431 0 : if (!TAILQ_EMPTY(&tgt->transports)) {
432 0 : transport = TAILQ_FIRST(&tgt->transports);
433 0 : TAILQ_REMOVE(&tgt->transports, transport, link);
434 0 : spdk_nvmf_transport_destroy(transport, _nvmf_tgt_destroy_next_transport, tgt);
435 0 : } else {
436 0 : spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn = tgt->destroy_cb_fn;
437 0 : void *destroy_cb_arg = tgt->destroy_cb_arg;
438 :
439 0 : pthread_mutex_destroy(&tgt->mutex);
440 0 : free(tgt);
441 :
442 0 : if (destroy_cb_fn) {
443 0 : destroy_cb_fn(destroy_cb_arg, 0);
444 0 : }
445 : }
446 0 : }
447 :
448 : static void
449 0 : nvmf_tgt_destroy_cb(void *io_device)
450 : {
451 0 : struct spdk_nvmf_tgt *tgt = io_device;
452 : struct spdk_nvmf_subsystem *subsystem, *subsystem_next;
453 : int rc;
454 : struct spdk_nvmf_referral *referral;
455 :
456 0 : while ((referral = TAILQ_FIRST(&tgt->referrals))) {
457 0 : TAILQ_REMOVE(&tgt->referrals, referral, link);
458 0 : free(referral);
459 : }
460 :
461 0 : nvmf_tgt_stop_mdns_prr(tgt);
462 :
463 : /* We will be freeing subsystems in this loop, so we always need to get the next one
464 : * ahead of time, since we can't call get_next() on a subsystem that's been freed.
465 : */
466 0 : for (subsystem = spdk_nvmf_subsystem_get_first(tgt),
467 0 : subsystem_next = spdk_nvmf_subsystem_get_next(subsystem);
468 0 : subsystem != NULL;
469 0 : subsystem = subsystem_next,
470 0 : subsystem_next = spdk_nvmf_subsystem_get_next(subsystem_next)) {
471 0 : nvmf_subsystem_remove_all_listeners(subsystem, true);
472 :
473 0 : rc = spdk_nvmf_subsystem_destroy(subsystem, nvmf_tgt_destroy_cb, tgt);
474 0 : if (rc) {
475 0 : if (rc == -EINPROGRESS) {
476 : /* If rc is -EINPROGRESS, nvmf_tgt_destroy_cb will be called again when subsystem #i
477 : * is destroyed, nvmf_tgt_destroy_cb will continue to destroy other subsystems if any */
478 0 : return;
479 : } else {
480 0 : SPDK_ERRLOG("Failed to destroy subsystem %s, rc %d\n", subsystem->subnqn, rc);
481 : }
482 0 : }
483 0 : }
484 0 : spdk_bit_array_free(&tgt->subsystem_ids);
485 0 : _nvmf_tgt_destroy_next_transport(tgt);
486 0 : }
487 :
488 : void
489 0 : spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt,
490 : spdk_nvmf_tgt_destroy_done_fn cb_fn,
491 : void *cb_arg)
492 : {
493 0 : assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING));
494 :
495 0 : tgt->destroy_cb_fn = cb_fn;
496 0 : tgt->destroy_cb_arg = cb_arg;
497 :
498 0 : TAILQ_REMOVE(&g_nvmf_tgts, tgt, link);
499 :
500 0 : spdk_io_device_unregister(tgt, nvmf_tgt_destroy_cb);
501 0 : }
502 :
503 : const char *
504 0 : spdk_nvmf_tgt_get_name(struct spdk_nvmf_tgt *tgt)
505 : {
506 0 : return tgt->name;
507 : }
508 :
509 : struct spdk_nvmf_tgt *
510 0 : spdk_nvmf_get_tgt(const char *name)
511 : {
512 : struct spdk_nvmf_tgt *tgt;
513 0 : uint32_t num_targets = 0;
514 :
515 0 : TAILQ_FOREACH(tgt, &g_nvmf_tgts, link) {
516 0 : if (name) {
517 0 : if (!strncmp(tgt->name, name, NVMF_TGT_NAME_MAX_LENGTH)) {
518 0 : return tgt;
519 : }
520 0 : }
521 0 : num_targets++;
522 0 : }
523 :
524 : /*
525 : * special case. If there is only one target and
526 : * no name was specified, return the only available
527 : * target. If there is more than one target, name must
528 : * be specified.
529 : */
530 0 : if (!name && num_targets == 1) {
531 0 : return TAILQ_FIRST(&g_nvmf_tgts);
532 : }
533 :
534 0 : return NULL;
535 0 : }
536 :
537 : struct spdk_nvmf_tgt *
538 0 : spdk_nvmf_get_first_tgt(void)
539 : {
540 0 : return TAILQ_FIRST(&g_nvmf_tgts);
541 : }
542 :
543 : struct spdk_nvmf_tgt *
544 0 : spdk_nvmf_get_next_tgt(struct spdk_nvmf_tgt *prev)
545 : {
546 0 : return TAILQ_NEXT(prev, link);
547 : }
548 :
549 : static void
550 0 : nvmf_write_nvme_subsystem_config(struct spdk_json_write_ctx *w,
551 : struct spdk_nvmf_subsystem *subsystem)
552 : {
553 : struct spdk_nvmf_host *host;
554 : struct spdk_nvmf_ns *ns;
555 : struct spdk_nvmf_ns_opts ns_opts;
556 : uint32_t max_namespaces;
557 : struct spdk_nvmf_transport *transport;
558 :
559 0 : assert(spdk_nvmf_subsystem_get_type(subsystem) == SPDK_NVMF_SUBTYPE_NVME);
560 :
561 : /* { */
562 0 : spdk_json_write_object_begin(w);
563 0 : spdk_json_write_named_string(w, "method", "nvmf_create_subsystem");
564 :
565 : /* "params" : { */
566 0 : spdk_json_write_named_object_begin(w, "params");
567 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
568 0 : spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_subsystem_get_allow_any_host(subsystem));
569 0 : spdk_json_write_named_string(w, "serial_number", spdk_nvmf_subsystem_get_sn(subsystem));
570 0 : spdk_json_write_named_string(w, "model_number", spdk_nvmf_subsystem_get_mn(subsystem));
571 :
572 0 : max_namespaces = spdk_nvmf_subsystem_get_max_namespaces(subsystem);
573 0 : if (max_namespaces != 0) {
574 0 : spdk_json_write_named_uint32(w, "max_namespaces", max_namespaces);
575 0 : }
576 :
577 0 : spdk_json_write_named_uint32(w, "min_cntlid", spdk_nvmf_subsystem_get_min_cntlid(subsystem));
578 0 : spdk_json_write_named_uint32(w, "max_cntlid", spdk_nvmf_subsystem_get_max_cntlid(subsystem));
579 0 : spdk_json_write_named_bool(w, "ana_reporting", spdk_nvmf_subsystem_get_ana_reporting(subsystem));
580 :
581 : /* } "params" */
582 0 : spdk_json_write_object_end(w);
583 :
584 : /* } */
585 0 : spdk_json_write_object_end(w);
586 :
587 0 : for (host = spdk_nvmf_subsystem_get_first_host(subsystem); host != NULL;
588 0 : host = spdk_nvmf_subsystem_get_next_host(subsystem, host)) {
589 :
590 0 : spdk_json_write_object_begin(w);
591 0 : spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_host");
592 :
593 : /* "params" : { */
594 0 : spdk_json_write_named_object_begin(w, "params");
595 :
596 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
597 0 : spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host));
598 0 : if (host->dhchap_key != NULL) {
599 0 : spdk_json_write_named_string(w, "dhchap_key",
600 0 : spdk_key_get_name(host->dhchap_key));
601 0 : }
602 0 : if (host->dhchap_ctrlr_key != NULL) {
603 0 : spdk_json_write_named_string(w, "dhchap_ctrlr_key",
604 0 : spdk_key_get_name(host->dhchap_ctrlr_key));
605 0 : }
606 0 : TAILQ_FOREACH(transport, &subsystem->tgt->transports, link) {
607 0 : if (transport->ops->subsystem_dump_host != NULL) {
608 0 : transport->ops->subsystem_dump_host(transport, subsystem, host->nqn, w);
609 0 : }
610 0 : }
611 :
612 : /* } "params" */
613 0 : spdk_json_write_object_end(w);
614 :
615 : /* } */
616 0 : spdk_json_write_object_end(w);
617 0 : }
618 :
619 0 : for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL;
620 0 : ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) {
621 0 : spdk_nvmf_ns_get_opts(ns, &ns_opts, sizeof(ns_opts));
622 :
623 0 : spdk_json_write_object_begin(w);
624 0 : spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_ns");
625 :
626 : /* "params" : { */
627 0 : spdk_json_write_named_object_begin(w, "params");
628 :
629 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
630 :
631 : /* "namespace" : { */
632 0 : spdk_json_write_named_object_begin(w, "namespace");
633 :
634 0 : spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns));
635 0 : spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns)));
636 :
637 0 : if (ns->ptpl_file != NULL) {
638 0 : spdk_json_write_named_string(w, "ptpl_file", ns->ptpl_file);
639 0 : }
640 :
641 0 : if (!spdk_mem_all_zero(ns_opts.nguid, sizeof(ns_opts.nguid))) {
642 : SPDK_STATIC_ASSERT(sizeof(ns_opts.nguid) == sizeof(uint64_t) * 2, "size mismatch");
643 0 : spdk_json_write_named_string_fmt(w, "nguid", "%016"PRIX64"%016"PRIX64, from_be64(&ns_opts.nguid[0]),
644 0 : from_be64(&ns_opts.nguid[8]));
645 0 : }
646 :
647 0 : if (!spdk_mem_all_zero(ns_opts.eui64, sizeof(ns_opts.eui64))) {
648 : SPDK_STATIC_ASSERT(sizeof(ns_opts.eui64) == sizeof(uint64_t), "size mismatch");
649 0 : spdk_json_write_named_string_fmt(w, "eui64", "%016"PRIX64, from_be64(&ns_opts.eui64));
650 0 : }
651 :
652 0 : if (!spdk_uuid_is_null(&ns_opts.uuid)) {
653 0 : spdk_json_write_named_uuid(w, "uuid", &ns_opts.uuid);
654 0 : }
655 :
656 0 : if (spdk_nvmf_subsystem_get_ana_reporting(subsystem)) {
657 0 : spdk_json_write_named_uint32(w, "anagrpid", ns_opts.anagrpid);
658 0 : }
659 :
660 0 : spdk_json_write_named_bool(w, "no_auto_visible", !ns->always_visible);
661 :
662 : /* "namespace" */
663 0 : spdk_json_write_object_end(w);
664 :
665 : /* } "params" */
666 0 : spdk_json_write_object_end(w);
667 :
668 : /* } */
669 0 : spdk_json_write_object_end(w);
670 :
671 0 : TAILQ_FOREACH(host, &ns->hosts, link) {
672 0 : spdk_json_write_object_begin(w);
673 0 : spdk_json_write_named_string(w, "method", "nvmf_ns_add_host");
674 0 : spdk_json_write_named_object_begin(w, "params");
675 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
676 0 : spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns));
677 0 : spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host));
678 0 : spdk_json_write_object_end(w);
679 0 : spdk_json_write_object_end(w);
680 0 : }
681 0 : }
682 0 : }
683 :
684 : static void
685 0 : nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w,
686 : struct spdk_nvmf_subsystem *subsystem)
687 : {
688 : struct spdk_nvmf_subsystem_listener *listener;
689 : struct spdk_nvmf_transport *transport;
690 : const struct spdk_nvme_transport_id *trid;
691 :
692 0 : if (spdk_nvmf_subsystem_get_type(subsystem) == SPDK_NVMF_SUBTYPE_NVME) {
693 0 : nvmf_write_nvme_subsystem_config(w, subsystem);
694 0 : }
695 :
696 0 : for (listener = spdk_nvmf_subsystem_get_first_listener(subsystem); listener != NULL;
697 0 : listener = spdk_nvmf_subsystem_get_next_listener(subsystem, listener)) {
698 0 : transport = listener->transport;
699 0 : trid = spdk_nvmf_subsystem_listener_get_trid(listener);
700 :
701 0 : spdk_json_write_object_begin(w);
702 0 : spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_listener");
703 :
704 : /* "params" : { */
705 0 : spdk_json_write_named_object_begin(w, "params");
706 :
707 0 : spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem));
708 :
709 0 : spdk_json_write_named_object_begin(w, "listen_address");
710 0 : nvmf_transport_listen_dump_trid(trid, w);
711 0 : spdk_json_write_object_end(w);
712 0 : if (transport->ops->listen_dump_opts) {
713 0 : transport->ops->listen_dump_opts(transport, trid, w);
714 0 : }
715 :
716 0 : spdk_json_write_named_bool(w, "secure_channel", listener->opts.secure_channel);
717 :
718 0 : if (listener->opts.sock_impl) {
719 0 : spdk_json_write_named_string(w, "sock_impl", listener->opts.sock_impl);
720 0 : }
721 :
722 : /* } "params" */
723 0 : spdk_json_write_object_end(w);
724 :
725 : /* } */
726 0 : spdk_json_write_object_end(w);
727 0 : }
728 0 : }
729 :
730 : void
731 0 : spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt)
732 : {
733 : struct spdk_nvmf_subsystem *subsystem;
734 : struct spdk_nvmf_transport *transport;
735 : struct spdk_nvmf_referral *referral;
736 :
737 0 : spdk_json_write_object_begin(w);
738 0 : spdk_json_write_named_string(w, "method", "nvmf_set_max_subsystems");
739 :
740 0 : spdk_json_write_named_object_begin(w, "params");
741 0 : spdk_json_write_named_uint32(w, "max_subsystems", tgt->max_subsystems);
742 0 : spdk_json_write_object_end(w);
743 :
744 0 : spdk_json_write_object_end(w);
745 :
746 0 : spdk_json_write_object_begin(w);
747 0 : spdk_json_write_named_string(w, "method", "nvmf_set_crdt");
748 0 : spdk_json_write_named_object_begin(w, "params");
749 0 : spdk_json_write_named_uint32(w, "crdt1", tgt->crdt[0]);
750 0 : spdk_json_write_named_uint32(w, "crdt2", tgt->crdt[1]);
751 0 : spdk_json_write_named_uint32(w, "crdt3", tgt->crdt[2]);
752 0 : spdk_json_write_object_end(w);
753 0 : spdk_json_write_object_end(w);
754 :
755 : /* write transports */
756 0 : TAILQ_FOREACH(transport, &tgt->transports, link) {
757 0 : spdk_json_write_object_begin(w);
758 0 : spdk_json_write_named_string(w, "method", "nvmf_create_transport");
759 0 : nvmf_transport_dump_opts(transport, w, true);
760 0 : spdk_json_write_object_end(w);
761 0 : }
762 :
763 0 : TAILQ_FOREACH(referral, &tgt->referrals, link) {
764 0 : spdk_json_write_object_begin(w);
765 0 : spdk_json_write_named_string(w, "method", "nvmf_discovery_add_referral");
766 :
767 0 : spdk_json_write_named_object_begin(w, "params");
768 0 : spdk_json_write_named_object_begin(w, "address");
769 0 : nvmf_transport_listen_dump_trid(&referral->trid, w);
770 0 : spdk_json_write_object_end(w);
771 0 : spdk_json_write_named_bool(w, "secure_channel",
772 0 : referral->entry.treq.secure_channel ==
773 : SPDK_NVMF_TREQ_SECURE_CHANNEL_REQUIRED);
774 0 : spdk_json_write_named_string(w, "subnqn", referral->trid.subnqn);
775 0 : spdk_json_write_object_end(w);
776 :
777 0 : spdk_json_write_object_end(w);
778 0 : }
779 :
780 0 : subsystem = spdk_nvmf_subsystem_get_first(tgt);
781 0 : while (subsystem) {
782 0 : nvmf_write_subsystem_config_json(w, subsystem);
783 0 : subsystem = spdk_nvmf_subsystem_get_next(subsystem);
784 : }
785 0 : }
786 :
787 : static void
788 0 : nvmf_listen_opts_copy(struct spdk_nvmf_listen_opts *opts,
789 : const struct spdk_nvmf_listen_opts *opts_src, size_t opts_size)
790 : {
791 0 : assert(opts);
792 0 : assert(opts_src);
793 :
794 0 : opts->opts_size = opts_size;
795 :
796 : #define SET_FIELD(field) \
797 : if (offsetof(struct spdk_nvmf_listen_opts, field) + sizeof(opts->field) <= opts_size) { \
798 : opts->field = opts_src->field; \
799 : } \
800 :
801 0 : SET_FIELD(transport_specific);
802 0 : SET_FIELD(secure_channel);
803 0 : SET_FIELD(ana_state);
804 0 : SET_FIELD(sock_impl);
805 : #undef SET_FIELD
806 :
807 : /* Do not remove this statement, you should always update this statement when you adding a new field,
808 : * and do not forget to add the SET_FIELD statement for your added field. */
809 : SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_listen_opts) == 32, "Incorrect size");
810 0 : }
811 :
812 : void
813 0 : spdk_nvmf_listen_opts_init(struct spdk_nvmf_listen_opts *opts, size_t opts_size)
814 : {
815 0 : struct spdk_nvmf_listen_opts opts_local = {};
816 :
817 : /* local version of opts should have defaults set here */
818 0 : opts_local.ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
819 0 : nvmf_listen_opts_copy(opts, &opts_local, opts_size);
820 0 : }
821 :
822 : int
823 0 : spdk_nvmf_tgt_listen_ext(struct spdk_nvmf_tgt *tgt, const struct spdk_nvme_transport_id *trid,
824 : struct spdk_nvmf_listen_opts *opts)
825 : {
826 : struct spdk_nvmf_transport *transport;
827 : int rc;
828 0 : struct spdk_nvmf_listen_opts opts_local = {};
829 :
830 0 : if (!opts) {
831 0 : SPDK_ERRLOG("opts should not be NULL\n");
832 0 : return -EINVAL;
833 : }
834 :
835 0 : if (!opts->opts_size) {
836 0 : SPDK_ERRLOG("The opts_size in opts structure should not be zero\n");
837 0 : return -EINVAL;
838 : }
839 :
840 0 : transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring);
841 0 : if (!transport) {
842 0 : SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n",
843 : trid->trstring);
844 0 : return -EINVAL;
845 : }
846 :
847 0 : nvmf_listen_opts_copy(&opts_local, opts, opts->opts_size);
848 0 : rc = spdk_nvmf_transport_listen(transport, trid, &opts_local);
849 0 : if (rc < 0) {
850 0 : SPDK_ERRLOG("Unable to listen on address '%s'\n", trid->traddr);
851 0 : }
852 :
853 0 : return rc;
854 0 : }
855 :
856 : int
857 0 : spdk_nvmf_tgt_stop_listen(struct spdk_nvmf_tgt *tgt,
858 : struct spdk_nvme_transport_id *trid)
859 : {
860 : struct spdk_nvmf_transport *transport;
861 : int rc;
862 :
863 0 : transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring);
864 0 : if (!transport) {
865 0 : SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n",
866 : trid->trstring);
867 0 : return -EINVAL;
868 : }
869 :
870 0 : rc = spdk_nvmf_transport_stop_listen(transport, trid);
871 0 : if (rc < 0) {
872 0 : SPDK_ERRLOG("Failed to stop listening on address '%s'\n", trid->traddr);
873 0 : return rc;
874 : }
875 0 : return 0;
876 0 : }
877 :
878 : struct spdk_nvmf_tgt_add_transport_ctx {
879 : struct spdk_nvmf_tgt *tgt;
880 : struct spdk_nvmf_transport *transport;
881 : spdk_nvmf_tgt_add_transport_done_fn cb_fn;
882 : void *cb_arg;
883 : int status;
884 : };
885 :
886 : static void
887 0 : _nvmf_tgt_remove_transport_done(struct spdk_io_channel_iter *i, int status)
888 : {
889 0 : struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
890 :
891 0 : ctx->cb_fn(ctx->cb_arg, ctx->status);
892 0 : free(ctx);
893 0 : }
894 :
895 : static void
896 0 : _nvmf_tgt_remove_transport(struct spdk_io_channel_iter *i)
897 : {
898 0 : struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
899 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
900 0 : struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch);
901 : struct spdk_nvmf_transport_poll_group *tgroup, *tmp;
902 :
903 0 : TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) {
904 0 : if (tgroup->transport == ctx->transport) {
905 0 : TAILQ_REMOVE(&group->tgroups, tgroup, link);
906 0 : nvmf_transport_poll_group_destroy(tgroup);
907 0 : }
908 0 : }
909 :
910 0 : spdk_for_each_channel_continue(i, 0);
911 0 : }
912 :
913 : static void
914 0 : _nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status)
915 : {
916 0 : struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
917 :
918 0 : if (status) {
919 0 : ctx->status = status;
920 0 : spdk_for_each_channel(ctx->tgt,
921 : _nvmf_tgt_remove_transport,
922 0 : ctx,
923 : _nvmf_tgt_remove_transport_done);
924 0 : return;
925 : }
926 :
927 0 : ctx->transport->tgt = ctx->tgt;
928 0 : TAILQ_INSERT_TAIL(&ctx->tgt->transports, ctx->transport, link);
929 0 : ctx->cb_fn(ctx->cb_arg, status);
930 0 : free(ctx);
931 0 : }
932 :
933 : static void
934 0 : _nvmf_tgt_add_transport(struct spdk_io_channel_iter *i)
935 : {
936 0 : struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
937 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
938 0 : struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch);
939 : int rc;
940 :
941 0 : rc = nvmf_poll_group_add_transport(group, ctx->transport);
942 0 : spdk_for_each_channel_continue(i, rc);
943 0 : }
944 :
945 : void
946 0 : spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt,
947 : struct spdk_nvmf_transport *transport,
948 : spdk_nvmf_tgt_add_transport_done_fn cb_fn,
949 : void *cb_arg)
950 : {
951 : struct spdk_nvmf_tgt_add_transport_ctx *ctx;
952 :
953 : SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_add_transport, transport, tgt->name);
954 :
955 0 : if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->name)) {
956 0 : cb_fn(cb_arg, -EEXIST);
957 0 : return; /* transport already created */
958 : }
959 :
960 0 : ctx = calloc(1, sizeof(*ctx));
961 0 : if (!ctx) {
962 0 : cb_fn(cb_arg, -ENOMEM);
963 0 : return;
964 : }
965 :
966 0 : ctx->tgt = tgt;
967 0 : ctx->transport = transport;
968 0 : ctx->cb_fn = cb_fn;
969 0 : ctx->cb_arg = cb_arg;
970 :
971 0 : spdk_for_each_channel(tgt,
972 : _nvmf_tgt_add_transport,
973 0 : ctx,
974 : _nvmf_tgt_add_transport_done);
975 0 : }
976 :
977 : struct nvmf_tgt_pause_ctx {
978 : struct spdk_nvmf_tgt *tgt;
979 : spdk_nvmf_tgt_pause_polling_cb_fn cb_fn;
980 : void *cb_arg;
981 : };
982 :
983 : static void
984 0 : _nvmf_tgt_pause_polling_done(struct spdk_io_channel_iter *i, int status)
985 : {
986 0 : struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
987 :
988 0 : ctx->tgt->state = NVMF_TGT_PAUSED;
989 :
990 0 : ctx->cb_fn(ctx->cb_arg, status);
991 0 : free(ctx);
992 0 : }
993 :
994 : static void
995 0 : _nvmf_tgt_pause_polling(struct spdk_io_channel_iter *i)
996 : {
997 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
998 0 : struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch);
999 :
1000 0 : spdk_poller_unregister(&group->poller);
1001 :
1002 0 : spdk_for_each_channel_continue(i, 0);
1003 0 : }
1004 :
1005 : int
1006 0 : spdk_nvmf_tgt_pause_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_pause_polling_cb_fn cb_fn,
1007 : void *cb_arg)
1008 : {
1009 : struct nvmf_tgt_pause_ctx *ctx;
1010 :
1011 : SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_pause_polling, tgt, tgt->name);
1012 :
1013 0 : switch (tgt->state) {
1014 : case NVMF_TGT_PAUSING:
1015 : case NVMF_TGT_RESUMING:
1016 0 : return -EBUSY;
1017 : case NVMF_TGT_RUNNING:
1018 0 : break;
1019 : default:
1020 0 : return -EINVAL;
1021 : }
1022 :
1023 0 : ctx = calloc(1, sizeof(*ctx));
1024 0 : if (!ctx) {
1025 0 : return -ENOMEM;
1026 : }
1027 :
1028 :
1029 0 : tgt->state = NVMF_TGT_PAUSING;
1030 :
1031 0 : ctx->tgt = tgt;
1032 0 : ctx->cb_fn = cb_fn;
1033 0 : ctx->cb_arg = cb_arg;
1034 :
1035 0 : spdk_for_each_channel(tgt,
1036 : _nvmf_tgt_pause_polling,
1037 0 : ctx,
1038 : _nvmf_tgt_pause_polling_done);
1039 0 : return 0;
1040 0 : }
1041 :
1042 : static void
1043 0 : _nvmf_tgt_resume_polling_done(struct spdk_io_channel_iter *i, int status)
1044 : {
1045 0 : struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
1046 :
1047 0 : ctx->tgt->state = NVMF_TGT_RUNNING;
1048 :
1049 0 : ctx->cb_fn(ctx->cb_arg, status);
1050 0 : free(ctx);
1051 0 : }
1052 :
1053 : static void
1054 0 : _nvmf_tgt_resume_polling(struct spdk_io_channel_iter *i)
1055 : {
1056 0 : struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
1057 0 : struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch);
1058 :
1059 0 : assert(group->poller == NULL);
1060 0 : group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0);
1061 :
1062 0 : spdk_for_each_channel_continue(i, 0);
1063 0 : }
1064 :
1065 : int
1066 0 : spdk_nvmf_tgt_resume_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_resume_polling_cb_fn cb_fn,
1067 : void *cb_arg)
1068 : {
1069 : struct nvmf_tgt_pause_ctx *ctx;
1070 :
1071 : SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_resume_polling, tgt, tgt->name);
1072 :
1073 0 : switch (tgt->state) {
1074 : case NVMF_TGT_PAUSING:
1075 : case NVMF_TGT_RESUMING:
1076 0 : return -EBUSY;
1077 : case NVMF_TGT_PAUSED:
1078 0 : break;
1079 : default:
1080 0 : return -EINVAL;
1081 : }
1082 :
1083 0 : ctx = calloc(1, sizeof(*ctx));
1084 0 : if (!ctx) {
1085 0 : return -ENOMEM;
1086 : }
1087 :
1088 0 : tgt->state = NVMF_TGT_RESUMING;
1089 :
1090 0 : ctx->tgt = tgt;
1091 0 : ctx->cb_fn = cb_fn;
1092 0 : ctx->cb_arg = cb_arg;
1093 :
1094 0 : spdk_for_each_channel(tgt,
1095 : _nvmf_tgt_resume_polling,
1096 0 : ctx,
1097 : _nvmf_tgt_resume_polling_done);
1098 0 : return 0;
1099 0 : }
1100 :
1101 : struct spdk_nvmf_subsystem *
1102 0 : spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn)
1103 : {
1104 : struct spdk_nvmf_subsystem subsystem;
1105 :
1106 0 : if (!subnqn) {
1107 0 : return NULL;
1108 : }
1109 :
1110 : /* Ensure that subnqn is null terminated */
1111 0 : if (!memchr(subnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) {
1112 0 : SPDK_ERRLOG("Connect SUBNQN is not null terminated\n");
1113 0 : return NULL;
1114 : }
1115 :
1116 0 : snprintf(subsystem.subnqn, sizeof(subsystem.subnqn), "%s", subnqn);
1117 0 : return RB_FIND(subsystem_tree, &tgt->subsystems, &subsystem);
1118 0 : }
1119 :
1120 : struct spdk_nvmf_transport *
1121 0 : spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, const char *transport_name)
1122 : {
1123 : struct spdk_nvmf_transport *transport;
1124 :
1125 0 : TAILQ_FOREACH(transport, &tgt->transports, link) {
1126 0 : if (!strncasecmp(transport->ops->name, transport_name, SPDK_NVMF_TRSTRING_MAX_LEN)) {
1127 0 : return transport;
1128 : }
1129 0 : }
1130 0 : return NULL;
1131 0 : }
1132 :
1133 : struct nvmf_new_qpair_ctx {
1134 : struct spdk_nvmf_qpair *qpair;
1135 : struct spdk_nvmf_poll_group *group;
1136 : };
1137 :
1138 : static void
1139 0 : _nvmf_poll_group_add(void *_ctx)
1140 : {
1141 0 : struct nvmf_new_qpair_ctx *ctx = _ctx;
1142 0 : struct spdk_nvmf_qpair *qpair = ctx->qpair;
1143 0 : struct spdk_nvmf_poll_group *group = ctx->group;
1144 :
1145 0 : free(_ctx);
1146 :
1147 0 : if (spdk_nvmf_poll_group_add(group, qpair) != 0) {
1148 0 : SPDK_ERRLOG("Unable to add the qpair to a poll group.\n");
1149 0 : spdk_nvmf_qpair_disconnect(qpair);
1150 0 : }
1151 0 : }
1152 :
1153 : void
1154 0 : spdk_nvmf_tgt_new_qpair(struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair)
1155 : {
1156 : struct spdk_nvmf_poll_group *group;
1157 : struct nvmf_new_qpair_ctx *ctx;
1158 :
1159 0 : group = spdk_nvmf_get_optimal_poll_group(qpair);
1160 0 : if (group == NULL) {
1161 0 : if (tgt->next_poll_group == NULL) {
1162 0 : tgt->next_poll_group = TAILQ_FIRST(&tgt->poll_groups);
1163 0 : if (tgt->next_poll_group == NULL) {
1164 0 : SPDK_ERRLOG("No poll groups exist.\n");
1165 0 : spdk_nvmf_qpair_disconnect(qpair);
1166 0 : return;
1167 : }
1168 0 : }
1169 0 : group = tgt->next_poll_group;
1170 0 : tgt->next_poll_group = TAILQ_NEXT(group, link);
1171 0 : }
1172 :
1173 0 : ctx = calloc(1, sizeof(*ctx));
1174 0 : if (!ctx) {
1175 0 : SPDK_ERRLOG("Unable to send message to poll group.\n");
1176 0 : spdk_nvmf_qpair_disconnect(qpair);
1177 0 : return;
1178 : }
1179 :
1180 0 : ctx->qpair = qpair;
1181 0 : ctx->group = group;
1182 :
1183 0 : pthread_mutex_lock(&group->mutex);
1184 0 : group->current_unassociated_qpairs++;
1185 0 : pthread_mutex_unlock(&group->mutex);
1186 :
1187 0 : spdk_thread_send_msg(group->thread, _nvmf_poll_group_add, ctx);
1188 0 : }
1189 :
1190 : struct spdk_nvmf_poll_group *
1191 0 : spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt)
1192 : {
1193 : struct spdk_io_channel *ch;
1194 :
1195 0 : ch = spdk_get_io_channel(tgt);
1196 0 : if (!ch) {
1197 0 : SPDK_ERRLOG("Unable to get I/O channel for target\n");
1198 0 : return NULL;
1199 : }
1200 :
1201 0 : return spdk_io_channel_get_ctx(ch);
1202 0 : }
1203 :
1204 : void
1205 0 : spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group,
1206 : spdk_nvmf_poll_group_destroy_done_fn cb_fn,
1207 : void *cb_arg)
1208 : {
1209 0 : assert(group->destroy_cb_fn == NULL);
1210 0 : group->destroy_cb_fn = cb_fn;
1211 0 : group->destroy_cb_arg = cb_arg;
1212 :
1213 : /* This function will put the io_channel associated with this poll group */
1214 0 : nvmf_tgt_destroy_poll_group_qpairs(group);
1215 0 : }
1216 :
1217 : int
1218 0 : spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group,
1219 : struct spdk_nvmf_qpair *qpair)
1220 : {
1221 : int rc;
1222 : struct spdk_nvmf_transport_poll_group *tgroup;
1223 :
1224 0 : TAILQ_INIT(&qpair->outstanding);
1225 0 : qpair->group = group;
1226 0 : qpair->ctrlr = NULL;
1227 0 : qpair->disconnect_started = false;
1228 :
1229 0 : tgroup = nvmf_get_transport_poll_group(group, qpair->transport);
1230 0 : if (tgroup == NULL) {
1231 0 : return -1;
1232 : }
1233 :
1234 0 : rc = nvmf_transport_poll_group_add(tgroup, qpair);
1235 :
1236 : /* We add the qpair to the group only it is successfully added into the tgroup */
1237 0 : if (rc == 0) {
1238 : SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_add_qpair, qpair, spdk_thread_get_id(group->thread));
1239 0 : TAILQ_INSERT_TAIL(&group->qpairs, qpair, link);
1240 0 : nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_CONNECTING);
1241 0 : }
1242 :
1243 0 : return rc;
1244 0 : }
1245 :
1246 : static void
1247 0 : _nvmf_ctrlr_destruct(void *ctx)
1248 : {
1249 0 : struct spdk_nvmf_ctrlr *ctrlr = ctx;
1250 :
1251 0 : nvmf_ctrlr_destruct(ctrlr);
1252 0 : }
1253 :
1254 : static void
1255 0 : _nvmf_ctrlr_free_from_qpair(void *ctx)
1256 : {
1257 0 : struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;
1258 0 : struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr;
1259 : uint32_t count;
1260 :
1261 0 : spdk_bit_array_clear(ctrlr->qpair_mask, qpair_ctx->qid);
1262 0 : SPDK_DEBUGLOG(nvmf, "qpair_mask cleared, qid %u\n", qpair_ctx->qid);
1263 0 : count = spdk_bit_array_count_set(ctrlr->qpair_mask);
1264 0 : if (count == 0) {
1265 0 : assert(!ctrlr->in_destruct);
1266 0 : SPDK_DEBUGLOG(nvmf, "Last qpair %u, destroy ctrlr 0x%hx\n", qpair_ctx->qid, ctrlr->cntlid);
1267 0 : ctrlr->in_destruct = true;
1268 0 : spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr);
1269 0 : }
1270 0 : free(qpair_ctx);
1271 0 : }
1272 :
1273 : static void
1274 0 : _nvmf_transport_qpair_fini_complete(void *cb_ctx)
1275 : {
1276 0 : struct nvmf_qpair_disconnect_ctx *qpair_ctx = cb_ctx;
1277 : struct spdk_nvmf_ctrlr *ctrlr;
1278 :
1279 0 : ctrlr = qpair_ctx->ctrlr;
1280 0 : SPDK_DEBUGLOG(nvmf, "Finish destroying qid %u\n", qpair_ctx->qid);
1281 :
1282 0 : if (ctrlr) {
1283 0 : if (qpair_ctx->qid == 0) {
1284 : /* Admin qpair is removed, so set the pointer to NULL.
1285 : * This operation is safe since we are on ctrlr thread now, admin qpair's thread is the same
1286 : * as controller's thread */
1287 0 : assert(ctrlr->thread == spdk_get_thread());
1288 0 : ctrlr->admin_qpair = NULL;
1289 0 : }
1290 : /* Free qpair id from controller's bit mask and destroy the controller if it is the last qpair */
1291 0 : if (ctrlr->thread) {
1292 0 : spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_free_from_qpair, qpair_ctx);
1293 0 : } else {
1294 0 : _nvmf_ctrlr_free_from_qpair(qpair_ctx);
1295 : }
1296 0 : } else {
1297 0 : free(qpair_ctx);
1298 : }
1299 0 : }
1300 :
1301 : void
1302 0 : spdk_nvmf_poll_group_remove(struct spdk_nvmf_qpair *qpair)
1303 : {
1304 : struct spdk_nvmf_transport_poll_group *tgroup;
1305 : int rc;
1306 :
1307 : SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_remove_qpair, qpair,
1308 : spdk_thread_get_id(qpair->group->thread));
1309 0 : nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR);
1310 :
1311 : /* Find the tgroup and remove the qpair from the tgroup */
1312 0 : tgroup = nvmf_get_transport_poll_group(qpair->group, qpair->transport);
1313 0 : if (tgroup != NULL) {
1314 0 : rc = nvmf_transport_poll_group_remove(tgroup, qpair);
1315 0 : if (rc && (rc != ENOTSUP)) {
1316 0 : SPDK_ERRLOG("Cannot remove qpair=%p from transport group=%p\n",
1317 : qpair, tgroup);
1318 0 : }
1319 0 : }
1320 :
1321 0 : TAILQ_REMOVE(&qpair->group->qpairs, qpair, link);
1322 0 : qpair->group = NULL;
1323 0 : }
1324 :
1325 : static void
1326 0 : _nvmf_qpair_sgroup_req_clean(struct spdk_nvmf_subsystem_poll_group *sgroup,
1327 : const struct spdk_nvmf_qpair *qpair)
1328 : {
1329 : struct spdk_nvmf_request *req, *tmp;
1330 0 : TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) {
1331 0 : if (req->qpair == qpair) {
1332 0 : TAILQ_REMOVE(&sgroup->queued, req, link);
1333 0 : if (nvmf_transport_req_free(req)) {
1334 0 : SPDK_ERRLOG("Transport request free error!\n");
1335 0 : }
1336 0 : }
1337 0 : }
1338 0 : }
1339 :
1340 : static void
1341 0 : _nvmf_qpair_destroy(void *ctx, int status)
1342 : {
1343 0 : struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;
1344 0 : struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair;
1345 0 : struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr;
1346 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1347 : uint32_t sid;
1348 :
1349 0 : assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING);
1350 0 : qpair_ctx->qid = qpair->qid;
1351 :
1352 0 : if (qpair->connect_received) {
1353 0 : if (0 == qpair->qid) {
1354 0 : assert(qpair->group->stat.current_admin_qpairs > 0);
1355 0 : qpair->group->stat.current_admin_qpairs--;
1356 0 : } else {
1357 0 : assert(qpair->group->stat.current_io_qpairs > 0);
1358 0 : qpair->group->stat.current_io_qpairs--;
1359 : }
1360 0 : } else {
1361 0 : pthread_mutex_lock(&qpair->group->mutex);
1362 0 : qpair->group->current_unassociated_qpairs--;
1363 0 : pthread_mutex_unlock(&qpair->group->mutex);
1364 : }
1365 :
1366 0 : if (ctrlr) {
1367 0 : sgroup = &qpair->group->sgroups[ctrlr->subsys->id];
1368 0 : _nvmf_qpair_sgroup_req_clean(sgroup, qpair);
1369 0 : } else {
1370 0 : for (sid = 0; sid < qpair->group->num_sgroups; sid++) {
1371 0 : sgroup = &qpair->group->sgroups[sid];
1372 0 : assert(sgroup != NULL);
1373 0 : _nvmf_qpair_sgroup_req_clean(sgroup, qpair);
1374 0 : }
1375 : }
1376 :
1377 0 : nvmf_qpair_auth_destroy(qpair);
1378 0 : qpair_ctx->ctrlr = ctrlr;
1379 0 : spdk_nvmf_poll_group_remove(qpair);
1380 0 : nvmf_transport_qpair_fini(qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx);
1381 0 : }
1382 :
1383 : static void
1384 0 : _nvmf_qpair_disconnect_msg(void *ctx)
1385 : {
1386 0 : struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;
1387 :
1388 0 : spdk_nvmf_qpair_disconnect(qpair_ctx->qpair);
1389 0 : free(ctx);
1390 0 : }
1391 :
1392 : int
1393 0 : spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair)
1394 : {
1395 0 : struct spdk_nvmf_poll_group *group = qpair->group;
1396 : struct nvmf_qpair_disconnect_ctx *qpair_ctx;
1397 :
1398 0 : if (__atomic_test_and_set(&qpair->disconnect_started, __ATOMIC_RELAXED)) {
1399 0 : return -EINPROGRESS;
1400 : }
1401 :
1402 : /* If we get a qpair in the uninitialized state, we can just destroy it immediately */
1403 0 : if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) {
1404 0 : nvmf_transport_qpair_fini(qpair, NULL, NULL);
1405 0 : return 0;
1406 : }
1407 :
1408 0 : assert(group != NULL);
1409 0 : if (spdk_get_thread() != group->thread) {
1410 : /* clear the atomic so we can set it on the next call on the proper thread. */
1411 0 : __atomic_clear(&qpair->disconnect_started, __ATOMIC_RELAXED);
1412 0 : qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx));
1413 0 : if (!qpair_ctx) {
1414 0 : SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n");
1415 0 : return -ENOMEM;
1416 : }
1417 0 : qpair_ctx->qpair = qpair;
1418 0 : spdk_thread_send_msg(group->thread, _nvmf_qpair_disconnect_msg, qpair_ctx);
1419 0 : return 0;
1420 : }
1421 :
1422 : SPDK_DTRACE_PROBE2_TICKS(nvmf_qpair_disconnect, qpair, spdk_thread_get_id(group->thread));
1423 0 : assert(spdk_nvmf_qpair_is_active(qpair));
1424 0 : nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_DEACTIVATING);
1425 :
1426 0 : qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx));
1427 0 : if (!qpair_ctx) {
1428 0 : SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n");
1429 0 : return -ENOMEM;
1430 : }
1431 :
1432 0 : qpair_ctx->qpair = qpair;
1433 :
1434 : /* Check for outstanding I/O */
1435 0 : if (!TAILQ_EMPTY(&qpair->outstanding)) {
1436 : SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_drain_qpair, qpair, spdk_thread_get_id(group->thread));
1437 0 : qpair->state_cb = _nvmf_qpair_destroy;
1438 0 : qpair->state_cb_arg = qpair_ctx;
1439 0 : nvmf_qpair_abort_pending_zcopy_reqs(qpair);
1440 0 : nvmf_qpair_free_aer(qpair);
1441 0 : return 0;
1442 : }
1443 :
1444 0 : _nvmf_qpair_destroy(qpair_ctx, 0);
1445 :
1446 0 : return 0;
1447 0 : }
1448 :
1449 : int
1450 0 : spdk_nvmf_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair,
1451 : struct spdk_nvme_transport_id *trid)
1452 : {
1453 0 : memset(trid, 0, sizeof(*trid));
1454 0 : return nvmf_transport_qpair_get_peer_trid(qpair, trid);
1455 : }
1456 :
1457 : int
1458 0 : spdk_nvmf_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair,
1459 : struct spdk_nvme_transport_id *trid)
1460 : {
1461 0 : memset(trid, 0, sizeof(*trid));
1462 0 : return nvmf_transport_qpair_get_local_trid(qpair, trid);
1463 : }
1464 :
1465 : int
1466 0 : spdk_nvmf_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair,
1467 : struct spdk_nvme_transport_id *trid)
1468 : {
1469 0 : memset(trid, 0, sizeof(*trid));
1470 0 : return nvmf_transport_qpair_get_listen_trid(qpair, trid);
1471 : }
1472 :
1473 : static int
1474 1 : poll_group_update_subsystem(struct spdk_nvmf_poll_group *group,
1475 : struct spdk_nvmf_subsystem *subsystem)
1476 : {
1477 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1478 : uint32_t i, j;
1479 : struct spdk_nvmf_ns *ns;
1480 : struct spdk_nvmf_registrant *reg, *tmp;
1481 : struct spdk_io_channel *ch;
1482 : struct spdk_nvmf_subsystem_pg_ns_info *ns_info;
1483 : struct spdk_nvmf_ctrlr *ctrlr;
1484 : bool ns_changed;
1485 :
1486 : /* Make sure our poll group has memory for this subsystem allocated */
1487 1 : if (subsystem->id >= group->num_sgroups) {
1488 0 : return -ENOMEM;
1489 : }
1490 :
1491 1 : sgroup = &group->sgroups[subsystem->id];
1492 :
1493 : /* Make sure the array of namespace information is the correct size */
1494 1 : if (sgroup->num_ns == 0 && subsystem->max_nsid > 0) {
1495 : /* First allocation */
1496 1 : sgroup->ns_info = calloc(subsystem->max_nsid, sizeof(struct spdk_nvmf_subsystem_pg_ns_info));
1497 1 : if (!sgroup->ns_info) {
1498 0 : return -ENOMEM;
1499 : }
1500 1 : sgroup->num_ns = subsystem->max_nsid;
1501 1 : }
1502 :
1503 1 : ns_changed = false;
1504 :
1505 : /* Detect bdevs that were added or removed */
1506 2 : for (i = 0; i < sgroup->num_ns; i++) {
1507 1 : ns = subsystem->ns[i];
1508 1 : ns_info = &sgroup->ns_info[i];
1509 1 : ch = ns_info->channel;
1510 :
1511 1 : if (ns == NULL && ch == NULL) {
1512 : /* Both NULL. Leave empty */
1513 1 : } else if (ns == NULL && ch != NULL) {
1514 : /* There was a channel here, but the namespace is gone. */
1515 0 : ns_changed = true;
1516 0 : spdk_put_io_channel(ch);
1517 0 : ns_info->channel = NULL;
1518 1 : } else if (ns != NULL && ch == NULL) {
1519 : /* A namespace appeared but there is no channel yet */
1520 1 : ns_changed = true;
1521 1 : ch = spdk_bdev_get_io_channel(ns->desc);
1522 1 : if (ch == NULL) {
1523 0 : SPDK_ERRLOG("Could not allocate I/O channel.\n");
1524 0 : return -ENOMEM;
1525 : }
1526 1 : ns_info->channel = ch;
1527 1 : } else if (spdk_uuid_compare(&ns_info->uuid, spdk_bdev_get_uuid(ns->bdev)) != 0) {
1528 : /* A namespace was here before, but was replaced by a new one. */
1529 0 : ns_changed = true;
1530 0 : spdk_put_io_channel(ns_info->channel);
1531 0 : memset(ns_info, 0, sizeof(*ns_info));
1532 :
1533 0 : ch = spdk_bdev_get_io_channel(ns->desc);
1534 0 : if (ch == NULL) {
1535 0 : SPDK_ERRLOG("Could not allocate I/O channel.\n");
1536 0 : return -ENOMEM;
1537 : }
1538 0 : ns_info->channel = ch;
1539 0 : } else if (ns_info->num_blocks != spdk_bdev_get_num_blocks(ns->bdev)) {
1540 : /* Namespace is still there but size has changed */
1541 0 : SPDK_DEBUGLOG(nvmf, "Namespace resized: subsystem_id %u,"
1542 : " nsid %u, pg %p, old %" PRIu64 ", new %" PRIu64 "\n",
1543 : subsystem->id,
1544 : ns->nsid,
1545 : group,
1546 : ns_info->num_blocks,
1547 : spdk_bdev_get_num_blocks(ns->bdev));
1548 0 : ns_changed = true;
1549 0 : } else if (ns_info->anagrpid != ns->anagrpid) {
1550 : /* Namespace is still there but ANA group ID has changed */
1551 0 : SPDK_DEBUGLOG(nvmf, "ANA group ID changed: subsystem_id %u,"
1552 : "nsid %u, pg %p, old %u, new %u\n",
1553 : subsystem->id,
1554 : ns->nsid,
1555 : group,
1556 : ns_info->anagrpid,
1557 : ns->anagrpid);
1558 0 : ns_changed = true;
1559 0 : }
1560 :
1561 1 : if (ns == NULL) {
1562 0 : memset(ns_info, 0, sizeof(*ns_info));
1563 0 : } else {
1564 1 : ns_info->uuid = *spdk_bdev_get_uuid(ns->bdev);
1565 1 : ns_info->num_blocks = spdk_bdev_get_num_blocks(ns->bdev);
1566 1 : ns_info->anagrpid = ns->anagrpid;
1567 1 : ns_info->crkey = ns->crkey;
1568 1 : ns_info->rtype = ns->rtype;
1569 1 : if (ns->holder) {
1570 0 : ns_info->holder_id = ns->holder->hostid;
1571 0 : }
1572 :
1573 1 : memset(&ns_info->reg_hostid, 0, SPDK_NVMF_MAX_NUM_REGISTRANTS * sizeof(struct spdk_uuid));
1574 1 : j = 0;
1575 1 : TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) {
1576 0 : if (j >= SPDK_NVMF_MAX_NUM_REGISTRANTS) {
1577 0 : SPDK_ERRLOG("Maximum %u registrants can support.\n", SPDK_NVMF_MAX_NUM_REGISTRANTS);
1578 0 : return -EINVAL;
1579 : }
1580 0 : ns_info->reg_hostid[j++] = reg->hostid;
1581 0 : }
1582 : }
1583 1 : }
1584 :
1585 1 : if (ns_changed) {
1586 1 : TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) {
1587 0 : if (ctrlr->thread != spdk_get_thread()) {
1588 0 : continue;
1589 : }
1590 : /* It is possible that a ctrlr was added but the admin_qpair hasn't been
1591 : * assigned yet.
1592 : */
1593 0 : if (!ctrlr->admin_qpair) {
1594 0 : continue;
1595 : }
1596 0 : if (ctrlr->admin_qpair->group == group) {
1597 0 : nvmf_ctrlr_async_event_ns_notice(ctrlr);
1598 0 : nvmf_ctrlr_async_event_ana_change_notice(ctrlr);
1599 0 : }
1600 0 : }
1601 1 : }
1602 :
1603 1 : return 0;
1604 1 : }
1605 :
1606 : int
1607 0 : nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group,
1608 : struct spdk_nvmf_subsystem *subsystem)
1609 : {
1610 0 : return poll_group_update_subsystem(group, subsystem);
1611 : }
1612 :
1613 : int
1614 1 : nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group,
1615 : struct spdk_nvmf_subsystem *subsystem,
1616 : spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)
1617 : {
1618 1 : int rc = 0;
1619 1 : struct spdk_nvmf_subsystem_poll_group *sgroup = &group->sgroups[subsystem->id];
1620 : struct spdk_nvmf_request *req, *tmp;
1621 : uint32_t i;
1622 :
1623 1 : if (!TAILQ_EMPTY(&sgroup->queued)) {
1624 0 : SPDK_ERRLOG("sgroup->queued not empty when adding subsystem\n");
1625 0 : TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) {
1626 0 : TAILQ_REMOVE(&sgroup->queued, req, link);
1627 0 : if (nvmf_transport_req_free(req)) {
1628 0 : SPDK_ERRLOG("Transport request free error!\n");
1629 0 : }
1630 0 : }
1631 0 : }
1632 :
1633 1 : rc = poll_group_update_subsystem(group, subsystem);
1634 1 : if (rc) {
1635 0 : nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL);
1636 0 : goto fini;
1637 : }
1638 :
1639 1 : sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE;
1640 :
1641 2 : for (i = 0; i < sgroup->num_ns; i++) {
1642 1 : sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE;
1643 2 : }
1644 :
1645 : fini:
1646 1 : if (cb_fn) {
1647 0 : cb_fn(cb_arg, rc);
1648 0 : }
1649 :
1650 : SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_add_subsystem, spdk_thread_get_id(group->thread),
1651 : subsystem->subnqn);
1652 :
1653 1 : return rc;
1654 : }
1655 :
1656 : static void
1657 0 : _nvmf_poll_group_remove_subsystem_cb(void *ctx, int status)
1658 : {
1659 0 : struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx;
1660 : struct spdk_nvmf_subsystem *subsystem;
1661 : struct spdk_nvmf_poll_group *group;
1662 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1663 0 : spdk_nvmf_poll_group_mod_done cpl_fn = NULL;
1664 0 : void *cpl_ctx = NULL;
1665 : uint32_t nsid;
1666 :
1667 0 : group = qpair_ctx->group;
1668 0 : subsystem = qpair_ctx->subsystem;
1669 0 : cpl_fn = qpair_ctx->cpl_fn;
1670 0 : cpl_ctx = qpair_ctx->cpl_ctx;
1671 0 : sgroup = &group->sgroups[subsystem->id];
1672 :
1673 0 : if (status) {
1674 0 : goto fini;
1675 : }
1676 :
1677 0 : for (nsid = 0; nsid < sgroup->num_ns; nsid++) {
1678 0 : if (sgroup->ns_info[nsid].channel) {
1679 0 : spdk_put_io_channel(sgroup->ns_info[nsid].channel);
1680 0 : sgroup->ns_info[nsid].channel = NULL;
1681 0 : }
1682 0 : }
1683 :
1684 0 : sgroup->num_ns = 0;
1685 0 : free(sgroup->ns_info);
1686 0 : sgroup->ns_info = NULL;
1687 : fini:
1688 0 : free(qpair_ctx);
1689 0 : if (cpl_fn) {
1690 0 : cpl_fn(cpl_ctx, status);
1691 0 : }
1692 0 : }
1693 :
1694 : static void nvmf_poll_group_remove_subsystem_msg(void *ctx);
1695 :
1696 : static void
1697 0 : nvmf_poll_group_remove_subsystem_msg(void *ctx)
1698 : {
1699 : struct spdk_nvmf_qpair *qpair, *qpair_tmp;
1700 : struct spdk_nvmf_subsystem *subsystem;
1701 : struct spdk_nvmf_poll_group *group;
1702 0 : struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx;
1703 0 : bool qpairs_found = false;
1704 0 : int rc = 0;
1705 :
1706 0 : group = qpair_ctx->group;
1707 0 : subsystem = qpair_ctx->subsystem;
1708 :
1709 0 : TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) {
1710 0 : if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) {
1711 0 : qpairs_found = true;
1712 0 : rc = spdk_nvmf_qpair_disconnect(qpair);
1713 0 : if (rc && rc != -EINPROGRESS) {
1714 0 : break;
1715 : }
1716 0 : }
1717 0 : }
1718 :
1719 0 : if (!qpairs_found) {
1720 0 : _nvmf_poll_group_remove_subsystem_cb(ctx, 0);
1721 0 : return;
1722 : }
1723 :
1724 : /* Some qpairs are in process of being disconnected. Send a message and try to remove them again */
1725 0 : spdk_thread_send_msg(spdk_get_thread(), nvmf_poll_group_remove_subsystem_msg, ctx);
1726 0 : }
1727 :
1728 : void
1729 0 : nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group,
1730 : struct spdk_nvmf_subsystem *subsystem,
1731 : spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)
1732 : {
1733 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1734 : struct nvmf_qpair_disconnect_many_ctx *ctx;
1735 : uint32_t i;
1736 :
1737 : SPDK_DTRACE_PROBE3_TICKS(nvmf_poll_group_remove_subsystem, group, spdk_thread_get_id(group->thread),
1738 : subsystem->subnqn);
1739 :
1740 0 : ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx));
1741 0 : if (!ctx) {
1742 0 : SPDK_ERRLOG("Unable to allocate memory for context to remove poll subsystem\n");
1743 0 : if (cb_fn) {
1744 0 : cb_fn(cb_arg, -1);
1745 0 : }
1746 0 : return;
1747 : }
1748 :
1749 0 : ctx->group = group;
1750 0 : ctx->subsystem = subsystem;
1751 0 : ctx->cpl_fn = cb_fn;
1752 0 : ctx->cpl_ctx = cb_arg;
1753 :
1754 0 : sgroup = &group->sgroups[subsystem->id];
1755 0 : sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE;
1756 :
1757 0 : for (i = 0; i < sgroup->num_ns; i++) {
1758 0 : sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_INACTIVE;
1759 0 : }
1760 :
1761 0 : nvmf_poll_group_remove_subsystem_msg(ctx);
1762 0 : }
1763 :
1764 : void
1765 0 : nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group,
1766 : struct spdk_nvmf_subsystem *subsystem,
1767 : uint32_t nsid,
1768 : spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)
1769 : {
1770 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1771 0 : struct spdk_nvmf_subsystem_pg_ns_info *ns_info = NULL;
1772 0 : int rc = 0;
1773 : uint32_t i;
1774 :
1775 0 : if (subsystem->id >= group->num_sgroups) {
1776 0 : rc = -1;
1777 0 : goto fini;
1778 : }
1779 :
1780 0 : sgroup = &group->sgroups[subsystem->id];
1781 0 : if (sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSED) {
1782 0 : goto fini;
1783 : }
1784 0 : sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSING;
1785 :
1786 0 : if (nsid == SPDK_NVME_GLOBAL_NS_TAG) {
1787 0 : for (i = 0; i < sgroup->num_ns; i++) {
1788 0 : ns_info = &sgroup->ns_info[i];
1789 0 : ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING;
1790 0 : }
1791 0 : } else {
1792 : /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */
1793 0 : if (nsid - 1 < sgroup->num_ns) {
1794 0 : ns_info = &sgroup->ns_info[nsid - 1];
1795 0 : ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING;
1796 0 : }
1797 : }
1798 :
1799 0 : if (sgroup->mgmt_io_outstanding > 0) {
1800 0 : assert(sgroup->cb_fn == NULL);
1801 0 : sgroup->cb_fn = cb_fn;
1802 0 : assert(sgroup->cb_arg == NULL);
1803 0 : sgroup->cb_arg = cb_arg;
1804 0 : return;
1805 : }
1806 :
1807 0 : if (nsid == SPDK_NVME_GLOBAL_NS_TAG) {
1808 0 : for (i = 0; i < sgroup->num_ns; i++) {
1809 0 : ns_info = &sgroup->ns_info[i];
1810 :
1811 0 : if (ns_info->io_outstanding > 0) {
1812 0 : assert(sgroup->cb_fn == NULL);
1813 0 : sgroup->cb_fn = cb_fn;
1814 0 : assert(sgroup->cb_arg == NULL);
1815 0 : sgroup->cb_arg = cb_arg;
1816 0 : return;
1817 : }
1818 0 : }
1819 0 : } else {
1820 0 : if (ns_info != NULL && ns_info->io_outstanding > 0) {
1821 0 : assert(sgroup->cb_fn == NULL);
1822 0 : sgroup->cb_fn = cb_fn;
1823 0 : assert(sgroup->cb_arg == NULL);
1824 0 : sgroup->cb_arg = cb_arg;
1825 0 : return;
1826 : }
1827 : }
1828 :
1829 0 : assert(sgroup->mgmt_io_outstanding == 0);
1830 0 : sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED;
1831 : fini:
1832 0 : if (cb_fn) {
1833 0 : cb_fn(cb_arg, rc);
1834 0 : }
1835 0 : }
1836 :
1837 : void
1838 0 : nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group,
1839 : struct spdk_nvmf_subsystem *subsystem,
1840 : spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)
1841 : {
1842 : struct spdk_nvmf_request *req, *tmp;
1843 : struct spdk_nvmf_subsystem_poll_group *sgroup;
1844 0 : int rc = 0;
1845 : uint32_t i;
1846 :
1847 0 : if (subsystem->id >= group->num_sgroups) {
1848 0 : rc = -1;
1849 0 : goto fini;
1850 : }
1851 :
1852 0 : sgroup = &group->sgroups[subsystem->id];
1853 :
1854 0 : if (sgroup->state == SPDK_NVMF_SUBSYSTEM_ACTIVE) {
1855 0 : goto fini;
1856 : }
1857 :
1858 0 : rc = poll_group_update_subsystem(group, subsystem);
1859 0 : if (rc) {
1860 0 : goto fini;
1861 : }
1862 :
1863 0 : for (i = 0; i < sgroup->num_ns; i++) {
1864 0 : sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE;
1865 0 : }
1866 :
1867 0 : sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE;
1868 :
1869 : /* Release all queued requests */
1870 0 : TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) {
1871 0 : TAILQ_REMOVE(&sgroup->queued, req, link);
1872 0 : if (spdk_nvmf_request_using_zcopy(req)) {
1873 0 : spdk_nvmf_request_zcopy_start(req);
1874 0 : } else {
1875 0 : spdk_nvmf_request_exec(req);
1876 : }
1877 :
1878 0 : }
1879 : fini:
1880 0 : if (cb_fn) {
1881 0 : cb_fn(cb_arg, rc);
1882 0 : }
1883 0 : }
1884 :
1885 :
1886 : struct spdk_nvmf_poll_group *
1887 0 : spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair)
1888 : {
1889 : struct spdk_nvmf_transport_poll_group *tgroup;
1890 :
1891 0 : tgroup = nvmf_transport_get_optimal_poll_group(qpair->transport, qpair);
1892 :
1893 0 : if (tgroup == NULL) {
1894 0 : return NULL;
1895 : }
1896 :
1897 0 : return tgroup->group;
1898 0 : }
1899 :
1900 : void
1901 0 : spdk_nvmf_poll_group_dump_stat(struct spdk_nvmf_poll_group *group, struct spdk_json_write_ctx *w)
1902 : {
1903 : struct spdk_nvmf_transport_poll_group *tgroup;
1904 :
1905 0 : spdk_json_write_object_begin(w);
1906 :
1907 0 : spdk_json_write_named_string(w, "name", spdk_thread_get_name(spdk_get_thread()));
1908 0 : spdk_json_write_named_uint32(w, "admin_qpairs", group->stat.admin_qpairs);
1909 0 : spdk_json_write_named_uint32(w, "io_qpairs", group->stat.io_qpairs);
1910 0 : spdk_json_write_named_uint32(w, "current_admin_qpairs", group->stat.current_admin_qpairs);
1911 0 : spdk_json_write_named_uint32(w, "current_io_qpairs", group->stat.current_io_qpairs);
1912 0 : spdk_json_write_named_uint64(w, "pending_bdev_io", group->stat.pending_bdev_io);
1913 0 : spdk_json_write_named_uint64(w, "completed_nvme_io", group->stat.completed_nvme_io);
1914 :
1915 0 : spdk_json_write_named_array_begin(w, "transports");
1916 :
1917 0 : TAILQ_FOREACH(tgroup, &group->tgroups, link) {
1918 0 : spdk_json_write_object_begin(w);
1919 : /*
1920 : * The trtype field intentionally contains a transport name as this is more informative.
1921 : * The field has not been renamed for backward compatibility.
1922 : */
1923 0 : spdk_json_write_named_string(w, "trtype", spdk_nvmf_get_transport_name(tgroup->transport));
1924 :
1925 0 : if (tgroup->transport->ops->poll_group_dump_stat) {
1926 0 : tgroup->transport->ops->poll_group_dump_stat(tgroup, w);
1927 0 : }
1928 :
1929 0 : spdk_json_write_object_end(w);
1930 0 : }
1931 :
1932 0 : spdk_json_write_array_end(w);
1933 0 : spdk_json_write_object_end(w);
1934 0 : }
|
