Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2020 Intel Corporation.
3 : * Copyright (c) 2021 Mellanox Technologies LTD.
4 : * Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
5 : * All rights reserved.
6 : */
7 :
8 : #include "nvme_internal.h"
9 :
10 : struct spdk_nvme_poll_group *
11 9 : spdk_nvme_poll_group_create(void *ctx, struct spdk_nvme_accel_fn_table *table)
12 : {
13 : struct spdk_nvme_poll_group *group;
14 : int rc;
15 :
16 9 : group = calloc(1, sizeof(*group));
17 9 : if (group == NULL) {
18 1 : return NULL;
19 : }
20 :
21 8 : group->accel_fn_table.table_size = sizeof(struct spdk_nvme_accel_fn_table);
22 8 : if (table && table->table_size != 0) {
23 0 : group->accel_fn_table.table_size = table->table_size;
24 : #define SET_FIELD(field) \
25 : if (offsetof(struct spdk_nvme_accel_fn_table, field) + sizeof(table->field) <= table->table_size) { \
26 : group->accel_fn_table.field = table->field; \
27 : } \
28 :
29 0 : SET_FIELD(append_crc32c);
30 0 : SET_FIELD(append_copy);
31 0 : SET_FIELD(finish_sequence);
32 0 : SET_FIELD(reverse_sequence);
33 0 : SET_FIELD(abort_sequence);
34 : /* Do not remove this statement, you should always update this statement when you adding a new field,
35 : * and do not forget to add the SET_FIELD statement for your added field. */
36 : SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_accel_fn_table) == 56, "Incorrect size");
37 :
38 : #undef SET_FIELD
39 0 : }
40 :
41 : /* Make sure either all or none of the sequence manipulation callbacks are implemented */
42 16 : if ((group->accel_fn_table.finish_sequence && group->accel_fn_table.reverse_sequence &&
43 8 : group->accel_fn_table.abort_sequence) !=
44 8 : (group->accel_fn_table.finish_sequence || group->accel_fn_table.reverse_sequence ||
45 8 : group->accel_fn_table.abort_sequence)) {
46 0 : SPDK_ERRLOG("Invalid accel_fn_table configuration: either all or none of the "
47 : "sequence callbacks must be provided\n");
48 0 : free(group);
49 0 : return NULL;
50 : }
51 :
52 : /* Make sure that sequence callbacks are implemented if append* callbacks are provided */
53 8 : if ((group->accel_fn_table.append_crc32c || group->accel_fn_table.append_copy) &&
54 8 : !group->accel_fn_table.finish_sequence) {
55 0 : SPDK_ERRLOG("Invalid accel_fn_table configuration: append_crc32c and/or append_copy require sequence "
56 : "callbacks to be provided\n");
57 0 : free(group);
58 0 : return NULL;
59 : }
60 :
61 : /* If interrupt is enabled, this fd_group will be used to manage events triggerd on file
62 : * descriptors of all the qpairs in this poll group */
63 8 : rc = spdk_fd_group_create(&group->fgrp);
64 8 : if (rc) {
65 : /* Ignore this for non-Linux platforms, as fd_groups aren't supported there. */
66 : #if defined(__linux__)
67 : SPDK_ERRLOG("Cannot create fd group for the nvme poll group\n");
68 : free(group);
69 : return NULL;
70 : #endif
71 8 : }
72 :
73 8 : group->disconnect_qpair_fd = -1;
74 8 : group->ctx = ctx;
75 8 : STAILQ_INIT(&group->tgroups);
76 :
77 8 : return group;
78 9 : }
79 :
80 : int
81 0 : spdk_nvme_poll_group_get_fd(struct spdk_nvme_poll_group *group)
82 : {
83 0 : if (!group->fgrp) {
84 0 : SPDK_ERRLOG("No fd group present for the nvme poll group.\n");
85 0 : assert(false);
86 : return -EINVAL;
87 : }
88 :
89 0 : return spdk_fd_group_get_fd(group->fgrp);
90 : }
91 :
92 : struct spdk_fd_group *
93 0 : spdk_nvme_poll_group_get_fd_group(struct spdk_nvme_poll_group *group)
94 : {
95 0 : return group->fgrp;
96 : }
97 :
98 : struct spdk_nvme_poll_group *
99 0 : spdk_nvme_qpair_get_optimal_poll_group(struct spdk_nvme_qpair *qpair)
100 : {
101 : struct spdk_nvme_transport_poll_group *tgroup;
102 :
103 0 : tgroup = nvme_transport_qpair_get_optimal_poll_group(qpair->transport, qpair);
104 :
105 0 : if (tgroup == NULL) {
106 0 : return NULL;
107 : }
108 :
109 0 : return tgroup->group;
110 0 : }
111 :
112 : #ifdef __linux__
113 : static int
114 : nvme_poll_group_read_disconnect_qpair_fd(void *arg)
115 : {
116 : return 0;
117 : }
118 :
119 : void
120 : nvme_poll_group_write_disconnect_qpair_fd(struct spdk_nvme_poll_group *group)
121 : {
122 : uint64_t notify = 1;
123 : int rc;
124 :
125 : if (!group->enable_interrupts) {
126 : return;
127 : }
128 :
129 : /* Write to the disconnect qpair fd. This will generate event on the epoll fd of poll
130 : * group. We then check for disconnected qpairs in spdk_nvme_poll_group_wait() */
131 : rc = write(group->disconnect_qpair_fd, ¬ify, sizeof(notify));
132 : if (rc < 0) {
133 : SPDK_ERRLOG("failed to write the disconnect qpair fd: %s.\n", strerror(errno));
134 : }
135 : }
136 :
137 : static int
138 : nvme_poll_group_add_disconnect_qpair_fd(struct spdk_nvme_poll_group *group)
139 : {
140 : struct spdk_event_handler_opts opts = {};
141 : int fd;
142 :
143 : fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
144 : if (fd < 0) {
145 : return fd;
146 : }
147 :
148 : assert(group->disconnect_qpair_fd == -1);
149 : group->disconnect_qpair_fd = fd;
150 :
151 : spdk_fd_group_get_default_event_handler_opts(&opts, sizeof(opts));
152 : opts.fd_type = SPDK_FD_TYPE_EVENTFD;
153 :
154 : return SPDK_FD_GROUP_ADD_EXT(group->fgrp, fd, nvme_poll_group_read_disconnect_qpair_fd,
155 : group, &opts);
156 : }
157 :
158 : #else
159 :
160 : void
161 0 : nvme_poll_group_write_disconnect_qpair_fd(struct spdk_nvme_poll_group *group)
162 : {
163 0 : }
164 :
165 : static int
166 0 : nvme_poll_group_add_disconnect_qpair_fd(struct spdk_nvme_poll_group *group)
167 : {
168 0 : return -ENOTSUP;
169 : }
170 :
171 : #endif
172 :
173 : int
174 13 : spdk_nvme_poll_group_add(struct spdk_nvme_poll_group *group, struct spdk_nvme_qpair *qpair)
175 : {
176 : struct spdk_nvme_transport_poll_group *tgroup;
177 : const struct spdk_nvme_transport *transport;
178 : int rc;
179 :
180 13 : if (nvme_qpair_get_state(qpair) != NVME_QPAIR_DISCONNECTED) {
181 1 : return -EINVAL;
182 : }
183 :
184 12 : if (!group->enable_interrupts_is_valid) {
185 4 : group->enable_interrupts_is_valid = true;
186 4 : group->enable_interrupts = qpair->ctrlr->opts.enable_interrupts;
187 4 : if (group->enable_interrupts) {
188 0 : rc = nvme_poll_group_add_disconnect_qpair_fd(group);
189 0 : if (rc != 0) {
190 0 : return rc;
191 : }
192 0 : }
193 12 : } else if (qpair->ctrlr->opts.enable_interrupts != group->enable_interrupts) {
194 1 : SPDK_ERRLOG("Queue pair %s interrupts cannot be added to poll group\n",
195 : qpair->ctrlr->opts.enable_interrupts ? "without" : "with");
196 1 : return -EINVAL;
197 : }
198 :
199 21 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
200 13 : if (tgroup->transport == qpair->transport) {
201 3 : break;
202 : }
203 10 : }
204 :
205 : /* See if a new transport has been added (dlopen style) and we need to update the poll group */
206 11 : if (!tgroup) {
207 8 : transport = nvme_get_first_transport();
208 18 : while (transport != NULL) {
209 16 : if (transport == qpair->transport) {
210 6 : tgroup = nvme_transport_poll_group_create(transport);
211 6 : if (tgroup == NULL) {
212 0 : return -ENOMEM;
213 : }
214 6 : tgroup->group = group;
215 6 : STAILQ_INSERT_TAIL(&group->tgroups, tgroup, link);
216 6 : break;
217 : }
218 10 : transport = nvme_get_next_transport(transport);
219 : }
220 8 : }
221 :
222 11 : return tgroup ? nvme_transport_poll_group_add(tgroup, qpair) : -ENODEV;
223 13 : }
224 :
225 : int
226 9 : spdk_nvme_poll_group_remove(struct spdk_nvme_poll_group *group, struct spdk_nvme_qpair *qpair)
227 : {
228 : struct spdk_nvme_transport_poll_group *tgroup;
229 :
230 17 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
231 16 : if (tgroup->transport == qpair->transport) {
232 8 : return nvme_transport_poll_group_remove(tgroup, qpair);
233 : }
234 8 : }
235 :
236 1 : return -ENODEV;
237 9 : }
238 :
239 : static int
240 0 : nvme_qpair_process_completion_wrapper(void *arg)
241 : {
242 0 : struct spdk_nvme_qpair *qpair = arg;
243 :
244 0 : return spdk_nvme_qpair_process_completions(qpair, 0);
245 : }
246 :
247 : static int
248 1 : nvme_poll_group_add_qpair_fd(struct spdk_nvme_qpair *qpair)
249 : {
250 : struct spdk_nvme_poll_group *group;
251 1 : struct spdk_event_handler_opts opts = {
252 : .opts_size = SPDK_SIZEOF(&opts, fd_type),
253 : };
254 : int fd;
255 :
256 1 : group = qpair->poll_group->group;
257 1 : if (group->enable_interrupts == false) {
258 1 : return 0;
259 : }
260 :
261 0 : fd = spdk_nvme_qpair_get_fd(qpair, &opts);
262 0 : if (fd < 0) {
263 0 : SPDK_ERRLOG("Cannot get fd for the qpair: %d\n", fd);
264 0 : return -EINVAL;
265 : }
266 :
267 0 : return SPDK_FD_GROUP_ADD_EXT(group->fgrp, fd, nvme_qpair_process_completion_wrapper,
268 : qpair, &opts);
269 1 : }
270 :
271 : static void
272 0 : nvme_poll_group_remove_qpair_fd(struct spdk_nvme_qpair *qpair)
273 : {
274 : struct spdk_nvme_poll_group *group;
275 : int fd;
276 :
277 0 : group = qpair->poll_group->group;
278 0 : if (group->enable_interrupts == false) {
279 0 : return;
280 : }
281 :
282 0 : fd = spdk_nvme_qpair_get_fd(qpair, NULL);
283 0 : if (fd < 0) {
284 0 : SPDK_ERRLOG("Cannot get fd for the qpair: %d\n", fd);
285 0 : assert(false);
286 : return;
287 : }
288 :
289 0 : spdk_fd_group_remove(group->fgrp, fd);
290 0 : }
291 :
292 : int
293 1 : nvme_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
294 : {
295 : int rc;
296 :
297 1 : rc = nvme_transport_poll_group_connect_qpair(qpair);
298 1 : if (rc != 0) {
299 0 : return rc;
300 : }
301 :
302 1 : rc = nvme_poll_group_add_qpair_fd(qpair);
303 1 : if (rc != 0) {
304 0 : nvme_transport_poll_group_disconnect_qpair(qpair);
305 0 : return rc;
306 : }
307 :
308 1 : return 0;
309 1 : }
310 :
311 : int
312 0 : nvme_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)
313 : {
314 0 : nvme_poll_group_remove_qpair_fd(qpair);
315 :
316 0 : return nvme_transport_poll_group_disconnect_qpair(qpair);
317 : }
318 :
319 : int
320 0 : spdk_nvme_poll_group_wait(struct spdk_nvme_poll_group *group,
321 : spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
322 : {
323 : struct spdk_nvme_transport_poll_group *tgroup;
324 0 : int num_events, timeout = -1;
325 :
326 0 : if (disconnected_qpair_cb == NULL) {
327 0 : return -EINVAL;
328 : }
329 :
330 0 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
331 0 : nvme_transport_poll_group_check_disconnected_qpairs(tgroup, disconnected_qpair_cb);
332 0 : }
333 :
334 0 : num_events = spdk_fd_group_wait(group->fgrp, timeout);
335 :
336 0 : return num_events;
337 0 : }
338 :
339 : int64_t
340 2 : spdk_nvme_poll_group_process_completions(struct spdk_nvme_poll_group *group,
341 : uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
342 : {
343 : struct spdk_nvme_transport_poll_group *tgroup;
344 2 : int64_t local_completions = 0, error_reason = 0, num_completions = 0;
345 :
346 2 : if (disconnected_qpair_cb == NULL) {
347 0 : return -EINVAL;
348 : }
349 :
350 2 : if (spdk_unlikely(group->in_process_completions)) {
351 0 : return 0;
352 : }
353 2 : group->in_process_completions = true;
354 :
355 3 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
356 2 : local_completions = nvme_transport_poll_group_process_completions(tgroup, completions_per_qpair,
357 1 : disconnected_qpair_cb);
358 1 : if (local_completions < 0 && error_reason == 0) {
359 0 : error_reason = local_completions;
360 0 : } else {
361 1 : num_completions += local_completions;
362 : /* Just to be safe */
363 1 : assert(num_completions >= 0);
364 : }
365 1 : }
366 2 : group->in_process_completions = false;
367 :
368 2 : return error_reason ? error_reason : num_completions;
369 2 : }
370 :
371 : int
372 0 : spdk_nvme_poll_group_all_connected(struct spdk_nvme_poll_group *group)
373 : {
374 : struct spdk_nvme_transport_poll_group *tgroup;
375 : struct spdk_nvme_qpair *qpair;
376 0 : int rc = 0;
377 :
378 0 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
379 0 : if (!STAILQ_EMPTY(&tgroup->disconnected_qpairs)) {
380 : /* Treat disconnected qpairs as highest priority for notification.
381 : * This means we can just return immediately here.
382 : */
383 0 : return -EIO;
384 : }
385 0 : STAILQ_FOREACH(qpair, &tgroup->connected_qpairs, poll_group_stailq) {
386 0 : if (nvme_qpair_get_state(qpair) < NVME_QPAIR_CONNECTING) {
387 0 : return -EIO;
388 0 : } else if (nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING) {
389 0 : rc = -EAGAIN;
390 : /* Break so that we can check the remaining transport groups,
391 : * in case any of them have a disconnected qpair.
392 : */
393 0 : break;
394 : }
395 0 : }
396 0 : }
397 :
398 0 : return rc;
399 0 : }
400 :
401 : void *
402 0 : spdk_nvme_poll_group_get_ctx(struct spdk_nvme_poll_group *group)
403 : {
404 0 : return group->ctx;
405 : }
406 :
407 : int
408 9 : spdk_nvme_poll_group_destroy(struct spdk_nvme_poll_group *group)
409 : {
410 : struct spdk_nvme_transport_poll_group *tgroup, *tmp_tgroup;
411 9 : struct spdk_fd_group *fgrp = group->fgrp;
412 :
413 10 : STAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp_tgroup) {
414 2 : STAILQ_REMOVE(&group->tgroups, tgroup, spdk_nvme_transport_poll_group, link);
415 2 : if (nvme_transport_poll_group_destroy(tgroup) != 0) {
416 1 : STAILQ_INSERT_TAIL(&group->tgroups, tgroup, link);
417 1 : return -EBUSY;
418 : }
419 :
420 1 : }
421 :
422 8 : if (fgrp) {
423 0 : if (group->enable_interrupts) {
424 0 : spdk_fd_group_remove(fgrp, group->disconnect_qpair_fd);
425 0 : close(group->disconnect_qpair_fd);
426 0 : }
427 0 : spdk_fd_group_destroy(fgrp);
428 0 : }
429 :
430 8 : free(group);
431 :
432 8 : return 0;
433 9 : }
434 :
435 : int
436 2 : spdk_nvme_poll_group_get_stats(struct spdk_nvme_poll_group *group,
437 : struct spdk_nvme_poll_group_stat **stats)
438 : {
439 : struct spdk_nvme_transport_poll_group *tgroup;
440 : struct spdk_nvme_poll_group_stat *result;
441 2 : uint32_t transports_count = 0;
442 : /* Not all transports used by this poll group may support statistics reporting */
443 2 : uint32_t reported_stats_count = 0;
444 : int rc;
445 :
446 2 : assert(group);
447 2 : assert(stats);
448 :
449 2 : result = calloc(1, sizeof(*result));
450 2 : if (!result) {
451 0 : SPDK_ERRLOG("Failed to allocate memory for poll group statistics\n");
452 0 : return -ENOMEM;
453 : }
454 :
455 5 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
456 3 : transports_count++;
457 3 : }
458 :
459 2 : result->transport_stat = calloc(transports_count, sizeof(*result->transport_stat));
460 2 : if (!result->transport_stat) {
461 0 : SPDK_ERRLOG("Failed to allocate memory for poll group statistics\n");
462 0 : free(result);
463 0 : return -ENOMEM;
464 : }
465 :
466 5 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
467 3 : rc = nvme_transport_poll_group_get_stats(tgroup, &result->transport_stat[reported_stats_count]);
468 3 : if (rc == 0) {
469 3 : reported_stats_count++;
470 3 : }
471 3 : }
472 :
473 2 : if (reported_stats_count == 0) {
474 1 : free(result->transport_stat);
475 1 : free(result);
476 1 : SPDK_DEBUGLOG(nvme, "No transport statistics available\n");
477 1 : return -ENOTSUP;
478 : }
479 :
480 1 : result->num_transports = reported_stats_count;
481 1 : *stats = result;
482 :
483 1 : return 0;
484 2 : }
485 :
486 : void
487 1 : spdk_nvme_poll_group_free_stats(struct spdk_nvme_poll_group *group,
488 : struct spdk_nvme_poll_group_stat *stat)
489 : {
490 : struct spdk_nvme_transport_poll_group *tgroup;
491 : uint32_t i;
492 1 : uint32_t freed_stats __attribute__((unused)) = 0;
493 :
494 1 : assert(group);
495 1 : assert(stat);
496 :
497 4 : for (i = 0; i < stat->num_transports; i++) {
498 3 : STAILQ_FOREACH(tgroup, &group->tgroups, link) {
499 3 : if (nvme_transport_get_trtype(tgroup->transport) == stat->transport_stat[i]->trtype) {
500 3 : nvme_transport_poll_group_free_stats(tgroup, stat->transport_stat[i]);
501 3 : freed_stats++;
502 3 : break;
503 : }
504 0 : }
505 3 : }
506 :
507 1 : assert(freed_stats == stat->num_transports);
508 :
509 1 : free(stat->transport_stat);
510 1 : free(stat);
511 1 : }
|