LCOV - code coverage report
Current view: top level - lib/nvme - nvme_poll_group.c (source / functions) Hit Total Coverage
Test: ut_cov_unit.info Lines: 136 238 57.1 %
Date: 2024-12-09 22:52:56 Functions: 9 20 45.0 %

          Line data    Source code
       1             : /*   SPDX-License-Identifier: BSD-3-Clause
       2             :  *   Copyright (C) 2020 Intel Corporation.
       3             :  *   Copyright (c) 2021 Mellanox Technologies LTD.
       4             :  *   Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
       5             :  *   All rights reserved.
       6             :  */
       7             : 
       8             : #include "nvme_internal.h"
       9             : 
      10             : struct spdk_nvme_poll_group *
      11           9 : spdk_nvme_poll_group_create(void *ctx, struct spdk_nvme_accel_fn_table *table)
      12             : {
      13             :         struct spdk_nvme_poll_group *group;
      14             :         int rc;
      15             : 
      16           9 :         group = calloc(1, sizeof(*group));
      17           9 :         if (group == NULL) {
      18           1 :                 return NULL;
      19             :         }
      20             : 
      21           8 :         group->accel_fn_table.table_size = sizeof(struct spdk_nvme_accel_fn_table);
      22           8 :         if (table && table->table_size != 0) {
      23           0 :                 group->accel_fn_table.table_size = table->table_size;
      24             : #define SET_FIELD(field) \
      25             :         if (offsetof(struct spdk_nvme_accel_fn_table, field) + sizeof(table->field) <= table->table_size) { \
      26             :                 group->accel_fn_table.field = table->field; \
      27             :         } \
      28             : 
      29           0 :                 SET_FIELD(append_crc32c);
      30           0 :                 SET_FIELD(append_copy);
      31           0 :                 SET_FIELD(finish_sequence);
      32           0 :                 SET_FIELD(reverse_sequence);
      33           0 :                 SET_FIELD(abort_sequence);
      34             :                 /* Do not remove this statement, you should always update this statement when you adding a new field,
      35             :                  * and do not forget to add the SET_FIELD statement for your added field. */
      36             :                 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_accel_fn_table) == 56, "Incorrect size");
      37             : 
      38             : #undef SET_FIELD
      39           0 :         }
      40             : 
      41             :         /* Make sure either all or none of the sequence manipulation callbacks are implemented */
      42          16 :         if ((group->accel_fn_table.finish_sequence && group->accel_fn_table.reverse_sequence &&
      43           8 :              group->accel_fn_table.abort_sequence) !=
      44           8 :             (group->accel_fn_table.finish_sequence || group->accel_fn_table.reverse_sequence ||
      45           8 :              group->accel_fn_table.abort_sequence)) {
      46           0 :                 SPDK_ERRLOG("Invalid accel_fn_table configuration: either all or none of the "
      47             :                             "sequence callbacks must be provided\n");
      48           0 :                 free(group);
      49           0 :                 return NULL;
      50             :         }
      51             : 
      52             :         /* Make sure that sequence callbacks are implemented if append* callbacks are provided */
      53           8 :         if ((group->accel_fn_table.append_crc32c || group->accel_fn_table.append_copy) &&
      54           8 :             !group->accel_fn_table.finish_sequence) {
      55           0 :                 SPDK_ERRLOG("Invalid accel_fn_table configuration: append_crc32c and/or append_copy require sequence "
      56             :                             "callbacks to be provided\n");
      57           0 :                 free(group);
      58           0 :                 return NULL;
      59             :         }
      60             : 
      61             :         /* If interrupt is enabled, this fd_group will be used to manage events triggerd on file
      62             :          * descriptors of all the qpairs in this poll group */
      63           8 :         rc = spdk_fd_group_create(&group->fgrp);
      64           8 :         if (rc) {
      65             :                 /* Ignore this for non-Linux platforms, as fd_groups aren't supported there. */
      66             : #if defined(__linux__)
      67             :                 SPDK_ERRLOG("Cannot create fd group for the nvme poll group\n");
      68             :                 free(group);
      69             :                 return NULL;
      70             : #endif
      71           8 :         }
      72             : 
      73           8 :         group->disconnect_qpair_fd = -1;
      74           8 :         group->ctx = ctx;
      75           8 :         STAILQ_INIT(&group->tgroups);
      76             : 
      77           8 :         return group;
      78           9 : }
      79             : 
      80             : int
      81           0 : spdk_nvme_poll_group_get_fd(struct spdk_nvme_poll_group *group)
      82             : {
      83           0 :         if (!group->fgrp) {
      84           0 :                 SPDK_ERRLOG("No fd group present for the nvme poll group.\n");
      85           0 :                 assert(false);
      86             :                 return -EINVAL;
      87             :         }
      88             : 
      89           0 :         return spdk_fd_group_get_fd(group->fgrp);
      90             : }
      91             : 
      92             : struct spdk_fd_group *
      93           0 : spdk_nvme_poll_group_get_fd_group(struct spdk_nvme_poll_group *group)
      94             : {
      95           0 :         return group->fgrp;
      96             : }
      97             : 
      98             : struct spdk_nvme_poll_group *
      99           0 : spdk_nvme_qpair_get_optimal_poll_group(struct spdk_nvme_qpair *qpair)
     100             : {
     101             :         struct spdk_nvme_transport_poll_group *tgroup;
     102             : 
     103           0 :         tgroup = nvme_transport_qpair_get_optimal_poll_group(qpair->transport, qpair);
     104             : 
     105           0 :         if (tgroup == NULL) {
     106           0 :                 return NULL;
     107             :         }
     108             : 
     109           0 :         return tgroup->group;
     110           0 : }
     111             : 
     112             : #ifdef __linux__
     113             : static int
     114             : nvme_poll_group_read_disconnect_qpair_fd(void *arg)
     115             : {
     116             :         return 0;
     117             : }
     118             : 
     119             : void
     120             : nvme_poll_group_write_disconnect_qpair_fd(struct spdk_nvme_poll_group *group)
     121             : {
     122             :         uint64_t notify = 1;
     123             :         int rc;
     124             : 
     125             :         if (!group->enable_interrupts) {
     126             :                 return;
     127             :         }
     128             : 
     129             :         /* Write to the disconnect qpair fd. This will generate event on the epoll fd of poll
     130             :          * group. We then check for disconnected qpairs in spdk_nvme_poll_group_wait() */
     131             :         rc = write(group->disconnect_qpair_fd, &notify, sizeof(notify));
     132             :         if (rc < 0) {
     133             :                 SPDK_ERRLOG("failed to write the disconnect qpair fd: %s.\n", strerror(errno));
     134             :         }
     135             : }
     136             : 
     137             : static int
     138             : nvme_poll_group_add_disconnect_qpair_fd(struct spdk_nvme_poll_group *group)
     139             : {
     140             :         struct spdk_event_handler_opts opts = {};
     141             :         int fd;
     142             : 
     143             :         fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
     144             :         if (fd < 0) {
     145             :                 return fd;
     146             :         }
     147             : 
     148             :         assert(group->disconnect_qpair_fd == -1);
     149             :         group->disconnect_qpair_fd = fd;
     150             : 
     151             :         spdk_fd_group_get_default_event_handler_opts(&opts, sizeof(opts));
     152             :         opts.fd_type = SPDK_FD_TYPE_EVENTFD;
     153             : 
     154             :         return SPDK_FD_GROUP_ADD_EXT(group->fgrp, fd, nvme_poll_group_read_disconnect_qpair_fd,
     155             :                                      group, &opts);
     156             : }
     157             : 
     158             : #else
     159             : 
     160             : void
     161           0 : nvme_poll_group_write_disconnect_qpair_fd(struct spdk_nvme_poll_group *group)
     162             : {
     163           0 : }
     164             : 
     165             : static int
     166           0 : nvme_poll_group_add_disconnect_qpair_fd(struct spdk_nvme_poll_group *group)
     167             : {
     168           0 :         return -ENOTSUP;
     169             : }
     170             : 
     171             : #endif
     172             : 
     173             : int
     174          13 : spdk_nvme_poll_group_add(struct spdk_nvme_poll_group *group, struct spdk_nvme_qpair *qpair)
     175             : {
     176             :         struct spdk_nvme_transport_poll_group *tgroup;
     177             :         const struct spdk_nvme_transport *transport;
     178             :         int rc;
     179             : 
     180          13 :         if (nvme_qpair_get_state(qpair) != NVME_QPAIR_DISCONNECTED) {
     181           1 :                 return -EINVAL;
     182             :         }
     183             : 
     184          12 :         if (!group->enable_interrupts_is_valid) {
     185           4 :                 group->enable_interrupts_is_valid = true;
     186           4 :                 group->enable_interrupts = qpair->ctrlr->opts.enable_interrupts;
     187           4 :                 if (group->enable_interrupts) {
     188           0 :                         rc = nvme_poll_group_add_disconnect_qpair_fd(group);
     189           0 :                         if (rc != 0) {
     190           0 :                                 return rc;
     191             :                         }
     192           0 :                 }
     193          12 :         } else if (qpair->ctrlr->opts.enable_interrupts != group->enable_interrupts) {
     194           1 :                 SPDK_ERRLOG("Queue pair %s interrupts cannot be added to poll group\n",
     195             :                             qpair->ctrlr->opts.enable_interrupts ? "without" : "with");
     196           1 :                 return -EINVAL;
     197             :         }
     198             : 
     199          21 :         STAILQ_FOREACH(tgroup, &group->tgroups, link) {
     200          13 :                 if (tgroup->transport == qpair->transport) {
     201           3 :                         break;
     202             :                 }
     203          10 :         }
     204             : 
     205             :         /* See if a new transport has been added (dlopen style) and we need to update the poll group */
     206          11 :         if (!tgroup) {
     207           8 :                 transport = nvme_get_first_transport();
     208          18 :                 while (transport != NULL) {
     209          16 :                         if (transport == qpair->transport) {
     210           6 :                                 tgroup = nvme_transport_poll_group_create(transport);
     211           6 :                                 if (tgroup == NULL) {
     212           0 :                                         return -ENOMEM;
     213             :                                 }
     214           6 :                                 tgroup->group = group;
     215           6 :                                 STAILQ_INSERT_TAIL(&group->tgroups, tgroup, link);
     216           6 :                                 break;
     217             :                         }
     218          10 :                         transport = nvme_get_next_transport(transport);
     219             :                 }
     220           8 :         }
     221             : 
     222          11 :         return tgroup ? nvme_transport_poll_group_add(tgroup, qpair) : -ENODEV;
     223          13 : }
     224             : 
     225             : int
     226           9 : spdk_nvme_poll_group_remove(struct spdk_nvme_poll_group *group, struct spdk_nvme_qpair *qpair)
     227             : {
     228             :         struct spdk_nvme_transport_poll_group *tgroup;
     229             : 
     230          17 :         STAILQ_FOREACH(tgroup, &group->tgroups, link) {
     231          16 :                 if (tgroup->transport == qpair->transport) {
     232           8 :                         return nvme_transport_poll_group_remove(tgroup, qpair);
     233             :                 }
     234           8 :         }
     235             : 
     236           1 :         return -ENODEV;
     237           9 : }
     238             : 
     239             : static int
     240           0 : nvme_qpair_process_completion_wrapper(void *arg)
     241             : {
     242           0 :         struct spdk_nvme_qpair *qpair = arg;
     243             : 
     244           0 :         return spdk_nvme_qpair_process_completions(qpair, 0);
     245             : }
     246             : 
     247             : static int
     248           1 : nvme_poll_group_add_qpair_fd(struct spdk_nvme_qpair *qpair)
     249             : {
     250             :         struct spdk_nvme_poll_group *group;
     251           1 :         struct spdk_event_handler_opts opts = {
     252             :                 .opts_size = SPDK_SIZEOF(&opts, fd_type),
     253             :         };
     254             :         int fd;
     255             : 
     256           1 :         group = qpair->poll_group->group;
     257           1 :         if (group->enable_interrupts == false) {
     258           1 :                 return 0;
     259             :         }
     260             : 
     261           0 :         fd = spdk_nvme_qpair_get_fd(qpair, &opts);
     262           0 :         if (fd < 0) {
     263           0 :                 SPDK_ERRLOG("Cannot get fd for the qpair: %d\n", fd);
     264           0 :                 return -EINVAL;
     265             :         }
     266             : 
     267           0 :         return SPDK_FD_GROUP_ADD_EXT(group->fgrp, fd, nvme_qpair_process_completion_wrapper,
     268             :                                      qpair, &opts);
     269           1 : }
     270             : 
     271             : static void
     272           0 : nvme_poll_group_remove_qpair_fd(struct spdk_nvme_qpair *qpair)
     273             : {
     274             :         struct spdk_nvme_poll_group *group;
     275             :         int fd;
     276             : 
     277           0 :         group = qpair->poll_group->group;
     278           0 :         if (group->enable_interrupts == false) {
     279           0 :                 return;
     280             :         }
     281             : 
     282           0 :         fd = spdk_nvme_qpair_get_fd(qpair, NULL);
     283           0 :         if (fd < 0) {
     284           0 :                 SPDK_ERRLOG("Cannot get fd for the qpair: %d\n", fd);
     285           0 :                 assert(false);
     286             :                 return;
     287             :         }
     288             : 
     289           0 :         spdk_fd_group_remove(group->fgrp, fd);
     290           0 : }
     291             : 
     292             : int
     293           1 : nvme_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
     294             : {
     295             :         int rc;
     296             : 
     297           1 :         rc = nvme_transport_poll_group_connect_qpair(qpair);
     298           1 :         if (rc != 0) {
     299           0 :                 return rc;
     300             :         }
     301             : 
     302           1 :         rc = nvme_poll_group_add_qpair_fd(qpair);
     303           1 :         if (rc != 0) {
     304           0 :                 nvme_transport_poll_group_disconnect_qpair(qpair);
     305           0 :                 return rc;
     306             :         }
     307             : 
     308           1 :         return 0;
     309           1 : }
     310             : 
     311             : int
     312           0 : nvme_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)
     313             : {
     314           0 :         nvme_poll_group_remove_qpair_fd(qpair);
     315             : 
     316           0 :         return nvme_transport_poll_group_disconnect_qpair(qpair);
     317             : }
     318             : 
     319             : int
     320           0 : spdk_nvme_poll_group_wait(struct spdk_nvme_poll_group *group,
     321             :                           spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
     322             : {
     323             :         struct spdk_nvme_transport_poll_group *tgroup;
     324           0 :         int num_events, timeout = -1;
     325             : 
     326           0 :         if (disconnected_qpair_cb == NULL) {
     327           0 :                 return -EINVAL;
     328             :         }
     329             : 
     330           0 :         STAILQ_FOREACH(tgroup, &group->tgroups, link) {
     331           0 :                 nvme_transport_poll_group_check_disconnected_qpairs(tgroup, disconnected_qpair_cb);
     332           0 :         }
     333             : 
     334           0 :         num_events = spdk_fd_group_wait(group->fgrp, timeout);
     335             : 
     336           0 :         return num_events;
     337           0 : }
     338             : 
     339             : int64_t
     340           2 : spdk_nvme_poll_group_process_completions(struct spdk_nvme_poll_group *group,
     341             :                 uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
     342             : {
     343             :         struct spdk_nvme_transport_poll_group *tgroup;
     344           2 :         int64_t local_completions = 0, error_reason = 0, num_completions = 0;
     345             : 
     346           2 :         if (disconnected_qpair_cb == NULL) {
     347           0 :                 return -EINVAL;
     348             :         }
     349             : 
     350           2 :         if (spdk_unlikely(group->in_process_completions)) {
     351           0 :                 return 0;
     352             :         }
     353           2 :         group->in_process_completions = true;
     354             : 
     355           3 :         STAILQ_FOREACH(tgroup, &group->tgroups, link) {
     356           2 :                 local_completions = nvme_transport_poll_group_process_completions(tgroup, completions_per_qpair,
     357           1 :                                     disconnected_qpair_cb);
     358           1 :                 if (local_completions < 0 && error_reason == 0) {
     359           0 :                         error_reason = local_completions;
     360           0 :                 } else {
     361           1 :                         num_completions += local_completions;
     362             :                         /* Just to be safe */
     363           1 :                         assert(num_completions >= 0);
     364             :                 }
     365           1 :         }
     366           2 :         group->in_process_completions = false;
     367             : 
     368           2 :         return error_reason ? error_reason : num_completions;
     369           2 : }
     370             : 
     371             : int
     372           0 : spdk_nvme_poll_group_all_connected(struct spdk_nvme_poll_group *group)
     373             : {
     374             :         struct spdk_nvme_transport_poll_group *tgroup;
     375             :         struct spdk_nvme_qpair *qpair;
     376           0 :         int rc = 0;
     377             : 
     378           0 :         STAILQ_FOREACH(tgroup, &group->tgroups, link) {
     379           0 :                 if (!STAILQ_EMPTY(&tgroup->disconnected_qpairs)) {
     380             :                         /* Treat disconnected qpairs as highest priority for notification.
     381             :                          * This means we can just return immediately here.
     382             :                          */
     383           0 :                         return -EIO;
     384             :                 }
     385           0 :                 STAILQ_FOREACH(qpair, &tgroup->connected_qpairs, poll_group_stailq) {
     386           0 :                         if (nvme_qpair_get_state(qpair) < NVME_QPAIR_CONNECTING) {
     387           0 :                                 return -EIO;
     388           0 :                         } else if (nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING) {
     389           0 :                                 rc = -EAGAIN;
     390             :                                 /* Break so that we can check the remaining transport groups,
     391             :                                  * in case any of them have a disconnected qpair.
     392             :                                  */
     393           0 :                                 break;
     394             :                         }
     395           0 :                 }
     396           0 :         }
     397             : 
     398           0 :         return rc;
     399           0 : }
     400             : 
     401             : void *
     402           0 : spdk_nvme_poll_group_get_ctx(struct spdk_nvme_poll_group *group)
     403             : {
     404           0 :         return group->ctx;
     405             : }
     406             : 
     407             : int
     408           9 : spdk_nvme_poll_group_destroy(struct spdk_nvme_poll_group *group)
     409             : {
     410             :         struct spdk_nvme_transport_poll_group *tgroup, *tmp_tgroup;
     411           9 :         struct spdk_fd_group *fgrp = group->fgrp;
     412             : 
     413          10 :         STAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp_tgroup) {
     414           2 :                 STAILQ_REMOVE(&group->tgroups, tgroup, spdk_nvme_transport_poll_group, link);
     415           2 :                 if (nvme_transport_poll_group_destroy(tgroup) != 0) {
     416           1 :                         STAILQ_INSERT_TAIL(&group->tgroups, tgroup, link);
     417           1 :                         return -EBUSY;
     418             :                 }
     419             : 
     420           1 :         }
     421             : 
     422           8 :         if (fgrp) {
     423           0 :                 if (group->enable_interrupts) {
     424           0 :                         spdk_fd_group_remove(fgrp, group->disconnect_qpair_fd);
     425           0 :                         close(group->disconnect_qpair_fd);
     426           0 :                 }
     427           0 :                 spdk_fd_group_destroy(fgrp);
     428           0 :         }
     429             : 
     430           8 :         free(group);
     431             : 
     432           8 :         return 0;
     433           9 : }
     434             : 
     435             : int
     436           2 : spdk_nvme_poll_group_get_stats(struct spdk_nvme_poll_group *group,
     437             :                                struct spdk_nvme_poll_group_stat **stats)
     438             : {
     439             :         struct spdk_nvme_transport_poll_group *tgroup;
     440             :         struct spdk_nvme_poll_group_stat *result;
     441           2 :         uint32_t transports_count = 0;
     442             :         /* Not all transports used by this poll group may support statistics reporting */
     443           2 :         uint32_t reported_stats_count = 0;
     444             :         int rc;
     445             : 
     446           2 :         assert(group);
     447           2 :         assert(stats);
     448             : 
     449           2 :         result = calloc(1, sizeof(*result));
     450           2 :         if (!result) {
     451           0 :                 SPDK_ERRLOG("Failed to allocate memory for poll group statistics\n");
     452           0 :                 return -ENOMEM;
     453             :         }
     454             : 
     455           5 :         STAILQ_FOREACH(tgroup, &group->tgroups, link) {
     456           3 :                 transports_count++;
     457           3 :         }
     458             : 
     459           2 :         result->transport_stat = calloc(transports_count, sizeof(*result->transport_stat));
     460           2 :         if (!result->transport_stat) {
     461           0 :                 SPDK_ERRLOG("Failed to allocate memory for poll group statistics\n");
     462           0 :                 free(result);
     463           0 :                 return -ENOMEM;
     464             :         }
     465             : 
     466           5 :         STAILQ_FOREACH(tgroup, &group->tgroups, link) {
     467           3 :                 rc = nvme_transport_poll_group_get_stats(tgroup, &result->transport_stat[reported_stats_count]);
     468           3 :                 if (rc == 0) {
     469           3 :                         reported_stats_count++;
     470           3 :                 }
     471           3 :         }
     472             : 
     473           2 :         if (reported_stats_count == 0) {
     474           1 :                 free(result->transport_stat);
     475           1 :                 free(result);
     476           1 :                 SPDK_DEBUGLOG(nvme, "No transport statistics available\n");
     477           1 :                 return -ENOTSUP;
     478             :         }
     479             : 
     480           1 :         result->num_transports = reported_stats_count;
     481           1 :         *stats = result;
     482             : 
     483           1 :         return 0;
     484           2 : }
     485             : 
     486             : void
     487           1 : spdk_nvme_poll_group_free_stats(struct spdk_nvme_poll_group *group,
     488             :                                 struct spdk_nvme_poll_group_stat *stat)
     489             : {
     490             :         struct spdk_nvme_transport_poll_group *tgroup;
     491             :         uint32_t i;
     492           1 :         uint32_t freed_stats __attribute__((unused)) = 0;
     493             : 
     494           1 :         assert(group);
     495           1 :         assert(stat);
     496             : 
     497           4 :         for (i = 0; i < stat->num_transports; i++) {
     498           3 :                 STAILQ_FOREACH(tgroup, &group->tgroups, link) {
     499           3 :                         if (nvme_transport_get_trtype(tgroup->transport) == stat->transport_stat[i]->trtype) {
     500           3 :                                 nvme_transport_poll_group_free_stats(tgroup, stat->transport_stat[i]);
     501           3 :                                 freed_stats++;
     502           3 :                                 break;
     503             :                         }
     504           0 :                 }
     505           3 :         }
     506             : 
     507           1 :         assert(freed_stats == stat->num_transports);
     508             : 
     509           1 :         free(stat->transport_stat);
     510           1 :         free(stat);
     511           1 : }

Generated by: LCOV version 1.15