Line data    Source code

       1             : /*   SPDX-License-Identifier: BSD-3-Clause
       2             :  *   Copyright (C) 2019 Intel Corporation.
       3             :  *   All rights reserved.
       4             :  */
       5             : 
       6             : #include "nvme_internal.h"
       7             : #include "nvme_io_msg.h"
       8             : 
       9             : #define SPDK_NVME_MSG_IO_PROCESS_SIZE 8
      10             : 
      11             : /**
      12             :  * Send message to IO queue.
      13             :  */
      14             : int
      15           9 : nvme_io_msg_send(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, spdk_nvme_io_msg_fn fn,
      16             :                  void *arg)
      17             : {
      18             :         int rc;
      19             :         struct spdk_nvme_io_msg *io;
      20             : 
      21             :         /* Protect requests ring against preemptive producers */
      22           9 :         pthread_mutex_lock(&ctrlr->external_io_msgs_lock);
      23             : 
      24           9 :         io = (struct spdk_nvme_io_msg *)calloc(1, sizeof(struct spdk_nvme_io_msg));
      25           9 :         if (!io) {
      26           0 :                 SPDK_ERRLOG("IO msg allocation failed.");
      27           0 :                 pthread_mutex_unlock(&ctrlr->external_io_msgs_lock);
      28           0 :                 return -ENOMEM;
      29             :         }
      30             : 
      31           9 :         io->ctrlr = ctrlr;
      32           9 :         io->nsid = nsid;
      33           9 :         io->fn = fn;
      34           9 :         io->arg = arg;
      35             : 
      36           9 :         rc = spdk_ring_enqueue(ctrlr->external_io_msgs, (void **)&io, 1, NULL);
      37           9 :         if (rc != 1) {
      38           0 :                 assert(false);
      39             :                 free(io);
      40             :                 pthread_mutex_unlock(&ctrlr->external_io_msgs_lock);
      41             :                 return -ENOMEM;
      42             :         }
      43             : 
      44           9 :         pthread_mutex_unlock(&ctrlr->external_io_msgs_lock);
      45             : 
      46           9 :         return 0;
      47           9 : }
      48             : 
      49             : int
      50           2 : nvme_io_msg_process(struct spdk_nvme_ctrlr *ctrlr)
      51             : {
      52             :         int i;
      53             :         int count;
      54             :         struct spdk_nvme_io_msg *io;
      55             :         void *requests[SPDK_NVME_MSG_IO_PROCESS_SIZE];
      56             : 
      57           2 :         if (!spdk_process_is_primary()) {
      58           0 :                 return 0;
      59             :         }
      60             : 
      61           2 :         if (!ctrlr->external_io_msgs || !ctrlr->external_io_msgs_qpair || ctrlr->prepare_for_reset) {
      62             :                 /* Not ready or pending reset */
      63           1 :                 return 0;
      64             :         }
      65             : 
      66           1 :         if (ctrlr->needs_io_msg_update) {
      67           0 :                 ctrlr->needs_io_msg_update = false;
      68           0 :                 nvme_io_msg_ctrlr_update(ctrlr);
      69           0 :         }
      70             : 
      71           1 :         spdk_nvme_qpair_process_completions(ctrlr->external_io_msgs_qpair, 0);
      72             : 
      73           1 :         count = spdk_ring_dequeue(ctrlr->external_io_msgs, requests,
      74             :                                   SPDK_NVME_MSG_IO_PROCESS_SIZE);
      75           1 :         if (count == 0) {
      76           0 :                 return 0;
      77             :         }
      78             : 
      79           9 :         for (i = 0; i < count; i++) {
      80           8 :                 io = requests[i];
      81             : 
      82           8 :                 assert(io != NULL);
      83             : 
      84           8 :                 io->fn(io->ctrlr, io->nsid, io->arg);
      85           8 :                 free(io);
      86           8 :         }
      87             : 
      88           1 :         return count;
      89           2 : }
      90             : 
      91             : static bool
      92           9 : nvme_io_msg_is_producer_registered(struct spdk_nvme_ctrlr *ctrlr,
      93             :                                    struct nvme_io_msg_producer *io_msg_producer)
      94             : {
      95             :         struct nvme_io_msg_producer *tmp;
      96             : 
      97          10 :         STAILQ_FOREACH(tmp, &ctrlr->io_producers, link) {
      98           6 :                 if (tmp == io_msg_producer) {
      99           5 :                         return true;
     100             :                 }
     101           1 :         }
     102           4 :         return false;
     103           9 : }
     104             : 
     105             : int
     106           5 : nvme_io_msg_ctrlr_register(struct spdk_nvme_ctrlr *ctrlr,
     107             :                            struct nvme_io_msg_producer *io_msg_producer)
     108             : {
     109           5 :         if (io_msg_producer == NULL) {
     110           0 :                 SPDK_ERRLOG("io_msg_producer cannot be NULL\n");
     111           0 :                 return -EINVAL;
     112             :         }
     113             : 
     114           5 :         nvme_ctrlr_lock(ctrlr);
     115           5 :         if (nvme_io_msg_is_producer_registered(ctrlr, io_msg_producer)) {
     116           1 :                 nvme_ctrlr_unlock(ctrlr);
     117           1 :                 return -EEXIST;
     118             :         }
     119             : 
     120           4 :         if (!STAILQ_EMPTY(&ctrlr->io_producers) || ctrlr->is_resetting) {
     121             :                 /* There are registered producers - IO messaging already started */
     122           1 :                 STAILQ_INSERT_TAIL(&ctrlr->io_producers, io_msg_producer, link);
     123           1 :                 nvme_ctrlr_unlock(ctrlr);
     124           1 :                 return 0;
     125             :         }
     126             : 
     127           3 :         pthread_mutex_init(&ctrlr->external_io_msgs_lock, NULL);
     128             : 
     129             :         /**
     130             :          * Initialize ring and qpair for controller
     131             :          */
     132           3 :         ctrlr->external_io_msgs = spdk_ring_create(SPDK_RING_TYPE_MP_SC, 65536, SPDK_ENV_NUMA_ID_ANY);
     133           3 :         if (!ctrlr->external_io_msgs) {
     134           0 :                 SPDK_ERRLOG("Unable to allocate memory for message ring\n");
     135           0 :                 nvme_ctrlr_unlock(ctrlr);
     136           0 :                 return -ENOMEM;
     137             :         }
     138             : 
     139           3 :         ctrlr->external_io_msgs_qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, NULL, 0);
     140           3 :         if (ctrlr->external_io_msgs_qpair == NULL) {
     141           0 :                 SPDK_ERRLOG("spdk_nvme_ctrlr_alloc_io_qpair() failed\n");
     142           0 :                 spdk_ring_free(ctrlr->external_io_msgs);
     143           0 :                 ctrlr->external_io_msgs = NULL;
     144           0 :                 nvme_ctrlr_unlock(ctrlr);
     145           0 :                 return -ENOMEM;
     146             :         }
     147             : 
     148           3 :         STAILQ_INSERT_TAIL(&ctrlr->io_producers, io_msg_producer, link);
     149           3 :         nvme_ctrlr_unlock(ctrlr);
     150             : 
     151           3 :         return 0;
     152           5 : }
     153             : 
     154             : void
     155           0 : nvme_io_msg_ctrlr_update(struct spdk_nvme_ctrlr *ctrlr)
     156             : {
     157             :         struct nvme_io_msg_producer *io_msg_producer;
     158             : 
     159           0 :         if (!spdk_process_is_primary()) {
     160           0 :                 ctrlr->needs_io_msg_update = true;
     161           0 :                 return;
     162             :         }
     163             : 
     164             :         /* Update all producers */
     165           0 :         nvme_ctrlr_lock(ctrlr);
     166           0 :         STAILQ_FOREACH(io_msg_producer, &ctrlr->io_producers, link) {
     167           0 :                 io_msg_producer->update(ctrlr);
     168           0 :         }
     169           0 :         nvme_ctrlr_unlock(ctrlr);
     170           0 : }
     171             : 
     172             : void
     173           3 : nvme_io_msg_ctrlr_detach(struct spdk_nvme_ctrlr *ctrlr)
     174             : {
     175             :         struct nvme_io_msg_producer *io_msg_producer, *tmp;
     176             : 
     177           3 :         if (!spdk_process_is_primary()) {
     178           0 :                 return;
     179             :         }
     180             : 
     181             :         /* Stop all producers */
     182           3 :         STAILQ_FOREACH_SAFE(io_msg_producer, &ctrlr->io_producers, link, tmp) {
     183           0 :                 io_msg_producer->stop(ctrlr);
     184           0 :                 STAILQ_REMOVE(&ctrlr->io_producers, io_msg_producer, nvme_io_msg_producer, link);
     185           0 :         }
     186             : 
     187           3 :         if (ctrlr->external_io_msgs) {
     188           3 :                 spdk_ring_free(ctrlr->external_io_msgs);
     189           3 :                 ctrlr->external_io_msgs = NULL;
     190           3 :         }
     191             : 
     192           3 :         if (ctrlr->external_io_msgs_qpair) {
     193           3 :                 spdk_nvme_ctrlr_free_io_qpair(ctrlr->external_io_msgs_qpair);
     194           3 :                 ctrlr->external_io_msgs_qpair = NULL;
     195           3 :         }
     196             : 
     197           3 :         pthread_mutex_destroy(&ctrlr->external_io_msgs_lock);
     198           3 : }
     199             : 
     200             : void
     201           4 : nvme_io_msg_ctrlr_unregister(struct spdk_nvme_ctrlr *ctrlr,
     202             :                              struct nvme_io_msg_producer *io_msg_producer)
     203             : {
     204           4 :         assert(io_msg_producer != NULL);
     205             : 
     206           4 :         nvme_ctrlr_lock(ctrlr);
     207           4 :         if (!nvme_io_msg_is_producer_registered(ctrlr, io_msg_producer)) {
     208           0 :                 nvme_ctrlr_unlock(ctrlr);
     209           0 :                 return;
     210             :         }
     211             : 
     212           4 :         STAILQ_REMOVE(&ctrlr->io_producers, io_msg_producer, nvme_io_msg_producer, link);
     213           4 :         if (STAILQ_EMPTY(&ctrlr->io_producers)) {
     214           3 :                 nvme_io_msg_ctrlr_detach(ctrlr);
     215           3 :         }
     216           4 :         nvme_ctrlr_unlock(ctrlr);
     217           4 : }