Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright (C) 2016 Intel Corporation.
3 : : * All rights reserved.
4 : : */
5 : :
6 : : #include "spdk/stdinc.h"
7 : :
8 : : #include "spdk/nvme.h"
9 : : #include "spdk/vmd.h"
10 : : #include "spdk/nvme_zns.h"
11 : : #include "spdk/env.h"
12 : : #include "spdk/string.h"
13 : : #include "spdk/log.h"
14 : :
15 : : #define DATA_BUFFER_STRING "Hello world!"
16 : :
17 : : struct ctrlr_entry {
18 : : struct spdk_nvme_ctrlr *ctrlr;
19 : : TAILQ_ENTRY(ctrlr_entry) link;
20 : : char name[1024];
21 : : };
22 : :
23 : : struct ns_entry {
24 : : struct spdk_nvme_ctrlr *ctrlr;
25 : : struct spdk_nvme_ns *ns;
26 : : TAILQ_ENTRY(ns_entry) link;
27 : : struct spdk_nvme_qpair *qpair;
28 : : };
29 : :
30 : : static TAILQ_HEAD(, ctrlr_entry) g_controllers = TAILQ_HEAD_INITIALIZER(g_controllers);
31 : : static TAILQ_HEAD(, ns_entry) g_namespaces = TAILQ_HEAD_INITIALIZER(g_namespaces);
32 : : static struct spdk_nvme_transport_id g_trid = {};
33 : :
34 : : static bool g_vmd = false;
35 : :
36 : : static void
37 : 15 : register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns)
38 : : {
39 : : struct ns_entry *entry;
40 : :
41 [ - + ]: 15 : if (!spdk_nvme_ns_is_active(ns)) {
42 : 0 : return;
43 : : }
44 : :
45 : 15 : entry = malloc(sizeof(struct ns_entry));
46 [ - + ]: 15 : if (entry == NULL) {
47 : 0 : perror("ns_entry malloc");
48 : 0 : exit(1);
49 : : }
50 : :
51 : 15 : entry->ctrlr = ctrlr;
52 : 15 : entry->ns = ns;
53 : 15 : TAILQ_INSERT_TAIL(&g_namespaces, entry, link);
54 : :
55 [ - + ]: 15 : printf(" Namespace ID: %d size: %juGB\n", spdk_nvme_ns_get_id(ns),
56 : 15 : spdk_nvme_ns_get_size(ns) / 1000000000);
57 : : }
58 : :
59 : : struct hello_world_sequence {
60 : : struct ns_entry *ns_entry;
61 : : char *buf;
62 : : unsigned using_cmb_io;
63 : : int is_completed;
64 : : };
65 : :
66 : : static void
67 : 15 : read_complete(void *arg, const struct spdk_nvme_cpl *completion)
68 : : {
69 : 15 : struct hello_world_sequence *sequence = arg;
70 : :
71 : : /* Assume the I/O was successful */
72 : 15 : sequence->is_completed = 1;
73 : : /* See if an error occurred. If so, display information
74 : : * about it, and set completion value so that I/O
75 : : * caller is aware that an error occurred.
76 : : */
77 [ + - - + ]: 15 : if (spdk_nvme_cpl_is_error(completion)) {
78 : 0 : spdk_nvme_qpair_print_completion(sequence->ns_entry->qpair, (struct spdk_nvme_cpl *)completion);
79 [ # # ]: 0 : fprintf(stderr, "I/O error status: %s\n", spdk_nvme_cpl_get_status_string(&completion->status));
80 [ # # ]: 0 : fprintf(stderr, "Read I/O failed, aborting run\n");
81 : 0 : sequence->is_completed = 2;
82 : 0 : exit(1);
83 : : }
84 : :
85 [ - + - + ]: 15 : if (strcmp(sequence->buf, DATA_BUFFER_STRING)) {
86 [ # # ]: 0 : fprintf(stderr, "Read data doesn't match write data\n");
87 : 0 : exit(1);
88 : : }
89 : :
90 : : /*
91 : : * The read I/O has completed. Print the contents of the
92 : : * buffer, free the buffer, then mark the sequence as
93 : : * completed. This will trigger the hello_world() function
94 : : * to exit its polling loop.
95 : : */
96 [ - + ]: 15 : printf("%s\n", sequence->buf);
97 : 15 : spdk_free(sequence->buf);
98 : 15 : }
99 : :
100 : : static void
101 : 15 : write_complete(void *arg, const struct spdk_nvme_cpl *completion)
102 : : {
103 : 15 : struct hello_world_sequence *sequence = arg;
104 : 15 : struct ns_entry *ns_entry = sequence->ns_entry;
105 : : int rc;
106 : :
107 : : /* See if an error occurred. If so, display information
108 : : * about it, and set completion value so that I/O
109 : : * caller is aware that an error occurred.
110 : : */
111 [ + - - + ]: 15 : if (spdk_nvme_cpl_is_error(completion)) {
112 : 0 : spdk_nvme_qpair_print_completion(sequence->ns_entry->qpair, (struct spdk_nvme_cpl *)completion);
113 [ # # # # ]: 0 : fprintf(stderr, "I/O error status: %s\n", spdk_nvme_cpl_get_status_string(&completion->status));
114 [ # # # # ]: 0 : fprintf(stderr, "Write I/O failed, aborting run\n");
115 : 0 : sequence->is_completed = 2;
116 : 0 : exit(1);
117 : : }
118 : : /*
119 : : * The write I/O has completed. Free the buffer associated with
120 : : * the write I/O and allocate a new zeroed buffer for reading
121 : : * the data back from the NVMe namespace.
122 : : */
123 [ - + ]: 15 : if (sequence->using_cmb_io) {
124 : 0 : spdk_nvme_ctrlr_unmap_cmb(ns_entry->ctrlr);
125 : : } else {
126 : 15 : spdk_free(sequence->buf);
127 : : }
128 : 15 : sequence->buf = spdk_zmalloc(0x1000, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
129 : :
130 : 15 : rc = spdk_nvme_ns_cmd_read(ns_entry->ns, ns_entry->qpair, sequence->buf,
131 : : 0, /* LBA start */
132 : : 1, /* number of LBAs */
133 : : read_complete, (void *)sequence, 0);
134 [ - + ]: 15 : if (rc != 0) {
135 [ # # # # ]: 0 : fprintf(stderr, "starting read I/O failed\n");
136 : 0 : exit(1);
137 : : }
138 : 15 : }
139 : :
140 : : static void
141 : 0 : reset_zone_complete(void *arg, const struct spdk_nvme_cpl *completion)
142 : : {
143 : 0 : struct hello_world_sequence *sequence = arg;
144 : :
145 : : /* Assume the I/O was successful */
146 : 0 : sequence->is_completed = 1;
147 : : /* See if an error occurred. If so, display information
148 : : * about it, and set completion value so that I/O
149 : : * caller is aware that an error occurred.
150 : : */
151 [ # # # # ]: 0 : if (spdk_nvme_cpl_is_error(completion)) {
152 : 0 : spdk_nvme_qpair_print_completion(sequence->ns_entry->qpair, (struct spdk_nvme_cpl *)completion);
153 [ # # # # ]: 0 : fprintf(stderr, "I/O error status: %s\n", spdk_nvme_cpl_get_status_string(&completion->status));
154 [ # # # # ]: 0 : fprintf(stderr, "Reset zone I/O failed, aborting run\n");
155 : 0 : sequence->is_completed = 2;
156 : 0 : exit(1);
157 : : }
158 : 0 : }
159 : :
160 : : static void
161 : 0 : reset_zone_and_wait_for_completion(struct hello_world_sequence *sequence)
162 : : {
163 [ # # ]: 0 : if (spdk_nvme_zns_reset_zone(sequence->ns_entry->ns, sequence->ns_entry->qpair,
164 : : 0, /* starting LBA of the zone to reset */
165 : : false, /* don't reset all zones */
166 : : reset_zone_complete,
167 : : sequence)) {
168 [ # # # # ]: 0 : fprintf(stderr, "starting reset zone I/O failed\n");
169 : 0 : exit(1);
170 : : }
171 [ # # ]: 0 : while (!sequence->is_completed) {
172 : 0 : spdk_nvme_qpair_process_completions(sequence->ns_entry->qpair, 0);
173 : : }
174 : 0 : sequence->is_completed = 0;
175 : 0 : }
176 : :
177 : : static void
178 : 9 : hello_world(void)
179 : : {
180 : : struct ns_entry *ns_entry;
181 : 4 : struct hello_world_sequence sequence;
182 : : int rc;
183 : 4 : size_t sz;
184 : :
185 [ + + ]: 24 : TAILQ_FOREACH(ns_entry, &g_namespaces, link) {
186 : : /*
187 : : * Allocate an I/O qpair that we can use to submit read/write requests
188 : : * to namespaces on the controller. NVMe controllers typically support
189 : : * many qpairs per controller. Any I/O qpair allocated for a controller
190 : : * can submit I/O to any namespace on that controller.
191 : : *
192 : : * The SPDK NVMe driver provides no synchronization for qpair accesses -
193 : : * the application must ensure only a single thread submits I/O to a
194 : : * qpair, and that same thread must also check for completions on that
195 : : * qpair. This enables extremely efficient I/O processing by making all
196 : : * I/O operations completely lockless.
197 : : */
198 : 15 : ns_entry->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ns_entry->ctrlr, NULL, 0);
199 [ - + ]: 15 : if (ns_entry->qpair == NULL) {
200 [ # # ]: 0 : printf("ERROR: spdk_nvme_ctrlr_alloc_io_qpair() failed\n");
201 : 0 : return;
202 : : }
203 : :
204 : : /*
205 : : * Use spdk_dma_zmalloc to allocate a 4KB zeroed buffer. This memory
206 : : * will be pinned, which is required for data buffers used for SPDK NVMe
207 : : * I/O operations.
208 : : */
209 : 15 : sequence.using_cmb_io = 1;
210 : 15 : sequence.buf = spdk_nvme_ctrlr_map_cmb(ns_entry->ctrlr, &sz);
211 [ - + - - ]: 15 : if (sequence.buf == NULL || sz < 0x1000) {
212 : 15 : sequence.using_cmb_io = 0;
213 : 15 : sequence.buf = spdk_zmalloc(0x1000, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
214 : : }
215 [ - + ]: 15 : if (sequence.buf == NULL) {
216 [ # # ]: 0 : printf("ERROR: write buffer allocation failed\n");
217 : 0 : return;
218 : : }
219 [ - + ]: 15 : if (sequence.using_cmb_io) {
220 [ # # ]: 0 : printf("INFO: using controller memory buffer for IO\n");
221 : : } else {
222 [ - + ]: 15 : printf("INFO: using host memory buffer for IO\n");
223 : : }
224 : 15 : sequence.is_completed = 0;
225 : 15 : sequence.ns_entry = ns_entry;
226 : :
227 : : /*
228 : : * If the namespace is a Zoned Namespace, rather than a regular
229 : : * NVM namespace, we need to reset the first zone, before we
230 : : * write to it. This not needed for regular NVM namespaces.
231 : : */
232 [ - + ]: 15 : if (spdk_nvme_ns_get_csi(ns_entry->ns) == SPDK_NVME_CSI_ZNS) {
233 : 0 : reset_zone_and_wait_for_completion(&sequence);
234 : : }
235 : :
236 : : /*
237 : : * Print DATA_BUFFER_STRING to sequence.buf. We will write this data to LBA
238 : : * 0 on the namespace, and then later read it back into a separate buffer
239 : : * to demonstrate the full I/O path.
240 : : */
241 [ - + ]: 15 : snprintf(sequence.buf, 0x1000, "%s", DATA_BUFFER_STRING);
242 : :
243 : : /*
244 : : * Write the data buffer to LBA 0 of this namespace. "write_complete" and
245 : : * "&sequence" are specified as the completion callback function and
246 : : * argument respectively. write_complete() will be called with the
247 : : * value of &sequence as a parameter when the write I/O is completed.
248 : : * This allows users to potentially specify different completion
249 : : * callback routines for each I/O, as well as pass a unique handle
250 : : * as an argument so the application knows which I/O has completed.
251 : : *
252 : : * Note that the SPDK NVMe driver will only check for completions
253 : : * when the application calls spdk_nvme_qpair_process_completions().
254 : : * It is the responsibility of the application to trigger the polling
255 : : * process.
256 : : */
257 : 15 : rc = spdk_nvme_ns_cmd_write(ns_entry->ns, ns_entry->qpair, sequence.buf,
258 : : 0, /* LBA start */
259 : : 1, /* number of LBAs */
260 : : write_complete, &sequence, 0);
261 [ - + ]: 15 : if (rc != 0) {
262 [ # # # # ]: 0 : fprintf(stderr, "starting write I/O failed\n");
263 : 0 : exit(1);
264 : : }
265 : :
266 : : /*
267 : : * Poll for completions. 0 here means process all available completions.
268 : : * In certain usage models, the caller may specify a positive integer
269 : : * instead of 0 to signify the maximum number of completions it should
270 : : * process. This function will never block - if there are no
271 : : * completions pending on the specified qpair, it will return immediately.
272 : : *
273 : : * When the write I/O completes, write_complete() will submit a new I/O
274 : : * to read LBA 0 into a separate buffer, specifying read_complete() as its
275 : : * completion routine. When the read I/O completes, read_complete() will
276 : : * print the buffer contents and set sequence.is_completed = 1. That will
277 : : * break this loop and then exit the program.
278 : : */
279 [ + + ]: 19669 : while (!sequence.is_completed) {
280 : 19654 : spdk_nvme_qpair_process_completions(ns_entry->qpair, 0);
281 : : }
282 : :
283 : : /*
284 : : * Free the I/O qpair. This typically is done when an application exits.
285 : : * But SPDK does support freeing and then reallocating qpairs during
286 : : * operation. It is the responsibility of the caller to ensure all
287 : : * pending I/O are completed before trying to free the qpair.
288 : : */
289 : 15 : spdk_nvme_ctrlr_free_io_qpair(ns_entry->qpair);
290 : : }
291 : : }
292 : :
293 : : static bool
294 : 3 : probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
295 : : struct spdk_nvme_ctrlr_opts *opts)
296 : : {
297 [ - + ]: 3 : printf("Attaching to %s\n", trid->traddr);
298 : :
299 : 3 : return true;
300 : : }
301 : :
302 : : static void
303 : 13 : attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
304 : : struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
305 : : {
306 : : int nsid;
307 : : struct ctrlr_entry *entry;
308 : : struct spdk_nvme_ns *ns;
309 : : const struct spdk_nvme_ctrlr_data *cdata;
310 : :
311 : 13 : entry = malloc(sizeof(struct ctrlr_entry));
312 [ - + ]: 13 : if (entry == NULL) {
313 : 0 : perror("ctrlr_entry malloc");
314 : 0 : exit(1);
315 : : }
316 : :
317 [ - + ]: 13 : printf("Attached to %s\n", trid->traddr);
318 : :
319 : : /*
320 : : * spdk_nvme_ctrlr is the logical abstraction in SPDK for an NVMe
321 : : * controller. During initialization, the IDENTIFY data for the
322 : : * controller is read using an NVMe admin command, and that data
323 : : * can be retrieved using spdk_nvme_ctrlr_get_data() to get
324 : : * detailed information on the controller. Refer to the NVMe
325 : : * specification for more details on IDENTIFY for NVMe controllers.
326 : : */
327 : 13 : cdata = spdk_nvme_ctrlr_get_data(ctrlr);
328 : :
329 [ - + ]: 13 : snprintf(entry->name, sizeof(entry->name), "%-20.20s (%-20.20s)", cdata->mn, cdata->sn);
330 : :
331 : 13 : entry->ctrlr = ctrlr;
332 : 13 : TAILQ_INSERT_TAIL(&g_controllers, entry, link);
333 : :
334 : : /*
335 : : * Each controller has one or more namespaces. An NVMe namespace is basically
336 : : * equivalent to a SCSI LUN. The controller's IDENTIFY data tells us how
337 : : * many namespaces exist on the controller. For Intel(R) P3X00 controllers,
338 : : * it will just be one namespace.
339 : : *
340 : : * Note that in NVMe, namespace IDs start at 1, not 0.
341 : : */
342 [ + + ]: 28 : for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); nsid != 0;
343 : 15 : nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
344 : 15 : ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
345 [ - + ]: 15 : if (ns == NULL) {
346 : 0 : continue;
347 : : }
348 : 15 : register_ns(ctrlr, ns);
349 : : }
350 : 13 : }
351 : :
352 : : static void
353 : 9 : cleanup(void)
354 : : {
355 : : struct ns_entry *ns_entry, *tmp_ns_entry;
356 : : struct ctrlr_entry *ctrlr_entry, *tmp_ctrlr_entry;
357 : 9 : struct spdk_nvme_detach_ctx *detach_ctx = NULL;
358 : :
359 [ + + ]: 24 : TAILQ_FOREACH_SAFE(ns_entry, &g_namespaces, link, tmp_ns_entry) {
360 [ + + ]: 15 : TAILQ_REMOVE(&g_namespaces, ns_entry, link);
361 : 15 : free(ns_entry);
362 : : }
363 : :
364 [ + + ]: 22 : TAILQ_FOREACH_SAFE(ctrlr_entry, &g_controllers, link, tmp_ctrlr_entry) {
365 [ + + ]: 13 : TAILQ_REMOVE(&g_controllers, ctrlr_entry, link);
366 : 13 : spdk_nvme_detach_async(ctrlr_entry->ctrlr, &detach_ctx);
367 : 13 : free(ctrlr_entry);
368 : : }
369 : :
370 [ + + ]: 9 : if (detach_ctx) {
371 : 3 : spdk_nvme_detach_poll(detach_ctx);
372 : : }
373 : 9 : }
374 : :
375 : : static void
376 : 0 : usage(const char *program_name)
377 : : {
378 [ # # ]: 0 : printf("%s [options]", program_name);
379 [ # # ]: 0 : printf("\t\n");
380 [ # # ]: 0 : printf("options:\n");
381 [ # # ]: 0 : printf("\t[-d DPDK huge memory size in MB]\n");
382 [ # # ]: 0 : printf("\t[-g use single file descriptor for DPDK memory segments]\n");
383 [ # # ]: 0 : printf("\t[-i shared memory group ID]\n");
384 [ # # ]: 0 : printf("\t[-r remote NVMe over Fabrics target address]\n");
385 [ # # ]: 0 : printf("\t[-V enumerate VMD]\n");
386 : : #ifdef DEBUG
387 [ # # ]: 0 : printf("\t[-L enable debug logging]\n");
388 : : #else
389 : : printf("\t[-L enable debug logging (flag disabled, must reconfigure with --enable-debug)]\n");
390 : : #endif
391 : 0 : }
392 : :
393 : : static int
394 : 9 : parse_args(int argc, char **argv, struct spdk_env_opts *env_opts)
395 : : {
396 : : int op, rc;
397 : :
398 : 9 : spdk_nvme_trid_populate_transport(&g_trid, SPDK_NVME_TRANSPORT_PCIE);
399 [ - + ]: 9 : snprintf(g_trid.subnqn, sizeof(g_trid.subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
400 : :
401 [ + + + + : 22 : while ((op = getopt(argc, argv, "d:ghi:r:L:V")) != -1) {
+ + ]
402 [ + + + + : 13 : switch (op) {
+ - - - ]
403 : 1 : case 'V':
404 : 1 : g_vmd = true;
405 : 1 : break;
406 : 6 : case 'i':
407 : 6 : env_opts->shm_id = spdk_strtol(optarg, 10);
408 [ - + ]: 6 : if (env_opts->shm_id < 0) {
409 [ # # # # ]: 0 : fprintf(stderr, "Invalid shared memory ID\n");
410 : 0 : return env_opts->shm_id;
411 : : }
412 : 6 : break;
413 : 2 : case 'g':
414 : 2 : env_opts->hugepage_single_segments = true;
415 : 2 : break;
416 : 2 : case 'r':
417 [ - + ]: 2 : if (spdk_nvme_transport_id_parse(&g_trid, optarg) != 0) {
418 [ # # # # ]: 0 : fprintf(stderr, "Error parsing transport address\n");
419 : 0 : return 1;
420 : : }
421 : 2 : break;
422 : 2 : case 'd':
423 : 2 : env_opts->mem_size = spdk_strtol(optarg, 10);
424 [ - + ]: 2 : if (env_opts->mem_size < 0) {
425 [ # # # # ]: 0 : fprintf(stderr, "Invalid DPDK memory size\n");
426 : 0 : return env_opts->mem_size;
427 : : }
428 : 2 : break;
429 : 0 : case 'L':
430 : 0 : rc = spdk_log_set_flag(optarg);
431 [ # # ]: 0 : if (rc < 0) {
432 [ # # # # ]: 0 : fprintf(stderr, "unknown flag\n");
433 : 0 : usage(argv[0]);
434 : 0 : exit(EXIT_FAILURE);
435 : : }
436 : : #ifdef DEBUG
437 : 0 : spdk_log_set_print_level(SPDK_LOG_DEBUG);
438 : : #endif
439 : 0 : break;
440 : 0 : case 'h':
441 : 0 : usage(argv[0]);
442 : 0 : exit(EXIT_SUCCESS);
443 : 0 : default:
444 : 0 : usage(argv[0]);
445 : 0 : return 1;
446 : : }
447 : : }
448 : :
449 : 9 : return 0;
450 : : }
451 : :
452 : : int
453 : 9 : main(int argc, char **argv)
454 : : {
455 : : int rc;
456 : 4 : struct spdk_env_opts opts;
457 : :
458 : : /*
459 : : * SPDK relies on an abstraction around the local environment
460 : : * named env that handles memory allocation and PCI device operations.
461 : : * This library must be initialized first.
462 : : *
463 : : */
464 : 9 : spdk_env_opts_init(&opts);
465 : 9 : rc = parse_args(argc, argv, &opts);
466 [ - + ]: 9 : if (rc != 0) {
467 : 0 : return rc;
468 : : }
469 : :
470 : 9 : opts.name = "hello_world";
471 [ - + ]: 9 : if (spdk_env_init(&opts) < 0) {
472 [ # # # # ]: 0 : fprintf(stderr, "Unable to initialize SPDK env\n");
473 : 0 : return 1;
474 : : }
475 : :
476 [ - + ]: 9 : printf("Initializing NVMe Controllers\n");
477 : :
478 [ - + + + : 9 : if (g_vmd && spdk_vmd_init()) {
- + ]
479 [ # # # # ]: 0 : fprintf(stderr, "Failed to initialize VMD."
480 : : " Some NVMe devices can be unavailable.\n");
481 : : }
482 : :
483 : : /*
484 : : * Start the SPDK NVMe enumeration process. probe_cb will be called
485 : : * for each NVMe controller found, giving our application a choice on
486 : : * whether to attach to each controller. attach_cb will then be
487 : : * called for each controller after the SPDK NVMe driver has completed
488 : : * initializing the controller we chose to attach.
489 : : */
490 : 9 : rc = spdk_nvme_probe(&g_trid, NULL, probe_cb, attach_cb, NULL);
491 [ - + ]: 9 : if (rc != 0) {
492 [ # # # # ]: 0 : fprintf(stderr, "spdk_nvme_probe() failed\n");
493 : 0 : rc = 1;
494 : 0 : goto exit;
495 : : }
496 : :
497 [ - + ]: 9 : if (TAILQ_EMPTY(&g_controllers)) {
498 [ # # # # ]: 0 : fprintf(stderr, "no NVMe controllers found\n");
499 : 0 : rc = 1;
500 : 0 : goto exit;
501 : : }
502 : :
503 [ - + ]: 9 : printf("Initialization complete.\n");
504 : 9 : hello_world();
505 : :
506 : 9 : exit:
507 : 9 : fflush(stdout);
508 : 9 : cleanup();
509 [ - + + + ]: 9 : if (g_vmd) {
510 : 1 : spdk_vmd_fini();
511 : : }
512 : :
513 : 9 : spdk_env_fini();
514 : 9 : return rc;
515 : : }
|