Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2016 Intel Corporation.
3 : * All rights reserved.
4 : * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : */
6 :
7 : /** \file
8 : * Thread
9 : */
10 :
11 : #ifndef SPDK_THREAD_H_
12 : #define SPDK_THREAD_H_
13 :
14 : #ifdef __linux__
15 : #include <sys/epoll.h>
16 : #endif
17 :
18 : #include "spdk/fd_group.h"
19 : #include "spdk/stdinc.h"
20 : #include "spdk/assert.h"
21 : #include "spdk/cpuset.h"
22 : #include "spdk/env.h"
23 : #include "spdk/util.h"
24 : #include "spdk/likely.h"
25 :
26 : #ifdef __cplusplus
27 : extern "C" {
28 : #endif
29 :
30 : /**
31 : * Pollers should always return a value of this type
32 : * indicating whether they did real work or not.
33 : */
34 : enum spdk_thread_poller_rc {
35 : SPDK_POLLER_IDLE,
36 : SPDK_POLLER_BUSY,
37 : };
38 :
39 : /**
40 : * A stackless, lightweight thread.
41 : */
42 : struct spdk_thread;
43 :
44 : /**
45 : * A function repeatedly called on the same spdk_thread.
46 : */
47 : struct spdk_poller;
48 :
49 : struct spdk_io_channel_iter;
50 :
51 : /**
52 : * A function that is called each time a new thread is created.
53 : * The implementer of this function should frequently call
54 : * spdk_thread_poll() on the thread provided.
55 : *
56 : * \param thread The new spdk_thread.
57 : */
58 : typedef int (*spdk_new_thread_fn)(struct spdk_thread *thread);
59 :
60 : /**
61 : * SPDK thread operation type.
62 : */
63 : enum spdk_thread_op {
64 : /* Called each time a new thread is created. The implementer of this operation
65 : * should frequently call spdk_thread_poll() on the thread provided.
66 : */
67 : SPDK_THREAD_OP_NEW,
68 :
69 : /* Called when SPDK thread needs to be rescheduled. (e.g., when cpumask of the
70 : * SPDK thread is updated.
71 : */
72 : SPDK_THREAD_OP_RESCHED,
73 : };
74 :
75 : /**
76 : * Function to be called for SPDK thread operation.
77 : */
78 : typedef int (*spdk_thread_op_fn)(struct spdk_thread *thread, enum spdk_thread_op op);
79 :
80 : /**
81 : * Function to check whether the SPDK thread operation is supported.
82 : */
83 : typedef bool (*spdk_thread_op_supported_fn)(enum spdk_thread_op op);
84 :
85 : /**
86 : * A function that will be called on the target thread.
87 : *
88 : * \param ctx Context passed as arg to spdk_thread_pass_msg().
89 : */
90 : typedef void (*spdk_msg_fn)(void *ctx);
91 :
92 : /**
93 : * Function to be called to pass a message to a thread.
94 : *
95 : * \param fn Callback function for a thread.
96 : * \param ctx Context passed to fn.
97 : * \param thread_ctx Context for the thread.
98 : */
99 : typedef void (*spdk_thread_pass_msg)(spdk_msg_fn fn, void *ctx,
100 : void *thread_ctx);
101 :
102 : /**
103 : * Callback function for a poller.
104 : *
105 : * \param ctx Context passed as arg to spdk_poller_register().
106 : * \return value of type `enum spdk_thread_poller_rc` (ex: SPDK_POLLER_IDLE
107 : * if no work was done or SPDK_POLLER_BUSY if work was done.)
108 : */
109 : typedef int (*spdk_poller_fn)(void *ctx);
110 :
111 : /**
112 : * Callback function to set poller into interrupt mode or back to poll mode.
113 : *
114 : * \param poller Poller to set interrupt or poll mode.
115 : * \param cb_arg Argument passed to the callback function.
116 : * \param interrupt_mode Set interrupt mode for true, or poll mode for false
117 : */
118 : typedef void (*spdk_poller_set_interrupt_mode_cb)(struct spdk_poller *poller, void *cb_arg,
119 : bool interrupt_mode);
120 :
121 : /**
122 : * Mark that the poller is capable of entering interrupt mode.
123 : *
124 : * When registering the poller set interrupt callback, the callback will get
125 : * executed immediately if its spdk_thread is in the interrupt mode.
126 : *
127 : * Callers may pass NULL for the cb_fn, signifying that no callback is
128 : * necessary when the interrupt mode changes.
129 : *
130 : * \param poller The poller to register callback function.
131 : * \param cb_fn Callback function called when the poller must transition into or out of interrupt mode
132 : * \param cb_arg Argument passed to the callback function.
133 : */
134 : void spdk_poller_register_interrupt(struct spdk_poller *poller,
135 : spdk_poller_set_interrupt_mode_cb cb_fn,
136 : void *cb_arg);
137 :
138 : /**
139 : * I/O channel creation callback.
140 : *
141 : * \param io_device I/O device associated with this channel.
142 : * \param ctx_buf Context for the I/O device.
143 : */
144 : typedef int (*spdk_io_channel_create_cb)(void *io_device, void *ctx_buf);
145 :
146 : /**
147 : * I/O channel destruction callback.
148 : *
149 : * \param io_device I/O device associated with this channel.
150 : * \param ctx_buf Context for the I/O device.
151 : */
152 : typedef void (*spdk_io_channel_destroy_cb)(void *io_device, void *ctx_buf);
153 :
154 : /**
155 : * I/O device unregister callback.
156 : *
157 : * \param io_device Unregistered I/O device.
158 : */
159 : typedef void (*spdk_io_device_unregister_cb)(void *io_device);
160 :
161 : /**
162 : * Called on the appropriate thread for each channel associated with io_device.
163 : *
164 : * \param i I/O channel iterator.
165 : */
166 : typedef void (*spdk_channel_msg)(struct spdk_io_channel_iter *i);
167 :
168 : /**
169 : * spdk_for_each_channel() callback.
170 : *
171 : * \param i I/O channel iterator.
172 : * \param status 0 if it completed successfully, or negative errno if it failed.
173 : */
174 : typedef void (*spdk_channel_for_each_cpl)(struct spdk_io_channel_iter *i, int status);
175 :
176 : #define SPDK_IO_CHANNEL_STRUCT_SIZE 96
177 :
178 : /**
179 : * Message memory pool size definitions
180 : */
181 : #define SPDK_MSG_MEMPOOL_CACHE_SIZE 1024
182 : /* Power of 2 minus 1 is optimal for memory consumption */
183 : #define SPDK_DEFAULT_MSG_MEMPOOL_SIZE (262144 - 1)
184 :
185 : /**
186 : * Initialize the threading library. Must be called once prior to allocating any threads.
187 : *
188 : * \param new_thread_fn Called each time a new SPDK thread is created. The implementer
189 : * is expected to frequently call spdk_thread_poll() on the provided thread.
190 : * \param ctx_sz For each thread allocated, an additional region of memory of
191 : * size ctx_size will also be allocated, for use by the thread scheduler. A pointer
192 : * to this region may be obtained by calling spdk_thread_get_ctx().
193 : *
194 : * \return 0 on success. Negated errno on failure.
195 : */
196 : int spdk_thread_lib_init(spdk_new_thread_fn new_thread_fn, size_t ctx_sz);
197 :
198 : /**
199 : * Initialize the threading library. Must be called once prior to allocating any threads
200 : *
201 : * Both thread_op_fn and thread_op_type_supported_fn have to be specified or not
202 : * specified together.
203 : *
204 : * \param thread_op_fn Called for SPDK thread operation.
205 : * \param thread_op_supported_fn Called to check whether the SPDK thread operation is supported.
206 : * \param ctx_sz For each thread allocated, for use by the thread scheduler. A pointer
207 : * to this region may be obtained by calling spdk_thread_get_ctx().
208 : * \param msg_mempool_size Size of the allocated spdk_msg_mempool.
209 : *
210 : * \return 0 on success. Negated errno on failure.
211 : */
212 : int spdk_thread_lib_init_ext(spdk_thread_op_fn thread_op_fn,
213 : spdk_thread_op_supported_fn thread_op_supported_fn,
214 : size_t ctx_sz, size_t msg_mempool_size);
215 :
216 : /**
217 : * Release all resources associated with this library.
218 : */
219 : void spdk_thread_lib_fini(void);
220 :
221 : /**
222 : * Creates a new SPDK thread object.
223 : *
224 : * Note that the first thread created via spdk_thread_create() will be designated as
225 : * the app thread. Other SPDK libraries may place restrictions on certain APIs to
226 : * only be called in the context of this app thread.
227 : *
228 : * \param name Human-readable name for the thread; can be retrieved with spdk_thread_get_name().
229 : * The string is copied, so the pointed-to data only needs to be valid during the
230 : * spdk_thread_create() call. May be NULL to specify no name.
231 : * \param cpumask Optional mask of CPU cores on which to schedule this thread. This is only
232 : * a suggestion to the scheduler. The value is copied, so cpumask may be released when
233 : * this function returns. May be NULL if no mask is required.
234 : *
235 : * \return a pointer to the allocated thread on success or NULL on failure..
236 : */
237 : struct spdk_thread *spdk_thread_create(const char *name, const struct spdk_cpuset *cpumask);
238 :
239 : /**
240 : * Return the app thread.
241 : *
242 : * The app thread is the first thread created using spdk_thread_create().
243 : *
244 : * \return a pointer to the app thread, or NULL if no thread has been created yet.
245 : */
246 : struct spdk_thread *spdk_thread_get_app_thread(void);
247 :
248 : /**
249 : * Check if the specified spdk_thread is the app thread.
250 : *
251 : * \param thread The thread to check. If NULL, check the current spdk_thread.
252 : * \return true if the specified spdk_thread is the app thread, false otherwise.
253 : */
254 : bool spdk_thread_is_app_thread(struct spdk_thread *thread);
255 :
256 : /**
257 : * Force the current system thread to act as if executing the given SPDK thread.
258 : *
259 : * \param thread The thread to set.
260 : */
261 : void spdk_set_thread(struct spdk_thread *thread);
262 :
263 : /**
264 : * Bind or unbind spdk_thread to its current CPU core.
265 : *
266 : * If spdk_thread is bound, it couldn't be rescheduled to other CPU cores until it is unbound.
267 : *
268 : * \param thread The thread to bind or not.
269 : * \param bind true for bind, false for unbind.
270 : */
271 : void spdk_thread_bind(struct spdk_thread *thread, bool bind);
272 :
273 : /**
274 : * Returns whether the thread is bound to its current CPU core.
275 : *
276 : * \param thread The thread to query.
277 : *
278 : * \return true if bound, false otherwise
279 : */
280 : bool spdk_thread_is_bound(struct spdk_thread *thread);
281 :
282 : /**
283 : * Mark the thread as exited, failing all future spdk_thread_send_msg(),
284 : * spdk_poller_register(), and spdk_get_io_channel() calls. May only be called
285 : * within an spdk poller or message.
286 : *
287 : * All I/O channel references associated with the thread must be released
288 : * using spdk_put_io_channel(), and all active pollers associated with the thread
289 : * should be unregistered using spdk_poller_unregister(), prior to calling
290 : * this function. This function will complete these processing. The completion can
291 : * be queried by spdk_thread_is_exited().
292 : *
293 : * Note that this function must not be called on the app thread until after it
294 : * has been called for all other threads.
295 : *
296 : * \param thread The thread to exit.
297 : *
298 : * \return always 0. (return value was deprecated but keep it for ABI compatibility.)
299 : */
300 : int spdk_thread_exit(struct spdk_thread *thread);
301 :
302 : /**
303 : * Returns whether the thread is marked as exited.
304 : *
305 : * A thread is exited only after it has spdk_thread_exit() called on it, and
306 : * it has been polled until any outstanding operations targeting this
307 : * thread have completed. This may include poller unregistrations, io channel
308 : * unregistrations, or outstanding spdk_thread_send_msg calls.
309 : *
310 : * \param thread The thread to query.
311 : *
312 : * \return true if marked as exited, false otherwise.
313 : */
314 : bool spdk_thread_is_exited(struct spdk_thread *thread);
315 :
316 : /**
317 : * Returns whether the thread is still running.
318 : *
319 : * A thread is considered running until it has * spdk_thread_exit() called on it.
320 : *
321 : * \param thread The thread to query.
322 : *
323 : * \return true if still running, false otherwise.
324 : */
325 : bool spdk_thread_is_running(struct spdk_thread *thread);
326 :
327 : /**
328 : * Destroy a thread, releasing all of its resources. May only be called
329 : * on a thread previously marked as exited.
330 : *
331 : * \param thread The thread to destroy.
332 : *
333 : */
334 : void spdk_thread_destroy(struct spdk_thread *thread);
335 :
336 : /**
337 : * Return a pointer to this thread's context.
338 : *
339 : * \param thread The thread on which to get the context.
340 : *
341 : * \return a pointer to the per-thread context, or NULL if there is
342 : * no per-thread context.
343 : */
344 : void *spdk_thread_get_ctx(struct spdk_thread *thread);
345 :
346 : /**
347 : * Get the thread's cpumask.
348 : *
349 : * \param thread The thread to get the cpumask for.
350 : *
351 : * \return cpuset pointer
352 : */
353 : struct spdk_cpuset *spdk_thread_get_cpumask(struct spdk_thread *thread);
354 :
355 : /**
356 : * Set the current thread's cpumask to the specified value. The thread may be
357 : * rescheduled to one of the CPUs specified in the cpumask.
358 : *
359 : * This API requires SPDK thread operation supports SPDK_THREAD_OP_RESCHED.
360 : *
361 : * \param cpumask The new cpumask for the thread.
362 : *
363 : * \return 0 on success, negated errno otherwise.
364 : */
365 : int spdk_thread_set_cpumask(struct spdk_cpuset *cpumask);
366 :
367 : /**
368 : * Return the thread object associated with the context handle previously
369 : * obtained by calling spdk_thread_get_ctx().
370 : *
371 : * \param ctx A context previously obtained by calling spdk_thread_get_ctx()
372 : *
373 : * \return The associated thread.
374 : */
375 : struct spdk_thread *spdk_thread_get_from_ctx(void *ctx);
376 :
377 : /**
378 : * Perform one iteration worth of processing on the thread. This includes
379 : * both expired and continuous pollers as well as messages. If the thread
380 : * has exited, return immediately.
381 : *
382 : * \param thread The thread to process
383 : * \param max_msgs The maximum number of messages that will be processed.
384 : * Use 0 to process the default number of messages (8).
385 : * \param now The current time, in ticks. Optional. If 0 is passed, this
386 : * function will call spdk_get_ticks() to get the current time.
387 : * The current time is used as start time and this function
388 : * will call spdk_get_ticks() at its end to know end time to
389 : * measure run time of this function.
390 : *
391 : * \return 1 if work was done. 0 if no work was done.
392 : */
393 : int spdk_thread_poll(struct spdk_thread *thread, uint32_t max_msgs, uint64_t now);
394 :
395 : /**
396 : * Return the number of ticks until the next timed poller
397 : * would expire. Timed pollers are pollers for which
398 : * period_microseconds is greater than 0.
399 : *
400 : * \param thread The thread to check poller expiration times on
401 : *
402 : * \return Number of ticks. If no timed pollers, return 0.
403 : */
404 : uint64_t spdk_thread_next_poller_expiration(struct spdk_thread *thread);
405 :
406 : /**
407 : * Returns whether there are any active pollers (pollers for which
408 : * period_microseconds equals 0) registered to be run on the thread.
409 : *
410 : * \param thread The thread to check.
411 : *
412 : * \return 1 if there is at least one active poller, 0 otherwise.
413 : */
414 : int spdk_thread_has_active_pollers(struct spdk_thread *thread);
415 :
416 : /**
417 : * Returns whether there are any pollers registered to be run
418 : * on the thread.
419 : *
420 : * \param thread The thread to check.
421 : *
422 : * \return true if there is any active poller, false otherwise.
423 : */
424 : bool spdk_thread_has_pollers(struct spdk_thread *thread);
425 :
426 : /**
427 : * Returns whether there are scheduled operations to be run on the thread.
428 : *
429 : * \param thread The thread to check.
430 : *
431 : * \return true if there are no scheduled operations, false otherwise.
432 : */
433 : bool spdk_thread_is_idle(struct spdk_thread *thread);
434 :
435 : /**
436 : * Get count of allocated threads.
437 : */
438 : uint32_t spdk_thread_get_count(void);
439 :
440 : /**
441 : * Get a handle to the current thread.
442 : *
443 : * This handle may be passed to other threads and used as the target of
444 : * spdk_thread_send_msg().
445 : *
446 : * \sa spdk_io_channel_get_thread()
447 : *
448 : * \return a pointer to the current thread on success or NULL on failure.
449 : */
450 : struct spdk_thread *spdk_get_thread(void);
451 :
452 : /**
453 : * Get a thread's name.
454 : *
455 : * \param thread Thread to query.
456 : *
457 : * \return the name of the thread.
458 : */
459 : const char *spdk_thread_get_name(const struct spdk_thread *thread);
460 :
461 : /**
462 : * Get a thread's ID.
463 : *
464 : * \param thread Thread to query.
465 : *
466 : * \return the ID of the thread..
467 : */
468 : uint64_t spdk_thread_get_id(const struct spdk_thread *thread);
469 :
470 : /**
471 : * Get the thread by the ID.
472 : *
473 : * \param id ID of the thread.
474 : * \return Thread whose ID matches or NULL otherwise.
475 : */
476 : struct spdk_thread *spdk_thread_get_by_id(uint64_t id);
477 :
478 : struct spdk_thread_stats {
479 : uint64_t busy_tsc;
480 : uint64_t idle_tsc;
481 : };
482 :
483 : /**
484 : * Get statistics about the current thread.
485 : *
486 : * Copy cumulative thread stats values to the provided thread stats structure.
487 : *
488 : * \param stats User's thread_stats structure.
489 : */
490 : int spdk_thread_get_stats(struct spdk_thread_stats *stats);
491 :
492 : /**
493 : * Return the TSC value from the end of the last time this thread was polled.
494 : *
495 : * \param thread Thread to query. If NULL, use current thread.
496 : *
497 : * \return TSC value from the end of the last time this thread was polled.
498 : */
499 : uint64_t spdk_thread_get_last_tsc(struct spdk_thread *thread);
500 :
501 : /**
502 : * Send a message to the given thread.
503 : *
504 : * The message will be sent asynchronously - i.e. spdk_thread_send_msg will always return
505 : * prior to `fn` being called.
506 : *
507 : * \param thread The target thread.
508 : * \param fn This function will be called on the given thread.
509 : * \param ctx This context will be passed to fn when called.
510 : *
511 : * \return 0 on success
512 : * \return -ENOMEM if the message could not be allocated
513 : * \return -EIO if the message could not be sent to the destination thread
514 : */
515 : int spdk_thread_send_msg(const struct spdk_thread *thread, spdk_msg_fn fn, void *ctx);
516 :
517 : /**
518 : * Send a message to the given thread. Only one critical message can be outstanding at the same
519 : * time. It's intended to use this function in any cases that might interrupt the execution of the
520 : * application, such as signal handlers.
521 : *
522 : * The message will be sent asynchronously - i.e. spdk_thread_send_critical_msg will always return
523 : * prior to `fn` being called.
524 : *
525 : * \param thread The target thread.
526 : * \param fn This function will be called on the given thread.
527 : *
528 : * \return 0 on success
529 : * \return -EIO if the message could not be sent to the destination thread, due to an already
530 : * outstanding critical message
531 : */
532 : int spdk_thread_send_critical_msg(struct spdk_thread *thread, spdk_msg_fn fn);
533 :
534 : /**
535 : * Run the msg callback on the given thread. If this happens to be the current
536 : * thread, the callback is executed immediately; otherwise a message is sent to
537 : * the thread, and it's run asynchronously.
538 : *
539 : * \param thread The target thread.
540 : * \param fn This function will be called on the given thread.
541 : * \param ctx This context will be passed to fn when called.
542 : *
543 : * \return 0 on success
544 : * \return -ENOMEM if the message could not be allocated
545 : * \return -EIO if the message could not be sent to the destination thread
546 : */
547 : static inline int
548 116 : spdk_thread_exec_msg(const struct spdk_thread *thread, spdk_msg_fn fn, void *ctx)
549 : {
550 116 : assert(thread != NULL);
551 :
552 116 : if (spdk_get_thread() == thread) {
553 114 : fn(ctx);
554 114 : return 0;
555 : }
556 :
557 2 : return spdk_thread_send_msg(thread, fn, ctx);
558 : }
559 :
560 : /**
561 : * Send a message to each thread, serially.
562 : *
563 : * The message is sent asynchronously - i.e. spdk_for_each_thread will return
564 : * prior to `fn` being called on each thread.
565 : *
566 : * \param fn This is the function that will be called on each thread.
567 : * \param ctx This context will be passed to fn when called.
568 : * \param cpl This will be called on the originating thread after `fn` has been
569 : * called on each thread.
570 : */
571 : void spdk_for_each_thread(spdk_msg_fn fn, void *ctx, spdk_msg_fn cpl);
572 :
573 : /**
574 : * Set current spdk_thread into interrupt mode or back to poll mode.
575 : *
576 : * Only valid when thread interrupt facility is enabled by
577 : * spdk_interrupt_mode_enable().
578 : *
579 : * \param enable_interrupt Set interrupt mode for true, or poll mode for false
580 : */
581 : void spdk_thread_set_interrupt_mode(bool enable_interrupt);
582 :
583 : /**
584 : * Register a poller on the current thread.
585 : *
586 : * The poller can be unregistered by calling spdk_poller_unregister().
587 : *
588 : * \param fn This function will be called every `period_microseconds`.
589 : * \param arg Argument passed to fn.
590 : * \param period_microseconds How often to call `fn`. If 0, call `fn` as often
591 : * as possible.
592 : *
593 : * \return a pointer to the poller registered on the current thread on success
594 : * or NULL on failure.
595 : */
596 : struct spdk_poller *spdk_poller_register(spdk_poller_fn fn,
597 : void *arg,
598 : uint64_t period_microseconds);
599 :
600 : /**
601 : * Register a poller on the current thread with arbitrary name.
602 : *
603 : * The poller can be unregistered by calling spdk_poller_unregister().
604 : *
605 : * \param fn This function will be called every `period_microseconds`.
606 : * \param arg Argument passed to fn.
607 : * \param period_microseconds How often to call `fn`. If 0, call `fn` as often
608 : * as possible.
609 : * \param name Human readable name for the poller. Pointer of the poller function
610 : * name is set if NULL.
611 : *
612 : * \return a pointer to the poller registered on the current thread on success
613 : * or NULL on failure.
614 : */
615 : struct spdk_poller *spdk_poller_register_named(spdk_poller_fn fn,
616 : void *arg,
617 : uint64_t period_microseconds,
618 : const char *name);
619 :
620 : /*
621 : * \brief Register a poller on the current thread with setting its name
622 : * to the string of the poller function name. The poller being registered
623 : * should return a value of type `enum spdk_thread_poller_rc`. See
624 : * \ref spdk_poller_fn for more information.
625 : */
626 : #define SPDK_POLLER_REGISTER(fn, arg, period_microseconds) \
627 : spdk_poller_register_named(fn, arg, period_microseconds, #fn)
628 :
629 : /**
630 : * Unregister a poller on the current thread.
631 : *
632 : * This function will also write NULL to the spdk_poller pointer pointed
633 : * to by ppoller, to help encourage a poller pointer not getting reused
634 : * after it has been unregistered.
635 : *
636 : * It is OK to pass a ppoller parameter that points to NULL, in this case
637 : * the function is a nop.
638 : *
639 : * \param ppoller The poller to unregister.
640 : */
641 : void spdk_poller_unregister(struct spdk_poller **ppoller);
642 :
643 : /**
644 : * Pause a poller on the current thread.
645 : *
646 : * The poller is not run until it is resumed with spdk_poller_resume(). It is
647 : * perfectly fine to pause an already paused poller.
648 : *
649 : * \param poller The poller to pause.
650 : */
651 : void spdk_poller_pause(struct spdk_poller *poller);
652 :
653 : /**
654 : * Resume a poller on the current thread.
655 : *
656 : * Resumes a poller paused with spdk_poller_pause(). It is perfectly fine to
657 : * resume an unpaused poller.
658 : *
659 : * \param poller The poller to resume.
660 : */
661 : void spdk_poller_resume(struct spdk_poller *poller);
662 :
663 : /**
664 : * Register the opaque io_device context as an I/O device.
665 : *
666 : * After an I/O device is registered, it can return I/O channels using the
667 : * spdk_get_io_channel() function.
668 : *
669 : * \param io_device The pointer to io_device context.
670 : * \param create_cb Callback function invoked to allocate any resources required
671 : * for a new I/O channel.
672 : * \param destroy_cb Callback function invoked to release the resources for an
673 : * I/O channel.
674 : * \param ctx_size The size of the context buffer allocated to store references
675 : * to allocated I/O channel resources.
676 : * \param name A string name for the device used only for debugging. Optional -
677 : * may be NULL.
678 : */
679 : void spdk_io_device_register(void *io_device, spdk_io_channel_create_cb create_cb,
680 : spdk_io_channel_destroy_cb destroy_cb, uint32_t ctx_size,
681 : const char *name);
682 :
683 : /**
684 : * Unregister the opaque io_device context as an I/O device.
685 : *
686 : * The actual unregistration might be deferred until all active I/O channels are
687 : * destroyed.
688 : *
689 : * \param io_device The pointer to io_device context.
690 : * \param unregister_cb An optional callback function invoked to release any
691 : * references to this I/O device.
692 : */
693 : void spdk_io_device_unregister(void *io_device, spdk_io_device_unregister_cb unregister_cb);
694 :
695 : /**
696 : * Get an I/O channel for the specified io_device to be used by the calling thread.
697 : *
698 : * The io_device context pointer specified must have previously been registered
699 : * using spdk_io_device_register(). If an existing I/O channel does not exist
700 : * yet for the given io_device on the calling thread, it will allocate an I/O
701 : * channel and invoke the create_cb function pointer specified in spdk_io_device_register().
702 : * If an I/O channel already exists for the given io_device on the calling thread,
703 : * its reference is returned rather than creating a new I/O channel.
704 : *
705 : * \param io_device The pointer to io_device context.
706 : *
707 : * \return a pointer to the I/O channel for this device on success or NULL on failure.
708 : */
709 : struct spdk_io_channel *spdk_get_io_channel(void *io_device);
710 :
711 : /**
712 : * Release a reference to an I/O channel. This happens asynchronously.
713 : *
714 : * This must be called on the same thread that called spdk_get_io_channel()
715 : * for the specified I/O channel. If this releases the last reference to the
716 : * I/O channel, The destroy_cb function specified in spdk_io_device_register()
717 : * will be invoked to release any associated resources.
718 : *
719 : * \param ch I/O channel to release a reference.
720 : */
721 : void spdk_put_io_channel(struct spdk_io_channel *ch);
722 :
723 : /**
724 : * Get the context buffer associated with an I/O channel.
725 : *
726 : * \param ch I/O channel.
727 : *
728 : * \return a pointer to the context buffer.
729 : */
730 : static inline void *
731 78214 : spdk_io_channel_get_ctx(struct spdk_io_channel *ch)
732 : {
733 78214 : if (spdk_unlikely(!ch)) {
734 0 : assert(false);
735 : return NULL;
736 : }
737 :
738 78214 : return (uint8_t *)ch + SPDK_IO_CHANNEL_STRUCT_SIZE;
739 : }
740 :
741 : /**
742 : * Get I/O channel from the context buffer. This is the inverse of
743 : * spdk_io_channel_get_ctx().
744 : *
745 : * \param ctx The pointer to the context buffer.
746 : *
747 : * \return a pointer to the I/O channel associated with the context buffer.
748 : */
749 : struct spdk_io_channel *spdk_io_channel_from_ctx(void *ctx);
750 :
751 : /**
752 : * Get the thread associated with an I/O channel.
753 : *
754 : * \param ch I/O channel.
755 : *
756 : * \return a pointer to the thread associated with the I/O channel
757 : */
758 : struct spdk_thread *spdk_io_channel_get_thread(struct spdk_io_channel *ch);
759 :
760 : /**
761 : * Call 'fn' on each channel associated with io_device.
762 : *
763 : * This happens asynchronously, so fn may be called after spdk_for_each_channel
764 : * returns. 'fn' will be called for each channel serially, such that two calls
765 : * to 'fn' will not overlap in time. After 'fn' has been called, call
766 : * spdk_for_each_channel_continue() to continue iterating.
767 : *
768 : * \param io_device 'fn' will be called on each channel associated with this io_device.
769 : * \param fn Called on the appropriate thread for each channel associated with io_device.
770 : * \param ctx Context buffer registered to spdk_io_channel_iter that can be obtained
771 : * form the function spdk_io_channel_iter_get_ctx().
772 : * \param cpl Called on the thread that spdk_for_each_channel was initially called
773 : * from when 'fn' has been called on each channel.
774 : */
775 : void spdk_for_each_channel(void *io_device, spdk_channel_msg fn, void *ctx,
776 : spdk_channel_for_each_cpl cpl);
777 :
778 : /**
779 : * Get io_device from the I/O channel iterator.
780 : *
781 : * \param i I/O channel iterator.
782 : *
783 : * \return a pointer to the io_device.
784 : */
785 : void *spdk_io_channel_iter_get_io_device(struct spdk_io_channel_iter *i);
786 :
787 : /**
788 : * Get I/O channel from the I/O channel iterator.
789 : *
790 : * \param i I/O channel iterator.
791 : *
792 : * \return a pointer to the I/O channel.
793 : */
794 : struct spdk_io_channel *spdk_io_channel_iter_get_channel(struct spdk_io_channel_iter *i);
795 :
796 : /**
797 : * Get context buffer from the I/O channel iterator.
798 : *
799 : * \param i I/O channel iterator.
800 : *
801 : * \return a pointer to the context buffer.
802 : */
803 : void *spdk_io_channel_iter_get_ctx(struct spdk_io_channel_iter *i);
804 :
805 : /**
806 : * Get the io_device for the specified I/O channel.
807 : *
808 : * \param ch I/O channel.
809 : *
810 : * \return a pointer to the io_device for the I/O channel
811 : */
812 : void *spdk_io_channel_get_io_device(struct spdk_io_channel *ch);
813 :
814 : /**
815 : * Helper function to iterate all channels for spdk_for_each_channel().
816 : *
817 : * \param i I/O channel iterator.
818 : * \param status Status for the I/O channel iterator;
819 : * for non 0 status remaining iterations are terminated.
820 : */
821 : void spdk_for_each_channel_continue(struct spdk_io_channel_iter *i, int status);
822 :
823 : /**
824 : * A representative for registered interrupt file descriptor.
825 : */
826 : struct spdk_interrupt;
827 :
828 : /**
829 : * Callback function registered for interrupt file descriptor.
830 : *
831 : * \param ctx Context passed as arg to spdk_interrupt_register().
832 : *
833 : * \return 0 to indicate that interrupt took place but no events were found;
834 : * positive to indicate that interrupt took place and some events were processed;
835 : * negative if no event information is provided.
836 : */
837 : typedef int (*spdk_interrupt_fn)(void *ctx);
838 :
839 : /**
840 : * Register an spdk_interrupt on the current thread.
841 : *
842 : * The provided function will be called any time a SPDK_INTERRUPT_EVENT_IN event
843 : * triggers on the associated file descriptor.
844 : *
845 : * \param efd File descriptor of the spdk_interrupt.
846 : * \param fn Called each time there are events in spdk_interrupt.
847 : * \param arg Function argument for fn.
848 : * \param name Human readable name for the spdk_interrupt. Pointer of the spdk_interrupt
849 : * name is set if NULL.
850 : *
851 : * \return a pointer to the spdk_interrupt registered on the current thread on success
852 : * or NULL on failure.
853 : */
854 : struct spdk_interrupt *spdk_interrupt_register(int efd, spdk_interrupt_fn fn,
855 : void *arg, const char *name);
856 :
857 : /**
858 : * Register an spdk_interrupt with specific event types on the current thread.
859 : *
860 : * The provided function will be called any time one of specified event types triggers on
861 : * the associated file descriptor.
862 : * Event types argument is a bit mask composed by ORing together
863 : * enum spdk_interrupt_event_types values.
864 : *
865 : * \param efd File descriptor of the spdk_interrupt.
866 : * \param events Event notification types.
867 : * \param fn Called each time there are events in spdk_interrupt.
868 : * \param arg Function argument for fn.
869 : * \param name Human readable name for the spdk_interrupt. Pointer of the spdk_interrupt
870 : * name is set if NULL.
871 : *
872 : * \return a pointer to the spdk_interrupt registered on the current thread on success
873 : * or NULL on failure.
874 : */
875 : struct spdk_interrupt *spdk_interrupt_register_for_events(int efd, uint32_t events,
876 : spdk_interrupt_fn fn, void *arg, const char *name);
877 :
878 : /*
879 : * \brief Register an spdk_interrupt on the current thread with setting its name
880 : * to the string of the spdk_interrupt function name.
881 : */
882 : #define SPDK_INTERRUPT_REGISTER(efd, fn, arg) \
883 : spdk_interrupt_register(efd, fn, arg, #fn)
884 :
885 : /*
886 : * \brief Register an spdk_interrupt on the current thread with specific event types
887 : * and with setting its name to the string of the spdk_interrupt function name.
888 : */
889 : #define SPDK_INTERRUPT_REGISTER_FOR_EVENTS(efd, events, fn, arg) \
890 : spdk_interrupt_register_for_events(efd, events, fn, arg, #fn)
891 :
892 : /**
893 : * Unregister an spdk_interrupt on the current thread.
894 : *
895 : * \param pintr The spdk_interrupt to unregister.
896 : */
897 : void spdk_interrupt_unregister(struct spdk_interrupt **pintr);
898 :
899 : enum spdk_interrupt_event_types {
900 : #ifdef __linux__
901 : SPDK_INTERRUPT_EVENT_IN = EPOLLIN,
902 : SPDK_INTERRUPT_EVENT_OUT = EPOLLOUT,
903 : SPDK_INTERRUPT_EVENT_ET = EPOLLET
904 : #else
905 : SPDK_INTERRUPT_EVENT_IN = 0x001,
906 : SPDK_INTERRUPT_EVENT_OUT = 0x004,
907 : SPDK_INTERRUPT_EVENT_ET = 1u << 31
908 : #endif
909 : };
910 :
911 : /**
912 : * Change the event_types associated with the spdk_interrupt on the current thread.
913 : *
914 : * \param intr The pointer to the spdk_interrupt registered on the current thread.
915 : * \param event_types New event_types for the spdk_interrupt.
916 : *
917 : * \return 0 if success or -errno if failed.
918 : */
919 : int spdk_interrupt_set_event_types(struct spdk_interrupt *intr,
920 : enum spdk_interrupt_event_types event_types);
921 :
922 : /**
923 : * Return a file descriptor that becomes ready whenever any of the registered
924 : * interrupt file descriptors are ready
925 : *
926 : * \param thread The thread to get.
927 : *
928 : * \return The spdk_interrupt fd of thread itself.
929 : */
930 : int spdk_thread_get_interrupt_fd(struct spdk_thread *thread);
931 :
932 : /**
933 : * Return an fd_group that becomes ready whenever any of the registered
934 : * interrupt file descriptors are ready
935 : *
936 : *
937 : * \param thread The thread to get.
938 : *
939 : * \return The spdk_fd_group of the thread itself.
940 : */
941 : struct spdk_fd_group *spdk_thread_get_interrupt_fd_group(struct spdk_thread *thread);
942 :
943 : /**
944 : * Set SPDK run as event driven mode
945 : *
946 : * \return 0 on success or -errno on failure
947 : */
948 : int spdk_interrupt_mode_enable(void);
949 :
950 : /**
951 : * Reports whether interrupt mode is set.
952 : *
953 : * \return True if interrupt mode is set, false otherwise.
954 : */
955 : bool spdk_interrupt_mode_is_enabled(void);
956 :
957 : /**
958 : * A spinlock augmented with safety checks for use with SPDK.
959 : *
960 : * SPDK code that uses spdk_spinlock runs from an SPDK thread, which itself is associated with a
961 : * pthread. There are typically many SPDK threads associated with each pthread. The SPDK application
962 : * may migrate SPDK threads between pthreads from time to time to balance the load on those threads.
963 : * Migration of SPDK threads only happens when the thread is off CPU, and as such it is only safe to
964 : * hold a lock so long as an SPDK thread stays on CPU.
965 : *
966 : * It is not safe to lock a spinlock, return from the event or poller, then unlock it at some later
967 : * time because:
968 : *
969 : * - Even though the SPDK thread may be the same, the SPDK thread may be running on different
970 : * pthreads during lock and unlock. A pthread spinlock may consider this to be an unlock by a
971 : * non-owner, which results in undefined behavior.
972 : * - A lock that is acquired by a poller or event may be needed by another poller or event that
973 : * runs on the same pthread. This can lead to deadlock or detection of deadlock.
974 : * - A lock that is acquired by a poller or event that is needed by another poller or event that
975 : * runs on a second pthread will block the second pthread from doing any useful work until the
976 : * lock is released. Because the lock holder and the lock acquirer are on the same pthread, this
977 : * would lead to deadlock.
978 : *
979 : * If an SPDK spinlock is used erroneously, the program will abort.
980 : */
981 : struct spdk_spinlock {
982 : pthread_spinlock_t spinlock;
983 : struct spdk_thread *thread;
984 : struct spdk_spinlock_internal *internal;
985 : bool initialized;
986 : bool destroyed;
987 : };
988 :
989 : /**
990 : * Initialize an spdk_spinlock.
991 : *
992 : * \param sspin The SPDK spinlock to initialize.
993 : */
994 : void spdk_spin_init(struct spdk_spinlock *sspin);
995 :
996 : /**
997 : * Destroy an spdk_spinlock.
998 : *
999 : * \param sspin The SPDK spinlock to initialize.
1000 : */
1001 : void spdk_spin_destroy(struct spdk_spinlock *sspin);
1002 :
1003 : /**
1004 : * Lock an SPDK spin lock.
1005 : *
1006 : * \param sspin An SPDK spinlock.
1007 : */
1008 : void spdk_spin_lock(struct spdk_spinlock *sspin);
1009 :
1010 : /**
1011 : * Unlock an SPDK spinlock.
1012 : *
1013 : * \param sspin An SPDK spinlock.
1014 : */
1015 : void spdk_spin_unlock(struct spdk_spinlock *sspin);
1016 :
1017 : /**
1018 : * Determine if the caller holds this SPDK spinlock.
1019 : *
1020 : * \param sspin An SPDK spinlock.
1021 : * \return true if spinlock is held by this thread, else false
1022 : */
1023 : bool spdk_spin_held(struct spdk_spinlock *sspin);
1024 :
1025 : struct spdk_iobuf_opts {
1026 : /** Maximum number of small buffers */
1027 : uint64_t small_pool_count;
1028 : /** Maximum number of large buffers */
1029 : uint64_t large_pool_count;
1030 : /** Size of a single small buffer */
1031 : uint32_t small_bufsize;
1032 : /** Size of a single large buffer */
1033 : uint32_t large_bufsize;
1034 :
1035 : /**
1036 : * The size of spdk_iobuf_opts according to the caller of this library is used for ABI
1037 : * compatibility. The library uses this field to know how many fields in this
1038 : * structure are valid. And the library will populate any remaining fields with default values.
1039 : * New added fields should be put at the end of the struct.
1040 : */
1041 : size_t opts_size;
1042 :
1043 : };
1044 :
1045 : struct spdk_iobuf_pool_stats {
1046 : /** Buffer got from local per-thread cache */
1047 : uint64_t cache;
1048 : /** Buffer got from the main shared pool */
1049 : uint64_t main;
1050 : /** Buffer missed and request to get buffer was queued */
1051 : uint64_t retry;
1052 : };
1053 :
1054 : struct spdk_iobuf_module_stats {
1055 : struct spdk_iobuf_pool_stats small_pool;
1056 : struct spdk_iobuf_pool_stats large_pool;
1057 : const char *module;
1058 : };
1059 :
1060 : struct spdk_iobuf_entry;
1061 :
1062 : typedef void (*spdk_iobuf_get_cb)(struct spdk_iobuf_entry *entry, void *buf);
1063 :
1064 : /** iobuf queue entry */
1065 : struct spdk_iobuf_entry {
1066 : spdk_iobuf_get_cb cb_fn;
1067 : const void *module;
1068 : STAILQ_ENTRY(spdk_iobuf_entry) stailq;
1069 : };
1070 :
1071 : struct spdk_iobuf_buffer {
1072 : STAILQ_ENTRY(spdk_iobuf_buffer) stailq;
1073 : };
1074 :
1075 : typedef STAILQ_HEAD(, spdk_iobuf_entry) spdk_iobuf_entry_stailq_t;
1076 : typedef STAILQ_HEAD(, spdk_iobuf_buffer) spdk_iobuf_buffer_stailq_t;
1077 :
1078 : struct spdk_iobuf_pool {
1079 : /** Buffer pool */
1080 : struct spdk_ring *pool;
1081 : /** Buffer cache */
1082 : spdk_iobuf_buffer_stailq_t cache;
1083 : /** Number of elements in the cache */
1084 : uint32_t cache_count;
1085 : /** Size of the cache */
1086 : uint32_t cache_size;
1087 : /** Buffer wait queue */
1088 : spdk_iobuf_entry_stailq_t *queue;
1089 : /** Buffer size */
1090 : uint32_t bufsize;
1091 : /** Pool usage statistics */
1092 : struct spdk_iobuf_pool_stats stats;
1093 : };
1094 :
1095 : /** iobuf channel */
1096 : struct spdk_iobuf_channel {
1097 : /** Small buffer memory pool */
1098 : struct spdk_iobuf_pool small;
1099 : /** Large buffer memory pool */
1100 : struct spdk_iobuf_pool large;
1101 : /** Module pointer */
1102 : const void *module;
1103 : /** Parent IO channel */
1104 : struct spdk_io_channel *parent;
1105 : };
1106 :
1107 : /**
1108 : * Initialize and allocate iobuf pools.
1109 : *
1110 : * \return 0 on success, negative errno otherwise.
1111 : */
1112 : int spdk_iobuf_initialize(void);
1113 :
1114 : typedef void (*spdk_iobuf_finish_cb)(void *cb_arg);
1115 :
1116 : /**
1117 : * Clean up and free iobuf pools.
1118 : *
1119 : * \param cb_fn Callback to be executed once the clean up is completed.
1120 : * \param cb_arg Callback argument.
1121 : */
1122 : void spdk_iobuf_finish(spdk_iobuf_finish_cb cb_fn, void *cb_arg);
1123 :
1124 : /**
1125 : * Set iobuf options. These options will be used during `spdk_iobuf_initialize()`.
1126 : *
1127 : * \param opts Options describing the size of the pools to reserve.
1128 : *
1129 : * \return 0 on success, negative errno otherwise.
1130 : */
1131 : int spdk_iobuf_set_opts(const struct spdk_iobuf_opts *opts);
1132 :
1133 : /**
1134 : * Get iobuf options.
1135 : *
1136 : * \param opts Output parameter for options.
1137 : * \param opts_size sizeof(*opts)
1138 : */
1139 : void spdk_iobuf_get_opts(struct spdk_iobuf_opts *opts, size_t opts_size);
1140 :
1141 : /**
1142 : * Register a module as an iobuf pool user. Only registered users can request buffers from the
1143 : * iobuf pool.
1144 : *
1145 : * \name Name of the module.
1146 : *
1147 : * \return 0 on success, negative errno otherwise.
1148 : */
1149 : int spdk_iobuf_register_module(const char *name);
1150 :
1151 : /**
1152 : * Unregister an iobuf pool user from a module.
1153 : *
1154 : * \name Name of the module.
1155 : *
1156 : * \return 0 on success, negative errno otherwise.
1157 : */
1158 : int spdk_iobuf_unregister_module(const char *name);
1159 :
1160 : /**
1161 : * Initialize an iobuf channel.
1162 : *
1163 : * \param ch iobuf channel to initialize.
1164 : * \param name Name of the module registered via `spdk_iobuf_register_module()`.
1165 : * \param small_cache_size Number of small buffers to be cached by this channel.
1166 : * \param large_cache_size Number of large buffers to be cached by this channel.
1167 : *
1168 : * \return 0 on success, negative errno otherwise.
1169 : */
1170 : int spdk_iobuf_channel_init(struct spdk_iobuf_channel *ch, const char *name,
1171 : uint32_t small_cache_size, uint32_t large_cache_size);
1172 :
1173 : /**
1174 : * Release resources tied to an iobuf channel.
1175 : *
1176 : * \param ch iobuf channel.
1177 : */
1178 : void spdk_iobuf_channel_fini(struct spdk_iobuf_channel *ch);
1179 :
1180 : typedef int (*spdk_iobuf_for_each_entry_fn)(struct spdk_iobuf_channel *ch,
1181 : struct spdk_iobuf_entry *entry, void *ctx);
1182 :
1183 : /**
1184 : * Iterate over all entries on a given queue and execute a callback on those that were requested
1185 : * using `ch`. The iteration is stopped if the callback returns non-zero status.
1186 : *
1187 : * \param ch iobuf channel to iterate over.
1188 : * \param pool Pool to iterate over (`small` or `large`).
1189 : * \param cb_fn Callback to execute on each entry on the queue that was requested using `ch`.
1190 : * \param cb_ctx Argument passed to `cb_fn`.
1191 : *
1192 : * \return status of the last callback.
1193 : */
1194 : int spdk_iobuf_for_each_entry(struct spdk_iobuf_channel *ch, struct spdk_iobuf_pool *pool,
1195 : spdk_iobuf_for_each_entry_fn cb_fn, void *cb_ctx);
1196 :
1197 : /**
1198 : * Abort an outstanding request waiting for a buffer.
1199 : *
1200 : * \param ch iobuf channel on which the entry is waiting.
1201 : * \param entry Entry to remove from the wait queue.
1202 : * \param len Length of the requested buffer (must be the exact same value as specified in
1203 : * `spdk_iobuf_get()`.
1204 : */
1205 : void spdk_iobuf_entry_abort(struct spdk_iobuf_channel *ch, struct spdk_iobuf_entry *entry,
1206 : uint64_t len);
1207 :
1208 : /**
1209 : * Get a buffer from the iobuf pool. If no buffers are available and entry with cb_fn provided
1210 : * then the request is queued until a buffer becomes available.
1211 : *
1212 : * \param ch iobuf channel.
1213 : * \param len Length of the buffer to retrieve. The user is responsible for making sure the length
1214 : * doesn't exceed large_bufsize.
1215 : * \param entry Wait queue entry (optional).
1216 : * \param cb_fn Callback to be executed once a buffer becomes available. If a buffer is available
1217 : * immediately, it is NOT executed. Mandatory only if entry provided.
1218 : *
1219 : * \return pointer to a buffer or NULL if no buffers are currently available.
1220 : */
1221 : void *spdk_iobuf_get(struct spdk_iobuf_channel *ch, uint64_t len, struct spdk_iobuf_entry *entry,
1222 : spdk_iobuf_get_cb cb_fn);
1223 :
1224 : /**
1225 : * Release a buffer back to the iobuf pool. If there are outstanding requests waiting for a buffer,
1226 : * this buffer will be passed to one of them.
1227 : *
1228 : * \param ch iobuf channel.
1229 : * \param buf Buffer to release
1230 : * \param len Length of the buffer (must be the exact same value as specified in `spdk_iobuf_get()`).
1231 : */
1232 : void spdk_iobuf_put(struct spdk_iobuf_channel *ch, void *buf, uint64_t len);
1233 :
1234 : typedef void (*spdk_iobuf_get_stats_cb)(struct spdk_iobuf_module_stats *modules,
1235 : uint32_t num_modules, void *cb_arg);
1236 :
1237 : /**
1238 : * Get iobuf statistics.
1239 : *
1240 : * \param cb_fn Callback to be executed once stats are gathered.
1241 : * \param cb_arg Argument passed to the callback function.
1242 : *
1243 : * \return 0 on success, negative errno otherwise.
1244 : */
1245 : int spdk_iobuf_get_stats(spdk_iobuf_get_stats_cb cb_fn, void *cb_arg);
1246 :
1247 : #ifdef __cplusplus
1248 : }
1249 : #endif
1250 :
1251 : #endif /* SPDK_THREAD_H_ */
|