Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright (C) 2017 Intel Corporation. All rights reserved.
3 : : * All rights reserved.
4 : : */
5 : :
6 : : #include "spdk/stdinc.h"
7 : :
8 : : #include <linux/virtio_scsi.h>
9 : :
10 : : #include "spdk/env.h"
11 : : #include "spdk/thread.h"
12 : : #include "spdk/scsi.h"
13 : : #include "spdk/scsi_spec.h"
14 : : #include "spdk/util.h"
15 : : #include "spdk/likely.h"
16 : :
17 : : #include "spdk/vhost.h"
18 : : #include "vhost_internal.h"
19 : :
20 : : /* Features supported by SPDK VHOST lib. */
21 : : #define SPDK_VHOST_SCSI_FEATURES (SPDK_VHOST_FEATURES | \
22 : : (1ULL << VIRTIO_SCSI_F_INOUT) | \
23 : : (1ULL << VIRTIO_SCSI_F_HOTPLUG) | \
24 : : (1ULL << VIRTIO_SCSI_F_CHANGE ) | \
25 : : (1ULL << VIRTIO_SCSI_F_T10_PI ))
26 : :
27 : : /* Features that are specified in VIRTIO SCSI but currently not supported:
28 : : * - Live migration not supported yet
29 : : * - T10 PI
30 : : */
31 : : #define SPDK_VHOST_SCSI_DISABLED_FEATURES (SPDK_VHOST_DISABLED_FEATURES | \
32 : : (1ULL << VIRTIO_SCSI_F_T10_PI ))
33 : :
34 : : /* Vhost-user-scsi support protocol features */
35 : : #define SPDK_VHOST_SCSI_PROTOCOL_FEATURES (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)
36 : :
37 : : #define MGMT_POLL_PERIOD_US (1000 * 5)
38 : :
39 : : #define VIRTIO_SCSI_CONTROLQ 0
40 : : #define VIRTIO_SCSI_EVENTQ 1
41 : : #define VIRTIO_SCSI_REQUESTQ 2
42 : :
43 : : enum spdk_scsi_dev_vhost_status {
44 : : /* Target ID is empty. */
45 : : VHOST_SCSI_DEV_EMPTY,
46 : :
47 : : /* Target is still being added. */
48 : : VHOST_SCSI_DEV_ADDING,
49 : :
50 : : /* Target ID occupied. */
51 : : VHOST_SCSI_DEV_PRESENT,
52 : :
53 : : /* Target ID is occupied but removal is in progress. */
54 : : VHOST_SCSI_DEV_REMOVING,
55 : :
56 : : /* In session - device (SCSI target) seen but removed. */
57 : : VHOST_SCSI_DEV_REMOVED,
58 : : };
59 : :
60 : : /** Context for a SCSI target in a vhost device */
61 : : struct spdk_scsi_dev_vhost_state {
62 : : struct spdk_scsi_dev *dev;
63 : : enum spdk_scsi_dev_vhost_status status;
64 : : spdk_vhost_event_fn remove_cb;
65 : : void *remove_ctx;
66 : : };
67 : :
68 : : struct spdk_vhost_scsi_dev {
69 : : int ref;
70 : : bool registered;
71 : : struct spdk_vhost_dev vdev;
72 : : struct spdk_scsi_dev_vhost_state scsi_dev_state[SPDK_VHOST_SCSI_CTRLR_MAX_DEVS];
73 : : };
74 : :
75 : : /** Context for a SCSI target in a vhost session */
76 : : struct spdk_scsi_dev_session_state {
77 : : struct spdk_scsi_dev *dev;
78 : : enum spdk_scsi_dev_vhost_status status;
79 : : };
80 : :
81 : : struct spdk_vhost_scsi_session {
82 : : struct spdk_vhost_session vsession;
83 : :
84 : : struct spdk_vhost_scsi_dev *svdev;
85 : : /** Local copy of the device state */
86 : : struct spdk_scsi_dev_session_state scsi_dev_state[SPDK_VHOST_SCSI_CTRLR_MAX_DEVS];
87 : : struct spdk_poller *requestq_poller;
88 : : struct spdk_poller *mgmt_poller;
89 : : struct spdk_poller *stop_poller;
90 : : };
91 : :
92 : : struct spdk_vhost_scsi_task {
93 : : struct spdk_scsi_task scsi;
94 : : struct iovec iovs[SPDK_VHOST_IOVS_MAX];
95 : :
96 : : union {
97 : : struct virtio_scsi_cmd_resp *resp;
98 : : struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
99 : : };
100 : :
101 : : struct spdk_vhost_scsi_session *svsession;
102 : : struct spdk_scsi_dev *scsi_dev;
103 : :
104 : : /** Number of bytes that were written. */
105 : : uint32_t used_len;
106 : :
107 : : int req_idx;
108 : :
109 : : /* If set, the task is currently used for I/O processing. */
110 : : bool used;
111 : :
112 : : struct spdk_vhost_virtqueue *vq;
113 : : };
114 : :
115 : : static int vhost_scsi_start(struct spdk_vhost_dev *vdev,
116 : : struct spdk_vhost_session *vsession, void *unused);
117 : : static int vhost_scsi_stop(struct spdk_vhost_dev *vdev,
118 : : struct spdk_vhost_session *vsession, void *unused);
119 : : static void vhost_scsi_dump_info_json(struct spdk_vhost_dev *vdev,
120 : : struct spdk_json_write_ctx *w);
121 : : static void vhost_scsi_write_config_json(struct spdk_vhost_dev *vdev,
122 : : struct spdk_json_write_ctx *w);
123 : : static int vhost_scsi_dev_remove(struct spdk_vhost_dev *vdev);
124 : : static int vhost_scsi_dev_param_changed(struct spdk_vhost_dev *vdev,
125 : : unsigned scsi_tgt_num);
126 : : static int alloc_vq_task_pool(struct spdk_vhost_session *vsession, uint16_t qid);
127 : :
128 : : static const struct spdk_vhost_user_dev_backend spdk_vhost_scsi_user_device_backend = {
129 : : .session_ctx_size = sizeof(struct spdk_vhost_scsi_session) - sizeof(struct spdk_vhost_session),
130 : : .start_session = vhost_scsi_start,
131 : : .stop_session = vhost_scsi_stop,
132 : : .alloc_vq_tasks = alloc_vq_task_pool,
133 : : };
134 : :
135 : : static const struct spdk_vhost_dev_backend spdk_vhost_scsi_device_backend = {
136 : : .type = VHOST_BACKEND_SCSI,
137 : : .dump_info_json = vhost_scsi_dump_info_json,
138 : : .write_config_json = vhost_scsi_write_config_json,
139 : : .remove_device = vhost_scsi_dev_remove,
140 : : .set_coalescing = vhost_user_set_coalescing,
141 : : .get_coalescing = vhost_user_get_coalescing,
142 : : };
143 : :
144 : : static inline void
145 : 4038172 : scsi_task_init(struct spdk_vhost_scsi_task *task)
146 : : {
147 [ - + # # ]: 4038172 : memset(&task->scsi, 0, sizeof(task->scsi));
148 : : /* Tmf_resp pointer and resp pointer are in a union.
149 : : * Here means task->tmf_resp = task->resp = NULL.
150 : : */
151 [ # # # # : 4038172 : task->resp = NULL;
# # ]
152 [ # # # # ]: 4038172 : task->used = true;
153 [ # # # # ]: 4038172 : task->used_len = 0;
154 : 4038172 : }
155 : :
156 : : static void
157 : 4038172 : vhost_scsi_task_put(struct spdk_vhost_scsi_task *task)
158 : : {
159 [ # # ]: 4038172 : spdk_scsi_task_put(&task->scsi);
160 : 4038172 : }
161 : :
162 : : static void
163 : 4038172 : vhost_scsi_task_free_cb(struct spdk_scsi_task *scsi_task)
164 : : {
165 : 4038172 : struct spdk_vhost_scsi_task *task = SPDK_CONTAINEROF(scsi_task, struct spdk_vhost_scsi_task, scsi);
166 [ # # # # : 4038172 : struct spdk_vhost_session *vsession = &task->svsession->vsession;
# # ]
167 : :
168 [ - + # # : 4038172 : assert(vsession->task_cnt > 0);
# # # # ]
169 [ # # # # ]: 4038172 : vsession->task_cnt--;
170 [ # # # # ]: 4038172 : task->used = false;
171 : 4038172 : }
172 : :
173 : : static void
174 : 10 : vhost_scsi_dev_unregister(void *arg1)
175 : : {
176 : 10 : struct spdk_vhost_scsi_dev *svdev = arg1;
177 : :
178 [ + - # # ]: 10 : if (vhost_dev_unregister(&svdev->vdev) == 0) {
179 : 10 : free(svdev);
180 : 0 : }
181 : 10 : }
182 : :
183 : : static void
184 : 59 : remove_scsi_tgt(struct spdk_vhost_scsi_dev *svdev,
185 : : unsigned scsi_tgt_num)
186 : : {
187 : : struct spdk_scsi_dev_vhost_state *state;
188 : : struct spdk_scsi_dev *dev;
189 : :
190 [ # # # # ]: 59 : state = &svdev->scsi_dev_state[scsi_tgt_num];
191 [ # # # # ]: 59 : dev = state->dev;
192 [ # # # # ]: 59 : state->dev = NULL;
193 [ - + # # : 59 : assert(state->status == VHOST_SCSI_DEV_REMOVING);
# # # # ]
194 [ # # # # ]: 59 : state->status = VHOST_SCSI_DEV_EMPTY;
195 : 59 : spdk_scsi_dev_destruct(dev, NULL, NULL);
196 [ + + # # : 59 : if (state->remove_cb) {
# # ]
197 [ # # # # : 38 : state->remove_cb(&svdev->vdev, state->remove_ctx);
# # # # #
# # # #
# ]
198 [ # # # # ]: 38 : state->remove_cb = NULL;
199 : 0 : }
200 [ - + - + : 59 : SPDK_INFOLOG(vhost, "removed target 'Target %u'\n", scsi_tgt_num);
# # ]
201 : :
202 [ + + - + : 59 : if (--svdev->ref == 0 && svdev->registered == false) {
+ + # # #
# # # #
# ]
203 : : /* `remove_scsi_tgt` is running under vhost-user lock, so we
204 : : * unregister the device in next poll.
205 : : */
206 : 10 : spdk_thread_send_msg(spdk_get_thread(), vhost_scsi_dev_unregister, svdev);
207 : 0 : }
208 : 59 : }
209 : :
210 : : static void
211 : 4 : vhost_scsi_dev_process_removed_cpl_cb(struct spdk_vhost_dev *vdev, void *ctx)
212 : : {
213 : 4 : unsigned scsi_tgt_num = (unsigned)(uintptr_t)ctx;
214 : 4 : struct spdk_vhost_scsi_dev *svdev = SPDK_CONTAINEROF(vdev,
215 : : struct spdk_vhost_scsi_dev, vdev);
216 : :
217 : : /* all sessions have already detached the device */
218 [ - + # # : 4 : if (svdev->scsi_dev_state[scsi_tgt_num].status != VHOST_SCSI_DEV_REMOVING) {
# # # # #
# ]
219 : : /* device was already removed in the meantime */
220 : 0 : return;
221 : : }
222 : :
223 : 4 : remove_scsi_tgt(svdev, scsi_tgt_num);
224 : 0 : }
225 : :
226 : : static int
227 : 4 : vhost_scsi_session_process_removed(struct spdk_vhost_dev *vdev,
228 : : struct spdk_vhost_session *vsession, void *ctx)
229 : : {
230 : 4 : unsigned scsi_tgt_num = (unsigned)(uintptr_t)ctx;
231 : 4 : struct spdk_vhost_scsi_session *svsession = (struct spdk_vhost_scsi_session *)vsession;
232 [ # # # # : 4 : struct spdk_scsi_dev_session_state *state = &svsession->scsi_dev_state[scsi_tgt_num];
# # ]
233 : :
234 [ - + # # : 4 : if (state->dev != NULL) {
# # ]
235 : : /* there's still a session that references this device,
236 : : * so abort our foreach chain here. We'll be called
237 : : * again from this session's management poller after it
238 : : * is removed in there
239 : : */
240 : 0 : return -1;
241 : : }
242 : :
243 : 4 : return 0;
244 : 0 : }
245 : :
246 : : static void
247 : 149429 : process_removed_devs(struct spdk_vhost_scsi_session *svsession)
248 : : {
249 : : struct spdk_scsi_dev *dev;
250 : : struct spdk_scsi_dev_session_state *state;
251 : : int i;
252 : :
253 [ + + # # ]: 1344861 : for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; ++i) {
254 [ # # # # : 1195432 : state = &svsession->scsi_dev_state[i];
# # ]
255 [ # # # # ]: 1195432 : dev = state->dev;
256 : :
257 [ + + + + : 1195432 : if (dev && state->status == VHOST_SCSI_DEV_REMOVING &&
# # # # #
# ]
258 [ + - ]: 4 : !spdk_scsi_dev_has_pending_tasks(dev, NULL)) {
259 : : /* detach the device from this session */
260 : 4 : spdk_scsi_dev_free_io_channels(dev);
261 [ # # # # ]: 4 : state->dev = NULL;
262 [ # # # # ]: 4 : state->status = VHOST_SCSI_DEV_REMOVED;
263 : : /* try to detach it globally */
264 [ # # # # : 4 : vhost_user_dev_foreach_session(&svsession->svdev->vdev,
# # ]
265 : : vhost_scsi_session_process_removed,
266 : : vhost_scsi_dev_process_removed_cpl_cb,
267 : 4 : (void *)(uintptr_t)i);
268 : 0 : }
269 : 0 : }
270 : 149429 : }
271 : :
272 : : static void
273 : 8 : eventq_enqueue(struct spdk_vhost_scsi_session *svsession, unsigned scsi_dev_num,
274 : : uint32_t event, uint32_t reason)
275 : : {
276 [ # # ]: 8 : struct spdk_vhost_session *vsession = &svsession->vsession;
277 : : struct spdk_vhost_virtqueue *vq;
278 : 8 : struct vring_desc *desc, *desc_table;
279 : : struct virtio_scsi_event *desc_ev;
280 : 8 : uint32_t desc_table_size, req_size = 0;
281 : 8 : uint16_t req;
282 : : int rc;
283 : :
284 [ - + # # ]: 8 : assert(scsi_dev_num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS);
285 [ # # # # : 8 : vq = &vsession->virtqueue[VIRTIO_SCSI_EVENTQ];
# # ]
286 : :
287 [ + - - + : 8 : if (vq->vring.desc == NULL || vhost_vq_avail_ring_get(vq, &req, 1) != 1) {
# # # # #
# # # ]
288 [ # # # # ]: 0 : SPDK_ERRLOG("%s: failed to send virtio event (no avail ring entries?).\n",
289 : : vsession->name);
290 : 0 : return;
291 : : }
292 : :
293 : 8 : rc = vhost_vq_get_desc(vsession, vq, req, &desc, &desc_table, &desc_table_size);
294 [ + - - + : 8 : if (rc != 0 || desc->len < sizeof(*desc_ev)) {
# # # # ]
295 [ # # # # ]: 0 : SPDK_ERRLOG("%s: invalid eventq descriptor at index %"PRIu16".\n",
296 : : vsession->name, req);
297 : 0 : goto out;
298 : : }
299 : :
300 [ # # # # ]: 8 : desc_ev = vhost_gpa_to_vva(vsession, desc->addr, sizeof(*desc_ev));
301 [ - + ]: 8 : if (desc_ev == NULL) {
302 [ # # # # : 0 : SPDK_ERRLOG("%s: eventq descriptor at index %"PRIu16" points "
# # # # ]
303 : : "to unmapped guest memory address %p.\n",
304 : : vsession->name, req, (void *)(uintptr_t)desc->addr);
305 : 0 : goto out;
306 : : }
307 : :
308 [ # # # # ]: 8 : desc_ev->event = event;
309 [ # # # # : 8 : desc_ev->lun[0] = 1;
# # # # ]
310 [ # # # # : 8 : desc_ev->lun[1] = scsi_dev_num;
# # # # ]
311 : : /* virtio LUN id 0 can refer either to the entire device
312 : : * or actual LUN 0 (the only supported by vhost for now)
313 : : */
314 [ # # # # : 8 : desc_ev->lun[2] = 0 >> 8;
# # # # #
# ]
315 [ # # # # : 8 : desc_ev->lun[3] = 0 & 0xFF;
# # # # ]
316 : : /* virtio doesn't specify any strict format for LUN id (bytes 2 and 3)
317 : : * current implementation relies on linux kernel sources
318 : : */
319 [ - + # # : 8 : memset(&desc_ev->lun[4], 0, 4);
# # # # ]
320 [ # # # # ]: 8 : desc_ev->reason = reason;
321 : 8 : req_size = sizeof(*desc_ev);
322 : :
323 : 8 : out:
324 : 8 : vhost_vq_used_ring_enqueue(vsession, vq, req, req_size);
325 : 0 : }
326 : :
327 : : static void
328 : 3988135 : submit_completion(struct spdk_vhost_scsi_task *task)
329 : : {
330 [ # # # # : 3988135 : struct spdk_vhost_session *vsession = &task->svsession->vsession;
# # ]
331 : :
332 [ # # # # : 3988135 : vhost_vq_used_ring_enqueue(vsession, task->vq, task->req_idx,
# # # # ]
333 [ # # # # ]: 0 : task->used_len);
334 [ - + - + : 3988135 : SPDK_DEBUGLOG(vhost_scsi, "Finished task (%p) req_idx=%d\n", task, task->req_idx);
# # # # #
# ]
335 : :
336 : 3988135 : vhost_scsi_task_put(task);
337 : 3988135 : }
338 : :
339 : : static void
340 : 4 : vhost_scsi_task_mgmt_cpl(struct spdk_scsi_task *scsi_task)
341 : : {
342 : 4 : struct spdk_vhost_scsi_task *task = SPDK_CONTAINEROF(scsi_task, struct spdk_vhost_scsi_task, scsi);
343 : :
344 : 4 : submit_completion(task);
345 : 4 : }
346 : :
347 : : static void
348 : 3988131 : vhost_scsi_task_cpl(struct spdk_scsi_task *scsi_task)
349 : : {
350 : 3988131 : struct spdk_vhost_scsi_task *task = SPDK_CONTAINEROF(scsi_task, struct spdk_vhost_scsi_task, scsi);
351 : :
352 : : /* The SCSI task has completed. Do final processing and then post
353 : : notification to the virtqueue's "used" ring.
354 : : */
355 [ # # # # : 3988131 : task->resp->status = task->scsi.status;
# # # # #
# # # # #
# # ]
356 : :
357 [ + + # # : 3988131 : if (task->scsi.status != SPDK_SCSI_STATUS_GOOD) {
# # # # ]
358 [ - + - + : 72311 : memcpy(task->resp->sense, task->scsi.sense_data, task->scsi.sense_data_len);
# # # # #
# # # # #
# # # # #
# # # ]
359 [ # # # # : 72311 : task->resp->sense_len = task->scsi.sense_data_len;
# # # # #
# # # # #
# # ]
360 [ - + - + : 72311 : SPDK_DEBUGLOG(vhost_scsi, "Task (%p) req_idx=%d failed - status=%u\n", task, task->req_idx,
# # # # #
# # # # #
# # ]
361 : : task->scsi.status);
362 : 0 : }
363 [ - + # # : 3988131 : assert(task->scsi.transfer_len == task->scsi.length);
# # # # #
# # # # #
# # ]
364 [ # # # # : 3988131 : task->resp->resid = task->scsi.length - task->scsi.data_transferred;
# # # # #
# # # # #
# # # # #
# # # ]
365 : :
366 : 3988131 : submit_completion(task);
367 : 3988131 : }
368 : :
369 : : static void
370 : 3915688 : task_submit(struct spdk_vhost_scsi_task *task)
371 : : {
372 [ # # # # : 3915688 : task->resp->response = VIRTIO_SCSI_S_OK;
# # # # #
# ]
373 [ # # # # : 3915688 : spdk_scsi_dev_queue_task(task->scsi_dev, &task->scsi);
# # ]
374 : 3915688 : }
375 : :
376 : : static void
377 : 4 : mgmt_task_submit(struct spdk_vhost_scsi_task *task, enum spdk_scsi_task_func func)
378 : : {
379 [ # # # # : 4 : task->tmf_resp->response = VIRTIO_SCSI_S_OK;
# # # # #
# ]
380 [ # # # # : 4 : task->scsi.function = func;
# # ]
381 [ # # # # : 4 : spdk_scsi_dev_queue_mgmt_task(task->scsi_dev, &task->scsi);
# # ]
382 : 4 : }
383 : :
384 : : static void
385 : 6867 : invalid_request(struct spdk_vhost_scsi_task *task)
386 : : {
387 [ # # # # : 6867 : struct spdk_vhost_session *vsession = &task->svsession->vsession;
# # ]
388 : :
389 [ # # # # : 6867 : vhost_vq_used_ring_enqueue(vsession, task->vq, task->req_idx,
# # # # ]
390 [ # # # # ]: 0 : task->used_len);
391 : 6867 : vhost_scsi_task_put(task);
392 : :
393 [ - + - + : 6867 : SPDK_DEBUGLOG(vhost_scsi, "Invalid request (status=%" PRIu8")\n",
- - # # #
# # # # #
# # # # #
# # # #
# ]
394 : : task->resp ? task->resp->response : -1);
395 : 6867 : }
396 : :
397 : : static int
398 : 4038170 : vhost_scsi_task_init_target(struct spdk_vhost_scsi_task *task, const __u8 *lun)
399 : : {
400 [ # # # # ]: 4038170 : struct spdk_vhost_scsi_session *svsession = task->svsession;
401 : : struct spdk_scsi_dev_session_state *state;
402 [ - + # # : 4038170 : uint16_t lun_id = (((uint16_t)lun[2] << 8) | lun[3]) & 0x3FFF;
# # # # #
# # # ]
403 : :
404 [ - + - + : 4038170 : SPDK_LOGDUMP(vhost_scsi_queue, "LUN", lun, 8);
# # ]
405 : :
406 : : /* First byte must be 1 and second is target */
407 [ + + + + : 4038170 : if (lun[0] != 1 || lun[1] >= SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
# # # # #
# # # ]
408 : 49785 : return -1;
409 : : }
410 : :
411 [ # # # # : 3988385 : state = &svsession->scsi_dev_state[lun[1]];
# # # # #
# ]
412 [ # # # # : 3988385 : task->scsi_dev = state->dev;
# # # # ]
413 [ + + - + : 3988385 : if (state->dev == NULL || state->status != VHOST_SCSI_DEV_PRESENT) {
# # # # #
# # # ]
414 : : /* If dev has been hotdetached, return 0 to allow sending
415 : : * additional hotremove event via sense codes.
416 : : */
417 [ - + # # ]: 249 : return state->status != VHOST_SCSI_DEV_EMPTY ? 0 : -1;
418 : : }
419 : :
420 [ # # # # : 3988136 : task->scsi.target_port = spdk_scsi_dev_find_port_by_id(task->scsi_dev, 0);
# # # # #
# ]
421 [ # # # # : 3988136 : task->scsi.lun = spdk_scsi_dev_get_lun(state->dev, lun_id);
# # # # #
# ]
422 : 3988136 : return 0;
423 : 0 : }
424 : :
425 : : static void
426 : 43174 : process_ctrl_request(struct spdk_vhost_scsi_task *task)
427 : : {
428 [ # # # # : 43174 : struct spdk_vhost_session *vsession = &task->svsession->vsession;
# # ]
429 : 43174 : struct vring_desc *desc, *desc_table;
430 : : struct virtio_scsi_ctrl_tmf_req *ctrl_req;
431 : : struct virtio_scsi_ctrl_an_resp *an_resp;
432 : 43174 : uint32_t desc_table_size, used_len = 0;
433 : : int rc;
434 : :
435 [ # # ]: 43174 : spdk_scsi_task_construct(&task->scsi, vhost_scsi_task_mgmt_cpl, vhost_scsi_task_free_cb);
436 [ # # # # : 43174 : rc = vhost_vq_get_desc(vsession, task->vq, task->req_idx, &desc, &desc_table,
# # # # ]
437 : : &desc_table_size);
438 [ - + ]: 43174 : if (spdk_unlikely(rc != 0)) {
439 [ # # # # : 0 : SPDK_ERRLOG("%s: invalid controlq descriptor at index %d.\n",
# # # # ]
440 : : vsession->name, task->req_idx);
441 : 0 : goto out;
442 : : }
443 : :
444 [ # # # # ]: 43174 : ctrl_req = vhost_gpa_to_vva(vsession, desc->addr, sizeof(*ctrl_req));
445 [ - + ]: 43174 : if (ctrl_req == NULL) {
446 [ # # # # : 0 : SPDK_ERRLOG("%s: invalid task management request at index %d.\n",
# # # # ]
447 : : vsession->name, task->req_idx);
448 : 0 : goto out;
449 : : }
450 : :
451 [ - + - + : 43174 : SPDK_DEBUGLOG(vhost_scsi_queue,
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
452 : : "Processing controlq descriptor: desc %d/%p, desc_addr %p, len %d, flags %d, last_used_idx %d; kickfd %d; size %d\n",
453 : : task->req_idx, desc, (void *)desc->addr, desc->len, desc->flags, task->vq->last_used_idx,
454 : : task->vq->vring.kickfd, task->vq->vring.size);
455 [ - + - + : 43174 : SPDK_LOGDUMP(vhost_scsi_queue, "Request descriptor", (uint8_t *)ctrl_req, desc->len);
# # # # #
# ]
456 : :
457 [ # # ]: 43174 : vhost_scsi_task_init_target(task, ctrl_req->lun);
458 : :
459 : 43174 : vhost_vring_desc_get_next(&desc, desc_table, desc_table_size);
460 [ + + ]: 43174 : if (spdk_unlikely(desc == NULL)) {
461 [ # # # # : 2 : SPDK_ERRLOG("%s: no response descriptor for controlq request %d.\n",
# # # # ]
462 : : vsession->name, task->req_idx);
463 : 2 : goto out;
464 : : }
465 : :
466 : : /* Process the TMF request */
467 [ + + + # : 43172 : switch (ctrl_req->type) {
# # # ]
468 : 10984 : case VIRTIO_SCSI_T_TMF:
469 [ # # # # : 10984 : task->tmf_resp = vhost_gpa_to_vva(vsession, desc->addr, sizeof(*task->tmf_resp));
# # # # #
# ]
470 [ + - - + : 10984 : if (spdk_unlikely(desc->len < sizeof(struct virtio_scsi_ctrl_tmf_resp) || task->tmf_resp == NULL)) {
# # # # #
# # # #
# ]
471 [ # # # # : 0 : SPDK_ERRLOG("%s: TMF response descriptor at index %d points to invalid guest memory region\n",
# # # # ]
472 : : vsession->name, task->req_idx);
473 : 0 : goto out;
474 : : }
475 : :
476 : : /* Check if we are processing a valid request */
477 [ + + # # : 10984 : if (task->scsi_dev == NULL) {
# # ]
478 [ # # # # : 10969 : task->tmf_resp->response = VIRTIO_SCSI_S_BAD_TARGET;
# # # # #
# ]
479 : 10969 : break;
480 : : }
481 : :
482 [ + + # # : 15 : switch (ctrl_req->subtype) {
# # ]
483 : 4 : case VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET:
484 : : /* Handle LUN reset */
485 [ - + - + : 4 : SPDK_DEBUGLOG(vhost_scsi_queue, "%s: LUN reset\n", vsession->name);
# # # # #
# ]
486 : :
487 : 4 : mgmt_task_submit(task, SPDK_SCSI_TASK_FUNC_LUN_RESET);
488 : 4 : return;
489 : 11 : default:
490 [ # # # # : 11 : task->tmf_resp->response = VIRTIO_SCSI_S_ABORTED;
# # # # #
# ]
491 : : /* Unsupported command */
492 [ - + - + : 11 : SPDK_DEBUGLOG(vhost_scsi_queue, "%s: unsupported TMF command %x\n",
# # # # #
# # # #
# ]
493 : : vsession->name, ctrl_req->subtype);
494 : 11 : break;
495 : : }
496 : 11 : break;
497 : 21444 : case VIRTIO_SCSI_T_AN_QUERY:
498 : : case VIRTIO_SCSI_T_AN_SUBSCRIBE: {
499 [ # # # # ]: 21444 : an_resp = vhost_gpa_to_vva(vsession, desc->addr, sizeof(*an_resp));
500 [ + - - + : 21444 : if (spdk_unlikely(desc->len < sizeof(struct virtio_scsi_ctrl_an_resp) || an_resp == NULL)) {
# # # # ]
501 [ # # # # ]: 0 : SPDK_WARNLOG("%s: asynchronous response descriptor points to invalid guest memory region\n",
502 : : vsession->name);
503 : 0 : goto out;
504 : : }
505 : :
506 [ # # # # ]: 21444 : an_resp->response = VIRTIO_SCSI_S_ABORTED;
507 : 21444 : break;
508 : : }
509 : 10744 : default:
510 [ - + - + : 10744 : SPDK_DEBUGLOG(vhost_scsi_queue, "%s: Unsupported control command %x\n",
# # # # #
# # # #
# ]
511 : : vsession->name, ctrl_req->type);
512 : 10744 : break;
513 : : }
514 : :
515 : 43168 : used_len = sizeof(struct virtio_scsi_ctrl_tmf_resp);
516 : 43170 : out:
517 [ # # # # : 43170 : vhost_vq_used_ring_enqueue(vsession, task->vq, task->req_idx, used_len);
# # # # ]
518 : 43170 : vhost_scsi_task_put(task);
519 : 0 : }
520 : :
521 : : /*
522 : : * Process task's descriptor chain and setup data related fields.
523 : : * Return
524 : : * -1 if request is invalid and must be aborted,
525 : : * 0 if all data are set.
526 : : */
527 : : static int
528 : 3994998 : task_data_setup(struct spdk_vhost_scsi_task *task,
529 : : struct virtio_scsi_cmd_req **req)
530 : : {
531 [ # # # # : 3994998 : struct spdk_vhost_session *vsession = &task->svsession->vsession;
# # ]
532 : 3994998 : struct vring_desc *desc, *desc_table;
533 [ # # ]: 3994998 : struct iovec *iovs = task->iovs;
534 : 3994998 : uint16_t iovcnt = 0;
535 : 3994998 : uint32_t desc_table_len, len = 0;
536 : : int rc;
537 : :
538 [ # # ]: 3994998 : spdk_scsi_task_construct(&task->scsi, vhost_scsi_task_cpl, vhost_scsi_task_free_cb);
539 : :
540 [ # # # # : 3994998 : rc = vhost_vq_get_desc(vsession, task->vq, task->req_idx, &desc, &desc_table, &desc_table_len);
# # # # ]
541 : : /* First descriptor must be readable */
542 [ + - + + : 3994998 : if (spdk_unlikely(rc != 0 || vhost_vring_desc_is_wr(desc) ||
+ + - + #
# ]
543 : : desc->len < sizeof(struct virtio_scsi_cmd_req))) {
544 [ # # # # : 2 : SPDK_WARNLOG("%s: invalid first request descriptor at index %"PRIu16".\n",
# # # # ]
545 : : vsession->name, task->req_idx);
546 : 2 : goto invalid_task;
547 : : }
548 : :
549 [ # # # # : 3994996 : *req = vhost_gpa_to_vva(vsession, desc->addr, sizeof(**req));
# # ]
550 [ - + # # ]: 3994996 : if (spdk_unlikely(*req == NULL)) {
551 [ # # # # : 0 : SPDK_WARNLOG("%s: request descriptor at index %d points to invalid guest memory region\n",
# # # # ]
552 : : vsession->name, task->req_idx);
553 : 0 : goto invalid_task;
554 : : }
555 : :
556 : : /* Each request must have at least 2 descriptors (e.g. request and response) */
557 : 3994996 : vhost_vring_desc_get_next(&desc, desc_table, desc_table_len);
558 [ - + ]: 3994996 : if (desc == NULL) {
559 [ # # # # : 0 : SPDK_WARNLOG("%s: descriptor chain at index %d contains neither payload nor response buffer.\n",
# # # # ]
560 : : vsession->name, task->req_idx);
561 : 0 : goto invalid_task;
562 : : }
563 [ + + # # : 3994996 : task->scsi.dxfer_dir = vhost_vring_desc_is_wr(desc) ? SPDK_SCSI_DIR_FROM_DEV :
# # ]
564 : : SPDK_SCSI_DIR_TO_DEV;
565 [ # # # # : 3994996 : task->scsi.iovs = iovs;
# # ]
566 : :
567 [ + + # # : 3994996 : if (task->scsi.dxfer_dir == SPDK_SCSI_DIR_FROM_DEV) {
# # # # ]
568 : : /*
569 : : * FROM_DEV (READ): [RD_req][WR_resp][WR_buf0]...[WR_bufN]
570 : : */
571 [ # # # # : 2033006 : task->resp = vhost_gpa_to_vva(vsession, desc->addr, sizeof(*task->resp));
# # # # #
# ]
572 [ + - - + : 2033006 : if (spdk_unlikely(desc->len < sizeof(struct virtio_scsi_cmd_resp) || task->resp == NULL)) {
# # # # #
# # # #
# ]
573 [ # # # # : 0 : SPDK_WARNLOG("%s: response descriptor at index %d points to invalid guest memory region\n",
# # # # ]
574 : : vsession->name, task->req_idx);
575 : 0 : goto invalid_task;
576 : : }
577 : 2033006 : rc = vhost_vring_desc_get_next(&desc, desc_table, desc_table_len);
578 [ - + ]: 2033006 : if (spdk_unlikely(rc != 0)) {
579 [ # # # # : 0 : SPDK_WARNLOG("%s: invalid descriptor chain at request index %d (descriptor id overflow?).\n",
# # # # ]
580 : : vsession->name, task->req_idx);
581 : 0 : goto invalid_task;
582 : : }
583 : :
584 [ + + ]: 2033006 : if (desc == NULL) {
585 : : /*
586 : : * TEST UNIT READY command and some others might not contain any payload and this is not an error.
587 : : */
588 [ - + - + : 175 : SPDK_DEBUGLOG(vhost_scsi_data,
# # # # #
# ]
589 : : "No payload descriptors for FROM DEV command req_idx=%"PRIu16".\n", task->req_idx);
590 [ - + - + : 175 : SPDK_LOGDUMP(vhost_scsi_data, "CDB=", (*req)->cdb, VIRTIO_SCSI_CDB_SIZE);
# # # # #
# ]
591 [ # # # # ]: 175 : task->used_len = sizeof(struct virtio_scsi_cmd_resp);
592 [ # # # # : 175 : task->scsi.iovcnt = 1;
# # ]
593 [ # # # # : 175 : task->scsi.iovs[0].iov_len = 0;
# # # # #
# # # ]
594 [ # # # # : 175 : task->scsi.length = 0;
# # ]
595 [ # # # # : 175 : task->scsi.transfer_len = 0;
# # ]
596 : 175 : return 0;
597 : : }
598 : :
599 : : /* All remaining descriptors are data. */
600 [ + + ]: 7038807 : while (desc) {
601 [ - + ]: 5005976 : if (spdk_unlikely(!vhost_vring_desc_is_wr(desc))) {
602 [ # # # # ]: 0 : SPDK_WARNLOG("%s: FROM DEV cmd: descriptor nr %" PRIu16" in payload chain is read only.\n",
603 : : vsession->name, iovcnt);
604 : 0 : goto invalid_task;
605 : : }
606 : :
607 [ - + ]: 5005976 : if (spdk_unlikely(vhost_vring_desc_to_iov(vsession, iovs, &iovcnt, desc))) {
608 : 0 : goto invalid_task;
609 : : }
610 [ # # # # ]: 5005976 : len += desc->len;
611 : :
612 : 5005976 : rc = vhost_vring_desc_get_next(&desc, desc_table, desc_table_len);
613 [ - + ]: 5005976 : if (spdk_unlikely(rc != 0)) {
614 [ # # # # : 0 : SPDK_WARNLOG("%s: invalid payload in descriptor chain starting at index %d.\n",
# # # # ]
615 : : vsession->name, task->req_idx);
616 : 0 : goto invalid_task;
617 : : }
618 : : }
619 : :
620 [ # # # # ]: 2032831 : task->used_len = sizeof(struct virtio_scsi_cmd_resp) + len;
621 : 0 : } else {
622 [ - + - + : 1961990 : SPDK_DEBUGLOG(vhost_scsi_data, "TO DEV");
# # ]
623 : : /*
624 : : * TO_DEV (WRITE):[RD_req][RD_buf0]...[RD_bufN][WR_resp]
625 : : * No need to check descriptor WR flag as this is done while setting scsi.dxfer_dir.
626 : : */
627 : :
628 : : /* Process descriptors up to response. */
629 [ + + ]: 6368127 : while (!vhost_vring_desc_is_wr(desc)) {
630 [ - + ]: 4406137 : if (spdk_unlikely(vhost_vring_desc_to_iov(vsession, iovs, &iovcnt, desc))) {
631 : 0 : goto invalid_task;
632 : : }
633 [ # # # # ]: 4406137 : len += desc->len;
634 : :
635 : 4406137 : vhost_vring_desc_get_next(&desc, desc_table, desc_table_len);
636 [ - + ]: 4406137 : if (spdk_unlikely(desc == NULL)) {
637 [ # # # # ]: 0 : SPDK_WARNLOG("%s: TO_DEV cmd: no response descriptor.\n", vsession->name);
638 : 0 : goto invalid_task;
639 : : }
640 : : }
641 : :
642 [ # # # # : 1961990 : task->resp = vhost_gpa_to_vva(vsession, desc->addr, sizeof(*task->resp));
# # # # #
# ]
643 [ + - - + : 1961990 : if (spdk_unlikely(desc->len < sizeof(struct virtio_scsi_cmd_resp) || task->resp == NULL)) {
# # # # #
# # # #
# ]
644 [ # # # # : 0 : SPDK_WARNLOG("%s: response descriptor at index %d points to invalid guest memory region\n",
# # # # ]
645 : : vsession->name, task->req_idx);
646 : 0 : goto invalid_task;
647 : : }
648 : :
649 [ # # # # ]: 1961990 : task->used_len = sizeof(struct virtio_scsi_cmd_resp);
650 : : }
651 : :
652 [ # # # # : 3994821 : task->scsi.iovcnt = iovcnt;
# # ]
653 [ # # # # : 3994821 : task->scsi.length = len;
# # ]
654 [ # # # # : 3994821 : task->scsi.transfer_len = len;
# # ]
655 : 3994821 : return 0;
656 : :
657 : 2 : invalid_task:
658 [ - + - + : 2 : SPDK_DEBUGLOG(vhost_scsi_data, "%s: Invalid task at index %"PRIu16".\n",
# # # # #
# # # #
# ]
659 : : vsession->name, task->req_idx);
660 : 2 : return -1;
661 : 0 : }
662 : :
663 : : static int
664 : 3994998 : process_request(struct spdk_vhost_scsi_task *task)
665 : : {
666 : 3994998 : struct virtio_scsi_cmd_req *req;
667 : : int result;
668 : :
669 : 3994998 : result = task_data_setup(task, &req);
670 [ + + ]: 3994998 : if (result) {
671 : 2 : return result;
672 : : }
673 : :
674 [ # # ]: 3994996 : result = vhost_scsi_task_init_target(task, req->lun);
675 [ + + ]: 3994996 : if (spdk_unlikely(result != 0)) {
676 [ # # # # : 6865 : task->resp->response = VIRTIO_SCSI_S_BAD_TARGET;
# # # # #
# ]
677 : 6865 : return -1;
678 : : }
679 : :
680 [ # # # # : 3988131 : task->scsi.cdb = req->cdb;
# # # # ]
681 [ - + - + : 3988131 : SPDK_LOGDUMP(vhost_scsi_data, "request CDB", req->cdb, VIRTIO_SCSI_CDB_SIZE);
# # # # ]
682 : :
683 [ + + # # : 3988131 : if (spdk_unlikely(task->scsi.lun == NULL)) {
# # # # ]
684 [ # # ]: 72443 : spdk_scsi_task_process_null_lun(&task->scsi);
685 [ # # # # : 72443 : task->resp->response = VIRTIO_SCSI_S_OK;
# # # # #
# ]
686 : 72443 : return 1;
687 : : }
688 : :
689 : 3915688 : return 0;
690 : 0 : }
691 : :
692 : : static void
693 : 4038172 : process_scsi_task(struct spdk_vhost_session *vsession,
694 : : struct spdk_vhost_virtqueue *vq,
695 : : uint16_t req_idx)
696 : : {
697 : : struct spdk_vhost_scsi_task *task;
698 : : int result;
699 : :
700 [ # # # # : 4038172 : task = &((struct spdk_vhost_scsi_task *)vq->tasks)[req_idx];
# # ]
701 [ - + - + : 4038172 : if (spdk_unlikely(task->used)) {
# # # # ]
702 [ # # # # ]: 0 : SPDK_ERRLOG("%s: request with idx '%"PRIu16"' is already pending.\n",
703 : : vsession->name, req_idx);
704 : 0 : vhost_vq_used_ring_enqueue(vsession, vq, req_idx, 0);
705 : 0 : return;
706 : : }
707 : :
708 [ # # # # ]: 4038172 : vsession->task_cnt++;
709 : 4038172 : scsi_task_init(task);
710 : :
711 [ + + # # : 4038172 : if (spdk_unlikely(vq->vring_idx == VIRTIO_SCSI_CONTROLQ)) {
# # ]
712 : 43174 : process_ctrl_request(task);
713 : 0 : } else {
714 : 3994998 : result = process_request(task);
715 [ + + ]: 3994998 : if (likely(result == 0)) {
716 : 3915688 : task_submit(task);
717 [ - + - + : 3915688 : SPDK_DEBUGLOG(vhost_scsi, "====== Task %p req_idx %d submitted ======\n", task,
# # # # #
# ]
718 : : task->req_idx);
719 [ + + ]: 79310 : } else if (result > 0) {
720 [ # # ]: 72443 : vhost_scsi_task_cpl(&task->scsi);
721 [ - + - + : 72443 : SPDK_DEBUGLOG(vhost_scsi, "====== Task %p req_idx %d finished early ======\n", task,
# # # # #
# ]
722 : : task->req_idx);
723 : 0 : } else {
724 : 6867 : invalid_request(task);
725 [ - + - + : 6867 : SPDK_DEBUGLOG(vhost_scsi, "====== Task %p req_idx %d failed ======\n", task,
# # # # #
# ]
726 : : task->req_idx);
727 : : }
728 : : }
729 : 0 : }
730 : :
731 : : static int
732 : 162647114 : submit_inflight_desc(struct spdk_vhost_scsi_session *svsession,
733 : : struct spdk_vhost_virtqueue *vq)
734 : : {
735 : : struct spdk_vhost_session *vsession;
736 : : spdk_vhost_resubmit_info *resubmit;
737 : : spdk_vhost_resubmit_desc *resubmit_list;
738 : : uint16_t req_idx;
739 : : int i, resubmit_cnt;
740 : :
741 [ # # # # : 162647114 : resubmit = vq->vring_inflight.resubmit_inflight;
# # ]
742 [ - + - - : 162647114 : if (spdk_likely(resubmit == NULL || resubmit->resubmit_list == NULL ||
- + - - #
# # # #
# ]
743 : : resubmit->resubmit_num == 0)) {
744 : 162647114 : return 0;
745 : : }
746 : :
747 [ # # # # ]: 0 : resubmit_list = resubmit->resubmit_list;
748 [ # # ]: 0 : vsession = &svsession->vsession;
749 : :
750 [ # # # # : 0 : for (i = resubmit->resubmit_num - 1; i >= 0; --i) {
# # # # #
# ]
751 [ # # # # : 0 : req_idx = resubmit_list[i].index;
# # ]
752 [ # # # # : 0 : SPDK_DEBUGLOG(vhost_scsi, "====== Start processing resubmit request idx %"PRIu16"======\n",
# # ]
753 : : req_idx);
754 : :
755 [ # # # # : 0 : if (spdk_unlikely(req_idx >= vq->vring.size)) {
# # # # ]
756 [ # # # # : 0 : SPDK_ERRLOG("%s: request idx '%"PRIu16"' exceeds virtqueue size (%"PRIu16").\n",
# # # # #
# ]
757 : : vsession->name, req_idx, vq->vring.size);
758 : 0 : vhost_vq_used_ring_enqueue(vsession, vq, req_idx, 0);
759 : 0 : continue;
760 : : }
761 : :
762 : 0 : process_scsi_task(vsession, vq, req_idx);
763 : 0 : }
764 [ # # # # ]: 0 : resubmit_cnt = resubmit->resubmit_num;
765 [ # # # # ]: 0 : resubmit->resubmit_num = 0;
766 : 0 : return resubmit_cnt;
767 : 0 : }
768 : :
769 : : static int
770 : 162647114 : process_vq(struct spdk_vhost_scsi_session *svsession, struct spdk_vhost_virtqueue *vq)
771 : : {
772 [ # # ]: 162647114 : struct spdk_vhost_session *vsession = &svsession->vsession;
773 : 162647114 : uint16_t reqs[32];
774 : : uint16_t reqs_cnt, i;
775 : : int resubmit_cnt;
776 : :
777 : 162647114 : resubmit_cnt = submit_inflight_desc(svsession, vq);
778 : :
779 : 162647114 : reqs_cnt = vhost_vq_avail_ring_get(vq, reqs, SPDK_COUNTOF(reqs));
780 [ - + # # ]: 162647114 : assert(reqs_cnt <= 32);
781 : :
782 [ + + ]: 166685286 : for (i = 0; i < reqs_cnt; i++) {
783 [ - + - + : 4038172 : SPDK_DEBUGLOG(vhost_scsi, "====== Starting processing request idx %"PRIu16"======\n",
# # # # #
# # # ]
784 : : reqs[i]);
785 : :
786 [ - + # # : 4038172 : if (spdk_unlikely(reqs[i] >= vq->vring.size)) {
# # # # #
# # # #
# ]
787 [ # # # # : 0 : SPDK_ERRLOG("%s: request idx '%"PRIu16"' exceeds virtqueue size (%"PRIu16").\n",
# # # # #
# # # # #
# # ]
788 : : vsession->name, reqs[i], vq->vring.size);
789 [ # # # # : 0 : vhost_vq_used_ring_enqueue(vsession, vq, reqs[i], 0);
# # ]
790 : 0 : continue;
791 : : }
792 : :
793 [ # # # # : 4038172 : rte_vhost_set_inflight_desc_split(vsession->vid, vq->vring_idx, reqs[i]);
# # # # #
# # # #
# ]
794 : :
795 [ # # # # : 4038172 : process_scsi_task(vsession, vq, reqs[i]);
# # ]
796 : 0 : }
797 : :
798 [ + + ]: 162647114 : return reqs_cnt > 0 ? reqs_cnt : resubmit_cnt;
799 : : }
800 : :
801 : : static int
802 : 149429 : vdev_mgmt_worker(void *arg)
803 : : {
804 : 149429 : struct spdk_vhost_scsi_session *svsession = arg;
805 [ # # ]: 149429 : struct spdk_vhost_session *vsession = &svsession->vsession;
806 : 149429 : int rc = 0;
807 : :
808 : 149429 : process_removed_devs(svsession);
809 : :
810 [ + + # # : 149429 : if (vsession->virtqueue[VIRTIO_SCSI_EVENTQ].vring.desc) {
# # # # #
# # # # #
# # ]
811 [ # # # # : 130649 : vhost_vq_used_signal(vsession, &vsession->virtqueue[VIRTIO_SCSI_EVENTQ]);
# # ]
812 : 0 : }
813 : :
814 [ + + # # : 149429 : if (vsession->virtqueue[VIRTIO_SCSI_CONTROLQ].vring.desc) {
# # # # #
# # # # #
# # ]
815 [ # # # # : 130649 : rc = process_vq(svsession, &vsession->virtqueue[VIRTIO_SCSI_CONTROLQ]);
# # ]
816 [ # # # # : 130649 : vhost_vq_used_signal(vsession, &vsession->virtqueue[VIRTIO_SCSI_CONTROLQ]);
# # ]
817 : 0 : }
818 : :
819 : 149429 : return rc > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;
820 : : }
821 : :
822 : : static int
823 : 83743420 : vdev_worker(void *arg)
824 : : {
825 : 83743420 : struct spdk_vhost_scsi_session *svsession = arg;
826 [ # # ]: 83743420 : struct spdk_vhost_session *vsession = &svsession->vsession;
827 : : uint32_t q_idx;
828 : 83743420 : int rc = 0;
829 : :
830 [ + + # # : 246259885 : for (q_idx = VIRTIO_SCSI_REQUESTQ; q_idx < vsession->max_queues; q_idx++) {
# # ]
831 [ # # # # : 162516465 : rc = process_vq(svsession, &vsession->virtqueue[q_idx]);
# # ]
832 [ # # # # : 162516465 : vhost_session_vq_used_signal(&vsession->virtqueue[q_idx]);
# # ]
833 : 0 : }
834 : :
835 : 83743420 : return rc > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;
836 : : }
837 : :
838 : : static struct spdk_vhost_scsi_dev *
839 : 3559 : to_scsi_dev(struct spdk_vhost_dev *ctrlr)
840 : : {
841 [ - + ]: 3559 : if (ctrlr == NULL) {
842 : 0 : return NULL;
843 : : }
844 : :
845 [ - + # # : 3559 : if (ctrlr->backend->type != VHOST_BACKEND_SCSI) {
# # # # #
# ]
846 [ # # # # ]: 0 : SPDK_ERRLOG("%s: not a vhost-scsi device.\n", ctrlr->name);
847 : 0 : return NULL;
848 : : }
849 : :
850 : 3559 : return SPDK_CONTAINEROF(ctrlr, struct spdk_vhost_scsi_dev, vdev);
851 : 0 : }
852 : :
853 : : static struct spdk_vhost_scsi_session *
854 : 324 : to_scsi_session(struct spdk_vhost_session *vsession)
855 : : {
856 [ - + # # : 324 : assert(vsession->vdev->backend->type == VHOST_BACKEND_SCSI);
# # # # #
# # # # #
# # ]
857 : 324 : return (struct spdk_vhost_scsi_session *)vsession;
858 : : }
859 : :
860 : : int
861 : 7 : vhost_scsi_controller_start(const char *name)
862 : : {
863 : : struct spdk_vhost_dev *vdev;
864 : : struct spdk_vhost_scsi_dev *svdev;
865 : : int rc;
866 : :
867 : 7 : spdk_vhost_lock();
868 : 7 : vdev = spdk_vhost_dev_find(name);
869 [ - + ]: 7 : if (vdev == NULL) {
870 : 0 : spdk_vhost_unlock();
871 : 0 : return -ENODEV;
872 : : }
873 : :
874 : 7 : svdev = to_scsi_dev(vdev);
875 [ - + # # ]: 7 : assert(svdev != NULL);
876 : :
877 [ - + - + : 7 : if (svdev->registered == true) {
# # # # ]
878 : : /* already started, nothing to do */
879 : 0 : spdk_vhost_unlock();
880 : 0 : return 0;
881 : : }
882 : :
883 : 7 : rc = vhost_user_dev_start(vdev);
884 [ - + ]: 7 : if (rc != 0) {
885 : 0 : spdk_vhost_unlock();
886 : 0 : return rc;
887 : : }
888 [ # # # # ]: 7 : svdev->registered = true;
889 : :
890 : 7 : spdk_vhost_unlock();
891 : 7 : return 0;
892 : 0 : }
893 : :
894 : : static int
895 : 37 : vhost_scsi_dev_construct(const char *name, const char *cpumask, bool delay)
896 : : {
897 : 37 : struct spdk_vhost_scsi_dev *svdev = calloc(1, sizeof(*svdev));
898 : : int rc;
899 : :
900 [ - + ]: 37 : if (svdev == NULL) {
901 : 0 : return -ENOMEM;
902 : : }
903 : :
904 [ # # # # : 37 : svdev->vdev.virtio_features = SPDK_VHOST_SCSI_FEATURES;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
905 [ # # # # : 37 : svdev->vdev.disabled_features = SPDK_VHOST_SCSI_DISABLED_FEATURES;
# # # # #
# # # ]
906 [ # # # # : 37 : svdev->vdev.protocol_features = SPDK_VHOST_SCSI_PROTOCOL_FEATURES;
# # # # ]
907 : :
908 [ # # ]: 37 : rc = vhost_dev_register(&svdev->vdev, name, cpumask, NULL,
909 : : &spdk_vhost_scsi_device_backend,
910 [ # # ]: 0 : &spdk_vhost_scsi_user_device_backend, delay);
911 [ + + ]: 37 : if (rc) {
912 : 3 : free(svdev);
913 : 3 : return rc;
914 : : }
915 : :
916 [ + + # # ]: 34 : if (delay == false) {
917 [ # # # # ]: 27 : svdev->registered = true;
918 : 0 : }
919 : :
920 : 34 : return rc;
921 : 0 : }
922 : :
923 : : int
924 : 30 : spdk_vhost_scsi_dev_construct(const char *name, const char *cpumask)
925 : : {
926 : 30 : return vhost_scsi_dev_construct(name, cpumask, false);
927 : : }
928 : :
929 : : int
930 : 7 : spdk_vhost_scsi_dev_construct_no_start(const char *name, const char *cpumask)
931 : : {
932 : 7 : return vhost_scsi_dev_construct(name, cpumask, true);
933 : : }
934 : :
935 : : static int
936 : 34 : vhost_scsi_dev_remove(struct spdk_vhost_dev *vdev)
937 : : {
938 : 34 : struct spdk_vhost_scsi_dev *svdev = to_scsi_dev(vdev);
939 : 34 : int rc = 0, i;
940 : :
941 [ - + # # ]: 34 : assert(svdev != NULL);
942 : :
943 [ - + ]: 34 : if (vhost_user_dev_busy(vdev)) {
944 : 0 : return -EBUSY;
945 : : }
946 : :
947 [ + + # # ]: 306 : for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; ++i) {
948 [ + + # # : 272 : if (svdev->scsi_dev_state[i].dev) {
# # # # #
# ]
949 : 12 : rc = spdk_vhost_scsi_dev_remove_tgt(vdev, i, NULL, NULL);
950 [ - + ]: 12 : if (rc != 0) {
951 [ # # # # ]: 0 : SPDK_ERRLOG("%s: failed to force-remove target %d\n", vdev->name, i);
952 : 0 : return rc;
953 : : }
954 : 0 : }
955 : 0 : }
956 : :
957 [ # # # # ]: 34 : svdev->registered = false;
958 : :
959 [ + + # # : 34 : if (svdev->ref == 0) {
# # ]
960 : 24 : rc = vhost_dev_unregister(vdev);
961 [ - + ]: 24 : if (rc != 0) {
962 : 0 : return rc;
963 : : }
964 : 24 : free(svdev);
965 : 0 : }
966 : :
967 : 34 : return rc;
968 : 0 : }
969 : :
970 : : struct spdk_scsi_dev *
971 : 3352 : spdk_vhost_scsi_dev_get_tgt(struct spdk_vhost_dev *vdev, uint8_t num)
972 : : {
973 : : struct spdk_vhost_scsi_dev *svdev;
974 : :
975 [ - + # # ]: 3352 : assert(num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS);
976 : 3352 : svdev = to_scsi_dev(vdev);
977 [ - + # # ]: 3352 : assert(svdev != NULL);
978 [ + + # # : 3352 : if (svdev->scsi_dev_state[num].status != VHOST_SCSI_DEV_PRESENT) {
# # # # #
# ]
979 : 2720 : return NULL;
980 : : }
981 : :
982 [ - + # # : 632 : assert(svdev->scsi_dev_state[num].dev != NULL);
# # # # #
# # # ]
983 [ # # # # : 632 : return svdev->scsi_dev_state[num].dev;
# # # # ]
984 : 0 : }
985 : :
986 : : static unsigned
987 : 59 : get_scsi_dev_num(const struct spdk_vhost_scsi_dev *svdev,
988 : : const struct spdk_scsi_lun *lun)
989 : : {
990 : : const struct spdk_scsi_dev *scsi_dev;
991 : : unsigned scsi_dev_num;
992 : :
993 [ - + # # ]: 59 : assert(lun != NULL);
994 [ - + # # ]: 59 : assert(svdev != NULL);
995 : 59 : scsi_dev = spdk_scsi_lun_get_dev(lun);
996 [ + + ]: 465 : for (scsi_dev_num = 0; scsi_dev_num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; scsi_dev_num++) {
997 [ + + # # : 415 : if (svdev->scsi_dev_state[scsi_dev_num].dev == scsi_dev) {
# # # # #
# ]
998 : 9 : break;
999 : : }
1000 : 0 : }
1001 : :
1002 : 59 : return scsi_dev_num;
1003 : : }
1004 : :
1005 : : static void
1006 : 0 : vhost_scsi_lun_resize(const struct spdk_scsi_lun *lun, void *arg)
1007 : : {
1008 : 0 : struct spdk_vhost_scsi_dev *svdev = arg;
1009 : : unsigned scsi_dev_num;
1010 : :
1011 : 0 : scsi_dev_num = get_scsi_dev_num(svdev, lun);
1012 [ # # ]: 0 : if (scsi_dev_num == SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
1013 : : /* The entire device has been already removed. */
1014 : 0 : return;
1015 : : }
1016 : :
1017 [ # # ]: 0 : vhost_scsi_dev_param_changed(&svdev->vdev, scsi_dev_num);
1018 : 0 : }
1019 : :
1020 : : static void
1021 : 59 : vhost_scsi_lun_hotremove(const struct spdk_scsi_lun *lun, void *arg)
1022 : : {
1023 : 59 : struct spdk_vhost_scsi_dev *svdev = arg;
1024 : : unsigned scsi_dev_num;
1025 : :
1026 : 59 : scsi_dev_num = get_scsi_dev_num(svdev, lun);
1027 [ + + ]: 59 : if (scsi_dev_num == SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
1028 : : /* The entire device has been already removed. */
1029 : 50 : return;
1030 : : }
1031 : :
1032 : : /* remove entire device */
1033 [ # # ]: 9 : spdk_vhost_scsi_dev_remove_tgt(&svdev->vdev, scsi_dev_num, NULL, NULL);
1034 : 0 : }
1035 : :
1036 : : static void
1037 : 44 : vhost_scsi_dev_add_tgt_cpl_cb(struct spdk_vhost_dev *vdev, void *ctx)
1038 : : {
1039 : 44 : unsigned scsi_tgt_num = (unsigned)(uintptr_t)ctx;
1040 : 44 : struct spdk_vhost_scsi_dev *svdev = SPDK_CONTAINEROF(vdev,
1041 : : struct spdk_vhost_scsi_dev, vdev);
1042 : : struct spdk_scsi_dev_vhost_state *vhost_sdev;
1043 : :
1044 [ # # # # ]: 44 : vhost_sdev = &svdev->scsi_dev_state[scsi_tgt_num];
1045 : :
1046 : : /* All sessions have added the target */
1047 [ - + # # : 44 : assert(vhost_sdev->status == VHOST_SCSI_DEV_ADDING);
# # # # ]
1048 [ # # # # ]: 44 : vhost_sdev->status = VHOST_SCSI_DEV_PRESENT;
1049 [ # # # # ]: 44 : svdev->ref++;
1050 : 44 : }
1051 : :
1052 : : static int
1053 : 4 : vhost_scsi_session_add_tgt(struct spdk_vhost_dev *vdev,
1054 : : struct spdk_vhost_session *vsession, void *ctx)
1055 : : {
1056 : 4 : unsigned scsi_tgt_num = (unsigned)(uintptr_t)ctx;
1057 : 4 : struct spdk_vhost_scsi_session *svsession = (struct spdk_vhost_scsi_session *)vsession;
1058 [ # # # # : 4 : struct spdk_scsi_dev_session_state *session_sdev = &svsession->scsi_dev_state[scsi_tgt_num];
# # ]
1059 : : struct spdk_scsi_dev_vhost_state *vhost_sdev;
1060 : : int rc;
1061 : :
1062 [ - + + - : 4 : if (!vsession->started || session_sdev->dev != NULL) {
- + # # #
# # # #
# ]
1063 : : /* Nothing to do. */
1064 : 0 : return 0;
1065 : : }
1066 : :
1067 [ # # # # : 4 : vhost_sdev = &svsession->svdev->scsi_dev_state[scsi_tgt_num];
# # # # ]
1068 [ # # # # : 4 : session_sdev->dev = vhost_sdev->dev;
# # # # ]
1069 [ # # # # ]: 4 : session_sdev->status = VHOST_SCSI_DEV_PRESENT;
1070 : :
1071 [ # # # # : 4 : rc = spdk_scsi_dev_allocate_io_channels(svsession->scsi_dev_state[scsi_tgt_num].dev);
# # # # #
# ]
1072 [ - + ]: 4 : if (rc != 0) {
1073 [ # # # # ]: 0 : SPDK_ERRLOG("%s: Couldn't allocate io channel for SCSI target %u.\n",
1074 : : vsession->name, scsi_tgt_num);
1075 : :
1076 : : /* unset the SCSI target so that all I/O to it will be rejected */
1077 [ # # # # ]: 0 : session_sdev->dev = NULL;
1078 : : /* Set status to EMPTY so that we won't reply with SCSI hotremove
1079 : : * sense codes - the device hasn't ever been added.
1080 : : */
1081 [ # # # # ]: 0 : session_sdev->status = VHOST_SCSI_DEV_EMPTY;
1082 : :
1083 : : /* Return with no error. We'll continue allocating io_channels for
1084 : : * other sessions on this device in hopes they succeed. The sessions
1085 : : * that failed to allocate io_channels simply won't be able to
1086 : : * detect the SCSI target, nor do any I/O to it.
1087 : : */
1088 : 0 : return 0;
1089 : : }
1090 : :
1091 [ + - ]: 4 : if (vhost_dev_has_feature(vsession, VIRTIO_SCSI_F_HOTPLUG)) {
1092 : 4 : eventq_enqueue(svsession, scsi_tgt_num,
1093 : : VIRTIO_SCSI_T_TRANSPORT_RESET, VIRTIO_SCSI_EVT_RESET_RESCAN);
1094 : 0 : } else {
1095 [ # # # # ]: 0 : SPDK_NOTICELOG("%s: driver does not support hotplug. "
1096 : : "Please restart it or perform a rescan.\n",
1097 : : vsession->name);
1098 : : }
1099 : :
1100 : 4 : return 0;
1101 : 0 : }
1102 : :
1103 : : int
1104 : 63 : spdk_vhost_scsi_dev_add_tgt(struct spdk_vhost_dev *vdev, int scsi_tgt_num,
1105 : : const char *bdev_name)
1106 : : {
1107 : : struct spdk_vhost_scsi_dev *svdev;
1108 : : struct spdk_scsi_dev_vhost_state *state;
1109 : 63 : char target_name[SPDK_SCSI_DEV_MAX_NAME];
1110 : 63 : int lun_id_list[1];
1111 : 63 : const char *bdev_names_list[1];
1112 : :
1113 : 63 : svdev = to_scsi_dev(vdev);
1114 [ - + ]: 63 : if (!svdev) {
1115 : 0 : SPDK_ERRLOG("Before adding a SCSI target, there should be a SCSI device.");
1116 : 0 : return -EINVAL;
1117 : : }
1118 : :
1119 [ + + ]: 63 : if (scsi_tgt_num < 0) {
1120 [ + + # # ]: 49 : for (scsi_tgt_num = 0; scsi_tgt_num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; scsi_tgt_num++) {
1121 [ + + # # : 48 : if (svdev->scsi_dev_state[scsi_tgt_num].dev == NULL) {
# # # # #
# ]
1122 : 10 : break;
1123 : : }
1124 : 0 : }
1125 : :
1126 [ + + ]: 11 : if (scsi_tgt_num == SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
1127 [ # # # # ]: 1 : SPDK_ERRLOG("%s: all SCSI target slots are already in use.\n", vdev->name);
1128 : 1 : return -ENOSPC;
1129 : : }
1130 : 0 : } else {
1131 [ + + ]: 52 : if (scsi_tgt_num >= SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
1132 [ # # # # ]: 1 : SPDK_ERRLOG("%s: SCSI target number is too big (got %d, max %d), started from 0.\n",
1133 : : vdev->name, scsi_tgt_num, SPDK_VHOST_SCSI_CTRLR_MAX_DEVS - 1);
1134 : 1 : return -EINVAL;
1135 : : }
1136 : : }
1137 : :
1138 [ - + ]: 61 : if (bdev_name == NULL) {
1139 : 0 : SPDK_ERRLOG("No lun name specified\n");
1140 : 0 : return -EINVAL;
1141 : : }
1142 : :
1143 [ # # # # ]: 61 : state = &svdev->scsi_dev_state[scsi_tgt_num];
1144 [ + + # # : 61 : if (state->dev != NULL) {
# # ]
1145 [ # # # # ]: 1 : SPDK_ERRLOG("%s: SCSI target %u already occupied\n", vdev->name, scsi_tgt_num);
1146 : 1 : return -EEXIST;
1147 : : }
1148 : :
1149 : : /*
1150 : : * At this stage only one LUN per target
1151 : : */
1152 [ - + ]: 60 : snprintf(target_name, sizeof(target_name), "Target %u", scsi_tgt_num);
1153 [ # # # # ]: 60 : lun_id_list[0] = 0;
1154 [ # # # # ]: 60 : bdev_names_list[0] = (char *)bdev_name;
1155 : :
1156 [ # # # # ]: 60 : state->status = VHOST_SCSI_DEV_ADDING;
1157 [ # # # # : 60 : state->dev = spdk_scsi_dev_construct_ext(target_name, bdev_names_list, lun_id_list, 1,
# # ]
1158 : : SPDK_SPC_PROTOCOL_IDENTIFIER_SAS,
1159 : 0 : vhost_scsi_lun_resize, svdev,
1160 : 0 : vhost_scsi_lun_hotremove, svdev);
1161 : :
1162 [ + + # # : 60 : if (state->dev == NULL) {
# # ]
1163 [ # # # # ]: 1 : state->status = VHOST_SCSI_DEV_EMPTY;
1164 [ # # # # ]: 1 : SPDK_ERRLOG("%s: couldn't create SCSI target %u using bdev '%s'\n",
1165 : : vdev->name, scsi_tgt_num, bdev_name);
1166 : 1 : return -EINVAL;
1167 : : }
1168 [ # # # # ]: 59 : spdk_scsi_dev_add_port(state->dev, 0, "vhost");
1169 : :
1170 [ - + - + : 59 : SPDK_INFOLOG(vhost, "%s: added SCSI target %u using bdev '%s'\n",
# # # # #
# ]
1171 : : vdev->name, scsi_tgt_num, bdev_name);
1172 : :
1173 [ - + + + : 59 : if (svdev->registered) {
# # # # ]
1174 : 44 : vhost_user_dev_foreach_session(vdev, vhost_scsi_session_add_tgt,
1175 : : vhost_scsi_dev_add_tgt_cpl_cb,
1176 : 44 : (void *)(uintptr_t)scsi_tgt_num);
1177 : 0 : } else {
1178 [ # # # # ]: 15 : state->status = VHOST_SCSI_DEV_PRESENT;
1179 [ # # # # ]: 15 : svdev->ref++;
1180 : : }
1181 : :
1182 : 59 : return scsi_tgt_num;
1183 : 0 : }
1184 : :
1185 : : struct scsi_tgt_hotplug_ctx {
1186 : : unsigned scsi_tgt_num;
1187 : : bool async_fini;
1188 : : };
1189 : :
1190 : : static void
1191 : 59 : vhost_scsi_dev_remove_tgt_cpl_cb(struct spdk_vhost_dev *vdev, void *_ctx)
1192 : : {
1193 : 59 : struct scsi_tgt_hotplug_ctx *ctx = _ctx;
1194 : 59 : struct spdk_vhost_scsi_dev *svdev = SPDK_CONTAINEROF(vdev,
1195 : : struct spdk_vhost_scsi_dev, vdev);
1196 : :
1197 [ - + + + : 59 : if (!ctx->async_fini) {
# # # # ]
1198 : : /* there aren't any active sessions, so remove the dev and exit */
1199 [ # # # # ]: 55 : remove_scsi_tgt(svdev, ctx->scsi_tgt_num);
1200 : 0 : }
1201 : :
1202 : 59 : free(ctx);
1203 : 59 : }
1204 : :
1205 : : static int
1206 : 4 : vhost_scsi_session_remove_tgt(struct spdk_vhost_dev *vdev,
1207 : : struct spdk_vhost_session *vsession, void *_ctx)
1208 : : {
1209 : 4 : struct scsi_tgt_hotplug_ctx *ctx = _ctx;
1210 [ # # # # ]: 4 : unsigned scsi_tgt_num = ctx->scsi_tgt_num;
1211 : 4 : struct spdk_vhost_scsi_session *svsession = (struct spdk_vhost_scsi_session *)vsession;
1212 [ # # # # : 4 : struct spdk_scsi_dev_session_state *state = &svsession->scsi_dev_state[scsi_tgt_num];
# # ]
1213 : :
1214 [ - + + - : 4 : if (!vsession->started || state->dev == NULL) {
- + # # #
# # # #
# ]
1215 : : /* Nothing to do */
1216 : 0 : return 0;
1217 : : }
1218 : :
1219 : : /* Mark the target for removal */
1220 [ - + # # : 4 : assert(state->status == VHOST_SCSI_DEV_PRESENT);
# # # # ]
1221 [ - + # # ]: 4 : state->status = VHOST_SCSI_DEV_REMOVING;
1222 : :
1223 : : /* Send a hotremove virtio event */
1224 [ + - ]: 4 : if (vhost_dev_has_feature(vsession, VIRTIO_SCSI_F_HOTPLUG)) {
1225 : 4 : eventq_enqueue(svsession, scsi_tgt_num,
1226 : : VIRTIO_SCSI_T_TRANSPORT_RESET, VIRTIO_SCSI_EVT_RESET_REMOVED);
1227 : 0 : }
1228 : :
1229 : : /* Wait for the session's management poller to remove the target after
1230 : : * all its pending I/O has finished.
1231 : : */
1232 [ # # # # ]: 4 : ctx->async_fini = true;
1233 : 4 : return 0;
1234 : 0 : }
1235 : :
1236 : : int
1237 : 62 : spdk_vhost_scsi_dev_remove_tgt(struct spdk_vhost_dev *vdev, unsigned scsi_tgt_num,
1238 : : spdk_vhost_event_fn cb_fn, void *cb_arg)
1239 : : {
1240 : : struct spdk_vhost_scsi_dev *svdev;
1241 : : struct spdk_scsi_dev_vhost_state *scsi_dev_state;
1242 : : struct scsi_tgt_hotplug_ctx *ctx;
1243 : :
1244 [ + + ]: 62 : if (scsi_tgt_num >= SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
1245 [ # # # # ]: 1 : SPDK_ERRLOG("%s: invalid SCSI target number %d\n", vdev->name, scsi_tgt_num);
1246 : 1 : return -EINVAL;
1247 : : }
1248 : :
1249 : 61 : svdev = to_scsi_dev(vdev);
1250 [ - + ]: 61 : if (!svdev) {
1251 : 0 : SPDK_ERRLOG("An invalid SCSI device that removing from a SCSI target.");
1252 : 0 : return -EINVAL;
1253 : : }
1254 : :
1255 [ # # # # ]: 61 : scsi_dev_state = &svdev->scsi_dev_state[scsi_tgt_num];
1256 : :
1257 [ + + # # : 61 : if (scsi_dev_state->status != VHOST_SCSI_DEV_PRESENT) {
# # ]
1258 : 2 : return -EBUSY;
1259 : : }
1260 : :
1261 [ + - - + : 59 : if (scsi_dev_state->dev == NULL || scsi_dev_state->status == VHOST_SCSI_DEV_ADDING) {
# # # # #
# # # ]
1262 [ # # # # ]: 0 : SPDK_ERRLOG("%s: SCSI target %u is not occupied\n", vdev->name, scsi_tgt_num);
1263 : 0 : return -ENODEV;
1264 : : }
1265 : :
1266 [ - + # # : 59 : assert(scsi_dev_state->status != VHOST_SCSI_DEV_EMPTY);
# # # # ]
1267 : 59 : ctx = calloc(1, sizeof(*ctx));
1268 [ - + ]: 59 : if (ctx == NULL) {
1269 : 0 : SPDK_ERRLOG("calloc failed\n");
1270 : 0 : return -ENOMEM;
1271 : : }
1272 : :
1273 [ # # # # ]: 59 : ctx->scsi_tgt_num = scsi_tgt_num;
1274 [ # # # # ]: 59 : ctx->async_fini = false;
1275 : :
1276 [ # # # # ]: 59 : scsi_dev_state->remove_cb = cb_fn;
1277 [ # # # # ]: 59 : scsi_dev_state->remove_ctx = cb_arg;
1278 [ # # # # ]: 59 : scsi_dev_state->status = VHOST_SCSI_DEV_REMOVING;
1279 : :
1280 : 59 : vhost_user_dev_foreach_session(vdev, vhost_scsi_session_remove_tgt,
1281 : 0 : vhost_scsi_dev_remove_tgt_cpl_cb, ctx);
1282 : 59 : return 0;
1283 : 0 : }
1284 : :
1285 : : static int
1286 : 0 : vhost_scsi_session_param_changed(struct spdk_vhost_dev *vdev,
1287 : : struct spdk_vhost_session *vsession, void *ctx)
1288 : : {
1289 : 0 : unsigned scsi_tgt_num = (unsigned)(uintptr_t)ctx;
1290 : 0 : struct spdk_vhost_scsi_session *svsession = (struct spdk_vhost_scsi_session *)vsession;
1291 [ # # # # : 0 : struct spdk_scsi_dev_session_state *state = &svsession->scsi_dev_state[scsi_tgt_num];
# # ]
1292 : :
1293 [ # # # # : 0 : if (!vsession->started || state->dev == NULL) {
# # # # #
# # # #
# ]
1294 : : /* Nothing to do */
1295 : 0 : return 0;
1296 : : }
1297 : :
1298 : : /* Send a parameter change virtio event */
1299 [ # # ]: 0 : if (vhost_dev_has_feature(vsession, VIRTIO_SCSI_F_CHANGE)) {
1300 : : /*
1301 : : * virtio 1.0 spec says:
1302 : : * By sending this event, the device signals a change in the configuration
1303 : : * parameters of a logical unit, for example the capacity or cache mode.
1304 : : * event is set to VIRTIO_SCSI_T_PARAM_CHANGE. lun addresses a logical unit
1305 : : * in the SCSI host. The same event SHOULD also be reported as a unit
1306 : : * attention condition. reason contains the additional sense code and
1307 : : * additional sense code qualifier, respectively in bits 0…7 and 8…15.
1308 : : * Note: For example, a change in * capacity will be reported as asc
1309 : : * 0x2a, ascq 0x09 (CAPACITY DATA HAS CHANGED).
1310 : : */
1311 [ # # # # ]: 0 : eventq_enqueue(svsession, scsi_tgt_num, VIRTIO_SCSI_T_PARAM_CHANGE, 0x2a | (0x09 << 8));
1312 : 0 : }
1313 : :
1314 : 0 : return 0;
1315 : 0 : }
1316 : :
1317 : : static int
1318 : 0 : vhost_scsi_dev_param_changed(struct spdk_vhost_dev *vdev, unsigned scsi_tgt_num)
1319 : : {
1320 : : struct spdk_vhost_scsi_dev *svdev;
1321 : : struct spdk_scsi_dev_vhost_state *scsi_dev_state;
1322 : :
1323 [ # # ]: 0 : if (scsi_tgt_num >= SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
1324 [ # # # # ]: 0 : SPDK_ERRLOG("%s: invalid SCSI target number %d\n", vdev->name, scsi_tgt_num);
1325 : 0 : return -EINVAL;
1326 : : }
1327 : :
1328 : 0 : svdev = to_scsi_dev(vdev);
1329 [ # # ]: 0 : if (!svdev) {
1330 : 0 : SPDK_ERRLOG("An invalid SCSI device that removing from a SCSI target.");
1331 : 0 : return -EINVAL;
1332 : : }
1333 : :
1334 [ # # # # ]: 0 : scsi_dev_state = &svdev->scsi_dev_state[scsi_tgt_num];
1335 : :
1336 [ # # # # : 0 : if (scsi_dev_state->status != VHOST_SCSI_DEV_PRESENT) {
# # ]
1337 : 0 : return -EBUSY;
1338 : : }
1339 : :
1340 [ # # # # : 0 : if (scsi_dev_state->dev == NULL || scsi_dev_state->status == VHOST_SCSI_DEV_ADDING) {
# # # # #
# # # ]
1341 [ # # # # ]: 0 : SPDK_ERRLOG("%s: SCSI target %u is not occupied\n", vdev->name, scsi_tgt_num);
1342 : 0 : return -ENODEV;
1343 : : }
1344 : :
1345 [ # # # # : 0 : assert(scsi_dev_state->status != VHOST_SCSI_DEV_EMPTY);
# # # # ]
1346 : :
1347 : 0 : vhost_user_dev_foreach_session(vdev, vhost_scsi_session_param_changed,
1348 : 0 : NULL, (void *)(uintptr_t)scsi_tgt_num);
1349 : 0 : return 0;
1350 : 0 : }
1351 : :
1352 : : static void
1353 : 42 : free_task_pool(struct spdk_vhost_scsi_session *svsession)
1354 : : {
1355 [ # # ]: 42 : struct spdk_vhost_session *vsession = &svsession->vsession;
1356 : : struct spdk_vhost_virtqueue *vq;
1357 : : uint16_t i;
1358 : :
1359 [ + + # # : 245 : for (i = 0; i < vsession->max_queues; i++) {
# # ]
1360 [ # # # # : 203 : vq = &vsession->virtqueue[i];
# # ]
1361 [ + + # # : 203 : if (vq->tasks == NULL) {
# # ]
1362 : 24 : continue;
1363 : : }
1364 : :
1365 [ # # # # ]: 179 : spdk_free(vq->tasks);
1366 [ # # # # ]: 179 : vq->tasks = NULL;
1367 : 0 : }
1368 : 42 : }
1369 : :
1370 : : static int
1371 : 179 : alloc_vq_task_pool(struct spdk_vhost_session *vsession, uint16_t qid)
1372 : : {
1373 : 179 : struct spdk_vhost_scsi_session *svsession = to_scsi_session(vsession);
1374 : : struct spdk_vhost_virtqueue *vq;
1375 : : struct spdk_vhost_scsi_task *task;
1376 : : uint32_t task_cnt;
1377 : : uint32_t j;
1378 : :
1379 [ - + ]: 179 : if (qid >= SPDK_VHOST_MAX_VQUEUES) {
1380 : 0 : return -EINVAL;
1381 : : }
1382 : :
1383 [ # # # # : 179 : vq = &vsession->virtqueue[qid];
# # ]
1384 [ - + # # : 179 : if (vq->vring.desc == NULL) {
# # # # #
# ]
1385 : 0 : return 0;
1386 : : }
1387 : :
1388 [ # # # # : 179 : task_cnt = vq->vring.size;
# # ]
1389 [ - + ]: 179 : if (task_cnt > SPDK_VHOST_MAX_VQ_SIZE) {
1390 : : /* sanity check */
1391 [ # # # # ]: 0 : SPDK_ERRLOG("%s: virtqueue %"PRIu16" is too big. (size = %"PRIu32", max = %"PRIu32")\n",
1392 : : vsession->name, qid, task_cnt, SPDK_VHOST_MAX_VQ_SIZE);
1393 : 0 : return -1;
1394 : : }
1395 [ # # # # ]: 179 : vq->tasks = spdk_zmalloc(sizeof(struct spdk_vhost_scsi_task) * task_cnt,
1396 : : SPDK_CACHE_LINE_SIZE, NULL,
1397 : : SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
1398 [ - + # # : 179 : if (vq->tasks == NULL) {
# # ]
1399 [ # # # # ]: 0 : SPDK_ERRLOG("%s: failed to allocate %"PRIu32" tasks for virtqueue %"PRIu16"\n",
1400 : : vsession->name, task_cnt, qid);
1401 : 0 : return -1;
1402 : : }
1403 : :
1404 [ + + ]: 71859 : for (j = 0; j < task_cnt; j++) {
1405 [ # # # # : 71680 : task = &((struct spdk_vhost_scsi_task *)vq->tasks)[j];
# # ]
1406 [ # # # # ]: 71680 : task->svsession = svsession;
1407 [ # # # # ]: 71680 : task->vq = vq;
1408 [ # # # # ]: 71680 : task->req_idx = j;
1409 : 0 : }
1410 : :
1411 : 179 : return 0;
1412 : 0 : }
1413 : :
1414 : : static int
1415 : 103 : vhost_scsi_start(struct spdk_vhost_dev *vdev,
1416 : : struct spdk_vhost_session *vsession, void *unused)
1417 : : {
1418 : 103 : struct spdk_vhost_scsi_session *svsession = to_scsi_session(vsession);
1419 : : struct spdk_vhost_scsi_dev *svdev;
1420 : : struct spdk_scsi_dev_vhost_state *state;
1421 : : uint32_t i;
1422 : : int rc;
1423 : :
1424 : : /* return if start is already in progress */
1425 [ + + # # : 103 : if (svsession->requestq_poller) {
# # ]
1426 [ - + - + : 2 : SPDK_INFOLOG(vhost, "%s: start in progress\n", vsession->name);
# # # # #
# ]
1427 : 2 : return -EINPROGRESS;
1428 : : }
1429 : :
1430 : : /* validate all I/O queues are in a contiguous index range */
1431 [ + + # # : 101 : if (vsession->max_queues < VIRTIO_SCSI_REQUESTQ + 1) {
# # ]
1432 [ - + - + : 59 : SPDK_INFOLOG(vhost, "%s: max_queues %u, no I/O queues\n", vsession->name, vsession->max_queues);
# # # # #
# # # #
# ]
1433 : 59 : return -1;
1434 : : }
1435 [ + + # # : 84 : for (i = VIRTIO_SCSI_REQUESTQ; i < vsession->max_queues; i++) {
# # ]
1436 [ - + # # : 42 : if (vsession->virtqueue[i].vring.desc == NULL) {
# # # # #
# # # # #
# # ]
1437 [ # # # # ]: 0 : SPDK_ERRLOG("%s: queue %"PRIu32" is empty\n", vsession->name, i);
1438 : 0 : return -1;
1439 : : }
1440 : 0 : }
1441 : :
1442 [ # # # # ]: 42 : svdev = to_scsi_dev(vsession->vdev);
1443 [ - + # # ]: 42 : assert(svdev != NULL);
1444 [ # # # # ]: 42 : svsession->svdev = svdev;
1445 : :
1446 [ + + ]: 378 : for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
1447 [ # # # # ]: 336 : state = &svdev->scsi_dev_state[i];
1448 [ + + - + : 336 : if (state->dev == NULL || state->status == VHOST_SCSI_DEV_REMOVING) {
# # # # #
# # # ]
1449 : 283 : continue;
1450 : : }
1451 : :
1452 [ - + # # : 53 : assert(svsession->scsi_dev_state[i].status == VHOST_SCSI_DEV_EMPTY);
# # # # #
# # # #
# ]
1453 [ # # # # : 53 : svsession->scsi_dev_state[i].dev = state->dev;
# # # # #
# # # #
# ]
1454 [ # # # # : 53 : svsession->scsi_dev_state[i].status = VHOST_SCSI_DEV_PRESENT;
# # # # #
# ]
1455 [ # # # # ]: 53 : rc = spdk_scsi_dev_allocate_io_channels(state->dev);
1456 [ - + ]: 53 : if (rc != 0) {
1457 [ # # # # ]: 0 : SPDK_ERRLOG("%s: failed to alloc io_channel for SCSI target %"PRIu32"\n",
1458 : : vsession->name, i);
1459 : : /* unset the SCSI target so that all I/O to it will be rejected */
1460 [ # # # # : 0 : svsession->scsi_dev_state[i].dev = NULL;
# # # # #
# ]
1461 : : /* set EMPTY state so that we won't reply with SCSI hotremove
1462 : : * sense codes - the device hasn't ever been added.
1463 : : */
1464 [ # # # # : 0 : svsession->scsi_dev_state[i].status = VHOST_SCSI_DEV_EMPTY;
# # # # #
# ]
1465 : 0 : continue;
1466 : : }
1467 : 0 : }
1468 [ - + - + : 42 : SPDK_INFOLOG(vhost, "%s: started poller on lcore %d\n",
# # # # #
# ]
1469 : : vsession->name, spdk_env_get_current_core());
1470 : :
1471 [ # # # # ]: 42 : svsession->requestq_poller = SPDK_POLLER_REGISTER(vdev_worker, svsession, 0);
1472 [ # # # # ]: 42 : svsession->mgmt_poller = SPDK_POLLER_REGISTER(vdev_mgmt_worker, svsession,
1473 : : MGMT_POLL_PERIOD_US);
1474 : 42 : return 0;
1475 : 0 : }
1476 : :
1477 : : static int
1478 : 42 : destroy_session_poller_cb(void *arg)
1479 : : {
1480 : 42 : struct spdk_vhost_scsi_session *svsession = arg;
1481 [ # # ]: 42 : struct spdk_vhost_session *vsession = &svsession->vsession;
1482 [ # # # # ]: 42 : struct spdk_vhost_user_dev *user_dev = to_user_dev(vsession->vdev);
1483 : : struct spdk_scsi_dev_session_state *state;
1484 : : uint32_t i;
1485 : :
1486 [ + - - + : 42 : if (vsession->task_cnt > 0 || (pthread_mutex_trylock(&user_dev->lock) != 0)) {
- + # # #
# # # ]
1487 [ # # # # : 0 : assert(vsession->stop_retry_count > 0);
# # # # ]
1488 [ # # ]: 0 : vsession->stop_retry_count--;
1489 [ # # # # : 0 : if (vsession->stop_retry_count == 0) {
# # ]
1490 [ # # # # : 0 : SPDK_ERRLOG("%s: Timedout when destroy session (task_cnt %d)\n", vsession->name,
# # # # ]
1491 : : vsession->task_cnt);
1492 [ # # ]: 0 : spdk_poller_unregister(&svsession->stop_poller);
1493 : 0 : vhost_user_session_stop_done(vsession, -ETIMEDOUT);
1494 : 0 : }
1495 : :
1496 : 0 : return SPDK_POLLER_BUSY;
1497 : : }
1498 : :
1499 [ + + # # : 245 : for (i = 0; i < vsession->max_queues; i++) {
# # ]
1500 [ # # # # : 203 : vhost_vq_used_signal(vsession, &vsession->virtqueue[i]);
# # ]
1501 : 0 : }
1502 : :
1503 [ + + ]: 378 : for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
1504 : : enum spdk_scsi_dev_vhost_status prev_status;
1505 : :
1506 [ # # # # : 336 : state = &svsession->scsi_dev_state[i];
# # ]
1507 : : /* clear the REMOVED status so that we won't send hotremove events anymore */
1508 [ # # # # ]: 336 : prev_status = state->status;
1509 [ # # # # ]: 336 : state->status = VHOST_SCSI_DEV_EMPTY;
1510 [ + + # # : 336 : if (state->dev == NULL) {
# # ]
1511 : 283 : continue;
1512 : : }
1513 : :
1514 [ # # # # ]: 53 : spdk_scsi_dev_free_io_channels(state->dev);
1515 : :
1516 [ # # # # ]: 53 : state->dev = NULL;
1517 : :
1518 [ - + ]: 53 : if (prev_status == VHOST_SCSI_DEV_REMOVING) {
1519 : : /* try to detach it globally */
1520 [ # # # # ]: 0 : pthread_mutex_unlock(&user_dev->lock);
1521 [ # # # # ]: 0 : vhost_user_dev_foreach_session(vsession->vdev,
1522 : : vhost_scsi_session_process_removed,
1523 : : vhost_scsi_dev_process_removed_cpl_cb,
1524 : 0 : (void *)(uintptr_t)i);
1525 [ # # # # ]: 0 : pthread_mutex_lock(&user_dev->lock);
1526 : 0 : }
1527 : 0 : }
1528 : :
1529 [ - + - + : 42 : SPDK_INFOLOG(vhost, "%s: stopping poller on lcore %d\n",
# # # # #
# ]
1530 : : vsession->name, spdk_env_get_current_core());
1531 : :
1532 : 42 : free_task_pool(svsession);
1533 : :
1534 [ # # ]: 42 : spdk_poller_unregister(&svsession->stop_poller);
1535 : 42 : vhost_user_session_stop_done(vsession, 0);
1536 : :
1537 [ - + # # ]: 42 : pthread_mutex_unlock(&user_dev->lock);
1538 : 42 : return SPDK_POLLER_BUSY;
1539 : 0 : }
1540 : :
1541 : : static int
1542 : 42 : vhost_scsi_stop(struct spdk_vhost_dev *vdev,
1543 : : struct spdk_vhost_session *vsession, void *unused)
1544 : : {
1545 : 42 : struct spdk_vhost_scsi_session *svsession = to_scsi_session(vsession);
1546 : :
1547 : : /* return if stop is already in progress */
1548 [ - + # # : 42 : if (svsession->stop_poller) {
# # ]
1549 : 0 : return -EINPROGRESS;
1550 : : }
1551 : :
1552 : : /* Stop receiving new I/O requests */
1553 [ # # ]: 42 : spdk_poller_unregister(&svsession->requestq_poller);
1554 : :
1555 : : /* Stop receiving controlq requests, also stop processing the
1556 : : * asynchronous hotremove events. All the remaining events
1557 : : * will be finalized by the stop_poller below.
1558 : : */
1559 [ # # ]: 42 : spdk_poller_unregister(&svsession->mgmt_poller);
1560 : :
1561 [ # # # # : 42 : svsession->vsession.stop_retry_count = (SPDK_VHOST_SESSION_STOP_RETRY_TIMEOUT_IN_SEC * 1000 *
# # ]
1562 : : 1000) / SPDK_VHOST_SESSION_STOP_RETRY_PERIOD_IN_US;
1563 : :
1564 : : /* Wait for all pending I/Os to complete, then process all the
1565 : : * remaining hotremove events one last time.
1566 : : */
1567 [ # # # # ]: 42 : svsession->stop_poller = SPDK_POLLER_REGISTER(destroy_session_poller_cb,
1568 : : svsession, SPDK_VHOST_SESSION_STOP_RETRY_PERIOD_IN_US);
1569 : :
1570 : 42 : return 0;
1571 : 0 : }
1572 : :
1573 : : static void
1574 : 403 : vhost_scsi_dump_info_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w)
1575 : : {
1576 : : struct spdk_scsi_dev *sdev;
1577 : : struct spdk_scsi_lun *lun;
1578 : : uint32_t dev_idx;
1579 : :
1580 [ - + # # ]: 403 : assert(vdev != NULL);
1581 : 403 : spdk_json_write_named_array_begin(w, "scsi");
1582 [ + + ]: 3627 : for (dev_idx = 0; dev_idx < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; dev_idx++) {
1583 : 3224 : sdev = spdk_vhost_scsi_dev_get_tgt(vdev, dev_idx);
1584 [ + + ]: 3224 : if (!sdev) {
1585 : 2624 : continue;
1586 : : }
1587 : :
1588 : 600 : spdk_json_write_object_begin(w);
1589 : :
1590 : 600 : spdk_json_write_named_uint32(w, "scsi_dev_num", dev_idx);
1591 : :
1592 : 600 : spdk_json_write_named_uint32(w, "id", spdk_scsi_dev_get_id(sdev));
1593 : :
1594 : 600 : spdk_json_write_named_string(w, "target_name", spdk_scsi_dev_get_name(sdev));
1595 : :
1596 : 600 : spdk_json_write_named_array_begin(w, "luns");
1597 : :
1598 [ + + ]: 1200 : for (lun = spdk_scsi_dev_get_first_lun(sdev); lun != NULL;
1599 : 600 : lun = spdk_scsi_dev_get_next_lun(lun)) {
1600 : 600 : spdk_json_write_object_begin(w);
1601 : :
1602 : 600 : spdk_json_write_named_int32(w, "id", spdk_scsi_lun_get_id(lun));
1603 : :
1604 : 600 : spdk_json_write_named_string(w, "bdev_name", spdk_scsi_lun_get_bdev_name(lun));
1605 : :
1606 : 600 : spdk_json_write_object_end(w);
1607 : 0 : }
1608 : :
1609 : 600 : spdk_json_write_array_end(w);
1610 : 600 : spdk_json_write_object_end(w);
1611 : 0 : }
1612 : :
1613 : 403 : spdk_json_write_array_end(w);
1614 : 403 : }
1615 : :
1616 : : static void
1617 : 16 : vhost_scsi_write_config_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w)
1618 : : {
1619 : : struct spdk_scsi_dev *scsi_dev;
1620 : : struct spdk_scsi_lun *lun;
1621 : : uint32_t i;
1622 : :
1623 : 16 : spdk_json_write_object_begin(w);
1624 : 16 : spdk_json_write_named_string(w, "method", "vhost_create_scsi_controller");
1625 : :
1626 : 16 : spdk_json_write_named_object_begin(w, "params");
1627 [ # # # # ]: 16 : spdk_json_write_named_string(w, "ctrlr", vdev->name);
1628 : 16 : spdk_json_write_named_string(w, "cpumask",
1629 [ # # # # ]: 0 : spdk_cpuset_fmt(spdk_thread_get_cpumask(vdev->thread)));
1630 : 16 : spdk_json_write_named_bool(w, "delay", true);
1631 : 16 : spdk_json_write_object_end(w);
1632 : :
1633 : 16 : spdk_json_write_object_end(w);
1634 : :
1635 [ + + ]: 144 : for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
1636 : 128 : scsi_dev = spdk_vhost_scsi_dev_get_tgt(vdev, i);
1637 [ + + ]: 128 : if (scsi_dev == NULL) {
1638 : 96 : continue;
1639 : : }
1640 : :
1641 : 32 : lun = spdk_scsi_dev_get_lun(scsi_dev, 0);
1642 [ - + # # ]: 32 : assert(lun != NULL);
1643 : :
1644 : 32 : spdk_json_write_object_begin(w);
1645 : 32 : spdk_json_write_named_string(w, "method", "vhost_scsi_controller_add_target");
1646 : :
1647 : 32 : spdk_json_write_named_object_begin(w, "params");
1648 [ # # # # ]: 32 : spdk_json_write_named_string(w, "ctrlr", vdev->name);
1649 : 32 : spdk_json_write_named_uint32(w, "scsi_target_num", i);
1650 : :
1651 : 32 : spdk_json_write_named_string(w, "bdev_name", spdk_scsi_lun_get_bdev_name(lun));
1652 : 32 : spdk_json_write_object_end(w);
1653 : :
1654 : 32 : spdk_json_write_object_end(w);
1655 : 0 : }
1656 : :
1657 : 16 : spdk_json_write_object_begin(w);
1658 : 16 : spdk_json_write_named_string(w, "method", "vhost_start_scsi_controller");
1659 : :
1660 : 16 : spdk_json_write_named_object_begin(w, "params");
1661 [ # # # # ]: 16 : spdk_json_write_named_string(w, "ctrlr", vdev->name);
1662 : 16 : spdk_json_write_object_end(w);
1663 : :
1664 : 16 : spdk_json_write_object_end(w);
1665 : 16 : }
1666 : :
1667 : 751 : SPDK_LOG_REGISTER_COMPONENT(vhost_scsi)
1668 : 751 : SPDK_LOG_REGISTER_COMPONENT(vhost_scsi_queue)
1669 : 751 : SPDK_LOG_REGISTER_COMPONENT(vhost_scsi_data)
|