Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright (C) 2017 Intel Corporation. All rights reserved.
3 : : * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
4 : : * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : : */
6 : :
7 : : #include "spdk/stdinc.h"
8 : :
9 : : #include "nvmf_internal.h"
10 : :
11 : : #include "spdk/bdev.h"
12 : : #include "spdk/endian.h"
13 : : #include "spdk/thread.h"
14 : : #include "spdk/likely.h"
15 : : #include "spdk/nvme.h"
16 : : #include "spdk/nvmf_cmd.h"
17 : : #include "spdk/nvmf_spec.h"
18 : : #include "spdk/trace.h"
19 : : #include "spdk/scsi_spec.h"
20 : : #include "spdk/string.h"
21 : : #include "spdk/util.h"
22 : :
23 : : #include "spdk/log.h"
24 : :
25 : : static bool
26 : 7066 : nvmf_subsystem_bdev_io_type_supported(struct spdk_nvmf_subsystem *subsystem,
27 : : enum spdk_bdev_io_type io_type)
28 : : {
29 : : struct spdk_nvmf_ns *ns;
30 : :
31 [ + + ]: 11821 : for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL;
32 : 4755 : ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) {
33 [ + + + - : 5109 : if (ns->bdev == NULL) {
+ - ]
34 : 0 : continue;
35 : : }
36 : :
37 [ + + + - : 5109 : if (!spdk_bdev_io_type_supported(ns->bdev, io_type)) {
+ - ]
38 [ - + - + : 354 : SPDK_DEBUGLOG(nvmf,
# # # # #
# # # # #
# # ]
39 : : "Subsystem %s namespace %u (%s) does not support io_type %d\n",
40 : : spdk_nvmf_subsystem_get_nqn(subsystem),
41 : : ns->opts.nsid, spdk_bdev_get_name(ns->bdev), (int)io_type);
42 : 354 : return false;
43 : : }
44 : 1558 : }
45 : :
46 [ + + + + : 6712 : SPDK_DEBUGLOG(nvmf, "All devices in Subsystem %s support io_type %d\n",
+ - ]
47 : : spdk_nvmf_subsystem_get_nqn(subsystem), (int)io_type);
48 : 6712 : return true;
49 : 1558 : }
50 : :
51 : : bool
52 : 3533 : nvmf_ctrlr_dsm_supported(struct spdk_nvmf_ctrlr *ctrlr)
53 : : {
54 [ + - + - ]: 3533 : return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_UNMAP);
55 : : }
56 : :
57 : : bool
58 : 3533 : nvmf_ctrlr_write_zeroes_supported(struct spdk_nvmf_ctrlr *ctrlr)
59 : : {
60 [ + - + - ]: 3533 : return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_WRITE_ZEROES);
61 : : }
62 : :
63 : : bool
64 : 0 : nvmf_ctrlr_copy_supported(struct spdk_nvmf_ctrlr *ctrlr)
65 : : {
66 [ # # # # ]: 0 : return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_COPY);
67 : : }
68 : :
69 : : static void
70 : 26721110 : nvmf_bdev_ctrlr_complete_cmd(struct spdk_bdev_io *bdev_io, bool success,
71 : : void *cb_arg)
72 : : {
73 : 26721110 : struct spdk_nvmf_request *req = cb_arg;
74 [ + - + - : 26721110 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
+ - ]
75 : 26721110 : int sc = 0, sct = 0;
76 : 26721110 : uint32_t cdw0 = 0;
77 : :
78 [ + + + - : 26721110 : if (spdk_unlikely(req->first_fused)) {
+ - - + ]
79 [ # # # # ]: 3117 : struct spdk_nvmf_request *first_req = req->first_fused_req;
80 [ # # # # : 3117 : struct spdk_nvme_cpl *first_response = &first_req->rsp->nvme_cpl;
# # ]
81 : 3117 : int first_sc = 0, first_sct = 0;
82 : :
83 : : /* get status for both operations */
84 : 3117 : spdk_bdev_io_get_nvme_fused_status(bdev_io, &cdw0, &first_sct, &first_sc, &sct, &sc);
85 [ # # # # ]: 3117 : first_response->cdw0 = cdw0;
86 [ # # # # : 3117 : first_response->status.sc = first_sc;
# # ]
87 [ # # # # : 3117 : first_response->status.sct = first_sct;
# # ]
88 : :
89 : : /* first request should be completed */
90 : 3117 : spdk_nvmf_request_complete(first_req);
91 [ # # # # ]: 3117 : req->first_fused_req = NULL;
92 [ # # # # : 3117 : req->first_fused = false;
# # ]
93 : 0 : } else {
94 : 26717993 : spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc);
95 : : }
96 : :
97 [ + - + - ]: 26721110 : response->cdw0 = cdw0;
98 [ + - + - : 26721110 : response->status.sc = sc;
+ - ]
99 [ + - + - : 26721110 : response->status.sct = sct;
+ - ]
100 : :
101 : 26721110 : spdk_nvmf_request_complete(req);
102 : 26721110 : spdk_bdev_free_io(bdev_io);
103 : 26721110 : }
104 : :
105 : : static void
106 : 54 : nvmf_bdev_ctrlr_complete_admin_cmd(struct spdk_bdev_io *bdev_io, bool success,
107 : : void *cb_arg)
108 : : {
109 : 54 : struct spdk_nvmf_request *req = cb_arg;
110 : :
111 [ + + # # : 54 : if (req->cmd_cb_fn) {
# # ]
112 [ # # # # : 48 : req->cmd_cb_fn(req);
# # # # ]
113 : 0 : }
114 : :
115 [ # # ]: 54 : nvmf_bdev_ctrlr_complete_cmd(bdev_io, success, req);
116 : 54 : }
117 : :
118 : : void
119 : 3818 : nvmf_bdev_ctrlr_identify_ns(struct spdk_nvmf_ns *ns, struct spdk_nvme_ns_data *nsdata,
120 : : bool dif_insert_or_strip)
121 : : {
122 [ + - + - ]: 3818 : struct spdk_bdev *bdev = ns->bdev;
123 : : uint64_t num_blocks;
124 : : uint32_t phys_blocklen;
125 : : uint32_t max_copy;
126 : :
127 : 3818 : num_blocks = spdk_bdev_get_num_blocks(bdev);
128 : :
129 [ + - + - ]: 3818 : nsdata->nsze = num_blocks;
130 [ + - + - ]: 3818 : nsdata->ncap = num_blocks;
131 [ + - + - ]: 3818 : nsdata->nuse = num_blocks;
132 [ + - + - ]: 3818 : nsdata->nlbaf = 0;
133 [ + - + - ]: 3818 : nsdata->flbas.format = 0;
134 [ + - + - ]: 3818 : nsdata->flbas.msb_format = 0;
135 [ + - + - : 3818 : nsdata->nacwu = spdk_bdev_get_acwu(bdev) - 1; /* nacwu is 0-based */
+ - ]
136 [ + + - + ]: 3818 : if (!dif_insert_or_strip) {
137 [ + - + - : 3785 : nsdata->lbaf[0].ms = spdk_bdev_get_md_size(bdev);
+ - + - ]
138 [ + - + - : 3785 : nsdata->lbaf[0].lbads = spdk_u32log2(spdk_bdev_get_block_size(bdev));
+ - + - ]
139 [ + + + - : 3785 : if (nsdata->lbaf[0].ms != 0) {
+ - + - +
- ]
140 [ # # # # ]: 3 : nsdata->flbas.extended = 1;
141 [ # # # # ]: 3 : nsdata->mc.extended = 1;
142 [ # # # # ]: 3 : nsdata->mc.pointer = 0;
143 [ # # # # ]: 3 : nsdata->dps.md_start = spdk_bdev_is_dif_head_of_md(bdev);
144 : : /* NVMf library doesn't process PRACT and PRCHK flags, we
145 : : * leave the use of extended LBA buffer to users.
146 : : */
147 [ # # # # ]: 3 : nsdata->dps.pit = SPDK_NVME_FMT_NVM_PROTECTION_DISABLE;
148 : 0 : }
149 : 779 : } else {
150 [ # # # # : 33 : nsdata->lbaf[0].ms = 0;
# # # # ]
151 [ # # # # : 33 : nsdata->lbaf[0].lbads = spdk_u32log2(spdk_bdev_get_data_block_size(bdev));
# # # # ]
152 : : }
153 : :
154 : 3818 : phys_blocklen = spdk_bdev_get_physical_block_size(bdev);
155 [ + + # # ]: 3818 : assert(phys_blocklen > 0);
156 : : /* Linux driver uses min(nawupf, npwg) to set physical_block_size */
157 [ + - + - ]: 3818 : nsdata->nsfeat.optperf = 1;
158 [ + - + - ]: 3818 : nsdata->nsfeat.ns_atomic_write_unit = 1;
159 [ + + + - : 3818 : nsdata->npwg = (phys_blocklen >> nsdata->lbaf[0].lbads) - 1;
+ - + - +
- + - +
- ]
160 [ + - + - : 3818 : nsdata->nawupf = nsdata->npwg;
+ - + - ]
161 [ + - + - : 3818 : nsdata->npwa = nsdata->npwg;
+ - + - ]
162 [ + - + - : 3818 : nsdata->npdg = nsdata->npwg;
+ - + - ]
163 [ + - + - : 3818 : nsdata->npda = nsdata->npwg;
+ - + - ]
164 : :
165 [ + + ]: 3818 : if (spdk_bdev_get_write_unit_size(bdev) == 1) {
166 [ + - + - ]: 3818 : nsdata->noiob = spdk_bdev_get_optimal_io_boundary(bdev);
167 : 779 : }
168 [ + - + - ]: 3818 : nsdata->nmic.can_share = 1;
169 [ + + ]: 3818 : if (nvmf_ns_is_ptpl_capable(ns)) {
170 [ # # # # : 6 : nsdata->nsrescap.rescap.persist = 1;
# # ]
171 : 0 : }
172 [ + - + - : 3818 : nsdata->nsrescap.rescap.write_exclusive = 1;
+ - ]
173 [ + - + - : 3818 : nsdata->nsrescap.rescap.exclusive_access = 1;
+ - ]
174 [ + - + - : 3818 : nsdata->nsrescap.rescap.write_exclusive_reg_only = 1;
+ - ]
175 [ + - + - : 3818 : nsdata->nsrescap.rescap.exclusive_access_reg_only = 1;
+ - ]
176 [ + - + - : 3818 : nsdata->nsrescap.rescap.write_exclusive_all_reg = 1;
+ - ]
177 [ + - + - : 3818 : nsdata->nsrescap.rescap.exclusive_access_all_reg = 1;
+ - ]
178 [ + - + - : 3818 : nsdata->nsrescap.rescap.ignore_existing_key = 1;
+ - ]
179 : :
180 : : SPDK_STATIC_ASSERT(sizeof(nsdata->nguid) == sizeof(ns->opts.nguid), "size mismatch");
181 [ + - + - : 3818 : memcpy(nsdata->nguid, ns->opts.nguid, sizeof(nsdata->nguid));
+ - + - +
- ]
182 : :
183 : : SPDK_STATIC_ASSERT(sizeof(nsdata->eui64) == sizeof(ns->opts.eui64), "size mismatch");
184 [ + - + - : 3818 : memcpy(&nsdata->eui64, ns->opts.eui64, sizeof(nsdata->eui64));
+ - + - +
- ]
185 : :
186 : : /* For now we support just one source range for copy command */
187 [ + - + - ]: 3818 : nsdata->msrc = 0;
188 : :
189 : 3818 : max_copy = spdk_bdev_get_max_copy(bdev);
190 [ + + - + ]: 3818 : if (max_copy == 0 || max_copy > UINT16_MAX) {
191 : : /* Zero means copy size is unlimited */
192 [ + - + - ]: 3151 : nsdata->mcl = UINT16_MAX;
193 [ + - + - ]: 3151 : nsdata->mssrl = UINT16_MAX;
194 : 779 : } else {
195 [ # # # # ]: 667 : nsdata->mcl = max_copy;
196 [ # # # # ]: 667 : nsdata->mssrl = max_copy;
197 : : }
198 : 3818 : }
199 : :
200 : : void
201 : 0 : nvmf_bdev_ctrlr_identify_iocs_nvm(struct spdk_nvmf_ns *ns,
202 : : struct spdk_nvme_nvm_ns_data *nsdata_nvm)
203 : : {
204 [ # # # # ]: 0 : struct spdk_bdev *bdev = ns->bdev;
205 : : uint8_t _16bpists;
206 : : uint32_t sts, pif;
207 : :
208 [ # # ]: 0 : if (spdk_bdev_get_dif_type(bdev) == SPDK_DIF_DISABLE) {
209 : 0 : return;
210 : : }
211 : :
212 : 0 : pif = spdk_bdev_get_dif_pi_format(bdev);
213 : :
214 : : /*
215 : : * 16BPISTS shall be 1 for 32/64b Guard PI.
216 : : * STCRS shall be 1 if 16BPISTS is 1.
217 : : * 16 is the minimum value of STS for 32b Guard PI.
218 : : */
219 [ # # # # ]: 0 : switch (pif) {
220 : 0 : case SPDK_DIF_PI_FORMAT_16:
221 : 0 : _16bpists = 0;
222 : 0 : sts = 0;
223 : 0 : break;
224 : 0 : case SPDK_DIF_PI_FORMAT_32:
225 : 0 : _16bpists = 1;
226 : 0 : sts = 16;
227 : 0 : break;
228 : 0 : case SPDK_DIF_PI_FORMAT_64:
229 : 0 : _16bpists = 1;
230 : 0 : sts = 0;
231 : 0 : break;
232 : 0 : default:
233 : 0 : SPDK_WARNLOG("PI format %u is not supported\n", pif);
234 : 0 : return;
235 : : }
236 : :
237 : : /* For 16b Guard PI, Storage Tag is not available because we set STS to 0.
238 : : * In this case, we do not have to set 16BPISTM to 1. For simplicity,
239 : : * set 16BPISTM to 0 and set LBSTM to all zeroes.
240 : : *
241 : : * We will revisit here when we find any OS uses Storage Tag.
242 : : */
243 [ # # # # ]: 0 : nsdata_nvm->lbstm = 0;
244 [ # # # # ]: 0 : nsdata_nvm->pic._16bpistm = 0;
245 : :
246 [ # # # # ]: 0 : nsdata_nvm->pic._16bpists = _16bpists;
247 [ # # # # ]: 0 : nsdata_nvm->pic.stcrs = 0;
248 [ # # # # : 0 : nsdata_nvm->elbaf[0].sts = sts;
# # # # ]
249 [ # # # # : 0 : nsdata_nvm->elbaf[0].pif = pif;
# # # # ]
250 : 0 : }
251 : :
252 : : static void
253 : 28000005 : nvmf_bdev_ctrlr_get_rw_params(const struct spdk_nvme_cmd *cmd, uint64_t *start_lba,
254 : : uint64_t *num_blocks)
255 : : {
256 : : /* SLBA: CDW10 and CDW11 */
257 [ + - + - : 28000005 : *start_lba = from_le64(&cmd->cdw10);
+ - ]
258 : :
259 : : /* NLB: CDW12 bits 15:00, 0's based */
260 [ + - + - : 28000005 : *num_blocks = (from_le32(&cmd->cdw12) & 0xFFFFu) + 1;
+ - ]
261 : 28000005 : }
262 : :
263 : : static void
264 : 11839025 : nvmf_bdev_ctrlr_get_rw_ext_params(const struct spdk_nvme_cmd *cmd,
265 : : struct spdk_bdev_ext_io_opts *opts)
266 : : {
267 : : /* Get CDW12 values */
268 [ + - + - : 11839025 : opts->nvme_cdw12.raw = from_le32(&cmd->cdw12);
+ - + - +
- ]
269 : :
270 : : /* Get CDW13 values */
271 [ + - + - : 11839025 : opts->nvme_cdw13.raw = from_le32(&cmd->cdw13);
+ - + - +
- ]
272 : 11839025 : }
273 : :
274 : : static bool
275 : 27996903 : nvmf_bdev_ctrlr_lba_in_range(uint64_t bdev_num_blocks, uint64_t io_start_lba,
276 : : uint64_t io_num_blocks)
277 : : {
278 [ + + - + ]: 27996903 : if (io_start_lba + io_num_blocks > bdev_num_blocks ||
279 [ + + ]: 27963219 : io_start_lba + io_num_blocks < io_start_lba) {
280 : 33690 : return false;
281 : : }
282 : :
283 : 27963213 : return true;
284 : 145044 : }
285 : :
286 : : static void
287 : 114266 : nvmf_ctrlr_process_io_cmd_resubmit(void *arg)
288 : : {
289 : 114266 : struct spdk_nvmf_request *req = arg;
290 : : int rc;
291 : :
292 : 114266 : rc = nvmf_ctrlr_process_io_cmd(req);
293 [ - + ]: 114266 : if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) {
294 : 0 : spdk_nvmf_request_complete(req);
295 : 0 : }
296 : 114266 : }
297 : :
298 : : static void
299 : 0 : nvmf_ctrlr_process_admin_cmd_resubmit(void *arg)
300 : : {
301 : 0 : struct spdk_nvmf_request *req = arg;
302 : : int rc;
303 : :
304 : 0 : rc = nvmf_ctrlr_process_admin_cmd(req);
305 [ # # ]: 0 : if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) {
306 : 0 : spdk_nvmf_request_complete(req);
307 : 0 : }
308 : 0 : }
309 : :
310 : : static void
311 : 183006 : nvmf_bdev_ctrl_queue_io(struct spdk_nvmf_request *req, struct spdk_bdev *bdev,
312 : : struct spdk_io_channel *ch, spdk_bdev_io_wait_cb cb_fn, void *cb_arg)
313 : : {
314 : : int rc;
315 : :
316 [ # # # # : 183006 : req->bdev_io_wait.bdev = bdev;
# # ]
317 [ # # # # : 183006 : req->bdev_io_wait.cb_fn = cb_fn;
# # ]
318 [ # # # # : 183006 : req->bdev_io_wait.cb_arg = cb_arg;
# # ]
319 : :
320 [ # # ]: 183006 : rc = spdk_bdev_queue_io_wait(bdev, ch, &req->bdev_io_wait);
321 [ - + ]: 183006 : if (rc != 0) {
322 [ # # ]: 0 : assert(false);
323 : : }
324 [ # # # # : 183006 : req->qpair->group->stat.pending_bdev_io++;
# # # # #
# # # ]
325 : 183006 : }
326 : :
327 : : bool
328 : 0 : nvmf_bdev_zcopy_enabled(struct spdk_bdev *bdev)
329 : : {
330 : 0 : return spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_ZCOPY);
331 : : }
332 : :
333 : : int
334 : 14992702 : nvmf_bdev_ctrlr_read_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
335 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
336 : : {
337 : 59427211 : struct spdk_bdev_ext_io_opts opts = {
338 : : .size = SPDK_SIZEOF(&opts, accel_sequence),
339 [ + - + - ]: 14992702 : .memory_domain = req->memory_domain,
340 [ + - + - ]: 14992702 : .memory_domain_ctx = req->memory_domain_ctx,
341 [ + - + - ]: 14992702 : .accel_sequence = req->accel_sequence,
342 : : };
343 : 14992702 : uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev);
344 : 14992702 : uint32_t block_size = spdk_bdev_get_block_size(bdev);
345 [ + - + - : 14992702 : struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
+ - ]
346 [ + - + - : 14992702 : struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
+ - ]
347 : 181199 : uint64_t start_lba;
348 : 181199 : uint64_t num_blocks;
349 : : int rc;
350 : :
351 : 14992702 : nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks);
352 : :
353 [ + + ]: 14992702 : if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) {
354 : 7395 : SPDK_ERRLOG("end of media\n");
355 [ # # # # : 7395 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
356 [ # # # # : 7395 : rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE;
# # ]
357 : 7395 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
358 : : }
359 : :
360 [ + + + - : 14985307 : if (spdk_unlikely(num_blocks * block_size > req->length)) {
- + ]
361 [ # # # # ]: 0 : SPDK_ERRLOG("Read NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n",
362 : : num_blocks, block_size, req->length);
363 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
364 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
# # ]
365 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
366 : : }
367 : :
368 [ + + # # ]: 14985307 : assert(!spdk_nvmf_request_using_zcopy(req));
369 : :
370 [ + - + - : 14985307 : rc = spdk_bdev_readv_blocks_ext(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks,
+ - ]
371 : 72522 : nvmf_bdev_ctrlr_complete_cmd, req, &opts);
372 [ + + ]: 14985307 : if (spdk_unlikely(rc)) {
373 [ + - ]: 54406 : if (rc == -ENOMEM) {
374 : 54406 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
375 : 54406 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
376 : : }
377 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
378 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
379 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
380 : : }
381 : :
382 : 14930901 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
383 : 72522 : }
384 : :
385 : : int
386 : 11839022 : nvmf_bdev_ctrlr_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
387 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
388 : : {
389 : 46886678 : struct spdk_bdev_ext_io_opts opts = {
390 : : .size = SPDK_SIZEOF(&opts, nvme_cdw13),
391 [ + - + - ]: 11839022 : .memory_domain = req->memory_domain,
392 [ + - + - ]: 11839022 : .memory_domain_ctx = req->memory_domain_ctx,
393 [ + - + - ]: 11839022 : .accel_sequence = req->accel_sequence,
394 : : };
395 : 11839022 : uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev);
396 : 11839022 : uint32_t block_size = spdk_bdev_get_block_size(bdev);
397 [ + - + - : 11839022 : struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
+ - ]
398 [ + - + - : 11839022 : struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
+ - ]
399 : 156470 : uint64_t start_lba;
400 : 156470 : uint64_t num_blocks;
401 : : int rc;
402 : :
403 : 11839022 : nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks);
404 : 11839022 : nvmf_bdev_ctrlr_get_rw_ext_params(cmd, &opts);
405 : :
406 [ + + ]: 11839022 : if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) {
407 : 7487 : SPDK_ERRLOG("end of media\n");
408 [ # # # # : 7487 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
409 [ # # # # : 7487 : rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE;
# # ]
410 : 7487 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
411 : : }
412 : :
413 [ + + + - : 11831535 : if (spdk_unlikely(num_blocks * block_size > req->length)) {
- + ]
414 [ # # # # ]: 0 : SPDK_ERRLOG("Write NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n",
415 : : num_blocks, block_size, req->length);
416 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
417 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
# # ]
418 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
419 : : }
420 : :
421 [ + + # # ]: 11831535 : assert(!spdk_nvmf_request_using_zcopy(req));
422 : :
423 [ + - + - : 11831535 : rc = spdk_bdev_writev_blocks_ext(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks,
+ - ]
424 : 72522 : nvmf_bdev_ctrlr_complete_cmd, req, &opts);
425 [ + + ]: 11831535 : if (spdk_unlikely(rc)) {
426 [ + - ]: 59860 : if (rc == -ENOMEM) {
427 : 59860 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
428 : 59860 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
429 : : }
430 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
431 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
432 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
433 : : }
434 : :
435 : 11771675 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
436 : 72522 : }
437 : :
438 : : int
439 : 7345 : nvmf_bdev_ctrlr_compare_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
440 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
441 : : {
442 : 7345 : uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev);
443 : 7345 : uint32_t block_size = spdk_bdev_get_block_size(bdev);
444 [ # # # # : 7345 : struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
# # ]
445 [ # # # # : 7345 : struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
# # ]
446 : 12 : uint64_t start_lba;
447 : 12 : uint64_t num_blocks;
448 : : int rc;
449 : :
450 : 7345 : nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks);
451 : :
452 [ + + ]: 7345 : if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) {
453 : 7336 : SPDK_ERRLOG("end of media\n");
454 [ # # # # : 7336 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
455 [ # # # # : 7336 : rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE;
# # ]
456 : 7336 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
457 : : }
458 : :
459 [ + + # # : 9 : if (spdk_unlikely(num_blocks * block_size > req->length)) {
# # ]
460 [ # # # # ]: 3 : SPDK_ERRLOG("Compare NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n",
461 : : num_blocks, block_size, req->length);
462 [ # # # # : 3 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
463 [ # # # # : 3 : rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
# # ]
464 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
465 : : }
466 : :
467 [ # # # # : 6 : rc = spdk_bdev_comparev_blocks(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks,
# # ]
468 : 0 : nvmf_bdev_ctrlr_complete_cmd, req);
469 [ + + ]: 6 : if (spdk_unlikely(rc)) {
470 [ - + ]: 3 : if (rc == -ENOMEM) {
471 : 0 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
472 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
473 : : }
474 [ # # # # : 3 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
475 [ # # # # : 3 : rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
476 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
477 : : }
478 : :
479 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
480 : 0 : }
481 : :
482 : : int
483 : 3129 : nvmf_bdev_ctrlr_compare_and_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
484 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *cmp_req, struct spdk_nvmf_request *write_req)
485 : : {
486 : 3129 : uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev);
487 : 3129 : uint32_t block_size = spdk_bdev_get_block_size(bdev);
488 [ # # # # : 3129 : struct spdk_nvme_cmd *cmp_cmd = &cmp_req->cmd->nvme_cmd;
# # ]
489 [ # # # # : 3129 : struct spdk_nvme_cmd *write_cmd = &write_req->cmd->nvme_cmd;
# # ]
490 [ # # # # : 3129 : struct spdk_nvme_cpl *rsp = &write_req->rsp->nvme_cpl;
# # ]
491 : 12 : uint64_t write_start_lba, cmp_start_lba;
492 : 12 : uint64_t write_num_blocks, cmp_num_blocks;
493 : : int rc;
494 : :
495 : 3129 : nvmf_bdev_ctrlr_get_rw_params(cmp_cmd, &cmp_start_lba, &cmp_num_blocks);
496 : 3129 : nvmf_bdev_ctrlr_get_rw_params(write_cmd, &write_start_lba, &write_num_blocks);
497 : :
498 [ + - + + ]: 3129 : if (spdk_unlikely(write_start_lba != cmp_start_lba || write_num_blocks != cmp_num_blocks)) {
499 : 3 : SPDK_ERRLOG("Fused command start lba / num blocks mismatch\n");
500 [ # # # # : 3 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
501 [ # # # # : 3 : rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD;
# # ]
502 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
503 : : }
504 : :
505 [ + + ]: 3126 : if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, write_start_lba,
506 : : write_num_blocks))) {
507 : 3 : SPDK_ERRLOG("end of media\n");
508 [ # # # # : 3 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
509 [ # # # # : 3 : rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE;
# # ]
510 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
511 : : }
512 : :
513 [ + + # # : 3123 : if (spdk_unlikely(write_num_blocks * block_size > write_req->length)) {
# # ]
514 [ # # # # ]: 3 : SPDK_ERRLOG("Write NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n",
515 : : write_num_blocks, block_size, write_req->length);
516 [ # # # # : 3 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
517 [ # # # # : 3 : rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
# # ]
518 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
519 : : }
520 : :
521 [ # # # # : 3120 : rc = spdk_bdev_comparev_and_writev_blocks(desc, ch, cmp_req->iov, cmp_req->iovcnt, write_req->iov,
# # # # ]
522 [ # # # # ]: 3120 : write_req->iovcnt, write_start_lba, write_num_blocks, nvmf_bdev_ctrlr_complete_cmd, write_req);
523 [ - + ]: 3120 : if (spdk_unlikely(rc)) {
524 [ # # ]: 0 : if (rc == -ENOMEM) {
525 : 0 : nvmf_bdev_ctrl_queue_io(cmp_req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, cmp_req);
526 : 0 : nvmf_bdev_ctrl_queue_io(write_req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, write_req);
527 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
528 : : }
529 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
530 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
531 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
532 : : }
533 : :
534 : 3120 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
535 : 0 : }
536 : :
537 : : int
538 : 14969 : nvmf_bdev_ctrlr_write_zeroes_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
539 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
540 : : {
541 : 14969 : uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev);
542 [ # # # # : 14969 : struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
# # ]
543 [ # # # # : 14969 : struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
# # ]
544 [ # # # # : 14969 : uint64_t max_write_zeroes_size = req->qpair->ctrlr->subsys->max_write_zeroes_size_kib;
# # # # #
# # # # #
# # ]
545 : 12 : uint64_t start_lba;
546 : 12 : uint64_t num_blocks;
547 : : int rc;
548 : :
549 : 14969 : nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks);
550 [ + + + + : 14969 : if (spdk_unlikely(max_write_zeroes_size > 0 &&
+ - # # ]
551 : : num_blocks > (max_write_zeroes_size << 10) / spdk_bdev_get_block_size(bdev))) {
552 : 3 : SPDK_ERRLOG("invalid write zeroes size, should not exceed %" PRIu64 "Kib\n", max_write_zeroes_size);
553 [ # # # # : 3 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
554 [ # # # # : 3 : rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD;
# # ]
555 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
556 : : }
557 : :
558 [ + + ]: 14966 : if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) {
559 : 11448 : SPDK_ERRLOG("end of media\n");
560 [ # # # # : 11448 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
561 [ # # # # : 11448 : rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE;
# # ]
562 : 11448 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
563 : : }
564 : :
565 [ - + # # : 3518 : if (spdk_unlikely(cmd->cdw12_bits.write_zeroes.deac)) {
# # # # #
# ]
566 : 0 : SPDK_ERRLOG("Write Zeroes Deallocate is not supported\n");
567 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
568 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD;
# # ]
569 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
570 : : }
571 : :
572 : 3518 : rc = spdk_bdev_write_zeroes_blocks(desc, ch, start_lba, num_blocks,
573 : 0 : nvmf_bdev_ctrlr_complete_cmd, req);
574 [ + + ]: 3518 : if (spdk_unlikely(rc)) {
575 [ - + ]: 3 : if (rc == -ENOMEM) {
576 : 0 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
577 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
578 : : }
579 [ # # # # : 3 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
580 [ # # # # : 3 : rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
581 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
582 : : }
583 : :
584 : 3515 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
585 : 0 : }
586 : :
587 : : int
588 : 11851 : nvmf_bdev_ctrlr_flush_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
589 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
590 : : {
591 [ # # # # : 11851 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
# # ]
592 : : int rc;
593 : :
594 : : /* As for NVMeoF controller, SPDK always set volatile write
595 : : * cache bit to 1, return success for those block devices
596 : : * which can't support FLUSH command.
597 : : */
598 [ + + ]: 11851 : if (!spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_FLUSH)) {
599 [ # # # # : 3 : response->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
600 [ # # # # : 3 : response->status.sc = SPDK_NVME_SC_SUCCESS;
# # ]
601 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
602 : : }
603 : :
604 : 11848 : rc = spdk_bdev_flush_blocks(desc, ch, 0, spdk_bdev_get_num_blocks(bdev),
605 : 0 : nvmf_bdev_ctrlr_complete_cmd, req);
606 [ + + ]: 11848 : if (spdk_unlikely(rc)) {
607 [ - + ]: 3 : if (rc == -ENOMEM) {
608 : 0 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
609 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
610 : : }
611 [ # # # # : 3 : response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
612 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
613 : : }
614 : 11845 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
615 : 0 : }
616 : :
617 : : struct nvmf_bdev_ctrlr_unmap {
618 : : struct spdk_nvmf_request *req;
619 : : uint32_t count;
620 : : struct spdk_bdev_desc *desc;
621 : : struct spdk_bdev *bdev;
622 : : struct spdk_io_channel *ch;
623 : : uint32_t range_index;
624 : : };
625 : :
626 : : static void
627 : 68768 : nvmf_bdev_ctrlr_unmap_cpl(struct spdk_bdev_io *bdev_io, bool success,
628 : : void *cb_arg)
629 : : {
630 : 68768 : struct nvmf_bdev_ctrlr_unmap *unmap_ctx = cb_arg;
631 [ # # # # ]: 68768 : struct spdk_nvmf_request *req = unmap_ctx->req;
632 [ # # # # : 68768 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
# # ]
633 : 0 : int sc, sct;
634 : 0 : uint32_t cdw0;
635 : :
636 [ # # ]: 68768 : unmap_ctx->count--;
637 : :
638 [ + - # # : 68768 : if (response->status.sct == SPDK_NVME_SCT_GENERIC &&
# # # # #
# ]
639 [ + - # # : 68768 : response->status.sc == SPDK_NVME_SC_SUCCESS) {
# # ]
640 : 68768 : spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc);
641 [ # # # # ]: 68768 : response->cdw0 = cdw0;
642 [ # # # # : 68768 : response->status.sc = sc;
# # ]
643 [ # # # # : 68768 : response->status.sct = sct;
# # ]
644 : 0 : }
645 : :
646 [ + - # # : 68768 : if (unmap_ctx->count == 0) {
# # ]
647 : 68768 : spdk_nvmf_request_complete(req);
648 : 68768 : free(unmap_ctx);
649 : 0 : }
650 : 68768 : spdk_bdev_free_io(bdev_io);
651 : 68768 : }
652 : :
653 : : static int nvmf_bdev_ctrlr_unmap(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
654 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req,
655 : : struct nvmf_bdev_ctrlr_unmap *unmap_ctx);
656 : : static void
657 : 68734 : nvmf_bdev_ctrlr_unmap_resubmit(void *arg)
658 : : {
659 : 68734 : struct nvmf_bdev_ctrlr_unmap *unmap_ctx = arg;
660 [ # # # # ]: 68734 : struct spdk_nvmf_request *req = unmap_ctx->req;
661 [ # # # # ]: 68734 : struct spdk_bdev_desc *desc = unmap_ctx->desc;
662 [ # # # # ]: 68734 : struct spdk_bdev *bdev = unmap_ctx->bdev;
663 [ # # # # ]: 68734 : struct spdk_io_channel *ch = unmap_ctx->ch;
664 : :
665 : 68734 : nvmf_bdev_ctrlr_unmap(bdev, desc, ch, req, unmap_ctx);
666 : 68734 : }
667 : :
668 : : static int
669 : 141228 : nvmf_bdev_ctrlr_unmap(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
670 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req,
671 : : struct nvmf_bdev_ctrlr_unmap *unmap_ctx)
672 : : {
673 : : uint16_t nr, i;
674 [ # # # # : 141228 : struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
# # ]
675 [ # # # # : 141228 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
# # ]
676 [ # # # # : 141228 : uint64_t max_discard_size = req->qpair->ctrlr->subsys->max_discard_size_kib;
# # # # #
# # # # #
# # ]
677 : 141228 : uint32_t block_size = spdk_bdev_get_block_size(bdev);
678 : 0 : struct spdk_iov_xfer ix;
679 : : uint64_t lba;
680 : : uint32_t lba_count;
681 : : int rc;
682 : :
683 [ # # # # : 141228 : nr = cmd->cdw10_bits.dsm.nr + 1;
# # # # #
# ]
684 [ + + # # : 141228 : if (nr * sizeof(struct spdk_nvme_dsm_range) > req->length) {
# # ]
685 : 3726 : SPDK_ERRLOG("Dataset Management number of ranges > SGL length\n");
686 [ # # # # : 3726 : response->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
# # ]
687 : 3726 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
688 : : }
689 : :
690 [ + + ]: 137502 : if (unmap_ctx == NULL) {
691 : 68768 : unmap_ctx = calloc(1, sizeof(*unmap_ctx));
692 [ - + ]: 68768 : if (!unmap_ctx) {
693 [ # # # # : 0 : response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
694 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
695 : : }
696 : :
697 [ # # # # ]: 68768 : unmap_ctx->req = req;
698 [ # # # # ]: 68768 : unmap_ctx->desc = desc;
699 [ # # # # ]: 68768 : unmap_ctx->ch = ch;
700 [ # # # # ]: 68768 : unmap_ctx->bdev = bdev;
701 : :
702 [ # # # # : 68768 : response->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
703 [ # # # # : 68768 : response->status.sc = SPDK_NVME_SC_SUCCESS;
# # ]
704 : 0 : } else {
705 [ # # ]: 68734 : unmap_ctx->count--; /* dequeued */
706 : : }
707 : :
708 [ # # # # : 137502 : spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);
# # ]
709 : :
710 [ + + # # : 206270 : for (i = unmap_ctx->range_index; i < nr; i++) {
# # ]
711 : 137502 : struct spdk_nvme_dsm_range dsm_range = { 0 };
712 : :
713 : 137502 : spdk_iov_xfer_to_buf(&ix, &dsm_range, sizeof(dsm_range));
714 : :
715 [ # # ]: 137502 : lba = dsm_range.starting_lba;
716 [ # # ]: 137502 : lba_count = dsm_range.length;
717 [ - + - - : 137502 : if (max_discard_size > 0 && lba_count > (max_discard_size << 10) / block_size) {
- - # # ]
718 : 0 : SPDK_ERRLOG("invalid unmap size, should not exceed %" PRIu64 "Kib\n", max_discard_size);
719 [ # # # # : 0 : response->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
720 [ # # # # : 0 : response->status.sc = SPDK_NVME_SC_INVALID_FIELD;
# # ]
721 : 68734 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
722 : : }
723 : :
724 [ # # ]: 137502 : unmap_ctx->count++;
725 : :
726 : 137502 : rc = spdk_bdev_unmap_blocks(desc, ch, lba, lba_count,
727 : 0 : nvmf_bdev_ctrlr_unmap_cpl, unmap_ctx);
728 [ + + ]: 137502 : if (rc) {
729 [ + - ]: 68734 : if (rc == -ENOMEM) {
730 : 68734 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_bdev_ctrlr_unmap_resubmit, unmap_ctx);
731 : : /* Unmap was not yet submitted to bdev */
732 : : /* unmap_ctx->count will be decremented when the request is dequeued */
733 : 68734 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
734 : : }
735 [ # # # # : 0 : response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
736 [ # # ]: 0 : unmap_ctx->count--;
737 : : /* We can't return here - we may have to wait for any other
738 : : * unmaps already sent to complete */
739 : 0 : break;
740 : : }
741 [ # # ]: 68768 : unmap_ctx->range_index++;
742 : 0 : }
743 : :
744 [ - + # # : 68768 : if (unmap_ctx->count == 0) {
# # ]
745 : 0 : free(unmap_ctx);
746 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
747 : : }
748 : :
749 : 68768 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
750 : 0 : }
751 : :
752 : : int
753 : 76155 : nvmf_bdev_ctrlr_dsm_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
754 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
755 : : {
756 [ # # # # : 76155 : struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
# # ]
757 [ # # # # : 76155 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
# # ]
758 : :
759 [ + + # # : 76155 : if (cmd->cdw11_bits.dsm.ad) {
# # # # #
# ]
760 : 72494 : return nvmf_bdev_ctrlr_unmap(bdev, desc, ch, req, NULL);
761 : : }
762 : :
763 [ # # # # : 3661 : response->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
764 [ # # # # : 3661 : response->status.sc = SPDK_NVME_SC_SUCCESS;
# # ]
765 : 3661 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
766 : 0 : }
767 : :
768 : : int
769 : 7368 : nvmf_bdev_ctrlr_copy_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
770 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
771 : : {
772 [ # # # # : 7368 : struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
# # ]
773 [ # # # # : 7368 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
# # ]
774 [ # # # # : 7368 : uint64_t sdlba = ((uint64_t)cmd->cdw11 << 32) + cmd->cdw10;
# # # # #
# # # #
# ]
775 : 7368 : struct spdk_nvme_scc_source_range range = { 0 };
776 : 15 : struct spdk_iov_xfer ix;
777 : : int rc;
778 : :
779 [ - + - + : 7368 : SPDK_DEBUGLOG(nvmf, "Copy command: SDLBA %lu, NR %u, desc format %u, PRINFOR %u, "
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
780 : : "DTYPE %u, STCW %u, PRINFOW %u, FUA %u, LR %u\n",
781 : : sdlba,
782 : : cmd->cdw12_bits.copy.nr,
783 : : cmd->cdw12_bits.copy.df,
784 : : cmd->cdw12_bits.copy.prinfor,
785 : : cmd->cdw12_bits.copy.dtype,
786 : : cmd->cdw12_bits.copy.stcw,
787 : : cmd->cdw12_bits.copy.prinfow,
788 : : cmd->cdw12_bits.copy.fua,
789 : : cmd->cdw12_bits.copy.lr);
790 : :
791 [ + + # # : 7368 : if (spdk_unlikely(req->length != (cmd->cdw12_bits.copy.nr + 1) *
# # # # #
# # # # #
# # ]
792 : : sizeof(struct spdk_nvme_scc_source_range))) {
793 [ # # # # : 7344 : response->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
794 [ # # # # : 7344 : response->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
# # ]
795 : 7344 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
796 : : }
797 : :
798 : : /*
799 : : * We support only one source range, and rely on this with the xfer
800 : : * below.
801 : : */
802 [ + + # # : 24 : if (cmd->cdw12_bits.copy.nr > 0) {
# # # # #
# ]
803 [ # # # # : 3 : response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;
# # ]
804 [ # # # # : 3 : response->status.sc = SPDK_NVME_SC_CMD_SIZE_LIMIT_SIZE_EXCEEDED;
# # ]
805 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
806 : : }
807 : :
808 [ + + # # : 21 : if (cmd->cdw12_bits.copy.df != 0) {
# # # # #
# ]
809 [ # # # # : 3 : response->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
810 [ # # # # : 3 : response->status.sc = SPDK_NVME_SC_INVALID_FIELD;
# # ]
811 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
812 : : }
813 : :
814 [ # # # # : 18 : spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);
# # ]
815 : 18 : spdk_iov_xfer_to_buf(&ix, &range, sizeof(range));
816 : :
817 [ # # # # : 18 : rc = spdk_bdev_copy_blocks(desc, ch, sdlba, range.slba, range.nlb + 1,
# # ]
818 : 0 : nvmf_bdev_ctrlr_complete_cmd, req);
819 [ + + ]: 18 : if (spdk_unlikely(rc)) {
820 [ - + ]: 3 : if (rc == -ENOMEM) {
821 : 0 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
822 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
823 : : }
824 : :
825 [ # # # # : 3 : response->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
826 [ # # # # : 3 : response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
827 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
828 : : }
829 : :
830 : 15 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
831 : 0 : }
832 : :
833 : : int
834 : 2050762 : nvmf_bdev_ctrlr_nvme_passthru_io(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
835 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req)
836 : : {
837 : : int rc;
838 : :
839 [ # # # # : 2050762 : rc = spdk_bdev_nvme_iov_passthru_md(desc, ch, &req->cmd->nvme_cmd, req->iov, req->iovcnt,
# # # # #
# # # ]
840 [ # # # # ]: 2050762 : req->length, NULL, 0, nvmf_bdev_ctrlr_complete_cmd, req);
841 : :
842 [ + + ]: 2050762 : if (spdk_unlikely(rc)) {
843 [ + + ]: 2050756 : if (rc == -ENOMEM) {
844 : 3 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
845 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
846 : : }
847 [ # # # # : 2050753 : req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
# # # # #
# # # ]
848 [ # # # # : 2050753 : req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE;
# # # # #
# # # ]
849 [ # # # # : 2050753 : req->rsp->nvme_cpl.status.dnr = 1;
# # # # #
# # # ]
850 : 2050753 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
851 : : }
852 : :
853 : 6 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
854 : 0 : }
855 : :
856 : : int
857 : 60 : spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
858 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req,
859 : : spdk_nvmf_nvme_passthru_cmd_cb cb_fn)
860 : : {
861 : : int rc;
862 : :
863 [ - + # # : 60 : if (spdk_unlikely(req->iovcnt > 1)) {
# # ]
864 [ # # # # : 0 : req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
# # # # #
# # # ]
865 [ # # # # : 0 : req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # # # #
# # # ]
866 [ # # # # : 0 : req->rsp->nvme_cpl.status.dnr = 1;
# # # # #
# # # ]
867 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
868 : : }
869 : :
870 [ # # # # ]: 60 : req->cmd_cb_fn = cb_fn;
871 : :
872 [ # # # # : 60 : rc = spdk_bdev_nvme_admin_passthru(desc, ch, &req->cmd->nvme_cmd, req->iov[0].iov_base, req->length,
# # # # #
# # # # #
# # # # #
# ]
873 : 0 : nvmf_bdev_ctrlr_complete_admin_cmd, req);
874 [ + + ]: 60 : if (spdk_unlikely(rc)) {
875 [ + + ]: 6 : if (rc == -ENOMEM) {
876 : 3 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_admin_cmd_resubmit, req);
877 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
878 : : }
879 [ # # # # : 3 : req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
# # # # #
# # # ]
880 [ + - ]: 3 : if (rc == -ENOTSUP) {
881 [ # # # # : 3 : req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE;
# # # # #
# # # ]
882 : 0 : } else {
883 [ # # # # : 0 : req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # # # #
# # # ]
884 : : }
885 : :
886 [ # # # # : 3 : req->rsp->nvme_cpl.status.dnr = 1;
# # # # #
# # # ]
887 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
888 : : }
889 : :
890 : 54 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
891 : 0 : }
892 : :
893 : : static void
894 : 73186 : nvmf_bdev_ctrlr_complete_abort_cmd(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
895 : : {
896 : 73186 : struct spdk_nvmf_request *req = cb_arg;
897 : :
898 [ + + # # ]: 73186 : if (success) {
899 [ # # # # : 70850 : req->rsp->nvme_cpl.cdw0 &= ~1U;
# # # # #
# ]
900 : 0 : }
901 : :
902 : 73186 : spdk_nvmf_request_complete(req);
903 : 73186 : spdk_bdev_free_io(bdev_io);
904 : 73186 : }
905 : :
906 : : int
907 : 73186 : spdk_nvmf_bdev_ctrlr_abort_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
908 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req,
909 : : struct spdk_nvmf_request *req_to_abort)
910 : : {
911 : : int rc;
912 : :
913 [ - + # # : 73186 : assert((req->rsp->nvme_cpl.cdw0 & 1U) != 0);
# # # # #
# # # #
# ]
914 : :
915 : 73186 : rc = spdk_bdev_abort(desc, ch, req_to_abort, nvmf_bdev_ctrlr_complete_abort_cmd, req);
916 [ + - ]: 73186 : if (spdk_likely(rc == 0)) {
917 : 73186 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
918 [ # # ]: 0 : } else if (rc == -ENOMEM) {
919 : 0 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_admin_cmd_resubmit, req);
920 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
921 : : } else {
922 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
923 : : }
924 : 0 : }
925 : :
926 : : bool
927 : 561538 : nvmf_bdev_ctrlr_get_dif_ctx(struct spdk_bdev *bdev, struct spdk_nvme_cmd *cmd,
928 : : struct spdk_dif_ctx *dif_ctx)
929 : : {
930 : 561538 : uint32_t init_ref_tag, dif_check_flags = 0;
931 : : int rc;
932 : 6 : struct spdk_dif_ctx_init_ext_opts dif_opts;
933 : :
934 [ + + ]: 561538 : if (spdk_bdev_get_md_size(bdev) == 0) {
935 : 3 : return false;
936 : : }
937 : :
938 : : /* Initial Reference Tag is the lower 32 bits of the start LBA. */
939 [ # # # # ]: 561535 : init_ref_tag = (uint32_t)from_le64(&cmd->cdw10);
940 : :
941 [ + + ]: 561535 : if (spdk_bdev_is_dif_check_enabled(bdev, SPDK_DIF_CHECK_TYPE_REFTAG)) {
942 [ # # ]: 524768 : dif_check_flags |= SPDK_DIF_FLAGS_REFTAG_CHECK;
943 : 0 : }
944 : :
945 [ + + ]: 561535 : if (spdk_bdev_is_dif_check_enabled(bdev, SPDK_DIF_CHECK_TYPE_GUARD)) {
946 [ # # ]: 561532 : dif_check_flags |= SPDK_DIF_FLAGS_GUARD_CHECK;
947 : 0 : }
948 : :
949 : 561535 : dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
950 [ # # ]: 561535 : dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;
951 : 1123070 : rc = spdk_dif_ctx_init(dif_ctx,
952 : 0 : spdk_bdev_get_block_size(bdev),
953 : 0 : spdk_bdev_get_md_size(bdev),
954 : 561535 : spdk_bdev_is_md_interleaved(bdev),
955 : 561535 : spdk_bdev_is_dif_head_of_md(bdev),
956 : 0 : spdk_bdev_get_dif_type(bdev),
957 : 0 : dif_check_flags,
958 : 0 : init_ref_tag, 0, 0, 0, 0, &dif_opts);
959 : :
960 : 561535 : return (rc == 0) ? true : false;
961 : 0 : }
962 : :
963 : : static void
964 : 1139697 : nvmf_bdev_ctrlr_zcopy_start_complete(struct spdk_bdev_io *bdev_io, bool success,
965 : : void *cb_arg)
966 : : {
967 : 1139697 : struct spdk_nvmf_request *req = cb_arg;
968 : 0 : struct iovec *iov;
969 : 1139697 : int iovcnt = 0;
970 : :
971 [ + + # # ]: 1139697 : if (spdk_unlikely(!success)) {
972 : 16873 : int sc = 0, sct = 0;
973 : 16873 : uint32_t cdw0 = 0;
974 [ # # # # : 16873 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
# # ]
975 : 16873 : spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc);
976 : :
977 [ # # # # ]: 16873 : response->cdw0 = cdw0;
978 [ # # # # : 16873 : response->status.sc = sc;
# # ]
979 [ # # # # : 16873 : response->status.sct = sct;
# # ]
980 : :
981 : 16873 : spdk_bdev_free_io(bdev_io);
982 : 16873 : spdk_nvmf_request_complete(req);
983 : 16873 : return;
984 : : }
985 : :
986 : 1122824 : spdk_bdev_io_get_iovec(bdev_io, &iov, &iovcnt);
987 : :
988 [ - + # # ]: 1122824 : assert(iovcnt <= NVMF_REQ_MAX_BUFFERS);
989 [ - + # # ]: 1122824 : assert(iovcnt > 0);
990 : :
991 [ # # # # ]: 1122824 : req->iovcnt = iovcnt;
992 : :
993 [ - + # # : 1122824 : assert(req->iov == iov);
# # ]
994 : :
995 [ # # # # ]: 1122824 : req->zcopy_bdev_io = bdev_io; /* Preserve the bdev_io for the end zcopy */
996 : :
997 : 1122824 : spdk_nvmf_request_complete(req);
998 : : /* Don't free the bdev_io here as it is needed for the END ZCOPY */
999 : 0 : }
1000 : :
1001 : : int
1002 : 1139706 : nvmf_bdev_ctrlr_zcopy_start(struct spdk_bdev *bdev,
1003 : : struct spdk_bdev_desc *desc,
1004 : : struct spdk_io_channel *ch,
1005 : : struct spdk_nvmf_request *req)
1006 : : {
1007 [ # # # # : 1139706 : struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
# # ]
1008 : 1139706 : uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev);
1009 : 1139706 : uint32_t block_size = spdk_bdev_get_block_size(bdev);
1010 : 9 : uint64_t start_lba;
1011 : 9 : uint64_t num_blocks;
1012 : : int rc;
1013 : :
1014 [ # # # # : 1139706 : nvmf_bdev_ctrlr_get_rw_params(&req->cmd->nvme_cmd, &start_lba, &num_blocks);
# # ]
1015 : :
1016 [ + + ]: 1139706 : if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) {
1017 : 3 : SPDK_ERRLOG("end of media\n");
1018 [ # # # # : 3 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
1019 [ # # # # : 3 : rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE;
# # ]
1020 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
1021 : : }
1022 : :
1023 [ + + # # : 1139703 : if (spdk_unlikely(num_blocks * block_size > req->length)) {
# # ]
1024 [ # # # # ]: 3 : SPDK_ERRLOG("Read NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n",
1025 : : num_blocks, block_size, req->length);
1026 [ # # # # : 3 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
1027 [ # # # # : 3 : rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID;
# # ]
1028 : 3 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
1029 : : }
1030 : :
1031 [ # # # # : 1139700 : bool populate = (req->cmd->nvme_cmd.opc == SPDK_NVME_OPC_READ) ? true : false;
# # # # ]
1032 : :
1033 [ # # # # : 1139700 : rc = spdk_bdev_zcopy_start(desc, ch, req->iov, req->iovcnt, start_lba,
# # ]
1034 [ # # ]: 0 : num_blocks, populate, nvmf_bdev_ctrlr_zcopy_start_complete, req);
1035 [ - + ]: 1139700 : if (spdk_unlikely(rc != 0)) {
1036 [ # # ]: 0 : if (rc == -ENOMEM) {
1037 : 0 : nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req);
1038 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
1039 : : }
1040 [ # # # # : 0 : rsp->status.sct = SPDK_NVME_SCT_GENERIC;
# # ]
1041 [ # # # # : 0 : rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
# # ]
1042 : 0 : return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
1043 : : }
1044 : :
1045 : 1139700 : return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
1046 : 0 : }
1047 : :
1048 : : static void
1049 : 1122824 : nvmf_bdev_ctrlr_zcopy_end_complete(struct spdk_bdev_io *bdev_io, bool success,
1050 : : void *cb_arg)
1051 : : {
1052 : 1122824 : struct spdk_nvmf_request *req = cb_arg;
1053 : :
1054 [ + + # # ]: 1122824 : if (spdk_unlikely(!success)) {
1055 : 16431 : int sc = 0, sct = 0;
1056 : 16431 : uint32_t cdw0 = 0;
1057 [ # # # # : 16431 : struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
# # ]
1058 : 16431 : spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc);
1059 : :
1060 [ # # # # ]: 16431 : response->cdw0 = cdw0;
1061 [ # # # # : 16431 : response->status.sc = sc;
# # ]
1062 [ # # # # : 16431 : response->status.sct = sct;
# # ]
1063 : 0 : }
1064 : :
1065 : 1122824 : spdk_bdev_free_io(bdev_io);
1066 [ # # # # ]: 1122824 : req->zcopy_bdev_io = NULL;
1067 : 1122824 : spdk_nvmf_request_complete(req);
1068 : 1122824 : }
1069 : :
1070 : : void
1071 : 1122824 : nvmf_bdev_ctrlr_zcopy_end(struct spdk_nvmf_request *req, bool commit)
1072 : : {
1073 : : int rc __attribute__((unused));
1074 : :
1075 [ # # # # : 1122824 : rc = spdk_bdev_zcopy_end(req->zcopy_bdev_io, commit, nvmf_bdev_ctrlr_zcopy_end_complete, req);
# # ]
1076 : :
1077 : : /* The only way spdk_bdev_zcopy_end() can fail is if we pass a bdev_io type that isn't ZCOPY */
1078 [ - + # # ]: 1122824 : assert(rc == 0);
1079 : 1122824 : }
|