Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2018 Intel Corporation. All rights reserved.
3 : * Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved.
4 : * Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : */
6 :
7 : #ifndef SPDK_INTERNAL_NVME_TCP_H
8 : #define SPDK_INTERNAL_NVME_TCP_H
9 :
10 : #include "spdk/likely.h"
11 : #include "spdk/sock.h"
12 : #include "spdk/dif.h"
13 : #include "spdk/hexlify.h"
14 : #include "spdk/nvmf_spec.h"
15 : #include "spdk/util.h"
16 : #include "spdk/base64.h"
17 :
18 : #include "sgl.h"
19 :
20 : #include "openssl/evp.h"
21 : #include "openssl/kdf.h"
22 : #include "openssl/sha.h"
23 :
24 : #define SPDK_CRC32C_XOR 0xffffffffUL
25 : #define SPDK_NVME_TCP_DIGEST_LEN 4
26 : #define SPDK_NVME_TCP_DIGEST_ALIGNMENT 4
27 : #define SPDK_NVME_TCP_QPAIR_EXIT_TIMEOUT 30
28 : #define SPDK_NVMF_TCP_RECV_BUF_SIZE_FACTOR 8
29 : #define SPDK_NVME_TCP_IN_CAPSULE_DATA_MAX_SIZE 8192u
30 : /*
31 : * Maximum number of SGL elements.
32 : */
33 : #define NVME_TCP_MAX_SGL_DESCRIPTORS (16)
34 :
35 : #define MAKE_DIGEST_WORD(BUF, CRC32C) \
36 : ( ((*((uint8_t *)(BUF)+0)) = (uint8_t)((uint32_t)(CRC32C) >> 0)), \
37 : ((*((uint8_t *)(BUF)+1)) = (uint8_t)((uint32_t)(CRC32C) >> 8)), \
38 : ((*((uint8_t *)(BUF)+2)) = (uint8_t)((uint32_t)(CRC32C) >> 16)), \
39 : ((*((uint8_t *)(BUF)+3)) = (uint8_t)((uint32_t)(CRC32C) >> 24)))
40 :
41 : #define MATCH_DIGEST_WORD(BUF, CRC32C) \
42 : ( ((((uint32_t) *((uint8_t *)(BUF)+0)) << 0) \
43 : | (((uint32_t) *((uint8_t *)(BUF)+1)) << 8) \
44 : | (((uint32_t) *((uint8_t *)(BUF)+2)) << 16) \
45 : | (((uint32_t) *((uint8_t *)(BUF)+3)) << 24)) \
46 : == (CRC32C))
47 :
48 : #define DGET32(B) \
49 : ((( (uint32_t) *((uint8_t *)(B)+0)) << 0) \
50 : | (((uint32_t) *((uint8_t *)(B)+1)) << 8) \
51 : | (((uint32_t) *((uint8_t *)(B)+2)) << 16) \
52 : | (((uint32_t) *((uint8_t *)(B)+3)) << 24))
53 :
54 : #define DSET32(B,D) \
55 : (((*((uint8_t *)(B)+0)) = (uint8_t)((uint32_t)(D) >> 0)), \
56 : ((*((uint8_t *)(B)+1)) = (uint8_t)((uint32_t)(D) >> 8)), \
57 : ((*((uint8_t *)(B)+2)) = (uint8_t)((uint32_t)(D) >> 16)), \
58 : ((*((uint8_t *)(B)+3)) = (uint8_t)((uint32_t)(D) >> 24)))
59 :
60 : /* The PSK identity comprises of following components:
61 : * 4-character format specifier "NVMe" +
62 : * 1-character TLS protocol version indicator +
63 : * 1-character PSK type indicator, specifying the used PSK +
64 : * 2-characters hash specifier +
65 : * NQN of the host (SPDK_NVMF_NQN_MAX_LEN -> 223) +
66 : * NQN of the subsystem (SPDK_NVMF_NQN_MAX_LEN -> 223) +
67 : * 2 space character separators +
68 : * 1 null terminator =
69 : * 457 characters. */
70 : #define NVMF_PSK_IDENTITY_LEN (SPDK_NVMF_NQN_MAX_LEN + SPDK_NVMF_NQN_MAX_LEN + 11)
71 :
72 : /* The maximum size of hkdf_info is defined by RFC 8446, 514B (2 + 256 + 256). */
73 : #define NVME_TCP_HKDF_INFO_MAX_LEN 514
74 :
75 : #define PSK_ID_PREFIX "NVMe0R"
76 :
77 : enum nvme_tcp_cipher_suite {
78 : NVME_TCP_CIPHER_AES_128_GCM_SHA256,
79 : NVME_TCP_CIPHER_AES_256_GCM_SHA384,
80 : };
81 :
82 : typedef void (*nvme_tcp_qpair_xfer_complete_cb)(void *cb_arg);
83 :
84 : struct nvme_tcp_pdu {
85 : union {
86 : /* to hold error pdu data */
87 : uint8_t raw[SPDK_NVME_TCP_TERM_REQ_PDU_MAX_SIZE];
88 : struct spdk_nvme_tcp_common_pdu_hdr common;
89 : struct spdk_nvme_tcp_ic_req ic_req;
90 : struct spdk_nvme_tcp_term_req_hdr term_req;
91 : struct spdk_nvme_tcp_cmd capsule_cmd;
92 : struct spdk_nvme_tcp_h2c_data_hdr h2c_data;
93 : struct spdk_nvme_tcp_ic_resp ic_resp;
94 : struct spdk_nvme_tcp_rsp capsule_resp;
95 : struct spdk_nvme_tcp_c2h_data_hdr c2h_data;
96 : struct spdk_nvme_tcp_r2t_hdr r2t;
97 :
98 : } hdr;
99 :
100 : bool has_hdgst;
101 : bool ddgst_enable;
102 : uint32_t data_digest_crc32;
103 : uint8_t data_digest[SPDK_NVME_TCP_DIGEST_LEN];
104 :
105 : uint8_t ch_valid_bytes;
106 : uint8_t psh_valid_bytes;
107 : uint8_t psh_len;
108 :
109 : nvme_tcp_qpair_xfer_complete_cb cb_fn;
110 : void *cb_arg;
111 :
112 : /* The sock request ends with a 0 length iovec. Place the actual iovec immediately
113 : * after it. There is a static assert below to check if the compiler inserted
114 : * any unwanted padding */
115 : struct spdk_sock_request sock_req;
116 : struct iovec iov[NVME_TCP_MAX_SGL_DESCRIPTORS * 2];
117 :
118 : struct iovec data_iov[NVME_TCP_MAX_SGL_DESCRIPTORS];
119 : uint32_t data_iovcnt;
120 : uint32_t data_len;
121 :
122 : uint32_t rw_offset;
123 : TAILQ_ENTRY(nvme_tcp_pdu) tailq;
124 : uint32_t remaining;
125 : uint32_t padding_len;
126 :
127 : struct spdk_dif_ctx *dif_ctx;
128 :
129 : void *req; /* data tied to a tcp request */
130 : void *qpair;
131 : SLIST_ENTRY(nvme_tcp_pdu) slist;
132 : };
133 : SPDK_STATIC_ASSERT(offsetof(struct nvme_tcp_pdu,
134 : sock_req) + sizeof(struct spdk_sock_request) == offsetof(struct nvme_tcp_pdu, iov),
135 : "Compiler inserted padding between iov and sock_req");
136 :
137 : enum nvme_tcp_pdu_recv_state {
138 : /* Ready to wait for PDU */
139 : NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY,
140 :
141 : /* Active tqpair waiting for any PDU common header */
142 : NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH,
143 :
144 : /* Active tqpair waiting for any PDU specific header */
145 : NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH,
146 :
147 : /* Active tqpair waiting for a tcp request, only use in target side */
148 : NVME_TCP_PDU_RECV_STATE_AWAIT_REQ,
149 :
150 : /* Active tqpair waiting for a free buffer to store PDU */
151 : NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_BUF,
152 :
153 : /* Active tqpair waiting for payload */
154 : NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD,
155 :
156 : /* Active tqpair waiting for all outstanding PDUs to complete */
157 : NVME_TCP_PDU_RECV_STATE_QUIESCING,
158 :
159 : /* Active tqpair does not wait for payload */
160 : NVME_TCP_PDU_RECV_STATE_ERROR,
161 : };
162 :
163 : enum nvme_tcp_error_codes {
164 : NVME_TCP_PDU_IN_PROGRESS = 0,
165 : NVME_TCP_CONNECTION_FATAL = -1,
166 : NVME_TCP_PDU_FATAL = -2,
167 : };
168 :
169 : static const bool g_nvme_tcp_hdgst[] = {
170 : [SPDK_NVME_TCP_PDU_TYPE_IC_REQ] = false,
171 : [SPDK_NVME_TCP_PDU_TYPE_IC_RESP] = false,
172 : [SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ] = false,
173 : [SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ] = false,
174 : [SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD] = true,
175 : [SPDK_NVME_TCP_PDU_TYPE_CAPSULE_RESP] = true,
176 : [SPDK_NVME_TCP_PDU_TYPE_H2C_DATA] = true,
177 : [SPDK_NVME_TCP_PDU_TYPE_C2H_DATA] = true,
178 : [SPDK_NVME_TCP_PDU_TYPE_R2T] = true
179 : };
180 :
181 : static const bool g_nvme_tcp_ddgst[] = {
182 : [SPDK_NVME_TCP_PDU_TYPE_IC_REQ] = false,
183 : [SPDK_NVME_TCP_PDU_TYPE_IC_RESP] = false,
184 : [SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ] = false,
185 : [SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ] = false,
186 : [SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD] = true,
187 : [SPDK_NVME_TCP_PDU_TYPE_CAPSULE_RESP] = false,
188 : [SPDK_NVME_TCP_PDU_TYPE_H2C_DATA] = true,
189 : [SPDK_NVME_TCP_PDU_TYPE_C2H_DATA] = true,
190 : [SPDK_NVME_TCP_PDU_TYPE_R2T] = false
191 : };
192 :
193 : static uint32_t
194 4 : nvme_tcp_pdu_calc_header_digest(struct nvme_tcp_pdu *pdu)
195 : {
196 : uint32_t crc32c;
197 4 : uint32_t hlen = pdu->hdr.common.hlen;
198 :
199 4 : crc32c = spdk_crc32c_update(&pdu->hdr.raw, hlen, ~0);
200 4 : crc32c = crc32c ^ SPDK_CRC32C_XOR;
201 4 : return crc32c;
202 : }
203 :
204 : static uint32_t
205 3 : nvme_tcp_pdu_calc_data_digest(struct nvme_tcp_pdu *pdu)
206 : {
207 3 : uint32_t crc32c = SPDK_CRC32C_XOR;
208 : uint32_t mod;
209 :
210 3 : assert(pdu->data_len != 0);
211 :
212 3 : if (spdk_likely(!pdu->dif_ctx)) {
213 3 : crc32c = spdk_crc32c_iov_update(pdu->data_iov, pdu->data_iovcnt, crc32c);
214 3 : } else {
215 0 : spdk_dif_update_crc32c_stream(pdu->data_iov, pdu->data_iovcnt,
216 0 : 0, pdu->data_len, &crc32c, pdu->dif_ctx);
217 : }
218 :
219 3 : mod = pdu->data_len % SPDK_NVME_TCP_DIGEST_ALIGNMENT;
220 3 : if (mod != 0) {
221 0 : uint32_t pad_length = SPDK_NVME_TCP_DIGEST_ALIGNMENT - mod;
222 0 : uint8_t pad[3] = {0, 0, 0};
223 :
224 0 : assert(pad_length > 0);
225 0 : assert(pad_length <= sizeof(pad));
226 0 : crc32c = spdk_crc32c_update(pad, pad_length, crc32c);
227 0 : }
228 3 : return crc32c;
229 : }
230 :
231 : static inline void
232 52 : _nvme_tcp_sgl_get_buf(struct spdk_iov_sgl *s, void **_buf, uint32_t *_buf_len)
233 : {
234 52 : if (_buf != NULL) {
235 52 : *_buf = (uint8_t *)s->iov->iov_base + s->iov_offset;
236 52 : }
237 52 : if (_buf_len != NULL) {
238 52 : *_buf_len = s->iov->iov_len - s->iov_offset;
239 52 : }
240 52 : }
241 :
242 : static inline bool
243 29 : _nvme_tcp_sgl_append_multi(struct spdk_iov_sgl *s, struct iovec *iov, int iovcnt)
244 : {
245 : int i;
246 :
247 75 : for (i = 0; i < iovcnt; i++) {
248 46 : if (!spdk_iov_sgl_append(s, iov[i].iov_base, iov[i].iov_len)) {
249 0 : return false;
250 : }
251 46 : }
252 :
253 29 : return true;
254 29 : }
255 :
256 : static inline uint32_t
257 0 : _get_iov_array_size(struct iovec *iov, int iovcnt)
258 : {
259 : int i;
260 0 : uint32_t size = 0;
261 :
262 0 : for (i = 0; i < iovcnt; i++) {
263 0 : size += iov[i].iov_len;
264 0 : }
265 :
266 0 : return size;
267 : }
268 :
269 : static inline bool
270 1 : _nvme_tcp_sgl_append_multi_with_md(struct spdk_iov_sgl *s, struct iovec *iov, int iovcnt,
271 : uint32_t data_len, const struct spdk_dif_ctx *dif_ctx)
272 : {
273 : int rc;
274 1 : uint32_t mapped_len = 0;
275 :
276 1 : if (s->iov_offset >= data_len) {
277 0 : s->iov_offset -= _get_iov_array_size(iov, iovcnt);
278 0 : } else {
279 2 : rc = spdk_dif_set_md_interleave_iovs(s->iov, s->iovcnt, iov, iovcnt,
280 1 : s->iov_offset, data_len - s->iov_offset,
281 1 : &mapped_len, dif_ctx);
282 1 : if (rc < 0) {
283 0 : SPDK_ERRLOG("Failed to setup iovs for DIF insert/strip.\n");
284 0 : return false;
285 : }
286 :
287 1 : s->total_size += mapped_len;
288 1 : s->iov_offset = 0;
289 1 : assert(s->iovcnt >= rc);
290 1 : s->iovcnt -= rc;
291 1 : s->iov += rc;
292 :
293 1 : if (s->iovcnt == 0) {
294 0 : return false;
295 : }
296 : }
297 :
298 1 : return true;
299 1 : }
300 :
301 : static int
302 49 : nvme_tcp_build_iovs(struct iovec *iov, int iovcnt, struct nvme_tcp_pdu *pdu,
303 : bool hdgst_enable, bool ddgst_enable, uint32_t *_mapped_length)
304 : {
305 : uint32_t hlen;
306 : uint32_t plen __attribute__((unused));
307 : struct spdk_iov_sgl sgl;
308 :
309 49 : if (iovcnt == 0) {
310 0 : return 0;
311 : }
312 :
313 49 : spdk_iov_sgl_init(&sgl, iov, iovcnt, 0);
314 49 : hlen = pdu->hdr.common.hlen;
315 :
316 : /* Header Digest */
317 49 : if (g_nvme_tcp_hdgst[pdu->hdr.common.pdu_type] && hdgst_enable) {
318 7 : hlen += SPDK_NVME_TCP_DIGEST_LEN;
319 7 : }
320 :
321 49 : plen = hlen;
322 49 : if (!pdu->data_len) {
323 : /* PDU header + possible header digest */
324 19 : spdk_iov_sgl_append(&sgl, (uint8_t *)&pdu->hdr.raw, hlen);
325 19 : goto end;
326 : }
327 :
328 : /* Padding */
329 30 : if (pdu->padding_len > 0) {
330 1 : hlen += pdu->padding_len;
331 1 : plen = hlen;
332 1 : }
333 :
334 30 : if (!spdk_iov_sgl_append(&sgl, (uint8_t *)&pdu->hdr.raw, hlen)) {
335 0 : goto end;
336 : }
337 :
338 : /* Data Segment */
339 30 : plen += pdu->data_len;
340 30 : if (spdk_likely(!pdu->dif_ctx)) {
341 29 : if (!_nvme_tcp_sgl_append_multi(&sgl, pdu->data_iov, pdu->data_iovcnt)) {
342 0 : goto end;
343 : }
344 29 : } else {
345 2 : if (!_nvme_tcp_sgl_append_multi_with_md(&sgl, pdu->data_iov, pdu->data_iovcnt,
346 1 : pdu->data_len, pdu->dif_ctx)) {
347 0 : goto end;
348 : }
349 : }
350 :
351 : /* Data Digest */
352 30 : if (g_nvme_tcp_ddgst[pdu->hdr.common.pdu_type] && ddgst_enable) {
353 6 : plen += SPDK_NVME_TCP_DIGEST_LEN;
354 6 : spdk_iov_sgl_append(&sgl, pdu->data_digest, SPDK_NVME_TCP_DIGEST_LEN);
355 6 : }
356 :
357 30 : assert(plen == pdu->hdr.common.plen);
358 :
359 : end:
360 49 : if (_mapped_length != NULL) {
361 49 : *_mapped_length = sgl.total_size;
362 49 : }
363 :
364 49 : return iovcnt - sgl.iovcnt;
365 49 : }
366 :
367 : static int
368 0 : nvme_tcp_build_payload_iovs(struct iovec *iov, int iovcnt, struct nvme_tcp_pdu *pdu,
369 : bool ddgst_enable, uint32_t *_mapped_length)
370 : {
371 : struct spdk_iov_sgl sgl;
372 :
373 0 : if (iovcnt == 0) {
374 0 : return 0;
375 : }
376 :
377 0 : spdk_iov_sgl_init(&sgl, iov, iovcnt, pdu->rw_offset);
378 :
379 0 : if (spdk_likely(!pdu->dif_ctx)) {
380 0 : if (!_nvme_tcp_sgl_append_multi(&sgl, pdu->data_iov, pdu->data_iovcnt)) {
381 0 : goto end;
382 : }
383 0 : } else {
384 0 : if (!_nvme_tcp_sgl_append_multi_with_md(&sgl, pdu->data_iov, pdu->data_iovcnt,
385 0 : pdu->data_len, pdu->dif_ctx)) {
386 0 : goto end;
387 : }
388 : }
389 :
390 : /* Data Digest */
391 0 : if (ddgst_enable) {
392 0 : spdk_iov_sgl_append(&sgl, pdu->data_digest, SPDK_NVME_TCP_DIGEST_LEN);
393 0 : }
394 :
395 : end:
396 0 : if (_mapped_length != NULL) {
397 0 : *_mapped_length = sgl.total_size;
398 0 : }
399 0 : return iovcnt - sgl.iovcnt;
400 0 : }
401 :
402 : static int
403 4 : nvme_tcp_read_data(struct spdk_sock *sock, int bytes,
404 : void *buf)
405 : {
406 : int ret;
407 :
408 4 : ret = spdk_sock_recv(sock, buf, bytes);
409 :
410 4 : if (ret > 0) {
411 4 : return ret;
412 : }
413 :
414 0 : if (ret < 0) {
415 0 : if (errno == EAGAIN || errno == EWOULDBLOCK) {
416 0 : return 0;
417 : }
418 :
419 : /* For connect reset issue, do not output error log */
420 0 : if (errno != ECONNRESET) {
421 0 : SPDK_ERRLOG("spdk_sock_recv() failed, errno %d: %s\n",
422 : errno, spdk_strerror(errno));
423 0 : }
424 0 : }
425 :
426 : /* connection closed */
427 0 : return NVME_TCP_CONNECTION_FATAL;
428 4 : }
429 :
430 : static int
431 0 : nvme_tcp_readv_data(struct spdk_sock *sock, struct iovec *iov, int iovcnt)
432 : {
433 : int ret;
434 :
435 0 : assert(sock != NULL);
436 0 : if (iov == NULL || iovcnt == 0) {
437 0 : return 0;
438 : }
439 :
440 0 : if (iovcnt == 1) {
441 0 : return nvme_tcp_read_data(sock, iov->iov_len, iov->iov_base);
442 : }
443 :
444 0 : ret = spdk_sock_readv(sock, iov, iovcnt);
445 :
446 0 : if (ret > 0) {
447 0 : return ret;
448 : }
449 :
450 0 : if (ret < 0) {
451 0 : if (errno == EAGAIN || errno == EWOULDBLOCK) {
452 0 : return 0;
453 : }
454 :
455 : /* For connect reset issue, do not output error log */
456 0 : if (errno != ECONNRESET) {
457 0 : SPDK_ERRLOG("spdk_sock_readv() failed, errno %d: %s\n",
458 : errno, spdk_strerror(errno));
459 0 : }
460 0 : }
461 :
462 : /* connection closed */
463 0 : return NVME_TCP_CONNECTION_FATAL;
464 0 : }
465 :
466 :
467 : static int
468 0 : nvme_tcp_read_payload_data(struct spdk_sock *sock, struct nvme_tcp_pdu *pdu)
469 : {
470 : struct iovec iov[NVME_TCP_MAX_SGL_DESCRIPTORS + 1];
471 : int iovcnt;
472 :
473 0 : iovcnt = nvme_tcp_build_payload_iovs(iov, NVME_TCP_MAX_SGL_DESCRIPTORS + 1, pdu,
474 0 : pdu->ddgst_enable, NULL);
475 0 : assert(iovcnt >= 0);
476 :
477 0 : return nvme_tcp_readv_data(sock, iov, iovcnt);
478 : }
479 :
480 : static void
481 36 : _nvme_tcp_pdu_set_data(struct nvme_tcp_pdu *pdu, void *data, uint32_t data_len)
482 : {
483 36 : pdu->data_iov[0].iov_base = data;
484 36 : pdu->data_iov[0].iov_len = data_len;
485 36 : pdu->data_iovcnt = 1;
486 36 : }
487 :
488 : static void
489 30 : nvme_tcp_pdu_set_data(struct nvme_tcp_pdu *pdu, void *data, uint32_t data_len)
490 : {
491 30 : _nvme_tcp_pdu_set_data(pdu, data, data_len);
492 30 : pdu->data_len = data_len;
493 30 : }
494 :
495 : static void
496 22 : nvme_tcp_pdu_set_data_buf(struct nvme_tcp_pdu *pdu,
497 : struct iovec *iov, int iovcnt,
498 : uint32_t data_offset, uint32_t data_len)
499 : {
500 : uint32_t buf_offset, buf_len, remain_len, len;
501 : uint8_t *buf;
502 : struct spdk_iov_sgl pdu_sgl, buf_sgl;
503 :
504 22 : pdu->data_len = data_len;
505 :
506 22 : if (spdk_likely(!pdu->dif_ctx)) {
507 16 : buf_offset = data_offset;
508 16 : buf_len = data_len;
509 16 : } else {
510 6 : spdk_dif_ctx_set_data_offset(pdu->dif_ctx, data_offset);
511 12 : spdk_dif_get_range_with_md(data_offset, data_len,
512 6 : &buf_offset, &buf_len, pdu->dif_ctx);
513 : }
514 :
515 22 : if (iovcnt == 1) {
516 6 : _nvme_tcp_pdu_set_data(pdu, (void *)((uint64_t)iov[0].iov_base + buf_offset), buf_len);
517 6 : } else {
518 16 : spdk_iov_sgl_init(&pdu_sgl, pdu->data_iov, NVME_TCP_MAX_SGL_DESCRIPTORS, 0);
519 16 : spdk_iov_sgl_init(&buf_sgl, iov, iovcnt, 0);
520 :
521 16 : spdk_iov_sgl_advance(&buf_sgl, buf_offset);
522 16 : remain_len = buf_len;
523 :
524 67 : while (remain_len > 0) {
525 52 : _nvme_tcp_sgl_get_buf(&buf_sgl, (void *)&buf, &len);
526 52 : len = spdk_min(len, remain_len);
527 :
528 52 : spdk_iov_sgl_advance(&buf_sgl, len);
529 52 : remain_len -= len;
530 :
531 52 : if (!spdk_iov_sgl_append(&pdu_sgl, buf, len)) {
532 1 : break;
533 : }
534 : }
535 :
536 16 : assert(remain_len == 0);
537 16 : assert(pdu_sgl.total_size == buf_len);
538 :
539 16 : pdu->data_iovcnt = NVME_TCP_MAX_SGL_DESCRIPTORS - pdu_sgl.iovcnt;
540 : }
541 22 : }
542 :
543 : static void
544 3 : nvme_tcp_pdu_calc_psh_len(struct nvme_tcp_pdu *pdu, bool hdgst_enable)
545 : {
546 : uint8_t psh_len, pdo, padding_len;
547 :
548 3 : psh_len = pdu->hdr.common.hlen;
549 :
550 3 : if (g_nvme_tcp_hdgst[pdu->hdr.common.pdu_type] && hdgst_enable) {
551 0 : pdu->has_hdgst = true;
552 0 : psh_len += SPDK_NVME_TCP_DIGEST_LEN;
553 0 : }
554 3 : if (pdu->hdr.common.plen > psh_len) {
555 0 : switch (pdu->hdr.common.pdu_type) {
556 : case SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD:
557 : case SPDK_NVME_TCP_PDU_TYPE_H2C_DATA:
558 : case SPDK_NVME_TCP_PDU_TYPE_C2H_DATA:
559 0 : pdo = pdu->hdr.common.pdo;
560 0 : padding_len = pdo - psh_len;
561 0 : if (padding_len > 0) {
562 0 : psh_len = pdo;
563 0 : }
564 0 : break;
565 : default:
566 : /* There is no padding for other PDU types */
567 0 : break;
568 : }
569 0 : }
570 :
571 3 : psh_len -= sizeof(struct spdk_nvme_tcp_common_pdu_hdr);
572 3 : pdu->psh_len = psh_len;
573 3 : }
574 :
575 : static inline int
576 4 : nvme_tcp_generate_psk_identity(char *out_id, size_t out_id_len, const char *hostnqn,
577 : const char *subnqn, enum nvme_tcp_cipher_suite tls_cipher_suite)
578 : {
579 : int rc;
580 :
581 4 : assert(out_id != NULL);
582 :
583 4 : if (out_id_len < strlen(PSK_ID_PREFIX) + strlen(hostnqn) + strlen(subnqn) + 5) {
584 1 : SPDK_ERRLOG("Out buffer too small!\n");
585 1 : return -1;
586 : }
587 :
588 3 : if (tls_cipher_suite == NVME_TCP_CIPHER_AES_128_GCM_SHA256) {
589 4 : rc = snprintf(out_id, out_id_len, "%s%s %s %s", PSK_ID_PREFIX, "01",
590 2 : hostnqn, subnqn);
591 3 : } else if (tls_cipher_suite == NVME_TCP_CIPHER_AES_256_GCM_SHA384) {
592 0 : rc = snprintf(out_id, out_id_len, "%s%s %s %s", PSK_ID_PREFIX, "02",
593 0 : hostnqn, subnqn);
594 0 : } else {
595 1 : SPDK_ERRLOG("Unknown cipher suite requested!\n");
596 1 : return -EOPNOTSUPP;
597 : }
598 :
599 2 : if (rc < 0) {
600 0 : SPDK_ERRLOG("Could not generate PSK identity\n");
601 0 : return -1;
602 : }
603 :
604 2 : return 0;
605 4 : }
606 :
607 : enum nvme_tcp_hash_algorithm {
608 : NVME_TCP_HASH_ALGORITHM_NONE,
609 : NVME_TCP_HASH_ALGORITHM_SHA256,
610 : NVME_TCP_HASH_ALGORITHM_SHA384,
611 : };
612 :
613 : static inline int
614 8 : nvme_tcp_derive_retained_psk(const uint8_t *psk_in, uint64_t psk_in_size, const char *hostnqn,
615 : uint8_t *psk_out, uint64_t psk_out_len, enum nvme_tcp_hash_algorithm psk_retained_hash)
616 : {
617 : EVP_PKEY_CTX *ctx;
618 : uint64_t digest_len;
619 8 : uint8_t hkdf_info[NVME_TCP_HKDF_INFO_MAX_LEN] = {};
620 8 : const char *label = "tls13 HostNQN";
621 : size_t pos, labellen, nqnlen;
622 : const EVP_MD *hash;
623 : int rc, hkdf_info_size;
624 :
625 8 : labellen = strlen(label);
626 8 : nqnlen = strlen(hostnqn);
627 8 : assert(nqnlen <= SPDK_NVMF_NQN_MAX_LEN);
628 :
629 8 : *(uint16_t *)&hkdf_info[0] = htons(psk_in_size);
630 8 : pos = sizeof(uint16_t);
631 8 : hkdf_info[pos] = (uint8_t)labellen;
632 8 : pos += sizeof(uint8_t);
633 8 : memcpy(&hkdf_info[pos], label, labellen);
634 8 : pos += labellen;
635 8 : hkdf_info[pos] = (uint8_t)nqnlen;
636 8 : pos += sizeof(uint8_t);
637 8 : memcpy(&hkdf_info[pos], hostnqn, nqnlen);
638 8 : pos += nqnlen;
639 8 : hkdf_info_size = pos;
640 :
641 8 : switch (psk_retained_hash) {
642 : case NVME_TCP_HASH_ALGORITHM_SHA256:
643 6 : digest_len = SHA256_DIGEST_LENGTH;
644 6 : hash = EVP_sha256();
645 6 : break;
646 : case NVME_TCP_HASH_ALGORITHM_SHA384:
647 1 : digest_len = SHA384_DIGEST_LENGTH;
648 1 : hash = EVP_sha384();
649 1 : break;
650 : default:
651 1 : SPDK_ERRLOG("Unknown PSK hash requested!\n");
652 1 : return -EOPNOTSUPP;
653 : }
654 :
655 7 : if (digest_len > psk_out_len) {
656 1 : SPDK_ERRLOG("Insufficient buffer size for out key!\n");
657 1 : return -EINVAL;
658 : }
659 :
660 6 : ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_HKDF, NULL);
661 6 : if (!ctx) {
662 0 : SPDK_ERRLOG("Unable to initialize EVP_PKEY_CTX!\n");
663 0 : return -ENOMEM;
664 : }
665 :
666 : /* EVP_PKEY_* functions returns 1 as a success code and 0 or negative on failure. */
667 6 : if (EVP_PKEY_derive_init(ctx) != 1) {
668 0 : SPDK_ERRLOG("Unable to initialize key derivation ctx for HKDF!\n");
669 0 : rc = -ENOMEM;
670 0 : goto end;
671 : }
672 6 : if (EVP_PKEY_CTX_set_hkdf_md(ctx, hash) != 1) {
673 0 : SPDK_ERRLOG("Unable to set hash for HKDF!\n");
674 0 : rc = -EOPNOTSUPP;
675 0 : goto end;
676 : }
677 6 : if (EVP_PKEY_CTX_set1_hkdf_key(ctx, psk_in, psk_in_size) != 1) {
678 0 : SPDK_ERRLOG("Unable to set PSK key for HKDF!\n");
679 0 : rc = -ENOBUFS;
680 0 : goto end;
681 : }
682 :
683 6 : if (EVP_PKEY_CTX_add1_hkdf_info(ctx, hkdf_info, hkdf_info_size) != 1) {
684 0 : SPDK_ERRLOG("Unable to set info label for HKDF!\n");
685 0 : rc = -ENOBUFS;
686 0 : goto end;
687 : }
688 6 : if (EVP_PKEY_CTX_set1_hkdf_salt(ctx, NULL, 0) != 1) {
689 0 : SPDK_ERRLOG("Unable to set salt for HKDF!\n");
690 0 : rc = -EINVAL;
691 0 : goto end;
692 : }
693 6 : if (EVP_PKEY_derive(ctx, psk_out, &digest_len) != 1) {
694 0 : SPDK_ERRLOG("Unable to derive the PSK key!\n");
695 0 : rc = -EINVAL;
696 0 : goto end;
697 : }
698 :
699 6 : rc = digest_len;
700 :
701 : end:
702 6 : EVP_PKEY_CTX_free(ctx);
703 6 : return rc;
704 8 : }
705 :
706 : static inline int
707 4 : nvme_tcp_derive_tls_psk(const uint8_t *psk_in, uint64_t psk_in_size, const char *psk_identity,
708 : uint8_t *psk_out, uint64_t psk_out_size, enum nvme_tcp_cipher_suite tls_cipher_suite)
709 : {
710 : EVP_PKEY_CTX *ctx;
711 4 : uint64_t digest_len = 0;
712 4 : char hkdf_info[NVME_TCP_HKDF_INFO_MAX_LEN] = {};
713 4 : const char *label = "tls13 nvme-tls-psk";
714 : size_t pos, labellen, idlen;
715 : const EVP_MD *hash;
716 : int rc, hkdf_info_size;
717 :
718 4 : if (tls_cipher_suite == NVME_TCP_CIPHER_AES_128_GCM_SHA256) {
719 2 : digest_len = SHA256_DIGEST_LENGTH;
720 2 : hash = EVP_sha256();
721 4 : } else if (tls_cipher_suite == NVME_TCP_CIPHER_AES_256_GCM_SHA384) {
722 1 : digest_len = SHA384_DIGEST_LENGTH;
723 1 : hash = EVP_sha384();
724 1 : } else {
725 1 : SPDK_ERRLOG("Unknown cipher suite requested!\n");
726 1 : return -EOPNOTSUPP;
727 : }
728 :
729 3 : labellen = strlen(label);
730 3 : idlen = strlen(psk_identity);
731 3 : if (idlen > UINT8_MAX) {
732 0 : SPDK_ERRLOG("Invalid PSK ID: too long\n");
733 0 : return -1;
734 : }
735 :
736 3 : *(uint16_t *)&hkdf_info[0] = htons(psk_in_size);
737 3 : pos = sizeof(uint16_t);
738 3 : hkdf_info[pos] = (uint8_t)labellen;
739 3 : pos += sizeof(uint8_t);
740 3 : memcpy(&hkdf_info[pos], label, labellen);
741 3 : pos += labellen;
742 3 : hkdf_info[pos] = (uint8_t)idlen;
743 3 : pos += sizeof(uint8_t);
744 3 : memcpy(&hkdf_info[pos], psk_identity, idlen);
745 3 : pos += idlen;
746 3 : hkdf_info_size = pos;
747 :
748 3 : if (digest_len > psk_out_size) {
749 1 : SPDK_ERRLOG("Insufficient buffer size for out key!\n");
750 1 : return -1;
751 : }
752 :
753 2 : ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_HKDF, NULL);
754 2 : if (!ctx) {
755 0 : SPDK_ERRLOG("Unable to initialize EVP_PKEY_CTX!\n");
756 0 : return -1;
757 : }
758 :
759 2 : if (EVP_PKEY_derive_init(ctx) != 1) {
760 0 : SPDK_ERRLOG("Unable to initialize key derivation ctx for HKDF!\n");
761 0 : rc = -ENOMEM;
762 0 : goto end;
763 : }
764 2 : if (EVP_PKEY_CTX_set_hkdf_md(ctx, hash) != 1) {
765 0 : SPDK_ERRLOG("Unable to set hash method for HKDF!\n");
766 0 : rc = -EOPNOTSUPP;
767 0 : goto end;
768 : }
769 2 : if (EVP_PKEY_CTX_set1_hkdf_key(ctx, psk_in, psk_in_size) != 1) {
770 0 : SPDK_ERRLOG("Unable to set PSK key for HKDF!\n");
771 0 : rc = -ENOBUFS;
772 0 : goto end;
773 : }
774 2 : if (EVP_PKEY_CTX_add1_hkdf_info(ctx, hkdf_info, hkdf_info_size) != 1) {
775 0 : SPDK_ERRLOG("Unable to set info label for HKDF!\n");
776 0 : rc = -ENOBUFS;
777 0 : goto end;
778 : }
779 2 : if (EVP_PKEY_CTX_set1_hkdf_salt(ctx, NULL, 0) != 1) {
780 0 : SPDK_ERRLOG("Unable to set salt for HKDF!\n");
781 0 : rc = -EINVAL;
782 0 : goto end;
783 : }
784 2 : if (EVP_PKEY_derive(ctx, psk_out, &digest_len) != 1) {
785 0 : SPDK_ERRLOG("Unable to derive the PSK key!\n");
786 0 : rc = -EINVAL;
787 0 : goto end;
788 : }
789 :
790 2 : rc = digest_len;
791 :
792 : end:
793 2 : EVP_PKEY_CTX_free(ctx);
794 2 : return rc;
795 4 : }
796 :
797 : static inline int
798 1 : nvme_tcp_parse_interchange_psk(const char *psk_in, uint8_t *psk_out, size_t psk_out_size,
799 : uint64_t *psk_out_decoded_size, uint8_t *hash)
800 : {
801 1 : const char *delim = ":";
802 1 : char psk_cpy[SPDK_TLS_PSK_MAX_LEN] = {};
803 1 : char *sp = NULL;
804 1 : uint8_t psk_base64_decoded[SPDK_TLS_PSK_MAX_LEN] = {};
805 1 : uint64_t psk_configured_size = 0;
806 : uint32_t crc32_calc, crc32;
807 : char *psk_base64;
808 1 : uint64_t psk_base64_decoded_size = 0;
809 : int rc;
810 :
811 : /* Verify PSK format. */
812 1 : if (sscanf(psk_in, "NVMeTLSkey-1:%02hhx:", hash) != 1 || psk_in[strlen(psk_in) - 1] != delim[0]) {
813 0 : SPDK_ERRLOG("Invalid format of PSK interchange!\n");
814 0 : return -EINVAL;
815 : }
816 :
817 1 : if (strlen(psk_in) >= SPDK_TLS_PSK_MAX_LEN) {
818 0 : SPDK_ERRLOG("PSK interchange exceeds maximum %d characters!\n", SPDK_TLS_PSK_MAX_LEN);
819 0 : return -EINVAL;
820 : }
821 1 : if (*hash != NVME_TCP_HASH_ALGORITHM_NONE && *hash != NVME_TCP_HASH_ALGORITHM_SHA256 &&
822 0 : *hash != NVME_TCP_HASH_ALGORITHM_SHA384) {
823 0 : SPDK_ERRLOG("Invalid PSK length!\n");
824 0 : return -EINVAL;
825 : }
826 :
827 : /* Check provided hash function string. */
828 1 : memcpy(psk_cpy, psk_in, strlen(psk_in));
829 1 : strtok_r(psk_cpy, delim, &sp);
830 1 : strtok_r(NULL, delim, &sp);
831 :
832 1 : psk_base64 = strtok_r(NULL, delim, &sp);
833 1 : if (psk_base64 == NULL) {
834 0 : SPDK_ERRLOG("Could not get base64 string from PSK interchange!\n");
835 0 : return -EINVAL;
836 : }
837 :
838 1 : rc = spdk_base64_decode(psk_base64_decoded, &psk_base64_decoded_size, psk_base64);
839 1 : if (rc) {
840 0 : SPDK_ERRLOG("Could not decode base64 PSK!\n");
841 0 : return -EINVAL;
842 : }
843 :
844 1 : switch (*hash) {
845 : case NVME_TCP_HASH_ALGORITHM_SHA256:
846 1 : psk_configured_size = SHA256_DIGEST_LENGTH;
847 1 : break;
848 : case NVME_TCP_HASH_ALGORITHM_SHA384:
849 0 : psk_configured_size = SHA384_DIGEST_LENGTH;
850 0 : break;
851 : case NVME_TCP_HASH_ALGORITHM_NONE:
852 0 : if (psk_base64_decoded_size == SHA256_DIGEST_LENGTH + SPDK_CRC32_SIZE_BYTES) {
853 0 : psk_configured_size = SHA256_DIGEST_LENGTH;
854 0 : } else if (psk_base64_decoded_size == SHA384_DIGEST_LENGTH + SPDK_CRC32_SIZE_BYTES) {
855 0 : psk_configured_size = SHA384_DIGEST_LENGTH;
856 0 : }
857 0 : break;
858 : default:
859 0 : SPDK_ERRLOG("Invalid key: unsupported key hash\n");
860 0 : assert(0);
861 : return -EINVAL;
862 : }
863 1 : if (psk_base64_decoded_size != psk_configured_size + SPDK_CRC32_SIZE_BYTES) {
864 0 : SPDK_ERRLOG("Invalid key: unsupported key length\n");
865 0 : return -EINVAL;
866 : }
867 :
868 1 : crc32 = from_le32(&psk_base64_decoded[psk_configured_size]);
869 :
870 1 : crc32_calc = spdk_crc32_ieee_update(psk_base64_decoded, psk_configured_size, ~0);
871 1 : crc32_calc = ~crc32_calc;
872 :
873 1 : if (crc32 != crc32_calc) {
874 0 : SPDK_ERRLOG("CRC-32 checksums do not match!\n");
875 0 : return -EINVAL;
876 : }
877 :
878 1 : if (psk_configured_size > psk_out_size) {
879 0 : SPDK_ERRLOG("Insufficient buffer size: %lu for configured PSK of size: %lu!\n",
880 : psk_out_size, psk_configured_size);
881 0 : return -ENOBUFS;
882 : }
883 1 : memcpy(psk_out, psk_base64_decoded, psk_configured_size);
884 1 : *psk_out_decoded_size = psk_configured_size;
885 :
886 1 : return rc;
887 1 : }
888 :
889 : #endif /* SPDK_INTERNAL_NVME_TCP_H */
|
