LCOV - code coverage report
Current view: top level - lib/iscsi - iscsi.c (source / functions) Hit Total Coverage
Test: ut_cov_unit.info Lines: 1020 2432 41.9 %
Date: 2024-07-12 09:47:37 Functions: 45 99 45.5 %

          Line data    Source code
       1             : /*   SPDX-License-Identifier: BSD-3-Clause
       2             :  *   Copyright (C) 2008-2012 Daisuke Aoyama <aoyama@peach.ne.jp>.
       3             :  *   Copyright (C) 2016 Intel Corporation.
       4             :  *   All rights reserved.
       5             :  */
       6             : 
       7             : #include "spdk/stdinc.h"
       8             : 
       9             : #include "spdk/base64.h"
      10             : #include "spdk/crc32.h"
      11             : #include "spdk/endian.h"
      12             : #include "spdk/env.h"
      13             : #include "spdk/likely.h"
      14             : #include "spdk/trace.h"
      15             : #include "spdk/sock.h"
      16             : #include "spdk/string.h"
      17             : #include "spdk/queue.h"
      18             : 
      19             : #include "iscsi/md5.h"
      20             : #include "iscsi/iscsi.h"
      21             : #include "iscsi/param.h"
      22             : #include "iscsi/tgt_node.h"
      23             : #include "iscsi/task.h"
      24             : #include "iscsi/conn.h"
      25             : #include "spdk/scsi.h"
      26             : #include "spdk/bdev.h"
      27             : #include "iscsi/portal_grp.h"
      28             : 
      29             : #include "spdk/log.h"
      30             : 
      31             : #include "spdk_internal/sgl.h"
      32             : 
      33             : #define MAX_TMPBUF 1024
      34             : 
      35             : struct spdk_iscsi_globals g_iscsi = {
      36             :         .mutex = PTHREAD_MUTEX_INITIALIZER,
      37             :         .portal_head = TAILQ_HEAD_INITIALIZER(g_iscsi.portal_head),
      38             :         .pg_head = TAILQ_HEAD_INITIALIZER(g_iscsi.pg_head),
      39             :         .ig_head = TAILQ_HEAD_INITIALIZER(g_iscsi.ig_head),
      40             :         .target_head = TAILQ_HEAD_INITIALIZER(g_iscsi.target_head),
      41             :         .auth_group_head = TAILQ_HEAD_INITIALIZER(g_iscsi.auth_group_head),
      42             :         .poll_group_head = TAILQ_HEAD_INITIALIZER(g_iscsi.poll_group_head),
      43             : };
      44             : 
      45             : #define MATCH_DIGEST_WORD(BUF, CRC32C) \
      46             :         (    ((((uint32_t) *((uint8_t *)(BUF)+0)) << 0)           \
      47             :             | (((uint32_t) *((uint8_t *)(BUF)+1)) << 8)           \
      48             :             | (((uint32_t) *((uint8_t *)(BUF)+2)) << 16)  \
      49             :             | (((uint32_t) *((uint8_t *)(BUF)+3)) << 24)) \
      50             :             == (CRC32C))
      51             : 
      52             : #ifndef SPDK_CONFIG_HAVE_ARC4RANDOM
      53             : static void
      54           0 : srandomdev(void)
      55             : {
      56             :         unsigned long seed;
      57             :         time_t now;
      58             :         pid_t pid;
      59             : 
      60           0 :         pid = getpid();
      61           0 :         now = time(NULL);
      62           0 :         seed = pid ^ now;
      63           0 :         srandom(seed);
      64           0 : }
      65             : 
      66             : static int g_arc4random_initialized = 0;
      67             : 
      68             : static uint32_t
      69           0 : arc4random(void)
      70             : {
      71             :         uint32_t r;
      72             :         uint32_t r1, r2;
      73             : 
      74           0 :         if (!g_arc4random_initialized) {
      75           0 :                 srandomdev();
      76           0 :                 g_arc4random_initialized = 1;
      77             :         }
      78           0 :         r1 = (uint32_t)(random() & 0xffff);
      79           0 :         r2 = (uint32_t)(random() & 0xffff);
      80           0 :         r = (r1 << 16) | r2;
      81           0 :         return r;
      82             : }
      83             : #endif /* SPDK_CONFIG_HAVE_ARC4RANDOM */
      84             : 
      85             : static void
      86           0 : gen_random(uint8_t *buf, size_t len)
      87             : {
      88             :         uint32_t r;
      89             :         size_t idx;
      90             : 
      91           0 :         for (idx = 0; idx < len; idx++) {
      92           0 :                 r = arc4random();
      93           0 :                 buf[idx] = (uint8_t) r;
      94             :         }
      95           0 : }
      96             : 
      97             : static uint64_t
      98           2 : iscsi_get_isid(const uint8_t isid[6])
      99             : {
     100           2 :         return (uint64_t)isid[0] << 40 |
     101           2 :                (uint64_t)isid[1] << 32 |
     102           2 :                (uint64_t)isid[2] << 24 |
     103           2 :                (uint64_t)isid[3] << 16 |
     104           4 :                (uint64_t)isid[4] << 8 |
     105           2 :                (uint64_t)isid[5];
     106             : }
     107             : 
     108             : static int
     109           0 : bin2hex(char *buf, size_t len, const uint8_t *data, size_t data_len)
     110             : {
     111           0 :         const char *digits = "0123456789ABCDEF";
     112           0 :         size_t total = 0;
     113             :         size_t idx;
     114             : 
     115           0 :         if (len < 3) {
     116           0 :                 return -1;
     117             :         }
     118           0 :         buf[total] = '0';
     119           0 :         total++;
     120           0 :         buf[total] = 'x';
     121           0 :         total++;
     122           0 :         buf[total] = '\0';
     123             : 
     124           0 :         for (idx = 0; idx < data_len; idx++) {
     125           0 :                 if (total + 3 > len) {
     126           0 :                         buf[total] = '\0';
     127           0 :                         return - 1;
     128             :                 }
     129           0 :                 buf[total] = digits[(data[idx] >> 4) & 0x0fU];
     130           0 :                 total++;
     131           0 :                 buf[total] = digits[data[idx] & 0x0fU];
     132           0 :                 total++;
     133             :         }
     134           0 :         buf[total] = '\0';
     135           0 :         return total;
     136             : }
     137             : 
     138             : static int
     139           0 : hex2bin(uint8_t *data, size_t data_len, const char *str)
     140             : {
     141           0 :         const char *digits = "0123456789ABCDEF";
     142             :         const char *dp;
     143             :         const char *p;
     144           0 :         size_t total = 0;
     145             :         int n0, n1;
     146             : 
     147           0 :         p = str;
     148           0 :         if (p[0] != '0' && (p[1] != 'x' && p[1] != 'X')) {
     149           0 :                 return -1;
     150             :         }
     151           0 :         p += 2;
     152             : 
     153           0 :         while (p[0] != '\0' && p[1] != '\0') {
     154           0 :                 if (total >= data_len) {
     155           0 :                         return -1;
     156             :                 }
     157           0 :                 dp = strchr(digits, toupper((int) p[0]));
     158           0 :                 if (dp == NULL) {
     159           0 :                         return -1;
     160             :                 }
     161           0 :                 n0 = (int)(dp - digits);
     162           0 :                 dp = strchr(digits, toupper((int) p[1]));
     163           0 :                 if (dp == NULL) {
     164           0 :                         return -1;
     165             :                 }
     166           0 :                 n1 = (int)(dp - digits);
     167             : 
     168           0 :                 data[total] = (uint8_t)(((n0 & 0x0fU) << 4) | (n1 & 0x0fU));
     169           0 :                 total++;
     170           0 :                 p += 2;
     171             :         }
     172           0 :         return total;
     173             : }
     174             : 
     175             : static int
     176          14 : iscsi_reject(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu,
     177             :              int reason)
     178             : {
     179             :         struct spdk_iscsi_pdu *rsp_pdu;
     180             :         struct iscsi_bhs_reject *rsph;
     181             :         uint8_t *data;
     182             :         int total_ahs_len;
     183             :         int data_len;
     184             :         int alloc_len;
     185             : 
     186          14 :         pdu->is_rejected = true;
     187             : 
     188          14 :         total_ahs_len = pdu->bhs.total_ahs_len;
     189          14 :         data_len = 0;
     190          14 :         alloc_len = ISCSI_BHS_LEN + (4 * total_ahs_len);
     191             : 
     192          14 :         if (conn->header_digest) {
     193           0 :                 alloc_len += ISCSI_DIGEST_LEN;
     194             :         }
     195             : 
     196          14 :         data = calloc(1, alloc_len);
     197          14 :         if (!data) {
     198           0 :                 SPDK_ERRLOG("calloc() failed for data segment\n");
     199           0 :                 return -ENOMEM;
     200             :         }
     201             : 
     202          14 :         SPDK_DEBUGLOG(iscsi, "Reject PDU reason=%d\n", reason);
     203             : 
     204          14 :         if (conn->sess != NULL) {
     205          13 :                 SPDK_DEBUGLOG(iscsi,
     206             :                               "StatSN=%u, ExpCmdSN=%u, MaxCmdSN=%u\n",
     207             :                               conn->StatSN, conn->sess->ExpCmdSN,
     208             :                               conn->sess->MaxCmdSN);
     209             :         } else {
     210           1 :                 SPDK_DEBUGLOG(iscsi, "StatSN=%u\n", conn->StatSN);
     211             :         }
     212             : 
     213          14 :         memcpy(data, &pdu->bhs, ISCSI_BHS_LEN);
     214          14 :         data_len += ISCSI_BHS_LEN;
     215             : 
     216          14 :         if (total_ahs_len != 0) {
     217           0 :                 total_ahs_len = spdk_min((4 * total_ahs_len), ISCSI_AHS_LEN);
     218           0 :                 memcpy(data + data_len, pdu->ahs, total_ahs_len);
     219           0 :                 data_len += total_ahs_len;
     220             :         }
     221             : 
     222          14 :         if (conn->header_digest) {
     223           0 :                 memcpy(data + data_len, pdu->header_digest, ISCSI_DIGEST_LEN);
     224           0 :                 data_len += ISCSI_DIGEST_LEN;
     225             :         }
     226             : 
     227          14 :         rsp_pdu = iscsi_get_pdu(conn);
     228          14 :         if (rsp_pdu == NULL) {
     229           0 :                 free(data);
     230           0 :                 return -ENOMEM;
     231             :         }
     232             : 
     233          14 :         rsph = (struct iscsi_bhs_reject *)&rsp_pdu->bhs;
     234          14 :         rsp_pdu->data = data;
     235          14 :         rsph->opcode = ISCSI_OP_REJECT;
     236          14 :         rsph->flags |= 0x80; /* bit 0 is default to 1 */
     237          14 :         rsph->reason = reason;
     238          14 :         DSET24(rsph->data_segment_len, data_len);
     239             : 
     240          14 :         rsph->ffffffff = 0xffffffffU;
     241          14 :         to_be32(&rsph->stat_sn, conn->StatSN);
     242          14 :         conn->StatSN++;
     243             : 
     244          14 :         if (conn->sess != NULL) {
     245          13 :                 to_be32(&rsph->exp_cmd_sn, conn->sess->ExpCmdSN);
     246          13 :                 to_be32(&rsph->max_cmd_sn, conn->sess->MaxCmdSN);
     247             :         } else {
     248           1 :                 to_be32(&rsph->exp_cmd_sn, 1);
     249           1 :                 to_be32(&rsph->max_cmd_sn, 1);
     250             :         }
     251             : 
     252          14 :         SPDK_LOGDUMP(iscsi, "PDU", (void *)&rsp_pdu->bhs, ISCSI_BHS_LEN);
     253             : 
     254          14 :         iscsi_conn_write_pdu(conn, rsp_pdu, iscsi_conn_pdu_generic_complete, NULL);
     255             : 
     256          14 :         return 0;
     257             : }
     258             : 
     259             : uint32_t
     260           0 : iscsi_pdu_calc_header_digest(struct spdk_iscsi_pdu *pdu)
     261             : {
     262             :         uint32_t crc32c;
     263           0 :         uint32_t ahs_len_bytes = pdu->bhs.total_ahs_len * 4;
     264             : 
     265           0 :         crc32c = SPDK_CRC32C_INITIAL;
     266           0 :         crc32c = spdk_crc32c_update(&pdu->bhs, ISCSI_BHS_LEN, crc32c);
     267             : 
     268           0 :         if (ahs_len_bytes) {
     269           0 :                 crc32c = spdk_crc32c_update(pdu->ahs, ahs_len_bytes, crc32c);
     270             :         }
     271             : 
     272             :         /* BHS and AHS are always 4-byte multiples in length, so no padding is necessary. */
     273             : 
     274             :         /* Finalize CRC by inverting all bits. */
     275           0 :         return crc32c ^ SPDK_CRC32C_XOR;
     276             : }
     277             : 
     278             : /* Calculate CRC for each partial data segment. */
     279             : static void
     280           1 : iscsi_pdu_calc_partial_data_digest(struct spdk_iscsi_pdu *pdu)
     281             : {
     282           1 :         struct iovec iov;
     283             :         uint32_t num_blocks;
     284             : 
     285           1 :         if (spdk_likely(!pdu->dif_insert_or_strip)) {
     286           1 :                 pdu->crc32c = spdk_crc32c_update(pdu->data,
     287           1 :                                                  pdu->data_valid_bytes - pdu->data_offset,
     288             :                                                  pdu->crc32c);
     289             :         } else {
     290           0 :                 iov.iov_base = pdu->data;
     291           0 :                 iov.iov_len = pdu->data_buf_len;
     292           0 :                 num_blocks = pdu->data_buf_len / pdu->dif_ctx.block_size;
     293             : 
     294           0 :                 spdk_dif_update_crc32c(&iov, 1, num_blocks, &pdu->crc32c, &pdu->dif_ctx);
     295             :         }
     296           1 : }
     297             : 
     298             : static uint32_t
     299           1 : iscsi_pdu_calc_partial_data_digest_done(struct spdk_iscsi_pdu *pdu)
     300             : {
     301           1 :         uint32_t crc32c = pdu->crc32c;
     302             :         uint32_t mod;
     303             : 
     304             :         /* Include padding bytes into CRC if any. */
     305           1 :         mod = pdu->data_valid_bytes % ISCSI_ALIGNMENT;
     306           1 :         if (mod != 0) {
     307           0 :                 uint32_t pad_length = ISCSI_ALIGNMENT - mod;
     308           0 :                 uint8_t pad[3] = {0, 0, 0};
     309             : 
     310           0 :                 assert(pad_length > 0);
     311           0 :                 assert(pad_length <= sizeof(pad));
     312           0 :                 crc32c = spdk_crc32c_update(pad, pad_length, crc32c);
     313             :         }
     314             : 
     315             :         /* Finalize CRC by inverting all bits. */
     316           1 :         return crc32c ^ SPDK_CRC32C_XOR;
     317             : }
     318             : 
     319             : uint32_t
     320           0 : iscsi_pdu_calc_data_digest(struct spdk_iscsi_pdu *pdu)
     321             : {
     322           0 :         uint32_t data_len = DGET24(pdu->bhs.data_segment_len);
     323           0 :         uint32_t crc32c;
     324             :         uint32_t mod;
     325           0 :         struct iovec iov;
     326             :         uint32_t num_blocks;
     327             : 
     328             :         /* Initialize CRC. */
     329           0 :         crc32c = SPDK_CRC32C_INITIAL;
     330             : 
     331             :         /* Calculate CRC for the whole data segment. */
     332           0 :         if (spdk_likely(!pdu->dif_insert_or_strip)) {
     333           0 :                 crc32c = spdk_crc32c_update(pdu->data, data_len, crc32c);
     334             :         } else {
     335           0 :                 iov.iov_base = pdu->data;
     336           0 :                 iov.iov_len = pdu->data_buf_len;
     337           0 :                 num_blocks = pdu->data_buf_len / pdu->dif_ctx.block_size;
     338             : 
     339           0 :                 spdk_dif_update_crc32c(&iov, 1, num_blocks, &crc32c, &pdu->dif_ctx);
     340             :         }
     341             : 
     342             :         /* Include padding bytes into CRC if any. */
     343           0 :         mod = data_len % ISCSI_ALIGNMENT;
     344           0 :         if (mod != 0) {
     345           0 :                 uint32_t pad_length = ISCSI_ALIGNMENT - mod;
     346           0 :                 uint8_t pad[3] = {0, 0, 0};
     347           0 :                 assert(pad_length > 0);
     348           0 :                 assert(pad_length <= sizeof(pad));
     349           0 :                 crc32c = spdk_crc32c_update(pad, pad_length, crc32c);
     350             :         }
     351             : 
     352             :         /* Finalize CRC by inverting all bits. */
     353           0 :         return crc32c ^ SPDK_CRC32C_XOR;
     354             : }
     355             : 
     356             : static int
     357          17 : iscsi_conn_read_data_segment(struct spdk_iscsi_conn *conn,
     358             :                              struct spdk_iscsi_pdu *pdu,
     359             :                              uint32_t data_offset, uint32_t data_len)
     360             : {
     361          17 :         struct iovec buf_iov, iovs[32];
     362             :         int rc, _rc;
     363             : 
     364          17 :         if (spdk_likely(!pdu->dif_insert_or_strip)) {
     365          17 :                 return iscsi_conn_read_data(conn, data_len, pdu->data + data_offset);
     366             :         } else {
     367           0 :                 buf_iov.iov_base = pdu->data;
     368           0 :                 buf_iov.iov_len = pdu->data_buf_len;
     369           0 :                 rc = spdk_dif_set_md_interleave_iovs(iovs, 32, &buf_iov, 1,
     370             :                                                      data_offset, data_len, NULL,
     371           0 :                                                      &pdu->dif_ctx);
     372           0 :                 if (rc > 0) {
     373           0 :                         rc = iscsi_conn_readv_data(conn, iovs, rc);
     374           0 :                         if (rc > 0) {
     375           0 :                                 _rc = spdk_dif_generate_stream(&buf_iov, 1, data_offset, rc,
     376             :                                                                &pdu->dif_ctx);
     377           0 :                                 if (_rc != 0) {
     378           0 :                                         SPDK_ERRLOG("DIF generate failed\n");
     379           0 :                                         rc = _rc;
     380             :                                 }
     381             :                         }
     382             :                 } else {
     383           0 :                         SPDK_ERRLOG("Setup iovs for interleaved metadata failed\n");
     384             :                 }
     385           0 :                 return rc;
     386             :         }
     387             : }
     388             : 
     389             : /* Build iovec array to leave metadata space for every data block
     390             :  * when reading data segment from socket.
     391             :  */
     392             : static inline bool
     393           4 : _iscsi_sgl_append_with_md(struct spdk_iov_sgl *s,
     394             :                           void *buf, uint32_t buf_len, uint32_t data_len,
     395             :                           struct spdk_dif_ctx *dif_ctx)
     396             : {
     397             :         int rc;
     398           4 :         uint32_t total_size = 0;
     399           4 :         struct iovec buf_iov;
     400             : 
     401           4 :         if (s->iov_offset >= data_len) {
     402           1 :                 s->iov_offset -= data_len;
     403             :         } else {
     404           3 :                 buf_iov.iov_base = buf;
     405           3 :                 buf_iov.iov_len = buf_len;
     406           3 :                 rc = spdk_dif_set_md_interleave_iovs(s->iov, s->iovcnt, &buf_iov, 1,
     407           3 :                                                      s->iov_offset, data_len - s->iov_offset,
     408             :                                                      &total_size, dif_ctx);
     409           3 :                 if (rc < 0) {
     410           0 :                         SPDK_ERRLOG("Failed to setup iovs for DIF strip\n");
     411           0 :                         return false;
     412             :                 }
     413             : 
     414           3 :                 s->total_size += total_size;
     415           3 :                 s->iov_offset = 0;
     416           3 :                 assert(s->iovcnt >= rc);
     417           3 :                 s->iovcnt -= rc;
     418           3 :                 s->iov += rc;
     419             : 
     420           3 :                 if (s->iovcnt == 0) {
     421           1 :                         return false;
     422             :                 }
     423             :         }
     424             : 
     425           3 :         return true;
     426             : }
     427             : 
     428             : int
     429          16 : iscsi_build_iovs(struct spdk_iscsi_conn *conn, struct iovec *iovs, int iovcnt,
     430             :                  struct spdk_iscsi_pdu *pdu, uint32_t *_mapped_length)
     431             : {
     432          16 :         struct spdk_iov_sgl sgl;
     433             :         int enable_digest;
     434             :         uint32_t total_ahs_len;
     435             :         uint32_t data_len;
     436             : 
     437          16 :         if (iovcnt == 0) {
     438           0 :                 return 0;
     439             :         }
     440             : 
     441          16 :         total_ahs_len = pdu->bhs.total_ahs_len;
     442          16 :         data_len = DGET24(pdu->bhs.data_segment_len);
     443          16 :         data_len = ISCSI_ALIGN(data_len);
     444             : 
     445          16 :         enable_digest = 1;
     446          16 :         if (pdu->bhs.opcode == ISCSI_OP_LOGIN_RSP) {
     447             :                 /* this PDU should be sent without digest */
     448           0 :                 enable_digest = 0;
     449             :         }
     450             : 
     451          16 :         spdk_iov_sgl_init(&sgl, iovs, iovcnt, pdu->writev_offset);
     452             : 
     453             :         /* BHS */
     454          16 :         if (!spdk_iov_sgl_append(&sgl, (uint8_t *)&pdu->bhs, ISCSI_BHS_LEN)) {
     455           1 :                 goto end;
     456             :         }
     457             :         /* AHS */
     458          15 :         if (total_ahs_len > 0) {
     459           0 :                 if (!spdk_iov_sgl_append(&sgl, pdu->ahs, 4 * total_ahs_len)) {
     460           0 :                         goto end;
     461             :                 }
     462             :         }
     463             : 
     464             :         /* Header Digest */
     465          15 :         if (enable_digest && conn->header_digest) {
     466          15 :                 if (!spdk_iov_sgl_append(&sgl, pdu->header_digest, ISCSI_DIGEST_LEN)) {
     467           1 :                         goto end;
     468             :                 }
     469             :         }
     470             : 
     471             :         /* Data Segment */
     472          14 :         if (data_len > 0) {
     473          14 :                 if (!pdu->dif_insert_or_strip) {
     474          10 :                         if (!spdk_iov_sgl_append(&sgl, pdu->data, data_len)) {
     475           1 :                                 goto end;
     476             :                         }
     477             :                 } else {
     478           4 :                         if (!_iscsi_sgl_append_with_md(&sgl, pdu->data, pdu->data_buf_len,
     479             :                                                        data_len, &pdu->dif_ctx)) {
     480           1 :                                 goto end;
     481             :                         }
     482             :                 }
     483             :         }
     484             : 
     485             :         /* Data Digest */
     486          12 :         if (enable_digest && conn->data_digest && data_len != 0) {
     487          12 :                 spdk_iov_sgl_append(&sgl, pdu->data_digest, ISCSI_DIGEST_LEN);
     488             :         }
     489             : 
     490          16 : end:
     491          16 :         if (_mapped_length != NULL) {
     492          16 :                 *_mapped_length = sgl.total_size;
     493             :         }
     494             : 
     495          16 :         return iovcnt - sgl.iovcnt;
     496             : }
     497             : 
     498             : void
     499           0 : iscsi_free_sess(struct spdk_iscsi_sess *sess)
     500             : {
     501           0 :         if (sess == NULL) {
     502           0 :                 return;
     503             :         }
     504             : 
     505           0 :         sess->tag = 0;
     506           0 :         sess->target = NULL;
     507           0 :         sess->session_type = SESSION_TYPE_INVALID;
     508           0 :         iscsi_param_free(sess->params);
     509           0 :         free(sess->conns);
     510           0 :         spdk_scsi_port_free(&sess->initiator_port);
     511           0 :         spdk_mempool_put(g_iscsi.session_pool, (void *)sess);
     512             : }
     513             : 
     514             : static int
     515           0 : create_iscsi_sess(struct spdk_iscsi_conn *conn,
     516             :                   struct spdk_iscsi_tgt_node *target,
     517             :                   enum session_type session_type)
     518             : {
     519             :         struct spdk_iscsi_sess *sess;
     520             :         int rc;
     521             : 
     522           0 :         sess = spdk_mempool_get(g_iscsi.session_pool);
     523           0 :         if (!sess) {
     524           0 :                 SPDK_ERRLOG("Unable to get session object\n");
     525           0 :                 SPDK_ERRLOG("MaxSessions set to %d\n", g_iscsi.MaxSessions);
     526           0 :                 return -ENOMEM;
     527             :         }
     528             : 
     529             :         /* configuration values */
     530           0 :         pthread_mutex_lock(&g_iscsi.mutex);
     531             : 
     532           0 :         sess->MaxConnections = g_iscsi.MaxConnectionsPerSession;
     533           0 :         sess->MaxOutstandingR2T = DEFAULT_MAXOUTSTANDINGR2T;
     534             : 
     535           0 :         sess->DefaultTime2Wait = g_iscsi.DefaultTime2Wait;
     536           0 :         sess->DefaultTime2Retain = g_iscsi.DefaultTime2Retain;
     537           0 :         sess->FirstBurstLength = g_iscsi.FirstBurstLength;
     538           0 :         sess->MaxBurstLength = SPDK_ISCSI_MAX_BURST_LENGTH;
     539           0 :         sess->InitialR2T = DEFAULT_INITIALR2T;
     540           0 :         sess->ImmediateData = g_iscsi.ImmediateData;
     541           0 :         sess->DataPDUInOrder = DEFAULT_DATAPDUINORDER;
     542           0 :         sess->DataSequenceInOrder = DEFAULT_DATASEQUENCEINORDER;
     543           0 :         sess->ErrorRecoveryLevel = g_iscsi.ErrorRecoveryLevel;
     544             : 
     545           0 :         pthread_mutex_unlock(&g_iscsi.mutex);
     546             : 
     547           0 :         sess->tag = conn->pg_tag;
     548             : 
     549           0 :         sess->conns = calloc(sess->MaxConnections, sizeof(*sess->conns));
     550           0 :         if (!sess->conns) {
     551           0 :                 spdk_mempool_put(g_iscsi.session_pool, (void *)sess);
     552           0 :                 SPDK_ERRLOG("calloc() failed for connection array\n");
     553           0 :                 return -ENOMEM;
     554             :         }
     555             : 
     556           0 :         sess->connections = 0;
     557             : 
     558           0 :         sess->conns[sess->connections] = conn;
     559           0 :         sess->connections++;
     560             : 
     561           0 :         sess->params = NULL;
     562           0 :         sess->target = target;
     563           0 :         sess->isid = 0;
     564           0 :         sess->session_type = session_type;
     565           0 :         sess->current_text_itt = 0xffffffffU;
     566             : 
     567             :         /* set default params */
     568           0 :         rc = iscsi_sess_params_init(&sess->params);
     569           0 :         if (rc < 0) {
     570           0 :                 SPDK_ERRLOG("iscsi_sess_params_init() failed\n");
     571           0 :                 goto error_return;
     572             :         }
     573             :         /* replace with config value */
     574           0 :         rc = iscsi_param_set_int(sess->params, "MaxConnections",
     575             :                                  sess->MaxConnections);
     576           0 :         if (rc < 0) {
     577           0 :                 SPDK_ERRLOG("iscsi_param_set_int() failed\n");
     578           0 :                 goto error_return;
     579             :         }
     580             : 
     581           0 :         rc = iscsi_param_set_int(sess->params, "MaxOutstandingR2T",
     582             :                                  sess->MaxOutstandingR2T);
     583           0 :         if (rc < 0) {
     584           0 :                 SPDK_ERRLOG("iscsi_param_set_int() failed\n");
     585           0 :                 goto error_return;
     586             :         }
     587             : 
     588           0 :         rc = iscsi_param_set_int(sess->params, "DefaultTime2Wait",
     589             :                                  sess->DefaultTime2Wait);
     590           0 :         if (rc < 0) {
     591           0 :                 SPDK_ERRLOG("iscsi_param_set_int() failed\n");
     592           0 :                 goto error_return;
     593             :         }
     594             : 
     595           0 :         rc = iscsi_param_set_int(sess->params, "DefaultTime2Retain",
     596             :                                  sess->DefaultTime2Retain);
     597           0 :         if (rc < 0) {
     598           0 :                 SPDK_ERRLOG("iscsi_param_set_int() failed\n");
     599           0 :                 goto error_return;
     600             :         }
     601             : 
     602           0 :         rc = iscsi_param_set_int(sess->params, "FirstBurstLength",
     603             :                                  sess->FirstBurstLength);
     604           0 :         if (rc < 0) {
     605           0 :                 SPDK_ERRLOG("iscsi_param_set_int() failed\n");
     606           0 :                 goto error_return;
     607             :         }
     608             : 
     609           0 :         rc = iscsi_param_set_int(sess->params, "MaxBurstLength",
     610             :                                  sess->MaxBurstLength);
     611           0 :         if (rc < 0) {
     612           0 :                 SPDK_ERRLOG("iscsi_param_set_int() failed\n");
     613           0 :                 goto error_return;
     614             :         }
     615             : 
     616           0 :         rc = iscsi_param_set(sess->params, "InitialR2T",
     617           0 :                              sess->InitialR2T ? "Yes" : "No");
     618           0 :         if (rc < 0) {
     619           0 :                 SPDK_ERRLOG("iscsi_param_set() failed\n");
     620           0 :                 goto error_return;
     621             :         }
     622             : 
     623           0 :         rc = iscsi_param_set(sess->params, "ImmediateData",
     624           0 :                              sess->ImmediateData ? "Yes" : "No");
     625           0 :         if (rc < 0) {
     626           0 :                 SPDK_ERRLOG("iscsi_param_set() failed\n");
     627           0 :                 goto error_return;
     628             :         }
     629             : 
     630           0 :         rc = iscsi_param_set(sess->params, "DataPDUInOrder",
     631           0 :                              sess->DataPDUInOrder ? "Yes" : "No");
     632           0 :         if (rc < 0) {
     633           0 :                 SPDK_ERRLOG("iscsi_param_set() failed\n");
     634           0 :                 goto error_return;
     635             :         }
     636             : 
     637           0 :         rc = iscsi_param_set(sess->params, "DataSequenceInOrder",
     638           0 :                              sess->DataSequenceInOrder ? "Yes" : "No");
     639           0 :         if (rc < 0) {
     640           0 :                 SPDK_ERRLOG("iscsi_param_set() failed\n");
     641           0 :                 goto error_return;
     642             :         }
     643             : 
     644           0 :         rc = iscsi_param_set_int(sess->params, "ErrorRecoveryLevel",
     645             :                                  sess->ErrorRecoveryLevel);
     646           0 :         if (rc < 0) {
     647           0 :                 SPDK_ERRLOG("iscsi_param_set_int() failed\n");
     648           0 :                 goto error_return;
     649             :         }
     650             : 
     651             :         /* realloc buffer */
     652           0 :         rc = iscsi_param_set_int(conn->params, "MaxRecvDataSegmentLength",
     653           0 :                                  conn->MaxRecvDataSegmentLength);
     654           0 :         if (rc < 0) {
     655           0 :                 SPDK_ERRLOG("iscsi_param_set_int() failed\n");
     656           0 :                 goto error_return;
     657             :         }
     658             : 
     659             :         /* sess for first connection of session */
     660           0 :         conn->sess = sess;
     661           0 :         return 0;
     662             : 
     663           0 : error_return:
     664           0 :         iscsi_free_sess(sess);
     665           0 :         conn->sess = NULL;
     666           0 :         return -1;
     667             : }
     668             : 
     669             : static struct spdk_iscsi_sess *
     670           2 : get_iscsi_sess_by_tsih(uint16_t tsih)
     671             : {
     672             :         struct spdk_iscsi_sess *session;
     673             : 
     674           2 :         if (tsih == 0 || tsih > g_iscsi.MaxSessions) {
     675           1 :                 return NULL;
     676             :         }
     677             : 
     678           1 :         session = g_iscsi.session[tsih - 1];
     679           1 :         assert(tsih == session->tsih);
     680             : 
     681           1 :         return session;
     682             : }
     683             : 
     684             : static uint8_t
     685           2 : append_iscsi_sess(struct spdk_iscsi_conn *conn,
     686             :                   const char *initiator_port_name, uint16_t tsih, uint16_t cid)
     687             : {
     688             :         struct spdk_iscsi_sess *sess;
     689             : 
     690           2 :         SPDK_DEBUGLOG(iscsi, "append session: init port name=%s, tsih=%u, cid=%u\n",
     691             :                       initiator_port_name, tsih, cid);
     692             : 
     693           2 :         sess = get_iscsi_sess_by_tsih(tsih);
     694           2 :         if (sess == NULL) {
     695           1 :                 SPDK_ERRLOG("spdk_get_iscsi_sess_by_tsih failed\n");
     696           1 :                 return ISCSI_LOGIN_CONN_ADD_FAIL;
     697             :         }
     698           1 :         if ((conn->pg_tag != sess->tag) ||
     699           0 :             (strcasecmp(initiator_port_name, spdk_scsi_port_get_name(sess->initiator_port)) != 0) ||
     700           0 :             (conn->target != sess->target)) {
     701             :                 /* no match */
     702           1 :                 SPDK_ERRLOG("no MCS session for init port name=%s, tsih=%d, cid=%d\n",
     703             :                             initiator_port_name, tsih, cid);
     704           1 :                 return ISCSI_LOGIN_CONN_ADD_FAIL;
     705             :         }
     706             : 
     707           0 :         if (sess->connections >= sess->MaxConnections) {
     708             :                 /* no slot for connection */
     709           0 :                 SPDK_ERRLOG("too many connections for init port name=%s, tsih=%d, cid=%d\n",
     710             :                             initiator_port_name, tsih, cid);
     711           0 :                 return ISCSI_LOGIN_TOO_MANY_CONNECTIONS;
     712             :         }
     713             : 
     714           0 :         SPDK_DEBUGLOG(iscsi, "Connections (tsih %d): %d\n", sess->tsih, sess->connections);
     715           0 :         conn->sess = sess;
     716             : 
     717             :         /*
     718             :          * TODO: need a mutex or other sync mechanism to protect the session's
     719             :          *  connection list.
     720             :          */
     721           0 :         sess->conns[sess->connections] = conn;
     722           0 :         sess->connections++;
     723             : 
     724           0 :         return 0;
     725             : }
     726             : 
     727             : static int
     728           0 : iscsi_append_text(const char *key, const char *val, uint8_t *data,
     729             :                   int alloc_len, int data_len)
     730             : {
     731             :         int total;
     732             :         int len;
     733             : 
     734           0 :         total = data_len;
     735           0 :         if (alloc_len < 1) {
     736           0 :                 return 0;
     737             :         }
     738           0 :         if (total > alloc_len) {
     739           0 :                 total = alloc_len;
     740           0 :                 data[total - 1] = '\0';
     741           0 :                 return total;
     742             :         }
     743             : 
     744           0 :         if (alloc_len - total < 1) {
     745           0 :                 SPDK_ERRLOG("data space small %d\n", alloc_len);
     746           0 :                 return total;
     747             :         }
     748           0 :         len = snprintf((char *) data + total, alloc_len - total, "%s=%s", key, val);
     749           0 :         total += len + 1;
     750             : 
     751           0 :         return total;
     752             : }
     753             : 
     754             : static int
     755           0 : iscsi_append_param(struct spdk_iscsi_conn *conn, const char *key,
     756             :                    uint8_t *data, int alloc_len, int data_len)
     757             : {
     758             :         struct iscsi_param *param;
     759             : 
     760           0 :         param = iscsi_param_find(conn->params, key);
     761           0 :         if (param == NULL) {
     762           0 :                 param = iscsi_param_find(conn->sess->params, key);
     763           0 :                 if (param == NULL) {
     764           0 :                         SPDK_DEBUGLOG(iscsi, "no key %.64s\n", key);
     765           0 :                         return data_len;
     766             :                 }
     767             :         }
     768           0 :         return iscsi_append_text(param->key, param->val, data,
     769             :                                  alloc_len, data_len);
     770             : }
     771             : 
     772             : static int
     773           0 : iscsi_auth_params(struct spdk_iscsi_conn *conn,
     774             :                   struct iscsi_param *params, const char *method, uint8_t *data,
     775             :                   int alloc_len, int data_len)
     776             : {
     777             :         char *in_val;
     778           0 :         char *in_next;
     779             :         char *new_val;
     780             :         const char *algorithm;
     781             :         const char *name;
     782             :         const char *response;
     783             :         const char *identifier;
     784             :         const char *challenge;
     785             :         int total;
     786             :         int rc;
     787             : 
     788           0 :         if (conn == NULL || params == NULL || method == NULL) {
     789           0 :                 return -1;
     790             :         }
     791           0 :         if (strcasecmp(method, "CHAP") == 0) {
     792             :                 /* method OK */
     793             :         } else {
     794           0 :                 SPDK_ERRLOG("unsupported AuthMethod %.64s\n", method);
     795           0 :                 return -1;
     796             :         }
     797             : 
     798           0 :         total = data_len;
     799           0 :         if (alloc_len < 1) {
     800           0 :                 return 0;
     801             :         }
     802           0 :         if (total > alloc_len) {
     803           0 :                 total = alloc_len;
     804           0 :                 data[total - 1] = '\0';
     805           0 :                 return total;
     806             :         }
     807             : 
     808             :         /* for temporary store */
     809           0 :         in_val = malloc(ISCSI_TEXT_MAX_VAL_LEN + 1);
     810           0 :         if (!in_val) {
     811           0 :                 SPDK_ERRLOG("malloc() failed for temporary store\n");
     812           0 :                 return -ENOMEM;
     813             :         }
     814             : 
     815             :         /* CHAP method (RFC1994) */
     816           0 :         if ((algorithm = iscsi_param_get_val(params, "CHAP_A")) != NULL) {
     817           0 :                 if (conn->auth.chap_phase != ISCSI_CHAP_PHASE_WAIT_A) {
     818           0 :                         SPDK_ERRLOG("CHAP sequence error\n");
     819           0 :                         goto error_return;
     820             :                 }
     821             : 
     822             :                 /* CHAP_A is LIST type */
     823           0 :                 snprintf(in_val, ISCSI_TEXT_MAX_VAL_LEN + 1, "%s", algorithm);
     824           0 :                 in_next = in_val;
     825           0 :                 while ((new_val = spdk_strsepq(&in_next, ",")) != NULL) {
     826           0 :                         if (strcasecmp(new_val, "5") == 0) {
     827             :                                 /* CHAP with MD5 */
     828           0 :                                 break;
     829             :                         }
     830             :                 }
     831           0 :                 if (new_val == NULL) {
     832           0 :                         snprintf(in_val, ISCSI_TEXT_MAX_VAL_LEN + 1, "%s", "Reject");
     833           0 :                         new_val = in_val;
     834           0 :                         iscsi_append_text("CHAP_A", new_val, data, alloc_len, total);
     835           0 :                         goto error_return;
     836             :                 }
     837             :                 /* selected algorithm is 5 (MD5) */
     838           0 :                 SPDK_DEBUGLOG(iscsi, "got CHAP_A=%s\n", new_val);
     839           0 :                 total = iscsi_append_text("CHAP_A", new_val, data, alloc_len, total);
     840             : 
     841             :                 /* Identifier is one octet */
     842           0 :                 gen_random(conn->auth.chap_id, 1);
     843           0 :                 snprintf(in_val, ISCSI_TEXT_MAX_VAL_LEN, "%d",
     844           0 :                          (int) conn->auth.chap_id[0]);
     845           0 :                 total = iscsi_append_text("CHAP_I", in_val, data, alloc_len, total);
     846             : 
     847             :                 /* Challenge Value is a variable stream of octets */
     848             :                 /* (binary length MUST not exceed 1024 bytes) */
     849           0 :                 conn->auth.chap_challenge_len = ISCSI_CHAP_CHALLENGE_LEN;
     850           0 :                 gen_random(conn->auth.chap_challenge, conn->auth.chap_challenge_len);
     851           0 :                 bin2hex(in_val, ISCSI_TEXT_MAX_VAL_LEN,
     852           0 :                         conn->auth.chap_challenge, conn->auth.chap_challenge_len);
     853           0 :                 total = iscsi_append_text("CHAP_C", in_val, data, alloc_len, total);
     854             : 
     855           0 :                 conn->auth.chap_phase = ISCSI_CHAP_PHASE_WAIT_NR;
     856           0 :         } else if ((name = iscsi_param_get_val(params, "CHAP_N")) != NULL) {
     857           0 :                 uint8_t resmd5[SPDK_MD5DIGEST_LEN];
     858           0 :                 uint8_t tgtmd5[SPDK_MD5DIGEST_LEN];
     859           0 :                 struct spdk_md5ctx md5ctx;
     860           0 :                 size_t decoded_len = 0;
     861             : 
     862           0 :                 if (conn->auth.chap_phase != ISCSI_CHAP_PHASE_WAIT_NR) {
     863           0 :                         SPDK_ERRLOG("CHAP sequence error\n");
     864           0 :                         goto error_return;
     865             :                 }
     866             : 
     867           0 :                 response = iscsi_param_get_val(params, "CHAP_R");
     868           0 :                 if (response == NULL) {
     869           0 :                         SPDK_ERRLOG("no response\n");
     870           0 :                         goto error_return;
     871             :                 }
     872           0 :                 if (response[0] == '0' &&
     873           0 :                     (response[1] == 'x' || response[1] == 'X')) {
     874           0 :                         rc = hex2bin(resmd5, SPDK_MD5DIGEST_LEN, response);
     875           0 :                         if (rc < 0 || rc != SPDK_MD5DIGEST_LEN) {
     876           0 :                                 SPDK_ERRLOG("response format error\n");
     877           0 :                                 goto error_return;
     878             :                         }
     879           0 :                 } else if (response[0] == '0' &&
     880           0 :                            (response[1] == 'b' || response[1] == 'B')) {
     881           0 :                         response += 2;
     882           0 :                         rc = spdk_base64_decode(resmd5, &decoded_len, response);
     883           0 :                         if (rc < 0 || decoded_len != SPDK_MD5DIGEST_LEN) {
     884           0 :                                 SPDK_ERRLOG("response format error\n");
     885           0 :                                 goto error_return;
     886             :                         }
     887             :                 } else {
     888           0 :                         SPDK_ERRLOG("response format error\n");
     889           0 :                         goto error_return;
     890             :                 }
     891           0 :                 SPDK_DEBUGLOG(iscsi, "got CHAP_N/CHAP_R\n");
     892             : 
     893           0 :                 SPDK_DEBUGLOG(iscsi, "ag_tag=%d\n", conn->chap_group);
     894             : 
     895           0 :                 rc = iscsi_chap_get_authinfo(&conn->auth, name, conn->chap_group);
     896           0 :                 if (rc < 0) {
     897             :                         /* SPDK_ERRLOG("auth user or secret is missing\n"); */
     898           0 :                         SPDK_ERRLOG("iscsi_chap_get_authinfo() failed\n");
     899           0 :                         goto error_return;
     900             :                 }
     901           0 :                 if (conn->auth.user[0] == '\0' || conn->auth.secret[0] == '\0') {
     902             :                         /* SPDK_ERRLOG("auth user or secret is missing\n"); */
     903           0 :                         SPDK_ERRLOG("auth failed (name %.64s)\n", name);
     904           0 :                         goto error_return;
     905             :                 }
     906             : 
     907           0 :                 md5init(&md5ctx);
     908             :                 /* Identifier */
     909           0 :                 md5update(&md5ctx, conn->auth.chap_id, 1);
     910             :                 /* followed by secret */
     911           0 :                 md5update(&md5ctx, conn->auth.secret,
     912           0 :                           strlen(conn->auth.secret));
     913             :                 /* followed by Challenge Value */
     914           0 :                 md5update(&md5ctx, conn->auth.chap_challenge,
     915           0 :                           conn->auth.chap_challenge_len);
     916             :                 /* tgtmd5 is expecting Response Value */
     917           0 :                 md5final(tgtmd5, &md5ctx);
     918             : 
     919           0 :                 bin2hex(in_val, ISCSI_TEXT_MAX_VAL_LEN, tgtmd5, SPDK_MD5DIGEST_LEN);
     920             : 
     921             : #if 0
     922             :                 SPDK_DEBUGLOG(iscsi, "tgtmd5=%s, resmd5=%s\n", in_val, response);
     923             :                 spdk_dump("tgtmd5", tgtmd5, SPDK_MD5DIGEST_LEN);
     924             :                 spdk_dump("resmd5", resmd5, SPDK_MD5DIGEST_LEN);
     925             : #endif
     926             : 
     927             :                 /* compare MD5 digest */
     928           0 :                 if (memcmp(tgtmd5, resmd5, SPDK_MD5DIGEST_LEN) != 0) {
     929             :                         /* not match */
     930             :                         /* SPDK_ERRLOG("auth user or secret is missing\n"); */
     931           0 :                         SPDK_ERRLOG("auth failed (name %.64s)\n", name);
     932           0 :                         goto error_return;
     933             :                 }
     934             :                 /* OK initiator's secret */
     935           0 :                 conn->authenticated = true;
     936             : 
     937             :                 /* mutual CHAP? */
     938           0 :                 identifier = iscsi_param_get_val(params, "CHAP_I");
     939           0 :                 if (identifier != NULL) {
     940           0 :                         conn->auth.chap_mid[0] = (uint8_t) strtol(identifier, NULL, 10);
     941           0 :                         challenge = iscsi_param_get_val(params, "CHAP_C");
     942           0 :                         if (challenge == NULL) {
     943           0 :                                 SPDK_ERRLOG("CHAP sequence error\n");
     944           0 :                                 goto error_return;
     945             :                         }
     946           0 :                         if (challenge[0] == '0' &&
     947           0 :                             (challenge[1] == 'x' || challenge[1] == 'X')) {
     948           0 :                                 rc = hex2bin(conn->auth.chap_mchallenge,
     949             :                                              ISCSI_CHAP_CHALLENGE_LEN, challenge);
     950           0 :                                 if (rc < 0) {
     951           0 :                                         SPDK_ERRLOG("challenge format error\n");
     952           0 :                                         goto error_return;
     953             :                                 }
     954           0 :                                 conn->auth.chap_mchallenge_len = rc;
     955           0 :                         } else if (challenge[0] == '0' &&
     956           0 :                                    (challenge[1] == 'b' || challenge[1] == 'B')) {
     957           0 :                                 challenge += 2;
     958           0 :                                 rc = spdk_base64_decode(conn->auth.chap_mchallenge,
     959             :                                                         &decoded_len, challenge);
     960           0 :                                 if (rc < 0) {
     961           0 :                                         SPDK_ERRLOG("challenge format error\n");
     962           0 :                                         goto error_return;
     963             :                                 }
     964           0 :                                 conn->auth.chap_mchallenge_len = decoded_len;
     965             :                         } else {
     966           0 :                                 SPDK_ERRLOG("challenge format error\n");
     967           0 :                                 goto error_return;
     968             :                         }
     969             : #if 0
     970             :                         spdk_dump("MChallenge", conn->auth.chap_mchallenge,
     971             :                                   conn->auth.chap_mchallenge_len);
     972             : #endif
     973           0 :                         SPDK_DEBUGLOG(iscsi, "got CHAP_I/CHAP_C\n");
     974             : 
     975           0 :                         if (conn->auth.muser[0] == '\0' || conn->auth.msecret[0] == '\0') {
     976             :                                 /* SPDK_ERRLOG("mutual auth user or secret is missing\n"); */
     977           0 :                                 SPDK_ERRLOG("auth failed (name %.64s)\n", name);
     978           0 :                                 goto error_return;
     979             :                         }
     980             : 
     981           0 :                         md5init(&md5ctx);
     982             :                         /* Identifier */
     983           0 :                         md5update(&md5ctx, conn->auth.chap_mid, 1);
     984             :                         /* followed by secret */
     985           0 :                         md5update(&md5ctx, conn->auth.msecret,
     986           0 :                                   strlen(conn->auth.msecret));
     987             :                         /* followed by Challenge Value */
     988           0 :                         md5update(&md5ctx, conn->auth.chap_mchallenge,
     989           0 :                                   conn->auth.chap_mchallenge_len);
     990             :                         /* tgtmd5 is Response Value */
     991           0 :                         md5final(tgtmd5, &md5ctx);
     992             : 
     993           0 :                         bin2hex(in_val, ISCSI_TEXT_MAX_VAL_LEN, tgtmd5, SPDK_MD5DIGEST_LEN);
     994             : 
     995           0 :                         total = iscsi_append_text("CHAP_N", conn->auth.muser, data,
     996             :                                                   alloc_len, total);
     997           0 :                         total = iscsi_append_text("CHAP_R", in_val, data, alloc_len, total);
     998             :                 } else {
     999             :                         /* not mutual */
    1000           0 :                         if (conn->mutual_chap) {
    1001           0 :                                 SPDK_ERRLOG("required mutual CHAP\n");
    1002           0 :                                 goto error_return;
    1003             :                         }
    1004             :                 }
    1005             : 
    1006           0 :                 conn->auth.chap_phase = ISCSI_CHAP_PHASE_END;
    1007             :         } else {
    1008             :                 /* not found CHAP keys */
    1009           0 :                 SPDK_DEBUGLOG(iscsi, "start CHAP\n");
    1010           0 :                 conn->auth.chap_phase = ISCSI_CHAP_PHASE_WAIT_A;
    1011             :         }
    1012             : 
    1013           0 :         free(in_val);
    1014           0 :         return total;
    1015             : 
    1016           0 : error_return:
    1017           0 :         conn->auth.chap_phase = ISCSI_CHAP_PHASE_WAIT_A;
    1018           0 :         free(in_val);
    1019           0 :         return -1;
    1020             : }
    1021             : 
    1022             : static int
    1023           0 : iscsi_check_values(struct spdk_iscsi_conn *conn)
    1024             : {
    1025           0 :         if (conn->sess->FirstBurstLength > conn->sess->MaxBurstLength) {
    1026           0 :                 SPDK_ERRLOG("FirstBurstLength(%d) > MaxBurstLength(%d)\n",
    1027             :                             conn->sess->FirstBurstLength,
    1028             :                             conn->sess->MaxBurstLength);
    1029           0 :                 return -1;
    1030             :         }
    1031           0 :         if (conn->sess->FirstBurstLength > g_iscsi.FirstBurstLength) {
    1032           0 :                 SPDK_ERRLOG("FirstBurstLength(%d) > iSCSI target restriction(%d)\n",
    1033             :                             conn->sess->FirstBurstLength, g_iscsi.FirstBurstLength);
    1034           0 :                 return -1;
    1035             :         }
    1036           0 :         if (conn->sess->MaxBurstLength > 0x00ffffff) {
    1037           0 :                 SPDK_ERRLOG("MaxBurstLength(%d) > 0x00ffffff\n",
    1038             :                             conn->sess->MaxBurstLength);
    1039           0 :                 return -1;
    1040             :         }
    1041             : 
    1042           0 :         if (conn->MaxRecvDataSegmentLength < 512) {
    1043           0 :                 SPDK_ERRLOG("MaxRecvDataSegmentLength(%d) < 512\n",
    1044             :                             conn->MaxRecvDataSegmentLength);
    1045           0 :                 return -1;
    1046             :         }
    1047           0 :         if (conn->MaxRecvDataSegmentLength > 0x00ffffff) {
    1048           0 :                 SPDK_ERRLOG("MaxRecvDataSegmentLength(%d) > 0x00ffffff\n",
    1049             :                             conn->MaxRecvDataSegmentLength);
    1050           0 :                 return -1;
    1051             :         }
    1052           0 :         return 0;
    1053             : }
    1054             : 
    1055             : static int
    1056           0 : iscsi_conn_params_update(struct spdk_iscsi_conn *conn)
    1057             : {
    1058             :         int rc;
    1059             :         uint32_t recv_buf_size;
    1060             : 
    1061             :         /* update internal variables */
    1062           0 :         rc = iscsi_copy_param2var(conn);
    1063           0 :         if (rc < 0) {
    1064           0 :                 SPDK_ERRLOG("iscsi_copy_param2var() failed\n");
    1065           0 :                 if (conn->state < ISCSI_CONN_STATE_EXITING) {
    1066           0 :                         conn->state = ISCSI_CONN_STATE_EXITING;
    1067             :                 }
    1068           0 :                 return rc;
    1069             :         }
    1070             : 
    1071             :         /* check value */
    1072           0 :         rc = iscsi_check_values(conn);
    1073           0 :         if (rc < 0) {
    1074           0 :                 SPDK_ERRLOG("iscsi_check_values() failed\n");
    1075           0 :                 if (conn->state < ISCSI_CONN_STATE_EXITING) {
    1076           0 :                         conn->state = ISCSI_CONN_STATE_EXITING;
    1077             :                 }
    1078             :         }
    1079             : 
    1080           0 :         if (conn->sock == NULL) {
    1081           0 :                 SPDK_INFOLOG(iscsi, "socket is already closed.\n");
    1082           0 :                 return -ENXIO;
    1083             :         }
    1084             : 
    1085             :         /* The socket receive buffer may need to be adjusted based on the new parameters */
    1086             : 
    1087             :         /* Don't allow the recv buffer to be 0 or very large. */
    1088           0 :         recv_buf_size = spdk_max(0x1000, spdk_min(0x2000, conn->sess->FirstBurstLength));
    1089             : 
    1090             :         /* Add in extra space for the PDU */
    1091           0 :         recv_buf_size += ISCSI_BHS_LEN + ISCSI_AHS_LEN;
    1092             : 
    1093           0 :         if (conn->header_digest) {
    1094           0 :                 recv_buf_size += ISCSI_DIGEST_LEN;
    1095             :         }
    1096             : 
    1097           0 :         if (conn->data_digest) {
    1098           0 :                 recv_buf_size += ISCSI_DIGEST_LEN;
    1099             :         }
    1100             : 
    1101             :         /* Set up to buffer up to 4 commands with immediate data at once */
    1102           0 :         if (spdk_sock_set_recvbuf(conn->sock, recv_buf_size * 4) < 0) {
    1103             :                 /* Not fatal. */
    1104             :         }
    1105             : 
    1106           0 :         return rc;
    1107             : }
    1108             : 
    1109             : static void
    1110           0 : iscsi_conn_login_pdu_err_complete(void *arg)
    1111             : {
    1112           0 :         struct spdk_iscsi_conn *conn = arg;
    1113             : 
    1114           0 :         if (conn->full_feature) {
    1115           0 :                 iscsi_conn_params_update(conn);
    1116             :         }
    1117           0 : }
    1118             : 
    1119             : static void
    1120           0 : iscsi_conn_login_pdu_success_complete(void *arg)
    1121             : {
    1122           0 :         struct spdk_iscsi_conn *conn = arg;
    1123             : 
    1124             : 
    1125           0 :         if (conn->state >= ISCSI_CONN_STATE_EXITING) {
    1126             :                 /* Connection is being exited before this callback is executed. */
    1127           0 :                 SPDK_DEBUGLOG(iscsi, "Connection is already exited.\n");
    1128           0 :                 return;
    1129             :         }
    1130           0 :         if (conn->full_feature) {
    1131           0 :                 if (iscsi_conn_params_update(conn) != 0) {
    1132           0 :                         return;
    1133             :                 }
    1134             :         }
    1135           0 :         if (conn->full_feature != 0) {
    1136           0 :                 iscsi_conn_schedule(conn);
    1137             :         }
    1138             : }
    1139             : 
    1140             : /*
    1141             :  * The response function of spdk_iscsi_op_login
    1142             :  */
    1143             : static void
    1144           3 : iscsi_op_login_response(struct spdk_iscsi_conn *conn,
    1145             :                         struct spdk_iscsi_pdu *rsp_pdu, struct iscsi_param *params,
    1146             :                         iscsi_conn_xfer_complete_cb cb_fn)
    1147             : {
    1148             :         struct iscsi_bhs_login_rsp *rsph;
    1149             : 
    1150           3 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    1151           3 :         rsph->version_max = ISCSI_VERSION;
    1152           3 :         rsph->version_act = ISCSI_VERSION;
    1153           3 :         DSET24(rsph->data_segment_len, rsp_pdu->data_segment_len);
    1154             : 
    1155           3 :         to_be32(&rsph->stat_sn, conn->StatSN);
    1156           3 :         conn->StatSN++;
    1157             : 
    1158           3 :         if (conn->sess != NULL) {
    1159           0 :                 to_be32(&rsph->exp_cmd_sn, conn->sess->ExpCmdSN);
    1160           0 :                 to_be32(&rsph->max_cmd_sn, conn->sess->MaxCmdSN);
    1161             :         } else {
    1162           3 :                 to_be32(&rsph->exp_cmd_sn, rsp_pdu->cmd_sn);
    1163           3 :                 to_be32(&rsph->max_cmd_sn, rsp_pdu->cmd_sn);
    1164             :         }
    1165             : 
    1166           3 :         SPDK_LOGDUMP(iscsi, "PDU", (uint8_t *)rsph, ISCSI_BHS_LEN);
    1167           3 :         SPDK_LOGDUMP(iscsi, "DATA", rsp_pdu->data, rsp_pdu->data_segment_len);
    1168             : 
    1169             :         /* Set T/CSG/NSG to reserved if login error. */
    1170           3 :         if (rsph->status_class != 0) {
    1171           3 :                 rsph->flags &= ~(ISCSI_LOGIN_TRANSIT | ISCSI_LOGIN_CURRENT_STAGE_MASK |
    1172             :                                  ISCSI_LOGIN_NEXT_STAGE_MASK);
    1173             :         }
    1174           3 :         iscsi_param_free(params);
    1175           3 :         iscsi_conn_write_pdu(conn, rsp_pdu, cb_fn, conn);
    1176           3 : }
    1177             : 
    1178             : /*
    1179             :  * The function which is used to initialize the internal response data
    1180             :  * structure of iscsi login function.
    1181             :  * return:
    1182             :  * 0, success;
    1183             :  * otherwise, error;
    1184             :  */
    1185             : static int
    1186           4 : iscsi_op_login_rsp_init(struct spdk_iscsi_conn *conn,
    1187             :                         struct spdk_iscsi_pdu *pdu, struct spdk_iscsi_pdu *rsp_pdu)
    1188             : {
    1189             :         struct iscsi_bhs_login_req *reqh;
    1190             :         struct iscsi_bhs_login_rsp *rsph;
    1191             : 
    1192           4 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    1193           4 :         rsph->opcode = ISCSI_OP_LOGIN_RSP;
    1194           4 :         rsph->status_class = ISCSI_CLASS_SUCCESS;
    1195           4 :         rsph->status_detail = ISCSI_LOGIN_ACCEPT;
    1196           4 :         rsp_pdu->data_segment_len = 0;
    1197             : 
    1198             :         /* The default MaxRecvDataSegmentLength 8192 is used during login. - RFC3720 */
    1199           4 :         rsp_pdu->data = calloc(1, 8192);
    1200           4 :         if (!rsp_pdu->data) {
    1201           0 :                 SPDK_ERRLOG("calloc() failed for data segment\n");
    1202           0 :                 rsph->status_class = ISCSI_CLASS_TARGET_ERROR;
    1203           0 :                 rsph->status_detail = ISCSI_LOGIN_STATUS_NO_RESOURCES;
    1204           0 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1205             :         }
    1206           4 :         rsp_pdu->data_buf_len = 8192;
    1207             : 
    1208           4 :         reqh = (struct iscsi_bhs_login_req *)&pdu->bhs;
    1209           4 :         rsph->flags |= (reqh->flags & (ISCSI_LOGIN_TRANSIT | ISCSI_LOGIN_CONTINUE |
    1210             :                                        ISCSI_LOGIN_CURRENT_STAGE_MASK));
    1211           4 :         if (ISCSI_BHS_LOGIN_GET_TBIT(rsph->flags)) {
    1212           2 :                 rsph->flags |= (reqh->flags & ISCSI_LOGIN_NEXT_STAGE_MASK);
    1213             :         }
    1214             : 
    1215             :         /* We don't need to convert from network byte order. Just store it */
    1216           4 :         memcpy(&rsph->isid, reqh->isid, 6);
    1217           4 :         rsph->tsih = reqh->tsih;
    1218           4 :         rsph->itt = reqh->itt;
    1219           4 :         rsp_pdu->cmd_sn = from_be32(&reqh->cmd_sn);
    1220             : 
    1221           4 :         if (rsph->tsih) {
    1222           0 :                 rsph->stat_sn = reqh->exp_stat_sn;
    1223             :         }
    1224             : 
    1225           4 :         SPDK_LOGDUMP(iscsi, "PDU", (uint8_t *)&pdu->bhs, ISCSI_BHS_LEN);
    1226             : 
    1227           4 :         SPDK_DEBUGLOG(iscsi,
    1228             :                       "T=%d, C=%d, CSG=%d, NSG=%d, Min=%d, Max=%d, ITT=%x\n",
    1229             :                       ISCSI_BHS_LOGIN_GET_TBIT(rsph->flags),
    1230             :                       ISCSI_BHS_LOGIN_GET_CBIT(rsph->flags),
    1231             :                       ISCSI_BHS_LOGIN_GET_CSG(rsph->flags),
    1232             :                       ISCSI_BHS_LOGIN_GET_NSG(rsph->flags),
    1233             :                       reqh->version_min, reqh->version_max, from_be32(&rsph->itt));
    1234             : 
    1235           4 :         if (conn->sess != NULL) {
    1236           0 :                 SPDK_DEBUGLOG(iscsi,
    1237             :                               "CmdSN=%u, ExpStatSN=%u, StatSN=%u, ExpCmdSN=%u,"
    1238             :                               "MaxCmdSN=%u\n", rsp_pdu->cmd_sn,
    1239             :                               from_be32(&rsph->stat_sn), conn->StatSN,
    1240             :                               conn->sess->ExpCmdSN,
    1241             :                               conn->sess->MaxCmdSN);
    1242             :         } else {
    1243           4 :                 SPDK_DEBUGLOG(iscsi,
    1244             :                               "CmdSN=%u, ExpStatSN=%u, StatSN=%u\n",
    1245             :                               rsp_pdu->cmd_sn, from_be32(&rsph->stat_sn),
    1246             :                               conn->StatSN);
    1247             :         }
    1248             : 
    1249           4 :         if (ISCSI_BHS_LOGIN_GET_TBIT(rsph->flags) &&
    1250           2 :             ISCSI_BHS_LOGIN_GET_CBIT(rsph->flags)) {
    1251           1 :                 SPDK_ERRLOG("transit error\n");
    1252           1 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1253           1 :                 rsph->status_detail = ISCSI_LOGIN_INITIATOR_ERROR;
    1254           1 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1255             :         }
    1256             :         /* make sure reqh->version_max < ISCSI_VERSION */
    1257           3 :         if (reqh->version_min > ISCSI_VERSION) {
    1258           1 :                 SPDK_ERRLOG("unsupported version min %d/max %d, expecting %d\n", reqh->version_min,
    1259             :                             reqh->version_max, ISCSI_VERSION);
    1260             :                 /* Unsupported version */
    1261             :                 /* set all reserved flag to zero */
    1262           1 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1263           1 :                 rsph->status_detail = ISCSI_LOGIN_UNSUPPORTED_VERSION;
    1264           1 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1265             :         }
    1266             : 
    1267           2 :         if ((ISCSI_BHS_LOGIN_GET_NSG(rsph->flags) == ISCSI_NSG_RESERVED_CODE) &&
    1268           1 :             ISCSI_BHS_LOGIN_GET_TBIT(rsph->flags)) {
    1269             :                 /* set NSG and other bits to zero */
    1270           1 :                 rsph->flags &= ~(ISCSI_LOGIN_NEXT_STAGE_MASK | ISCSI_LOGIN_TRANSIT |
    1271             :                                  ISCSI_LOGIN_CURRENT_STAGE_MASK);
    1272           1 :                 SPDK_ERRLOG("Received reserved NSG code: %d\n", ISCSI_NSG_RESERVED_CODE);
    1273             :                 /* Initiator error */
    1274           1 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1275           1 :                 rsph->status_detail = ISCSI_LOGIN_INITIATOR_ERROR;
    1276           1 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1277             :         }
    1278             : 
    1279           1 :         return 0;
    1280             : }
    1281             : 
    1282             : static int
    1283           0 : iscsi_op_login_store_incoming_params(struct spdk_iscsi_conn *conn,
    1284             :                                      struct spdk_iscsi_pdu *pdu, struct spdk_iscsi_pdu *rsp_pdu,
    1285             :                                      struct iscsi_param **params)
    1286             : {
    1287             :         struct iscsi_bhs_login_req *reqh;
    1288             :         struct iscsi_bhs_login_rsp *rsph;
    1289             :         int rc;
    1290             : 
    1291           0 :         reqh = (struct iscsi_bhs_login_req *)&pdu->bhs;
    1292           0 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    1293             : 
    1294           0 :         rc = iscsi_parse_params(params, pdu->data,
    1295           0 :                                 pdu->data_segment_len, ISCSI_BHS_LOGIN_GET_CBIT(reqh->flags),
    1296             :                                 &conn->partial_text_parameter);
    1297           0 :         if (rc < 0) {
    1298           0 :                 SPDK_ERRLOG("iscsi_parse_params() failed\n");
    1299           0 :                 iscsi_param_free(*params);
    1300           0 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1301           0 :                 rsph->status_detail = ISCSI_LOGIN_INITIATOR_ERROR;
    1302           0 :                 return SPDK_ISCSI_LOGIN_ERROR_PARAMETER;
    1303             :         }
    1304             : 
    1305           0 :         return 0;
    1306             : }
    1307             : 
    1308             : /*
    1309             :  * This function is used to initialize the port info
    1310             :  * return
    1311             :  * 0: success
    1312             :  * otherwise: error
    1313             :  */
    1314             : static int
    1315           0 : iscsi_op_login_initialize_port(struct spdk_iscsi_conn *conn,
    1316             :                                struct spdk_iscsi_pdu *rsp_pdu,
    1317             :                                char *initiator_port_name,
    1318             :                                uint32_t name_length,
    1319             :                                struct iscsi_param *params)
    1320             : {
    1321             :         const char *val;
    1322             :         struct iscsi_bhs_login_rsp *rsph;
    1323           0 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    1324             : 
    1325             :         /* Initiator Name and Port */
    1326           0 :         val = iscsi_param_get_val(params, "InitiatorName");
    1327           0 :         if (val == NULL) {
    1328           0 :                 SPDK_ERRLOG("InitiatorName is empty\n");
    1329             :                 /* Missing parameter */
    1330           0 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1331           0 :                 rsph->status_detail = ISCSI_LOGIN_MISSING_PARMS;
    1332           0 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1333             :         }
    1334           0 :         snprintf(conn->initiator_name, sizeof(conn->initiator_name), "%s", val);
    1335           0 :         snprintf(initiator_port_name, name_length,
    1336           0 :                  "%s,i,0x%12.12" PRIx64, val, iscsi_get_isid(rsph->isid));
    1337           0 :         spdk_strlwr(conn->initiator_name);
    1338           0 :         spdk_strlwr(initiator_port_name);
    1339           0 :         SPDK_DEBUGLOG(iscsi, "Initiator name: %s\n", conn->initiator_name);
    1340           0 :         SPDK_DEBUGLOG(iscsi, "Initiator port: %s\n", initiator_port_name);
    1341             : 
    1342           0 :         return 0;
    1343             : }
    1344             : 
    1345             : /*
    1346             :  * This function is used to judge the session type
    1347             :  * return
    1348             :  * 0: success
    1349             :  * Other value: error
    1350             :  */
    1351             : static int
    1352           0 : iscsi_op_login_session_type(struct spdk_iscsi_conn *conn,
    1353             :                             struct spdk_iscsi_pdu *rsp_pdu,
    1354             :                             enum session_type *session_type,
    1355             :                             struct iscsi_param *params)
    1356             : {
    1357             :         const char *session_type_str;
    1358             :         struct iscsi_bhs_login_rsp *rsph;
    1359             : 
    1360           0 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    1361           0 :         session_type_str = iscsi_param_get_val(params, "SessionType");
    1362           0 :         if (session_type_str == NULL) {
    1363           0 :                 if (rsph->tsih != 0) {
    1364           0 :                         *session_type = SESSION_TYPE_NORMAL;
    1365             :                 } else {
    1366           0 :                         SPDK_ERRLOG("SessionType is empty\n");
    1367             :                         /* Missing parameter */
    1368           0 :                         rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1369           0 :                         rsph->status_detail = ISCSI_LOGIN_MISSING_PARMS;
    1370           0 :                         return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1371             :                 }
    1372             :         } else {
    1373           0 :                 if (strcasecmp(session_type_str, "Discovery") == 0) {
    1374           0 :                         *session_type = SESSION_TYPE_DISCOVERY;
    1375           0 :                 } else if (strcasecmp(session_type_str, "Normal") == 0) {
    1376           0 :                         *session_type = SESSION_TYPE_NORMAL;
    1377             :                 } else {
    1378           0 :                         *session_type = SESSION_TYPE_INVALID;
    1379           0 :                         SPDK_ERRLOG("SessionType is invalid\n");
    1380             :                         /* Missing parameter */
    1381           0 :                         rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1382           0 :                         rsph->status_detail = ISCSI_LOGIN_MISSING_PARMS;
    1383           0 :                         return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1384             :                 }
    1385             :         }
    1386           0 :         SPDK_DEBUGLOG(iscsi, "Session Type: %s\n", session_type_str);
    1387             : 
    1388           0 :         return 0;
    1389             : }
    1390             : 
    1391             : /*
    1392             :  * This function is used to check the target info
    1393             :  * return:
    1394             :  * 0: success
    1395             :  * otherwise: error
    1396             :  */
    1397             : static int
    1398           7 : iscsi_op_login_check_target(struct spdk_iscsi_conn *conn,
    1399             :                             struct spdk_iscsi_pdu *rsp_pdu,
    1400             :                             const char *target_name,
    1401             :                             struct spdk_iscsi_tgt_node **target)
    1402             : {
    1403             :         struct iscsi_bhs_login_rsp *rsph;
    1404           7 :         char buf[MAX_TMPBUF] = {};
    1405             : 
    1406           7 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    1407           7 :         *target = iscsi_find_tgt_node(target_name);
    1408           7 :         if (*target == NULL) {
    1409           1 :                 SPDK_WARNLOG("target %s not found\n", target_name);
    1410             :                 /* Not found */
    1411           1 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1412           1 :                 rsph->status_detail = ISCSI_LOGIN_TARGET_NOT_FOUND;
    1413           1 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1414             :         }
    1415           6 :         if (iscsi_tgt_node_is_destructed(*target)) {
    1416           0 :                 SPDK_ERRLOG("target %s is removed\n", target_name);
    1417           0 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1418           0 :                 rsph->status_detail = ISCSI_LOGIN_TARGET_REMOVED;
    1419           0 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1420             :         }
    1421           6 :         if (iscsi_tgt_node_is_redirected(conn, *target, buf, MAX_TMPBUF)) {
    1422           0 :                 SPDK_INFOLOG(iscsi, "target %s is redirected\n", target_name);
    1423           0 :                 rsp_pdu->data_segment_len = iscsi_append_text("TargetAddress",
    1424             :                                             buf,
    1425             :                                             rsp_pdu->data,
    1426           0 :                                             rsp_pdu->data_buf_len,
    1427           0 :                                             rsp_pdu->data_segment_len);
    1428           0 :                 rsph->status_class = ISCSI_CLASS_REDIRECT;
    1429           0 :                 rsph->status_detail = ISCSI_LOGIN_TARGET_TEMPORARILY_MOVED;
    1430           0 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1431             :         }
    1432           6 :         if (!iscsi_tgt_node_access(conn, *target, conn->initiator_name,
    1433           6 :                                    conn->initiator_addr)) {
    1434           1 :                 SPDK_ERRLOG("access denied\n");
    1435           1 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1436           1 :                 rsph->status_detail = ISCSI_LOGIN_AUTHORIZATION_FAIL;
    1437           1 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1438             :         }
    1439             : 
    1440           5 :         return 0;
    1441             : }
    1442             : 
    1443             : /*
    1444             :  * This function use to check the session
    1445             :  * return:
    1446             :  * 0, success
    1447             :  * otherwise: error
    1448             :  */
    1449             : static int
    1450           4 : iscsi_op_login_check_session(struct spdk_iscsi_conn *conn,
    1451             :                              struct spdk_iscsi_pdu *rsp_pdu,
    1452             :                              char *initiator_port_name, int cid)
    1453             : 
    1454             : {
    1455           4 :         int rc = 0;
    1456             :         struct iscsi_bhs_login_rsp *rsph;
    1457             : 
    1458           4 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    1459             :         /* check existing session */
    1460           4 :         SPDK_DEBUGLOG(iscsi, "isid=%"PRIx64", tsih=%u, cid=%u\n",
    1461             :                       iscsi_get_isid(rsph->isid), from_be16(&rsph->tsih), cid);
    1462           4 :         if (rsph->tsih != 0) {
    1463             :                 /* multiple connections */
    1464           2 :                 rc = append_iscsi_sess(conn, initiator_port_name,
    1465           2 :                                        from_be16(&rsph->tsih), cid);
    1466           2 :                 if (rc != 0) {
    1467           2 :                         SPDK_ERRLOG("isid=%"PRIx64", tsih=%u, cid=%u:"
    1468             :                                     "spdk_append_iscsi_sess() failed\n",
    1469             :                                     iscsi_get_isid(rsph->isid), from_be16(&rsph->tsih),
    1470             :                                     cid);
    1471             :                         /* Can't include in session */
    1472           2 :                         rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1473           2 :                         rsph->status_detail = rc;
    1474           2 :                         return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1475             :                 }
    1476           2 :         } else if (!g_iscsi.AllowDuplicateIsid) {
    1477             :                 /* new session, drop old sess by the initiator */
    1478           1 :                 iscsi_drop_conns(conn, initiator_port_name, 0 /* drop old */);
    1479             :         }
    1480             : 
    1481           2 :         return rc;
    1482             : }
    1483             : 
    1484             : /*
    1485             :  * This function is used to del the original param and update it with new
    1486             :  * value
    1487             :  * return:
    1488             :  * 0: success
    1489             :  * otherwise: error
    1490             :  */
    1491             : static int
    1492           0 : iscsi_op_login_update_param(struct spdk_iscsi_conn *conn,
    1493             :                             const char *key, const char *value,
    1494             :                             const char *list)
    1495             : {
    1496           0 :         int rc = 0;
    1497             :         struct iscsi_param *new_param, *orig_param;
    1498             :         int index;
    1499             : 
    1500           0 :         orig_param = iscsi_param_find(conn->params, key);
    1501           0 :         if (orig_param == NULL) {
    1502           0 :                 SPDK_ERRLOG("orig_param %s not found\n", key);
    1503           0 :                 return SPDK_ISCSI_LOGIN_ERROR_PARAMETER;
    1504             :         }
    1505             : 
    1506           0 :         index = orig_param->state_index;
    1507           0 :         rc = iscsi_param_del(&conn->params, key);
    1508           0 :         if (rc < 0) {
    1509           0 :                 SPDK_ERRLOG("iscsi_param_del(%s) failed\n", key);
    1510           0 :                 return SPDK_ISCSI_LOGIN_ERROR_PARAMETER;
    1511             :         }
    1512           0 :         rc = iscsi_param_add(&conn->params, key, value, list, ISPT_LIST);
    1513           0 :         if (rc < 0) {
    1514           0 :                 SPDK_ERRLOG("iscsi_param_add() failed\n");
    1515           0 :                 return SPDK_ISCSI_LOGIN_ERROR_PARAMETER;
    1516             :         }
    1517           0 :         new_param = iscsi_param_find(conn->params, key);
    1518           0 :         if (new_param == NULL) {
    1519           0 :                 SPDK_ERRLOG("iscsi_param_find() failed\n");
    1520           0 :                 return SPDK_ISCSI_LOGIN_ERROR_PARAMETER;
    1521             :         }
    1522           0 :         new_param->state_index = index;
    1523           0 :         return rc;
    1524             : }
    1525             : 
    1526             : static int
    1527           2 : iscsi_negotiate_chap_param(struct spdk_iscsi_conn *conn)
    1528             : {
    1529           2 :         int rc = 0;
    1530             : 
    1531           2 :         if (conn->disable_chap) {
    1532           0 :                 rc = iscsi_op_login_update_param(conn, "AuthMethod", "None", "None");
    1533           2 :         } else if (conn->require_chap) {
    1534           0 :                 rc = iscsi_op_login_update_param(conn, "AuthMethod", "CHAP", "CHAP");
    1535             :         }
    1536             : 
    1537           2 :         return rc;
    1538             : }
    1539             : 
    1540             : /*
    1541             :  * The function which is used to handle the part of session discovery
    1542             :  * return:
    1543             :  * 0, success;
    1544             :  * otherwise: error;
    1545             :  */
    1546             : static int
    1547           0 : iscsi_op_login_session_discovery_chap(struct spdk_iscsi_conn *conn)
    1548             : {
    1549           0 :         return iscsi_negotiate_chap_param(conn);
    1550             : }
    1551             : 
    1552             : /*
    1553             :  * This function is used to update the param related with chap
    1554             :  * return:
    1555             :  * 0: success
    1556             :  * otherwise: error
    1557             :  */
    1558             : static int
    1559           2 : iscsi_op_login_negotiate_chap_param(struct spdk_iscsi_conn *conn,
    1560             :                                     struct spdk_iscsi_tgt_node *target)
    1561             : {
    1562           2 :         conn->disable_chap = target->disable_chap;
    1563           2 :         conn->require_chap = target->require_chap;
    1564           2 :         conn->mutual_chap = target->mutual_chap;
    1565           2 :         conn->chap_group = target->chap_group;
    1566             : 
    1567           2 :         return iscsi_negotiate_chap_param(conn);
    1568             : }
    1569             : 
    1570             : static int
    1571           2 : iscsi_op_login_negotiate_digest_param(struct spdk_iscsi_conn *conn,
    1572             :                                       struct spdk_iscsi_tgt_node *target)
    1573             : {
    1574             :         int rc;
    1575             : 
    1576           2 :         if (target->header_digest) {
    1577             :                 /*
    1578             :                  * User specified header digests, so update the list of
    1579             :                  *  HeaderDigest values to remove "None" so that only
    1580             :                  *  initiators who support CRC32C can connect.
    1581             :                  */
    1582           0 :                 rc = iscsi_op_login_update_param(conn, "HeaderDigest", "CRC32C", "CRC32C");
    1583           0 :                 if (rc < 0) {
    1584           0 :                         return rc;
    1585             :                 }
    1586             :         }
    1587             : 
    1588           2 :         if (target->data_digest) {
    1589             :                 /*
    1590             :                  * User specified data digests, so update the list of
    1591             :                  *  DataDigest values to remove "None" so that only
    1592             :                  *  initiators who support CRC32C can connect.
    1593             :                  */
    1594           0 :                 rc = iscsi_op_login_update_param(conn, "DataDigest", "CRC32C", "CRC32C");
    1595           0 :                 if (rc < 0) {
    1596           0 :                         return rc;
    1597             :                 }
    1598             :         }
    1599             : 
    1600           2 :         return 0;
    1601             : }
    1602             : 
    1603             : /*
    1604             :  * The function which is used to handle the part of normal login session
    1605             :  * return:
    1606             :  * 0, success;
    1607             :  * SPDK_ISCSI_LOGIN_ERROR_PARAMETER, parameter error;
    1608             :  */
    1609             : static int
    1610           7 : iscsi_op_login_session_normal(struct spdk_iscsi_conn *conn,
    1611             :                               struct spdk_iscsi_pdu *rsp_pdu,
    1612             :                               char *initiator_port_name,
    1613             :                               struct iscsi_param *params,
    1614             :                               int cid)
    1615             : {
    1616           7 :         struct spdk_iscsi_tgt_node *target = NULL;
    1617             :         const char *target_name;
    1618             :         const char *target_short_name;
    1619             :         struct iscsi_bhs_login_rsp *rsph;
    1620           7 :         int rc = 0;
    1621             : 
    1622           7 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    1623           7 :         target_name = iscsi_param_get_val(params, "TargetName");
    1624             : 
    1625           7 :         if (target_name == NULL) {
    1626           3 :                 SPDK_ERRLOG("TargetName is empty\n");
    1627             :                 /* Missing parameter */
    1628           3 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1629           3 :                 rsph->status_detail = ISCSI_LOGIN_MISSING_PARMS;
    1630           3 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1631             :         }
    1632             : 
    1633           4 :         memset(conn->target_short_name, 0, MAX_TARGET_NAME);
    1634           4 :         target_short_name = strstr(target_name, ":");
    1635           4 :         if (target_short_name != NULL) {
    1636           4 :                 target_short_name++; /* Advance past the ':' */
    1637           4 :                 if (strlen(target_short_name) >= MAX_TARGET_NAME) {
    1638           0 :                         SPDK_ERRLOG("Target Short Name (%s) is more than %u characters\n",
    1639             :                                     target_short_name, MAX_TARGET_NAME);
    1640             :                         /* Invalid request */
    1641           0 :                         rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1642           0 :                         rsph->status_detail = ISCSI_LOGIN_INVALID_LOGIN_REQUEST;
    1643           0 :                         return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1644             :                 }
    1645           4 :                 snprintf(conn->target_short_name, MAX_TARGET_NAME, "%s",
    1646             :                          target_short_name);
    1647             :         } else {
    1648           0 :                 snprintf(conn->target_short_name, MAX_TARGET_NAME, "%s", target_name);
    1649             :         }
    1650             : 
    1651           4 :         pthread_mutex_lock(&g_iscsi.mutex);
    1652           4 :         rc = iscsi_op_login_check_target(conn, rsp_pdu, target_name, &target);
    1653           4 :         pthread_mutex_unlock(&g_iscsi.mutex);
    1654             : 
    1655           4 :         if (rc < 0) {
    1656           0 :                 return rc;
    1657             :         }
    1658             : 
    1659           4 :         conn->target = target;
    1660           4 :         conn->dev = target->dev;
    1661           8 :         conn->target_port = spdk_scsi_dev_find_port_by_id(target->dev,
    1662           4 :                             conn->pg_tag);
    1663             : 
    1664           4 :         rc = iscsi_op_login_check_session(conn, rsp_pdu,
    1665             :                                           initiator_port_name, cid);
    1666           4 :         if (rc < 0) {
    1667           2 :                 return rc;
    1668             :         }
    1669             : 
    1670             :         /* force target flags */
    1671           2 :         pthread_mutex_lock(&target->mutex);
    1672           2 :         rc = iscsi_op_login_negotiate_chap_param(conn, target);
    1673           2 :         pthread_mutex_unlock(&target->mutex);
    1674             : 
    1675           2 :         if (rc == 0) {
    1676           2 :                 rc = iscsi_op_login_negotiate_digest_param(conn, target);
    1677             :         }
    1678             : 
    1679           2 :         if (rc != 0) {
    1680             :                 /* Invalid request */
    1681           0 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1682           0 :                 rsph->status_detail = ISCSI_LOGIN_INVALID_LOGIN_REQUEST;
    1683             :         }
    1684             : 
    1685           2 :         return rc;
    1686             : }
    1687             : 
    1688             : /*
    1689             :  * This function is used to set the info in the connection data structure
    1690             :  * return
    1691             :  * 0: success
    1692             :  * otherwise: error
    1693             :  */
    1694             : static int
    1695           0 : iscsi_op_login_set_conn_info(struct spdk_iscsi_conn *conn,
    1696             :                              struct spdk_iscsi_pdu *rsp_pdu,
    1697             :                              char *initiator_port_name,
    1698             :                              enum session_type session_type, int cid)
    1699             : {
    1700           0 :         int rc = 0;
    1701             :         struct spdk_iscsi_tgt_node *target;
    1702             :         struct iscsi_bhs_login_rsp *rsph;
    1703           0 :         struct spdk_scsi_port *initiator_port;
    1704             : 
    1705           0 :         target = conn->target;
    1706             : 
    1707           0 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    1708           0 :         conn->authenticated = false;
    1709           0 :         conn->auth.chap_phase = ISCSI_CHAP_PHASE_WAIT_A;
    1710           0 :         conn->cid = cid;
    1711             : 
    1712           0 :         if (conn->sess == NULL) {
    1713             :                 /* create initiator port */
    1714           0 :                 initiator_port = spdk_scsi_port_create(iscsi_get_isid(rsph->isid), 0, initiator_port_name);
    1715           0 :                 if (initiator_port == NULL) {
    1716           0 :                         SPDK_ERRLOG("create_port() failed\n");
    1717           0 :                         rsph->status_class = ISCSI_CLASS_TARGET_ERROR;
    1718           0 :                         rsph->status_detail = ISCSI_LOGIN_STATUS_NO_RESOURCES;
    1719           0 :                         return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1720             :                 }
    1721             : 
    1722             :                 /* new session */
    1723           0 :                 rc = create_iscsi_sess(conn, target, session_type);
    1724           0 :                 if (rc < 0) {
    1725           0 :                         spdk_scsi_port_free(&initiator_port);
    1726           0 :                         SPDK_ERRLOG("create_sess() failed\n");
    1727           0 :                         rsph->status_class = ISCSI_CLASS_TARGET_ERROR;
    1728           0 :                         rsph->status_detail = ISCSI_LOGIN_STATUS_NO_RESOURCES;
    1729           0 :                         return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1730             :                 }
    1731             :                 /* initialize parameters */
    1732           0 :                 conn->sess->initiator_port = initiator_port;
    1733           0 :                 conn->StatSN = from_be32(&rsph->stat_sn);
    1734           0 :                 conn->sess->isid = iscsi_get_isid(rsph->isid);
    1735             : 
    1736             :                 /* Initiator port TransportID */
    1737           0 :                 spdk_scsi_port_set_iscsi_transport_id(conn->sess->initiator_port,
    1738           0 :                                                       conn->initiator_name,
    1739           0 :                                                       conn->sess->isid);
    1740             : 
    1741             :                 /* Discovery sessions will not have a target. */
    1742           0 :                 if (target != NULL) {
    1743           0 :                         conn->sess->queue_depth = target->queue_depth;
    1744             :                 } else {
    1745             :                         /*
    1746             :                          * Assume discovery sessions have an effective command
    1747             :                          *  windows size of 1.
    1748             :                          */
    1749           0 :                         conn->sess->queue_depth = 1;
    1750             :                 }
    1751           0 :                 conn->sess->ExpCmdSN = rsp_pdu->cmd_sn;
    1752           0 :                 conn->sess->MaxCmdSN = rsp_pdu->cmd_sn + conn->sess->queue_depth - 1;
    1753             :         }
    1754             : 
    1755           0 :         conn->initiator_port = conn->sess->initiator_port;
    1756             : 
    1757           0 :         return 0;
    1758             : }
    1759             : 
    1760             : /*
    1761             :  * This function is used to set the target info
    1762             :  * return
    1763             :  * 0: success
    1764             :  * otherwise: error
    1765             :  */
    1766             : static int
    1767           0 : iscsi_op_login_set_target_info(struct spdk_iscsi_conn *conn,
    1768             :                                struct spdk_iscsi_pdu *rsp_pdu,
    1769             :                                enum session_type session_type)
    1770             : {
    1771           0 :         char buf[MAX_TMPBUF];
    1772             :         const char *val;
    1773           0 :         int rc = 0;
    1774           0 :         struct spdk_iscsi_tgt_node *target = conn->target;
    1775             : 
    1776             :         /* declarative parameters */
    1777           0 :         if (target != NULL) {
    1778           0 :                 pthread_mutex_lock(&target->mutex);
    1779           0 :                 if (target->alias[0] != '\0') {
    1780           0 :                         snprintf(buf, sizeof buf, "%s", target->alias);
    1781             :                 } else {
    1782           0 :                         snprintf(buf, sizeof buf, "%s", "");
    1783             :                 }
    1784           0 :                 pthread_mutex_unlock(&target->mutex);
    1785           0 :                 rc = iscsi_param_set(conn->sess->params, "TargetAlias", buf);
    1786           0 :                 if (rc < 0) {
    1787           0 :                         SPDK_ERRLOG("iscsi_param_set() failed\n");
    1788           0 :                         return SPDK_ISCSI_LOGIN_ERROR_PARAMETER;
    1789             :                 }
    1790             :         }
    1791           0 :         snprintf(buf, sizeof buf, "%s:%s,%d", conn->portal_host, conn->portal_port,
    1792             :                  conn->pg_tag);
    1793           0 :         rc = iscsi_param_set(conn->sess->params, "TargetAddress", buf);
    1794           0 :         if (rc < 0) {
    1795           0 :                 SPDK_ERRLOG("iscsi_param_set() failed\n");
    1796           0 :                 return SPDK_ISCSI_LOGIN_ERROR_PARAMETER;
    1797             :         }
    1798           0 :         snprintf(buf, sizeof buf, "%d", conn->pg_tag);
    1799           0 :         rc = iscsi_param_set(conn->sess->params, "TargetPortalGroupTag", buf);
    1800           0 :         if (rc < 0) {
    1801           0 :                 SPDK_ERRLOG("iscsi_param_set() failed\n");
    1802           0 :                 return SPDK_ISCSI_LOGIN_ERROR_PARAMETER;
    1803             :         }
    1804             : 
    1805             :         /* write in response */
    1806           0 :         if (target != NULL) {
    1807           0 :                 val = iscsi_param_get_val(conn->sess->params, "TargetAlias");
    1808           0 :                 if (val != NULL && strlen(val) != 0) {
    1809           0 :                         rsp_pdu->data_segment_len = iscsi_append_param(conn,
    1810             :                                                     "TargetAlias",
    1811             :                                                     rsp_pdu->data,
    1812           0 :                                                     rsp_pdu->data_buf_len,
    1813           0 :                                                     rsp_pdu->data_segment_len);
    1814             :                 }
    1815           0 :                 if (session_type == SESSION_TYPE_DISCOVERY) {
    1816           0 :                         rsp_pdu->data_segment_len = iscsi_append_param(conn,
    1817             :                                                     "TargetAddress",
    1818             :                                                     rsp_pdu->data,
    1819           0 :                                                     rsp_pdu->data_buf_len,
    1820           0 :                                                     rsp_pdu->data_segment_len);
    1821             :                 }
    1822           0 :                 rsp_pdu->data_segment_len = iscsi_append_param(conn,
    1823             :                                             "TargetPortalGroupTag",
    1824             :                                             rsp_pdu->data,
    1825           0 :                                             rsp_pdu->data_buf_len,
    1826           0 :                                             rsp_pdu->data_segment_len);
    1827             :         }
    1828             : 
    1829           0 :         return rc;
    1830             : }
    1831             : 
    1832             : /*
    1833             :  * This function is used to handle the login of iscsi initiator when there is
    1834             :  * no session
    1835             :  * return:
    1836             :  * 0, success;
    1837             :  * SPDK_ISCSI_LOGIN_ERROR_PARAMETER, parameter error;
    1838             :  * SPDK_ISCSI_LOGIN_ERROR_RESPONSE,  used to notify the login fail.
    1839             :  */
    1840             : static int
    1841           0 : iscsi_op_login_phase_none(struct spdk_iscsi_conn *conn,
    1842             :                           struct spdk_iscsi_pdu *rsp_pdu,
    1843             :                           struct iscsi_param *params, int cid)
    1844             : {
    1845           0 :         enum session_type session_type;
    1846           0 :         char initiator_port_name[MAX_INITIATOR_PORT_NAME];
    1847             :         struct iscsi_bhs_login_rsp *rsph;
    1848           0 :         int rc = 0;
    1849           0 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    1850             : 
    1851           0 :         conn->target = NULL;
    1852           0 :         conn->dev = NULL;
    1853             : 
    1854           0 :         rc = iscsi_op_login_initialize_port(conn, rsp_pdu, initiator_port_name,
    1855             :                                             MAX_INITIATOR_PORT_NAME, params);
    1856           0 :         if (rc < 0) {
    1857           0 :                 return rc;
    1858             :         }
    1859             : 
    1860           0 :         rc = iscsi_op_login_session_type(conn, rsp_pdu, &session_type, params);
    1861           0 :         if (rc < 0) {
    1862           0 :                 return rc;
    1863             :         }
    1864             : 
    1865             :         /* Target Name and Port */
    1866           0 :         if (session_type == SESSION_TYPE_NORMAL) {
    1867           0 :                 rc = iscsi_op_login_session_normal(conn, rsp_pdu,
    1868             :                                                    initiator_port_name,
    1869             :                                                    params, cid);
    1870           0 :                 if (rc < 0) {
    1871           0 :                         return rc;
    1872             :                 }
    1873             : 
    1874           0 :         } else if (session_type == SESSION_TYPE_DISCOVERY) {
    1875           0 :                 rsph->tsih = 0;
    1876             : 
    1877             :                 /* force target flags */
    1878           0 :                 pthread_mutex_lock(&g_iscsi.mutex);
    1879           0 :                 rc = iscsi_op_login_session_discovery_chap(conn);
    1880           0 :                 pthread_mutex_unlock(&g_iscsi.mutex);
    1881           0 :                 if (rc < 0) {
    1882           0 :                         return rc;
    1883             :                 }
    1884             :         } else {
    1885           0 :                 SPDK_ERRLOG("unknown session type\n");
    1886             :                 /* Missing parameter */
    1887           0 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1888           0 :                 rsph->status_detail = ISCSI_LOGIN_MISSING_PARMS;
    1889           0 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1890             :         }
    1891             : 
    1892           0 :         rc = iscsi_op_login_set_conn_info(conn, rsp_pdu, initiator_port_name,
    1893             :                                           session_type, cid);
    1894           0 :         if (rc < 0) {
    1895           0 :                 return rc;
    1896             :         }
    1897             : 
    1898             :         /* limit conns on discovery session */
    1899           0 :         if (session_type == SESSION_TYPE_DISCOVERY) {
    1900           0 :                 conn->sess->MaxConnections = 1;
    1901           0 :                 rc = iscsi_param_set_int(conn->sess->params,
    1902             :                                          "MaxConnections",
    1903           0 :                                          conn->sess->MaxConnections);
    1904           0 :                 if (rc < 0) {
    1905           0 :                         SPDK_ERRLOG("iscsi_param_set_int() failed\n");
    1906           0 :                         return SPDK_ISCSI_LOGIN_ERROR_PARAMETER;
    1907             :                 }
    1908             :         }
    1909             : 
    1910           0 :         return iscsi_op_login_set_target_info(conn, rsp_pdu, session_type);
    1911             : }
    1912             : 
    1913             : /*
    1914             :  * This function is used to set the csg bit case in rsp
    1915             :  * return:
    1916             :  * 0, success
    1917             :  * otherwise: error
    1918             :  */
    1919             : static int
    1920           0 : iscsi_op_login_rsp_handle_csg_bit(struct spdk_iscsi_conn *conn,
    1921             :                                   struct spdk_iscsi_pdu *rsp_pdu,
    1922             :                                   struct iscsi_param *params)
    1923             : {
    1924             :         const char *auth_method;
    1925             :         int rc;
    1926             :         struct iscsi_bhs_login_rsp *rsph;
    1927           0 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    1928             : 
    1929           0 :         switch (ISCSI_BHS_LOGIN_GET_CSG(rsph->flags)) {
    1930           0 :         case ISCSI_SECURITY_NEGOTIATION_PHASE:
    1931             :                 /* SecurityNegotiation */
    1932           0 :                 auth_method = iscsi_param_get_val(conn->params, "AuthMethod");
    1933           0 :                 if (auth_method == NULL) {
    1934           0 :                         SPDK_ERRLOG("AuthMethod is empty\n");
    1935             :                         /* Missing parameter */
    1936           0 :                         rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1937           0 :                         rsph->status_detail = ISCSI_LOGIN_MISSING_PARMS;
    1938           0 :                         return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1939             :                 }
    1940           0 :                 if (strcasecmp(auth_method, "None") == 0) {
    1941           0 :                         conn->authenticated = true;
    1942             :                 } else {
    1943           0 :                         rc = iscsi_auth_params(conn, params, auth_method,
    1944           0 :                                                rsp_pdu->data, rsp_pdu->data_buf_len,
    1945           0 :                                                rsp_pdu->data_segment_len);
    1946           0 :                         if (rc < 0) {
    1947           0 :                                 SPDK_ERRLOG("iscsi_auth_params() failed\n");
    1948             :                                 /* Authentication failure */
    1949           0 :                                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1950           0 :                                 rsph->status_detail = ISCSI_LOGIN_AUTHENT_FAIL;
    1951           0 :                                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1952             :                         }
    1953           0 :                         rsp_pdu->data_segment_len = rc;
    1954           0 :                         if (!conn->authenticated) {
    1955             :                                 /* not complete */
    1956           0 :                                 rsph->flags &= ~ISCSI_LOGIN_TRANSIT;
    1957             :                         } else {
    1958           0 :                                 if (conn->auth.chap_phase != ISCSI_CHAP_PHASE_END) {
    1959           0 :                                         SPDK_DEBUGLOG(iscsi, "CHAP phase not complete");
    1960             :                                 }
    1961             :                         }
    1962             : 
    1963           0 :                         SPDK_LOGDUMP(iscsi, "Negotiated Auth Params",
    1964             :                                      rsp_pdu->data, rsp_pdu->data_segment_len);
    1965             :                 }
    1966           0 :                 break;
    1967             : 
    1968           0 :         case ISCSI_OPERATIONAL_NEGOTIATION_PHASE:
    1969             :                 /* LoginOperationalNegotiation */
    1970           0 :                 if (conn->state == ISCSI_CONN_STATE_INVALID) {
    1971           0 :                         if (conn->require_chap) {
    1972             :                                 /* Authentication failure */
    1973           0 :                                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1974           0 :                                 rsph->status_detail = ISCSI_LOGIN_AUTHENT_FAIL;
    1975           0 :                                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1976             :                         } else {
    1977             :                                 /* AuthMethod=None */
    1978           0 :                                 conn->authenticated = true;
    1979             :                         }
    1980             :                 }
    1981           0 :                 if (!conn->authenticated) {
    1982           0 :                         SPDK_ERRLOG("authentication error\n");
    1983             :                         /* Authentication failure */
    1984           0 :                         rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1985           0 :                         rsph->status_detail = ISCSI_LOGIN_AUTHENT_FAIL;
    1986           0 :                         return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1987             :                 }
    1988           0 :                 break;
    1989             : 
    1990           0 :         case ISCSI_FULL_FEATURE_PHASE:
    1991             :                 /* FullFeaturePhase */
    1992           0 :                 SPDK_ERRLOG("XXX Login in FullFeaturePhase\n");
    1993             :                 /* Initiator error */
    1994           0 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    1995           0 :                 rsph->status_detail = ISCSI_LOGIN_INITIATOR_ERROR;
    1996           0 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    1997             : 
    1998           0 :         default:
    1999           0 :                 SPDK_ERRLOG("unknown stage\n");
    2000             :                 /* Initiator error */
    2001           0 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    2002           0 :                 rsph->status_detail = ISCSI_LOGIN_INITIATOR_ERROR;
    2003           0 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    2004             :         }
    2005             : 
    2006           0 :         return 0;
    2007             : }
    2008             : 
    2009             : /* This function is used to notify the session info
    2010             :  * return
    2011             :  * 0: success
    2012             :  * otherwise: error
    2013             :  */
    2014             : static int
    2015           0 : iscsi_op_login_notify_session_info(struct spdk_iscsi_conn *conn,
    2016             :                                    struct spdk_iscsi_pdu *rsp_pdu)
    2017             : {
    2018             :         struct iscsi_bhs_login_rsp *rsph;
    2019             : 
    2020           0 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    2021           0 :         if (conn->sess->session_type == SESSION_TYPE_NORMAL) {
    2022             :                 /* normal session */
    2023           0 :                 SPDK_DEBUGLOG(iscsi, "Login from %s (%s) on %s tgt_node%d"
    2024             :                               " (%s:%s,%d), ISID=%"PRIx64", TSIH=%u,"
    2025             :                               " CID=%u, HeaderDigest=%s, DataDigest=%s\n",
    2026             :                               conn->initiator_name, conn->initiator_addr,
    2027             :                               conn->target->name, conn->target->num,
    2028             :                               conn->portal_host, conn->portal_port, conn->pg_tag,
    2029             :                               conn->sess->isid, conn->sess->tsih, conn->cid,
    2030             :                               (iscsi_param_eq_val(conn->params, "HeaderDigest", "CRC32C")
    2031             :                                ? "on" : "off"),
    2032             :                               (iscsi_param_eq_val(conn->params, "DataDigest", "CRC32C")
    2033             :                                ? "on" : "off"));
    2034           0 :         } else if (conn->sess->session_type == SESSION_TYPE_DISCOVERY) {
    2035             :                 /* discovery session */
    2036           0 :                 SPDK_DEBUGLOG(iscsi, "Login(discovery) from %s (%s) on"
    2037             :                               " (%s:%s,%d), ISID=%"PRIx64", TSIH=%u,"
    2038             :                               " CID=%u, HeaderDigest=%s, DataDigest=%s\n",
    2039             :                               conn->initiator_name, conn->initiator_addr,
    2040             :                               conn->portal_host, conn->portal_port, conn->pg_tag,
    2041             :                               conn->sess->isid, conn->sess->tsih, conn->cid,
    2042             :                               (iscsi_param_eq_val(conn->params, "HeaderDigest", "CRC32C")
    2043             :                                ? "on" : "off"),
    2044             :                               (iscsi_param_eq_val(conn->params, "DataDigest", "CRC32C")
    2045             :                                ? "on" : "off"));
    2046             :         } else {
    2047           0 :                 SPDK_ERRLOG("unknown session type\n");
    2048             :                 /* Initiator error */
    2049           0 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    2050           0 :                 rsph->status_detail = ISCSI_LOGIN_INITIATOR_ERROR;
    2051           0 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    2052             :         }
    2053             : 
    2054           0 :         return 0;
    2055             : }
    2056             : 
    2057             : /*
    2058             :  * This function is to handle the tbit cases
    2059             :  * return
    2060             :  * 0: success
    2061             :  * otherwise error
    2062             :  */
    2063             : static int
    2064           0 : iscsi_op_login_rsp_handle_t_bit(struct spdk_iscsi_conn *conn,
    2065             :                                 struct spdk_iscsi_pdu *rsp_pdu)
    2066             : {
    2067             :         int rc;
    2068             :         struct iscsi_bhs_login_rsp *rsph;
    2069           0 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    2070             : 
    2071           0 :         switch (ISCSI_BHS_LOGIN_GET_NSG(rsph->flags)) {
    2072           0 :         case ISCSI_SECURITY_NEGOTIATION_PHASE:
    2073             :                 /* SecurityNegotiation */
    2074           0 :                 conn->login_phase = ISCSI_SECURITY_NEGOTIATION_PHASE;
    2075           0 :                 break;
    2076             : 
    2077           0 :         case ISCSI_OPERATIONAL_NEGOTIATION_PHASE:
    2078             :                 /* LoginOperationalNegotiation */
    2079           0 :                 conn->login_phase = ISCSI_OPERATIONAL_NEGOTIATION_PHASE;
    2080           0 :                 break;
    2081             : 
    2082           0 :         case ISCSI_FULL_FEATURE_PHASE:
    2083             :                 /* FullFeaturePhase */
    2084           0 :                 conn->login_phase = ISCSI_FULL_FEATURE_PHASE;
    2085           0 :                 to_be16(&rsph->tsih, conn->sess->tsih);
    2086             : 
    2087           0 :                 rc = iscsi_op_login_notify_session_info(conn, rsp_pdu);
    2088           0 :                 if (rc < 0) {
    2089           0 :                         return rc;
    2090             :                 }
    2091             : 
    2092           0 :                 conn->full_feature = 1;
    2093           0 :                 if (conn->sess->session_type == SESSION_TYPE_DISCOVERY) {
    2094           0 :                         spdk_trace_owner_append_description(conn->trace_id, "discovery");
    2095             :                 } else {
    2096           0 :                         assert(conn->target != NULL);
    2097           0 :                         spdk_trace_owner_append_description(conn->trace_id, conn->target->name);
    2098             :                 }
    2099           0 :                 break;
    2100             : 
    2101           0 :         default:
    2102           0 :                 SPDK_ERRLOG("unknown stage\n");
    2103             :                 /* Initiator error */
    2104           0 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    2105           0 :                 rsph->status_detail = ISCSI_LOGIN_INITIATOR_ERROR;
    2106           0 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    2107             :         }
    2108             : 
    2109           0 :         return 0;
    2110             : }
    2111             : 
    2112             : /*
    2113             :  * This function is used to set the values of the internal data structure used
    2114             :  * by spdk_iscsi_op_login function
    2115             :  * return:
    2116             :  * 0, used to notify the a successful login
    2117             :  * SPDK_ISCSI_LOGIN_ERROR_RESPONSE,  used to notify a failure login.
    2118             :  */
    2119             : static int
    2120           0 : iscsi_op_login_rsp_handle(struct spdk_iscsi_conn *conn,
    2121             :                           struct spdk_iscsi_pdu *rsp_pdu, struct iscsi_param **params)
    2122             : {
    2123             :         int rc;
    2124             :         struct iscsi_bhs_login_rsp *rsph;
    2125           0 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    2126             : 
    2127             :         /* negotiate parameters */
    2128           0 :         rc = iscsi_negotiate_params(conn, params, rsp_pdu->data,
    2129           0 :                                     rsp_pdu->data_buf_len,
    2130           0 :                                     rsp_pdu->data_segment_len);
    2131           0 :         if (rc < 0) {
    2132             :                 /*
    2133             :                  * iscsi_negotiate_params just returns -1 on failure,
    2134             :                  *  so translate this into meaningful response codes and
    2135             :                  *  return values.
    2136             :                  */
    2137           0 :                 rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    2138           0 :                 rsph->status_detail = ISCSI_LOGIN_INITIATOR_ERROR;
    2139           0 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    2140             :         }
    2141             : 
    2142           0 :         rsp_pdu->data_segment_len = rc;
    2143           0 :         SPDK_LOGDUMP(iscsi, "Negotiated Params", rsp_pdu->data, rc);
    2144             : 
    2145             :         /* handle the CSG bit case */
    2146           0 :         rc = iscsi_op_login_rsp_handle_csg_bit(conn, rsp_pdu, *params);
    2147           0 :         if (rc < 0) {
    2148           0 :                 return rc;
    2149             :         }
    2150             : 
    2151             :         /* handle the T bit case */
    2152           0 :         if (ISCSI_BHS_LOGIN_GET_TBIT(rsph->flags)) {
    2153           0 :                 rc = iscsi_op_login_rsp_handle_t_bit(conn, rsp_pdu);
    2154             :         }
    2155             : 
    2156           0 :         return rc;
    2157             : }
    2158             : 
    2159             : static int
    2160           7 : iscsi_pdu_hdr_op_login(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    2161             : {
    2162             :         int rc;
    2163             :         struct iscsi_bhs_login_req *reqh;
    2164             :         struct spdk_iscsi_pdu *rsp_pdu;
    2165             : 
    2166           7 :         if (conn->full_feature && conn->sess != NULL &&
    2167           1 :             conn->sess->session_type == SESSION_TYPE_DISCOVERY) {
    2168           1 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    2169             :         }
    2170             : 
    2171           6 :         reqh = (struct iscsi_bhs_login_req *)&pdu->bhs;
    2172           6 :         pdu->cmd_sn = from_be32(&reqh->cmd_sn);
    2173             : 
    2174             :         /* During login processing, use the 8KB default FirstBurstLength as
    2175             :          *  our maximum data segment length value.
    2176             :          */
    2177           6 :         if (pdu->data_segment_len > SPDK_ISCSI_FIRST_BURST_LENGTH) {
    2178           1 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    2179             :         }
    2180             : 
    2181           5 :         rsp_pdu = iscsi_get_pdu(conn);
    2182           5 :         if (rsp_pdu == NULL) {
    2183           1 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    2184             :         }
    2185           4 :         rc = iscsi_op_login_rsp_init(conn, pdu, rsp_pdu);
    2186           4 :         if (rc < 0) {
    2187           3 :                 iscsi_op_login_response(conn, rsp_pdu, NULL, iscsi_conn_login_pdu_err_complete);
    2188           3 :                 return 0;
    2189             :         }
    2190             : 
    2191           1 :         conn->login_rsp_pdu = rsp_pdu;
    2192           1 :         return 0;
    2193             : }
    2194             : 
    2195             : static int
    2196           0 : iscsi_pdu_payload_op_login(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    2197             : {
    2198             :         int rc;
    2199             :         struct iscsi_bhs_login_req *reqh;
    2200             :         struct spdk_iscsi_pdu *rsp_pdu;
    2201           0 :         struct iscsi_param *params = NULL;
    2202             :         int cid;
    2203             : 
    2204           0 :         if (conn->login_rsp_pdu == NULL) {
    2205           0 :                 return 0;
    2206             :         }
    2207             : 
    2208           0 :         spdk_poller_unregister(&conn->login_timer);
    2209           0 :         rsp_pdu = conn->login_rsp_pdu;
    2210             : 
    2211           0 :         reqh = (struct iscsi_bhs_login_req *)&pdu->bhs;
    2212           0 :         cid = from_be16(&reqh->cid);
    2213             : 
    2214           0 :         rc = iscsi_op_login_store_incoming_params(conn, pdu, rsp_pdu, &params);
    2215           0 :         if (rc < 0) {
    2216           0 :                 iscsi_op_login_response(conn, rsp_pdu, NULL, iscsi_conn_login_pdu_err_complete);
    2217           0 :                 return 0;
    2218             :         }
    2219             : 
    2220           0 :         if (conn->state == ISCSI_CONN_STATE_INVALID) {
    2221           0 :                 rc = iscsi_op_login_phase_none(conn, rsp_pdu, params, cid);
    2222           0 :                 if (rc == SPDK_ISCSI_LOGIN_ERROR_RESPONSE || rc == SPDK_ISCSI_LOGIN_ERROR_PARAMETER) {
    2223           0 :                         iscsi_op_login_response(conn, rsp_pdu, params, iscsi_conn_login_pdu_err_complete);
    2224           0 :                         return 0;
    2225             :                 }
    2226             :         }
    2227             : 
    2228           0 :         rc = iscsi_op_login_rsp_handle(conn, rsp_pdu, &params);
    2229           0 :         if (rc == SPDK_ISCSI_LOGIN_ERROR_RESPONSE) {
    2230           0 :                 iscsi_op_login_response(conn, rsp_pdu, params, iscsi_conn_login_pdu_err_complete);
    2231           0 :                 return 0;
    2232             :         }
    2233             : 
    2234           0 :         conn->state = ISCSI_CONN_STATE_RUNNING;
    2235           0 :         iscsi_op_login_response(conn, rsp_pdu, params, iscsi_conn_login_pdu_success_complete);
    2236           0 :         return 0;
    2237             : }
    2238             : 
    2239             : static int
    2240           7 : iscsi_pdu_hdr_op_text(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    2241             : {
    2242             :         uint32_t task_tag;
    2243             :         uint32_t ExpStatSN;
    2244             :         int F_bit, C_bit;
    2245             :         struct iscsi_bhs_text_req *reqh;
    2246             : 
    2247           7 :         if (pdu->data_segment_len > iscsi_get_max_immediate_data_size()) {
    2248           1 :                 SPDK_ERRLOG("data segment len(=%zu) > immediate data len(=%"PRIu32")\n",
    2249             :                             pdu->data_segment_len, iscsi_get_max_immediate_data_size());
    2250           1 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    2251             :         }
    2252             : 
    2253           6 :         reqh = (struct iscsi_bhs_text_req *)&pdu->bhs;
    2254             : 
    2255           6 :         F_bit = !!(reqh->flags & ISCSI_FLAG_FINAL);
    2256           6 :         C_bit = !!(reqh->flags & ISCSI_TEXT_CONTINUE);
    2257           6 :         task_tag = from_be32(&reqh->itt);
    2258           6 :         ExpStatSN = from_be32(&reqh->exp_stat_sn);
    2259             : 
    2260           6 :         SPDK_DEBUGLOG(iscsi, "I=%d, F=%d, C=%d, ITT=%x, TTT=%x\n",
    2261             :                       reqh->immediate, F_bit, C_bit, task_tag, from_be32(&reqh->ttt));
    2262             : 
    2263           6 :         SPDK_DEBUGLOG(iscsi,
    2264             :                       "CmdSN=%u, ExpStatSN=%u, StatSN=%u, ExpCmdSN=%u, MaxCmdSN=%u\n",
    2265             :                       pdu->cmd_sn, ExpStatSN, conn->StatSN, conn->sess->ExpCmdSN,
    2266             :                       conn->sess->MaxCmdSN);
    2267             : 
    2268           6 :         if (ExpStatSN != conn->StatSN) {
    2269             : #if 0
    2270             :                 SPDK_ERRLOG("StatSN(%u) error\n", ExpStatSN);
    2271             :                 return -1;
    2272             : #else
    2273             :                 /* StarPort have a bug */
    2274           3 :                 SPDK_DEBUGLOG(iscsi, "StatSN(%u) rewound\n", ExpStatSN);
    2275           3 :                 conn->StatSN = ExpStatSN;
    2276             : #endif
    2277             :         }
    2278             : 
    2279           6 :         if (F_bit && C_bit) {
    2280           1 :                 SPDK_ERRLOG("final and continue\n");
    2281           1 :                 return -1;
    2282             :         }
    2283             : 
    2284             :         /*
    2285             :          * If this is the first text op in a sequence, save the ITT so we can
    2286             :          * compare it against the ITT for subsequent ops in the same sequence.
    2287             :          * If a subsequent text op in same sequence has a different ITT, reject
    2288             :          * that PDU.
    2289             :          */
    2290           5 :         if (conn->sess->current_text_itt == 0xffffffffU) {
    2291           1 :                 conn->sess->current_text_itt = task_tag;
    2292           4 :         } else if (conn->sess->current_text_itt != task_tag) {
    2293           1 :                 SPDK_ERRLOG("The correct itt is %u, and the current itt is %u...\n",
    2294             :                             conn->sess->current_text_itt, task_tag);
    2295           1 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    2296             :         }
    2297             : 
    2298           4 :         return 0;
    2299             : }
    2300             : 
    2301             : static void
    2302           0 : iscsi_conn_text_pdu_complete(void *arg)
    2303             : {
    2304           0 :         struct spdk_iscsi_conn *conn = arg;
    2305             : 
    2306           0 :         iscsi_conn_params_update(conn);
    2307           0 : }
    2308             : 
    2309             : static int
    2310           1 : iscsi_pdu_payload_op_text(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    2311             : {
    2312           1 :         struct iscsi_param *params = NULL;
    2313             :         struct spdk_iscsi_pdu *rsp_pdu;
    2314             :         uint8_t *data;
    2315             :         uint64_t lun;
    2316             :         uint32_t task_tag;
    2317             :         const char *val;
    2318             :         int F_bit, C_bit;
    2319             :         int data_len;
    2320             :         int alloc_len;
    2321             :         int rc;
    2322             :         struct iscsi_bhs_text_req *reqh;
    2323             :         struct iscsi_bhs_text_resp *rsph;
    2324             : 
    2325           1 :         data_len = 0;
    2326           1 :         alloc_len = conn->MaxRecvDataSegmentLength;
    2327             : 
    2328           1 :         reqh = (struct iscsi_bhs_text_req *)&pdu->bhs;
    2329             : 
    2330           1 :         F_bit = !!(reqh->flags & ISCSI_FLAG_FINAL);
    2331           1 :         C_bit = !!(reqh->flags & ISCSI_TEXT_CONTINUE);
    2332           1 :         lun = from_be64(&reqh->lun);
    2333           1 :         task_tag = from_be32(&reqh->itt);
    2334             : 
    2335             :         /* store incoming parameters */
    2336           1 :         rc = iscsi_parse_params(&params, pdu->data, pdu->data_segment_len,
    2337             :                                 C_bit, &conn->partial_text_parameter);
    2338           1 :         if (rc < 0) {
    2339           0 :                 SPDK_ERRLOG("iscsi_parse_params() failed\n");
    2340           0 :                 iscsi_param_free(params);
    2341           0 :                 return -1;
    2342             :         }
    2343             : 
    2344           1 :         if (pdu->data_segment_len == 0 && params == NULL) {
    2345           1 :                 params = conn->params_text;
    2346           1 :                 conn->params_text = NULL;
    2347             :         }
    2348             : 
    2349           1 :         data = calloc(1, alloc_len);
    2350           1 :         if (!data) {
    2351           0 :                 SPDK_ERRLOG("calloc() failed for data segment\n");
    2352           0 :                 iscsi_param_free(params);
    2353           0 :                 return -ENOMEM;
    2354             :         }
    2355             : 
    2356             :         /* negotiate parameters */
    2357           1 :         data_len = iscsi_negotiate_params(conn, &params,
    2358             :                                           data, alloc_len, data_len);
    2359           1 :         if (data_len < 0) {
    2360           0 :                 SPDK_ERRLOG("iscsi_negotiate_params() failed\n");
    2361           0 :                 iscsi_param_free(params);
    2362           0 :                 free(data);
    2363           0 :                 return -1;
    2364             :         }
    2365             : 
    2366             :         /* sendtargets is special case */
    2367           1 :         val = iscsi_param_get_val(params, "SendTargets");
    2368           1 :         if (val != NULL) {
    2369           0 :                 if (iscsi_param_eq_val(conn->sess->params,
    2370             :                                        "SessionType", "Discovery")) {
    2371           0 :                         if (strcasecmp(val, "") == 0) {
    2372           0 :                                 val = "ALL";
    2373             :                         }
    2374             : 
    2375           0 :                         data_len = iscsi_send_tgts(conn,
    2376           0 :                                                    conn->initiator_name,
    2377             :                                                    val, data, alloc_len,
    2378             :                                                    data_len);
    2379             :                 } else {
    2380           0 :                         if (strcasecmp(val, "") == 0) {
    2381           0 :                                 val = conn->target->name;
    2382             :                         }
    2383             : 
    2384           0 :                         if (strcasecmp(val, "ALL") == 0) {
    2385             :                                 /* not in discovery session */
    2386           0 :                                 data_len = iscsi_append_text("SendTargets", "Reject",
    2387             :                                                              data, alloc_len, data_len);
    2388             :                         } else {
    2389           0 :                                 data_len = iscsi_send_tgts(conn,
    2390           0 :                                                            conn->initiator_name,
    2391             :                                                            val, data, alloc_len,
    2392             :                                                            data_len);
    2393             :                         }
    2394             :                 }
    2395             : 
    2396           0 :                 if (conn->send_tgt_completed_size != 0) {
    2397           0 :                         F_bit = 0;
    2398           0 :                         C_bit = 1;
    2399             :                 }
    2400             :         } else {
    2401           1 :                 if (iscsi_param_eq_val(conn->sess->params, "SessionType", "Discovery")) {
    2402           0 :                         iscsi_param_free(params);
    2403           0 :                         free(data);
    2404           0 :                         return SPDK_ISCSI_CONNECTION_FATAL;
    2405             :                 }
    2406             :         }
    2407             : 
    2408           1 :         if (spdk_likely(conn->send_tgt_completed_size == 0)) {
    2409           1 :                 iscsi_param_free(params);
    2410             :         } else {
    2411           0 :                 conn->params_text = params;
    2412             :         }
    2413           1 :         SPDK_LOGDUMP(iscsi, "Negotiated Params", data, data_len);
    2414             : 
    2415             :         /* response PDU */
    2416           1 :         rsp_pdu = iscsi_get_pdu(conn);
    2417           1 :         if (rsp_pdu == NULL) {
    2418           0 :                 free(data);
    2419           0 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    2420             :         }
    2421           1 :         rsph = (struct iscsi_bhs_text_resp *)&rsp_pdu->bhs;
    2422             : 
    2423           1 :         rsp_pdu->data = data;
    2424           1 :         rsph->opcode = ISCSI_OP_TEXT_RSP;
    2425             : 
    2426           1 :         if (F_bit) {
    2427           0 :                 rsph->flags |= ISCSI_FLAG_FINAL;
    2428             :         }
    2429             : 
    2430           1 :         if (C_bit) {
    2431           1 :                 rsph->flags |= ISCSI_TEXT_CONTINUE;
    2432             :         }
    2433             : 
    2434           1 :         DSET24(rsph->data_segment_len, data_len);
    2435           1 :         to_be64(&rsph->lun, lun);
    2436           1 :         to_be32(&rsph->itt, task_tag);
    2437             : 
    2438           1 :         if (F_bit) {
    2439           0 :                 rsph->ttt = 0xffffffffU;
    2440           0 :                 conn->sess->current_text_itt = 0xffffffffU;
    2441             :         } else {
    2442           1 :                 to_be32(&rsph->ttt, 1 + conn->id);
    2443             :         }
    2444             : 
    2445           1 :         to_be32(&rsph->stat_sn, conn->StatSN);
    2446           1 :         conn->StatSN++;
    2447             : 
    2448           1 :         if (reqh->immediate == 0) {
    2449           1 :                 conn->sess->MaxCmdSN++;
    2450             :         }
    2451             : 
    2452           1 :         to_be32(&rsph->exp_cmd_sn, conn->sess->ExpCmdSN);
    2453           1 :         to_be32(&rsph->max_cmd_sn, conn->sess->MaxCmdSN);
    2454             : 
    2455           1 :         iscsi_conn_write_pdu(conn, rsp_pdu, iscsi_conn_text_pdu_complete, conn);
    2456           1 :         return 0;
    2457             : }
    2458             : 
    2459             : static void
    2460           0 : iscsi_conn_logout_pdu_complete(void *arg)
    2461             : {
    2462           0 :         struct spdk_iscsi_conn *conn = arg;
    2463             : 
    2464           0 :         if (conn->sess == NULL) {
    2465             :                 /*
    2466             :                  * login failed but initiator still sent a logout rather than
    2467             :                  *  just closing the TCP connection.
    2468             :                  */
    2469           0 :                 SPDK_DEBUGLOG(iscsi, "Logout(login failed) from %s (%s) on"
    2470             :                               " (%s:%s,%d)\n",
    2471             :                               conn->initiator_name, conn->initiator_addr,
    2472             :                               conn->portal_host, conn->portal_port, conn->pg_tag);
    2473           0 :         } else if (iscsi_param_eq_val(conn->sess->params, "SessionType", "Normal")) {
    2474           0 :                 SPDK_DEBUGLOG(iscsi, "Logout from %s (%s) on %s tgt_node%d"
    2475             :                               " (%s:%s,%d), ISID=%"PRIx64", TSIH=%u,"
    2476             :                               " CID=%u, HeaderDigest=%s, DataDigest=%s\n",
    2477             :                               conn->initiator_name, conn->initiator_addr,
    2478             :                               conn->target->name, conn->target->num,
    2479             :                               conn->portal_host, conn->portal_port, conn->pg_tag,
    2480             :                               conn->sess->isid, conn->sess->tsih, conn->cid,
    2481             :                               (iscsi_param_eq_val(conn->params, "HeaderDigest", "CRC32C")
    2482             :                                ? "on" : "off"),
    2483             :                               (iscsi_param_eq_val(conn->params, "DataDigest", "CRC32C")
    2484             :                                ? "on" : "off"));
    2485             :         } else {
    2486             :                 /* discovery session */
    2487           0 :                 SPDK_DEBUGLOG(iscsi, "Logout(discovery) from %s (%s) on"
    2488             :                               " (%s:%s,%d), ISID=%"PRIx64", TSIH=%u,"
    2489             :                               " CID=%u, HeaderDigest=%s, DataDigest=%s\n",
    2490             :                               conn->initiator_name, conn->initiator_addr,
    2491             :                               conn->portal_host, conn->portal_port, conn->pg_tag,
    2492             :                               conn->sess->isid, conn->sess->tsih, conn->cid,
    2493             :                               (iscsi_param_eq_val(conn->params, "HeaderDigest", "CRC32C")
    2494             :                                ? "on" : "off"),
    2495             :                               (iscsi_param_eq_val(conn->params, "DataDigest", "CRC32C")
    2496             :                                ? "on" : "off"));
    2497             :         }
    2498           0 : }
    2499             : 
    2500             : static int
    2501           5 : iscsi_pdu_hdr_op_logout(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    2502             : {
    2503             :         struct spdk_iscsi_pdu *rsp_pdu;
    2504             :         uint32_t task_tag;
    2505             :         uint32_t ExpStatSN;
    2506             :         int response;
    2507             :         struct iscsi_bhs_logout_req *reqh;
    2508             :         struct iscsi_bhs_logout_resp *rsph;
    2509             :         uint16_t cid;
    2510             : 
    2511           5 :         reqh = (struct iscsi_bhs_logout_req *)&pdu->bhs;
    2512             : 
    2513           5 :         cid = from_be16(&reqh->cid);
    2514           5 :         task_tag = from_be32(&reqh->itt);
    2515           5 :         ExpStatSN = from_be32(&reqh->exp_stat_sn);
    2516             : 
    2517           5 :         SPDK_DEBUGLOG(iscsi, "reason=%d, ITT=%x, cid=%d\n",
    2518             :                       reqh->reason, task_tag, cid);
    2519             : 
    2520           5 :         if (conn->sess != NULL) {
    2521           4 :                 if (conn->sess->session_type == SESSION_TYPE_DISCOVERY &&
    2522           1 :                     reqh->reason != ISCSI_LOGOUT_REASON_CLOSE_SESSION) {
    2523           1 :                         SPDK_ERRLOG("Target can accept logout only with reason \"close the session\" "
    2524             :                                     "on discovery session. %d is not acceptable reason.\n",
    2525             :                                     reqh->reason);
    2526           1 :                         return SPDK_ISCSI_CONNECTION_FATAL;
    2527             :                 }
    2528             : 
    2529           3 :                 SPDK_DEBUGLOG(iscsi,
    2530             :                               "CmdSN=%u, ExpStatSN=%u, StatSN=%u, ExpCmdSN=%u, MaxCmdSN=%u\n",
    2531             :                               pdu->cmd_sn, ExpStatSN, conn->StatSN,
    2532             :                               conn->sess->ExpCmdSN, conn->sess->MaxCmdSN);
    2533             : 
    2534           3 :                 if (pdu->cmd_sn != conn->sess->ExpCmdSN) {
    2535           3 :                         SPDK_DEBUGLOG(iscsi, "CmdSN(%u) might have dropped\n", pdu->cmd_sn);
    2536             :                         /* ignore error */
    2537             :                 }
    2538             :         } else {
    2539           1 :                 SPDK_DEBUGLOG(iscsi, "CmdSN=%u, ExpStatSN=%u, StatSN=%u\n",
    2540             :                               pdu->cmd_sn, ExpStatSN, conn->StatSN);
    2541             :         }
    2542             : 
    2543           4 :         if (ExpStatSN != conn->StatSN) {
    2544           3 :                 SPDK_DEBUGLOG(iscsi, "StatSN(%u/%u) might have dropped\n",
    2545             :                               ExpStatSN, conn->StatSN);
    2546             :                 /* ignore error */
    2547             :         }
    2548             : 
    2549           4 :         if (conn->cid == cid) {
    2550             :                 /* connection or session closed successfully */
    2551           3 :                 response = 0;
    2552           3 :                 iscsi_conn_logout(conn);
    2553             :         } else {
    2554           1 :                 response = 1;
    2555             :         }
    2556             : 
    2557             :         /* response PDU */
    2558           4 :         rsp_pdu = iscsi_get_pdu(conn);
    2559           4 :         if (rsp_pdu == NULL) {
    2560           1 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    2561             :         }
    2562           3 :         rsph = (struct iscsi_bhs_logout_resp *)&rsp_pdu->bhs;
    2563           3 :         rsp_pdu->data = NULL;
    2564           3 :         rsph->opcode = ISCSI_OP_LOGOUT_RSP;
    2565           3 :         rsph->flags |= 0x80; /* bit 0 must be 1 */
    2566           3 :         rsph->response = response;
    2567           3 :         DSET24(rsph->data_segment_len, 0);
    2568           3 :         to_be32(&rsph->itt, task_tag);
    2569             : 
    2570           3 :         if (conn->sess != NULL) {
    2571           2 :                 to_be32(&rsph->stat_sn, conn->StatSN);
    2572           2 :                 conn->StatSN++;
    2573             : 
    2574           2 :                 if (conn->sess->connections == 1) {
    2575           2 :                         conn->sess->MaxCmdSN++;
    2576             :                 }
    2577             : 
    2578           2 :                 to_be32(&rsph->exp_cmd_sn, conn->sess->ExpCmdSN);
    2579           2 :                 to_be32(&rsph->max_cmd_sn, conn->sess->MaxCmdSN);
    2580             :         } else {
    2581           1 :                 to_be32(&rsph->stat_sn, conn->StatSN);
    2582           1 :                 conn->StatSN++;
    2583           1 :                 to_be32(&rsph->exp_cmd_sn, pdu->cmd_sn);
    2584           1 :                 to_be32(&rsph->max_cmd_sn, pdu->cmd_sn);
    2585             :         }
    2586             : 
    2587           3 :         rsph->time_2_wait = 0;
    2588           3 :         rsph->time_2_retain = 0;
    2589             : 
    2590           3 :         iscsi_conn_write_pdu(conn, rsp_pdu, iscsi_conn_logout_pdu_complete, conn);
    2591             : 
    2592           3 :         return 0;
    2593             : }
    2594             : 
    2595             : static int
    2596           9 : iscsi_send_r2t(struct spdk_iscsi_conn *conn,
    2597             :                struct spdk_iscsi_task *task, int offset,
    2598             :                int len, uint32_t transfer_tag, uint32_t *R2TSN)
    2599             : {
    2600             :         struct spdk_iscsi_pdu *rsp_pdu;
    2601             :         struct iscsi_bhs_r2t *rsph;
    2602             :         uint64_t fmt_lun;
    2603             : 
    2604             :         /* R2T PDU */
    2605           9 :         rsp_pdu = iscsi_get_pdu(conn);
    2606           9 :         if (rsp_pdu == NULL) {
    2607           0 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    2608             :         }
    2609           9 :         rsph = (struct iscsi_bhs_r2t *)&rsp_pdu->bhs;
    2610           9 :         rsp_pdu->data = NULL;
    2611           9 :         rsph->opcode = ISCSI_OP_R2T;
    2612           9 :         rsph->flags |= 0x80; /* bit 0 is default to 1 */
    2613           9 :         fmt_lun = spdk_scsi_lun_id_int_to_fmt(task->lun_id);
    2614           9 :         to_be64(&rsph->lun, fmt_lun);
    2615           9 :         to_be32(&rsph->itt, task->tag);
    2616           9 :         to_be32(&rsph->ttt, transfer_tag);
    2617             : 
    2618           9 :         to_be32(&rsph->stat_sn, conn->StatSN);
    2619           9 :         to_be32(&rsph->exp_cmd_sn, conn->sess->ExpCmdSN);
    2620           9 :         to_be32(&rsph->max_cmd_sn, conn->sess->MaxCmdSN);
    2621             : 
    2622           9 :         to_be32(&rsph->r2t_sn, *R2TSN);
    2623           9 :         *R2TSN += 1;
    2624             : 
    2625           9 :         task->r2t_datasn = 0; /* next expected datasn to ack */
    2626             : 
    2627           9 :         to_be32(&rsph->buffer_offset, (uint32_t)offset);
    2628           9 :         to_be32(&rsph->desired_xfer_len, (uint32_t)len);
    2629           9 :         task->desired_data_transfer_length = (size_t)len;
    2630             : 
    2631             :         /* we need to hold onto this task/cmd because until the PDU has been
    2632             :          * written out */
    2633           9 :         rsp_pdu->task = task;
    2634           9 :         task->scsi.ref++;
    2635             : 
    2636           9 :         iscsi_conn_write_pdu(conn, rsp_pdu, iscsi_conn_pdu_generic_complete, NULL);
    2637             : 
    2638           9 :         return 0;
    2639             : }
    2640             : 
    2641             : /* This function is used to remove the r2t pdu from snack_pdu_list by < task, r2t_sn> info */
    2642             : static struct spdk_iscsi_pdu *
    2643           0 : iscsi_remove_r2t_pdu_from_snack_list(struct spdk_iscsi_conn *conn,
    2644             :                                      struct spdk_iscsi_task *task,
    2645             :                                      uint32_t r2t_sn)
    2646             : {
    2647             :         struct spdk_iscsi_pdu *pdu;
    2648             :         struct iscsi_bhs_r2t *r2t_header;
    2649             : 
    2650           0 :         TAILQ_FOREACH(pdu, &conn->snack_pdu_list, tailq) {
    2651           0 :                 if (pdu->bhs.opcode == ISCSI_OP_R2T) {
    2652           0 :                         r2t_header = (struct iscsi_bhs_r2t *)&pdu->bhs;
    2653           0 :                         if (pdu->task == task &&
    2654           0 :                             from_be32(&r2t_header->r2t_sn) == r2t_sn) {
    2655           0 :                                 TAILQ_REMOVE(&conn->snack_pdu_list, pdu, tailq);
    2656           0 :                                 return pdu;
    2657             :                         }
    2658             :                 }
    2659             :         }
    2660             : 
    2661           0 :         return NULL;
    2662             : }
    2663             : 
    2664             : /* This function is used re-send the r2t packet */
    2665             : static int
    2666           0 : iscsi_send_r2t_recovery(struct spdk_iscsi_conn *conn,
    2667             :                         struct spdk_iscsi_task *task, uint32_t r2t_sn,
    2668             :                         bool send_new_r2tsn)
    2669             : {
    2670             :         struct spdk_iscsi_pdu *pdu;
    2671             :         struct iscsi_bhs_r2t *rsph;
    2672             :         uint32_t transfer_len;
    2673             :         uint32_t len;
    2674             :         int rc;
    2675             : 
    2676             :         /* remove the r2t pdu from the snack_list */
    2677           0 :         pdu = iscsi_remove_r2t_pdu_from_snack_list(conn, task, r2t_sn);
    2678           0 :         if (!pdu) {
    2679           0 :                 SPDK_DEBUGLOG(iscsi, "No pdu is found\n");
    2680           0 :                 return -1;
    2681             :         }
    2682             : 
    2683             :         /* flag
    2684             :          * false: only need to re-send the old r2t with changing statsn
    2685             :          * true: we send a r2t with new r2tsn
    2686             :          */
    2687           0 :         if (!send_new_r2tsn) {
    2688           0 :                 to_be32(&pdu->bhs.stat_sn, conn->StatSN);
    2689           0 :                 iscsi_conn_write_pdu(conn, pdu, iscsi_conn_pdu_generic_complete, NULL);
    2690             :         } else {
    2691           0 :                 rsph = (struct iscsi_bhs_r2t *)&pdu->bhs;
    2692           0 :                 transfer_len = from_be32(&rsph->desired_xfer_len);
    2693             : 
    2694             :                 /* still need to increase the acked r2tsn */
    2695           0 :                 task->acked_r2tsn++;
    2696           0 :                 len = spdk_min(conn->sess->MaxBurstLength,
    2697             :                                (transfer_len - task->next_expected_r2t_offset));
    2698             : 
    2699             :                 /* remove the old_r2t_pdu */
    2700           0 :                 iscsi_conn_free_pdu(conn, pdu);
    2701             : 
    2702             :                 /* re-send a new r2t pdu */
    2703           0 :                 rc = iscsi_send_r2t(conn, task, task->next_expected_r2t_offset,
    2704             :                                     len, task->ttt, &task->R2TSN);
    2705           0 :                 if (rc < 0) {
    2706           0 :                         return SPDK_ISCSI_CONNECTION_FATAL;
    2707             :                 }
    2708             :         }
    2709             : 
    2710           0 :         return 0;
    2711             : }
    2712             : 
    2713             : static int
    2714          20 : add_transfer_task(struct spdk_iscsi_conn *conn, struct spdk_iscsi_task *task)
    2715             : {
    2716             :         uint32_t transfer_len;
    2717             :         size_t max_burst_len;
    2718             :         size_t segment_len;
    2719             :         size_t data_len;
    2720             :         int len;
    2721             :         int rc;
    2722             :         int data_out_req;
    2723             : 
    2724          20 :         transfer_len = task->scsi.transfer_len;
    2725          20 :         data_len = iscsi_task_get_pdu(task)->data_segment_len;
    2726          20 :         max_burst_len = conn->sess->MaxBurstLength;
    2727          20 :         segment_len = SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH;
    2728          20 :         data_out_req = 1 + (transfer_len - data_len - 1) / segment_len;
    2729          20 :         task->data_out_cnt = data_out_req;
    2730             : 
    2731             :         /*
    2732             :          * If we already have too many tasks using R2T, then queue this task
    2733             :          *  and start sending R2T for it after some of the tasks using R2T/data
    2734             :          *  out buffers complete.
    2735             :          */
    2736          20 :         if (conn->pending_r2t >= g_iscsi.MaxR2TPerConnection) {
    2737           4 :                 TAILQ_INSERT_TAIL(&conn->queued_r2t_tasks, task, link);
    2738           4 :                 return 0;
    2739             :         }
    2740             : 
    2741          16 :         conn->data_out_cnt += data_out_req;
    2742          16 :         conn->pending_r2t++;
    2743             : 
    2744          16 :         task->next_expected_r2t_offset = data_len;
    2745          16 :         task->current_r2t_length = 0;
    2746          16 :         task->R2TSN = 0;
    2747             :         /* According to RFC3720 10.8.5, 0xffffffff is
    2748             :          * reserved for TTT in R2T.
    2749             :          */
    2750          16 :         if (++conn->ttt == 0xffffffffu) {
    2751           0 :                 conn->ttt = 0;
    2752             :         }
    2753          16 :         task->ttt = conn->ttt;
    2754             : 
    2755          22 :         while (data_len != transfer_len) {
    2756           8 :                 len = spdk_min(max_burst_len, (transfer_len - data_len));
    2757           8 :                 rc = iscsi_send_r2t(conn, task, data_len, len,
    2758             :                                     task->ttt, &task->R2TSN);
    2759           8 :                 if (rc < 0) {
    2760           0 :                         SPDK_ERRLOG("iscsi_send_r2t() failed\n");
    2761           0 :                         return rc;
    2762             :                 }
    2763           8 :                 data_len += len;
    2764           8 :                 task->next_r2t_offset = data_len;
    2765           8 :                 task->outstanding_r2t++;
    2766           8 :                 if (conn->sess->MaxOutstandingR2T == task->outstanding_r2t) {
    2767           2 :                         break;
    2768             :                 }
    2769             :         }
    2770             : 
    2771          16 :         TAILQ_INSERT_TAIL(&conn->active_r2t_tasks, task, link);
    2772          16 :         task->is_r2t_active = true;
    2773          16 :         return 0;
    2774             : }
    2775             : 
    2776             : /* If there are additional large writes queued for R2Ts, start them now.
    2777             :  *  This is called when a large write is just completed or when multiple LUNs
    2778             :  *  are attached and large write tasks for the specific LUN are cleared.
    2779             :  */
    2780             : static void
    2781           9 : start_queued_transfer_tasks(struct spdk_iscsi_conn *conn)
    2782             : {
    2783             :         struct spdk_iscsi_task *task, *tmp;
    2784             : 
    2785          12 :         TAILQ_FOREACH_SAFE(task, &conn->queued_r2t_tasks, link, tmp) {
    2786           4 :                 if (conn->pending_r2t < g_iscsi.MaxR2TPerConnection) {
    2787           3 :                         TAILQ_REMOVE(&conn->queued_r2t_tasks, task, link);
    2788           3 :                         add_transfer_task(conn, task);
    2789             :                 } else {
    2790           1 :                         break;
    2791             :                 }
    2792             :         }
    2793           9 : }
    2794             : 
    2795             : bool
    2796           5 : iscsi_del_transfer_task(struct spdk_iscsi_conn *conn, uint32_t task_tag)
    2797             : {
    2798             :         struct spdk_iscsi_task *task, *tmp;
    2799             : 
    2800           5 :         TAILQ_FOREACH_SAFE(task, &conn->active_r2t_tasks, link, tmp) {
    2801           5 :                 if (task->tag == task_tag) {
    2802           5 :                         assert(conn->data_out_cnt >= task->data_out_cnt);
    2803           5 :                         conn->data_out_cnt -= task->data_out_cnt;
    2804             : 
    2805           5 :                         assert(conn->pending_r2t > 0);
    2806           5 :                         conn->pending_r2t--;
    2807             : 
    2808           5 :                         assert(task->is_r2t_active == true);
    2809           5 :                         TAILQ_REMOVE(&conn->active_r2t_tasks, task, link);
    2810           5 :                         task->is_r2t_active = false;
    2811           5 :                         iscsi_task_put(task);
    2812             : 
    2813           5 :                         start_queued_transfer_tasks(conn);
    2814           5 :                         return true;
    2815             :                 }
    2816             :         }
    2817           0 :         return false;
    2818             : }
    2819             : 
    2820             : void
    2821           4 : iscsi_clear_all_transfer_task(struct spdk_iscsi_conn *conn,
    2822             :                               struct spdk_scsi_lun *lun,
    2823             :                               struct spdk_iscsi_pdu *pdu)
    2824             : {
    2825             :         struct spdk_iscsi_task *task, *task_tmp;
    2826             :         struct spdk_iscsi_pdu *pdu_tmp;
    2827             : 
    2828          16 :         TAILQ_FOREACH_SAFE(task, &conn->active_r2t_tasks, link, task_tmp) {
    2829          12 :                 pdu_tmp = iscsi_task_get_pdu(task);
    2830          12 :                 if ((lun == NULL || lun == task->scsi.lun) &&
    2831           6 :                     (pdu == NULL || spdk_sn32_lt(pdu_tmp->cmd_sn, pdu->cmd_sn))) {
    2832           6 :                         task->outstanding_r2t = 0;
    2833           6 :                         task->next_r2t_offset = 0;
    2834           6 :                         task->next_expected_r2t_offset = 0;
    2835           6 :                         task->current_data_offset = 0;
    2836           6 :                         assert(conn->data_out_cnt >= task->data_out_cnt);
    2837           6 :                         conn->data_out_cnt -= task->data_out_cnt;
    2838           6 :                         assert(conn->pending_r2t > 0);
    2839           6 :                         conn->pending_r2t--;
    2840             : 
    2841           6 :                         TAILQ_REMOVE(&conn->active_r2t_tasks, task, link);
    2842           6 :                         task->is_r2t_active = false;
    2843           6 :                         if (lun != NULL && spdk_scsi_lun_is_removing(lun)) {
    2844           5 :                                 spdk_scsi_task_process_null_lun(&task->scsi);
    2845           5 :                                 iscsi_task_response(conn, task);
    2846             :                         }
    2847           6 :                         iscsi_task_put(task);
    2848             :                 }
    2849             :         }
    2850             : 
    2851           7 :         TAILQ_FOREACH_SAFE(task, &conn->queued_r2t_tasks, link, task_tmp) {
    2852           3 :                 pdu_tmp = iscsi_task_get_pdu(task);
    2853           3 :                 if ((lun == NULL || lun == task->scsi.lun) &&
    2854           0 :                     (pdu == NULL || spdk_sn32_lt(pdu_tmp->cmd_sn, pdu->cmd_sn))) {
    2855           0 :                         TAILQ_REMOVE(&conn->queued_r2t_tasks, task, link);
    2856           0 :                         task->is_r2t_active = false;
    2857           0 :                         if (lun != NULL && spdk_scsi_lun_is_removing(lun)) {
    2858           0 :                                 spdk_scsi_task_process_null_lun(&task->scsi);
    2859           0 :                                 iscsi_task_response(conn, task);
    2860             :                         }
    2861           0 :                         iscsi_task_put(task);
    2862             :                 }
    2863             :         }
    2864             : 
    2865           4 :         start_queued_transfer_tasks(conn);
    2866           4 : }
    2867             : 
    2868             : static struct spdk_iscsi_task *
    2869          65 : get_transfer_task(struct spdk_iscsi_conn *conn, uint32_t transfer_tag)
    2870             : {
    2871             :         struct spdk_iscsi_task *task;
    2872             : 
    2873         121 :         TAILQ_FOREACH(task, &conn->active_r2t_tasks, link) {
    2874         105 :                 if (task->ttt == transfer_tag) {
    2875          49 :                         return task;
    2876             :                 }
    2877             :         }
    2878             : 
    2879          16 :         return NULL;
    2880             : }
    2881             : 
    2882             : static void
    2883           0 : iscsi_conn_datain_pdu_complete(void *arg)
    2884             : {
    2885           0 :         struct spdk_iscsi_conn *conn = arg;
    2886             : 
    2887           0 :         iscsi_conn_handle_queued_datain_tasks(conn);
    2888           0 : }
    2889             : 
    2890             : static int
    2891           2 : iscsi_send_datain(struct spdk_iscsi_conn *conn,
    2892             :                   struct spdk_iscsi_task *task, int datain_flag,
    2893             :                   int residual_len, int offset, int DataSN, int len)
    2894             : {
    2895             :         struct spdk_iscsi_pdu *rsp_pdu;
    2896             :         struct iscsi_bhs_data_in *rsph;
    2897             :         uint32_t task_tag;
    2898             :         uint32_t transfer_tag;
    2899             :         int F_bit, U_bit, O_bit, S_bit;
    2900             :         struct spdk_iscsi_task *primary;
    2901             :         struct spdk_scsi_lun *lun_dev;
    2902             : 
    2903           2 :         primary = iscsi_task_get_primary(task);
    2904             : 
    2905             :         /* DATA PDU */
    2906           2 :         rsp_pdu = iscsi_get_pdu(conn);
    2907           2 :         rsph = (struct iscsi_bhs_data_in *)&rsp_pdu->bhs;
    2908           2 :         rsp_pdu->data = task->scsi.iovs[0].iov_base + offset;
    2909           2 :         rsp_pdu->data_buf_len = task->scsi.iovs[0].iov_len - offset;
    2910           2 :         rsp_pdu->data_valid_bytes = len;
    2911           2 :         rsp_pdu->data_from_mempool = true;
    2912             : 
    2913           2 :         task_tag = task->tag;
    2914           2 :         transfer_tag = 0xffffffffU;
    2915             : 
    2916           2 :         F_bit = datain_flag & ISCSI_FLAG_FINAL;
    2917           2 :         O_bit = datain_flag & ISCSI_DATAIN_OVERFLOW;
    2918           2 :         U_bit = datain_flag & ISCSI_DATAIN_UNDERFLOW;
    2919           2 :         S_bit = datain_flag & ISCSI_DATAIN_STATUS;
    2920             : 
    2921             :         /*
    2922             :          * we need to hold onto this task/cmd because until the
    2923             :          * PDU has been written out
    2924             :          */
    2925           2 :         rsp_pdu->task = task;
    2926           2 :         task->scsi.ref++;
    2927             : 
    2928           2 :         rsph->opcode = ISCSI_OP_SCSI_DATAIN;
    2929             : 
    2930           2 :         if (F_bit) {
    2931           2 :                 rsph->flags |= ISCSI_FLAG_FINAL;
    2932             :         }
    2933             : 
    2934             :         /* we leave the A_bit clear */
    2935             : 
    2936           2 :         if (F_bit && S_bit)  {
    2937           1 :                 if (O_bit) {
    2938           0 :                         rsph->flags |= ISCSI_DATAIN_OVERFLOW;
    2939             :                 }
    2940             : 
    2941           1 :                 if (U_bit) {
    2942           1 :                         rsph->flags |= ISCSI_DATAIN_UNDERFLOW;
    2943             :                 }
    2944             :         }
    2945             : 
    2946           2 :         if (S_bit) {
    2947           1 :                 rsph->flags |= ISCSI_DATAIN_STATUS;
    2948           1 :                 rsph->status = task->scsi.status;
    2949             :         }
    2950             : 
    2951           2 :         DSET24(rsph->data_segment_len, len);
    2952             : 
    2953           2 :         to_be32(&rsph->itt, task_tag);
    2954           2 :         to_be32(&rsph->ttt, transfer_tag);
    2955             : 
    2956           2 :         if (S_bit) {
    2957           1 :                 to_be32(&rsph->stat_sn, conn->StatSN);
    2958           1 :                 conn->StatSN++;
    2959             :         }
    2960             : 
    2961           2 :         if (F_bit && S_bit && !iscsi_task_is_immediate(primary)) {
    2962           1 :                 conn->sess->MaxCmdSN++;
    2963             :         }
    2964             : 
    2965           2 :         to_be32(&rsph->exp_cmd_sn, conn->sess->ExpCmdSN);
    2966           2 :         to_be32(&rsph->max_cmd_sn, conn->sess->MaxCmdSN);
    2967             : 
    2968           2 :         to_be32(&rsph->data_sn, DataSN);
    2969             : 
    2970           2 :         if (conn->sess->ErrorRecoveryLevel >= 1) {
    2971           0 :                 primary->datain_datasn = DataSN;
    2972             :         }
    2973           2 :         DataSN++;
    2974             : 
    2975           2 :         offset += task->scsi.offset;
    2976           2 :         to_be32(&rsph->buffer_offset, (uint32_t)offset);
    2977             : 
    2978           2 :         if (F_bit && S_bit) {
    2979           1 :                 to_be32(&rsph->res_cnt, residual_len);
    2980             :         }
    2981             : 
    2982           2 :         lun_dev = spdk_scsi_dev_get_lun(conn->dev, task->lun_id);
    2983           2 :         if (spdk_likely(lun_dev != NULL)) {
    2984           2 :                 if (spdk_unlikely(spdk_scsi_lun_get_dif_ctx(lun_dev, &task->scsi,
    2985             :                                   &rsp_pdu->dif_ctx))) {
    2986           0 :                         rsp_pdu->dif_insert_or_strip = true;
    2987             :                 }
    2988             :         }
    2989             : 
    2990           2 :         iscsi_conn_write_pdu(conn, rsp_pdu, iscsi_conn_datain_pdu_complete, conn);
    2991             : 
    2992           2 :         return DataSN;
    2993             : }
    2994             : 
    2995             : static int
    2996           4 : iscsi_transfer_in(struct spdk_iscsi_conn *conn, struct spdk_iscsi_task *task)
    2997             : {
    2998             :         uint32_t DataSN;
    2999             :         uint32_t transfer_len;
    3000             :         uint32_t data_len;
    3001             :         uint32_t segment_len;
    3002             :         uint32_t offset;
    3003           4 :         uint32_t residual_len = 0;
    3004             :         int sent_status;
    3005             :         uint32_t len;
    3006           4 :         int datain_flag = 0;
    3007             :         int datain_seq_cnt;
    3008             :         int i;
    3009             :         uint32_t sequence_end;
    3010             :         struct spdk_iscsi_task *primary;
    3011             : 
    3012           4 :         primary = iscsi_task_get_primary(task);
    3013           4 :         segment_len = conn->MaxRecvDataSegmentLength;
    3014           4 :         data_len = task->scsi.data_transferred;
    3015           4 :         transfer_len = task->scsi.length;
    3016             : 
    3017           4 :         if (task->scsi.status != SPDK_SCSI_STATUS_GOOD) {
    3018           1 :                 return 0;
    3019             :         }
    3020             : 
    3021           3 :         if (data_len < transfer_len) {
    3022             :                 /* underflow */
    3023           3 :                 SPDK_DEBUGLOG(iscsi, "Underflow %u/%u\n", data_len, transfer_len);
    3024           3 :                 residual_len = transfer_len - data_len;
    3025           3 :                 transfer_len = data_len;
    3026           3 :                 datain_flag |= ISCSI_DATAIN_UNDERFLOW;
    3027           0 :         } else if (data_len > transfer_len) {
    3028             :                 /* overflow */
    3029           0 :                 SPDK_DEBUGLOG(iscsi, "Overflow %u/%u\n", data_len, transfer_len);
    3030           0 :                 residual_len = data_len - transfer_len;
    3031           0 :                 datain_flag |= ISCSI_DATAIN_OVERFLOW;
    3032             :         } else {
    3033           0 :                 SPDK_DEBUGLOG(iscsi, "Transfer %u\n", transfer_len);
    3034           0 :                 residual_len = 0;
    3035             :         }
    3036             : 
    3037           3 :         DataSN = primary->datain_datasn;
    3038           3 :         sent_status = 0;
    3039             : 
    3040             :         /* calculate the number of sequences for all data-in pdus */
    3041           3 :         datain_seq_cnt = 1 + ((transfer_len - 1) / (int)conn->sess->MaxBurstLength);
    3042        4101 :         for (i = 0; i < datain_seq_cnt; i++) {
    3043        4098 :                 offset = i * conn->sess->MaxBurstLength;
    3044        4098 :                 sequence_end = spdk_min(((i + 1) * conn->sess->MaxBurstLength),
    3045             :                                         transfer_len);
    3046             : 
    3047             :                 /* send data split by segment_len */
    3048        4100 :                 for (; offset < sequence_end; offset += segment_len) {
    3049           2 :                         len = spdk_min(segment_len, (sequence_end - offset));
    3050             : 
    3051           2 :                         datain_flag &= ~(ISCSI_FLAG_FINAL | ISCSI_DATAIN_STATUS);
    3052             : 
    3053           2 :                         if (offset + len == sequence_end) {
    3054             :                                 /* last PDU in a sequence */
    3055           2 :                                 datain_flag |= ISCSI_FLAG_FINAL;
    3056           2 :                                 if (task->scsi.sense_data_len == 0) {
    3057             :                                         /* The last pdu in all data-in pdus */
    3058           1 :                                         if ((offset + len) == transfer_len &&
    3059           1 :                                             (primary->bytes_completed == primary->scsi.transfer_len)) {
    3060           1 :                                                 datain_flag |= ISCSI_DATAIN_STATUS;
    3061           1 :                                                 sent_status = 1;
    3062             :                                         }
    3063             :                                 }
    3064             :                         }
    3065             : 
    3066           2 :                         SPDK_DEBUGLOG(iscsi, "Transfer=%d, Offset=%d, Len=%d\n",
    3067             :                                       sequence_end, offset, len);
    3068           2 :                         SPDK_DEBUGLOG(iscsi, "StatSN=%u, DataSN=%u, Offset=%u, Len=%d\n",
    3069             :                                       conn->StatSN, DataSN, offset, len);
    3070             : 
    3071           2 :                         DataSN = iscsi_send_datain(conn, task, datain_flag, residual_len,
    3072             :                                                    offset, DataSN, len);
    3073             :                 }
    3074             :         }
    3075             : 
    3076           3 :         if (task != primary) {
    3077           0 :                 primary->scsi.data_transferred += task->scsi.data_transferred;
    3078             :         }
    3079           3 :         primary->datain_datasn = DataSN;
    3080             : 
    3081           3 :         return sent_status;
    3082             : }
    3083             : 
    3084             : void
    3085           9 : iscsi_task_response(struct spdk_iscsi_conn *conn,
    3086             :                     struct spdk_iscsi_task *task)
    3087             : {
    3088             :         struct spdk_iscsi_pdu *rsp_pdu;
    3089             :         struct iscsi_bhs_scsi_resp *rsph;
    3090             :         uint32_t task_tag;
    3091             :         uint32_t transfer_len;
    3092             :         size_t residual_len;
    3093             :         size_t data_len;
    3094             :         int O_bit, U_bit;
    3095             :         int rc;
    3096             :         struct spdk_iscsi_task *primary;
    3097             : 
    3098           9 :         primary = iscsi_task_get_primary(task);
    3099             : 
    3100           9 :         transfer_len = primary->scsi.transfer_len;
    3101           9 :         task_tag = task->tag;
    3102             : 
    3103             :         /* transfer data from logical unit */
    3104             :         /* (direction is view of initiator side) */
    3105           9 :         if (iscsi_task_is_read(primary)) {
    3106           4 :                 rc = iscsi_transfer_in(conn, task);
    3107           4 :                 if (rc > 0) {
    3108             :                         /* sent status by last DATAIN PDU */
    3109           1 :                         return;
    3110             :                 }
    3111             : 
    3112           3 :                 if (primary->bytes_completed != primary->scsi.transfer_len) {
    3113           0 :                         return;
    3114             :                 }
    3115             :         }
    3116             : 
    3117           8 :         O_bit = U_bit = 0;
    3118           8 :         residual_len = 0;
    3119           8 :         data_len = primary->scsi.data_transferred;
    3120             : 
    3121           8 :         if ((transfer_len != 0) &&
    3122           8 :             (task->scsi.status == SPDK_SCSI_STATUS_GOOD)) {
    3123           7 :                 if (data_len < transfer_len) {
    3124             :                         /* underflow */
    3125           7 :                         SPDK_DEBUGLOG(iscsi, "Underflow %zu/%u\n", data_len, transfer_len);
    3126           7 :                         residual_len = transfer_len - data_len;
    3127           7 :                         U_bit = 1;
    3128           0 :                 } else if (data_len > transfer_len) {
    3129             :                         /* overflow */
    3130           0 :                         SPDK_DEBUGLOG(iscsi, "Overflow %zu/%u\n", data_len, transfer_len);
    3131           0 :                         residual_len = data_len - transfer_len;
    3132           0 :                         O_bit = 1;
    3133             :                 } else {
    3134           0 :                         SPDK_DEBUGLOG(iscsi, "Transfer %u\n", transfer_len);
    3135             :                 }
    3136             :         }
    3137             : 
    3138             :         /* response PDU */
    3139           8 :         rsp_pdu = iscsi_get_pdu(conn);
    3140           8 :         assert(rsp_pdu != NULL);
    3141           8 :         rsph = (struct iscsi_bhs_scsi_resp *)&rsp_pdu->bhs;
    3142           8 :         assert(task->scsi.sense_data_len <= sizeof(rsp_pdu->sense.data));
    3143           8 :         memcpy(rsp_pdu->sense.data, task->scsi.sense_data, task->scsi.sense_data_len);
    3144           8 :         to_be16(&rsp_pdu->sense.length, task->scsi.sense_data_len);
    3145           8 :         rsp_pdu->data = (uint8_t *)&rsp_pdu->sense;
    3146           8 :         rsp_pdu->data_from_mempool = true;
    3147             : 
    3148             :         /*
    3149             :          * we need to hold onto this task/cmd because until the
    3150             :          * PDU has been written out
    3151             :          */
    3152           8 :         rsp_pdu->task = task;
    3153           8 :         task->scsi.ref++;
    3154             : 
    3155           8 :         rsph->opcode = ISCSI_OP_SCSI_RSP;
    3156           8 :         rsph->flags |= 0x80; /* bit 0 is default to 1 */
    3157             : 
    3158           8 :         if (O_bit) {
    3159           0 :                 rsph->flags |= ISCSI_SCSI_OVERFLOW;
    3160             :         }
    3161             : 
    3162           8 :         if (U_bit) {
    3163           7 :                 rsph->flags |= ISCSI_SCSI_UNDERFLOW;
    3164             :         }
    3165             : 
    3166           8 :         rsph->status = task->scsi.status;
    3167           8 :         if (task->scsi.sense_data_len) {
    3168             :                 /* SenseLength (2 bytes) + SenseData  */
    3169           2 :                 DSET24(rsph->data_segment_len, 2 + task->scsi.sense_data_len);
    3170             :         }
    3171           8 :         to_be32(&rsph->itt, task_tag);
    3172             : 
    3173           8 :         to_be32(&rsph->stat_sn, conn->StatSN);
    3174           8 :         conn->StatSN++;
    3175             : 
    3176           8 :         if (!iscsi_task_is_immediate(primary)) {
    3177           8 :                 conn->sess->MaxCmdSN++;
    3178             :         }
    3179             : 
    3180           8 :         to_be32(&rsph->exp_cmd_sn, conn->sess->ExpCmdSN);
    3181           8 :         to_be32(&rsph->max_cmd_sn, conn->sess->MaxCmdSN);
    3182             : 
    3183           8 :         to_be32(&rsph->bi_read_res_cnt, 0);
    3184           8 :         to_be32(&rsph->res_cnt, residual_len);
    3185             : 
    3186           8 :         iscsi_conn_write_pdu(conn, rsp_pdu, iscsi_conn_pdu_generic_complete, NULL);
    3187             : }
    3188             : 
    3189             : /*
    3190             :  *  This function compare the input pdu's bhs with the pdu's bhs associated by
    3191             :  *  active_r2t_tasks and queued_r2t_tasks in a connection
    3192             :  */
    3193             : static bool
    3194           0 : iscsi_compare_pdu_bhs_within_existed_r2t_tasks(struct spdk_iscsi_conn *conn,
    3195             :                 struct spdk_iscsi_pdu *pdu)
    3196             : {
    3197             :         struct spdk_iscsi_task  *task;
    3198             : 
    3199           0 :         TAILQ_FOREACH(task, &conn->active_r2t_tasks, link) {
    3200           0 :                 if (!memcmp(&pdu->bhs, iscsi_task_get_bhs(task), ISCSI_BHS_LEN)) {
    3201           0 :                         return true;
    3202             :                 }
    3203             :         }
    3204             : 
    3205           0 :         TAILQ_FOREACH(task, &conn->queued_r2t_tasks, link) {
    3206           0 :                 if (!memcmp(&pdu->bhs, iscsi_task_get_bhs(task), ISCSI_BHS_LEN)) {
    3207           0 :                         return true;
    3208             :                 }
    3209             :         }
    3210             : 
    3211           0 :         return false;
    3212             : }
    3213             : 
    3214             : void
    3215           4 : iscsi_queue_task(struct spdk_iscsi_conn *conn, struct spdk_iscsi_task *task)
    3216             : {
    3217           4 :         spdk_trace_record(TRACE_ISCSI_TASK_QUEUE, conn->trace_id, task->scsi.length,
    3218             :                           (uintptr_t)task, (uintptr_t)task->pdu);
    3219           4 :         task->is_queued = true;
    3220           4 :         spdk_scsi_dev_queue_task(conn->dev, &task->scsi);
    3221           4 : }
    3222             : 
    3223             : static int
    3224           0 : iscsi_pdu_payload_op_scsi_read(struct spdk_iscsi_conn *conn, struct spdk_iscsi_task *task)
    3225             : {
    3226           0 :         if (task->scsi.transfer_len <= SPDK_BDEV_LARGE_BUF_MAX_SIZE) {
    3227           0 :                 task->parent = NULL;
    3228           0 :                 task->scsi.offset = 0;
    3229           0 :                 task->scsi.length = task->scsi.transfer_len;
    3230           0 :                 spdk_scsi_task_set_data(&task->scsi, NULL, 0);
    3231             : 
    3232           0 :                 iscsi_queue_task(conn, task);
    3233           0 :                 return 0;
    3234             :         } else {
    3235           0 :                 TAILQ_INIT(&task->subtask_list);
    3236           0 :                 task->current_data_offset = 0;
    3237           0 :                 TAILQ_INSERT_TAIL(&conn->queued_datain_tasks, task, link);
    3238             : 
    3239           0 :                 return iscsi_conn_handle_queued_datain_tasks(conn);
    3240             :         }
    3241             : }
    3242             : 
    3243             : static int
    3244           4 : iscsi_submit_write_subtask(struct spdk_iscsi_conn *conn, struct spdk_iscsi_task *task,
    3245             :                            struct spdk_iscsi_pdu *pdu, struct spdk_mobj *mobj)
    3246             : {
    3247             :         struct spdk_iscsi_task *subtask;
    3248             : 
    3249           4 :         subtask = iscsi_task_get(conn, task, iscsi_task_cpl);
    3250           4 :         if (subtask == NULL) {
    3251           0 :                 SPDK_ERRLOG("Unable to acquire subtask\n");
    3252           0 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    3253             :         }
    3254           4 :         subtask->scsi.offset = task->current_data_offset;
    3255           4 :         subtask->scsi.length = mobj->data_len;
    3256           4 :         iscsi_task_associate_pdu(subtask, pdu);
    3257             : 
    3258           4 :         task->current_data_offset += mobj->data_len;
    3259             : 
    3260           4 :         if (spdk_likely(!pdu->dif_insert_or_strip)) {
    3261           4 :                 spdk_scsi_task_set_data(&subtask->scsi, mobj->buf, mobj->data_len);
    3262             :         } else {
    3263           0 :                 spdk_scsi_task_set_data(&subtask->scsi, mobj->buf, pdu->data_buf_len);
    3264             :         }
    3265             : 
    3266           4 :         iscsi_queue_task(conn, subtask);
    3267           4 :         return 0;
    3268             : }
    3269             : 
    3270             : static int
    3271           2 : iscsi_pdu_payload_op_scsi_write(struct spdk_iscsi_conn *conn, struct spdk_iscsi_task *task)
    3272             : {
    3273             :         struct spdk_iscsi_pdu *pdu;
    3274             :         struct iscsi_bhs_scsi_req *reqh;
    3275             :         uint32_t transfer_len;
    3276             :         struct spdk_mobj *mobj;
    3277             :         int rc;
    3278             : 
    3279           2 :         pdu = iscsi_task_get_pdu(task);
    3280           2 :         reqh = (struct iscsi_bhs_scsi_req *)&pdu->bhs;
    3281             : 
    3282           2 :         transfer_len = task->scsi.transfer_len;
    3283             : 
    3284           2 :         if (reqh->final_bit &&
    3285           2 :             pdu->data_segment_len < transfer_len) {
    3286             :                 /* needs R2T */
    3287           2 :                 rc = add_transfer_task(conn, task);
    3288           2 :                 if (rc < 0) {
    3289           0 :                         SPDK_ERRLOG("add_transfer_task() failed\n");
    3290           0 :                         iscsi_task_put(task);
    3291           0 :                         return SPDK_ISCSI_CONNECTION_FATAL;
    3292             :                 }
    3293             : 
    3294             :                 /* immediate writes */
    3295           2 :                 if (pdu->data_segment_len != 0) {
    3296           1 :                         mobj = pdu->mobj[0];
    3297           1 :                         assert(mobj != NULL);
    3298             : 
    3299           1 :                         if (!pdu->dif_insert_or_strip &&
    3300           1 :                             mobj->data_len < SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH) {
    3301             :                                 /* continue aggregation until the first data buffer is full. */
    3302           1 :                                 iscsi_task_set_mobj(task, mobj);
    3303           1 :                                 pdu->mobj[0] = NULL;
    3304             :                         } else {
    3305             :                                 /* we are doing the first partial write task */
    3306           0 :                                 rc = iscsi_submit_write_subtask(conn, task, pdu, mobj);
    3307           0 :                                 if (rc < 0) {
    3308           0 :                                         iscsi_task_put(task);
    3309           0 :                                         return SPDK_ISCSI_CONNECTION_FATAL;
    3310             :                                 }
    3311             :                         }
    3312             :                 }
    3313           2 :                 return 0;
    3314             :         }
    3315             : 
    3316           0 :         if (pdu->data_segment_len == transfer_len) {
    3317             :                 /* we are doing small writes with no R2T */
    3318           0 :                 if (spdk_likely(!pdu->dif_insert_or_strip)) {
    3319           0 :                         spdk_scsi_task_set_data(&task->scsi, pdu->data, pdu->data_segment_len);
    3320             :                 } else {
    3321           0 :                         spdk_scsi_task_set_data(&task->scsi, pdu->data, pdu->data_buf_len);
    3322             :                 }
    3323           0 :                 task->scsi.length = transfer_len;
    3324             :         }
    3325             : 
    3326           0 :         iscsi_queue_task(conn, task);
    3327           0 :         return 0;
    3328             : }
    3329             : 
    3330             : static int
    3331          14 : iscsi_pdu_hdr_op_scsi(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    3332             : {
    3333             :         struct spdk_iscsi_task  *task;
    3334             :         struct spdk_scsi_dev    *dev;
    3335             :         uint8_t *cdb;
    3336             :         uint64_t lun;
    3337             :         uint32_t task_tag;
    3338             :         uint32_t transfer_len;
    3339             :         int R_bit, W_bit;
    3340             :         int lun_i;
    3341             :         struct iscsi_bhs_scsi_req *reqh;
    3342             : 
    3343          14 :         if (conn->sess->session_type != SESSION_TYPE_NORMAL) {
    3344           2 :                 SPDK_ERRLOG("ISCSI_OP_SCSI not allowed in discovery and invalid session\n");
    3345           2 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    3346             :         }
    3347             : 
    3348          12 :         reqh = (struct iscsi_bhs_scsi_req *)&pdu->bhs;
    3349             : 
    3350          12 :         R_bit = reqh->read_bit;
    3351          12 :         W_bit = reqh->write_bit;
    3352          12 :         lun = from_be64(&reqh->lun);
    3353          12 :         task_tag = from_be32(&reqh->itt);
    3354          12 :         transfer_len = from_be32(&reqh->expected_data_xfer_len);
    3355          12 :         cdb = reqh->cdb;
    3356             : 
    3357          12 :         SPDK_LOGDUMP(iscsi, "CDB", cdb, 16);
    3358             : 
    3359          12 :         task = iscsi_task_get(conn, NULL, iscsi_task_cpl);
    3360          12 :         if (!task) {
    3361           0 :                 SPDK_ERRLOG("Unable to acquire task\n");
    3362           0 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    3363             :         }
    3364             : 
    3365          12 :         iscsi_task_associate_pdu(task, pdu);
    3366          12 :         lun_i = spdk_scsi_lun_id_fmt_to_int(lun);
    3367          12 :         task->lun_id = lun_i;
    3368          12 :         dev = conn->dev;
    3369          12 :         task->scsi.lun = spdk_scsi_dev_get_lun(dev, lun_i);
    3370             : 
    3371          12 :         if ((R_bit != 0) && (W_bit != 0)) {
    3372           1 :                 SPDK_ERRLOG("Bidirectional CDB is not supported\n");
    3373           1 :                 iscsi_task_put(task);
    3374           1 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    3375             :         }
    3376             : 
    3377          11 :         task->scsi.cdb = cdb;
    3378          11 :         task->tag = task_tag;
    3379          11 :         task->scsi.transfer_len = transfer_len;
    3380          11 :         task->scsi.target_port = conn->target_port;
    3381          11 :         task->scsi.initiator_port = conn->initiator_port;
    3382          11 :         task->parent = NULL;
    3383          11 :         task->scsi.status = SPDK_SCSI_STATUS_GOOD;
    3384             : 
    3385          11 :         if (task->scsi.lun == NULL) {
    3386           1 :                 spdk_scsi_task_process_null_lun(&task->scsi);
    3387           1 :                 iscsi_task_cpl(&task->scsi);
    3388           1 :                 return 0;
    3389             :         }
    3390             : 
    3391             :         /* no bi-directional support */
    3392          10 :         if (R_bit) {
    3393           1 :                 task->scsi.dxfer_dir = SPDK_SCSI_DIR_FROM_DEV;
    3394           9 :         } else if (W_bit) {
    3395           7 :                 task->scsi.dxfer_dir = SPDK_SCSI_DIR_TO_DEV;
    3396             : 
    3397           7 :                 if ((conn->sess->ErrorRecoveryLevel >= 1) &&
    3398           0 :                     (iscsi_compare_pdu_bhs_within_existed_r2t_tasks(conn, pdu))) {
    3399           0 :                         iscsi_task_response(conn, task);
    3400           0 :                         iscsi_task_put(task);
    3401           0 :                         return 0;
    3402             :                 }
    3403             : 
    3404           7 :                 if (pdu->data_segment_len > iscsi_get_max_immediate_data_size()) {
    3405           1 :                         SPDK_ERRLOG("data segment len(=%zu) > immediate data len(=%"PRIu32")\n",
    3406             :                                     pdu->data_segment_len, iscsi_get_max_immediate_data_size());
    3407           1 :                         iscsi_task_put(task);
    3408           1 :                         return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    3409             :                 }
    3410             : 
    3411           6 :                 if (pdu->data_segment_len > transfer_len) {
    3412           1 :                         SPDK_ERRLOG("data segment len(=%zu) > task transfer len(=%d)\n",
    3413             :                                     pdu->data_segment_len, transfer_len);
    3414           1 :                         iscsi_task_put(task);
    3415           1 :                         return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    3416             :                 }
    3417             : 
    3418             :                 /* check the ImmediateData and also pdu->data_segment_len */
    3419           5 :                 if ((!conn->sess->ImmediateData && (pdu->data_segment_len > 0)) ||
    3420           4 :                     (pdu->data_segment_len > conn->sess->FirstBurstLength)) {
    3421           2 :                         iscsi_task_put(task);
    3422           2 :                         return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    3423             :                 }
    3424             : 
    3425           3 :                 if (spdk_unlikely(spdk_scsi_lun_get_dif_ctx(task->scsi.lun, &task->scsi, &pdu->dif_ctx))) {
    3426           0 :                         pdu->dif_insert_or_strip = true;
    3427           3 :                 } else if (reqh->final_bit && pdu->data_segment_len < transfer_len) {
    3428           2 :                         pdu->data_buf_len = spdk_min(transfer_len,
    3429             :                                                      SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH);
    3430             :                 }
    3431             :         } else {
    3432             :                 /* neither R nor W bit set */
    3433           2 :                 task->scsi.dxfer_dir = SPDK_SCSI_DIR_NONE;
    3434           2 :                 if (transfer_len > 0) {
    3435           1 :                         iscsi_task_put(task);
    3436           1 :                         SPDK_ERRLOG("Reject scsi cmd with EDTL > 0 but (R | W) == 0\n");
    3437           1 :                         return iscsi_reject(conn, pdu, ISCSI_REASON_INVALID_PDU_FIELD);
    3438             :                 }
    3439             :         }
    3440             : 
    3441           5 :         pdu->task = task;
    3442           5 :         return 0;
    3443             : }
    3444             : 
    3445             : static int
    3446           2 : iscsi_pdu_payload_op_scsi(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    3447             : {
    3448             :         struct spdk_iscsi_task *task;
    3449             : 
    3450           2 :         if (pdu->task == NULL) {
    3451           0 :                 return 0;
    3452             :         }
    3453             : 
    3454           2 :         task = pdu->task;
    3455             : 
    3456           2 :         if (spdk_scsi_dev_get_lun(conn->dev, task->lun_id) == NULL) {
    3457           0 :                 spdk_scsi_task_process_null_lun(&task->scsi);
    3458           0 :                 iscsi_task_cpl(&task->scsi);
    3459           0 :                 return 0;
    3460             :         }
    3461             : 
    3462           2 :         switch (task->scsi.dxfer_dir) {
    3463           0 :         case SPDK_SCSI_DIR_FROM_DEV:
    3464           0 :                 return iscsi_pdu_payload_op_scsi_read(conn, task);
    3465           2 :         case SPDK_SCSI_DIR_TO_DEV:
    3466           2 :                 return iscsi_pdu_payload_op_scsi_write(conn, task);
    3467           0 :         case SPDK_SCSI_DIR_NONE:
    3468           0 :                 iscsi_queue_task(conn, task);
    3469           0 :                 return 0;
    3470           0 :         default:
    3471           0 :                 assert(false);
    3472             :                 iscsi_task_put(task);
    3473             :                 break;
    3474             :         }
    3475             : 
    3476             :         return SPDK_ISCSI_CONNECTION_FATAL;
    3477             : }
    3478             : 
    3479             : void
    3480           7 : iscsi_task_mgmt_response(struct spdk_iscsi_conn *conn,
    3481             :                          struct spdk_iscsi_task *task)
    3482             : {
    3483             :         struct spdk_iscsi_pdu *rsp_pdu;
    3484             :         struct iscsi_bhs_task_req *reqh;
    3485             :         struct iscsi_bhs_task_resp *rsph;
    3486             : 
    3487           7 :         if (task->pdu == NULL) {
    3488             :                 /*
    3489             :                  * This was an internally generated task management command,
    3490             :                  *  usually from LUN cleanup when a connection closes.
    3491             :                  */
    3492           0 :                 return;
    3493             :         }
    3494             : 
    3495           7 :         reqh = (struct iscsi_bhs_task_req *)&task->pdu->bhs;
    3496             :         /* response PDU */
    3497           7 :         rsp_pdu = iscsi_get_pdu(conn);
    3498           7 :         rsph = (struct iscsi_bhs_task_resp *)&rsp_pdu->bhs;
    3499           7 :         rsph->opcode = ISCSI_OP_TASK_RSP;
    3500           7 :         rsph->flags |= 0x80; /* bit 0 default to 1 */
    3501           7 :         switch (task->scsi.response) {
    3502           0 :         case SPDK_SCSI_TASK_MGMT_RESP_COMPLETE:
    3503           0 :                 rsph->response = ISCSI_TASK_FUNC_RESP_COMPLETE;
    3504           0 :                 break;
    3505           0 :         case SPDK_SCSI_TASK_MGMT_RESP_SUCCESS:
    3506           0 :                 rsph->response = ISCSI_TASK_FUNC_RESP_COMPLETE;
    3507           0 :                 break;
    3508           1 :         case SPDK_SCSI_TASK_MGMT_RESP_REJECT:
    3509           1 :                 rsph->response = ISCSI_TASK_FUNC_REJECTED;
    3510           1 :                 break;
    3511           1 :         case SPDK_SCSI_TASK_MGMT_RESP_INVALID_LUN:
    3512           1 :                 rsph->response = ISCSI_TASK_FUNC_RESP_LUN_NOT_EXIST;
    3513           1 :                 break;
    3514           0 :         case SPDK_SCSI_TASK_MGMT_RESP_TARGET_FAILURE:
    3515           0 :                 rsph->response = ISCSI_TASK_FUNC_REJECTED;
    3516           0 :                 break;
    3517           5 :         case SPDK_SCSI_TASK_MGMT_RESP_REJECT_FUNC_NOT_SUPPORTED:
    3518           5 :                 rsph->response = ISCSI_TASK_FUNC_RESP_FUNC_NOT_SUPPORTED;
    3519           5 :                 break;
    3520             :         }
    3521           7 :         rsph->itt = reqh->itt;
    3522             : 
    3523           7 :         to_be32(&rsph->stat_sn, conn->StatSN);
    3524           7 :         conn->StatSN++;
    3525             : 
    3526           7 :         if (reqh->immediate == 0) {
    3527           7 :                 conn->sess->MaxCmdSN++;
    3528             :         }
    3529             : 
    3530           7 :         to_be32(&rsph->exp_cmd_sn, conn->sess->ExpCmdSN);
    3531           7 :         to_be32(&rsph->max_cmd_sn, conn->sess->MaxCmdSN);
    3532             : 
    3533           7 :         iscsi_conn_write_pdu(conn, rsp_pdu, iscsi_conn_pdu_generic_complete, NULL);
    3534             : }
    3535             : 
    3536             : static void
    3537           0 : iscsi_queue_mgmt_task(struct spdk_iscsi_conn *conn, struct spdk_iscsi_task *task)
    3538             : {
    3539             :         struct spdk_scsi_lun *lun;
    3540             : 
    3541           0 :         lun = spdk_scsi_dev_get_lun(conn->dev, task->lun_id);
    3542           0 :         if (lun == NULL) {
    3543           0 :                 task->scsi.response = SPDK_SCSI_TASK_MGMT_RESP_INVALID_LUN;
    3544           0 :                 iscsi_task_mgmt_response(conn, task);
    3545           0 :                 iscsi_task_put(task);
    3546           0 :                 return;
    3547             :         }
    3548             : 
    3549           0 :         spdk_scsi_dev_queue_mgmt_task(conn->dev, &task->scsi);
    3550             : }
    3551             : 
    3552             : static int
    3553           0 : _iscsi_op_abort_task(void *arg)
    3554             : {
    3555           0 :         struct spdk_iscsi_task *task = arg;
    3556             :         int rc;
    3557             : 
    3558           0 :         rc = iscsi_conn_abort_queued_datain_task(task->conn, task->scsi.abort_id);
    3559           0 :         if (rc != 0) {
    3560           0 :                 return SPDK_POLLER_BUSY;
    3561             :         }
    3562             : 
    3563           0 :         spdk_poller_unregister(&task->mgmt_poller);
    3564           0 :         iscsi_queue_mgmt_task(task->conn, task);
    3565           0 :         return SPDK_POLLER_BUSY;
    3566             : }
    3567             : 
    3568             : static void
    3569           0 : iscsi_op_abort_task(struct spdk_iscsi_task *task, uint32_t ref_task_tag)
    3570             : {
    3571           0 :         task->scsi.abort_id = ref_task_tag;
    3572           0 :         task->scsi.function = SPDK_SCSI_TASK_FUNC_ABORT_TASK;
    3573           0 :         task->mgmt_poller = SPDK_POLLER_REGISTER(_iscsi_op_abort_task, task, 10);
    3574           0 : }
    3575             : 
    3576             : static int
    3577           0 : _iscsi_op_abort_task_set(void *arg)
    3578             : {
    3579           0 :         struct spdk_iscsi_task *task = arg;
    3580             :         int rc;
    3581             : 
    3582           0 :         rc = iscsi_conn_abort_queued_datain_tasks(task->conn, task->scsi.lun,
    3583             :                         task->pdu);
    3584           0 :         if (rc != 0) {
    3585           0 :                 return SPDK_POLLER_BUSY;
    3586             :         }
    3587             : 
    3588           0 :         spdk_poller_unregister(&task->mgmt_poller);
    3589           0 :         iscsi_queue_mgmt_task(task->conn, task);
    3590           0 :         return SPDK_POLLER_BUSY;
    3591             : }
    3592             : 
    3593             : void
    3594           0 : iscsi_op_abort_task_set(struct spdk_iscsi_task *task, uint8_t function)
    3595             : {
    3596           0 :         task->scsi.function = function;
    3597           0 :         task->mgmt_poller = SPDK_POLLER_REGISTER(_iscsi_op_abort_task_set, task, 10);
    3598           0 : }
    3599             : 
    3600             : static int
    3601           8 : iscsi_pdu_hdr_op_task(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    3602             : {
    3603             :         struct iscsi_bhs_task_req *reqh;
    3604             :         uint64_t lun;
    3605             :         uint32_t task_tag;
    3606             :         uint32_t ref_task_tag;
    3607             :         uint8_t function;
    3608             :         int lun_i;
    3609             :         struct spdk_iscsi_task *task;
    3610             :         struct spdk_scsi_dev *dev;
    3611             : 
    3612           8 :         if (conn->sess->session_type != SESSION_TYPE_NORMAL) {
    3613           1 :                 SPDK_ERRLOG("ISCSI_OP_TASK not allowed in discovery and invalid session\n");
    3614           1 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    3615             :         }
    3616             : 
    3617           7 :         reqh = (struct iscsi_bhs_task_req *)&pdu->bhs;
    3618           7 :         function = reqh->flags & ISCSI_TASK_FUNCTION_MASK;
    3619           7 :         lun = from_be64(&reqh->lun);
    3620           7 :         task_tag = from_be32(&reqh->itt);
    3621           7 :         ref_task_tag = from_be32(&reqh->ref_task_tag);
    3622             : 
    3623           7 :         SPDK_DEBUGLOG(iscsi, "I=%d, func=%d, ITT=%x, ref TT=%x, LUN=0x%16.16"PRIx64"\n",
    3624             :                       reqh->immediate, function, task_tag, ref_task_tag, lun);
    3625             : 
    3626           7 :         SPDK_DEBUGLOG(iscsi, "StatSN=%u, ExpCmdSN=%u, MaxCmdSN=%u\n",
    3627             :                       conn->StatSN, conn->sess->ExpCmdSN, conn->sess->MaxCmdSN);
    3628             : 
    3629           7 :         lun_i = spdk_scsi_lun_id_fmt_to_int(lun);
    3630           7 :         dev = conn->dev;
    3631             : 
    3632           7 :         task = iscsi_task_get(conn, NULL, iscsi_task_mgmt_cpl);
    3633           7 :         if (!task) {
    3634           0 :                 SPDK_ERRLOG("Unable to acquire task\n");
    3635           0 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    3636             :         }
    3637             : 
    3638           7 :         iscsi_task_associate_pdu(task, pdu);
    3639           7 :         task->scsi.target_port = conn->target_port;
    3640           7 :         task->scsi.initiator_port = conn->initiator_port;
    3641           7 :         task->tag = task_tag;
    3642           7 :         task->scsi.lun = spdk_scsi_dev_get_lun(dev, lun_i);
    3643           7 :         task->lun_id = lun_i;
    3644             : 
    3645           7 :         if (task->scsi.lun == NULL) {
    3646           1 :                 task->scsi.response = SPDK_SCSI_TASK_MGMT_RESP_INVALID_LUN;
    3647           1 :                 iscsi_task_mgmt_response(conn, task);
    3648           1 :                 iscsi_task_put(task);
    3649           1 :                 return 0;
    3650             :         }
    3651             : 
    3652           6 :         switch (function) {
    3653             :         /* abort task identified by Referenced Task Tag field */
    3654           0 :         case ISCSI_TASK_FUNC_ABORT_TASK:
    3655           0 :                 SPDK_NOTICELOG("ABORT_TASK\n");
    3656             : 
    3657           0 :                 iscsi_del_transfer_task(conn, ref_task_tag);
    3658           0 :                 iscsi_op_abort_task(task, ref_task_tag);
    3659           0 :                 return 0;
    3660             : 
    3661             :         /* abort all tasks issued via this session on the LUN */
    3662           0 :         case ISCSI_TASK_FUNC_ABORT_TASK_SET:
    3663           0 :                 SPDK_NOTICELOG("ABORT_TASK_SET\n");
    3664             : 
    3665           0 :                 iscsi_clear_all_transfer_task(conn, task->scsi.lun, pdu);
    3666           0 :                 iscsi_op_abort_task_set(task, SPDK_SCSI_TASK_FUNC_ABORT_TASK_SET);
    3667           0 :                 return 0;
    3668             : 
    3669           1 :         case ISCSI_TASK_FUNC_CLEAR_TASK_SET:
    3670           1 :                 task->scsi.response = SPDK_SCSI_TASK_MGMT_RESP_REJECT_FUNC_NOT_SUPPORTED;
    3671           1 :                 SPDK_NOTICELOG("CLEAR_TASK_SET (Unsupported)\n");
    3672           1 :                 break;
    3673             : 
    3674           1 :         case ISCSI_TASK_FUNC_CLEAR_ACA:
    3675           1 :                 task->scsi.response = SPDK_SCSI_TASK_MGMT_RESP_REJECT_FUNC_NOT_SUPPORTED;
    3676           1 :                 SPDK_NOTICELOG("CLEAR_ACA (Unsupported)\n");
    3677           1 :                 break;
    3678             : 
    3679           0 :         case ISCSI_TASK_FUNC_LOGICAL_UNIT_RESET:
    3680           0 :                 SPDK_NOTICELOG("LOGICAL_UNIT_RESET\n");
    3681             : 
    3682           0 :                 iscsi_clear_all_transfer_task(conn, task->scsi.lun, pdu);
    3683           0 :                 iscsi_op_abort_task_set(task, SPDK_SCSI_TASK_FUNC_LUN_RESET);
    3684           0 :                 return 0;
    3685             : 
    3686           1 :         case ISCSI_TASK_FUNC_TARGET_WARM_RESET:
    3687           1 :                 SPDK_NOTICELOG("TARGET_WARM_RESET (Unsupported)\n");
    3688           1 :                 task->scsi.response = SPDK_SCSI_TASK_MGMT_RESP_REJECT_FUNC_NOT_SUPPORTED;
    3689           1 :                 break;
    3690             : 
    3691           1 :         case ISCSI_TASK_FUNC_TARGET_COLD_RESET:
    3692           1 :                 SPDK_NOTICELOG("TARGET_COLD_RESET (Unsupported)\n");
    3693           1 :                 task->scsi.response = SPDK_SCSI_TASK_MGMT_RESP_REJECT_FUNC_NOT_SUPPORTED;
    3694           1 :                 break;
    3695             : 
    3696           1 :         case ISCSI_TASK_FUNC_TASK_REASSIGN:
    3697           1 :                 SPDK_NOTICELOG("TASK_REASSIGN (Unsupported)\n");
    3698           1 :                 task->scsi.response = SPDK_SCSI_TASK_MGMT_RESP_REJECT_FUNC_NOT_SUPPORTED;
    3699           1 :                 break;
    3700             : 
    3701           1 :         default:
    3702           1 :                 SPDK_ERRLOG("unsupported function %d\n", function);
    3703           1 :                 task->scsi.response = SPDK_SCSI_TASK_MGMT_RESP_REJECT;
    3704           1 :                 break;
    3705             :         }
    3706             : 
    3707           6 :         iscsi_task_mgmt_response(conn, task);
    3708           6 :         iscsi_task_put(task);
    3709           6 :         return 0;
    3710             : }
    3711             : 
    3712             : static int
    3713           4 : iscsi_pdu_hdr_op_nopout(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    3714             : {
    3715             :         struct iscsi_bhs_nop_out *reqh;
    3716             :         uint32_t task_tag;
    3717             :         uint32_t transfer_tag;
    3718             :         int I_bit;
    3719             : 
    3720           4 :         if (conn->sess->session_type == SESSION_TYPE_DISCOVERY) {
    3721           1 :                 SPDK_ERRLOG("ISCSI_OP_NOPOUT not allowed in discovery session\n");
    3722           1 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    3723             :         }
    3724             : 
    3725           3 :         reqh = (struct iscsi_bhs_nop_out *)&pdu->bhs;
    3726           3 :         I_bit = reqh->immediate;
    3727             : 
    3728           3 :         if (pdu->data_segment_len > SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH) {
    3729           1 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    3730             :         }
    3731             : 
    3732           2 :         task_tag = from_be32(&reqh->itt);
    3733           2 :         transfer_tag = from_be32(&reqh->ttt);
    3734             : 
    3735           2 :         SPDK_DEBUGLOG(iscsi, "I=%d, ITT=%x, TTT=%x\n",
    3736             :                       I_bit, task_tag, transfer_tag);
    3737             : 
    3738           2 :         SPDK_DEBUGLOG(iscsi, "CmdSN=%u, StatSN=%u, ExpCmdSN=%u, MaxCmdSN=%u\n",
    3739             :                       pdu->cmd_sn, conn->StatSN, conn->sess->ExpCmdSN,
    3740             :                       conn->sess->MaxCmdSN);
    3741             : 
    3742           2 :         if (transfer_tag != 0xFFFFFFFF && transfer_tag != (uint32_t)conn->id) {
    3743           2 :                 SPDK_ERRLOG("invalid transfer tag 0x%x\n", transfer_tag);
    3744             :                 /*
    3745             :                  * Technically we should probably fail the connection here, but for now
    3746             :                  *  just print the error message and continue.
    3747             :                  */
    3748             :         }
    3749             : 
    3750           2 :         if (task_tag == 0xffffffffU && I_bit == 0) {
    3751           1 :                 SPDK_ERRLOG("got NOPOUT ITT=0xffffffff, I=0\n");
    3752           1 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    3753             :         }
    3754             : 
    3755           1 :         return 0;
    3756             : }
    3757             : 
    3758             : static int
    3759           0 : iscsi_pdu_payload_op_nopout(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    3760             : {
    3761             :         struct spdk_iscsi_pdu *rsp_pdu;
    3762             :         struct iscsi_bhs_nop_out *reqh;
    3763             :         struct iscsi_bhs_nop_in *rsph;
    3764             :         uint8_t *data;
    3765             :         uint64_t lun;
    3766             :         uint32_t task_tag;
    3767             :         int I_bit;
    3768             :         int data_len;
    3769             : 
    3770           0 :         reqh = (struct iscsi_bhs_nop_out *)&pdu->bhs;
    3771           0 :         I_bit = reqh->immediate;
    3772             : 
    3773           0 :         data_len = pdu->data_segment_len;
    3774           0 :         if (data_len > conn->MaxRecvDataSegmentLength) {
    3775           0 :                 data_len = conn->MaxRecvDataSegmentLength;
    3776             :         }
    3777             : 
    3778           0 :         lun = from_be64(&reqh->lun);
    3779           0 :         task_tag = from_be32(&reqh->itt);
    3780             : 
    3781             :         /*
    3782             :          * We don't actually check to see if this is a response to the NOP-In
    3783             :          * that we sent.  Our goal is to just verify that the initiator is
    3784             :          * alive and responding to commands, not to verify that it tags
    3785             :          * NOP-Outs correctly
    3786             :          */
    3787           0 :         conn->nop_outstanding = false;
    3788             : 
    3789           0 :         if (task_tag == 0xffffffffU) {
    3790           0 :                 assert(I_bit == 1);
    3791           0 :                 SPDK_DEBUGLOG(iscsi, "got NOPOUT ITT=0xffffffff\n");
    3792           0 :                 return 0;
    3793             :         }
    3794             : 
    3795           0 :         data = calloc(1, data_len);
    3796           0 :         if (!data) {
    3797           0 :                 SPDK_ERRLOG("calloc() failed for ping data\n");
    3798           0 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    3799             :         }
    3800             : 
    3801             :         /* response of NOPOUT */
    3802           0 :         if (data_len > 0) {
    3803             :                 /* copy ping data */
    3804           0 :                 memcpy(data, pdu->data, data_len);
    3805             :         }
    3806             : 
    3807             :         /* response PDU */
    3808           0 :         rsp_pdu = iscsi_get_pdu(conn);
    3809           0 :         assert(rsp_pdu != NULL);
    3810             : 
    3811           0 :         rsph = (struct iscsi_bhs_nop_in *)&rsp_pdu->bhs;
    3812           0 :         rsp_pdu->data = data;
    3813           0 :         rsph->opcode = ISCSI_OP_NOPIN;
    3814           0 :         rsph->flags |= 0x80; /* bit 0 default to 1 */
    3815           0 :         DSET24(rsph->data_segment_len, data_len);
    3816           0 :         to_be64(&rsph->lun, lun);
    3817           0 :         to_be32(&rsph->itt, task_tag);
    3818           0 :         to_be32(&rsph->ttt, 0xffffffffU);
    3819             : 
    3820           0 :         to_be32(&rsph->stat_sn, conn->StatSN);
    3821           0 :         conn->StatSN++;
    3822             : 
    3823           0 :         if (I_bit == 0) {
    3824           0 :                 conn->sess->MaxCmdSN++;
    3825             :         }
    3826             : 
    3827           0 :         to_be32(&rsph->exp_cmd_sn, conn->sess->ExpCmdSN);
    3828           0 :         to_be32(&rsph->max_cmd_sn, conn->sess->MaxCmdSN);
    3829             : 
    3830           0 :         iscsi_conn_write_pdu(conn, rsp_pdu, iscsi_conn_pdu_generic_complete, NULL);
    3831           0 :         conn->last_nopin = spdk_get_ticks();
    3832             : 
    3833           0 :         return 0;
    3834             : }
    3835             : 
    3836             : /* This function returns the spdk_scsi_task by searching the snack list via
    3837             :  * task transfertag and the pdu's opcode
    3838             :  */
    3839             : static struct spdk_iscsi_task *
    3840           0 : get_scsi_task_from_ttt(struct spdk_iscsi_conn *conn, uint32_t transfer_tag)
    3841             : {
    3842             :         struct spdk_iscsi_pdu *pdu;
    3843             :         struct iscsi_bhs_data_in *datain_bhs;
    3844             : 
    3845           0 :         TAILQ_FOREACH(pdu, &conn->snack_pdu_list, tailq) {
    3846           0 :                 if (pdu->bhs.opcode == ISCSI_OP_SCSI_DATAIN) {
    3847           0 :                         datain_bhs = (struct iscsi_bhs_data_in *)&pdu->bhs;
    3848           0 :                         if (from_be32(&datain_bhs->ttt) == transfer_tag) {
    3849           0 :                                 return pdu->task;
    3850             :                         }
    3851             :                 }
    3852             :         }
    3853             : 
    3854           0 :         return NULL;
    3855             : }
    3856             : 
    3857             : /* This function returns the spdk_scsi_task by searching the snack list via
    3858             :  * initiator task tag and the pdu's opcode
    3859             :  */
    3860             : static struct spdk_iscsi_task *
    3861           0 : get_scsi_task_from_itt(struct spdk_iscsi_conn *conn,
    3862             :                        uint32_t task_tag, enum iscsi_op opcode)
    3863             : {
    3864             :         struct spdk_iscsi_pdu *pdu;
    3865             : 
    3866           0 :         TAILQ_FOREACH(pdu, &conn->snack_pdu_list, tailq) {
    3867           0 :                 if (pdu->bhs.opcode == opcode &&
    3868           0 :                     pdu->task != NULL &&
    3869           0 :                     pdu->task->tag == task_tag) {
    3870           0 :                         return pdu->task;
    3871             :                 }
    3872             :         }
    3873             : 
    3874           0 :         return NULL;
    3875             : }
    3876             : 
    3877             : /* This function is used to handle the r2t snack */
    3878             : static int
    3879           0 : iscsi_handle_r2t_snack(struct spdk_iscsi_conn *conn,
    3880             :                        struct spdk_iscsi_task *task,
    3881             :                        struct spdk_iscsi_pdu *pdu, uint32_t beg_run,
    3882             :                        uint32_t run_length, int32_t task_tag)
    3883             : {
    3884             :         int32_t last_r2tsn;
    3885             :         int i;
    3886             : 
    3887           0 :         if (beg_run < task->acked_r2tsn) {
    3888           0 :                 SPDK_ERRLOG("ITT: 0x%08x, R2T SNACK requests retransmission of"
    3889             :                             "R2TSN: from 0x%08x to 0x%08x. But it has already"
    3890             :                             "ack to R2TSN:0x%08x, protocol error.\n",
    3891             :                             task_tag, beg_run, (beg_run + run_length),
    3892             :                             (task->acked_r2tsn - 1));
    3893           0 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    3894             :         }
    3895             : 
    3896           0 :         if (run_length) {
    3897           0 :                 if ((beg_run + run_length) > task->R2TSN) {
    3898           0 :                         SPDK_ERRLOG("ITT: 0x%08x, received R2T SNACK with"
    3899             :                                     "BegRun: 0x%08x, RunLength: 0x%08x, exceeds"
    3900             :                                     "current R2TSN: 0x%08x, protocol error.\n",
    3901             :                                     task_tag, beg_run, run_length,
    3902             :                                     task->R2TSN);
    3903             : 
    3904           0 :                         return iscsi_reject(conn, pdu, ISCSI_REASON_INVALID_PDU_FIELD);
    3905             :                 }
    3906           0 :                 last_r2tsn = (beg_run + run_length);
    3907             :         } else {
    3908           0 :                 last_r2tsn = task->R2TSN;
    3909             :         }
    3910             : 
    3911           0 :         for (i = beg_run; i < last_r2tsn; i++) {
    3912           0 :                 if (iscsi_send_r2t_recovery(conn, task, i, false) < 0) {
    3913           0 :                         SPDK_ERRLOG("The r2t_sn=%d of r2t_task=%p is not sent\n", i, task);
    3914             :                 }
    3915             :         }
    3916           0 :         return 0;
    3917             : }
    3918             : 
    3919             : /* This function is used to recover the data in packet */
    3920             : static int
    3921           0 : iscsi_handle_recovery_datain(struct spdk_iscsi_conn *conn,
    3922             :                              struct spdk_iscsi_task *task,
    3923             :                              struct spdk_iscsi_pdu *pdu, uint32_t beg_run,
    3924             :                              uint32_t run_length, uint32_t task_tag)
    3925             : {
    3926             :         struct spdk_iscsi_pdu *old_pdu, *pdu_temp;
    3927             :         uint32_t i;
    3928             :         struct iscsi_bhs_data_in *datain_header;
    3929             :         uint32_t last_statsn;
    3930             : 
    3931           0 :         task = iscsi_task_get_primary(task);
    3932             : 
    3933           0 :         SPDK_DEBUGLOG(iscsi, "iscsi_handle_recovery_datain\n");
    3934             : 
    3935           0 :         if (beg_run < task->acked_data_sn) {
    3936           0 :                 SPDK_ERRLOG("ITT: 0x%08x, DATA IN SNACK requests retransmission of"
    3937             :                             "DATASN: from 0x%08x to 0x%08x but already acked to "
    3938             :                             "DATASN: 0x%08x protocol error\n",
    3939             :                             task_tag, beg_run,
    3940             :                             (beg_run + run_length), (task->acked_data_sn - 1));
    3941             : 
    3942           0 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    3943             :         }
    3944             : 
    3945           0 :         if (run_length == 0) {
    3946             :                 /* as the DataSN begins at 0 */
    3947           0 :                 run_length = task->datain_datasn + 1;
    3948             :         }
    3949             : 
    3950           0 :         if ((beg_run + run_length - 1) > task->datain_datasn) {
    3951           0 :                 SPDK_ERRLOG("Initiator requests BegRun: 0x%08x, RunLength:"
    3952             :                             "0x%08x greater than maximum DataSN: 0x%08x.\n",
    3953             :                             beg_run, run_length, task->datain_datasn);
    3954             : 
    3955           0 :                 return -1;
    3956             :         } else {
    3957           0 :                 last_statsn = beg_run + run_length - 1;
    3958             :         }
    3959             : 
    3960           0 :         for (i = beg_run; i <= last_statsn; i++) {
    3961           0 :                 TAILQ_FOREACH_SAFE(old_pdu, &conn->snack_pdu_list, tailq, pdu_temp) {
    3962           0 :                         if (old_pdu->bhs.opcode == ISCSI_OP_SCSI_DATAIN) {
    3963           0 :                                 datain_header = (struct iscsi_bhs_data_in *)&old_pdu->bhs;
    3964           0 :                                 if (from_be32(&datain_header->itt) == task_tag &&
    3965           0 :                                     from_be32(&datain_header->data_sn) == i) {
    3966           0 :                                         TAILQ_REMOVE(&conn->snack_pdu_list, old_pdu, tailq);
    3967           0 :                                         iscsi_conn_write_pdu(conn, old_pdu, old_pdu->cb_fn, old_pdu->cb_arg);
    3968           0 :                                         break;
    3969             :                                 }
    3970             :                         }
    3971             :                 }
    3972             :         }
    3973           0 :         return 0;
    3974             : }
    3975             : 
    3976             : /* This function is used to handle the status snack */
    3977             : static int
    3978           0 : iscsi_handle_status_snack(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    3979             : {
    3980             :         uint32_t beg_run;
    3981             :         uint32_t run_length;
    3982             :         struct iscsi_bhs_snack_req *reqh;
    3983             :         uint32_t i;
    3984             :         uint32_t last_statsn;
    3985             :         bool found_pdu;
    3986             :         struct spdk_iscsi_pdu *old_pdu;
    3987             : 
    3988           0 :         reqh = (struct iscsi_bhs_snack_req *)&pdu->bhs;
    3989           0 :         beg_run = from_be32(&reqh->beg_run);
    3990           0 :         run_length = from_be32(&reqh->run_len);
    3991             : 
    3992           0 :         SPDK_DEBUGLOG(iscsi, "beg_run=%d, run_length=%d, conn->StatSN="
    3993             :                       "%d, conn->exp_statsn=%d\n", beg_run, run_length,
    3994             :                       conn->StatSN, conn->exp_statsn);
    3995             : 
    3996           0 :         if (!beg_run) {
    3997           0 :                 beg_run = conn->exp_statsn;
    3998           0 :         } else if (beg_run < conn->exp_statsn) {
    3999           0 :                 SPDK_ERRLOG("Got Status SNACK Begrun: 0x%08x, RunLength: 0x%08x "
    4000             :                             "but already got ExpStatSN: 0x%08x on CID:%hu.\n",
    4001             :                             beg_run, run_length, conn->StatSN, conn->cid);
    4002             : 
    4003           0 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_INVALID_PDU_FIELD);
    4004             :         }
    4005             : 
    4006           0 :         last_statsn = (!run_length) ? conn->StatSN : (beg_run + run_length);
    4007             : 
    4008           0 :         for (i = beg_run; i < last_statsn; i++) {
    4009           0 :                 found_pdu = false;
    4010           0 :                 TAILQ_FOREACH(old_pdu, &conn->snack_pdu_list, tailq) {
    4011           0 :                         if (from_be32(&old_pdu->bhs.stat_sn) == i) {
    4012           0 :                                 found_pdu = true;
    4013           0 :                                 break;
    4014             :                         }
    4015             :                 }
    4016             : 
    4017           0 :                 if (!found_pdu) {
    4018           0 :                         SPDK_ERRLOG("Unable to find StatSN: 0x%08x. For a Status"
    4019             :                                     "SNACK, assuming this is a proactive SNACK "
    4020             :                                     "for an untransmitted StatSN, ignoring.\n",
    4021             :                                     beg_run);
    4022             :                 } else {
    4023           0 :                         TAILQ_REMOVE(&conn->snack_pdu_list, old_pdu, tailq);
    4024           0 :                         iscsi_conn_write_pdu(conn, old_pdu, old_pdu->cb_fn, old_pdu->cb_arg);
    4025             :                 }
    4026             :         }
    4027             : 
    4028           0 :         return 0;
    4029             : }
    4030             : 
    4031             : /* This function is used to handle the data ack snack */
    4032             : static int
    4033           0 : iscsi_handle_data_ack(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    4034             : {
    4035             :         uint32_t transfer_tag;
    4036             :         uint32_t beg_run;
    4037             :         uint32_t run_length;
    4038             :         struct spdk_iscsi_pdu *old_pdu;
    4039             :         uint32_t old_datasn;
    4040             :         struct iscsi_bhs_snack_req *reqh;
    4041             :         struct spdk_iscsi_task *task;
    4042             :         struct iscsi_bhs_data_in *datain_header;
    4043             :         struct spdk_iscsi_task *primary;
    4044             : 
    4045           0 :         reqh = (struct iscsi_bhs_snack_req *)&pdu->bhs;
    4046           0 :         transfer_tag = from_be32(&reqh->ttt);
    4047           0 :         beg_run = from_be32(&reqh->beg_run);
    4048           0 :         run_length = from_be32(&reqh->run_len);
    4049           0 :         task = NULL;
    4050           0 :         datain_header = NULL;
    4051             : 
    4052           0 :         SPDK_DEBUGLOG(iscsi, "beg_run=%d,transfer_tag=%d,run_len=%d\n",
    4053             :                       beg_run, transfer_tag, run_length);
    4054             : 
    4055           0 :         task = get_scsi_task_from_ttt(conn, transfer_tag);
    4056           0 :         if (!task) {
    4057           0 :                 SPDK_ERRLOG("Data ACK SNACK for TTT: 0x%08x is invalid.\n",
    4058             :                             transfer_tag);
    4059           0 :                 goto reject_return;
    4060             :         }
    4061             : 
    4062           0 :         primary = iscsi_task_get_primary(task);
    4063           0 :         if ((run_length != 0) || (beg_run < primary->acked_data_sn)) {
    4064           0 :                 SPDK_ERRLOG("TTT: 0x%08x Data ACK SNACK BegRUN: %d is less than "
    4065             :                             "the next expected acked DataSN: %d\n",
    4066             :                             transfer_tag, beg_run, primary->acked_data_sn);
    4067           0 :                 goto reject_return;
    4068             :         }
    4069             : 
    4070           0 :         primary->acked_data_sn = beg_run;
    4071             : 
    4072             :         /* To free the pdu */
    4073           0 :         TAILQ_FOREACH(old_pdu, &conn->snack_pdu_list, tailq) {
    4074           0 :                 if (old_pdu->bhs.opcode == ISCSI_OP_SCSI_DATAIN) {
    4075           0 :                         datain_header = (struct iscsi_bhs_data_in *) &old_pdu->bhs;
    4076           0 :                         old_datasn = from_be32(&datain_header->data_sn);
    4077           0 :                         if ((from_be32(&datain_header->ttt) == transfer_tag) &&
    4078           0 :                             (old_datasn == beg_run - 1)) {
    4079           0 :                                 TAILQ_REMOVE(&conn->snack_pdu_list, old_pdu, tailq);
    4080           0 :                                 iscsi_conn_free_pdu(conn, old_pdu);
    4081           0 :                                 break;
    4082             :                         }
    4083             :                 }
    4084             :         }
    4085             : 
    4086           0 :         SPDK_DEBUGLOG(iscsi, "Received Data ACK SNACK for TTT: 0x%08x,"
    4087             :                       " updated acked DataSN to 0x%08x.\n", transfer_tag,
    4088             :                       (task->acked_data_sn - 1));
    4089             : 
    4090           0 :         return 0;
    4091             : 
    4092           0 : reject_return:
    4093           0 :         return iscsi_reject(conn, pdu, ISCSI_REASON_INVALID_SNACK);
    4094             : }
    4095             : 
    4096             : /* This function is used to handle the snack request from the initiator */
    4097             : static int
    4098           0 : iscsi_pdu_hdr_op_snack(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    4099             : {
    4100             :         struct iscsi_bhs_snack_req *reqh;
    4101             :         struct spdk_iscsi_task *task;
    4102             :         int type;
    4103             :         uint32_t task_tag;
    4104             :         uint32_t beg_run;
    4105             :         uint32_t run_length;
    4106             :         int rc;
    4107             : 
    4108           0 :         if (conn->sess->session_type == SESSION_TYPE_DISCOVERY) {
    4109           0 :                 SPDK_ERRLOG("ISCSI_OP_SNACK not allowed in  discovery session\n");
    4110           0 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    4111             :         }
    4112             : 
    4113           0 :         reqh = (struct iscsi_bhs_snack_req *)&pdu->bhs;
    4114           0 :         if (!conn->sess->ErrorRecoveryLevel) {
    4115           0 :                 SPDK_ERRLOG("Got a SNACK request in ErrorRecoveryLevel=0\n");
    4116           0 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    4117             :         }
    4118             : 
    4119           0 :         type = reqh->flags & ISCSI_FLAG_SNACK_TYPE_MASK;
    4120           0 :         SPDK_DEBUGLOG(iscsi, "The value of type is %d\n", type);
    4121             : 
    4122           0 :         switch (type) {
    4123           0 :         case 0:
    4124           0 :                 reqh = (struct iscsi_bhs_snack_req *)&pdu->bhs;
    4125           0 :                 task_tag = from_be32(&reqh->itt);
    4126           0 :                 beg_run = from_be32(&reqh->beg_run);
    4127           0 :                 run_length = from_be32(&reqh->run_len);
    4128             : 
    4129           0 :                 SPDK_DEBUGLOG(iscsi, "beg_run=%d, run_length=%d, "
    4130             :                               "task_tag=%x, transfer_tag=%u\n", beg_run,
    4131             :                               run_length, task_tag, from_be32(&reqh->ttt));
    4132             : 
    4133           0 :                 task = get_scsi_task_from_itt(conn, task_tag,
    4134             :                                               ISCSI_OP_SCSI_DATAIN);
    4135           0 :                 if (task) {
    4136           0 :                         return iscsi_handle_recovery_datain(conn, task, pdu,
    4137             :                                                             beg_run, run_length, task_tag);
    4138             :                 }
    4139           0 :                 task = get_scsi_task_from_itt(conn, task_tag, ISCSI_OP_R2T);
    4140           0 :                 if (task) {
    4141           0 :                         return iscsi_handle_r2t_snack(conn, task, pdu, beg_run,
    4142             :                                                       run_length, task_tag);
    4143             :                 }
    4144           0 :                 SPDK_ERRLOG("It is Neither datain nor r2t recovery request\n");
    4145           0 :                 rc = -1;
    4146           0 :                 break;
    4147           0 :         case ISCSI_FLAG_SNACK_TYPE_STATUS:
    4148           0 :                 rc = iscsi_handle_status_snack(conn, pdu);
    4149           0 :                 break;
    4150           0 :         case ISCSI_FLAG_SNACK_TYPE_DATA_ACK:
    4151           0 :                 rc = iscsi_handle_data_ack(conn, pdu);
    4152           0 :                 break;
    4153           0 :         case ISCSI_FLAG_SNACK_TYPE_RDATA:
    4154           0 :                 SPDK_ERRLOG("R-Data SNACK is Not Supported int spdk\n");
    4155           0 :                 rc = iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    4156           0 :                 break;
    4157           0 :         default:
    4158           0 :                 SPDK_ERRLOG("Unknown SNACK type %d, protocol error\n", type);
    4159           0 :                 rc = iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    4160           0 :                 break;
    4161             :         }
    4162             : 
    4163           0 :         return rc;
    4164             : }
    4165             : 
    4166             : static inline uint32_t
    4167          47 : iscsi_get_mobj_max_data_len(struct spdk_mobj *mobj)
    4168             : {
    4169          47 :         if (mobj->mp == g_iscsi.pdu_immediate_data_pool) {
    4170           0 :                 return iscsi_get_max_immediate_data_size();
    4171             :         } else {
    4172          47 :                 return SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH;
    4173             :         }
    4174             : }
    4175             : 
    4176             : static int
    4177          22 : iscsi_pdu_hdr_op_data(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    4178             : {
    4179             :         struct spdk_iscsi_task  *task;
    4180             :         struct iscsi_bhs_data_out *reqh;
    4181             :         struct spdk_scsi_lun    *lun_dev;
    4182             :         struct spdk_mobj        *mobj;
    4183             :         uint32_t transfer_tag;
    4184             :         uint32_t task_tag;
    4185             :         uint32_t transfer_len;
    4186             :         uint32_t DataSN;
    4187             :         uint32_t buffer_offset;
    4188             :         uint32_t len;
    4189             :         uint32_t current_desired_data_transfer_length;
    4190             :         int F_bit;
    4191             :         int rc;
    4192             : 
    4193          22 :         if (conn->sess->session_type == SESSION_TYPE_DISCOVERY) {
    4194           1 :                 SPDK_ERRLOG("ISCSI_OP_SCSI_DATAOUT not allowed in discovery session\n");
    4195           1 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    4196             :         }
    4197             : 
    4198          21 :         reqh = (struct iscsi_bhs_data_out *)&pdu->bhs;
    4199          21 :         F_bit = !!(reqh->flags & ISCSI_FLAG_FINAL);
    4200          21 :         transfer_tag = from_be32(&reqh->ttt);
    4201          21 :         task_tag = from_be32(&reqh->itt);
    4202          21 :         DataSN = from_be32(&reqh->data_sn);
    4203          21 :         buffer_offset = from_be32(&reqh->buffer_offset);
    4204             : 
    4205          21 :         if (pdu->data_segment_len > SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH) {
    4206           1 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    4207             :         }
    4208             : 
    4209          20 :         task = get_transfer_task(conn, transfer_tag);
    4210          20 :         if (task == NULL) {
    4211           1 :                 SPDK_ERRLOG("Not found task for transfer_tag=%x\n", transfer_tag);
    4212           1 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_INVALID_PDU_FIELD);
    4213             :         }
    4214             : 
    4215          19 :         lun_dev = spdk_scsi_dev_get_lun(conn->dev, task->lun_id);
    4216          19 :         current_desired_data_transfer_length = task->desired_data_transfer_length;
    4217             : 
    4218          19 :         if (pdu->data_segment_len > task->desired_data_transfer_length) {
    4219           2 :                 SPDK_ERRLOG("the dataout pdu data length is larger than the value sent by R2T PDU\n");
    4220           2 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    4221             :         }
    4222             : 
    4223          17 :         if (task->tag != task_tag) {
    4224           1 :                 SPDK_ERRLOG("The r2t task tag is %u, and the dataout task tag is %u\n",
    4225             :                             task->tag, task_tag);
    4226           1 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_INVALID_PDU_FIELD);
    4227             :         }
    4228             : 
    4229          16 :         if (DataSN != task->r2t_datasn) {
    4230           1 :                 SPDK_ERRLOG("DataSN(%u) exp=%d error\n", DataSN, task->r2t_datasn);
    4231           1 :                 if (conn->sess->ErrorRecoveryLevel >= 1) {
    4232           0 :                         rc = iscsi_send_r2t_recovery(conn, task, task->acked_r2tsn, true);
    4233           0 :                         if (rc == 0) {
    4234           0 :                                 return 0;
    4235             :                         }
    4236             :                 }
    4237           1 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    4238             :         }
    4239             : 
    4240          15 :         if (buffer_offset != task->next_expected_r2t_offset) {
    4241           1 :                 SPDK_ERRLOG("offset(%u) error\n", buffer_offset);
    4242           1 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    4243             :         }
    4244             : 
    4245          14 :         transfer_len = task->scsi.transfer_len;
    4246          14 :         task->current_r2t_length += pdu->data_segment_len;
    4247          14 :         task->next_expected_r2t_offset += pdu->data_segment_len;
    4248          14 :         task->r2t_datasn++;
    4249             : 
    4250          14 :         if (task->current_r2t_length > conn->sess->MaxBurstLength) {
    4251           1 :                 SPDK_ERRLOG("R2T burst(%u) > MaxBurstLength(%u)\n",
    4252             :                             task->current_r2t_length,
    4253             :                             conn->sess->MaxBurstLength);
    4254           1 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    4255             :         }
    4256             : 
    4257          13 :         if (F_bit) {
    4258             :                 /*
    4259             :                  * This R2T burst is done. Clear the length before we
    4260             :                  *  receive a PDU for the next R2t burst.
    4261             :                  */
    4262           3 :                 task->current_r2t_length = 0;
    4263             :         }
    4264             : 
    4265          13 :         if (task->next_expected_r2t_offset == transfer_len) {
    4266           2 :                 task->acked_r2tsn++;
    4267          11 :         } else if (F_bit && (task->next_r2t_offset < transfer_len)) {
    4268           1 :                 task->acked_r2tsn++;
    4269           1 :                 len = spdk_min(conn->sess->MaxBurstLength,
    4270             :                                (transfer_len - task->next_r2t_offset));
    4271           1 :                 rc = iscsi_send_r2t(conn, task, task->next_r2t_offset, len,
    4272             :                                     task->ttt, &task->R2TSN);
    4273           1 :                 if (rc < 0) {
    4274           0 :                         SPDK_ERRLOG("iscsi_send_r2t() failed\n");
    4275             :                 }
    4276           1 :                 task->next_r2t_offset += len;
    4277             :         }
    4278             : 
    4279          13 :         if (lun_dev == NULL) {
    4280           1 :                 SPDK_DEBUGLOG(iscsi, "LUN %d is removed, reject this PDU.\n",
    4281             :                               task->lun_id);
    4282           1 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    4283          12 :         } else if (spdk_unlikely(spdk_scsi_lun_get_dif_ctx(lun_dev, &task->scsi, &pdu->dif_ctx))) {
    4284           0 :                 pdu->dif_insert_or_strip = true;
    4285             :         }
    4286             : 
    4287          12 :         mobj = iscsi_task_get_mobj(task);
    4288          12 :         if (mobj == NULL) {
    4289           3 :                 if (!pdu->dif_insert_or_strip) {
    4290             :                         /* More Data-OUT PDUs may follow. Increase the buffer size up to
    4291             :                          * SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH to merge them into a
    4292             :                          * single subtask.
    4293             :                          */
    4294           3 :                         pdu->data_buf_len = spdk_min(current_desired_data_transfer_length,
    4295             :                                                      SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH);
    4296             :                 }
    4297             :         } else {
    4298             :                 /* Set up the data buffer from the one saved by the primary task. */
    4299           9 :                 pdu->mobj[0] = mobj;
    4300           9 :                 pdu->data = (void *)((uint64_t)mobj->buf + mobj->data_len);
    4301           9 :                 pdu->data_from_mempool = true;
    4302           9 :                 pdu->data_buf_len = SPDK_BDEV_BUF_SIZE_WITH_MD(iscsi_get_mobj_max_data_len(mobj));
    4303             : 
    4304           9 :                 iscsi_task_set_mobj(task, NULL);
    4305             :         }
    4306             : 
    4307          12 :         return 0;
    4308             : }
    4309             : 
    4310             : static int
    4311          10 : iscsi_pdu_payload_op_data(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    4312             : {
    4313             :         struct spdk_iscsi_task *task;
    4314             :         struct iscsi_bhs_data_out *reqh;
    4315             :         struct spdk_mobj *mobj;
    4316             :         uint32_t transfer_tag;
    4317             :         int F_bit;
    4318             :         int rc;
    4319             : 
    4320          10 :         reqh = (struct iscsi_bhs_data_out *)&pdu->bhs;
    4321          10 :         F_bit = !!(reqh->flags & ISCSI_FLAG_FINAL);
    4322          10 :         transfer_tag = from_be32(&reqh->ttt);
    4323             : 
    4324          10 :         task = get_transfer_task(conn, transfer_tag);
    4325          10 :         if (spdk_unlikely(task == NULL)) {
    4326           0 :                 SPDK_ERRLOG("Not found for transfer_tag=%x\n", transfer_tag);
    4327           0 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_INVALID_PDU_FIELD);
    4328             :         }
    4329             : 
    4330          10 :         if (spdk_scsi_dev_get_lun(conn->dev, task->lun_id) == NULL) {
    4331           0 :                 SPDK_DEBUGLOG(iscsi, "LUN %d is removed, reject this PDU.\n",
    4332             :                               task->lun_id);
    4333           0 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    4334             :         }
    4335             : 
    4336             :         /* If current PDU is final in a sequence, submit all received data,
    4337             :          * otherwise, continue aggregation until the first data buffer is full.
    4338             :          * We do not use SGL and instead create a subtask per data buffer. Hence further
    4339             :          * aggregation does not improve any performance.
    4340             :          */
    4341          10 :         mobj = pdu->mobj[0];
    4342          10 :         assert(mobj != NULL);
    4343             : 
    4344          10 :         if (F_bit || mobj->data_len >= SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH ||
    4345           7 :             pdu->dif_insert_or_strip) {
    4346           3 :                 rc = iscsi_submit_write_subtask(conn, task, pdu, mobj);
    4347           3 :                 if (rc != 0) {
    4348           0 :                         return rc;
    4349             :                 }
    4350             :         } else {
    4351           7 :                 assert(pdu->mobj[1] == NULL);
    4352           7 :                 iscsi_task_set_mobj(task, mobj);
    4353           7 :                 pdu->mobj[0] = NULL;
    4354           7 :                 return 0;
    4355             :         }
    4356             : 
    4357           3 :         mobj = pdu->mobj[1];
    4358           3 :         if (mobj == NULL) {
    4359           1 :                 return 0;
    4360             :         }
    4361             : 
    4362           2 :         assert(pdu->dif_insert_or_strip == false);
    4363           2 :         assert(mobj->data_len < SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH);
    4364             : 
    4365           2 :         if (F_bit) {
    4366           1 :                 return iscsi_submit_write_subtask(conn, task, pdu, mobj);
    4367             :         } else {
    4368           1 :                 iscsi_task_set_mobj(task, mobj);
    4369           1 :                 pdu->mobj[1] = NULL;
    4370           1 :                 return 0;
    4371             :         }
    4372             : }
    4373             : 
    4374             : static void
    4375           0 : init_login_reject_response(struct spdk_iscsi_pdu *pdu, struct spdk_iscsi_pdu *rsp_pdu)
    4376             : {
    4377             :         struct iscsi_bhs_login_rsp *rsph;
    4378             : 
    4379           0 :         memset(rsp_pdu, 0, sizeof(struct spdk_iscsi_pdu));
    4380           0 :         rsph = (struct iscsi_bhs_login_rsp *)&rsp_pdu->bhs;
    4381           0 :         rsph->version_max = ISCSI_VERSION;
    4382           0 :         rsph->version_act = ISCSI_VERSION;
    4383           0 :         rsph->opcode = ISCSI_OP_LOGIN_RSP;
    4384           0 :         rsph->status_class = ISCSI_CLASS_INITIATOR_ERROR;
    4385           0 :         rsph->status_detail = ISCSI_LOGIN_INVALID_LOGIN_REQUEST;
    4386           0 :         rsph->itt = pdu->bhs.itt;
    4387           0 : }
    4388             : 
    4389             : static void
    4390           0 : iscsi_pdu_dump(struct spdk_iscsi_pdu *pdu)
    4391             : {
    4392           0 :         spdk_log_dump(stderr, "PDU", (uint8_t *)&pdu->bhs, ISCSI_BHS_LEN);
    4393           0 : }
    4394             : 
    4395             : /* This function is used to refree the pdu when it is acknowledged */
    4396             : static void
    4397           0 : remove_acked_pdu(struct spdk_iscsi_conn *conn, uint32_t ExpStatSN)
    4398             : {
    4399             :         struct spdk_iscsi_pdu *pdu, *pdu_temp;
    4400             :         uint32_t stat_sn;
    4401             : 
    4402           0 :         conn->exp_statsn = spdk_min(ExpStatSN, conn->StatSN);
    4403           0 :         TAILQ_FOREACH_SAFE(pdu, &conn->snack_pdu_list, tailq, pdu_temp) {
    4404           0 :                 stat_sn = from_be32(&pdu->bhs.stat_sn);
    4405           0 :                 if (spdk_sn32_lt(stat_sn, conn->exp_statsn)) {
    4406           0 :                         TAILQ_REMOVE(&conn->snack_pdu_list, pdu, tailq);
    4407           0 :                         iscsi_conn_free_pdu(conn, pdu);
    4408             :                 }
    4409             :         }
    4410           0 : }
    4411             : 
    4412             : static int
    4413          14 : iscsi_update_cmdsn(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    4414             : {
    4415             :         int opcode;
    4416             :         uint32_t ExpStatSN;
    4417             :         int I_bit;
    4418             :         struct spdk_iscsi_sess *sess;
    4419             :         struct iscsi_bhs_scsi_req *reqh;
    4420             : 
    4421          14 :         sess = conn->sess;
    4422          14 :         if (!sess) {
    4423           0 :                 SPDK_ERRLOG("Connection has no associated session!\n");
    4424           0 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    4425             :         }
    4426             : 
    4427          14 :         opcode = pdu->bhs.opcode;
    4428          14 :         reqh = (struct iscsi_bhs_scsi_req *)&pdu->bhs;
    4429             : 
    4430          14 :         pdu->cmd_sn = from_be32(&reqh->cmd_sn);
    4431             : 
    4432          14 :         I_bit = reqh->immediate;
    4433          14 :         if (I_bit == 0) {
    4434          21 :                 if (spdk_sn32_lt(pdu->cmd_sn, sess->ExpCmdSN) ||
    4435           7 :                     spdk_sn32_gt(pdu->cmd_sn, sess->MaxCmdSN)) {
    4436           7 :                         if (sess->session_type == SESSION_TYPE_NORMAL &&
    4437             :                             opcode != ISCSI_OP_SCSI_DATAOUT) {
    4438           0 :                                 SPDK_ERRLOG("CmdSN(%u) ignore (ExpCmdSN=%u, MaxCmdSN=%u)\n",
    4439             :                                             pdu->cmd_sn, sess->ExpCmdSN, sess->MaxCmdSN);
    4440             : 
    4441           0 :                                 if (sess->ErrorRecoveryLevel >= 1) {
    4442           0 :                                         SPDK_DEBUGLOG(iscsi, "Skip the error in ERL 1 and 2\n");
    4443             :                                 } else {
    4444           0 :                                         return SPDK_PDU_FATAL;
    4445             :                                 }
    4446             :                         }
    4447             :                 }
    4448           0 :         } else if (pdu->cmd_sn != sess->ExpCmdSN) {
    4449           0 :                 SPDK_ERRLOG("CmdSN(%u) error ExpCmdSN=%u\n", pdu->cmd_sn, sess->ExpCmdSN);
    4450             : 
    4451           0 :                 if (sess->ErrorRecoveryLevel >= 1) {
    4452           0 :                         SPDK_DEBUGLOG(iscsi, "Skip the error in ERL 1 and 2\n");
    4453           0 :                 } else if (opcode != ISCSI_OP_NOPOUT) {
    4454             :                         /*
    4455             :                          * The Linux initiator does not send valid CmdSNs for
    4456             :                          *  nopout under heavy load, so do not close the
    4457             :                          *  connection in that case.
    4458             :                          */
    4459           0 :                         return SPDK_ISCSI_CONNECTION_FATAL;
    4460             :                 }
    4461             :         }
    4462             : 
    4463          14 :         ExpStatSN = from_be32(&reqh->exp_stat_sn);
    4464          14 :         if (spdk_sn32_gt(ExpStatSN, conn->StatSN)) {
    4465           0 :                 SPDK_DEBUGLOG(iscsi, "StatSN(%u) advanced\n", ExpStatSN);
    4466           0 :                 ExpStatSN = conn->StatSN;
    4467             :         }
    4468             : 
    4469          14 :         if (sess->ErrorRecoveryLevel >= 1) {
    4470           0 :                 remove_acked_pdu(conn, ExpStatSN);
    4471             :         }
    4472             : 
    4473          14 :         if (!I_bit && opcode != ISCSI_OP_SCSI_DATAOUT) {
    4474           3 :                 sess->ExpCmdSN++;
    4475             :         }
    4476             : 
    4477          14 :         return 0;
    4478             : }
    4479             : 
    4480             : static int
    4481          14 : iscsi_pdu_hdr_handle(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    4482             : {
    4483             :         int opcode;
    4484             :         int rc;
    4485          14 :         struct spdk_iscsi_pdu *rsp_pdu = NULL;
    4486             : 
    4487          14 :         if (pdu == NULL) {
    4488           0 :                 return -1;
    4489             :         }
    4490             : 
    4491          14 :         opcode = pdu->bhs.opcode;
    4492             : 
    4493          14 :         SPDK_DEBUGLOG(iscsi, "opcode %x\n", opcode);
    4494             : 
    4495          14 :         if (opcode == ISCSI_OP_LOGIN) {
    4496           0 :                 return iscsi_pdu_hdr_op_login(conn, pdu);
    4497             :         }
    4498             : 
    4499             :         /* connection in login phase but receive non-login opcode
    4500             :          * return response code 0x020b to initiator.
    4501             :          * */
    4502          14 :         if (!conn->full_feature && conn->state == ISCSI_CONN_STATE_RUNNING) {
    4503           0 :                 rsp_pdu = iscsi_get_pdu(conn);
    4504           0 :                 if (rsp_pdu == NULL) {
    4505           0 :                         return SPDK_ISCSI_CONNECTION_FATAL;
    4506             :                 }
    4507           0 :                 init_login_reject_response(pdu, rsp_pdu);
    4508           0 :                 iscsi_conn_write_pdu(conn, rsp_pdu, iscsi_conn_pdu_generic_complete, NULL);
    4509           0 :                 SPDK_ERRLOG("Received opcode %d in login phase\n", opcode);
    4510           0 :                 return SPDK_ISCSI_LOGIN_ERROR_RESPONSE;
    4511          14 :         } else if (conn->state == ISCSI_CONN_STATE_INVALID) {
    4512           0 :                 SPDK_ERRLOG("before Full Feature\n");
    4513           0 :                 iscsi_pdu_dump(pdu);
    4514           0 :                 return SPDK_ISCSI_CONNECTION_FATAL;
    4515             :         }
    4516             : 
    4517          14 :         rc = iscsi_update_cmdsn(conn, pdu);
    4518          14 :         if (rc != 0) {
    4519           0 :                 return rc;
    4520             :         }
    4521             : 
    4522          14 :         switch (opcode) {
    4523           0 :         case ISCSI_OP_NOPOUT:
    4524           0 :                 rc = iscsi_pdu_hdr_op_nopout(conn, pdu);
    4525           0 :                 break;
    4526             : 
    4527           2 :         case ISCSI_OP_SCSI:
    4528           2 :                 rc = iscsi_pdu_hdr_op_scsi(conn, pdu);
    4529           2 :                 break;
    4530           0 :         case ISCSI_OP_TASK:
    4531           0 :                 rc = iscsi_pdu_hdr_op_task(conn, pdu);
    4532           0 :                 break;
    4533             : 
    4534           1 :         case ISCSI_OP_TEXT:
    4535           1 :                 rc = iscsi_pdu_hdr_op_text(conn, pdu);
    4536           1 :                 break;
    4537             : 
    4538           0 :         case ISCSI_OP_LOGOUT:
    4539           0 :                 rc = iscsi_pdu_hdr_op_logout(conn, pdu);
    4540           0 :                 break;
    4541             : 
    4542          11 :         case ISCSI_OP_SCSI_DATAOUT:
    4543          11 :                 rc = iscsi_pdu_hdr_op_data(conn, pdu);
    4544          11 :                 break;
    4545             : 
    4546           0 :         case ISCSI_OP_SNACK:
    4547           0 :                 rc = iscsi_pdu_hdr_op_snack(conn, pdu);
    4548           0 :                 break;
    4549             : 
    4550           0 :         default:
    4551           0 :                 SPDK_ERRLOG("unsupported opcode %x\n", opcode);
    4552           0 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    4553             :         }
    4554             : 
    4555          14 :         if (rc < 0) {
    4556           1 :                 SPDK_ERRLOG("processing PDU header (opcode=%x) failed on %s(%s)\n",
    4557             :                             opcode,
    4558             :                             conn->target_port != NULL ? spdk_scsi_port_get_name(conn->target_port) : "NULL",
    4559             :                             conn->initiator_port != NULL ? spdk_scsi_port_get_name(conn->initiator_port) : "NULL");
    4560             :         }
    4561             : 
    4562          14 :         return rc;
    4563             : }
    4564             : 
    4565             : static int
    4566          13 : iscsi_pdu_payload_handle(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    4567             : {
    4568             :         int opcode;
    4569          13 :         int rc = 0;
    4570             : 
    4571          13 :         opcode = pdu->bhs.opcode;
    4572             : 
    4573          13 :         SPDK_DEBUGLOG(iscsi, "opcode %x\n", opcode);
    4574             : 
    4575          13 :         switch (opcode) {
    4576           0 :         case ISCSI_OP_LOGIN:
    4577           0 :                 rc = iscsi_pdu_payload_op_login(conn, pdu);
    4578           0 :                 break;
    4579           0 :         case ISCSI_OP_NOPOUT:
    4580           0 :                 rc = iscsi_pdu_payload_op_nopout(conn, pdu);
    4581           0 :                 break;
    4582           2 :         case ISCSI_OP_SCSI:
    4583           2 :                 rc = iscsi_pdu_payload_op_scsi(conn, pdu);
    4584           2 :                 break;
    4585           0 :         case ISCSI_OP_TASK:
    4586           0 :                 break;
    4587           1 :         case ISCSI_OP_TEXT:
    4588           1 :                 rc = iscsi_pdu_payload_op_text(conn, pdu);
    4589           1 :                 break;
    4590           0 :         case ISCSI_OP_LOGOUT:
    4591           0 :                 break;
    4592          10 :         case ISCSI_OP_SCSI_DATAOUT:
    4593          10 :                 rc = iscsi_pdu_payload_op_data(conn, pdu);
    4594          10 :                 break;
    4595           0 :         case ISCSI_OP_SNACK:
    4596           0 :                 break;
    4597           0 :         default:
    4598           0 :                 SPDK_ERRLOG("unsupported opcode %x\n", opcode);
    4599           0 :                 return iscsi_reject(conn, pdu, ISCSI_REASON_PROTOCOL_ERROR);
    4600             :         }
    4601             : 
    4602          13 :         if (rc < 0) {
    4603           0 :                 SPDK_ERRLOG("processing PDU payload (opcode=%x) failed on %s(%s)\n",
    4604             :                             opcode,
    4605             :                             conn->target_port != NULL ? spdk_scsi_port_get_name(conn->target_port) : "NULL",
    4606             :                             conn->initiator_port != NULL ? spdk_scsi_port_get_name(conn->initiator_port) : "NULL");
    4607             :         }
    4608             : 
    4609          13 :         return rc;
    4610             : }
    4611             : 
    4612             : /* Return zero if completed to read payload, positive number if still in progress,
    4613             :  * or negative number if any error.
    4614             :  */
    4615             : static int
    4616          19 : iscsi_pdu_payload_read(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
    4617             : {
    4618             :         struct spdk_mempool *pool;
    4619             :         struct spdk_mobj *mobj;
    4620             :         uint32_t data_len;
    4621             :         uint32_t read_len;
    4622             :         uint32_t crc32c;
    4623             :         int rc;
    4624             :         uint32_t data_buf_len;
    4625             : 
    4626          19 :         data_len = pdu->data_segment_len;
    4627          19 :         read_len = data_len - pdu->data_valid_bytes;
    4628          19 :         data_buf_len = pdu->data_buf_len;
    4629             : 
    4630          19 :         mobj = pdu->mobj[0];
    4631          19 :         if (mobj == NULL) {
    4632           5 :                 if (data_buf_len <= iscsi_get_max_immediate_data_size()) {
    4633           1 :                         pool = g_iscsi.pdu_immediate_data_pool;
    4634           1 :                         data_buf_len = SPDK_BDEV_BUF_SIZE_WITH_MD(iscsi_get_max_immediate_data_size());
    4635           4 :                 } else if (data_buf_len <= SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH) {
    4636           3 :                         pool = g_iscsi.pdu_data_out_pool;
    4637           3 :                         data_buf_len = SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH);
    4638             :                 } else {
    4639           1 :                         SPDK_ERRLOG("Data(%d) > MaxSegment(%d)\n",
    4640             :                                     data_len, SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH);
    4641           1 :                         return -1;
    4642             :                 }
    4643           4 :                 mobj = iscsi_datapool_get(pool);
    4644           4 :                 if (mobj == NULL) {
    4645           0 :                         return 1;
    4646             :                 }
    4647             : 
    4648           4 :                 pdu->data_buf_len = data_buf_len;
    4649             : 
    4650           4 :                 pdu->mobj[0] = mobj;
    4651           4 :                 pdu->data = mobj->buf;
    4652           4 :                 pdu->data_from_mempool = true;
    4653          14 :         } else if (mobj->data_len == iscsi_get_mobj_max_data_len(mobj) && read_len > 0) {
    4654           3 :                 mobj = pdu->mobj[1];
    4655           3 :                 if (mobj == NULL) {
    4656             :                         /* The first data buffer just ran out. Allocate the second data buffer and
    4657             :                          * continue reading the data segment.
    4658             :                          */
    4659           3 :                         assert(pdu->data_from_mempool == true);
    4660           3 :                         assert(!pdu->dif_insert_or_strip);
    4661             : 
    4662           3 :                         if (conn->data_digest) {
    4663           0 :                                 iscsi_pdu_calc_partial_data_digest(pdu);
    4664             :                         }
    4665           3 :                         mobj = iscsi_datapool_get(g_iscsi.pdu_data_out_pool);
    4666           3 :                         if (mobj == NULL) {
    4667           0 :                                 return 1;
    4668             :                         }
    4669           3 :                         pdu->mobj[1] = mobj;
    4670           3 :                         pdu->data = mobj->buf;
    4671           3 :                         pdu->data_offset = pdu->data_valid_bytes;
    4672           3 :                         pdu->data_buf_len = SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_ISCSI_MAX_RECV_DATA_SEGMENT_LENGTH);
    4673             :                 }
    4674             :         }
    4675             : 
    4676             :         /* copy the actual data into local buffer */
    4677          18 :         read_len = spdk_min(read_len, iscsi_get_mobj_max_data_len(mobj) - mobj->data_len);
    4678             : 
    4679          18 :         if (read_len > 0) {
    4680          17 :                 rc = iscsi_conn_read_data_segment(conn,
    4681             :                                                   pdu,
    4682          17 :                                                   pdu->data_valid_bytes - pdu->data_offset,
    4683             :                                                   read_len);
    4684          17 :                 if (rc < 0) {
    4685           0 :                         return rc;
    4686             :                 }
    4687             : 
    4688          17 :                 mobj->data_len += rc;
    4689          17 :                 pdu->data_valid_bytes += rc;
    4690          17 :                 if (pdu->data_valid_bytes < data_len) {
    4691           3 :                         return 1;
    4692             :                 }
    4693             :         }
    4694             : 
    4695             :         /* copy out the data digest */
    4696          15 :         if (conn->data_digest &&
    4697           1 :             pdu->ddigest_valid_bytes < ISCSI_DIGEST_LEN) {
    4698           1 :                 rc = iscsi_conn_read_data(conn,
    4699           1 :                                           ISCSI_DIGEST_LEN - pdu->ddigest_valid_bytes,
    4700           1 :                                           pdu->data_digest + pdu->ddigest_valid_bytes);
    4701           1 :                 if (rc < 0) {
    4702           0 :                         return rc;
    4703             :                 }
    4704             : 
    4705           1 :                 pdu->ddigest_valid_bytes += rc;
    4706           1 :                 if (pdu->ddigest_valid_bytes < ISCSI_DIGEST_LEN) {
    4707           0 :                         return 1;
    4708             :                 }
    4709             :         }
    4710             : 
    4711             :         /* check data digest */
    4712          15 :         if (conn->data_digest) {
    4713           1 :                 iscsi_pdu_calc_partial_data_digest(pdu);
    4714           1 :                 crc32c = iscsi_pdu_calc_partial_data_digest_done(pdu);
    4715             : 
    4716           1 :                 rc = MATCH_DIGEST_WORD(pdu->data_digest, crc32c);
    4717           1 :                 if (rc == 0) {
    4718           0 :                         SPDK_ERRLOG("data digest error (%s)\n", conn->initiator_name);
    4719           0 :                         return -1;
    4720             :                 }
    4721             :         }
    4722             : 
    4723          15 :         return 0;
    4724             : }
    4725             : 
    4726             : static int
    4727           0 : iscsi_read_pdu(struct spdk_iscsi_conn *conn)
    4728             : {
    4729             :         enum iscsi_pdu_recv_state prev_state;
    4730             :         struct spdk_iscsi_pdu *pdu;
    4731             :         uint32_t crc32c;
    4732             :         int ahs_len;
    4733             :         int rc;
    4734             : 
    4735             :         do {
    4736           0 :                 prev_state = conn->pdu_recv_state;
    4737           0 :                 pdu = conn->pdu_in_progress;
    4738             : 
    4739           0 :                 switch (conn->pdu_recv_state) {
    4740           0 :                 case ISCSI_PDU_RECV_STATE_AWAIT_PDU_READY:
    4741           0 :                         assert(conn->pdu_in_progress == NULL);
    4742             : 
    4743           0 :                         conn->pdu_in_progress = iscsi_get_pdu(conn);
    4744           0 :                         if (conn->pdu_in_progress == NULL) {
    4745           0 :                                 return SPDK_ISCSI_CONNECTION_FATAL;
    4746             :                         }
    4747           0 :                         conn->pdu_recv_state = ISCSI_PDU_RECV_STATE_AWAIT_PDU_HDR;
    4748           0 :                         break;
    4749           0 :                 case ISCSI_PDU_RECV_STATE_AWAIT_PDU_HDR:
    4750           0 :                         if (pdu->bhs_valid_bytes < ISCSI_BHS_LEN) {
    4751           0 :                                 rc = iscsi_conn_read_data(conn,
    4752           0 :                                                           ISCSI_BHS_LEN - pdu->bhs_valid_bytes,
    4753           0 :                                                           (uint8_t *)&pdu->bhs + pdu->bhs_valid_bytes);
    4754           0 :                                 if (rc < 0) {
    4755           0 :                                         conn->pdu_recv_state = ISCSI_PDU_RECV_STATE_ERROR;
    4756           0 :                                         break;
    4757             :                                 }
    4758           0 :                                 pdu->bhs_valid_bytes += rc;
    4759           0 :                                 if (pdu->bhs_valid_bytes < ISCSI_BHS_LEN) {
    4760           0 :                                         return 0;
    4761             :                                 }
    4762             :                         }
    4763             : 
    4764             :                         /* conn->is_logged_out must be checked after completing to process
    4765             :                          * logout request, i.e., before processing PDU header in this state
    4766             :                          * machine, otherwise logout response may not be sent to initiator
    4767             :                          * and initiator may get logout timeout.
    4768             :                          */
    4769           0 :                         if (spdk_unlikely(conn->is_logged_out)) {
    4770           0 :                                 SPDK_DEBUGLOG(iscsi, "pdu received after logout\n");
    4771           0 :                                 conn->pdu_recv_state = ISCSI_PDU_RECV_STATE_ERROR;
    4772           0 :                                 break;
    4773             :                         }
    4774             : 
    4775           0 :                         pdu->data_segment_len = ISCSI_ALIGN(DGET24(pdu->bhs.data_segment_len));
    4776           0 :                         pdu->data_buf_len = pdu->data_segment_len;
    4777             : 
    4778             :                         /* AHS */
    4779           0 :                         ahs_len = pdu->bhs.total_ahs_len * 4;
    4780           0 :                         if (ahs_len > ISCSI_AHS_LEN) {
    4781           0 :                                 SPDK_DEBUGLOG(iscsi, "pdu ahs length %d is invalid\n", ahs_len);
    4782           0 :                                 conn->pdu_recv_state = ISCSI_PDU_RECV_STATE_ERROR;
    4783           0 :                                 break;
    4784             :                         }
    4785             : 
    4786           0 :                         if (pdu->ahs_valid_bytes < ahs_len) {
    4787           0 :                                 rc = iscsi_conn_read_data(conn,
    4788           0 :                                                           ahs_len - pdu->ahs_valid_bytes,
    4789           0 :                                                           pdu->ahs + pdu->ahs_valid_bytes);
    4790           0 :                                 if (rc < 0) {
    4791           0 :                                         conn->pdu_recv_state = ISCSI_PDU_RECV_STATE_ERROR;
    4792           0 :                                         break;
    4793             :                                 }
    4794             : 
    4795           0 :                                 pdu->ahs_valid_bytes += rc;
    4796           0 :                                 if (pdu->ahs_valid_bytes < ahs_len) {
    4797           0 :                                         return 0;
    4798             :                                 }
    4799             :                         }
    4800             : 
    4801             :                         /* Header Digest */
    4802           0 :                         if (conn->header_digest &&
    4803           0 :                             pdu->hdigest_valid_bytes < ISCSI_DIGEST_LEN) {
    4804           0 :                                 rc = iscsi_conn_read_data(conn,
    4805           0 :                                                           ISCSI_DIGEST_LEN - pdu->hdigest_valid_bytes,
    4806           0 :                                                           pdu->header_digest + pdu->hdigest_valid_bytes);
    4807           0 :                                 if (rc < 0) {
    4808           0 :                                         conn->pdu_recv_state = ISCSI_PDU_RECV_STATE_ERROR;
    4809           0 :                                         break;
    4810             :                                 }
    4811             : 
    4812           0 :                                 pdu->hdigest_valid_bytes += rc;
    4813           0 :                                 if (pdu->hdigest_valid_bytes < ISCSI_DIGEST_LEN) {
    4814           0 :                                         return 0;
    4815             :                                 }
    4816             :                         }
    4817             : 
    4818           0 :                         if (conn->header_digest) {
    4819           0 :                                 crc32c = iscsi_pdu_calc_header_digest(pdu);
    4820           0 :                                 rc = MATCH_DIGEST_WORD(pdu->header_digest, crc32c);
    4821           0 :                                 if (rc == 0) {
    4822           0 :                                         SPDK_ERRLOG("header digest error (%s)\n", conn->initiator_name);
    4823           0 :                                         conn->pdu_recv_state = ISCSI_PDU_RECV_STATE_ERROR;
    4824           0 :                                         break;
    4825             :                                 }
    4826             :                         }
    4827             : 
    4828           0 :                         rc = iscsi_pdu_hdr_handle(conn, pdu);
    4829           0 :                         if (rc < 0) {
    4830           0 :                                 SPDK_ERRLOG("Critical error is detected. Close the connection\n");
    4831           0 :                                 conn->pdu_recv_state = ISCSI_PDU_RECV_STATE_ERROR;
    4832           0 :                                 break;
    4833             :                         }
    4834             : 
    4835           0 :                         conn->pdu_recv_state = ISCSI_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD;
    4836           0 :                         break;
    4837           0 :                 case ISCSI_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD:
    4838           0 :                         if (pdu->data_segment_len != 0) {
    4839           0 :                                 rc = iscsi_pdu_payload_read(conn, pdu);
    4840           0 :                                 if (rc > 0) {
    4841           0 :                                         return 0;
    4842           0 :                                 } else if (rc < 0) {
    4843           0 :                                         conn->pdu_recv_state = ISCSI_PDU_RECV_STATE_ERROR;
    4844           0 :                                         break;
    4845             :                                 }
    4846             :                         }
    4847             : 
    4848             :                         /* All data for this PDU has now been read from the socket. */
    4849           0 :                         spdk_trace_record(TRACE_ISCSI_READ_PDU, conn->trace_id, pdu->data_valid_bytes,
    4850             :                                           (uintptr_t)pdu, pdu->bhs.opcode);
    4851             : 
    4852           0 :                         if (!pdu->is_rejected) {
    4853           0 :                                 rc = iscsi_pdu_payload_handle(conn, pdu);
    4854             :                         } else {
    4855           0 :                                 rc = 0;
    4856             :                         }
    4857           0 :                         if (rc == 0) {
    4858           0 :                                 spdk_trace_record(TRACE_ISCSI_TASK_EXECUTED, conn->trace_id, 0, (uintptr_t)pdu);
    4859           0 :                                 iscsi_put_pdu(pdu);
    4860           0 :                                 conn->pdu_in_progress = NULL;
    4861           0 :                                 conn->pdu_recv_state = ISCSI_PDU_RECV_STATE_AWAIT_PDU_READY;
    4862           0 :                                 return 1;
    4863             :                         } else {
    4864           0 :                                 conn->pdu_recv_state = ISCSI_PDU_RECV_STATE_ERROR;
    4865             :                         }
    4866           0 :                         break;
    4867           0 :                 case ISCSI_PDU_RECV_STATE_ERROR:
    4868           0 :                         return SPDK_ISCSI_CONNECTION_FATAL;
    4869           0 :                 default:
    4870           0 :                         assert(false);
    4871             :                         SPDK_ERRLOG("code should not come here\n");
    4872             :                         break;
    4873             :                 }
    4874           0 :         } while (prev_state != conn->pdu_recv_state);
    4875             : 
    4876           0 :         return 0;
    4877             : }
    4878             : 
    4879             : #define GET_PDU_LOOP_COUNT      16
    4880             : 
    4881             : int
    4882           0 : iscsi_handle_incoming_pdus(struct spdk_iscsi_conn *conn)
    4883             : {
    4884             :         int i, rc;
    4885             : 
    4886             :         /* Read new PDUs from network */
    4887           0 :         for (i = 0; i < GET_PDU_LOOP_COUNT; i++) {
    4888           0 :                 rc = iscsi_read_pdu(conn);
    4889           0 :                 if (rc == 0) {
    4890           0 :                         break;
    4891           0 :                 } else if (rc < 0) {
    4892           0 :                         return rc;
    4893             :                 }
    4894             : 
    4895           0 :                 if (conn->is_stopped) {
    4896           0 :                         break;
    4897             :                 }
    4898             :         }
    4899             : 
    4900           0 :         return i;
    4901             : }

Generated by: LCOV version 1.15