1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  23  *
  24  * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
  25  */
  26 
  27 #include <sys/cpuvar.h>
  28 #include <sys/types.h>
  29 #include <sys/conf.h>
  30 #include <sys/stat.h>
  31 #include <sys/file.h>
  32 #include <sys/ddi.h>
  33 #include <sys/sunddi.h>
  34 #include <sys/modctl.h>
  35 #include <sys/sysmacros.h>
  36 #include <sys/socket.h>
  37 #include <sys/strsubr.h>
  38 #include <sys/nvpair.h>
  39 
  40 #include <sys/stmf.h>
  41 #include <sys/stmf_ioctl.h>
  42 #include <sys/portif.h>
  43 #include <sys/idm/idm.h>
  44 #include <sys/idm/idm_conn_sm.h>
  45 
  46 #include "iscsit_isns.h"
  47 #include "iscsit.h"
  48 
  49 #define ISCSIT_VERSION          BUILD_DATE "-1.18dev"
  50 #define ISCSIT_NAME_VERSION     "COMSTAR ISCSIT v" ISCSIT_VERSION
  51 
  52 /*
  53  * DDI entry points.
  54  */
  55 static int iscsit_drv_attach(dev_info_t *, ddi_attach_cmd_t);
  56 static int iscsit_drv_detach(dev_info_t *, ddi_detach_cmd_t);
  57 static int iscsit_drv_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
  58 static int iscsit_drv_open(dev_t *, int, int, cred_t *);
  59 static int iscsit_drv_close(dev_t, int, int, cred_t *);
  60 static boolean_t iscsit_drv_busy(void);
  61 static int iscsit_drv_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
  62 
  63 extern struct mod_ops mod_miscops;
  64 
  65 
  66 static struct cb_ops iscsit_cb_ops = {
  67         iscsit_drv_open,        /* cb_open */
  68         iscsit_drv_close,       /* cb_close */
  69         nodev,                  /* cb_strategy */
  70         nodev,                  /* cb_print */
  71         nodev,                  /* cb_dump */
  72         nodev,                  /* cb_read */
  73         nodev,                  /* cb_write */
  74         iscsit_drv_ioctl,       /* cb_ioctl */
  75         nodev,                  /* cb_devmap */
  76         nodev,                  /* cb_mmap */
  77         nodev,                  /* cb_segmap */
  78         nochpoll,               /* cb_chpoll */
  79         ddi_prop_op,            /* cb_prop_op */
  80         NULL,                   /* cb_streamtab */
  81         D_MP,                   /* cb_flag */
  82         CB_REV,                 /* cb_rev */
  83         nodev,                  /* cb_aread */
  84         nodev,                  /* cb_awrite */
  85 };
  86 
  87 static struct dev_ops iscsit_dev_ops = {
  88         DEVO_REV,               /* devo_rev */
  89         0,                      /* devo_refcnt */
  90         iscsit_drv_getinfo,     /* devo_getinfo */
  91         nulldev,                /* devo_identify */
  92         nulldev,                /* devo_probe */
  93         iscsit_drv_attach,      /* devo_attach */
  94         iscsit_drv_detach,      /* devo_detach */
  95         nodev,                  /* devo_reset */
  96         &iscsit_cb_ops,             /* devo_cb_ops */
  97         NULL,                   /* devo_bus_ops */
  98         NULL,                   /* devo_power */
  99         ddi_quiesce_not_needed, /* quiesce */
 100 };
 101 
 102 static struct modldrv modldrv = {
 103         &mod_driverops,
 104         "iSCSI Target",
 105         &iscsit_dev_ops,
 106 };
 107 
 108 static struct modlinkage modlinkage = {
 109         MODREV_1,
 110         &modldrv,
 111         NULL,
 112 };
 113 
 114 
 115 iscsit_global_t iscsit_global;
 116 
 117 kmem_cache_t    *iscsit_status_pdu_cache;
 118 
 119 boolean_t       iscsit_sm_logging = B_FALSE;
 120 
 121 kmutex_t        login_sm_session_mutex;
 122 
 123 static idm_status_t iscsit_init(dev_info_t *dip);
 124 static idm_status_t iscsit_enable_svc(iscsit_hostinfo_t *hostinfo);
 125 static void iscsit_disable_svc(void);
 126 
 127 static int
 128 iscsit_check_cmdsn_and_queue(idm_pdu_t *rx_pdu);
 129 
 130 static void
 131 iscsit_add_pdu_to_queue(iscsit_sess_t *ist, idm_pdu_t *rx_pdu);
 132 
 133 static idm_pdu_t *
 134 iscsit_remove_pdu_from_queue(iscsit_sess_t *ist, uint32_t cmdsn);
 135 
 136 static void
 137 iscsit_process_pdu_in_queue(iscsit_sess_t *ist);
 138 
 139 static void
 140 iscsit_rxpdu_queue_monitor_session(iscsit_sess_t *ist);
 141 
 142 static void
 143 iscsit_rxpdu_queue_monitor(void *arg);
 144 
 145 static void
 146 iscsit_post_staged_pdu(idm_pdu_t *rx_pdu);
 147 
 148 static void
 149 iscsit_post_scsi_cmd(idm_conn_t *ic, idm_pdu_t *rx_pdu);
 150 
 151 static void
 152 iscsit_op_scsi_task_mgmt(iscsit_conn_t *ict, idm_pdu_t *rx_pdu);
 153 
 154 static void
 155 iscsit_pdu_op_noop(iscsit_conn_t *ict, idm_pdu_t *rx_pdu);
 156 
 157 static void
 158 iscsit_pdu_op_login_cmd(iscsit_conn_t *ict, idm_pdu_t *rx_pdu);
 159 
 160 void
 161 iscsit_pdu_op_text_cmd(iscsit_conn_t *ict, idm_pdu_t *rx_pdu);
 162 
 163 static void
 164 iscsit_pdu_op_logout_cmd(iscsit_conn_t *ict, idm_pdu_t *rx_pdu);
 165 
 166 int iscsit_cmd_window();
 167 
 168 static  int
 169 iscsit_sna_lt(uint32_t sn1, uint32_t sn2);
 170 
 171 void
 172 iscsit_set_cmdsn(iscsit_conn_t *ict, idm_pdu_t *rx_pdu);
 173 
 174 static void
 175 iscsit_deferred_dispatch(idm_pdu_t *rx_pdu);
 176 
 177 static void
 178 iscsit_deferred(void *rx_pdu_void);
 179 
 180 static idm_status_t
 181 iscsit_conn_accept(idm_conn_t *ic);
 182 
 183 static idm_status_t
 184 iscsit_ffp_enabled(idm_conn_t *ic);
 185 
 186 static idm_status_t
 187 iscsit_ffp_disabled(idm_conn_t *ic, idm_ffp_disable_t disable_class);
 188 
 189 static idm_status_t
 190 iscsit_conn_lost(idm_conn_t *ic);
 191 
 192 static idm_status_t
 193 iscsit_conn_destroy(idm_conn_t *ic);
 194 
 195 static stmf_data_buf_t *
 196 iscsit_dbuf_alloc(scsi_task_t *task, uint32_t size, uint32_t *pminsize,
 197     uint32_t flags);
 198 
 199 static void
 200 iscsit_dbuf_free(stmf_dbuf_store_t *ds, stmf_data_buf_t *dbuf);
 201 
 202 static void
 203 iscsit_buf_xfer_cb(idm_buf_t *idb, idm_status_t status);
 204 
 205 static void
 206 iscsit_send_good_status_done(idm_pdu_t *pdu, idm_status_t status);
 207 
 208 static void
 209 iscsit_send_status_done(idm_pdu_t *pdu, idm_status_t status);
 210 
 211 static stmf_status_t
 212 iscsit_idm_to_stmf(idm_status_t idmrc);
 213 
 214 static iscsit_task_t *
 215 iscsit_task_alloc(iscsit_conn_t *ict);
 216 
 217 static void
 218 iscsit_task_free(iscsit_task_t *itask);
 219 
 220 static iscsit_task_t *
 221 iscsit_tm_task_alloc(iscsit_conn_t *ict);
 222 
 223 static void
 224 iscsit_tm_task_free(iscsit_task_t *itask);
 225 
 226 static idm_status_t
 227 iscsit_task_start(iscsit_task_t *itask);
 228 
 229 static void
 230 iscsit_task_done(iscsit_task_t *itask);
 231 
 232 static int
 233 iscsit_status_pdu_constructor(void *pdu_void, void *arg, int flags);
 234 
 235 static void
 236 iscsit_pp_cb(struct stmf_port_provider *pp, int cmd, void *arg, uint32_t flags);
 237 
 238 static it_cfg_status_t
 239 iscsit_config_merge(it_config_t *cfg);
 240 
 241 static idm_status_t
 242 iscsit_login_fail(idm_conn_t *ic);
 243 
 244 static boolean_t iscsit_cmdsn_in_window(iscsit_conn_t *ict, uint32_t cmdsn);
 245 static void iscsit_send_direct_scsi_resp(iscsit_conn_t *ict, idm_pdu_t *rx_pdu,
 246     uint8_t response, uint8_t cmd_status);
 247 static void iscsit_send_task_mgmt_resp(idm_pdu_t *tm_resp_pdu,
 248     uint8_t tm_status);
 249 
 250 /*
 251  * MC/S: Out-of-order commands are staged on a session-wide wait
 252  * queue until a system-tunable threshold is reached. A separate
 253  * thread is used to scan the staging queue on all the session,
 254  * If a delayed PDU does not arrive within a timeout, the target
 255  * will advance to the staged PDU that is next in sequence, skipping
 256  * over the missing PDU(s) to go past a hole in the sequence.
 257  */
 258 volatile int rxpdu_queue_threshold = ISCSIT_RXPDU_QUEUE_THRESHOLD;
 259 
 260 static kmutex_t         iscsit_rxpdu_queue_monitor_mutex;
 261 kthread_t               *iscsit_rxpdu_queue_monitor_thr_id;
 262 static kt_did_t         iscsit_rxpdu_queue_monitor_thr_did;
 263 static boolean_t        iscsit_rxpdu_queue_monitor_thr_running;
 264 static kcondvar_t       iscsit_rxpdu_queue_monitor_cv;
 265 
 266 int
 267 _init(void)
 268 {
 269         int rc;
 270 
 271         rw_init(&iscsit_global.global_rwlock, NULL, RW_DRIVER, NULL);
 272         mutex_init(&iscsit_global.global_state_mutex, NULL,
 273             MUTEX_DRIVER, NULL);
 274         iscsit_global.global_svc_state = ISE_DETACHED;
 275 
 276         mutex_init(&iscsit_rxpdu_queue_monitor_mutex, NULL,
 277             MUTEX_DRIVER, NULL);
 278         mutex_init(&login_sm_session_mutex, NULL, MUTEX_DRIVER, NULL);
 279         iscsit_rxpdu_queue_monitor_thr_id = NULL;
 280         iscsit_rxpdu_queue_monitor_thr_running = B_FALSE;
 281         cv_init(&iscsit_rxpdu_queue_monitor_cv, NULL, CV_DEFAULT, NULL);
 282 
 283         if ((rc = mod_install(&modlinkage)) != 0) {
 284                 mutex_destroy(&iscsit_global.global_state_mutex);
 285                 rw_destroy(&iscsit_global.global_rwlock);
 286                 return (rc);
 287         }
 288 
 289         return (rc);
 290 }
 291 
 292 int
 293 _info(struct modinfo *modinfop)
 294 {
 295         return (mod_info(&modlinkage, modinfop));
 296 }
 297 
 298 int
 299 _fini(void)
 300 {
 301         int rc;
 302 
 303         rc = mod_remove(&modlinkage);
 304 
 305         if (rc == 0) {
 306                 mutex_destroy(&iscsit_rxpdu_queue_monitor_mutex);
 307                 mutex_destroy(&login_sm_session_mutex);
 308                 cv_destroy(&iscsit_rxpdu_queue_monitor_cv);
 309                 mutex_destroy(&iscsit_global.global_state_mutex);
 310                 rw_destroy(&iscsit_global.global_rwlock);
 311         }
 312 
 313         return (rc);
 314 }
 315 
 316 /*
 317  * DDI entry points.
 318  */
 319 
 320 /* ARGSUSED */
 321 static int
 322 iscsit_drv_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
 323     void **result)
 324 {
 325         ulong_t instance = getminor((dev_t)arg);
 326 
 327         switch (cmd) {
 328         case DDI_INFO_DEVT2DEVINFO:
 329                 *result = iscsit_global.global_dip;
 330                 return (DDI_SUCCESS);
 331 
 332         case DDI_INFO_DEVT2INSTANCE:
 333                 *result = (void *)instance;
 334                 return (DDI_SUCCESS);
 335 
 336         default:
 337                 break;
 338         }
 339 
 340         return (DDI_FAILURE);
 341 }
 342 
 343 static int
 344 iscsit_drv_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 345 {
 346         if (cmd != DDI_ATTACH) {
 347                 return (DDI_FAILURE);
 348         }
 349 
 350         if (ddi_get_instance(dip) != 0) {
 351                 /* we only allow instance 0 to attach */
 352                 return (DDI_FAILURE);
 353         }
 354 
 355         /* create the minor node */
 356         if (ddi_create_minor_node(dip, ISCSIT_MODNAME, S_IFCHR, 0,
 357             DDI_PSEUDO, 0) != DDI_SUCCESS) {
 358                 cmn_err(CE_WARN, "iscsit_drv_attach: "
 359                     "failed creating minor node");
 360                 return (DDI_FAILURE);
 361         }
 362 
 363         if (iscsit_init(dip) != IDM_STATUS_SUCCESS) {
 364                 cmn_err(CE_WARN, "iscsit_drv_attach: "
 365                     "failed to initialize");
 366                 ddi_remove_minor_node(dip, NULL);
 367                 return (DDI_FAILURE);
 368         }
 369 
 370         iscsit_global.global_svc_state = ISE_DISABLED;
 371         iscsit_global.global_dip = dip;
 372 
 373         return (DDI_SUCCESS);
 374 }
 375 
 376 /*ARGSUSED*/
 377 static int
 378 iscsit_drv_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 379 {
 380         if (cmd != DDI_DETACH)
 381                 return (DDI_FAILURE);
 382 
 383         /*
 384          * drv_detach is called in a context that owns the
 385          * device node for the /dev/pseudo device.  If this thread blocks
 386          * for any resource, other threads that need the /dev/pseudo device
 387          * may end up in a deadlock with this thread.Hence, we use a
 388          * separate lock just for the structures that drv_detach needs
 389          * to access.
 390          */
 391         mutex_enter(&iscsit_global.global_state_mutex);
 392         if (iscsit_drv_busy()) {
 393                 mutex_exit(&iscsit_global.global_state_mutex);
 394                 return (EBUSY);
 395         }
 396 
 397         iscsit_global.global_dip = NULL;
 398         ddi_remove_minor_node(dip, NULL);
 399 
 400         ldi_ident_release(iscsit_global.global_li);
 401         iscsit_global.global_svc_state = ISE_DETACHED;
 402 
 403         mutex_exit(&iscsit_global.global_state_mutex);
 404 
 405         return (DDI_SUCCESS);
 406 }
 407 
 408 /*ARGSUSED*/
 409 static int
 410 iscsit_drv_open(dev_t *devp, int flag, int otyp, cred_t *credp)
 411 {
 412         return (0);
 413 }
 414 
 415 /* ARGSUSED */
 416 static int
 417 iscsit_drv_close(dev_t dev, int flag, int otyp, cred_t *credp)
 418 {
 419         return (0);
 420 }
 421 
 422 static boolean_t
 423 iscsit_drv_busy(void)
 424 {
 425         ASSERT(MUTEX_HELD(&iscsit_global.global_state_mutex));
 426 
 427         switch (iscsit_global.global_svc_state) {
 428         case ISE_DISABLED:
 429         case ISE_DETACHED:
 430                 return (B_FALSE);
 431         default:
 432                 return (B_TRUE);
 433         }
 434         /* NOTREACHED */
 435 }
 436 
 437 /* ARGSUSED */
 438 static int
 439 iscsit_drv_ioctl(dev_t drv, int cmd, intptr_t argp, int flag, cred_t *cred,
 440     int *retval)
 441 {
 442         iscsit_ioc_set_config_t         setcfg;
 443         iscsit_ioc_set_config32_t       setcfg32;
 444         char                            *cfg_pnvlist = NULL;
 445         nvlist_t                        *cfg_nvlist = NULL;
 446         it_config_t                     *cfg = NULL;
 447         idm_status_t                    idmrc;
 448         int                             rc = 0;
 449 
 450         if (drv_priv(cred) != 0) {
 451                 return (EPERM);
 452         }
 453 
 454         mutex_enter(&iscsit_global.global_state_mutex);
 455 
 456         /*
 457          * Validate ioctl requests against global service state
 458          */
 459         switch (iscsit_global.global_svc_state) {
 460         case ISE_ENABLED:
 461                 if (cmd == ISCSIT_IOC_DISABLE_SVC) {
 462                         iscsit_global.global_svc_state = ISE_DISABLING;
 463                 } else if (cmd == ISCSIT_IOC_ENABLE_SVC) {
 464                         /* Already enabled */
 465                         mutex_exit(&iscsit_global.global_state_mutex);
 466                         return (0);
 467                 } else {
 468                         iscsit_global.global_svc_state = ISE_BUSY;
 469                 }
 470                 break;
 471         case ISE_DISABLED:
 472                 if (cmd == ISCSIT_IOC_ENABLE_SVC) {
 473                         iscsit_global.global_svc_state = ISE_ENABLING;
 474                 } else if (cmd == ISCSIT_IOC_DISABLE_SVC) {
 475                         /* Already disabled */
 476                         mutex_exit(&iscsit_global.global_state_mutex);
 477                         return (0);
 478                 } else {
 479                         rc = EFAULT;
 480                 }
 481                 break;
 482         case ISE_BUSY:
 483         case ISE_ENABLING:
 484         case ISE_DISABLING:
 485                 rc = EAGAIN;
 486                 break;
 487         case ISE_DETACHED:
 488         default:
 489                 rc = EFAULT;
 490                 break;
 491         }
 492 
 493         mutex_exit(&iscsit_global.global_state_mutex);
 494         if (rc != 0)
 495                 return (rc);
 496 
 497         /* Handle ioctl request (enable/disable have already been handled) */
 498         switch (cmd) {
 499         case ISCSIT_IOC_SET_CONFIG:
 500                 /* Any errors must set state back to ISE_ENABLED */
 501                 switch (ddi_model_convert_from(flag & FMODELS)) {
 502                 case DDI_MODEL_ILP32:
 503                         if (ddi_copyin((void *)argp, &setcfg32,
 504                             sizeof (iscsit_ioc_set_config32_t), flag) != 0) {
 505                                 rc = EFAULT;
 506                                 goto cleanup;
 507                         }
 508 
 509                         setcfg.set_cfg_pnvlist =
 510                             (char *)((uintptr_t)setcfg32.set_cfg_pnvlist);
 511                         setcfg.set_cfg_vers = setcfg32.set_cfg_vers;
 512                         setcfg.set_cfg_pnvlist_len =
 513                             setcfg32.set_cfg_pnvlist_len;
 514                         break;
 515                 case DDI_MODEL_NONE:
 516                         if (ddi_copyin((void *)argp, &setcfg,
 517                             sizeof (iscsit_ioc_set_config_t), flag) != 0) {
 518                                 rc = EFAULT;
 519                                 goto cleanup;
 520                         }
 521                         break;
 522                 default:
 523                         rc = EFAULT;
 524                         goto cleanup;
 525                 }
 526 
 527                 /* Check API version */
 528                 if (setcfg.set_cfg_vers != ISCSIT_API_VERS0) {
 529                         rc = EINVAL;
 530                         goto cleanup;
 531                 }
 532 
 533                 /* Config is in packed nvlist format so unpack it */
 534                 cfg_pnvlist = kmem_alloc(setcfg.set_cfg_pnvlist_len,
 535                     KM_SLEEP);
 536                 ASSERT(cfg_pnvlist != NULL);
 537 
 538                 if (ddi_copyin(setcfg.set_cfg_pnvlist, cfg_pnvlist,
 539                     setcfg.set_cfg_pnvlist_len, flag) != 0) {
 540                         rc = EFAULT;
 541                         goto cleanup;
 542                 }
 543 
 544                 rc = nvlist_unpack(cfg_pnvlist, setcfg.set_cfg_pnvlist_len,
 545                     &cfg_nvlist, KM_SLEEP);
 546                 if (rc != 0) {
 547                         goto cleanup;
 548                 }
 549 
 550                 /* Translate nvlist */
 551                 rc = it_nv_to_config(cfg_nvlist, &cfg);
 552                 if (rc != 0) {
 553                         cmn_err(CE_WARN, "Configuration is invalid");
 554                         goto cleanup;
 555                 }
 556 
 557                 /* Update config */
 558                 rc = iscsit_config_merge(cfg);
 559                 /* FALLTHROUGH */
 560 
 561 cleanup:
 562                 if (cfg)
 563                         it_config_free_cmn(cfg);
 564                 if (cfg_pnvlist)
 565                         kmem_free(cfg_pnvlist, setcfg.set_cfg_pnvlist_len);
 566                 if (cfg_nvlist)
 567                         nvlist_free(cfg_nvlist);
 568 
 569                 /*
 570                  * Now that the reconfig is complete set our state back to
 571                  * enabled.
 572                  */
 573                 mutex_enter(&iscsit_global.global_state_mutex);
 574                 iscsit_global.global_svc_state = ISE_ENABLED;
 575                 mutex_exit(&iscsit_global.global_state_mutex);
 576                 break;
 577         case ISCSIT_IOC_ENABLE_SVC: {
 578                 iscsit_hostinfo_t hostinfo;
 579 
 580                 if (ddi_copyin((void *)argp, &hostinfo.length,
 581                     sizeof (hostinfo.length), flag) != 0) {
 582                         mutex_enter(&iscsit_global.global_state_mutex);
 583                         iscsit_global.global_svc_state = ISE_DISABLED;
 584                         mutex_exit(&iscsit_global.global_state_mutex);
 585                         return (EFAULT);
 586                 }
 587 
 588                 if (hostinfo.length > sizeof (hostinfo.fqhn))
 589                         hostinfo.length = sizeof (hostinfo.fqhn);
 590 
 591                 if (ddi_copyin((void *)((caddr_t)argp +
 592                     sizeof (hostinfo.length)), &hostinfo.fqhn,
 593                     hostinfo.length, flag) != 0) {
 594                         mutex_enter(&iscsit_global.global_state_mutex);
 595                         iscsit_global.global_svc_state = ISE_DISABLED;
 596                         mutex_exit(&iscsit_global.global_state_mutex);
 597                         return (EFAULT);
 598                 }
 599 
 600                 idmrc = iscsit_enable_svc(&hostinfo);
 601                 mutex_enter(&iscsit_global.global_state_mutex);
 602                 if (idmrc == IDM_STATUS_SUCCESS) {
 603                         iscsit_global.global_svc_state = ISE_ENABLED;
 604                 } else {
 605                         rc = EIO;
 606                         iscsit_global.global_svc_state = ISE_DISABLED;
 607                 }
 608                 mutex_exit(&iscsit_global.global_state_mutex);
 609                 break;
 610         }
 611         case ISCSIT_IOC_DISABLE_SVC:
 612                 iscsit_disable_svc();
 613                 mutex_enter(&iscsit_global.global_state_mutex);
 614                 iscsit_global.global_svc_state = ISE_DISABLED;
 615                 mutex_exit(&iscsit_global.global_state_mutex);
 616                 break;
 617 
 618         default:
 619                 rc = EINVAL;
 620                 mutex_enter(&iscsit_global.global_state_mutex);
 621                 iscsit_global.global_svc_state = ISE_ENABLED;
 622                 mutex_exit(&iscsit_global.global_state_mutex);
 623         }
 624 
 625         return (rc);
 626 }
 627 
 628 static idm_status_t
 629 iscsit_init(dev_info_t *dip)
 630 {
 631         int                     rc;
 632 
 633         rc = ldi_ident_from_dip(dip, &iscsit_global.global_li);
 634         ASSERT(rc == 0);  /* Failure indicates invalid argument */
 635 
 636         iscsit_global.global_svc_state = ISE_DISABLED;
 637 
 638         return (IDM_STATUS_SUCCESS);
 639 }
 640 
 641 /*
 642  * iscsit_enable_svc
 643  *
 644  * registers all the configured targets and target portals with STMF
 645  */
 646 static idm_status_t
 647 iscsit_enable_svc(iscsit_hostinfo_t *hostinfo)
 648 {
 649         stmf_port_provider_t    *pp;
 650         stmf_dbuf_store_t       *dbuf_store;
 651         boolean_t               did_iscsit_isns_init;
 652         idm_status_t            retval = IDM_STATUS_SUCCESS;
 653 
 654         ASSERT(iscsit_global.global_svc_state == ISE_ENABLING);
 655 
 656         /*
 657          * Make sure that can tell if we have partially allocated
 658          * in case we need to exit and tear down anything allocated.
 659          */
 660         iscsit_global.global_tsih_pool = NULL;
 661         iscsit_global.global_dbuf_store = NULL;
 662         iscsit_status_pdu_cache = NULL;
 663         pp = NULL;
 664         iscsit_global.global_pp = NULL;
 665         iscsit_global.global_default_tpg = NULL;
 666         did_iscsit_isns_init = B_FALSE;
 667         iscsit_global.global_dispatch_taskq = NULL;
 668 
 669         /* Setup remaining fields in iscsit_global_t */
 670         idm_refcnt_init(&iscsit_global.global_refcnt,
 671             &iscsit_global);
 672 
 673         avl_create(&iscsit_global.global_discovery_sessions,
 674             iscsit_sess_avl_compare, sizeof (iscsit_sess_t),
 675             offsetof(iscsit_sess_t, ist_tgt_ln));
 676 
 677         avl_create(&iscsit_global.global_target_list,
 678             iscsit_tgt_avl_compare, sizeof (iscsit_tgt_t),
 679             offsetof(iscsit_tgt_t, target_global_ln));
 680 
 681         list_create(&iscsit_global.global_deleted_target_list,
 682             sizeof (iscsit_tgt_t),
 683             offsetof(iscsit_tgt_t, target_global_deleted_ln));
 684 
 685         avl_create(&iscsit_global.global_tpg_list,
 686             iscsit_tpg_avl_compare, sizeof (iscsit_tpg_t),
 687             offsetof(iscsit_tpg_t, tpg_global_ln));
 688 
 689         avl_create(&iscsit_global.global_ini_list,
 690             iscsit_ini_avl_compare, sizeof (iscsit_ini_t),
 691             offsetof(iscsit_ini_t, ini_global_ln));
 692 
 693         iscsit_global.global_tsih_pool = vmem_create("iscsit_tsih_pool",
 694             (void *)1, ISCSI_MAX_TSIH, 1, NULL, NULL, NULL, 0,
 695             VM_SLEEP | VMC_IDENTIFIER);
 696 
 697         /*
 698          * Setup STMF dbuf store.  Our buffers are bound to a specific
 699          * connection so we really can't let STMF cache buffers for us.
 700          * Consequently we'll just allocate one global buffer store.
 701          */
 702         dbuf_store = stmf_alloc(STMF_STRUCT_DBUF_STORE, 0, 0);
 703         if (dbuf_store == NULL) {
 704                 retval = IDM_STATUS_FAIL;
 705                 goto tear_down_and_return;
 706         }
 707         dbuf_store->ds_alloc_data_buf = iscsit_dbuf_alloc;
 708         dbuf_store->ds_free_data_buf = iscsit_dbuf_free;
 709         dbuf_store->ds_port_private = NULL;
 710         iscsit_global.global_dbuf_store = dbuf_store;
 711 
 712         /* Status PDU cache */
 713         iscsit_status_pdu_cache = kmem_cache_create("iscsit_status_pdu_cache",
 714             sizeof (idm_pdu_t) + sizeof (iscsi_scsi_rsp_hdr_t), 8,
 715             &iscsit_status_pdu_constructor,
 716             NULL, NULL, NULL, NULL, KM_SLEEP);
 717 
 718         /* Default TPG and portal */
 719         iscsit_global.global_default_tpg = iscsit_tpg_createdefault();
 720         if (iscsit_global.global_default_tpg == NULL) {
 721                 retval = IDM_STATUS_FAIL;
 722                 goto tear_down_and_return;
 723         }
 724 
 725         /* initialize isns client */
 726         (void) iscsit_isns_init(hostinfo);
 727         did_iscsit_isns_init = B_TRUE;
 728 
 729         /* Register port provider */
 730         pp = stmf_alloc(STMF_STRUCT_PORT_PROVIDER, 0, 0);
 731         if (pp == NULL) {
 732                 retval = IDM_STATUS_FAIL;
 733                 goto tear_down_and_return;
 734         }
 735 
 736         pp->pp_portif_rev = PORTIF_REV_1;
 737         pp->pp_instance = 0;
 738         pp->pp_name = ISCSIT_MODNAME;
 739         pp->pp_cb = iscsit_pp_cb;
 740 
 741         iscsit_global.global_pp = pp;
 742 
 743 
 744         if (stmf_register_port_provider(pp) != STMF_SUCCESS) {
 745                 retval = IDM_STATUS_FAIL;
 746                 goto tear_down_and_return;
 747         }
 748 
 749         iscsit_global.global_dispatch_taskq = taskq_create("iscsit_dispatch",
 750             1, minclsyspri, 16, 16, TASKQ_PREPOPULATE);
 751 
 752         /* Scan staged PDUs, meaningful in MC/S situations */
 753         iscsit_rxpdu_queue_monitor_start();
 754 
 755         return (IDM_STATUS_SUCCESS);
 756 
 757 tear_down_and_return:
 758 
 759         if (iscsit_global.global_dispatch_taskq) {
 760                 taskq_destroy(iscsit_global.global_dispatch_taskq);
 761                 iscsit_global.global_dispatch_taskq = NULL;
 762         }
 763 
 764         if (did_iscsit_isns_init)
 765                 iscsit_isns_fini();
 766 
 767         if (iscsit_global.global_default_tpg) {
 768                 iscsit_tpg_destroydefault(iscsit_global.global_default_tpg);
 769                 iscsit_global.global_default_tpg = NULL;
 770         }
 771 
 772         if (iscsit_global.global_pp)
 773                 iscsit_global.global_pp = NULL;
 774 
 775         if (pp)
 776                 stmf_free(pp);
 777 
 778         if (iscsit_status_pdu_cache) {
 779                 kmem_cache_destroy(iscsit_status_pdu_cache);
 780                 iscsit_status_pdu_cache = NULL;
 781         }
 782 
 783         if (iscsit_global.global_dbuf_store) {
 784                 stmf_free(iscsit_global.global_dbuf_store);
 785                 iscsit_global.global_dbuf_store = NULL;
 786         }
 787 
 788         if (iscsit_global.global_tsih_pool) {
 789                 vmem_destroy(iscsit_global.global_tsih_pool);
 790                 iscsit_global.global_tsih_pool = NULL;
 791         }
 792 
 793         avl_destroy(&iscsit_global.global_ini_list);
 794         avl_destroy(&iscsit_global.global_tpg_list);
 795         list_destroy(&iscsit_global.global_deleted_target_list);
 796         avl_destroy(&iscsit_global.global_target_list);
 797         avl_destroy(&iscsit_global.global_discovery_sessions);
 798 
 799         idm_refcnt_destroy(&iscsit_global.global_refcnt);
 800 
 801         return (retval);
 802 }
 803 
 804 /*
 805  * iscsit_disable_svc
 806  *
 807  * clean up all existing connections and deregister targets from STMF
 808  */
 809 static void
 810 iscsit_disable_svc(void)
 811 {
 812         iscsit_sess_t   *sess;
 813 
 814         ASSERT(iscsit_global.global_svc_state == ISE_DISABLING);
 815 
 816         iscsit_rxpdu_queue_monitor_stop();
 817 
 818         /* tear down discovery sessions */
 819         for (sess = avl_first(&iscsit_global.global_discovery_sessions);
 820             sess != NULL;
 821             sess = AVL_NEXT(&iscsit_global.global_discovery_sessions, sess))
 822                 iscsit_sess_close(sess);
 823 
 824         /*
 825          * Passing NULL to iscsit_config_merge tells it to go to an empty
 826          * config.
 827          */
 828         (void) iscsit_config_merge(NULL);
 829 
 830         /*
 831          * Wait until there are no more global references
 832          */
 833         idm_refcnt_wait_ref(&iscsit_global.global_refcnt);
 834         idm_refcnt_destroy(&iscsit_global.global_refcnt);
 835 
 836         /*
 837          * Default TPG must be destroyed after global_refcnt is 0.
 838          */
 839         iscsit_tpg_destroydefault(iscsit_global.global_default_tpg);
 840 
 841         avl_destroy(&iscsit_global.global_discovery_sessions);
 842         list_destroy(&iscsit_global.global_deleted_target_list);
 843         avl_destroy(&iscsit_global.global_target_list);
 844         avl_destroy(&iscsit_global.global_tpg_list);
 845         avl_destroy(&iscsit_global.global_ini_list);
 846 
 847         taskq_destroy(iscsit_global.global_dispatch_taskq);
 848 
 849         iscsit_isns_fini();
 850 
 851         stmf_free(iscsit_global.global_dbuf_store);
 852         iscsit_global.global_dbuf_store = NULL;
 853 
 854         (void) stmf_deregister_port_provider(iscsit_global.global_pp);
 855         stmf_free(iscsit_global.global_pp);
 856         iscsit_global.global_pp = NULL;
 857 
 858         kmem_cache_destroy(iscsit_status_pdu_cache);
 859         iscsit_status_pdu_cache = NULL;
 860 
 861         vmem_destroy(iscsit_global.global_tsih_pool);
 862         iscsit_global.global_tsih_pool = NULL;
 863 }
 864 
 865 void
 866 iscsit_global_hold()
 867 {
 868         /*
 869          * To take out a global hold, we must either own the global
 870          * state mutex or we must be running inside of an ioctl that
 871          * has set the global state to ISE_BUSY, ISE_DISABLING, or
 872          * ISE_ENABLING.  We don't track the "owner" for these flags,
 873          * so just checking if they are set is enough for now.
 874          */
 875         ASSERT((iscsit_global.global_svc_state == ISE_ENABLING) ||
 876             (iscsit_global.global_svc_state == ISE_DISABLING) ||
 877             (iscsit_global.global_svc_state == ISE_BUSY) ||
 878             MUTEX_HELD(&iscsit_global.global_state_mutex));
 879 
 880         idm_refcnt_hold(&iscsit_global.global_refcnt);
 881 }
 882 
 883 void
 884 iscsit_global_rele()
 885 {
 886         idm_refcnt_rele(&iscsit_global.global_refcnt);
 887 }
 888 
 889 void
 890 iscsit_global_wait_ref()
 891 {
 892         idm_refcnt_wait_ref(&iscsit_global.global_refcnt);
 893 }
 894 
 895 /*
 896  * IDM callbacks
 897  */
 898 
 899 /*ARGSUSED*/
 900 void
 901 iscsit_rx_pdu(idm_conn_t *ic, idm_pdu_t *rx_pdu)
 902 {
 903         iscsit_conn_t *ict = ic->ic_handle;
 904         switch (IDM_PDU_OPCODE(rx_pdu)) {
 905         case ISCSI_OP_SCSI_CMD:
 906                 ASSERT(0); /* Shouldn't happen */
 907                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
 908                 break;
 909         case ISCSI_OP_SNACK_CMD:
 910                 /*
 911                  * We'll need to handle this when we support ERL1/2.  For
 912                  * now we treat it as a protocol error.
 913                  */
 914                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
 915                 idm_conn_event(ic, CE_TRANSPORT_FAIL, NULL);
 916                 break;
 917         case ISCSI_OP_SCSI_TASK_MGT_MSG:
 918                 if (iscsit_check_cmdsn_and_queue(rx_pdu)) {
 919                         iscsit_set_cmdsn(ict, rx_pdu);
 920                         iscsit_op_scsi_task_mgmt(ict, rx_pdu);
 921                 }
 922                 break;
 923         case ISCSI_OP_NOOP_OUT:
 924         case ISCSI_OP_LOGIN_CMD:
 925         case ISCSI_OP_TEXT_CMD:
 926         case ISCSI_OP_LOGOUT_CMD:
 927                 /*
 928                  * If/when we switch to userland processing these PDU's
 929                  * will be handled by iscsitd.
 930                  */
 931                 iscsit_deferred_dispatch(rx_pdu);
 932                 break;
 933         default:
 934                 /* Protocol error */
 935                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
 936                 idm_conn_event(ic, CE_TRANSPORT_FAIL, NULL);
 937                 break;
 938         }
 939 }
 940 
 941 /*ARGSUSED*/
 942 void
 943 iscsit_rx_pdu_error(idm_conn_t *ic, idm_pdu_t *rx_pdu, idm_status_t status)
 944 {
 945         idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
 946 }
 947 
 948 void
 949 iscsit_task_aborted(idm_task_t *idt, idm_status_t status)
 950 {
 951         iscsit_task_t *itask = idt->idt_private;
 952 
 953         switch (status) {
 954         case IDM_STATUS_SUSPENDED:
 955                 break;
 956         case IDM_STATUS_ABORTED:
 957                 mutex_enter(&itask->it_mutex);
 958                 itask->it_aborted = B_TRUE;
 959                 /*
 960                  * We rely on the fact that STMF tracks outstanding
 961                  * buffer transfers and will free all of our buffers
 962                  * before freeing the task so we don't need to
 963                  * explicitly free the buffers from iscsit/idm
 964                  */
 965                 if (itask->it_stmf_abort) {
 966                         mutex_exit(&itask->it_mutex);
 967                         /*
 968                          * Task is no longer active
 969                          */
 970                         iscsit_task_done(itask);
 971 
 972                         /*
 973                          * STMF has already asked for this task to be aborted
 974                          *
 975                          * STMF specification is wrong... says to return
 976                          * STMF_ABORTED, the code actually looks for
 977                          * STMF_ABORT_SUCCESS.
 978                          */
 979                         stmf_task_lport_aborted(itask->it_stmf_task,
 980                             STMF_ABORT_SUCCESS, STMF_IOF_LPORT_DONE);
 981                         return;
 982                 } else {
 983                         mutex_exit(&itask->it_mutex);
 984                         /*
 985                          * Tell STMF to stop processing the task.
 986                          */
 987                         stmf_abort(STMF_QUEUE_TASK_ABORT, itask->it_stmf_task,
 988                             STMF_ABORTED, NULL);
 989                         return;
 990                 }
 991                 /*NOTREACHED*/
 992         default:
 993                 ASSERT(0);
 994         }
 995 }
 996 
 997 /*ARGSUSED*/
 998 idm_status_t
 999 iscsit_client_notify(idm_conn_t *ic, idm_client_notify_t icn,
1000     uintptr_t data)
1001 {
1002         idm_status_t rc = IDM_STATUS_SUCCESS;
1003 
1004         /*
1005          * IDM client notifications will never occur at interrupt level
1006          * since they are generated from the connection state machine which
1007          * running on taskq threads.
1008          *
1009          */
1010         switch (icn) {
1011         case CN_CONNECT_ACCEPT:
1012                 rc = iscsit_conn_accept(ic); /* No data */
1013                 break;
1014         case CN_FFP_ENABLED:
1015                 rc = iscsit_ffp_enabled(ic); /* No data */
1016                 break;
1017         case CN_FFP_DISABLED:
1018                 /*
1019                  * Data indicates whether this was the result of an
1020                  * explicit logout request.
1021                  */
1022                 rc = iscsit_ffp_disabled(ic, (idm_ffp_disable_t)data);
1023                 break;
1024         case CN_CONNECT_LOST:
1025                 rc = iscsit_conn_lost(ic);
1026                 break;
1027         case CN_CONNECT_DESTROY:
1028                 rc = iscsit_conn_destroy(ic);
1029                 break;
1030         case CN_LOGIN_FAIL:
1031                 /*
1032                  * Force the login state machine to completion
1033                  */
1034                 rc = iscsit_login_fail(ic);
1035                 break;
1036         default:
1037                 rc = IDM_STATUS_REJECT;
1038                 break;
1039         }
1040 
1041         return (rc);
1042 }
1043 
1044 /*
1045  * iscsit_update_statsn is invoked for all the PDUs which have the StatSN
1046  * field in the header. The StatSN is incremented if the IDM_PDU_ADVANCE_STATSN
1047  * flag is set in the pdu flags field. The StatSN is connection-wide and is
1048  * protected by the mutex ict_statsn_mutex. For Data-In PDUs, if the flag
1049  * IDM_TASK_PHASECOLLAPSE_REQ is set, the status (phase-collapse) is also filled
1050  */
1051 void
1052 iscsit_update_statsn(idm_task_t *idm_task, idm_pdu_t *pdu)
1053 {
1054         iscsi_scsi_rsp_hdr_t *rsp = (iscsi_scsi_rsp_hdr_t *)pdu->isp_hdr;
1055         iscsit_conn_t *ict = (iscsit_conn_t *)pdu->isp_ic->ic_handle;
1056         iscsit_task_t *itask = NULL;
1057         scsi_task_t *task = NULL;
1058 
1059         mutex_enter(&ict->ict_statsn_mutex);
1060         rsp->statsn = htonl(ict->ict_statsn);
1061         if (pdu->isp_flags & IDM_PDU_ADVANCE_STATSN)
1062                 ict->ict_statsn++;
1063         mutex_exit(&ict->ict_statsn_mutex);
1064 
1065         /*
1066          * The last SCSI Data PDU passed for a command may also contain the
1067          * status if the status indicates termination with no expections, i.e.
1068          * no sense data or response involved. If the command completes with
1069          * an error, then the response and sense data will be sent in a
1070          * separate iSCSI Response PDU.
1071          */
1072         if ((idm_task) && (idm_task->idt_flags & IDM_TASK_PHASECOLLAPSE_REQ)) {
1073                 itask = idm_task->idt_private;
1074                 task = itask->it_stmf_task;
1075 
1076                 rsp->cmd_status = task->task_scsi_status;
1077                 rsp->flags   |= ISCSI_FLAG_DATA_STATUS;
1078                 if (task->task_status_ctrl & TASK_SCTRL_OVER) {
1079                         rsp->flags |= ISCSI_FLAG_CMD_OVERFLOW;
1080                 } else if (task->task_status_ctrl & TASK_SCTRL_UNDER) {
1081                         rsp->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
1082                 }
1083                 rsp->residual_count = htonl(task->task_resid);
1084 
1085                 /*
1086                  * Removing the task from the session task list
1087                  * just before the status is sent in the last
1088                  * Data PDU transfer
1089                  */
1090                 iscsit_task_done(itask);
1091         }
1092 }
1093 
1094 void
1095 iscsit_build_hdr(idm_task_t *idm_task, idm_pdu_t *pdu, uint8_t opcode)
1096 {
1097         iscsit_task_t *itask = idm_task->idt_private;
1098         iscsi_data_rsp_hdr_t *dh = (iscsi_data_rsp_hdr_t *)pdu->isp_hdr;
1099 
1100         /*
1101          * We acquired iscsit_sess_t.ist_sn_mutex in iscsit_xfer_scsi_data
1102          */
1103         ASSERT(MUTEX_HELD(&itask->it_ict->ict_sess->ist_sn_mutex));
1104         /*
1105          * On incoming data, the target transfer tag and Lun is only
1106          * provided by the target if the A bit is set, Since the target
1107          * does not currently support Error Recovery Level 1, the A
1108          * bit is never set.
1109          */
1110         dh->opcode = opcode;
1111         dh->itt = itask->it_itt;
1112         dh->ttt = ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_SCSI_DATA_RSP) ?
1113             ISCSI_RSVD_TASK_TAG : itask->it_ttt;
1114 
1115         dh->expcmdsn = htonl(itask->it_ict->ict_sess->ist_expcmdsn);
1116         dh->maxcmdsn = htonl(itask->it_ict->ict_sess->ist_maxcmdsn);
1117 
1118         /*
1119          * IDM must set:
1120          *
1121          * data.flags and rtt.flags
1122          * data.dlength
1123          * data.datasn
1124          * data.offset
1125          * statsn, residual_count and cmd_status (for phase collapse)
1126          * rtt.rttsn
1127          * rtt.data_offset
1128          * rtt.data_length
1129          */
1130 }
1131 
1132 void
1133 iscsit_keepalive(idm_conn_t *ic)
1134 {
1135         idm_pdu_t               *nop_in_pdu;
1136         iscsi_nop_in_hdr_t      *nop_in;
1137         iscsit_conn_t           *ict = ic->ic_handle;
1138 
1139         /*
1140          * IDM noticed the connection has been idle for too long so it's
1141          * time to provoke some activity.  Build and transmit an iSCSI
1142          * nop-in PDU -- when the initiator responds it will be counted
1143          * as "activity" and keep the connection alive.
1144          *
1145          * We don't actually care about the response here at the iscsit level
1146          * so we will just throw it away without looking at it when it arrives.
1147          */
1148         nop_in_pdu = idm_pdu_alloc(sizeof (*nop_in), 0);
1149         idm_pdu_init(nop_in_pdu, ic, NULL, NULL);
1150         nop_in = (iscsi_nop_in_hdr_t *)nop_in_pdu->isp_hdr;
1151         bzero(nop_in, sizeof (*nop_in));
1152         nop_in->opcode = ISCSI_OP_NOOP_IN;
1153         nop_in->flags = ISCSI_FLAG_FINAL;
1154         nop_in->itt = ISCSI_RSVD_TASK_TAG;
1155         /*
1156          * When the target sends a NOP-In as a Ping, the target transfer tag
1157          * is set to a valid (not reserved) value and the initiator task tag
1158          * is set to ISCSI_RSVD_TASK_TAG (0xffffffff). In this case the StatSN
1159          * will always contain the next sequence number but the StatSN for the
1160          * connection is not advanced after this PDU is sent.
1161          */
1162         nop_in_pdu->isp_flags |= IDM_PDU_SET_STATSN;
1163         /*
1164          * This works because we don't currently allocate ttt's anywhere else
1165          * in iscsit so as long as we stay out of IDM's range we are safe.
1166          * If we need to allocate ttt's for other PDU's in the future this will
1167          * need to be improved.
1168          */
1169         mutex_enter(&ict->ict_mutex);
1170         nop_in->ttt = ict->ict_keepalive_ttt;
1171         ict->ict_keepalive_ttt++;
1172         if (ict->ict_keepalive_ttt == ISCSI_RSVD_TASK_TAG)
1173                 ict->ict_keepalive_ttt = IDM_TASKIDS_MAX;
1174         mutex_exit(&ict->ict_mutex);
1175 
1176         iscsit_pdu_tx(nop_in_pdu);
1177 }
1178 
1179 static idm_status_t
1180 iscsit_conn_accept(idm_conn_t *ic)
1181 {
1182         iscsit_conn_t *ict;
1183 
1184         /*
1185          * We need to get a global hold here to ensure that the service
1186          * doesn't get shutdown prior to establishing a session. This
1187          * gets released in iscsit_conn_destroy().
1188          */
1189         mutex_enter(&iscsit_global.global_state_mutex);
1190         if (iscsit_global.global_svc_state != ISE_ENABLED) {
1191                 mutex_exit(&iscsit_global.global_state_mutex);
1192                 return (IDM_STATUS_FAIL);
1193         }
1194         iscsit_global_hold();
1195         mutex_exit(&iscsit_global.global_state_mutex);
1196 
1197         /*
1198          * Allocate an associated iscsit structure to represent this
1199          * connection.  We shouldn't really create a session until we
1200          * get the first login PDU.
1201          */
1202         ict = kmem_zalloc(sizeof (*ict), KM_SLEEP);
1203 
1204         ict->ict_ic = ic;
1205         ict->ict_statsn = 1;
1206         ict->ict_keepalive_ttt = IDM_TASKIDS_MAX; /* Avoid IDM TT range */
1207         ic->ic_handle = ict;
1208         mutex_init(&ict->ict_mutex, NULL, MUTEX_DRIVER, NULL);
1209         mutex_init(&ict->ict_statsn_mutex, NULL, MUTEX_DRIVER, NULL);
1210         idm_refcnt_init(&ict->ict_refcnt, ict);
1211 
1212         /*
1213          * Initialize login state machine
1214          */
1215         if (iscsit_login_sm_init(ict) != IDM_STATUS_SUCCESS) {
1216                 iscsit_global_rele();
1217                 /*
1218                  * Cleanup the ict after idm notifies us about this failure
1219                  */
1220                 return (IDM_STATUS_FAIL);
1221         }
1222 
1223         return (IDM_STATUS_SUCCESS);
1224 }
1225 
1226 idm_status_t
1227 iscsit_conn_reinstate(iscsit_conn_t *reinstate_ict, iscsit_conn_t *new_ict)
1228 {
1229         idm_status_t    result;
1230 
1231         /*
1232          * Note in new connection state that this connection is
1233          * reinstating an existing connection.
1234          */
1235         new_ict->ict_reinstating = B_TRUE;
1236         new_ict->ict_reinstate_conn = reinstate_ict;
1237         new_ict->ict_statsn = reinstate_ict->ict_statsn;
1238 
1239         /*
1240          * Now generate connection state machine event to existing connection
1241          * so that it starts the cleanup process.
1242          */
1243         result = idm_conn_reinstate_event(reinstate_ict->ict_ic,
1244             new_ict->ict_ic);
1245 
1246         return (result);
1247 }
1248 
1249 void
1250 iscsit_conn_hold(iscsit_conn_t *ict)
1251 {
1252         idm_refcnt_hold(&ict->ict_refcnt);
1253 }
1254 
1255 void
1256 iscsit_conn_rele(iscsit_conn_t *ict)
1257 {
1258         idm_refcnt_rele(&ict->ict_refcnt);
1259 }
1260 
1261 void
1262 iscsit_conn_dispatch_hold(iscsit_conn_t *ict)
1263 {
1264         idm_refcnt_hold(&ict->ict_dispatch_refcnt);
1265 }
1266 
1267 void
1268 iscsit_conn_dispatch_rele(iscsit_conn_t *ict)
1269 {
1270         idm_refcnt_rele(&ict->ict_dispatch_refcnt);
1271 }
1272 
1273 static idm_status_t
1274 iscsit_login_fail(idm_conn_t *ic)
1275 {
1276         iscsit_conn_t *ict = ic->ic_handle;
1277 
1278         /* Generate login state machine event */
1279         iscsit_login_sm_event(ict, ILE_LOGIN_CONN_ERROR, NULL);
1280 
1281         return (IDM_STATUS_SUCCESS);
1282 }
1283 
1284 static idm_status_t
1285 iscsit_ffp_enabled(idm_conn_t *ic)
1286 {
1287         iscsit_conn_t *ict = ic->ic_handle;
1288 
1289         /* Generate session state machine event */
1290         iscsit_sess_sm_event(ict->ict_sess, SE_CONN_LOGGED_IN, ict);
1291 
1292         return (IDM_STATUS_SUCCESS);
1293 }
1294 
1295 static idm_status_t
1296 iscsit_ffp_disabled(idm_conn_t *ic, idm_ffp_disable_t disable_class)
1297 {
1298         iscsit_conn_t *ict = ic->ic_handle;
1299 
1300         /* Generate session state machine event */
1301         switch (disable_class) {
1302         case FD_CONN_FAIL:
1303                 iscsit_sess_sm_event(ict->ict_sess, SE_CONN_FFP_FAIL, ict);
1304                 break;
1305         case FD_CONN_LOGOUT:
1306                 iscsit_sess_sm_event(ict->ict_sess, SE_CONN_FFP_DISABLE, ict);
1307                 break;
1308         case FD_SESS_LOGOUT:
1309                 iscsit_sess_sm_event(ict->ict_sess, SE_SESSION_CLOSE, ict);
1310                 break;
1311         default:
1312                 ASSERT(0);
1313         }
1314 
1315         return (IDM_STATUS_SUCCESS);
1316 }
1317 
1318 static idm_status_t
1319 iscsit_conn_lost(idm_conn_t *ic)
1320 {
1321         iscsit_conn_t   *ict    = ic->ic_handle;
1322         iscsit_sess_t   *ist    = ict->ict_sess;
1323         iscsit_cbuf_t   *cbuf;
1324         idm_pdu_t       *rx_pdu;
1325         int i;
1326 
1327         mutex_enter(&ict->ict_mutex);
1328         ict->ict_lost = B_TRUE;
1329         mutex_exit(&ict->ict_mutex);
1330         /*
1331          * scrub the staging queue for all PDUs on this connection
1332          */
1333         if (ist != NULL) {
1334                 mutex_enter(&ist->ist_sn_mutex);
1335                 for (cbuf = ist->ist_rxpdu_queue, i = 0;
1336                     ((cbuf->cb_num_elems > 0) && (i < ISCSIT_RXPDU_QUEUE_LEN));
1337                     i++) {
1338                         if (((rx_pdu = cbuf->cb_buffer[i]) != NULL) &&
1339                             (rx_pdu->isp_ic == ic)) {
1340                                 /* conn is lost, drop the pdu */
1341                                 DTRACE_PROBE3(scrubbing__staging__queue,
1342                                     iscsit_sess_t *, ist, idm_conn_t *, ic,
1343                                     idm_pdu_t *, rx_pdu);
1344                                 idm_pdu_complete(rx_pdu, IDM_STATUS_FAIL);
1345                                 cbuf->cb_buffer[i] = NULL;
1346                                 cbuf->cb_num_elems--;
1347                                 iscsit_conn_dispatch_rele(ict);
1348                         }
1349                 }
1350                 mutex_exit(&ist->ist_sn_mutex);
1351         }
1352         /*
1353          * Make sure there aren't any PDU's transitioning from the receive
1354          * handler to the dispatch taskq.
1355          */
1356         idm_refcnt_wait_ref(&ict->ict_dispatch_refcnt);
1357 
1358         return (IDM_STATUS_SUCCESS);
1359 }
1360 
1361 static idm_status_t
1362 iscsit_conn_destroy(idm_conn_t *ic)
1363 {
1364         iscsit_conn_t *ict = ic->ic_handle;
1365 
1366         mutex_enter(&ict->ict_mutex);
1367         ict->ict_destroyed = B_TRUE;
1368         mutex_exit(&ict->ict_mutex);
1369 
1370         /* Generate session state machine event */
1371         if (ict->ict_sess != NULL) {
1372                 /*
1373                  * Session state machine will call iscsit_conn_destroy_done()
1374                  * when it has removed references to this connection.
1375                  */
1376                 iscsit_sess_sm_event(ict->ict_sess, SE_CONN_FAIL, ict);
1377         }
1378 
1379         idm_refcnt_wait_ref(&ict->ict_refcnt);
1380         /*
1381          * The session state machine does not need to post
1382          * events to IDM any longer, so it is safe to set
1383          * the idm connection reference to NULL
1384          */
1385         ict->ict_ic = NULL;
1386 
1387         /* Reap the login state machine */
1388         iscsit_login_sm_fini(ict);
1389 
1390         /* Clean up any text command remnants */
1391         iscsit_text_cmd_fini(ict);
1392 
1393         mutex_destroy(&ict->ict_mutex);
1394         idm_refcnt_destroy(&ict->ict_refcnt);
1395         kmem_free(ict, sizeof (*ict));
1396 
1397         iscsit_global_rele();
1398 
1399         return (IDM_STATUS_SUCCESS);
1400 }
1401 
1402 void
1403 iscsit_conn_logout(iscsit_conn_t *ict)
1404 {
1405         /*
1406          * If the iscsi connection is active, then
1407          * logout the IDM connection by sending a
1408          * CE_LOGOUT_SESSION_SUCCESS, else, no action
1409          * needs to be taken because the connection
1410          * is already in the teardown process.
1411          */
1412         mutex_enter(&ict->ict_mutex);
1413         if (ict->ict_lost == B_FALSE && ict->ict_destroyed == B_FALSE) {
1414                 idm_conn_event(ict->ict_ic, CE_LOGOUT_SESSION_SUCCESS, NULL);
1415         }
1416         mutex_exit(&ict->ict_mutex);
1417 }
1418 
1419 /*
1420  * STMF-related functions
1421  *
1422  * iSCSI to STMF mapping
1423  *
1424  * Session == ?
1425  * Connection == bound to local port but not itself a local port
1426  * Target
1427  * Target portal (group?) == local port (really but we're not going to do this)
1428  *      iscsit needs to map connections to local ports (whatever we decide
1429  *      they are)
1430  * Target == ?
1431  */
1432 
1433 /*ARGSUSED*/
1434 static stmf_data_buf_t *
1435 iscsit_dbuf_alloc(scsi_task_t *task, uint32_t size, uint32_t *pminsize,
1436     uint32_t flags)
1437 {
1438         iscsit_task_t *itask = task->task_port_private;
1439         idm_buf_t *idm_buffer;
1440         iscsit_buf_t    *ibuf;
1441         stmf_data_buf_t *result;
1442         uint32_t        bsize;
1443 
1444         /*
1445          * If the requested size is larger than MaxBurstLength and the
1446          * given pminsize is also larger than MaxBurstLength, then the
1447          * allocation fails (dbuf = NULL) and pminsize is modified to
1448          * be equal to MaxBurstLength. stmf/sbd then should re-invoke
1449          * this function with the corrected values for transfer.
1450          */
1451         ASSERT(pminsize);
1452         if (size <= itask->it_ict->ict_op.op_max_burst_length) {
1453                 bsize = size;
1454         } else if (*pminsize <= itask->it_ict->ict_op.op_max_burst_length) {
1455                 bsize = itask->it_ict->ict_op.op_max_burst_length;
1456         } else {
1457                 *pminsize = itask->it_ict->ict_op.op_max_burst_length;
1458                 return (NULL);
1459         }
1460 
1461         /* Alloc buffer */
1462         idm_buffer = idm_buf_alloc(itask->it_ict->ict_ic, NULL, bsize);
1463         if (idm_buffer != NULL) {
1464                 result = stmf_alloc(STMF_STRUCT_DATA_BUF,
1465                     sizeof (iscsit_buf_t), 0);
1466                 if (result != NULL) {
1467                         /* Fill in stmf_data_buf_t */
1468                         ibuf = result->db_port_private;
1469                         ibuf->ibuf_idm_buf = idm_buffer;
1470                         ibuf->ibuf_stmf_buf = result;
1471                         ibuf->ibuf_is_immed = B_FALSE;
1472                         result->db_flags = DB_DONT_CACHE;
1473                         result->db_buf_size = bsize;
1474                         result->db_data_size = bsize;
1475                         result->db_sglist_length = 1;
1476                         result->db_sglist[0].seg_addr = idm_buffer->idb_buf;
1477                         result->db_sglist[0].seg_length =
1478                             idm_buffer->idb_buflen;
1479                         return (result);
1480                 }
1481 
1482                 /* Couldn't get the stmf_data_buf_t so free the buffer */
1483                 idm_buf_free(idm_buffer);
1484         }
1485 
1486         return (NULL);
1487 }
1488 
1489 /*ARGSUSED*/
1490 static void
1491 iscsit_dbuf_free(stmf_dbuf_store_t *ds, stmf_data_buf_t *dbuf)
1492 {
1493         iscsit_buf_t *ibuf = dbuf->db_port_private;
1494 
1495         if (ibuf->ibuf_is_immed) {
1496                 /*
1497                  * The iscsit_buf_t structure itself will be freed with its
1498                  * associated task.  Here we just need to free the PDU that
1499                  * held the immediate data.
1500                  */
1501                 idm_pdu_complete(ibuf->ibuf_immed_data_pdu, IDM_STATUS_SUCCESS);
1502                 ibuf->ibuf_immed_data_pdu = 0;
1503         } else {
1504                 idm_buf_free(ibuf->ibuf_idm_buf);
1505                 stmf_free(dbuf);
1506         }
1507 }
1508 
1509 /*ARGSUSED*/
1510 stmf_status_t
1511 iscsit_xfer_scsi_data(scsi_task_t *task, stmf_data_buf_t *dbuf,
1512     uint32_t ioflags)
1513 {
1514         iscsit_task_t *iscsit_task = task->task_port_private;
1515         iscsit_sess_t *ict_sess = iscsit_task->it_ict->ict_sess;
1516         iscsit_buf_t *ibuf = dbuf->db_port_private;
1517         int idm_rc;
1518 
1519         /*
1520          * If we are aborting then we can ignore this request
1521          */
1522         if (iscsit_task->it_stmf_abort) {
1523                 return (STMF_SUCCESS);
1524         }
1525 
1526         /*
1527          * If it's not immediate data then start the transfer
1528          */
1529         ASSERT(ibuf->ibuf_is_immed == B_FALSE);
1530         if (dbuf->db_flags & DB_DIRECTION_TO_RPORT) {
1531                 /*
1532                  * The DB_SEND_STATUS_GOOD flag in the STMF data buffer allows
1533                  * the port provider to phase-collapse, i.e. send the status
1534                  * along with the final data PDU for the command. The port
1535                  * provider passes this request to the transport layer by
1536                  * setting a flag IDM_TASK_PHASECOLLAPSE_REQ in the task.
1537                  */
1538                 if (dbuf->db_flags & DB_SEND_STATUS_GOOD)
1539                         iscsit_task->it_idm_task->idt_flags |=
1540                             IDM_TASK_PHASECOLLAPSE_REQ;
1541                 /*
1542                  * IDM will call iscsit_build_hdr so lock now to serialize
1543                  * access to the SN values.  We need to lock here to enforce
1544                  * lock ordering
1545                  */
1546                 mutex_enter(&ict_sess->ist_sn_mutex);
1547                 idm_rc = idm_buf_tx_to_ini(iscsit_task->it_idm_task,
1548                     ibuf->ibuf_idm_buf, dbuf->db_relative_offset,
1549                     dbuf->db_data_size, &iscsit_buf_xfer_cb, dbuf);
1550                 mutex_exit(&ict_sess->ist_sn_mutex);
1551 
1552                 return (iscsit_idm_to_stmf(idm_rc));
1553         } else if (dbuf->db_flags & DB_DIRECTION_FROM_RPORT) {
1554                 /* Grab the SN lock (see comment above) */
1555                 mutex_enter(&ict_sess->ist_sn_mutex);
1556                 idm_rc = idm_buf_rx_from_ini(iscsit_task->it_idm_task,
1557                     ibuf->ibuf_idm_buf, dbuf->db_relative_offset,
1558                     dbuf->db_data_size, &iscsit_buf_xfer_cb, dbuf);
1559                 mutex_exit(&ict_sess->ist_sn_mutex);
1560 
1561                 return (iscsit_idm_to_stmf(idm_rc));
1562         }
1563 
1564         /* What are we supposed to do if there is no direction? */
1565         return (STMF_INVALID_ARG);
1566 }
1567 
1568 static void
1569 iscsit_buf_xfer_cb(idm_buf_t *idb, idm_status_t status)
1570 {
1571         iscsit_task_t *itask = idb->idb_task_binding->idt_private;
1572         stmf_data_buf_t *dbuf = idb->idb_cb_arg;
1573 
1574         dbuf->db_xfer_status = iscsit_idm_to_stmf(status);
1575 
1576         /*
1577          * If the task has been aborted then we don't need to call STMF
1578          */
1579         if (itask->it_stmf_abort) {
1580                 return;
1581         }
1582 
1583         /*
1584          * For ISCSI over TCP (not iSER), the last SCSI Data PDU passed
1585          * for a successful command contains the status as requested by
1586          * by COMSTAR (via the DB_SEND_STATUS_GOOD flag). But the iSER
1587          * transport does not support phase-collapse. So pretend we are
1588          * COMSTAR and send the status in a separate PDU now.
1589          */
1590         if (idb->idb_task_binding->idt_flags & IDM_TASK_PHASECOLLAPSE_SUCCESS) {
1591                 /*
1592                  * Mark task complete and notify COMSTAR
1593                  * that the status has been sent.
1594                  */
1595                 itask->it_idm_task->idt_state = TASK_COMPLETE;
1596                 stmf_send_status_done(itask->it_stmf_task,
1597                     iscsit_idm_to_stmf(status), STMF_IOF_LPORT_DONE);
1598         } else if ((dbuf->db_flags & DB_SEND_STATUS_GOOD) &&
1599             status == IDM_STATUS_SUCCESS) {
1600 
1601                 /*
1602                  * The iscsi target port provider - for iSER, emulates the
1603                  * DB_SEND_STATUS_GOOD optimization if requested by STMF;
1604                  * it sends the status in a separate PDU after the data
1605                  * transfer. In this case the port provider should first
1606                  * call stmf_data_xfer_done() to mark the transfer complete
1607                  * and then send the status. Although STMF will free the
1608                  * buffer at the time the task is freed, even if the transfer
1609                  * is not marked complete, this behavior makes statistics
1610                  * gathering and task state tracking more difficult than it
1611                  * needs to be.
1612                  */
1613                 stmf_data_xfer_done(itask->it_stmf_task, dbuf, 0);
1614                 if (iscsit_send_scsi_status(itask->it_stmf_task, 0)
1615                     != STMF_SUCCESS) {
1616                         stmf_send_status_done(itask->it_stmf_task,
1617                             STMF_FAILURE, STMF_IOF_LPORT_DONE);
1618                 }
1619         } else {
1620                 stmf_data_xfer_done(itask->it_stmf_task, dbuf, 0);
1621                 /* don't touch dbuf after stmf_data_xfer_done */
1622         }
1623 }
1624 
1625 
1626 /*ARGSUSED*/
1627 stmf_status_t
1628 iscsit_send_scsi_status(scsi_task_t *task, uint32_t ioflags)
1629 {
1630         iscsit_task_t *itask = task->task_port_private;
1631         iscsi_scsi_rsp_hdr_t *rsp;
1632         idm_pdu_t *pdu;
1633         int resp_datalen;
1634 
1635         /*
1636          * If this task is aborted then we don't need to respond.
1637          */
1638         if (itask->it_stmf_abort) {
1639                 return (STMF_SUCCESS);
1640         }
1641 
1642         /*
1643          * If this is a task management status, handle it elsewhere.
1644          */
1645         if (task->task_mgmt_function != TM_NONE) {
1646                 /*
1647                  * Don't wait for the PDU completion to tell STMF
1648                  * the task is done -- it doesn't really matter and
1649                  * it makes life complicated if STMF later asks us to
1650                  * abort the request and we don't know whether the
1651                  * status has been sent or not.
1652                  */
1653                 itask->it_tm_responded = B_TRUE;
1654                 iscsit_send_task_mgmt_resp(itask->it_tm_pdu,
1655                     (task->task_completion_status == STMF_SUCCESS) ?
1656                     SCSI_TCP_TM_RESP_COMPLETE : SCSI_TCP_TM_RESP_FUNC_NOT_SUPP);
1657                 stmf_send_status_done(task, STMF_SUCCESS,
1658                     STMF_IOF_LPORT_DONE);
1659                 return (STMF_SUCCESS);
1660         }
1661 
1662         /*
1663          * Remove the task from the session task list
1664          */
1665         iscsit_task_done(itask);
1666 
1667         /*
1668          * Send status
1669          */
1670         mutex_enter(&itask->it_idm_task->idt_mutex);
1671         if ((itask->it_idm_task->idt_state == TASK_ACTIVE) &&
1672             (task->task_completion_status == STMF_SUCCESS) &&
1673             (task->task_sense_length == 0) &&
1674             (task->task_resid == 0)) {
1675                 itask->it_idm_task->idt_state = TASK_COMPLETE;
1676                 /* PDU callback releases task hold */
1677                 idm_task_hold(itask->it_idm_task);
1678                 mutex_exit(&itask->it_idm_task->idt_mutex);
1679                 /*
1680                  * Fast path.  Cached status PDU's are already
1681                  * initialized.  We just need to fill in
1682                  * connection and task information. StatSN is
1683                  * incremented by 1 for every status sent a
1684                  * connection.
1685                  */
1686                 pdu = kmem_cache_alloc(iscsit_status_pdu_cache, KM_SLEEP);
1687                 pdu->isp_ic = itask->it_ict->ict_ic;
1688                 pdu->isp_private = itask;
1689                 pdu->isp_flags |= IDM_PDU_SET_STATSN | IDM_PDU_ADVANCE_STATSN;
1690 
1691                 rsp = (iscsi_scsi_rsp_hdr_t *)pdu->isp_hdr;
1692                 rsp->itt = itask->it_itt;
1693                 /*
1694                  * ExpDataSN is the number of R2T and Data-In (read)
1695                  * PDUs the target has sent for the SCSI command.
1696                  *
1697                  * Since there is no support for bidirectional transfer
1698                  * yet, either idt_exp_datasn or idt_exp_rttsn, but not
1699                  * both is valid at any time
1700                  */
1701                 rsp->expdatasn = (itask->it_idm_task->idt_exp_datasn != 0) ?
1702                     htonl(itask->it_idm_task->idt_exp_datasn):
1703                     htonl(itask->it_idm_task->idt_exp_rttsn);
1704                 rsp->cmd_status = task->task_scsi_status;
1705                 iscsit_pdu_tx(pdu);
1706                 return (STMF_SUCCESS);
1707         } else {
1708                 if (itask->it_idm_task->idt_state != TASK_ACTIVE) {
1709                         mutex_exit(&itask->it_idm_task->idt_mutex);
1710                         return (STMF_FAILURE);
1711                 }
1712                 itask->it_idm_task->idt_state = TASK_COMPLETE;
1713                 /* PDU callback releases task hold */
1714                 idm_task_hold(itask->it_idm_task);
1715                 mutex_exit(&itask->it_idm_task->idt_mutex);
1716 
1717                 resp_datalen = (task->task_sense_length == 0) ? 0 :
1718                     (task->task_sense_length + sizeof (uint16_t));
1719 
1720                 pdu = idm_pdu_alloc(sizeof (iscsi_hdr_t), resp_datalen);
1721                 idm_pdu_init(pdu, itask->it_ict->ict_ic, itask,
1722                     iscsit_send_status_done);
1723                 pdu->isp_flags |= IDM_PDU_SET_STATSN | IDM_PDU_ADVANCE_STATSN;
1724 
1725                 rsp = (iscsi_scsi_rsp_hdr_t *)pdu->isp_hdr;
1726                 bzero(rsp, sizeof (*rsp));
1727                 rsp->opcode = ISCSI_OP_SCSI_RSP;
1728 
1729                 rsp->flags = ISCSI_FLAG_FINAL;
1730                 if (task->task_status_ctrl & TASK_SCTRL_OVER) {
1731                         rsp->flags |= ISCSI_FLAG_CMD_OVERFLOW;
1732                 } else if (task->task_status_ctrl & TASK_SCTRL_UNDER) {
1733                         rsp->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
1734                 }
1735 
1736                 rsp->bi_residual_count = 0;
1737                 rsp->residual_count = htonl(task->task_resid);
1738                 rsp->itt = itask->it_itt;
1739                 rsp->response = ISCSI_STATUS_CMD_COMPLETED;
1740                 rsp->expdatasn = (itask->it_idm_task->idt_exp_datasn != 0) ?
1741                     htonl(itask->it_idm_task->idt_exp_datasn):
1742                     htonl(itask->it_idm_task->idt_exp_rttsn);
1743                 rsp->cmd_status = task->task_scsi_status;
1744                 if (task->task_sense_length != 0) {
1745                         /*
1746                          * Add a byte to provide the sense length in
1747                          * the response
1748                          */
1749                         *(uint16_t *)((void *)pdu->isp_data) =
1750                             htons(task->task_sense_length);
1751                         bcopy(task->task_sense_data,
1752                             (uint8_t *)pdu->isp_data +
1753                             sizeof (uint16_t),
1754                             task->task_sense_length);
1755                         hton24(rsp->dlength, resp_datalen);
1756                 }
1757 
1758                 DTRACE_PROBE5(iscsi__scsi__response,
1759                     iscsit_conn_t *, itask->it_ict,
1760                     uint8_t, rsp->response,
1761                     uint8_t, rsp->cmd_status,
1762                     idm_pdu_t *, pdu,
1763                     scsi_task_t *, task);
1764 
1765                 iscsit_pdu_tx(pdu);
1766 
1767                 return (STMF_SUCCESS);
1768         }
1769 }
1770 
1771 /*ARGSUSED*/
1772 static void
1773 iscsit_send_good_status_done(idm_pdu_t *pdu, idm_status_t status)
1774 {
1775         iscsit_task_t   *itask;
1776         boolean_t       aborted;
1777 
1778         itask = pdu->isp_private;
1779         aborted = itask->it_stmf_abort;
1780 
1781         /*
1782          * After releasing the hold the task may be freed at any time so
1783          * don't touch it.
1784          */
1785         idm_task_rele(itask->it_idm_task);
1786         if (!aborted) {
1787                 stmf_send_status_done(itask->it_stmf_task,
1788                     iscsit_idm_to_stmf(pdu->isp_status), STMF_IOF_LPORT_DONE);
1789         }
1790         kmem_cache_free(iscsit_status_pdu_cache, pdu);
1791 }
1792 
1793 /*ARGSUSED*/
1794 static void
1795 iscsit_send_status_done(idm_pdu_t *pdu, idm_status_t status)
1796 {
1797         iscsit_task_t    *itask;
1798         boolean_t       aborted;
1799 
1800         itask = pdu->isp_private;
1801         aborted = itask->it_stmf_abort;
1802 
1803         /*
1804          * After releasing the hold the task may be freed at any time so
1805          * don't touch it.
1806          */
1807         idm_task_rele(itask->it_idm_task);
1808         if (!aborted) {
1809                 stmf_send_status_done(itask->it_stmf_task,
1810                     iscsit_idm_to_stmf(pdu->isp_status), STMF_IOF_LPORT_DONE);
1811         }
1812         idm_pdu_free(pdu);
1813 }
1814 
1815 
1816 void
1817 iscsit_lport_task_free(scsi_task_t *task)
1818 {
1819         iscsit_task_t *itask = task->task_port_private;
1820 
1821         /* We only call idm_task_start for regular tasks, not task management */
1822         if (task->task_mgmt_function == TM_NONE) {
1823                 idm_task_done(itask->it_idm_task);
1824                 iscsit_task_free(itask);
1825                 return;
1826         } else {
1827                 iscsit_tm_task_free(itask);
1828         }
1829 }
1830 
1831 /*ARGSUSED*/
1832 stmf_status_t
1833 iscsit_abort(stmf_local_port_t *lport, int abort_cmd, void *arg, uint32_t flags)
1834 {
1835         scsi_task_t     *st = (scsi_task_t *)arg;
1836         iscsit_task_t   *iscsit_task;
1837         idm_task_t      *idt;
1838 
1839         /*
1840          * If this is a task management request then there's really not much to
1841          * do.
1842          */
1843         if (st->task_mgmt_function != TM_NONE) {
1844                 return (STMF_ABORT_SUCCESS);
1845         }
1846 
1847         /*
1848          * Regular task, start cleaning up
1849          */
1850         iscsit_task = st->task_port_private;
1851         idt = iscsit_task->it_idm_task;
1852         mutex_enter(&iscsit_task->it_mutex);
1853         iscsit_task->it_stmf_abort = B_TRUE;
1854         if (iscsit_task->it_aborted) {
1855                 mutex_exit(&iscsit_task->it_mutex);
1856                 /*
1857                  * Task is no longer active
1858                  */
1859                 iscsit_task_done(iscsit_task);
1860 
1861                 /*
1862                  * STMF specification is wrong... says to return
1863                  * STMF_ABORTED, the code actually looks for
1864                  * STMF_ABORT_SUCCESS.
1865                  */
1866                 return (STMF_ABORT_SUCCESS);
1867         } else {
1868                 mutex_exit(&iscsit_task->it_mutex);
1869                 /*
1870                  * Call IDM to abort the task.  Due to a variety of
1871                  * circumstances the task may already be in the process of
1872                  * aborting.
1873                  * We'll let IDM worry about rationalizing all that except
1874                  * for one particular instance.  If the state of the task
1875                  * is TASK_COMPLETE, we need to indicate to the framework
1876                  * that we are in fact done.  This typically happens with
1877                  * framework-initiated task management type requests
1878                  * (e.g. abort task).
1879                  */
1880                 if (idt->idt_state == TASK_COMPLETE) {
1881                         idm_refcnt_wait_ref(&idt->idt_refcnt);
1882                         return (STMF_ABORT_SUCCESS);
1883                 } else {
1884                         idm_task_abort(idt->idt_ic, idt, AT_TASK_MGMT_ABORT);
1885                         return (STMF_SUCCESS);
1886                 }
1887         }
1888 
1889         /*NOTREACHED*/
1890 }
1891 
1892 /*ARGSUSED*/
1893 void
1894 iscsit_ctl(stmf_local_port_t *lport, int cmd, void *arg)
1895 {
1896         iscsit_tgt_t            *iscsit_tgt;
1897 
1898         ASSERT((cmd == STMF_CMD_LPORT_ONLINE) ||
1899             (cmd == STMF_ACK_LPORT_ONLINE_COMPLETE) ||
1900             (cmd == STMF_CMD_LPORT_OFFLINE) ||
1901             (cmd == STMF_ACK_LPORT_OFFLINE_COMPLETE));
1902 
1903         iscsit_tgt = (iscsit_tgt_t *)lport->lport_port_private;
1904 
1905         switch (cmd) {
1906         case STMF_CMD_LPORT_ONLINE:
1907                 iscsit_tgt_sm_event(iscsit_tgt, TE_STMF_ONLINE_REQ);
1908                 break;
1909         case STMF_CMD_LPORT_OFFLINE:
1910                 iscsit_tgt_sm_event(iscsit_tgt, TE_STMF_OFFLINE_REQ);
1911                 break;
1912         case STMF_ACK_LPORT_ONLINE_COMPLETE:
1913                 iscsit_tgt_sm_event(iscsit_tgt, TE_STMF_ONLINE_COMPLETE_ACK);
1914                 break;
1915         case STMF_ACK_LPORT_OFFLINE_COMPLETE:
1916                 iscsit_tgt_sm_event(iscsit_tgt, TE_STMF_OFFLINE_COMPLETE_ACK);
1917                 break;
1918 
1919         default:
1920                 break;
1921         }
1922 }
1923 
1924 static stmf_status_t
1925 iscsit_idm_to_stmf(idm_status_t idmrc)
1926 {
1927         switch (idmrc) {
1928         case IDM_STATUS_SUCCESS:
1929                 return (STMF_SUCCESS);
1930         default:
1931                 return (STMF_FAILURE);
1932         }
1933         /*NOTREACHED*/
1934 }
1935 
1936 void
1937 iscsit_op_scsi_cmd(idm_conn_t *ic, idm_pdu_t *rx_pdu)
1938 {
1939         iscsit_conn_t           *ict = ic->ic_handle;
1940 
1941         if (iscsit_check_cmdsn_and_queue(rx_pdu)) {
1942                 iscsit_post_scsi_cmd(ic, rx_pdu);
1943         }
1944         iscsit_process_pdu_in_queue(ict->ict_sess);
1945 }
1946 
1947 /*
1948  * ISCSI protocol
1949  */
1950 
1951 void
1952 iscsit_post_scsi_cmd(idm_conn_t *ic, idm_pdu_t *rx_pdu)
1953 {
1954         iscsit_conn_t           *ict;
1955         iscsit_task_t           *itask;
1956         scsi_task_t             *task;
1957         iscsit_buf_t            *ibuf;
1958         iscsi_scsi_cmd_hdr_t    *iscsi_scsi =
1959             (iscsi_scsi_cmd_hdr_t *)rx_pdu->isp_hdr;
1960         iscsi_addl_hdr_t        *ahs_hdr;
1961         uint16_t                addl_cdb_len = 0;
1962 
1963         ict = ic->ic_handle;
1964 
1965         itask = iscsit_task_alloc(ict);
1966         if (itask == NULL) {
1967                 /* Finish processing request */
1968                 iscsit_set_cmdsn(ict, rx_pdu);
1969 
1970                 iscsit_send_direct_scsi_resp(ict, rx_pdu,
1971                     ISCSI_STATUS_CMD_COMPLETED, STATUS_BUSY);
1972                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
1973                 return;
1974         }
1975 
1976         /*
1977          * Note CmdSN and ITT in task.  IDM will have already validated this
1978          * request against the connection state so we don't need to check
1979          * that (the connection may have changed state in the meantime but
1980          * we will catch that when we try to send a response)
1981          */
1982         itask->it_cmdsn = ntohl(iscsi_scsi->cmdsn);
1983         itask->it_itt = iscsi_scsi->itt;
1984 
1985         /*
1986          * Check for extended CDB AHS
1987          */
1988         if (iscsi_scsi->hlength > 0) {
1989                 ahs_hdr = (iscsi_addl_hdr_t *)iscsi_scsi;
1990                 addl_cdb_len = ((ahs_hdr->ahs_hlen_hi << 8) |
1991                     ahs_hdr->ahs_hlen_lo) - 1; /* Adjust for reserved byte */
1992                 if (((addl_cdb_len + 4) / sizeof (uint32_t)) >
1993                     iscsi_scsi->hlength) {
1994                         /* Mangled header info, drop it */
1995                         idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
1996                         return;
1997                 }
1998         }
1999 
2000         ict = rx_pdu->isp_ic->ic_handle; /* IDM client private */
2001 
2002         /*
2003          * Add task to session list.  This function will also check to
2004          * ensure that the task does not already exist.
2005          */
2006         if (iscsit_task_start(itask) != IDM_STATUS_SUCCESS) {
2007                 /*
2008                  * Task exists, free all resources and reject.  Don't
2009                  * update expcmdsn in this case because RFC 3720 says
2010                  * "The CmdSN of the rejected command PDU (if it is a
2011                  * non-immediate command) MUST NOT be considered received
2012                  * by the target (i.e., a command sequence gap must be
2013                  * assumed for the CmdSN), even though the CmdSN of the
2014                  * rejected command PDU may be reliably ascertained.  Upon
2015                  * receiving the Reject, the initiator MUST plug the CmdSN
2016                  * gap in order to continue to use the session.  The gap
2017                  * may be plugged either by transmitting a command PDU
2018                  * with the same CmdSN, or by aborting the task (see section
2019                  * 6.9 on how an abort may plug a CmdSN gap)." (Section 6.3)
2020                  */
2021                 iscsit_task_free(itask);
2022                 iscsit_send_reject(ict, rx_pdu, ISCSI_REJECT_TASK_IN_PROGRESS);
2023                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2024                 return;
2025         }
2026 
2027         /* Update sequence numbers */
2028         iscsit_set_cmdsn(ict, rx_pdu);
2029 
2030         /*
2031          * Allocate STMF task
2032          */
2033         itask->it_stmf_task = stmf_task_alloc(
2034             itask->it_ict->ict_sess->ist_lport,
2035             itask->it_ict->ict_sess->ist_stmf_sess, iscsi_scsi->lun,
2036             16 + addl_cdb_len, 0);
2037         if (itask->it_stmf_task == NULL) {
2038                 /*
2039                  * Either stmf really couldn't get memory for a task or,
2040                  * more likely, the LU is currently in reset.  Either way
2041                  * we have no choice but to fail the request.
2042                  */
2043                 iscsit_task_done(itask);
2044                 iscsit_task_free(itask);
2045                 iscsit_send_direct_scsi_resp(ict, rx_pdu,
2046                     ISCSI_STATUS_CMD_COMPLETED, STATUS_BUSY);
2047                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2048                 return;
2049         }
2050 
2051         task = itask->it_stmf_task;
2052         task->task_port_private = itask;
2053 
2054         bcopy(iscsi_scsi->lun, task->task_lun_no, sizeof (task->task_lun_no));
2055 
2056         /*
2057          * iSCSI and Comstar use the same values.  Should we rely on this
2058          * or translate them bit-wise?
2059          */
2060 
2061         task->task_flags =
2062             (((iscsi_scsi->flags & ISCSI_FLAG_CMD_READ) ? TF_READ_DATA : 0) |
2063             ((iscsi_scsi->flags & ISCSI_FLAG_CMD_WRITE) ? TF_WRITE_DATA : 0) |
2064             ((rx_pdu->isp_datalen == 0) ? 0 : TF_INITIAL_BURST));
2065 
2066         switch (iscsi_scsi->flags & ISCSI_FLAG_CMD_ATTR_MASK) {
2067         case ISCSI_ATTR_UNTAGGED:
2068                 break;
2069         case ISCSI_ATTR_SIMPLE:
2070                 task->task_additional_flags |= TF_ATTR_SIMPLE_QUEUE;
2071                 break;
2072         case ISCSI_ATTR_ORDERED:
2073                 task->task_additional_flags |= TF_ATTR_ORDERED_QUEUE;
2074                 break;
2075         case ISCSI_ATTR_HEAD_OF_QUEUE:
2076                 task->task_additional_flags |= TF_ATTR_HEAD_OF_QUEUE;
2077                 break;
2078         case ISCSI_ATTR_ACA:
2079                 task->task_additional_flags |= TF_ATTR_ACA;
2080                 break;
2081         default:
2082                 /* Protocol error but just take it, treat as untagged */
2083                 break;
2084         }
2085 
2086 
2087         task->task_additional_flags = 0;
2088         task->task_priority = 0;
2089         task->task_mgmt_function = TM_NONE;
2090 
2091         /*
2092          * This "task_max_nbufs" doesn't map well to BIDI.  We probably need
2093          * parameter for each direction.  "MaxOutstandingR2T" may very well
2094          * be set to one which could prevent us from doing simultaneous
2095          * transfers in each direction.
2096          */
2097         task->task_max_nbufs = (iscsi_scsi->flags & ISCSI_FLAG_CMD_WRITE) ?
2098             ict->ict_op.op_max_outstanding_r2t : STMF_BUFS_MAX;
2099         task->task_cmd_seq_no = ntohl(iscsi_scsi->itt);
2100         task->task_expected_xfer_length = ntohl(iscsi_scsi->data_length);
2101 
2102         /* Copy CDB */
2103         bcopy(iscsi_scsi->scb, task->task_cdb, 16);
2104         if (addl_cdb_len > 0) {
2105                 bcopy(ahs_hdr->ahs_extscb, task->task_cdb + 16, addl_cdb_len);
2106         }
2107 
2108         DTRACE_ISCSI_3(scsi__command, idm_conn_t *, ic,
2109             iscsi_scsi_cmd_hdr_t *, (iscsi_scsi_cmd_hdr_t *)rx_pdu->isp_hdr,
2110             scsi_task_t *, task);
2111 
2112         /*
2113          * Copy the transport header into the task handle from the PDU
2114          * handle. The transport header describes this task's remote tagged
2115          * buffer.
2116          */
2117         if (rx_pdu->isp_transport_hdrlen != 0) {
2118                 bcopy(rx_pdu->isp_transport_hdr,
2119                     itask->it_idm_task->idt_transport_hdr,
2120                     rx_pdu->isp_transport_hdrlen);
2121         }
2122 
2123         /*
2124          * Tell IDM about our new active task
2125          */
2126         idm_task_start(itask->it_idm_task, (uintptr_t)itask->it_itt);
2127 
2128         /*
2129          * If we have any immediate data then setup the immediate buffer
2130          * context that comes with the task
2131          */
2132         if (rx_pdu->isp_datalen) {
2133                 ibuf = itask->it_immed_data;
2134                 ibuf->ibuf_immed_data_pdu = rx_pdu;
2135                 ibuf->ibuf_stmf_buf->db_data_size = rx_pdu->isp_datalen;
2136                 ibuf->ibuf_stmf_buf->db_buf_size = rx_pdu->isp_datalen;
2137                 ibuf->ibuf_stmf_buf->db_relative_offset = 0;
2138                 ibuf->ibuf_stmf_buf->db_sglist[0].seg_length =
2139                     rx_pdu->isp_datalen;
2140                 ibuf->ibuf_stmf_buf->db_sglist[0].seg_addr = rx_pdu->isp_data;
2141 
2142                 DTRACE_ISCSI_8(xfer__start, idm_conn_t *, ic,
2143                     uintptr_t, ibuf->ibuf_stmf_buf->db_sglist[0].seg_addr,
2144                     uint32_t, ibuf->ibuf_stmf_buf->db_relative_offset,
2145                     uint64_t, 0, uint32_t, 0, uint32_t, 0, /* no raddr */
2146                     uint32_t, rx_pdu->isp_datalen, int, XFER_BUF_TX_TO_INI);
2147 
2148                 /*
2149                  * For immediate data transfer, there is no callback from
2150                  * stmf to indicate that the initial burst of data is
2151                  * transferred successfully. In some cases, the task can
2152                  * get freed before execution returns from stmf_post_task.
2153                  * Although this xfer-start/done probe accurately tracks
2154                  * the size of the transfer, it does only provide a best
2155                  * effort on the timing of the transfer.
2156                  */
2157                 DTRACE_ISCSI_8(xfer__done, idm_conn_t *, ic,
2158                     uintptr_t, ibuf->ibuf_stmf_buf->db_sglist[0].seg_addr,
2159                     uint32_t, ibuf->ibuf_stmf_buf->db_relative_offset,
2160                     uint64_t, 0, uint32_t, 0, uint32_t, 0, /* no raddr */
2161                     uint32_t, rx_pdu->isp_datalen, int, XFER_BUF_TX_TO_INI);
2162                 stmf_post_task(task, ibuf->ibuf_stmf_buf);
2163         } else {
2164 
2165                 stmf_post_task(task, NULL);
2166                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2167         }
2168 }
2169 
2170 void
2171 iscsit_deferred_dispatch(idm_pdu_t *rx_pdu)
2172 {
2173         iscsit_conn_t *ict = rx_pdu->isp_ic->ic_handle;
2174 
2175         /*
2176          * If this isn't a login packet, we need a session.  Otherwise
2177          * this is a protocol error (perhaps one IDM should've caught?).
2178          */
2179         if (IDM_PDU_OPCODE(rx_pdu) != ISCSI_OP_LOGIN_CMD &&
2180             ict->ict_sess == NULL) {
2181                 DTRACE_PROBE2(iscsi__idm__deferred__no__session,
2182                     iscsit_conn_t *, ict, idm_pdu_t *, rx_pdu);
2183                 idm_pdu_complete(rx_pdu, IDM_STATUS_FAIL);
2184                 return;
2185         }
2186 
2187         /*
2188          * If the connection has been lost then ignore new PDU's
2189          */
2190         mutex_enter(&ict->ict_mutex);
2191         if (ict->ict_lost) {
2192                 mutex_exit(&ict->ict_mutex);
2193                 idm_pdu_complete(rx_pdu, IDM_STATUS_FAIL);
2194                 return;
2195         }
2196 
2197         /*
2198          * Grab a hold on the connection to prevent it from going away
2199          * between now and when the taskq function is called.
2200          */
2201         iscsit_conn_dispatch_hold(ict);
2202         mutex_exit(&ict->ict_mutex);
2203 
2204         taskq_dispatch_ent(iscsit_global.global_dispatch_taskq,
2205             iscsit_deferred, rx_pdu, 0, &rx_pdu->isp_tqent);
2206 }
2207 
2208 static void
2209 iscsit_deferred(void *rx_pdu_void)
2210 {
2211         idm_pdu_t               *rx_pdu = rx_pdu_void;
2212         idm_conn_t              *ic = rx_pdu->isp_ic;
2213         iscsit_conn_t           *ict = ic->ic_handle;
2214 
2215         /*
2216          * NOP and Task Management Commands can be marked for immediate
2217          * delivery. Commands marked as 'Immediate' are to be considered
2218          * for execution as soon as they arrive on the target. So these
2219          * should not be checked for sequence order and put in a queue.
2220          * The CmdSN is not advanced for Immediate Commands.
2221          */
2222         switch (IDM_PDU_OPCODE(rx_pdu)) {
2223         case ISCSI_OP_NOOP_OUT:
2224                 if (iscsit_check_cmdsn_and_queue(rx_pdu)) {
2225                         iscsit_set_cmdsn(ict, rx_pdu);
2226                         iscsit_pdu_op_noop(ict, rx_pdu);
2227                 }
2228                 break;
2229         case ISCSI_OP_LOGIN_CMD:
2230                 iscsit_pdu_op_login_cmd(ict, rx_pdu);
2231                 iscsit_conn_dispatch_rele(ict);
2232                 return;
2233         case ISCSI_OP_TEXT_CMD:
2234                 if (iscsit_check_cmdsn_and_queue(rx_pdu)) {
2235                         iscsit_set_cmdsn(ict, rx_pdu);
2236                         iscsit_pdu_op_text_cmd(ict, rx_pdu);
2237                 }
2238                 break;
2239         case ISCSI_OP_LOGOUT_CMD:
2240                 if (iscsit_check_cmdsn_and_queue(rx_pdu)) {
2241                         iscsit_set_cmdsn(ict, rx_pdu);
2242                         iscsit_pdu_op_logout_cmd(ict, rx_pdu);
2243                 }
2244                 break;
2245         default:
2246                 /* Protocol error.  IDM should have caught this */
2247                 idm_pdu_complete(rx_pdu, IDM_STATUS_FAIL);
2248                 ASSERT(0);
2249                 break;
2250         }
2251         /*
2252          * Check if there are other PDUs in the session staging queue
2253          * waiting to be posted to SCSI layer.
2254          */
2255         iscsit_process_pdu_in_queue(ict->ict_sess);
2256 
2257         iscsit_conn_dispatch_rele(ict);
2258 }
2259 
2260 static void
2261 iscsit_send_direct_scsi_resp(iscsit_conn_t *ict, idm_pdu_t *rx_pdu,
2262     uint8_t response, uint8_t cmd_status)
2263 {
2264         idm_pdu_t                       *rsp_pdu;
2265         idm_conn_t                      *ic;
2266         iscsi_scsi_rsp_hdr_t            *resp;
2267         iscsi_scsi_cmd_hdr_t            *req =
2268             (iscsi_scsi_cmd_hdr_t *)rx_pdu->isp_hdr;
2269 
2270         ic = ict->ict_ic;
2271 
2272         rsp_pdu = idm_pdu_alloc(sizeof (iscsi_scsi_rsp_hdr_t), 0);
2273         idm_pdu_init(rsp_pdu, ic, NULL, NULL);
2274         /*
2275          * StatSN is incremented by 1 for every response sent on
2276          * a connection except for responses sent as a result of
2277          * a retry or SNACK
2278          */
2279         rsp_pdu->isp_flags |= IDM_PDU_SET_STATSN | IDM_PDU_ADVANCE_STATSN;
2280 
2281         resp = (iscsi_scsi_rsp_hdr_t *)rsp_pdu->isp_hdr;
2282 
2283         resp->opcode = ISCSI_OP_SCSI_RSP;
2284         resp->flags = ISCSI_FLAG_FINAL;
2285         resp->response = response;
2286         resp->cmd_status = cmd_status;
2287         resp->itt = req->itt;
2288         if ((response == ISCSI_STATUS_CMD_COMPLETED) &&
2289             (req->data_length != 0) &&
2290             ((req->flags & ISCSI_FLAG_CMD_READ) ||
2291             (req->flags & ISCSI_FLAG_CMD_WRITE))) {
2292                 resp->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
2293                 resp->residual_count = req->data_length;
2294         }
2295 
2296         DTRACE_PROBE4(iscsi__scsi__direct__response,
2297             iscsit_conn_t *, ict,
2298             uint8_t, resp->response,
2299             uint8_t, resp->cmd_status,
2300             idm_pdu_t *, rsp_pdu);
2301 
2302         iscsit_pdu_tx(rsp_pdu);
2303 }
2304 
2305 void
2306 iscsit_send_task_mgmt_resp(idm_pdu_t *tm_resp_pdu, uint8_t tm_status)
2307 {
2308         iscsi_scsi_task_mgt_rsp_hdr_t   *tm_resp;
2309 
2310         /*
2311          * The target must take note of the last-sent StatSN.
2312          * The StatSN is to be incremented after sending a
2313          * task management response. Digest recovery can only
2314          * work if StatSN is incremented.
2315          */
2316         tm_resp_pdu->isp_flags |= IDM_PDU_SET_STATSN | IDM_PDU_ADVANCE_STATSN;
2317         tm_resp = (iscsi_scsi_task_mgt_rsp_hdr_t *)tm_resp_pdu->isp_hdr;
2318         tm_resp->response = tm_status;
2319 
2320         DTRACE_PROBE3(iscsi__scsi__tm__response,
2321             iscsit_conn_t *, tm_resp_pdu->isp_ic->ic_handle,
2322             uint8_t, tm_resp->response,
2323             idm_pdu_t *, tm_resp_pdu);
2324         iscsit_pdu_tx(tm_resp_pdu);
2325 }
2326 
2327 void
2328 iscsit_op_scsi_task_mgmt(iscsit_conn_t *ict, idm_pdu_t *rx_pdu)
2329 {
2330         idm_pdu_t                       *tm_resp_pdu;
2331         iscsit_task_t                   *itask;
2332         iscsit_task_t                   *tm_itask;
2333         scsi_task_t                     *task;
2334         iscsi_scsi_task_mgt_hdr_t       *iscsi_tm =
2335             (iscsi_scsi_task_mgt_hdr_t *)rx_pdu->isp_hdr;
2336         iscsi_scsi_task_mgt_rsp_hdr_t   *iscsi_tm_rsp =
2337             (iscsi_scsi_task_mgt_rsp_hdr_t *)rx_pdu->isp_hdr;
2338         uint32_t                        rtt, cmdsn, refcmdsn;
2339         uint8_t                         tm_func;
2340 
2341         /*
2342          * Setup response PDU (response field will get filled in later)
2343          */
2344         tm_resp_pdu = idm_pdu_alloc(sizeof (iscsi_scsi_task_mgt_rsp_hdr_t), 0);
2345         if (tm_resp_pdu == NULL) {
2346                 /* Can't respond, just drop it */
2347                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2348                 return;
2349         }
2350         idm_pdu_init(tm_resp_pdu, ict->ict_ic, NULL, NULL);
2351         iscsi_tm_rsp = (iscsi_scsi_task_mgt_rsp_hdr_t *)tm_resp_pdu->isp_hdr;
2352         bzero(iscsi_tm_rsp, sizeof (iscsi_scsi_task_mgt_rsp_hdr_t));
2353         iscsi_tm_rsp->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
2354         iscsi_tm_rsp->flags = ISCSI_FLAG_FINAL;
2355         iscsi_tm_rsp->itt = rx_pdu->isp_hdr->itt;
2356 
2357         /*
2358          * Figure out what we're being asked to do.
2359          */
2360         DTRACE_PROBE4(iscsi__scsi__tm__request,
2361             iscsit_conn_t *, ict,
2362             uint8_t, (iscsi_tm->function & ISCSI_FLAG_TASK_MGMT_FUNCTION_MASK),
2363             uint32_t, iscsi_tm->rtt,
2364             idm_pdu_t *, rx_pdu);
2365         switch (iscsi_tm->function & ISCSI_FLAG_TASK_MGMT_FUNCTION_MASK) {
2366         case ISCSI_TM_FUNC_ABORT_TASK:
2367                 /*
2368                  * STMF doesn't currently support the "abort task" task
2369                  * management command although it does support aborting
2370                  * an individual task.  We'll get STMF to abort the task
2371                  * for us but handle the details of the task management
2372                  * command ourselves.
2373                  *
2374                  * Find the task associated with the referenced task tag.
2375                  */
2376                 rtt = iscsi_tm->rtt;
2377                 itask = (iscsit_task_t *)idm_task_find_by_handle(ict->ict_ic,
2378                     (uintptr_t)rtt);
2379 
2380                 if (itask == NULL) {
2381                         cmdsn = ntohl(iscsi_tm->cmdsn);
2382                         refcmdsn = ntohl(iscsi_tm->refcmdsn);
2383 
2384                         /*
2385                          * Task was not found. But the SCSI command could be
2386                          * on the rxpdu wait queue. If RefCmdSN is within
2387                          * the CmdSN window and less than CmdSN of the TM
2388                          * function, return "Function Complete". Otherwise,
2389                          * return "Task Does Not Exist".
2390                          */
2391 
2392                         if (iscsit_cmdsn_in_window(ict, refcmdsn) &&
2393                             iscsit_sna_lt(refcmdsn, cmdsn)) {
2394                                 mutex_enter(&ict->ict_sess->ist_sn_mutex);
2395                                 (void) iscsit_remove_pdu_from_queue(
2396                                     ict->ict_sess, refcmdsn);
2397                                 iscsit_conn_dispatch_rele(ict);
2398                                 mutex_exit(&ict->ict_sess->ist_sn_mutex);
2399                                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2400                                     SCSI_TCP_TM_RESP_COMPLETE);
2401                         } else {
2402                                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2403                                     SCSI_TCP_TM_RESP_NO_TASK);
2404                         }
2405                 } else {
2406 
2407                         /*
2408                          * Tell STMF to abort the task.  This will do no harm
2409                          * if the task is already complete.
2410                          */
2411                         stmf_abort(STMF_QUEUE_TASK_ABORT, itask->it_stmf_task,
2412                             STMF_ABORTED, NULL);
2413 
2414                         /*
2415                          * Make sure the task hasn't already completed
2416                          */
2417                         mutex_enter(&itask->it_idm_task->idt_mutex);
2418                         if ((itask->it_idm_task->idt_state == TASK_COMPLETE) ||
2419                             (itask->it_idm_task->idt_state == TASK_IDLE)) {
2420                                 /*
2421                                  * Task is complete, return "Task Does Not
2422                                  * Exist"
2423                                  */
2424                                 mutex_exit(&itask->it_idm_task->idt_mutex);
2425                                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2426                                     SCSI_TCP_TM_RESP_NO_TASK);
2427                         } else {
2428                                 /*
2429                                  * STMF is now aborting the task, return
2430                                  * "Function Complete"
2431                                  */
2432                                 mutex_exit(&itask->it_idm_task->idt_mutex);
2433                                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2434                                     SCSI_TCP_TM_RESP_COMPLETE);
2435                         }
2436                         idm_task_rele(itask->it_idm_task);
2437                 }
2438                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2439                 return;
2440 
2441         case ISCSI_TM_FUNC_ABORT_TASK_SET:
2442                 tm_func = TM_ABORT_TASK_SET;
2443                 break;
2444 
2445         case ISCSI_TM_FUNC_CLEAR_ACA:
2446                 tm_func = TM_CLEAR_ACA;
2447                 break;
2448 
2449         case ISCSI_TM_FUNC_CLEAR_TASK_SET:
2450                 tm_func = TM_CLEAR_TASK_SET;
2451                 break;
2452 
2453         case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
2454                 tm_func = TM_LUN_RESET;
2455                 break;
2456 
2457         case ISCSI_TM_FUNC_TARGET_WARM_RESET:
2458                 tm_func = TM_TARGET_WARM_RESET;
2459                 break;
2460 
2461         case ISCSI_TM_FUNC_TARGET_COLD_RESET:
2462                 tm_func = TM_TARGET_COLD_RESET;
2463                 break;
2464 
2465         case ISCSI_TM_FUNC_TASK_REASSIGN:
2466                 /*
2467                  * We do not currently support allegiance reassignment.  When
2468                  * we start supporting ERL1+, we will need to.
2469                  */
2470                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2471                     SCSI_TCP_TM_RESP_NO_ALLG_REASSN);
2472                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2473                 return;
2474 
2475         default:
2476                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2477                     SCSI_TCP_TM_RESP_REJECTED);
2478                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2479                 return;
2480         }
2481 
2482         tm_itask = iscsit_tm_task_alloc(ict);
2483         if (tm_itask == NULL) {
2484                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2485                     SCSI_TCP_TM_RESP_REJECTED);
2486                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2487                 return;
2488         }
2489 
2490 
2491         task = stmf_task_alloc(ict->ict_sess->ist_lport,
2492             ict->ict_sess->ist_stmf_sess, iscsi_tm->lun,
2493             0, STMF_TASK_EXT_NONE);
2494         if (task == NULL) {
2495                 /*
2496                  * If this happens, either the LU is in reset, couldn't
2497                  * get memory, or some other condition in which we simply
2498                  * can't complete this request.  It would be nice to return
2499                  * an error code like "busy" but the closest we have is
2500                  * "rejected".
2501                  */
2502                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2503                     SCSI_TCP_TM_RESP_REJECTED);
2504                 iscsit_tm_task_free(tm_itask);
2505                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2506                 return;
2507         }
2508 
2509         tm_itask->it_tm_pdu = tm_resp_pdu;
2510         tm_itask->it_stmf_task = task;
2511         task->task_port_private = tm_itask;
2512         task->task_mgmt_function = tm_func;
2513         task->task_additional_flags = TASK_AF_NO_EXPECTED_XFER_LENGTH;
2514         task->task_priority = 0;
2515         task->task_max_nbufs = STMF_BUFS_MAX;
2516         task->task_cmd_seq_no = iscsi_tm->itt;
2517         task->task_expected_xfer_length = 0;
2518 
2519         stmf_post_task(task, NULL);
2520         idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2521 }
2522 
2523 static void
2524 iscsit_pdu_op_noop(iscsit_conn_t *ict, idm_pdu_t *rx_pdu)
2525 {
2526         iscsi_nop_out_hdr_t *out = (iscsi_nop_out_hdr_t *)rx_pdu->isp_hdr;
2527         iscsi_nop_in_hdr_t *in;
2528         int resp_datalen;
2529         idm_pdu_t *resp;
2530 
2531         /* Ignore the response from initiator */
2532         if ((out->itt == ISCSI_RSVD_TASK_TAG) ||
2533             (out->ttt != ISCSI_RSVD_TASK_TAG)) {
2534                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2535                 return;
2536         }
2537 
2538         /* Allocate a PDU to respond */
2539         resp_datalen = ntoh24(out->dlength);
2540         resp = idm_pdu_alloc(sizeof (iscsi_hdr_t), resp_datalen);
2541         idm_pdu_init(resp, ict->ict_ic, NULL, NULL);
2542         if (resp_datalen > 0) {
2543                 bcopy(rx_pdu->isp_data, resp->isp_data, resp_datalen);
2544         }
2545 
2546         /*
2547          * When sending a NOP-In as a response to a NOP-Out from the initiator,
2548          * the target must respond with the same initiator task tag that was
2549          * provided in the NOP-Out request, the target transfer tag must be
2550          * ISCSI_RSVD_TASK_TAG (0xffffffff) and StatSN will contain the next
2551          * status sequence number. The StatSN for the connection is advanced
2552          * after this PDU is sent.
2553          */
2554         in = (iscsi_nop_in_hdr_t *)resp->isp_hdr;
2555         bzero(in, sizeof (*in));
2556         in->opcode = ISCSI_OP_NOOP_IN;
2557         in->flags = ISCSI_FLAG_FINAL;
2558         bcopy(out->lun, in->lun, 8);
2559         in->itt              = out->itt;
2560         in->ttt              = ISCSI_RSVD_TASK_TAG;
2561         hton24(in->dlength, resp_datalen);
2562         resp->isp_flags |= IDM_PDU_SET_STATSN | IDM_PDU_ADVANCE_STATSN;
2563         /* Any other field in resp to be set? */
2564         iscsit_pdu_tx(resp);
2565         idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2566 }
2567 
2568 static void
2569 iscsit_pdu_op_login_cmd(iscsit_conn_t   *ict, idm_pdu_t *rx_pdu)
2570 {
2571 
2572         /*
2573          * Submit PDU to login state machine.  State machine will free the
2574          * PDU.
2575          */
2576         iscsit_login_sm_event(ict, ILE_LOGIN_RCV, rx_pdu);
2577 }
2578 
2579 void
2580 iscsit_pdu_op_logout_cmd(iscsit_conn_t  *ict, idm_pdu_t *rx_pdu)
2581 {
2582         iscsi_logout_hdr_t      *logout_req =
2583             (iscsi_logout_hdr_t *)rx_pdu->isp_hdr;
2584         iscsi_logout_rsp_hdr_t  *logout_rsp;
2585         idm_pdu_t *resp;
2586 
2587         /* Allocate a PDU to respond */
2588         resp = idm_pdu_alloc(sizeof (iscsi_hdr_t), 0);
2589         idm_pdu_init(resp, ict->ict_ic, NULL, NULL);
2590         /*
2591          * The StatSN is to be sent to the initiator,
2592          * it is not required to increment the number
2593          * as the connection is terminating.
2594          */
2595         resp->isp_flags |= IDM_PDU_SET_STATSN;
2596         /*
2597          * Logout results in the immediate termination of all tasks except
2598          * if the logout reason is ISCSI_LOGOUT_REASON_RECOVERY.  The
2599          * connection state machine will drive this task cleanup automatically
2600          * so we don't need to handle that here.
2601          */
2602         logout_rsp = (iscsi_logout_rsp_hdr_t *)resp->isp_hdr;
2603         bzero(logout_rsp, sizeof (*logout_rsp));
2604         logout_rsp->opcode = ISCSI_OP_LOGOUT_RSP;
2605         logout_rsp->flags = ISCSI_FLAG_FINAL;
2606         logout_rsp->itt = logout_req->itt;
2607         if ((logout_req->flags & ISCSI_FLAG_LOGOUT_REASON_MASK) >
2608             ISCSI_LOGOUT_REASON_RECOVERY) {
2609                 logout_rsp->response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED;
2610         } else {
2611                 logout_rsp->response = ISCSI_LOGOUT_SUCCESS;
2612         }
2613 
2614         iscsit_pdu_tx(resp);
2615         idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2616 }
2617 
2618 /*
2619  * Calculate the number of outstanding commands we can process
2620  */
2621 int
2622 iscsit_cmd_window()
2623 {
2624         /*
2625          * Instead of using a pre-defined constant for the command window,
2626          * it should be made confiurable and dynamic. With MC/S, sequence
2627          * numbers will be used up at a much faster rate than with SC/S.
2628          */
2629         return  (ISCSIT_MAX_WINDOW);
2630 }
2631 
2632 /*
2633  * Set local registers based on incoming PDU
2634  */
2635 void
2636 iscsit_set_cmdsn(iscsit_conn_t *ict, idm_pdu_t *rx_pdu)
2637 {
2638         iscsit_sess_t *ist;
2639         iscsi_scsi_cmd_hdr_t *req;
2640 
2641         ist = ict->ict_sess;
2642 
2643         req = (iscsi_scsi_cmd_hdr_t *)rx_pdu->isp_hdr;
2644         if (req->opcode & ISCSI_OP_IMMEDIATE) {
2645                 /* no cmdsn increment for immediate PDUs */
2646                 return;
2647         }
2648 
2649         /* Ensure that the ExpCmdSN advances in an orderly manner */
2650         mutex_enter(&ist->ist_sn_mutex);
2651         ist->ist_expcmdsn = ntohl(req->cmdsn) + 1;
2652         ist->ist_maxcmdsn = ntohl(req->cmdsn) + iscsit_cmd_window();
2653         mutex_exit(&ist->ist_sn_mutex);
2654 }
2655 
2656 /*
2657  * Wrapper funtion, calls iscsi_calc_rspsn and idm_pdu_tx
2658  */
2659 void
2660 iscsit_pdu_tx(idm_pdu_t *pdu)
2661 {
2662         iscsit_conn_t *ict = pdu->isp_ic->ic_handle;
2663         iscsi_scsi_rsp_hdr_t *rsp = (iscsi_scsi_rsp_hdr_t *)pdu->isp_hdr;
2664         iscsit_sess_t *ist = ict->ict_sess;
2665 
2666         /*
2667          * The command sequence numbers are session-wide and must stay
2668          * consistent across the transfer, so protect the cmdsn with a
2669          * mutex lock on the session. The status sequence number will
2670          * be updated just before the transport layer transmits the PDU.
2671          */
2672 
2673         mutex_enter(&ict->ict_sess->ist_sn_mutex);
2674         /* Set ExpCmdSN and MaxCmdSN */
2675         rsp->maxcmdsn = htonl(ist->ist_maxcmdsn);
2676         rsp->expcmdsn = htonl(ist->ist_expcmdsn);
2677         idm_pdu_tx(pdu);
2678         mutex_exit(&ict->ict_sess->ist_sn_mutex);
2679 }
2680 
2681 /*
2682  * Internal functions
2683  */
2684 
2685 void
2686 iscsit_send_async_event(iscsit_conn_t *ict, uint8_t event)
2687 {
2688         idm_pdu_t               *abt;
2689         iscsi_async_evt_hdr_t   *async_abt;
2690 
2691         /*
2692          * Get a PDU to build the abort request.
2693          */
2694         abt = idm_pdu_alloc(sizeof (iscsi_hdr_t), 0);
2695         if (abt == NULL) {
2696                 idm_conn_event(ict->ict_ic, CE_TRANSPORT_FAIL, NULL);
2697                 return;
2698         }
2699 
2700         /*
2701          * A asynchronous message is sent by the target to request a logout.
2702          * The StatSN for the connection is advanced after the PDU is sent
2703          * to allow for initiator and target state synchronization.
2704          */
2705         idm_pdu_init(abt, ict->ict_ic, NULL, NULL);
2706         abt->isp_datalen = 0;
2707         abt->isp_flags |= IDM_PDU_SET_STATSN | IDM_PDU_ADVANCE_STATSN;
2708 
2709         async_abt = (iscsi_async_evt_hdr_t *)abt->isp_hdr;
2710         bzero(async_abt, sizeof (*async_abt));
2711         async_abt->opcode = ISCSI_OP_ASYNC_EVENT;
2712         async_abt->async_event = event;
2713         async_abt->flags = ISCSI_FLAG_FINAL;
2714         async_abt->rsvd4[0] = 0xff;
2715         async_abt->rsvd4[1] = 0xff;
2716         async_abt->rsvd4[2] = 0xff;
2717         async_abt->rsvd4[3] = 0xff;
2718 
2719         switch (event) {
2720         case ISCSI_ASYNC_EVENT_REQUEST_LOGOUT:
2721                 async_abt->param3 = htons(IDM_LOGOUT_SECONDS);
2722                 break;
2723         case ISCSI_ASYNC_EVENT_SCSI_EVENT:
2724         case ISCSI_ASYNC_EVENT_DROPPING_CONNECTION:
2725         case ISCSI_ASYNC_EVENT_DROPPING_ALL_CONNECTIONS:
2726         case ISCSI_ASYNC_EVENT_PARAM_NEGOTIATION:
2727         default:
2728                 ASSERT(0);
2729         }
2730 
2731         iscsit_pdu_tx(abt);
2732 }
2733 
2734 void
2735 iscsit_send_reject(iscsit_conn_t *ict, idm_pdu_t *rejected_pdu, uint8_t reason)
2736 {
2737         idm_pdu_t               *reject_pdu;
2738         iscsi_reject_rsp_hdr_t  *reject;
2739 
2740         /*
2741          * Get a PDU to build the abort request.
2742          */
2743         reject_pdu = idm_pdu_alloc(sizeof (iscsi_hdr_t),
2744             rejected_pdu->isp_hdrlen);
2745         if (reject_pdu == NULL) {
2746                 idm_conn_event(ict->ict_ic, CE_TRANSPORT_FAIL, NULL);
2747                 return;
2748         }
2749         idm_pdu_init(reject_pdu, ict->ict_ic, NULL, NULL);
2750         /* StatSN is advanced after a Reject PDU */
2751         reject_pdu->isp_flags |= IDM_PDU_SET_STATSN | IDM_PDU_ADVANCE_STATSN;
2752         reject_pdu->isp_datalen = rejected_pdu->isp_hdrlen;
2753         bcopy(rejected_pdu->isp_hdr, reject_pdu->isp_data,
2754             rejected_pdu->isp_hdrlen);
2755 
2756         reject = (iscsi_reject_rsp_hdr_t *)reject_pdu->isp_hdr;
2757         bzero(reject, sizeof (*reject));
2758         reject->opcode = ISCSI_OP_REJECT_MSG;
2759         reject->reason = reason;
2760         reject->flags = ISCSI_FLAG_FINAL;
2761         hton24(reject->dlength, rejected_pdu->isp_hdrlen);
2762         reject->must_be_ff[0] = 0xff;
2763         reject->must_be_ff[1] = 0xff;
2764         reject->must_be_ff[2] = 0xff;
2765         reject->must_be_ff[3] = 0xff;
2766 
2767         iscsit_pdu_tx(reject_pdu);
2768 }
2769 
2770 
2771 static iscsit_task_t *
2772 iscsit_task_alloc(iscsit_conn_t *ict)
2773 {
2774         iscsit_task_t *itask;
2775         iscsit_buf_t *immed_ibuf;
2776 
2777         /*
2778          * Possible items to pre-alloc if we cache iscsit_task_t's:
2779          *
2780          * Status PDU w/ sense buffer
2781          * stmf_data_buf_t for immediate data
2782          */
2783         itask = kmem_alloc(sizeof (iscsit_task_t) + sizeof (iscsit_buf_t) +
2784             sizeof (stmf_data_buf_t), KM_NOSLEEP);
2785         if (itask != NULL) {
2786                 mutex_init(&itask->it_mutex, NULL, MUTEX_DRIVER, NULL);
2787                 itask->it_aborted = itask->it_stmf_abort =
2788                     itask->it_tm_task = 0;
2789 
2790                 immed_ibuf = (iscsit_buf_t *)(itask + 1);
2791                 bzero(immed_ibuf, sizeof (*immed_ibuf));
2792                 immed_ibuf->ibuf_is_immed = B_TRUE;
2793                 immed_ibuf->ibuf_stmf_buf = (stmf_data_buf_t *)(immed_ibuf + 1);
2794 
2795                 bzero(immed_ibuf->ibuf_stmf_buf, sizeof (stmf_data_buf_t));
2796                 immed_ibuf->ibuf_stmf_buf->db_port_private = immed_ibuf;
2797                 immed_ibuf->ibuf_stmf_buf->db_sglist_length = 1;
2798                 immed_ibuf->ibuf_stmf_buf->db_flags = DB_DIRECTION_FROM_RPORT |
2799                     DB_DONT_CACHE;
2800                 itask->it_immed_data = immed_ibuf;
2801                 itask->it_idm_task = idm_task_alloc(ict->ict_ic);
2802                 if (itask->it_idm_task != NULL) {
2803                         itask->it_idm_task->idt_private = itask;
2804                         itask->it_ict = ict;
2805                         itask->it_ttt = itask->it_idm_task->idt_tt;
2806                         return (itask);
2807                 } else {
2808                         kmem_free(itask, sizeof (iscsit_task_t) +
2809                             sizeof (iscsit_buf_t) + sizeof (stmf_data_buf_t));
2810                 }
2811         }
2812 
2813         return (NULL);
2814 }
2815 
2816 static void
2817 iscsit_task_free(iscsit_task_t *itask)
2818 {
2819         idm_task_free(itask->it_idm_task);
2820         mutex_destroy(&itask->it_mutex);
2821         kmem_free(itask, sizeof (iscsit_task_t) +
2822             sizeof (iscsit_buf_t) + sizeof (stmf_data_buf_t));
2823 }
2824 
2825 static iscsit_task_t *
2826 iscsit_tm_task_alloc(iscsit_conn_t *ict)
2827 {
2828         iscsit_task_t *itask;
2829 
2830         itask = kmem_zalloc(sizeof (iscsit_task_t), KM_NOSLEEP);
2831         if (itask != NULL) {
2832                 idm_conn_hold(ict->ict_ic);
2833                 mutex_init(&itask->it_mutex, NULL, MUTEX_DRIVER, NULL);
2834                 itask->it_aborted = itask->it_stmf_abort =
2835                     itask->it_tm_responded = 0;
2836                 itask->it_tm_pdu = NULL;
2837                 itask->it_tm_task = 1;
2838                 itask->it_ict = ict;
2839         }
2840 
2841         return (itask);
2842 }
2843 
2844 static void
2845 iscsit_tm_task_free(iscsit_task_t *itask)
2846 {
2847         /*
2848          * If we responded then the call to idm_pdu_complete will free the
2849          * PDU.  Otherwise we got aborted before the TM function could
2850          * complete and we need to free the PDU explicitly.
2851          */
2852         if (itask->it_tm_pdu != NULL && !itask->it_tm_responded)
2853                 idm_pdu_free(itask->it_tm_pdu);
2854         idm_conn_rele(itask->it_ict->ict_ic);
2855         mutex_destroy(&itask->it_mutex);
2856         kmem_free(itask, sizeof (iscsit_task_t));
2857 }
2858 
2859 static idm_status_t
2860 iscsit_task_start(iscsit_task_t *itask)
2861 {
2862         iscsit_sess_t *ist = itask->it_ict->ict_sess;
2863         avl_index_t             where;
2864 
2865         /*
2866          * Sanity check the ITT and ensure that this task does not already
2867          * exist.  If not then add the task to the session task list.
2868          */
2869         mutex_enter(&ist->ist_mutex);
2870         mutex_enter(&itask->it_mutex);
2871         itask->it_active = 1;
2872         if (avl_find(&ist->ist_task_list, itask, &where) == NULL) {
2873                 /* New task, add to AVL */
2874                 avl_insert(&ist->ist_task_list, itask, where);
2875                 mutex_exit(&itask->it_mutex);
2876                 mutex_exit(&ist->ist_mutex);
2877                 return (IDM_STATUS_SUCCESS);
2878         }
2879         mutex_exit(&itask->it_mutex);
2880         mutex_exit(&ist->ist_mutex);
2881 
2882         return (IDM_STATUS_REJECT);
2883 }
2884 
2885 static void
2886 iscsit_task_done(iscsit_task_t *itask)
2887 {
2888         iscsit_sess_t *ist = itask->it_ict->ict_sess;
2889 
2890         mutex_enter(&ist->ist_mutex);
2891         mutex_enter(&itask->it_mutex);
2892         if (itask->it_active) {
2893                 avl_remove(&ist->ist_task_list, itask);
2894                 itask->it_active = 0;
2895         }
2896         mutex_exit(&itask->it_mutex);
2897         mutex_exit(&ist->ist_mutex);
2898 }
2899 
2900 /*
2901  * iscsit status PDU cache
2902  */
2903 
2904 /*ARGSUSED*/
2905 static int
2906 iscsit_status_pdu_constructor(void *pdu_void, void *arg, int flags)
2907 {
2908         idm_pdu_t *pdu = pdu_void;
2909         iscsi_scsi_rsp_hdr_t *rsp;
2910 
2911         bzero(pdu, sizeof (idm_pdu_t));
2912         pdu->isp_callback = iscsit_send_good_status_done;
2913         pdu->isp_magic = IDM_PDU_MAGIC;
2914         pdu->isp_hdr = (iscsi_hdr_t *)(pdu + 1); /* Ptr arithmetic */
2915         pdu->isp_hdrlen = sizeof (iscsi_hdr_t);
2916 
2917         /* Setup status response */
2918         rsp = (iscsi_scsi_rsp_hdr_t *)pdu->isp_hdr;
2919         bzero(rsp, sizeof (*rsp));
2920         rsp->opcode = ISCSI_OP_SCSI_RSP;
2921         rsp->flags = ISCSI_FLAG_FINAL;
2922         rsp->response = ISCSI_STATUS_CMD_COMPLETED;
2923 
2924         return (0);
2925 }
2926 
2927 /*
2928  * iscsit private data handler
2929  */
2930 
2931 /*ARGSUSED*/
2932 static void
2933 iscsit_pp_cb(struct stmf_port_provider *pp, int cmd, void *arg, uint32_t flags)
2934 {
2935         it_config_t             *cfg;
2936         nvlist_t                *nvl;
2937         iscsit_service_enabled_t        old_state;
2938 
2939         if ((cmd != STMF_PROVIDER_DATA_UPDATED) || (arg == NULL)) {
2940                 return;
2941         }
2942 
2943         nvl = (nvlist_t *)arg;
2944 
2945         /* Translate nvlist */
2946         if (it_nv_to_config(nvl, &cfg) != 0) {
2947                 cmn_err(CE_WARN, "Configuration is invalid");
2948                 return;
2949         }
2950 
2951         /* Check that no iSCSI ioctl is currently running */
2952         mutex_enter(&iscsit_global.global_state_mutex);
2953         old_state = iscsit_global.global_svc_state;
2954         switch (iscsit_global.global_svc_state) {
2955         case ISE_ENABLED:
2956         case ISE_DISABLED:
2957                 iscsit_global.global_svc_state = ISE_BUSY;
2958                 break;
2959         case ISE_ENABLING:
2960                 /*
2961                  * It is OK for the iscsit_pp_cb to be called from inside of
2962                  * an iSCSI ioctl only if we are currently executing inside
2963                  * of stmf_register_port_provider.
2964                  */
2965                 ASSERT((flags & STMF_PCB_PREG_COMPLETE) != 0);
2966                 break;
2967         default:
2968                 cmn_err(CE_WARN, "iscsit_pp_cb called when global_svc_state"
2969                     " is not ENABLED(0x%x) -- ignoring",
2970                     iscsit_global.global_svc_state);
2971                 mutex_exit(&iscsit_global.global_state_mutex);
2972                 it_config_free_cmn(cfg);
2973                 return;
2974         }
2975         mutex_exit(&iscsit_global.global_state_mutex);
2976 
2977         /* Update config */
2978         (void) iscsit_config_merge(cfg);
2979 
2980         it_config_free_cmn(cfg);
2981 
2982         /* Restore old iSCSI driver global state */
2983         mutex_enter(&iscsit_global.global_state_mutex);
2984         ASSERT(iscsit_global.global_svc_state == ISE_BUSY ||
2985             iscsit_global.global_svc_state == ISE_ENABLING);
2986         iscsit_global.global_svc_state = old_state;
2987         mutex_exit(&iscsit_global.global_state_mutex);
2988 }
2989 
2990 
2991 static it_cfg_status_t
2992 iscsit_config_merge(it_config_t *in_cfg)
2993 {
2994         it_cfg_status_t status;
2995         it_config_t     *cfg;
2996         it_config_t     tmp_cfg;
2997         list_t          tpg_del_list;
2998 
2999         if (in_cfg) {
3000                 cfg = in_cfg;
3001         } else {
3002                 /* Make empty config */
3003                 bzero(&tmp_cfg, sizeof (tmp_cfg));
3004                 cfg = &tmp_cfg;
3005         }
3006 
3007         list_create(&tpg_del_list,  sizeof (iscsit_tpg_t),
3008             offsetof(iscsit_tpg_t, tpg_delete_ln));
3009 
3010         /*
3011          * Update targets, initiator contexts, target portal groups,
3012          * and iSNS client
3013          */
3014         ISCSIT_GLOBAL_LOCK(RW_WRITER);
3015         if (((status = iscsit_config_merge_tpg(cfg, &tpg_del_list))
3016             != 0) ||
3017             ((status = iscsit_config_merge_tgt(cfg)) != 0) ||
3018             ((status = iscsit_config_merge_ini(cfg)) != 0) ||
3019             ((status = isnst_config_merge(cfg)) != 0)) {
3020                 ISCSIT_GLOBAL_UNLOCK();
3021                 return (status);
3022         }
3023 
3024         /* Update other global config parameters */
3025         if (iscsit_global.global_props) {
3026                 nvlist_free(iscsit_global.global_props);
3027                 iscsit_global.global_props = NULL;
3028         }
3029         if (in_cfg) {
3030                 (void) nvlist_dup(cfg->config_global_properties,
3031                     &iscsit_global.global_props, KM_SLEEP);
3032         }
3033         ISCSIT_GLOBAL_UNLOCK();
3034 
3035         iscsit_config_destroy_tpgs(&tpg_del_list);
3036 
3037         list_destroy(&tpg_del_list);
3038 
3039         return (ITCFG_SUCCESS);
3040 }
3041 
3042 /*
3043  * iscsit_sna_lt[e]
3044  *
3045  * Compare serial numbers using serial number arithmetic as defined in
3046  * RFC 1982.
3047  *
3048  * NOTE: This code is duplicated in the isns server. It ought to be common.
3049  */
3050 
3051 static int
3052 iscsit_sna_lt(uint32_t sn1, uint32_t sn2)
3053 {
3054         return ((sn1 != sn2) &&
3055             (((sn1 < sn2) && ((sn2 - sn1) < ISCSIT_SNA32_CHECK)) ||
3056             ((sn1 > sn2) && ((sn1 - sn2) > ISCSIT_SNA32_CHECK))));
3057 }
3058 
3059 static int
3060 iscsit_sna_lte(uint32_t sn1, uint32_t sn2)
3061 {
3062         return ((sn1 == sn2) ||
3063             (((sn1 < sn2) && ((sn2 - sn1) < ISCSIT_SNA32_CHECK)) ||
3064             ((sn1 > sn2) && ((sn1 - sn2) > ISCSIT_SNA32_CHECK))));
3065 }
3066 
3067 
3068 static boolean_t
3069 iscsit_cmdsn_in_window(iscsit_conn_t *ict, uint32_t cmdsn)
3070 {
3071         iscsit_sess_t   *ist = ict->ict_sess;
3072         int             rval = B_TRUE;
3073 
3074         ist = ict->ict_sess;
3075 
3076         mutex_enter(&ist->ist_sn_mutex);
3077 
3078         /*
3079          * If cmdsn is less than ist_expcmdsn - iscsit_cmd_window() or
3080          * greater than ist_expcmdsn, it's not in the window.
3081          */
3082 
3083         if (iscsit_sna_lt(cmdsn, (ist->ist_expcmdsn - iscsit_cmd_window())) ||
3084             !iscsit_sna_lte(cmdsn, ist->ist_expcmdsn)) {
3085                 rval = B_FALSE;
3086         }
3087 
3088         mutex_exit(&ist->ist_sn_mutex);
3089 
3090         return (rval);
3091 }
3092 
3093 /*
3094  * iscsit_check_cmdsn_and_queue
3095  *
3096  * Independent of the order in which the iSCSI target receives non-immediate
3097  * command PDU across the entire session and any multiple connections within
3098  * the session, the target must deliver the commands to the SCSI layer in
3099  * CmdSN order. So out-of-order non-immediate commands are queued up on a
3100  * session-wide wait queue. Duplicate commands are ignored.
3101  *
3102  */
3103 static int
3104 iscsit_check_cmdsn_and_queue(idm_pdu_t *rx_pdu)
3105 {
3106         idm_conn_t              *ic = rx_pdu->isp_ic;
3107         iscsit_conn_t           *ict = ic->ic_handle;
3108         iscsit_sess_t           *ist = ict->ict_sess;
3109         iscsi_scsi_cmd_hdr_t    *hdr = (iscsi_scsi_cmd_hdr_t *)rx_pdu->isp_hdr;
3110 
3111         mutex_enter(&ist->ist_sn_mutex);
3112         if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
3113                 /* do not queue, handle it immediately */
3114                 DTRACE_PROBE2(immediate__cmd, iscsit_sess_t *, ist,
3115                     idm_pdu_t *, rx_pdu);
3116                 mutex_exit(&ist->ist_sn_mutex);
3117                 return (ISCSIT_CMDSN_EQ_EXPCMDSN);
3118         }
3119         if (iscsit_sna_lt(ist->ist_expcmdsn, ntohl(hdr->cmdsn))) {
3120                 /*
3121                  * Out-of-order commands (cmdSN higher than ExpCmdSN)
3122                  * are staged on a fixed-size circular buffer until
3123                  * the missing command is delivered to the SCSI layer.
3124                  * Irrespective of the order of insertion into the
3125                  * staging queue, the commands are processed out of the
3126                  * queue in cmdSN order only.
3127                  */
3128                 rx_pdu->isp_queue_time = gethrtime();
3129                 iscsit_add_pdu_to_queue(ist, rx_pdu);
3130                 mutex_exit(&ist->ist_sn_mutex);
3131                 return (ISCSIT_CMDSN_GT_EXPCMDSN);
3132         } else if (iscsit_sna_lt(ntohl(hdr->cmdsn), ist->ist_expcmdsn)) {
3133                 DTRACE_PROBE3(cmdsn__lt__expcmdsn, iscsit_sess_t *, ist,
3134                     iscsit_conn_t *, ict, idm_pdu_t *, rx_pdu);
3135                 mutex_exit(&ist->ist_sn_mutex);
3136                 return (ISCSIT_CMDSN_LT_EXPCMDSN);
3137         } else {
3138                 mutex_exit(&ist->ist_sn_mutex);
3139                 return (ISCSIT_CMDSN_EQ_EXPCMDSN);
3140         }
3141 }
3142 
3143 /*
3144  * iscsit_add_pdu_to_queue() adds PDUs into the array indexed by
3145  * their cmdsn value. The length of the array is kept above the
3146  * maximum window size. The window keeps the cmdsn within a range
3147  * such that there are no collisons. e.g. the assumption is that
3148  * the windowing checks make it impossible to receive PDUs that
3149  * index into the same location in the array.
3150  */
3151 static void
3152 iscsit_add_pdu_to_queue(iscsit_sess_t *ist, idm_pdu_t *rx_pdu)
3153 {
3154         iscsit_cbuf_t   *cbuf   = ist->ist_rxpdu_queue;
3155         iscsit_conn_t   *ict    = rx_pdu->isp_ic->ic_handle;
3156         uint32_t        cmdsn   =
3157             ((iscsi_scsi_cmd_hdr_t *)rx_pdu->isp_hdr)->cmdsn;
3158         uint32_t        index;
3159 
3160         ASSERT(MUTEX_HELD(&ist->ist_sn_mutex));
3161         /*
3162          * If the connection is being torn down, then
3163          * don't add the PDU to the staging queue
3164          */
3165         mutex_enter(&ict->ict_mutex);
3166         if (ict->ict_lost) {
3167                 mutex_exit(&ict->ict_mutex);
3168                 idm_pdu_complete(rx_pdu, IDM_STATUS_FAIL);
3169                 return;
3170         }
3171         iscsit_conn_dispatch_hold(ict);
3172         mutex_exit(&ict->ict_mutex);
3173 
3174         index = ntohl(cmdsn) % ISCSIT_RXPDU_QUEUE_LEN;
3175         /*
3176          * In the normal case, assuming that the Initiator is not
3177          * buggy and that we don't have packet duplication occuring,
3178          * the entry in the array will be NULL.  However, we may have
3179          * received a duplicate PDU with cmdsn > expsn , and in that
3180          * case we just ignore this PDU -- the previously received one
3181          * remains queued for processing.  We need to be careful not
3182          * to leak this one however.
3183          */
3184         if (cbuf->cb_buffer[index] != NULL) {
3185                 idm_pdu_complete(rx_pdu, IDM_STATUS_FAIL);
3186         } else {
3187                 cbuf->cb_buffer[index] = rx_pdu;
3188                 cbuf->cb_num_elems++;
3189         }
3190 }
3191 
3192 static idm_pdu_t *
3193 iscsit_remove_pdu_from_queue(iscsit_sess_t *ist, uint32_t cmdsn)
3194 {
3195         iscsit_cbuf_t   *cbuf   = ist->ist_rxpdu_queue;
3196         idm_pdu_t       *pdu    = NULL;
3197         uint32_t        index;
3198 
3199         ASSERT(MUTEX_HELD(&ist->ist_sn_mutex));
3200         index = cmdsn % ISCSIT_RXPDU_QUEUE_LEN;
3201         if ((pdu = cbuf->cb_buffer[index]) != NULL) {
3202                 ASSERT(cmdsn ==
3203                     ntohl(((iscsi_scsi_cmd_hdr_t *)pdu->isp_hdr)->cmdsn));
3204                 cbuf->cb_buffer[index] = NULL;
3205                 cbuf->cb_num_elems--;
3206                 return (pdu);
3207         }
3208         return (NULL);
3209 }
3210 
3211 /*
3212  * iscsit_process_pdu_in_queue() finds the next pdu in sequence
3213  * and posts it to the SCSI layer
3214  */
3215 static void
3216 iscsit_process_pdu_in_queue(iscsit_sess_t *ist)
3217 {
3218         iscsit_cbuf_t   *cbuf   = ist->ist_rxpdu_queue;
3219         idm_pdu_t       *pdu = NULL;
3220         uint32_t        expcmdsn;
3221 
3222         for (;;) {
3223                 mutex_enter(&ist->ist_sn_mutex);
3224                 if (cbuf->cb_num_elems == 0) {
3225                         mutex_exit(&ist->ist_sn_mutex);
3226                         break;
3227                 }
3228                 expcmdsn = ist->ist_expcmdsn;
3229                 if ((pdu = iscsit_remove_pdu_from_queue(ist, expcmdsn))
3230                     == NULL) {
3231                         mutex_exit(&ist->ist_sn_mutex);
3232                         break;
3233                 }
3234                 mutex_exit(&ist->ist_sn_mutex);
3235                 iscsit_post_staged_pdu(pdu);
3236         }
3237 }
3238 
3239 static void
3240 iscsit_post_staged_pdu(idm_pdu_t *rx_pdu)
3241 {
3242         iscsit_conn_t   *ict    = rx_pdu->isp_ic->ic_handle;
3243 
3244         /* Post the PDU to the SCSI layer */
3245         switch (IDM_PDU_OPCODE(rx_pdu)) {
3246         case ISCSI_OP_NOOP_OUT:
3247                 iscsit_set_cmdsn(ict, rx_pdu);
3248                 iscsit_pdu_op_noop(ict, rx_pdu);
3249                 break;
3250         case ISCSI_OP_TEXT_CMD:
3251                 iscsit_set_cmdsn(ict, rx_pdu);
3252                 iscsit_pdu_op_text_cmd(ict, rx_pdu);
3253                 break;
3254         case ISCSI_OP_SCSI_TASK_MGT_MSG:
3255                 iscsit_set_cmdsn(ict, rx_pdu);
3256                 iscsit_op_scsi_task_mgmt(ict, rx_pdu);
3257                 break;
3258         case ISCSI_OP_SCSI_CMD:
3259                 /* cmdSN will be incremented after creating itask */
3260                 iscsit_post_scsi_cmd(rx_pdu->isp_ic, rx_pdu);
3261                 break;
3262         case ISCSI_OP_LOGOUT_CMD:
3263                 iscsit_set_cmdsn(ict, rx_pdu);
3264                 iscsit_pdu_op_logout_cmd(ict, rx_pdu);
3265                 break;
3266         default:
3267                 /* No other PDUs should be placed on the queue */
3268                 ASSERT(0);
3269         }
3270         iscsit_conn_dispatch_rele(ict); /* release hold on the conn */
3271 }
3272 
3273 /* ARGSUSED */
3274 void
3275 iscsit_rxpdu_queue_monitor_start(void)
3276 {
3277         mutex_enter(&iscsit_rxpdu_queue_monitor_mutex);
3278         if (iscsit_rxpdu_queue_monitor_thr_running) {
3279                 mutex_exit(&iscsit_rxpdu_queue_monitor_mutex);
3280                 return;
3281         }
3282         iscsit_rxpdu_queue_monitor_thr_id =
3283             thread_create(NULL, 0, iscsit_rxpdu_queue_monitor, NULL,
3284             0, &p0, TS_RUN, minclsyspri);
3285         while (!iscsit_rxpdu_queue_monitor_thr_running) {
3286                 cv_wait(&iscsit_rxpdu_queue_monitor_cv,
3287                     &iscsit_rxpdu_queue_monitor_mutex);
3288         }
3289         mutex_exit(&iscsit_rxpdu_queue_monitor_mutex);
3290 
3291 }
3292 
3293 /* ARGSUSED */
3294 void
3295 iscsit_rxpdu_queue_monitor_stop(void)
3296 {
3297         mutex_enter(&iscsit_rxpdu_queue_monitor_mutex);
3298         if (iscsit_rxpdu_queue_monitor_thr_running) {
3299                 iscsit_rxpdu_queue_monitor_thr_running = B_FALSE;
3300                 cv_signal(&iscsit_rxpdu_queue_monitor_cv);
3301                 mutex_exit(&iscsit_rxpdu_queue_monitor_mutex);
3302 
3303                 thread_join(iscsit_rxpdu_queue_monitor_thr_did);
3304                 return;
3305         }
3306         mutex_exit(&iscsit_rxpdu_queue_monitor_mutex);
3307 }
3308 
3309 /*
3310  * A separate thread is used to scan the staging queue on all the
3311  * sessions, If a delayed PDU does not arrive within a timeout, the
3312  * target will advance to the staged PDU that is next in sequence
3313  * and exceeded the threshold wait time. It is up to the initiator
3314  * to note that the target has not acknowledged a particular cmdsn
3315  * and take appropriate action.
3316  */
3317 /* ARGSUSED */
3318 static void
3319 iscsit_rxpdu_queue_monitor(void *arg)
3320 {
3321         iscsit_tgt_t    *tgt;
3322         iscsit_sess_t   *ist;
3323 
3324         mutex_enter(&iscsit_rxpdu_queue_monitor_mutex);
3325         iscsit_rxpdu_queue_monitor_thr_did = curthread->t_did;
3326         iscsit_rxpdu_queue_monitor_thr_running = B_TRUE;
3327         cv_signal(&iscsit_rxpdu_queue_monitor_cv);
3328 
3329         while (iscsit_rxpdu_queue_monitor_thr_running) {
3330                 ISCSIT_GLOBAL_LOCK(RW_READER);
3331                 for (tgt = avl_first(&iscsit_global.global_target_list);
3332                     tgt != NULL;
3333                     tgt = AVL_NEXT(&iscsit_global.global_target_list, tgt)) {
3334                         mutex_enter(&tgt->target_mutex);
3335                         for (ist = avl_first(&tgt->target_sess_list);
3336                             ist != NULL;
3337                             ist = AVL_NEXT(&tgt->target_sess_list, ist)) {
3338 
3339                                 iscsit_rxpdu_queue_monitor_session(ist);
3340                         }
3341                         mutex_exit(&tgt->target_mutex);
3342                 }
3343                 ISCSIT_GLOBAL_UNLOCK();
3344                 if (iscsit_rxpdu_queue_monitor_thr_running == B_FALSE) {
3345                         break;
3346                 }
3347                 (void) cv_reltimedwait(&iscsit_rxpdu_queue_monitor_cv,
3348                     &iscsit_rxpdu_queue_monitor_mutex,
3349                     ISCSIT_RXPDU_QUEUE_MONITOR_INTERVAL * drv_usectohz(1000000),
3350                     TR_CLOCK_TICK);
3351         }
3352         mutex_exit(&iscsit_rxpdu_queue_monitor_mutex);
3353         thread_exit();
3354 }
3355 
3356 static void
3357 iscsit_rxpdu_queue_monitor_session(iscsit_sess_t *ist)
3358 {
3359         iscsit_cbuf_t   *cbuf   = ist->ist_rxpdu_queue;
3360         idm_pdu_t       *next_pdu = NULL;
3361         uint32_t        index, next_cmdsn, i;
3362 
3363         /*
3364          * Assume that all PDUs in the staging queue have a cmdsn >= expcmdsn.
3365          * Starting with the expcmdsn, iterate over the staged PDUs to find
3366          * the next PDU with a wait time greater than the threshold. If found
3367          * advance the staged PDU to the SCSI layer, skipping over the missing
3368          * PDU(s) to get past the hole in the command sequence. It is up to
3369          * the initiator to note that the target has not acknowledged a cmdsn
3370          * and take appropriate action.
3371          *
3372          * Since the PDU(s) arrive in any random order, it is possible that
3373          * that the actual wait time for a particular PDU is much longer than
3374          * the defined threshold. e.g. Consider a case where commands are sent
3375          * over 4 different connections, and cmdsn = 1004 arrives first, then
3376          * 1003, and 1002 and 1001 are lost due to a connection failure.
3377          * So now 1003 is waiting for 1002 to be delivered, and although the
3378          * wait time of 1004 > wait time of 1003, only 1003 will be considered
3379          * by the monitor thread. 1004 will be automatically processed by
3380          * iscsit_process_pdu_in_queue() once the scan is complete and the
3381          * expcmdsn becomes current.
3382          */
3383         mutex_enter(&ist->ist_sn_mutex);
3384         cbuf = ist->ist_rxpdu_queue;
3385         if (cbuf->cb_num_elems == 0) {
3386                 mutex_exit(&ist->ist_sn_mutex);
3387                 return;
3388         }
3389         for (next_pdu = NULL, i = 0; ; i++) {
3390                 next_cmdsn = ist->ist_expcmdsn + i; /* start at expcmdsn */
3391                 index = next_cmdsn % ISCSIT_RXPDU_QUEUE_LEN;
3392                 if ((next_pdu = cbuf->cb_buffer[index]) != NULL) {
3393                         /*
3394                          * If the PDU wait time has not exceeded threshold
3395                          * stop scanning the staging queue until the timer
3396                          * fires again
3397                          */
3398                         if ((gethrtime() - next_pdu->isp_queue_time)
3399                             < (rxpdu_queue_threshold * NANOSEC)) {
3400                                 mutex_exit(&ist->ist_sn_mutex);
3401                                 return;
3402                         }
3403                         /*
3404                          * Remove the next PDU from the queue and post it
3405                          * to the SCSI layer, skipping over the missing
3406                          * PDU. Stop scanning the staging queue until
3407                          * the monitor timer fires again
3408                          */
3409                         (void) iscsit_remove_pdu_from_queue(ist, next_cmdsn);
3410                         mutex_exit(&ist->ist_sn_mutex);
3411                         DTRACE_PROBE3(advanced__to__blocked__cmdsn,
3412                             iscsit_sess_t *, ist, idm_pdu_t *, next_pdu,
3413                             uint32_t, next_cmdsn);
3414                         iscsit_post_staged_pdu(next_pdu);
3415                         /* Deliver any subsequent PDUs immediately */
3416                         iscsit_process_pdu_in_queue(ist);
3417                         return;
3418                 }
3419                 /*
3420                  * Skipping over i PDUs, e.g. a case where commands 1001 and
3421                  * 1002 are lost in the network, skip over both and post 1003
3422                  * expcmdsn then becomes 1004 at the end of the scan.
3423                  */
3424                 DTRACE_PROBE2(skipping__over__cmdsn, iscsit_sess_t *, ist,
3425                     uint32_t, next_cmdsn);
3426         }
3427         /*
3428          * following the assumption, staged cmdsn >= expcmdsn, this statement
3429          * is never reached.
3430          */
3431 }