1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  23  *
  24  * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
  25  */
  26 
  27 #include <sys/cpuvar.h>
  28 #include <sys/types.h>
  29 #include <sys/conf.h>
  30 #include <sys/stat.h>
  31 #include <sys/file.h>
  32 #include <sys/ddi.h>
  33 #include <sys/sunddi.h>
  34 #include <sys/modctl.h>
  35 #include <sys/sysmacros.h>
  36 #include <sys/socket.h>
  37 #include <sys/strsubr.h>
  38 #include <sys/nvpair.h>
  39 
  40 #include <sys/stmf.h>
  41 #include <sys/stmf_ioctl.h>
  42 #include <sys/portif.h>
  43 #include <sys/idm/idm.h>
  44 #include <sys/idm/idm_conn_sm.h>
  45 
  46 #include "iscsit_isns.h"
  47 #include "iscsit.h"
  48 
  49 #define ISCSIT_VERSION          BUILD_DATE "-1.18dev"
  50 #define ISCSIT_NAME_VERSION     "COMSTAR ISCSIT v" ISCSIT_VERSION
  51 
  52 /*
  53  * DDI entry points.
  54  */
  55 static int iscsit_drv_attach(dev_info_t *, ddi_attach_cmd_t);
  56 static int iscsit_drv_detach(dev_info_t *, ddi_detach_cmd_t);
  57 static int iscsit_drv_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
  58 static int iscsit_drv_open(dev_t *, int, int, cred_t *);
  59 static int iscsit_drv_close(dev_t, int, int, cred_t *);
  60 static boolean_t iscsit_drv_busy(void);
  61 static int iscsit_drv_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
  62 
  63 extern struct mod_ops mod_miscops;
  64 
  65 
  66 static struct cb_ops iscsit_cb_ops = {
  67         iscsit_drv_open,        /* cb_open */
  68         iscsit_drv_close,       /* cb_close */
  69         nodev,                  /* cb_strategy */
  70         nodev,                  /* cb_print */
  71         nodev,                  /* cb_dump */
  72         nodev,                  /* cb_read */
  73         nodev,                  /* cb_write */
  74         iscsit_drv_ioctl,       /* cb_ioctl */
  75         nodev,                  /* cb_devmap */
  76         nodev,                  /* cb_mmap */
  77         nodev,                  /* cb_segmap */
  78         nochpoll,               /* cb_chpoll */
  79         ddi_prop_op,            /* cb_prop_op */
  80         NULL,                   /* cb_streamtab */
  81         D_MP,                   /* cb_flag */
  82         CB_REV,                 /* cb_rev */
  83         nodev,                  /* cb_aread */
  84         nodev,                  /* cb_awrite */
  85 };
  86 
  87 static struct dev_ops iscsit_dev_ops = {
  88         DEVO_REV,               /* devo_rev */
  89         0,                      /* devo_refcnt */
  90         iscsit_drv_getinfo,     /* devo_getinfo */
  91         nulldev,                /* devo_identify */
  92         nulldev,                /* devo_probe */
  93         iscsit_drv_attach,      /* devo_attach */
  94         iscsit_drv_detach,      /* devo_detach */
  95         nodev,                  /* devo_reset */
  96         &iscsit_cb_ops,             /* devo_cb_ops */
  97         NULL,                   /* devo_bus_ops */
  98         NULL,                   /* devo_power */
  99         ddi_quiesce_not_needed, /* quiesce */
 100 };
 101 
 102 static struct modldrv modldrv = {
 103         &mod_driverops,
 104         "iSCSI Target",
 105         &iscsit_dev_ops,
 106 };
 107 
 108 static struct modlinkage modlinkage = {
 109         MODREV_1,
 110         &modldrv,
 111         NULL,
 112 };
 113 
 114 
 115 iscsit_global_t iscsit_global;
 116 
 117 kmem_cache_t    *iscsit_status_pdu_cache;
 118 
 119 boolean_t       iscsit_sm_logging = B_FALSE;
 120 
 121 kmutex_t        login_sm_session_mutex;
 122 
 123 static idm_status_t iscsit_init(dev_info_t *dip);
 124 static idm_status_t iscsit_enable_svc(iscsit_hostinfo_t *hostinfo);
 125 static void iscsit_disable_svc(void);
 126 
 127 static int
 128 iscsit_check_cmdsn_and_queue(idm_pdu_t *rx_pdu);
 129 
 130 static void
 131 iscsit_add_pdu_to_queue(iscsit_sess_t *ist, idm_pdu_t *rx_pdu);
 132 
 133 static idm_pdu_t *
 134 iscsit_remove_pdu_from_queue(iscsit_sess_t *ist, uint32_t cmdsn);
 135 
 136 static void
 137 iscsit_process_pdu_in_queue(iscsit_sess_t *ist);
 138 
 139 static void
 140 iscsit_rxpdu_queue_monitor_session(iscsit_sess_t *ist);
 141 
 142 static void
 143 iscsit_rxpdu_queue_monitor(void *arg);
 144 
 145 static void
 146 iscsit_post_staged_pdu(idm_pdu_t *rx_pdu);
 147 
 148 static void
 149 iscsit_post_scsi_cmd(idm_conn_t *ic, idm_pdu_t *rx_pdu);
 150 
 151 static void
 152 iscsit_op_scsi_task_mgmt(iscsit_conn_t *ict, idm_pdu_t *rx_pdu);
 153 
 154 static void
 155 iscsit_pdu_op_noop(iscsit_conn_t *ict, idm_pdu_t *rx_pdu);
 156 
 157 static void
 158 iscsit_pdu_op_login_cmd(iscsit_conn_t *ict, idm_pdu_t *rx_pdu);
 159 
 160 void
 161 iscsit_pdu_op_text_cmd(iscsit_conn_t *ict, idm_pdu_t *rx_pdu);
 162 
 163 static void
 164 iscsit_pdu_op_logout_cmd(iscsit_conn_t *ict, idm_pdu_t *rx_pdu);
 165 
 166 int iscsit_cmd_window();
 167 
 168 static  int
 169 iscsit_sna_lt(uint32_t sn1, uint32_t sn2);
 170 
 171 void
 172 iscsit_set_cmdsn(iscsit_conn_t *ict, idm_pdu_t *rx_pdu);
 173 
 174 static void
 175 iscsit_deferred_dispatch(idm_pdu_t *rx_pdu);
 176 
 177 static void
 178 iscsit_deferred(void *rx_pdu_void);
 179 
 180 static idm_status_t
 181 iscsit_conn_accept(idm_conn_t *ic);
 182 
 183 static idm_status_t
 184 iscsit_ffp_enabled(idm_conn_t *ic);
 185 
 186 static idm_status_t
 187 iscsit_ffp_disabled(idm_conn_t *ic, idm_ffp_disable_t disable_class);
 188 
 189 static idm_status_t
 190 iscsit_conn_lost(idm_conn_t *ic);
 191 
 192 static idm_status_t
 193 iscsit_conn_destroy(idm_conn_t *ic);
 194 
 195 static stmf_data_buf_t *
 196 iscsit_dbuf_alloc(scsi_task_t *task, uint32_t size, uint32_t *pminsize,
 197     uint32_t flags);
 198 
 199 static void
 200 iscsit_dbuf_free(stmf_dbuf_store_t *ds, stmf_data_buf_t *dbuf);
 201 
 202 static void
 203 iscsit_buf_xfer_cb(idm_buf_t *idb, idm_status_t status);
 204 
 205 static void
 206 iscsit_send_good_status_done(idm_pdu_t *pdu, idm_status_t status);
 207 
 208 static void
 209 iscsit_send_status_done(idm_pdu_t *pdu, idm_status_t status);
 210 
 211 static stmf_status_t
 212 iscsit_idm_to_stmf(idm_status_t idmrc);
 213 
 214 static iscsit_task_t *
 215 iscsit_task_alloc(iscsit_conn_t *ict);
 216 
 217 static void
 218 iscsit_task_free(iscsit_task_t *itask);
 219 
 220 static iscsit_task_t *
 221 iscsit_tm_task_alloc(iscsit_conn_t *ict);
 222 
 223 static void
 224 iscsit_tm_task_free(iscsit_task_t *itask);
 225 
 226 static idm_status_t
 227 iscsit_task_start(iscsit_task_t *itask);
 228 
 229 static void
 230 iscsit_task_done(iscsit_task_t *itask);
 231 
 232 static int
 233 iscsit_status_pdu_constructor(void *pdu_void, void *arg, int flags);
 234 
 235 static void
 236 iscsit_pp_cb(struct stmf_port_provider *pp, int cmd, void *arg, uint32_t flags);
 237 
 238 static it_cfg_status_t
 239 iscsit_config_merge(it_config_t *cfg);
 240 
 241 static idm_status_t
 242 iscsit_login_fail(idm_conn_t *ic);
 243 
 244 static boolean_t iscsit_cmdsn_in_window(iscsit_conn_t *ict, uint32_t cmdsn);
 245 static void iscsit_send_direct_scsi_resp(iscsit_conn_t *ict, idm_pdu_t *rx_pdu,
 246     uint8_t response, uint8_t cmd_status);
 247 static void iscsit_send_task_mgmt_resp(idm_pdu_t *tm_resp_pdu,
 248     uint8_t tm_status);
 249 
 250 /*
 251  * MC/S: Out-of-order commands are staged on a session-wide wait
 252  * queue until a system-tunable threshold is reached. A separate
 253  * thread is used to scan the staging queue on all the session,
 254  * If a delayed PDU does not arrive within a timeout, the target
 255  * will advance to the staged PDU that is next in sequence, skipping
 256  * over the missing PDU(s) to go past a hole in the sequence.
 257  */
 258 volatile int rxpdu_queue_threshold = ISCSIT_RXPDU_QUEUE_THRESHOLD;
 259 
 260 static kmutex_t         iscsit_rxpdu_queue_monitor_mutex;
 261 kthread_t               *iscsit_rxpdu_queue_monitor_thr_id;
 262 static kt_did_t         iscsit_rxpdu_queue_monitor_thr_did;
 263 static boolean_t        iscsit_rxpdu_queue_monitor_thr_running;
 264 static kcondvar_t       iscsit_rxpdu_queue_monitor_cv;
 265 
 266 int
 267 _init(void)
 268 {
 269         int rc;
 270 
 271         rw_init(&iscsit_global.global_rwlock, NULL, RW_DRIVER, NULL);
 272         mutex_init(&iscsit_global.global_state_mutex, NULL,
 273             MUTEX_DRIVER, NULL);
 274         iscsit_global.global_svc_state = ISE_DETACHED;
 275 
 276         mutex_init(&iscsit_rxpdu_queue_monitor_mutex, NULL,
 277             MUTEX_DRIVER, NULL);
 278         mutex_init(&login_sm_session_mutex, NULL, MUTEX_DRIVER, NULL);
 279         iscsit_rxpdu_queue_monitor_thr_id = NULL;
 280         iscsit_rxpdu_queue_monitor_thr_running = B_FALSE;
 281         cv_init(&iscsit_rxpdu_queue_monitor_cv, NULL, CV_DEFAULT, NULL);
 282 
 283         if ((rc = mod_install(&modlinkage)) != 0) {
 284                 mutex_destroy(&iscsit_global.global_state_mutex);
 285                 rw_destroy(&iscsit_global.global_rwlock);
 286                 return (rc);
 287         }
 288 
 289         return (rc);
 290 }
 291 
 292 int
 293 _info(struct modinfo *modinfop)
 294 {
 295         return (mod_info(&modlinkage, modinfop));
 296 }
 297 
 298 int
 299 _fini(void)
 300 {
 301         int rc;
 302 
 303         rc = mod_remove(&modlinkage);
 304 
 305         if (rc == 0) {
 306                 mutex_destroy(&iscsit_rxpdu_queue_monitor_mutex);
 307                 mutex_destroy(&login_sm_session_mutex);
 308                 cv_destroy(&iscsit_rxpdu_queue_monitor_cv);
 309                 mutex_destroy(&iscsit_global.global_state_mutex);
 310                 rw_destroy(&iscsit_global.global_rwlock);
 311         }
 312 
 313         return (rc);
 314 }
 315 
 316 /*
 317  * DDI entry points.
 318  */
 319 
 320 /* ARGSUSED */
 321 static int
 322 iscsit_drv_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
 323     void **result)
 324 {
 325         ulong_t instance = getminor((dev_t)arg);
 326 
 327         switch (cmd) {
 328         case DDI_INFO_DEVT2DEVINFO:
 329                 *result = iscsit_global.global_dip;
 330                 return (DDI_SUCCESS);
 331 
 332         case DDI_INFO_DEVT2INSTANCE:
 333                 *result = (void *)instance;
 334                 return (DDI_SUCCESS);
 335 
 336         default:
 337                 break;
 338         }
 339 
 340         return (DDI_FAILURE);
 341 }
 342 
 343 static int
 344 iscsit_drv_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 345 {
 346         if (cmd != DDI_ATTACH) {
 347                 return (DDI_FAILURE);
 348         }
 349 
 350         if (ddi_get_instance(dip) != 0) {
 351                 /* we only allow instance 0 to attach */
 352                 return (DDI_FAILURE);
 353         }
 354 
 355         /* create the minor node */
 356         if (ddi_create_minor_node(dip, ISCSIT_MODNAME, S_IFCHR, 0,
 357             DDI_PSEUDO, 0) != DDI_SUCCESS) {
 358                 cmn_err(CE_WARN, "iscsit_drv_attach: "
 359                     "failed creating minor node");
 360                 return (DDI_FAILURE);
 361         }
 362 
 363         if (iscsit_init(dip) != IDM_STATUS_SUCCESS) {
 364                 cmn_err(CE_WARN, "iscsit_drv_attach: "
 365                     "failed to initialize");
 366                 ddi_remove_minor_node(dip, NULL);
 367                 return (DDI_FAILURE);
 368         }
 369 
 370         iscsit_global.global_svc_state = ISE_DISABLED;
 371         iscsit_global.global_dip = dip;
 372 
 373         return (DDI_SUCCESS);
 374 }
 375 
 376 /*ARGSUSED*/
 377 static int
 378 iscsit_drv_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 379 {
 380         if (cmd != DDI_DETACH)
 381                 return (DDI_FAILURE);
 382 
 383         /*
 384          * drv_detach is called in a context that owns the
 385          * device node for the /dev/pseudo device.  If this thread blocks
 386          * for any resource, other threads that need the /dev/pseudo device
 387          * may end up in a deadlock with this thread.Hence, we use a
 388          * separate lock just for the structures that drv_detach needs
 389          * to access.
 390          */
 391         mutex_enter(&iscsit_global.global_state_mutex);
 392         if (iscsit_drv_busy()) {
 393                 mutex_exit(&iscsit_global.global_state_mutex);
 394                 return (EBUSY);
 395         }
 396 
 397         iscsit_global.global_dip = NULL;
 398         ddi_remove_minor_node(dip, NULL);
 399 
 400         ldi_ident_release(iscsit_global.global_li);
 401         iscsit_global.global_svc_state = ISE_DETACHED;
 402 
 403         mutex_exit(&iscsit_global.global_state_mutex);
 404 
 405         return (DDI_SUCCESS);
 406 }
 407 
 408 /*ARGSUSED*/
 409 static int
 410 iscsit_drv_open(dev_t *devp, int flag, int otyp, cred_t *credp)
 411 {
 412         return (0);
 413 }
 414 
 415 /* ARGSUSED */
 416 static int
 417 iscsit_drv_close(dev_t dev, int flag, int otyp, cred_t *credp)
 418 {
 419         return (0);
 420 }
 421 
 422 static boolean_t
 423 iscsit_drv_busy(void)
 424 {
 425         ASSERT(MUTEX_HELD(&iscsit_global.global_state_mutex));
 426 
 427         switch (iscsit_global.global_svc_state) {
 428         case ISE_DISABLED:
 429         case ISE_DETACHED:
 430                 return (B_FALSE);
 431         default:
 432                 return (B_TRUE);
 433         }
 434         /* NOTREACHED */
 435 }
 436 
 437 /* ARGSUSED */
 438 static int
 439 iscsit_drv_ioctl(dev_t drv, int cmd, intptr_t argp, int flag, cred_t *cred,
 440     int *retval)
 441 {
 442         iscsit_ioc_set_config_t         setcfg;
 443         iscsit_ioc_set_config32_t       setcfg32;
 444         char                            *cfg_pnvlist = NULL;
 445         nvlist_t                        *cfg_nvlist = NULL;
 446         it_config_t                     *cfg = NULL;
 447         idm_status_t                    idmrc;
 448         int                             rc = 0;
 449 
 450         if (drv_priv(cred) != 0) {
 451                 return (EPERM);
 452         }
 453 
 454         mutex_enter(&iscsit_global.global_state_mutex);
 455 
 456         /*
 457          * Validate ioctl requests against global service state
 458          */
 459         switch (iscsit_global.global_svc_state) {
 460         case ISE_ENABLED:
 461                 if (cmd == ISCSIT_IOC_DISABLE_SVC) {
 462                         iscsit_global.global_svc_state = ISE_DISABLING;
 463                 } else if (cmd == ISCSIT_IOC_ENABLE_SVC) {
 464                         /* Already enabled */
 465                         mutex_exit(&iscsit_global.global_state_mutex);
 466                         return (0);
 467                 } else {
 468                         iscsit_global.global_svc_state = ISE_BUSY;
 469                 }
 470                 break;
 471         case ISE_DISABLED:
 472                 if (cmd == ISCSIT_IOC_ENABLE_SVC) {
 473                         iscsit_global.global_svc_state = ISE_ENABLING;
 474                 } else if (cmd == ISCSIT_IOC_DISABLE_SVC) {
 475                         /* Already disabled */
 476                         mutex_exit(&iscsit_global.global_state_mutex);
 477                         return (0);
 478                 } else {
 479                         rc = EFAULT;
 480                 }
 481                 break;
 482         case ISE_BUSY:
 483         case ISE_ENABLING:
 484         case ISE_DISABLING:
 485                 rc = EAGAIN;
 486                 break;
 487         case ISE_DETACHED:
 488         default:
 489                 rc = EFAULT;
 490                 break;
 491         }
 492 
 493         mutex_exit(&iscsit_global.global_state_mutex);
 494         if (rc != 0)
 495                 return (rc);
 496 
 497         /* Handle ioctl request (enable/disable have already been handled) */
 498         switch (cmd) {
 499         case ISCSIT_IOC_SET_CONFIG:
 500                 /* Any errors must set state back to ISE_ENABLED */
 501                 switch (ddi_model_convert_from(flag & FMODELS)) {
 502                 case DDI_MODEL_ILP32:
 503                         if (ddi_copyin((void *)argp, &setcfg32,
 504                             sizeof (iscsit_ioc_set_config32_t), flag) != 0) {
 505                                 rc = EFAULT;
 506                                 goto cleanup;
 507                         }
 508 
 509                         setcfg.set_cfg_pnvlist =
 510                             (char *)((uintptr_t)setcfg32.set_cfg_pnvlist);
 511                         setcfg.set_cfg_vers = setcfg32.set_cfg_vers;
 512                         setcfg.set_cfg_pnvlist_len =
 513                             setcfg32.set_cfg_pnvlist_len;
 514                         break;
 515                 case DDI_MODEL_NONE:
 516                         if (ddi_copyin((void *)argp, &setcfg,
 517                             sizeof (iscsit_ioc_set_config_t), flag) != 0) {
 518                                 rc = EFAULT;
 519                                 goto cleanup;
 520                         }
 521                         break;
 522                 default:
 523                         rc = EFAULT;
 524                         goto cleanup;
 525                 }
 526 
 527                 /* Check API version */
 528                 if (setcfg.set_cfg_vers != ISCSIT_API_VERS0) {
 529                         rc = EINVAL;
 530                         goto cleanup;
 531                 }
 532 
 533                 /* Config is in packed nvlist format so unpack it */
 534                 cfg_pnvlist = kmem_alloc(setcfg.set_cfg_pnvlist_len,
 535                     KM_SLEEP);
 536                 ASSERT(cfg_pnvlist != NULL);
 537 
 538                 if (ddi_copyin(setcfg.set_cfg_pnvlist, cfg_pnvlist,
 539                     setcfg.set_cfg_pnvlist_len, flag) != 0) {
 540                         rc = EFAULT;
 541                         goto cleanup;
 542                 }
 543 
 544                 rc = nvlist_unpack(cfg_pnvlist, setcfg.set_cfg_pnvlist_len,
 545                     &cfg_nvlist, KM_SLEEP);
 546                 if (rc != 0) {
 547                         goto cleanup;
 548                 }
 549 
 550                 /* Translate nvlist */
 551                 rc = it_nv_to_config(cfg_nvlist, &cfg);
 552                 if (rc != 0) {
 553                         cmn_err(CE_WARN, "Configuration is invalid");
 554                         goto cleanup;
 555                 }
 556 
 557                 /* Update config */
 558                 rc = iscsit_config_merge(cfg);
 559                 /* FALLTHROUGH */
 560 
 561 cleanup:
 562                 if (cfg)
 563                         it_config_free_cmn(cfg);
 564                 if (cfg_pnvlist)
 565                         kmem_free(cfg_pnvlist, setcfg.set_cfg_pnvlist_len);
 566                 nvlist_free(cfg_nvlist);
 567 
 568                 /*
 569                  * Now that the reconfig is complete set our state back to
 570                  * enabled.
 571                  */
 572                 mutex_enter(&iscsit_global.global_state_mutex);
 573                 iscsit_global.global_svc_state = ISE_ENABLED;
 574                 mutex_exit(&iscsit_global.global_state_mutex);
 575                 break;
 576         case ISCSIT_IOC_ENABLE_SVC: {
 577                 iscsit_hostinfo_t hostinfo;
 578 
 579                 if (ddi_copyin((void *)argp, &hostinfo.length,
 580                     sizeof (hostinfo.length), flag) != 0) {
 581                         mutex_enter(&iscsit_global.global_state_mutex);
 582                         iscsit_global.global_svc_state = ISE_DISABLED;
 583                         mutex_exit(&iscsit_global.global_state_mutex);
 584                         return (EFAULT);
 585                 }
 586 
 587                 if (hostinfo.length > sizeof (hostinfo.fqhn))
 588                         hostinfo.length = sizeof (hostinfo.fqhn);
 589 
 590                 if (ddi_copyin((void *)((caddr_t)argp +
 591                     sizeof (hostinfo.length)), &hostinfo.fqhn,
 592                     hostinfo.length, flag) != 0) {
 593                         mutex_enter(&iscsit_global.global_state_mutex);
 594                         iscsit_global.global_svc_state = ISE_DISABLED;
 595                         mutex_exit(&iscsit_global.global_state_mutex);
 596                         return (EFAULT);
 597                 }
 598 
 599                 idmrc = iscsit_enable_svc(&hostinfo);
 600                 mutex_enter(&iscsit_global.global_state_mutex);
 601                 if (idmrc == IDM_STATUS_SUCCESS) {
 602                         iscsit_global.global_svc_state = ISE_ENABLED;
 603                 } else {
 604                         rc = EIO;
 605                         iscsit_global.global_svc_state = ISE_DISABLED;
 606                 }
 607                 mutex_exit(&iscsit_global.global_state_mutex);
 608                 break;
 609         }
 610         case ISCSIT_IOC_DISABLE_SVC:
 611                 iscsit_disable_svc();
 612                 mutex_enter(&iscsit_global.global_state_mutex);
 613                 iscsit_global.global_svc_state = ISE_DISABLED;
 614                 mutex_exit(&iscsit_global.global_state_mutex);
 615                 break;
 616 
 617         default:
 618                 rc = EINVAL;
 619                 mutex_enter(&iscsit_global.global_state_mutex);
 620                 iscsit_global.global_svc_state = ISE_ENABLED;
 621                 mutex_exit(&iscsit_global.global_state_mutex);
 622         }
 623 
 624         return (rc);
 625 }
 626 
 627 static idm_status_t
 628 iscsit_init(dev_info_t *dip)
 629 {
 630         int                     rc;
 631 
 632         rc = ldi_ident_from_dip(dip, &iscsit_global.global_li);
 633         ASSERT(rc == 0);  /* Failure indicates invalid argument */
 634 
 635         iscsit_global.global_svc_state = ISE_DISABLED;
 636 
 637         return (IDM_STATUS_SUCCESS);
 638 }
 639 
 640 /*
 641  * iscsit_enable_svc
 642  *
 643  * registers all the configured targets and target portals with STMF
 644  */
 645 static idm_status_t
 646 iscsit_enable_svc(iscsit_hostinfo_t *hostinfo)
 647 {
 648         stmf_port_provider_t    *pp;
 649         stmf_dbuf_store_t       *dbuf_store;
 650         boolean_t               did_iscsit_isns_init;
 651         idm_status_t            retval = IDM_STATUS_SUCCESS;
 652 
 653         ASSERT(iscsit_global.global_svc_state == ISE_ENABLING);
 654 
 655         /*
 656          * Make sure that can tell if we have partially allocated
 657          * in case we need to exit and tear down anything allocated.
 658          */
 659         iscsit_global.global_tsih_pool = NULL;
 660         iscsit_global.global_dbuf_store = NULL;
 661         iscsit_status_pdu_cache = NULL;
 662         pp = NULL;
 663         iscsit_global.global_pp = NULL;
 664         iscsit_global.global_default_tpg = NULL;
 665         did_iscsit_isns_init = B_FALSE;
 666         iscsit_global.global_dispatch_taskq = NULL;
 667 
 668         /* Setup remaining fields in iscsit_global_t */
 669         idm_refcnt_init(&iscsit_global.global_refcnt,
 670             &iscsit_global);
 671 
 672         avl_create(&iscsit_global.global_discovery_sessions,
 673             iscsit_sess_avl_compare, sizeof (iscsit_sess_t),
 674             offsetof(iscsit_sess_t, ist_tgt_ln));
 675 
 676         avl_create(&iscsit_global.global_target_list,
 677             iscsit_tgt_avl_compare, sizeof (iscsit_tgt_t),
 678             offsetof(iscsit_tgt_t, target_global_ln));
 679 
 680         list_create(&iscsit_global.global_deleted_target_list,
 681             sizeof (iscsit_tgt_t),
 682             offsetof(iscsit_tgt_t, target_global_deleted_ln));
 683 
 684         avl_create(&iscsit_global.global_tpg_list,
 685             iscsit_tpg_avl_compare, sizeof (iscsit_tpg_t),
 686             offsetof(iscsit_tpg_t, tpg_global_ln));
 687 
 688         avl_create(&iscsit_global.global_ini_list,
 689             iscsit_ini_avl_compare, sizeof (iscsit_ini_t),
 690             offsetof(iscsit_ini_t, ini_global_ln));
 691 
 692         iscsit_global.global_tsih_pool = vmem_create("iscsit_tsih_pool",
 693             (void *)1, ISCSI_MAX_TSIH, 1, NULL, NULL, NULL, 0,
 694             VM_SLEEP | VMC_IDENTIFIER);
 695 
 696         /*
 697          * Setup STMF dbuf store.  Our buffers are bound to a specific
 698          * connection so we really can't let STMF cache buffers for us.
 699          * Consequently we'll just allocate one global buffer store.
 700          */
 701         dbuf_store = stmf_alloc(STMF_STRUCT_DBUF_STORE, 0, 0);
 702         if (dbuf_store == NULL) {
 703                 retval = IDM_STATUS_FAIL;
 704                 goto tear_down_and_return;
 705         }
 706         dbuf_store->ds_alloc_data_buf = iscsit_dbuf_alloc;
 707         dbuf_store->ds_free_data_buf = iscsit_dbuf_free;
 708         dbuf_store->ds_port_private = NULL;
 709         iscsit_global.global_dbuf_store = dbuf_store;
 710 
 711         /* Status PDU cache */
 712         iscsit_status_pdu_cache = kmem_cache_create("iscsit_status_pdu_cache",
 713             sizeof (idm_pdu_t) + sizeof (iscsi_scsi_rsp_hdr_t), 8,
 714             &iscsit_status_pdu_constructor,
 715             NULL, NULL, NULL, NULL, KM_SLEEP);
 716 
 717         /* Default TPG and portal */
 718         iscsit_global.global_default_tpg = iscsit_tpg_createdefault();
 719         if (iscsit_global.global_default_tpg == NULL) {
 720                 retval = IDM_STATUS_FAIL;
 721                 goto tear_down_and_return;
 722         }
 723 
 724         /* initialize isns client */
 725         (void) iscsit_isns_init(hostinfo);
 726         did_iscsit_isns_init = B_TRUE;
 727 
 728         /* Register port provider */
 729         pp = stmf_alloc(STMF_STRUCT_PORT_PROVIDER, 0, 0);
 730         if (pp == NULL) {
 731                 retval = IDM_STATUS_FAIL;
 732                 goto tear_down_and_return;
 733         }
 734 
 735         pp->pp_portif_rev = PORTIF_REV_1;
 736         pp->pp_instance = 0;
 737         pp->pp_name = ISCSIT_MODNAME;
 738         pp->pp_cb = iscsit_pp_cb;
 739 
 740         iscsit_global.global_pp = pp;
 741 
 742 
 743         if (stmf_register_port_provider(pp) != STMF_SUCCESS) {
 744                 retval = IDM_STATUS_FAIL;
 745                 goto tear_down_and_return;
 746         }
 747 
 748         iscsit_global.global_dispatch_taskq = taskq_create("iscsit_dispatch",
 749             1, minclsyspri, 16, 16, TASKQ_PREPOPULATE);
 750 
 751         /* Scan staged PDUs, meaningful in MC/S situations */
 752         iscsit_rxpdu_queue_monitor_start();
 753 
 754         return (IDM_STATUS_SUCCESS);
 755 
 756 tear_down_and_return:
 757 
 758         if (iscsit_global.global_dispatch_taskq) {
 759                 taskq_destroy(iscsit_global.global_dispatch_taskq);
 760                 iscsit_global.global_dispatch_taskq = NULL;
 761         }
 762 
 763         if (did_iscsit_isns_init)
 764                 iscsit_isns_fini();
 765 
 766         if (iscsit_global.global_default_tpg) {
 767                 iscsit_tpg_destroydefault(iscsit_global.global_default_tpg);
 768                 iscsit_global.global_default_tpg = NULL;
 769         }
 770 
 771         if (iscsit_global.global_pp)
 772                 iscsit_global.global_pp = NULL;
 773 
 774         if (pp)
 775                 stmf_free(pp);
 776 
 777         if (iscsit_status_pdu_cache) {
 778                 kmem_cache_destroy(iscsit_status_pdu_cache);
 779                 iscsit_status_pdu_cache = NULL;
 780         }
 781 
 782         if (iscsit_global.global_dbuf_store) {
 783                 stmf_free(iscsit_global.global_dbuf_store);
 784                 iscsit_global.global_dbuf_store = NULL;
 785         }
 786 
 787         if (iscsit_global.global_tsih_pool) {
 788                 vmem_destroy(iscsit_global.global_tsih_pool);
 789                 iscsit_global.global_tsih_pool = NULL;
 790         }
 791 
 792         avl_destroy(&iscsit_global.global_ini_list);
 793         avl_destroy(&iscsit_global.global_tpg_list);
 794         list_destroy(&iscsit_global.global_deleted_target_list);
 795         avl_destroy(&iscsit_global.global_target_list);
 796         avl_destroy(&iscsit_global.global_discovery_sessions);
 797 
 798         idm_refcnt_destroy(&iscsit_global.global_refcnt);
 799 
 800         return (retval);
 801 }
 802 
 803 /*
 804  * iscsit_disable_svc
 805  *
 806  * clean up all existing connections and deregister targets from STMF
 807  */
 808 static void
 809 iscsit_disable_svc(void)
 810 {
 811         iscsit_sess_t   *sess;
 812 
 813         ASSERT(iscsit_global.global_svc_state == ISE_DISABLING);
 814 
 815         iscsit_rxpdu_queue_monitor_stop();
 816 
 817         /* tear down discovery sessions */
 818         for (sess = avl_first(&iscsit_global.global_discovery_sessions);
 819             sess != NULL;
 820             sess = AVL_NEXT(&iscsit_global.global_discovery_sessions, sess))
 821                 iscsit_sess_close(sess);
 822 
 823         /*
 824          * Passing NULL to iscsit_config_merge tells it to go to an empty
 825          * config.
 826          */
 827         (void) iscsit_config_merge(NULL);
 828 
 829         /*
 830          * Wait until there are no more global references
 831          */
 832         idm_refcnt_wait_ref(&iscsit_global.global_refcnt);
 833         idm_refcnt_destroy(&iscsit_global.global_refcnt);
 834 
 835         /*
 836          * Default TPG must be destroyed after global_refcnt is 0.
 837          */
 838         iscsit_tpg_destroydefault(iscsit_global.global_default_tpg);
 839 
 840         avl_destroy(&iscsit_global.global_discovery_sessions);
 841         list_destroy(&iscsit_global.global_deleted_target_list);
 842         avl_destroy(&iscsit_global.global_target_list);
 843         avl_destroy(&iscsit_global.global_tpg_list);
 844         avl_destroy(&iscsit_global.global_ini_list);
 845 
 846         taskq_destroy(iscsit_global.global_dispatch_taskq);
 847 
 848         iscsit_isns_fini();
 849 
 850         stmf_free(iscsit_global.global_dbuf_store);
 851         iscsit_global.global_dbuf_store = NULL;
 852 
 853         (void) stmf_deregister_port_provider(iscsit_global.global_pp);
 854         stmf_free(iscsit_global.global_pp);
 855         iscsit_global.global_pp = NULL;
 856 
 857         kmem_cache_destroy(iscsit_status_pdu_cache);
 858         iscsit_status_pdu_cache = NULL;
 859 
 860         vmem_destroy(iscsit_global.global_tsih_pool);
 861         iscsit_global.global_tsih_pool = NULL;
 862 }
 863 
 864 void
 865 iscsit_global_hold()
 866 {
 867         /*
 868          * To take out a global hold, we must either own the global
 869          * state mutex or we must be running inside of an ioctl that
 870          * has set the global state to ISE_BUSY, ISE_DISABLING, or
 871          * ISE_ENABLING.  We don't track the "owner" for these flags,
 872          * so just checking if they are set is enough for now.
 873          */
 874         ASSERT((iscsit_global.global_svc_state == ISE_ENABLING) ||
 875             (iscsit_global.global_svc_state == ISE_DISABLING) ||
 876             (iscsit_global.global_svc_state == ISE_BUSY) ||
 877             MUTEX_HELD(&iscsit_global.global_state_mutex));
 878 
 879         idm_refcnt_hold(&iscsit_global.global_refcnt);
 880 }
 881 
 882 void
 883 iscsit_global_rele()
 884 {
 885         idm_refcnt_rele(&iscsit_global.global_refcnt);
 886 }
 887 
 888 void
 889 iscsit_global_wait_ref()
 890 {
 891         idm_refcnt_wait_ref(&iscsit_global.global_refcnt);
 892 }
 893 
 894 /*
 895  * IDM callbacks
 896  */
 897 
 898 /*ARGSUSED*/
 899 void
 900 iscsit_rx_pdu(idm_conn_t *ic, idm_pdu_t *rx_pdu)
 901 {
 902         iscsit_conn_t *ict = ic->ic_handle;
 903         switch (IDM_PDU_OPCODE(rx_pdu)) {
 904         case ISCSI_OP_SCSI_CMD:
 905                 ASSERT(0); /* Shouldn't happen */
 906                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
 907                 break;
 908         case ISCSI_OP_SNACK_CMD:
 909                 /*
 910                  * We'll need to handle this when we support ERL1/2.  For
 911                  * now we treat it as a protocol error.
 912                  */
 913                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
 914                 idm_conn_event(ic, CE_TRANSPORT_FAIL, NULL);
 915                 break;
 916         case ISCSI_OP_SCSI_TASK_MGT_MSG:
 917                 if (iscsit_check_cmdsn_and_queue(rx_pdu)) {
 918                         iscsit_set_cmdsn(ict, rx_pdu);
 919                         iscsit_op_scsi_task_mgmt(ict, rx_pdu);
 920                 }
 921                 break;
 922         case ISCSI_OP_NOOP_OUT:
 923         case ISCSI_OP_LOGIN_CMD:
 924         case ISCSI_OP_TEXT_CMD:
 925         case ISCSI_OP_LOGOUT_CMD:
 926                 /*
 927                  * If/when we switch to userland processing these PDU's
 928                  * will be handled by iscsitd.
 929                  */
 930                 iscsit_deferred_dispatch(rx_pdu);
 931                 break;
 932         default:
 933                 /* Protocol error */
 934                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
 935                 idm_conn_event(ic, CE_TRANSPORT_FAIL, NULL);
 936                 break;
 937         }
 938 }
 939 
 940 /*ARGSUSED*/
 941 void
 942 iscsit_rx_pdu_error(idm_conn_t *ic, idm_pdu_t *rx_pdu, idm_status_t status)
 943 {
 944         idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
 945 }
 946 
 947 void
 948 iscsit_task_aborted(idm_task_t *idt, idm_status_t status)
 949 {
 950         iscsit_task_t *itask = idt->idt_private;
 951 
 952         switch (status) {
 953         case IDM_STATUS_SUSPENDED:
 954                 break;
 955         case IDM_STATUS_ABORTED:
 956                 mutex_enter(&itask->it_mutex);
 957                 itask->it_aborted = B_TRUE;
 958                 /*
 959                  * We rely on the fact that STMF tracks outstanding
 960                  * buffer transfers and will free all of our buffers
 961                  * before freeing the task so we don't need to
 962                  * explicitly free the buffers from iscsit/idm
 963                  */
 964                 if (itask->it_stmf_abort) {
 965                         mutex_exit(&itask->it_mutex);
 966                         /*
 967                          * Task is no longer active
 968                          */
 969                         iscsit_task_done(itask);
 970 
 971                         /*
 972                          * STMF has already asked for this task to be aborted
 973                          *
 974                          * STMF specification is wrong... says to return
 975                          * STMF_ABORTED, the code actually looks for
 976                          * STMF_ABORT_SUCCESS.
 977                          */
 978                         stmf_task_lport_aborted(itask->it_stmf_task,
 979                             STMF_ABORT_SUCCESS, STMF_IOF_LPORT_DONE);
 980                         return;
 981                 } else {
 982                         mutex_exit(&itask->it_mutex);
 983                         /*
 984                          * Tell STMF to stop processing the task.
 985                          */
 986                         stmf_abort(STMF_QUEUE_TASK_ABORT, itask->it_stmf_task,
 987                             STMF_ABORTED, NULL);
 988                         return;
 989                 }
 990                 /*NOTREACHED*/
 991         default:
 992                 ASSERT(0);
 993         }
 994 }
 995 
 996 /*ARGSUSED*/
 997 idm_status_t
 998 iscsit_client_notify(idm_conn_t *ic, idm_client_notify_t icn,
 999     uintptr_t data)
1000 {
1001         idm_status_t rc = IDM_STATUS_SUCCESS;
1002 
1003         /*
1004          * IDM client notifications will never occur at interrupt level
1005          * since they are generated from the connection state machine which
1006          * running on taskq threads.
1007          *
1008          */
1009         switch (icn) {
1010         case CN_CONNECT_ACCEPT:
1011                 rc = iscsit_conn_accept(ic); /* No data */
1012                 break;
1013         case CN_FFP_ENABLED:
1014                 rc = iscsit_ffp_enabled(ic); /* No data */
1015                 break;
1016         case CN_FFP_DISABLED:
1017                 /*
1018                  * Data indicates whether this was the result of an
1019                  * explicit logout request.
1020                  */
1021                 rc = iscsit_ffp_disabled(ic, (idm_ffp_disable_t)data);
1022                 break;
1023         case CN_CONNECT_LOST:
1024                 rc = iscsit_conn_lost(ic);
1025                 break;
1026         case CN_CONNECT_DESTROY:
1027                 rc = iscsit_conn_destroy(ic);
1028                 break;
1029         case CN_LOGIN_FAIL:
1030                 /*
1031                  * Force the login state machine to completion
1032                  */
1033                 rc = iscsit_login_fail(ic);
1034                 break;
1035         default:
1036                 rc = IDM_STATUS_REJECT;
1037                 break;
1038         }
1039 
1040         return (rc);
1041 }
1042 
1043 /*
1044  * iscsit_update_statsn is invoked for all the PDUs which have the StatSN
1045  * field in the header. The StatSN is incremented if the IDM_PDU_ADVANCE_STATSN
1046  * flag is set in the pdu flags field. The StatSN is connection-wide and is
1047  * protected by the mutex ict_statsn_mutex. For Data-In PDUs, if the flag
1048  * IDM_TASK_PHASECOLLAPSE_REQ is set, the status (phase-collapse) is also filled
1049  */
1050 void
1051 iscsit_update_statsn(idm_task_t *idm_task, idm_pdu_t *pdu)
1052 {
1053         iscsi_scsi_rsp_hdr_t *rsp = (iscsi_scsi_rsp_hdr_t *)pdu->isp_hdr;
1054         iscsit_conn_t *ict = (iscsit_conn_t *)pdu->isp_ic->ic_handle;
1055         iscsit_task_t *itask = NULL;
1056         scsi_task_t *task = NULL;
1057 
1058         mutex_enter(&ict->ict_statsn_mutex);
1059         rsp->statsn = htonl(ict->ict_statsn);
1060         if (pdu->isp_flags & IDM_PDU_ADVANCE_STATSN)
1061                 ict->ict_statsn++;
1062         mutex_exit(&ict->ict_statsn_mutex);
1063 
1064         /*
1065          * The last SCSI Data PDU passed for a command may also contain the
1066          * status if the status indicates termination with no expections, i.e.
1067          * no sense data or response involved. If the command completes with
1068          * an error, then the response and sense data will be sent in a
1069          * separate iSCSI Response PDU.
1070          */
1071         if ((idm_task) && (idm_task->idt_flags & IDM_TASK_PHASECOLLAPSE_REQ)) {
1072                 itask = idm_task->idt_private;
1073                 task = itask->it_stmf_task;
1074 
1075                 rsp->cmd_status = task->task_scsi_status;
1076                 rsp->flags   |= ISCSI_FLAG_DATA_STATUS;
1077                 if (task->task_status_ctrl & TASK_SCTRL_OVER) {
1078                         rsp->flags |= ISCSI_FLAG_CMD_OVERFLOW;
1079                 } else if (task->task_status_ctrl & TASK_SCTRL_UNDER) {
1080                         rsp->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
1081                 }
1082                 rsp->residual_count = htonl(task->task_resid);
1083 
1084                 /*
1085                  * Removing the task from the session task list
1086                  * just before the status is sent in the last
1087                  * Data PDU transfer
1088                  */
1089                 iscsit_task_done(itask);
1090         }
1091 }
1092 
1093 void
1094 iscsit_build_hdr(idm_task_t *idm_task, idm_pdu_t *pdu, uint8_t opcode)
1095 {
1096         iscsit_task_t *itask = idm_task->idt_private;
1097         iscsi_data_rsp_hdr_t *dh = (iscsi_data_rsp_hdr_t *)pdu->isp_hdr;
1098 
1099         /*
1100          * We acquired iscsit_sess_t.ist_sn_mutex in iscsit_xfer_scsi_data
1101          */
1102         ASSERT(MUTEX_HELD(&itask->it_ict->ict_sess->ist_sn_mutex));
1103         /*
1104          * On incoming data, the target transfer tag and Lun is only
1105          * provided by the target if the A bit is set, Since the target
1106          * does not currently support Error Recovery Level 1, the A
1107          * bit is never set.
1108          */
1109         dh->opcode = opcode;
1110         dh->itt = itask->it_itt;
1111         dh->ttt = ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_SCSI_DATA_RSP) ?
1112             ISCSI_RSVD_TASK_TAG : itask->it_ttt;
1113 
1114         dh->expcmdsn = htonl(itask->it_ict->ict_sess->ist_expcmdsn);
1115         dh->maxcmdsn = htonl(itask->it_ict->ict_sess->ist_maxcmdsn);
1116 
1117         /*
1118          * IDM must set:
1119          *
1120          * data.flags and rtt.flags
1121          * data.dlength
1122          * data.datasn
1123          * data.offset
1124          * statsn, residual_count and cmd_status (for phase collapse)
1125          * rtt.rttsn
1126          * rtt.data_offset
1127          * rtt.data_length
1128          */
1129 }
1130 
1131 void
1132 iscsit_keepalive(idm_conn_t *ic)
1133 {
1134         idm_pdu_t               *nop_in_pdu;
1135         iscsi_nop_in_hdr_t      *nop_in;
1136         iscsit_conn_t           *ict = ic->ic_handle;
1137 
1138         /*
1139          * IDM noticed the connection has been idle for too long so it's
1140          * time to provoke some activity.  Build and transmit an iSCSI
1141          * nop-in PDU -- when the initiator responds it will be counted
1142          * as "activity" and keep the connection alive.
1143          *
1144          * We don't actually care about the response here at the iscsit level
1145          * so we will just throw it away without looking at it when it arrives.
1146          */
1147         nop_in_pdu = idm_pdu_alloc(sizeof (*nop_in), 0);
1148         idm_pdu_init(nop_in_pdu, ic, NULL, NULL);
1149         nop_in = (iscsi_nop_in_hdr_t *)nop_in_pdu->isp_hdr;
1150         bzero(nop_in, sizeof (*nop_in));
1151         nop_in->opcode = ISCSI_OP_NOOP_IN;
1152         nop_in->flags = ISCSI_FLAG_FINAL;
1153         nop_in->itt = ISCSI_RSVD_TASK_TAG;
1154         /*
1155          * When the target sends a NOP-In as a Ping, the target transfer tag
1156          * is set to a valid (not reserved) value and the initiator task tag
1157          * is set to ISCSI_RSVD_TASK_TAG (0xffffffff). In this case the StatSN
1158          * will always contain the next sequence number but the StatSN for the
1159          * connection is not advanced after this PDU is sent.
1160          */
1161         nop_in_pdu->isp_flags |= IDM_PDU_SET_STATSN;
1162         /*
1163          * This works because we don't currently allocate ttt's anywhere else
1164          * in iscsit so as long as we stay out of IDM's range we are safe.
1165          * If we need to allocate ttt's for other PDU's in the future this will
1166          * need to be improved.
1167          */
1168         mutex_enter(&ict->ict_mutex);
1169         nop_in->ttt = ict->ict_keepalive_ttt;
1170         ict->ict_keepalive_ttt++;
1171         if (ict->ict_keepalive_ttt == ISCSI_RSVD_TASK_TAG)
1172                 ict->ict_keepalive_ttt = IDM_TASKIDS_MAX;
1173         mutex_exit(&ict->ict_mutex);
1174 
1175         iscsit_pdu_tx(nop_in_pdu);
1176 }
1177 
1178 static idm_status_t
1179 iscsit_conn_accept(idm_conn_t *ic)
1180 {
1181         iscsit_conn_t *ict;
1182 
1183         /*
1184          * We need to get a global hold here to ensure that the service
1185          * doesn't get shutdown prior to establishing a session. This
1186          * gets released in iscsit_conn_destroy().
1187          */
1188         mutex_enter(&iscsit_global.global_state_mutex);
1189         if (iscsit_global.global_svc_state != ISE_ENABLED) {
1190                 mutex_exit(&iscsit_global.global_state_mutex);
1191                 return (IDM_STATUS_FAIL);
1192         }
1193         iscsit_global_hold();
1194         mutex_exit(&iscsit_global.global_state_mutex);
1195 
1196         /*
1197          * Allocate an associated iscsit structure to represent this
1198          * connection.  We shouldn't really create a session until we
1199          * get the first login PDU.
1200          */
1201         ict = kmem_zalloc(sizeof (*ict), KM_SLEEP);
1202 
1203         ict->ict_ic = ic;
1204         ict->ict_statsn = 1;
1205         ict->ict_keepalive_ttt = IDM_TASKIDS_MAX; /* Avoid IDM TT range */
1206         ic->ic_handle = ict;
1207         mutex_init(&ict->ict_mutex, NULL, MUTEX_DRIVER, NULL);
1208         mutex_init(&ict->ict_statsn_mutex, NULL, MUTEX_DRIVER, NULL);
1209         idm_refcnt_init(&ict->ict_refcnt, ict);
1210 
1211         /*
1212          * Initialize login state machine
1213          */
1214         if (iscsit_login_sm_init(ict) != IDM_STATUS_SUCCESS) {
1215                 iscsit_global_rele();
1216                 /*
1217                  * Cleanup the ict after idm notifies us about this failure
1218                  */
1219                 return (IDM_STATUS_FAIL);
1220         }
1221 
1222         return (IDM_STATUS_SUCCESS);
1223 }
1224 
1225 idm_status_t
1226 iscsit_conn_reinstate(iscsit_conn_t *reinstate_ict, iscsit_conn_t *new_ict)
1227 {
1228         idm_status_t    result;
1229 
1230         /*
1231          * Note in new connection state that this connection is
1232          * reinstating an existing connection.
1233          */
1234         new_ict->ict_reinstating = B_TRUE;
1235         new_ict->ict_reinstate_conn = reinstate_ict;
1236         new_ict->ict_statsn = reinstate_ict->ict_statsn;
1237 
1238         /*
1239          * Now generate connection state machine event to existing connection
1240          * so that it starts the cleanup process.
1241          */
1242         result = idm_conn_reinstate_event(reinstate_ict->ict_ic,
1243             new_ict->ict_ic);
1244 
1245         return (result);
1246 }
1247 
1248 void
1249 iscsit_conn_hold(iscsit_conn_t *ict)
1250 {
1251         idm_refcnt_hold(&ict->ict_refcnt);
1252 }
1253 
1254 void
1255 iscsit_conn_rele(iscsit_conn_t *ict)
1256 {
1257         idm_refcnt_rele(&ict->ict_refcnt);
1258 }
1259 
1260 void
1261 iscsit_conn_dispatch_hold(iscsit_conn_t *ict)
1262 {
1263         idm_refcnt_hold(&ict->ict_dispatch_refcnt);
1264 }
1265 
1266 void
1267 iscsit_conn_dispatch_rele(iscsit_conn_t *ict)
1268 {
1269         idm_refcnt_rele(&ict->ict_dispatch_refcnt);
1270 }
1271 
1272 static idm_status_t
1273 iscsit_login_fail(idm_conn_t *ic)
1274 {
1275         iscsit_conn_t *ict = ic->ic_handle;
1276 
1277         /* Generate login state machine event */
1278         iscsit_login_sm_event(ict, ILE_LOGIN_CONN_ERROR, NULL);
1279 
1280         return (IDM_STATUS_SUCCESS);
1281 }
1282 
1283 static idm_status_t
1284 iscsit_ffp_enabled(idm_conn_t *ic)
1285 {
1286         iscsit_conn_t *ict = ic->ic_handle;
1287 
1288         /* Generate session state machine event */
1289         iscsit_sess_sm_event(ict->ict_sess, SE_CONN_LOGGED_IN, ict);
1290 
1291         return (IDM_STATUS_SUCCESS);
1292 }
1293 
1294 static idm_status_t
1295 iscsit_ffp_disabled(idm_conn_t *ic, idm_ffp_disable_t disable_class)
1296 {
1297         iscsit_conn_t *ict = ic->ic_handle;
1298 
1299         /* Generate session state machine event */
1300         switch (disable_class) {
1301         case FD_CONN_FAIL:
1302                 iscsit_sess_sm_event(ict->ict_sess, SE_CONN_FFP_FAIL, ict);
1303                 break;
1304         case FD_CONN_LOGOUT:
1305                 iscsit_sess_sm_event(ict->ict_sess, SE_CONN_FFP_DISABLE, ict);
1306                 break;
1307         case FD_SESS_LOGOUT:
1308                 iscsit_sess_sm_event(ict->ict_sess, SE_SESSION_CLOSE, ict);
1309                 break;
1310         default:
1311                 ASSERT(0);
1312         }
1313 
1314         return (IDM_STATUS_SUCCESS);
1315 }
1316 
1317 static idm_status_t
1318 iscsit_conn_lost(idm_conn_t *ic)
1319 {
1320         iscsit_conn_t   *ict    = ic->ic_handle;
1321         iscsit_sess_t   *ist    = ict->ict_sess;
1322         iscsit_cbuf_t   *cbuf;
1323         idm_pdu_t       *rx_pdu;
1324         int i;
1325 
1326         mutex_enter(&ict->ict_mutex);
1327         ict->ict_lost = B_TRUE;
1328         mutex_exit(&ict->ict_mutex);
1329         /*
1330          * scrub the staging queue for all PDUs on this connection
1331          */
1332         if (ist != NULL) {
1333                 mutex_enter(&ist->ist_sn_mutex);
1334                 for (cbuf = ist->ist_rxpdu_queue, i = 0;
1335                     ((cbuf->cb_num_elems > 0) && (i < ISCSIT_RXPDU_QUEUE_LEN));
1336                     i++) {
1337                         if (((rx_pdu = cbuf->cb_buffer[i]) != NULL) &&
1338                             (rx_pdu->isp_ic == ic)) {
1339                                 /* conn is lost, drop the pdu */
1340                                 DTRACE_PROBE3(scrubbing__staging__queue,
1341                                     iscsit_sess_t *, ist, idm_conn_t *, ic,
1342                                     idm_pdu_t *, rx_pdu);
1343                                 idm_pdu_complete(rx_pdu, IDM_STATUS_FAIL);
1344                                 cbuf->cb_buffer[i] = NULL;
1345                                 cbuf->cb_num_elems--;
1346                                 iscsit_conn_dispatch_rele(ict);
1347                         }
1348                 }
1349                 mutex_exit(&ist->ist_sn_mutex);
1350         }
1351         /*
1352          * Make sure there aren't any PDU's transitioning from the receive
1353          * handler to the dispatch taskq.
1354          */
1355         idm_refcnt_wait_ref(&ict->ict_dispatch_refcnt);
1356 
1357         return (IDM_STATUS_SUCCESS);
1358 }
1359 
1360 static idm_status_t
1361 iscsit_conn_destroy(idm_conn_t *ic)
1362 {
1363         iscsit_conn_t *ict = ic->ic_handle;
1364 
1365         mutex_enter(&ict->ict_mutex);
1366         ict->ict_destroyed = B_TRUE;
1367         mutex_exit(&ict->ict_mutex);
1368 
1369         /* Generate session state machine event */
1370         if (ict->ict_sess != NULL) {
1371                 /*
1372                  * Session state machine will call iscsit_conn_destroy_done()
1373                  * when it has removed references to this connection.
1374                  */
1375                 iscsit_sess_sm_event(ict->ict_sess, SE_CONN_FAIL, ict);
1376         }
1377 
1378         idm_refcnt_wait_ref(&ict->ict_refcnt);
1379         /*
1380          * The session state machine does not need to post
1381          * events to IDM any longer, so it is safe to set
1382          * the idm connection reference to NULL
1383          */
1384         ict->ict_ic = NULL;
1385 
1386         /* Reap the login state machine */
1387         iscsit_login_sm_fini(ict);
1388 
1389         /* Clean up any text command remnants */
1390         iscsit_text_cmd_fini(ict);
1391 
1392         mutex_destroy(&ict->ict_mutex);
1393         idm_refcnt_destroy(&ict->ict_refcnt);
1394         kmem_free(ict, sizeof (*ict));
1395 
1396         iscsit_global_rele();
1397 
1398         return (IDM_STATUS_SUCCESS);
1399 }
1400 
1401 void
1402 iscsit_conn_logout(iscsit_conn_t *ict)
1403 {
1404         /*
1405          * If the iscsi connection is active, then
1406          * logout the IDM connection by sending a
1407          * CE_LOGOUT_SESSION_SUCCESS, else, no action
1408          * needs to be taken because the connection
1409          * is already in the teardown process.
1410          */
1411         mutex_enter(&ict->ict_mutex);
1412         if (ict->ict_lost == B_FALSE && ict->ict_destroyed == B_FALSE) {
1413                 idm_conn_event(ict->ict_ic, CE_LOGOUT_SESSION_SUCCESS, NULL);
1414         }
1415         mutex_exit(&ict->ict_mutex);
1416 }
1417 
1418 /*
1419  * STMF-related functions
1420  *
1421  * iSCSI to STMF mapping
1422  *
1423  * Session == ?
1424  * Connection == bound to local port but not itself a local port
1425  * Target
1426  * Target portal (group?) == local port (really but we're not going to do this)
1427  *      iscsit needs to map connections to local ports (whatever we decide
1428  *      they are)
1429  * Target == ?
1430  */
1431 
1432 /*ARGSUSED*/
1433 static stmf_data_buf_t *
1434 iscsit_dbuf_alloc(scsi_task_t *task, uint32_t size, uint32_t *pminsize,
1435     uint32_t flags)
1436 {
1437         iscsit_task_t *itask = task->task_port_private;
1438         idm_buf_t *idm_buffer;
1439         iscsit_buf_t    *ibuf;
1440         stmf_data_buf_t *result;
1441         uint32_t        bsize;
1442 
1443         /*
1444          * If the requested size is larger than MaxBurstLength and the
1445          * given pminsize is also larger than MaxBurstLength, then the
1446          * allocation fails (dbuf = NULL) and pminsize is modified to
1447          * be equal to MaxBurstLength. stmf/sbd then should re-invoke
1448          * this function with the corrected values for transfer.
1449          */
1450         ASSERT(pminsize);
1451         if (size <= itask->it_ict->ict_op.op_max_burst_length) {
1452                 bsize = size;
1453         } else if (*pminsize <= itask->it_ict->ict_op.op_max_burst_length) {
1454                 bsize = itask->it_ict->ict_op.op_max_burst_length;
1455         } else {
1456                 *pminsize = itask->it_ict->ict_op.op_max_burst_length;
1457                 return (NULL);
1458         }
1459 
1460         /* Alloc buffer */
1461         idm_buffer = idm_buf_alloc(itask->it_ict->ict_ic, NULL, bsize);
1462         if (idm_buffer != NULL) {
1463                 result = stmf_alloc(STMF_STRUCT_DATA_BUF,
1464                     sizeof (iscsit_buf_t), 0);
1465                 if (result != NULL) {
1466                         /* Fill in stmf_data_buf_t */
1467                         ibuf = result->db_port_private;
1468                         ibuf->ibuf_idm_buf = idm_buffer;
1469                         ibuf->ibuf_stmf_buf = result;
1470                         ibuf->ibuf_is_immed = B_FALSE;
1471                         result->db_flags = DB_DONT_CACHE;
1472                         result->db_buf_size = bsize;
1473                         result->db_data_size = bsize;
1474                         result->db_sglist_length = 1;
1475                         result->db_sglist[0].seg_addr = idm_buffer->idb_buf;
1476                         result->db_sglist[0].seg_length =
1477                             idm_buffer->idb_buflen;
1478                         return (result);
1479                 }
1480 
1481                 /* Couldn't get the stmf_data_buf_t so free the buffer */
1482                 idm_buf_free(idm_buffer);
1483         }
1484 
1485         return (NULL);
1486 }
1487 
1488 /*ARGSUSED*/
1489 static void
1490 iscsit_dbuf_free(stmf_dbuf_store_t *ds, stmf_data_buf_t *dbuf)
1491 {
1492         iscsit_buf_t *ibuf = dbuf->db_port_private;
1493 
1494         if (ibuf->ibuf_is_immed) {
1495                 /*
1496                  * The iscsit_buf_t structure itself will be freed with its
1497                  * associated task.  Here we just need to free the PDU that
1498                  * held the immediate data.
1499                  */
1500                 idm_pdu_complete(ibuf->ibuf_immed_data_pdu, IDM_STATUS_SUCCESS);
1501                 ibuf->ibuf_immed_data_pdu = 0;
1502         } else {
1503                 idm_buf_free(ibuf->ibuf_idm_buf);
1504                 stmf_free(dbuf);
1505         }
1506 }
1507 
1508 /*ARGSUSED*/
1509 stmf_status_t
1510 iscsit_xfer_scsi_data(scsi_task_t *task, stmf_data_buf_t *dbuf,
1511     uint32_t ioflags)
1512 {
1513         iscsit_task_t *iscsit_task = task->task_port_private;
1514         iscsit_sess_t *ict_sess = iscsit_task->it_ict->ict_sess;
1515         iscsit_buf_t *ibuf = dbuf->db_port_private;
1516         int idm_rc;
1517 
1518         /*
1519          * If we are aborting then we can ignore this request
1520          */
1521         if (iscsit_task->it_stmf_abort) {
1522                 return (STMF_SUCCESS);
1523         }
1524 
1525         /*
1526          * If it's not immediate data then start the transfer
1527          */
1528         ASSERT(ibuf->ibuf_is_immed == B_FALSE);
1529         if (dbuf->db_flags & DB_DIRECTION_TO_RPORT) {
1530                 /*
1531                  * The DB_SEND_STATUS_GOOD flag in the STMF data buffer allows
1532                  * the port provider to phase-collapse, i.e. send the status
1533                  * along with the final data PDU for the command. The port
1534                  * provider passes this request to the transport layer by
1535                  * setting a flag IDM_TASK_PHASECOLLAPSE_REQ in the task.
1536                  */
1537                 if (dbuf->db_flags & DB_SEND_STATUS_GOOD)
1538                         iscsit_task->it_idm_task->idt_flags |=
1539                             IDM_TASK_PHASECOLLAPSE_REQ;
1540                 /*
1541                  * IDM will call iscsit_build_hdr so lock now to serialize
1542                  * access to the SN values.  We need to lock here to enforce
1543                  * lock ordering
1544                  */
1545                 mutex_enter(&ict_sess->ist_sn_mutex);
1546                 idm_rc = idm_buf_tx_to_ini(iscsit_task->it_idm_task,
1547                     ibuf->ibuf_idm_buf, dbuf->db_relative_offset,
1548                     dbuf->db_data_size, &iscsit_buf_xfer_cb, dbuf);
1549                 mutex_exit(&ict_sess->ist_sn_mutex);
1550 
1551                 return (iscsit_idm_to_stmf(idm_rc));
1552         } else if (dbuf->db_flags & DB_DIRECTION_FROM_RPORT) {
1553                 /* Grab the SN lock (see comment above) */
1554                 mutex_enter(&ict_sess->ist_sn_mutex);
1555                 idm_rc = idm_buf_rx_from_ini(iscsit_task->it_idm_task,
1556                     ibuf->ibuf_idm_buf, dbuf->db_relative_offset,
1557                     dbuf->db_data_size, &iscsit_buf_xfer_cb, dbuf);
1558                 mutex_exit(&ict_sess->ist_sn_mutex);
1559 
1560                 return (iscsit_idm_to_stmf(idm_rc));
1561         }
1562 
1563         /* What are we supposed to do if there is no direction? */
1564         return (STMF_INVALID_ARG);
1565 }
1566 
1567 static void
1568 iscsit_buf_xfer_cb(idm_buf_t *idb, idm_status_t status)
1569 {
1570         iscsit_task_t *itask = idb->idb_task_binding->idt_private;
1571         stmf_data_buf_t *dbuf = idb->idb_cb_arg;
1572 
1573         dbuf->db_xfer_status = iscsit_idm_to_stmf(status);
1574 
1575         /*
1576          * If the task has been aborted then we don't need to call STMF
1577          */
1578         if (itask->it_stmf_abort) {
1579                 return;
1580         }
1581 
1582         /*
1583          * For ISCSI over TCP (not iSER), the last SCSI Data PDU passed
1584          * for a successful command contains the status as requested by
1585          * by COMSTAR (via the DB_SEND_STATUS_GOOD flag). But the iSER
1586          * transport does not support phase-collapse. So pretend we are
1587          * COMSTAR and send the status in a separate PDU now.
1588          */
1589         if (idb->idb_task_binding->idt_flags & IDM_TASK_PHASECOLLAPSE_SUCCESS) {
1590                 /*
1591                  * Mark task complete and notify COMSTAR
1592                  * that the status has been sent.
1593                  */
1594                 itask->it_idm_task->idt_state = TASK_COMPLETE;
1595                 stmf_send_status_done(itask->it_stmf_task,
1596                     iscsit_idm_to_stmf(status), STMF_IOF_LPORT_DONE);
1597         } else if ((dbuf->db_flags & DB_SEND_STATUS_GOOD) &&
1598             status == IDM_STATUS_SUCCESS) {
1599 
1600                 /*
1601                  * The iscsi target port provider - for iSER, emulates the
1602                  * DB_SEND_STATUS_GOOD optimization if requested by STMF;
1603                  * it sends the status in a separate PDU after the data
1604                  * transfer. In this case the port provider should first
1605                  * call stmf_data_xfer_done() to mark the transfer complete
1606                  * and then send the status. Although STMF will free the
1607                  * buffer at the time the task is freed, even if the transfer
1608                  * is not marked complete, this behavior makes statistics
1609                  * gathering and task state tracking more difficult than it
1610                  * needs to be.
1611                  */
1612                 stmf_data_xfer_done(itask->it_stmf_task, dbuf, 0);
1613                 if (iscsit_send_scsi_status(itask->it_stmf_task, 0)
1614                     != STMF_SUCCESS) {
1615                         stmf_send_status_done(itask->it_stmf_task,
1616                             STMF_FAILURE, STMF_IOF_LPORT_DONE);
1617                 }
1618         } else {
1619                 stmf_data_xfer_done(itask->it_stmf_task, dbuf, 0);
1620                 /* don't touch dbuf after stmf_data_xfer_done */
1621         }
1622 }
1623 
1624 
1625 /*ARGSUSED*/
1626 stmf_status_t
1627 iscsit_send_scsi_status(scsi_task_t *task, uint32_t ioflags)
1628 {
1629         iscsit_task_t *itask = task->task_port_private;
1630         iscsi_scsi_rsp_hdr_t *rsp;
1631         idm_pdu_t *pdu;
1632         int resp_datalen;
1633 
1634         /*
1635          * If this task is aborted then we don't need to respond.
1636          */
1637         if (itask->it_stmf_abort) {
1638                 return (STMF_SUCCESS);
1639         }
1640 
1641         /*
1642          * If this is a task management status, handle it elsewhere.
1643          */
1644         if (task->task_mgmt_function != TM_NONE) {
1645                 /*
1646                  * Don't wait for the PDU completion to tell STMF
1647                  * the task is done -- it doesn't really matter and
1648                  * it makes life complicated if STMF later asks us to
1649                  * abort the request and we don't know whether the
1650                  * status has been sent or not.
1651                  */
1652                 itask->it_tm_responded = B_TRUE;
1653                 iscsit_send_task_mgmt_resp(itask->it_tm_pdu,
1654                     (task->task_completion_status == STMF_SUCCESS) ?
1655                     SCSI_TCP_TM_RESP_COMPLETE : SCSI_TCP_TM_RESP_FUNC_NOT_SUPP);
1656                 stmf_send_status_done(task, STMF_SUCCESS,
1657                     STMF_IOF_LPORT_DONE);
1658                 return (STMF_SUCCESS);
1659         }
1660 
1661         /*
1662          * Remove the task from the session task list
1663          */
1664         iscsit_task_done(itask);
1665 
1666         /*
1667          * Send status
1668          */
1669         mutex_enter(&itask->it_idm_task->idt_mutex);
1670         if ((itask->it_idm_task->idt_state == TASK_ACTIVE) &&
1671             (task->task_completion_status == STMF_SUCCESS) &&
1672             (task->task_sense_length == 0) &&
1673             (task->task_resid == 0)) {
1674                 itask->it_idm_task->idt_state = TASK_COMPLETE;
1675                 /* PDU callback releases task hold */
1676                 idm_task_hold(itask->it_idm_task);
1677                 mutex_exit(&itask->it_idm_task->idt_mutex);
1678                 /*
1679                  * Fast path.  Cached status PDU's are already
1680                  * initialized.  We just need to fill in
1681                  * connection and task information. StatSN is
1682                  * incremented by 1 for every status sent a
1683                  * connection.
1684                  */
1685                 pdu = kmem_cache_alloc(iscsit_status_pdu_cache, KM_SLEEP);
1686                 pdu->isp_ic = itask->it_ict->ict_ic;
1687                 pdu->isp_private = itask;
1688                 pdu->isp_flags |= IDM_PDU_SET_STATSN | IDM_PDU_ADVANCE_STATSN;
1689 
1690                 rsp = (iscsi_scsi_rsp_hdr_t *)pdu->isp_hdr;
1691                 rsp->itt = itask->it_itt;
1692                 /*
1693                  * ExpDataSN is the number of R2T and Data-In (read)
1694                  * PDUs the target has sent for the SCSI command.
1695                  *
1696                  * Since there is no support for bidirectional transfer
1697                  * yet, either idt_exp_datasn or idt_exp_rttsn, but not
1698                  * both is valid at any time
1699                  */
1700                 rsp->expdatasn = (itask->it_idm_task->idt_exp_datasn != 0) ?
1701                     htonl(itask->it_idm_task->idt_exp_datasn):
1702                     htonl(itask->it_idm_task->idt_exp_rttsn);
1703                 rsp->cmd_status = task->task_scsi_status;
1704                 iscsit_pdu_tx(pdu);
1705                 return (STMF_SUCCESS);
1706         } else {
1707                 if (itask->it_idm_task->idt_state != TASK_ACTIVE) {
1708                         mutex_exit(&itask->it_idm_task->idt_mutex);
1709                         return (STMF_FAILURE);
1710                 }
1711                 itask->it_idm_task->idt_state = TASK_COMPLETE;
1712                 /* PDU callback releases task hold */
1713                 idm_task_hold(itask->it_idm_task);
1714                 mutex_exit(&itask->it_idm_task->idt_mutex);
1715 
1716                 resp_datalen = (task->task_sense_length == 0) ? 0 :
1717                     (task->task_sense_length + sizeof (uint16_t));
1718 
1719                 pdu = idm_pdu_alloc(sizeof (iscsi_hdr_t), resp_datalen);
1720                 idm_pdu_init(pdu, itask->it_ict->ict_ic, itask,
1721                     iscsit_send_status_done);
1722                 pdu->isp_flags |= IDM_PDU_SET_STATSN | IDM_PDU_ADVANCE_STATSN;
1723 
1724                 rsp = (iscsi_scsi_rsp_hdr_t *)pdu->isp_hdr;
1725                 bzero(rsp, sizeof (*rsp));
1726                 rsp->opcode = ISCSI_OP_SCSI_RSP;
1727 
1728                 rsp->flags = ISCSI_FLAG_FINAL;
1729                 if (task->task_status_ctrl & TASK_SCTRL_OVER) {
1730                         rsp->flags |= ISCSI_FLAG_CMD_OVERFLOW;
1731                 } else if (task->task_status_ctrl & TASK_SCTRL_UNDER) {
1732                         rsp->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
1733                 }
1734 
1735                 rsp->bi_residual_count = 0;
1736                 rsp->residual_count = htonl(task->task_resid);
1737                 rsp->itt = itask->it_itt;
1738                 rsp->response = ISCSI_STATUS_CMD_COMPLETED;
1739                 rsp->expdatasn = (itask->it_idm_task->idt_exp_datasn != 0) ?
1740                     htonl(itask->it_idm_task->idt_exp_datasn):
1741                     htonl(itask->it_idm_task->idt_exp_rttsn);
1742                 rsp->cmd_status = task->task_scsi_status;
1743                 if (task->task_sense_length != 0) {
1744                         /*
1745                          * Add a byte to provide the sense length in
1746                          * the response
1747                          */
1748                         *(uint16_t *)((void *)pdu->isp_data) =
1749                             htons(task->task_sense_length);
1750                         bcopy(task->task_sense_data,
1751                             (uint8_t *)pdu->isp_data +
1752                             sizeof (uint16_t),
1753                             task->task_sense_length);
1754                         hton24(rsp->dlength, resp_datalen);
1755                 }
1756 
1757                 DTRACE_PROBE5(iscsi__scsi__response,
1758                     iscsit_conn_t *, itask->it_ict,
1759                     uint8_t, rsp->response,
1760                     uint8_t, rsp->cmd_status,
1761                     idm_pdu_t *, pdu,
1762                     scsi_task_t *, task);
1763 
1764                 iscsit_pdu_tx(pdu);
1765 
1766                 return (STMF_SUCCESS);
1767         }
1768 }
1769 
1770 /*ARGSUSED*/
1771 static void
1772 iscsit_send_good_status_done(idm_pdu_t *pdu, idm_status_t status)
1773 {
1774         iscsit_task_t   *itask;
1775         boolean_t       aborted;
1776 
1777         itask = pdu->isp_private;
1778         aborted = itask->it_stmf_abort;
1779 
1780         /*
1781          * After releasing the hold the task may be freed at any time so
1782          * don't touch it.
1783          */
1784         idm_task_rele(itask->it_idm_task);
1785         if (!aborted) {
1786                 stmf_send_status_done(itask->it_stmf_task,
1787                     iscsit_idm_to_stmf(pdu->isp_status), STMF_IOF_LPORT_DONE);
1788         }
1789         kmem_cache_free(iscsit_status_pdu_cache, pdu);
1790 }
1791 
1792 /*ARGSUSED*/
1793 static void
1794 iscsit_send_status_done(idm_pdu_t *pdu, idm_status_t status)
1795 {
1796         iscsit_task_t    *itask;
1797         boolean_t       aborted;
1798 
1799         itask = pdu->isp_private;
1800         aborted = itask->it_stmf_abort;
1801 
1802         /*
1803          * After releasing the hold the task may be freed at any time so
1804          * don't touch it.
1805          */
1806         idm_task_rele(itask->it_idm_task);
1807         if (!aborted) {
1808                 stmf_send_status_done(itask->it_stmf_task,
1809                     iscsit_idm_to_stmf(pdu->isp_status), STMF_IOF_LPORT_DONE);
1810         }
1811         idm_pdu_free(pdu);
1812 }
1813 
1814 
1815 void
1816 iscsit_lport_task_free(scsi_task_t *task)
1817 {
1818         iscsit_task_t *itask = task->task_port_private;
1819 
1820         /* We only call idm_task_start for regular tasks, not task management */
1821         if (task->task_mgmt_function == TM_NONE) {
1822                 idm_task_done(itask->it_idm_task);
1823                 iscsit_task_free(itask);
1824                 return;
1825         } else {
1826                 iscsit_tm_task_free(itask);
1827         }
1828 }
1829 
1830 /*ARGSUSED*/
1831 stmf_status_t
1832 iscsit_abort(stmf_local_port_t *lport, int abort_cmd, void *arg, uint32_t flags)
1833 {
1834         scsi_task_t     *st = (scsi_task_t *)arg;
1835         iscsit_task_t   *iscsit_task;
1836         idm_task_t      *idt;
1837 
1838         /*
1839          * If this is a task management request then there's really not much to
1840          * do.
1841          */
1842         if (st->task_mgmt_function != TM_NONE) {
1843                 return (STMF_ABORT_SUCCESS);
1844         }
1845 
1846         /*
1847          * Regular task, start cleaning up
1848          */
1849         iscsit_task = st->task_port_private;
1850         idt = iscsit_task->it_idm_task;
1851         mutex_enter(&iscsit_task->it_mutex);
1852         iscsit_task->it_stmf_abort = B_TRUE;
1853         if (iscsit_task->it_aborted) {
1854                 mutex_exit(&iscsit_task->it_mutex);
1855                 /*
1856                  * Task is no longer active
1857                  */
1858                 iscsit_task_done(iscsit_task);
1859 
1860                 /*
1861                  * STMF specification is wrong... says to return
1862                  * STMF_ABORTED, the code actually looks for
1863                  * STMF_ABORT_SUCCESS.
1864                  */
1865                 return (STMF_ABORT_SUCCESS);
1866         } else {
1867                 mutex_exit(&iscsit_task->it_mutex);
1868                 /*
1869                  * Call IDM to abort the task.  Due to a variety of
1870                  * circumstances the task may already be in the process of
1871                  * aborting.
1872                  * We'll let IDM worry about rationalizing all that except
1873                  * for one particular instance.  If the state of the task
1874                  * is TASK_COMPLETE, we need to indicate to the framework
1875                  * that we are in fact done.  This typically happens with
1876                  * framework-initiated task management type requests
1877                  * (e.g. abort task).
1878                  */
1879                 if (idt->idt_state == TASK_COMPLETE) {
1880                         idm_refcnt_wait_ref(&idt->idt_refcnt);
1881                         return (STMF_ABORT_SUCCESS);
1882                 } else {
1883                         idm_task_abort(idt->idt_ic, idt, AT_TASK_MGMT_ABORT);
1884                         return (STMF_SUCCESS);
1885                 }
1886         }
1887 
1888         /*NOTREACHED*/
1889 }
1890 
1891 /*ARGSUSED*/
1892 void
1893 iscsit_ctl(stmf_local_port_t *lport, int cmd, void *arg)
1894 {
1895         iscsit_tgt_t            *iscsit_tgt;
1896 
1897         ASSERT((cmd == STMF_CMD_LPORT_ONLINE) ||
1898             (cmd == STMF_ACK_LPORT_ONLINE_COMPLETE) ||
1899             (cmd == STMF_CMD_LPORT_OFFLINE) ||
1900             (cmd == STMF_ACK_LPORT_OFFLINE_COMPLETE));
1901 
1902         iscsit_tgt = (iscsit_tgt_t *)lport->lport_port_private;
1903 
1904         switch (cmd) {
1905         case STMF_CMD_LPORT_ONLINE:
1906                 iscsit_tgt_sm_event(iscsit_tgt, TE_STMF_ONLINE_REQ);
1907                 break;
1908         case STMF_CMD_LPORT_OFFLINE:
1909                 iscsit_tgt_sm_event(iscsit_tgt, TE_STMF_OFFLINE_REQ);
1910                 break;
1911         case STMF_ACK_LPORT_ONLINE_COMPLETE:
1912                 iscsit_tgt_sm_event(iscsit_tgt, TE_STMF_ONLINE_COMPLETE_ACK);
1913                 break;
1914         case STMF_ACK_LPORT_OFFLINE_COMPLETE:
1915                 iscsit_tgt_sm_event(iscsit_tgt, TE_STMF_OFFLINE_COMPLETE_ACK);
1916                 break;
1917 
1918         default:
1919                 break;
1920         }
1921 }
1922 
1923 static stmf_status_t
1924 iscsit_idm_to_stmf(idm_status_t idmrc)
1925 {
1926         switch (idmrc) {
1927         case IDM_STATUS_SUCCESS:
1928                 return (STMF_SUCCESS);
1929         default:
1930                 return (STMF_FAILURE);
1931         }
1932         /*NOTREACHED*/
1933 }
1934 
1935 void
1936 iscsit_op_scsi_cmd(idm_conn_t *ic, idm_pdu_t *rx_pdu)
1937 {
1938         iscsit_conn_t           *ict = ic->ic_handle;
1939 
1940         if (iscsit_check_cmdsn_and_queue(rx_pdu)) {
1941                 iscsit_post_scsi_cmd(ic, rx_pdu);
1942         }
1943         iscsit_process_pdu_in_queue(ict->ict_sess);
1944 }
1945 
1946 /*
1947  * ISCSI protocol
1948  */
1949 
1950 void
1951 iscsit_post_scsi_cmd(idm_conn_t *ic, idm_pdu_t *rx_pdu)
1952 {
1953         iscsit_conn_t           *ict;
1954         iscsit_task_t           *itask;
1955         scsi_task_t             *task;
1956         iscsit_buf_t            *ibuf;
1957         iscsi_scsi_cmd_hdr_t    *iscsi_scsi =
1958             (iscsi_scsi_cmd_hdr_t *)rx_pdu->isp_hdr;
1959         iscsi_addl_hdr_t        *ahs_hdr;
1960         uint16_t                addl_cdb_len = 0;
1961 
1962         ict = ic->ic_handle;
1963 
1964         itask = iscsit_task_alloc(ict);
1965         if (itask == NULL) {
1966                 /* Finish processing request */
1967                 iscsit_set_cmdsn(ict, rx_pdu);
1968 
1969                 iscsit_send_direct_scsi_resp(ict, rx_pdu,
1970                     ISCSI_STATUS_CMD_COMPLETED, STATUS_BUSY);
1971                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
1972                 return;
1973         }
1974 
1975         /*
1976          * Note CmdSN and ITT in task.  IDM will have already validated this
1977          * request against the connection state so we don't need to check
1978          * that (the connection may have changed state in the meantime but
1979          * we will catch that when we try to send a response)
1980          */
1981         itask->it_cmdsn = ntohl(iscsi_scsi->cmdsn);
1982         itask->it_itt = iscsi_scsi->itt;
1983 
1984         /*
1985          * Check for extended CDB AHS
1986          */
1987         if (iscsi_scsi->hlength > 0) {
1988                 ahs_hdr = (iscsi_addl_hdr_t *)iscsi_scsi;
1989                 addl_cdb_len = ((ahs_hdr->ahs_hlen_hi << 8) |
1990                     ahs_hdr->ahs_hlen_lo) - 1; /* Adjust for reserved byte */
1991                 if (((addl_cdb_len + 4) / sizeof (uint32_t)) >
1992                     iscsi_scsi->hlength) {
1993                         /* Mangled header info, drop it */
1994                         idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
1995                         return;
1996                 }
1997         }
1998 
1999         ict = rx_pdu->isp_ic->ic_handle; /* IDM client private */
2000 
2001         /*
2002          * Add task to session list.  This function will also check to
2003          * ensure that the task does not already exist.
2004          */
2005         if (iscsit_task_start(itask) != IDM_STATUS_SUCCESS) {
2006                 /*
2007                  * Task exists, free all resources and reject.  Don't
2008                  * update expcmdsn in this case because RFC 3720 says
2009                  * "The CmdSN of the rejected command PDU (if it is a
2010                  * non-immediate command) MUST NOT be considered received
2011                  * by the target (i.e., a command sequence gap must be
2012                  * assumed for the CmdSN), even though the CmdSN of the
2013                  * rejected command PDU may be reliably ascertained.  Upon
2014                  * receiving the Reject, the initiator MUST plug the CmdSN
2015                  * gap in order to continue to use the session.  The gap
2016                  * may be plugged either by transmitting a command PDU
2017                  * with the same CmdSN, or by aborting the task (see section
2018                  * 6.9 on how an abort may plug a CmdSN gap)." (Section 6.3)
2019                  */
2020                 iscsit_task_free(itask);
2021                 iscsit_send_reject(ict, rx_pdu, ISCSI_REJECT_TASK_IN_PROGRESS);
2022                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2023                 return;
2024         }
2025 
2026         /* Update sequence numbers */
2027         iscsit_set_cmdsn(ict, rx_pdu);
2028 
2029         /*
2030          * Allocate STMF task
2031          */
2032         itask->it_stmf_task = stmf_task_alloc(
2033             itask->it_ict->ict_sess->ist_lport,
2034             itask->it_ict->ict_sess->ist_stmf_sess, iscsi_scsi->lun,
2035             16 + addl_cdb_len, 0);
2036         if (itask->it_stmf_task == NULL) {
2037                 /*
2038                  * Either stmf really couldn't get memory for a task or,
2039                  * more likely, the LU is currently in reset.  Either way
2040                  * we have no choice but to fail the request.
2041                  */
2042                 iscsit_task_done(itask);
2043                 iscsit_task_free(itask);
2044                 iscsit_send_direct_scsi_resp(ict, rx_pdu,
2045                     ISCSI_STATUS_CMD_COMPLETED, STATUS_BUSY);
2046                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2047                 return;
2048         }
2049 
2050         task = itask->it_stmf_task;
2051         task->task_port_private = itask;
2052 
2053         bcopy(iscsi_scsi->lun, task->task_lun_no, sizeof (task->task_lun_no));
2054 
2055         /*
2056          * iSCSI and Comstar use the same values.  Should we rely on this
2057          * or translate them bit-wise?
2058          */
2059 
2060         task->task_flags =
2061             (((iscsi_scsi->flags & ISCSI_FLAG_CMD_READ) ? TF_READ_DATA : 0) |
2062             ((iscsi_scsi->flags & ISCSI_FLAG_CMD_WRITE) ? TF_WRITE_DATA : 0) |
2063             ((rx_pdu->isp_datalen == 0) ? 0 : TF_INITIAL_BURST));
2064 
2065         switch (iscsi_scsi->flags & ISCSI_FLAG_CMD_ATTR_MASK) {
2066         case ISCSI_ATTR_UNTAGGED:
2067                 break;
2068         case ISCSI_ATTR_SIMPLE:
2069                 task->task_additional_flags |= TF_ATTR_SIMPLE_QUEUE;
2070                 break;
2071         case ISCSI_ATTR_ORDERED:
2072                 task->task_additional_flags |= TF_ATTR_ORDERED_QUEUE;
2073                 break;
2074         case ISCSI_ATTR_HEAD_OF_QUEUE:
2075                 task->task_additional_flags |= TF_ATTR_HEAD_OF_QUEUE;
2076                 break;
2077         case ISCSI_ATTR_ACA:
2078                 task->task_additional_flags |= TF_ATTR_ACA;
2079                 break;
2080         default:
2081                 /* Protocol error but just take it, treat as untagged */
2082                 break;
2083         }
2084 
2085 
2086         task->task_additional_flags = 0;
2087         task->task_priority = 0;
2088         task->task_mgmt_function = TM_NONE;
2089 
2090         /*
2091          * This "task_max_nbufs" doesn't map well to BIDI.  We probably need
2092          * parameter for each direction.  "MaxOutstandingR2T" may very well
2093          * be set to one which could prevent us from doing simultaneous
2094          * transfers in each direction.
2095          */
2096         task->task_max_nbufs = (iscsi_scsi->flags & ISCSI_FLAG_CMD_WRITE) ?
2097             ict->ict_op.op_max_outstanding_r2t : STMF_BUFS_MAX;
2098         task->task_cmd_seq_no = ntohl(iscsi_scsi->itt);
2099         task->task_expected_xfer_length = ntohl(iscsi_scsi->data_length);
2100 
2101         /* Copy CDB */
2102         bcopy(iscsi_scsi->scb, task->task_cdb, 16);
2103         if (addl_cdb_len > 0) {
2104                 bcopy(ahs_hdr->ahs_extscb, task->task_cdb + 16, addl_cdb_len);
2105         }
2106 
2107         DTRACE_ISCSI_3(scsi__command, idm_conn_t *, ic,
2108             iscsi_scsi_cmd_hdr_t *, (iscsi_scsi_cmd_hdr_t *)rx_pdu->isp_hdr,
2109             scsi_task_t *, task);
2110 
2111         /*
2112          * Copy the transport header into the task handle from the PDU
2113          * handle. The transport header describes this task's remote tagged
2114          * buffer.
2115          */
2116         if (rx_pdu->isp_transport_hdrlen != 0) {
2117                 bcopy(rx_pdu->isp_transport_hdr,
2118                     itask->it_idm_task->idt_transport_hdr,
2119                     rx_pdu->isp_transport_hdrlen);
2120         }
2121 
2122         /*
2123          * Tell IDM about our new active task
2124          */
2125         idm_task_start(itask->it_idm_task, (uintptr_t)itask->it_itt);
2126 
2127         /*
2128          * If we have any immediate data then setup the immediate buffer
2129          * context that comes with the task
2130          */
2131         if (rx_pdu->isp_datalen) {
2132                 ibuf = itask->it_immed_data;
2133                 ibuf->ibuf_immed_data_pdu = rx_pdu;
2134                 ibuf->ibuf_stmf_buf->db_data_size = rx_pdu->isp_datalen;
2135                 ibuf->ibuf_stmf_buf->db_buf_size = rx_pdu->isp_datalen;
2136                 ibuf->ibuf_stmf_buf->db_relative_offset = 0;
2137                 ibuf->ibuf_stmf_buf->db_sglist[0].seg_length =
2138                     rx_pdu->isp_datalen;
2139                 ibuf->ibuf_stmf_buf->db_sglist[0].seg_addr = rx_pdu->isp_data;
2140 
2141                 DTRACE_ISCSI_8(xfer__start, idm_conn_t *, ic,
2142                     uintptr_t, ibuf->ibuf_stmf_buf->db_sglist[0].seg_addr,
2143                     uint32_t, ibuf->ibuf_stmf_buf->db_relative_offset,
2144                     uint64_t, 0, uint32_t, 0, uint32_t, 0, /* no raddr */
2145                     uint32_t, rx_pdu->isp_datalen, int, XFER_BUF_TX_TO_INI);
2146 
2147                 /*
2148                  * For immediate data transfer, there is no callback from
2149                  * stmf to indicate that the initial burst of data is
2150                  * transferred successfully. In some cases, the task can
2151                  * get freed before execution returns from stmf_post_task.
2152                  * Although this xfer-start/done probe accurately tracks
2153                  * the size of the transfer, it does only provide a best
2154                  * effort on the timing of the transfer.
2155                  */
2156                 DTRACE_ISCSI_8(xfer__done, idm_conn_t *, ic,
2157                     uintptr_t, ibuf->ibuf_stmf_buf->db_sglist[0].seg_addr,
2158                     uint32_t, ibuf->ibuf_stmf_buf->db_relative_offset,
2159                     uint64_t, 0, uint32_t, 0, uint32_t, 0, /* no raddr */
2160                     uint32_t, rx_pdu->isp_datalen, int, XFER_BUF_TX_TO_INI);
2161                 stmf_post_task(task, ibuf->ibuf_stmf_buf);
2162         } else {
2163 
2164                 stmf_post_task(task, NULL);
2165                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2166         }
2167 }
2168 
2169 void
2170 iscsit_deferred_dispatch(idm_pdu_t *rx_pdu)
2171 {
2172         iscsit_conn_t *ict = rx_pdu->isp_ic->ic_handle;
2173 
2174         /*
2175          * If this isn't a login packet, we need a session.  Otherwise
2176          * this is a protocol error (perhaps one IDM should've caught?).
2177          */
2178         if (IDM_PDU_OPCODE(rx_pdu) != ISCSI_OP_LOGIN_CMD &&
2179             ict->ict_sess == NULL) {
2180                 DTRACE_PROBE2(iscsi__idm__deferred__no__session,
2181                     iscsit_conn_t *, ict, idm_pdu_t *, rx_pdu);
2182                 idm_pdu_complete(rx_pdu, IDM_STATUS_FAIL);
2183                 return;
2184         }
2185 
2186         /*
2187          * If the connection has been lost then ignore new PDU's
2188          */
2189         mutex_enter(&ict->ict_mutex);
2190         if (ict->ict_lost) {
2191                 mutex_exit(&ict->ict_mutex);
2192                 idm_pdu_complete(rx_pdu, IDM_STATUS_FAIL);
2193                 return;
2194         }
2195 
2196         /*
2197          * Grab a hold on the connection to prevent it from going away
2198          * between now and when the taskq function is called.
2199          */
2200         iscsit_conn_dispatch_hold(ict);
2201         mutex_exit(&ict->ict_mutex);
2202 
2203         taskq_dispatch_ent(iscsit_global.global_dispatch_taskq,
2204             iscsit_deferred, rx_pdu, 0, &rx_pdu->isp_tqent);
2205 }
2206 
2207 static void
2208 iscsit_deferred(void *rx_pdu_void)
2209 {
2210         idm_pdu_t               *rx_pdu = rx_pdu_void;
2211         idm_conn_t              *ic = rx_pdu->isp_ic;
2212         iscsit_conn_t           *ict = ic->ic_handle;
2213 
2214         /*
2215          * NOP and Task Management Commands can be marked for immediate
2216          * delivery. Commands marked as 'Immediate' are to be considered
2217          * for execution as soon as they arrive on the target. So these
2218          * should not be checked for sequence order and put in a queue.
2219          * The CmdSN is not advanced for Immediate Commands.
2220          */
2221         switch (IDM_PDU_OPCODE(rx_pdu)) {
2222         case ISCSI_OP_NOOP_OUT:
2223                 if (iscsit_check_cmdsn_and_queue(rx_pdu)) {
2224                         iscsit_set_cmdsn(ict, rx_pdu);
2225                         iscsit_pdu_op_noop(ict, rx_pdu);
2226                 }
2227                 break;
2228         case ISCSI_OP_LOGIN_CMD:
2229                 iscsit_pdu_op_login_cmd(ict, rx_pdu);
2230                 iscsit_conn_dispatch_rele(ict);
2231                 return;
2232         case ISCSI_OP_TEXT_CMD:
2233                 if (iscsit_check_cmdsn_and_queue(rx_pdu)) {
2234                         iscsit_set_cmdsn(ict, rx_pdu);
2235                         iscsit_pdu_op_text_cmd(ict, rx_pdu);
2236                 }
2237                 break;
2238         case ISCSI_OP_LOGOUT_CMD:
2239                 if (iscsit_check_cmdsn_and_queue(rx_pdu)) {
2240                         iscsit_set_cmdsn(ict, rx_pdu);
2241                         iscsit_pdu_op_logout_cmd(ict, rx_pdu);
2242                 }
2243                 break;
2244         default:
2245                 /* Protocol error.  IDM should have caught this */
2246                 idm_pdu_complete(rx_pdu, IDM_STATUS_FAIL);
2247                 ASSERT(0);
2248                 break;
2249         }
2250         /*
2251          * Check if there are other PDUs in the session staging queue
2252          * waiting to be posted to SCSI layer.
2253          */
2254         iscsit_process_pdu_in_queue(ict->ict_sess);
2255 
2256         iscsit_conn_dispatch_rele(ict);
2257 }
2258 
2259 static void
2260 iscsit_send_direct_scsi_resp(iscsit_conn_t *ict, idm_pdu_t *rx_pdu,
2261     uint8_t response, uint8_t cmd_status)
2262 {
2263         idm_pdu_t                       *rsp_pdu;
2264         idm_conn_t                      *ic;
2265         iscsi_scsi_rsp_hdr_t            *resp;
2266         iscsi_scsi_cmd_hdr_t            *req =
2267             (iscsi_scsi_cmd_hdr_t *)rx_pdu->isp_hdr;
2268 
2269         ic = ict->ict_ic;
2270 
2271         rsp_pdu = idm_pdu_alloc(sizeof (iscsi_scsi_rsp_hdr_t), 0);
2272         idm_pdu_init(rsp_pdu, ic, NULL, NULL);
2273         /*
2274          * StatSN is incremented by 1 for every response sent on
2275          * a connection except for responses sent as a result of
2276          * a retry or SNACK
2277          */
2278         rsp_pdu->isp_flags |= IDM_PDU_SET_STATSN | IDM_PDU_ADVANCE_STATSN;
2279 
2280         resp = (iscsi_scsi_rsp_hdr_t *)rsp_pdu->isp_hdr;
2281 
2282         resp->opcode = ISCSI_OP_SCSI_RSP;
2283         resp->flags = ISCSI_FLAG_FINAL;
2284         resp->response = response;
2285         resp->cmd_status = cmd_status;
2286         resp->itt = req->itt;
2287         if ((response == ISCSI_STATUS_CMD_COMPLETED) &&
2288             (req->data_length != 0) &&
2289             ((req->flags & ISCSI_FLAG_CMD_READ) ||
2290             (req->flags & ISCSI_FLAG_CMD_WRITE))) {
2291                 resp->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
2292                 resp->residual_count = req->data_length;
2293         }
2294 
2295         DTRACE_PROBE4(iscsi__scsi__direct__response,
2296             iscsit_conn_t *, ict,
2297             uint8_t, resp->response,
2298             uint8_t, resp->cmd_status,
2299             idm_pdu_t *, rsp_pdu);
2300 
2301         iscsit_pdu_tx(rsp_pdu);
2302 }
2303 
2304 void
2305 iscsit_send_task_mgmt_resp(idm_pdu_t *tm_resp_pdu, uint8_t tm_status)
2306 {
2307         iscsi_scsi_task_mgt_rsp_hdr_t   *tm_resp;
2308 
2309         /*
2310          * The target must take note of the last-sent StatSN.
2311          * The StatSN is to be incremented after sending a
2312          * task management response. Digest recovery can only
2313          * work if StatSN is incremented.
2314          */
2315         tm_resp_pdu->isp_flags |= IDM_PDU_SET_STATSN | IDM_PDU_ADVANCE_STATSN;
2316         tm_resp = (iscsi_scsi_task_mgt_rsp_hdr_t *)tm_resp_pdu->isp_hdr;
2317         tm_resp->response = tm_status;
2318 
2319         DTRACE_PROBE3(iscsi__scsi__tm__response,
2320             iscsit_conn_t *, tm_resp_pdu->isp_ic->ic_handle,
2321             uint8_t, tm_resp->response,
2322             idm_pdu_t *, tm_resp_pdu);
2323         iscsit_pdu_tx(tm_resp_pdu);
2324 }
2325 
2326 void
2327 iscsit_op_scsi_task_mgmt(iscsit_conn_t *ict, idm_pdu_t *rx_pdu)
2328 {
2329         idm_pdu_t                       *tm_resp_pdu;
2330         iscsit_task_t                   *itask;
2331         iscsit_task_t                   *tm_itask;
2332         scsi_task_t                     *task;
2333         iscsi_scsi_task_mgt_hdr_t       *iscsi_tm =
2334             (iscsi_scsi_task_mgt_hdr_t *)rx_pdu->isp_hdr;
2335         iscsi_scsi_task_mgt_rsp_hdr_t   *iscsi_tm_rsp =
2336             (iscsi_scsi_task_mgt_rsp_hdr_t *)rx_pdu->isp_hdr;
2337         uint32_t                        rtt, cmdsn, refcmdsn;
2338         uint8_t                         tm_func;
2339 
2340         /*
2341          * Setup response PDU (response field will get filled in later)
2342          */
2343         tm_resp_pdu = idm_pdu_alloc(sizeof (iscsi_scsi_task_mgt_rsp_hdr_t), 0);
2344         if (tm_resp_pdu == NULL) {
2345                 /* Can't respond, just drop it */
2346                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2347                 return;
2348         }
2349         idm_pdu_init(tm_resp_pdu, ict->ict_ic, NULL, NULL);
2350         iscsi_tm_rsp = (iscsi_scsi_task_mgt_rsp_hdr_t *)tm_resp_pdu->isp_hdr;
2351         bzero(iscsi_tm_rsp, sizeof (iscsi_scsi_task_mgt_rsp_hdr_t));
2352         iscsi_tm_rsp->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
2353         iscsi_tm_rsp->flags = ISCSI_FLAG_FINAL;
2354         iscsi_tm_rsp->itt = rx_pdu->isp_hdr->itt;
2355 
2356         /*
2357          * Figure out what we're being asked to do.
2358          */
2359         DTRACE_PROBE4(iscsi__scsi__tm__request,
2360             iscsit_conn_t *, ict,
2361             uint8_t, (iscsi_tm->function & ISCSI_FLAG_TASK_MGMT_FUNCTION_MASK),
2362             uint32_t, iscsi_tm->rtt,
2363             idm_pdu_t *, rx_pdu);
2364         switch (iscsi_tm->function & ISCSI_FLAG_TASK_MGMT_FUNCTION_MASK) {
2365         case ISCSI_TM_FUNC_ABORT_TASK:
2366                 /*
2367                  * STMF doesn't currently support the "abort task" task
2368                  * management command although it does support aborting
2369                  * an individual task.  We'll get STMF to abort the task
2370                  * for us but handle the details of the task management
2371                  * command ourselves.
2372                  *
2373                  * Find the task associated with the referenced task tag.
2374                  */
2375                 rtt = iscsi_tm->rtt;
2376                 itask = (iscsit_task_t *)idm_task_find_by_handle(ict->ict_ic,
2377                     (uintptr_t)rtt);
2378 
2379                 if (itask == NULL) {
2380                         cmdsn = ntohl(iscsi_tm->cmdsn);
2381                         refcmdsn = ntohl(iscsi_tm->refcmdsn);
2382 
2383                         /*
2384                          * Task was not found. But the SCSI command could be
2385                          * on the rxpdu wait queue. If RefCmdSN is within
2386                          * the CmdSN window and less than CmdSN of the TM
2387                          * function, return "Function Complete". Otherwise,
2388                          * return "Task Does Not Exist".
2389                          */
2390 
2391                         if (iscsit_cmdsn_in_window(ict, refcmdsn) &&
2392                             iscsit_sna_lt(refcmdsn, cmdsn)) {
2393                                 mutex_enter(&ict->ict_sess->ist_sn_mutex);
2394                                 (void) iscsit_remove_pdu_from_queue(
2395                                     ict->ict_sess, refcmdsn);
2396                                 iscsit_conn_dispatch_rele(ict);
2397                                 mutex_exit(&ict->ict_sess->ist_sn_mutex);
2398                                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2399                                     SCSI_TCP_TM_RESP_COMPLETE);
2400                         } else {
2401                                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2402                                     SCSI_TCP_TM_RESP_NO_TASK);
2403                         }
2404                 } else {
2405 
2406                         /*
2407                          * Tell STMF to abort the task.  This will do no harm
2408                          * if the task is already complete.
2409                          */
2410                         stmf_abort(STMF_QUEUE_TASK_ABORT, itask->it_stmf_task,
2411                             STMF_ABORTED, NULL);
2412 
2413                         /*
2414                          * Make sure the task hasn't already completed
2415                          */
2416                         mutex_enter(&itask->it_idm_task->idt_mutex);
2417                         if ((itask->it_idm_task->idt_state == TASK_COMPLETE) ||
2418                             (itask->it_idm_task->idt_state == TASK_IDLE)) {
2419                                 /*
2420                                  * Task is complete, return "Task Does Not
2421                                  * Exist"
2422                                  */
2423                                 mutex_exit(&itask->it_idm_task->idt_mutex);
2424                                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2425                                     SCSI_TCP_TM_RESP_NO_TASK);
2426                         } else {
2427                                 /*
2428                                  * STMF is now aborting the task, return
2429                                  * "Function Complete"
2430                                  */
2431                                 mutex_exit(&itask->it_idm_task->idt_mutex);
2432                                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2433                                     SCSI_TCP_TM_RESP_COMPLETE);
2434                         }
2435                         idm_task_rele(itask->it_idm_task);
2436                 }
2437                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2438                 return;
2439 
2440         case ISCSI_TM_FUNC_ABORT_TASK_SET:
2441                 tm_func = TM_ABORT_TASK_SET;
2442                 break;
2443 
2444         case ISCSI_TM_FUNC_CLEAR_ACA:
2445                 tm_func = TM_CLEAR_ACA;
2446                 break;
2447 
2448         case ISCSI_TM_FUNC_CLEAR_TASK_SET:
2449                 tm_func = TM_CLEAR_TASK_SET;
2450                 break;
2451 
2452         case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
2453                 tm_func = TM_LUN_RESET;
2454                 break;
2455 
2456         case ISCSI_TM_FUNC_TARGET_WARM_RESET:
2457                 tm_func = TM_TARGET_WARM_RESET;
2458                 break;
2459 
2460         case ISCSI_TM_FUNC_TARGET_COLD_RESET:
2461                 tm_func = TM_TARGET_COLD_RESET;
2462                 break;
2463 
2464         case ISCSI_TM_FUNC_TASK_REASSIGN:
2465                 /*
2466                  * We do not currently support allegiance reassignment.  When
2467                  * we start supporting ERL1+, we will need to.
2468                  */
2469                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2470                     SCSI_TCP_TM_RESP_NO_ALLG_REASSN);
2471                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2472                 return;
2473 
2474         default:
2475                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2476                     SCSI_TCP_TM_RESP_REJECTED);
2477                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2478                 return;
2479         }
2480 
2481         tm_itask = iscsit_tm_task_alloc(ict);
2482         if (tm_itask == NULL) {
2483                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2484                     SCSI_TCP_TM_RESP_REJECTED);
2485                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2486                 return;
2487         }
2488 
2489 
2490         task = stmf_task_alloc(ict->ict_sess->ist_lport,
2491             ict->ict_sess->ist_stmf_sess, iscsi_tm->lun,
2492             0, STMF_TASK_EXT_NONE);
2493         if (task == NULL) {
2494                 /*
2495                  * If this happens, either the LU is in reset, couldn't
2496                  * get memory, or some other condition in which we simply
2497                  * can't complete this request.  It would be nice to return
2498                  * an error code like "busy" but the closest we have is
2499                  * "rejected".
2500                  */
2501                 iscsit_send_task_mgmt_resp(tm_resp_pdu,
2502                     SCSI_TCP_TM_RESP_REJECTED);
2503                 iscsit_tm_task_free(tm_itask);
2504                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2505                 return;
2506         }
2507 
2508         tm_itask->it_tm_pdu = tm_resp_pdu;
2509         tm_itask->it_stmf_task = task;
2510         task->task_port_private = tm_itask;
2511         task->task_mgmt_function = tm_func;
2512         task->task_additional_flags = TASK_AF_NO_EXPECTED_XFER_LENGTH;
2513         task->task_priority = 0;
2514         task->task_max_nbufs = STMF_BUFS_MAX;
2515         task->task_cmd_seq_no = iscsi_tm->itt;
2516         task->task_expected_xfer_length = 0;
2517 
2518         stmf_post_task(task, NULL);
2519         idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2520 }
2521 
2522 static void
2523 iscsit_pdu_op_noop(iscsit_conn_t *ict, idm_pdu_t *rx_pdu)
2524 {
2525         iscsi_nop_out_hdr_t *out = (iscsi_nop_out_hdr_t *)rx_pdu->isp_hdr;
2526         iscsi_nop_in_hdr_t *in;
2527         int resp_datalen;
2528         idm_pdu_t *resp;
2529 
2530         /* Ignore the response from initiator */
2531         if ((out->itt == ISCSI_RSVD_TASK_TAG) ||
2532             (out->ttt != ISCSI_RSVD_TASK_TAG)) {
2533                 idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2534                 return;
2535         }
2536 
2537         /* Allocate a PDU to respond */
2538         resp_datalen = ntoh24(out->dlength);
2539         resp = idm_pdu_alloc(sizeof (iscsi_hdr_t), resp_datalen);
2540         idm_pdu_init(resp, ict->ict_ic, NULL, NULL);
2541         if (resp_datalen > 0) {
2542                 bcopy(rx_pdu->isp_data, resp->isp_data, resp_datalen);
2543         }
2544 
2545         /*
2546          * When sending a NOP-In as a response to a NOP-Out from the initiator,
2547          * the target must respond with the same initiator task tag that was
2548          * provided in the NOP-Out request, the target transfer tag must be
2549          * ISCSI_RSVD_TASK_TAG (0xffffffff) and StatSN will contain the next
2550          * status sequence number. The StatSN for the connection is advanced
2551          * after this PDU is sent.
2552          */
2553         in = (iscsi_nop_in_hdr_t *)resp->isp_hdr;
2554         bzero(in, sizeof (*in));
2555         in->opcode = ISCSI_OP_NOOP_IN;
2556         in->flags = ISCSI_FLAG_FINAL;
2557         bcopy(out->lun, in->lun, 8);
2558         in->itt              = out->itt;
2559         in->ttt              = ISCSI_RSVD_TASK_TAG;
2560         hton24(in->dlength, resp_datalen);
2561         resp->isp_flags |= IDM_PDU_SET_STATSN | IDM_PDU_ADVANCE_STATSN;
2562         /* Any other field in resp to be set? */
2563         iscsit_pdu_tx(resp);
2564         idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2565 }
2566 
2567 static void
2568 iscsit_pdu_op_login_cmd(iscsit_conn_t   *ict, idm_pdu_t *rx_pdu)
2569 {
2570 
2571         /*
2572          * Submit PDU to login state machine.  State machine will free the
2573          * PDU.
2574          */
2575         iscsit_login_sm_event(ict, ILE_LOGIN_RCV, rx_pdu);
2576 }
2577 
2578 void
2579 iscsit_pdu_op_logout_cmd(iscsit_conn_t  *ict, idm_pdu_t *rx_pdu)
2580 {
2581         iscsi_logout_hdr_t      *logout_req =
2582             (iscsi_logout_hdr_t *)rx_pdu->isp_hdr;
2583         iscsi_logout_rsp_hdr_t  *logout_rsp;
2584         idm_pdu_t *resp;
2585 
2586         /* Allocate a PDU to respond */
2587         resp = idm_pdu_alloc(sizeof (iscsi_hdr_t), 0);
2588         idm_pdu_init(resp, ict->ict_ic, NULL, NULL);
2589         /*
2590          * The StatSN is to be sent to the initiator,
2591          * it is not required to increment the number
2592          * as the connection is terminating.
2593          */
2594         resp->isp_flags |= IDM_PDU_SET_STATSN;
2595         /*
2596          * Logout results in the immediate termination of all tasks except
2597          * if the logout reason is ISCSI_LOGOUT_REASON_RECOVERY.  The
2598          * connection state machine will drive this task cleanup automatically
2599          * so we don't need to handle that here.
2600          */
2601         logout_rsp = (iscsi_logout_rsp_hdr_t *)resp->isp_hdr;
2602         bzero(logout_rsp, sizeof (*logout_rsp));
2603         logout_rsp->opcode = ISCSI_OP_LOGOUT_RSP;
2604         logout_rsp->flags = ISCSI_FLAG_FINAL;
2605         logout_rsp->itt = logout_req->itt;
2606         if ((logout_req->flags & ISCSI_FLAG_LOGOUT_REASON_MASK) >
2607             ISCSI_LOGOUT_REASON_RECOVERY) {
2608                 logout_rsp->response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED;
2609         } else {
2610                 logout_rsp->response = ISCSI_LOGOUT_SUCCESS;
2611         }
2612 
2613         iscsit_pdu_tx(resp);
2614         idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
2615 }
2616 
2617 /*
2618  * Calculate the number of outstanding commands we can process
2619  */
2620 int
2621 iscsit_cmd_window()
2622 {
2623         /*
2624          * Instead of using a pre-defined constant for the command window,
2625          * it should be made confiurable and dynamic. With MC/S, sequence
2626          * numbers will be used up at a much faster rate than with SC/S.
2627          */
2628         return  (ISCSIT_MAX_WINDOW);
2629 }
2630 
2631 /*
2632  * Set local registers based on incoming PDU
2633  */
2634 void
2635 iscsit_set_cmdsn(iscsit_conn_t *ict, idm_pdu_t *rx_pdu)
2636 {
2637         iscsit_sess_t *ist;
2638         iscsi_scsi_cmd_hdr_t *req;
2639 
2640         ist = ict->ict_sess;
2641 
2642         req = (iscsi_scsi_cmd_hdr_t *)rx_pdu->isp_hdr;
2643         if (req->opcode & ISCSI_OP_IMMEDIATE) {
2644                 /* no cmdsn increment for immediate PDUs */
2645                 return;
2646         }
2647 
2648         /* Ensure that the ExpCmdSN advances in an orderly manner */
2649         mutex_enter(&ist->ist_sn_mutex);
2650         ist->ist_expcmdsn = ntohl(req->cmdsn) + 1;
2651         ist->ist_maxcmdsn = ntohl(req->cmdsn) + iscsit_cmd_window();
2652         mutex_exit(&ist->ist_sn_mutex);
2653 }
2654 
2655 /*
2656  * Wrapper funtion, calls iscsi_calc_rspsn and idm_pdu_tx
2657  */
2658 void
2659 iscsit_pdu_tx(idm_pdu_t *pdu)
2660 {
2661         iscsit_conn_t *ict = pdu->isp_ic->ic_handle;
2662         iscsi_scsi_rsp_hdr_t *rsp = (iscsi_scsi_rsp_hdr_t *)pdu->isp_hdr;
2663         iscsit_sess_t *ist = ict->ict_sess;
2664 
2665         /*
2666          * The command sequence numbers are session-wide and must stay
2667          * consistent across the transfer, so protect the cmdsn with a
2668          * mutex lock on the session. The status sequence number will
2669          * be updated just before the transport layer transmits the PDU.
2670          */
2671 
2672         mutex_enter(&ict->ict_sess->ist_sn_mutex);
2673         /* Set ExpCmdSN and MaxCmdSN */
2674         rsp->maxcmdsn = htonl(ist->ist_maxcmdsn);
2675         rsp->expcmdsn = htonl(ist->ist_expcmdsn);
2676         idm_pdu_tx(pdu);
2677         mutex_exit(&ict->ict_sess->ist_sn_mutex);
2678 }
2679 
2680 /*
2681  * Internal functions
2682  */
2683 
2684 void
2685 iscsit_send_async_event(iscsit_conn_t *ict, uint8_t event)
2686 {
2687         idm_pdu_t               *abt;
2688         iscsi_async_evt_hdr_t   *async_abt;
2689 
2690         /*
2691          * Get a PDU to build the abort request.
2692          */
2693         abt = idm_pdu_alloc(sizeof (iscsi_hdr_t), 0);
2694         if (abt == NULL) {
2695                 idm_conn_event(ict->ict_ic, CE_TRANSPORT_FAIL, NULL);
2696                 return;
2697         }
2698 
2699         /*
2700          * A asynchronous message is sent by the target to request a logout.
2701          * The StatSN for the connection is advanced after the PDU is sent
2702          * to allow for initiator and target state synchronization.
2703          */
2704         idm_pdu_init(abt, ict->ict_ic, NULL, NULL);
2705         abt->isp_datalen = 0;
2706         abt->isp_flags |= IDM_PDU_SET_STATSN | IDM_PDU_ADVANCE_STATSN;
2707 
2708         async_abt = (iscsi_async_evt_hdr_t *)abt->isp_hdr;
2709         bzero(async_abt, sizeof (*async_abt));
2710         async_abt->opcode = ISCSI_OP_ASYNC_EVENT;
2711         async_abt->async_event = event;
2712         async_abt->flags = ISCSI_FLAG_FINAL;
2713         async_abt->rsvd4[0] = 0xff;
2714         async_abt->rsvd4[1] = 0xff;
2715         async_abt->rsvd4[2] = 0xff;
2716         async_abt->rsvd4[3] = 0xff;
2717 
2718         switch (event) {
2719         case ISCSI_ASYNC_EVENT_REQUEST_LOGOUT:
2720                 async_abt->param3 = htons(IDM_LOGOUT_SECONDS);
2721                 break;
2722         case ISCSI_ASYNC_EVENT_SCSI_EVENT:
2723         case ISCSI_ASYNC_EVENT_DROPPING_CONNECTION:
2724         case ISCSI_ASYNC_EVENT_DROPPING_ALL_CONNECTIONS:
2725         case ISCSI_ASYNC_EVENT_PARAM_NEGOTIATION:
2726         default:
2727                 ASSERT(0);
2728         }
2729 
2730         iscsit_pdu_tx(abt);
2731 }
2732 
2733 void
2734 iscsit_send_reject(iscsit_conn_t *ict, idm_pdu_t *rejected_pdu, uint8_t reason)
2735 {
2736         idm_pdu_t               *reject_pdu;
2737         iscsi_reject_rsp_hdr_t  *reject;
2738 
2739         /*
2740          * Get a PDU to build the abort request.
2741          */
2742         reject_pdu = idm_pdu_alloc(sizeof (iscsi_hdr_t),
2743             rejected_pdu->isp_hdrlen);
2744         if (reject_pdu == NULL) {
2745                 idm_conn_event(ict->ict_ic, CE_TRANSPORT_FAIL, NULL);
2746                 return;
2747         }
2748         idm_pdu_init(reject_pdu, ict->ict_ic, NULL, NULL);
2749         /* StatSN is advanced after a Reject PDU */
2750         reject_pdu->isp_flags |= IDM_PDU_SET_STATSN | IDM_PDU_ADVANCE_STATSN;
2751         reject_pdu->isp_datalen = rejected_pdu->isp_hdrlen;
2752         bcopy(rejected_pdu->isp_hdr, reject_pdu->isp_data,
2753             rejected_pdu->isp_hdrlen);
2754 
2755         reject = (iscsi_reject_rsp_hdr_t *)reject_pdu->isp_hdr;
2756         bzero(reject, sizeof (*reject));
2757         reject->opcode = ISCSI_OP_REJECT_MSG;
2758         reject->reason = reason;
2759         reject->flags = ISCSI_FLAG_FINAL;
2760         hton24(reject->dlength, rejected_pdu->isp_hdrlen);
2761         reject->must_be_ff[0] = 0xff;
2762         reject->must_be_ff[1] = 0xff;
2763         reject->must_be_ff[2] = 0xff;
2764         reject->must_be_ff[3] = 0xff;
2765 
2766         iscsit_pdu_tx(reject_pdu);
2767 }
2768 
2769 
2770 static iscsit_task_t *
2771 iscsit_task_alloc(iscsit_conn_t *ict)
2772 {
2773         iscsit_task_t *itask;
2774         iscsit_buf_t *immed_ibuf;
2775 
2776         /*
2777          * Possible items to pre-alloc if we cache iscsit_task_t's:
2778          *
2779          * Status PDU w/ sense buffer
2780          * stmf_data_buf_t for immediate data
2781          */
2782         itask = kmem_alloc(sizeof (iscsit_task_t) + sizeof (iscsit_buf_t) +
2783             sizeof (stmf_data_buf_t), KM_NOSLEEP);
2784         if (itask != NULL) {
2785                 mutex_init(&itask->it_mutex, NULL, MUTEX_DRIVER, NULL);
2786                 itask->it_aborted = itask->it_stmf_abort =
2787                     itask->it_tm_task = 0;
2788 
2789                 immed_ibuf = (iscsit_buf_t *)(itask + 1);
2790                 bzero(immed_ibuf, sizeof (*immed_ibuf));
2791                 immed_ibuf->ibuf_is_immed = B_TRUE;
2792                 immed_ibuf->ibuf_stmf_buf = (stmf_data_buf_t *)(immed_ibuf + 1);
2793 
2794                 bzero(immed_ibuf->ibuf_stmf_buf, sizeof (stmf_data_buf_t));
2795                 immed_ibuf->ibuf_stmf_buf->db_port_private = immed_ibuf;
2796                 immed_ibuf->ibuf_stmf_buf->db_sglist_length = 1;
2797                 immed_ibuf->ibuf_stmf_buf->db_flags = DB_DIRECTION_FROM_RPORT |
2798                     DB_DONT_CACHE;
2799                 itask->it_immed_data = immed_ibuf;
2800                 itask->it_idm_task = idm_task_alloc(ict->ict_ic);
2801                 if (itask->it_idm_task != NULL) {
2802                         itask->it_idm_task->idt_private = itask;
2803                         itask->it_ict = ict;
2804                         itask->it_ttt = itask->it_idm_task->idt_tt;
2805                         return (itask);
2806                 } else {
2807                         kmem_free(itask, sizeof (iscsit_task_t) +
2808                             sizeof (iscsit_buf_t) + sizeof (stmf_data_buf_t));
2809                 }
2810         }
2811 
2812         return (NULL);
2813 }
2814 
2815 static void
2816 iscsit_task_free(iscsit_task_t *itask)
2817 {
2818         idm_task_free(itask->it_idm_task);
2819         mutex_destroy(&itask->it_mutex);
2820         kmem_free(itask, sizeof (iscsit_task_t) +
2821             sizeof (iscsit_buf_t) + sizeof (stmf_data_buf_t));
2822 }
2823 
2824 static iscsit_task_t *
2825 iscsit_tm_task_alloc(iscsit_conn_t *ict)
2826 {
2827         iscsit_task_t *itask;
2828 
2829         itask = kmem_zalloc(sizeof (iscsit_task_t), KM_NOSLEEP);
2830         if (itask != NULL) {
2831                 idm_conn_hold(ict->ict_ic);
2832                 mutex_init(&itask->it_mutex, NULL, MUTEX_DRIVER, NULL);
2833                 itask->it_aborted = itask->it_stmf_abort =
2834                     itask->it_tm_responded = 0;
2835                 itask->it_tm_pdu = NULL;
2836                 itask->it_tm_task = 1;
2837                 itask->it_ict = ict;
2838         }
2839 
2840         return (itask);
2841 }
2842 
2843 static void
2844 iscsit_tm_task_free(iscsit_task_t *itask)
2845 {
2846         /*
2847          * If we responded then the call to idm_pdu_complete will free the
2848          * PDU.  Otherwise we got aborted before the TM function could
2849          * complete and we need to free the PDU explicitly.
2850          */
2851         if (itask->it_tm_pdu != NULL && !itask->it_tm_responded)
2852                 idm_pdu_free(itask->it_tm_pdu);
2853         idm_conn_rele(itask->it_ict->ict_ic);
2854         mutex_destroy(&itask->it_mutex);
2855         kmem_free(itask, sizeof (iscsit_task_t));
2856 }
2857 
2858 static idm_status_t
2859 iscsit_task_start(iscsit_task_t *itask)
2860 {
2861         iscsit_sess_t *ist = itask->it_ict->ict_sess;
2862         avl_index_t             where;
2863 
2864         /*
2865          * Sanity check the ITT and ensure that this task does not already
2866          * exist.  If not then add the task to the session task list.
2867          */
2868         mutex_enter(&ist->ist_mutex);
2869         mutex_enter(&itask->it_mutex);
2870         itask->it_active = 1;
2871         if (avl_find(&ist->ist_task_list, itask, &where) == NULL) {
2872                 /* New task, add to AVL */
2873                 avl_insert(&ist->ist_task_list, itask, where);
2874                 mutex_exit(&itask->it_mutex);
2875                 mutex_exit(&ist->ist_mutex);
2876                 return (IDM_STATUS_SUCCESS);
2877         }
2878         mutex_exit(&itask->it_mutex);
2879         mutex_exit(&ist->ist_mutex);
2880 
2881         return (IDM_STATUS_REJECT);
2882 }
2883 
2884 static void
2885 iscsit_task_done(iscsit_task_t *itask)
2886 {
2887         iscsit_sess_t *ist = itask->it_ict->ict_sess;
2888 
2889         mutex_enter(&ist->ist_mutex);
2890         mutex_enter(&itask->it_mutex);
2891         if (itask->it_active) {
2892                 avl_remove(&ist->ist_task_list, itask);
2893                 itask->it_active = 0;
2894         }
2895         mutex_exit(&itask->it_mutex);
2896         mutex_exit(&ist->ist_mutex);
2897 }
2898 
2899 /*
2900  * iscsit status PDU cache
2901  */
2902 
2903 /*ARGSUSED*/
2904 static int
2905 iscsit_status_pdu_constructor(void *pdu_void, void *arg, int flags)
2906 {
2907         idm_pdu_t *pdu = pdu_void;
2908         iscsi_scsi_rsp_hdr_t *rsp;
2909 
2910         bzero(pdu, sizeof (idm_pdu_t));
2911         pdu->isp_callback = iscsit_send_good_status_done;
2912         pdu->isp_magic = IDM_PDU_MAGIC;
2913         pdu->isp_hdr = (iscsi_hdr_t *)(pdu + 1); /* Ptr arithmetic */
2914         pdu->isp_hdrlen = sizeof (iscsi_hdr_t);
2915 
2916         /* Setup status response */
2917         rsp = (iscsi_scsi_rsp_hdr_t *)pdu->isp_hdr;
2918         bzero(rsp, sizeof (*rsp));
2919         rsp->opcode = ISCSI_OP_SCSI_RSP;
2920         rsp->flags = ISCSI_FLAG_FINAL;
2921         rsp->response = ISCSI_STATUS_CMD_COMPLETED;
2922 
2923         return (0);
2924 }
2925 
2926 /*
2927  * iscsit private data handler
2928  */
2929 
2930 /*ARGSUSED*/
2931 static void
2932 iscsit_pp_cb(struct stmf_port_provider *pp, int cmd, void *arg, uint32_t flags)
2933 {
2934         it_config_t             *cfg;
2935         nvlist_t                *nvl;
2936         iscsit_service_enabled_t        old_state;
2937 
2938         if ((cmd != STMF_PROVIDER_DATA_UPDATED) || (arg == NULL)) {
2939                 return;
2940         }
2941 
2942         nvl = (nvlist_t *)arg;
2943 
2944         /* Translate nvlist */
2945         if (it_nv_to_config(nvl, &cfg) != 0) {
2946                 cmn_err(CE_WARN, "Configuration is invalid");
2947                 return;
2948         }
2949 
2950         /* Check that no iSCSI ioctl is currently running */
2951         mutex_enter(&iscsit_global.global_state_mutex);
2952         old_state = iscsit_global.global_svc_state;
2953         switch (iscsit_global.global_svc_state) {
2954         case ISE_ENABLED:
2955         case ISE_DISABLED:
2956                 iscsit_global.global_svc_state = ISE_BUSY;
2957                 break;
2958         case ISE_ENABLING:
2959                 /*
2960                  * It is OK for the iscsit_pp_cb to be called from inside of
2961                  * an iSCSI ioctl only if we are currently executing inside
2962                  * of stmf_register_port_provider.
2963                  */
2964                 ASSERT((flags & STMF_PCB_PREG_COMPLETE) != 0);
2965                 break;
2966         default:
2967                 cmn_err(CE_WARN, "iscsit_pp_cb called when global_svc_state"
2968                     " is not ENABLED(0x%x) -- ignoring",
2969                     iscsit_global.global_svc_state);
2970                 mutex_exit(&iscsit_global.global_state_mutex);
2971                 it_config_free_cmn(cfg);
2972                 return;
2973         }
2974         mutex_exit(&iscsit_global.global_state_mutex);
2975 
2976         /* Update config */
2977         (void) iscsit_config_merge(cfg);
2978 
2979         it_config_free_cmn(cfg);
2980 
2981         /* Restore old iSCSI driver global state */
2982         mutex_enter(&iscsit_global.global_state_mutex);
2983         ASSERT(iscsit_global.global_svc_state == ISE_BUSY ||
2984             iscsit_global.global_svc_state == ISE_ENABLING);
2985         iscsit_global.global_svc_state = old_state;
2986         mutex_exit(&iscsit_global.global_state_mutex);
2987 }
2988 
2989 
2990 static it_cfg_status_t
2991 iscsit_config_merge(it_config_t *in_cfg)
2992 {
2993         it_cfg_status_t status;
2994         it_config_t     *cfg;
2995         it_config_t     tmp_cfg;
2996         list_t          tpg_del_list;
2997 
2998         if (in_cfg) {
2999                 cfg = in_cfg;
3000         } else {
3001                 /* Make empty config */
3002                 bzero(&tmp_cfg, sizeof (tmp_cfg));
3003                 cfg = &tmp_cfg;
3004         }
3005 
3006         list_create(&tpg_del_list,  sizeof (iscsit_tpg_t),
3007             offsetof(iscsit_tpg_t, tpg_delete_ln));
3008 
3009         /*
3010          * Update targets, initiator contexts, target portal groups,
3011          * and iSNS client
3012          */
3013         ISCSIT_GLOBAL_LOCK(RW_WRITER);
3014         if (((status = iscsit_config_merge_tpg(cfg, &tpg_del_list))
3015             != 0) ||
3016             ((status = iscsit_config_merge_tgt(cfg)) != 0) ||
3017             ((status = iscsit_config_merge_ini(cfg)) != 0) ||
3018             ((status = isnst_config_merge(cfg)) != 0)) {
3019                 ISCSIT_GLOBAL_UNLOCK();
3020                 return (status);
3021         }
3022 
3023         /* Update other global config parameters */
3024         if (iscsit_global.global_props) {
3025                 nvlist_free(iscsit_global.global_props);
3026                 iscsit_global.global_props = NULL;
3027         }
3028         if (in_cfg) {
3029                 (void) nvlist_dup(cfg->config_global_properties,
3030                     &iscsit_global.global_props, KM_SLEEP);
3031         }
3032         ISCSIT_GLOBAL_UNLOCK();
3033 
3034         iscsit_config_destroy_tpgs(&tpg_del_list);
3035 
3036         list_destroy(&tpg_del_list);
3037 
3038         return (ITCFG_SUCCESS);
3039 }
3040 
3041 /*
3042  * iscsit_sna_lt[e]
3043  *
3044  * Compare serial numbers using serial number arithmetic as defined in
3045  * RFC 1982.
3046  *
3047  * NOTE: This code is duplicated in the isns server. It ought to be common.
3048  */
3049 
3050 static int
3051 iscsit_sna_lt(uint32_t sn1, uint32_t sn2)
3052 {
3053         return ((sn1 != sn2) &&
3054             (((sn1 < sn2) && ((sn2 - sn1) < ISCSIT_SNA32_CHECK)) ||
3055             ((sn1 > sn2) && ((sn1 - sn2) > ISCSIT_SNA32_CHECK))));
3056 }
3057 
3058 static int
3059 iscsit_sna_lte(uint32_t sn1, uint32_t sn2)
3060 {
3061         return ((sn1 == sn2) ||
3062             (((sn1 < sn2) && ((sn2 - sn1) < ISCSIT_SNA32_CHECK)) ||
3063             ((sn1 > sn2) && ((sn1 - sn2) > ISCSIT_SNA32_CHECK))));
3064 }
3065 
3066 
3067 static boolean_t
3068 iscsit_cmdsn_in_window(iscsit_conn_t *ict, uint32_t cmdsn)
3069 {
3070         iscsit_sess_t   *ist = ict->ict_sess;
3071         int             rval = B_TRUE;
3072 
3073         ist = ict->ict_sess;
3074 
3075         mutex_enter(&ist->ist_sn_mutex);
3076 
3077         /*
3078          * If cmdsn is less than ist_expcmdsn - iscsit_cmd_window() or
3079          * greater than ist_expcmdsn, it's not in the window.
3080          */
3081 
3082         if (iscsit_sna_lt(cmdsn, (ist->ist_expcmdsn - iscsit_cmd_window())) ||
3083             !iscsit_sna_lte(cmdsn, ist->ist_expcmdsn)) {
3084                 rval = B_FALSE;
3085         }
3086 
3087         mutex_exit(&ist->ist_sn_mutex);
3088 
3089         return (rval);
3090 }
3091 
3092 /*
3093  * iscsit_check_cmdsn_and_queue
3094  *
3095  * Independent of the order in which the iSCSI target receives non-immediate
3096  * command PDU across the entire session and any multiple connections within
3097  * the session, the target must deliver the commands to the SCSI layer in
3098  * CmdSN order. So out-of-order non-immediate commands are queued up on a
3099  * session-wide wait queue. Duplicate commands are ignored.
3100  *
3101  */
3102 static int
3103 iscsit_check_cmdsn_and_queue(idm_pdu_t *rx_pdu)
3104 {
3105         idm_conn_t              *ic = rx_pdu->isp_ic;
3106         iscsit_conn_t           *ict = ic->ic_handle;
3107         iscsit_sess_t           *ist = ict->ict_sess;
3108         iscsi_scsi_cmd_hdr_t    *hdr = (iscsi_scsi_cmd_hdr_t *)rx_pdu->isp_hdr;
3109 
3110         mutex_enter(&ist->ist_sn_mutex);
3111         if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
3112                 /* do not queue, handle it immediately */
3113                 DTRACE_PROBE2(immediate__cmd, iscsit_sess_t *, ist,
3114                     idm_pdu_t *, rx_pdu);
3115                 mutex_exit(&ist->ist_sn_mutex);
3116                 return (ISCSIT_CMDSN_EQ_EXPCMDSN);
3117         }
3118         if (iscsit_sna_lt(ist->ist_expcmdsn, ntohl(hdr->cmdsn))) {
3119                 /*
3120                  * Out-of-order commands (cmdSN higher than ExpCmdSN)
3121                  * are staged on a fixed-size circular buffer until
3122                  * the missing command is delivered to the SCSI layer.
3123                  * Irrespective of the order of insertion into the
3124                  * staging queue, the commands are processed out of the
3125                  * queue in cmdSN order only.
3126                  */
3127                 rx_pdu->isp_queue_time = gethrtime();
3128                 iscsit_add_pdu_to_queue(ist, rx_pdu);
3129                 mutex_exit(&ist->ist_sn_mutex);
3130                 return (ISCSIT_CMDSN_GT_EXPCMDSN);
3131         } else if (iscsit_sna_lt(ntohl(hdr->cmdsn), ist->ist_expcmdsn)) {
3132                 DTRACE_PROBE3(cmdsn__lt__expcmdsn, iscsit_sess_t *, ist,
3133                     iscsit_conn_t *, ict, idm_pdu_t *, rx_pdu);
3134                 mutex_exit(&ist->ist_sn_mutex);
3135                 return (ISCSIT_CMDSN_LT_EXPCMDSN);
3136         } else {
3137                 mutex_exit(&ist->ist_sn_mutex);
3138                 return (ISCSIT_CMDSN_EQ_EXPCMDSN);
3139         }
3140 }
3141 
3142 /*
3143  * iscsit_add_pdu_to_queue() adds PDUs into the array indexed by
3144  * their cmdsn value. The length of the array is kept above the
3145  * maximum window size. The window keeps the cmdsn within a range
3146  * such that there are no collisons. e.g. the assumption is that
3147  * the windowing checks make it impossible to receive PDUs that
3148  * index into the same location in the array.
3149  */
3150 static void
3151 iscsit_add_pdu_to_queue(iscsit_sess_t *ist, idm_pdu_t *rx_pdu)
3152 {
3153         iscsit_cbuf_t   *cbuf   = ist->ist_rxpdu_queue;
3154         iscsit_conn_t   *ict    = rx_pdu->isp_ic->ic_handle;
3155         uint32_t        cmdsn   =
3156             ((iscsi_scsi_cmd_hdr_t *)rx_pdu->isp_hdr)->cmdsn;
3157         uint32_t        index;
3158 
3159         ASSERT(MUTEX_HELD(&ist->ist_sn_mutex));
3160         /*
3161          * If the connection is being torn down, then
3162          * don't add the PDU to the staging queue
3163          */
3164         mutex_enter(&ict->ict_mutex);
3165         if (ict->ict_lost) {
3166                 mutex_exit(&ict->ict_mutex);
3167                 idm_pdu_complete(rx_pdu, IDM_STATUS_FAIL);
3168                 return;
3169         }
3170         iscsit_conn_dispatch_hold(ict);
3171         mutex_exit(&ict->ict_mutex);
3172 
3173         index = ntohl(cmdsn) % ISCSIT_RXPDU_QUEUE_LEN;
3174         /*
3175          * In the normal case, assuming that the Initiator is not
3176          * buggy and that we don't have packet duplication occuring,
3177          * the entry in the array will be NULL.  However, we may have
3178          * received a duplicate PDU with cmdsn > expsn , and in that
3179          * case we just ignore this PDU -- the previously received one
3180          * remains queued for processing.  We need to be careful not
3181          * to leak this one however.
3182          */
3183         if (cbuf->cb_buffer[index] != NULL) {
3184                 idm_pdu_complete(rx_pdu, IDM_STATUS_FAIL);
3185         } else {
3186                 cbuf->cb_buffer[index] = rx_pdu;
3187                 cbuf->cb_num_elems++;
3188         }
3189 }
3190 
3191 static idm_pdu_t *
3192 iscsit_remove_pdu_from_queue(iscsit_sess_t *ist, uint32_t cmdsn)
3193 {
3194         iscsit_cbuf_t   *cbuf   = ist->ist_rxpdu_queue;
3195         idm_pdu_t       *pdu    = NULL;
3196         uint32_t        index;
3197 
3198         ASSERT(MUTEX_HELD(&ist->ist_sn_mutex));
3199         index = cmdsn % ISCSIT_RXPDU_QUEUE_LEN;
3200         if ((pdu = cbuf->cb_buffer[index]) != NULL) {
3201                 ASSERT(cmdsn ==
3202                     ntohl(((iscsi_scsi_cmd_hdr_t *)pdu->isp_hdr)->cmdsn));
3203                 cbuf->cb_buffer[index] = NULL;
3204                 cbuf->cb_num_elems--;
3205                 return (pdu);
3206         }
3207         return (NULL);
3208 }
3209 
3210 /*
3211  * iscsit_process_pdu_in_queue() finds the next pdu in sequence
3212  * and posts it to the SCSI layer
3213  */
3214 static void
3215 iscsit_process_pdu_in_queue(iscsit_sess_t *ist)
3216 {
3217         iscsit_cbuf_t   *cbuf   = ist->ist_rxpdu_queue;
3218         idm_pdu_t       *pdu = NULL;
3219         uint32_t        expcmdsn;
3220 
3221         for (;;) {
3222                 mutex_enter(&ist->ist_sn_mutex);
3223                 if (cbuf->cb_num_elems == 0) {
3224                         mutex_exit(&ist->ist_sn_mutex);
3225                         break;
3226                 }
3227                 expcmdsn = ist->ist_expcmdsn;
3228                 if ((pdu = iscsit_remove_pdu_from_queue(ist, expcmdsn))
3229                     == NULL) {
3230                         mutex_exit(&ist->ist_sn_mutex);
3231                         break;
3232                 }
3233                 mutex_exit(&ist->ist_sn_mutex);
3234                 iscsit_post_staged_pdu(pdu);
3235         }
3236 }
3237 
3238 static void
3239 iscsit_post_staged_pdu(idm_pdu_t *rx_pdu)
3240 {
3241         iscsit_conn_t   *ict    = rx_pdu->isp_ic->ic_handle;
3242 
3243         /* Post the PDU to the SCSI layer */
3244         switch (IDM_PDU_OPCODE(rx_pdu)) {
3245         case ISCSI_OP_NOOP_OUT:
3246                 iscsit_set_cmdsn(ict, rx_pdu);
3247                 iscsit_pdu_op_noop(ict, rx_pdu);
3248                 break;
3249         case ISCSI_OP_TEXT_CMD:
3250                 iscsit_set_cmdsn(ict, rx_pdu);
3251                 iscsit_pdu_op_text_cmd(ict, rx_pdu);
3252                 break;
3253         case ISCSI_OP_SCSI_TASK_MGT_MSG:
3254                 iscsit_set_cmdsn(ict, rx_pdu);
3255                 iscsit_op_scsi_task_mgmt(ict, rx_pdu);
3256                 break;
3257         case ISCSI_OP_SCSI_CMD:
3258                 /* cmdSN will be incremented after creating itask */
3259                 iscsit_post_scsi_cmd(rx_pdu->isp_ic, rx_pdu);
3260                 break;
3261         case ISCSI_OP_LOGOUT_CMD:
3262                 iscsit_set_cmdsn(ict, rx_pdu);
3263                 iscsit_pdu_op_logout_cmd(ict, rx_pdu);
3264                 break;
3265         default:
3266                 /* No other PDUs should be placed on the queue */
3267                 ASSERT(0);
3268         }
3269         iscsit_conn_dispatch_rele(ict); /* release hold on the conn */
3270 }
3271 
3272 /* ARGSUSED */
3273 void
3274 iscsit_rxpdu_queue_monitor_start(void)
3275 {
3276         mutex_enter(&iscsit_rxpdu_queue_monitor_mutex);
3277         if (iscsit_rxpdu_queue_monitor_thr_running) {
3278                 mutex_exit(&iscsit_rxpdu_queue_monitor_mutex);
3279                 return;
3280         }
3281         iscsit_rxpdu_queue_monitor_thr_id =
3282             thread_create(NULL, 0, iscsit_rxpdu_queue_monitor, NULL,
3283             0, &p0, TS_RUN, minclsyspri);
3284         while (!iscsit_rxpdu_queue_monitor_thr_running) {
3285                 cv_wait(&iscsit_rxpdu_queue_monitor_cv,
3286                     &iscsit_rxpdu_queue_monitor_mutex);
3287         }
3288         mutex_exit(&iscsit_rxpdu_queue_monitor_mutex);
3289 
3290 }
3291 
3292 /* ARGSUSED */
3293 void
3294 iscsit_rxpdu_queue_monitor_stop(void)
3295 {
3296         mutex_enter(&iscsit_rxpdu_queue_monitor_mutex);
3297         if (iscsit_rxpdu_queue_monitor_thr_running) {
3298                 iscsit_rxpdu_queue_monitor_thr_running = B_FALSE;
3299                 cv_signal(&iscsit_rxpdu_queue_monitor_cv);
3300                 mutex_exit(&iscsit_rxpdu_queue_monitor_mutex);
3301 
3302                 thread_join(iscsit_rxpdu_queue_monitor_thr_did);
3303                 return;
3304         }
3305         mutex_exit(&iscsit_rxpdu_queue_monitor_mutex);
3306 }
3307 
3308 /*
3309  * A separate thread is used to scan the staging queue on all the
3310  * sessions, If a delayed PDU does not arrive within a timeout, the
3311  * target will advance to the staged PDU that is next in sequence
3312  * and exceeded the threshold wait time. It is up to the initiator
3313  * to note that the target has not acknowledged a particular cmdsn
3314  * and take appropriate action.
3315  */
3316 /* ARGSUSED */
3317 static void
3318 iscsit_rxpdu_queue_monitor(void *arg)
3319 {
3320         iscsit_tgt_t    *tgt;
3321         iscsit_sess_t   *ist;
3322 
3323         mutex_enter(&iscsit_rxpdu_queue_monitor_mutex);
3324         iscsit_rxpdu_queue_monitor_thr_did = curthread->t_did;
3325         iscsit_rxpdu_queue_monitor_thr_running = B_TRUE;
3326         cv_signal(&iscsit_rxpdu_queue_monitor_cv);
3327 
3328         while (iscsit_rxpdu_queue_monitor_thr_running) {
3329                 ISCSIT_GLOBAL_LOCK(RW_READER);
3330                 for (tgt = avl_first(&iscsit_global.global_target_list);
3331                     tgt != NULL;
3332                     tgt = AVL_NEXT(&iscsit_global.global_target_list, tgt)) {
3333                         mutex_enter(&tgt->target_mutex);
3334                         for (ist = avl_first(&tgt->target_sess_list);
3335                             ist != NULL;
3336                             ist = AVL_NEXT(&tgt->target_sess_list, ist)) {
3337 
3338                                 iscsit_rxpdu_queue_monitor_session(ist);
3339                         }
3340                         mutex_exit(&tgt->target_mutex);
3341                 }
3342                 ISCSIT_GLOBAL_UNLOCK();
3343                 if (iscsit_rxpdu_queue_monitor_thr_running == B_FALSE) {
3344                         break;
3345                 }
3346                 (void) cv_reltimedwait(&iscsit_rxpdu_queue_monitor_cv,
3347                     &iscsit_rxpdu_queue_monitor_mutex,
3348                     ISCSIT_RXPDU_QUEUE_MONITOR_INTERVAL * drv_usectohz(1000000),
3349                     TR_CLOCK_TICK);
3350         }
3351         mutex_exit(&iscsit_rxpdu_queue_monitor_mutex);
3352         thread_exit();
3353 }
3354 
3355 static void
3356 iscsit_rxpdu_queue_monitor_session(iscsit_sess_t *ist)
3357 {
3358         iscsit_cbuf_t   *cbuf   = ist->ist_rxpdu_queue;
3359         idm_pdu_t       *next_pdu = NULL;
3360         uint32_t        index, next_cmdsn, i;
3361 
3362         /*
3363          * Assume that all PDUs in the staging queue have a cmdsn >= expcmdsn.
3364          * Starting with the expcmdsn, iterate over the staged PDUs to find
3365          * the next PDU with a wait time greater than the threshold. If found
3366          * advance the staged PDU to the SCSI layer, skipping over the missing
3367          * PDU(s) to get past the hole in the command sequence. It is up to
3368          * the initiator to note that the target has not acknowledged a cmdsn
3369          * and take appropriate action.
3370          *
3371          * Since the PDU(s) arrive in any random order, it is possible that
3372          * that the actual wait time for a particular PDU is much longer than
3373          * the defined threshold. e.g. Consider a case where commands are sent
3374          * over 4 different connections, and cmdsn = 1004 arrives first, then
3375          * 1003, and 1002 and 1001 are lost due to a connection failure.
3376          * So now 1003 is waiting for 1002 to be delivered, and although the
3377          * wait time of 1004 > wait time of 1003, only 1003 will be considered
3378          * by the monitor thread. 1004 will be automatically processed by
3379          * iscsit_process_pdu_in_queue() once the scan is complete and the
3380          * expcmdsn becomes current.
3381          */
3382         mutex_enter(&ist->ist_sn_mutex);
3383         cbuf = ist->ist_rxpdu_queue;
3384         if (cbuf->cb_num_elems == 0) {
3385                 mutex_exit(&ist->ist_sn_mutex);
3386                 return;
3387         }
3388         for (next_pdu = NULL, i = 0; ; i++) {
3389                 next_cmdsn = ist->ist_expcmdsn + i; /* start at expcmdsn */
3390                 index = next_cmdsn % ISCSIT_RXPDU_QUEUE_LEN;
3391                 if ((next_pdu = cbuf->cb_buffer[index]) != NULL) {
3392                         /*
3393                          * If the PDU wait time has not exceeded threshold
3394                          * stop scanning the staging queue until the timer
3395                          * fires again
3396                          */
3397                         if ((gethrtime() - next_pdu->isp_queue_time)
3398                             < (rxpdu_queue_threshold * NANOSEC)) {
3399                                 mutex_exit(&ist->ist_sn_mutex);
3400                                 return;
3401                         }
3402                         /*
3403                          * Remove the next PDU from the queue and post it
3404                          * to the SCSI layer, skipping over the missing
3405                          * PDU. Stop scanning the staging queue until
3406                          * the monitor timer fires again
3407                          */
3408                         (void) iscsit_remove_pdu_from_queue(ist, next_cmdsn);
3409                         mutex_exit(&ist->ist_sn_mutex);
3410                         DTRACE_PROBE3(advanced__to__blocked__cmdsn,
3411                             iscsit_sess_t *, ist, idm_pdu_t *, next_pdu,
3412                             uint32_t, next_cmdsn);
3413                         iscsit_post_staged_pdu(next_pdu);
3414                         /* Deliver any subsequent PDUs immediately */
3415                         iscsit_process_pdu_in_queue(ist);
3416                         return;
3417                 }
3418                 /*
3419                  * Skipping over i PDUs, e.g. a case where commands 1001 and
3420                  * 1002 are lost in the network, skip over both and post 1003
3421                  * expcmdsn then becomes 1004 at the end of the scan.
3422                  */
3423                 DTRACE_PROBE2(skipping__over__cmdsn, iscsit_sess_t *, ist,
3424                     uint32_t, next_cmdsn);
3425         }
3426         /*
3427          * following the assumption, staged cmdsn >= expcmdsn, this statement
3428          * is never reached.
3429          */
3430 }