1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 /*
  27  * SunOS 5.x Multithreaded STREAMS DLPI FCIP Module
  28  * This is a pseudo driver module to handle encapsulation of IP and ARP
  29  * datagrams over FibreChannel interfaces. FCIP is a cloneable STREAMS
  30  * driver module which interfaces with IP/ARP using DLPI. This module
  31  * is a Style-2 DLS provider.
  32  *
  33  * The implementation of this module is based on RFC 2625 which gives
  34  * details on the encapsulation of IP/ARP data over FibreChannel.
  35  * The fcip module needs to resolve an IP address to a port address before
  36  * sending data to a destination port. A FC device port has 2 addresses
  37  * associated with it: A 8 byte World Wide unique Port Name and a 3 byte
  38  * volatile Port number or Port_ID.
  39  *
  40  * The mapping between a IP address and the World Wide Port Name is handled
  41  * by the ARP layer since the IP over FC draft requires the MAC address to
  42  * be the least significant six bytes of the WorldWide Port Names. The
  43  * fcip module however needs to identify the destination port uniquely when
  44  * the destination FC device has multiple FC ports.
  45  *
  46  * The FC layer mapping between the World Wide Port Name and the Port_ID
  47  * will be handled through the use of a fabric name server or through the
  48  * use of the FARP ELS command as described in the draft. Since the Port_IDs
  49  * are volatile, the mapping between the World Wide Port Name and Port_IDs
  50  * must be maintained and validated before use each time a datagram
  51  * needs to be sent to the destination ports. The FC transport module
  52  * informs the fcip module of all changes to states of ports on the
  53  * fabric through registered callbacks. This enables the fcip module
  54  * to maintain the WW_PN to Port_ID mappings current.
  55  *
  56  * For details on how this module interfaces with the FibreChannel Transport
  57  * modules, refer to PSARC/1997/385. Chapter 3 of the FibreChannel Transport
  58  * Programming guide details the APIs between ULPs and the Transport.
  59  *
  60  * Now for some Caveats:
  61  *
  62  * RFC 2625 requires that a FibreChannel Port name (the Port WWN) have
  63  * the NAA bits set to '0001' indicating a IEEE 48bit address which
  64  * corresponds to a ULA (Universal LAN MAC address). But with FibreChannel
  65  * adapters containing 2 or more ports, IEEE naming cannot identify the
  66  * ports on an adapter uniquely so we will in the first implementation
  67  * be operating only on Port 0 of each adapter.
  68  */
  69 
  70 #include        <sys/types.h>
  71 #include        <sys/errno.h>
  72 #include        <sys/debug.h>
  73 #include        <sys/time.h>
  74 #include        <sys/sysmacros.h>
  75 #include        <sys/systm.h>
  76 #include        <sys/user.h>
  77 #include        <sys/stropts.h>
  78 #include        <sys/stream.h>
  79 #include        <sys/strlog.h>
  80 #include        <sys/strsubr.h>
  81 #include        <sys/cmn_err.h>
  82 #include        <sys/cpu.h>
  83 #include        <sys/kmem.h>
  84 #include        <sys/conf.h>
  85 #include        <sys/ddi.h>
  86 #include        <sys/sunddi.h>
  87 #include        <sys/ksynch.h>
  88 #include        <sys/stat.h>
  89 #include        <sys/kstat.h>
  90 #include        <sys/vtrace.h>
  91 #include        <sys/strsun.h>
  92 #include        <sys/varargs.h>
  93 #include        <sys/modctl.h>
  94 #include        <sys/thread.h>
  95 #include        <sys/var.h>
  96 #include        <sys/proc.h>
  97 #include        <inet/common.h>
  98 #include        <netinet/ip6.h>
  99 #include        <inet/ip.h>
 100 #include        <inet/arp.h>
 101 #include        <inet/mi.h>
 102 #include        <inet/nd.h>
 103 #include        <sys/dlpi.h>
 104 #include        <sys/ethernet.h>
 105 #include        <sys/file.h>
 106 #include        <sys/syslog.h>
 107 #include        <sys/disp.h>
 108 #include        <sys/taskq.h>
 109 
 110 /*
 111  * Leadville includes
 112  */
 113 
 114 #include        <sys/fibre-channel/fc.h>
 115 #include        <sys/fibre-channel/impl/fc_ulpif.h>
 116 #include        <sys/fibre-channel/ulp/fcip.h>
 117 
 118 /*
 119  * TNF Probe/trace facility include
 120  */
 121 #if defined(lint) || defined(FCIP_TNF_ENABLED)
 122 #include <sys/tnf_probe.h>
 123 #endif
 124 
 125 #define FCIP_ESBALLOC
 126 
 127 /*
 128  * Function prototypes
 129  */
 130 
 131 /* standard loadable modules entry points */
 132 static int      fcip_attach(dev_info_t *, ddi_attach_cmd_t);
 133 static int      fcip_detach(dev_info_t *, ddi_detach_cmd_t);
 134 static void     fcip_dodetach(struct fcipstr *slp);
 135 static int fcip_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd,
 136     void *arg, void **result);
 137 
 138 
 139 /* streams specific */
 140 static void fcip_setipq(struct fcip *fptr);
 141 static int fcip_wput(queue_t *, mblk_t *);
 142 static int fcip_wsrv(queue_t *);
 143 static void fcip_proto(queue_t *, mblk_t *);
 144 static void fcip_ioctl(queue_t *, mblk_t *);
 145 static int fcip_open(queue_t *wq, dev_t *devp, int flag,
 146                 int sflag, cred_t *credp);
 147 static int fcip_close(queue_t *rq, int flag, int otyp, cred_t *credp);
 148 static int fcip_start(queue_t *wq, mblk_t *mp, struct fcip *fptr,
 149     struct fcip_dest *fdestp, int flags);
 150 static void fcip_sendup(struct fcip *fptr, mblk_t *mp,
 151     struct fcipstr *(*acceptfunc)());
 152 static struct fcipstr *fcip_accept(struct fcipstr *slp, struct fcip *fptr,
 153     int type, la_wwn_t *dhostp);
 154 static mblk_t *fcip_addudind(struct fcip *fptr, mblk_t *mp,
 155     fcph_network_hdr_t *nhdr, int type);
 156 static int fcip_setup_mac_addr(struct fcip *fptr);
 157 static void fcip_kstat_init(struct fcip *fptr);
 158 static int fcip_stat_update(kstat_t *, int);
 159 
 160 
 161 /* dlpi specific */
 162 static void fcip_spareq(queue_t *wq, mblk_t *mp);
 163 static void fcip_pareq(queue_t *wq, mblk_t *mp);
 164 static void fcip_ubreq(queue_t *wq, mblk_t *mp);
 165 static void fcip_breq(queue_t *wq, mblk_t *mp);
 166 static void fcip_dreq(queue_t *wq, mblk_t *mp);
 167 static void fcip_areq(queue_t *wq, mblk_t *mp);
 168 static void fcip_udreq(queue_t *wq, mblk_t *mp);
 169 static void fcip_ireq(queue_t *wq, mblk_t *mp);
 170 static void fcip_dl_ioc_hdr_info(queue_t *wq, mblk_t *mp);
 171 
 172 
 173 /* solaris sundry, DR/CPR etc */
 174 static int fcip_cache_constructor(void *buf, void *arg, int size);
 175 static void fcip_cache_destructor(void *buf, void *size);
 176 static int fcip_handle_suspend(fcip_port_info_t *fport, fc_detach_cmd_t cmd);
 177 static int fcip_handle_resume(fcip_port_info_t *fport,
 178     fc_ulp_port_info_t *port_info, fc_attach_cmd_t cmd);
 179 static fcip_port_info_t *fcip_softstate_free(fcip_port_info_t *fport);
 180 static int fcip_port_attach_handler(struct fcip *fptr);
 181 
 182 
 183 /*
 184  * ulp - transport interface function prototypes
 185  */
 186 static int fcip_port_attach(opaque_t ulp_handle, fc_ulp_port_info_t *,
 187     fc_attach_cmd_t cmd, uint32_t sid);
 188 static int fcip_port_detach(opaque_t ulp_handle, fc_ulp_port_info_t *,
 189     fc_detach_cmd_t cmd);
 190 static int fcip_port_ioctl(opaque_t ulp_handle,  opaque_t port_handle,
 191     dev_t dev, int cmd, intptr_t data, int mode, cred_t *credp, int *rval,
 192     uint32_t claimed);
 193 static void fcip_statec_cb(opaque_t ulp_handle, opaque_t phandle,
 194     uint32_t port_state, uint32_t port_top, fc_portmap_t changelist[],
 195     uint32_t listlen, uint32_t sid);
 196 static int fcip_els_cb(opaque_t ulp_handle, opaque_t phandle,
 197     fc_unsol_buf_t *buf, uint32_t claimed);
 198 static int fcip_data_cb(opaque_t ulp_handle, opaque_t phandle,
 199     fc_unsol_buf_t *payload, uint32_t claimed);
 200 
 201 
 202 /* Routing table specific */
 203 static void fcip_handle_topology(struct fcip *fptr);
 204 static int fcip_init_port(struct fcip *fptr);
 205 struct fcip_routing_table *fcip_lookup_rtable(struct fcip *fptr,
 206     la_wwn_t *pwwn, int matchflag);
 207 static void fcip_rt_update(struct fcip *fptr, fc_portmap_t *devlist,
 208     uint32_t listlen);
 209 static void fcip_rt_flush(struct fcip *fptr);
 210 static void fcip_rte_remove_deferred(void *arg);
 211 static int fcip_do_plogi(struct fcip *fptr, struct fcip_routing_table *frp);
 212 
 213 
 214 /* dest table specific */
 215 static struct fcip_dest *fcip_get_dest(struct fcip *fptr,
 216     la_wwn_t *dlphys);
 217 static struct fcip_dest *fcip_add_dest(struct fcip *fptr,
 218     struct fcip_routing_table *frp);
 219 static int fcip_dest_add_broadcast_entry(struct fcip *fptr, int new_flag);
 220 static uint32_t fcip_get_broadcast_did(struct fcip *fptr);
 221 static void fcip_cleanup_dest(struct fcip *fptr);
 222 
 223 
 224 /* helper functions */
 225 static fcip_port_info_t *fcip_get_port(opaque_t phandle);
 226 static int fcip_wwn_compare(la_wwn_t *wwn1, la_wwn_t *wwn2, int flag);
 227 static void fcip_ether_to_str(struct ether_addr *e, caddr_t s);
 228 static int fcip_port_get_num_pkts(struct fcip *fptr);
 229 static int fcip_check_port_busy(struct fcip *fptr);
 230 static void fcip_check_remove_minor_node(void);
 231 static int fcip_set_wwn(la_wwn_t *pwwn);
 232 static int fcip_plogi_in_progress(struct fcip *fptr);
 233 static int fcip_check_port_exists(struct fcip *fptr);
 234 static int fcip_is_supported_fc_topology(int fc_topology);
 235 
 236 
 237 /* pkt specific */
 238 static fcip_pkt_t *fcip_pkt_alloc(struct fcip *fptr, mblk_t *bp,
 239     int flags, int datalen);
 240 static void fcip_pkt_free(struct fcip_pkt *fcip_pkt, int flags);
 241 static fcip_pkt_t *fcip_ipkt_alloc(struct fcip *fptr, int cmdlen,
 242     int resplen, opaque_t pd, int flags);
 243 static void fcip_ipkt_free(fcip_pkt_t *fcip_pkt);
 244 static void fcip_ipkt_callback(fc_packet_t *fc_pkt);
 245 static void fcip_free_pkt_dma(fcip_pkt_t *fcip_pkt);
 246 static void fcip_pkt_callback(fc_packet_t *fc_pkt);
 247 static void fcip_init_unicast_pkt(fcip_pkt_t *fcip_pkt, fc_portid_t sid,
 248     fc_portid_t did, void (*comp) ());
 249 static int fcip_transport(fcip_pkt_t *fcip_pkt);
 250 static void fcip_pkt_timeout(void *arg);
 251 static void fcip_timeout(void *arg);
 252 static void fcip_fdestp_enqueue_pkt(struct fcip_dest *fdestp,
 253     fcip_pkt_t *fcip_pkt);
 254 static int fcip_fdestp_dequeue_pkt(struct fcip_dest *fdestp,
 255     fcip_pkt_t *fcip_pkt);
 256 static int fcip_sendup_constructor(void *buf, void *arg, int flags);
 257 static void fcip_sendup_thr(void *arg);
 258 static int fcip_sendup_alloc_enque(struct fcip *ftpr, mblk_t *mp,
 259     struct fcipstr *(*f)());
 260 
 261 /*
 262  * zero copy inbound data handling
 263  */
 264 #ifdef FCIP_ESBALLOC
 265 static void fcip_ubfree(char *arg);
 266 #endif /* FCIP_ESBALLOC */
 267 
 268 #if !defined(FCIP_ESBALLOC)
 269 static void *fcip_allocb(size_t size, uint_t pri);
 270 #endif
 271 
 272 
 273 /* FCIP FARP support functions */
 274 static struct fcip_dest *fcip_do_farp(struct fcip *fptr, la_wwn_t *pwwn,
 275     char *ip_addr, size_t ip_addr_len, int flags);
 276 static void fcip_init_broadcast_pkt(fcip_pkt_t *fcip_pkt, void (*comp) (),
 277     int is_els);
 278 static int fcip_handle_farp_request(struct fcip *fptr, la_els_farp_t *fcmd);
 279 static int fcip_handle_farp_response(struct fcip *fptr, la_els_farp_t *fcmd);
 280 static void fcip_cache_arp_broadcast(struct fcip *ftpr, fc_unsol_buf_t *buf);
 281 static void fcip_port_ns(void *arg);
 282 
 283 #ifdef DEBUG
 284 
 285 #include <sys/debug.h>
 286 
 287 #define FCIP_DEBUG_DEFAULT      0x1
 288 #define FCIP_DEBUG_ATTACH       0x2
 289 #define FCIP_DEBUG_INIT         0x4
 290 #define FCIP_DEBUG_DETACH       0x8
 291 #define FCIP_DEBUG_DLPI         0x10
 292 #define FCIP_DEBUG_ELS          0x20
 293 #define FCIP_DEBUG_DOWNSTREAM   0x40
 294 #define FCIP_DEBUG_UPSTREAM     0x80
 295 #define FCIP_DEBUG_MISC         0x100
 296 
 297 #define FCIP_DEBUG_STARTUP      (FCIP_DEBUG_ATTACH|FCIP_DEBUG_INIT)
 298 #define FCIP_DEBUG_DATAOUT      (FCIP_DEBUG_DLPI|FCIP_DEBUG_DOWNSTREAM)
 299 #define FCIP_DEBUG_DATAIN       (FCIP_DEBUG_ELS|FCIP_DEBUG_UPSTREAM)
 300 
 301 static int fcip_debug = FCIP_DEBUG_DEFAULT;
 302 
 303 #define FCIP_DEBUG(level, args) \
 304         if (fcip_debug & (level))   cmn_err args;
 305 
 306 #else   /* DEBUG */
 307 
 308 #define FCIP_DEBUG(level, args)         /* do nothing */
 309 
 310 #endif  /* DEBUG */
 311 
 312 #define KIOIP   KSTAT_INTR_PTR(fcip->fcip_intrstats)
 313 
 314 /*
 315  * Endian independent ethernet to WWN copy
 316  */
 317 #define ether_to_wwn(E, W)      \
 318         bzero((void *)(W), sizeof (la_wwn_t)); \
 319         bcopy((void *)(E), (void *)&((W)->raw_wwn[2]), ETHERADDRL); \
 320         (W)->raw_wwn[0] |= 0x10
 321 
 322 /*
 323  * wwn_to_ether : Endian independent, copies a WWN to struct ether_addr.
 324  * The args to the macro are pointers to WWN and ether_addr structures
 325  */
 326 #define wwn_to_ether(W, E)      \
 327         bcopy((void *)&((W)->raw_wwn[2]), (void *)E, ETHERADDRL)
 328 
 329 /*
 330  * The module_info structure contains identification and limit values.
 331  * All queues associated with a certain driver share the same module_info
 332  * structures. This structure defines the characteristics of that driver/
 333  * module's queues. The module name must be unique. The max and min packet
 334  * sizes limit the no. of characters in M_DATA messages. The Hi and Lo
 335  * water marks are for flow control when a module has a service procedure.
 336  */
 337 static struct module_info       fcipminfo = {
 338         FCIPIDNUM,      /* mi_idnum : Module ID num */
 339         FCIPNAME,       /* mi_idname: Module Name */
 340         FCIPMINPSZ,     /* mi_minpsz: Min packet size */
 341         FCIPMAXPSZ,     /* mi_maxpsz: Max packet size */
 342         FCIPHIWAT,      /* mi_hiwat : High water mark */
 343         FCIPLOWAT       /* mi_lowat : Low water mark */
 344 };
 345 
 346 /*
 347  * The qinit structres contain the module put, service. open and close
 348  * procedure pointers. All modules and drivers with the same streamtab
 349  * file (i.e same fmodsw or cdevsw entry points) point to the same
 350  * upstream (read) and downstream (write) qinit structs.
 351  */
 352 static struct qinit     fcip_rinit = {
 353         NULL,           /* qi_putp */
 354         NULL,           /* qi_srvp */
 355         fcip_open,      /* qi_qopen */
 356         fcip_close,     /* qi_qclose */
 357         NULL,           /* qi_qadmin */
 358         &fcipminfo, /* qi_minfo */
 359         NULL            /* qi_mstat */
 360 };
 361 
 362 static struct qinit     fcip_winit = {
 363         fcip_wput,      /* qi_putp */
 364         fcip_wsrv,      /* qi_srvp */
 365         NULL,           /* qi_qopen */
 366         NULL,           /* qi_qclose */
 367         NULL,           /* qi_qadmin */
 368         &fcipminfo, /* qi_minfo */
 369         NULL            /* qi_mstat */
 370 };
 371 
 372 /*
 373  * streamtab contains pointers to the read and write qinit structures
 374  */
 375 
 376 static struct streamtab fcip_info = {
 377         &fcip_rinit,        /* st_rdinit */
 378         &fcip_winit,        /* st_wrinit */
 379         NULL,           /* st_muxrinit */
 380         NULL,           /* st_muxwrinit */
 381 };
 382 
 383 static struct cb_ops  fcip_cb_ops = {
 384         nodev,                          /* open */
 385         nodev,                          /* close */
 386         nodev,                          /* strategy */
 387         nodev,                          /* print */
 388         nodev,                          /* dump */
 389         nodev,                          /* read */
 390         nodev,                          /* write */
 391         nodev,                          /* ioctl */
 392         nodev,                          /* devmap */
 393         nodev,                          /* mmap */
 394         nodev,                          /* segmap */
 395         nochpoll,                       /* poll */
 396         ddi_prop_op,                    /* cb_prop_op */
 397         &fcip_info,                 /* streamtab  */
 398         D_MP | D_HOTPLUG,               /* Driver compatibility flag */
 399         CB_REV,                         /* rev */
 400         nodev,                          /* int (*cb_aread)() */
 401         nodev                           /* int (*cb_awrite)() */
 402 };
 403 
 404 /*
 405  * autoconfiguration routines.
 406  */
 407 static struct dev_ops fcip_ops = {
 408         DEVO_REV,               /* devo_rev, */
 409         0,                      /* refcnt  */
 410         fcip_getinfo,           /* info */
 411         nulldev,                /* identify */
 412         nulldev,                /* probe */
 413         fcip_attach,            /* attach */
 414         fcip_detach,            /* detach */
 415         nodev,                  /* RESET */
 416         &fcip_cb_ops,               /* driver operations */
 417         NULL,                   /* bus operations */
 418         ddi_power               /* power management */
 419 };
 420 
 421 #define FCIP_VERSION    "1.61"
 422 #define FCIP_NAME       "SunFC FCIP v" FCIP_VERSION
 423 
 424 #define PORT_DRIVER     "fp"
 425 
 426 #define GETSTRUCT(struct, number)       \
 427         ((struct *)kmem_zalloc((size_t)(sizeof (struct) * (number)), \
 428                 KM_SLEEP))
 429 
 430 static struct modldrv modldrv = {
 431         &mod_driverops,                     /* Type of module - driver */
 432         FCIP_NAME,                      /* Name of module */
 433         &fcip_ops,                  /* driver ops */
 434 };
 435 
 436 static struct modlinkage modlinkage = {
 437         MODREV_1, (void *)&modldrv, NULL
 438 };
 439 
 440 
 441 /*
 442  * Now for some global statics
 443  */
 444 static uint32_t fcip_ub_nbufs = FCIP_UB_NBUFS;
 445 static uint32_t fcip_ub_size = FCIP_UB_SIZE;
 446 static int fcip_pkt_ttl_ticks = FCIP_PKT_TTL;
 447 static int fcip_tick_incr = 1;
 448 static int fcip_wait_cmds = FCIP_WAIT_CMDS;
 449 static int fcip_num_attaching = 0;
 450 static int fcip_port_attach_pending = 0;
 451 static int fcip_create_nodes_on_demand = 1;     /* keep it similar to fcp */
 452 static int fcip_cache_on_arp_broadcast = 0;
 453 static int fcip_farp_supported = 0;
 454 static int fcip_minor_node_created = 0;
 455 
 456 /*
 457  * Supported FCAs
 458  */
 459 #define QLC_PORT_1_ID_BITS              0x100
 460 #define QLC_PORT_2_ID_BITS              0x101
 461 #define QLC_PORT_NAA                    0x2
 462 #define QLC_MODULE_NAME                 "qlc"
 463 #define IS_QLC_PORT(port_dip)           \
 464                         (strcmp(ddi_driver_name(ddi_get_parent((port_dip))),\
 465                         QLC_MODULE_NAME) == 0)
 466 
 467 
 468 /*
 469  * fcip softstate structures head.
 470  */
 471 
 472 static void *fcip_softp = NULL;
 473 
 474 /*
 475  * linked list of active (inuse) driver streams
 476  */
 477 
 478 static int fcip_num_instances = 0;
 479 static dev_info_t *fcip_module_dip = (dev_info_t *)0;
 480 
 481 
 482 /*
 483  * Ethernet broadcast address: Broadcast addressing in IP over fibre
 484  * channel should be the IEEE ULA (also the low 6 bytes of the Port WWN).
 485  *
 486  * The broadcast addressing varies for differing topologies a node may be in:
 487  *      - On a private loop the ARP broadcast is a class 3 sequence sent
 488  *        using OPNfr (Open Broadcast Replicate primitive) followed by
 489  *        the ARP frame to D_ID 0xFFFFFF
 490  *
 491  *      - On a public Loop the broadcast sequence is sent to AL_PA 0x00
 492  *        (no OPNfr primitive).
 493  *
 494  *      - For direct attach and point to point topologies we just send
 495  *        the frame to D_ID 0xFFFFFF
 496  *
 497  * For public loop the handling would probably be different - for now
 498  * I'll just declare this struct - It can be deleted if not necessary.
 499  *
 500  */
 501 
 502 
 503 /*
 504  * DL_INFO_ACK template for the fcip module. The dl_info_ack_t structure is
 505  * returned as a part of an  DL_INFO_ACK message which is a M_PCPROTO message
 506  * returned in response to a DL_INFO_REQ message sent to us from a DLS user
 507  * Let us fake an ether header as much as possible.
 508  *
 509  * dl_addr_length is the Provider's DLSAP addr which is SAP addr +
 510  *                Physical addr of the provider. We set this to
 511  *                ushort_t + sizeof (la_wwn_t) for Fibre Channel ports.
 512  * dl_mac_type    Lets just use DL_ETHER - we can try using DL_IPFC, a new
 513  *                dlpi.h define later.
 514  * dl_sap_length  -2 indicating the SAP address follows the Physical addr
 515  *                component in the DLSAP addr.
 516  * dl_service_mode: DLCLDS - connectionless data link service.
 517  *
 518  */
 519 
 520 static dl_info_ack_t fcip_infoack = {
 521         DL_INFO_ACK,                            /* dl_primitive */
 522         FCIPMTU,                                /* dl_max_sdu */
 523         0,                                      /* dl_min_sdu */
 524         FCIPADDRL,                              /* dl_addr_length */
 525         DL_ETHER,                               /* dl_mac_type */
 526         0,                                      /* dl_reserved */
 527         0,                                      /* dl_current_state */
 528         -2,                                     /* dl_sap_length */
 529         DL_CLDLS,                               /* dl_service_mode */
 530         0,                                      /* dl_qos_length */
 531         0,                                      /* dl_qos_offset */
 532         0,                                      /* dl_range_length */
 533         0,                                      /* dl_range_offset */
 534         DL_STYLE2,                              /* dl_provider_style */
 535         sizeof (dl_info_ack_t),                 /* dl_addr_offset */
 536         DL_VERSION_2,                           /* dl_version */
 537         ETHERADDRL,                             /* dl_brdcst_addr_length */
 538         sizeof (dl_info_ack_t) + FCIPADDRL,     /* dl_brdcst_addr_offset */
 539         0                                       /* dl_growth */
 540 };
 541 
 542 /*
 543  * FCIP broadcast address definition.
 544  */
 545 static  struct ether_addr       fcipnhbroadcastaddr = {
 546         0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 547 };
 548 
 549 /*
 550  * RFC2625 requires the broadcast ARP address in the ARP data payload to
 551  * be set to 0x00 00 00 00 00 00 for ARP broadcast packets
 552  */
 553 static  struct ether_addr       fcip_arpbroadcast_addr = {
 554         0x00, 0x00, 0x00, 0x00, 0x00, 0x00
 555 };
 556 
 557 
 558 #define ether_bcopy(src, dest)  bcopy((src), (dest), ETHERADDRL);
 559 
 560 /*
 561  * global kernel locks
 562  */
 563 static kcondvar_t       fcip_global_cv;
 564 static kmutex_t         fcip_global_mutex;
 565 
 566 /*
 567  * fctl external defines
 568  */
 569 extern int fc_ulp_add(fc_ulp_modinfo_t *);
 570 
 571 /*
 572  * fctl data structures
 573  */
 574 
 575 #define FCIP_REV        0x07
 576 
 577 /* linked list of port info structures */
 578 static fcip_port_info_t *fcip_port_head = NULL;
 579 
 580 /* linked list of fcip structures */
 581 static struct fcipstr   *fcipstrup = NULL;
 582 static krwlock_t        fcipstruplock;
 583 
 584 
 585 /*
 586  * Module information structure. This structure gives the FC Transport modules
 587  * information about an ULP that registers with it.
 588  */
 589 static fc_ulp_modinfo_t fcip_modinfo = {
 590         0,                      /* for xref checks? */
 591         FCTL_ULP_MODREV_4,      /* FCIP revision */
 592         FC_TYPE_IS8802_SNAP,    /* type 5 for SNAP encapsulated datagrams */
 593         FCIP_NAME,              /* module name as in the modldrv struct */
 594         0x0,                    /* get all statec callbacks for now */
 595         fcip_port_attach,       /* port attach callback */
 596         fcip_port_detach,       /* port detach callback */
 597         fcip_port_ioctl,        /* port ioctl callback */
 598         fcip_els_cb,            /* els callback */
 599         fcip_data_cb,           /* data callback */
 600         fcip_statec_cb          /* state change callback */
 601 };
 602 
 603 
 604 /*
 605  * Solaris 9 and up, the /kernel/drv/fp.conf file will have the following entry
 606  *
 607  * ddi-forceattach=1;
 608  *
 609  * This will ensure that fp is loaded at bootup. No additional checks are needed
 610  */
 611 int
 612 _init(void)
 613 {
 614         int     rval;
 615 
 616         FCIP_TNF_LOAD();
 617 
 618         /*
 619          * Initialize the mutexs used by port attach and other callbacks.
 620          * The transport can call back into our port_attach_callback
 621          * routine even before _init() completes and bad things can happen.
 622          */
 623         mutex_init(&fcip_global_mutex, NULL, MUTEX_DRIVER, NULL);
 624         cv_init(&fcip_global_cv, NULL, CV_DRIVER, NULL);
 625         rw_init(&fcipstruplock, NULL, RW_DRIVER, NULL);
 626 
 627         mutex_enter(&fcip_global_mutex);
 628         fcip_port_attach_pending = 1;
 629         mutex_exit(&fcip_global_mutex);
 630 
 631         /*
 632          * Now attempt to register fcip with the transport.
 633          * If fc_ulp_add fails, fcip module will not be loaded.
 634          */
 635         rval = fc_ulp_add(&fcip_modinfo);
 636         if (rval != FC_SUCCESS) {
 637                 mutex_destroy(&fcip_global_mutex);
 638                 cv_destroy(&fcip_global_cv);
 639                 rw_destroy(&fcipstruplock);
 640                 switch (rval) {
 641                 case FC_ULP_SAMEMODULE:
 642                         FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN,
 643                             "!fcip: module is already registered with"
 644                             " transport"));
 645                         rval = EEXIST;
 646                         break;
 647                 case FC_ULP_SAMETYPE:
 648                         FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN,
 649                             "!fcip: Another module of the same ULP type 0x%x"
 650                             " is already registered with the transport",
 651                             fcip_modinfo.ulp_type));
 652                         rval = EEXIST;
 653                         break;
 654                 case FC_BADULP:
 655                         FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN,
 656                             "!fcip: Current fcip version 0x%x does not match"
 657                             " fctl version",
 658                             fcip_modinfo.ulp_rev));
 659                         rval = ENODEV;
 660                         break;
 661                 default:
 662                         FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN,
 663                             "!fcip: fc_ulp_add failed with status 0x%x", rval));
 664                         rval = ENODEV;
 665                         break;
 666                 }
 667                 FCIP_TNF_UNLOAD(&modlinkage);
 668                 return (rval);
 669         }
 670 
 671         if ((rval = ddi_soft_state_init(&fcip_softp, sizeof (struct fcip),
 672                         FCIP_NUM_INSTANCES)) != 0) {
 673                 mutex_destroy(&fcip_global_mutex);
 674                 cv_destroy(&fcip_global_cv);
 675                 rw_destroy(&fcipstruplock);
 676                 (void) fc_ulp_remove(&fcip_modinfo);
 677                 FCIP_TNF_UNLOAD(&modlinkage);
 678                 return (rval);
 679         }
 680 
 681         if ((rval = mod_install(&modlinkage)) != 0) {
 682                 FCIP_TNF_UNLOAD(&modlinkage);
 683                 (void) fc_ulp_remove(&fcip_modinfo);
 684                 mutex_destroy(&fcip_global_mutex);
 685                 cv_destroy(&fcip_global_cv);
 686                 rw_destroy(&fcipstruplock);
 687                 ddi_soft_state_fini(&fcip_softp);
 688         }
 689         return (rval);
 690 }
 691 
 692 /*
 693  * Unload the port driver if this was the only ULP loaded and then
 694  * deregister with the transport.
 695  */
 696 int
 697 _fini(void)
 698 {
 699         int     rval;
 700         int     rval1;
 701 
 702         /*
 703          * Do not permit the module to be unloaded before a port
 704          * attach callback has happened.
 705          */
 706         mutex_enter(&fcip_global_mutex);
 707         if (fcip_num_attaching || fcip_port_attach_pending) {
 708                 mutex_exit(&fcip_global_mutex);
 709                 return (EBUSY);
 710         }
 711         mutex_exit(&fcip_global_mutex);
 712 
 713         if ((rval = mod_remove(&modlinkage)) != 0) {
 714                 return (rval);
 715         }
 716 
 717         /*
 718          * unregister with the transport layer
 719          */
 720         rval1 = fc_ulp_remove(&fcip_modinfo);
 721 
 722         /*
 723          * If the ULP was not registered with the transport, init should
 724          * have failed. If transport has no knowledge of our existence
 725          * we should simply bail out and succeed
 726          */
 727 #ifdef DEBUG
 728         if (rval1 == FC_BADULP) {
 729                 FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN,
 730                 "fcip: ULP was never registered with the transport"));
 731                 rval = ENODEV;
 732         } else if (rval1 == FC_BADTYPE) {
 733                 FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN,
 734                         "fcip: No ULP of this type 0x%x was registered with "
 735                         "transport", fcip_modinfo.ulp_type));
 736                 rval = ENODEV;
 737         }
 738 #endif /* DEBUG */
 739 
 740         mutex_destroy(&fcip_global_mutex);
 741         rw_destroy(&fcipstruplock);
 742         cv_destroy(&fcip_global_cv);
 743         ddi_soft_state_fini(&fcip_softp);
 744 
 745         FCIP_TNF_UNLOAD(&modlinkage);
 746 
 747         return (rval);
 748 }
 749 
 750 /*
 751  * Info about this loadable module
 752  */
 753 int
 754 _info(struct modinfo *modinfop)
 755 {
 756         return (mod_info(&modlinkage, modinfop));
 757 }
 758 
 759 /*
 760  * The port attach callback is invoked by the port driver when a FCA
 761  * port comes online and binds with the transport layer. The transport
 762  * then callsback into all ULP modules registered with it. The Port attach
 763  * call back will also provide the ULP module with the Port's WWN and S_ID
 764  */
 765 /* ARGSUSED */
 766 static int
 767 fcip_port_attach(opaque_t ulp_handle, fc_ulp_port_info_t *port_info,
 768     fc_attach_cmd_t cmd, uint32_t sid)
 769 {
 770         int                     rval = FC_FAILURE;
 771         int                     instance;
 772         struct fcip             *fptr;
 773         fcip_port_info_t        *fport = NULL;
 774         fcip_port_info_t        *cur_fport;
 775         fc_portid_t             src_id;
 776 
 777         switch (cmd) {
 778         case FC_CMD_ATTACH: {
 779                 la_wwn_t        *ww_pn = NULL;
 780                 /*
 781                  * It was determined that, as per spec, the lower 48 bits of
 782                  * the port-WWN will always be unique. This will make the MAC
 783                  * address (i.e the lower 48 bits of the WWN), that IP/ARP
 784                  * depend on, unique too. Hence we should be able to remove the
 785                  * restriction of attaching to only one of the ports of
 786                  * multi port FCAs.
 787                  *
 788                  * Earlier, fcip used to attach only to qlc module and fail
 789                  * silently for attach failures resulting from unknown FCAs or
 790                  * unsupported FCA ports. Now, we'll do no such checks.
 791                  */
 792                 ww_pn = &port_info->port_pwwn;
 793 
 794                 FCIP_TNF_PROBE_2((fcip_port_attach, "fcip io", /* CSTYLED */,
 795                         tnf_string, msg, "port id bits",
 796                         tnf_opaque, nport_id, ww_pn->w.nport_id));
 797                 FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_NOTE,
 798                     "port id bits: 0x%x", ww_pn->w.nport_id));
 799                 /*
 800                  * A port has come online
 801                  */
 802                 mutex_enter(&fcip_global_mutex);
 803                 fcip_num_instances++;
 804                 fcip_num_attaching++;
 805 
 806                 if (fcip_port_head == NULL) {
 807                         /* OK to sleep here ? */
 808                         fport = kmem_zalloc(sizeof (fcip_port_info_t),
 809                                                 KM_NOSLEEP);
 810                         if (fport == NULL) {
 811                                 fcip_num_instances--;
 812                                 fcip_num_attaching--;
 813                                 ASSERT(fcip_num_attaching >= 0);
 814                                 mutex_exit(&fcip_global_mutex);
 815                                 rval = FC_FAILURE;
 816                                 cmn_err(CE_WARN, "!fcip(%d): port attach "
 817                                     "failed: alloc failed",
 818                                     ddi_get_instance(port_info->port_dip));
 819                                 goto done;
 820                         }
 821                         fcip_port_head = fport;
 822                 } else {
 823                         /*
 824                          * traverse the port list and also check for
 825                          * duplicate port attaches - Nothing wrong in being
 826                          * paranoid Heh Heh.
 827                          */
 828                         cur_fport = fcip_port_head;
 829                         while (cur_fport != NULL) {
 830                                 if (cur_fport->fcipp_handle ==
 831                                     port_info->port_handle) {
 832                                         fcip_num_instances--;
 833                                         fcip_num_attaching--;
 834                                         ASSERT(fcip_num_attaching >= 0);
 835                                         mutex_exit(&fcip_global_mutex);
 836                                         FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_WARN,
 837                                             "!fcip(%d): port already "
 838                                             "attached!!", ddi_get_instance(
 839                                             port_info->port_dip)));
 840                                         rval = FC_FAILURE;
 841                                         goto done;
 842                                 }
 843                                 cur_fport = cur_fport->fcipp_next;
 844                         }
 845                         fport = kmem_zalloc(sizeof (fcip_port_info_t),
 846                                                 KM_NOSLEEP);
 847                         if (fport == NULL) {
 848                                 rval = FC_FAILURE;
 849                                 fcip_num_instances--;
 850                                 fcip_num_attaching--;
 851                                 ASSERT(fcip_num_attaching >= 0);
 852                                 mutex_exit(&fcip_global_mutex);
 853                                 cmn_err(CE_WARN, "!fcip(%d): port attach "
 854                                     "failed: alloc failed",
 855                                     ddi_get_instance(port_info->port_dip));
 856                                 goto done;
 857                         }
 858                         fport->fcipp_next = fcip_port_head;
 859                         fcip_port_head = fport;
 860                 }
 861 
 862                 mutex_exit(&fcip_global_mutex);
 863 
 864                 /*
 865                  * now fill in the details about the port itself
 866                  */
 867                 fport->fcipp_linkage = *port_info->port_linkage;
 868                 fport->fcipp_handle = port_info->port_handle;
 869                 fport->fcipp_dip = port_info->port_dip;
 870                 fport->fcipp_topology = port_info->port_flags;
 871                 fport->fcipp_pstate = port_info->port_state;
 872                 fport->fcipp_naa = port_info->port_pwwn.w.naa_id;
 873                 bcopy(&port_info->port_pwwn, &fport->fcipp_pwwn,
 874                     sizeof (la_wwn_t));
 875                 bcopy(&port_info->port_nwwn, &fport->fcipp_nwwn,
 876                     sizeof (la_wwn_t));
 877                 fport->fcipp_fca_pkt_size = port_info->port_fca_pkt_size;
 878                 fport->fcipp_cmd_dma_attr = *port_info->port_cmd_dma_attr;
 879                 fport->fcipp_resp_dma_attr = *port_info->port_resp_dma_attr;
 880                 fport->fcipp_fca_acc_attr = *port_info->port_acc_attr;
 881                 src_id.port_id = sid;
 882                 src_id.priv_lilp_posit = 0;
 883                 fport->fcipp_sid = src_id;
 884 
 885                 /*
 886                  * allocate soft state for this instance
 887                  */
 888                 instance = ddi_get_instance(fport->fcipp_dip);
 889                 if (ddi_soft_state_zalloc(fcip_softp,
 890                     instance) != DDI_SUCCESS) {
 891                         rval = FC_FAILURE;
 892                         cmn_err(CE_WARN, "!fcip(%d): port attach failed: "
 893                             "soft state alloc failed", instance);
 894                         goto failure;
 895                 }
 896 
 897                 fptr = ddi_get_soft_state(fcip_softp, instance);
 898 
 899                 if (fptr == NULL) {
 900                         rval = FC_FAILURE;
 901                         cmn_err(CE_WARN, "!fcip(%d): port attach failed: "
 902                             "failure to get soft state", instance);
 903                         goto failure;
 904                 }
 905 
 906                 /*
 907                  * initialize all mutexes and locks required for this module
 908                  */
 909                 mutex_init(&fptr->fcip_mutex, NULL, MUTEX_DRIVER, NULL);
 910                 mutex_init(&fptr->fcip_ub_mutex, NULL, MUTEX_DRIVER, NULL);
 911                 mutex_init(&fptr->fcip_rt_mutex, NULL, MUTEX_DRIVER, NULL);
 912                 mutex_init(&fptr->fcip_dest_mutex, NULL, MUTEX_DRIVER, NULL);
 913                 mutex_init(&fptr->fcip_sendup_mutex, NULL, MUTEX_DRIVER, NULL);
 914                 cv_init(&fptr->fcip_farp_cv, NULL, CV_DRIVER, NULL);
 915                 cv_init(&fptr->fcip_sendup_cv, NULL, CV_DRIVER, NULL);
 916                 cv_init(&fptr->fcip_ub_cv, NULL, CV_DRIVER, NULL);
 917 
 918                 mutex_enter(&fptr->fcip_mutex);
 919 
 920                 fptr->fcip_dip = fport->fcipp_dip;        /* parent's dip */
 921                 fptr->fcip_instance = instance;
 922                 fptr->fcip_ub_upstream = 0;
 923 
 924                 if (FC_PORT_STATE_MASK(port_info->port_state) ==
 925                     FC_STATE_ONLINE) {
 926                         fptr->fcip_port_state = FCIP_PORT_ONLINE;
 927                         if (fptr->fcip_flags & FCIP_LINK_DOWN) {
 928                                 fptr->fcip_flags &= ~FCIP_LINK_DOWN;
 929                         }
 930                 } else {
 931                         fptr->fcip_port_state = FCIP_PORT_OFFLINE;
 932                 }
 933 
 934                 fptr->fcip_flags |= FCIP_ATTACHING;
 935                 fptr->fcip_port_info = fport;
 936 
 937                 /*
 938                  * Extract our MAC addr from our port's WWN. The lower 48
 939                  * bits will be our MAC address
 940                  */
 941                 wwn_to_ether(&fport->fcipp_nwwn, &fptr->fcip_macaddr);
 942 
 943                 fport->fcipp_fcip = fptr;
 944 
 945                 FCIP_DEBUG(FCIP_DEBUG_ATTACH,
 946                     (CE_NOTE, "fcipdest : 0x%lx, rtable : 0x%lx",
 947                     (long)(sizeof (fptr->fcip_dest)),
 948                     (long)(sizeof (fptr->fcip_rtable))));
 949 
 950                 bzero(fptr->fcip_dest, sizeof (fptr->fcip_dest));
 951                 bzero(fptr->fcip_rtable, sizeof (fptr->fcip_rtable));
 952 
 953                 /*
 954                  * create a taskq to handle sundry jobs for the driver
 955                  * This way we can have jobs run in parallel
 956                  */
 957                 fptr->fcip_tq = taskq_create("fcip_tasks",
 958                     FCIP_NUM_THREADS, MINCLSYSPRI, FCIP_MIN_TASKS,
 959                     FCIP_MAX_TASKS, TASKQ_PREPOPULATE);
 960 
 961                 mutex_exit(&fptr->fcip_mutex);
 962 
 963                 /*
 964                  * create a separate thread to handle all unsolicited
 965                  * callback handling. This is because unsolicited_callback
 966                  * can happen from an interrupt context and the upstream
 967                  * modules can put new messages right back in the same
 968                  * thread context. This usually works fine, but sometimes
 969                  * we may have to block to obtain the dest struct entries
 970                  * for some remote ports.
 971                  */
 972                 mutex_enter(&fptr->fcip_sendup_mutex);
 973                 if (thread_create(NULL, DEFAULTSTKSZ,
 974                     (void (*)())fcip_sendup_thr, (caddr_t)fptr, 0, &p0,
 975                     TS_RUN, minclsyspri) == NULL) {
 976                         mutex_exit(&fptr->fcip_sendup_mutex);
 977                         cmn_err(CE_WARN,
 978                             "!unable to create fcip sendup thread for "
 979                             " instance: 0x%x", instance);
 980                         rval = FC_FAILURE;
 981                         goto done;
 982                 }
 983                 fptr->fcip_sendup_thr_initted = 1;
 984                 fptr->fcip_sendup_head = fptr->fcip_sendup_tail = NULL;
 985                 mutex_exit(&fptr->fcip_sendup_mutex);
 986 
 987 
 988                 /* Let the attach handler do the rest */
 989                 if (fcip_port_attach_handler(fptr) != FC_SUCCESS) {
 990                         /*
 991                          * We have already cleaned up so return
 992                          */
 993                         rval = FC_FAILURE;
 994                         cmn_err(CE_WARN, "!fcip(%d): port attach failed",
 995                             instance);
 996                         goto done;
 997                 }
 998 
 999                 FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_CONT,
1000                     "!fcip attach for port instance (0x%x) successful",
1001                     instance));
1002 
1003                 rval = FC_SUCCESS;
1004                 goto done;
1005         }
1006         case FC_CMD_POWER_UP:
1007         /* FALLTHROUGH */
1008         case FC_CMD_RESUME:
1009                 mutex_enter(&fcip_global_mutex);
1010                 fport = fcip_port_head;
1011                 while (fport != NULL) {
1012                         if (fport->fcipp_handle == port_info->port_handle) {
1013                                 break;
1014                         }
1015                         fport = fport->fcipp_next;
1016                 }
1017                 if (fport == NULL) {
1018                         rval = FC_SUCCESS;
1019                         mutex_exit(&fcip_global_mutex);
1020                         goto done;
1021                 }
1022                 rval = fcip_handle_resume(fport, port_info, cmd);
1023                 mutex_exit(&fcip_global_mutex);
1024                 goto done;
1025 
1026         default:
1027                 FCIP_TNF_PROBE_2((fcip_port_attach, "fcip io", /* CSTYLED */,
1028                         tnf_string, msg, "unknown command type",
1029                         tnf_uint, cmd, cmd));
1030                 FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_WARN,
1031                     "unknown cmd type 0x%x in port_attach", cmd));
1032                 rval = FC_FAILURE;
1033                 goto done;
1034         }
1035 
1036 failure:
1037         if (fport) {
1038                 mutex_enter(&fcip_global_mutex);
1039                 fcip_num_attaching--;
1040                 ASSERT(fcip_num_attaching >= 0);
1041                 (void) fcip_softstate_free(fport);
1042                 fcip_port_attach_pending = 0;
1043                 mutex_exit(&fcip_global_mutex);
1044         }
1045         return (rval);
1046 
1047 done:
1048         mutex_enter(&fcip_global_mutex);
1049         fcip_port_attach_pending = 0;
1050         mutex_exit(&fcip_global_mutex);
1051         return (rval);
1052 }
1053 
1054 /*
1055  * fcip_port_attach_handler : Completes the port attach operation after
1056  * the ulp_port_attach routine has completed its ground work. The job
1057  * of this function among other things is to obtain and handle topology
1058  * specifics, initialize a port, setup broadcast address entries in
1059  * the fcip tables etc. This routine cleans up behind itself on failures.
1060  * Returns FC_SUCCESS or FC_FAILURE.
1061  */
1062 static int
1063 fcip_port_attach_handler(struct fcip *fptr)
1064 {
1065         fcip_port_info_t                *fport = fptr->fcip_port_info;
1066         int                             rval = FC_FAILURE;
1067 
1068         ASSERT(fport != NULL);
1069 
1070         mutex_enter(&fcip_global_mutex);
1071 
1072         FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_NOTE,
1073             "fcip module dip: %p instance: %d",
1074             (void *)fcip_module_dip, ddi_get_instance(fptr->fcip_dip)));
1075 
1076         if (fcip_module_dip == NULL) {
1077                 clock_t         fcip_lbolt;
1078 
1079                 fcip_lbolt = ddi_get_lbolt();
1080                 /*
1081                  * we need to use the fcip devinfo for creating
1082                  * the clone device node, but the fcip attach
1083                  * (from its conf file entry claiming to be a
1084                  * child of pseudo) may not have happened yet.
1085                  * wait here for 10 seconds and fail port attach
1086                  * if the fcip devinfo is not attached yet
1087                  */
1088                 fcip_lbolt += drv_usectohz(FCIP_INIT_DELAY);
1089 
1090                 FCIP_DEBUG(FCIP_DEBUG_ATTACH,
1091                     (CE_WARN, "cv_timedwait lbolt %lx", fcip_lbolt));
1092 
1093                 (void) cv_timedwait(&fcip_global_cv, &fcip_global_mutex,
1094                     fcip_lbolt);
1095 
1096                 if (fcip_module_dip == NULL) {
1097                         mutex_exit(&fcip_global_mutex);
1098 
1099                         FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_WARN,
1100                                 "fcip attach did not happen"));
1101                         goto port_attach_cleanup;
1102                 }
1103         }
1104 
1105         if ((!fcip_minor_node_created) &&
1106             fcip_is_supported_fc_topology(fport->fcipp_topology)) {
1107                 /*
1108                  * Checking for same topologies which are considered valid
1109                  * by fcip_handle_topology(). Dont create a minor node if
1110                  * nothing is hanging off the FC port.
1111                  */
1112                 if (ddi_create_minor_node(fcip_module_dip, "fcip", S_IFCHR,
1113                     ddi_get_instance(fptr->fcip_dip), DDI_PSEUDO,
1114                     CLONE_DEV) == DDI_FAILURE) {
1115                         mutex_exit(&fcip_global_mutex);
1116                         FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_WARN,
1117                             "failed to create minor node for fcip(%d)",
1118                             ddi_get_instance(fptr->fcip_dip)));
1119                         goto port_attach_cleanup;
1120                 }
1121                 fcip_minor_node_created++;
1122         }
1123         mutex_exit(&fcip_global_mutex);
1124 
1125         /*
1126          * initialize port for traffic
1127          */
1128         if (fcip_init_port(fptr) != FC_SUCCESS) {
1129                 /* fcip_init_port has already cleaned up its stuff */
1130 
1131                 mutex_enter(&fcip_global_mutex);
1132 
1133                 if ((fcip_num_instances == 1) &&
1134                     (fcip_minor_node_created == 1)) {
1135                         /* Remove minor node iff this is the last instance */
1136                         ddi_remove_minor_node(fcip_module_dip, NULL);
1137                 }
1138 
1139                 mutex_exit(&fcip_global_mutex);
1140 
1141                 goto port_attach_cleanup;
1142         }
1143 
1144         mutex_enter(&fptr->fcip_mutex);
1145         fptr->fcip_flags &= ~FCIP_ATTACHING;
1146         fptr->fcip_flags |= FCIP_INITED;
1147         fptr->fcip_timeout_ticks = 0;
1148 
1149         /*
1150          * start the timeout threads
1151          */
1152         fptr->fcip_timeout_id = timeout(fcip_timeout, fptr,
1153             drv_sectohz(1));
1154 
1155         mutex_exit(&fptr->fcip_mutex);
1156         mutex_enter(&fcip_global_mutex);
1157         fcip_num_attaching--;
1158         ASSERT(fcip_num_attaching >= 0);
1159         mutex_exit(&fcip_global_mutex);
1160         rval = FC_SUCCESS;
1161         return (rval);
1162 
1163 port_attach_cleanup:
1164         mutex_enter(&fcip_global_mutex);
1165         (void) fcip_softstate_free(fport);
1166         fcip_num_attaching--;
1167         ASSERT(fcip_num_attaching >= 0);
1168         mutex_exit(&fcip_global_mutex);
1169         rval = FC_FAILURE;
1170         return (rval);
1171 }
1172 
1173 
1174 /*
1175  * Handler for DDI_RESUME operations. Port must be ready to restart IP
1176  * traffic on resume
1177  */
1178 static int
1179 fcip_handle_resume(fcip_port_info_t *fport, fc_ulp_port_info_t *port_info,
1180     fc_attach_cmd_t cmd)
1181 {
1182         int             rval = FC_SUCCESS;
1183         struct fcip     *fptr = fport->fcipp_fcip;
1184         struct fcipstr  *tslp;
1185         int             index;
1186 
1187 
1188         ASSERT(fptr != NULL);
1189 
1190         mutex_enter(&fptr->fcip_mutex);
1191 
1192         if (cmd == FC_CMD_POWER_UP) {
1193                 fptr->fcip_flags &= ~(FCIP_POWER_DOWN);
1194                 if (fptr->fcip_flags & FCIP_SUSPENDED) {
1195                         mutex_exit(&fptr->fcip_mutex);
1196                         return (FC_SUCCESS);
1197                 }
1198         } else if (cmd == FC_CMD_RESUME) {
1199                 fptr->fcip_flags &= ~(FCIP_SUSPENDED);
1200         } else {
1201                 mutex_exit(&fptr->fcip_mutex);
1202                 return (FC_FAILURE);
1203         }
1204 
1205         /*
1206          * set the current port state and topology
1207          */
1208         fport->fcipp_topology = port_info->port_flags;
1209         fport->fcipp_pstate = port_info->port_state;
1210 
1211         rw_enter(&fcipstruplock, RW_READER);
1212         for (tslp = fcipstrup; tslp; tslp = tslp->sl_nextp) {
1213                 if (tslp->sl_fcip == fptr) {
1214                         break;
1215                 }
1216         }
1217         rw_exit(&fcipstruplock);
1218 
1219         /*
1220          * No active streams on this port
1221          */
1222         if (tslp == NULL) {
1223                 rval = FC_SUCCESS;
1224                 goto done;
1225         }
1226 
1227         mutex_enter(&fptr->fcip_rt_mutex);
1228         for (index = 0; index < FCIP_RT_HASH_ELEMS; index++) {
1229                 struct fcip_routing_table       *frp;
1230 
1231                 frp = fptr->fcip_rtable[index];
1232                 while (frp) {
1233                         uint32_t                did;
1234                         /*
1235                          * Mark the broadcast RTE available again. It
1236                          * was marked SUSPENDED during SUSPEND.
1237                          */
1238                         did = fcip_get_broadcast_did(fptr);
1239                         if (frp->fcipr_d_id.port_id == did) {
1240                                 frp->fcipr_state = 0;
1241                                 index = FCIP_RT_HASH_ELEMS;
1242                                 break;
1243                         }
1244                         frp = frp->fcipr_next;
1245                 }
1246         }
1247         mutex_exit(&fptr->fcip_rt_mutex);
1248 
1249         /*
1250          * fcip_handle_topology will update the port entries in the
1251          * routing table.
1252          * fcip_handle_topology also takes care of resetting the
1253          * fcipr_state field in the routing table structure. The entries
1254          * were set to RT_INVALID during suspend.
1255          */
1256         fcip_handle_topology(fptr);
1257 
1258 done:
1259         /*
1260          * Restart the timeout thread
1261          */
1262         fptr->fcip_timeout_id = timeout(fcip_timeout, fptr,
1263             drv_sectohz(1));
1264         mutex_exit(&fptr->fcip_mutex);
1265         return (rval);
1266 }
1267 
1268 
1269 /*
1270  * Insert a destination port entry into the routing table for
1271  * this port
1272  */
1273 static void
1274 fcip_rt_update(struct fcip *fptr, fc_portmap_t *devlist, uint32_t listlen)
1275 {
1276         struct fcip_routing_table       *frp;
1277         fcip_port_info_t                *fport = fptr->fcip_port_info;
1278         int                             hash_bucket, i;
1279         fc_portmap_t                    *pmap;
1280         char                            wwn_buf[20];
1281 
1282         FCIP_TNF_PROBE_2((fcip_rt_update, "fcip io", /* CSTYLED */,
1283                 tnf_string, msg, "enter",
1284                 tnf_int, listlen, listlen));
1285 
1286         ASSERT(!mutex_owned(&fptr->fcip_mutex));
1287         mutex_enter(&fptr->fcip_rt_mutex);
1288 
1289         for (i = 0; i < listlen; i++) {
1290                 pmap = &(devlist[i]);
1291 
1292                 frp = fcip_lookup_rtable(fptr, &(pmap->map_pwwn),
1293                     FCIP_COMPARE_PWWN);
1294                 /*
1295                  * If an entry for a port in the devlist exists in the
1296                  * in the per port routing table, make sure the data
1297                  * is current. We need to do this irrespective of the
1298                  * underlying port topology.
1299                  */
1300                 switch (pmap->map_type) {
1301                 /* FALLTHROUGH */
1302                 case PORT_DEVICE_NOCHANGE:
1303                 /* FALLTHROUGH */
1304                 case PORT_DEVICE_USER_LOGIN:
1305                 /* FALLTHROUGH */
1306                 case PORT_DEVICE_CHANGED:
1307                 /* FALLTHROUGH */
1308                 case PORT_DEVICE_NEW:
1309                         if (frp == NULL) {
1310                                 goto add_new_entry;
1311                         } else if (frp) {
1312                                 goto update_entry;
1313                         } else {
1314                                 continue;
1315                         }
1316 
1317                 case PORT_DEVICE_OLD:
1318                 /* FALLTHROUGH */
1319                 case PORT_DEVICE_USER_LOGOUT:
1320                         /*
1321                          * Mark entry for removal from Routing Table if
1322                          * one exists. Let the timeout thread actually
1323                          * remove the entry after we've given up hopes
1324                          * of the port ever showing up.
1325                          */
1326                         if (frp) {
1327                                 uint32_t                did;
1328 
1329                                 /*
1330                                  * Mark the routing table as invalid to bail
1331                                  * the packets early that are in transit
1332                                  */
1333                                 did = fptr->fcip_broadcast_did;
1334                                 if (frp->fcipr_d_id.port_id != did) {
1335                                         frp->fcipr_pd = NULL;
1336                                         frp->fcipr_state = FCIP_RT_INVALID;
1337                                         frp->fcipr_invalid_timeout =
1338                                             fptr->fcip_timeout_ticks +
1339                                             FCIP_RTE_TIMEOUT;
1340                                 }
1341                         }
1342                         continue;
1343 
1344                 default:
1345                         FCIP_DEBUG(FCIP_DEBUG_INIT, (CE_WARN,
1346                             "unknown map flags in rt_update"));
1347                         continue;
1348                 }
1349 add_new_entry:
1350                 ASSERT(frp == NULL);
1351                 hash_bucket = FCIP_RT_HASH(pmap->map_pwwn.raw_wwn);
1352 
1353                 ASSERT(hash_bucket < FCIP_RT_HASH_ELEMS);
1354 
1355                 FCIP_TNF_PROBE_2((fcip_rt_update, "cfip io", /* CSTYLED */,
1356                         tnf_string, msg,
1357                         "add new entry",
1358                         tnf_int, hashbucket, hash_bucket));
1359 
1360                 frp = (struct fcip_routing_table *)
1361                     kmem_zalloc(sizeof (struct fcip_routing_table), KM_SLEEP);
1362                 /* insert at beginning of hash bucket */
1363                 frp->fcipr_next = fptr->fcip_rtable[hash_bucket];
1364                 fptr->fcip_rtable[hash_bucket] = frp;
1365                 fc_wwn_to_str(&pmap->map_pwwn, wwn_buf);
1366                 FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_NOTE,
1367                     "added entry for pwwn %s and d_id 0x%x",
1368                     wwn_buf, pmap->map_did.port_id));
1369 update_entry:
1370                 bcopy((void *)&pmap->map_pwwn,
1371                     (void *)&frp->fcipr_pwwn, sizeof (la_wwn_t));
1372                 bcopy((void *)&pmap->map_nwwn, (void *)&frp->fcipr_nwwn,
1373                     sizeof (la_wwn_t));
1374                 frp->fcipr_d_id = pmap->map_did;
1375                 frp->fcipr_state = pmap->map_state;
1376                 frp->fcipr_pd = pmap->map_pd;
1377 
1378                 /*
1379                  * If there is no pd for a destination port that is not
1380                  * a broadcast entry, the port is pretty much unusable - so
1381                  * mark the port for removal so we can try adding back the
1382                  * entry again.
1383                  */
1384                 if ((frp->fcipr_pd == NULL) &&
1385                     (frp->fcipr_d_id.port_id != fptr->fcip_broadcast_did)) {
1386                         frp->fcipr_state = PORT_DEVICE_INVALID;
1387                         frp->fcipr_invalid_timeout = fptr->fcip_timeout_ticks +
1388                             (FCIP_RTE_TIMEOUT / 2);
1389                 }
1390                 frp->fcipr_fca_dev =
1391                     fc_ulp_get_fca_device(fport->fcipp_handle, pmap->map_did);
1392 
1393                 /*
1394                  * login to the remote port. Don't worry about
1395                  * plogi failures for now
1396                  */
1397                 if (pmap->map_pd != NULL) {
1398                         (void) fcip_do_plogi(fptr, frp);
1399                 } else if (FC_TOP_EXTERNAL(fport->fcipp_topology)) {
1400                         fc_wwn_to_str(&frp->fcipr_pwwn, wwn_buf);
1401                         FCIP_DEBUG(FCIP_DEBUG_MISC, (CE_NOTE,
1402                             "logging into pwwn %s, d_id 0x%x",
1403                             wwn_buf, frp->fcipr_d_id.port_id));
1404                         (void) fcip_do_plogi(fptr, frp);
1405                 }
1406 
1407                 FCIP_TNF_BYTE_ARRAY(fcip_rt_update, "fcip io", "detail",
1408                         "new wwn in rt", pwwn,
1409                         &frp->fcipr_pwwn, sizeof (la_wwn_t));
1410         }
1411         mutex_exit(&fptr->fcip_rt_mutex);
1412 }
1413 
1414 
1415 /*
1416  * return a matching routing table entry for a given fcip instance
1417  */
1418 struct fcip_routing_table *
1419 fcip_lookup_rtable(struct fcip *fptr, la_wwn_t *wwn, int matchflag)
1420 {
1421         struct fcip_routing_table       *frp = NULL;
1422         int                             hash_bucket;
1423 
1424 
1425         FCIP_TNF_PROBE_1((fcip_lookup_rtable, "fcip io", /* CSTYLED */,
1426                 tnf_string, msg, "enter"));
1427         FCIP_TNF_BYTE_ARRAY(fcip_lookup_rtable, "fcip io", "detail",
1428                 "rtable lookup for", wwn,
1429                 &wwn->raw_wwn, sizeof (la_wwn_t));
1430         FCIP_TNF_PROBE_2((fcip_lookup_rtable, "fcip io", /* CSTYLED */,
1431                 tnf_string, msg, "match by",
1432                 tnf_int, matchflag, matchflag));
1433 
1434         ASSERT(mutex_owned(&fptr->fcip_rt_mutex));
1435 
1436         hash_bucket = FCIP_RT_HASH(wwn->raw_wwn);
1437         frp = fptr->fcip_rtable[hash_bucket];
1438         while (frp != NULL) {
1439 
1440                 FCIP_TNF_BYTE_ARRAY(fcip_lookup_rtable, "fcip io", "detail",
1441                         "rtable entry", nwwn,
1442                         &(frp->fcipr_nwwn.raw_wwn), sizeof (la_wwn_t));
1443 
1444                 if (fcip_wwn_compare(&frp->fcipr_pwwn, wwn, matchflag) == 0) {
1445                         break;
1446                 }
1447 
1448                 frp = frp->fcipr_next;
1449         }
1450         FCIP_TNF_PROBE_2((fcip_lookup_rtable, "fcip io", /* CSTYLED */,
1451                 tnf_string, msg, "lookup result",
1452                 tnf_opaque, frp, frp));
1453         return (frp);
1454 }
1455 
1456 /*
1457  * Attach of fcip under pseudo. The actual setup of the interface
1458  * actually happens in fcip_port_attach on a callback from the
1459  * transport. The port_attach callback however can proceed only
1460  * after the devinfo for fcip has been created under pseudo
1461  */
1462 static int
1463 fcip_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
1464 {
1465         switch ((int)cmd) {
1466 
1467         case DDI_ATTACH: {
1468                 ASSERT(fcip_module_dip == NULL);
1469                 fcip_module_dip = dip;
1470 
1471                 /*
1472                  * this call originates as a result of fcip's conf
1473                  * file entry and will result in a fcip instance being
1474                  * a child of pseudo. We should ensure here that the port
1475                  * driver (fp) has been loaded and initted since we would
1476                  * never get a port attach callback without fp being loaded.
1477                  * If we are unable to succesfully load and initalize fp -
1478                  * just fail this attach.
1479                  */
1480                 mutex_enter(&fcip_global_mutex);
1481 
1482                 FCIP_DEBUG(FCIP_DEBUG_ATTACH,
1483                     (CE_WARN, "global cv - signaling"));
1484 
1485                 cv_signal(&fcip_global_cv);
1486 
1487                 FCIP_DEBUG(FCIP_DEBUG_ATTACH,
1488                     (CE_WARN, "global cv - signaled"));
1489                 mutex_exit(&fcip_global_mutex);
1490                 return (DDI_SUCCESS);
1491         }
1492         case DDI_RESUME:
1493                 /*
1494                  * Resume appears trickier
1495                  */
1496                 return (DDI_SUCCESS);
1497         default:
1498                 return (DDI_FAILURE);
1499         }
1500 }
1501 
1502 
1503 /*
1504  * The detach entry point to permit unloading fcip. We make sure
1505  * there are no active streams before we proceed with the detach
1506  */
1507 /* ARGSUSED */
1508 static int
1509 fcip_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1510 {
1511         struct fcip             *fptr;
1512         fcip_port_info_t        *fport;
1513         int                     detached;
1514 
1515         switch (cmd) {
1516         case DDI_DETACH: {
1517                 /*
1518                  * If we got here, any active streams should have been
1519                  * unplumbed but check anyway
1520                  */
1521                 mutex_enter(&fcip_global_mutex);
1522                 if (fcipstrup != NULL) {
1523                         mutex_exit(&fcip_global_mutex);
1524                         return (DDI_FAILURE);
1525                 }
1526 
1527                 if (fcip_port_head != NULL) {
1528                         /*
1529                          * Check to see if we have unattached/unbound
1530                          * ports. If all the ports are unattached/unbound go
1531                          * ahead and unregister with the transport
1532                          */
1533                         fport = fcip_port_head;
1534                         while (fport != NULL) {
1535                                 fptr = fport->fcipp_fcip;
1536                                 if (fptr == NULL) {
1537                                         continue;
1538                                 }
1539                                 mutex_enter(&fptr->fcip_mutex);
1540                                 fptr->fcip_flags |= FCIP_DETACHING;
1541                                 if (fptr->fcip_ipq ||
1542                                     fptr->fcip_flags & (FCIP_IN_TIMEOUT |
1543                                     FCIP_IN_CALLBACK | FCIP_ATTACHING |
1544                                     FCIP_SUSPENDED | FCIP_POWER_DOWN |
1545                                     FCIP_REG_INPROGRESS)) {
1546                                         FCIP_TNF_PROBE_1((fcip_detach,
1547                                             "fcip io", /* CSTYLED */,
1548                                             tnf_string, msg,
1549                                             "fcip instance busy"));
1550 
1551                                         mutex_exit(&fptr->fcip_mutex);
1552                                         FCIP_DEBUG(FCIP_DEBUG_DETACH, (CE_WARN,
1553                                             "fcip instance busy"));
1554                                         break;
1555                                 }
1556                                 /*
1557                                  * Check for any outstanding pkts. If yes
1558                                  * fail the detach
1559                                  */
1560                                 mutex_enter(&fptr->fcip_dest_mutex);
1561                                 if (fcip_port_get_num_pkts(fptr) > 0) {
1562                                         mutex_exit(&fptr->fcip_dest_mutex);
1563                                         mutex_exit(&fptr->fcip_mutex);
1564                                         FCIP_DEBUG(FCIP_DEBUG_DETACH, (CE_WARN,
1565                                             "fcip instance busy - pkts "
1566                                             "pending"));
1567                                         break;
1568                                 }
1569                                 mutex_exit(&fptr->fcip_dest_mutex);
1570 
1571                                 mutex_enter(&fptr->fcip_rt_mutex);
1572                                 if (fcip_plogi_in_progress(fptr)) {
1573                                         mutex_exit(&fptr->fcip_rt_mutex);
1574                                         mutex_exit(&fptr->fcip_mutex);
1575                                         FCIP_DEBUG(FCIP_DEBUG_DETACH, (CE_WARN,
1576                                             "fcip instance busy - plogi in "
1577                                             "progress"));
1578                                         break;
1579                                 }
1580                                 mutex_exit(&fptr->fcip_rt_mutex);
1581 
1582                                 mutex_exit(&fptr->fcip_mutex);
1583                                 fport = fport->fcipp_next;
1584                         }
1585                         /*
1586                          * if fport is non NULL - we have active ports
1587                          */
1588                         if (fport != NULL) {
1589                                 /*
1590                                  * Remove the DETACHING flags on the ports
1591                                  */
1592                                 fport = fcip_port_head;
1593                                 while (fport != NULL) {
1594                                         fptr = fport->fcipp_fcip;
1595                                         mutex_enter(&fptr->fcip_mutex);
1596                                         fptr->fcip_flags &= ~(FCIP_DETACHING);
1597                                         mutex_exit(&fptr->fcip_mutex);
1598                                         fport = fport->fcipp_next;
1599                                 }
1600                                 mutex_exit(&fcip_global_mutex);
1601                                 return (DDI_FAILURE);
1602                         }
1603                 }
1604 
1605                 /*
1606                  * free up all softstate structures
1607                  */
1608                 fport = fcip_port_head;
1609                 while (fport != NULL) {
1610                         detached = 1;
1611 
1612                         fptr = fport->fcipp_fcip;
1613                         if (fptr) {
1614                                 mutex_enter(&fptr->fcip_mutex);
1615                                 /*
1616                                  * Check to see if somebody beat us to the
1617                                  * punch
1618                                  */
1619                                 detached = fptr->fcip_flags & FCIP_DETACHED;
1620                                 fptr->fcip_flags &= ~(FCIP_DETACHING);
1621                                 fptr->fcip_flags |= FCIP_DETACHED;
1622                                 mutex_exit(&fptr->fcip_mutex);
1623                         }
1624 
1625                         if (!detached) {
1626                                 fport = fcip_softstate_free(fport);
1627                         } else {
1628                                 /*
1629                                  * If the port was marked as detached
1630                                  * but it was still in the list, that
1631                                  * means another thread has marked it
1632                                  * but we got in while it released the
1633                                  * fcip_global_mutex in softstate_free.
1634                                  * Given that, we're still safe to use
1635                                  * fport->fcipp_next to find out what
1636                                  * the next port on the list is.
1637                                  */
1638                                 fport = fport->fcipp_next;
1639                         }
1640 
1641                         FCIP_DEBUG(FCIP_DEBUG_DETACH,
1642                             (CE_NOTE, "detaching port"));
1643 
1644                         FCIP_TNF_PROBE_1((fcip_detach,
1645                                 "fcip io", /* CSTYLED */, tnf_string,
1646                                 msg, "detaching port"));
1647                 }
1648 
1649                 /*
1650                  * If we haven't removed all the port structures, we
1651                  * aren't yet ready to be detached.
1652                  */
1653                 if (fcip_port_head != NULL) {
1654                         mutex_exit(&fcip_global_mutex);
1655                         return (DDI_FAILURE);
1656                 }
1657 
1658                 fcip_num_instances = 0;
1659                 mutex_exit(&fcip_global_mutex);
1660                 fcip_module_dip = NULL;
1661                 return (DDI_SUCCESS);
1662         }
1663         case DDI_SUSPEND:
1664                 return (DDI_SUCCESS);
1665         default:
1666                 return (DDI_FAILURE);
1667         }
1668 }
1669 
1670 /*
1671  * The port_detach callback is called from the transport when a
1672  * FC port is being removed from the transport's control. This routine
1673  * provides fcip with an opportunity to cleanup all activities and
1674  * structures on the port marked for removal.
1675  */
1676 /* ARGSUSED */
1677 static int
1678 fcip_port_detach(opaque_t ulp_handle, fc_ulp_port_info_t *port_info,
1679     fc_detach_cmd_t cmd)
1680 {
1681         int                     rval = FC_FAILURE;
1682         fcip_port_info_t        *fport;
1683         struct fcip             *fptr;
1684         struct fcipstr          *strp;
1685 
1686         switch (cmd) {
1687         case FC_CMD_DETACH: {
1688                 mutex_enter(&fcip_global_mutex);
1689 
1690                 if (fcip_port_head == NULL) {
1691                         /*
1692                          * we are all done but our fini has not been
1693                          * called yet!! Let's hope we have no active
1694                          * fcip instances here. - strange secnario but
1695                          * no harm in having this return a success.
1696                          */
1697                         fcip_check_remove_minor_node();
1698 
1699                         mutex_exit(&fcip_global_mutex);
1700                         return (FC_SUCCESS);
1701                 } else {
1702                         /*
1703                          * traverse the port list
1704                          */
1705                         fport = fcip_port_head;
1706                         while (fport != NULL) {
1707                                 if (fport->fcipp_handle ==
1708                                     port_info->port_handle) {
1709                                         fptr = fport->fcipp_fcip;
1710 
1711                                         /*
1712                                          * Fail the port detach if there is
1713                                          * still an attached, bound stream on
1714                                          * this interface.
1715                                          */
1716 
1717                                         rw_enter(&fcipstruplock, RW_READER);
1718 
1719                                         for (strp = fcipstrup; strp != NULL;
1720                                             strp = strp->sl_nextp) {
1721                                                 if (strp->sl_fcip == fptr) {
1722                                                         rw_exit(&fcipstruplock);
1723                                                         mutex_exit(
1724                                                             &fcip_global_mutex);
1725                                                         return (FC_FAILURE);
1726                                                 }
1727                                         }
1728 
1729                                         rw_exit(&fcipstruplock);
1730 
1731                                         /*
1732                                          * fail port detach if we are in
1733                                          * the middle of a deferred port attach
1734                                          * or if the port has outstanding pkts
1735                                          */
1736                                         if (fptr != NULL) {
1737                                                 mutex_enter(&fptr->fcip_mutex);
1738                                                 if (fcip_check_port_busy
1739                                                     (fptr) ||
1740                                                     (fptr->fcip_flags &
1741                                                     FCIP_DETACHED)) {
1742                                                         mutex_exit(
1743                                                             &fptr->fcip_mutex);
1744                                                         mutex_exit(
1745                                                             &fcip_global_mutex);
1746                                                         return (FC_FAILURE);
1747                                                 }
1748 
1749                                                 fptr->fcip_flags |=
1750                                                     FCIP_DETACHED;
1751                                                 mutex_exit(&fptr->fcip_mutex);
1752                                         }
1753                                         (void) fcip_softstate_free(fport);
1754 
1755                                         fcip_check_remove_minor_node();
1756                                         mutex_exit(&fcip_global_mutex);
1757                                         return (FC_SUCCESS);
1758                                 }
1759                                 fport = fport->fcipp_next;
1760                         }
1761                         ASSERT(fport == NULL);
1762                 }
1763                 mutex_exit(&fcip_global_mutex);
1764                 break;
1765         }
1766         case FC_CMD_POWER_DOWN:
1767         /* FALLTHROUGH */
1768         case FC_CMD_SUSPEND:
1769                 mutex_enter(&fcip_global_mutex);
1770                 fport = fcip_port_head;
1771                 while (fport != NULL) {
1772                         if (fport->fcipp_handle == port_info->port_handle) {
1773                                 break;
1774                         }
1775                         fport = fport->fcipp_next;
1776                 }
1777                 if (fport == NULL) {
1778                         mutex_exit(&fcip_global_mutex);
1779                         break;
1780                 }
1781                 rval = fcip_handle_suspend(fport, cmd);
1782                 mutex_exit(&fcip_global_mutex);
1783                 break;
1784         default:
1785                 FCIP_DEBUG(FCIP_DEBUG_DETACH,
1786                     (CE_WARN, "unknown port detach command!!"));
1787                 break;
1788         }
1789         return (rval);
1790 }
1791 
1792 
1793 /*
1794  * Returns 0 if the port is not busy, else returns non zero.
1795  */
1796 static int
1797 fcip_check_port_busy(struct fcip *fptr)
1798 {
1799         int rval = 0, num_pkts = 0;
1800 
1801         ASSERT(fptr != NULL);
1802         ASSERT(MUTEX_HELD(&fptr->fcip_mutex));
1803 
1804         mutex_enter(&fptr->fcip_dest_mutex);
1805 
1806         if (fptr->fcip_flags & FCIP_PORT_BUSY ||
1807             ((num_pkts = fcip_port_get_num_pkts(fptr)) > 0) ||
1808             fptr->fcip_num_ipkts_pending) {
1809                 rval = 1;
1810                 FCIP_DEBUG(FCIP_DEBUG_DETACH,
1811                     (CE_NOTE, "!fcip_check_port_busy: port is busy "
1812                     "fcip_flags: 0x%x, num_pkts: 0x%x, ipkts_pending: 0x%lx!",
1813                     fptr->fcip_flags, num_pkts, fptr->fcip_num_ipkts_pending));
1814         }
1815 
1816         mutex_exit(&fptr->fcip_dest_mutex);
1817         return (rval);
1818 }
1819 
1820 /*
1821  * Helper routine to remove fcip's minor node
1822  * There is one minor node per system and it should be removed if there are no
1823  * other fcip instances (which has a 1:1 mapping for fp instances) present
1824  */
1825 static void
1826 fcip_check_remove_minor_node(void)
1827 {
1828         ASSERT(MUTEX_HELD(&fcip_global_mutex));
1829 
1830         /*
1831          * If there are no more fcip (fp) instances, remove the
1832          * minor node for fcip.
1833          * Reset fcip_minor_node_created to invalidate it.
1834          */
1835         if (fcip_num_instances == 0 && (fcip_module_dip != NULL)) {
1836                 ddi_remove_minor_node(fcip_module_dip, NULL);
1837                 fcip_minor_node_created = 0;
1838         }
1839 }
1840 
1841 /*
1842  * This routine permits the suspend operation during a CPR/System
1843  * power management operation. The routine basically quiesces I/Os
1844  * on all active interfaces
1845  */
1846 static int
1847 fcip_handle_suspend(fcip_port_info_t *fport, fc_detach_cmd_t cmd)
1848 {
1849         struct fcip     *fptr = fport->fcipp_fcip;
1850         timeout_id_t    tid;
1851         int             index;
1852         int             tryagain = 0;
1853         int             count;
1854         struct fcipstr  *tslp;
1855 
1856 
1857         ASSERT(fptr != NULL);
1858         mutex_enter(&fptr->fcip_mutex);
1859 
1860         /*
1861          * Fail if we are in the middle of a callback. Don't use delay during
1862          * suspend since clock intrs are not available so busy wait
1863          */
1864         count = 0;
1865         while (count++ < 15 &&
1866             ((fptr->fcip_flags & FCIP_IN_CALLBACK) ||
1867             (fptr->fcip_flags & FCIP_IN_TIMEOUT))) {
1868                 mutex_exit(&fptr->fcip_mutex);
1869                 drv_usecwait(1000000);
1870                 mutex_enter(&fptr->fcip_mutex);
1871         }
1872 
1873         if (fptr->fcip_flags & FCIP_IN_CALLBACK ||
1874             fptr->fcip_flags & FCIP_IN_TIMEOUT) {
1875                 mutex_exit(&fptr->fcip_mutex);
1876                 return (FC_FAILURE);
1877         }
1878 
1879         if (cmd == FC_CMD_POWER_DOWN) {
1880                 if (fptr->fcip_flags & FCIP_SUSPENDED) {
1881                         fptr->fcip_flags |= FCIP_POWER_DOWN;
1882                         mutex_exit(&fptr->fcip_mutex);
1883                         goto success;
1884                 } else {
1885                         fptr->fcip_flags |= FCIP_POWER_DOWN;
1886                 }
1887         } else if (cmd == FC_CMD_SUSPEND) {
1888                 fptr->fcip_flags |= FCIP_SUSPENDED;
1889         } else {
1890                 mutex_exit(&fptr->fcip_mutex);
1891                 return (FC_FAILURE);
1892         }
1893 
1894         mutex_exit(&fptr->fcip_mutex);
1895         /*
1896          * If no streams are plumbed - its the easiest case - Just
1897          * bail out without having to do much
1898          */
1899 
1900         rw_enter(&fcipstruplock, RW_READER);
1901         for (tslp = fcipstrup; tslp; tslp = tslp->sl_nextp) {
1902                 if (tslp->sl_fcip == fptr) {
1903                         break;
1904                 }
1905         }
1906         rw_exit(&fcipstruplock);
1907 
1908         /*
1909          * No active streams on this port
1910          */
1911         if (tslp == NULL) {
1912                 goto success;
1913         }
1914 
1915         /*
1916          * Walk through each Routing table structure and check if
1917          * the destination table has any outstanding commands. If yes
1918          * wait for the commands to drain. Since we go through each
1919          * routing table entry in succession, it may be wise to wait
1920          * only a few seconds for each entry.
1921          */
1922         mutex_enter(&fptr->fcip_rt_mutex);
1923         while (!tryagain) {
1924 
1925                 tryagain = 0;
1926                 for (index = 0; index < FCIP_RT_HASH_ELEMS; index++) {
1927                         struct fcip_routing_table       *frp;
1928                         struct fcip_dest                *fdestp;
1929                         la_wwn_t                        *pwwn;
1930                         int                             hash_bucket;
1931 
1932                         frp = fptr->fcip_rtable[index];
1933                         while (frp) {
1934                                 /*
1935                                  * Mark the routing table as SUSPENDED. Even
1936                                  * mark the broadcast entry SUSPENDED to
1937                                  * prevent any ARP or other broadcasts. We
1938                                  * can reset the state of the broadcast
1939                                  * RTE when we resume.
1940                                  */
1941                                 frp->fcipr_state = FCIP_RT_SUSPENDED;
1942                                 pwwn = &frp->fcipr_pwwn;
1943 
1944                                 /*
1945                                  * Get hold of destination pointer
1946                                  */
1947                                 mutex_enter(&fptr->fcip_dest_mutex);
1948 
1949                                 hash_bucket = FCIP_DEST_HASH(pwwn->raw_wwn);
1950                                 ASSERT(hash_bucket < FCIP_DEST_HASH_ELEMS);
1951 
1952                                 fdestp = fptr->fcip_dest[hash_bucket];
1953                                 while (fdestp != NULL) {
1954                                         mutex_enter(&fdestp->fcipd_mutex);
1955                                         if (fdestp->fcipd_rtable) {
1956                                                 if (fcip_wwn_compare(pwwn,
1957                                                     &fdestp->fcipd_pwwn,
1958                                                     FCIP_COMPARE_PWWN) == 0) {
1959                                                         mutex_exit(
1960                                                         &fdestp->fcipd_mutex);
1961                                                         break;
1962                                                 }
1963                                         }
1964                                         mutex_exit(&fdestp->fcipd_mutex);
1965                                         fdestp = fdestp->fcipd_next;
1966                                 }
1967 
1968                                 mutex_exit(&fptr->fcip_dest_mutex);
1969                                 if (fdestp == NULL) {
1970                                         frp = frp->fcipr_next;
1971                                         continue;
1972                                 }
1973 
1974                                 /*
1975                                  * Wait for fcip_wait_cmds seconds for
1976                                  * the commands to drain.
1977                                  */
1978                                 count = 0;
1979                                 mutex_enter(&fdestp->fcipd_mutex);
1980                                 while (fdestp->fcipd_ncmds &&
1981                                     count < fcip_wait_cmds) {
1982                                         mutex_exit(&fdestp->fcipd_mutex);
1983                                         mutex_exit(&fptr->fcip_rt_mutex);
1984                                         drv_usecwait(1000000);
1985                                         mutex_enter(&fptr->fcip_rt_mutex);
1986                                         mutex_enter(&fdestp->fcipd_mutex);
1987                                         count++;
1988                                 }
1989                                 /*
1990                                  * Check if we were able to drain all cmds
1991                                  * successfully. Else continue with other
1992                                  * ports and try during the second pass
1993                                  */
1994                                 if (fdestp->fcipd_ncmds) {
1995                                         tryagain++;
1996                                 }
1997                                 mutex_exit(&fdestp->fcipd_mutex);
1998 
1999                                 frp = frp->fcipr_next;
2000                         }
2001                 }
2002                 if (tryagain == 0) {
2003                         break;
2004                 }
2005         }
2006         mutex_exit(&fptr->fcip_rt_mutex);
2007 
2008         if (tryagain) {
2009                 mutex_enter(&fptr->fcip_mutex);
2010                 fptr->fcip_flags &= ~(FCIP_SUSPENDED | FCIP_POWER_DOWN);
2011                 mutex_exit(&fptr->fcip_mutex);
2012                 return (FC_FAILURE);
2013         }
2014 
2015 success:
2016         mutex_enter(&fptr->fcip_mutex);
2017         tid = fptr->fcip_timeout_id;
2018         fptr->fcip_timeout_id = NULL;
2019         mutex_exit(&fptr->fcip_mutex);
2020 
2021         (void) untimeout(tid);
2022 
2023         return (FC_SUCCESS);
2024 }
2025 
2026 /*
2027  * the getinfo(9E) entry point
2028  */
2029 /* ARGSUSED */
2030 static int
2031 fcip_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
2032 {
2033         int rval = DDI_FAILURE;
2034 
2035         switch (cmd) {
2036         case DDI_INFO_DEVT2DEVINFO:
2037                 *result = fcip_module_dip;
2038                 if (*result)
2039                         rval = DDI_SUCCESS;
2040                 break;
2041 
2042         case DDI_INFO_DEVT2INSTANCE:
2043                 *result = (void *)0;
2044                 rval = DDI_SUCCESS;
2045                 break;
2046         default:
2047                 break;
2048         }
2049 
2050         return (rval);
2051 }
2052 
2053 /*
2054  * called from fcip_attach to initialize kstats for the link
2055  */
2056 /* ARGSUSED */
2057 static void
2058 fcip_kstat_init(struct fcip *fptr)
2059 {
2060         int instance;
2061         char buf[16];
2062         struct fcipstat *fcipstatp;
2063 
2064         ASSERT(mutex_owned(&fptr->fcip_mutex));
2065 
2066         instance = ddi_get_instance(fptr->fcip_dip);
2067         (void) sprintf(buf, "fcip%d", instance);
2068 
2069 #ifdef  kstat
2070         fptr->fcip_kstatp = kstat_create("fcip", instance, buf, "net",
2071             KSTAT_TYPE_NAMED,
2072             (sizeof (struct fcipstat)/ sizeof (kstat_named_t)),
2073             KSTAT_FLAG_PERSISTENT);
2074 #else
2075         fptr->fcip_kstatp = kstat_create("fcip", instance, buf, "net",
2076             KSTAT_TYPE_NAMED,
2077             (sizeof (struct fcipstat)/ sizeof (kstat_named_t)), 0);
2078 #endif
2079         if (fptr->fcip_kstatp == NULL) {
2080                 FCIP_DEBUG(FCIP_DEBUG_INIT, (CE_WARN, "kstat created failed"));
2081                 return;
2082         }
2083 
2084         fcipstatp = (struct  fcipstat *)fptr->fcip_kstatp->ks_data;
2085         kstat_named_init(&fcipstatp->fcips_ipackets,     "ipackets",
2086                 KSTAT_DATA_ULONG);
2087         kstat_named_init(&fcipstatp->fcips_ierrors,      "ierrors",
2088                 KSTAT_DATA_ULONG);
2089         kstat_named_init(&fcipstatp->fcips_opackets,     "opackets",
2090                 KSTAT_DATA_ULONG);
2091         kstat_named_init(&fcipstatp->fcips_oerrors,      "oerrors",
2092                 KSTAT_DATA_ULONG);
2093         kstat_named_init(&fcipstatp->fcips_collisions,   "collisions",
2094                 KSTAT_DATA_ULONG);
2095         kstat_named_init(&fcipstatp->fcips_nocanput,     "nocanput",
2096                 KSTAT_DATA_ULONG);
2097         kstat_named_init(&fcipstatp->fcips_allocbfail,   "allocbfail",
2098                 KSTAT_DATA_ULONG);
2099 
2100         kstat_named_init(&fcipstatp->fcips_defer, "defer",
2101                 KSTAT_DATA_ULONG);
2102         kstat_named_init(&fcipstatp->fcips_fram, "fram",
2103                 KSTAT_DATA_ULONG);
2104         kstat_named_init(&fcipstatp->fcips_crc, "crc",
2105                 KSTAT_DATA_ULONG);
2106         kstat_named_init(&fcipstatp->fcips_oflo, "oflo",
2107                 KSTAT_DATA_ULONG);
2108         kstat_named_init(&fcipstatp->fcips_uflo, "uflo",
2109                 KSTAT_DATA_ULONG);
2110         kstat_named_init(&fcipstatp->fcips_missed, "missed",
2111                 KSTAT_DATA_ULONG);
2112         kstat_named_init(&fcipstatp->fcips_tlcol, "tlcol",
2113                 KSTAT_DATA_ULONG);
2114         kstat_named_init(&fcipstatp->fcips_trtry, "trtry",
2115                 KSTAT_DATA_ULONG);
2116         kstat_named_init(&fcipstatp->fcips_tnocar, "tnocar",
2117                 KSTAT_DATA_ULONG);
2118         kstat_named_init(&fcipstatp->fcips_inits, "inits",
2119                 KSTAT_DATA_ULONG);
2120         kstat_named_init(&fcipstatp->fcips_notbufs, "notbufs",
2121                 KSTAT_DATA_ULONG);
2122         kstat_named_init(&fcipstatp->fcips_norbufs, "norbufs",
2123                 KSTAT_DATA_ULONG);
2124         kstat_named_init(&fcipstatp->fcips_allocbfail, "allocbfail",
2125                 KSTAT_DATA_ULONG);
2126 
2127         /*
2128          * required by kstat for MIB II objects(RFC 1213)
2129          */
2130         kstat_named_init(&fcipstatp->fcips_rcvbytes, "fcips_rcvbytes",
2131                 KSTAT_DATA_ULONG);      /* # octets received */
2132                                         /* MIB - ifInOctets */
2133         kstat_named_init(&fcipstatp->fcips_xmtbytes, "fcips_xmtbytes",
2134                 KSTAT_DATA_ULONG);      /* # octets xmitted */
2135                                         /* MIB - ifOutOctets */
2136         kstat_named_init(&fcipstatp->fcips_multircv,     "fcips_multircv",
2137                 KSTAT_DATA_ULONG);      /* # multicast packets */
2138                                         /* delivered to upper layer */
2139                                         /* MIB - ifInNUcastPkts */
2140         kstat_named_init(&fcipstatp->fcips_multixmt,     "fcips_multixmt",
2141                 KSTAT_DATA_ULONG);      /* # multicast packets */
2142                                         /* requested to be sent */
2143                                         /* MIB - ifOutNUcastPkts */
2144         kstat_named_init(&fcipstatp->fcips_brdcstrcv, "fcips_brdcstrcv",
2145                 KSTAT_DATA_ULONG); /* # broadcast packets */
2146                                         /* delivered to upper layer */
2147                                         /* MIB - ifInNUcastPkts */
2148         kstat_named_init(&fcipstatp->fcips_brdcstxmt, "fcips_brdcstxmt",
2149                 KSTAT_DATA_ULONG);      /* # broadcast packets */
2150                                         /* requested to be sent */
2151                                         /* MIB - ifOutNUcastPkts */
2152         kstat_named_init(&fcipstatp->fcips_norcvbuf,     "fcips_norcvbuf",
2153                 KSTAT_DATA_ULONG);      /* # rcv packets discarded */
2154                                         /* MIB - ifInDiscards */
2155         kstat_named_init(&fcipstatp->fcips_noxmtbuf,     "fcips_noxmtbuf",
2156                 KSTAT_DATA_ULONG);      /* # xmt packets discarded */
2157 
2158         fptr->fcip_kstatp->ks_update = fcip_stat_update;
2159         fptr->fcip_kstatp->ks_private = (void *) fptr;
2160         kstat_install(fptr->fcip_kstatp);
2161 }
2162 
2163 /*
2164  * Update the defined kstats for netstat et al to use
2165  */
2166 /* ARGSUSED */
2167 static int
2168 fcip_stat_update(kstat_t *fcip_statp, int val)
2169 {
2170         struct fcipstat *fcipstatp;
2171         struct fcip     *fptr;
2172 
2173         fptr = (struct fcip *)fcip_statp->ks_private;
2174         fcipstatp = (struct fcipstat *)fcip_statp->ks_data;
2175 
2176         if (val == KSTAT_WRITE) {
2177                 fptr->fcip_ipackets  = fcipstatp->fcips_ipackets.value.ul;
2178                 fptr->fcip_ierrors   = fcipstatp->fcips_ierrors.value.ul;
2179                 fptr->fcip_opackets  = fcipstatp->fcips_opackets.value.ul;
2180                 fptr->fcip_oerrors   = fcipstatp->fcips_oerrors.value.ul;
2181                 fptr->fcip_collisions        = fcipstatp->fcips_collisions.value.ul;
2182                 fptr->fcip_defer     = fcipstatp->fcips_defer.value.ul;
2183                 fptr->fcip_fram      = fcipstatp->fcips_fram.value.ul;
2184                 fptr->fcip_crc       = fcipstatp->fcips_crc.value.ul;
2185                 fptr->fcip_oflo      = fcipstatp->fcips_oflo.value.ul;
2186                 fptr->fcip_uflo      = fcipstatp->fcips_uflo.value.ul;
2187                 fptr->fcip_missed    = fcipstatp->fcips_missed.value.ul;
2188                 fptr->fcip_tlcol     = fcipstatp->fcips_tlcol.value.ul;
2189                 fptr->fcip_trtry     = fcipstatp->fcips_trtry.value.ul;
2190                 fptr->fcip_tnocar    = fcipstatp->fcips_tnocar.value.ul;
2191                 fptr->fcip_inits     = fcipstatp->fcips_inits.value.ul;
2192                 fptr->fcip_notbufs   = fcipstatp->fcips_notbufs.value.ul;
2193                 fptr->fcip_norbufs   = fcipstatp->fcips_norbufs.value.ul;
2194                 fptr->fcip_nocanput  = fcipstatp->fcips_nocanput.value.ul;
2195                 fptr->fcip_allocbfail        = fcipstatp->fcips_allocbfail.value.ul;
2196                 fptr->fcip_rcvbytes  = fcipstatp->fcips_rcvbytes.value.ul;
2197                 fptr->fcip_xmtbytes  = fcipstatp->fcips_xmtbytes.value.ul;
2198                 fptr->fcip_multircv  = fcipstatp->fcips_multircv.value.ul;
2199                 fptr->fcip_multixmt  = fcipstatp->fcips_multixmt.value.ul;
2200                 fptr->fcip_brdcstrcv = fcipstatp->fcips_brdcstrcv.value.ul;
2201                 fptr->fcip_norcvbuf  = fcipstatp->fcips_norcvbuf.value.ul;
2202                 fptr->fcip_noxmtbuf  = fcipstatp->fcips_noxmtbuf.value.ul;
2203                 fptr->fcip_allocbfail        = fcipstatp->fcips_allocbfail.value.ul;
2204                 fptr->fcip_allocbfail        = fcipstatp->fcips_allocbfail.value.ul;
2205                 fptr->fcip_allocbfail        = fcipstatp->fcips_allocbfail.value.ul;
2206                 fptr->fcip_allocbfail        = fcipstatp->fcips_allocbfail.value.ul;
2207                 fptr->fcip_allocbfail        = fcipstatp->fcips_allocbfail.value.ul;
2208                 fptr->fcip_allocbfail        = fcipstatp->fcips_allocbfail.value.ul;
2209                 fptr->fcip_allocbfail        = fcipstatp->fcips_allocbfail.value.ul;
2210                 fptr->fcip_allocbfail        = fcipstatp->fcips_allocbfail.value.ul;
2211 
2212         } else {
2213                 fcipstatp->fcips_ipackets.value.ul   = fptr->fcip_ipackets;
2214                 fcipstatp->fcips_ierrors.value.ul    = fptr->fcip_ierrors;
2215                 fcipstatp->fcips_opackets.value.ul   = fptr->fcip_opackets;
2216                 fcipstatp->fcips_oerrors.value.ul    = fptr->fcip_oerrors;
2217                 fcipstatp->fcips_collisions.value.ul = fptr->fcip_collisions;
2218                 fcipstatp->fcips_nocanput.value.ul   = fptr->fcip_nocanput;
2219                 fcipstatp->fcips_allocbfail.value.ul = fptr->fcip_allocbfail;
2220                 fcipstatp->fcips_defer.value.ul      = fptr->fcip_defer;
2221                 fcipstatp->fcips_fram.value.ul       = fptr->fcip_fram;
2222                 fcipstatp->fcips_crc.value.ul        = fptr->fcip_crc;
2223                 fcipstatp->fcips_oflo.value.ul       = fptr->fcip_oflo;
2224                 fcipstatp->fcips_uflo.value.ul       = fptr->fcip_uflo;
2225                 fcipstatp->fcips_missed.value.ul     = fptr->fcip_missed;
2226                 fcipstatp->fcips_tlcol.value.ul      = fptr->fcip_tlcol;
2227                 fcipstatp->fcips_trtry.value.ul      = fptr->fcip_trtry;
2228                 fcipstatp->fcips_tnocar.value.ul     = fptr->fcip_tnocar;
2229                 fcipstatp->fcips_inits.value.ul      = fptr->fcip_inits;
2230                 fcipstatp->fcips_norbufs.value.ul    = fptr->fcip_norbufs;
2231                 fcipstatp->fcips_notbufs.value.ul    = fptr->fcip_notbufs;
2232                 fcipstatp->fcips_rcvbytes.value.ul   = fptr->fcip_rcvbytes;
2233                 fcipstatp->fcips_xmtbytes.value.ul   = fptr->fcip_xmtbytes;
2234                 fcipstatp->fcips_multircv.value.ul   = fptr->fcip_multircv;
2235                 fcipstatp->fcips_multixmt.value.ul   = fptr->fcip_multixmt;
2236                 fcipstatp->fcips_brdcstrcv.value.ul  = fptr->fcip_brdcstrcv;
2237                 fcipstatp->fcips_brdcstxmt.value.ul  = fptr->fcip_brdcstxmt;
2238                 fcipstatp->fcips_norcvbuf.value.ul   = fptr->fcip_norcvbuf;
2239                 fcipstatp->fcips_noxmtbuf.value.ul   = fptr->fcip_noxmtbuf;
2240 
2241         }
2242         return (0);
2243 }
2244 
2245 
2246 /*
2247  * fcip_statec_cb: handles all required state change callback notifications
2248  * it receives from the transport
2249  */
2250 /* ARGSUSED */
2251 static void
2252 fcip_statec_cb(opaque_t ulp_handle, opaque_t phandle,
2253     uint32_t port_state, uint32_t port_top, fc_portmap_t changelist[],
2254     uint32_t listlen, uint32_t sid)
2255 {
2256         fcip_port_info_t        *fport;
2257         struct fcip             *fptr;
2258         struct fcipstr          *slp;
2259         queue_t                 *wrq;
2260         int                     instance;
2261         int                     index;
2262         struct fcip_routing_table       *frtp;
2263 
2264         fport = fcip_get_port(phandle);
2265 
2266         if (fport == NULL) {
2267                 return;
2268         }
2269 
2270         fptr = fport->fcipp_fcip;
2271         ASSERT(fptr != NULL);
2272 
2273         if (fptr == NULL) {
2274                 return;
2275         }
2276 
2277         instance = ddi_get_instance(fport->fcipp_dip);
2278 
2279         FCIP_TNF_PROBE_4((fcip_statec_cb, "fcip io", /* CSTYLED */,
2280                 tnf_string, msg, "state change callback",
2281                 tnf_uint, instance, instance,
2282                 tnf_uint, S_ID, sid,
2283                 tnf_int, count, listlen));
2284         FCIP_DEBUG(FCIP_DEBUG_ELS,
2285             (CE_NOTE, "fcip%d, state change callback: state:0x%x, "
2286             "S_ID:0x%x, count:0x%x", instance, port_state, sid, listlen));
2287 
2288         mutex_enter(&fptr->fcip_mutex);
2289 
2290         if ((fptr->fcip_flags & (FCIP_DETACHING | FCIP_DETACHED)) ||
2291             (fptr->fcip_flags & (FCIP_SUSPENDED | FCIP_POWER_DOWN))) {
2292                 mutex_exit(&fptr->fcip_mutex);
2293                 return;
2294         }
2295 
2296         /*
2297          * set fcip flags to indicate we are in the middle of a
2298          * state change callback so we can wait till the statechange
2299          * is handled before succeeding/failing the SUSPEND/POWER DOWN.
2300          */
2301         fptr->fcip_flags |= FCIP_IN_SC_CB;
2302 
2303         fport->fcipp_pstate = port_state;
2304 
2305         /*
2306          * Check if topology changed. If Yes - Modify the broadcast
2307          * RTE entries to understand the new broadcast D_IDs
2308          */
2309         if (fport->fcipp_topology != port_top &&
2310             (port_top != FC_TOP_UNKNOWN)) {
2311                 /* REMOVE later */
2312                 FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_NOTE,
2313                     "topology changed: Old topology: 0x%x New topology 0x%x",
2314                     fport->fcipp_topology, port_top));
2315                 /*
2316                  * If topology changed - attempt a rediscovery of
2317                  * devices. Helps specially in Fabric/Public loops
2318                  * and if on_demand_node_creation is disabled
2319                  */
2320                 fport->fcipp_topology = port_top;
2321                 fcip_handle_topology(fptr);
2322         }
2323 
2324         mutex_exit(&fptr->fcip_mutex);
2325 
2326         switch (FC_PORT_STATE_MASK(port_state)) {
2327         case FC_STATE_ONLINE:
2328         /* FALLTHROUGH */
2329         case FC_STATE_LIP:
2330         /* FALLTHROUGH */
2331         case FC_STATE_LIP_LBIT_SET:
2332 
2333                 /*
2334                  * nothing to do here actually other than if we
2335                  * were actually logged onto a port in the devlist
2336                  * (which indicates active communication between
2337                  * the host port and the port in the changelist).
2338                  * If however we are in a private loop or point to
2339                  * point mode, we need to check for any IP capable
2340                  * ports and update our routing table.
2341                  */
2342                 switch (port_top) {
2343                 case FC_TOP_FABRIC:
2344                         /*
2345                          * This indicates a fabric port with a NameServer.
2346                          * Check the devlist to see if we are in active
2347                          * communication with a port on the devlist.
2348                          */
2349                         FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_NOTE,
2350                             "Statec_cb: fabric topology"));
2351                         fcip_rt_update(fptr, changelist, listlen);
2352                         break;
2353                 case FC_TOP_NO_NS:
2354                         /*
2355                          * No nameserver - so treat it like a Private loop
2356                          * or point to point topology and get a map of
2357                          * devices on the link and get IP capable ports to
2358                          * to update the routing table.
2359                          */
2360                         FCIP_DEBUG(FCIP_DEBUG_ELS,
2361                             (CE_NOTE, "Statec_cb: NO_NS topology"));
2362                 /* FALLTHROUGH */
2363                 case FC_TOP_PRIVATE_LOOP:
2364                         FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_NOTE,
2365                             "Statec_cb: Pvt_Loop topology"));
2366                 /* FALLTHROUGH */
2367                 case FC_TOP_PT_PT:
2368                         /*
2369                          * call get_port_map() and update routing table
2370                          */
2371                         fcip_rt_update(fptr, changelist, listlen);
2372                         break;
2373                 default:
2374                         FCIP_DEBUG(FCIP_DEBUG_ELS,
2375                             (CE_NOTE, "Statec_cb: Unknown topology"));
2376                 }
2377 
2378                 /*
2379                  * We should now enable the Queues and permit I/Os
2380                  * to flow through downstream. The update of routing
2381                  * table should have flushed out any port entries that
2382                  * don't exist or are not available after the state change
2383                  */
2384                 mutex_enter(&fptr->fcip_mutex);
2385                 fptr->fcip_port_state = FCIP_PORT_ONLINE;
2386                 if (fptr->fcip_flags & FCIP_LINK_DOWN) {
2387                         fptr->fcip_flags &= ~FCIP_LINK_DOWN;
2388                 }
2389                 mutex_exit(&fptr->fcip_mutex);
2390 
2391                 /*
2392                  * Enable write queues
2393                  */
2394                 rw_enter(&fcipstruplock, RW_READER);
2395                 for (slp = fcipstrup; slp != NULL; slp = slp->sl_nextp) {
2396                         if (slp && slp->sl_fcip == fptr) {
2397                                 wrq = WR(slp->sl_rq);
2398                                 if (wrq->q_flag & QFULL) {
2399                                         qenable(wrq);
2400                                 }
2401                         }
2402                 }
2403                 rw_exit(&fcipstruplock);
2404                 break;
2405         case FC_STATE_OFFLINE:
2406                 /*
2407                  * mark the port_state OFFLINE and wait for it to
2408                  * become online. Any new messages in this state will
2409                  * simply be queued back up. If the port does not
2410                  * come online in a short while, we can begin failing
2411                  * messages and flush the routing table
2412                  */
2413                 mutex_enter(&fptr->fcip_mutex);
2414                 fptr->fcip_mark_offline = fptr->fcip_timeout_ticks +
2415                     FCIP_OFFLINE_TIMEOUT;
2416                 fptr->fcip_port_state = FCIP_PORT_OFFLINE;
2417                 mutex_exit(&fptr->fcip_mutex);
2418 
2419                 /*
2420                  * Mark all Routing table entries as invalid to prevent
2421                  * any commands from trickling through to ports that
2422                  * have disappeared from under us
2423                  */
2424                 mutex_enter(&fptr->fcip_rt_mutex);
2425                 for (index = 0; index < FCIP_RT_HASH_ELEMS; index++) {
2426                         frtp = fptr->fcip_rtable[index];
2427                         while (frtp) {
2428                                 frtp->fcipr_state = PORT_DEVICE_INVALID;
2429                                 frtp = frtp->fcipr_next;
2430                         }
2431                 }
2432                 mutex_exit(&fptr->fcip_rt_mutex);
2433 
2434                 break;
2435 
2436         case FC_STATE_RESET_REQUESTED:
2437                 /*
2438                  * Release all Unsolicited buffers back to transport/FCA.
2439                  * This also means the port state is marked offline - so
2440                  * we may have to do what OFFLINE state requires us to do.
2441                  * Care must be taken to wait for any active unsolicited
2442                  * buffer with the other Streams modules - so wait for
2443                  * a freeb if the unsolicited buffer is passed back all
2444                  * the way upstream.
2445                  */
2446                 mutex_enter(&fptr->fcip_mutex);
2447 
2448 #ifdef FCIP_ESBALLOC
2449                 while (fptr->fcip_ub_upstream) {
2450                         cv_wait(&fptr->fcip_ub_cv, &fptr->fcip_mutex);
2451                 }
2452 #endif  /* FCIP_ESBALLOC */
2453 
2454                 fptr->fcip_mark_offline = fptr->fcip_timeout_ticks +
2455                     FCIP_OFFLINE_TIMEOUT;
2456                 fptr->fcip_port_state = FCIP_PORT_OFFLINE;
2457                 mutex_exit(&fptr->fcip_mutex);
2458                 break;
2459 
2460         case FC_STATE_DEVICE_CHANGE:
2461                 if (listlen) {
2462                         fcip_rt_update(fptr, changelist, listlen);
2463                 }
2464                 break;
2465         case FC_STATE_RESET:
2466                 /*
2467                  * Not much to do I guess - wait for port to become
2468                  * ONLINE. If the port doesn't become online in a short
2469                  * while, the upper layers abort any request themselves.
2470                  * We can just putback the messages in the streams queues
2471                  * if the link is offline
2472                  */
2473                 break;
2474         }
2475         mutex_enter(&fptr->fcip_mutex);
2476         fptr->fcip_flags &= ~(FCIP_IN_SC_CB);
2477         mutex_exit(&fptr->fcip_mutex);
2478 }
2479 
2480 /*
2481  * Given a port handle, return the fcip_port_info structure corresponding
2482  * to that port handle. The transport allocates and communicates with
2483  * ULPs using port handles
2484  */
2485 static fcip_port_info_t *
2486 fcip_get_port(opaque_t phandle)
2487 {
2488         fcip_port_info_t *fport;
2489 
2490         ASSERT(phandle != NULL);
2491 
2492         mutex_enter(&fcip_global_mutex);
2493         fport = fcip_port_head;
2494 
2495         while (fport != NULL) {
2496                 if (fport->fcipp_handle == phandle) {
2497                         /* found */
2498                         break;
2499                 }
2500                 fport = fport->fcipp_next;
2501         }
2502 
2503         mutex_exit(&fcip_global_mutex);
2504 
2505         return (fport);
2506 }
2507 
2508 /*
2509  * Handle inbound ELS requests received by the transport. We are only
2510  * intereseted in FARP/InARP mostly.
2511  */
2512 /* ARGSUSED */
2513 static int
2514 fcip_els_cb(opaque_t ulp_handle, opaque_t phandle,
2515     fc_unsol_buf_t *buf, uint32_t claimed)
2516 {
2517         fcip_port_info_t        *fport;
2518         struct fcip             *fptr;
2519         int                     instance;
2520         uchar_t                 r_ctl;
2521         uchar_t                 ls_code;
2522         la_els_farp_t           farp_cmd;
2523         la_els_farp_t           *fcmd;
2524         int                     rval = FC_UNCLAIMED;
2525 
2526         fport = fcip_get_port(phandle);
2527         if (fport == NULL) {
2528                 return (FC_UNCLAIMED);
2529         }
2530 
2531         fptr = fport->fcipp_fcip;
2532         ASSERT(fptr != NULL);
2533         if (fptr == NULL) {
2534                 return (FC_UNCLAIMED);
2535         }
2536 
2537         instance = ddi_get_instance(fport->fcipp_dip);
2538 
2539         mutex_enter(&fptr->fcip_mutex);
2540         if ((fptr->fcip_flags & (FCIP_DETACHING | FCIP_DETACHED)) ||
2541             (fptr->fcip_flags & (FCIP_SUSPENDED | FCIP_POWER_DOWN))) {
2542                 mutex_exit(&fptr->fcip_mutex);
2543                 return (FC_UNCLAIMED);
2544         }
2545 
2546         /*
2547          * set fcip flags to indicate we are in the middle of a
2548          * ELS callback so we can wait till the statechange
2549          * is handled before succeeding/failing the SUSPEND/POWER DOWN.
2550          */
2551         fptr->fcip_flags |= FCIP_IN_ELS_CB;
2552         mutex_exit(&fptr->fcip_mutex);
2553 
2554         FCIP_TNF_PROBE_2((fcip_els_cb, "fcip io", /* CSTYLED */,
2555                 tnf_string, msg, "ELS callback",
2556                 tnf_uint, instance, instance));
2557 
2558         FCIP_DEBUG(FCIP_DEBUG_ELS,
2559             (CE_NOTE, "fcip%d, ELS callback , ", instance));
2560 
2561         r_ctl = buf->ub_frame.r_ctl;
2562         switch (r_ctl & R_CTL_ROUTING) {
2563         case R_CTL_EXTENDED_SVC:
2564                 if (r_ctl == R_CTL_ELS_REQ) {
2565                         ls_code = buf->ub_buffer[0];
2566                         if (ls_code == LA_ELS_FARP_REQ) {
2567                                 /*
2568                                  * Inbound FARP broadcast request
2569                                  */
2570                                 if (buf->ub_bufsize != sizeof (la_els_farp_t)) {
2571                                         FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_WARN,
2572                                             "Invalid FARP req buffer size "
2573                                             "expected 0x%lx, got 0x%x",
2574                                             (long)(sizeof (la_els_farp_t)),
2575                                             buf->ub_bufsize));
2576                                         rval = FC_UNCLAIMED;
2577                                         goto els_cb_done;
2578                                 }
2579                                 fcmd = (la_els_farp_t *)buf;
2580                                 if (fcip_wwn_compare(&fcmd->resp_nwwn,
2581                                     &fport->fcipp_nwwn,
2582                                     FCIP_COMPARE_NWWN) != 0) {
2583                                         rval = FC_UNCLAIMED;
2584                                         goto els_cb_done;
2585                                 }
2586                                 /*
2587                                  * copy the FARP request and release the
2588                                  * unsolicited buffer
2589                                  */
2590                                 fcmd = &farp_cmd;
2591                                 bcopy((void *)buf, (void *)fcmd,
2592                                     sizeof (la_els_farp_t));
2593                                 (void) fc_ulp_ubrelease(fport->fcipp_handle, 1,
2594                                     &buf->ub_token);
2595 
2596                                 if (fcip_farp_supported &&
2597                                     fcip_handle_farp_request(fptr, fcmd) ==
2598                                     FC_SUCCESS) {
2599                                         /*
2600                                          * We successfully sent out a FARP
2601                                          * reply to the requesting port
2602                                          */
2603                                         rval = FC_SUCCESS;
2604                                         goto els_cb_done;
2605                                 } else {
2606                                         rval = FC_UNCLAIMED;
2607                                         goto els_cb_done;
2608                                 }
2609                         }
2610                 } else if (r_ctl == R_CTL_ELS_RSP) {
2611                         ls_code = buf->ub_buffer[0];
2612                         if (ls_code == LA_ELS_FARP_REPLY) {
2613                                 /*
2614                                  * We received a REPLY to our FARP request
2615                                  */
2616                                 if (buf->ub_bufsize != sizeof (la_els_farp_t)) {
2617                                         FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_WARN,
2618                                             "Invalid FARP req buffer size "
2619                                             "expected 0x%lx, got 0x%x",
2620                                             (long)(sizeof (la_els_farp_t)),
2621                                             buf->ub_bufsize));
2622                                         rval = FC_UNCLAIMED;
2623                                         goto els_cb_done;
2624                                 }
2625                                 fcmd = &farp_cmd;
2626                                 bcopy((void *)buf, (void *)fcmd,
2627                                     sizeof (la_els_farp_t));
2628                                 (void) fc_ulp_ubrelease(fport->fcipp_handle, 1,
2629                                     &buf->ub_token);
2630                                 if (fcip_farp_supported &&
2631                                     fcip_handle_farp_response(fptr, fcmd) ==
2632                                     FC_SUCCESS) {
2633                                         FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_NOTE,
2634                                             "Successfully recevied a FARP "
2635                                             "response"));
2636                                         mutex_enter(&fptr->fcip_mutex);
2637                                         fptr->fcip_farp_rsp_flag = 1;
2638                                         cv_signal(&fptr->fcip_farp_cv);
2639                                         mutex_exit(&fptr->fcip_mutex);
2640                                         rval = FC_SUCCESS;
2641                                         goto els_cb_done;
2642                                 } else {
2643                                         FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_WARN,
2644                                             "Unable to handle a FARP response "
2645                                             "receive"));
2646                                         rval = FC_UNCLAIMED;
2647                                         goto els_cb_done;
2648                                 }
2649                         }
2650                 }
2651                 break;
2652         default:
2653                 break;
2654         }
2655 els_cb_done:
2656         mutex_enter(&fptr->fcip_mutex);
2657         fptr->fcip_flags &= ~(FCIP_IN_ELS_CB);
2658         mutex_exit(&fptr->fcip_mutex);
2659         return (rval);
2660 }
2661 
2662 
2663 /*
2664  * Handle inbound FARP requests
2665  */
2666 static int
2667 fcip_handle_farp_request(struct fcip *fptr, la_els_farp_t *fcmd)
2668 {
2669         fcip_pkt_t              *fcip_pkt;
2670         fc_packet_t             *fc_pkt;
2671         fcip_port_info_t        *fport = fptr->fcip_port_info;
2672         int                     rval = FC_FAILURE;
2673         opaque_t                fca_dev;
2674         fc_portmap_t            map;
2675         struct fcip_routing_table *frp;
2676         struct fcip_dest *fdestp;
2677 
2678         /*
2679          * Add an entry for the remote port into our routing and destination
2680          * tables.
2681          */
2682         map.map_did = fcmd->req_id;
2683         map.map_hard_addr.hard_addr = fcmd->req_id.port_id;
2684         map.map_state = PORT_DEVICE_VALID;
2685         map.map_type = PORT_DEVICE_NEW;
2686         map.map_flags = 0;
2687         map.map_pd = NULL;
2688         bcopy((void *)&fcmd->req_pwwn, (void *)&map.map_pwwn,
2689             sizeof (la_wwn_t));
2690         bcopy((void *)&fcmd->req_nwwn, (void *)&map.map_nwwn,
2691             sizeof (la_wwn_t));
2692         fcip_rt_update(fptr, &map, 1);
2693         mutex_enter(&fptr->fcip_rt_mutex);
2694         frp = fcip_lookup_rtable(fptr, &fcmd->req_pwwn, FCIP_COMPARE_NWWN);
2695         mutex_exit(&fptr->fcip_rt_mutex);
2696 
2697         fdestp = fcip_add_dest(fptr, frp);
2698 
2699         fcip_pkt = fcip_ipkt_alloc(fptr, sizeof (la_els_farp_t),
2700             sizeof (la_els_farp_t), NULL, KM_SLEEP);
2701         if (fcip_pkt == NULL) {
2702                 rval = FC_FAILURE;
2703                 goto farp_done;
2704         }
2705         /*
2706          * Fill in our port's PWWN and NWWN
2707          */
2708         fcmd->resp_pwwn = fport->fcipp_pwwn;
2709         fcmd->resp_nwwn = fport->fcipp_nwwn;
2710 
2711         fcip_init_unicast_pkt(fcip_pkt, fport->fcipp_sid,
2712             fcmd->req_id, NULL);
2713 
2714         fca_dev =
2715             fc_ulp_get_fca_device(fport->fcipp_handle, fcmd->req_id);
2716         fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
2717         fc_pkt->pkt_cmd_fhdr.r_ctl = R_CTL_ELS_RSP;
2718         fc_pkt->pkt_fca_device = fca_dev;
2719         fcip_pkt->fcip_pkt_dest = fdestp;
2720 
2721         /*
2722          * Attempt a PLOGI again
2723          */
2724         if (fcmd->resp_flags & FARP_INIT_P_LOGI) {
2725                 if (fcip_do_plogi(fptr, frp) != FC_SUCCESS) {
2726                         /*
2727                          * Login to the remote port failed. There is no
2728                          * point continuing with the FARP request further
2729                          * so bail out here.
2730                          */
2731                         frp->fcipr_state = PORT_DEVICE_INVALID;
2732                         rval = FC_FAILURE;
2733                         goto farp_done;
2734                 }
2735         }
2736 
2737         FCIP_CP_OUT(fcmd, fc_pkt->pkt_cmd, fc_pkt->pkt_cmd_acc,
2738             sizeof (la_els_farp_t));
2739 
2740         rval = fc_ulp_issue_els(fport->fcipp_handle, fc_pkt);
2741         if (rval != FC_SUCCESS) {
2742                 FCIP_TNF_PROBE_2((fcip_handle_farp_request, "fcip io",
2743                     /* CSTYLED */, tnf_string, msg,
2744                     "fcip_transport of farp reply failed",
2745                     tnf_uint, rval, rval));
2746                 FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_WARN,
2747                     "fcip_transport of farp reply failed 0x%x", rval));
2748         }
2749 
2750 farp_done:
2751         return (rval);
2752 }
2753 
2754 
2755 /*
2756  * Handle FARP responses to our FARP requests. When we receive a FARP
2757  * reply, we need to add the entry for the Port that replied into our
2758  * routing and destination hash tables. It is possible that the remote
2759  * port did not login into us (FARP responses can be received without
2760  * a PLOGI)
2761  */
2762 static int
2763 fcip_handle_farp_response(struct fcip *fptr, la_els_farp_t *fcmd)
2764 {
2765         int                     rval = FC_FAILURE;
2766         fc_portmap_t            map;
2767         struct fcip_routing_table *frp;
2768         struct fcip_dest *fdestp;
2769 
2770         /*
2771          * Add an entry for the remote port into our routing and destination
2772          * tables.
2773          */
2774         map.map_did = fcmd->dest_id;
2775         map.map_hard_addr.hard_addr = fcmd->dest_id.port_id;
2776         map.map_state = PORT_DEVICE_VALID;
2777         map.map_type = PORT_DEVICE_NEW;
2778         map.map_flags = 0;
2779         map.map_pd = NULL;
2780         bcopy((void *)&fcmd->resp_pwwn, (void *)&map.map_pwwn,
2781             sizeof (la_wwn_t));
2782         bcopy((void *)&fcmd->resp_nwwn, (void *)&map.map_nwwn,
2783             sizeof (la_wwn_t));
2784         fcip_rt_update(fptr, &map, 1);
2785         mutex_enter(&fptr->fcip_rt_mutex);
2786         frp = fcip_lookup_rtable(fptr, &fcmd->resp_pwwn, FCIP_COMPARE_NWWN);
2787         mutex_exit(&fptr->fcip_rt_mutex);
2788 
2789         fdestp = fcip_add_dest(fptr, frp);
2790 
2791         if (fdestp != NULL) {
2792                 rval = FC_SUCCESS;
2793         }
2794         return (rval);
2795 }
2796 
2797 
2798 #define FCIP_HDRS_LENGTH        \
2799         sizeof (fcph_network_hdr_t)+sizeof (llc_snap_hdr_t)+sizeof (ipha_t)
2800 
2801 /*
2802  * fcip_data_cb is the heart of most IP operations. This routine is called
2803  * by the transport when any unsolicited IP data arrives at a port (which
2804  * is almost all IP data). This routine then strips off the Network header
2805  * from the payload (after authenticating the received payload ofcourse),
2806  * creates a message blk and sends the data upstream. You will see ugly
2807  * #defines because of problems with using esballoc() as opposed to
2808  * allocb to prevent an extra copy of data. We should probably move to
2809  * esballoc entirely when the MTU eventually will be larger than 1500 bytes
2810  * since copies will get more expensive then. At 1500 byte MTUs, there is
2811  * no noticable difference between using allocb and esballoc. The other
2812  * caveat is that the qlc firmware still cannot tell us accurately the
2813  * no. of valid bytes in the unsol buffer it DMA'ed so we have to resort
2814  * to looking into the IP header and hoping that the no. of bytes speficified
2815  * in the header was actually received.
2816  */
2817 /* ARGSUSED */
2818 static int
2819 fcip_data_cb(opaque_t ulp_handle, opaque_t phandle,
2820     fc_unsol_buf_t *buf, uint32_t claimed)
2821 {
2822         fcip_port_info_t                *fport;
2823         struct fcip                     *fptr;
2824         fcph_network_hdr_t              *nhdr;
2825         llc_snap_hdr_t                  *snaphdr;
2826         mblk_t                          *bp;
2827         uint32_t                        len;
2828         uint32_t                        hdrlen;
2829         ushort_t                        type;
2830         ipha_t                          *iphdr;
2831         int                             rval;
2832 
2833 #ifdef FCIP_ESBALLOC
2834         frtn_t                          *free_ubuf;
2835         struct fcip_esballoc_arg        *fesb_argp;
2836 #endif /* FCIP_ESBALLOC */
2837 
2838         fport = fcip_get_port(phandle);
2839         if (fport == NULL) {
2840                 return (FC_UNCLAIMED);
2841         }
2842 
2843         fptr = fport->fcipp_fcip;
2844         ASSERT(fptr != NULL);
2845 
2846         if (fptr == NULL) {
2847                 return (FC_UNCLAIMED);
2848         }
2849 
2850         mutex_enter(&fptr->fcip_mutex);
2851         if ((fptr->fcip_flags & (FCIP_DETACHING | FCIP_DETACHED)) ||
2852             (fptr->fcip_flags & (FCIP_SUSPENDED | FCIP_POWER_DOWN))) {
2853                 mutex_exit(&fptr->fcip_mutex);
2854                 rval = FC_UNCLAIMED;
2855                 goto data_cb_done;
2856         }
2857 
2858         /*
2859          * set fcip flags to indicate we are in the middle of a
2860          * data callback so we can wait till the statechange
2861          * is handled before succeeding/failing the SUSPEND/POWER DOWN.
2862          */
2863         fptr->fcip_flags |= FCIP_IN_DATA_CB;
2864         mutex_exit(&fptr->fcip_mutex);
2865 
2866         FCIP_TNF_PROBE_2((fcip_data_cb, "fcip io", /* CSTYLED */,
2867                 tnf_string, msg, "data callback",
2868                 tnf_int, instance, ddi_get_instance(fport->fcipp_dip)));
2869         FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
2870             (CE_NOTE, "fcip%d, data callback",
2871             ddi_get_instance(fport->fcipp_dip)));
2872 
2873         /*
2874          * get to the network and snap headers in the payload
2875          */
2876         nhdr = (fcph_network_hdr_t *)buf->ub_buffer;
2877         snaphdr = (llc_snap_hdr_t *)(buf->ub_buffer +
2878             sizeof (fcph_network_hdr_t));
2879 
2880         hdrlen = sizeof (fcph_network_hdr_t) + sizeof (llc_snap_hdr_t);
2881 
2882         /*
2883          * get the IP header to obtain the no. of bytes we need to read
2884          * off from the unsol buffer. This obviously is because not all
2885          * data fills up the unsol buffer completely and the firmware
2886          * doesn't tell us how many valid bytes are in there as well
2887          */
2888         iphdr = (ipha_t *)(buf->ub_buffer + hdrlen);
2889         snaphdr->pid = BE_16(snaphdr->pid);
2890         type = snaphdr->pid;
2891 
2892         FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
2893             (CE_CONT, "SNAPHDR: dsap %x, ssap %x, ctrl %x\n",
2894             snaphdr->dsap, snaphdr->ssap, snaphdr->ctrl));
2895 
2896         FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
2897             (CE_CONT, "oui[0] 0x%x oui[1] 0x%x oui[2] 0x%x pid 0x%x\n",
2898             snaphdr->oui[0], snaphdr->oui[1], snaphdr->oui[2], snaphdr->pid));
2899 
2900         /* Authneticate, Authenticate */
2901         if (type == ETHERTYPE_IP) {
2902                 len = hdrlen + BE_16(iphdr->ipha_length);
2903         } else if (type == ETHERTYPE_ARP) {
2904                 len = hdrlen + 28;
2905         } else {
2906                 len = buf->ub_bufsize;
2907         }
2908 
2909         FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
2910             (CE_CONT, "effective packet length is %d bytes.\n", len));
2911 
2912         if (len < hdrlen || len > FCIP_UB_SIZE) {
2913                 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
2914                     (CE_NOTE, "Incorrect buffer size %d bytes", len));
2915                 rval = FC_UNCLAIMED;
2916                 goto data_cb_done;
2917         }
2918 
2919         if (buf->ub_frame.type != FC_TYPE_IS8802_SNAP) {
2920                 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, (CE_NOTE, "Not IP/ARP data"));
2921                 rval = FC_UNCLAIMED;
2922                 goto data_cb_done;
2923         }
2924 
2925         FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, (CE_NOTE, "checking wwn"));
2926 
2927         if ((fcip_wwn_compare(&nhdr->net_dest_addr, &fport->fcipp_pwwn,
2928             FCIP_COMPARE_NWWN) != 0) &&
2929             (!IS_BROADCAST_ADDR(&nhdr->net_dest_addr))) {
2930                 rval = FC_UNCLAIMED;
2931                 goto data_cb_done;
2932         } else if (fcip_cache_on_arp_broadcast &&
2933             IS_BROADCAST_ADDR(&nhdr->net_dest_addr)) {
2934                 fcip_cache_arp_broadcast(fptr, buf);
2935         }
2936 
2937         FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, (CE_NOTE, "Allocate streams block"));
2938 
2939         /*
2940          * Using esballoc instead of allocb should be faster, atleast at
2941          * larger MTUs than 1500 bytes. Someday we'll get there :)
2942          */
2943 #if defined(FCIP_ESBALLOC)
2944         /*
2945          * allocate memory for the frtn function arg. The Function
2946          * (fcip_ubfree) arg is a struct fcip_esballoc_arg type
2947          * which contains pointers to the unsol buffer and the
2948          * opaque port handle for releasing the unsol buffer back to
2949          * the FCA for reuse
2950          */
2951         fesb_argp = (struct fcip_esballoc_arg *)
2952             kmem_zalloc(sizeof (struct fcip_esballoc_arg), KM_NOSLEEP);
2953 
2954         if (fesb_argp == NULL) {
2955                 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
2956                     (CE_WARN, "esballoc of mblk failed in data_cb"));
2957                 rval = FC_UNCLAIMED;
2958                 goto data_cb_done;
2959         }
2960         /*
2961          * Check with KM_NOSLEEP
2962          */
2963         free_ubuf = (frtn_t *)kmem_zalloc(sizeof (frtn_t), KM_NOSLEEP);
2964         if (free_ubuf == NULL) {
2965                 kmem_free(fesb_argp, sizeof (struct fcip_esballoc_arg));
2966                 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
2967                     (CE_WARN, "esballoc of mblk failed in data_cb"));
2968                 rval = FC_UNCLAIMED;
2969                 goto data_cb_done;
2970         }
2971 
2972         fesb_argp->frtnp = free_ubuf;
2973         fesb_argp->buf = buf;
2974         fesb_argp->phandle = phandle;
2975         free_ubuf->free_func = fcip_ubfree;
2976         free_ubuf->free_arg = (char *)fesb_argp;
2977         if ((bp = (mblk_t *)esballoc((unsigned char *)buf->ub_buffer,
2978             len, BPRI_MED, free_ubuf)) == NULL) {
2979                 kmem_free(fesb_argp, sizeof (struct fcip_esballoc_arg));
2980                 kmem_free(free_ubuf, sizeof (frtn_t));
2981                 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
2982                     (CE_WARN, "esballoc of mblk failed in data_cb"));
2983                 rval = FC_UNCLAIMED;
2984                 goto data_cb_done;
2985         }
2986 #elif !defined(FCIP_ESBALLOC)
2987         /*
2988          * allocate streams mblk and copy the contents of the
2989          * unsolicited buffer into this newly alloc'ed mblk
2990          */
2991         if ((bp = (mblk_t *)fcip_allocb((size_t)len, BPRI_LO)) == NULL) {
2992                 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
2993                     (CE_WARN, "alloc of mblk failed in data_cb"));
2994                 rval = FC_UNCLAIMED;
2995                 goto data_cb_done;
2996         }
2997 
2998         /*
2999          * Unsolicited buffers handed up to us from the FCA must be
3000          * endian clean so just bcopy the data into our mblk. Else
3001          * we may have to either copy the data byte by byte or
3002          * use the ddi_rep_get* routines to do the copy for us.
3003          */
3004         bcopy(buf->ub_buffer, bp->b_rptr, len);
3005 
3006         /*
3007          * for esballoc'ed mblks - free the UB in the frtn function
3008          * along with the memory allocated for the function arg.
3009          * for allocb'ed mblk - release the unsolicited buffer here
3010          */
3011         (void) fc_ulp_ubrelease(phandle, 1, &buf->ub_token);
3012 
3013 #endif  /* FCIP_ESBALLOC */
3014 
3015         bp->b_wptr = bp->b_rptr + len;
3016         fptr->fcip_ipackets++;
3017 
3018         if (type == ETHERTYPE_IP) {
3019                 mutex_enter(&fptr->fcip_mutex);
3020                 fptr->fcip_ub_upstream++;
3021                 mutex_exit(&fptr->fcip_mutex);
3022                 bp->b_rptr += hdrlen;
3023 
3024                 /*
3025                  * Check if ipq is valid in the sendup thread
3026                  */
3027                 if (fcip_sendup_alloc_enque(fptr, bp, NULL) != FC_SUCCESS) {
3028                         freemsg(bp);
3029                 }
3030         } else {
3031                 /*
3032                  * We won't get ethernet 802.3 packets in FCIP but we may get
3033                  * types other than ETHERTYPE_IP, such as ETHERTYPE_ARP. Let
3034                  * fcip_sendup() do the matching.
3035                  */
3036                 mutex_enter(&fptr->fcip_mutex);
3037                 fptr->fcip_ub_upstream++;
3038                 mutex_exit(&fptr->fcip_mutex);
3039                 if (fcip_sendup_alloc_enque(fptr, bp,
3040                     fcip_accept) != FC_SUCCESS) {
3041                         freemsg(bp);
3042                 }
3043         }
3044 
3045         rval = FC_SUCCESS;
3046 
3047         /*
3048          * Unset fcip_flags to indicate we are out of callback and return
3049          */
3050 data_cb_done:
3051         mutex_enter(&fptr->fcip_mutex);
3052         fptr->fcip_flags &= ~(FCIP_IN_DATA_CB);
3053         mutex_exit(&fptr->fcip_mutex);
3054         return (rval);
3055 }
3056 
3057 #if !defined(FCIP_ESBALLOC)
3058 /*
3059  * Allocate a message block for the inbound data to be sent upstream.
3060  */
3061 static void *
3062 fcip_allocb(size_t size, uint_t pri)
3063 {
3064         mblk_t  *mp;
3065 
3066         if ((mp = allocb(size, pri)) == NULL) {
3067                 return (NULL);
3068         }
3069         return (mp);
3070 }
3071 
3072 #endif
3073 
3074 /*
3075  * This helper routine kmem cache alloc's a sendup element for enquing
3076  * into the sendup list for callbacks upstream from the dedicated sendup
3077  * thread. We enque the msg buf into the sendup list and cv_signal the
3078  * sendup thread to finish the callback for us.
3079  */
3080 static int
3081 fcip_sendup_alloc_enque(struct fcip *fptr, mblk_t *mp, struct fcipstr *(*f)())
3082 {
3083         struct fcip_sendup_elem         *msg_elem;
3084         int                             rval = FC_FAILURE;
3085 
3086         FCIP_TNF_PROBE_1((fcip_sendup_alloc_enque, "fcip io", /* CSTYLED */,
3087                 tnf_string, msg, "sendup msg enque"));
3088         msg_elem = kmem_cache_alloc(fptr->fcip_sendup_cache, KM_NOSLEEP);
3089         if (msg_elem == NULL) {
3090                 /* drop pkt to floor - update stats */
3091                 rval = FC_FAILURE;
3092                 goto sendup_alloc_done;
3093         }
3094         msg_elem->fcipsu_mp = mp;
3095         msg_elem->fcipsu_func = f;
3096 
3097         mutex_enter(&fptr->fcip_sendup_mutex);
3098         if (fptr->fcip_sendup_head == NULL) {
3099                 fptr->fcip_sendup_head = fptr->fcip_sendup_tail = msg_elem;
3100         } else {
3101                 fptr->fcip_sendup_tail->fcipsu_next = msg_elem;
3102                 fptr->fcip_sendup_tail = msg_elem;
3103         }
3104         fptr->fcip_sendup_cnt++;
3105         cv_signal(&fptr->fcip_sendup_cv);
3106         mutex_exit(&fptr->fcip_sendup_mutex);
3107         rval = FC_SUCCESS;
3108 
3109 sendup_alloc_done:
3110         return (rval);
3111 }
3112 
3113 /*
3114  * One of the ways of performing the WWN to D_ID mapping required for
3115  * IPFC data is to cache the unsolicited ARP broadcast messages received
3116  * and update the routing table to add entry for the destination port
3117  * if we are the intended recipient of the ARP broadcast message. This is
3118  * one of the methods recommended in the rfc to obtain the WWN to D_ID mapping
3119  * but is not typically used unless enabled. The driver prefers to use the
3120  * nameserver/lilp map to obtain this mapping.
3121  */
3122 static void
3123 fcip_cache_arp_broadcast(struct fcip *fptr, fc_unsol_buf_t *buf)
3124 {
3125         fcip_port_info_t                *fport;
3126         fcph_network_hdr_t              *nhdr;
3127         struct fcip_routing_table       *frp;
3128         fc_portmap_t                    map;
3129 
3130         fport = fptr->fcip_port_info;
3131         if (fport == NULL) {
3132                 return;
3133         }
3134         ASSERT(fport != NULL);
3135 
3136         nhdr = (fcph_network_hdr_t *)buf->ub_buffer;
3137 
3138         mutex_enter(&fptr->fcip_rt_mutex);
3139         frp = fcip_lookup_rtable(fptr, &nhdr->net_src_addr, FCIP_COMPARE_NWWN);
3140         mutex_exit(&fptr->fcip_rt_mutex);
3141         if (frp == NULL) {
3142                 map.map_did.port_id = buf->ub_frame.s_id;
3143                 map.map_hard_addr.hard_addr = buf->ub_frame.s_id;
3144                 map.map_state = PORT_DEVICE_VALID;
3145                 map.map_type = PORT_DEVICE_NEW;
3146                 map.map_flags = 0;
3147                 map.map_pd = NULL;
3148                 bcopy((void *)&nhdr->net_src_addr, (void *)&map.map_pwwn,
3149                     sizeof (la_wwn_t));
3150                 bcopy((void *)&nhdr->net_src_addr, (void *)&map.map_nwwn,
3151                     sizeof (la_wwn_t));
3152                 fcip_rt_update(fptr, &map, 1);
3153                 mutex_enter(&fptr->fcip_rt_mutex);
3154                 frp = fcip_lookup_rtable(fptr, &nhdr->net_src_addr,
3155                     FCIP_COMPARE_NWWN);
3156                 mutex_exit(&fptr->fcip_rt_mutex);
3157 
3158                 (void) fcip_add_dest(fptr, frp);
3159         }
3160 
3161 }
3162 
3163 /*
3164  * This is a dedicated thread to do callbacks from fcip's data callback
3165  * routines into the modules upstream. The reason for this thread is
3166  * the data callback function can be called from an interrupt context and
3167  * the upstream modules *can* make calls downstream in the same thread
3168  * context. If the call is to a fabric port which is not yet in our
3169  * routing tables, we may have to query the nameserver/fabric for the
3170  * MAC addr to Port_ID mapping which may be blocking calls.
3171  */
3172 static void
3173 fcip_sendup_thr(void *arg)
3174 {
3175         struct fcip             *fptr = (struct fcip *)arg;
3176         struct fcip_sendup_elem *msg_elem;
3177         queue_t                 *ip4q = NULL;
3178 
3179         CALLB_CPR_INIT(&fptr->fcip_cpr_info, &fptr->fcip_sendup_mutex,
3180             callb_generic_cpr, "fcip_sendup_thr");
3181 
3182         mutex_enter(&fptr->fcip_sendup_mutex);
3183         for (;;) {
3184 
3185                 while (fptr->fcip_sendup_thr_initted &&
3186                     fptr->fcip_sendup_head == NULL) {
3187                         CALLB_CPR_SAFE_BEGIN(&fptr->fcip_cpr_info);
3188                         cv_wait(&fptr->fcip_sendup_cv,
3189                             &fptr->fcip_sendup_mutex);
3190                         CALLB_CPR_SAFE_END(&fptr->fcip_cpr_info,
3191                             &fptr->fcip_sendup_mutex);
3192                 }
3193 
3194                 if (fptr->fcip_sendup_thr_initted == 0) {
3195                         break;
3196                 }
3197 
3198                 FCIP_TNF_PROBE_1((fcip_sendup_thr, "fcip io", /* CSTYLED */,
3199                     tnf_string, msg, "fcip sendup thr - new msg"));
3200 
3201                 msg_elem = fptr->fcip_sendup_head;
3202                 fptr->fcip_sendup_head = msg_elem->fcipsu_next;
3203                 msg_elem->fcipsu_next = NULL;
3204                 mutex_exit(&fptr->fcip_sendup_mutex);
3205 
3206                 if (msg_elem->fcipsu_func == NULL) {
3207                         /*
3208                          * Message for ipq. Check to see if the ipq is
3209                          * is still valid. Since the thread is asynchronous,
3210                          * there could have been a close on the stream
3211                          */
3212                         mutex_enter(&fptr->fcip_mutex);
3213                         if (fptr->fcip_ipq && canputnext(fptr->fcip_ipq)) {
3214                                 ip4q = fptr->fcip_ipq;
3215                                 mutex_exit(&fptr->fcip_mutex);
3216                                 putnext(ip4q, msg_elem->fcipsu_mp);
3217                         } else {
3218                                 mutex_exit(&fptr->fcip_mutex);
3219                                 freemsg(msg_elem->fcipsu_mp);
3220                         }
3221                 } else {
3222                         fcip_sendup(fptr, msg_elem->fcipsu_mp,
3223                             msg_elem->fcipsu_func);
3224                 }
3225 
3226 #if !defined(FCIP_ESBALLOC)
3227                 /*
3228                  * for allocb'ed mblk - decrement upstream count here
3229                  */
3230                 mutex_enter(&fptr->fcip_mutex);
3231                 ASSERT(fptr->fcip_ub_upstream > 0);
3232                 fptr->fcip_ub_upstream--;
3233                 mutex_exit(&fptr->fcip_mutex);
3234 #endif /* FCIP_ESBALLOC */
3235 
3236                 kmem_cache_free(fptr->fcip_sendup_cache, (void *)msg_elem);
3237                 mutex_enter(&fptr->fcip_sendup_mutex);
3238                 fptr->fcip_sendup_cnt--;
3239         }
3240 
3241 
3242 #ifndef __lock_lint
3243         CALLB_CPR_EXIT(&fptr->fcip_cpr_info);
3244 #else
3245         mutex_exit(&fptr->fcip_sendup_mutex);
3246 #endif /* __lock_lint */
3247 
3248         /* Wake up fcip detach thread by the end */
3249         cv_signal(&fptr->fcip_sendup_cv);
3250 
3251         thread_exit();
3252 }
3253 
3254 #ifdef FCIP_ESBALLOC
3255 
3256 /*
3257  * called from the stream head when it is done using an unsolicited buffer.
3258  * We release this buffer then to the FCA for reuse.
3259  */
3260 static void
3261 fcip_ubfree(char *arg)
3262 {
3263         struct fcip_esballoc_arg *fesb_argp = (struct fcip_esballoc_arg *)arg;
3264         fc_unsol_buf_t  *ubuf;
3265         frtn_t          *frtnp;
3266         fcip_port_info_t                *fport;
3267         struct fcip                     *fptr;
3268 
3269 
3270         fport = fcip_get_port(fesb_argp->phandle);
3271         fptr = fport->fcipp_fcip;
3272 
3273         ASSERT(fesb_argp != NULL);
3274         ubuf = fesb_argp->buf;
3275         frtnp = fesb_argp->frtnp;
3276 
3277 
3278         FCIP_DEBUG(FCIP_DEBUG_UPSTREAM,
3279             (CE_WARN, "freeing ubuf after esballoc in fcip_ubfree"));
3280         (void) fc_ulp_ubrelease(fesb_argp->phandle, 1, &ubuf->ub_token);
3281 
3282         mutex_enter(&fptr->fcip_mutex);
3283         ASSERT(fptr->fcip_ub_upstream > 0);
3284         fptr->fcip_ub_upstream--;
3285         cv_signal(&fptr->fcip_ub_cv);
3286         mutex_exit(&fptr->fcip_mutex);
3287 
3288         kmem_free(frtnp, sizeof (frtn_t));
3289         kmem_free(fesb_argp, sizeof (struct fcip_esballoc_arg));
3290 }
3291 
3292 #endif /* FCIP_ESBALLOC */
3293 
3294 /*
3295  * handle data other than that of type ETHERTYPE_IP and send it on its
3296  * way upstream to the right streams module to handle
3297  */
3298 static void
3299 fcip_sendup(struct fcip *fptr, mblk_t *mp, struct fcipstr *(*acceptfunc)())
3300 {
3301         struct fcipstr  *slp, *nslp;
3302         la_wwn_t        *dhostp;
3303         mblk_t          *nmp;
3304         uint32_t        isgroupaddr;
3305         int             type;
3306         uint32_t        hdrlen;
3307         fcph_network_hdr_t      *nhdr;
3308         llc_snap_hdr_t          *snaphdr;
3309 
3310         FCIP_TNF_PROBE_1((fcip_sendup, "fcip io", /* CSTYLED */,
3311                 tnf_string, msg, "fcip sendup"));
3312         nhdr = (fcph_network_hdr_t *)mp->b_rptr;
3313         snaphdr =
3314             (llc_snap_hdr_t *)(mp->b_rptr + sizeof (fcph_network_hdr_t));
3315         dhostp = &nhdr->net_dest_addr;
3316         type = snaphdr->pid;
3317         hdrlen = sizeof (fcph_network_hdr_t) + sizeof (llc_snap_hdr_t);
3318 
3319         /* No group address with fibre channel */
3320         isgroupaddr = 0;
3321 
3322         /*
3323          * While holding a reader lock on the linked list of streams structures,
3324          * attempt to match the address criteria for each stream
3325          * and pass up the raw M_DATA ("fastpath") or a DL_UNITDATA_IND.
3326          */
3327 
3328         rw_enter(&fcipstruplock, RW_READER);
3329 
3330         if ((slp = (*acceptfunc)(fcipstrup, fptr, type, dhostp)) == NULL) {
3331                 rw_exit(&fcipstruplock);
3332                 FCIP_TNF_PROBE_1((fcip_sendup, "fcip io", /* CSTYLED */,
3333                     tnf_string, msg, "fcip sendup - no slp"));
3334                 freemsg(mp);
3335                 return;
3336         }
3337 
3338         /*
3339          * Loop on matching open streams until (*acceptfunc)() returns NULL.
3340          */
3341         for (; nslp = (*acceptfunc)(slp->sl_nextp, fptr, type, dhostp);
3342             slp = nslp) {
3343                 if (canputnext(slp->sl_rq)) {
3344                         if (nmp = dupmsg(mp)) {
3345                                 if ((slp->sl_flags & FCIP_SLFAST) &&
3346                                                         !isgroupaddr) {
3347                                         nmp->b_rptr += hdrlen;
3348                                         putnext(slp->sl_rq, nmp);
3349                                 } else if (slp->sl_flags & FCIP_SLRAW) {
3350                                         /* No headers when FCIP_SLRAW is set */
3351                                         putnext(slp->sl_rq, nmp);
3352                                 } else if ((nmp = fcip_addudind(fptr, nmp,
3353                                     nhdr, type))) {
3354                                         putnext(slp->sl_rq, nmp);
3355                                 }
3356                         }
3357                 }
3358         }
3359 
3360         /*
3361          * Do the last one.
3362          */
3363         if (canputnext(slp->sl_rq)) {
3364                 if (slp->sl_flags & FCIP_SLFAST) {
3365                         mp->b_rptr += hdrlen;
3366                         putnext(slp->sl_rq, mp);
3367                 } else if (slp->sl_flags & FCIP_SLRAW) {
3368                         putnext(slp->sl_rq, mp);
3369                 } else if ((mp = fcip_addudind(fptr, mp, nhdr, type))) {
3370                         putnext(slp->sl_rq, mp);
3371                 }
3372         } else {
3373                 freemsg(mp);
3374         }
3375         FCIP_TNF_PROBE_1((fcip_sendup, "fcip io", /* CSTYLED */,
3376             tnf_string, msg, "fcip sendup done"));
3377 
3378         rw_exit(&fcipstruplock);
3379 }
3380 
3381 /*
3382  * Match the stream based on type and wwn if necessary.
3383  * Destination wwn dhostp is passed to this routine is reserved
3384  * for future usage. We don't need to use it right now since port
3385  * to fcip instance mapping is unique and wwn is already validated when
3386  * packet comes to fcip.
3387  */
3388 /* ARGSUSED */
3389 static struct fcipstr *
3390 fcip_accept(struct fcipstr *slp, struct fcip *fptr, int type, la_wwn_t *dhostp)
3391 {
3392         t_uscalar_t     sap;
3393 
3394         FCIP_TNF_PROBE_1((fcip_accept, "fcip io", /* CSTYLED */,
3395             tnf_string, msg, "fcip accept"));
3396 
3397         for (; slp; slp = slp->sl_nextp) {
3398                 sap = slp->sl_sap;
3399                 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, (CE_CONT,
3400                     "fcip_accept: checking next sap = %x, type = %x",
3401                     sap, type));
3402 
3403                 if ((slp->sl_fcip == fptr) && (type == sap)) {
3404                         return (slp);
3405                 }
3406         }
3407         return (NULL);
3408 }
3409 
3410 /*
3411  * Handle DL_UNITDATA_IND messages
3412  */
3413 static mblk_t *
3414 fcip_addudind(struct fcip *fptr, mblk_t *mp, fcph_network_hdr_t *nhdr,
3415     int type)
3416 {
3417         dl_unitdata_ind_t       *dludindp;
3418         struct  fcipdladdr      *dlap;
3419         mblk_t  *nmp;
3420         int     size;
3421         uint32_t hdrlen;
3422         struct ether_addr       src_addr;
3423         struct ether_addr       dest_addr;
3424 
3425 
3426         hdrlen = (sizeof (llc_snap_hdr_t) + sizeof (fcph_network_hdr_t));
3427         mp->b_rptr += hdrlen;
3428 
3429         FCIP_TNF_PROBE_1((fcip_addudind, "fcip io", /* CSTYLED */,
3430             tnf_string, msg, "fcip addudind"));
3431 
3432         /*
3433          * Allocate an M_PROTO mblk for the DL_UNITDATA_IND.
3434          */
3435         size = sizeof (dl_unitdata_ind_t) + FCIPADDRL + FCIPADDRL;
3436         if ((nmp = allocb(size, BPRI_LO)) == NULL) {
3437                 fptr->fcip_allocbfail++;
3438                 freemsg(mp);
3439                 return (NULL);
3440         }
3441         DB_TYPE(nmp) = M_PROTO;
3442         nmp->b_wptr = nmp->b_datap->db_lim;
3443         nmp->b_rptr = nmp->b_wptr - size;
3444 
3445         /*
3446          * Construct a DL_UNITDATA_IND primitive.
3447          */
3448         dludindp = (dl_unitdata_ind_t *)nmp->b_rptr;
3449         dludindp->dl_primitive = DL_UNITDATA_IND;
3450         dludindp->dl_dest_addr_length = FCIPADDRL;
3451         dludindp->dl_dest_addr_offset = sizeof (dl_unitdata_ind_t);
3452         dludindp->dl_src_addr_length = FCIPADDRL;
3453         dludindp->dl_src_addr_offset = sizeof (dl_unitdata_ind_t) + FCIPADDRL;
3454         dludindp->dl_group_address = 0;              /* not DL_MULTI */
3455 
3456         dlap = (struct fcipdladdr *)(nmp->b_rptr + sizeof (dl_unitdata_ind_t));
3457         wwn_to_ether(&nhdr->net_dest_addr, &dest_addr);
3458         ether_bcopy(&dest_addr, &dlap->dl_phys);
3459         dlap->dl_sap = (uint16_t)type;
3460 
3461         dlap = (struct fcipdladdr *)(nmp->b_rptr + sizeof (dl_unitdata_ind_t)
3462                 + FCIPADDRL);
3463         wwn_to_ether(&nhdr->net_src_addr, &src_addr);
3464         ether_bcopy(&src_addr, &dlap->dl_phys);
3465         dlap->dl_sap = (uint16_t)type;
3466 
3467         /*
3468          * Link the M_PROTO and M_DATA together.
3469          */
3470         nmp->b_cont = mp;
3471         return (nmp);
3472 }
3473 
3474 
3475 /*
3476  * The open routine. For clone opens, we return the next available minor
3477  * no. for the stream to use
3478  */
3479 /* ARGSUSED */
3480 static int
3481 fcip_open(queue_t *rq, dev_t *devp, int flag, int sflag, cred_t *credp)
3482 {
3483         struct fcipstr  *slp;
3484         struct fcipstr  **prevslp;
3485         minor_t minor;
3486 
3487         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM, (CE_NOTE, "in fcip_open"));
3488         FCIP_TNF_PROBE_1((fcip_open, "fcip io", /* CSTYLED */,
3489                 tnf_string, msg, "enter"));
3490         /*
3491          * We need to ensure that the port driver is loaded before
3492          * we proceed
3493          */
3494         if (ddi_hold_installed_driver(ddi_name_to_major(PORT_DRIVER)) == NULL) {
3495                 /* no port driver instances found */
3496                 FCIP_DEBUG(FCIP_DEBUG_STARTUP, (CE_WARN,
3497                     "!ddi_hold_installed_driver of fp failed\n"));
3498                 return (ENXIO);
3499         }
3500         /* serialize opens */
3501         rw_enter(&fcipstruplock, RW_WRITER);
3502 
3503         prevslp = &fcipstrup;
3504         if (sflag == CLONEOPEN) {
3505                 minor = 0;
3506                 for (; (slp = *prevslp) != NULL; prevslp = &slp->sl_nextp) {
3507                         if (minor < slp->sl_minor) {
3508                                 break;
3509                         }
3510                         minor ++;
3511                 }
3512                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM, (CE_NOTE,
3513                     "getmajor returns 0x%x", getmajor(*devp)));
3514                 *devp = makedevice(getmajor(*devp), minor);
3515         } else {
3516                 minor = getminor(*devp);
3517         }
3518 
3519         /*
3520          * check if our qp's private area is already initialized. If yes
3521          * the stream is already open - just return
3522          */
3523         if (rq->q_ptr) {
3524                 goto done;
3525         }
3526 
3527         slp = GETSTRUCT(struct fcipstr, 1);
3528         slp->sl_minor = minor;
3529         slp->sl_rq = rq;
3530         slp->sl_sap = 0;
3531         slp->sl_flags = 0;
3532         slp->sl_state = DL_UNATTACHED;
3533         slp->sl_fcip = NULL;
3534 
3535         mutex_init(&slp->sl_lock, NULL, MUTEX_DRIVER, NULL);
3536 
3537         /*
3538          * link this new stream entry into list of active streams
3539          */
3540         slp->sl_nextp = *prevslp;
3541         *prevslp = slp;
3542 
3543         rq->q_ptr = WR(rq)->q_ptr = (char *)slp;
3544 
3545         /*
3546          * Disable automatic enabling of our write service procedures
3547          * we need to control this explicitly. This will prevent
3548          * anyone scheduling of our write service procedures.
3549          */
3550         noenable(WR(rq));
3551 
3552 done:
3553         rw_exit(&fcipstruplock);
3554         /*
3555          * enable our put and service routines on the read side
3556          */
3557         qprocson(rq);
3558 
3559         /*
3560          * There is only one instance of fcip (instance = 0)
3561          * for multiple instances of hardware
3562          */
3563         (void) qassociate(rq, 0);       /* don't allow drcompat to be pushed */
3564         return (0);
3565 }
3566 
3567 /*
3568  * close an opened stream. The minor no. will then be available for
3569  * future opens.
3570  */
3571 /* ARGSUSED */
3572 static int
3573 fcip_close(queue_t *rq, int flag, int otyp, cred_t *credp)
3574 {
3575         struct fcipstr *slp;
3576         struct fcipstr **prevslp;
3577 
3578         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM, (CE_NOTE, "in fcip_close"));
3579         FCIP_TNF_PROBE_1((fcip_close, "fcip io", /* CSTYLED */,
3580                 tnf_string, msg, "enter"));
3581         ASSERT(rq);
3582         /* we should also have the active stream pointer in q_ptr */
3583         ASSERT(rq->q_ptr);
3584 
3585         ddi_rele_driver(ddi_name_to_major(PORT_DRIVER));
3586         /*
3587          * disable our put and service procedures. We had enabled them
3588          * on open
3589          */
3590         qprocsoff(rq);
3591         slp = (struct fcipstr *)rq->q_ptr;
3592 
3593         /*
3594          * Implicitly detach stream  a stream from an interface.
3595          */
3596         if (slp->sl_fcip) {
3597                 fcip_dodetach(slp);
3598         }
3599 
3600         (void) qassociate(rq, -1);      /* undo association in open */
3601 
3602         rw_enter(&fcipstruplock, RW_WRITER);
3603 
3604         /*
3605          * unlink this stream from the active stream list and free it
3606          */
3607         for (prevslp = &fcipstrup; (slp = *prevslp) != NULL;
3608             prevslp = &slp->sl_nextp) {
3609                 if (slp == (struct fcipstr *)rq->q_ptr) {
3610                         break;
3611                 }
3612         }
3613 
3614         /* we should have found slp */
3615         ASSERT(slp);
3616 
3617         *prevslp = slp->sl_nextp;
3618         mutex_destroy(&slp->sl_lock);
3619         kmem_free(slp, sizeof (struct fcipstr));
3620         rq->q_ptr = WR(rq)->q_ptr = NULL;
3621 
3622         rw_exit(&fcipstruplock);
3623         return (0);
3624 }
3625 
3626 /*
3627  * This is not an extension of the DDI_DETACH request. This routine
3628  * only detaches a stream from an interface
3629  */
3630 static void
3631 fcip_dodetach(struct fcipstr *slp)
3632 {
3633         struct fcipstr  *tslp;
3634         struct fcip     *fptr;
3635 
3636         FCIP_DEBUG(FCIP_DEBUG_DETACH, (CE_NOTE, "in fcip_dodetach"));
3637         FCIP_TNF_PROBE_1((fcip_dodetach, "fcip io", /* CSTYLED */,
3638                 tnf_string, msg, "enter"));
3639         ASSERT(slp->sl_fcip != NULL);
3640 
3641         fptr = slp->sl_fcip;
3642         slp->sl_fcip = NULL;
3643 
3644         /*
3645          * we don't support promiscuous mode currently but check
3646          * for and disable any promiscuous mode operation
3647          */
3648         if (slp->sl_flags & SLALLPHYS) {
3649                 slp->sl_flags &= ~SLALLPHYS;
3650         }
3651 
3652         /*
3653          * disable ALLMULTI mode if all mulitcast addr are ON
3654          */
3655         if (slp->sl_flags & SLALLMULTI) {
3656                 slp->sl_flags &= ~SLALLMULTI;
3657         }
3658 
3659         /*
3660          * we are most likely going to perform multicast by
3661          * broadcasting to the well known addr (D_ID) 0xFFFFFF or
3662          * ALPA 0x00 in case of public loops
3663          */
3664 
3665 
3666         /*
3667          * detach unit from device structure.
3668          */
3669         for (tslp = fcipstrup; tslp != NULL; tslp = tslp->sl_nextp) {
3670                 if (tslp->sl_fcip == fptr) {
3671                         break;
3672                 }
3673         }
3674         if (tslp == NULL) {
3675                 FCIP_DEBUG(FCIP_DEBUG_DETACH, (CE_WARN,
3676                 "fcip_dodeatch - active stream struct not found"));
3677 
3678                 /* unregister with Fabric nameserver?? */
3679         }
3680         slp->sl_state = DL_UNATTACHED;
3681 
3682         fcip_setipq(fptr);
3683 }
3684 
3685 
3686 /*
3687  * Set or clear device ipq pointer.
3688  * Walk thru all the streams on this device, if a ETHERTYPE_IP
3689  * stream is found, assign device ipq to its sl_rq.
3690  */
3691 static void
3692 fcip_setipq(struct fcip *fptr)
3693 {
3694         struct fcipstr  *slp;
3695         int             ok = 1;
3696         queue_t         *ipq = NULL;
3697 
3698         FCIP_DEBUG(FCIP_DEBUG_INIT, (CE_NOTE, "entered fcip_setipq"));
3699 
3700         rw_enter(&fcipstruplock, RW_READER);
3701 
3702         for (slp = fcipstrup; slp != NULL; slp = slp->sl_nextp) {
3703                 if (slp->sl_fcip == fptr) {
3704                         if (slp->sl_flags & (SLALLPHYS|SLALLSAP)) {
3705                                 ok = 0;
3706                         }
3707                         if (slp->sl_sap == ETHERTYPE_IP) {
3708                                 if (ipq == NULL) {
3709                                         ipq = slp->sl_rq;
3710                                 } else {
3711                                         ok = 0;
3712                                 }
3713                         }
3714                 }
3715         }
3716 
3717         rw_exit(&fcipstruplock);
3718 
3719         if (fcip_check_port_exists(fptr)) {
3720                 /* fptr passed to us is stale */
3721                 return;
3722         }
3723 
3724         mutex_enter(&fptr->fcip_mutex);
3725         if (ok) {
3726                 fptr->fcip_ipq = ipq;
3727         } else {
3728                 fptr->fcip_ipq = NULL;
3729         }
3730         mutex_exit(&fptr->fcip_mutex);
3731 }
3732 
3733 
3734 /* ARGSUSED */
3735 static void
3736 fcip_ioctl(queue_t *wq, mblk_t *mp)
3737 {
3738         struct iocblk           *iocp = (struct iocblk *)mp->b_rptr;
3739         struct fcipstr          *slp = (struct fcipstr *)wq->q_ptr;
3740 
3741         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
3742             (CE_NOTE, "in fcip ioctl : %d", iocp->ioc_cmd));
3743         FCIP_TNF_PROBE_1((fcip_ioctl, "fcip io", /* CSTYLED */,
3744                 tnf_string, msg, "enter"));
3745 
3746         switch (iocp->ioc_cmd) {
3747         case DLIOCRAW:
3748                 slp->sl_flags |= FCIP_SLRAW;
3749                 miocack(wq, mp, 0, 0);
3750                 break;
3751 
3752         case DL_IOC_HDR_INFO:
3753                 fcip_dl_ioc_hdr_info(wq, mp);
3754                 break;
3755 
3756         default:
3757                 miocnak(wq, mp, 0, EINVAL);
3758                 break;
3759         }
3760 }
3761 
3762 /*
3763  * The streams 'Put' routine.
3764  */
3765 /* ARGSUSED */
3766 static int
3767 fcip_wput(queue_t *wq, mblk_t *mp)
3768 {
3769         struct fcipstr *slp = (struct fcipstr *)wq->q_ptr;
3770         struct fcip *fptr;
3771         struct fcip_dest *fdestp;
3772         fcph_network_hdr_t *headerp;
3773 
3774         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
3775             (CE_NOTE, "in fcip_wput :: type:%x", DB_TYPE(mp)));
3776 
3777         switch (DB_TYPE(mp)) {
3778         case M_DATA: {
3779 
3780                 fptr = slp->sl_fcip;
3781 
3782                 if (((slp->sl_flags & (FCIP_SLFAST|FCIP_SLRAW)) == 0) ||
3783                     (slp->sl_state != DL_IDLE) ||
3784                     (fptr == NULL)) {
3785                         /*
3786                          * set error in the message block and send a reply
3787                          * back upstream. Sun's merror routine does this
3788                          * for us more cleanly.
3789                          */
3790                         merror(wq, mp, EPROTO);
3791                         break;
3792                 }
3793 
3794                 /*
3795                  * if any messages are already enqueued or if the interface
3796                  * is in promiscuous mode, causing the packets to loop back
3797                  * up, then enqueue the message. Otherwise just transmit
3798                  * the message. putq() puts the message on fcip's
3799                  * write queue and qenable() puts the queue (wq) on
3800                  * the list of queues to be called by the streams scheduler.
3801                  */
3802                 if (wq->q_first) {
3803                         (void) putq(wq, mp);
3804                         fptr->fcip_wantw = 1;
3805                         qenable(wq);
3806                 } else if (fptr->fcip_flags & FCIP_PROMISC) {
3807                         /*
3808                          * Promiscous mode not supported but add this code in
3809                          * case it will be supported in future.
3810                          */
3811                         (void) putq(wq, mp);
3812                         qenable(wq);
3813                 } else {
3814 
3815                         headerp = (fcph_network_hdr_t *)mp->b_rptr;
3816                         fdestp = fcip_get_dest(fptr, &headerp->net_dest_addr);
3817 
3818                         if (fdestp == NULL) {
3819                                 merror(wq, mp, EPROTO);
3820                                 break;
3821                         }
3822 
3823                         ASSERT(fdestp != NULL);
3824 
3825                         (void) fcip_start(wq, mp, fptr, fdestp, KM_SLEEP);
3826                 }
3827                 break;
3828         }
3829         case M_PROTO:
3830         case M_PCPROTO:
3831                 /*
3832                  * to prevent recursive calls into fcip_proto
3833                  * (PROTO and PCPROTO messages are handled by fcip_proto)
3834                  * let the service procedure handle these messages by
3835                  * calling putq here.
3836                  */
3837                 (void) putq(wq, mp);
3838                 qenable(wq);
3839                 break;
3840 
3841         case M_IOCTL:
3842                 fcip_ioctl(wq, mp);
3843                 break;
3844 
3845         case M_FLUSH:
3846                 if (*mp->b_rptr & FLUSHW) {
3847                         flushq(wq, FLUSHALL);
3848                         *mp->b_rptr &= ~FLUSHW;
3849                 }
3850                 /*
3851                  * we have both FLUSHW and FLUSHR set with FLUSHRW
3852                  */
3853                 if (*mp->b_rptr & FLUSHR) {
3854                         /*
3855                          * send msg back upstream. qreply() takes care
3856                          * of using the RD(wq) queue on its reply
3857                          */
3858                         qreply(wq, mp);
3859                 } else {
3860                         freemsg(mp);
3861                 }
3862                 break;
3863 
3864         default:
3865                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
3866                     (CE_NOTE, "default msg type: %x", DB_TYPE(mp)));
3867                 freemsg(mp);
3868                 break;
3869         }
3870         return (0);
3871 }
3872 
3873 
3874 /*
3875  * Handle M_PROTO and M_PCPROTO messages
3876  */
3877 /* ARGSUSED */
3878 static void
3879 fcip_proto(queue_t *wq, mblk_t *mp)
3880 {
3881         union DL_primitives     *dlp;
3882         struct fcipstr          *slp;
3883         t_uscalar_t             prim;
3884 
3885         slp = (struct fcipstr *)wq->q_ptr;
3886         dlp = (union DL_primitives *)mp->b_rptr;
3887         prim = dlp->dl_primitive;            /* the DLPI command */
3888 
3889         FCIP_TNF_PROBE_5((fcip_proto, "fcip io", /* CSTYLED */,
3890                 tnf_string, msg, "enter",
3891                 tnf_opaque, wq, wq,
3892                 tnf_opaque, mp, mp,
3893                 tnf_opaque, MP_DB_TYPE, DB_TYPE(mp),
3894                 tnf_opaque, dl_primitive, dlp->dl_primitive));
3895 
3896         FCIP_DEBUG(FCIP_DEBUG_INIT, (CE_NOTE, "dl_primitve : %x", prim));
3897 
3898         mutex_enter(&slp->sl_lock);
3899 
3900         switch (prim) {
3901         case DL_UNITDATA_REQ:
3902                 FCIP_TNF_PROBE_1((fcip_proto, "fcip io", /* CSTYLED */,
3903                         tnf_string, msg, "unit data request"));
3904                 FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_NOTE, "unit data request"));
3905                 fcip_udreq(wq, mp);
3906                 break;
3907 
3908         case DL_ATTACH_REQ:
3909                 FCIP_TNF_PROBE_1((fcip_proto, "fcip io", /* CSTYLED */,
3910                         tnf_string, msg, "Attach request"));
3911                 FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_NOTE, "Attach request"));
3912                 fcip_areq(wq, mp);
3913                 break;
3914 
3915         case DL_DETACH_REQ:
3916                 FCIP_TNF_PROBE_1((fcip_proto, "fcip io", /* CSTYLED */,
3917                         tnf_string, msg, "Detach request"));
3918                 FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_NOTE, "Detach request"));
3919                 fcip_dreq(wq, mp);
3920                 break;
3921 
3922         case DL_BIND_REQ:
3923                 FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_NOTE, "Bind request"));
3924                 FCIP_TNF_PROBE_1((fcip_proto, "fcip io", /* CSTYLED */,
3925                         tnf_string, msg, "Bind request"));
3926                 fcip_breq(wq, mp);
3927                 break;
3928 
3929         case DL_UNBIND_REQ:
3930                 FCIP_TNF_PROBE_1((fcip_proto, "fcip io", /* CSTYLED */,
3931                         tnf_string, msg, "unbind request"));
3932                 FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_NOTE, "unbind request"));
3933                 fcip_ubreq(wq, mp);
3934                 break;
3935 
3936         case DL_INFO_REQ:
3937                 FCIP_TNF_PROBE_1((fcip_proto, "fcip io", /* CSTYLED */,
3938                         tnf_string, msg, "Info request"));
3939                 FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_NOTE, "Info request"));
3940                 fcip_ireq(wq, mp);
3941                 break;
3942 
3943         case DL_SET_PHYS_ADDR_REQ:
3944                 FCIP_TNF_PROBE_1((fcip_proto, "fcip io", /* CSTYLED */,
3945                         tnf_string, msg, "set phy addr request"));
3946                 FCIP_DEBUG(FCIP_DEBUG_DLPI,
3947                     (CE_NOTE, "set phy addr request"));
3948                 fcip_spareq(wq, mp);
3949                 break;
3950 
3951         case DL_PHYS_ADDR_REQ:
3952                 FCIP_TNF_PROBE_1((fcip_proto, "fcip io", /* CSTYLED */,
3953                         tnf_string, msg, "phy addr request"));
3954                 FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_NOTE, "phy addr request"));
3955                 fcip_pareq(wq, mp);
3956                 break;
3957 
3958         case DL_ENABMULTI_REQ:
3959                 FCIP_TNF_PROBE_1((fcip_proto, "fcip io", /* CSTYLED */,
3960                         tnf_string, msg, "Enable Multicast request"));
3961                 FCIP_DEBUG(FCIP_DEBUG_DLPI,
3962                     (CE_NOTE, "Enable Multicast request"));
3963                 dlerrorack(wq, mp, prim, DL_UNSUPPORTED, 0);
3964                 break;
3965 
3966         case DL_DISABMULTI_REQ:
3967                 FCIP_TNF_PROBE_1((fcip_proto, "fcip io", /* CSTYLED */,
3968                         tnf_string, msg, "Disable Multicast request"));
3969                 FCIP_DEBUG(FCIP_DEBUG_DLPI,
3970                     (CE_NOTE, "Disable Multicast request"));
3971                 dlerrorack(wq, mp, prim, DL_UNSUPPORTED, 0);
3972                 break;
3973 
3974         case DL_PROMISCON_REQ:
3975                 FCIP_TNF_PROBE_1((fcip_proto, "fcip io", /* CSTYLED */,
3976                         tnf_string, msg, "Promiscuous mode ON request"));
3977                 FCIP_DEBUG(FCIP_DEBUG_DLPI,
3978                     (CE_NOTE, "Promiscuous mode ON request"));
3979                 dlerrorack(wq, mp, prim, DL_UNSUPPORTED, 0);
3980                 break;
3981 
3982         case DL_PROMISCOFF_REQ:
3983                 FCIP_TNF_PROBE_1((fcip_proto, "fcip io", /* CSTYLED */,
3984                         tnf_string, msg, "Promiscuous mode OFF request"));
3985                 FCIP_DEBUG(FCIP_DEBUG_DLPI,
3986                     (CE_NOTE, "Promiscuous mode OFF request"));
3987                 dlerrorack(wq, mp, prim, DL_UNSUPPORTED, 0);
3988                 break;
3989 
3990         default:
3991                 FCIP_TNF_PROBE_1((fcip_proto, "fcip io", /* CSTYLED */,
3992                         tnf_string, msg, "Unsupported request"));
3993                 dlerrorack(wq, mp, prim, DL_UNSUPPORTED, 0);
3994                 break;
3995         }
3996         mutex_exit(&slp->sl_lock);
3997 }
3998 
3999 /*
4000  * Always enqueue M_PROTO and M_PCPROTO messages pn the wq and M_DATA
4001  * messages sometimes. Processing of M_PROTO and M_PCPROTO messages
4002  * require us to hold fcip's internal locks across (upstream) putnext
4003  * calls. Specifically fcip_intr could hold fcip_intrlock and fcipstruplock
4004  * when it calls putnext(). That thread could loop back around to call
4005  * fcip_wput and eventually fcip_init() to cause a recursive mutex panic
4006  *
4007  * M_DATA messages are enqueued only if we are out of xmit resources. Once
4008  * the transmit resources are available the service procedure is enabled
4009  * and an attempt is made to xmit all messages on the wq.
4010  */
4011 /* ARGSUSED */
4012 static int
4013 fcip_wsrv(queue_t *wq)
4014 {
4015         mblk_t          *mp;
4016         struct fcipstr  *slp;
4017         struct fcip     *fptr;
4018         struct fcip_dest *fdestp;
4019         fcph_network_hdr_t *headerp;
4020 
4021         slp = (struct fcipstr *)wq->q_ptr;
4022         fptr = slp->sl_fcip;
4023 
4024         FCIP_TNF_PROBE_2((fcip_wsrv, "fcip io", /* CSTYLED */,
4025                 tnf_string, msg, "enter",
4026                 tnf_opaque, wq, wq));
4027         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM, (CE_NOTE, "fcip wsrv"));
4028 
4029         while (mp = getq(wq)) {
4030                 switch (DB_TYPE(mp)) {
4031                 case M_DATA:
4032                         if (fptr && mp) {
4033                                 headerp = (fcph_network_hdr_t *)mp->b_rptr;
4034                                 fdestp = fcip_get_dest(fptr,
4035                                     &headerp->net_dest_addr);
4036                                 if (fdestp == NULL) {
4037                                         freemsg(mp);
4038                                         goto done;
4039                                 }
4040                                 if (fcip_start(wq, mp, fptr, fdestp,
4041                                     KM_SLEEP)) {
4042                                         goto done;
4043                                 }
4044                         } else {
4045                                 freemsg(mp);
4046                         }
4047                         break;
4048 
4049                 case M_PROTO:
4050                 case M_PCPROTO:
4051                         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
4052                             (CE_NOTE, "PROT msg in wsrv"));
4053                         fcip_proto(wq, mp);
4054                         break;
4055                 default:
4056                         break;
4057                 }
4058         }
4059 done:
4060         return (0);
4061 }
4062 
4063 
4064 /*
4065  * This routine is called from fcip_wsrv to send a message downstream
4066  * on the fibre towards its destination. This routine performs the
4067  * actual WWN to D_ID mapping by looking up the routing and destination
4068  * tables.
4069  */
4070 /* ARGSUSED */
4071 static int
4072 fcip_start(queue_t *wq, mblk_t *mp, struct fcip *fptr,
4073     struct fcip_dest *fdestp, int flags)
4074 {
4075         int                     rval;
4076         int                     free;
4077         fcip_pkt_t              *fcip_pkt;
4078         fc_packet_t             *fc_pkt;
4079         fcip_port_info_t        *fport = fptr->fcip_port_info;
4080         size_t                  datalen;
4081 
4082         FCIP_TNF_PROBE_4((fcip_start, "fcip io", /* CSTYLED */,
4083             tnf_string, msg, "enter", tnf_opaque, wq, wq,
4084             tnf_opaque, mp, mp,
4085             tnf_opaque, MP_DB_TYPE, DB_TYPE(mp)));
4086         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM, (CE_NOTE, "in fcipstart"));
4087 
4088         ASSERT(fdestp != NULL);
4089 
4090         /*
4091          * Only return if port has gone offline and not come back online
4092          * in a while
4093          */
4094         if (fptr->fcip_flags & FCIP_LINK_DOWN) {
4095                 freemsg(mp);
4096                 return (0);
4097         }
4098 
4099         /*
4100          * The message block coming in here already has the network and
4101          * llc_snap hdr stuffed in
4102          */
4103         /*
4104          * Traditionally ethernet drivers at sun handle 3 cases here -
4105          * 1. messages with one mblk
4106          * 2. messages with 2 mblks
4107          * 3. messages with >2 mblks
4108          * For now lets handle all the 3 cases in a single case where we
4109          * put them together in one mblk that has all the data
4110          */
4111 
4112         if (mp->b_cont != NULL) {
4113                 if (!pullupmsg(mp, -1)) {
4114                         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
4115                             (CE_WARN, "failed to concat message"));
4116                         freemsg(mp);
4117                         return (1);
4118                 }
4119         }
4120 
4121         datalen = msgsize(mp);
4122 
4123         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM, (CE_NOTE,
4124             "msgsize with nhdr & llcsnap hdr in fcip_pkt_alloc 0x%lx",
4125             datalen));
4126 
4127         /*
4128          * We cannot have requests larger than FCIPMTU+Headers
4129          */
4130         if (datalen > (FCIPMTU + sizeof (llc_snap_hdr_t) +
4131                 sizeof (fcph_network_hdr_t))) {
4132                 freemsg(mp);
4133                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM, (CE_NOTE,
4134                     "fcip_pkt_alloc: datalen is larger than "
4135                     "max possible size."));
4136                 return (1);
4137         }
4138 
4139         fcip_pkt = fcip_pkt_alloc(fptr, mp, flags, datalen);
4140         if (fcip_pkt == NULL) {
4141                 (void) putbq(wq, mp);
4142                 return (1);
4143         }
4144 
4145         fcip_pkt->fcip_pkt_mp = mp;
4146         fcip_pkt->fcip_pkt_wq = wq;
4147         fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
4148 
4149         mutex_enter(&fdestp->fcipd_mutex);
4150         /*
4151          * If the device dynamically disappeared, just fail the request.
4152          */
4153         if (fdestp->fcipd_rtable == NULL) {
4154                 mutex_exit(&fdestp->fcipd_mutex);
4155                 fcip_pkt_free(fcip_pkt, 1);
4156                 return (1);
4157         }
4158 
4159         /*
4160          * Now that we've assigned pkt_pd, we can call fc_ulp_init_packet
4161          */
4162 
4163         fc_pkt->pkt_pd = fdestp->fcipd_pd;
4164 
4165         if (fc_ulp_init_packet((opaque_t)fport->fcipp_handle,
4166             fc_pkt, flags) != FC_SUCCESS) {
4167                 mutex_exit(&fdestp->fcipd_mutex);
4168                 fcip_pkt_free(fcip_pkt, 1);
4169                 return (1);
4170         }
4171 
4172         fcip_fdestp_enqueue_pkt(fdestp, fcip_pkt);
4173         fcip_pkt->fcip_pkt_dest = fdestp;
4174         fc_pkt->pkt_fca_device = fdestp->fcipd_fca_dev;
4175 
4176         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM, (CE_NOTE,
4177             "setting cmdlen to 0x%x: rsp 0x%x : data 0x%x",
4178             fc_pkt->pkt_cmdlen, fc_pkt->pkt_rsplen, fc_pkt->pkt_datalen));
4179 
4180         fcip_init_unicast_pkt(fcip_pkt, fport->fcipp_sid,
4181             fdestp->fcipd_did, fcip_pkt_callback);
4182 
4183         fdestp->fcipd_ncmds++;
4184 
4185         mutex_exit(&fdestp->fcipd_mutex);
4186         if ((rval = fcip_transport(fcip_pkt)) == FC_SUCCESS) {
4187                 fptr->fcip_opackets++;
4188                 return (0);
4189         }
4190 
4191         free = (rval == FC_STATEC_BUSY || rval == FC_OFFLINE ||
4192             rval == FC_TRAN_BUSY) ? 0 : 1;
4193 
4194         mutex_enter(&fdestp->fcipd_mutex);
4195         rval = fcip_fdestp_dequeue_pkt(fdestp, fcip_pkt);
4196 
4197         if (!rval) {
4198                 fcip_pkt = NULL;
4199         } else {
4200                 fdestp->fcipd_ncmds--;
4201         }
4202         mutex_exit(&fdestp->fcipd_mutex);
4203 
4204         if (fcip_pkt != NULL) {
4205                 fcip_pkt_free(fcip_pkt, free);
4206         }
4207 
4208         if (!free) {
4209                 (void) putbq(wq, mp);
4210         }
4211 
4212         return (1);
4213 }
4214 
4215 
4216 /*
4217  * This routine enqueus a packet marked to be issued to the
4218  * transport in the dest structure. This enables us to timeout any
4219  * request stuck with the FCA/transport for long periods of time
4220  * without a response. fcip_pkt_timeout will attempt to clean up
4221  * any packets hung in this state of limbo.
4222  */
4223 static void
4224 fcip_fdestp_enqueue_pkt(struct fcip_dest *fdestp, fcip_pkt_t *fcip_pkt)
4225 {
4226         ASSERT(mutex_owned(&fdestp->fcipd_mutex));
4227         FCIP_TNF_PROBE_1((fcip_fdestp_enqueue_pkt, "fcip io", /* CSTYLED */,
4228                 tnf_string, msg, "destp enq pkt"));
4229 
4230         /*
4231          * Just hang it off the head of packet list
4232          */
4233         fcip_pkt->fcip_pkt_next = fdestp->fcipd_head;
4234         fcip_pkt->fcip_pkt_prev = NULL;
4235         fcip_pkt->fcip_pkt_flags |= FCIP_PKT_IN_LIST;
4236 
4237         if (fdestp->fcipd_head != NULL) {
4238                 ASSERT(fdestp->fcipd_head->fcip_pkt_prev == NULL);
4239                 fdestp->fcipd_head->fcip_pkt_prev = fcip_pkt;
4240         }
4241 
4242         fdestp->fcipd_head = fcip_pkt;
4243 }
4244 
4245 
4246 /*
4247  * dequeues any packets after the transport/FCA tells us it has
4248  * been successfully sent on its way. Ofcourse it doesn't mean that
4249  * the packet will actually reach its destination but its atleast
4250  * a step closer in that direction
4251  */
4252 static int
4253 fcip_fdestp_dequeue_pkt(struct fcip_dest *fdestp, fcip_pkt_t *fcip_pkt)
4254 {
4255         fcip_pkt_t      *fcipd_pkt;
4256 
4257         ASSERT(mutex_owned(&fdestp->fcipd_mutex));
4258         if (fcip_pkt->fcip_pkt_flags & FCIP_PKT_IN_TIMEOUT) {
4259                 fcipd_pkt = fdestp->fcipd_head;
4260                 while (fcipd_pkt) {
4261                         if (fcipd_pkt == fcip_pkt) {
4262                                 fcip_pkt_t      *pptr = NULL;
4263 
4264                                 if (fcipd_pkt == fdestp->fcipd_head) {
4265                                         ASSERT(fcipd_pkt->fcip_pkt_prev ==
4266                                             NULL);
4267                                         fdestp->fcipd_head =
4268                                             fcipd_pkt->fcip_pkt_next;
4269                                 } else {
4270                                         pptr = fcipd_pkt->fcip_pkt_prev;
4271                                         ASSERT(pptr != NULL);
4272                                         pptr->fcip_pkt_next =
4273                                             fcipd_pkt->fcip_pkt_next;
4274                                 }
4275                                 if (fcipd_pkt->fcip_pkt_next) {
4276                                         pptr = fcipd_pkt->fcip_pkt_next;
4277                                         pptr->fcip_pkt_prev =
4278                                             fcipd_pkt->fcip_pkt_prev;
4279                                 }
4280                                 fcip_pkt->fcip_pkt_flags &= ~FCIP_PKT_IN_LIST;
4281                                 break;
4282                         }
4283                         fcipd_pkt = fcipd_pkt->fcip_pkt_next;
4284                 }
4285         } else {
4286                 if (fcip_pkt->fcip_pkt_prev == NULL) {
4287                         ASSERT(fdestp->fcipd_head == fcip_pkt);
4288                         fdestp->fcipd_head = fcip_pkt->fcip_pkt_next;
4289                 } else {
4290                         fcip_pkt->fcip_pkt_prev->fcip_pkt_next =
4291                             fcip_pkt->fcip_pkt_next;
4292                 }
4293 
4294                 if (fcip_pkt->fcip_pkt_next) {
4295                         fcip_pkt->fcip_pkt_next->fcip_pkt_prev =
4296                             fcip_pkt->fcip_pkt_prev;
4297                 }
4298 
4299                 fcipd_pkt = fcip_pkt;
4300                 fcip_pkt->fcip_pkt_flags &= ~FCIP_PKT_IN_LIST;
4301         }
4302 
4303         return (fcipd_pkt == fcip_pkt);
4304 }
4305 
4306 /*
4307  * The transport routine - this is the routine that actually calls
4308  * into the FCA driver (through the transport ofcourse) to transmit a
4309  * datagram on the fibre. The dest struct assoicated with the port to
4310  * which the data is intended is already bound to the packet, this routine
4311  * only takes care of marking the packet a broadcast packet if it is
4312  * intended to be a broadcast request. This permits the transport to send
4313  * the packet down on the wire even if it doesn't have an entry for the
4314  * D_ID in its d_id hash tables.
4315  */
4316 static int
4317 fcip_transport(fcip_pkt_t *fcip_pkt)
4318 {
4319         struct fcip             *fptr;
4320         fc_packet_t             *fc_pkt;
4321         fcip_port_info_t        *fport;
4322         struct fcip_dest        *fdestp;
4323         uint32_t                did;
4324         int                     rval = FC_FAILURE;
4325         struct fcip_routing_table *frp = NULL;
4326 
4327         FCIP_TNF_PROBE_1((fcip_transport, "fcip io", /* CSTYLED */,
4328                 tnf_string, msg, "enter"));
4329 
4330         fptr = fcip_pkt->fcip_pkt_fptr;
4331         fport = fptr->fcip_port_info;
4332         fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
4333         fdestp = fcip_pkt->fcip_pkt_dest;
4334         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM, (CE_WARN, "fcip_transport called"));
4335 
4336         did = fptr->fcip_broadcast_did;
4337         if (fc_pkt->pkt_cmd_fhdr.d_id == did &&
4338             fc_pkt->pkt_tran_type != FC_PKT_BROADCAST) {
4339                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
4340                     (CE_NOTE, "trantype set to BROADCAST"));
4341                 fc_pkt->pkt_tran_type = FC_PKT_BROADCAST;
4342         }
4343 
4344         mutex_enter(&fptr->fcip_mutex);
4345         if ((fc_pkt->pkt_tran_type != FC_PKT_BROADCAST) &&
4346             (fc_pkt->pkt_pd == NULL)) {
4347                 mutex_exit(&fptr->fcip_mutex);
4348                 FCIP_TNF_PROBE_1((fcip_transport, "fcip io", /* CSTYLED */,
4349                     tnf_string, msg, "fcip transport no pd"));
4350                 return (rval);
4351         } else if (fptr->fcip_port_state == FCIP_PORT_OFFLINE) {
4352                 mutex_exit(&fptr->fcip_mutex);
4353                 FCIP_TNF_PROBE_1((fcip_transport, "fcip io", /* CSTYLED */,
4354                     tnf_string, msg, "fcip transport port offline"));
4355                 return (FC_TRAN_BUSY);
4356         }
4357         mutex_exit(&fptr->fcip_mutex);
4358 
4359         if (fdestp) {
4360                 struct fcip_routing_table       *frp;
4361 
4362                 frp = fdestp->fcipd_rtable;
4363                 mutex_enter(&fptr->fcip_rt_mutex);
4364                 mutex_enter(&fdestp->fcipd_mutex);
4365                 if (fc_pkt->pkt_pd != NULL) {
4366                         if ((frp == NULL) ||
4367                             (frp && FCIP_RTE_UNAVAIL(frp->fcipr_state))) {
4368                                 mutex_exit(&fdestp->fcipd_mutex);
4369                                 mutex_exit(&fptr->fcip_rt_mutex);
4370                                 if (frp &&
4371                                     (frp->fcipr_state == FCIP_RT_INVALID)) {
4372                                         FCIP_TNF_PROBE_1((fcip_transport,
4373                                             "fcip io", /* CSTYLED */,
4374                                             tnf_string, msg,
4375                                             "fcip transport - TRANBUSY"));
4376                                         return (FC_TRAN_BUSY);
4377                                 } else {
4378                                         FCIP_TNF_PROBE_1((fcip_transport,
4379                                             "fcip io", /* CSTYLED */,
4380                                             tnf_string, msg,
4381                                             "fcip transport: frp unavailable"));
4382                                         return (rval);
4383                                 }
4384                         }
4385                 }
4386                 mutex_exit(&fdestp->fcipd_mutex);
4387                 mutex_exit(&fptr->fcip_rt_mutex);
4388                 ASSERT(fcip_pkt->fcip_pkt_flags & FCIP_PKT_IN_LIST);
4389         }
4390 
4391         /* Explicitly invalidate this field till fcip decides to use it */
4392         fc_pkt->pkt_ulp_rscn_infop = NULL;
4393 
4394         rval = fc_ulp_transport(fport->fcipp_handle, fc_pkt);
4395         if (rval == FC_STATEC_BUSY || rval == FC_OFFLINE) {
4396                 /*
4397                  * Need to queue up the command for retry
4398                  */
4399                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
4400                     (CE_WARN, "ulp_transport failed: 0x%x", rval));
4401         } else if (rval == FC_LOGINREQ && (frp != NULL)) {
4402                 (void) fcip_do_plogi(fptr, frp);
4403         } else if (rval == FC_BADPACKET && (frp != NULL)) {
4404                 /*
4405                  * There is a distinct possiblity in our scheme of things
4406                  * that we have a routing table entry with a NULL pd struct.
4407                  * Mark the routing table entry for removal if it is not a
4408                  * broadcast entry
4409                  */
4410                 if ((frp->fcipr_d_id.port_id != 0x0) &&
4411                     (frp->fcipr_d_id.port_id != 0xffffff)) {
4412                         mutex_enter(&fptr->fcip_rt_mutex);
4413                         frp->fcipr_pd = NULL;
4414                         frp->fcipr_state = PORT_DEVICE_INVALID;
4415                         mutex_exit(&fptr->fcip_rt_mutex);
4416                 }
4417         }
4418 
4419         FCIP_TNF_PROBE_1((fcip_transport, "fcip io", /* CSTYLED */,
4420             tnf_string, msg, "fcip transport done"));
4421         return (rval);
4422 }
4423 
4424 /*
4425  * Call back routine. Called by the FCA/transport when the messages
4426  * has been put onto the wire towards its intended destination. We can
4427  * now free the fc_packet associated with the message
4428  */
4429 static void
4430 fcip_pkt_callback(fc_packet_t *fc_pkt)
4431 {
4432         int                     rval;
4433         fcip_pkt_t              *fcip_pkt;
4434         struct fcip_dest        *fdestp;
4435 
4436         fcip_pkt = (fcip_pkt_t *)fc_pkt->pkt_ulp_private;
4437         fdestp = fcip_pkt->fcip_pkt_dest;
4438 
4439         /*
4440          * take the lock early so that we don't have a race condition
4441          * with fcip_timeout
4442          *
4443          * fdestp->fcipd_mutex isn't really intended to lock per
4444          * packet struct - see bug 5105592 for permanent solution
4445          */
4446         mutex_enter(&fdestp->fcipd_mutex);
4447 
4448         fcip_pkt->fcip_pkt_flags |= FCIP_PKT_RETURNED;
4449         fcip_pkt->fcip_pkt_flags &= ~FCIP_PKT_IN_ABORT;
4450         if (fcip_pkt->fcip_pkt_flags & FCIP_PKT_IN_TIMEOUT) {
4451                 mutex_exit(&fdestp->fcipd_mutex);
4452                 return;
4453         }
4454 
4455         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM, (CE_NOTE, "pkt callback"));
4456 
4457         ASSERT(fdestp->fcipd_rtable != NULL);
4458         ASSERT(fcip_pkt->fcip_pkt_flags & FCIP_PKT_IN_LIST);
4459         rval = fcip_fdestp_dequeue_pkt(fdestp, fcip_pkt);
4460         fdestp->fcipd_ncmds--;
4461         mutex_exit(&fdestp->fcipd_mutex);
4462 
4463         if (rval) {
4464                 fcip_pkt_free(fcip_pkt, 1);
4465         }
4466 
4467         FCIP_TNF_PROBE_1((fcip_pkt_callback, "fcip io", /* CSTYLED */,
4468                 tnf_string, msg, "pkt callback done"));
4469         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM, (CE_NOTE, "pkt callback done"));
4470 }
4471 
4472 /*
4473  * Return 1 if the topology is supported, else return 0.
4474  * Topology support is consistent with what the whole
4475  * stack supports together.
4476  */
4477 static int
4478 fcip_is_supported_fc_topology(int fc_topology)
4479 {
4480         switch (fc_topology) {
4481 
4482         case FC_TOP_PRIVATE_LOOP :
4483         case FC_TOP_PUBLIC_LOOP :
4484         case FC_TOP_FABRIC :
4485         case FC_TOP_NO_NS :
4486                 return (1);
4487         default :
4488                 return (0);
4489         }
4490 }
4491 
4492 /*
4493  * handle any topology specific initializations here
4494  * this routine must be called while holding fcip_mutex
4495  */
4496 /* ARGSUSED */
4497 static void
4498 fcip_handle_topology(struct fcip *fptr)
4499 {
4500 
4501         fcip_port_info_t        *fport = fptr->fcip_port_info;
4502 
4503         ASSERT(mutex_owned(&fptr->fcip_mutex));
4504 
4505         /*
4506          * Since we know the port's topology - handle topology
4507          * specific details here. In Point to Point and Private Loop
4508          * topologies - we would probably not have a name server
4509          */
4510 
4511         FCIP_TNF_PROBE_3((fcip_handle_topology, "fcip io", /* CSTYLED */,
4512                 tnf_string, msg, "enter",
4513                 tnf_uint, port_state, fport->fcipp_pstate,
4514                 tnf_uint, topology, fport->fcipp_topology));
4515         FCIP_DEBUG(FCIP_DEBUG_INIT, (CE_NOTE, "port state: %x, topology %x",
4516                 fport->fcipp_pstate, fport->fcipp_topology));
4517 
4518         fptr->fcip_broadcast_did = fcip_get_broadcast_did(fptr);
4519         mutex_exit(&fptr->fcip_mutex);
4520         (void) fcip_dest_add_broadcast_entry(fptr, 0);
4521         mutex_enter(&fptr->fcip_mutex);
4522 
4523         if (!fcip_is_supported_fc_topology(fport->fcipp_topology)) {
4524                 FCIP_DEBUG(FCIP_DEBUG_INIT,
4525                     (CE_WARN, "fcip(0x%x): Unsupported port topology (0x%x)",
4526                     fptr->fcip_instance, fport->fcipp_topology));
4527                 return;
4528         }
4529 
4530         switch (fport->fcipp_topology) {
4531         case FC_TOP_PRIVATE_LOOP: {
4532 
4533                 fc_portmap_t            *port_map;
4534                 uint32_t                listlen, alloclen;
4535                 /*
4536                  * we may have to maintain routing. Get a list of
4537                  * all devices on this port that the transport layer is
4538                  * aware of. Check if any of them is a IS8802 type port,
4539                  * if yes get its WWN and DID mapping and cache it in
4540                  * the purport routing table. Since there is no
4541                  * State Change notification for private loop/point_point
4542                  * topologies - this table may not be accurate. The static
4543                  * routing table is updated on a state change callback.
4544                  */
4545                 FCIP_DEBUG(FCIP_DEBUG_INIT, (CE_WARN, "port state valid!!"));
4546                 fptr->fcip_port_state = FCIP_PORT_ONLINE;
4547                 listlen = alloclen = FCIP_MAX_PORTS;
4548                 port_map = (fc_portmap_t *)
4549                     kmem_zalloc((FCIP_MAX_PORTS * sizeof (fc_portmap_t)),
4550                     KM_SLEEP);
4551                 if (fc_ulp_getportmap(fport->fcipp_handle, &port_map,
4552                     &listlen, FC_ULP_PLOGI_PRESERVE) == FC_SUCCESS) {
4553                         mutex_exit(&fptr->fcip_mutex);
4554                         fcip_rt_update(fptr, port_map, listlen);
4555                         mutex_enter(&fptr->fcip_mutex);
4556                 }
4557                 if (listlen > alloclen) {
4558                         alloclen = listlen;
4559                 }
4560                 kmem_free(port_map, (alloclen * sizeof (fc_portmap_t)));
4561                 /*
4562                  * Now fall through and register with the transport
4563                  * that this port is IP capable
4564                  */
4565         }
4566         /* FALLTHROUGH */
4567         case FC_TOP_NO_NS:
4568                 /*
4569                  * If we don't have a nameserver, lets wait until we
4570                  * have to send out a packet to a remote port and then
4571                  * try and discover the port using ARP/FARP.
4572                  */
4573         /* FALLTHROUGH */
4574         case FC_TOP_PUBLIC_LOOP:
4575         case FC_TOP_FABRIC: {
4576                 fc_portmap_t    *port_map;
4577                 uint32_t        listlen, alloclen;
4578 
4579                 /* FC_TYPE of 0x05 goes to word 0, LSB */
4580                 fptr->fcip_port_state = FCIP_PORT_ONLINE;
4581 
4582                 if (!(fptr->fcip_flags & FCIP_REG_INPROGRESS)) {
4583                         fptr->fcip_flags |= FCIP_REG_INPROGRESS;
4584                         if (taskq_dispatch(fptr->fcip_tq, fcip_port_ns,
4585                             fptr, KM_NOSLEEP) == 0) {
4586                                 fptr->fcip_flags &= ~FCIP_REG_INPROGRESS;
4587                         }
4588                 }
4589 
4590                 /*
4591                  * If fcip_create_nodes_on_demand is overridden to force
4592                  * discovery of all nodes in Fabric/Public loop topologies
4593                  * we need to query for and obtain all nodes and log into
4594                  * them as with private loop devices
4595                  */
4596                 if (!fcip_create_nodes_on_demand) {
4597                         fptr->fcip_port_state = FCIP_PORT_ONLINE;
4598                         listlen = alloclen = FCIP_MAX_PORTS;
4599                         port_map = (fc_portmap_t *)
4600                             kmem_zalloc((FCIP_MAX_PORTS *
4601                             sizeof (fc_portmap_t)), KM_SLEEP);
4602                         if (fc_ulp_getportmap(fport->fcipp_handle, &port_map,
4603                             &listlen, FC_ULP_PLOGI_PRESERVE) == FC_SUCCESS) {
4604                                 mutex_exit(&fptr->fcip_mutex);
4605                                 fcip_rt_update(fptr, port_map, listlen);
4606                                 mutex_enter(&fptr->fcip_mutex);
4607                         }
4608                         if (listlen > alloclen) {
4609                                 alloclen = listlen;
4610                         }
4611                         kmem_free(port_map,
4612                             (alloclen * sizeof (fc_portmap_t)));
4613                 }
4614                 break;
4615         }
4616 
4617         default:
4618                 break;
4619         }
4620 }
4621 
4622 static void
4623 fcip_port_ns(void *arg)
4624 {
4625         struct  fcip            *fptr = (struct fcip *)arg;
4626         fcip_port_info_t        *fport = fptr->fcip_port_info;
4627         fc_ns_cmd_t             ns_cmd;
4628         uint32_t                types[8];
4629         ns_rfc_type_t           rfc;
4630 
4631         mutex_enter(&fptr->fcip_mutex);
4632         if ((fptr->fcip_flags & (FCIP_DETACHING | FCIP_DETACHED)) ||
4633             (fptr->fcip_flags & (FCIP_SUSPENDED | FCIP_POWER_DOWN))) {
4634                 fptr->fcip_flags &= ~FCIP_REG_INPROGRESS;
4635                 mutex_exit(&fptr->fcip_mutex);
4636                 return;
4637         }
4638         mutex_exit(&fptr->fcip_mutex);
4639 
4640         /*
4641          * Prepare the Name server structure to
4642          * register with the transport in case of
4643          * Fabric configuration.
4644          */
4645         bzero(&rfc, sizeof (rfc));
4646         bzero(types, sizeof (types));
4647 
4648         types[FC4_TYPE_WORD_POS(FC_TYPE_IS8802_SNAP)] = (1 <<
4649             FC4_TYPE_BIT_POS(FC_TYPE_IS8802_SNAP));
4650 
4651         rfc.rfc_port_id.port_id = fport->fcipp_sid.port_id;
4652         bcopy(types, rfc.rfc_types, sizeof (types));
4653 
4654         ns_cmd.ns_flags = 0;
4655         ns_cmd.ns_cmd = NS_RFT_ID;
4656         ns_cmd.ns_req_len = sizeof (rfc);
4657         ns_cmd.ns_req_payload = (caddr_t)&rfc;
4658         ns_cmd.ns_resp_len = 0;
4659         ns_cmd.ns_resp_payload = NULL;
4660 
4661         /*
4662          * Perform the Name Server Registration for FC IS8802_SNAP Type.
4663          * We don't expect a reply for registering port type
4664          */
4665         (void) fc_ulp_port_ns(fptr->fcip_port_info->fcipp_handle,
4666                 (opaque_t)0, &ns_cmd);
4667 
4668         mutex_enter(&fptr->fcip_mutex);
4669         fptr->fcip_flags &= ~FCIP_REG_INPROGRESS;
4670         mutex_exit(&fptr->fcip_mutex);
4671 }
4672 
4673 /*
4674  * setup this instance of fcip. This routine inits kstats, allocates
4675  * unsolicited buffers, determines' this port's siblings and handles
4676  * topology specific details which includes registering with the name
4677  * server and also setting up the routing table for this port for
4678  * private loops and point to point topologies
4679  */
4680 static int
4681 fcip_init_port(struct fcip *fptr)
4682 {
4683         int rval = FC_SUCCESS;
4684         fcip_port_info_t        *fport = fptr->fcip_port_info;
4685         static char buf[64];
4686         size_t  tok_buf_size;
4687 
4688         ASSERT(fport != NULL);
4689 
4690         FCIP_TNF_PROBE_1((fcip_init_port, "fcip io", /* CSTYLED */,
4691                 tnf_string, msg, "enter"));
4692         mutex_enter(&fptr->fcip_mutex);
4693 
4694         /*
4695          * setup mac address for this port. Don't be too worried if
4696          * the WWN is zero, there is probably nothing attached to
4697          * to the port. There is no point allocating unsolicited buffers
4698          * for an unused port so return success if we don't have a MAC
4699          * address. Do the port init on a state change notification.
4700          */
4701         if (fcip_setup_mac_addr(fptr) == FCIP_INVALID_WWN) {
4702                 fptr->fcip_port_state = FCIP_PORT_OFFLINE;
4703                 rval = FC_SUCCESS;
4704                 goto done;
4705         }
4706 
4707         /*
4708          * clear routing table hash list for this port
4709          */
4710         fcip_rt_flush(fptr);
4711 
4712         /*
4713          * init kstats for this instance
4714          */
4715         fcip_kstat_init(fptr);
4716 
4717         /*
4718          * Allocate unsolicited buffers
4719          */
4720         fptr->fcip_ub_nbufs = fcip_ub_nbufs;
4721         tok_buf_size = sizeof (*fptr->fcip_ub_tokens) * fcip_ub_nbufs;
4722 
4723         FCIP_TNF_PROBE_2((fcip_init_port, "fcip io", /* CSTYLED */,
4724                 tnf_string, msg, "debug",
4725                 tnf_int, tokBufsize, tok_buf_size));
4726 
4727         FCIP_DEBUG(FCIP_DEBUG_INIT,
4728             (CE_WARN, "tokBufsize: 0x%lx", tok_buf_size));
4729 
4730         fptr->fcip_ub_tokens = kmem_zalloc(tok_buf_size, KM_SLEEP);
4731 
4732         if (fptr->fcip_ub_tokens == NULL) {
4733                 rval = FC_FAILURE;
4734                 FCIP_DEBUG(FCIP_DEBUG_INIT,
4735                     (CE_WARN, "fcip(%d): failed to allocate unsol buf",
4736                     fptr->fcip_instance));
4737                 goto done;
4738         }
4739         rval = fc_ulp_uballoc(fport->fcipp_handle, &fptr->fcip_ub_nbufs,
4740                 fcip_ub_size, FC_TYPE_IS8802_SNAP, fptr->fcip_ub_tokens);
4741 
4742         if (rval != FC_SUCCESS) {
4743                 FCIP_DEBUG(FCIP_DEBUG_INIT,
4744                     (CE_WARN, "fcip(%d): fc_ulp_uballoc failed with 0x%x!!",
4745                     fptr->fcip_instance, rval));
4746         }
4747 
4748         switch (rval) {
4749         case FC_SUCCESS:
4750                 break;
4751 
4752         case FC_OFFLINE:
4753                 fptr->fcip_port_state = FCIP_PORT_OFFLINE;
4754                 rval = FC_FAILURE;
4755                 goto done;
4756 
4757         case FC_UB_ERROR:
4758                 FCIP_TNF_PROBE_1((fcip_init_port, "fcip io", /* CSTYLED */,
4759                         tnf_string, msg, "invalid ub alloc request"));
4760                 FCIP_DEBUG(FCIP_DEBUG_INIT,
4761                     (CE_WARN, "invalid ub alloc request !!"));
4762                 rval = FC_FAILURE;
4763                 goto done;
4764 
4765         case FC_FAILURE:
4766                 /*
4767                  * requested bytes could not be alloced
4768                  */
4769                 if (fptr->fcip_ub_nbufs != fcip_ub_nbufs) {
4770                         cmn_err(CE_WARN,
4771                             "!fcip(0x%x): Failed to alloc unsolicited bufs",
4772                             ddi_get_instance(fport->fcipp_dip));
4773                         rval = FC_FAILURE;
4774                         goto done;
4775                 }
4776                 break;
4777 
4778         default:
4779                 rval = FC_FAILURE;
4780                 break;
4781         }
4782 
4783         /*
4784          * Preallocate a Cache of fcip packets for transmit and receive
4785          * We don't want to be holding on to unsolicited buffers while
4786          * we transmit the message upstream
4787          */
4788         FCIP_DEBUG(FCIP_DEBUG_INIT, (CE_NOTE, "allocating fcip_pkt cache"));
4789 
4790         (void) sprintf(buf, "fcip%d_cache", fptr->fcip_instance);
4791         fptr->fcip_xmit_cache = kmem_cache_create(buf,
4792                 (fport->fcipp_fca_pkt_size + sizeof (fcip_pkt_t)),
4793                 8, fcip_cache_constructor, fcip_cache_destructor,
4794                 NULL, (void *)fport, NULL, 0);
4795 
4796         (void) sprintf(buf, "fcip%d_sendup_cache", fptr->fcip_instance);
4797         fptr->fcip_sendup_cache = kmem_cache_create(buf,
4798                 sizeof (struct fcip_sendup_elem),
4799                 8, fcip_sendup_constructor, NULL, NULL, (void *)fport, NULL, 0);
4800 
4801         if (fptr->fcip_xmit_cache == NULL) {
4802                 FCIP_TNF_PROBE_2((fcip_init_port, "fcip io", /* CSTYLED */,
4803                         tnf_string, msg, "unable to allocate xmit cache",
4804                         tnf_int, instance, fptr->fcip_instance));
4805                 FCIP_DEBUG(FCIP_DEBUG_INIT,
4806                     (CE_WARN, "fcip%d unable to allocate xmit cache",
4807                     fptr->fcip_instance));
4808                 rval = FC_FAILURE;
4809                 goto done;
4810         }
4811 
4812         /*
4813          * We may need to handle routing tables for point to point and
4814          * fcal topologies and register with NameServer for Fabric
4815          * topologies.
4816          */
4817         fcip_handle_topology(fptr);
4818         mutex_exit(&fptr->fcip_mutex);
4819         if (fcip_dest_add_broadcast_entry(fptr, 1) != FC_SUCCESS) {
4820                 FCIP_DEBUG(FCIP_DEBUG_INIT,
4821                     (CE_WARN, "fcip(0x%x):add broadcast entry failed!!",
4822                     fptr->fcip_instance));
4823                 mutex_enter(&fptr->fcip_mutex);
4824                 rval = FC_FAILURE;
4825                 goto done;
4826         }
4827 
4828         rval = FC_SUCCESS;
4829         return (rval);
4830 
4831 done:
4832         /*
4833          * we don't always come here from port_attach - so cleanup
4834          * anything done in the init_port routine
4835          */
4836         if (fptr->fcip_kstatp) {
4837                 kstat_delete(fptr->fcip_kstatp);
4838                 fptr->fcip_kstatp = NULL;
4839         }
4840 
4841         if (fptr->fcip_xmit_cache) {
4842                 kmem_cache_destroy(fptr->fcip_xmit_cache);
4843                 fptr->fcip_xmit_cache = NULL;
4844         }
4845 
4846         if (fptr->fcip_sendup_cache) {
4847                 kmem_cache_destroy(fptr->fcip_sendup_cache);
4848                 fptr->fcip_sendup_cache = NULL;
4849         }
4850 
4851         /* release unsolicited buffers */
4852         if (fptr->fcip_ub_tokens) {
4853                 uint64_t        *tokens = fptr->fcip_ub_tokens;
4854                 fptr->fcip_ub_tokens = NULL;
4855 
4856                 mutex_exit(&fptr->fcip_mutex);
4857                 (void) fc_ulp_ubfree(fport->fcipp_handle, fptr->fcip_ub_nbufs,
4858                         tokens);
4859                 kmem_free(tokens, tok_buf_size);
4860 
4861         } else {
4862                 mutex_exit(&fptr->fcip_mutex);
4863         }
4864 
4865         return (rval);
4866 }
4867 
4868 /*
4869  * Sets up a port's MAC address from its WWN
4870  */
4871 static int
4872 fcip_setup_mac_addr(struct fcip *fptr)
4873 {
4874         fcip_port_info_t        *fport = fptr->fcip_port_info;
4875 
4876         ASSERT(mutex_owned(&fptr->fcip_mutex));
4877 
4878         fptr->fcip_addrflags = 0;
4879 
4880         /*
4881          * we cannot choose a MAC address for our interface - we have
4882          * to live with whatever node WWN we get (minus the top two
4883          * MSbytes for the MAC address) from the transport layer. We will
4884          * treat the WWN as our factory MAC address.
4885          */
4886 
4887         if ((fport->fcipp_nwwn.w.wwn_hi != 0) ||
4888             (fport->fcipp_nwwn.w.wwn_lo != 0)) {
4889                 char            etherstr[ETHERSTRL];
4890 
4891                 wwn_to_ether(&fport->fcipp_nwwn, &fptr->fcip_macaddr);
4892                 fcip_ether_to_str(&fptr->fcip_macaddr, etherstr);
4893                 FCIP_DEBUG(FCIP_DEBUG_INIT,
4894                     (CE_NOTE, "setupmacaddr ouraddr %s", etherstr));
4895 
4896                 fptr->fcip_addrflags = (FCIP_FACTADDR_PRESENT |
4897                                                 FCIP_FACTADDR_USE);
4898         } else {
4899                 /*
4900                  * No WWN - just return failure - there's not much
4901                  * we can do since we cannot set the WWN.
4902                  */
4903                 FCIP_DEBUG(FCIP_DEBUG_INIT,
4904                     (CE_WARN, "Port does not have a valid WWN"));
4905                 return (FCIP_INVALID_WWN);
4906         }
4907         return (FC_SUCCESS);
4908 }
4909 
4910 
4911 /*
4912  * flush routing table entries
4913  */
4914 static void
4915 fcip_rt_flush(struct fcip *fptr)
4916 {
4917         int index;
4918 
4919         mutex_enter(&fptr->fcip_rt_mutex);
4920         for (index = 0; index < FCIP_RT_HASH_ELEMS; index++) {
4921                 struct fcip_routing_table       *frtp, *frtp_next;
4922                 frtp = fptr->fcip_rtable[index];
4923                 while (frtp) {
4924                         frtp_next = frtp->fcipr_next;
4925                         kmem_free(frtp, sizeof (struct fcip_routing_table));
4926                         frtp = frtp_next;
4927                 }
4928                 fptr->fcip_rtable[index] = NULL;
4929         }
4930         mutex_exit(&fptr->fcip_rt_mutex);
4931 }
4932 
4933 /*
4934  * Free up the fcip softstate and all allocated resources for the
4935  * fcip instance assoicated with a given port driver instance
4936  *
4937  * Given that the list of structures pointed to by fcip_port_head,
4938  * this function is called from multiple sources, and the
4939  * fcip_global_mutex that protects fcip_port_head must be dropped,
4940  * our best solution is to return a value that indicates the next
4941  * port in the list.  This way the caller doesn't need to worry
4942  * about the race condition where he saves off a pointer to the
4943  * next structure in the list and by the time this routine returns,
4944  * that next structure has already been freed.
4945  */
4946 static fcip_port_info_t *
4947 fcip_softstate_free(fcip_port_info_t *fport)
4948 {
4949         struct fcip             *fptr = NULL;
4950         int                     instance;
4951         timeout_id_t            tid;
4952         opaque_t                phandle = NULL;
4953         fcip_port_info_t        *prev_fport, *cur_fport, *next_fport = NULL;
4954 
4955         ASSERT(MUTEX_HELD(&fcip_global_mutex));
4956 
4957         if (fport) {
4958                 phandle = fport->fcipp_handle;
4959                 fptr = fport->fcipp_fcip;
4960         } else {
4961                 return (next_fport);
4962         }
4963 
4964         if (fptr) {
4965                 mutex_enter(&fptr->fcip_mutex);
4966                 instance = ddi_get_instance(fptr->fcip_dip);
4967 
4968                 /*
4969                  * dismantle timeout thread for this instance of fcip
4970                  */
4971                 tid = fptr->fcip_timeout_id;
4972                 fptr->fcip_timeout_id = NULL;
4973 
4974                 mutex_exit(&fptr->fcip_mutex);
4975                 (void) untimeout(tid);
4976                 mutex_enter(&fptr->fcip_mutex);
4977 
4978                 ASSERT(fcip_num_instances >= 0);
4979                 fcip_num_instances--;
4980 
4981                 /*
4982                  * stop sendup thread
4983                  */
4984                 mutex_enter(&fptr->fcip_sendup_mutex);
4985                 if (fptr->fcip_sendup_thr_initted) {
4986                         fptr->fcip_sendup_thr_initted = 0;
4987                         cv_signal(&fptr->fcip_sendup_cv);
4988                         cv_wait(&fptr->fcip_sendup_cv,
4989                             &fptr->fcip_sendup_mutex);
4990                 }
4991                 ASSERT(fptr->fcip_sendup_head == NULL);
4992                 fptr->fcip_sendup_head = fptr->fcip_sendup_tail = NULL;
4993                 mutex_exit(&fptr->fcip_sendup_mutex);
4994 
4995                 /*
4996                  * dismantle taskq
4997                  */
4998                 if (fptr->fcip_tq) {
4999                         taskq_t *tq = fptr->fcip_tq;
5000 
5001                         fptr->fcip_tq = NULL;
5002 
5003                         mutex_exit(&fptr->fcip_mutex);
5004                         taskq_destroy(tq);
5005                         mutex_enter(&fptr->fcip_mutex);
5006                 }
5007 
5008                 if (fptr->fcip_kstatp) {
5009                         kstat_delete(fptr->fcip_kstatp);
5010                         fptr->fcip_kstatp = NULL;
5011                 }
5012 
5013                 /* flush the routing table entries */
5014                 fcip_rt_flush(fptr);
5015 
5016                 if (fptr->fcip_xmit_cache) {
5017                         kmem_cache_destroy(fptr->fcip_xmit_cache);
5018                         fptr->fcip_xmit_cache = NULL;
5019                 }
5020 
5021                 if (fptr->fcip_sendup_cache) {
5022                         kmem_cache_destroy(fptr->fcip_sendup_cache);
5023                         fptr->fcip_sendup_cache = NULL;
5024                 }
5025 
5026                 fcip_cleanup_dest(fptr);
5027 
5028                 /* release unsolicited buffers */
5029                 if (fptr->fcip_ub_tokens) {
5030                         uint64_t        *tokens = fptr->fcip_ub_tokens;
5031 
5032                         fptr->fcip_ub_tokens = NULL;
5033                         mutex_exit(&fptr->fcip_mutex);
5034                         if (phandle) {
5035                                 /*
5036                                  * release the global mutex here to
5037                                  * permit any data pending callbacks to
5038                                  * complete. Else we will deadlock in the
5039                                  * FCA waiting for all unsol buffers to be
5040                                  * returned.
5041                                  */
5042                                 mutex_exit(&fcip_global_mutex);
5043                                 (void) fc_ulp_ubfree(phandle,
5044                                     fptr->fcip_ub_nbufs, tokens);
5045                                 mutex_enter(&fcip_global_mutex);
5046                         }
5047                         kmem_free(tokens, (sizeof (*tokens) * fcip_ub_nbufs));
5048                 } else {
5049                         mutex_exit(&fptr->fcip_mutex);
5050                 }
5051 
5052                 mutex_destroy(&fptr->fcip_mutex);
5053                 mutex_destroy(&fptr->fcip_ub_mutex);
5054                 mutex_destroy(&fptr->fcip_rt_mutex);
5055                 mutex_destroy(&fptr->fcip_dest_mutex);
5056                 mutex_destroy(&fptr->fcip_sendup_mutex);
5057                 cv_destroy(&fptr->fcip_farp_cv);
5058                 cv_destroy(&fptr->fcip_sendup_cv);
5059                 cv_destroy(&fptr->fcip_ub_cv);
5060 
5061                 ddi_soft_state_free(fcip_softp, instance);
5062         }
5063 
5064         /*
5065          * Now dequeue the fcip_port_info from the port list
5066          */
5067         cur_fport = fcip_port_head;
5068         prev_fport = NULL;
5069         while (cur_fport != NULL) {
5070                 if (cur_fport == fport) {
5071                         break;
5072                 }
5073                 prev_fport = cur_fport;
5074                 cur_fport = cur_fport->fcipp_next;
5075         }
5076 
5077         /*
5078          * Assert that we found a port in our port list
5079          */
5080         ASSERT(cur_fport == fport);
5081 
5082         if (prev_fport) {
5083                 /*
5084                  * Not the first port in the port list
5085                  */
5086                 prev_fport->fcipp_next = fport->fcipp_next;
5087         } else {
5088                 /*
5089                  * first port
5090                  */
5091                 fcip_port_head = fport->fcipp_next;
5092         }
5093         next_fport = fport->fcipp_next;
5094         kmem_free(fport, sizeof (fcip_port_info_t));
5095 
5096         return (next_fport);
5097 }
5098 
5099 
5100 /*
5101  * This is called by transport for any ioctl operations performed
5102  * on the devctl or other transport minor nodes. It is currently
5103  * unused for fcip
5104  */
5105 /* ARGSUSED */
5106 static int
5107 fcip_port_ioctl(opaque_t ulp_handle,  opaque_t port_handle, dev_t dev,
5108         int cmd, intptr_t data, int mode, cred_t *credp, int *rval,
5109         uint32_t claimed)
5110 {
5111         return (FC_UNCLAIMED);
5112 }
5113 
5114 /*
5115  * DL_INFO_REQ - returns information about the DLPI stream to the DLS user
5116  * requesting information about this interface
5117  */
5118 static void
5119 fcip_ireq(queue_t *wq, mblk_t *mp)
5120 {
5121         struct fcipstr          *slp;
5122         struct fcip             *fptr;
5123         dl_info_ack_t           *dlip;
5124         struct fcipdladdr       *dlap;
5125         la_wwn_t                *ep;
5126         int                     size;
5127         char                    etherstr[ETHERSTRL];
5128 
5129         slp = (struct fcipstr *)wq->q_ptr;
5130 
5131         fptr = slp->sl_fcip;
5132 
5133         FCIP_DEBUG(FCIP_DEBUG_DLPI,
5134             (CE_NOTE, "fcip_ireq: info request req rcvd"));
5135 
5136         FCIP_TNF_PROBE_1((fcip_ireq, "fcip io", /* CSTYLED */,
5137             tnf_string, msg, "fcip ireq entered"));
5138 
5139         if (MBLKL(mp) < DL_INFO_REQ_SIZE) {
5140                 dlerrorack(wq, mp, DL_INFO_REQ, DL_BADPRIM, 0);
5141                 return;
5142         }
5143 
5144         /*
5145          * Exchange current message for a DL_INFO_ACK
5146          */
5147         size = sizeof (dl_info_ack_t) + FCIPADDRL + ETHERADDRL;
5148         if ((mp = mexchange(wq, mp, size, M_PCPROTO, DL_INFO_ACK)) == NULL) {
5149                 return;
5150         }
5151 
5152         /*
5153          * FILL in the DL_INFO_ACK fields and reply
5154          */
5155         dlip = (dl_info_ack_t *)mp->b_rptr;
5156         *dlip = fcip_infoack;
5157         dlip->dl_current_state = slp->sl_state;
5158         dlap = (struct fcipdladdr *)(mp->b_rptr + dlip->dl_addr_offset);
5159         dlap->dl_sap = slp->sl_sap;
5160 
5161 
5162         if (fptr) {
5163                 fcip_ether_to_str(&fptr->fcip_macaddr, etherstr);
5164                 FCIP_DEBUG(FCIP_DEBUG_DLPI,
5165                     (CE_NOTE, "ireq - our mac: %s", etherstr));
5166                 ether_bcopy(&fptr->fcip_macaddr, &dlap->dl_phys);
5167         } else {
5168                 bzero((caddr_t)&dlap->dl_phys, ETHERADDRL);
5169         }
5170 
5171         ep = (la_wwn_t *)(mp->b_rptr + dlip->dl_brdcst_addr_offset);
5172         ether_bcopy(&fcip_arpbroadcast_addr, ep);
5173 
5174         FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_NOTE, "sending back info req.."));
5175         qreply(wq, mp);
5176 }
5177 
5178 
5179 /*
5180  * To handle DL_UNITDATA_REQ requests.
5181  */
5182 
5183 static void
5184 fcip_udreq(queue_t *wq, mblk_t *mp)
5185 {
5186         struct fcipstr          *slp;
5187         struct fcip             *fptr;
5188         fcip_port_info_t        *fport;
5189         dl_unitdata_req_t       *dludp;
5190         mblk_t                  *nmp;
5191         struct fcipdladdr       *dlap;
5192         fcph_network_hdr_t      *headerp;
5193         llc_snap_hdr_t          *lsnap;
5194         t_uscalar_t             off, len;
5195         struct fcip_dest        *fdestp;
5196         la_wwn_t                wwn;
5197         int                     hdr_size;
5198 
5199         FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_NOTE, "inside fcip_udreq"));
5200 
5201         FCIP_TNF_PROBE_1((fcip_udreq, "fcip io", /* CSTYLED */,
5202             tnf_string, msg, "fcip udreq entered"));
5203 
5204         slp = (struct fcipstr *)wq->q_ptr;
5205 
5206         if (slp->sl_state != DL_IDLE) {
5207                 dlerrorack(wq, mp, DL_UNITDATA_REQ, DL_OUTSTATE, 0);
5208                 return;
5209         }
5210 
5211         fptr = slp->sl_fcip;
5212 
5213         if (fptr == NULL) {
5214                 dlerrorack(wq, mp, DL_UNITDATA_REQ, DL_OUTSTATE, 0);
5215                 return;
5216         }
5217 
5218         fport = fptr->fcip_port_info;
5219 
5220         dludp = (dl_unitdata_req_t *)mp->b_rptr;
5221         off = dludp->dl_dest_addr_offset;
5222         len = dludp->dl_dest_addr_length;
5223 
5224         /*
5225          * Validate destination address format
5226          */
5227         if (!MBLKIN(mp, off, len) || (len != FCIPADDRL)) {
5228                 dluderrorind(wq, mp, (mp->b_rptr + off), len, DL_BADADDR, 0);
5229                 return;
5230         }
5231 
5232         /*
5233          * Error if no M_DATA follows
5234          */
5235         nmp = mp->b_cont;
5236         if (nmp == NULL) {
5237                 dluderrorind(wq, mp, (mp->b_rptr + off), len, DL_BADDATA, 0);
5238                 return;
5239         }
5240         dlap = (struct fcipdladdr *)(mp->b_rptr + off);
5241 
5242         /*
5243          * Now get the destination structure for the remote NPORT
5244          */
5245         ether_to_wwn(&dlap->dl_phys, &wwn);
5246         fdestp = fcip_get_dest(fptr, &wwn);
5247 
5248         if (fdestp == NULL) {
5249                 FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_NOTE,
5250                     "udreq - couldn't find dest struct for remote port");
5251                 dluderrorind(wq, mp, (mp->b_rptr + off), len, DL_BADDATA, 0));
5252                 return;
5253         }
5254 
5255         /*
5256          * Network header + SAP
5257          */
5258         hdr_size = sizeof (fcph_network_hdr_t) + sizeof (llc_snap_hdr_t);
5259 
5260         /* DB_REF gives the no. of msgs pointing to this block */
5261         if ((DB_REF(nmp) == 1) &&
5262             (MBLKHEAD(nmp) >= hdr_size) &&
5263             (((uintptr_t)mp->b_rptr & 0x1) == 0)) {
5264                 la_wwn_t wwn;
5265                 nmp->b_rptr -= hdr_size;
5266 
5267                 /* first put the network header */
5268                 headerp = (fcph_network_hdr_t *)nmp->b_rptr;
5269                 if (ether_cmp(&dlap->dl_phys, &fcip_arpbroadcast_addr) == 0) {
5270                         ether_to_wwn(&fcipnhbroadcastaddr, &wwn);
5271                 } else {
5272                         ether_to_wwn(&dlap->dl_phys, &wwn);
5273                 }
5274                 bcopy(&wwn, &headerp->net_dest_addr, sizeof (la_wwn_t));
5275                 ether_to_wwn(&fptr->fcip_macaddr, &wwn);
5276                 bcopy(&wwn, &headerp->net_src_addr, sizeof (la_wwn_t));
5277 
5278                 /* Now the snap header */
5279                 lsnap = (llc_snap_hdr_t *)(nmp->b_rptr +
5280                     sizeof (fcph_network_hdr_t));
5281                 lsnap->dsap = 0xAA;
5282                 lsnap->ssap = 0xAA;
5283                 lsnap->ctrl = 0x03;
5284                 lsnap->oui[0] = 0x00;
5285                 lsnap->oui[1] = 0x00;        /* 80 */
5286                 lsnap->oui[2] = 0x00;        /* C2 */
5287                 lsnap->pid = BE_16((dlap->dl_sap));
5288 
5289                 freeb(mp);
5290                 mp = nmp;
5291 
5292         } else {
5293                 la_wwn_t wwn;
5294 
5295                 DB_TYPE(mp) = M_DATA;
5296                 headerp = (fcph_network_hdr_t *)mp->b_rptr;
5297 
5298                 /*
5299                  * Only fill in the low 48bits of WWN for now - we can
5300                  * fill in the NAA_ID after we find the port in the
5301                  * routing tables
5302                  */
5303                 if (ether_cmp(&dlap->dl_phys, &fcip_arpbroadcast_addr) == 0) {
5304                         ether_to_wwn(&fcipnhbroadcastaddr, &wwn);
5305                 } else {
5306                         ether_to_wwn(&dlap->dl_phys, &wwn);
5307                 }
5308                 bcopy(&wwn, &headerp->net_dest_addr, sizeof (la_wwn_t));
5309                 /* need to send our PWWN */
5310                 bcopy(&fport->fcipp_pwwn, &headerp->net_src_addr,
5311                     sizeof (la_wwn_t));
5312 
5313                 lsnap = (llc_snap_hdr_t *)(nmp->b_rptr +
5314                     sizeof (fcph_network_hdr_t));
5315                 lsnap->dsap = 0xAA;
5316                 lsnap->ssap = 0xAA;
5317                 lsnap->ctrl = 0x03;
5318                 lsnap->oui[0] = 0x00;
5319                 lsnap->oui[1] = 0x00;
5320                 lsnap->oui[2] = 0x00;
5321                 lsnap->pid = BE_16(dlap->dl_sap);
5322 
5323                 mp->b_wptr = mp->b_rptr + hdr_size;
5324         }
5325 
5326         /*
5327          * Ethernet drivers have a lot of gunk here to put the Type
5328          * information (for Ethernet encapsulation (RFC 894) or the
5329          * Length (for 802.2/802.3) - I guess we'll just ignore that
5330          * here.
5331          */
5332 
5333         /*
5334          * Start the I/O on this port. If fcip_start failed for some reason
5335          * we call putbq in fcip_start so we don't need to check the
5336          * return value from fcip_start
5337          */
5338         (void) fcip_start(wq, mp, fptr, fdestp, KM_SLEEP);
5339 }
5340 
5341 /*
5342  * DL_ATTACH_REQ: attaches a PPA with a stream. ATTACH requets are needed
5343  * for style 2 DLS providers to identify the physical medium through which
5344  * the streams communication will happen
5345  */
5346 static void
5347 fcip_areq(queue_t *wq, mblk_t *mp)
5348 {
5349         struct fcipstr          *slp;
5350         union DL_primitives     *dlp;
5351         fcip_port_info_t        *fport;
5352         struct fcip             *fptr;
5353         int                     ppa;
5354 
5355         slp = (struct fcipstr *)wq->q_ptr;
5356         dlp = (union DL_primitives *)mp->b_rptr;
5357 
5358         if (MBLKL(mp) < DL_ATTACH_REQ_SIZE) {
5359                 dlerrorack(wq, mp, DL_ATTACH_REQ, DL_BADPRIM, 0);
5360                 return;
5361         }
5362 
5363         if (slp->sl_state != DL_UNATTACHED) {
5364                 dlerrorack(wq, mp, DL_ATTACH_REQ, DL_OUTSTATE, 0);
5365                 return;
5366         }
5367 
5368         ppa = dlp->attach_req.dl_ppa;
5369         FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_NOTE, "attach req: ppa %x", ppa));
5370 
5371         /*
5372          * check if the PPA is valid
5373          */
5374 
5375         mutex_enter(&fcip_global_mutex);
5376 
5377         for (fport = fcip_port_head; fport; fport = fport->fcipp_next) {
5378                 if ((fptr = fport->fcipp_fcip) == NULL) {
5379                         continue;
5380                 }
5381                 FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_NOTE, "ppa %x, inst %x", ppa,
5382                     ddi_get_instance(fptr->fcip_dip)));
5383 
5384                 if (ppa == ddi_get_instance(fptr->fcip_dip)) {
5385                         FCIP_DEBUG(FCIP_DEBUG_DLPI,
5386                             (CE_NOTE, "ppa found %x", ppa));
5387                         break;
5388                 }
5389         }
5390 
5391         if (fport == NULL) {
5392                 FCIP_DEBUG(FCIP_DEBUG_DLPI,
5393                     (CE_NOTE, "dlerrorack coz fport==NULL"));
5394 
5395                 mutex_exit(&fcip_global_mutex);
5396 
5397                 if (fc_ulp_get_port_handle(ppa) == NULL) {
5398                         dlerrorack(wq, mp, DL_ATTACH_REQ, DL_BADPPA, 0);
5399                         return;
5400                 }
5401 
5402                 /*
5403                  * Wait for Port attach callback to trigger.  If port_detach
5404                  * got in while we were waiting, then ddi_get_soft_state
5405                  * will return NULL, and we'll return error.
5406                  */
5407 
5408                 delay(drv_usectohz(FCIP_INIT_DELAY));
5409                 mutex_enter(&fcip_global_mutex);
5410 
5411                 fptr = ddi_get_soft_state(fcip_softp, ppa);
5412                 if (fptr == NULL) {
5413                         mutex_exit(&fcip_global_mutex);
5414                         dlerrorack(wq, mp, DL_ATTACH_REQ, DL_BADPPA, 0);
5415                         return;
5416                 }
5417         }
5418 
5419         /*
5420          * set link to device and update our state
5421          */
5422         slp->sl_fcip = fptr;
5423         slp->sl_state = DL_UNBOUND;
5424 
5425         mutex_exit(&fcip_global_mutex);
5426 
5427 #ifdef DEBUG
5428         mutex_enter(&fptr->fcip_mutex);
5429         if (fptr->fcip_flags & FCIP_LINK_DOWN) {
5430                 FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_WARN, "port not online yet"));
5431         }
5432         mutex_exit(&fptr->fcip_mutex);
5433 #endif
5434 
5435         dlokack(wq, mp, DL_ATTACH_REQ);
5436 }
5437 
5438 
5439 /*
5440  * DL_DETACH request - detaches a PPA from a stream
5441  */
5442 static void
5443 fcip_dreq(queue_t *wq, mblk_t *mp)
5444 {
5445         struct fcipstr          *slp;
5446 
5447         slp = (struct fcipstr *)wq->q_ptr;
5448 
5449         if (MBLKL(mp) < DL_DETACH_REQ_SIZE) {
5450                 dlerrorack(wq, mp, DL_DETACH_REQ, DL_BADPRIM, 0);
5451                 return;
5452         }
5453 
5454         if (slp->sl_state != DL_UNBOUND) {
5455                 dlerrorack(wq, mp, DL_DETACH_REQ, DL_OUTSTATE, 0);
5456                 return;
5457         }
5458 
5459         fcip_dodetach(slp);
5460         dlokack(wq, mp, DL_DETACH_REQ);
5461 }
5462 
5463 /*
5464  * DL_BIND request: requests a DLS provider to bind a DLSAP to the stream.
5465  * DLS users communicate with a physical interface through DLSAPs. Multiple
5466  * DLSAPs can be bound to the same stream (PPA)
5467  */
5468 static void
5469 fcip_breq(queue_t *wq, mblk_t *mp)
5470 {
5471         struct fcipstr          *slp;
5472         union DL_primitives     *dlp;
5473         struct fcip             *fptr;
5474         struct fcipdladdr       fcipaddr;
5475         t_uscalar_t             sap;
5476         int                     xidtest;
5477 
5478         slp = (struct fcipstr *)wq->q_ptr;
5479 
5480         if (MBLKL(mp) < DL_BIND_REQ_SIZE) {
5481                 dlerrorack(wq, mp, DL_BIND_REQ, DL_BADPRIM, 0);
5482                 return;
5483         }
5484 
5485         if (slp->sl_state != DL_UNBOUND) {
5486                 dlerrorack(wq, mp, DL_BIND_REQ, DL_OUTSTATE, 0);
5487                 return;
5488         }
5489 
5490         dlp = (union DL_primitives *)mp->b_rptr;
5491         fptr = slp->sl_fcip;
5492 
5493         if (fptr == NULL) {
5494                 dlerrorack(wq, mp, DL_BIND_REQ, DL_OUTSTATE, 0);
5495                 return;
5496         }
5497 
5498         sap = dlp->bind_req.dl_sap;
5499         FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_NOTE, "fcip_breq - sap: %x", sap));
5500         xidtest = dlp->bind_req.dl_xidtest_flg;
5501 
5502         if (xidtest) {
5503                 dlerrorack(wq, mp, DL_BIND_REQ, DL_NOAUTO, 0);
5504                 return;
5505         }
5506 
5507         FCIP_DEBUG(FCIP_DEBUG_DLPI, (CE_NOTE, "DLBIND: sap : %x", sap));
5508 
5509         if (sap > ETHERTYPE_MAX) {
5510                 dlerrorack(wq, mp, dlp->dl_primitive, DL_BADSAP, 0);
5511                 return;
5512         }
5513         /*
5514          * save SAP for this stream and change the link state
5515          */
5516         slp->sl_sap = sap;
5517         slp->sl_state = DL_IDLE;
5518 
5519         fcipaddr.dl_sap = sap;
5520         ether_bcopy(&fptr->fcip_macaddr, &fcipaddr.dl_phys);
5521         dlbindack(wq, mp, sap, &fcipaddr, FCIPADDRL, 0, 0);
5522 
5523         fcip_setipq(fptr);
5524 }
5525 
5526 /*
5527  * DL_UNBIND request to unbind a previously bound DLSAP, from this stream
5528  */
5529 static void
5530 fcip_ubreq(queue_t *wq, mblk_t *mp)
5531 {
5532         struct fcipstr  *slp;
5533 
5534         slp = (struct fcipstr *)wq->q_ptr;
5535 
5536         if (MBLKL(mp) < DL_UNBIND_REQ_SIZE) {
5537                 dlerrorack(wq, mp, DL_UNBIND_REQ, DL_BADPRIM, 0);
5538                 return;
5539         }
5540 
5541         if (slp->sl_state != DL_IDLE) {
5542                 dlerrorack(wq, mp, DL_UNBIND_REQ, DL_OUTSTATE, 0);
5543                 return;
5544         }
5545 
5546         slp->sl_state = DL_UNBOUND;
5547         slp->sl_sap = 0;
5548 
5549         (void) putnextctl1(RD(wq), M_FLUSH, FLUSHRW);
5550         dlokack(wq, mp, DL_UNBIND_REQ);
5551 
5552         fcip_setipq(slp->sl_fcip);
5553 }
5554 
5555 /*
5556  * Return our physical address
5557  */
5558 static void
5559 fcip_pareq(queue_t *wq, mblk_t *mp)
5560 {
5561         struct fcipstr          *slp;
5562         union DL_primitives     *dlp;
5563         int                     type;
5564         struct fcip             *fptr;
5565         fcip_port_info_t        *fport;
5566         struct ether_addr       addr;
5567 
5568         slp = (struct fcipstr *)wq->q_ptr;
5569 
5570         if (MBLKL(mp) < DL_PHYS_ADDR_REQ_SIZE) {
5571                 dlerrorack(wq, mp, DL_PHYS_ADDR_REQ, DL_BADPRIM, 0);
5572                 return;
5573         }
5574 
5575         dlp = (union DL_primitives *)mp->b_rptr;
5576         type = dlp->physaddr_req.dl_addr_type;
5577         fptr = slp->sl_fcip;
5578 
5579         if (fptr == NULL) {
5580                 dlerrorack(wq, mp, DL_PHYS_ADDR_REQ, DL_OUTSTATE, 0);
5581                 return;
5582         }
5583 
5584         fport = fptr->fcip_port_info;
5585 
5586         switch (type) {
5587         case DL_FACT_PHYS_ADDR:
5588                 FCIP_DEBUG(FCIP_DEBUG_DLPI,
5589                     (CE_NOTE, "returning factory phys addr"));
5590                 wwn_to_ether(&fport->fcipp_pwwn, &addr);
5591                 break;
5592 
5593         case DL_CURR_PHYS_ADDR:
5594                 FCIP_DEBUG(FCIP_DEBUG_DLPI,
5595                     (CE_NOTE, "returning current phys addr"));
5596                 ether_bcopy(&fptr->fcip_macaddr, &addr);
5597                 break;
5598 
5599         default:
5600                 FCIP_DEBUG(FCIP_DEBUG_DLPI,
5601                     (CE_NOTE, "Not known cmd type in phys addr"));
5602                 dlerrorack(wq, mp, DL_PHYS_ADDR_REQ, DL_NOTSUPPORTED, 0);
5603                 return;
5604         }
5605         dlphysaddrack(wq, mp, &addr, ETHERADDRL);
5606 }
5607 
5608 /*
5609  * Set physical address DLPI request
5610  */
5611 static void
5612 fcip_spareq(queue_t *wq, mblk_t *mp)
5613 {
5614         struct fcipstr          *slp;
5615         union DL_primitives     *dlp;
5616         t_uscalar_t             off, len;
5617         struct ether_addr       *addrp;
5618         la_wwn_t                wwn;
5619         struct fcip             *fptr;
5620         fc_ns_cmd_t             fcip_ns_cmd;
5621 
5622         slp = (struct fcipstr *)wq->q_ptr;
5623 
5624         if (MBLKL(mp) < DL_SET_PHYS_ADDR_REQ_SIZE) {
5625                 dlerrorack(wq, mp, DL_SET_PHYS_ADDR_REQ, DL_BADPRIM, 0);
5626                 return;
5627         }
5628 
5629         dlp = (union DL_primitives *)mp->b_rptr;
5630         len = dlp->set_physaddr_req.dl_addr_length;
5631         off = dlp->set_physaddr_req.dl_addr_offset;
5632 
5633         if (!MBLKIN(mp, off, len)) {
5634                 dlerrorack(wq, mp, DL_SET_PHYS_ADDR_REQ, DL_BADPRIM, 0);
5635                 return;
5636         }
5637 
5638         addrp = (struct ether_addr *)(mp->b_rptr + off);
5639 
5640         /*
5641          * If the length of physical address is not correct or address
5642          * specified is a broadcast address or multicast addr -
5643          * return an error.
5644          */
5645         if ((len != ETHERADDRL) ||
5646             ((addrp->ether_addr_octet[0] & 01) == 1) ||
5647             (ether_cmp(addrp, &fcip_arpbroadcast_addr) == 0)) {
5648                 dlerrorack(wq, mp, DL_SET_PHYS_ADDR_REQ, DL_BADADDR, 0);
5649                 return;
5650         }
5651 
5652         /*
5653          * check if a stream is attached to this device. Else return an error
5654          */
5655         if ((fptr = slp->sl_fcip) == NULL) {
5656                 dlerrorack(wq, mp, DL_SET_PHYS_ADDR_REQ, DL_OUTSTATE, 0);
5657                 return;
5658         }
5659 
5660         /*
5661          * set the new interface local address. We request the transport
5662          * layer to change the Port WWN for this device - return an error
5663          * if we don't succeed.
5664          */
5665 
5666         ether_to_wwn(addrp, &wwn);
5667         if (fcip_set_wwn(&wwn) == FC_SUCCESS) {
5668                 FCIP_DEBUG(FCIP_DEBUG_DLPI,
5669                     (CE_WARN, "WWN changed in spareq"));
5670         } else {
5671                 dlerrorack(wq, mp, DL_SET_PHYS_ADDR_REQ, DL_BADADDR, 0);
5672         }
5673 
5674         /*
5675          * register The new Port WWN and Node WWN with the transport
5676          * and Nameserver. Hope the transport ensures all current I/O
5677          * has stopped before actually attempting to register a new
5678          * port and Node WWN else we are hosed. Maybe a Link reset
5679          * will get everyone's attention.
5680          */
5681         fcip_ns_cmd.ns_flags = 0;
5682         fcip_ns_cmd.ns_cmd = NS_RPN_ID;
5683         fcip_ns_cmd.ns_req_len = sizeof (la_wwn_t);
5684         fcip_ns_cmd.ns_req_payload = (caddr_t)&wwn.raw_wwn[0];
5685         fcip_ns_cmd.ns_resp_len = 0;
5686         fcip_ns_cmd.ns_resp_payload = (caddr_t)0;
5687         if (fc_ulp_port_ns(fptr->fcip_port_info->fcipp_handle,
5688             (opaque_t)0, &fcip_ns_cmd) != FC_SUCCESS) {
5689                 FCIP_DEBUG(FCIP_DEBUG_DLPI,
5690                     (CE_WARN, "setting Port WWN failed"));
5691                 dlerrorack(wq, mp, DL_SET_PHYS_ADDR_REQ, DL_BADPRIM, 0);
5692                 return;
5693         }
5694 
5695         dlokack(wq, mp, DL_SET_PHYS_ADDR_REQ);
5696 }
5697 
5698 /*
5699  * change our port's WWN if permitted by hardware
5700  */
5701 /* ARGSUSED */
5702 static int
5703 fcip_set_wwn(la_wwn_t *pwwn)
5704 {
5705         /*
5706          * We're usually not allowed to change the WWN of adapters
5707          * but some adapters do permit us to change the WWN - don't
5708          * permit setting of WWNs (yet?) - This behavior could be
5709          * modified if needed
5710          */
5711         return (FC_FAILURE);
5712 }
5713 
5714 
5715 /*
5716  * This routine fills in the header for fastpath data requests. What this
5717  * does in simple terms is, instead of sending all data through the Unitdata
5718  * request dlpi code paths (which will then append the protocol specific
5719  * header - network and snap headers in our case), the upper layers issue
5720  * a M_IOCTL with a DL_IOC_HDR_INFO request and ask the streams endpoint
5721  * driver to give the header it needs appended and the upper layer
5722  * allocates and fills in the header and calls our put routine
5723  */
5724 static void
5725 fcip_dl_ioc_hdr_info(queue_t *wq, mblk_t *mp)
5726 {
5727         mblk_t                  *nmp;
5728         struct fcipstr          *slp;
5729         struct fcipdladdr       *dlap;
5730         dl_unitdata_req_t       *dlup;
5731         fcph_network_hdr_t      *headerp;
5732         la_wwn_t                wwn;
5733         llc_snap_hdr_t          *lsnap;
5734         struct fcip             *fptr;
5735         fcip_port_info_t        *fport;
5736         t_uscalar_t             off, len;
5737         size_t                  hdrlen;
5738         int                     error;
5739 
5740         slp = (struct fcipstr *)wq->q_ptr;
5741         fptr = slp->sl_fcip;
5742         if (fptr == NULL) {
5743                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
5744                     (CE_NOTE, "dliochdr : returns EINVAL1"));
5745                 miocnak(wq, mp, 0, EINVAL);
5746                 return;
5747         }
5748 
5749         error = miocpullup(mp, sizeof (dl_unitdata_req_t) + FCIPADDRL);
5750         if (error != 0) {
5751                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
5752                     (CE_NOTE, "dliochdr : returns %d", error));
5753                 miocnak(wq, mp, 0, error);
5754                 return;
5755         }
5756 
5757         fport = fptr->fcip_port_info;
5758 
5759         /*
5760          * check if the DL_UNITDATA_REQ destination addr has valid offset
5761          * and length values
5762          */
5763         dlup = (dl_unitdata_req_t *)mp->b_cont->b_rptr;
5764         off = dlup->dl_dest_addr_offset;
5765         len = dlup->dl_dest_addr_length;
5766         if (dlup->dl_primitive != DL_UNITDATA_REQ ||
5767             !MBLKIN(mp->b_cont, off, len) || (len != FCIPADDRL)) {
5768                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
5769                     (CE_NOTE, "dliochdr : returns EINVAL2"));
5770                 miocnak(wq, mp, 0, EINVAL);
5771                 return;
5772         }
5773 
5774         dlap = (struct fcipdladdr *)(mp->b_cont->b_rptr + off);
5775 
5776         /*
5777          * Allocate a new mblk to hold the ether header
5778          */
5779 
5780         /*
5781          * setup space for network header
5782          */
5783         hdrlen = (sizeof (llc_snap_hdr_t) + sizeof (fcph_network_hdr_t));
5784         if ((nmp = allocb(hdrlen, BPRI_MED)) == NULL) {
5785                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
5786                     (CE_NOTE, "dliochdr : returns ENOMEM"));
5787                 miocnak(wq, mp, 0, ENOMEM);
5788                 return;
5789         }
5790         nmp->b_wptr += hdrlen;
5791 
5792         /*
5793          * Fill in the Network Hdr and LLC SNAP header;
5794          */
5795         headerp = (fcph_network_hdr_t *)nmp->b_rptr;
5796         /*
5797          * just fill in the Node WWN here - we can fill in the NAA_ID when
5798          * we search the routing table
5799          */
5800         if (ether_cmp(&dlap->dl_phys, &fcip_arpbroadcast_addr) == 0) {
5801                 ether_to_wwn(&fcipnhbroadcastaddr, &wwn);
5802         } else {
5803                 ether_to_wwn(&dlap->dl_phys, &wwn);
5804         }
5805         bcopy(&wwn, &headerp->net_dest_addr, sizeof (la_wwn_t));
5806         bcopy(&fport->fcipp_pwwn, &headerp->net_src_addr, sizeof (la_wwn_t));
5807         lsnap = (llc_snap_hdr_t *)(nmp->b_rptr + sizeof (fcph_network_hdr_t));
5808         lsnap->dsap = 0xAA;
5809         lsnap->ssap = 0xAA;
5810         lsnap->ctrl = 0x03;
5811         lsnap->oui[0] = 0x00;
5812         lsnap->oui[1] = 0x00;
5813         lsnap->oui[2] = 0x00;
5814         lsnap->pid = BE_16(dlap->dl_sap);
5815 
5816         /*
5817          * Link new mblk in after the "request" mblks.
5818          */
5819         linkb(mp, nmp);
5820 
5821         slp->sl_flags |= FCIP_SLFAST;
5822 
5823         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
5824             (CE_NOTE, "dliochdr : returns success "));
5825         miocack(wq, mp, msgsize(mp->b_cont), 0);
5826 }
5827 
5828 
5829 /*
5830  * Establish a kmem cache for fcip packets
5831  */
5832 static int
5833 fcip_cache_constructor(void *buf, void *arg, int flags)
5834 {
5835         fcip_pkt_t              *fcip_pkt = buf;
5836         fc_packet_t             *fc_pkt;
5837         fcip_port_info_t        *fport = (fcip_port_info_t *)arg;
5838         int                     (*cb) (caddr_t);
5839         struct fcip             *fptr;
5840 
5841         cb = (flags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
5842 
5843         ASSERT(fport != NULL);
5844 
5845         fptr = fport->fcipp_fcip;
5846 
5847         /*
5848          * we allocated space for our private area at the end of the
5849          * fc packet. Make sure we point to it correctly. Ideally we
5850          * should just push fc_packet_private to the beginning or end
5851          * of the fc_packet structure
5852          */
5853         fcip_pkt->fcip_pkt_next = NULL;
5854         fcip_pkt->fcip_pkt_prev = NULL;
5855         fcip_pkt->fcip_pkt_dest = NULL;
5856         fcip_pkt->fcip_pkt_state = 0;
5857         fcip_pkt->fcip_pkt_reason = 0;
5858         fcip_pkt->fcip_pkt_flags = 0;
5859         fcip_pkt->fcip_pkt_fptr = fptr;
5860         fcip_pkt->fcip_pkt_dma_flags = 0;
5861 
5862         fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
5863         fc_pkt->pkt_ulp_rscn_infop = NULL;
5864 
5865         /*
5866          * We use pkt_cmd_dma for OUTBOUND requests. We don't expect
5867          * any responses for outbound IP data so no need to setup
5868          * response or data dma handles.
5869          */
5870         if (ddi_dma_alloc_handle(fport->fcipp_dip,
5871             &fport->fcipp_cmd_dma_attr, cb, NULL,
5872             &fc_pkt->pkt_cmd_dma) != DDI_SUCCESS) {
5873                 return (FCIP_FAILURE);
5874         }
5875 
5876         fc_pkt->pkt_cmd_acc = fc_pkt->pkt_resp_acc = NULL;
5877         fc_pkt->pkt_fca_private = (opaque_t)((caddr_t)buf +
5878             sizeof (fcip_pkt_t));
5879         fc_pkt->pkt_ulp_private = (opaque_t)fcip_pkt;
5880 
5881         fc_pkt->pkt_cmd_cookie_cnt = fc_pkt->pkt_resp_cookie_cnt =
5882             fc_pkt->pkt_data_cookie_cnt = 0;
5883         fc_pkt->pkt_cmd_cookie = fc_pkt->pkt_resp_cookie =
5884             fc_pkt->pkt_data_cookie = NULL;
5885 
5886         return (FCIP_SUCCESS);
5887 }
5888 
5889 /*
5890  * destroy the fcip kmem cache
5891  */
5892 static void
5893 fcip_cache_destructor(void *buf, void *arg)
5894 {
5895         fcip_pkt_t              *fcip_pkt = (fcip_pkt_t *)buf;
5896         fc_packet_t             *fc_pkt;
5897         fcip_port_info_t        *fport = (fcip_port_info_t *)arg;
5898         struct fcip             *fptr;
5899 
5900         ASSERT(fport != NULL);
5901 
5902         fptr = fport->fcipp_fcip;
5903 
5904         ASSERT(fptr == fcip_pkt->fcip_pkt_fptr);
5905         fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
5906 
5907         if (fc_pkt->pkt_cmd_dma) {
5908                 ddi_dma_free_handle(&fc_pkt->pkt_cmd_dma);
5909         }
5910 }
5911 
5912 /*
5913  * the fcip destination structure is hashed on Node WWN assuming
5914  * a  NAA_ID of 0x1 (IEEE)
5915  */
5916 static struct fcip_dest *
5917 fcip_get_dest(struct fcip *fptr, la_wwn_t *pwwn)
5918 {
5919         struct fcip_dest        *fdestp = NULL;
5920         fcip_port_info_t        *fport;
5921         int                     hash_bucket;
5922         opaque_t                pd;
5923         int                     rval;
5924         struct fcip_routing_table *frp;
5925         la_wwn_t                twwn;
5926         uint32_t                *twwnp = (uint32_t *)&twwn;
5927 
5928         hash_bucket = FCIP_DEST_HASH(pwwn->raw_wwn);
5929         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
5930             (CE_NOTE, "get dest hashbucket : 0x%x", hash_bucket));
5931         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
5932             (CE_NOTE, "0x%x 0x%x 0x%x 0x%x 0x%x 0x%x",
5933             pwwn->raw_wwn[2], pwwn->raw_wwn[3], pwwn->raw_wwn[4],
5934             pwwn->raw_wwn[5], pwwn->raw_wwn[6], pwwn->raw_wwn[7]));
5935 
5936         ASSERT(hash_bucket < FCIP_DEST_HASH_ELEMS);
5937 
5938         if (fcip_check_port_exists(fptr)) {
5939                 /* fptr is stale, return fdestp */
5940                 return (fdestp);
5941         }
5942         fport = fptr->fcip_port_info;
5943 
5944         /*
5945          * First check if we have active I/Os going on with the
5946          * destination port (an entry would exist in fcip_dest hash table)
5947          */
5948         mutex_enter(&fptr->fcip_dest_mutex);
5949         fdestp = fptr->fcip_dest[hash_bucket];
5950         while (fdestp != NULL) {
5951                 mutex_enter(&fdestp->fcipd_mutex);
5952                 if (fdestp->fcipd_rtable) {
5953                         if (fcip_wwn_compare(pwwn, &fdestp->fcipd_pwwn,
5954                             FCIP_COMPARE_NWWN) == 0) {
5955                                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
5956                                     (CE_NOTE, "found fdestp"));
5957                                 mutex_exit(&fdestp->fcipd_mutex);
5958                                 mutex_exit(&fptr->fcip_dest_mutex);
5959                                 return (fdestp);
5960                         }
5961                 }
5962                 mutex_exit(&fdestp->fcipd_mutex);
5963                 fdestp = fdestp->fcipd_next;
5964         }
5965         mutex_exit(&fptr->fcip_dest_mutex);
5966 
5967         /*
5968          * We did not find the destination port information in our
5969          * active port list so search for an entry in our routing
5970          * table.
5971          */
5972         mutex_enter(&fptr->fcip_rt_mutex);
5973         frp = fcip_lookup_rtable(fptr, pwwn, FCIP_COMPARE_NWWN);
5974         mutex_exit(&fptr->fcip_rt_mutex);
5975 
5976         if (frp == NULL || (frp && (!FCIP_RTE_UNAVAIL(frp->fcipr_state)) &&
5977             frp->fcipr_state != PORT_DEVICE_LOGGED_IN) ||
5978             (frp && frp->fcipr_pd == NULL)) {
5979                 /*
5980                  * No entry for the destination port in our routing
5981                  * table too. First query the transport to see if it
5982                  * already has structures for the destination port in
5983                  * its hash tables. This must be done for all topologies
5984                  * since we could have retired entries in the hash tables
5985                  * which may have to be re-added without a statechange
5986                  * callback happening. Its better to try and get an entry
5987                  * for the destination port rather than simply failing a
5988                  * request though it may be an overkill in private loop
5989                  * topologies.
5990                  * If a entry for the remote port exists in the transport's
5991                  * hash tables, we are fine and can add the entry to our
5992                  * routing and dest hash lists, Else for fabric configs we
5993                  * query the nameserver if one exists or issue FARP ELS.
5994                  */
5995 
5996                 /*
5997                  * We need to do a PortName based Nameserver
5998                  * query operation. So get the right PortWWN
5999                  * for the adapter.
6000                  */
6001                 bcopy(pwwn, &twwn, sizeof (la_wwn_t));
6002 
6003                 /*
6004                  * Try IEEE Name (Format 1) first, this is the default and
6005                  * Emulex uses this format.
6006                  */
6007                 pd = fc_ulp_get_remote_port(fport->fcipp_handle,
6008                                             &twwn, &rval, 1);
6009 
6010                 if (rval != FC_SUCCESS) {
6011                         /*
6012                          * If IEEE Name (Format 1) query failed, try IEEE
6013                          * Extended Name (Format 2) which Qlogic uses.
6014                          * And try port 1 on Qlogic FC-HBA first.
6015                          * Note: On x86, we need to byte swap the 32-bit
6016                          * word first, after the modification, swap it back.
6017                          */
6018                         *twwnp = BE_32(*twwnp);
6019                         twwn.w.nport_id = QLC_PORT_1_ID_BITS;
6020                         twwn.w.naa_id = QLC_PORT_NAA;
6021                         *twwnp = BE_32(*twwnp);
6022                         pd = fc_ulp_get_remote_port(fport->fcipp_handle,
6023                                                     &twwn, &rval, 1);
6024                 }
6025 
6026                 if (rval != FC_SUCCESS) {
6027                         /* If still failed, try port 2 on Qlogic FC-HBA. */
6028                         *twwnp = BE_32(*twwnp);
6029                         twwn.w.nport_id = QLC_PORT_2_ID_BITS;
6030                         *twwnp = BE_32(*twwnp);
6031                         pd = fc_ulp_get_remote_port(fport->fcipp_handle,
6032                                                     &twwn, &rval, 1);
6033                 }
6034 
6035                 if (rval == FC_SUCCESS) {
6036                         fc_portmap_t    map;
6037                         /*
6038                          * Add the newly found destination structure
6039                          * to our routing table. Create a map with
6040                          * the device we found. We could ask the
6041                          * transport to give us the list of all
6042                          * devices connected to our port but we
6043                          * probably don't need to know all the devices
6044                          * so let us just constuct a list with only
6045                          * one device instead.
6046                          */
6047 
6048                         fc_ulp_copy_portmap(&map, pd);
6049                         fcip_rt_update(fptr, &map, 1);
6050 
6051                         mutex_enter(&fptr->fcip_rt_mutex);
6052                         frp = fcip_lookup_rtable(fptr, pwwn,
6053                             FCIP_COMPARE_NWWN);
6054                         mutex_exit(&fptr->fcip_rt_mutex);
6055 
6056                         fdestp = fcip_add_dest(fptr, frp);
6057                 } else if (fcip_farp_supported &&
6058                         (FC_TOP_EXTERNAL(fport->fcipp_topology) ||
6059                         (fport->fcipp_topology == FC_TOP_PT_PT))) {
6060                         /*
6061                          * The Name server request failed so
6062                          * issue an FARP
6063                          */
6064                         fdestp = fcip_do_farp(fptr, pwwn, NULL,
6065                                 0, 0);
6066                 } else {
6067                     fdestp = NULL;
6068                 }
6069         } else if (frp && frp->fcipr_state == PORT_DEVICE_LOGGED_IN) {
6070                 /*
6071                  * Prepare a dest structure to return to caller
6072                  */
6073                 fdestp = fcip_add_dest(fptr, frp);
6074                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
6075                     (CE_NOTE, "in fcip get dest non fabric"));
6076         }
6077         return (fdestp);
6078 }
6079 
6080 
6081 /*
6082  * Endian clean WWN compare.
6083  * Returns 0 if they compare OK, else return non zero value.
6084  * flag can be bitwise OR of FCIP_COMPARE_NWWN, FCIP_COMPARE_PWWN,
6085  * FCIP_COMPARE_BROADCAST.
6086  */
6087 static int
6088 fcip_wwn_compare(la_wwn_t *wwn1, la_wwn_t *wwn2, int flag)
6089 {
6090         int rval = 0;
6091         if ((wwn1->raw_wwn[2] != wwn2->raw_wwn[2]) ||
6092             (wwn1->raw_wwn[3] != wwn2->raw_wwn[3]) ||
6093             (wwn1->raw_wwn[4] != wwn2->raw_wwn[4]) ||
6094             (wwn1->raw_wwn[5] != wwn2->raw_wwn[5]) ||
6095             (wwn1->raw_wwn[6] != wwn2->raw_wwn[6]) ||
6096             (wwn1->raw_wwn[7] != wwn2->raw_wwn[7])) {
6097                 rval = 1;
6098         } else if ((flag == FCIP_COMPARE_PWWN) &&
6099             (((wwn1->raw_wwn[0] & 0xf0) != (wwn2->raw_wwn[0] & 0xf0)) ||
6100             (wwn1->raw_wwn[1] != wwn2->raw_wwn[1]))) {
6101                 rval = 1;
6102         }
6103         return (rval);
6104 }
6105 
6106 
6107 /*
6108  * Add an entry for a remote port in the dest hash table. Dest hash table
6109  * has entries for ports in the routing hash table with which we decide
6110  * to establish IP communication with. The no. of entries in the dest hash
6111  * table must always be less than or equal to the entries in the routing
6112  * hash table. Every entry in the dest hash table ofcourse must have a
6113  * corresponding entry in the routing hash table
6114  */
6115 static struct fcip_dest *
6116 fcip_add_dest(struct fcip *fptr, struct fcip_routing_table *frp)
6117 {
6118         struct fcip_dest *fdestp = NULL;
6119         la_wwn_t        *pwwn;
6120         int hash_bucket;
6121         struct fcip_dest *fdest_new;
6122 
6123         if (frp == NULL) {
6124                 return (fdestp);
6125         }
6126 
6127         pwwn = &frp->fcipr_pwwn;
6128         mutex_enter(&fptr->fcip_dest_mutex);
6129         hash_bucket = FCIP_DEST_HASH(pwwn->raw_wwn);
6130         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
6131             (CE_NOTE, "add dest hash_bucket: 0x%x", hash_bucket));
6132 
6133         ASSERT(hash_bucket < FCIP_DEST_HASH_ELEMS);
6134 
6135         fdestp = fptr->fcip_dest[hash_bucket];
6136         while (fdestp != NULL) {
6137                 mutex_enter(&fdestp->fcipd_mutex);
6138                 if (fdestp->fcipd_rtable) {
6139                         if (fcip_wwn_compare(pwwn, &fdestp->fcipd_pwwn,
6140                             FCIP_COMPARE_PWWN) == 0) {
6141                                 mutex_exit(&fdestp->fcipd_mutex);
6142                                 mutex_exit(&fptr->fcip_dest_mutex);
6143                                 return (fdestp);
6144                         }
6145                 }
6146                 mutex_exit(&fdestp->fcipd_mutex);
6147                 fdestp = fdestp->fcipd_next;
6148         }
6149 
6150         ASSERT(fdestp == NULL);
6151 
6152         fdest_new = (struct fcip_dest *)
6153                         kmem_zalloc(sizeof (struct fcip_dest), KM_SLEEP);
6154 
6155         mutex_init(&fdest_new->fcipd_mutex, NULL, MUTEX_DRIVER, NULL);
6156         fdest_new->fcipd_next = fptr->fcip_dest[hash_bucket];
6157         fdest_new->fcipd_refcnt = 0;
6158         fdest_new->fcipd_rtable = frp;
6159         fdest_new->fcipd_ncmds = 0;
6160         fptr->fcip_dest[hash_bucket] = fdest_new;
6161         fdest_new->fcipd_flags = FCIP_PORT_NOTLOGGED;
6162 
6163         mutex_exit(&fptr->fcip_dest_mutex);
6164         return (fdest_new);
6165 }
6166 
6167 /*
6168  * Cleanup the dest hash table and remove all entries
6169  */
6170 static void
6171 fcip_cleanup_dest(struct fcip *fptr)
6172 {
6173         struct fcip_dest *fdestp = NULL;
6174         struct fcip_dest *fdest_delp = NULL;
6175         int i;
6176 
6177         mutex_enter(&fptr->fcip_dest_mutex);
6178 
6179         for (i = 0; i < FCIP_DEST_HASH_ELEMS; i++) {
6180                 fdestp = fptr->fcip_dest[i];
6181                 while (fdestp != NULL) {
6182                         mutex_destroy(&fdestp->fcipd_mutex);
6183                         fdest_delp = fdestp;
6184                         fdestp = fdestp->fcipd_next;
6185                         kmem_free(fdest_delp, sizeof (struct fcip_dest));
6186                         fptr->fcip_dest[i] = NULL;
6187                 }
6188         }
6189         mutex_exit(&fptr->fcip_dest_mutex);
6190 }
6191 
6192 
6193 /*
6194  * Send FARP requests for Fabric ports when we don't have the port
6195  * we wish to talk to in our routing hash table. FARP is specially required
6196  * to talk to FC switches for inband switch management. Most FC switches
6197  * today have a switch FC IP address for IP over FC inband switch management
6198  * but the WWN and Port_ID for this traffic is not available through the
6199  * Nameservers since the switch themeselves are transparent.
6200  */
6201 /* ARGSUSED */
6202 static struct fcip_dest *
6203 fcip_do_farp(struct fcip *fptr, la_wwn_t *pwwn, char *ip_addr,
6204     size_t ip_addr_len, int flags)
6205 {
6206         fcip_pkt_t              *fcip_pkt;
6207         fc_packet_t             *fc_pkt;
6208         fcip_port_info_t        *fport = fptr->fcip_port_info;
6209         la_els_farp_t           farp_cmd;
6210         la_els_farp_t           *fcmd;
6211         struct fcip_dest        *fdestp = NULL;
6212         int                     rval;
6213         clock_t                 farp_lbolt;
6214         la_wwn_t                broadcast_wwn;
6215         struct fcip_dest        *bdestp;
6216         struct fcip_routing_table       *frp;
6217 
6218         bdestp = fcip_get_dest(fptr, &broadcast_wwn);
6219 
6220         if (bdestp == NULL) {
6221                 return (fdestp);
6222         }
6223 
6224         fcip_pkt = fcip_ipkt_alloc(fptr, sizeof (la_els_farp_t),
6225             sizeof (la_els_farp_t), bdestp->fcipd_pd, KM_SLEEP);
6226 
6227         if (fcip_pkt == NULL) {
6228                 return (fdestp);
6229         }
6230 
6231         fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
6232         ether_to_wwn(&fcip_arpbroadcast_addr, &broadcast_wwn);
6233 
6234         mutex_enter(&bdestp->fcipd_mutex);
6235         if (bdestp->fcipd_rtable == NULL) {
6236                 mutex_exit(&bdestp->fcipd_mutex);
6237                 fcip_ipkt_free(fcip_pkt);
6238                 return (fdestp);
6239         }
6240 
6241         fcip_pkt->fcip_pkt_dest = bdestp;
6242         fc_pkt->pkt_fca_device = bdestp->fcipd_fca_dev;
6243 
6244         bdestp->fcipd_ncmds++;
6245         mutex_exit(&bdestp->fcipd_mutex);
6246 
6247         fcip_init_broadcast_pkt(fcip_pkt, NULL, 1);
6248         fcip_pkt->fcip_pkt_flags |= FCIP_PKT_IN_LIST;
6249 
6250         /*
6251          * Now initialize the FARP payload itself
6252          */
6253         fcmd = &farp_cmd;
6254         fcmd->ls_code.ls_code = LA_ELS_FARP_REQ;
6255         fcmd->ls_code.mbz = 0;
6256         /*
6257          * for now just match the Port WWN since the other match addr
6258          * code points are optional. We can explore matching the IP address
6259          * if needed
6260          */
6261         if (ip_addr) {
6262                 fcmd->match_addr = FARP_MATCH_WW_PN_IPv4;
6263         } else {
6264                 fcmd->match_addr = FARP_MATCH_WW_PN;
6265         }
6266 
6267         /*
6268          * Request the responder port to log into us - that way
6269          * the Transport is aware of the remote port when we create
6270          * an entry for it in our tables
6271          */
6272         fcmd->resp_flags = FARP_INIT_REPLY | FARP_INIT_P_LOGI;
6273         fcmd->req_id = fport->fcipp_sid;
6274         fcmd->dest_id.port_id = fc_pkt->pkt_cmd_fhdr.d_id;
6275         bcopy(&fport->fcipp_pwwn, &fcmd->req_pwwn, sizeof (la_wwn_t));
6276         bcopy(&fport->fcipp_nwwn, &fcmd->req_nwwn, sizeof (la_wwn_t));
6277         bcopy(pwwn, &fcmd->resp_pwwn, sizeof (la_wwn_t));
6278         /*
6279          * copy in source IP address if we get to know it
6280          */
6281         if (ip_addr) {
6282                 bcopy(ip_addr, fcmd->resp_ip, ip_addr_len);
6283         }
6284 
6285         fc_pkt->pkt_cmdlen = sizeof (la_els_farp_t);
6286         fc_pkt->pkt_rsplen = sizeof (la_els_farp_t);
6287         fc_pkt->pkt_tran_type = FC_PKT_EXCHANGE;
6288         fc_pkt->pkt_ulp_private = (opaque_t)fcip_pkt;
6289 
6290         /*
6291          * Endian safe copy
6292          */
6293         FCIP_CP_OUT(fcmd, fc_pkt->pkt_cmd, fc_pkt->pkt_cmd_acc,
6294             sizeof (la_els_farp_t));
6295 
6296         /*
6297          * send the packet in polled mode.
6298          */
6299         rval = fc_ulp_issue_els(fport->fcipp_handle, fc_pkt);
6300         if (rval != FC_SUCCESS) {
6301                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM, (CE_WARN,
6302                     "fcip_transport of farp pkt failed 0x%x", rval));
6303                 fcip_pkt->fcip_pkt_flags &= ~FCIP_PKT_IN_LIST;
6304                 fcip_ipkt_free(fcip_pkt);
6305 
6306                 mutex_enter(&bdestp->fcipd_mutex);
6307                 bdestp->fcipd_ncmds--;
6308                 mutex_exit(&bdestp->fcipd_mutex);
6309 
6310                 return (fdestp);
6311         }
6312 
6313         farp_lbolt = ddi_get_lbolt();
6314         farp_lbolt += drv_usectohz(FCIP_FARP_TIMEOUT);
6315 
6316         mutex_enter(&fptr->fcip_mutex);
6317         fptr->fcip_farp_rsp_flag = 0;
6318         while (!fptr->fcip_farp_rsp_flag) {
6319                 if (cv_timedwait(&fptr->fcip_farp_cv, &fptr->fcip_mutex,
6320                     farp_lbolt) == -1) {
6321                         /*
6322                          * No FARP response from any destination port
6323                          * so bail out.
6324                          */
6325                         fptr->fcip_farp_rsp_flag = 1;
6326                 } else {
6327                         /*
6328                          * We received a FARP response - check to see if the
6329                          * response was in reply to our FARP request.
6330                          */
6331 
6332                         mutex_enter(&fptr->fcip_rt_mutex);
6333                         frp = fcip_lookup_rtable(fptr, pwwn, FCIP_COMPARE_NWWN);
6334                         mutex_exit(&fptr->fcip_rt_mutex);
6335 
6336                         if ((frp != NULL) &&
6337                             !FCIP_RTE_UNAVAIL(frp->fcipr_state)) {
6338                                 fdestp = fcip_get_dest(fptr, pwwn);
6339                         } else {
6340                                 /*
6341                                  * Not our FARP response so go back and wait
6342                                  * again till FARP_TIMEOUT expires
6343                                  */
6344                                 fptr->fcip_farp_rsp_flag = 0;
6345                         }
6346                 }
6347         }
6348         mutex_exit(&fptr->fcip_mutex);
6349 
6350         fcip_pkt->fcip_pkt_flags |= FCIP_PKT_IN_LIST;
6351         fcip_ipkt_free(fcip_pkt);
6352         mutex_enter(&bdestp->fcipd_mutex);
6353         bdestp->fcipd_ncmds--;
6354         mutex_exit(&bdestp->fcipd_mutex);
6355         return (fdestp);
6356 }
6357 
6358 
6359 
6360 /*
6361  * Helper routine to PLOGI to a remote port we wish to talk to.
6362  * This may not be required since the port driver does logins anyway,
6363  * but this can be required in fabric cases since FARP requests/responses
6364  * don't require you to be logged in?
6365  */
6366 
6367 /* ARGSUSED */
6368 static int
6369 fcip_do_plogi(struct fcip *fptr, struct fcip_routing_table *frp)
6370 {
6371         fcip_pkt_t              *fcip_pkt;
6372         fc_packet_t             *fc_pkt;
6373         fcip_port_info_t        *fport = fptr->fcip_port_info;
6374         la_els_logi_t           logi;
6375         int                     rval;
6376         fc_frame_hdr_t          *fr_hdr;
6377 
6378         /*
6379          * Don't bother to login for broadcast RTE entries
6380          */
6381         if ((frp->fcipr_d_id.port_id == 0x0) ||
6382             (frp->fcipr_d_id.port_id == 0xffffff)) {
6383                 return (FC_FAILURE);
6384         }
6385 
6386         /*
6387          * We shouldn't pound in too many logins here
6388          *
6389          */
6390         if (frp->fcipr_state == FCIP_RT_LOGIN_PROGRESS ||
6391             frp->fcipr_state == PORT_DEVICE_LOGGED_IN) {
6392                 return (FC_SUCCESS);
6393         }
6394 
6395         fcip_pkt = fcip_ipkt_alloc(fptr, sizeof (la_els_logi_t),
6396             sizeof (la_els_logi_t), frp->fcipr_pd, KM_SLEEP);
6397 
6398         if (fcip_pkt == NULL) {
6399                 return (FC_FAILURE);
6400         }
6401 
6402         /*
6403          * Update back pointer for login state update
6404          */
6405         fcip_pkt->fcip_pkt_frp = frp;
6406         frp->fcipr_state = FCIP_RT_LOGIN_PROGRESS;
6407 
6408         fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
6409 
6410         /*
6411          * Initialize frame header for ELS
6412          */
6413         fr_hdr = &fc_pkt->pkt_cmd_fhdr;
6414         fr_hdr->r_ctl = R_CTL_ELS_REQ;
6415         fr_hdr->type = FC_TYPE_EXTENDED_LS;
6416         fr_hdr->f_ctl = F_CTL_SEQ_INITIATIVE | F_CTL_FIRST_SEQ;
6417         fr_hdr->df_ctl = 0;
6418         fr_hdr->s_id = fport->fcipp_sid.port_id;
6419         fr_hdr->d_id = frp->fcipr_d_id.port_id;
6420         fr_hdr->seq_cnt = 0;
6421         fr_hdr->ox_id = 0xffff;
6422         fr_hdr->rx_id = 0xffff;
6423         fr_hdr->ro = 0;
6424 
6425         fc_pkt->pkt_rsplen = sizeof (la_els_logi_t);
6426         fc_pkt->pkt_comp = fcip_ipkt_callback;
6427         fc_pkt->pkt_tran_type = FC_PKT_EXCHANGE;
6428         fc_pkt->pkt_timeout = 10;    /* 10 seconds */
6429         fcip_pkt->fcip_pkt_ttl = fptr->fcip_timeout_ticks + fc_pkt->pkt_timeout;
6430         fc_pkt->pkt_ulp_private = (opaque_t)fcip_pkt;
6431 
6432         /*
6433          * Everybody does class 3, so let's just set it.  If the transport
6434          * knows better, it will deal with the class appropriately.
6435          */
6436 
6437         fc_pkt->pkt_tran_flags = FC_TRAN_INTR | FC_TRAN_CLASS3;
6438 
6439         /*
6440          * we need only fill in the ls_code and the cmd frame header
6441          */
6442         bzero((void *)&logi, sizeof (la_els_logi_t));
6443         logi.ls_code.ls_code = LA_ELS_PLOGI;
6444         logi.ls_code.mbz = 0;
6445 
6446         FCIP_CP_OUT((uint8_t *)&logi, fc_pkt->pkt_cmd, fc_pkt->pkt_cmd_acc,
6447             sizeof (la_els_logi_t));
6448 
6449         rval = fc_ulp_login(fport->fcipp_handle, &fc_pkt, 1);
6450         if (rval != FC_SUCCESS) {
6451                 cmn_err(CE_WARN,
6452                     "!fc_ulp_login failed for d_id: 0x%x, rval: 0x%x",
6453                     frp->fcipr_d_id.port_id, rval);
6454                 fcip_ipkt_free(fcip_pkt);
6455         }
6456         return (rval);
6457 }
6458 
6459 /*
6460  * The packet callback routine - called from the transport/FCA after
6461  * it is done DMA'ing/sending out the packet contents on the wire so
6462  * that the alloc'ed packet can be freed
6463  */
6464 static void
6465 fcip_ipkt_callback(fc_packet_t *fc_pkt)
6466 {
6467         ls_code_t                       logi_req;
6468         ls_code_t                       logi_resp;
6469         fcip_pkt_t                      *fcip_pkt;
6470         fc_frame_hdr_t                  *fr_hdr;
6471         struct fcip                     *fptr;
6472         fcip_port_info_t                *fport;
6473         struct fcip_routing_table       *frp;
6474 
6475         fr_hdr = &fc_pkt->pkt_cmd_fhdr;
6476 
6477         FCIP_CP_IN(fc_pkt->pkt_resp, (uint8_t *)&logi_resp,
6478             fc_pkt->pkt_resp_acc, sizeof (logi_resp));
6479 
6480         FCIP_CP_IN(fc_pkt->pkt_cmd, (uint8_t *)&logi_req, fc_pkt->pkt_cmd_acc,
6481             sizeof (logi_req));
6482 
6483         fcip_pkt = (fcip_pkt_t *)fc_pkt->pkt_ulp_private;
6484         frp = fcip_pkt->fcip_pkt_frp;
6485         fptr = fcip_pkt->fcip_pkt_fptr;
6486         fport = fptr->fcip_port_info;
6487 
6488         ASSERT(logi_req.ls_code == LA_ELS_PLOGI);
6489 
6490         if (fc_pkt->pkt_state != FC_PKT_SUCCESS ||
6491             logi_resp.ls_code != LA_ELS_ACC) {
6492                 /* EMPTY */
6493 
6494                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM, (CE_WARN,
6495                     "opcode : 0x%x to d_id: 0x%x failed",
6496                     logi_req.ls_code, fr_hdr->d_id));
6497 
6498                 mutex_enter(&fptr->fcip_rt_mutex);
6499                 frp->fcipr_state = PORT_DEVICE_INVALID;
6500                 frp->fcipr_invalid_timeout = fptr->fcip_timeout_ticks +
6501                     (FCIP_RTE_TIMEOUT / 2);
6502                 mutex_exit(&fptr->fcip_rt_mutex);
6503         } else {
6504                 fc_portid_t     d_id;
6505 
6506                 d_id.port_id = fr_hdr->d_id;
6507                 d_id.priv_lilp_posit = 0;
6508 
6509                 /*
6510                  * Update PLOGI results; FCA Handle, and Port device handles
6511                  */
6512                 mutex_enter(&fptr->fcip_rt_mutex);
6513                 frp->fcipr_pd = fc_pkt->pkt_pd;
6514                 frp->fcipr_fca_dev =
6515                     fc_ulp_get_fca_device(fport->fcipp_handle, d_id);
6516                 frp->fcipr_state = PORT_DEVICE_LOGGED_IN;
6517                 mutex_exit(&fptr->fcip_rt_mutex);
6518         }
6519 
6520         fcip_ipkt_free(fcip_pkt);
6521 }
6522 
6523 
6524 /*
6525  * pkt_alloc routine for outbound IP datagrams. The cache constructor
6526  * Only initializes the pkt_cmd_dma (which is where the outbound datagram
6527  * is stuffed) since we don't expect response
6528  */
6529 static fcip_pkt_t *
6530 fcip_pkt_alloc(struct fcip *fptr, mblk_t *bp, int flags, int datalen)
6531 {
6532         fcip_pkt_t      *fcip_pkt;
6533         fc_packet_t     *fc_pkt;
6534         ddi_dma_cookie_t        pkt_cookie;
6535         ddi_dma_cookie_t        *cp;
6536         uint32_t                cnt;
6537         fcip_port_info_t        *fport = fptr->fcip_port_info;
6538 
6539         fcip_pkt = kmem_cache_alloc(fptr->fcip_xmit_cache, flags);
6540         if (fcip_pkt == NULL) {
6541                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM, (CE_WARN,
6542                     "fcip_pkt_alloc: kmem_cache_alloc failed"));
6543                 return (NULL);
6544         }
6545 
6546         fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
6547         fcip_pkt->fcip_pkt_fcpktp = fc_pkt;
6548         fc_pkt->pkt_tran_flags = 0;
6549         fcip_pkt->fcip_pkt_dma_flags = 0;
6550 
6551         /*
6552          * the cache constructor has allocated the dma handle
6553          */
6554         fc_pkt->pkt_cmd = (caddr_t)bp->b_rptr;
6555         if (ddi_dma_addr_bind_handle(fc_pkt->pkt_cmd_dma, NULL,
6556             (caddr_t)bp->b_rptr, datalen, DDI_DMA_WRITE | DDI_DMA_CONSISTENT,
6557             DDI_DMA_DONTWAIT, NULL, &pkt_cookie,
6558             &fc_pkt->pkt_cmd_cookie_cnt) != DDI_DMA_MAPPED) {
6559                         goto fail;
6560         }
6561 
6562         fcip_pkt->fcip_pkt_dma_flags |= FCIP_CMD_DMA_BOUND;
6563 
6564         if (fc_pkt->pkt_cmd_cookie_cnt >
6565             fport->fcipp_cmd_dma_attr.dma_attr_sgllen) {
6566                 goto fail;
6567         }
6568 
6569         ASSERT(fc_pkt->pkt_cmd_cookie_cnt != 0);
6570 
6571         cp = fc_pkt->pkt_cmd_cookie = (ddi_dma_cookie_t *)kmem_alloc(
6572             fc_pkt->pkt_cmd_cookie_cnt * sizeof (pkt_cookie),
6573             KM_NOSLEEP);
6574 
6575         if (cp == NULL) {
6576                 goto fail;
6577         }
6578 
6579         *cp = pkt_cookie;
6580         cp++;
6581         for (cnt = 1; cnt < fc_pkt->pkt_cmd_cookie_cnt; cnt++, cp++) {
6582                 ddi_dma_nextcookie(fc_pkt->pkt_cmd_dma, &pkt_cookie);
6583                 *cp = pkt_cookie;
6584         }
6585 
6586         fc_pkt->pkt_cmdlen = datalen;
6587 
6588         fcip_pkt->fcip_pkt_mp = NULL;
6589         fcip_pkt->fcip_pkt_wq = NULL;
6590         fcip_pkt->fcip_pkt_dest = NULL;
6591         fcip_pkt->fcip_pkt_next = NULL;
6592         fcip_pkt->fcip_pkt_prev = NULL;
6593         fcip_pkt->fcip_pkt_state = 0;
6594         fcip_pkt->fcip_pkt_reason = 0;
6595         fcip_pkt->fcip_pkt_flags = 0;
6596         fcip_pkt->fcip_pkt_frp = NULL;
6597 
6598         return (fcip_pkt);
6599 fail:
6600         if (fcip_pkt) {
6601                 fcip_pkt_free(fcip_pkt, 0);
6602         }
6603         return ((fcip_pkt_t *)0);
6604 }
6605 
6606 /*
6607  * Free a packet and all its associated resources
6608  */
6609 static void
6610 fcip_pkt_free(struct fcip_pkt *fcip_pkt, int free_mblk)
6611 {
6612         fc_packet_t     *fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
6613         struct fcip *fptr = fcip_pkt->fcip_pkt_fptr;
6614 
6615         if (fc_pkt->pkt_cmd_cookie != NULL) {
6616                 kmem_free(fc_pkt->pkt_cmd_cookie, fc_pkt->pkt_cmd_cookie_cnt *
6617                     sizeof (ddi_dma_cookie_t));
6618                 fc_pkt->pkt_cmd_cookie = NULL;
6619         }
6620 
6621         fcip_free_pkt_dma(fcip_pkt);
6622         if (free_mblk && fcip_pkt->fcip_pkt_mp) {
6623                 freemsg(fcip_pkt->fcip_pkt_mp);
6624                 fcip_pkt->fcip_pkt_mp = NULL;
6625         }
6626 
6627         (void) fc_ulp_uninit_packet(fptr->fcip_port_info->fcipp_handle, fc_pkt);
6628 
6629         kmem_cache_free(fptr->fcip_xmit_cache, (void *)fcip_pkt);
6630 }
6631 
6632 /*
6633  * Allocate a Packet for internal driver use. This is for requests
6634  * that originate from within the driver
6635  */
6636 static fcip_pkt_t *
6637 fcip_ipkt_alloc(struct fcip *fptr, int cmdlen, int resplen,
6638     opaque_t pd, int flags)
6639 {
6640         fcip_pkt_t              *fcip_pkt;
6641         fc_packet_t             *fc_pkt;
6642         int                     (*cb)(caddr_t);
6643         fcip_port_info_t        *fport = fptr->fcip_port_info;
6644         size_t                  real_len;
6645         uint_t                  held_here = 0;
6646         ddi_dma_cookie_t        pkt_cookie;
6647         ddi_dma_cookie_t        *cp;
6648         uint32_t                cnt;
6649 
6650         cb = (flags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
6651 
6652         fcip_pkt = kmem_zalloc((sizeof (fcip_pkt_t) +
6653             fport->fcipp_fca_pkt_size), flags);
6654 
6655         if (fcip_pkt == NULL) {
6656                 FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
6657                     (CE_WARN, "pkt alloc of ineternal pkt failed"));
6658                 goto fail;
6659         }
6660 
6661         fcip_pkt->fcip_pkt_flags = FCIP_PKT_INTERNAL;
6662         fcip_pkt->fcip_pkt_fptr = fptr;
6663         fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
6664         fcip_pkt->fcip_pkt_fcpktp = fc_pkt;
6665         fc_pkt->pkt_tran_flags = 0;
6666         fc_pkt->pkt_cmdlen = 0;
6667         fc_pkt->pkt_rsplen = 0;
6668         fc_pkt->pkt_datalen = 0;
6669         fc_pkt->pkt_fca_private = (opaque_t)((caddr_t)fcip_pkt +
6670             sizeof (fcip_pkt_t));
6671         fc_pkt->pkt_ulp_private = (opaque_t)fcip_pkt;
6672 
6673         if (cmdlen) {
6674                 if (ddi_dma_alloc_handle(fptr->fcip_dip,
6675                     &fport->fcipp_cmd_dma_attr, cb, NULL,
6676                     &fc_pkt->pkt_cmd_dma) != DDI_SUCCESS) {
6677                         goto fail;
6678                 }
6679 
6680                 if (ddi_dma_mem_alloc(fc_pkt->pkt_cmd_dma, cmdlen,
6681                     &fport->fcipp_fca_acc_attr, DDI_DMA_CONSISTENT,
6682                     cb, NULL, (caddr_t *)&fc_pkt->pkt_cmd,
6683                     &real_len, &fc_pkt->pkt_cmd_acc) != DDI_SUCCESS) {
6684                         goto fail;
6685                 }
6686 
6687                 fcip_pkt->fcip_pkt_dma_flags |= FCIP_CMD_DMA_MEM;
6688                 fc_pkt->pkt_cmdlen = cmdlen;
6689 
6690                 if (real_len < cmdlen) {
6691                         goto fail;
6692                 }
6693 
6694                 if (ddi_dma_addr_bind_handle(fc_pkt->pkt_cmd_dma, NULL,
6695                     (caddr_t)fc_pkt->pkt_cmd, real_len,
6696                     DDI_DMA_WRITE | DDI_DMA_CONSISTENT, cb, NULL,
6697                     &pkt_cookie, &fc_pkt->pkt_cmd_cookie_cnt) !=
6698                     DDI_DMA_MAPPED) {
6699                         goto fail;
6700                 }
6701 
6702                 fcip_pkt->fcip_pkt_dma_flags |= FCIP_CMD_DMA_BOUND;
6703 
6704                 if (fc_pkt->pkt_cmd_cookie_cnt >
6705                     fport->fcipp_cmd_dma_attr.dma_attr_sgllen) {
6706                         goto fail;
6707                 }
6708 
6709                 ASSERT(fc_pkt->pkt_cmd_cookie_cnt != 0);
6710 
6711                 cp = fc_pkt->pkt_cmd_cookie = (ddi_dma_cookie_t *)kmem_alloc(
6712                     fc_pkt->pkt_cmd_cookie_cnt * sizeof (pkt_cookie),
6713                     KM_NOSLEEP);
6714 
6715                 if (cp == NULL) {
6716                         goto fail;
6717                 }
6718 
6719                 *cp = pkt_cookie;
6720                 cp++;
6721                 for (cnt = 1; cnt < fc_pkt->pkt_cmd_cookie_cnt; cnt++, cp++) {
6722                         ddi_dma_nextcookie(fc_pkt->pkt_cmd_dma, &pkt_cookie);
6723                         *cp = pkt_cookie;
6724                 }
6725         }
6726 
6727         if (resplen) {
6728                 if (ddi_dma_alloc_handle(fptr->fcip_dip,
6729                     &fport->fcipp_resp_dma_attr, cb, NULL,
6730                     &fc_pkt->pkt_resp_dma) != DDI_SUCCESS) {
6731                         goto fail;
6732                 }
6733 
6734                 if (ddi_dma_mem_alloc(fc_pkt->pkt_resp_dma, resplen,
6735                     &fport->fcipp_fca_acc_attr, DDI_DMA_CONSISTENT,
6736                     cb, NULL, (caddr_t *)&fc_pkt->pkt_resp,
6737                     &real_len, &fc_pkt->pkt_resp_acc) != DDI_SUCCESS) {
6738                         goto fail;
6739                 }
6740 
6741                 fcip_pkt->fcip_pkt_dma_flags |= FCIP_RESP_DMA_MEM;
6742 
6743                 if (real_len < resplen) {
6744                         goto fail;
6745                 }
6746 
6747                 if (ddi_dma_addr_bind_handle(fc_pkt->pkt_resp_dma, NULL,
6748                     (caddr_t)fc_pkt->pkt_resp, real_len,
6749                     DDI_DMA_WRITE | DDI_DMA_CONSISTENT, cb, NULL,
6750                     &pkt_cookie, &fc_pkt->pkt_resp_cookie_cnt) !=
6751                     DDI_DMA_MAPPED) {
6752                         goto fail;
6753                 }
6754 
6755                 fcip_pkt->fcip_pkt_dma_flags |= FCIP_RESP_DMA_BOUND;
6756                 fc_pkt->pkt_rsplen = resplen;
6757 
6758                 if (fc_pkt->pkt_resp_cookie_cnt >
6759                     fport->fcipp_resp_dma_attr.dma_attr_sgllen) {
6760                         goto fail;
6761                 }
6762 
6763                 ASSERT(fc_pkt->pkt_resp_cookie_cnt != 0);
6764 
6765                 cp = fc_pkt->pkt_resp_cookie = (ddi_dma_cookie_t *)kmem_alloc(
6766                     fc_pkt->pkt_resp_cookie_cnt * sizeof (pkt_cookie),
6767                     KM_NOSLEEP);
6768 
6769                 if (cp == NULL) {
6770                         goto fail;
6771                 }
6772 
6773                 *cp = pkt_cookie;
6774                 cp++;
6775                 for (cnt = 1; cnt < fc_pkt->pkt_resp_cookie_cnt; cnt++, cp++) {
6776                         ddi_dma_nextcookie(fc_pkt->pkt_resp_dma, &pkt_cookie);
6777                         *cp = pkt_cookie;
6778                 }
6779         }
6780 
6781         /*
6782          * Initialize pkt_pd prior to calling fc_ulp_init_packet
6783          */
6784 
6785         fc_pkt->pkt_pd = pd;
6786 
6787         /*
6788          * Ask the FCA to bless the internal packet
6789          */
6790         if (fc_ulp_init_packet((opaque_t)fport->fcipp_handle,
6791             fc_pkt, flags) != FC_SUCCESS) {
6792                 goto fail;
6793         }
6794 
6795         /*
6796          * Keep track of # of ipkts alloc-ed
6797          * This function can get called with mutex either held or not. So, we'll
6798          * grab mutex if it is not already held by this thread.
6799          * This has to be cleaned up someday.
6800          */
6801         if (!MUTEX_HELD(&fptr->fcip_mutex)) {
6802                 held_here = 1;
6803                 mutex_enter(&fptr->fcip_mutex);
6804         }
6805 
6806         fptr->fcip_num_ipkts_pending++;
6807 
6808         if (held_here)
6809                 mutex_exit(&fptr->fcip_mutex);
6810 
6811         return (fcip_pkt);
6812 fail:
6813         if (fcip_pkt) {
6814                 fcip_ipkt_free(fcip_pkt);
6815         }
6816 
6817         return (NULL);
6818 }
6819 
6820 /*
6821  * free up an internal IP packet (like a FARP pkt etc)
6822  */
6823 static void
6824 fcip_ipkt_free(fcip_pkt_t *fcip_pkt)
6825 {
6826         fc_packet_t             *fc_pkt;
6827         struct fcip             *fptr = fcip_pkt->fcip_pkt_fptr;
6828         fcip_port_info_t        *fport = fptr->fcip_port_info;
6829 
6830         ASSERT(fptr != NULL);
6831         ASSERT(!mutex_owned(&fptr->fcip_mutex));
6832 
6833         /* One less ipkt to wait for */
6834         mutex_enter(&fptr->fcip_mutex);
6835         if (fptr->fcip_num_ipkts_pending)    /* Safety check */
6836                 fptr->fcip_num_ipkts_pending--;
6837         mutex_exit(&fptr->fcip_mutex);
6838 
6839         fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
6840 
6841         if (fc_pkt->pkt_cmd_cookie != NULL) {
6842                 kmem_free(fc_pkt->pkt_cmd_cookie, fc_pkt->pkt_cmd_cookie_cnt *
6843                     sizeof (ddi_dma_cookie_t));
6844                 fc_pkt->pkt_cmd_cookie = NULL;
6845         }
6846 
6847         if (fc_pkt->pkt_resp_cookie != NULL) {
6848                 kmem_free(fc_pkt->pkt_resp_cookie, fc_pkt->pkt_resp_cookie_cnt *
6849                     sizeof (ddi_dma_cookie_t));
6850                 fc_pkt->pkt_resp_cookie = NULL;
6851         }
6852 
6853         if (fc_ulp_uninit_packet(fport->fcipp_handle, fc_pkt) != FC_SUCCESS) {
6854                 FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_WARN,
6855                     "fc_ulp_uninit_pkt failed for internal fc pkt 0x%p",
6856                     (void *)fc_pkt));
6857         }
6858         fcip_free_pkt_dma(fcip_pkt);
6859         kmem_free(fcip_pkt, (sizeof (fcip_pkt_t) + fport->fcipp_fca_pkt_size));
6860 }
6861 
6862 /*
6863  * initialize a unicast request. This is a misnomer because even the
6864  * broadcast requests are initialized with this routine
6865  */
6866 static void
6867 fcip_init_unicast_pkt(fcip_pkt_t *fcip_pkt, fc_portid_t sid, fc_portid_t did,
6868     void (*comp) ())
6869 {
6870         fc_packet_t             *fc_pkt;
6871         fc_frame_hdr_t          *fr_hdr;
6872         struct fcip             *fptr = fcip_pkt->fcip_pkt_fptr;
6873 
6874         fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
6875         fr_hdr = &fc_pkt->pkt_cmd_fhdr;
6876 
6877         fr_hdr->r_ctl = R_CTL_DEVICE_DATA | R_CTL_UNSOL_DATA;
6878         fr_hdr->s_id = sid.port_id;
6879         fr_hdr->d_id = did.port_id;
6880         fr_hdr->type = FC_TYPE_IS8802_SNAP;
6881         fr_hdr->f_ctl = F_CTL_FIRST_SEQ | F_CTL_LAST_SEQ;
6882         fr_hdr->df_ctl = DF_CTL_NET_HDR;
6883         fr_hdr->seq_cnt = 0;
6884         fr_hdr->ox_id = 0xffff;
6885         fr_hdr->rx_id = 0xffff;
6886         fr_hdr->ro = 0;
6887         /*
6888          * reset all the length fields
6889          */
6890         fc_pkt->pkt_rsplen = 0;
6891         fc_pkt->pkt_datalen = 0;
6892         fc_pkt->pkt_comp = comp;
6893         if (comp) {
6894                 fc_pkt->pkt_tran_flags |= FC_TRAN_INTR;
6895         } else {
6896                 fc_pkt->pkt_tran_flags |= FC_TRAN_NO_INTR;
6897         }
6898         fc_pkt->pkt_tran_type = FC_PKT_OUTBOUND | FC_PKT_IP_WRITE;
6899         fc_pkt->pkt_timeout = fcip_pkt_ttl_ticks;
6900         fcip_pkt->fcip_pkt_ttl = fptr->fcip_timeout_ticks + fc_pkt->pkt_timeout;
6901 }
6902 
6903 
6904 /*
6905  * Initialize a fcip_packet for broadcast data transfers
6906  */
6907 static void
6908 fcip_init_broadcast_pkt(fcip_pkt_t *fcip_pkt, void (*comp) (), int is_els)
6909 {
6910         fc_packet_t             *fc_pkt;
6911         fc_frame_hdr_t          *fr_hdr;
6912         struct fcip             *fptr = fcip_pkt->fcip_pkt_fptr;
6913         fcip_port_info_t        *fport = fptr->fcip_port_info;
6914         uint32_t                sid;
6915         uint32_t                did;
6916 
6917         FCIP_TNF_PROBE_1((fcip_init_broadcast_pkt, "fcip io", /* CSTYLED */,
6918                 tnf_string, msg, "enter"));
6919         fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
6920         fr_hdr = &fc_pkt->pkt_cmd_fhdr;
6921         sid = fport->fcipp_sid.port_id;
6922 
6923         if (is_els) {
6924                 fr_hdr->r_ctl = R_CTL_ELS_REQ;
6925         } else {
6926                 fr_hdr->r_ctl = R_CTL_DEVICE_DATA | R_CTL_UNSOL_DATA;
6927         }
6928         fr_hdr->s_id = sid;
6929         /*
6930          * The destination broadcast address depends on the topology
6931          * of the underlying port
6932          */
6933         did = fptr->fcip_broadcast_did;
6934         /*
6935          * mark pkt a broadcast pkt
6936          */
6937         fc_pkt->pkt_tran_type = FC_PKT_BROADCAST;
6938 
6939         fr_hdr->d_id = did;
6940         fr_hdr->type = FC_TYPE_IS8802_SNAP;
6941         fr_hdr->f_ctl = F_CTL_FIRST_SEQ | F_CTL_LAST_SEQ | F_CTL_END_SEQ;
6942         fr_hdr->f_ctl &= ~(F_CTL_SEQ_INITIATIVE);
6943         fr_hdr->df_ctl = DF_CTL_NET_HDR;
6944         fr_hdr->seq_cnt = 0;
6945         fr_hdr->ox_id = 0xffff;
6946         fr_hdr->rx_id = 0xffff;
6947         fr_hdr->ro = 0;
6948         fc_pkt->pkt_comp = comp;
6949 
6950         if (comp) {
6951                 fc_pkt->pkt_tran_flags |= FC_TRAN_INTR;
6952         } else {
6953                 fc_pkt->pkt_tran_flags |= FC_TRAN_NO_INTR;
6954         }
6955 
6956         fc_pkt->pkt_tran_type = FC_PKT_BROADCAST;
6957         fc_pkt->pkt_timeout = fcip_pkt_ttl_ticks;
6958         fcip_pkt->fcip_pkt_ttl = fptr->fcip_timeout_ticks + fc_pkt->pkt_timeout;
6959 }
6960 
6961 
6962 
6963 /*
6964  * Free up all DMA resources associated with an allocated packet
6965  */
6966 static void
6967 fcip_free_pkt_dma(fcip_pkt_t *fcip_pkt)
6968 {
6969         fc_packet_t     *fc_pkt;
6970 
6971         fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
6972 
6973         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
6974             (CE_NOTE, "in freepktdma : flags 0x%x",
6975             fcip_pkt->fcip_pkt_dma_flags));
6976 
6977         if (fcip_pkt->fcip_pkt_dma_flags & FCIP_CMD_DMA_BOUND) {
6978                 (void) ddi_dma_unbind_handle(fc_pkt->pkt_cmd_dma);
6979         }
6980         if (fcip_pkt->fcip_pkt_dma_flags & FCIP_CMD_DMA_MEM) {
6981                 ddi_dma_mem_free(&fc_pkt->pkt_cmd_acc);
6982         }
6983 
6984         if (fcip_pkt->fcip_pkt_dma_flags & FCIP_RESP_DMA_BOUND) {
6985                 (void) ddi_dma_unbind_handle(fc_pkt->pkt_resp_dma);
6986         }
6987         if (fcip_pkt->fcip_pkt_dma_flags & FCIP_RESP_DMA_MEM) {
6988                 ddi_dma_mem_free(&fc_pkt->pkt_resp_acc);
6989         }
6990         /*
6991          * for internal commands, we need to free up the dma handles too.
6992          * This is done in the cache destructor for non internal cmds
6993          */
6994         if (fcip_pkt->fcip_pkt_flags & FCIP_PKT_INTERNAL) {
6995                 if (fc_pkt->pkt_cmd_dma) {
6996                         ddi_dma_free_handle(&fc_pkt->pkt_cmd_dma);
6997                 }
6998                 if (fc_pkt->pkt_resp_dma) {
6999                         ddi_dma_free_handle(&fc_pkt->pkt_resp_dma);
7000                 }
7001         }
7002 }
7003 
7004 
7005 /*
7006  * helper routine to generate a string, given an ether addr
7007  */
7008 static void
7009 fcip_ether_to_str(struct ether_addr *e, caddr_t s)
7010 {
7011         int i;
7012 
7013         for (i = 0; i < sizeof (struct ether_addr); i++, s += 2) {
7014                 FCIP_DEBUG(FCIP_DEBUG_MISC,
7015                     (CE_CONT, "0x%02X:", e->ether_addr_octet[i]));
7016                 (void) sprintf(s, "%02X", e->ether_addr_octet[i]);
7017         }
7018 
7019         *s = '\0';
7020 }
7021 
7022 /*
7023  * When a broadcast request comes from the upper streams modules, it
7024  * is ugly to look into every datagram to figure out if it is a broadcast
7025  * datagram or a unicast packet. Instead just add the broadcast entries
7026  * into our routing and dest tables and the standard hash table look ups
7027  * will find the entries. It is a lot cleaner this way. Also Solaris ifconfig
7028  * seems to be very ethernet specific and it requires broadcasts to the
7029  * ether broadcast addr of 0xffffffffff to succeed even though we specified
7030  * in the dl_info request that our broadcast MAC addr is 0x0000000000
7031  * (can't figure out why RFC2625 did this though). So add broadcast entries
7032  * for both MAC address
7033  */
7034 static int
7035 fcip_dest_add_broadcast_entry(struct fcip *fptr, int new_flag)
7036 {
7037         fc_portmap_t            map;
7038         struct fcip_routing_table *frp;
7039         uint32_t                did;
7040         la_wwn_t                broadcast_wwn;
7041 
7042         /*
7043          * get port_id of destination for broadcast - this is topology
7044          * dependent
7045          */
7046         did = fptr->fcip_broadcast_did;
7047 
7048         ether_to_wwn(&fcip_arpbroadcast_addr, &broadcast_wwn);
7049         bcopy((void *)&broadcast_wwn, (void *)&map.map_pwwn, sizeof (la_wwn_t));
7050         bcopy((void *)&broadcast_wwn, (void *)&map.map_nwwn, sizeof (la_wwn_t));
7051 
7052         map.map_did.port_id = did;
7053         map.map_hard_addr.hard_addr = did;
7054         map.map_state = PORT_DEVICE_VALID;
7055         if (new_flag) {
7056                 map.map_type = PORT_DEVICE_NEW;
7057         } else {
7058                 map.map_type = PORT_DEVICE_CHANGED;
7059         }
7060         map.map_flags = 0;
7061         map.map_pd = NULL;
7062         bzero(&map.map_fc4_types, sizeof (map.map_fc4_types));
7063         fcip_rt_update(fptr, &map, 1);
7064         mutex_enter(&fptr->fcip_rt_mutex);
7065         frp = fcip_lookup_rtable(fptr, &broadcast_wwn, FCIP_COMPARE_NWWN);
7066         mutex_exit(&fptr->fcip_rt_mutex);
7067         if (frp == NULL) {
7068                 return (FC_FAILURE);
7069         }
7070         (void) fcip_add_dest(fptr, frp);
7071         /*
7072          * The Upper IP layers expect the traditional broadcast MAC addr
7073          * of 0xff ff ff ff ff ff to work too if we want to plumb the fcip
7074          * stream through the /etc/hostname.fcipXX file. Instead of checking
7075          * each phys addr for a match with fcip's ARP header broadcast
7076          * addr (0x00 00 00 00 00 00), its simply easier to add another
7077          * broadcast entry for 0xff ff ff ff ff ff.
7078          */
7079         ether_to_wwn(&fcipnhbroadcastaddr, &broadcast_wwn);
7080         bcopy((void *)&broadcast_wwn, (void *)&map.map_pwwn, sizeof (la_wwn_t));
7081         bcopy((void *)&broadcast_wwn, (void *)&map.map_nwwn, sizeof (la_wwn_t));
7082         fcip_rt_update(fptr, &map, 1);
7083         mutex_enter(&fptr->fcip_rt_mutex);
7084         frp = fcip_lookup_rtable(fptr, &broadcast_wwn, FCIP_COMPARE_NWWN);
7085         mutex_exit(&fptr->fcip_rt_mutex);
7086         if (frp == NULL) {
7087                 return (FC_FAILURE);
7088         }
7089         (void) fcip_add_dest(fptr, frp);
7090         return (FC_SUCCESS);
7091 }
7092 
7093 /*
7094  * We need to obtain the D_ID of the broadcast port for transmitting all
7095  * our broadcast (and multicast) requests. The broadcast D_ID as we know
7096  * is dependent on the link topology
7097  */
7098 static uint32_t
7099 fcip_get_broadcast_did(struct fcip *fptr)
7100 {
7101         fcip_port_info_t        *fport = fptr->fcip_port_info;
7102         uint32_t                did = 0;
7103         uint32_t                sid;
7104 
7105         FCIP_TNF_PROBE_2((fcip_get_broadcast_did, "fcip io", /* CSTYLED */,
7106                 tnf_string, msg, "enter",
7107                 tnf_opaque, fptr, fptr));
7108 
7109         sid = fport->fcipp_sid.port_id;
7110 
7111         switch (fport->fcipp_topology) {
7112 
7113         case FC_TOP_PT_PT: {
7114                 fc_portmap_t    *port_map = NULL;
7115                 uint32_t        listlen = 0;
7116 
7117                 if (fc_ulp_getportmap(fport->fcipp_handle, &port_map,
7118                     &listlen, FC_ULP_PLOGI_DONTCARE) == FC_SUCCESS) {
7119                         FCIP_DEBUG(FCIP_DEBUG_INIT, (CE_NOTE,
7120                             "fcip_gpmap: listlen :  0x%x", listlen));
7121                         if (listlen == 1) {
7122                                 did = port_map->map_did.port_id;
7123                         }
7124                 }
7125                 if (port_map) {
7126                         kmem_free(port_map, listlen * sizeof (fc_portmap_t));
7127                 }
7128                 if (listlen != 1) {
7129                         /* Dummy return value */
7130                         return (0x00FFFFFF);
7131                 }
7132                 break;
7133         }
7134 
7135         case FC_TOP_NO_NS:
7136         /* FALLTHROUGH */
7137         case FC_TOP_FABRIC:
7138                 /*
7139                  * The broadcast address is the same whether or not
7140                  * the switch/fabric contains a Name service.
7141                  */
7142                 did = 0x00FFFFFF;
7143                 break;
7144 
7145         case FC_TOP_PUBLIC_LOOP:
7146                 /*
7147                  * The open replicate primitive must not be used. The
7148                  * broadcast sequence is simply sent to ALPA 0x00. The
7149                  * fabric controller then propagates the broadcast to all
7150                  * other ports. The fabric propagates the broadcast by
7151                  * using the OPNfr primitive.
7152                  */
7153                 did = 0x00;
7154                 break;
7155 
7156         case FC_TOP_PRIVATE_LOOP:
7157                 /*
7158                  * The source port for broadcast in private loop mode
7159                  * must send an OPN(fr) signal forcing all ports in the
7160                  * loop to replicate the frames that they receive.
7161                  */
7162                 did = 0x00FFFFFF;
7163                 break;
7164 
7165         case FC_TOP_UNKNOWN:
7166         /* FALLTHROUGH */
7167         default:
7168                 did = sid;
7169                 FCIP_DEBUG(FCIP_DEBUG_INIT, (CE_WARN,
7170                     "fcip(0x%x):unknown topology in init_broadcast_pkt",
7171                     fptr->fcip_instance));
7172                 break;
7173         }
7174         FCIP_TNF_PROBE_2((fcip_get_broadcast_did, "fcip io", /* CSTYLED */,
7175                 tnf_string, msg, "return",
7176                 tnf_opaque, did, did));
7177 
7178         return (did);
7179 }
7180 
7181 
7182 /*
7183  * fcip timeout performs 2 operations:
7184  * 1. timeout any packets sent to the FCA for which a callback hasn't
7185  *    happened. If you are wondering why we need a callback since all
7186  *    traffic in FCIP is unidirectional, hence all exchanges are unidirectional
7187  *    but wait, we can only free up the resources after we know the FCA has
7188  *    DMA'ed out the data. pretty obvious eh :)
7189  *
7190  * 2. Retire and routing table entries we marked up for retiring. This is
7191  *    to give the link a chance to recover instead of marking a port down
7192  *    when we have lost all communication with it after a link transition
7193  */
7194 static void
7195 fcip_timeout(void *arg)
7196 {
7197         struct fcip                     *fptr = (struct fcip *)arg;
7198         int                             i;
7199         fcip_pkt_t                      *fcip_pkt;
7200         struct fcip_dest                *fdestp;
7201         int                             index;
7202         struct fcip_routing_table       *frtp;
7203         int                             dispatch_rte_removal = 0;
7204 
7205         mutex_enter(&fptr->fcip_mutex);
7206 
7207         fptr->fcip_flags |= FCIP_IN_TIMEOUT;
7208         fptr->fcip_timeout_ticks += fcip_tick_incr;
7209 
7210         if (fptr->fcip_flags & (FCIP_DETACHED | FCIP_DETACHING | \
7211             FCIP_SUSPENDED | FCIP_POWER_DOWN)) {
7212                 fptr->fcip_flags &= ~(FCIP_IN_TIMEOUT);
7213                 mutex_exit(&fptr->fcip_mutex);
7214                 return;
7215         }
7216 
7217         if (fptr->fcip_port_state == FCIP_PORT_OFFLINE) {
7218                 if (fptr->fcip_timeout_ticks > fptr->fcip_mark_offline) {
7219                         fptr->fcip_flags |= FCIP_LINK_DOWN;
7220                 }
7221         }
7222         if (!fptr->fcip_flags & FCIP_RTE_REMOVING) {
7223                 dispatch_rte_removal = 1;
7224         }
7225         mutex_exit(&fptr->fcip_mutex);
7226 
7227         /*
7228          * Check if we have any Invalid routing table entries in our
7229          * hashtable we have marked off for deferred removal. If any,
7230          * we can spawn a taskq thread to do the cleanup for us. We
7231          * need to avoid cleanup in the timeout thread since we may
7232          * have to wait for outstanding commands to complete before
7233          * we retire a routing table entry. Also dispatch the taskq
7234          * thread only if we are already do not have a taskq thread
7235          * dispatched.
7236          */
7237         if (dispatch_rte_removal) {
7238                 mutex_enter(&fptr->fcip_rt_mutex);
7239                 for (index = 0; index < FCIP_RT_HASH_ELEMS; index++) {
7240                         frtp = fptr->fcip_rtable[index];
7241                         while (frtp) {
7242                                 if ((frtp->fcipr_state == FCIP_RT_INVALID) &&
7243                                     (fptr->fcip_timeout_ticks >
7244                                     frtp->fcipr_invalid_timeout)) {
7245                                         /*
7246                                          * If we cannot schedule a task thread
7247                                          * let us attempt again on the next
7248                                          * tick rather than call
7249                                          * fcip_rte_remove_deferred() from here
7250                                          * directly since the routine can sleep.
7251                                          */
7252                                         frtp->fcipr_state = FCIP_RT_RETIRED;
7253 
7254                                         mutex_enter(&fptr->fcip_mutex);
7255                                         fptr->fcip_flags |= FCIP_RTE_REMOVING;
7256                                         mutex_exit(&fptr->fcip_mutex);
7257 
7258                                         if (taskq_dispatch(fptr->fcip_tq,
7259                                             fcip_rte_remove_deferred, fptr,
7260                                             KM_NOSLEEP) == 0) {
7261                                                 /*
7262                                                  * failed - so mark the entry
7263                                                  * as invalid again.
7264                                                  */
7265                                                 frtp->fcipr_state =
7266                                                     FCIP_RT_INVALID;
7267 
7268                                                 mutex_enter(&fptr->fcip_mutex);
7269                                                 fptr->fcip_flags &=
7270                                                     ~FCIP_RTE_REMOVING;
7271                                                 mutex_exit(&fptr->fcip_mutex);
7272                                         }
7273                                 }
7274                                 frtp = frtp->fcipr_next;
7275                         }
7276                 }
7277                 mutex_exit(&fptr->fcip_rt_mutex);
7278         }
7279 
7280         mutex_enter(&fptr->fcip_dest_mutex);
7281 
7282         /*
7283          * Now timeout any packets stuck with the transport/FCA for too long
7284          */
7285         for (i = 0; i < FCIP_DEST_HASH_ELEMS; i++) {
7286                 fdestp = fptr->fcip_dest[i];
7287                 while (fdestp != NULL) {
7288                         mutex_enter(&fdestp->fcipd_mutex);
7289                         for (fcip_pkt = fdestp->fcipd_head; fcip_pkt != NULL;
7290                             fcip_pkt = fcip_pkt->fcip_pkt_next) {
7291                                 if (fcip_pkt->fcip_pkt_flags &
7292                                     (FCIP_PKT_RETURNED | FCIP_PKT_IN_TIMEOUT |
7293                                     FCIP_PKT_IN_ABORT)) {
7294                                         continue;
7295                                 }
7296                                 if (fptr->fcip_timeout_ticks >
7297                                     fcip_pkt->fcip_pkt_ttl) {
7298                                         fcip_pkt->fcip_pkt_flags |=
7299                                             FCIP_PKT_IN_TIMEOUT;
7300 
7301                                         mutex_exit(&fdestp->fcipd_mutex);
7302                                         if (taskq_dispatch(fptr->fcip_tq,
7303                                             fcip_pkt_timeout, fcip_pkt,
7304                                             KM_NOSLEEP) == 0) {
7305                                                 /*
7306                                                  * timeout immediately
7307                                                  */
7308                                                 fcip_pkt_timeout(fcip_pkt);
7309                                         }
7310                                         mutex_enter(&fdestp->fcipd_mutex);
7311                                         /*
7312                                          * The linked list is altered because
7313                                          * of one of the following reasons:
7314                                          *      a. Timeout code dequeued a pkt
7315                                          *      b. Pkt completion happened
7316                                          *
7317                                          * So restart the spin starting at
7318                                          * the head again; This is a bit
7319                                          * excessive, but okay since
7320                                          * fcip_timeout_ticks isn't incremented
7321                                          * for this spin, we will skip the
7322                                          * not-to-be-timedout packets quickly
7323                                          */
7324                                         fcip_pkt = fdestp->fcipd_head;
7325                                         if (fcip_pkt == NULL) {
7326                                                 break;
7327                                         }
7328                                 }
7329                         }
7330                         mutex_exit(&fdestp->fcipd_mutex);
7331                         fdestp = fdestp->fcipd_next;
7332                 }
7333         }
7334         mutex_exit(&fptr->fcip_dest_mutex);
7335 
7336         /*
7337          * reschedule the timeout thread
7338          */
7339         mutex_enter(&fptr->fcip_mutex);
7340 
7341         fptr->fcip_timeout_id = timeout(fcip_timeout, fptr,
7342             drv_sectohz(1));
7343         fptr->fcip_flags &= ~(FCIP_IN_TIMEOUT);
7344         mutex_exit(&fptr->fcip_mutex);
7345 }
7346 
7347 
7348 /*
7349  * This routine is either called from taskq or directly from fcip_timeout
7350  * does the actual job of aborting the packet
7351  */
7352 static void
7353 fcip_pkt_timeout(void *arg)
7354 {
7355         fcip_pkt_t              *fcip_pkt = (fcip_pkt_t *)arg;
7356         struct fcip_dest        *fdestp;
7357         struct fcip             *fptr;
7358         fc_packet_t             *fc_pkt;
7359         fcip_port_info_t        *fport;
7360         int                     rval;
7361 
7362         fdestp = fcip_pkt->fcip_pkt_dest;
7363         fptr = fcip_pkt->fcip_pkt_fptr;
7364         fport = fptr->fcip_port_info;
7365         fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt);
7366 
7367         /*
7368          * try to abort the pkt
7369          */
7370         fcip_pkt->fcip_pkt_flags |= FCIP_PKT_IN_ABORT;
7371         rval = fc_ulp_abort(fport->fcipp_handle, fc_pkt, KM_NOSLEEP);
7372 
7373         FCIP_DEBUG(FCIP_DEBUG_DOWNSTREAM,
7374             (CE_NOTE, "fc_ulp_abort returns: 0x%x", rval));
7375 
7376         if (rval == FC_SUCCESS) {
7377                 ASSERT(fdestp != NULL);
7378 
7379                 /*
7380                  * dequeue the pkt from the dest structure pkt list
7381                  */
7382                 fcip_pkt->fcip_pkt_flags &= ~FCIP_PKT_IN_ABORT;
7383                 mutex_enter(&fdestp->fcipd_mutex);
7384                 rval = fcip_fdestp_dequeue_pkt(fdestp, fcip_pkt);
7385                 ASSERT(rval == 1);
7386                 mutex_exit(&fdestp->fcipd_mutex);
7387 
7388                 /*
7389                  * Now cleanup the pkt and free the mblk
7390                  */
7391                 fcip_pkt_free(fcip_pkt, 1);
7392         } else {
7393                 /*
7394                  * abort failed - just mark the pkt as done and
7395                  * wait for it to complete in fcip_pkt_callback since
7396                  * the pkt has already been xmitted by the FCA
7397                  */
7398                 fcip_pkt->fcip_pkt_flags &= ~FCIP_PKT_IN_TIMEOUT;
7399                 if (fcip_pkt->fcip_pkt_flags & FCIP_PKT_RETURNED) {
7400                         fcip_pkt->fcip_pkt_flags &= ~FCIP_PKT_IN_ABORT;
7401                         mutex_enter(&fdestp->fcipd_mutex);
7402                         rval = fcip_fdestp_dequeue_pkt(fdestp, fcip_pkt);
7403                         ASSERT(rval == 1);
7404                         mutex_exit(&fdestp->fcipd_mutex);
7405 
7406                         fcip_pkt_free(fcip_pkt, 1);
7407                 }
7408                 return;
7409         }
7410 }
7411 
7412 
7413 /*
7414  * Remove  a routing table entry marked for deferred removal. This routine
7415  * unlike fcip_pkt_timeout, is always called from a taskq context
7416  */
7417 static void
7418 fcip_rte_remove_deferred(void *arg)
7419 {
7420         struct fcip                     *fptr = (struct fcip *)arg;
7421         int                             hash_bucket;
7422         struct fcip_dest                *fdestp;
7423         la_wwn_t                        *pwwn;
7424         int                             index;
7425         struct fcip_routing_table       *frtp, *frtp_next, *frtp_prev;
7426 
7427 
7428         mutex_enter(&fptr->fcip_rt_mutex);
7429         for (index = 0; index < FCIP_RT_HASH_ELEMS; index++) {
7430                 frtp = fptr->fcip_rtable[index];
7431                 frtp_prev = NULL;
7432                 while (frtp) {
7433                         frtp_next = frtp->fcipr_next;
7434 
7435                         if (frtp->fcipr_state == FCIP_RT_RETIRED) {
7436 
7437                                 pwwn = &frtp->fcipr_pwwn;
7438                                 /*
7439                                  * Get hold of destination pointer
7440                                  */
7441                                 mutex_enter(&fptr->fcip_dest_mutex);
7442 
7443                                 hash_bucket = FCIP_DEST_HASH(pwwn->raw_wwn);
7444                                 ASSERT(hash_bucket < FCIP_DEST_HASH_ELEMS);
7445 
7446                                 fdestp = fptr->fcip_dest[hash_bucket];
7447                                 while (fdestp != NULL) {
7448                                         mutex_enter(&fdestp->fcipd_mutex);
7449                                         if (fdestp->fcipd_rtable) {
7450                                                 if (fcip_wwn_compare(pwwn,
7451                                                     &fdestp->fcipd_pwwn,
7452                                                     FCIP_COMPARE_PWWN) == 0) {
7453                                                         mutex_exit(
7454                                                         &fdestp->fcipd_mutex);
7455                                                         break;
7456                                                 }
7457                                         }
7458                                         mutex_exit(&fdestp->fcipd_mutex);
7459                                         fdestp = fdestp->fcipd_next;
7460                                 }
7461 
7462                                 mutex_exit(&fptr->fcip_dest_mutex);
7463                                 if (fdestp == NULL) {
7464                                         frtp_prev = frtp;
7465                                         frtp = frtp_next;
7466                                         continue;
7467                                 }
7468 
7469                                 mutex_enter(&fdestp->fcipd_mutex);
7470                                 if (fdestp->fcipd_ncmds) {
7471                                         /*
7472                                          * Instead of waiting to drain commands
7473                                          * let us revisit this RT entry in
7474                                          * the next pass.
7475                                          */
7476                                         mutex_exit(&fdestp->fcipd_mutex);
7477                                         frtp_prev = frtp;
7478                                         frtp = frtp_next;
7479                                         continue;
7480                                 }
7481 
7482                                 /*
7483                                  * We are clean, so remove the RTE
7484                                  */
7485                                 fdestp->fcipd_rtable = NULL;
7486                                 mutex_exit(&fdestp->fcipd_mutex);
7487 
7488                                 FCIP_TNF_PROBE_2((fcip_rte_remove_deferred,
7489                                         "fcip io", /* CSTYLED */,
7490                                         tnf_string, msg,
7491                                         "remove retired routing entry",
7492                                         tnf_int, index, index));
7493 
7494                                 if (frtp_prev == NULL) {
7495                                         /* first element */
7496                                         fptr->fcip_rtable[index] =
7497                                             frtp->fcipr_next;
7498                                 } else {
7499                                         frtp_prev->fcipr_next =
7500                                             frtp->fcipr_next;
7501                                 }
7502                                 kmem_free(frtp,
7503                                     sizeof (struct fcip_routing_table));
7504 
7505                                 frtp = frtp_next;
7506                         } else {
7507                                 frtp_prev = frtp;
7508                                 frtp = frtp_next;
7509                         }
7510                 }
7511         }
7512         mutex_exit(&fptr->fcip_rt_mutex);
7513         /*
7514          * Clear the RTE_REMOVING flag
7515          */
7516         mutex_enter(&fptr->fcip_mutex);
7517         fptr->fcip_flags &= ~FCIP_RTE_REMOVING;
7518         mutex_exit(&fptr->fcip_mutex);
7519 }
7520 
7521 /*
7522  * Walk through all the dest hash table entries and count up the total
7523  * no. of packets outstanding against a given port
7524  */
7525 static int
7526 fcip_port_get_num_pkts(struct fcip *fptr)
7527 {
7528         int                     num_cmds = 0;
7529         int                     i;
7530         struct fcip_dest        *fdestp;
7531 
7532         ASSERT(mutex_owned(&fptr->fcip_dest_mutex));
7533 
7534         for (i = 0; i < FCIP_DEST_HASH_ELEMS; i++) {
7535                 fdestp = fptr->fcip_dest[i];
7536                 while (fdestp != NULL) {
7537                         mutex_enter(&fdestp->fcipd_mutex);
7538 
7539                         ASSERT(fdestp->fcipd_ncmds >= 0);
7540 
7541                         if (fdestp->fcipd_ncmds > 0) {
7542                                 num_cmds += fdestp->fcipd_ncmds;
7543                         }
7544                         mutex_exit(&fdestp->fcipd_mutex);
7545                         fdestp = fdestp->fcipd_next;
7546                 }
7547         }
7548 
7549         return (num_cmds);
7550 }
7551 
7552 
7553 /*
7554  * Walk through the routing table for this state instance and see if there is a
7555  * PLOGI in progress for any of the entries. Return success even if we find one.
7556  */
7557 static int
7558 fcip_plogi_in_progress(struct fcip *fptr)
7559 {
7560         int                             i;
7561         struct fcip_routing_table       *frp;
7562 
7563         ASSERT(mutex_owned(&fptr->fcip_rt_mutex));
7564 
7565         for (i = 0; i < FCIP_RT_HASH_ELEMS; i++) {
7566                 frp = fptr->fcip_rtable[i];
7567                 while (frp) {
7568                         if (frp->fcipr_state == FCIP_RT_LOGIN_PROGRESS) {
7569                                 /* Found an entry where PLOGI is in progress */
7570                                 return (1);
7571                         }
7572                         frp = frp->fcipr_next;
7573                 }
7574         }
7575 
7576         return (0);
7577 }
7578 
7579 /*
7580  * Walk through the fcip port global list and check if the given port exists in
7581  * the list. Returns "0" if port exists and "1" if otherwise.
7582  */
7583 static int
7584 fcip_check_port_exists(struct fcip *fptr)
7585 {
7586         fcip_port_info_t        *cur_fport;
7587         fcip_port_info_t        *fport;
7588 
7589         mutex_enter(&fcip_global_mutex);
7590         fport = fptr->fcip_port_info;
7591         cur_fport = fcip_port_head;
7592         while (cur_fport != NULL) {
7593                 if (cur_fport == fport) {
7594                         /* Found */
7595                         mutex_exit(&fcip_global_mutex);
7596                         return (0);
7597                 } else {
7598                         cur_fport = cur_fport->fcipp_next;
7599                 }
7600         }
7601         mutex_exit(&fcip_global_mutex);
7602 
7603         return (1);
7604 }
7605 
7606 /*
7607  * Constructor to initialize the sendup elements for callback into
7608  * modules upstream
7609  */
7610 
7611 /* ARGSUSED */
7612 static int
7613 fcip_sendup_constructor(void *buf, void *arg, int flags)
7614 {
7615         struct fcip_sendup_elem *msg_elem = (struct fcip_sendup_elem *)buf;
7616         fcip_port_info_t        *fport = (fcip_port_info_t *)arg;
7617 
7618         ASSERT(fport != NULL);
7619 
7620         msg_elem->fcipsu_mp = NULL;
7621         msg_elem->fcipsu_func = NULL;
7622         msg_elem->fcipsu_next = NULL;
7623 
7624         return (FCIP_SUCCESS);
7625 }