1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 #ifndef _SYS_IB_MGT_IBDM_IBDM_IMPL_H
  27 #define _SYS_IB_MGT_IBDM_IBDM_IMPL_H
  28 
  29 /*
  30  * ibdm_impl.h
  31  *
  32  *      This file contains definitions of the data structures, macros etc
  33  *      related to the IBDM module.
  34  */
  35 
  36 #include <sys/ib/mgt/ibdm/ibdm_ibnex.h>
  37 #include <sys/ib/ibtl/impl/ibtl_util.h>
  38 
  39 #ifdef __cplusplus
  40 extern "C" {
  41 #endif
  42 
  43 /* values for "cb_req_type" */
  44 #define IBDM_REQ_TYPE_INVALID           0x0
  45 #define IBDM_REQ_TYPE_CLASSPORTINFO     0x1
  46 #define IBDM_REQ_TYPE_IOUINFO           0x2
  47 #define IBDM_REQ_TYPE_IOCINFO           0x4
  48 #define IBDM_REQ_TYPE_SRVENTS           0x8
  49 #define IBDM_REQ_TYPE_IOU_DIAGCODE      0x10
  50 #define IBDM_REQ_TYPE_IOC_DIAGCODE      0x20
  51 
  52 typedef struct ibdm_taskq_args_s {
  53         ibmf_handle_t           tq_ibmf_handle;
  54         ibmf_msg_t              *tq_ibmf_msg;
  55         void                    *tq_args;
  56 } ibdm_taskq_args_t;
  57 _NOTE(SCHEME_PROTECTS_DATA("unique per call", ibdm_taskq_args_t))
  58 _NOTE(SCHEME_PROTECTS_DATA("unique per call", ib_mad_hdr_t))
  59 _NOTE(SCHEME_PROTECTS_DATA("unique per call", ibmf_msg_t))
  60 
  61 #define IBDM_GID_PRESENT                0x1
  62 #define IBDM_GID_NOT_PRESENT            0x0
  63 
  64 #define IBDM_IBMF_PKT_DUP_RESP          0x1
  65 #define IBDM_IBMF_PKT_REUSED            0x2
  66 #define IBDM_IBMF_PKT_UNEXP_RESP        0x4
  67 
  68 #define IBDM_MAX_SERV_ENTRIES_PER_REQ   4
  69 
  70 typedef struct ibdm_gid_s {
  71         uint64_t                gid_dgid_hi;
  72         uint64_t                gid_dgid_lo;
  73         struct ibdm_gid_s       *gid_next;
  74 } ibdm_gid_t;
  75 
  76 #define IBDM_GID_PROBE_NOT_DONE         0x00
  77 #define IBDM_GET_CLASSPORTINFO          0x01
  78 #define IBDM_GET_IOUNITINFO             0x02
  79 #define IBDM_GET_IOC_DETAILS            0x04
  80 #define IBDM_GID_PROBING_COMPLETE       0x08
  81 #define IBDM_GID_PROBING_SKIPPED        0x10
  82 #define IBDM_GID_PROBING_FAILED         0x20
  83 #define IBDM_SET_CLASSPORTINFO          0x40
  84 
  85 /*
  86  * Identifiers to distinguish a Cisco FC GW from others.
  87  * Used to filter a setclassportinfo request.
  88  */
  89 #define IBDM_CISCO_COMPANY_ID           (0x5ad)
  90 #define IBDM_CISCO_DEVICE_ID            (0xa87c)
  91 
  92 /*
  93  * the bit-shift value for OUI in GUID
  94  * A 64 bit globally unique identifier (GUID) composed of a 24 bit company id
  95  * and an 48 bit extension identifier, and this value is used to extract
  96  * the company id from the GUID.
  97  */
  98 #define IBDM_OUI_GUID_SHIFT             (40)
  99 
 100 /*
 101  * The state diagram for the gl_state
 102  *
 103  *                          (in case of Cisco FC GW)
 104  * IBDM_GID_PROBE_NOT_DONE  ---------- 40 -> IBDM_SET_CLASSPORTINFO
 105  *                          ----.               |
 106  *    |      |                  | (others)      |
 107  *    |      |                  1               |
 108  *    |      |                  |               1
 109  *    |      |                  `-------------. |
 110  *    |      |                                v v
 111  *    |      |                               IBDM_GET_CLASSPORTINFO
 112  *    |      |
 113  *    |      |                                  |
 114  *    |      2                                  3
 115  *    |      |                                  |
 116  *    |      v                                  v
 117  *    |     IBDM_GID_PROBING_FAILED          IBDM_GET_IOUNITINFO
 118  *    |                                         |
 119  *    6                                         4
 120  *    |                                         |
 121  *    v                                         v
 122  *  IBDM_GID_PROBING_SKIPPLED                IBDM_GET_IOC_DETAILS
 123  *                                              |
 124  *                                              5
 125  *                                              |
 126  *                                              v
 127  *                                           IBDM_GID_PROBE_COMPLETE
 128  *
 129  * Initial state : IBDM_GID_PROBE_NOT_DONE
 130  *     40 = Port sends setClassPortInfo to activate Cisco FC GW
 131  *      1 = Port supports DM MAD's and a request to ClassportInfo is sent
 132  *      3 = Received ClassPortInfo and sent IOUnitInfo
 133  *      4 = Recevied IOUunitInfo and sent IOC profile, diagcodes, and
 134  *              service entries requests
 135  *      5 = Received all the IOC information
 136  *      2 = Failed to probe the GID
 137  *              Port does not support DM MAD's
 138  *              Port did not respond property
 139  *      6 = A different GID for the same port, skip the probe
 140  *
 141  * Reprobe state transition :
 142  *
 143  * IBDM_GID_PROBE_COMPLETE
 144  *      |
 145  *      7
 146  *      |
 147  *      v
 148  * IBDM_GET_IOC_DETAILS
 149  *      |
 150  *      8
 151  *      |
 152  *      v
 153  * IBDM_GID_PROBE_COMPLETE
 154  *
 155  *      7 = Reprobe request for one or more IOCs initiated.
 156  *      8 = Reprobe done(IOC COntroller Profile & Service entries)
 157  */
 158 
 159 typedef struct ibdm_dp_gidinfo_s {
 160         kmutex_t                gl_mutex;
 161         uint_t                  gl_state;
 162         int                     gl_reprobe_flag;        /* pass this to taskq */
 163         struct ibdm_dp_gidinfo_s *gl_next;
 164         struct ibdm_dp_gidinfo_s *gl_prev;
 165         ibdm_iou_info_t         *gl_iou;
 166         int                     gl_pending_cmds;
 167         ibmf_qp_handle_t        gl_qp_hdl;
 168         uint64_t                gl_transactionID;
 169         ibdm_timeout_cb_args_t  gl_iou_cb_args;
 170         ib_lid_t                gl_dlid;
 171         ib_lid_t                gl_slid;
 172         uint64_t                gl_dgid_hi;
 173         uint64_t                gl_dgid_lo;
 174         uint64_t                gl_sgid_hi;
 175         uint64_t                gl_sgid_lo;
 176         ib_guid_t               gl_nodeguid;
 177         ib_guid_t               gl_portguid;
 178         ib_pkey_t               gl_p_key;
 179         boolean_t               gl_is_dm_capable;
 180         boolean_t               gl_redirected;
 181         uint32_t                gl_redirect_dlid;
 182         uint32_t                gl_redirect_QP;
 183         ib_pkey_t               gl_redirect_pkey;
 184         ib_qkey_t               gl_redirect_qkey;
 185         uint64_t                gl_redirectGID_hi;
 186         uint64_t                gl_redirectGID_lo;
 187         ibmf_handle_t           gl_ibmf_hdl;
 188         ibmf_saa_handle_t       gl_sa_hdl;
 189         timeout_id_t            gl_timeout_id;
 190         ibdm_timeout_cb_args_t  gl_cpi_cb_args;
 191         uint32_t                gl_ngids;
 192         ibdm_gid_t              *gl_gid;
 193         uint32_t                gl_resp_timeout;
 194         int                     gl_num_iocs;
 195         ibdm_hca_list_t         *gl_hca_list;
 196         int                     gl_disconnected;
 197         uint64_t                gl_min_transactionID;
 198         uint64_t                gl_max_transactionID;
 199         ibdm_iou_info_t         *gl_prev_iou;
 200         uint16_t                gl_devid;       /* device ID info */
 201         kcondvar_t              gl_probe_cv;    /* sync for Cisco FC GW */
 202         uint32_t                gl_flag;
 203         uint8_t                 gl_SL:4;        /* SL from path_record */
 204         uint8_t                 gl_redirectSL:4; /* SL from redirection */
 205 } ibdm_dp_gidinfo_t;
 206 _NOTE(MUTEX_PROTECTS_DATA(ibdm_dp_gidinfo_s::gl_mutex,
 207         ibdm_dp_gidinfo_s::{gl_state gl_timeout_id gl_pending_cmds}))
 208 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv", ibdm_dp_gidinfo_s))
 209 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibdm_dp_gidinfo_s::{gl_ibmf_hdl gl_sa_hdl}))
 210 _NOTE(MUTEX_PROTECTS_DATA(ibdm_dp_gidinfo_s::gl_mutex,
 211         ibdm_ioc_info_s::{ioc_timeout_id ioc_dc_timeout_id}))
 212 _NOTE(MUTEX_PROTECTS_DATA(ibdm_dp_gidinfo_s::gl_mutex,
 213         ibdm_srvents_info_s::se_timeout_id))
 214 
 215 /*
 216  * The transaction ID for the GID contains of two parts :
 217  *      1. Upper 32 bits which is unique for each GID.
 218  *      2. Lower 32 bits which is unique for each MAD.
 219  * The assumptions are :
 220  *      1. At most 2 power 32 DM capable GIDs on the IB fabric
 221  *      2. IBDM sends maximum of 2 power 32 MADs to the same DM GID
 222  * The limits are sufficient for practical configurations.
 223  */
 224 #define IBDM_GID_TRANSACTIONID_SHIFT    ((ulong_t)32)
 225 #define IBDM_GID_TRANSACTIONID_MASK     0xFFFFFFFF00000000ULL
 226 
 227 typedef struct ibdm_s {
 228         /* Protects IBDM's critical data */
 229         kmutex_t                ibdm_mutex;
 230         uint32_t                ibdm_hca_count;
 231         kmutex_t                ibdm_hl_mutex;
 232         kmutex_t                ibdm_ibnex_mutex;
 233         ibdm_hca_list_t         *ibdm_hca_list_head;
 234         ibdm_hca_list_t         *ibdm_hca_list_tail;
 235 
 236         ibdm_dp_gidinfo_t       *ibdm_dp_gidlist_head;
 237         ibdm_dp_gidinfo_t       *ibdm_dp_gidlist_tail;
 238 
 239         kcondvar_t              ibdm_probe_cv;
 240         kcondvar_t              ibdm_busy_cv;
 241         kcondvar_t              ibdm_port_settle_cv;
 242         uint32_t                ibdm_ngid_probes_in_progress;
 243         uint64_t                ibdm_transactionID;
 244         uint32_t                ibdm_ngids;
 245         uint32_t                ibdm_busy;
 246         int                     ibdm_state;
 247         ibt_clnt_hdl_t          ibdm_ibt_clnt_hdl;
 248 
 249         /*
 250          * These are callback routines registered by the IB nexus driver.
 251          * These callbacks are used to inform the IB nexus driver about
 252          * the arrival/removal of HCA and IOC's
 253          */
 254         ibdm_callback_t         ibdm_ibnex_callback;
 255 
 256         /* Flag indicating - prev_iou during sweep */
 257         int                     ibdm_prev_iou;
 258 } ibdm_t;
 259 _NOTE(MUTEX_PROTECTS_DATA(ibdm_s::ibdm_mutex, ibdm_s::{ibdm_ibt_clnt_hdl
 260         ibdm_busy ibdm_state}))
 261 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibdm_s::ibdm_ibt_clnt_hdl))
 262 _NOTE(MUTEX_PROTECTS_DATA(ibdm_s::ibdm_hl_mutex,
 263         ibdm_s::{ibdm_hca_list_head ibdm_hca_list_tail}))
 264 _NOTE(MUTEX_PROTECTS_DATA(ibdm_s::ibdm_ibnex_mutex,
 265         ibdm_s::ibdm_ibnex_callback))
 266 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv", ibdm_s))
 267 _NOTE(LOCK_ORDER(ibdm_s::ibdm_mutex ibdm_dp_gidinfo_s::gl_mutex))
 268 
 269 /* valid values for ibdm_state */
 270 #define IBDM_LOCKS_ALLOCED      0x01            /* global locks alloced */
 271 #define IBDM_CVS_ALLOCED        0x02            /* global "cv"s alloced */
 272 #define IBDM_IBT_ATTACHED       0x04            /* ibt_attach() called */
 273 #define IBDM_HCA_ATTACHED       0x08            /* ibdm_handle_hca() called */
 274 
 275 #define IBDM_8_BIT_MASK         0x00FF
 276 #define IBDM_16_BIT_MASK        0xFFFF
 277 #define IBDM_RETRY_COUNT        0x2
 278 
 279 #define IBDM_BUSY               0x1
 280 #define IBDM_PROBE_IN_PROGRESS  0x2
 281 #define IBDM_CISCO_PROBE        0x4
 282 #define IBDM_CISCO_PROBE_DONE   0x8
 283 
 284 /*
 285  * Device Management MAD packet format
 286  * +--------+------------+------------+------------+------------+
 287  * | offset |   byte 0   |   byte 1   |   byte 2   |   byte 3   |
 288  * +--------+------------+------------+------------+------------+ --
 289  * |   0    |                                                   |  ^
 290  * +--------+                                                   | sizeof(
 291  * |   ...  |              Common MAD Header                    | ib_mad_hdr_t)
 292  * +--------+                                                   |  | (A)
 293  * |   20   |                                                   |  v
 294  * +--------+------------+------------+------------+------------+ --
 295  * |   24   |                                                   |  ^
 296  * +--------+                                                   |  |
 297  * |   ...  |              RMPP Header                          |  |
 298  * +--------+                                                   |  |
 299  * |   32   |                                                   |  |
 300  * +--------+------------+------------+------------+------------+  |
 301  * |   36   |                                                   |  |
 302  * +--------+              Access_Key                           |
 303  * |   40   |                                                   | IBDM_DM_MAD_
 304  * +--------+------------+------------+------------+------------+ HDR_SZ
 305  * |   44   |  KeyType   |              reserved                |    (B)
 306  * +--------+------------+------------+------------+------------+  |
 307  * |   48   |                                                   |  |
 308  * +--------+                                                   |  |
 309  * |   52   |              Reserved                             |  |
 310  * +--------+                                                   |  |
 311  * |   56   |                                                   |  |
 312  * +--------+------------+------------+------------+------------+  |
 313  * |   60   |       Change_ID         |     ComponentMask       |  v
 314  * +--------+------------+------------+------------+------------+ --
 315  * |   64   |                                                   |  ^
 316  * +--------+                                                   | IBDM_MAD_SIZE
 317  * |   ...  |              Device Management Data               | - (A) - (B)
 318  * +--------+                                                   |  |
 319  * |  252   |                                                   |  v
 320  * +--------+------------+------------+------------+------------+ --
 321  */
 322 #define IBDM_MAD_SIZE           256
 323 #define IBDM_DM_MAD_HDR_SZ      40
 324 
 325 #define IBDM_DFT_TIMEOUT        4
 326 #define IBDM_DFT_NRETRIES       3
 327 
 328 #define IBDM_ENABLE_TASKQ_HANDLING      1
 329 #define IBDM_DISABLE_TASKQ_HANLDING     0
 330 
 331 typedef struct ibdm_saa_event_arg_s {
 332         ibmf_saa_handle_t ibmf_saa_handle;
 333         ibmf_saa_subnet_event_t ibmf_saa_event;
 334         ibmf_saa_event_details_t event_details;
 335         void *callback_arg;
 336 } ibdm_saa_event_arg_t;
 337 
 338 #define IBDM_TIMEOUT_VALUE(t)   drv_sectohz(t)
 339 
 340 #define IBDM_OUT_IBMFMSG_MADHDR(msg)\
 341                 (msg->im_msgbufs_send.im_bufs_mad_hdr)
 342 
 343 #define IBDM_IN_IBMFMSG_MADHDR(msg)\
 344                 (msg->im_msgbufs_recv.im_bufs_mad_hdr)
 345 
 346 #define IBDM_IN_IBMFMSG_STATUS(msg)\
 347                 b2h16(msg->im_msgbufs_recv.im_bufs_mad_hdr->Status)
 348 
 349 #define IBDM_IN_IBMFMSG_ATTR(msg)\
 350                 b2h16(msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID)
 351 
 352 #define IBDM_IN_IBMFMSG_ATTRMOD(msg)\
 353                 b2h32(msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier)
 354 
 355 #define IBDM_IN_IBMFMSG2IOU(msg)        (ib_dm_io_unitinfo_t *)\
 356                 (msg->im_msgbufs_recv.im_bufs_cl_data)
 357 
 358 #define IBDM_IN_IBMFMSG2IOC(msg)        (ib_dm_ioc_ctrl_profile_t *)\
 359                 (msg->im_msgbufs_recv.im_bufs_cl_data)
 360 
 361 #define IBDM_IN_IBMFMSG2SRVENT(msg)     (ib_dm_srv_t *)\
 362                 (msg->im_msgbufs_recv.im_bufs_cl_data)
 363 
 364 #define IBDM_IN_IBMFMSG2DIAGCODE(msg)   (uint32_t *)\
 365                 (msg->im_msgbufs_recv.im_bufs_cl_data)
 366 
 367 #define IBDM_GIDINFO2IOCINFO(gid_info, idx) \
 368                 (ibdm_ioc_info_t *)&gid_info->gl_iou->iou_ioc_info[idx];
 369 
 370 #define IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)\
 371                 ((ioc_no < 1) || (ioc_no >\
 372                         gid_info->gl_iou->iou_info.iou_num_ctrl_slots))
 373 
 374 #define IBDM_INVALID_PKEY(pkey) \
 375                 (((pkey) == IB_PKEY_INVALID_FULL) || \
 376                 ((pkey) == IB_PKEY_INVALID_LIMITED))
 377 
 378 #ifdef DEBUG
 379 
 380 void    ibdm_dump_mad_hdr(ib_mad_hdr_t *);
 381 void    ibdm_dump_ibmf_msg(ibmf_msg_t *, int);
 382 void    ibdm_dump_path_info(sa_path_record_t *);
 383 void    ibdm_dump_classportinfo(ib_mad_classportinfo_t *);
 384 void    ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *);
 385 void    ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *);
 386 void    ibdm_dump_service_entries(ib_dm_srv_t *);
 387 void    ibdm_dump_sweep_fabric_timestamp(int);
 388 
 389 #define ibdm_dump_mad_hdr(a)            ibdm_dump_mad_hdr(a)
 390 #define ibdm_dump_ibmf_msg(a, b)        ibdm_dump_ibmf_msg(a, b)
 391 #define ibdm_dump_path_info(a)          ibdm_dump_path_info(a)
 392 #define ibdm_dump_classportinfo(a)      ibdm_dump_classportinfo(a)
 393 #define ibdm_dump_iounitinfo(a)         ibdm_dump_iounitinfo(a)
 394 #define ibdm_dump_ioc_profile(a)        ibdm_dump_ioc_profile(a)
 395 #define ibdm_dump_service_entries(a)    ibdm_dump_service_entries(a)
 396 
 397 #else
 398 
 399 #define ibdm_dump_mad_hdr(a)
 400 #define ibdm_dump_ibmf_msg(a, b)
 401 #define ibdm_dump_path_info(a)
 402 #define ibdm_dump_classportinfo(a)
 403 #define ibdm_dump_iounitinfo(a)
 404 #define ibdm_dump_ioc_profile(a)
 405 #define ibdm_dump_service_entries(a)
 406 #define ibdm_dump_sweep_fabric_timestamp(a)
 407 
 408 #endif
 409 
 410 #ifdef __cplusplus
 411 }
 412 #endif
 413 
 414 #endif  /* _SYS_IB_MGT_IBDM_IBDM_IMPL_H */