1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/ib/mgt/ibmf/ibmf_saa_impl.h> 27 #include <sys/ib/mgt/ibmf/ibmf_saa_utils.h> 28 29 /* Global sa_access State Pointer */ 30 saa_state_t *saa_statep; 31 _NOTE(READ_ONLY_DATA(saa_statep)) 32 33 extern int ibmf_trace_level; 34 35 extern int ibmf_taskq_max_tasks; 36 37 static int 38 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 39 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags); 40 static int 41 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp, 42 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags); 43 static int 44 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 45 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, 46 hrtime_t trans_send_time, int transport_flags); 47 static int ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, 48 boolean_t sleep_flag, ibmf_msg_t **msgp, uint32_t *transport_flagsp, 49 ibmf_retrans_t *ibmf_retransp); 50 static int ibmf_saa_must_purge(saa_port_t *saa_portp); 51 static void ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp); 52 static void ibmf_saa_impl_destroy_port(saa_port_t *saa_portp); 53 static void ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp); 54 static void ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, 55 int status); 56 static void ibmf_saa_impl_async_event_cb(ibmf_handle_t ibmf_handle, 57 void *clnt_private, ibmf_async_event_t event_type); 58 static void ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num); 59 static void ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num); 60 static void ibmf_saa_impl_port_chg(ibt_async_event_t *event); 61 static void ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num); 62 static void ibmf_saa_impl_hca_detach(saa_port_t *saa_removed); 63 static void ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle, 64 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result, 65 size_t *length, boolean_t sleep_flag); 66 static int ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id); 67 static uint_t ibmf_saa_impl_get_attr_id_length(uint16_t attr_id); 68 static void ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp); 69 static int ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop, 70 ib_guid_t *guid_ret); 71 static void ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp, 72 ibt_hca_portinfo_t *portinfop); 73 static void ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, 74 ibmf_msg_t *msgp); 75 static int ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp); 76 77 int ibmf_saa_max_wait_time = IBMF_SAA_MAX_WAIT_TIME_IN_SECS; 78 int ibmf_saa_trans_wait_time = IBMF_SAA_TRANS_WAIT_TIME_IN_SECS; 79 int ibmf_saa_max_resp_time = IBMF_SAA_MAX_RESP_TIME; 80 int ibmf_saa_max_subnet_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT; 81 int ibmf_saa_retrans_retries = IBMF_SAA_RETRANS_RETRIES; 82 83 /* 84 * ibmf_saa_impl_init: 85 * Allocates memory for the ibmf_saa state structure and initializes the taskq. 86 * Called from the modules init() routine. 87 * 88 * Input Arguments 89 * none 90 * 91 * Output Arguments 92 * none 93 * 94 * Returns 95 * IBMF_NO_RESOURCES if taskq could not be created. 96 * IBMF_SUCCESS on success 97 * 98 */ 99 int 100 ibmf_saa_impl_init() 101 { 102 int res; 103 104 /* CONSTCOND */ 105 ASSERT(NO_COMPETING_THREADS); 106 107 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_init_start, 108 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() enter\n"); 109 110 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_statep)) 111 112 saa_statep = kmem_zalloc(sizeof (saa_state_t), KM_SLEEP); 113 114 /* create taskq for notifying event subscribers */ 115 saa_statep->saa_event_taskq = taskq_create( 116 "ibmf_saa_event_taskq", IBMF_TASKQ_NTHREADS, 117 MINCLSYSPRI, 1, ibmf_taskq_max_tasks, TASKQ_DYNAMIC | 118 TASKQ_PREPOPULATE); 119 if (saa_statep->saa_event_taskq == NULL) { 120 121 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L4, 122 ibmf_saa_impl_init_end_err, 123 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init(): %s\n", 124 tnf_string, msg, "event taskq create failed"); 125 126 kmem_free(saa_statep, sizeof (saa_state_t)); 127 128 res = IBMF_NO_RESOURCES; 129 130 goto bail; 131 } 132 133 mutex_init(&saa_statep->saa_port_list_mutex, NULL, MUTEX_DRIVER, 134 NULL); 135 136 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_statep)) 137 138 res = IBMF_SUCCESS; 139 bail: 140 141 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_init_end, 142 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() exit: status = %d\n", 143 tnf_int, res, res); 144 145 return (res); 146 } 147 148 /* 149 * ibmf_saa_impl_fini: 150 * If there are no registered clients, cleans up all memory associated with the 151 * state, including each of the port list entries. 152 * Called from the modules fini() routine. 153 * 154 * Input Arguments 155 * none 156 * 157 * Output Arguments 158 * none 159 * 160 * Returns 161 * EBUSY if there are outstanding transactions or registered clients 162 * 0 if cleanup was sucessfull 163 * 164 */ 165 int 166 ibmf_saa_impl_fini() 167 { 168 int ret = 0; 169 saa_port_t *saa_portp; 170 saa_port_t *next; 171 172 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_start, 173 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() enter\n"); 174 175 /* make sure there are no registered clients */ 176 mutex_enter(&saa_statep->saa_port_list_mutex); 177 178 saa_portp = saa_statep->saa_port_list; 179 while (saa_portp != NULL) { 180 181 mutex_enter(&saa_portp->saa_pt_mutex); 182 183 if (saa_portp->saa_pt_reference_count > 0) { 184 185 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 186 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "", 187 "ibmf_saa_impl_fini: %s, port %016" PRIx64 "\n", 188 tnf_string, msg, 189 "cannot unload ibmf_saa. Client on port still" 190 " registered", tnf_opaque, port, 191 saa_portp->saa_pt_port_guid); 192 193 mutex_exit(&saa_portp->saa_pt_mutex); 194 195 mutex_exit(&saa_statep->saa_port_list_mutex); 196 197 ret = EBUSY; 198 goto bail; 199 } 200 201 /* make sure there are no outstanding transactions */ 202 203 if (saa_portp->saa_pt_num_outstanding_trans > 0) { 204 205 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1, 206 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "", 207 "ibmf_saa_impl_fini: %s, port = %016" PRIx64 208 ", num transactions = %d\n", 209 tnf_string, msg, "Cannot unload ibmf_saa." 210 " Outstanding transactions on port.", 211 tnf_opaque, port, 212 saa_portp->saa_pt_port_guid, 213 tnf_uint, outstanding_transactions, 214 saa_portp->saa_pt_num_outstanding_trans); 215 216 mutex_exit(&saa_portp->saa_pt_mutex); 217 218 mutex_exit(&saa_statep->saa_port_list_mutex); 219 220 ret = EBUSY; 221 goto bail; 222 } 223 224 mutex_exit(&saa_portp->saa_pt_mutex); 225 226 saa_portp = saa_portp->next; 227 } 228 229 mutex_exit(&saa_statep->saa_port_list_mutex); 230 231 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(saa_statep->saa_port_list, 232 *saa_portp)) 233 234 /* 235 * no more clients nor pending transaction: 236 * unregister ibmf and destroy port entries 237 */ 238 while (saa_statep->saa_port_list != NULL) { 239 240 saa_portp = saa_statep->saa_port_list; 241 next = saa_portp->next; 242 243 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 244 ibmf_saa_impl_fini, IBMF_TNF_TRACE, "", 245 "ibmf_saa_impl_fini: %s, prefix = %016" PRIx64 "\n", 246 tnf_string, msg, "deinitializing port", 247 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 248 249 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp)) 250 251 mutex_enter(&saa_portp->saa_pt_mutex); 252 253 /* unregister from ibmf */ 254 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) { 255 256 mutex_exit(&saa_portp->saa_pt_mutex); 257 258 if (ibmf_saa_impl_ibmf_unreg(saa_portp) 259 != IBMF_SUCCESS) { 260 ret = EBUSY; 261 goto bail; 262 } 263 } else 264 mutex_exit(&saa_portp->saa_pt_mutex); 265 266 ibmf_saa_impl_destroy_port(saa_portp); 267 268 saa_statep->saa_port_list = next; 269 } 270 271 taskq_destroy(saa_statep->saa_event_taskq); 272 273 mutex_destroy(&saa_statep->saa_port_list_mutex); 274 275 kmem_free(saa_statep, sizeof (saa_state_t)); 276 277 bail: 278 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_end, 279 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() exit\n"); 280 281 return (ret); 282 } 283 284 /* 285 * ibmf_saa_is_valid 286 * Returns true the entry is valid. 287 * 288 * Input Arguments 289 * saa_portp pointer to state structure 290 * add_ref if B_TRUE ref count is incremented on a valid portp 291 * 292 * Output Arguments 293 * none 294 * 295 * Returns 296 * B_TRUE if entry was in a valid state, B_FALSE otherwise 297 */ 298 boolean_t 299 ibmf_saa_is_valid(saa_port_t *saa_portp, int add_ref) 300 { 301 boolean_t is_valid = B_TRUE; 302 303 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_start, 304 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() enter\n"); 305 306 mutex_enter(&saa_portp->saa_pt_mutex); 307 308 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID || 309 saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_PURGING) { 310 311 is_valid = B_FALSE; 312 313 } else if (add_ref == B_TRUE) { 314 /* 315 * increment reference count here to ensure that 316 * entry does not get purged behind our backs 317 */ 318 saa_portp->saa_pt_reference_count++; 319 } 320 mutex_exit(&saa_portp->saa_pt_mutex); 321 322 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_end, 323 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() exit\n"); 324 325 return (is_valid); 326 } 327 328 /* 329 * ibmf_saa_must_purge 330 * Determines if we can purge a portp (remove it from the list) based on the 331 * state and number of clients 332 * 333 * Input Arguments 334 * saa_portp pointer to state structure 335 * 336 * Output Arguments 337 * none 338 * 339 * Returns 340 * B_TRUE if the entry can be removed, B_FALSE otherwise 341 */ 342 static int 343 ibmf_saa_must_purge(saa_port_t *saa_portp) 344 { 345 int must_purge = B_FALSE; 346 347 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_start, 348 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() enter\n"); 349 350 mutex_enter(&saa_portp->saa_pt_mutex); 351 352 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID && 353 saa_portp->saa_pt_reference_count == 0) { 354 355 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_PURGING; 356 must_purge = B_TRUE; 357 } 358 359 mutex_exit(&saa_portp->saa_pt_mutex); 360 361 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_end, 362 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() exit\n"); 363 364 return (must_purge); 365 } 366 367 368 /* 369 * ibmf_saa_impl_purge: 370 * Removes invalid port state entries from the list 371 * 372 * Input Arguments 373 * none 374 * 375 * Output Arguments 376 * none 377 * 378 * Returns 379 * void 380 */ 381 void 382 ibmf_saa_impl_purge() 383 { 384 saa_port_t *cur_portp = NULL; 385 saa_port_t *prev_portp = NULL; 386 saa_port_t *rem_portp = NULL; 387 388 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_start, 389 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() enter\n"); 390 391 mutex_enter(&saa_statep->saa_port_list_mutex); 392 393 cur_portp = saa_statep->saa_port_list; 394 prev_portp = cur_portp; 395 396 while (cur_portp != NULL) { 397 398 if (ibmf_saa_must_purge(cur_portp) == B_TRUE) { 399 400 rem_portp = cur_portp; 401 402 /* unlink entry */ 403 if (cur_portp == saa_statep->saa_port_list) { 404 405 saa_statep->saa_port_list = cur_portp->next; 406 cur_portp = saa_statep->saa_port_list; 407 prev_portp = cur_portp; 408 409 } else { 410 411 prev_portp->next = cur_portp->next; 412 cur_portp = cur_portp->next; 413 } 414 415 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rem_portp)) 416 417 /* destroy entry */ 418 ASSERT(rem_portp != NULL); 419 ibmf_saa_impl_destroy_port(rem_portp); 420 421 } else { 422 423 prev_portp = cur_portp; 424 cur_portp = cur_portp->next; 425 } 426 } 427 428 mutex_exit(&saa_statep->saa_port_list_mutex); 429 430 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_end, 431 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() exit\n"); 432 } 433 434 /* 435 * saa_impl_add_client: 436 * Adds a client for a particular portp. Reference count has been incremented 437 * before this call. It is decremented by saa_impl_add_client() if the call 438 * fails. 439 * 440 * Input Arguments 441 * none 442 * 443 * Output Arguments 444 * none 445 * 446 * Returns 447 * IBMF_BUSY if there are already too many clients registered, 448 * IBMF_BAD_PORT_STATE if the port is invalid (generally because a previous 449 * client failed during registration for this port) 450 * IBMF_SUCCESS otherwise 451 */ 452 int 453 ibmf_saa_impl_add_client(saa_port_t *saa_portp) 454 { 455 int status = IBMF_SUCCESS; 456 457 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_add_client_start, 458 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client() enter\n"); 459 460 mutex_enter(&saa_portp->saa_pt_mutex); 461 462 /* 463 * check that we don't exceed max clients 464 */ 465 if (saa_portp->saa_pt_reference_count > 466 SAA_MAX_CLIENTS_PER_PORT) { 467 468 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 469 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, "", 470 "ibmf_saa_impl_add_client: %s, num_reg_clients %d\n", 471 tnf_string, msg, "too many clients registered for" 472 " port", tnf_uint, num_reg_clients, 473 saa_portp->saa_pt_reference_count); 474 475 status = IBMF_BUSY; 476 goto bail; 477 } 478 479 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 480 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "", 481 "ibmf_saa_impl_add_client: num_registered_clients %d\n", 482 tnf_uint, num_registered_clients, 483 saa_portp->saa_pt_reference_count); 484 485 /* 486 * wait until anyone who is currently registering 487 * this port with ibmf is done 488 */ 489 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) { 490 491 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 492 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "", 493 "ibmf_saa_impl_add_client: %s\n", 494 tnf_string, msg, "someone is registering. waiting" 495 " for them to finish"); 496 497 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv, 498 &saa_portp->saa_pt_mutex); 499 500 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 501 ibmf_saa_impl_add_client, 502 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client: %s\n", 503 tnf_string, msg, "done waiting"); 504 } 505 506 /* 507 * if port isn't ready here, fail. 508 */ 509 if (saa_portp->saa_pt_state != IBMF_SAA_PORT_STATE_READY) { 510 511 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 512 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, 513 "", "ibmf_saa_impl_add_client: %s\n", 514 tnf_string, msg, "port state not ready," 515 " removing client."); 516 517 status = IBMF_BAD_PORT_STATE; 518 goto bail; 519 } 520 521 bail: 522 mutex_exit(&saa_portp->saa_pt_mutex); 523 524 if (status != IBMF_SUCCESS) { 525 526 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 527 528 IBMF_SAA_ADD32_KSTATS(saa_portp, 529 clients_reg_failed, 1); 530 531 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 532 533 /* decrementing refcount is last thing we do on entry */ 534 535 mutex_enter(&saa_portp->saa_pt_mutex); 536 537 ASSERT(saa_portp->saa_pt_reference_count > 0); 538 saa_portp->saa_pt_reference_count--; 539 540 mutex_exit(&saa_portp->saa_pt_mutex); 541 } 542 543 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 544 ibmf_saa_impl_add_client_end, IBMF_TNF_TRACE, "", 545 "ibmf_saa_impl_add_client() exit\n"); 546 547 return (status); 548 } 549 550 /* 551 * ibmf_saa_impl_create_port() 552 * Create port entry with mimimal inits because 553 * we're holding the list mutex: NO BLOCKING CALLS HERE, please. 554 * 555 * Initialize port state to "registering", so that clients accessing 556 * same port concurrently will wait for the end of the ibmf registration. 557 * Note: this thread will access port members without locking mutex. 558 * 559 * Input Arguments 560 * pt_guid guid of port 561 * 562 * Output Arguments 563 * saa_portpp pointer to new saa_portp structure 564 * 565 * Returns 566 * IBMF_NO_MEMORY if memory could not be allocated 567 * IBMF_SUCCESS otherwise 568 */ 569 int 570 ibmf_saa_impl_create_port(ib_guid_t pt_guid, saa_port_t **saa_portpp) 571 { 572 int status = IBMF_SUCCESS; 573 saa_port_t *saa_portp = NULL; 574 575 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_create_port_start, 576 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port:" 577 " guid %016" PRIx64 "\n", 578 tnf_opaque, port_guid, pt_guid); 579 580 ASSERT(MUTEX_HELD(&saa_statep->saa_port_list_mutex)); 581 582 /* create & initialize new port */ 583 saa_portp = kmem_zalloc(sizeof (saa_port_t), KM_NOSLEEP); 584 585 if (saa_portp == NULL) { 586 587 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 588 ibmf_sa_session_open_err, IBMF_TNF_ERROR, "", 589 "ibmf_saa_impl_create_port: %s\n", 590 tnf_string, msg, "could not allocate memory for " 591 "new port"); 592 593 status = IBMF_NO_MEMORY; 594 goto bail; 595 } 596 597 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_sa_session_open, 598 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port: %s\n", 599 tnf_string, msg, "first client registering, initializing"); 600 601 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp)) 602 603 /* tell everyone that kstats are not initialized */ 604 saa_portp->saa_pt_kstatp = NULL; 605 606 /* 607 * set up mutexe and state variable to indicate to 608 * other clients that were currently in the process of 609 * setting up the port data. This will prevent a subsequent 610 * client from trying to to register with ibmf before the 611 * port data has been initialized. 612 */ 613 mutex_init(&saa_portp->saa_pt_mutex, NULL, MUTEX_DRIVER, NULL); 614 cv_init(&saa_portp->saa_pt_ibmf_reg_cv, NULL, CV_DRIVER, NULL); 615 616 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_REGISTERING; 617 618 /* create other mutexes */ 619 mutex_init(&saa_portp->saa_pt_kstat_mutex, NULL, MUTEX_DRIVER, NULL); 620 621 mutex_init(&saa_portp->saa_pt_event_sub_mutex, NULL, MUTEX_DRIVER, 622 NULL); 623 624 /* 625 * clients assume all arrive; set mask to this so we only notify 626 * if something failed 627 */ 628 saa_portp->saa_pt_event_sub_last_success_mask = 629 IBMF_SAA_PORT_EVENT_SUB_ALL_ARRIVE; 630 631 /* 632 * set port_guid now so any immediately subsequent clients 633 * registering on this port, guid will know we're already here 634 */ 635 saa_portp->saa_pt_port_guid = pt_guid; 636 saa_portp->saa_pt_reference_count = 1; 637 saa_portp->saa_pt_current_tid = pt_guid << 32; 638 639 saa_portp->saa_pt_redirect_active = B_FALSE; 640 641 /* set sa_uptime now in case we never receive anything from SA */ 642 saa_portp->saa_pt_sa_uptime = gethrtime(); 643 644 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_portp)) 645 646 /* Set new pointer in caller's */ 647 *saa_portpp = saa_portp; 648 649 bail: 650 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_create_port_end, 651 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port() exit\n"); 652 653 return (status); 654 } 655 656 /* 657 * ibmf_saa_impl_invalidate_port: 658 * invalidates port entry (assumes exist) and deletes kstat object 659 * kstat object is destroyed in order to allow creating port entry 660 * even if this entry is not purged 661 */ 662 static void 663 ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp) 664 { 665 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 666 ibmf_saa_impl_invalidate_port_start, 667 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() enter\n"); 668 669 ASSERT(saa_portp != NULL); 670 ASSERT(MUTEX_HELD(&saa_portp->saa_pt_mutex)); 671 672 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_INVALID; 673 ibmf_saa_impl_uninit_kstats(saa_portp); 674 675 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 676 ibmf_saa_impl_invalidate_port_end, 677 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() exit\n"); 678 } 679 680 /* 681 * ibmf_saa_impl_destroy_port: 682 * Frees the resources associated with an saa_portp structure. Assumes the 683 * saa_portp exists 684 * 685 * Input Arguments 686 * saa_portp pointer to saa_portp structure 687 * 688 * Output Arguments 689 * none 690 * 691 * Returns 692 * void 693 */ 694 static void 695 ibmf_saa_impl_destroy_port(saa_port_t *saa_portp) 696 { 697 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_start, 698 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() enter\n"); 699 700 ASSERT(saa_portp != NULL); 701 702 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 703 704 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 705 ibmf_saa_impl_destroy, IBMF_TNF_TRACE, "", 706 "ibmf_saa_impl_destroy(): destroying port_guid %016" PRIx64 "\n", 707 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 708 709 ibmf_saa_impl_uninit_kstats(saa_portp); 710 711 /* uninit synchronization variables used for registration */ 712 mutex_destroy(&saa_portp->saa_pt_mutex); 713 cv_destroy(&saa_portp->saa_pt_ibmf_reg_cv); 714 715 mutex_destroy(&saa_portp->saa_pt_event_sub_mutex); 716 mutex_destroy(&saa_portp->saa_pt_kstat_mutex); 717 718 kmem_free(saa_portp, sizeof (saa_port_t)); 719 720 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_end, 721 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() exit\n"); 722 } 723 724 /* 725 * ibmf_saa_impl_init_kstats: 726 * Create kstats structure. Should be called when memory is alloced for a new 727 * port entry. 728 */ 729 int 730 ibmf_saa_impl_init_kstats(saa_port_t *saa_portp) 731 { 732 char buf[128]; 733 ibmf_saa_kstat_t *ksp; 734 735 _NOTE(ASSUMING_PROTECTED(saa_portp->saa_pt_kstatp)) 736 737 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 738 ibmf_saa_impl_init_kstats_start, 739 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() enter\n"); 740 741 /* set up kstats structure */ 742 (void) sprintf(buf, "ibmf_saa_%016" PRIx64 "_stat", 743 saa_portp->saa_pt_port_guid); 744 745 saa_portp->saa_pt_kstatp = kstat_create("ibmf_saa", 746 0, buf, "misc", KSTAT_TYPE_NAMED, 747 sizeof (ibmf_saa_kstat_t) / sizeof (kstat_named_t), 748 KSTAT_FLAG_WRITABLE); 749 750 if (saa_portp->saa_pt_kstatp == NULL) 751 return (IBMF_NO_RESOURCES); 752 753 ksp = (ibmf_saa_kstat_t *)saa_portp->saa_pt_kstatp->ks_data; 754 755 kstat_named_init(&ksp->clients_registered, 756 "clients_registered", KSTAT_DATA_UINT32); 757 758 kstat_named_init(&ksp->clients_reg_failed, 759 "clients_reg_failed", KSTAT_DATA_UINT32); 760 761 kstat_named_init(&ksp->outstanding_requests, 762 "outstanding_requests", KSTAT_DATA_UINT32); 763 764 kstat_named_init(&ksp->total_requests, 765 "total_requests", KSTAT_DATA_UINT32); 766 767 kstat_named_init(&ksp->failed_requests, 768 "failed_requests", KSTAT_DATA_UINT32); 769 770 kstat_named_init(&ksp->requests_timedout, 771 "requests_timedout", KSTAT_DATA_UINT32); 772 773 kstat_install(saa_portp->saa_pt_kstatp); 774 775 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 776 ibmf_saa_impl_init_kstats_end, 777 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() exit\n"); 778 779 return (IBMF_SUCCESS); 780 } 781 782 /* 783 * ibmf_saa_impl_uninit_kstats: 784 * Free kstats context. Should be called when port is either destroyed 785 * or invalidated. 786 */ 787 static void 788 ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp) 789 { 790 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 791 ibmf_saa_impl_uninit_kstats_start, 792 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() enter\n"); 793 794 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 795 796 if (saa_portp->saa_pt_kstatp != NULL) { 797 kstat_delete(saa_portp->saa_pt_kstatp); 798 } 799 saa_portp->saa_pt_kstatp = NULL; 800 801 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 802 803 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 804 ibmf_saa_impl_uninit_kstats_end, 805 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() exit\n"); 806 } 807 808 /* 809 * ibmf_saa_impl_register_failed: 810 * invalidate entry and kick waiters 811 */ 812 void 813 ibmf_saa_impl_register_failed(saa_port_t *saa_portp) 814 { 815 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 816 ibmf_saa_impl_register_failed_start, 817 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() enter\n"); 818 819 mutex_enter(&saa_portp->saa_pt_mutex); 820 821 ibmf_saa_impl_invalidate_port(saa_portp); 822 823 cv_broadcast(&saa_portp->saa_pt_ibmf_reg_cv); 824 825 /* decrementing refcount is last thing we do on entry */ 826 827 ASSERT(saa_portp->saa_pt_reference_count > 0); 828 saa_portp->saa_pt_reference_count--; 829 830 mutex_exit(&saa_portp->saa_pt_mutex); 831 832 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 833 ibmf_saa_impl_register_failed_end, 834 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() exit\n"); 835 } 836 837 static int 838 ibmf_saa_impl_setup_qp_async_cb(saa_port_t *saa_portp, int setup_async_cb_only) 839 { 840 int status; 841 int unreg_status; 842 ib_pkey_t p_key; 843 ib_qkey_t q_key; 844 uint8_t portnum; 845 boolean_t qp_alloced = B_FALSE; 846 847 if (setup_async_cb_only == 0) { 848 849 /* allocate a qp through ibmf */ 850 status = ibmf_alloc_qp(saa_portp->saa_pt_ibmf_handle, 851 IB_PKEY_DEFAULT_LIMITED, IB_GSI_QKEY, 852 IBMF_ALT_QP_MAD_RMPP, &saa_portp->saa_pt_qp_handle); 853 854 if (status != IBMF_SUCCESS) { 855 856 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 857 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 858 "ibmf_saa_impl_setup_qp_async_cb: %s, " 859 "ibmf_status = %d\n", 860 tnf_string, msg, "Cannot alloc qp with ibmf", 861 tnf_int, status, status); 862 863 return (status); 864 } 865 866 qp_alloced = B_TRUE; 867 868 /* 869 * query the queue pair number; we will need it to unsubscribe 870 * from notice reports 871 */ 872 status = ibmf_query_qp(saa_portp->saa_pt_ibmf_handle, 873 saa_portp->saa_pt_qp_handle, &saa_portp->saa_pt_qpn, 874 &p_key, &q_key, &portnum, 0); 875 876 if (status != IBMF_SUCCESS) { 877 878 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 879 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 880 "ibmf_saa_impl_setup_qp_async_cb: %s, " 881 "ibmf_status = %d\n", 882 tnf_string, msg, 883 "Cannot query alt qp to get qp num", 884 tnf_int, status, status); 885 886 goto bail; 887 } 888 } 889 890 /* 891 * core ibmf is taking advantage of the fact that saa_portp is our 892 * callback arg. If this changes, the code in ibmf_recv would need to 893 * change as well 894 */ 895 status = ibmf_setup_async_cb(saa_portp->saa_pt_ibmf_handle, 896 saa_portp->saa_pt_qp_handle, ibmf_saa_report_cb, saa_portp, 0); 897 if (status != IBMF_SUCCESS) { 898 899 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 900 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 901 "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status = %d\n", 902 tnf_string, msg, "Cannot register async cb with ibmf", 903 tnf_int, status, status); 904 905 goto bail; 906 } 907 908 return (IBMF_SUCCESS); 909 910 bail: 911 if (qp_alloced == B_TRUE) { 912 /* free alternate qp */ 913 unreg_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle, 914 &saa_portp->saa_pt_qp_handle, 0); 915 if (unreg_status != IBMF_SUCCESS) { 916 917 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 918 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 919 "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status =" 920 " %d\n", tnf_string, msg, 921 "Cannot free alternate queue pair with ibmf", 922 tnf_int, unreg_status, unreg_status); 923 } 924 } 925 926 return (status); 927 } 928 929 /* 930 * ibmf_saa_impl_register_port: 931 */ 932 int 933 ibmf_saa_impl_register_port( 934 saa_port_t *saa_portp) 935 { 936 uint_t hca_count = 0; 937 ib_guid_t *hca_list = NULL; 938 int status = IBMF_SUCCESS; 939 int unreg_status = IBMF_SUCCESS; 940 int ibt_status = IBT_SUCCESS; 941 ibt_hca_portinfo_t *port_info_list = NULL; 942 uint_t port_count = 0; 943 uint_t port_size = 0; 944 int ihca, iport; 945 ib_guid_t port_guid; 946 boolean_t ibmf_reg = B_FALSE; 947 948 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 949 ibmf_saa_impl_register_port_start, IBMF_TNF_TRACE, "", 950 "ibmf_saa_impl_register_port() enter\n"); 951 952 ASSERT(saa_portp != NULL); 953 954 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 955 956 /* get the HCA list */ 957 958 hca_count = ibt_get_hca_list(&hca_list); 959 960 if (hca_count == 0) { 961 962 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 963 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 964 "ibmf_saa_impl_register_port: %s\n", 965 tnf_string, msg, "cannot register port (no HCAs).\n"); 966 967 status = IBMF_BAD_PORT; 968 goto bail; 969 } 970 971 /* lookup requested port guid in hca list */ 972 for (ihca = 0; ihca != hca_count; ihca++) { 973 974 ibt_status = ibt_query_hca_ports_byguid(hca_list[ihca], 975 0 /* all ports */, &port_info_list, 976 &port_count, &port_size); 977 978 if (ibt_status != IBT_SUCCESS) { 979 980 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 981 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 982 "ibmf_saa_impl_register_port: %s, %016" PRIx64 "\n", 983 tnf_string, msg, "Could not query hca. Exiting.", 984 tnf_opaque, guid, hca_list[ihca]); 985 986 status = IBMF_TRANSPORT_FAILURE; 987 break; 988 } 989 990 for (iport = 0; iport < port_count; iport++) { 991 992 /* get port guid associated with hca guid, port num */ 993 if (ibmf_saa_impl_get_port_guid( 994 port_info_list + iport, &port_guid) != IBMF_SUCCESS) 995 continue; 996 997 if (saa_portp->saa_pt_port_guid != port_guid) 998 continue; 999 1000 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 1001 ibmf_saa_impl_register_port, 1002 IBMF_TNF_TRACE, "", 1003 "ibmf_saa_impl_register_port: %s, hca_guid = %016" 1004 PRIx64 ", port_guid = %016" PRIx64 1005 ", number = %d\n", 1006 tnf_string, msg, "found port", 1007 tnf_opaque, hca_guid, hca_list[ihca], 1008 tnf_opaque, port_guid, port_guid, 1009 tnf_uint, port, iport + 1); 1010 1011 /* 1012 * we're here? then we found our port: 1013 * fill in ibmf registration info 1014 * and address parameters from the portinfo 1015 */ 1016 1017 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid 1018 = hca_list[ihca]; 1019 saa_portp->saa_pt_ibmf_reginfo.ir_port_num = iport+1; 1020 saa_portp->saa_pt_ibmf_reginfo.ir_client_class 1021 = SUBN_ADM_MANAGER; 1022 1023 saa_portp->saa_pt_node_guid = hca_list[ihca]; 1024 saa_portp->saa_pt_port_num = iport + 1; 1025 1026 ibmf_saa_impl_set_transaction_params( 1027 saa_portp, port_info_list + iport); 1028 break; 1029 } 1030 1031 ibt_free_portinfo(port_info_list, port_size); 1032 1033 if (iport != port_count) 1034 break; /* found our port */ 1035 } 1036 1037 ibt_free_hca_list(hca_list, hca_count); 1038 1039 if (ihca == hca_count) { 1040 1041 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1042 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1043 "ibmf_saa_impl_register_port: %s, port_guid %016" 1044 PRIx64 "\n", 1045 tnf_string, msg, "Could not find port, exiting", 1046 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 1047 1048 status = IBMF_BAD_PORT; 1049 } 1050 1051 if (status != IBMF_SUCCESS) { 1052 1053 goto bail; 1054 } 1055 1056 /* 1057 * Now we found the port we searched for, 1058 * and open an ibmf session on that port. 1059 */ 1060 1061 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1062 ibmf_saa_impl_register_port, IBMF_TNF_TRACE, "", 1063 "ibmf_saa_impl_register_port: %s, port_guid = %016" PRIx64 1064 ", port = %d\n", tnf_string, msg, "Registering with ibmf", 1065 tnf_opaque, port_guid, saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid, 1066 tnf_uint, port, saa_portp->saa_pt_ibmf_reginfo.ir_port_num); 1067 1068 status = ibmf_register(&saa_portp->saa_pt_ibmf_reginfo, 1069 IBMF_VERSION, IBMF_REG_FLAG_RMPP, 1070 ibmf_saa_impl_async_event_cb, saa_portp, 1071 &saa_portp->saa_pt_ibmf_handle, 1072 &saa_portp->saa_pt_ibmf_impl_features); 1073 1074 if (status != IBMF_SUCCESS) { 1075 1076 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1077 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1078 "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n", 1079 tnf_string, msg, "Could not register with ibmf", 1080 tnf_int, status, status); 1081 1082 goto bail; 1083 } 1084 1085 ibmf_reg = B_TRUE; 1086 1087 if (ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0) == IBMF_SUCCESS) 1088 return (IBMF_SUCCESS); 1089 1090 bail: 1091 if (ibmf_reg == B_TRUE) { 1092 /* unregister from ibmf */ 1093 unreg_status = ibmf_unregister( 1094 &saa_portp->saa_pt_ibmf_handle, 0); 1095 1096 if (unreg_status != IBMF_SUCCESS) { 1097 1098 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1099 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1100 "ibmf_saa_impl_register_port: %s, ibmf_status =" 1101 " %d\n", tnf_string, msg, 1102 "Cannot unregister from ibmf", 1103 tnf_int, unreg_status, unreg_status); 1104 } 1105 } 1106 1107 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_register_port_end, 1108 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_port() exit\n"); 1109 1110 return (status); 1111 } 1112 1113 /* 1114 * ibmf_saa_impl_getclassportinfo: 1115 */ 1116 void 1117 ibmf_saa_impl_get_classportinfo(saa_port_t *saa_portp) 1118 { 1119 int res; 1120 saa_impl_trans_info_t *trans_info; 1121 1122 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1123 ibmf_saa_impl_get_classportinfo_start, 1124 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() enter\n"); 1125 1126 /* 1127 * allocate memory for trans_info; send_request's callback will free up 1128 * memory since request is asynchronous 1129 */ 1130 trans_info = kmem_zalloc(sizeof (saa_impl_trans_info_t), KM_NOSLEEP); 1131 if (trans_info == NULL) { 1132 1133 mutex_enter(&saa_portp->saa_pt_mutex); 1134 1135 /* cpi transaction is handled as a client, decrement refcount */ 1136 ASSERT(saa_portp->saa_pt_reference_count > 0); 1137 saa_portp->saa_pt_reference_count--; 1138 1139 mutex_exit(&saa_portp->saa_pt_mutex); 1140 1141 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1142 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_ERROR, "", 1143 "ibmf_saa_impl_get_classportinfo: %s\n", tnf_string, msg, 1144 "Could not allocate memory for classportinfo trans_info"); 1145 1146 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1147 ibmf_saa_impl_get_classportinfo_end, IBMF_TNF_TRACE, "", 1148 "ibmf_saa_impl_get_classportinfo() exiting\n"); 1149 1150 return; 1151 } 1152 1153 /* no specific client associated with this transaction */ 1154 trans_info->si_trans_client_data = NULL; 1155 trans_info->si_trans_port = saa_portp; 1156 trans_info->si_trans_method = SA_SUBN_ADM_GET; 1157 trans_info->si_trans_attr_id = MAD_ATTR_ID_CLASSPORTINFO; 1158 1159 trans_info->si_trans_callback = ibmf_saa_impl_get_cpi_cb; 1160 trans_info->si_trans_callback_arg = saa_portp; 1161 1162 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 1163 1164 IBMF_SAA_ADD32_KSTATS(saa_portp, outstanding_requests, 1); 1165 IBMF_SAA_ADD32_KSTATS(saa_portp, total_requests, 1); 1166 1167 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 1168 1169 res = ibmf_saa_impl_send_request(trans_info); 1170 1171 if (res != IBMF_SUCCESS) { 1172 1173 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1174 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_TRACE, "", 1175 "ibmf_saa_impl_get_classportinfo: %s, res = 0x%x\n", 1176 tnf_string, msg, "ibmf_saa_impl_send_request failed", 1177 tnf_opaque, res, res); 1178 1179 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 1180 1181 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1); 1182 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1); 1183 1184 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 1185 1186 mutex_enter(&saa_portp->saa_pt_mutex); 1187 1188 /* cpi transaction is handled as a client, decrement refcount */ 1189 ASSERT(saa_portp->saa_pt_reference_count > 0); 1190 saa_portp->saa_pt_reference_count--; 1191 1192 mutex_exit(&saa_portp->saa_pt_mutex); 1193 1194 } 1195 1196 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1197 ibmf_saa_impl_get_classportinfo_end, 1198 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() exit\n"); 1199 } 1200 1201 /* 1202 * ibmf_saa_impl_get_cpi_cb: 1203 * 1204 * Called when the asynchronous getportinfo request receives its response. 1205 * Checks the status. If success, updates the times in the port's 1206 * ibmf_retrans structure that is used in ibmf_msg_transport calls. If failure, 1207 * just use default values. 1208 * 1209 * Input Arguments 1210 * arg user-specified pointer (points to the current port data) 1211 * length length of payload returned (should be size of classportinfo_t) 1212 * buffer pointer to classportinfo returned (should not be null) 1213 * status status of sa access request 1214 * 1215 * Output Arguments 1216 * none 1217 * 1218 * Returns void 1219 */ 1220 static void 1221 ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, int status) 1222 { 1223 saa_port_t *saa_portp; 1224 uint64_t base_time, resp_timeout, rttv_timeout; 1225 ib_mad_classportinfo_t *classportinfo; 1226 int resp_time_value; 1227 uint16_t sa_cap_mask; 1228 1229 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_start, 1230 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() enter\n"); 1231 1232 /* 1233 * access port entry: note that it may have become invalid 1234 * but we hold a ref count for cpi and the interactions on 1235 * the entry are harmless 1236 */ 1237 saa_portp = (saa_port_t *)arg; 1238 1239 /* process response */ 1240 1241 if ((status != IBMF_SUCCESS) || (buffer == NULL)) { 1242 1243 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1, 1244 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_ERROR, "", 1245 "ibmf_saa_impl_get_cpi_cb: %s, status = %d, buffer = " 1246 " 0x%p, length = %d\n", tnf_string, msg, 1247 "could not get classportinfo. Check node and path to sm" 1248 " lid", tnf_int, status, status, 1249 tnf_opaque, buffer, buffer, tnf_uint, length, length); 1250 1251 /* 1252 * IB spec (C13-13) indicates 20 can be used as default or 1253 * intial value for classportinfo->resptimeout value 1254 */ 1255 resp_time_value = 20; 1256 1257 sa_cap_mask = 0xFFFF; 1258 1259 } else if (buffer != NULL) { 1260 1261 classportinfo = (ib_mad_classportinfo_t *)buffer; 1262 1263 resp_time_value = classportinfo->RespTimeValue & 0x1f; 1264 1265 /* 1266 * Because some subnet managers might not provide sane 1267 * value for "resp_time_value", we limit it here. In 1268 * case this limit is too restrictive (very large fabric), 1269 * we allow the limit to be raised (/etc/system). 1270 */ 1271 if (resp_time_value > ibmf_saa_max_resp_time) { 1272 cmn_err(CE_CONT, "!ibmf_saa_max_resp_time (%d) " 1273 "exceeded.", ibmf_saa_max_resp_time); 1274 cmn_err(CE_CONT, "!Reducing subnet administrator " 1275 "resp_time value from %d to %d.", 1276 resp_time_value, ibmf_saa_max_resp_time); 1277 resp_time_value = ibmf_saa_max_resp_time; 1278 } 1279 1280 sa_cap_mask = classportinfo->CapabilityMask; 1281 1282 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1283 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "", 1284 "ibmf_saa_impl_get_cpi_cb: %s, timeout = 0x%x," 1285 " cap_mask = 0x%x\n", 1286 tnf_string, msg, "got classportinfo", 1287 tnf_opaque, timeout, resp_time_value, 1288 tnf_opaque, cap_mask, sa_cap_mask); 1289 1290 kmem_free(buffer, length); 1291 } 1292 1293 /* 1294 * using IB spec calculation from 13.4.6.2 1295 * use bit shifting for 2^x. 1296 */ 1297 base_time = (1 << resp_time_value); 1298 1299 resp_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000; 1300 1301 mutex_enter(&saa_portp->saa_pt_mutex); 1302 1303 base_time = 2 * (1 << saa_portp->saa_pt_timeout); 1304 1305 rttv_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000; 1306 1307 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = resp_timeout; 1308 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = rttv_timeout; 1309 saa_portp->saa_pt_sa_cap_mask = sa_cap_mask; 1310 1311 /* 1312 * cpi transaction is handled as a client, 1313 * decrement refcount; make sure it's the last 1314 * thing we do on this entry 1315 */ 1316 ASSERT(saa_portp->saa_pt_reference_count > 0); 1317 saa_portp->saa_pt_reference_count--; 1318 1319 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1320 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "", 1321 "ibmf_saa_impl_get_cpi_cb: %s, subnet_timeout = 0x%x, " 1322 "resp_time_value = 0x%x\n", 1323 tnf_string, msg, "updated resp timeout", 1324 tnf_opaque, subnet_timeout, saa_portp->saa_pt_timeout, 1325 tnf_opaque, resp_time_value, resp_time_value); 1326 1327 mutex_exit(&saa_portp->saa_pt_mutex); 1328 1329 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_end, 1330 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() exit\n"); 1331 } 1332 1333 /* 1334 * ibmf_saa_impl_send_request: 1335 * Sends a request to the sa. Can be used for both classportinfo and record 1336 * requests. Will set up all data structures for using the multi-packet 1337 * protocol, create the mad, and send it. Returns SA_SUCCESS if msg transport 1338 * worked, meaning succesful send for the async case and a succesful send and 1339 * recv for the sync case. 1340 */ 1341 int 1342 ibmf_saa_impl_send_request(saa_impl_trans_info_t *trans_info) 1343 { 1344 uint16_t attr_id; 1345 saa_client_data_t *client_data; 1346 saa_port_t *saa_portp; 1347 uint32_t transport_flags; 1348 ibmf_msg_cb_t ibmf_callback; 1349 void *ibmf_callback_arg; 1350 ibmf_msg_t *msgp; 1351 ibmf_retrans_t ibmf_retrans; 1352 uint16_t sa_cap_mask; 1353 boolean_t sleep_flag; 1354 int ibmf_status = IBMF_SUCCESS; 1355 int retry_count; 1356 uint16_t mad_status; 1357 boolean_t sa_is_redirected = B_FALSE; 1358 1359 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1360 ibmf_saa_impl_send_request_start, 1361 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() enter\n"); 1362 1363 attr_id = trans_info->si_trans_attr_id; 1364 client_data = trans_info->si_trans_client_data; 1365 saa_portp = trans_info->si_trans_port; 1366 1367 /* 1368 * don't send on invalid entry 1369 * Note that there is a window where it could become 1370 * invalid after this test is done, but we'd rely on ibmf errors... 1371 */ 1372 if (ibmf_saa_is_valid(saa_portp, B_FALSE) == B_FALSE) { 1373 1374 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1, 1375 ibmf_saa_impl_send_request, 1376 IBMF_TNF_ERROR, "", 1377 "ibmf_saa_impl_send_request: %s, hca_guid = %016" 1378 PRIx64 ", port_guid = %016" PRIx64 1379 ", number = %d\n", 1380 tnf_string, msg, "sending on invalid port", 1381 tnf_opaque, hca_guid, 1382 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid, 1383 tnf_opaque, port_guid, 1384 saa_portp->saa_pt_port_guid, 1385 tnf_uint, port, 1386 saa_portp->saa_pt_ibmf_reginfo.ir_port_num); 1387 1388 ibmf_status = IBMF_REQ_INVALID; 1389 goto bail; 1390 } 1391 1392 /* check whether SA supports this attribute */ 1393 mutex_enter(&saa_portp->saa_pt_mutex); 1394 1395 sa_cap_mask = saa_portp->saa_pt_sa_cap_mask; 1396 sa_is_redirected = saa_portp->saa_pt_redirect_active; 1397 1398 mutex_exit(&saa_portp->saa_pt_mutex); 1399 1400 ibmf_status = ibmf_saa_impl_check_sa_support(sa_cap_mask, attr_id); 1401 1402 if (ibmf_status != IBMF_SUCCESS) { 1403 1404 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1405 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "", 1406 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1407 tnf_string, msg, "SA does not support attribute", 1408 tnf_int, ibmf_status, ibmf_status); 1409 1410 goto bail; 1411 } 1412 1413 /* make only non-blocking calls if this is an async request */ 1414 if ((trans_info->si_trans_callback == NULL) && 1415 (trans_info->si_trans_sub_callback == NULL)) { 1416 ibmf_callback = NULL; 1417 ibmf_callback_arg = NULL; 1418 sleep_flag = B_TRUE; 1419 } else { 1420 ibmf_callback = ibmf_saa_async_cb; 1421 ibmf_callback_arg = (void *)trans_info; 1422 sleep_flag = B_FALSE; 1423 } 1424 1425 ibmf_status = ibmf_saa_impl_init_msg(trans_info, sleep_flag, &msgp, 1426 &transport_flags, &ibmf_retrans); 1427 if (ibmf_status != IBMF_SUCCESS) { 1428 1429 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1430 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "", 1431 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1432 tnf_string, msg, "init_msg() failed", 1433 tnf_int, ibmf_status, ibmf_status); 1434 1435 goto bail; 1436 } 1437 1438 mutex_enter(&saa_portp->saa_pt_mutex); 1439 1440 saa_portp->saa_pt_num_outstanding_trans++; 1441 1442 mutex_exit(&saa_portp->saa_pt_mutex); 1443 1444 /* 1445 * increment the number of outstanding transaction so 1446 * ibmf_close_sa_session() will wait. classportinfo requests 1447 * don't have associated clients so check for valid clientp 1448 */ 1449 if (client_data != NULL) { 1450 1451 mutex_enter(&client_data->saa_client_mutex); 1452 1453 client_data->saa_client_num_pending_trans++; 1454 1455 mutex_exit(&client_data->saa_client_mutex); 1456 } 1457 1458 /* 1459 * make the call to msg_transport. If synchronous and success, 1460 * check that the response mad isn't status busy. If so, repeat the 1461 * call 1462 */ 1463 retry_count = 0; 1464 1465 /* 1466 * set the send time here. We only set this once at the beginning of 1467 * the transaction. Retrying because of busys or mastersmlid changes 1468 * does not change the original send time. It is meant to be an 1469 * absolute time out value and will only be used if there are other 1470 * problems (i.e. a buggy SA) 1471 */ 1472 trans_info->si_trans_send_time = gethrtime(); 1473 1474 for (;;) { 1475 1476 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 1477 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 1478 ibmf_callback, ibmf_callback_arg, transport_flags); 1479 1480 if (ibmf_callback != NULL) 1481 break; 1482 1483 /* 1484 * stop here for non-sequenced transactions since they wouldn't 1485 * receive a timeout or busy response 1486 */ 1487 if (!(transport_flags & IBMF_MSG_TRANS_FLAG_SEQ)) 1488 break; 1489 1490 /* 1491 * if the transaction timed out and this was a synchronous 1492 * request there's a possiblity we were talking to the wrong 1493 * master smlid or that the SA has stopped responding on the 1494 * redirected desination (if redirect is active). 1495 * Check this and retry if necessary. 1496 */ 1497 if ((ibmf_status == IBMF_TRANS_TIMEOUT) && 1498 (sleep_flag == B_TRUE)) { 1499 if (sa_is_redirected == B_TRUE) { 1500 ibmf_status = ibmf_saa_impl_revert_to_qp1( 1501 saa_portp, msgp, ibmf_callback, 1502 ibmf_callback_arg, transport_flags); 1503 } else { 1504 ibmf_status = ibmf_saa_impl_new_smlid_retry( 1505 saa_portp, msgp, ibmf_callback, 1506 ibmf_callback_arg, transport_flags); 1507 } 1508 } 1509 1510 /* 1511 * if the transaction timed out (and retrying with a new SM LID 1512 * didn't help) check how long it's been since we received an SA 1513 * packet. If it hasn't been max_wait_time then retry the 1514 * request. 1515 */ 1516 if ((ibmf_status == IBMF_TRANS_TIMEOUT) && 1517 (sleep_flag == B_TRUE)) { 1518 1519 ibmf_status = ibmf_saa_check_sa_and_retry( 1520 saa_portp, msgp, ibmf_callback, ibmf_callback_arg, 1521 trans_info->si_trans_send_time, transport_flags); 1522 } 1523 1524 if (ibmf_status != IBMF_SUCCESS) 1525 break; 1526 1527 if (retry_count >= IBMF_SAA_MAX_BUSY_RETRY_COUNT) 1528 break; 1529 1530 /* sync transaction with status SUCCESS should have response */ 1531 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL); 1532 1533 mad_status = b2h16(msgp->im_msgbufs_recv. 1534 im_bufs_mad_hdr->Status); 1535 1536 if ((mad_status != MAD_STATUS_BUSY) && 1537 (mad_status != MAD_STATUS_REDIRECT_REQUIRED)) 1538 break; 1539 1540 if (mad_status == MAD_STATUS_REDIRECT_REQUIRED) { 1541 1542 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1543 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1544 "ibmf_saa_impl_send_request: %s, retry_count %d\n", 1545 tnf_string, msg, 1546 "response returned redirect status", 1547 tnf_int, retry_count, retry_count); 1548 1549 /* update address info and copy it into msgp */ 1550 ibmf_saa_impl_update_sa_address_info(saa_portp, msgp); 1551 } else { 1552 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1553 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1554 "ibmf_saa_impl_send_request: %s, retry_count %d\n", 1555 tnf_string, msg, "response returned busy status", 1556 tnf_int, retry_count, retry_count); 1557 } 1558 1559 retry_count++; 1560 1561 /* 1562 * since this is a blocking call, sleep for some time 1563 * to allow SA to transition from busy state (if busy) 1564 */ 1565 if (mad_status == MAD_STATUS_BUSY) 1566 delay(drv_sectohz(IBMF_SAA_BUSY_RETRY_SLEEP_SECS)); 1567 } 1568 1569 if (ibmf_status != IBMF_SUCCESS) { 1570 1571 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1572 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1573 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1574 tnf_string, msg, "ibmf_msg_transport() failed", 1575 tnf_int, ibmf_status, ibmf_status); 1576 1577 ibmf_saa_impl_free_msg(saa_portp->saa_pt_ibmf_handle, msgp); 1578 1579 mutex_enter(&saa_portp->saa_pt_mutex); 1580 1581 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 1582 saa_portp->saa_pt_num_outstanding_trans--; 1583 1584 mutex_exit(&saa_portp->saa_pt_mutex); 1585 1586 if (client_data != NULL) { 1587 1588 mutex_enter(&client_data->saa_client_mutex); 1589 1590 ASSERT(client_data->saa_client_num_pending_trans > 0); 1591 client_data->saa_client_num_pending_trans--; 1592 1593 if ((client_data->saa_client_num_pending_trans == 0) && 1594 (client_data->saa_client_state == 1595 SAA_CLIENT_STATE_WAITING)) 1596 cv_signal(&client_data->saa_client_state_cv); 1597 1598 mutex_exit(&client_data->saa_client_mutex); 1599 } 1600 1601 } else if (sleep_flag == B_TRUE) { 1602 1603 mutex_enter(&saa_portp->saa_pt_mutex); 1604 1605 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 1606 saa_portp->saa_pt_num_outstanding_trans--; 1607 1608 mutex_exit(&saa_portp->saa_pt_mutex); 1609 1610 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1611 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1612 "ibmf_saa_impl_send_request: %s\n", 1613 tnf_string, msg, "Message sent and received successfully"); 1614 1615 /* fill in response values and free the message */ 1616 ibmf_saa_impl_prepare_response(saa_portp->saa_pt_ibmf_handle, 1617 msgp, B_FALSE, &trans_info->si_trans_status, 1618 &trans_info->si_trans_result, 1619 &trans_info->si_trans_length, sleep_flag); 1620 1621 if (client_data != NULL) { 1622 mutex_enter(&client_data->saa_client_mutex); 1623 1624 ASSERT(client_data->saa_client_num_pending_trans > 0); 1625 client_data->saa_client_num_pending_trans--; 1626 1627 if ((client_data->saa_client_num_pending_trans == 0) && 1628 (client_data->saa_client_state == 1629 SAA_CLIENT_STATE_WAITING)) 1630 cv_signal(&client_data->saa_client_state_cv); 1631 1632 mutex_exit(&client_data->saa_client_mutex); 1633 } 1634 } else { 1635 1636 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1637 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1638 "ibmf_saa_impl_send_request: %s\n", 1639 tnf_string, msg, "Message sent successfully"); 1640 } 1641 1642 bail: 1643 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1644 ibmf_saa_impl_send_request_end, 1645 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() exiting" 1646 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 1647 1648 return (ibmf_status); 1649 } 1650 1651 /* 1652 * ibmf_saa_impl_init_msg: 1653 * Allocates an ibmf message and fills out the header fields and formatted data 1654 * fields. Also sets up the correct transport_flags and retrans argument for 1655 * the message transport call based on the request information. 1656 * 1657 * Input Arguments 1658 * trans_info saa_trans_info structure passed to send_request 1659 * sleep_flag B_TRUE if init_msg can sleep in function calls 1660 * 1661 * Output Arguments 1662 * msgp ibmf message that should be given to msg_transport 1663 * transport_flagsp transport flags that should be given to msg_transport 1664 * ibmf_retrans_t retrans parameter that should be given to msg_transport 1665 * 1666 * Returns 1667 * ibmf_status 1668 */ 1669 static int 1670 ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, boolean_t sleep_flag, 1671 ibmf_msg_t **msgp, uint32_t *transport_flagsp, 1672 ibmf_retrans_t *ibmf_retransp) 1673 { 1674 int ibmf_status; 1675 ibmf_msg_bufs_t *req_mad; 1676 ib_mad_hdr_t *mad_hdr; 1677 int ibmf_sleep_flag, km_sleep_flag; 1678 int free_res; 1679 ib_sa_hdr_t sa_hdr; 1680 ibmf_msg_t *ibmf_msg; 1681 uint16_t attr_id, pack_attr_id; 1682 uint8_t method; 1683 saa_client_data_t *client_data; 1684 saa_port_t *saa_portp; 1685 sa_multipath_record_t *multipath_template; 1686 size_t payload_length; 1687 uint32_t transport_flags; 1688 1689 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1690 ibmf_saa_impl_init_msg_start, 1691 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() entering\n"); 1692 1693 attr_id = trans_info->si_trans_attr_id; 1694 method = trans_info->si_trans_method; 1695 client_data = trans_info->si_trans_client_data; 1696 saa_portp = trans_info->si_trans_port; 1697 1698 if (sleep_flag == B_TRUE) { 1699 ibmf_sleep_flag = IBMF_ALLOC_SLEEP; 1700 km_sleep_flag = KM_SLEEP; 1701 } else { 1702 ibmf_sleep_flag = IBMF_ALLOC_NOSLEEP; 1703 km_sleep_flag = KM_NOSLEEP; 1704 } 1705 1706 ibmf_status = ibmf_alloc_msg(saa_portp->saa_pt_ibmf_handle, 1707 ibmf_sleep_flag, &ibmf_msg); 1708 if (ibmf_status != IBMF_SUCCESS) { 1709 1710 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1711 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1712 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n", 1713 tnf_string, msg, "Cannot allocate msg_buf.", 1714 tnf_int, ibmf_status, ibmf_status); 1715 1716 goto bail; 1717 } 1718 1719 req_mad = &ibmf_msg->im_msgbufs_send; 1720 1721 /* create a template (SA MAD) */ 1722 mad_hdr = kmem_zalloc(sizeof (ib_mad_hdr_t), km_sleep_flag); 1723 1724 if (mad_hdr == NULL) { 1725 1726 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1727 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1728 "ibmf_saa_impl_init_msg: %s\n", 1729 tnf_string, msg, "Cannot allocate mad header."); 1730 1731 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1732 &ibmf_msg); 1733 ASSERT(free_res == IBMF_SUCCESS); 1734 1735 ibmf_status = IBMF_NO_MEMORY; 1736 goto bail; 1737 } 1738 1739 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mad_hdr, 1740 *req_mad)) 1741 1742 bzero(mad_hdr, sizeof (ib_mad_hdr_t)); 1743 mad_hdr->BaseVersion = SAA_MAD_BASE_VERSION; 1744 mad_hdr->MgmtClass = MAD_MGMT_CLASS_SUBN_ADM; 1745 mad_hdr->ClassVersion = SAA_MAD_CLASS_VERSION; 1746 mad_hdr->R_Method = method; 1747 mad_hdr->AttributeID = h2b16(attr_id); 1748 1749 /* attribute modifier is all Fs since RIDs are no longer used */ 1750 mad_hdr->AttributeModifier = h2b32(0xffffffff); 1751 1752 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 1753 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1754 "ibmf_saa_impl_init_msg: %s, class = 0x%x, method = 0x%x," 1755 " attr_id = 0x%x\n", tnf_string, msg, "Sending MAD", 1756 tnf_opaque, class, mad_hdr->MgmtClass, 1757 tnf_opaque, method, mad_hdr->R_Method, 1758 tnf_opaque, attr_id, attr_id); 1759 1760 bzero(&sa_hdr, sizeof (ib_sa_hdr_t)); 1761 sa_hdr.ComponentMask = trans_info->si_trans_component_mask; 1762 1763 if (client_data != NULL) 1764 sa_hdr.SM_KEY = client_data->saa_client_sm_key; 1765 1766 /* 1767 * pack data for IB wire format; req_mad will have different pointers to 1768 * sa header and payload, mad_hdr will be the same 1769 */ 1770 req_mad->im_bufs_mad_hdr = mad_hdr; 1771 1772 ibmf_status = ibmf_saa_utils_pack_sa_hdr(&sa_hdr, 1773 &req_mad->im_bufs_cl_hdr, &req_mad->im_bufs_cl_hdr_len, 1774 km_sleep_flag); 1775 1776 if (ibmf_status != IBMF_SUCCESS) { 1777 1778 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1779 ibmf_saa_impl_init_msg, IBMF_TNF_ERROR, "", 1780 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n", 1781 tnf_string, msg, "ibmf_saa_utils_pack_sa_hdr() failed", 1782 tnf_int, ibmf_status, ibmf_status); 1783 1784 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1785 1786 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1787 &ibmf_msg); 1788 ASSERT(free_res == IBMF_SUCCESS); 1789 1790 goto bail; 1791 } 1792 1793 if (attr_id == SA_MULTIPATHRECORD_ATTRID) { 1794 1795 multipath_template = 1796 (sa_multipath_record_t *)trans_info->si_trans_template; 1797 1798 payload_length = sizeof (sa_multipath_record_t) + 1799 ((multipath_template->SGIDCount + 1800 multipath_template->DGIDCount) * sizeof (ib_gid_t)); 1801 1802 pack_attr_id = attr_id; 1803 } else { 1804 1805 /* trace record template is a path record */ 1806 pack_attr_id = (attr_id == SA_TRACERECORD_ATTRID) ? 1807 SA_PATHRECORD_ATTRID : attr_id; 1808 1809 payload_length = ibmf_saa_impl_get_attr_id_length(pack_attr_id); 1810 1811 if (payload_length == 0) { 1812 payload_length = trans_info->si_trans_template_length; 1813 1814 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 1815 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1816 "ibmf_saa_impl_init_msg: %s, length = %d\n", 1817 tnf_string, msg, 1818 "Unknown attribute. Using user-defined length.", 1819 tnf_uint, length, payload_length) 1820 } 1821 } 1822 1823 /* transport type depends on method */ 1824 switch (method) { 1825 1826 case SA_SUBN_ADM_GET: 1827 case SA_SUBN_ADM_DELETE: 1828 case SA_SUBN_ADM_GET_TABLE: 1829 case SA_SUBN_ADM_GET_TRACE_TABLE: 1830 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ; 1831 break; 1832 case SA_SUBN_ADM_SET: 1833 /* unsubscribes can be sequenced or unsequenced */ 1834 if (trans_info->si_trans_unseq_unsubscribe == B_TRUE) { 1835 transport_flags = 0; 1836 } else { 1837 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ; 1838 } 1839 break; 1840 case SA_SUBN_ADM_GET_MULTI: 1841 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ | 1842 IBMF_MSG_TRANS_FLAG_RMPP; 1843 break; 1844 default : 1845 ibmf_status = IBMF_UNSUPP_METHOD; 1846 goto bail; 1847 } 1848 1849 trans_info->si_trans_transport_flags = transport_flags; 1850 1851 if (trans_info->si_trans_template != NULL) { 1852 1853 ibmf_status = ibmf_saa_utils_pack_payload( 1854 trans_info->si_trans_template, payload_length, pack_attr_id, 1855 &req_mad->im_bufs_cl_data, &req_mad->im_bufs_cl_data_len, 1856 km_sleep_flag); 1857 if (ibmf_status != IBMF_SUCCESS) { 1858 1859 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1860 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1861 "ibmf_saa_impl_init_msg: %s, ibmf_status =" 1862 " %d\n", tnf_string, msg, 1863 "ibmf_saa_utils_pack_payload() failed", 1864 tnf_int, ibmf_status, ibmf_status); 1865 1866 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1867 1868 kmem_free(req_mad->im_bufs_cl_hdr, 1869 req_mad->im_bufs_cl_hdr_len); 1870 1871 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1872 &ibmf_msg); 1873 ASSERT(free_res == IBMF_SUCCESS); 1874 1875 goto bail; 1876 } 1877 1878 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1879 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1880 "ibmf_saa_impl_init_msg: %s, attr_id = 0x%x, length =" 1881 " %d\n", tnf_string, msg, "Packed payload successfully", 1882 tnf_opaque, attr_id, attr_id, 1883 tnf_uint, length, req_mad->im_bufs_cl_data_len); 1884 1885 /* non-RMPP transactions have template size limit */ 1886 if (((transport_flags & IBMF_MSG_TRANS_FLAG_RMPP) == 0) && 1887 ((req_mad->im_bufs_cl_data_len + req_mad->im_bufs_cl_hdr_len 1888 + sizeof (ib_mad_hdr_t)) > IBMF_MAD_SIZE)) { 1889 1890 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1891 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1892 "ibmf_saa_impl_init_msg: %s\n", tnf_string, msg, 1893 "Template too large to fit in single packet"); 1894 1895 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1896 1897 kmem_free(req_mad->im_bufs_cl_hdr, 1898 req_mad->im_bufs_cl_hdr_len); 1899 1900 kmem_free(req_mad->im_bufs_cl_data, 1901 req_mad->im_bufs_cl_data_len); 1902 1903 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1904 &ibmf_msg); 1905 ASSERT(free_res == IBMF_SUCCESS); 1906 1907 ibmf_status = IBMF_REQ_INVALID; 1908 goto bail; 1909 } 1910 } 1911 1912 mutex_enter(&saa_portp->saa_pt_mutex); 1913 1914 mad_hdr->TransactionID = h2b64(saa_portp->saa_pt_current_tid++); 1915 1916 bcopy(&saa_portp->saa_pt_ibmf_retrans, ibmf_retransp, 1917 sizeof (ibmf_retrans_t)); 1918 1919 /* copy local addressing information to message */ 1920 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &ibmf_msg->im_local_addr, 1921 sizeof (ibmf_addr_info_t)); 1922 1923 /* copy global addressing information to message if in use */ 1924 if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) { 1925 1926 ibmf_msg->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 1927 1928 bcopy(&saa_portp->saa_pt_ibmf_global_addr, 1929 &ibmf_msg->im_global_addr, 1930 sizeof (ibmf_global_addr_info_t)); 1931 } else { 1932 ibmf_msg->im_msg_flags = 0; 1933 } 1934 1935 mutex_exit(&saa_portp->saa_pt_mutex); 1936 1937 *msgp = ibmf_msg; 1938 *transport_flagsp = transport_flags; 1939 bail: 1940 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1941 ibmf_saa_impl_init_msg_end, 1942 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() exiting" 1943 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 1944 1945 return (ibmf_status); 1946 1947 } 1948 1949 /* 1950 * ibmf_saa_impl_new_smlid_retry: 1951 * 1952 * It's possible for the MasterSMLID to change while ibmf_saa is running. The 1953 * MasterSMLID is set when we first register with ibmf_saa. If a request 1954 * timesout, this function should be called to check whether the SM LID changed. 1955 * If so, it will call msg_transport again with the request. 1956 * 1957 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 1958 * same values passed to the original ibmf_msg_transport that timedout. The 1959 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 1960 * 1961 * If the lid did not change then this function returns IBMF_TRANS_TIMEOUT. 1962 * That way, callers can simply return the result of this function. 1963 * 1964 * Input Arguments 1965 * saa_portp pointer to saa_port structure 1966 * msgp ibmf message that timedout 1967 * ibmf_callback callback that should be called by msg_transport 1968 * ibmf_callback_arg args for ibmf_callback 1969 * transport_flags flags for ibmf_msg_transport 1970 * 1971 * Output Arguments 1972 * none 1973 * 1974 * Returns 1975 * IBMF_SUCCESS if lid changed and request was resent successfully, 1976 * IBMF_TRANS_TIMEOUT if lid did not change, 1977 * same values as ibmf_msg_transport() if lid changed but request could not be 1978 * resent. 1979 */ 1980 static int 1981 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 1982 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags) 1983 { 1984 ibt_hca_portinfo_t *ibt_portinfop; 1985 ib_lid_t master_sm_lid; 1986 int subnet_timeout; 1987 uint_t nports, size; 1988 ibmf_retrans_t ibmf_retrans; 1989 int ibmf_status; 1990 ibt_status_t ibt_status; 1991 1992 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1993 ibmf_saa_impl_new_smlid_retry_start, 1994 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() enter\n"); 1995 1996 _NOTE(ASSUMING_PROTECTED(*msgp)) 1997 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr)) 1998 1999 /* first query the portinfo to see if the lid changed */ 2000 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid, 2001 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size); 2002 2003 if (ibt_status != IBT_SUCCESS) { 2004 2005 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2006 ibmf_saa_impl_new_smlid_retry_err, IBMF_TNF_ERROR, "", 2007 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status =" 2008 " %d\n", tnf_string, msg, 2009 "ibt_query_hca_ports_byguid() failed", 2010 tnf_int, ibt_status, ibt_status); 2011 2012 ibmf_status = IBMF_TRANSPORT_FAILURE; 2013 2014 goto bail; 2015 } 2016 2017 master_sm_lid = ibt_portinfop->p_sm_lid; 2018 subnet_timeout = ibt_portinfop->p_subnet_timeout; 2019 2020 ibt_free_portinfo(ibt_portinfop, size); 2021 2022 /* if master smlid is different than the remote lid we sent to */ 2023 if (master_sm_lid != msgp->im_local_addr.ia_remote_lid) { 2024 2025 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L2, 2026 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2027 "ibmf_saa_impl_new_smlid_retry: %s, new_lid 0x%x," 2028 " old_lid 0x%x\n", tnf_string, msg, 2029 "master smlid has changed. retrying msg_transport", 2030 tnf_opaque, new_lid, master_sm_lid, 2031 tnf_opaque, old_lid, msgp->im_local_addr.ia_remote_lid); 2032 2033 mutex_enter(&saa_portp->saa_pt_mutex); 2034 2035 /* update the master sm lid value in ibmf_saa */ 2036 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2037 master_sm_lid; 2038 2039 /* new tid needed */ 2040 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 2041 h2b64(saa_portp->saa_pt_current_tid++); 2042 2043 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2044 sizeof (ibmf_retrans_t)); 2045 2046 /* update the subnet timeout since this may be a new sm/sa */ 2047 saa_portp->saa_pt_timeout = subnet_timeout; 2048 2049 /* place upper bound on subnet timeout in case of faulty SM */ 2050 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout) 2051 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout; 2052 2053 /* increment the reference count to account for the cpi call */ 2054 saa_portp->saa_pt_reference_count++; 2055 2056 mutex_exit(&saa_portp->saa_pt_mutex); 2057 2058 /* update the remote lid for this particular message */ 2059 msgp->im_local_addr.ia_remote_lid = master_sm_lid; 2060 2061 /* get the classportinfo again since this may be a new sm/sa */ 2062 ibmf_saa_impl_get_classportinfo(saa_portp); 2063 2064 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 2065 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2066 ibmf_callback, ibmf_callback_arg, transport_flags); 2067 2068 if (ibmf_status != IBMF_SUCCESS) { 2069 2070 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2071 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2072 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status = " 2073 "%d\n", tnf_string, msg, 2074 "ibmf_msg_transport() failed", 2075 tnf_int, ibmf_status, ibmf_status); 2076 } 2077 2078 goto bail; 2079 } 2080 2081 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2082 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2083 "ibmf_saa_impl_new_smlid_retry: %s, master_smlid = 0x%x\n", 2084 tnf_string, msg, 2085 "master smlid did not change. returning failure", 2086 tnf_opaque, master_smlid, master_sm_lid); 2087 2088 /* mark status as timeout since that was original failure */ 2089 ibmf_status = IBMF_TRANS_TIMEOUT; 2090 2091 bail: 2092 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2093 ibmf_saa_impl_new_smlid_retry_end, 2094 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() exiting" 2095 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 2096 2097 return (ibmf_status); 2098 } 2099 2100 /* 2101 * ibmf_saa_impl_revert_to_qp1() 2102 * 2103 * The SA that we had contact with via redirect may fail to respond. If this 2104 * occurs SA should revert back to qp1 and the SMLID set in the port. 2105 * msg_transport for the message that timed out will be retried with 2106 * these new parameters. 2107 * 2108 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 2109 * same values passed to the original ibmf_msg_transport that timedout. The 2110 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 2111 * 2112 * Input Arguments 2113 * saa_portp pointer to saa_port structure 2114 * msgp ibmf message that timedout 2115 * ibmf_callback callback that should be called by msg_transport 2116 * ibmf_callback_arg args for ibmf_callback 2117 * transport_flags flags for ibmf_msg_transport 2118 * 2119 * Output Arguments 2120 * none 2121 * 2122 * Returns 2123 * none 2124 */ 2125 static int 2126 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp, 2127 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags) 2128 { 2129 ibt_hca_portinfo_t *ibt_portinfop; 2130 ib_lid_t master_sm_lid, base_lid; 2131 uint8_t sm_sl; 2132 int subnet_timeout; 2133 uint_t nports, size; 2134 ibmf_retrans_t ibmf_retrans; 2135 int ibmf_status; 2136 ibt_status_t ibt_status; 2137 2138 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 2139 ibmf_saa_impl_revert_to_qp1_start, 2140 IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() enter\n"); 2141 2142 _NOTE(ASSUMING_PROTECTED(*msgp)) 2143 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr)) 2144 2145 /* first query the portinfo to see if the lid changed */ 2146 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid, 2147 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size); 2148 2149 if (ibt_status != IBT_SUCCESS) { 2150 2151 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2152 ibmf_saa_impl_revert_to_qp1_err, IBMF_TNF_ERROR, "", 2153 "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status =" 2154 " %d\n", tnf_string, msg, 2155 "ibt_query_hca_ports_byguid() failed", 2156 tnf_int, ibt_status, ibt_status); 2157 2158 ibmf_status = IBMF_TRANSPORT_FAILURE; 2159 2160 goto bail; 2161 } 2162 2163 master_sm_lid = ibt_portinfop->p_sm_lid; 2164 base_lid = ibt_portinfop->p_base_lid; 2165 sm_sl = ibt_portinfop->p_sm_sl; 2166 subnet_timeout = ibt_portinfop->p_subnet_timeout; 2167 2168 ibt_free_portinfo(ibt_portinfop, size); 2169 2170 2171 mutex_enter(&saa_portp->saa_pt_mutex); 2172 2173 saa_portp->saa_pt_redirect_active = B_FALSE; 2174 2175 /* update the address info in ibmf_saa */ 2176 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = base_lid; 2177 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = master_sm_lid; 2178 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = sm_sl; 2179 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1; 2180 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = IB_PKEY_DEFAULT_LIMITED; 2181 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY; 2182 saa_portp->saa_pt_ibmf_msg_flags = 0; 2183 2184 /* new tid needed */ 2185 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 2186 h2b64(saa_portp->saa_pt_current_tid++); 2187 2188 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2189 sizeof (ibmf_retrans_t)); 2190 2191 /* update the subnet timeout since this may be a new sm/sa */ 2192 saa_portp->saa_pt_timeout = subnet_timeout; 2193 2194 /* place upper bound on subnet timeout in case of faulty SM */ 2195 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout) 2196 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout; 2197 2198 /* increment the reference count to account for the cpi call */ 2199 saa_portp->saa_pt_reference_count++; 2200 2201 mutex_exit(&saa_portp->saa_pt_mutex); 2202 2203 /* update the address info for this particular message */ 2204 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr, 2205 sizeof (ibmf_addr_info_t)); 2206 msgp->im_msg_flags = 0; /* No GRH */ 2207 2208 /* get the classportinfo again since this may be a new sm/sa */ 2209 ibmf_saa_impl_get_classportinfo(saa_portp); 2210 2211 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 2212 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2213 ibmf_callback, ibmf_callback_args, transport_flags); 2214 2215 if (ibmf_status != IBMF_SUCCESS) { 2216 2217 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2218 ibmf_saa_impl_revert_to_qp1, IBMF_TNF_TRACE, "", 2219 "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status = " 2220 "%d\n", tnf_string, msg, 2221 "ibmf_msg_transport() failed", 2222 tnf_int, ibmf_status, ibmf_status); 2223 } 2224 2225 bail: 2226 2227 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2228 ibmf_saa_impl_revert_to_qp1_end, 2229 IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() exiting" 2230 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 2231 2232 return (ibmf_status); 2233 } 2234 2235 /* 2236 * ibmf_saa_impl_async_event_cb: 2237 * ibmf event callback, argument to ibmf_register 2238 * ibmf_handle is unused 2239 */ 2240 /* ARGSUSED */ 2241 static void 2242 ibmf_saa_impl_async_event_cb( 2243 ibmf_handle_t ibmf_handle, 2244 void *clnt_private, 2245 ibmf_async_event_t event_type) 2246 { 2247 saa_port_t *saa_portp; 2248 2249 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2250 ibmf_saa_impl_async_event_cb_start, IBMF_TNF_TRACE, "", 2251 "ibmf_saa_impl_async_event_cb: Handling event type 0x%x\n", 2252 tnf_opaque, event_type, event_type); 2253 2254 saa_portp = (saa_port_t *)clnt_private; 2255 ASSERT(saa_portp != NULL); 2256 2257 switch (event_type) { 2258 2259 case IBMF_CI_OFFLINE: 2260 ibmf_saa_impl_hca_detach(saa_portp); 2261 break; 2262 default: 2263 break; 2264 } 2265 2266 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, 2267 ibmf_saa_impl_async_event_cb_end, 2268 IBMF_TNF_TRACE, "", "ibmf_saa_impl_async_event_cb() exit\n"); 2269 } 2270 2271 2272 /* 2273 * ibmf_saa_impl_ibt_async_handler: 2274 * MUST NOT BE STATIC (referred from within IBMF) 2275 */ 2276 void 2277 ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code, ibt_async_event_t *event) 2278 { 2279 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2280 ibmf_saa_impl_ibt_async_handler_start, IBMF_TNF_TRACE, "", 2281 "ibmf_saa_impl_ibt_async_handler: Handling event code 0x%x\n", 2282 tnf_opaque, code, code); 2283 2284 switch (code) { 2285 2286 case IBT_EVENT_PORT_UP: 2287 ibmf_saa_impl_port_up(event->ev_hca_guid, event->ev_port); 2288 break; 2289 case IBT_ERROR_PORT_DOWN: 2290 ibmf_saa_impl_port_down(event->ev_hca_guid, event->ev_port); 2291 break; 2292 case IBT_PORT_CHANGE_EVENT: 2293 ibmf_saa_impl_port_chg(event); 2294 break; 2295 case IBT_CLNT_REREG_EVENT: 2296 ibmf_saa_impl_client_rereg(event->ev_hca_guid, event->ev_port); 2297 break; 2298 default: 2299 break; 2300 } 2301 2302 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_async_handler_end, 2303 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibt_async_handler() exit\n"); 2304 } 2305 2306 /* 2307 * ibmf_saa_impl_port_chg: 2308 */ 2309 static void 2310 ibmf_saa_impl_port_chg(ibt_async_event_t *event) 2311 { 2312 saa_port_t *saa_portp = NULL; 2313 boolean_t is_ready = B_FALSE; 2314 ibt_hca_portinfo_t *ibt_portinfop; 2315 uint_t nports, size; 2316 ibt_status_t ibt_status; 2317 ib_guid_t ci_guid; 2318 int port_num; 2319 2320 ci_guid = event->ev_hca_guid; 2321 port_num = event->ev_port; 2322 2323 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_chg_start, 2324 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg: Handling port chg" 2325 " guid %016" PRIx64 " port %d\n", 2326 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2327 2328 /* Get classportinfo of corresponding entry */ 2329 mutex_enter(&saa_statep->saa_port_list_mutex); 2330 2331 saa_portp = saa_statep->saa_port_list; 2332 while (saa_portp != NULL) { 2333 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid && 2334 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) { 2335 mutex_enter(&saa_portp->saa_pt_mutex); 2336 2337 is_ready = (saa_portp->saa_pt_state 2338 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE; 2339 2340 /* 2341 * increment reference count to account for cpi and 2342 * informinfos. All 4 informinfo's sent are treated as 2343 * one port client reference 2344 */ 2345 if (is_ready) 2346 saa_portp->saa_pt_reference_count ++; 2347 2348 mutex_exit(&saa_portp->saa_pt_mutex); 2349 2350 if (is_ready) 2351 break; /* normally, only 1 port entry */ 2352 } 2353 saa_portp = saa_portp->next; 2354 } 2355 2356 mutex_exit(&saa_statep->saa_port_list_mutex); 2357 2358 if (saa_portp != NULL) { 2359 /* first query the portinfo to see if the lid changed */ 2360 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num, 2361 &ibt_portinfop, &nports, &size); 2362 2363 if (ibt_status != IBT_SUCCESS) { 2364 2365 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2366 ibmf_saa_impl_port_chg_err, IBMF_TNF_ERROR, "", 2367 "ibmf_saa_impl_port_chg: %s, ibmf_status =" 2368 " %d\n", tnf_string, msg, 2369 "ibt_query_hca_ports_byguid() failed", 2370 tnf_int, ibt_status, ibt_status); 2371 2372 goto bail; 2373 } 2374 2375 mutex_enter(&saa_portp->saa_pt_mutex); 2376 if (event->ev_port_flags & IBT_PORT_CHANGE_SM_LID) { 2377 /* update the Master SM Lid value in ibmf_saa */ 2378 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2379 ibt_portinfop->p_sm_lid; 2380 } 2381 if (event->ev_port_flags & IBT_PORT_CHANGE_SM_SL) { 2382 /* update the Master SM SL value in ibmf_saa */ 2383 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = 2384 ibt_portinfop->p_sm_sl; 2385 } 2386 if (event->ev_port_flags & IBT_PORT_CHANGE_SUB_TIMEOUT) { 2387 /* update the Subnet timeout value in ibmf_saa */ 2388 saa_portp->saa_pt_timeout = 2389 ibt_portinfop->p_subnet_timeout; 2390 } 2391 mutex_exit(&saa_portp->saa_pt_mutex); 2392 2393 ibt_free_portinfo(ibt_portinfop, size); 2394 2395 /* get the classportinfo again */ 2396 ibmf_saa_impl_get_classportinfo(saa_portp); 2397 } 2398 bail: 2399 2400 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_chg_end, 2401 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg() exit\n"); 2402 } 2403 /* 2404 * ibmf_saa_impl_client_rereg: 2405 */ 2406 static void 2407 ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num) 2408 { 2409 saa_port_t *saa_portp = NULL; 2410 boolean_t is_ready = B_FALSE; 2411 ibt_hca_portinfo_t *ibt_portinfop; 2412 ib_lid_t master_sm_lid; 2413 uint_t nports, size; 2414 ibt_status_t ibt_status; 2415 boolean_t event_subs = B_FALSE; 2416 2417 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_rereg_start, 2418 IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg: Handling clnt " 2419 "rereg guid %016" PRIx64 " port %d\n", 2420 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2421 2422 /* Get classportinfo of corresponding entry */ 2423 mutex_enter(&saa_statep->saa_port_list_mutex); 2424 2425 saa_portp = saa_statep->saa_port_list; 2426 while (saa_portp != NULL) { 2427 2428 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid && 2429 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) { 2430 2431 mutex_enter(&saa_portp->saa_pt_mutex); 2432 2433 is_ready = (saa_portp->saa_pt_state 2434 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE; 2435 2436 /* 2437 * increment reference count to account for cpi and 2438 * informinfos. All 4 informinfo's sent are treated as 2439 * one port client reference 2440 */ 2441 if (is_ready) 2442 saa_portp->saa_pt_reference_count += 2; 2443 2444 mutex_exit(&saa_portp->saa_pt_mutex); 2445 2446 if (is_ready) 2447 break; /* normally, only 1 port entry */ 2448 } 2449 saa_portp = saa_portp->next; 2450 } 2451 2452 mutex_exit(&saa_statep->saa_port_list_mutex); 2453 2454 if (saa_portp != NULL && is_ready == B_TRUE) { 2455 2456 /* verify whether master sm lid changed */ 2457 2458 /* first query the portinfo to see if the lid changed */ 2459 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num, 2460 &ibt_portinfop, &nports, &size); 2461 2462 if (ibt_status != IBT_SUCCESS) { 2463 2464 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2465 ibmf_saa_impl_port_rereg_err, IBMF_TNF_ERROR, "", 2466 "ibmf_saa_impl_client_rereg: %s, ibmf_status =" 2467 " %d\n", tnf_string, msg, 2468 "ibt_query_hca_ports_byguid() failed", 2469 tnf_int, ibt_status, ibt_status); 2470 2471 goto bail; 2472 } 2473 2474 master_sm_lid = ibt_portinfop->p_sm_lid; 2475 2476 ibt_free_portinfo(ibt_portinfop, size); 2477 2478 /* check whether we need to subscribe for events */ 2479 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2480 2481 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2482 B_TRUE : B_FALSE; 2483 2484 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2485 2486 /* update the master smlid */ 2487 mutex_enter(&saa_portp->saa_pt_mutex); 2488 2489 /* update the master sm lid value in ibmf_saa */ 2490 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2491 master_sm_lid; 2492 2493 /* if we're not subscribed for events, dec reference count */ 2494 if (event_subs == B_FALSE) 2495 saa_portp->saa_pt_reference_count--; 2496 2497 mutex_exit(&saa_portp->saa_pt_mutex); 2498 2499 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2500 ibmf_saa_impl_port_rereg, IBMF_TNF_TRACE, "", 2501 "ibmf_saa_impl_client_rereg: %s, master_sm_lid = 0x%x\n", 2502 tnf_string, msg, 2503 "port is up. Sending classportinfo request", 2504 tnf_opaque, master_sm_lid, master_sm_lid); 2505 2506 /* get the classportinfo again */ 2507 ibmf_saa_impl_get_classportinfo(saa_portp); 2508 2509 /* 2510 * resubscribe to events if there are subscribers since SA may 2511 * have removed our subscription records when the port went down 2512 */ 2513 if (event_subs == B_TRUE) 2514 ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE); 2515 } 2516 2517 bail: 2518 2519 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_rereg_end, 2520 IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg() exit\n"); 2521 } 2522 /* 2523 * ibmf_saa_impl_port_up: 2524 */ 2525 static void 2526 ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num) 2527 { 2528 saa_port_t *saa_portp = NULL; 2529 int is_ready; 2530 ibt_hca_portinfo_t *ibt_portinfop; 2531 ib_lid_t master_sm_lid; 2532 uint_t nports, size; 2533 ibt_status_t ibt_status; 2534 boolean_t event_subs = B_FALSE; 2535 2536 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_up_start, 2537 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up: Handling port up" 2538 " guid %016" PRIx64 " port %d\n", 2539 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2540 2541 /* Get classportinfo of corresponding entry */ 2542 mutex_enter(&saa_statep->saa_port_list_mutex); 2543 2544 saa_portp = saa_statep->saa_port_list; 2545 while (saa_portp != NULL) { 2546 2547 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid && 2548 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) { 2549 2550 mutex_enter(&saa_portp->saa_pt_mutex); 2551 2552 is_ready = (saa_portp->saa_pt_state 2553 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE; 2554 2555 /* 2556 * increment reference count to account for cpi and 2557 * informinfos. All 4 informinfo's sent are treated as 2558 * one port client reference 2559 */ 2560 if (is_ready == B_TRUE) 2561 saa_portp->saa_pt_reference_count += 2; 2562 2563 mutex_exit(&saa_portp->saa_pt_mutex); 2564 2565 if (is_ready == B_TRUE) 2566 break; /* normally, only 1 port entry */ 2567 } 2568 saa_portp = saa_portp->next; 2569 } 2570 2571 mutex_exit(&saa_statep->saa_port_list_mutex); 2572 2573 if (saa_portp != NULL && is_ready == B_TRUE) { 2574 2575 /* verify whether master sm lid changed */ 2576 2577 /* first query the portinfo to see if the lid changed */ 2578 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num, 2579 &ibt_portinfop, &nports, &size); 2580 2581 if (ibt_status != IBT_SUCCESS) { 2582 2583 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2584 ibmf_saa_impl_port_up_err, IBMF_TNF_ERROR, "", 2585 "ibmf_saa_impl_port_up: %s, ibmf_status =" 2586 " %d\n", tnf_string, msg, 2587 "ibt_query_hca_ports_byguid() failed", 2588 tnf_int, ibt_status, ibt_status); 2589 2590 goto bail; 2591 } 2592 2593 master_sm_lid = ibt_portinfop->p_sm_lid; 2594 2595 ibt_free_portinfo(ibt_portinfop, size); 2596 2597 /* check whether we need to subscribe for events */ 2598 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2599 2600 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2601 B_TRUE : B_FALSE; 2602 2603 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2604 2605 /* update the master smlid */ 2606 mutex_enter(&saa_portp->saa_pt_mutex); 2607 2608 /* update the master sm lid value in ibmf_saa */ 2609 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2610 master_sm_lid; 2611 2612 /* if we're not subscribed for events, dec reference count */ 2613 if (event_subs == B_FALSE) 2614 saa_portp->saa_pt_reference_count--; 2615 2616 mutex_exit(&saa_portp->saa_pt_mutex); 2617 2618 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2619 ibmf_saa_impl_port_up, IBMF_TNF_TRACE, "", 2620 "ibmf_saa_impl_port_up: %s, master_sm_lid = 0x%x\n", 2621 tnf_string, msg, 2622 "port is up. Sending classportinfo request", 2623 tnf_opaque, master_sm_lid, master_sm_lid); 2624 2625 /* get the classportinfo again */ 2626 ibmf_saa_impl_get_classportinfo(saa_portp); 2627 2628 /* 2629 * resubscribe to events if there are subscribers since SA may 2630 * have removed our subscription records when the port went down 2631 */ 2632 if (event_subs == B_TRUE) 2633 ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE); 2634 } 2635 2636 bail: 2637 2638 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_up_end, 2639 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up() exit\n"); 2640 } 2641 2642 /* 2643 * ibmf_saa_impl_port_down: 2644 */ 2645 static void 2646 ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num) 2647 { 2648 2649 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_down_start, 2650 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down: Handling port down" 2651 " guid %016" PRIx64 " port %d\n", 2652 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2653 2654 2655 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_down_end, 2656 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down() exit\n"); 2657 } 2658 2659 /* 2660 * ibmf_saa_impl_hca_detach: 2661 * find entry, unregister if there are no clients 2662 * have to unregister since ibmf needs to close the hca and will only do this if 2663 * no clients are registered 2664 */ 2665 static void 2666 ibmf_saa_impl_hca_detach(saa_port_t *saa_removed) 2667 { 2668 saa_port_t *saa_portp; 2669 boolean_t must_unreg, must_unsub; 2670 2671 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_start, 2672 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: Detaching" 2673 " entry %016" PRIx64 "\n", tnf_opaque, entry, saa_removed); 2674 2675 /* find this entry */ 2676 mutex_enter(&saa_statep->saa_port_list_mutex); 2677 2678 saa_portp = saa_statep->saa_port_list; 2679 while (saa_portp != NULL) { 2680 2681 if (saa_portp == saa_removed) 2682 break; 2683 2684 saa_portp = saa_portp->next; 2685 } 2686 mutex_exit(&saa_statep->saa_port_list_mutex); 2687 2688 ASSERT(saa_portp != NULL); 2689 2690 if (saa_portp == NULL) { 2691 2692 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 2693 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2694 "ibmf_saa_impl_hca_detach: %s, entry %016" 2695 PRIx64 "\n", 2696 tnf_string, msg, 2697 "Port entry NOT found", 2698 tnf_opaque, entryp, saa_removed); 2699 2700 goto bail; 2701 } 2702 2703 /* if there are clients expecting Reports(), unsusbscribe */ 2704 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2705 2706 must_unsub = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2707 B_TRUE : B_FALSE; 2708 2709 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2710 2711 /* fail if outstanding transactions */ 2712 mutex_enter(&saa_portp->saa_pt_mutex); 2713 2714 if (saa_portp->saa_pt_num_outstanding_trans > 0) { 2715 2716 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L1, 2717 ibmf_saa_impl_fini_err, IBMF_TNF_TRACE, "", 2718 "ibmf_saa_impl_fini: %s, port = %016" PRIx64 2719 ", num transactions = %d\n", 2720 tnf_string, msg, "Detaching HCA." 2721 " Outstanding transactions on port.", 2722 tnf_opaque, port, 2723 saa_portp->saa_pt_port_guid, 2724 tnf_uint, outstanding_transactions, 2725 saa_portp->saa_pt_num_outstanding_trans); 2726 2727 mutex_exit(&saa_portp->saa_pt_mutex); 2728 2729 goto bail; 2730 } 2731 2732 2733 /* 2734 * increment reference count by one to account for unsubscribe requests 2735 * that are about to be sent. All four informinfo's are treated as one 2736 * port client reference. The count will be decremented by 2737 * subscribe_events() before the call returns. 2738 */ 2739 if (must_unsub == B_TRUE) 2740 saa_portp->saa_pt_reference_count++; 2741 2742 mutex_exit(&saa_portp->saa_pt_mutex); 2743 2744 /* 2745 * try and unsubscribe from SA. Generate synchronous, unsequenced 2746 * unsubscribe requests. 2747 */ 2748 if (must_unsub == B_TRUE) 2749 ibmf_saa_subscribe_events(saa_portp, B_FALSE, B_TRUE); 2750 2751 /* warning if registered clients */ 2752 mutex_enter(&saa_portp->saa_pt_mutex); 2753 2754 if (saa_portp->saa_pt_reference_count > 0) { 2755 2756 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 2757 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2758 "ibmf_saa_impl_hca_detach: %s, port %016" 2759 PRIx64 "\n", 2760 tnf_string, msg, 2761 "Detaching HCA for port with clients still" 2762 " registered", tnf_opaque, port, 2763 saa_portp->saa_pt_port_guid); 2764 } 2765 2766 /* synchronize on end of registration */ 2767 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) { 2768 2769 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1, 2770 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2771 "ibmf_saa_impl_hca_detach: %s\n", 2772 tnf_string, msg, "someone is registering. waiting" 2773 " for them to finish"); 2774 2775 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv, 2776 &saa_portp->saa_pt_mutex); 2777 2778 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1, 2779 ibmf_saa_impl_hca_detach, 2780 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: %s\n", 2781 tnf_string, msg, "done waiting"); 2782 } 2783 2784 /* unregister from ibmf */ 2785 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) { 2786 must_unreg = B_TRUE; 2787 } else 2788 must_unreg = B_FALSE; 2789 2790 ibmf_saa_impl_invalidate_port(saa_portp); 2791 2792 mutex_exit(&saa_portp->saa_pt_mutex); 2793 2794 if (must_unreg == B_TRUE) { 2795 if (ibmf_saa_impl_ibmf_unreg(saa_portp) != IBMF_SUCCESS) { 2796 mutex_enter(&saa_portp->saa_pt_mutex); 2797 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 2798 (void) ibmf_saa_impl_init_kstats(saa_portp); 2799 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 2800 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_READY; 2801 if (must_unsub == B_TRUE) 2802 saa_portp->saa_pt_reference_count++; 2803 mutex_exit(&saa_portp->saa_pt_mutex); 2804 2805 if (must_unsub == B_TRUE) { 2806 ibmf_saa_subscribe_events(saa_portp, B_TRUE, 2807 B_FALSE); 2808 } 2809 } 2810 } 2811 bail: 2812 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_end, 2813 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach() exit\n"); 2814 } 2815 2816 /* ARGSUSED */ 2817 void 2818 ibmf_saa_async_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, void *args) 2819 { 2820 saa_impl_trans_info_t *trans_info; 2821 int status; 2822 size_t length; 2823 void *result; 2824 saa_port_t *saa_portp; 2825 saa_client_data_t *client_data; 2826 int ibmf_status; 2827 boolean_t ignore_data; 2828 ibmf_retrans_t ibmf_retrans; 2829 boolean_t sa_is_redirected = B_FALSE; 2830 2831 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_start, 2832 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() enter\n"); 2833 2834 trans_info = (saa_impl_trans_info_t *)args; 2835 2836 client_data = trans_info->si_trans_client_data; 2837 saa_portp = trans_info->si_trans_port; 2838 2839 mutex_enter(&saa_portp->saa_pt_mutex); 2840 sa_is_redirected = saa_portp->saa_pt_redirect_active; 2841 mutex_exit(&saa_portp->saa_pt_mutex); 2842 2843 if ((msgp->im_msg_status == IBMF_TRANS_TIMEOUT) && 2844 (sa_is_redirected == B_TRUE)) { 2845 2846 /* 2847 * We should retry the request using SM_LID and QP1 if we 2848 * have been using redirect up until now 2849 */ 2850 ibmf_status = ibmf_saa_impl_revert_to_qp1( 2851 saa_portp, msgp, ibmf_saa_async_cb, args, 2852 trans_info->si_trans_transport_flags); 2853 2854 /* 2855 * If revert_to_qp1 returns success msg was resent. 2856 * Otherwise msg could not be resent. Continue normally 2857 */ 2858 if (ibmf_status == IBMF_SUCCESS) 2859 goto bail; 2860 2861 } else if (msgp->im_msg_status == IBMF_TRANS_TIMEOUT) { 2862 2863 2864 ibmf_status = ibmf_saa_impl_new_smlid_retry(saa_portp, msgp, 2865 ibmf_saa_async_cb, args, 2866 trans_info->si_trans_transport_flags); 2867 2868 /* 2869 * if smlid_retry() returns success sm lid changed and msg 2870 * was resent. Otherwise, lid did not change or msg could not 2871 * be resent. Continue normally. 2872 */ 2873 if (ibmf_status == IBMF_SUCCESS) 2874 goto bail; 2875 2876 /* 2877 * check whether we've received anything from the SA in a while. 2878 * If we have, this function will retry and return success. If 2879 * we haven't continue normally so that we return a timeout to 2880 * the client 2881 */ 2882 ibmf_status = ibmf_saa_check_sa_and_retry( 2883 saa_portp, msgp, ibmf_saa_async_cb, args, 2884 trans_info->si_trans_send_time, 2885 trans_info->si_trans_transport_flags); 2886 2887 if (ibmf_status == IBMF_SUCCESS) 2888 goto bail; 2889 } 2890 2891 /* 2892 * If SA returned success but mad status is busy, retry a few times. 2893 * If SA returned success but mad status says redirect is required, 2894 * update the address info and retry the request to the new SA address 2895 */ 2896 if (msgp->im_msg_status == IBMF_SUCCESS) { 2897 2898 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL); 2899 2900 if ((b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) == 2901 MAD_STATUS_BUSY) && 2902 (trans_info->si_trans_retry_busy_count < 2903 IBMF_SAA_MAX_BUSY_RETRY_COUNT)) { 2904 2905 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2906 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2907 "ibmf_saa_async_cb: %s, retry_count = %d\n", 2908 tnf_string, msg, 2909 "async response returned busy status", 2910 tnf_int, retry_count, 2911 trans_info->si_trans_retry_busy_count); 2912 2913 trans_info->si_trans_retry_busy_count++; 2914 2915 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2916 sizeof (ibmf_retrans_t)); 2917 2918 ibmf_status = ibmf_msg_transport( 2919 saa_portp->saa_pt_ibmf_handle, 2920 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2921 ibmf_saa_async_cb, args, 2922 trans_info->si_trans_transport_flags); 2923 2924 /* 2925 * if retry is successful, quit here since async_cb will 2926 * get called again; otherwise, let this function call 2927 * handle the cleanup 2928 */ 2929 if (ibmf_status == IBMF_SUCCESS) 2930 goto bail; 2931 } else if (b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) 2932 == MAD_STATUS_REDIRECT_REQUIRED) { 2933 2934 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L2, 2935 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2936 "ibmf_saa_async_cb: " 2937 "async response returned redirect status\n"); 2938 2939 /* update address info and copy it into msgp */ 2940 ibmf_saa_impl_update_sa_address_info(saa_portp, msgp); 2941 2942 /* retry with new address info */ 2943 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2944 sizeof (ibmf_retrans_t)); 2945 2946 ibmf_status = ibmf_msg_transport( 2947 saa_portp->saa_pt_ibmf_handle, 2948 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2949 ibmf_saa_async_cb, args, 2950 trans_info->si_trans_transport_flags); 2951 2952 /* 2953 * if retry is successful, quit here since async_cb will 2954 * get called again; otherwise, let this function call 2955 * handle the cleanup 2956 */ 2957 if (ibmf_status == IBMF_SUCCESS) 2958 goto bail; 2959 } 2960 } 2961 2962 mutex_enter(&saa_portp->saa_pt_mutex); 2963 2964 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 2965 saa_portp->saa_pt_num_outstanding_trans--; 2966 2967 mutex_exit(&saa_portp->saa_pt_mutex); 2968 2969 if ((trans_info->si_trans_callback == NULL) && 2970 (trans_info->si_trans_sub_callback == NULL)) 2971 ignore_data = B_TRUE; 2972 else 2973 ignore_data = B_FALSE; 2974 2975 ibmf_saa_impl_prepare_response(ibmf_handle, msgp, ignore_data, &status, 2976 &result, &length, B_FALSE); 2977 2978 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 2979 2980 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1); 2981 2982 if (status != IBMF_SUCCESS) 2983 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1); 2984 2985 if (status == IBMF_TRANS_TIMEOUT) 2986 IBMF_SAA_ADD32_KSTATS(saa_portp, requests_timedout, 1); 2987 2988 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 2989 2990 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2991 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2992 "ibmf_saa_async_cb: %s\n", tnf_string, msg, 2993 "Calling ibmf_saa client's callback"); 2994 2995 /* 2996 * there are three classes or trans_info users: ibmf_saa clients and 2997 * classportinfo requests; informinfo subscribe requests, and report 2998 * responses. For the first two, call the correct callback. For report 2999 * responses there's no need to notify anyone. 3000 */ 3001 if (trans_info->si_trans_callback != NULL) { 3002 /* ibmf_saa client or classportinfo request */ 3003 trans_info->si_trans_callback(trans_info->si_trans_callback_arg, 3004 length, result, status); 3005 } else if (trans_info->si_trans_sub_callback != NULL) { 3006 /* informinfo subscribe request */ 3007 trans_info->si_trans_sub_callback( 3008 trans_info->si_trans_callback_arg, length, result, status, 3009 trans_info->si_trans_sub_producer_type); 3010 } 3011 3012 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3013 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 3014 "ibmf_saa_async_cb: %s\n", tnf_string, msg, 3015 "Returned from callback"); 3016 3017 if (client_data != NULL) { 3018 mutex_enter(&client_data->saa_client_mutex); 3019 3020 ASSERT(client_data->saa_client_num_pending_trans > 0); 3021 client_data->saa_client_num_pending_trans--; 3022 3023 if ((client_data->saa_client_num_pending_trans == 0) && 3024 (client_data->saa_client_state == SAA_CLIENT_STATE_WAITING)) 3025 cv_signal(&client_data->saa_client_state_cv); 3026 3027 mutex_exit(&client_data->saa_client_mutex); 3028 } 3029 3030 kmem_free(trans_info, sizeof (saa_impl_trans_info_t)); 3031 3032 bail: 3033 3034 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_end, 3035 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() exit\n"); 3036 } 3037 3038 /* 3039 * ibmf_saa_check_sa_and_retry: 3040 * 3041 * If a particular transaction times out, we don't want to give up if we know 3042 * the SA is responding. Check the time since we last received a response. If 3043 * it's less than ibmf_saa_max_wait_time retry the request. 3044 * 3045 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 3046 * same values passed to the original ibmf_msg_transport that timed out. The 3047 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 3048 * 3049 * If max_wait_time seconds have passed, this function returns IBMF_TIMEOUT. 3050 * That way, callers can simply return the result of this function. 3051 * 3052 * Input Arguments 3053 * saa_portp pointer to saa_port structure 3054 * msgp ibmf message that timedout 3055 * ibmf_callback callback that should be called by msg_transport 3056 * ibmf_callback_arg args for ibmf_callback 3057 * transport_flags flags for ibmf_msg_transport 3058 * 3059 * Output Arguments 3060 * none 3061 * 3062 * Returns 3063 * IBMF_SUCCESS if we've recently received data from the SA and request was 3064 * resent. 3065 * IBMF_TRANS_TIMEOUT if no data has been received from the SA in max_wait_time 3066 * same values as ibmf_msg_transport() if data has been received but request 3067 * could not be resent. 3068 */ 3069 static int 3070 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 3071 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, 3072 hrtime_t trans_send_time, int transport_flags) 3073 { 3074 hrtime_t curr_time, sa_uptime; 3075 ibmf_retrans_t ibmf_retrans; 3076 int ibmf_status; 3077 3078 do { 3079 3080 mutex_enter(&saa_portp->saa_pt_mutex); 3081 3082 sa_uptime = saa_portp->saa_pt_sa_uptime; 3083 3084 /* if nothing received from SA since we sent */ 3085 curr_time = gethrtime(); 3086 3087 /* 3088 * check if it's been a very long time since this 3089 * particular transaction was sent 3090 */ 3091 if (((curr_time - trans_send_time) / 1000000000) > 3092 ibmf_saa_trans_wait_time) { 3093 3094 mutex_exit(&saa_portp->saa_pt_mutex); 3095 3096 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1, 3097 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "", 3098 "ibmf_saa_check_sa_and_retry: %s, msgp = " 3099 "%p sa_uptime = %" PRIu64 ", trans send time = %" 3100 PRIu64 ", curr_time = %" PRIu64 "\n", 3101 tnf_string, msg, 3102 "Nothing received for this transaction", 3103 tnf_opaque, msgp, msgp, 3104 tnf_long, sa_uptime, sa_uptime, 3105 tnf_long, trans_send_time, trans_send_time, 3106 tnf_long, curr_time, curr_time); 3107 3108 ibmf_status = IBMF_TRANS_TIMEOUT; 3109 3110 break; 3111 } 3112 3113 /* 3114 * check time since we received something, 3115 * and make sure that it hasn't been an extra long 3116 * time for this particular transaction 3117 */ 3118 if (((curr_time - sa_uptime) / 1000000000) < 3119 ibmf_saa_max_wait_time) { 3120 3121 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L2, 3122 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "", 3123 "ibmf_saa_check_sa_and_retry: %s, msgp = " 3124 "%p sa_uptime = %" PRIu64 " trans_send_time = %" 3125 PRIu64 " curr_time = %" PRIu64 "\n", 3126 tnf_string, msg, "Something received. Retrying", 3127 tnf_opaque, msgp, msgp, 3128 tnf_long, sa_uptime, sa_uptime, 3129 tnf_long, trans_send_time, trans_send_time, 3130 tnf_long, curr_time, curr_time); 3131 3132 /* 3133 * something received in WAIT_TIME_IN_SECS; 3134 * resend request 3135 */ 3136 3137 /* new tid needed */ 3138 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 3139 h2b64(saa_portp->saa_pt_current_tid++); 3140 3141 /* 3142 * We are going to retry the access to the SM but 3143 * Master SMLID could have changed due to a port change 3144 * event. So update the remote_lid of the message with 3145 * the SMLID from saa_portp for this port before the 3146 * retry. 3147 */ 3148 msgp->im_local_addr.ia_remote_lid = 3149 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid; 3150 3151 bcopy(&saa_portp->saa_pt_ibmf_retrans, 3152 &ibmf_retrans, sizeof (ibmf_retrans_t)); 3153 3154 mutex_exit(&saa_portp->saa_pt_mutex); 3155 3156 ibmf_status = ibmf_msg_transport( 3157 saa_portp->saa_pt_ibmf_handle, 3158 saa_portp->saa_pt_qp_handle, msgp, 3159 &ibmf_retrans, ibmf_callback, ibmf_callback_arg, 3160 transport_flags); 3161 3162 if (ibmf_status == IBMF_SUCCESS) 3163 goto bail; 3164 3165 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 3166 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "", 3167 "ibmf_saa_check_sa_and_retry: %s, ibmf_status = " 3168 "%d\n", tnf_string, msg, 3169 "ibmf_msg_transport() failed", 3170 tnf_int, ibmf_status, ibmf_status); 3171 } else { 3172 3173 mutex_exit(&saa_portp->saa_pt_mutex); 3174 3175 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L1, 3176 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "", 3177 "ibmf_saa_check_sa_and_retry: %s, msgp = " 3178 "%p sa_uptime = %" PRIu64 " curr_time = %" 3179 PRIu64 "\n", tnf_string, msg, 3180 "Nothing received. Timing out", 3181 tnf_opaque, msgp, msgp, 3182 tnf_long, sa_uptime, sa_uptime, 3183 tnf_long, curr_time, curr_time); 3184 3185 ibmf_status = IBMF_TRANS_TIMEOUT; 3186 3187 break; 3188 } 3189 } while (ibmf_status == IBMF_TRANS_TIMEOUT); 3190 3191 bail: 3192 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3193 ibmf_saa_check_sa_and_retry_end, 3194 IBMF_TNF_TRACE, "", "ibmf_saa_check_sa_and_retry() exiting" 3195 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 3196 3197 return (ibmf_status); 3198 } 3199 3200 3201 /* 3202 * ibmf_saa_impl_prepare_response: 3203 */ 3204 static void 3205 ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle, 3206 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result, 3207 size_t *length, boolean_t sleep_flag) 3208 { 3209 ibmf_msg_bufs_t *resp_buf; 3210 uint16_t attr_id; 3211 uint8_t method; 3212 boolean_t is_get_resp; 3213 uint16_t mad_status; 3214 uint16_t attr_offset; 3215 ib_sa_hdr_t *sa_hdr; 3216 3217 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3218 ibmf_saa_impl_prepare_response_start, 3219 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() enter\n"); 3220 3221 _NOTE(ASSUMING_PROTECTED(*msgp)) 3222 3223 *result = NULL; 3224 *length = 0; 3225 sa_hdr = NULL; 3226 3227 resp_buf = &msgp->im_msgbufs_recv; 3228 3229 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*resp_buf)) 3230 3231 if (msgp->im_msg_status != IBMF_SUCCESS) { 3232 3233 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 3234 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3235 "ibmf_saa_impl_prepare_response: %s, msg_status = %d\n", 3236 tnf_string, msg, "Bad ibmf status", 3237 tnf_int, msg_status, msgp->im_msg_status); 3238 3239 *status = msgp->im_msg_status; 3240 3241 goto exit; 3242 } 3243 3244 if (resp_buf->im_bufs_mad_hdr == NULL) { 3245 3246 /* 3247 * this was an unsequenced transaction (from an unsubscribe for 3248 * following a CI_OFFLINE event) 3249 */ 3250 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3251 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3252 "ibmf_saa_impl_prepare_response: %s\n", 3253 tnf_string, msg, "Unsequenced transaction callback"); 3254 3255 goto exit; 3256 } 3257 3258 if ((mad_status = b2h16(resp_buf->im_bufs_mad_hdr->Status)) != 3259 MAD_STATUS_NO_INVALID_FIELDS) { 3260 3261 /* convert mad packet status to IBMF status */ 3262 switch (mad_status) { 3263 3264 case SA_STATUS_ERR_NO_RESOURCES: 3265 *status = IBMF_NO_RESOURCES; 3266 break; 3267 case SA_STATUS_ERR_REQ_INVALID: 3268 *status = IBMF_REQ_INVALID; 3269 break; 3270 case SA_STATUS_ERR_NO_RECORDS: 3271 *status = IBMF_NO_RECORDS; 3272 break; 3273 case SA_STATUS_ERR_TOO_MANY_RECORDS: 3274 *status = IBMF_TOO_MANY_RECORDS; 3275 break; 3276 case SA_STATUS_ERR_REQ_INVALID_GID: 3277 *status = IBMF_INVALID_GID; 3278 break; 3279 case SA_STATUS_ERR_REQ_INSUFFICIENT_COMPONENTS: 3280 *status = IBMF_INSUFF_COMPS; 3281 break; 3282 case MAD_STATUS_UNSUPP_METHOD: 3283 *status = IBMF_UNSUPP_METHOD; 3284 break; 3285 case MAD_STATUS_UNSUPP_METHOD_ATTR: 3286 *status = IBMF_UNSUPP_METHOD_ATTR; 3287 break; 3288 case MAD_STATUS_INVALID_FIELD: 3289 *status = IBMF_INVALID_FIELD; 3290 break; 3291 default: 3292 *status = IBMF_REQ_INVALID; 3293 break; 3294 } 3295 3296 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 3297 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3298 "ibmf_saa_impl_prepare_response: %s, mad_status = %x\n", 3299 tnf_string, msg, "Bad MAD status", 3300 tnf_int, mad_status, mad_status); 3301 3302 goto exit; 3303 } 3304 3305 attr_id = b2h16(resp_buf->im_bufs_mad_hdr->AttributeID); 3306 method = resp_buf->im_bufs_mad_hdr->R_Method; 3307 3308 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3309 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3310 "ibmf_saa_impl_prepare_response: attr_id = 0x%x, method = " 3311 "0x%x\n", 3312 tnf_opaque, attr_id, attr_id, 3313 tnf_opaque, method, method); 3314 3315 /* 3316 * ignore any data from deleteresp since there's no way to know whether 3317 * real data was returned; also ignore data if this was a Report 3318 * response 3319 */ 3320 if (method == SA_SUBN_ADM_DELETE_RESP) { 3321 3322 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3323 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3324 "impf_saa_impl_prepare_response: %s\n", 3325 tnf_string, msg, 3326 "DeleteResp or NoticeResp returned. " 3327 "Ignoring response data"); 3328 3329 *status = IBMF_SUCCESS; 3330 3331 *length = 0; 3332 *result = NULL; 3333 3334 goto exit; 3335 } 3336 3337 if (attr_id == SA_MULTIPATHRECORD_ATTRID) { 3338 3339 /* 3340 * getmulti is only for requests; attribute should not 3341 * be returned from SA 3342 */ 3343 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 3344 ibmf_saa_impl_prepare_response_err, IBMF_TNF_ERROR, 3345 "", "ibmf_saa_impl_prepare_response: %s\n", 3346 tnf_string, msg, "SA returned getmulti record"); 3347 3348 *status = IBMF_REQ_INVALID; 3349 3350 goto exit; 3351 } 3352 3353 /* if we are supposed to ignore data, stop here */ 3354 if (ignore_data == B_TRUE) { 3355 3356 *status = IBMF_SUCCESS; 3357 3358 goto exit; 3359 } 3360 3361 is_get_resp = resp_buf->im_bufs_mad_hdr->R_Method == 3362 SA_SUBN_ADM_GET_RESP ? B_TRUE: B_FALSE; 3363 3364 /* unpack the sa header to get the attribute offset */ 3365 *status = ibmf_saa_utils_unpack_sa_hdr(resp_buf->im_bufs_cl_hdr, 3366 resp_buf->im_bufs_cl_hdr_len, &sa_hdr, sleep_flag); 3367 if (*status != IBMF_SUCCESS) { 3368 3369 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3370 ibmf_saa_impl_prepare_response_err, 3371 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s," 3372 " ibmf_status = %d\n", tnf_string, msg, 3373 "Could not unpack sa hdr", tnf_int, ibmf_status, *status); 3374 3375 goto exit; 3376 } 3377 3378 attr_offset = sa_hdr->AttributeOffset; 3379 3380 /* 3381 * unpack data payload; if unpack function doesn't return success 3382 * (because it could not allocate memory) forward this status to waiting 3383 * client 3384 */ 3385 *status = ibmf_saa_utils_unpack_payload(resp_buf->im_bufs_cl_data, 3386 resp_buf->im_bufs_cl_data_len, attr_id, result, length, 3387 attr_offset, is_get_resp, sleep_flag); 3388 if (*status == IBMF_SUCCESS) { 3389 3390 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 3391 ibmf_saa_impl_prepare_response, 3392 IBMF_TNF_TRACE, "", 3393 "ibmf_saa_impl_prepare_response: attr_id = " 3394 "0x%x, attr_offset = %d, packed_payload_len = %d, " 3395 "unpacked_payload_len = %d\n", 3396 tnf_opaque, attr_id, attr_id, 3397 tnf_opaque, attr_offset, attr_offset, 3398 tnf_opaque, packed_payload_len, 3399 resp_buf->im_bufs_cl_data_len, 3400 tnf_opaque, unpacked_payload_len, *length); 3401 } else { 3402 3403 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1, 3404 ibmf_saa_impl_prepare_response_err, 3405 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s," 3406 "attr_id = 0x%x, attr_offset = %d, packed_payload_len = %d," 3407 "status = %d\n", 3408 tnf_string, msg, "Could not unpack payload", 3409 tnf_opaque, attr_id, attr_id, 3410 tnf_int, attr_offset, attr_offset, 3411 tnf_int, packed_payload_len, 3412 resp_buf->im_bufs_cl_data_len, 3413 tnf_int, status, *status); 3414 } 3415 exit: 3416 if (sa_hdr != NULL) 3417 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3418 3419 ibmf_saa_impl_free_msg(ibmf_handle, msgp); 3420 3421 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3422 ibmf_saa_impl_prepare_response_end, 3423 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() exit," 3424 " status = 0x%d\n", tnf_int, status, *status); 3425 } 3426 3427 3428 /* 3429 * ibmf_saa_impl_check_sa_support: 3430 * Checks the capability mask (returned from the SA classportinfo response) to 3431 * determine whether the sa supports the specified attribute ID. 3432 * 3433 * Input Arguments 3434 * cap_mask 16-bit capability mask returned in SA's classportinfo 3435 * attr_id attribute ID of current request 3436 * 3437 * Returns 3438 * IBMF_NOT_SUPPORTED if capability mask indicates SA does not support attribute 3439 * IBMF_SUCCESS otherwise 3440 */ 3441 static int 3442 ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id) 3443 { 3444 boolean_t attr_supported = B_TRUE; 3445 3446 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3447 ibmf_saa_impl_check_sa_support, IBMF_TNF_TRACE, "", 3448 "ibmf_saa_impl_check_sa_support: cap_mask = 0x%x, " 3449 "attr_id = 0x%x\n", tnf_opaque, cap_mask, cap_mask, 3450 tnf_opaque, attr_id, attr_id); 3451 3452 switch (attr_id) { 3453 3454 case SA_SWITCHINFORECORD_ATTRID: 3455 case SA_LINEARFDBRECORD_ATTRID: 3456 case SA_RANDOMFDBRECORD_ATTRID: 3457 case SA_MULTICASTFDBRECORD_ATTRID: 3458 case SA_SMINFORECORD_ATTRID: 3459 case SA_INFORMINFORECORD_ATTRID: 3460 case SA_LINKRECORD_ATTRID: 3461 case SA_GUIDINFORECORD_ATTRID: 3462 case SA_TRACERECORD_ATTRID: 3463 case SA_SERVICEASSNRECORD_ATTRID: 3464 3465 if ((cap_mask & 3466 SA_CAPMASK_OPT_RECORDS_SUPPORTED) == 0) { 3467 3468 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1, 3469 ibmf_saa_impl_check_sa_support, 3470 IBMF_TNF_ERROR, "", 3471 "ibmf_saa_impl_check_sa_support: %s, " 3472 "cap_mask = 0x%x\n", tnf_string, msg, 3473 "SA does not support optional records", 3474 tnf_opaque, cap_mask, cap_mask, 3475 tnf_opaque, attr_id, attr_id); 3476 3477 attr_supported = B_FALSE; 3478 } 3479 break; 3480 3481 case SA_MULTIPATHRECORD_ATTRID: 3482 3483 if ((cap_mask & SA_CAPMASK_MULTIPATH_SUPPORTED) == 0) { 3484 3485 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 3486 ibmf_saa_impl_check_sa_support, 3487 IBMF_TNF_ERROR, "", 3488 "ibmf_saa_impl_check_sa_support: %s, " 3489 "cap_mask = 0x%x\n", tnf_string, msg, 3490 "SA does not support multipath records", 3491 tnf_opaque, cap_mask, cap_mask); 3492 3493 attr_supported = B_FALSE; 3494 } 3495 break; 3496 3497 case SA_MCMEMBERRECORD_ATTRID: 3498 3499 if ((cap_mask & SA_CAPMASK_UD_MCAST_SUPPORTED) == 0) { 3500 3501 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 3502 ibmf_saa_impl_check_sa_support, 3503 IBMF_TNF_ERROR, "", 3504 "ibmf_saa_impl_check_sa_support: %s, " 3505 "cap_mask = 0x%x\n", tnf_string, msg, 3506 "SA does not support ud multicast", 3507 tnf_opaque, cap_mask, cap_mask); 3508 3509 attr_supported = B_FALSE; 3510 } 3511 break; 3512 3513 default: 3514 break; 3515 } /* switch */ 3516 3517 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3518 ibmf_saa_impl_check_sa_support_end, IBMF_TNF_TRACE, "", 3519 "ibmf_saa_impl_check_sa_support() exiting, attr_supported = %d\n", 3520 tnf_opaque, attr_supported, attr_supported); 3521 3522 if (attr_supported == B_FALSE) 3523 return (IBMF_UNSUPP_METHOD_ATTR); 3524 else 3525 return (IBMF_SUCCESS); 3526 } 3527 3528 /* 3529 * ibmf_saa_impl_get_attr_id_length: 3530 * 3531 * Returns the host size of the specified sa record. Returns 0 for unknown 3532 * attributes. multipath record size is a dynamic value given as a parameter 3533 * specified with the ibmf_sa_access() call. 3534 */ 3535 static uint_t 3536 ibmf_saa_impl_get_attr_id_length(uint16_t attr_id) 3537 { 3538 uint_t attr_length; 3539 3540 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3541 ibmf_saa_impl_get_attr_id_length_start, 3542 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length() enter\n"); 3543 3544 /* this function should not be used for multipath record */ 3545 ASSERT(attr_id != SA_MULTIPATHRECORD_ATTRID); 3546 3547 switch (attr_id) { 3548 case SA_CLASSPORTINFO_ATTRID: 3549 attr_length = sizeof (ib_mad_classportinfo_t); 3550 break; 3551 case SA_NOTICE_ATTRID: 3552 attr_length = sizeof (ib_mad_notice_t); 3553 break; 3554 case SA_INFORMINFO_ATTRID: 3555 attr_length = sizeof (ib_mad_informinfo_t); 3556 break; 3557 case SA_NODERECORD_ATTRID: 3558 attr_length = sizeof (sa_node_record_t); 3559 break; 3560 case SA_PORTINFORECORD_ATTRID: 3561 attr_length = sizeof (sa_portinfo_record_t); 3562 break; 3563 case SA_SLTOVLRECORD_ATTRID: 3564 attr_length = sizeof (sa_SLtoVLmapping_record_t); 3565 break; 3566 case SA_SWITCHINFORECORD_ATTRID: 3567 attr_length = sizeof (sa_switchinfo_record_t); 3568 break; 3569 case SA_LINEARFDBRECORD_ATTRID: 3570 attr_length = sizeof (sa_linearft_record_t); 3571 break; 3572 case SA_RANDOMFDBRECORD_ATTRID: 3573 attr_length = sizeof (sa_randomft_record_t); 3574 break; 3575 case SA_MULTICASTFDBRECORD_ATTRID: 3576 attr_length = sizeof (sa_multicastft_record_t); 3577 break; 3578 case SA_SMINFORECORD_ATTRID: 3579 attr_length = sizeof (sa_sminfo_record_t); 3580 break; 3581 case SA_INFORMINFORECORD_ATTRID: 3582 attr_length = sizeof (sa_informinfo_record_t); 3583 break; 3584 case SA_LINKRECORD_ATTRID: 3585 attr_length = sizeof (sa_link_record_t); 3586 break; 3587 case SA_GUIDINFORECORD_ATTRID: 3588 attr_length = sizeof (sa_guidinfo_record_t); 3589 break; 3590 case SA_SERVICERECORD_ATTRID: 3591 attr_length = sizeof (sa_service_record_t); 3592 break; 3593 case SA_PARTITIONRECORD_ATTRID: 3594 attr_length = sizeof (sa_pkey_table_record_t); 3595 break; 3596 case SA_PATHRECORD_ATTRID: 3597 attr_length = sizeof (sa_path_record_t); 3598 break; 3599 case SA_VLARBRECORD_ATTRID: 3600 attr_length = sizeof (sa_VLarb_table_record_t); 3601 break; 3602 case SA_MCMEMBERRECORD_ATTRID: 3603 attr_length = sizeof (sa_mcmember_record_t); 3604 break; 3605 case SA_TRACERECORD_ATTRID: 3606 attr_length = sizeof (sa_trace_record_t); 3607 break; 3608 case SA_SERVICEASSNRECORD_ATTRID: 3609 attr_length = sizeof (sa_service_assn_record_t); 3610 break; 3611 default: 3612 /* should only get the above type of packets */ 3613 attr_length = 0; 3614 break; 3615 } 3616 3617 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3618 ibmf_saa_impl_get_attr_id_length_end, 3619 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length():" 3620 " attr_id: 0x%x size %d\n", 3621 tnf_opaque, attr_id, attr_id, tnf_uint, attr_length, attr_length); 3622 3623 return (attr_length); 3624 } 3625 3626 /* 3627 * ibmf_saa_impl_free_msg: 3628 * Takes a completed message and free memory associated with the message, 3629 * including the individual fields of the im_msgbufs_send. 3630 * ibmf_free_msg, called at the end of this function, takes a pointer to the 3631 * message pointer so that it can set the message pointer to NULL. This 3632 * function takes just the message pointer so the msgp will not be NULL after 3633 * this function returns. 3634 * 3635 * Input Arguments 3636 * ibmf_hdl ibmf handle used in ibmf_msg_alloc 3637 * msgp pointer to ibmf_msg_t to free 3638 * 3639 * Returns 3640 * void 3641 */ 3642 static void 3643 ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp) 3644 { 3645 int res; 3646 3647 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3648 ibmf_saa_impl_free_msg_start, 3649 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() enter: msg %p\n", 3650 tnf_opaque, msg, msgp); 3651 3652 ASSERT(msgp != NULL); 3653 3654 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, 3655 sizeof (ib_mad_hdr_t)); 3656 3657 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_hdr, 3658 msgp->im_msgbufs_send.im_bufs_cl_hdr_len); 3659 3660 if (msgp->im_msgbufs_send.im_bufs_cl_data_len > 0) 3661 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_data, 3662 msgp->im_msgbufs_send.im_bufs_cl_data_len); 3663 3664 res = ibmf_free_msg(ibmf_hdl, &msgp); 3665 ASSERT(res == IBMF_SUCCESS); 3666 3667 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3668 ibmf_saa_impl_free_msg_end, 3669 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() exit\n"); 3670 } 3671 3672 /* 3673 * ibmf_saa_impl_get_port_guid: 3674 */ 3675 static int 3676 ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop, 3677 ib_guid_t *guid_ret) 3678 { 3679 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3680 ibmf_saa_impl_get_port_guid_start, 3681 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_port_guid() enter\n"); 3682 3683 if (ibt_portinfop->p_linkstate != IBT_PORT_ACTIVE) { 3684 3685 return (IBMF_BAD_PORT_STATE); 3686 } 3687 3688 if (ibt_portinfop->p_sgid_tbl_sz == 0) { 3689 3690 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L2, 3691 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "", 3692 "ibmf_saa_impl_get_port_guid: %s\n", tnf_string, msg, 3693 "portinfo sgid table size is 0. Exiting.\n"); 3694 3695 return (IBMF_TRANSPORT_FAILURE); 3696 } 3697 3698 *guid_ret = ibt_portinfop->p_sgid_tbl[0].gid_guid; 3699 3700 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3701 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "", 3702 "ibmf_saa_impl_get_port_guid: Returning port_guid %016" PRIx64 "\n", 3703 tnf_opaque, port_guid, *guid_ret); 3704 3705 return (IBMF_SUCCESS); 3706 } 3707 3708 /* 3709 * ibmf_saa_impl_set_transaction_params: 3710 */ 3711 static void 3712 ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp, 3713 ibt_hca_portinfo_t *portinfop) 3714 { 3715 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3716 ibmf_saa_impl_set_transaction_params_start, 3717 IBMF_TNF_TRACE, "", 3718 "ibmf_saa_impl_set_transaction_params() enter\n"); 3719 3720 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 3721 3722 saa_portp->saa_pt_ibmf_retrans.retrans_retries = 3723 ibmf_saa_retrans_retries; 3724 /* 3725 * For the first transaction (generally getting the 3726 * classportinfo) have ibmf pick our timeouts. It should be using the 3727 * default IB spec values. 3728 * Once we get the classportinfo we'll update the correct response time 3729 * value (rtv) and round-trip time (rttv). ibmf should always calculate 3730 * trans_to since it depends on the particular transaction's number of 3731 * packets. 3732 */ 3733 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = 0; 3734 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = 0; 3735 saa_portp->saa_pt_ibmf_retrans.retrans_trans_to = 0; 3736 3737 /* 3738 * Assume that the SA supports all optional records. If it 3739 * does not, the request will get returned with ERR_NOT_SUPP. When 3740 * the classportinfo response comes back we will update the cap mask 3741 * to prevent unnecessary unsupported requests. 3742 */ 3743 saa_portp->saa_pt_sa_cap_mask = 0xFFFF; 3744 3745 saa_portp->saa_pt_ibmf_msg_flags = 0; 3746 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1; 3747 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = 3748 IB_PKEY_DEFAULT_LIMITED; 3749 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY; 3750 3751 /* 3752 * fill out addr information for MADs that will be sent 3753 * to SA on this port 3754 */ 3755 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = portinfop->p_base_lid; 3756 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = portinfop->p_sm_lid; 3757 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = portinfop->p_sm_sl; 3758 3759 /* place upper bound on subnet timeout in case of faulty SM */ 3760 saa_portp->saa_pt_timeout = portinfop->p_subnet_timeout; 3761 3762 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout) 3763 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout; 3764 3765 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 3766 ibmf_saa_impl_set_transaction_params, 3767 IBMF_TNF_TRACE, "", 3768 "ibmf_saa_impl_set_transaction_params: local_lid = 0x%x, " 3769 "sm_lid = 0x%x, sm_sl = 0x%x, sn_timeout = 0x%x\n", 3770 tnf_opaque, local_lid, portinfop->p_base_lid, 3771 tnf_opaque, sm_lid, portinfop->p_sm_lid, 3772 tnf_opaque, sm_sl, portinfop->p_sm_sl, 3773 tnf_opaque, subnet_timeout, portinfop->p_subnet_timeout); 3774 3775 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3776 ibmf_saa_impl_set_transaction_params_end, 3777 IBMF_TNF_TRACE, "", 3778 "ibmf_saa_impl_set_transaction_params() exit\n"); 3779 } 3780 3781 3782 /* 3783 * ibmf_saa_impl_update_sa_address_info 3784 */ 3785 static void 3786 ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, ibmf_msg_t *msgp) 3787 { 3788 void *result; 3789 ib_sa_hdr_t *sa_hdr; 3790 int rv; 3791 size_t length; 3792 uint16_t attr_id; 3793 ib_mad_classportinfo_t *cpi; 3794 ibmf_global_addr_info_t *gaddrp = &saa_portp->saa_pt_ibmf_global_addr; 3795 ibt_hca_portinfo_t *ibt_pinfo; 3796 uint_t nports, size; 3797 ibt_status_t ibt_status; 3798 3799 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3800 ibmf_saa_impl_update_sa_address_info, 3801 IBMF_TNF_TRACE, "", 3802 "ibmf_saa_impl_update_sa_address_info() enter\n"); 3803 3804 /* 3805 * decode the respons of msgp as a classportinfo attribute 3806 */ 3807 rv = ibmf_saa_utils_unpack_sa_hdr(msgp->im_msgbufs_recv.im_bufs_cl_hdr, 3808 msgp->im_msgbufs_recv.im_bufs_cl_hdr_len, &sa_hdr, KM_NOSLEEP); 3809 if (rv != IBMF_SUCCESS) { 3810 3811 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3812 ibmf_saa_impl_update_sa_address_err, 3813 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3814 "%s, ibmf_status = %d\n", tnf_string, msg, 3815 "Could not unpack sa hdr", tnf_int, ibmf_status, rv); 3816 3817 return; 3818 } 3819 3820 attr_id = b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID); 3821 if (attr_id != MAD_ATTR_ID_CLASSPORTINFO) { 3822 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3823 ibmf_saa_impl_update_sa_address_info_err, 3824 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3825 "%s, attrID = %x\n", tnf_string, msg, 3826 "Wrong attribute ID", tnf_int, ibmf_status, attr_id); 3827 3828 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3829 return; 3830 } 3831 rv = ibmf_saa_utils_unpack_payload( 3832 msgp->im_msgbufs_recv.im_bufs_cl_data, 3833 msgp->im_msgbufs_recv.im_bufs_cl_data_len, attr_id, &result, 3834 &length, sa_hdr->AttributeOffset, B_TRUE, KM_NOSLEEP); 3835 if (rv != IBMF_SUCCESS) { 3836 3837 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3838 ibmf_saa_impl_update_sa_address_err, 3839 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3840 "%s, ibmf_status = %d\n", tnf_string, msg, 3841 "Could not unpack payload", tnf_int, ibmf_status, rv); 3842 3843 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3844 return; 3845 } 3846 3847 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3848 3849 /* 3850 * Use the classportinfo contents to update the SA address info 3851 */ 3852 cpi = (ib_mad_classportinfo_t *)result; 3853 mutex_enter(&saa_portp->saa_pt_mutex); 3854 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = cpi->RedirectLID; 3855 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = cpi->RedirectQP; 3856 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = cpi->RedirectP_Key; 3857 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = cpi->RedirectQ_Key; 3858 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = cpi->RedirectSL; 3859 3860 saa_portp->saa_pt_redirect_active = B_TRUE; 3861 3862 if ((cpi->RedirectGID_hi != 0) || (cpi->RedirectGID_lo != 0)) { 3863 3864 mutex_exit(&saa_portp->saa_pt_mutex); 3865 ibt_status = ibt_query_hca_ports_byguid( 3866 saa_portp->saa_pt_node_guid, saa_portp->saa_pt_port_num, 3867 &ibt_pinfo, &nports, &size); 3868 if (ibt_status != IBT_SUCCESS) { 3869 3870 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3871 ibmf_saa_impl_update_sa_address_err, IBMF_TNF_TRACE, 3872 "", "ibmf_saa_impl_update_sa_address_info: " 3873 "%s, ibt_status = %d\n", tnf_string, msg, 3874 "Could not query hca port", 3875 tnf_int, ibt_status, ibt_status); 3876 3877 kmem_free(result, length); 3878 return; 3879 } 3880 3881 mutex_enter(&saa_portp->saa_pt_mutex); 3882 /* 3883 * Fill in global address info parameters 3884 * 3885 * NOTE: The HopLimit value is not specified through the 3886 * contents of ClassPortInfo. It may be possible to find 3887 * out the proper value to use even for SA beeing redirected 3888 * to another subnet. But we do only support redirect within 3889 * our local subnet 3890 */ 3891 gaddrp->ig_sender_gid.gid_prefix = 3892 ibt_pinfo->p_sgid_tbl[0].gid_prefix; 3893 gaddrp->ig_sender_gid.gid_guid = saa_portp->saa_pt_port_guid; 3894 gaddrp->ig_recver_gid.gid_prefix = cpi->RedirectGID_hi; 3895 gaddrp->ig_recver_gid.gid_guid = cpi->RedirectGID_lo; 3896 gaddrp->ig_flow_label = cpi->RedirectFL; 3897 gaddrp->ig_tclass = cpi->RedirectTC; 3898 gaddrp->ig_hop_limit = 0; 3899 3900 saa_portp->saa_pt_ibmf_msg_flags = 3901 IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 3902 3903 mutex_exit(&saa_portp->saa_pt_mutex); 3904 ibt_free_portinfo(ibt_pinfo, size); 3905 } else { 3906 saa_portp->saa_pt_ibmf_msg_flags = 0; 3907 mutex_exit(&saa_portp->saa_pt_mutex); 3908 } 3909 kmem_free(result, length); 3910 3911 /* 3912 * Update the address info of msgp with the new address parameters 3913 */ 3914 mutex_enter(&saa_portp->saa_pt_mutex); 3915 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr, 3916 sizeof (ibmf_addr_info_t)); 3917 if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) { 3918 3919 msgp->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 3920 3921 bcopy(&saa_portp->saa_pt_ibmf_global_addr, 3922 &msgp->im_global_addr, sizeof (ibmf_global_addr_info_t)); 3923 } else { 3924 msgp->im_msg_flags = 0; 3925 } 3926 mutex_exit(&saa_portp->saa_pt_mutex); 3927 } 3928 3929 /* 3930 * ibmf_saa_impl_ibmf_unreg: 3931 */ 3932 static int 3933 ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp) 3934 { 3935 int ibmf_status; 3936 3937 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_start, 3938 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() enter\n"); 3939 3940 /* teardown async cb */ 3941 ibmf_status = ibmf_tear_down_async_cb(saa_portp->saa_pt_ibmf_handle, 3942 saa_portp->saa_pt_qp_handle, 0); 3943 if (ibmf_status != IBMF_SUCCESS) { 3944 3945 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3946 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3947 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3948 tnf_string, msg, "Could not tear down async cb", 3949 tnf_int, ibmf_status, ibmf_status); 3950 3951 goto bail; 3952 } 3953 3954 /* free qp */ 3955 ibmf_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle, 3956 &saa_portp->saa_pt_qp_handle, 0); 3957 3958 if (ibmf_status != IBMF_SUCCESS) { 3959 3960 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3961 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3962 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3963 tnf_string, msg, "Could not free queue pair", 3964 tnf_int, ibmf_status, ibmf_status); 3965 3966 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 1); 3967 goto bail; 3968 } 3969 3970 ibmf_status = ibmf_unregister(&saa_portp->saa_pt_ibmf_handle, 0); 3971 3972 if (ibmf_status != IBMF_SUCCESS) { 3973 3974 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3975 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3976 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3977 tnf_string, msg, "ibmf_unregister() failed", 3978 tnf_int, ibmf_status, ibmf_status); 3979 3980 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0); 3981 } 3982 3983 bail: 3984 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_end, 3985 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() exit\n"); 3986 3987 return (ibmf_status); 3988 }