5255 uts shouldn't open-code ISP2

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 
  26 /*
  27  * hermon_srq.c
  28  *    Hermon Shared Receive Queue Processing Routines
  29  *
  30  *    Implements all the routines necessary for allocating, freeing, querying,
  31  *    modifying and posting shared receive queues.
  32  */
  33 
  34 #include <sys/sysmacros.h>
  35 #include <sys/types.h>
  36 #include <sys/conf.h>
  37 #include <sys/ddi.h>
  38 #include <sys/sunddi.h>
  39 #include <sys/modctl.h>
  40 #include <sys/bitmap.h>
  41 
  42 #include <sys/ib/adapters/hermon/hermon.h>
  43 
  44 static void hermon_srq_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl,
  45     hermon_qp_wq_type_t wq_type, uint_t *logwqesz, uint_t *max_sgl);
  46 
  47 /*
  48  * hermon_srq_alloc()
  49  *    Context: Can be called only from user or kernel context.
  50  */
  51 int
  52 hermon_srq_alloc(hermon_state_t *state, hermon_srq_info_t *srqinfo,
  53     uint_t sleepflag)
  54 {
  55         ibt_srq_hdl_t           ibt_srqhdl;
  56         hermon_pdhdl_t          pd;
  57         ibt_srq_sizes_t         *sizes;
  58         ibt_srq_sizes_t         *real_sizes;
  59         hermon_srqhdl_t         *srqhdl;
  60         ibt_srq_flags_t         flags;
  61         hermon_rsrc_t           *srqc, *rsrc;
  62         hermon_hw_srqc_t        srqc_entry;
  63         uint32_t                *buf;
  64         hermon_srqhdl_t         srq;
  65         hermon_umap_db_entry_t  *umapdb;
  66         ibt_mr_attr_t           mr_attr;
  67         hermon_mr_options_t     mr_op;
  68         hermon_mrhdl_t          mr;
  69         uint64_t                value, srq_desc_off;
  70         uint32_t                log_srq_size;
  71         uint32_t                uarpg;
  72         uint_t                  srq_is_umap;
  73         int                     flag, status;
  74         uint_t                  max_sgl;
  75         uint_t                  wqesz;
  76         uint_t                  srq_wr_sz;
  77         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sizes))
  78 
  79         /*
  80          * options-->wq_location used to be for location, now explicitly
  81          * LOCATION_NORMAL
  82          */
  83 
  84         /*
  85          * Extract the necessary info from the hermon_srq_info_t structure
  86          */
  87         real_sizes = srqinfo->srqi_real_sizes;
  88         sizes      = srqinfo->srqi_sizes;
  89         pd         = srqinfo->srqi_pd;
  90         ibt_srqhdl = srqinfo->srqi_ibt_srqhdl;
  91         flags      = srqinfo->srqi_flags;
  92         srqhdl     = srqinfo->srqi_srqhdl;
  93 
  94         /*
  95          * Determine whether SRQ is being allocated for userland access or
  96          * whether it is being allocated for kernel access.  If the SRQ is
  97          * being allocated for userland access, then lookup the UAR doorbell
  98          * page number for the current process.  Note:  If this is not found
  99          * (e.g. if the process has not previously open()'d the Hermon driver),
 100          * then an error is returned.
 101          */
 102         srq_is_umap = (flags & IBT_SRQ_USER_MAP) ? 1 : 0;
 103         if (srq_is_umap) {
 104                 status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(),
 105                     MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
 106                 if (status != DDI_SUCCESS) {
 107                         status = IBT_INVALID_PARAM;
 108                         goto srqalloc_fail3;
 109                 }
 110                 uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx;
 111         } else {
 112                 uarpg = state->hs_kernel_uar_index;
 113         }
 114 
 115         /* Increase PD refcnt */
 116         hermon_pd_refcnt_inc(pd);
 117 
 118         /* Allocate an SRQ context entry */
 119         status = hermon_rsrc_alloc(state, HERMON_SRQC, 1, sleepflag, &srqc);
 120         if (status != DDI_SUCCESS) {
 121                 status = IBT_INSUFF_RESOURCE;
 122                 goto srqalloc_fail1;
 123         }
 124 
 125         /* Allocate the SRQ Handle entry */
 126         status = hermon_rsrc_alloc(state, HERMON_SRQHDL, 1, sleepflag, &rsrc);
 127         if (status != DDI_SUCCESS) {
 128                 status = IBT_INSUFF_RESOURCE;
 129                 goto srqalloc_fail2;
 130         }
 131 
 132         srq = (hermon_srqhdl_t)rsrc->hr_addr;
 133         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*srq))
 134 
 135         bzero(srq, sizeof (struct hermon_sw_srq_s));
 136         /* Calculate the SRQ number */
 137 
 138         /* just use the index, implicit in Hermon */
 139         srq->srq_srqnum = srqc->hr_indx;
 140 
 141         /*
 142          * If this will be a user-mappable SRQ, then allocate an entry for
 143          * the "userland resources database".  This will later be added to
 144          * the database (after all further SRQ operations are successful).
 145          * If we fail here, we must undo the reference counts and the
 146          * previous resource allocation.
 147          */
 148         if (srq_is_umap) {
 149                 umapdb = hermon_umap_db_alloc(state->hs_instance,
 150                     srq->srq_srqnum, MLNX_UMAP_SRQMEM_RSRC,
 151                     (uint64_t)(uintptr_t)rsrc);
 152                 if (umapdb == NULL) {
 153                         status = IBT_INSUFF_RESOURCE;
 154                         goto srqalloc_fail3;
 155                 }
 156         }
 157 
 158         /*
 159          * Allocate the doorbell record.  Hermon just needs one for the
 160          * SRQ, and use uarpg (above) as the uar index
 161          */
 162 
 163         status = hermon_dbr_alloc(state, uarpg, &srq->srq_wq_dbr_acchdl,
 164             &srq->srq_wq_vdbr, &srq->srq_wq_pdbr, &srq->srq_rdbr_mapoffset);
 165         if (status != DDI_SUCCESS) {
 166                 status = IBT_INSUFF_RESOURCE;
 167                 goto srqalloc_fail4;
 168         }
 169 
 170         /*
 171          * Calculate the appropriate size for the SRQ.
 172          * Note:  All Hermon SRQs must be a power-of-2 in size.  Also
 173          * they may not be any smaller than HERMON_SRQ_MIN_SIZE.  This step
 174          * is to round the requested size up to the next highest power-of-2
 175          */
 176         srq_wr_sz = max(sizes->srq_wr_sz + 1, HERMON_SRQ_MIN_SIZE);
 177         log_srq_size = highbit(srq_wr_sz);
 178         if (ISP2(srq_wr_sz)) {
 179                 log_srq_size = log_srq_size - 1;
 180         }
 181 
 182         /*
 183          * Next we verify that the rounded-up size is valid (i.e. consistent
 184          * with the device limits and/or software-configured limits).  If not,
 185          * then obviously we have a lot of cleanup to do before returning.
 186          */
 187         if (log_srq_size > state->hs_cfg_profile->cp_log_max_srq_sz) {
 188                 status = IBT_HCA_WR_EXCEEDED;
 189                 goto srqalloc_fail4a;
 190         }
 191 
 192         /*
 193          * Next we verify that the requested number of SGL is valid (i.e.
 194          * consistent with the device limits and/or software-configured
 195          * limits).  If not, then obviously the same cleanup needs to be done.
 196          */
 197         max_sgl = state->hs_ibtfinfo.hca_attr->hca_max_srq_sgl;
 198         if (sizes->srq_sgl_sz > max_sgl) {
 199                 status = IBT_HCA_SGL_EXCEEDED;
 200                 goto srqalloc_fail4a;
 201         }
 202 
 203         /*
 204          * Determine the SRQ's WQE sizes.  This depends on the requested
 205          * number of SGLs.  Note: This also has the side-effect of
 206          * calculating the real number of SGLs (for the calculated WQE size)
 207          */
 208         hermon_srq_sgl_to_logwqesz(state, sizes->srq_sgl_sz,
 209             HERMON_QP_WQ_TYPE_RECVQ, &srq->srq_wq_log_wqesz,
 210             &srq->srq_wq_sgl);
 211 
 212         /*
 213          * Allocate the memory for SRQ work queues.  Note:  The location from
 214          * which we will allocate these work queues is always
 215          * QUEUE_LOCATION_NORMAL.  Since Hermon work queues are not
 216          * allowed to cross a 32-bit (4GB) boundary, the alignment of the work
 217          * queue memory is very important.  We used to allocate work queues
 218          * (the combined receive and send queues) so that they would be aligned
 219          * on their combined size.  That alignment guaranteed that they would
 220          * never cross the 4GB boundary (Hermon work queues are on the order of
 221          * MBs at maximum).  Now we are able to relax this alignment constraint
 222          * by ensuring that the IB address assigned to the queue memory (as a
 223          * result of the hermon_mr_register() call) is offset from zero.
 224          * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
 225          * guarantee the alignment, but when attempting to use IOMMU bypass
 226          * mode we found that we were not allowed to specify any alignment that
 227          * was more restrictive than the system page size.  So we avoided this
 228          * constraint by passing two alignment values, one for the memory
 229          * allocation itself and the other for the DMA handle (for later bind).
 230          * This used to cause more memory than necessary to be allocated (in
 231          * order to guarantee the more restrictive alignment contraint).  But
 232          * be guaranteeing the zero-based IB virtual address for the queue, we
 233          * are able to conserve this memory.
 234          *
 235          * Note: If SRQ is not user-mappable, then it may come from either
 236          * kernel system memory or from HCA-attached local DDR memory.
 237          *
 238          * Note2: We align this queue on a pagesize boundary.  This is required
 239          * to make sure that all the resulting IB addresses will start at 0, for
 240          * a zero-based queue.  By making sure we are aligned on at least a
 241          * page, any offset we use into our queue will be the same as when we
 242          * perform hermon_srq_modify() operations later.
 243          */
 244         wqesz = (1 << srq->srq_wq_log_wqesz);
 245         srq->srq_wqinfo.qa_size = (1 << log_srq_size) * wqesz;
 246         srq->srq_wqinfo.qa_alloc_align = PAGESIZE;
 247         srq->srq_wqinfo.qa_bind_align = PAGESIZE;
 248         if (srq_is_umap) {
 249                 srq->srq_wqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
 250         } else {
 251                 srq->srq_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
 252         }
 253         status = hermon_queue_alloc(state, &srq->srq_wqinfo, sleepflag);
 254         if (status != DDI_SUCCESS) {
 255                 status = IBT_INSUFF_RESOURCE;
 256                 goto srqalloc_fail4a;
 257         }
 258         buf = (uint32_t *)srq->srq_wqinfo.qa_buf_aligned;
 259         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
 260 
 261         /*
 262          * Register the memory for the SRQ work queues.  The memory for the SRQ
 263          * must be registered in the Hermon cMPT tables.  This gives us the LKey
 264          * to specify in the SRQ context later.  Note: If the work queue is to
 265          * be allocated from DDR memory, then only a "bypass" mapping is
 266          * appropriate.  And if the SRQ memory is user-mappable, then we force
 267          * DDI_DMA_CONSISTENT mapping.  Also, in order to meet the alignment
 268          * restriction, we pass the "mro_bind_override_addr" flag in the call
 269          * to hermon_mr_register().  This guarantees that the resulting IB vaddr
 270          * will be zero-based (modulo the offset into the first page).  If we
 271          * fail here, we still have the bunch of resource and reference count
 272          * cleanup to do.
 273          */
 274         flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
 275             IBT_MR_NOSLEEP;
 276         mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
 277         mr_attr.mr_len   = srq->srq_wqinfo.qa_size;
 278         mr_attr.mr_as    = NULL;
 279         mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
 280         mr_op.mro_bind_type   = state->hs_cfg_profile->cp_iommu_bypass;
 281         mr_op.mro_bind_dmahdl = srq->srq_wqinfo.qa_dmahdl;
 282         mr_op.mro_bind_override_addr = 1;
 283         status = hermon_mr_register(state, pd, &mr_attr, &mr,
 284             &mr_op, HERMON_SRQ_CMPT);
 285         if (status != DDI_SUCCESS) {
 286                 status = IBT_INSUFF_RESOURCE;
 287                 goto srqalloc_fail5;
 288         }
 289         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
 290 
 291         /*
 292          * Calculate the offset between the kernel virtual address space
 293          * and the IB virtual address space.  This will be used when
 294          * posting work requests to properly initialize each WQE.
 295          */
 296         srq_desc_off = (uint64_t)(uintptr_t)srq->srq_wqinfo.qa_buf_aligned -
 297             (uint64_t)mr->mr_bindinfo.bi_addr;
 298 
 299         srq->srq_wq_wqhdr = hermon_wrid_wqhdr_create(1 << log_srq_size);
 300 
 301         /*
 302          * Fill in all the return arguments (if necessary).  This includes
 303          * real queue size and real SGLs.
 304          */
 305         if (real_sizes != NULL) {
 306                 real_sizes->srq_wr_sz = (1 << log_srq_size) - 1;
 307                 real_sizes->srq_sgl_sz = srq->srq_wq_sgl;
 308         }
 309 
 310         /*
 311          * Fill in the SRQC entry.  This is the final step before passing
 312          * ownership of the SRQC entry to the Hermon hardware.  We use all of
 313          * the information collected/calculated above to fill in the
 314          * requisite portions of the SRQC.  Note: If this SRQ is going to be
 315          * used for userland access, then we need to set the UAR page number
 316          * appropriately (otherwise it's a "don't care")
 317          */
 318         bzero(&srqc_entry, sizeof (hermon_hw_srqc_t));
 319         srqc_entry.state           = HERMON_SRQ_STATE_HW_OWNER;
 320         srqc_entry.log_srq_size    = log_srq_size;
 321         srqc_entry.srqn            = srq->srq_srqnum;
 322         srqc_entry.log_rq_stride   = srq->srq_wq_log_wqesz - 4;
 323                                         /* 16-byte chunks */
 324 
 325         srqc_entry.page_offs       = srq->srq_wqinfo.qa_pgoffs >> 6;
 326         srqc_entry.log2_pgsz       = mr->mr_log2_pgsz;
 327         srqc_entry.mtt_base_addrh  = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF);
 328         srqc_entry.mtt_base_addrl  = mr->mr_mttaddr >> 3;
 329         srqc_entry.pd              = pd->pd_pdnum;
 330         srqc_entry.dbr_addrh = (uint32_t)((uint64_t)srq->srq_wq_pdbr >> 32);
 331         srqc_entry.dbr_addrl = (uint32_t)((uint64_t)srq->srq_wq_pdbr >> 2);
 332 
 333         /*
 334          * all others - specifically, xrcd, cqn_xrc, lwm, wqe_cnt, and wqe_cntr
 335          * are zero thanks to the bzero of the structure
 336          */
 337 
 338         /*
 339          * Write the SRQC entry to hardware.  Lastly, we pass ownership of
 340          * the entry to the hardware (using the Hermon SW2HW_SRQ firmware
 341          * command).  Note: In general, this operation shouldn't fail.  But
 342          * if it does, we have to undo everything we've done above before
 343          * returning error.
 344          */
 345         status = hermon_cmn_ownership_cmd_post(state, SW2HW_SRQ, &srqc_entry,
 346             sizeof (hermon_hw_srqc_t), srq->srq_srqnum,
 347             sleepflag);
 348         if (status != HERMON_CMD_SUCCESS) {
 349                 cmn_err(CE_CONT, "Hermon: SW2HW_SRQ command failed: %08x\n",
 350                     status);
 351                 if (status == HERMON_CMD_INVALID_STATUS) {
 352                         hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
 353                 }
 354                 status = ibc_get_ci_failure(0);
 355                 goto srqalloc_fail8;
 356         }
 357 
 358         /*
 359          * Fill in the rest of the Hermon SRQ handle.  We can update
 360          * the following fields for use in further operations on the SRQ.
 361          */
 362         srq->srq_srqcrsrcp = srqc;
 363         srq->srq_rsrcp          = rsrc;
 364         srq->srq_mrhdl          = mr;
 365         srq->srq_refcnt         = 0;
 366         srq->srq_is_umap   = srq_is_umap;
 367         srq->srq_uarpg          = uarpg;
 368         srq->srq_umap_dhp  = (devmap_cookie_t)NULL;
 369         srq->srq_pdhdl          = pd;
 370         srq->srq_wq_bufsz  = (1 << log_srq_size);
 371         srq->srq_wq_buf         = buf;
 372         srq->srq_desc_off  = srq_desc_off;
 373         srq->srq_hdlrarg   = (void *)ibt_srqhdl;
 374         srq->srq_state          = 0;
 375         srq->srq_real_sizes.srq_wr_sz = (1 << log_srq_size);
 376         srq->srq_real_sizes.srq_sgl_sz = srq->srq_wq_sgl;
 377 
 378         /*
 379          * Put SRQ handle in Hermon SRQNum-to-SRQhdl list.  Then fill in the
 380          * "srqhdl" and return success
 381          */
 382         hermon_icm_set_num_to_hdl(state, HERMON_SRQC, srqc->hr_indx, srq);
 383 
 384         /*
 385          * If this is a user-mappable SRQ, then we need to insert the
 386          * previously allocated entry into the "userland resources database".
 387          * This will allow for later lookup during devmap() (i.e. mmap())
 388          * calls.
 389          */
 390         if (srq->srq_is_umap) {
 391                 hermon_umap_db_add(umapdb);
 392         } else {        /* initialize work queue for kernel SRQs */
 393                 int i, len, last;
 394                 uint16_t *desc;
 395 
 396                 desc = (uint16_t *)buf;
 397                 len = wqesz / sizeof (*desc);
 398                 last = srq->srq_wq_bufsz - 1;
 399                 for (i = 0; i < last; i++) {
 400                         desc[1] = htons(i + 1);
 401                         desc += len;
 402                 }
 403                 srq->srq_wq_wqhdr->wq_tail = last;
 404                 srq->srq_wq_wqhdr->wq_head = 0;
 405         }
 406 
 407         *srqhdl = srq;
 408 
 409         return (status);
 410 
 411 /*
 412  * The following is cleanup for all possible failure cases in this routine
 413  */
 414 srqalloc_fail8:
 415         hermon_wrid_wqhdr_destroy(srq->srq_wq_wqhdr);
 416 srqalloc_fail7:
 417         if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
 418             HERMON_SLEEPFLAG_FOR_CONTEXT()) != DDI_SUCCESS) {
 419                 HERMON_WARNING(state, "failed to deregister SRQ memory");
 420         }
 421 srqalloc_fail5:
 422         hermon_queue_free(&srq->srq_wqinfo);
 423 srqalloc_fail4a:
 424         hermon_dbr_free(state, uarpg, srq->srq_wq_vdbr);
 425 srqalloc_fail4:
 426         if (srq_is_umap) {
 427                 hermon_umap_db_free(umapdb);
 428         }
 429 srqalloc_fail3:
 430         hermon_rsrc_free(state, &rsrc);
 431 srqalloc_fail2:
 432         hermon_rsrc_free(state, &srqc);
 433 srqalloc_fail1:
 434         hermon_pd_refcnt_dec(pd);
 435 srqalloc_fail:
 436         return (status);
 437 }
 438 
 439 
 440 /*
 441  * hermon_srq_free()
 442  *    Context: Can be called only from user or kernel context.
 443  */
 444 /* ARGSUSED */
 445 int
 446 hermon_srq_free(hermon_state_t *state, hermon_srqhdl_t *srqhdl,
 447     uint_t sleepflag)
 448 {
 449         hermon_rsrc_t           *srqc, *rsrc;
 450         hermon_umap_db_entry_t  *umapdb;
 451         uint64_t                value;
 452         hermon_srqhdl_t         srq;
 453         hermon_mrhdl_t          mr;
 454         hermon_pdhdl_t          pd;
 455         hermon_hw_srqc_t        srqc_entry;
 456         uint32_t                srqnum;
 457         uint_t                  maxprot;
 458         int                     status;
 459 
 460         /*
 461          * Pull all the necessary information from the Hermon Shared Receive
 462          * Queue handle.  This is necessary here because the resource for the
 463          * SRQ handle is going to be freed up as part of this operation.
 464          */
 465         srq     = *srqhdl;
 466         mutex_enter(&srq->srq_lock);
 467         srqc    = srq->srq_srqcrsrcp;
 468         rsrc    = srq->srq_rsrcp;
 469         pd      = srq->srq_pdhdl;
 470         mr      = srq->srq_mrhdl;
 471         srqnum  = srq->srq_srqnum;
 472 
 473         /*
 474          * If there are work queues still associated with the SRQ, then return
 475          * an error.  Otherwise, we will be holding the SRQ lock.
 476          */
 477         if (srq->srq_refcnt != 0) {
 478                 mutex_exit(&srq->srq_lock);
 479                 return (IBT_SRQ_IN_USE);
 480         }
 481 
 482         /*
 483          * If this was a user-mappable SRQ, then we need to remove its entry
 484          * from the "userland resources database".  If it is also currently
 485          * mmap()'d out to a user process, then we need to call
 486          * devmap_devmem_remap() to remap the SRQ memory to an invalid mapping.
 487          * We also need to invalidate the SRQ tracking information for the
 488          * user mapping.
 489          */
 490         if (srq->srq_is_umap) {
 491                 status = hermon_umap_db_find(state->hs_instance,
 492                     srq->srq_srqnum, MLNX_UMAP_SRQMEM_RSRC, &value,
 493                     HERMON_UMAP_DB_REMOVE, &umapdb);
 494                 if (status != DDI_SUCCESS) {
 495                         mutex_exit(&srq->srq_lock);
 496                         HERMON_WARNING(state, "failed to find in database");
 497                         return (ibc_get_ci_failure(0));
 498                 }
 499                 hermon_umap_db_free(umapdb);
 500                 if (srq->srq_umap_dhp != NULL) {
 501                         maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
 502                         status = devmap_devmem_remap(srq->srq_umap_dhp,
 503                             state->hs_dip, 0, 0, srq->srq_wqinfo.qa_size,
 504                             maxprot, DEVMAP_MAPPING_INVALID, NULL);
 505                         if (status != DDI_SUCCESS) {
 506                                 mutex_exit(&srq->srq_lock);
 507                                 HERMON_WARNING(state, "failed in SRQ memory "
 508                                     "devmap_devmem_remap()");
 509                                 return (ibc_get_ci_failure(0));
 510                         }
 511                         srq->srq_umap_dhp = (devmap_cookie_t)NULL;
 512                 }
 513         }
 514 
 515         /*
 516          * Put NULL into the Hermon SRQNum-to-SRQHdl list.  This will allow any
 517          * in-progress events to detect that the SRQ corresponding to this
 518          * number has been freed.
 519          */
 520         hermon_icm_set_num_to_hdl(state, HERMON_SRQC, srqc->hr_indx, NULL);
 521 
 522         mutex_exit(&srq->srq_lock);
 523         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*srq));
 524 
 525         /*
 526          * Reclaim SRQC entry from hardware (using the Hermon HW2SW_SRQ
 527          * firmware command).  If the ownership transfer fails for any reason,
 528          * then it is an indication that something (either in HW or SW) has
 529          * gone seriously wrong.
 530          */
 531         status = hermon_cmn_ownership_cmd_post(state, HW2SW_SRQ, &srqc_entry,
 532             sizeof (hermon_hw_srqc_t), srqnum, sleepflag);
 533         if (status != HERMON_CMD_SUCCESS) {
 534                 HERMON_WARNING(state, "failed to reclaim SRQC ownership");
 535                 cmn_err(CE_CONT, "Hermon: HW2SW_SRQ command failed: %08x\n",
 536                     status);
 537                 if (status == HERMON_CMD_INVALID_STATUS) {
 538                         hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
 539                 }
 540                 return (ibc_get_ci_failure(0));
 541         }
 542 
 543         /*
 544          * Deregister the memory for the Shared Receive Queue.  If this fails
 545          * for any reason, then it is an indication that something (either
 546          * in HW or SW) has gone seriously wrong.  So we print a warning
 547          * message and return.
 548          */
 549         status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
 550             sleepflag);
 551         if (status != DDI_SUCCESS) {
 552                 HERMON_WARNING(state, "failed to deregister SRQ memory");
 553                 return (IBT_FAILURE);
 554         }
 555 
 556         hermon_wrid_wqhdr_destroy(srq->srq_wq_wqhdr);
 557 
 558         /* Free the memory for the SRQ */
 559         hermon_queue_free(&srq->srq_wqinfo);
 560 
 561         /* Free the dbr */
 562         hermon_dbr_free(state, srq->srq_uarpg, srq->srq_wq_vdbr);
 563 
 564         /* Free the Hermon SRQ Handle */
 565         hermon_rsrc_free(state, &rsrc);
 566 
 567         /* Free the SRQC entry resource */
 568         hermon_rsrc_free(state, &srqc);
 569 
 570         /* Decrement the reference count on the protection domain (PD) */
 571         hermon_pd_refcnt_dec(pd);
 572 
 573         /* Set the srqhdl pointer to NULL and return success */
 574         *srqhdl = NULL;
 575 
 576         return (DDI_SUCCESS);
 577 }
 578 
 579 
 580 /*
 581  * hermon_srq_modify()
 582  *    Context: Can be called only from user or kernel context.
 583  */
 584 int
 585 hermon_srq_modify(hermon_state_t *state, hermon_srqhdl_t srq, uint_t size,
 586     uint_t *real_size, uint_t sleepflag)
 587 {
 588         hermon_qalloc_info_t    new_srqinfo, old_srqinfo;
 589         hermon_rsrc_t           *mtt, *old_mtt;
 590         hermon_bind_info_t      bind;
 591         hermon_bind_info_t      old_bind;
 592         hermon_mrhdl_t          mr;
 593         hermon_hw_srqc_t        srqc_entry;
 594         hermon_hw_dmpt_t        mpt_entry;
 595         uint64_t                *wre_new, *wre_old;
 596         uint64_t                mtt_addr;
 597         uint64_t                srq_pgoffs;
 598         uint64_t                srq_desc_off;
 599         uint32_t                *buf, srq_old_bufsz;
 600         uint32_t                wqesz;
 601         uint_t                  max_srq_size;
 602         uint_t                  mtt_pgsize_bits;
 603         uint_t                  log_srq_size, maxprot;
 604         int                     status;
 605 
 606         if ((state->hs_devlim.mod_wr_srq == 0) ||
 607             (state->hs_cfg_profile->cp_srq_resize_enabled == 0))
 608                 return (IBT_NOT_SUPPORTED);
 609 
 610         /*
 611          * If size requested is larger than device capability, return
 612          * Insufficient Resources
 613          */
 614         max_srq_size = (1 << state->hs_cfg_profile->cp_log_max_srq_sz);
 615         if (size > max_srq_size) {
 616                 return (IBT_HCA_WR_EXCEEDED);
 617         }
 618 
 619         /*
 620          * Calculate the appropriate size for the SRQ.
 621          * Note:  All Hermon SRQs must be a power-of-2 in size.  Also
 622          * they may not be any smaller than HERMON_SRQ_MIN_SIZE.  This step
 623          * is to round the requested size up to the next highest power-of-2
 624          */
 625         size = max(size, HERMON_SRQ_MIN_SIZE);
 626         log_srq_size = highbit(size);
 627         if (ISP2(size)) {
 628                 log_srq_size = log_srq_size - 1;
 629         }
 630 
 631         /*
 632          * Next we verify that the rounded-up size is valid (i.e. consistent
 633          * with the device limits and/or software-configured limits).
 634          */
 635         if (log_srq_size > state->hs_cfg_profile->cp_log_max_srq_sz) {
 636                 status = IBT_HCA_WR_EXCEEDED;
 637                 goto srqmodify_fail;
 638         }
 639 
 640         /*
 641          * Allocate the memory for newly resized Shared Receive Queue.
 642          *
 643          * Note: If SRQ is not user-mappable, then it may come from either
 644          * kernel system memory or from HCA-attached local DDR memory.
 645          *
 646          * Note2: We align this queue on a pagesize boundary.  This is required
 647          * to make sure that all the resulting IB addresses will start at 0,
 648          * for a zero-based queue.  By making sure we are aligned on at least a
 649          * page, any offset we use into our queue will be the same as it was
 650          * when we allocated it at hermon_srq_alloc() time.
 651          */
 652         wqesz = (1 << srq->srq_wq_log_wqesz);
 653         new_srqinfo.qa_size = (1 << log_srq_size) * wqesz;
 654         new_srqinfo.qa_alloc_align = PAGESIZE;
 655         new_srqinfo.qa_bind_align  = PAGESIZE;
 656         if (srq->srq_is_umap) {
 657                 new_srqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
 658         } else {
 659                 new_srqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
 660         }
 661         status = hermon_queue_alloc(state, &new_srqinfo, sleepflag);
 662         if (status != DDI_SUCCESS) {
 663                 status = IBT_INSUFF_RESOURCE;
 664                 goto srqmodify_fail;
 665         }
 666         buf = (uint32_t *)new_srqinfo.qa_buf_aligned;
 667         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
 668 
 669         /*
 670          * Allocate the memory for the new WRE list.  This will be used later
 671          * when we resize the wridlist based on the new SRQ size.
 672          */
 673         wre_new = kmem_zalloc((1 << log_srq_size) * sizeof (uint64_t),
 674             sleepflag);
 675         if (wre_new == NULL) {
 676                 status = IBT_INSUFF_RESOURCE;
 677                 goto srqmodify_fail;
 678         }
 679 
 680         /*
 681          * Fill in the "bind" struct.  This struct provides the majority
 682          * of the information that will be used to distinguish between an
 683          * "addr" binding (as is the case here) and a "buf" binding (see
 684          * below).  The "bind" struct is later passed to hermon_mr_mem_bind()
 685          * which does most of the "heavy lifting" for the Hermon memory
 686          * registration routines.
 687          */
 688         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(bind))
 689         bzero(&bind, sizeof (hermon_bind_info_t));
 690         bind.bi_type  = HERMON_BINDHDL_VADDR;
 691         bind.bi_addr  = (uint64_t)(uintptr_t)buf;
 692         bind.bi_len   = new_srqinfo.qa_size;
 693         bind.bi_as    = NULL;
 694         bind.bi_flags = sleepflag == HERMON_SLEEP ? IBT_MR_SLEEP :
 695             IBT_MR_NOSLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
 696         bind.bi_bypass = state->hs_cfg_profile->cp_iommu_bypass;
 697 
 698         status = hermon_mr_mtt_bind(state, &bind, new_srqinfo.qa_dmahdl, &mtt,
 699             &mtt_pgsize_bits, 0); /* no relaxed ordering */
 700         if (status != DDI_SUCCESS) {
 701                 status = status;
 702                 kmem_free(wre_new, (1 << log_srq_size) *
 703                     sizeof (uint64_t));
 704                 hermon_queue_free(&new_srqinfo);
 705                 goto srqmodify_fail;
 706         }
 707 
 708         /*
 709          * Calculate the offset between the kernel virtual address space
 710          * and the IB virtual address space.  This will be used when
 711          * posting work requests to properly initialize each WQE.
 712          *
 713          * Note: bind addr is zero-based (from alloc) so we calculate the
 714          * correct new offset here.
 715          */
 716         bind.bi_addr = bind.bi_addr & ((1 << mtt_pgsize_bits) - 1);
 717         srq_desc_off = (uint64_t)(uintptr_t)new_srqinfo.qa_buf_aligned -
 718             (uint64_t)bind.bi_addr;
 719         srq_pgoffs   = (uint_t)
 720             ((uintptr_t)new_srqinfo.qa_buf_aligned & HERMON_PAGEOFFSET);
 721 
 722         /*
 723          * Fill in the MPT entry.  This is the final step before passing
 724          * ownership of the MPT entry to the Hermon hardware.  We use all of
 725          * the information collected/calculated above to fill in the
 726          * requisite portions of the MPT.
 727          */
 728         bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t));
 729         mpt_entry.reg_win_len   = bind.bi_len;
 730         mtt_addr = (mtt->hr_indx << HERMON_MTT_SIZE_SHIFT);
 731         mpt_entry.mtt_addr_h = mtt_addr >> 32;
 732         mpt_entry.mtt_addr_l = mtt_addr >> 3;
 733 
 734         /*
 735          * for hermon we build up a new srqc and pass that (partially filled
 736          * to resize SRQ instead of modifying the (d)mpt directly
 737          */
 738 
 739 
 740 
 741         /*
 742          * Now we grab the SRQ lock.  Since we will be updating the actual
 743          * SRQ location and the producer/consumer indexes, we should hold
 744          * the lock.
 745          *
 746          * We do a HERMON_NOSLEEP here (and below), though, because we are
 747          * holding the "srq_lock" and if we got raised to interrupt level
 748          * by priority inversion, we would not want to block in this routine
 749          * waiting for success.
 750          */
 751         mutex_enter(&srq->srq_lock);
 752 
 753         /*
 754          * Copy old entries to new buffer
 755          */
 756         srq_old_bufsz = srq->srq_wq_bufsz;
 757         bcopy(srq->srq_wq_buf, buf, srq_old_bufsz * wqesz);
 758 
 759         /*
 760          * Setup MPT information for use in the MODIFY_MPT command
 761          */
 762         mr = srq->srq_mrhdl;
 763         mutex_enter(&mr->mr_lock);
 764 
 765         /*
 766          * now, setup the srqc information needed for resize - limit the
 767          * values, but use the same structure as the srqc
 768          */
 769 
 770         srqc_entry.log_srq_size   = log_srq_size;
 771         srqc_entry.page_offs      = srq_pgoffs >> 6;
 772         srqc_entry.log2_pgsz      = mr->mr_log2_pgsz;
 773         srqc_entry.mtt_base_addrl = (uint64_t)mtt_addr >> 32;
 774         srqc_entry.mtt_base_addrh = mtt_addr >> 3;
 775 
 776         /*
 777          * RESIZE_SRQ
 778          *
 779          * If this fails for any reason, then it is an indication that
 780          * something (either in HW or SW) has gone seriously wrong.  So we
 781          * print a warning message and return.
 782          */
 783         status = hermon_resize_srq_cmd_post(state, &srqc_entry,
 784             srq->srq_srqnum, sleepflag);
 785         if (status != HERMON_CMD_SUCCESS) {
 786                 cmn_err(CE_CONT, "Hermon: RESIZE_SRQ command failed: %08x\n",
 787                     status);
 788                 if (status == HERMON_CMD_INVALID_STATUS) {
 789                         hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
 790                 }
 791                 (void) hermon_mr_mtt_unbind(state, &bind, mtt);
 792                 kmem_free(wre_new, (1 << log_srq_size) *
 793                     sizeof (uint64_t));
 794                 hermon_queue_free(&new_srqinfo);
 795                 mutex_exit(&mr->mr_lock);
 796                 mutex_exit(&srq->srq_lock);
 797                 return (ibc_get_ci_failure(0));
 798         }
 799         /*
 800          * Update the Hermon Shared Receive Queue handle with all the new
 801          * information.  At the same time, save away all the necessary
 802          * information for freeing up the old resources
 803          */
 804         old_srqinfo        = srq->srq_wqinfo;
 805         old_mtt            = srq->srq_mrhdl->mr_mttrsrcp;
 806         bcopy(&srq->srq_mrhdl->mr_bindinfo, &old_bind,
 807             sizeof (hermon_bind_info_t));
 808 
 809         /* Now set the new info */
 810         srq->srq_wqinfo         = new_srqinfo;
 811         srq->srq_wq_buf         = buf;
 812         srq->srq_wq_bufsz  = (1 << log_srq_size);
 813         bcopy(&bind, &srq->srq_mrhdl->mr_bindinfo, sizeof (hermon_bind_info_t));
 814         srq->srq_mrhdl->mr_mttrsrcp = mtt;
 815         srq->srq_desc_off  = srq_desc_off;
 816         srq->srq_real_sizes.srq_wr_sz = (1 << log_srq_size);
 817 
 818         /* Update MR mtt pagesize */
 819         mr->mr_logmttpgsz = mtt_pgsize_bits;
 820         mutex_exit(&mr->mr_lock);
 821 
 822         /*
 823          * Initialize new wridlist, if needed.
 824          *
 825          * If a wridlist already is setup on an SRQ (the QP associated with an
 826          * SRQ has moved "from_reset") then we must update this wridlist based
 827          * on the new SRQ size.  We allocate the new size of Work Request ID
 828          * Entries, copy over the old entries to the new list, and
 829          * re-initialize the srq wridlist in non-umap case
 830          */
 831         wre_old = srq->srq_wq_wqhdr->wq_wrid;
 832 
 833         bcopy(wre_old, wre_new, srq_old_bufsz * sizeof (uint64_t));
 834 
 835         /* Setup new sizes in wre */
 836         srq->srq_wq_wqhdr->wq_wrid = wre_new;
 837 
 838         /*
 839          * If "old" SRQ was a user-mappable SRQ that is currently mmap()'d out
 840          * to a user process, then we need to call devmap_devmem_remap() to
 841          * invalidate the mapping to the SRQ memory.  We also need to
 842          * invalidate the SRQ tracking information for the user mapping.
 843          *
 844          * Note: On failure, the remap really shouldn't ever happen.  So, if it
 845          * does, it is an indication that something has gone seriously wrong.
 846          * So we print a warning message and return error (knowing, of course,
 847          * that the "old" SRQ memory will be leaked)
 848          */
 849         if ((srq->srq_is_umap) && (srq->srq_umap_dhp != NULL)) {
 850                 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
 851                 status = devmap_devmem_remap(srq->srq_umap_dhp,
 852                     state->hs_dip, 0, 0, srq->srq_wqinfo.qa_size, maxprot,
 853                     DEVMAP_MAPPING_INVALID, NULL);
 854                 if (status != DDI_SUCCESS) {
 855                         mutex_exit(&srq->srq_lock);
 856                         HERMON_WARNING(state, "failed in SRQ memory "
 857                             "devmap_devmem_remap()");
 858                         /* We can, however, free the memory for old wre */
 859                         kmem_free(wre_old, srq_old_bufsz * sizeof (uint64_t));
 860                         return (ibc_get_ci_failure(0));
 861                 }
 862                 srq->srq_umap_dhp = (devmap_cookie_t)NULL;
 863         }
 864 
 865         /*
 866          * Drop the SRQ lock now.  The only thing left to do is to free up
 867          * the old resources.
 868          */
 869         mutex_exit(&srq->srq_lock);
 870 
 871         /*
 872          * Unbind the MTT entries.
 873          */
 874         status = hermon_mr_mtt_unbind(state, &old_bind, old_mtt);
 875         if (status != DDI_SUCCESS) {
 876                 HERMON_WARNING(state, "failed to unbind old SRQ memory");
 877                 status = ibc_get_ci_failure(0);
 878                 goto srqmodify_fail;
 879         }
 880 
 881         /* Free the memory for old wre */
 882         kmem_free(wre_old, srq_old_bufsz * sizeof (uint64_t));
 883 
 884         /* Free the memory for the old SRQ */
 885         hermon_queue_free(&old_srqinfo);
 886 
 887         /*
 888          * Fill in the return arguments (if necessary).  This includes the
 889          * real new completion queue size.
 890          */
 891         if (real_size != NULL) {
 892                 *real_size = (1 << log_srq_size);
 893         }
 894 
 895         return (DDI_SUCCESS);
 896 
 897 srqmodify_fail:
 898         return (status);
 899 }
 900 
 901 
 902 /*
 903  * hermon_srq_refcnt_inc()
 904  *    Context: Can be called from interrupt or base context.
 905  */
 906 void
 907 hermon_srq_refcnt_inc(hermon_srqhdl_t srq)
 908 {
 909         mutex_enter(&srq->srq_lock);
 910         srq->srq_refcnt++;
 911         mutex_exit(&srq->srq_lock);
 912 }
 913 
 914 
 915 /*
 916  * hermon_srq_refcnt_dec()
 917  *    Context: Can be called from interrupt or base context.
 918  */
 919 void
 920 hermon_srq_refcnt_dec(hermon_srqhdl_t srq)
 921 {
 922         mutex_enter(&srq->srq_lock);
 923         srq->srq_refcnt--;
 924         mutex_exit(&srq->srq_lock);
 925 }
 926 
 927 
 928 /*
 929  * hermon_srqhdl_from_srqnum()
 930  *    Context: Can be called from interrupt or base context.
 931  *
 932  *    This routine is important because changing the unconstrained
 933  *    portion of the SRQ number is critical to the detection of a
 934  *    potential race condition in the SRQ handler code (i.e. the case
 935  *    where a SRQ is freed and alloc'd again before an event for the
 936  *    "old" SRQ can be handled).
 937  *
 938  *    While this is not a perfect solution (not sure that one exists)
 939  *    it does help to mitigate the chance that this race condition will
 940  *    cause us to deliver a "stale" event to the new SRQ owner.  Note:
 941  *    this solution does not scale well because the number of constrained
 942  *    bits increases (and, hence, the number of unconstrained bits
 943  *    decreases) as the number of supported SRQ grows.  For small and
 944  *    intermediate values, it should hopefully provide sufficient
 945  *    protection.
 946  */
 947 hermon_srqhdl_t
 948 hermon_srqhdl_from_srqnum(hermon_state_t *state, uint_t srqnum)
 949 {
 950         uint_t  srqindx, srqmask;
 951 
 952         /* Calculate the SRQ table index from the srqnum */
 953         srqmask = (1 << state->hs_cfg_profile->cp_log_num_srq) - 1;
 954         srqindx = srqnum & srqmask;
 955         return (hermon_icm_num_to_hdl(state, HERMON_SRQC, srqindx));
 956 }
 957 
 958 
 959 /*
 960  * hermon_srq_sgl_to_logwqesz()
 961  *    Context: Can be called from interrupt or base context.
 962  */
 963 static void
 964 hermon_srq_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl,
 965     hermon_qp_wq_type_t wq_type, uint_t *logwqesz, uint_t *max_sgl)
 966 {
 967         uint_t  max_size, log2, actual_sgl;
 968 
 969         switch (wq_type) {
 970         case HERMON_QP_WQ_TYPE_RECVQ:
 971                 /*
 972                  * Use requested maximum SGL to calculate max descriptor size
 973                  * (while guaranteeing that the descriptor size is a
 974                  * power-of-2 cachelines).
 975                  */
 976                 max_size = (HERMON_QP_WQE_MLX_SRQ_HDRS + (num_sgl << 4));
 977                 log2 = highbit(max_size);
 978                 if (ISP2(max_size)) {
 979                         log2 = log2 - 1;
 980                 }
 981 
 982                 /* Make sure descriptor is at least the minimum size */
 983                 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
 984 
 985                 /* Calculate actual number of SGL (given WQE size) */
 986                 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_SRQ_HDRS) >> 4;
 987                 break;
 988 
 989         default:
 990                 HERMON_WARNING(state, "unexpected work queue type");
 991                 break;
 992         }
 993 
 994         /* Fill in the return values */
 995         *logwqesz = log2;
 996         *max_sgl  = min(state->hs_cfg_profile->cp_srq_max_sgl, actual_sgl);
 997 }
--- EOF ---