5255 uts shouldn't open-code ISP2
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * hermon_qp.c
28 * Hermon Queue Pair Processing Routines
29 *
30 * Implements all the routines necessary for allocating, freeing, and
31 * querying the Hermon queue pairs.
32 */
33
34 #include <sys/types.h>
35 #include <sys/conf.h>
36 #include <sys/ddi.h>
37 #include <sys/sunddi.h>
38 #include <sys/modctl.h>
39 #include <sys/bitmap.h>
40 #include <sys/sysmacros.h>
41
42 #include <sys/ib/adapters/hermon/hermon.h>
43 #include <sys/ib/ib_pkt_hdrs.h>
44
45 static int hermon_qp_create_qpn(hermon_state_t *state, hermon_qphdl_t qp,
46 hermon_rsrc_t *qpc);
47 static int hermon_qpn_avl_compare(const void *q, const void *e);
48 static int hermon_special_qp_rsrc_alloc(hermon_state_t *state,
49 ibt_sqp_type_t type, uint_t port, hermon_rsrc_t **qp_rsrc);
50 static int hermon_special_qp_rsrc_free(hermon_state_t *state,
51 ibt_sqp_type_t type, uint_t port);
52 static void hermon_qp_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl,
53 uint_t real_max_sgl, hermon_qp_wq_type_t wq_type,
54 uint_t *logwqesz, uint_t *max_sgl);
55
56 /*
57 * hermon_qp_alloc()
58 * Context: Can be called only from user or kernel context.
59 */
60 int
61 hermon_qp_alloc(hermon_state_t *state, hermon_qp_info_t *qpinfo,
62 uint_t sleepflag)
63 {
64 hermon_rsrc_t *qpc, *rsrc;
65 hermon_rsrc_type_t rsrc_type;
66 hermon_umap_db_entry_t *umapdb;
67 hermon_qphdl_t qp;
68 ibt_qp_alloc_attr_t *attr_p;
69 ibt_qp_alloc_flags_t alloc_flags;
70 ibt_qp_type_t type;
71 hermon_qp_wq_type_t swq_type;
72 ibtl_qp_hdl_t ibt_qphdl;
73 ibt_chan_sizes_t *queuesz_p;
74 ib_qpn_t *qpn;
75 hermon_qphdl_t *qphdl;
76 ibt_mr_attr_t mr_attr;
77 hermon_mr_options_t mr_op;
78 hermon_srqhdl_t srq;
79 hermon_pdhdl_t pd;
80 hermon_cqhdl_t sq_cq, rq_cq;
81 hermon_mrhdl_t mr;
82 uint64_t value, qp_desc_off;
83 uint64_t *thewqe, thewqesz;
84 uint32_t *sq_buf, *rq_buf;
85 uint32_t log_qp_sq_size, log_qp_rq_size;
86 uint32_t sq_size, rq_size;
87 uint32_t sq_depth, rq_depth;
88 uint32_t sq_wqe_size, rq_wqe_size, wqesz_shift;
89 uint32_t max_sgl, max_recv_sgl, uarpg;
90 uint_t qp_is_umap;
91 uint_t qp_srq_en, i, j;
92 int status, flag;
93
94 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p, *queuesz_p))
95
96 /*
97 * Extract the necessary info from the hermon_qp_info_t structure
98 */
99 attr_p = qpinfo->qpi_attrp;
100 type = qpinfo->qpi_type;
101 ibt_qphdl = qpinfo->qpi_ibt_qphdl;
102 queuesz_p = qpinfo->qpi_queueszp;
103 qpn = qpinfo->qpi_qpn;
104 qphdl = &qpinfo->qpi_qphdl;
105 alloc_flags = attr_p->qp_alloc_flags;
106
107 /*
108 * Verify correctness of alloc_flags.
109 *
110 * 1. FEXCH and RSS are only allocated via qp_range.
111 */
112 if (alloc_flags & (IBT_QP_USES_FEXCH | IBT_QP_USES_RSS)) {
113 return (IBT_INVALID_PARAM);
114 }
115 rsrc_type = HERMON_QPC;
116 qp_is_umap = 0;
117
118 /* 2. Make sure only one of these flags is set. */
119 switch (alloc_flags &
120 (IBT_QP_USER_MAP | IBT_QP_USES_RFCI | IBT_QP_USES_FCMD)) {
121 case IBT_QP_USER_MAP:
122 qp_is_umap = 1;
123 break;
124 case IBT_QP_USES_RFCI:
125 if (type != IBT_UD_RQP)
126 return (IBT_INVALID_PARAM);
127
128 switch (attr_p->qp_fc.fc_hca_port) {
129 case 1:
130 rsrc_type = HERMON_QPC_RFCI_PORT1;
131 break;
132 case 2:
133 rsrc_type = HERMON_QPC_RFCI_PORT2;
134 break;
135 default:
136 return (IBT_INVALID_PARAM);
137 }
138 break;
139 case IBT_QP_USES_FCMD:
140 if (type != IBT_UD_RQP)
141 return (IBT_INVALID_PARAM);
142 break;
143 case 0:
144 break;
145 default:
146 return (IBT_INVALID_PARAM); /* conflicting flags set */
147 }
148
149 /*
150 * Determine whether QP is being allocated for userland access or
151 * whether it is being allocated for kernel access. If the QP is
152 * being allocated for userland access, then lookup the UAR
153 * page number for the current process. Note: If this is not found
154 * (e.g. if the process has not previously open()'d the Hermon driver),
155 * then an error is returned.
156 */
157 if (qp_is_umap) {
158 status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(),
159 MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
160 if (status != DDI_SUCCESS) {
161 return (IBT_INVALID_PARAM);
162 }
163 uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx;
164 } else {
165 uarpg = state->hs_kernel_uar_index;
166 }
167
168 /*
169 * Determine whether QP is being associated with an SRQ
170 */
171 qp_srq_en = (alloc_flags & IBT_QP_USES_SRQ) ? 1 : 0;
172 if (qp_srq_en) {
173 /*
174 * Check for valid SRQ handle pointers
175 */
176 if (attr_p->qp_ibc_srq_hdl == NULL) {
177 status = IBT_SRQ_HDL_INVALID;
178 goto qpalloc_fail;
179 }
180 srq = (hermon_srqhdl_t)attr_p->qp_ibc_srq_hdl;
181 }
182
183 /*
184 * Check for valid QP service type (only UD/RC/UC supported)
185 */
186 if (((type != IBT_UD_RQP) && (type != IBT_RC_RQP) &&
187 (type != IBT_UC_RQP))) {
188 status = IBT_QP_SRV_TYPE_INVALID;
189 goto qpalloc_fail;
190 }
191
192
193 /*
194 * Check for valid PD handle pointer
195 */
196 if (attr_p->qp_pd_hdl == NULL) {
197 status = IBT_PD_HDL_INVALID;
198 goto qpalloc_fail;
199 }
200 pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl;
201
202 /*
203 * If on an SRQ, check to make sure the PD is the same
204 */
205 if (qp_srq_en && (pd->pd_pdnum != srq->srq_pdhdl->pd_pdnum)) {
206 status = IBT_PD_HDL_INVALID;
207 goto qpalloc_fail;
208 }
209
210 /* Increment the reference count on the protection domain (PD) */
211 hermon_pd_refcnt_inc(pd);
212
213 /*
214 * Check for valid CQ handle pointers
215 *
216 * FCMD QPs do not require a receive cq handle.
217 */
218 if (attr_p->qp_ibc_scq_hdl == NULL) {
219 status = IBT_CQ_HDL_INVALID;
220 goto qpalloc_fail1;
221 }
222 sq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_scq_hdl;
223 if ((attr_p->qp_ibc_rcq_hdl == NULL)) {
224 if ((alloc_flags & IBT_QP_USES_FCMD) == 0) {
225 status = IBT_CQ_HDL_INVALID;
226 goto qpalloc_fail1;
227 }
228 rq_cq = sq_cq; /* just use the send cq */
229 } else
230 rq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_rcq_hdl;
231
232 /*
233 * Increment the reference count on the CQs. One or both of these
234 * could return error if we determine that the given CQ is already
235 * being used with a special (SMI/GSI) QP.
236 */
237 status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_NORMAL);
238 if (status != DDI_SUCCESS) {
239 status = IBT_CQ_HDL_INVALID;
240 goto qpalloc_fail1;
241 }
242 status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_NORMAL);
243 if (status != DDI_SUCCESS) {
244 status = IBT_CQ_HDL_INVALID;
245 goto qpalloc_fail2;
246 }
247
248 /*
249 * Allocate an QP context entry. This will be filled in with all
250 * the necessary parameters to define the Queue Pair. Unlike
251 * other Hermon hardware resources, ownership is not immediately
252 * given to hardware in the final step here. Instead, we must
253 * wait until the QP is later transitioned to the "Init" state before
254 * passing the QP to hardware. If we fail here, we must undo all
255 * the reference count (CQ and PD).
256 */
257 status = hermon_rsrc_alloc(state, rsrc_type, 1, sleepflag, &qpc);
258 if (status != DDI_SUCCESS) {
259 status = IBT_INSUFF_RESOURCE;
260 goto qpalloc_fail3;
261 }
262
263 /*
264 * Allocate the software structure for tracking the queue pair
265 * (i.e. the Hermon Queue Pair handle). If we fail here, we must
266 * undo the reference counts and the previous resource allocation.
267 */
268 status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc);
269 if (status != DDI_SUCCESS) {
270 status = IBT_INSUFF_RESOURCE;
271 goto qpalloc_fail4;
272 }
273 qp = (hermon_qphdl_t)rsrc->hr_addr;
274 bzero(qp, sizeof (struct hermon_sw_qp_s));
275 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
276
277 qp->qp_alloc_flags = alloc_flags;
278
279 /*
280 * Calculate the QP number from QPC index. This routine handles
281 * all of the operations necessary to keep track of used, unused,
282 * and released QP numbers.
283 */
284 if (type == IBT_UD_RQP) {
285 qp->qp_qpnum = qpc->hr_indx;
286 qp->qp_ring = qp->qp_qpnum << 8;
287 qp->qp_qpn_hdl = NULL;
288 } else {
289 status = hermon_qp_create_qpn(state, qp, qpc);
290 if (status != DDI_SUCCESS) {
291 status = IBT_INSUFF_RESOURCE;
292 goto qpalloc_fail5;
293 }
294 }
295
296 /*
297 * If this will be a user-mappable QP, then allocate an entry for
298 * the "userland resources database". This will later be added to
299 * the database (after all further QP operations are successful).
300 * If we fail here, we must undo the reference counts and the
301 * previous resource allocation.
302 */
303 if (qp_is_umap) {
304 umapdb = hermon_umap_db_alloc(state->hs_instance, qp->qp_qpnum,
305 MLNX_UMAP_QPMEM_RSRC, (uint64_t)(uintptr_t)rsrc);
306 if (umapdb == NULL) {
307 status = IBT_INSUFF_RESOURCE;
308 goto qpalloc_fail6;
309 }
310 }
311
312 /*
313 * Allocate the doorbell record. Hermon just needs one for the RQ,
314 * if the QP is not associated with an SRQ, and use uarpg (above) as
315 * the uar index
316 */
317
318 if (!qp_srq_en) {
319 status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl,
320 &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset);
321 if (status != DDI_SUCCESS) {
322 status = IBT_INSUFF_RESOURCE;
323 goto qpalloc_fail6;
324 }
325 }
326
327 qp->qp_uses_lso = (attr_p->qp_flags & IBT_USES_LSO);
328
329 /*
330 * We verify that the requested number of SGL is valid (i.e.
331 * consistent with the device limits and/or software-configured
332 * limits). If not, then obviously the same cleanup needs to be done.
333 */
334 if (type == IBT_UD_RQP) {
335 max_sgl = state->hs_ibtfinfo.hca_attr->hca_ud_send_sgl_sz;
336 swq_type = HERMON_QP_WQ_TYPE_SENDQ_UD;
337 } else {
338 max_sgl = state->hs_ibtfinfo.hca_attr->hca_conn_send_sgl_sz;
339 swq_type = HERMON_QP_WQ_TYPE_SENDQ_CONN;
340 }
341 max_recv_sgl = state->hs_ibtfinfo.hca_attr->hca_recv_sgl_sz;
342 if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) ||
343 (!qp_srq_en && (attr_p->qp_sizes.cs_rq_sgl > max_recv_sgl))) {
344 status = IBT_HCA_SGL_EXCEEDED;
345 goto qpalloc_fail7;
346 }
347
348 /*
349 * Determine this QP's WQE stride (for both the Send and Recv WQEs).
350 * This will depend on the requested number of SGLs. Note: this
351 * has the side-effect of also calculating the real number of SGLs
352 * (for the calculated WQE size).
353 *
354 * For QP's on an SRQ, we set these to 0.
355 */
356 if (qp_srq_en) {
357 qp->qp_rq_log_wqesz = 0;
358 qp->qp_rq_sgl = 0;
359 } else {
360 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl,
361 max_recv_sgl, HERMON_QP_WQ_TYPE_RECVQ,
362 &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl);
363 }
364 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
365 max_sgl, swq_type, &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
366
367 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
368
369 /* NOTE: currently policy in driver, later maybe IBTF interface */
370 qp->qp_no_prefetch = 0;
371
372 /*
373 * for prefetching, we need to add the number of wqes in
374 * the 2k area plus one to the number requested, but
375 * ONLY for send queue. If no_prefetch == 1 (prefetch off)
376 * it's exactly TWO wqes for the headroom
377 */
378 if (qp->qp_no_prefetch)
379 qp->qp_sq_headroom = 2 * sq_wqe_size;
380 else
381 qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE;
382 /*
383 * hdrm wqes must be integral since both sq_wqe_size &
384 * HERMON_QP_OH_SIZE are power of 2
385 */
386 qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size);
387
388
389 /*
390 * Calculate the appropriate size for the work queues.
391 * For send queue, add in the headroom wqes to the calculation.
392 * Note: All Hermon QP work queues must be a power-of-2 in size. Also
393 * they may not be any smaller than HERMON_QP_MIN_SIZE. This step is
394 * to round the requested size up to the next highest power-of-2
395 */
396 /* first, adjust to a minimum and tell the caller the change */
397 attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq,
398 HERMON_QP_MIN_SIZE);
399 attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq,
400 HERMON_QP_MIN_SIZE);
401 /*
402 * now, calculate the alloc size, taking into account
403 * the headroom for the sq
404 */
405 log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes);
406 /* if the total is a power of two, reduce it */
407 if (((attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes) &
408 (attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes - 1)) == 0) {
409 log_qp_sq_size = log_qp_sq_size - 1;
410 }
411
412 log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq);
413 if ((attr_p->qp_sizes.cs_rq & (attr_p->qp_sizes.cs_rq - 1)) == 0) {
414 log_qp_rq_size = log_qp_rq_size - 1;
415 }
416
417 /*
418 * Next we verify that the rounded-up size is valid (i.e. consistent
419 * with the device limits and/or software-configured limits). If not,
420 * then obviously we have a lot of cleanup to do before returning.
421 *
422 * NOTE: the first condition deals with the (test) case of cs_sq
423 * being just less than 2^32. In this case, the headroom addition
424 * to the requested cs_sq will pass the test when it should not.
425 * This test no longer lets that case slip through the check.
426 */
427 if ((attr_p->qp_sizes.cs_sq >
428 (1 << state->hs_cfg_profile->cp_log_max_qp_sz)) ||
429 (log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) ||
430 (!qp_srq_en && (log_qp_rq_size >
431 state->hs_cfg_profile->cp_log_max_qp_sz))) {
432 status = IBT_HCA_WR_EXCEEDED;
433 goto qpalloc_fail7;
434 }
435
436 /*
437 * Allocate the memory for QP work queues. Since Hermon work queues
438 * are not allowed to cross a 32-bit (4GB) boundary, the alignment of
439 * the work queue memory is very important. We used to allocate
440 * work queues (the combined receive and send queues) so that they
441 * would be aligned on their combined size. That alignment guaranteed
442 * that they would never cross the 4GB boundary (Hermon work queues
443 * are on the order of MBs at maximum). Now we are able to relax
444 * this alignment constraint by ensuring that the IB address assigned
445 * to the queue memory (as a result of the hermon_mr_register() call)
446 * is offset from zero.
447 * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
448 * guarantee the alignment, but when attempting to use IOMMU bypass
449 * mode we found that we were not allowed to specify any alignment
450 * that was more restrictive than the system page size.
451 * So we avoided this constraint by passing two alignment values,
452 * one for the memory allocation itself and the other for the DMA
453 * handle (for later bind). This used to cause more memory than
454 * necessary to be allocated (in order to guarantee the more
455 * restrictive alignment contraint). But by guaranteeing the
456 * zero-based IB virtual address for the queue, we are able to
457 * conserve this memory.
458 */
459 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
460 sq_depth = 1 << log_qp_sq_size;
461 sq_size = sq_depth * sq_wqe_size;
462
463 /* QP on SRQ sets these to 0 */
464 if (qp_srq_en) {
465 rq_wqe_size = 0;
466 rq_size = 0;
467 } else {
468 rq_wqe_size = 1 << qp->qp_rq_log_wqesz;
469 rq_depth = 1 << log_qp_rq_size;
470 rq_size = rq_depth * rq_wqe_size;
471 }
472
473 qp->qp_wqinfo.qa_size = sq_size + rq_size;
474
475 qp->qp_wqinfo.qa_alloc_align = PAGESIZE;
476 qp->qp_wqinfo.qa_bind_align = PAGESIZE;
477
478 if (qp_is_umap) {
479 qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
480 } else {
481 qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
482 }
483 status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag);
484 if (status != DDI_SUCCESS) {
485 status = IBT_INSUFF_RESOURCE;
486 goto qpalloc_fail7;
487 }
488
489 /*
490 * Sort WQs in memory according to stride (*q_wqe_size), largest first
491 * If they are equal, still put the SQ first
492 */
493 qp->qp_sq_baseaddr = 0;
494 qp->qp_rq_baseaddr = 0;
495 if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) {
496 sq_buf = qp->qp_wqinfo.qa_buf_aligned;
497
498 /* if this QP is on an SRQ, set the rq_buf to NULL */
499 if (qp_srq_en) {
500 rq_buf = NULL;
501 } else {
502 rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size);
503 qp->qp_rq_baseaddr = sq_size;
504 }
505 } else {
506 rq_buf = qp->qp_wqinfo.qa_buf_aligned;
507 sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size);
508 qp->qp_sq_baseaddr = rq_size;
509 }
510
511 if (qp_is_umap == 0) {
512 qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth);
513 if (qp->qp_sq_wqhdr == NULL) {
514 status = IBT_INSUFF_RESOURCE;
515 goto qpalloc_fail8;
516 }
517 if (qp_srq_en) {
518 qp->qp_rq_wqavl.wqa_wq = srq->srq_wq_wqhdr;
519 qp->qp_rq_wqavl.wqa_srq_en = 1;
520 qp->qp_rq_wqavl.wqa_srq = srq;
521 } else {
522 qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(rq_depth);
523 if (qp->qp_rq_wqhdr == NULL) {
524 status = IBT_INSUFF_RESOURCE;
525 goto qpalloc_fail8;
526 }
527 qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr;
528 }
529 qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum;
530 qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND;
531 qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr;
532 qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum;
533 qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV;
534 }
535
536 /*
537 * Register the memory for the QP work queues. The memory for the
538 * QP must be registered in the Hermon cMPT tables. This gives us the
539 * LKey to specify in the QP context later. Note: The memory for
540 * Hermon work queues (both Send and Recv) must be contiguous and
541 * registered as a single memory region. Note: If the QP memory is
542 * user-mappable, force DDI_DMA_CONSISTENT mapping. Also, in order to
543 * meet the alignment restriction, we pass the "mro_bind_override_addr"
544 * flag in the call to hermon_mr_register(). This guarantees that the
545 * resulting IB vaddr will be zero-based (modulo the offset into the
546 * first page). If we fail here, we still have the bunch of resource
547 * and reference count cleanup to do.
548 */
549 flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
550 IBT_MR_NOSLEEP;
551 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
552 mr_attr.mr_len = qp->qp_wqinfo.qa_size;
553 mr_attr.mr_as = NULL;
554 mr_attr.mr_flags = flag;
555 if (qp_is_umap) {
556 mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
557 } else {
558 /* HERMON_QUEUE_LOCATION_NORMAL */
559 mr_op.mro_bind_type =
560 state->hs_cfg_profile->cp_iommu_bypass;
561 }
562 mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl;
563 mr_op.mro_bind_override_addr = 1;
564 status = hermon_mr_register(state, pd, &mr_attr, &mr,
565 &mr_op, HERMON_QP_CMPT);
566 if (status != DDI_SUCCESS) {
567 status = IBT_INSUFF_RESOURCE;
568 goto qpalloc_fail9;
569 }
570
571 /*
572 * Calculate the offset between the kernel virtual address space
573 * and the IB virtual address space. This will be used when
574 * posting work requests to properly initialize each WQE.
575 */
576 qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
577 (uint64_t)mr->mr_bindinfo.bi_addr;
578
579 /*
580 * Fill in all the return arguments (if necessary). This includes
581 * real work queue sizes (in wqes), real SGLs, and QP number
582 */
583 if (queuesz_p != NULL) {
584 queuesz_p->cs_sq =
585 (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes;
586 queuesz_p->cs_sq_sgl = qp->qp_sq_sgl;
587
588 /* if this QP is on an SRQ, set these to 0 */
589 if (qp_srq_en) {
590 queuesz_p->cs_rq = 0;
591 queuesz_p->cs_rq_sgl = 0;
592 } else {
593 queuesz_p->cs_rq = (1 << log_qp_rq_size);
594 queuesz_p->cs_rq_sgl = qp->qp_rq_sgl;
595 }
596 }
597 if (qpn != NULL) {
598 *qpn = (ib_qpn_t)qp->qp_qpnum;
599 }
600
601 /*
602 * Fill in the rest of the Hermon Queue Pair handle.
603 */
604 qp->qp_qpcrsrcp = qpc;
605 qp->qp_rsrcp = rsrc;
606 qp->qp_state = HERMON_QP_RESET;
607 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
608 qp->qp_pdhdl = pd;
609 qp->qp_mrhdl = mr;
610 qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ?
611 HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED;
612 qp->qp_is_special = 0;
613 qp->qp_uarpg = uarpg;
614 qp->qp_umap_dhp = (devmap_cookie_t)NULL;
615 qp->qp_sq_cqhdl = sq_cq;
616 qp->qp_sq_bufsz = (1 << log_qp_sq_size);
617 qp->qp_sq_logqsz = log_qp_sq_size;
618 qp->qp_sq_buf = sq_buf;
619 qp->qp_desc_off = qp_desc_off;
620 qp->qp_rq_cqhdl = rq_cq;
621 qp->qp_rq_buf = rq_buf;
622 qp->qp_rlky = (attr_p->qp_flags & IBT_FAST_REG_RES_LKEY) !=
623 0;
624
625 /* if this QP is on an SRQ, set rq_bufsz to 0 */
626 if (qp_srq_en) {
627 qp->qp_rq_bufsz = 0;
628 qp->qp_rq_logqsz = 0;
629 } else {
630 qp->qp_rq_bufsz = (1 << log_qp_rq_size);
631 qp->qp_rq_logqsz = log_qp_rq_size;
632 }
633
634 qp->qp_forward_sqd_event = 0;
635 qp->qp_sqd_still_draining = 0;
636 qp->qp_hdlrarg = (void *)ibt_qphdl;
637 qp->qp_mcg_refcnt = 0;
638
639 /*
640 * If this QP is to be associated with an SRQ, set the SRQ handle
641 */
642 if (qp_srq_en) {
643 qp->qp_srqhdl = srq;
644 hermon_srq_refcnt_inc(qp->qp_srqhdl);
645 } else {
646 qp->qp_srqhdl = NULL;
647 }
648
649 /* Determine the QP service type */
650 qp->qp_type = type;
651 if (type == IBT_RC_RQP) {
652 qp->qp_serv_type = HERMON_QP_RC;
653 } else if (type == IBT_UD_RQP) {
654 if (alloc_flags & IBT_QP_USES_RFCI)
655 qp->qp_serv_type = HERMON_QP_RFCI;
656 else if (alloc_flags & IBT_QP_USES_FCMD)
657 qp->qp_serv_type = HERMON_QP_FCMND;
658 else
659 qp->qp_serv_type = HERMON_QP_UD;
660 } else {
661 qp->qp_serv_type = HERMON_QP_UC;
662 }
663
664 /*
665 * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed
666 */
667
668 /*
669 * Initialize the SQ WQEs - all that needs to be done is every 64 bytes
670 * set the quadword to all F's - high-order bit is owner (init to one)
671 * and the rest for the headroom definition of prefetching
672 *
673 */
674 wqesz_shift = qp->qp_sq_log_wqesz;
675 thewqesz = 1 << wqesz_shift;
676 thewqe = (uint64_t *)(void *)(qp->qp_sq_buf);
677 if (qp_is_umap == 0) {
678 for (i = 0; i < sq_depth; i++) {
679 /*
680 * for each stride, go through and every 64 bytes
681 * write the init value - having set the address
682 * once, just keep incrementing it
683 */
684 for (j = 0; j < thewqesz; j += 64, thewqe += 8) {
685 *(uint32_t *)thewqe = 0xFFFFFFFF;
686 }
687 }
688 }
689
690 /* Zero out the QP context */
691 bzero(&qp->qpc, sizeof (hermon_hw_qpc_t));
692
693 /*
694 * Put QP handle in Hermon QPNum-to-QPHdl list. Then fill in the
695 * "qphdl" and return success
696 */
697 hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx, qp);
698
699 /*
700 * If this is a user-mappable QP, then we need to insert the previously
701 * allocated entry into the "userland resources database". This will
702 * allow for later lookup during devmap() (i.e. mmap()) calls.
703 */
704 if (qp_is_umap) {
705 hermon_umap_db_add(umapdb);
706 }
707 mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER,
708 DDI_INTR_PRI(state->hs_intrmsi_pri));
709
710 *qphdl = qp;
711
712 return (DDI_SUCCESS);
713
714 /*
715 * The following is cleanup for all possible failure cases in this routine
716 */
717 qpalloc_fail9:
718 hermon_queue_free(&qp->qp_wqinfo);
719 qpalloc_fail8:
720 if (qp->qp_sq_wqhdr)
721 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
722 if (qp->qp_rq_wqhdr)
723 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
724 qpalloc_fail7:
725 if (qp_is_umap) {
726 hermon_umap_db_free(umapdb);
727 }
728 if (!qp_srq_en) {
729 hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr);
730 }
731
732 qpalloc_fail6:
733 /*
734 * Releasing the QPN will also free up the QPC context. Update
735 * the QPC context pointer to indicate this.
736 */
737 if (qp->qp_qpn_hdl) {
738 hermon_qp_release_qpn(state, qp->qp_qpn_hdl,
739 HERMON_QPN_RELEASE);
740 } else {
741 hermon_rsrc_free(state, &qpc);
742 }
743 qpc = NULL;
744 qpalloc_fail5:
745 hermon_rsrc_free(state, &rsrc);
746 qpalloc_fail4:
747 if (qpc) {
748 hermon_rsrc_free(state, &qpc);
749 }
750 qpalloc_fail3:
751 hermon_cq_refcnt_dec(rq_cq);
752 qpalloc_fail2:
753 hermon_cq_refcnt_dec(sq_cq);
754 qpalloc_fail1:
755 hermon_pd_refcnt_dec(pd);
756 qpalloc_fail:
757 return (status);
758 }
759
760
761
762 /*
763 * hermon_special_qp_alloc()
764 * Context: Can be called only from user or kernel context.
765 */
766 int
767 hermon_special_qp_alloc(hermon_state_t *state, hermon_qp_info_t *qpinfo,
768 uint_t sleepflag)
769 {
770 hermon_rsrc_t *qpc, *rsrc;
771 hermon_qphdl_t qp;
772 ibt_qp_alloc_attr_t *attr_p;
773 ibt_sqp_type_t type;
774 uint8_t port;
775 ibtl_qp_hdl_t ibt_qphdl;
776 ibt_chan_sizes_t *queuesz_p;
777 hermon_qphdl_t *qphdl;
778 ibt_mr_attr_t mr_attr;
779 hermon_mr_options_t mr_op;
780 hermon_pdhdl_t pd;
781 hermon_cqhdl_t sq_cq, rq_cq;
782 hermon_mrhdl_t mr;
783 uint64_t qp_desc_off;
784 uint64_t *thewqe, thewqesz;
785 uint32_t *sq_buf, *rq_buf;
786 uint32_t log_qp_sq_size, log_qp_rq_size;
787 uint32_t sq_size, rq_size, max_sgl;
788 uint32_t uarpg;
789 uint32_t sq_depth;
790 uint32_t sq_wqe_size, rq_wqe_size, wqesz_shift;
791 int status, flag, i, j;
792
793 /*
794 * Extract the necessary info from the hermon_qp_info_t structure
795 */
796 attr_p = qpinfo->qpi_attrp;
797 type = qpinfo->qpi_type;
798 port = qpinfo->qpi_port;
799 ibt_qphdl = qpinfo->qpi_ibt_qphdl;
800 queuesz_p = qpinfo->qpi_queueszp;
801 qphdl = &qpinfo->qpi_qphdl;
802
803 /*
804 * Check for valid special QP type (only SMI & GSI supported)
805 */
806 if ((type != IBT_SMI_SQP) && (type != IBT_GSI_SQP)) {
807 status = IBT_QP_SPECIAL_TYPE_INVALID;
808 goto spec_qpalloc_fail;
809 }
810
811 /*
812 * Check for valid port number
813 */
814 if (!hermon_portnum_is_valid(state, port)) {
815 status = IBT_HCA_PORT_INVALID;
816 goto spec_qpalloc_fail;
817 }
818 port = port - 1;
819
820 /*
821 * Check for valid PD handle pointer
822 */
823 if (attr_p->qp_pd_hdl == NULL) {
824 status = IBT_PD_HDL_INVALID;
825 goto spec_qpalloc_fail;
826 }
827 pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl;
828
829 /* Increment the reference count on the PD */
830 hermon_pd_refcnt_inc(pd);
831
832 /*
833 * Check for valid CQ handle pointers
834 */
835 if ((attr_p->qp_ibc_scq_hdl == NULL) ||
836 (attr_p->qp_ibc_rcq_hdl == NULL)) {
837 status = IBT_CQ_HDL_INVALID;
838 goto spec_qpalloc_fail1;
839 }
840 sq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_scq_hdl;
841 rq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_rcq_hdl;
842
843 /*
844 * Increment the reference count on the CQs. One or both of these
845 * could return error if we determine that the given CQ is already
846 * being used with a non-special QP (i.e. a normal QP).
847 */
848 status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_SPECIAL);
849 if (status != DDI_SUCCESS) {
850 status = IBT_CQ_HDL_INVALID;
851 goto spec_qpalloc_fail1;
852 }
853 status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_SPECIAL);
854 if (status != DDI_SUCCESS) {
855 status = IBT_CQ_HDL_INVALID;
856 goto spec_qpalloc_fail2;
857 }
858
859 /*
860 * Allocate the special QP resources. Essentially, this allocation
861 * amounts to checking if the request special QP has already been
862 * allocated. If successful, the QP context return is an actual
863 * QP context that has been "aliased" to act as a special QP of the
864 * appropriate type (and for the appropriate port). Just as in
865 * hermon_qp_alloc() above, ownership for this QP context is not
866 * immediately given to hardware in the final step here. Instead, we
867 * wait until the QP is later transitioned to the "Init" state before
868 * passing the QP to hardware. If we fail here, we must undo all
869 * the reference count (CQ and PD).
870 */
871 status = hermon_special_qp_rsrc_alloc(state, type, port, &qpc);
872 if (status != DDI_SUCCESS) {
873 goto spec_qpalloc_fail3;
874 }
875
876 /*
877 * Allocate the software structure for tracking the special queue
878 * pair (i.e. the Hermon Queue Pair handle). If we fail here, we
879 * must undo the reference counts and the previous resource allocation.
880 */
881 status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc);
882 if (status != DDI_SUCCESS) {
883 status = IBT_INSUFF_RESOURCE;
884 goto spec_qpalloc_fail4;
885 }
886 qp = (hermon_qphdl_t)rsrc->hr_addr;
887
888 bzero(qp, sizeof (struct hermon_sw_qp_s));
889
890 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
891 qp->qp_alloc_flags = attr_p->qp_alloc_flags;
892
893 /*
894 * Actual QP number is a combination of the index of the QPC and
895 * the port number. This is because the special QP contexts must
896 * be allocated two-at-a-time.
897 */
898 qp->qp_qpnum = qpc->hr_indx + port;
899 qp->qp_ring = qp->qp_qpnum << 8;
900
901 uarpg = state->hs_kernel_uar_index; /* must be for spec qp */
902 /*
903 * Allocate the doorbell record. Hermon uses only one for the RQ so
904 * alloc a qp doorbell, using uarpg (above) as the uar index
905 */
906
907 status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl,
908 &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset);
909 if (status != DDI_SUCCESS) {
910 status = IBT_INSUFF_RESOURCE;
911 goto spec_qpalloc_fail5;
912 }
913 /*
914 * Calculate the appropriate size for the work queues.
915 * Note: All Hermon QP work queues must be a power-of-2 in size. Also
916 * they may not be any smaller than HERMON_QP_MIN_SIZE. This step is
917 * to round the requested size up to the next highest power-of-2
918 */
919 attr_p->qp_sizes.cs_sq =
920 max(attr_p->qp_sizes.cs_sq, HERMON_QP_MIN_SIZE);
921 attr_p->qp_sizes.cs_rq =
922 max(attr_p->qp_sizes.cs_rq, HERMON_QP_MIN_SIZE);
923 log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq);
924 if ((attr_p->qp_sizes.cs_sq & (attr_p->qp_sizes.cs_sq - 1)) == 0) {
925 log_qp_sq_size = log_qp_sq_size - 1;
926 }
927 log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq);
928 if ((attr_p->qp_sizes.cs_rq & (attr_p->qp_sizes.cs_rq - 1)) == 0) {
929 log_qp_rq_size = log_qp_rq_size - 1;
930 }
931
932 /*
933 * Next we verify that the rounded-up size is valid (i.e. consistent
934 * with the device limits and/or software-configured limits). If not,
935 * then obviously we have a bit of cleanup to do before returning.
936 */
937 if ((log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) ||
938 (log_qp_rq_size > state->hs_cfg_profile->cp_log_max_qp_sz)) {
939 status = IBT_HCA_WR_EXCEEDED;
940 goto spec_qpalloc_fail5a;
941 }
942
943 /*
944 * Next we verify that the requested number of SGL is valid (i.e.
945 * consistent with the device limits and/or software-configured
946 * limits). If not, then obviously the same cleanup needs to be done.
947 */
948 max_sgl = state->hs_cfg_profile->cp_wqe_real_max_sgl;
949 if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) ||
950 (attr_p->qp_sizes.cs_rq_sgl > max_sgl)) {
951 status = IBT_HCA_SGL_EXCEEDED;
952 goto spec_qpalloc_fail5a;
953 }
954
955 /*
956 * Determine this QP's WQE stride (for both the Send and Recv WQEs).
957 * This will depend on the requested number of SGLs. Note: this
958 * has the side-effect of also calculating the real number of SGLs
959 * (for the calculated WQE size).
960 */
961 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl,
962 max_sgl, HERMON_QP_WQ_TYPE_RECVQ,
963 &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl);
964 if (type == IBT_SMI_SQP) {
965 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
966 max_sgl, HERMON_QP_WQ_TYPE_SENDMLX_QP0,
967 &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
968 } else {
969 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
970 max_sgl, HERMON_QP_WQ_TYPE_SENDMLX_QP1,
971 &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
972 }
973
974 /*
975 * Allocate the memory for QP work queues. Since Hermon work queues
976 * are not allowed to cross a 32-bit (4GB) boundary, the alignment of
977 * the work queue memory is very important. We used to allocate
978 * work queues (the combined receive and send queues) so that they
979 * would be aligned on their combined size. That alignment guaranteed
980 * that they would never cross the 4GB boundary (Hermon work queues
981 * are on the order of MBs at maximum). Now we are able to relax
982 * this alignment constraint by ensuring that the IB address assigned
983 * to the queue memory (as a result of the hermon_mr_register() call)
984 * is offset from zero.
985 * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
986 * guarantee the alignment, but when attempting to use IOMMU bypass
987 * mode we found that we were not allowed to specify any alignment
988 * that was more restrictive than the system page size.
989 * So we avoided this constraint by passing two alignment values,
990 * one for the memory allocation itself and the other for the DMA
991 * handle (for later bind). This used to cause more memory than
992 * necessary to be allocated (in order to guarantee the more
993 * restrictive alignment contraint). But by guaranteeing the
994 * zero-based IB virtual address for the queue, we are able to
995 * conserve this memory.
996 */
997 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
998 sq_depth = 1 << log_qp_sq_size;
999 sq_size = (1 << log_qp_sq_size) * sq_wqe_size;
1000
1001 rq_wqe_size = 1 << qp->qp_rq_log_wqesz;
1002 rq_size = (1 << log_qp_rq_size) * rq_wqe_size;
1003
1004 qp->qp_wqinfo.qa_size = sq_size + rq_size;
1005
1006 qp->qp_wqinfo.qa_alloc_align = PAGESIZE;
1007 qp->qp_wqinfo.qa_bind_align = PAGESIZE;
1008 qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
1009
1010 status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag);
1011 if (status != NULL) {
1012 status = IBT_INSUFF_RESOURCE;
1013 goto spec_qpalloc_fail5a;
1014 }
1015
1016 /*
1017 * Sort WQs in memory according to depth, stride (*q_wqe_size),
1018 * biggest first. If equal, the Send Queue still goes first
1019 */
1020 qp->qp_sq_baseaddr = 0;
1021 qp->qp_rq_baseaddr = 0;
1022 if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) {
1023 sq_buf = qp->qp_wqinfo.qa_buf_aligned;
1024 rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size);
1025 qp->qp_rq_baseaddr = sq_size;
1026 } else {
1027 rq_buf = qp->qp_wqinfo.qa_buf_aligned;
1028 sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size);
1029 qp->qp_sq_baseaddr = rq_size;
1030 }
1031
1032 qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth);
1033 if (qp->qp_sq_wqhdr == NULL) {
1034 status = IBT_INSUFF_RESOURCE;
1035 goto spec_qpalloc_fail6;
1036 }
1037 qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(1 << log_qp_rq_size);
1038 if (qp->qp_rq_wqhdr == NULL) {
1039 status = IBT_INSUFF_RESOURCE;
1040 goto spec_qpalloc_fail6;
1041 }
1042 qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum;
1043 qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND;
1044 qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr;
1045 qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum;
1046 qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV;
1047 qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr;
1048
1049 /*
1050 * Register the memory for the special QP work queues. The memory for
1051 * the special QP must be registered in the Hermon cMPT tables. This
1052 * gives us the LKey to specify in the QP context later. Note: The
1053 * memory for Hermon work queues (both Send and Recv) must be contiguous
1054 * and registered as a single memory region. Also, in order to meet the
1055 * alignment restriction, we pass the "mro_bind_override_addr" flag in
1056 * the call to hermon_mr_register(). This guarantees that the resulting
1057 * IB vaddr will be zero-based (modulo the offset into the first page).
1058 * If we fail here, we have a bunch of resource and reference count
1059 * cleanup to do.
1060 */
1061 flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
1062 IBT_MR_NOSLEEP;
1063 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
1064 mr_attr.mr_len = qp->qp_wqinfo.qa_size;
1065 mr_attr.mr_as = NULL;
1066 mr_attr.mr_flags = flag;
1067
1068 mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
1069 mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl;
1070 mr_op.mro_bind_override_addr = 1;
1071
1072 status = hermon_mr_register(state, pd, &mr_attr, &mr, &mr_op,
1073 HERMON_QP_CMPT);
1074 if (status != DDI_SUCCESS) {
1075 status = IBT_INSUFF_RESOURCE;
1076 goto spec_qpalloc_fail6;
1077 }
1078
1079 /*
1080 * Calculate the offset between the kernel virtual address space
1081 * and the IB virtual address space. This will be used when
1082 * posting work requests to properly initialize each WQE.
1083 */
1084 qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
1085 (uint64_t)mr->mr_bindinfo.bi_addr;
1086
1087 /* set the prefetch - initially, not prefetching */
1088 qp->qp_no_prefetch = 1;
1089
1090 if (qp->qp_no_prefetch)
1091 qp->qp_sq_headroom = 2 * sq_wqe_size;
1092 else
1093 qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE;
1094 /*
1095 * hdrm wqes must be integral since both sq_wqe_size &
1096 * HERMON_QP_OH_SIZE are power of 2
1097 */
1098 qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size);
1099 /*
1100 * Fill in all the return arguments (if necessary). This includes
1101 * real work queue sizes, real SGLs, and QP number (which will be
1102 * either zero or one, depending on the special QP type)
1103 */
1104 if (queuesz_p != NULL) {
1105 queuesz_p->cs_sq =
1106 (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes;
1107 queuesz_p->cs_sq_sgl = qp->qp_sq_sgl;
1108 queuesz_p->cs_rq = (1 << log_qp_rq_size);
1109 queuesz_p->cs_rq_sgl = qp->qp_rq_sgl;
1110 }
1111
1112 /*
1113 * Fill in the rest of the Hermon Queue Pair handle. We can update
1114 * the following fields for use in further operations on the QP.
1115 */
1116 qp->qp_qpcrsrcp = qpc;
1117 qp->qp_rsrcp = rsrc;
1118 qp->qp_state = HERMON_QP_RESET;
1119 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
1120 qp->qp_pdhdl = pd;
1121 qp->qp_mrhdl = mr;
1122 qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ?
1123 HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED;
1124 qp->qp_is_special = (type == IBT_SMI_SQP) ?
1125 HERMON_QP_SMI : HERMON_QP_GSI;
1126 qp->qp_uarpg = uarpg;
1127 qp->qp_umap_dhp = (devmap_cookie_t)NULL;
1128 qp->qp_sq_cqhdl = sq_cq;
1129 qp->qp_sq_bufsz = (1 << log_qp_sq_size);
1130 qp->qp_sq_buf = sq_buf;
1131 qp->qp_sq_logqsz = log_qp_sq_size;
1132 qp->qp_desc_off = qp_desc_off;
1133 qp->qp_rq_cqhdl = rq_cq;
1134 qp->qp_rq_bufsz = (1 << log_qp_rq_size);
1135 qp->qp_rq_buf = rq_buf;
1136 qp->qp_rq_logqsz = log_qp_rq_size;
1137 qp->qp_portnum = port;
1138 qp->qp_pkeyindx = 0;
1139 qp->qp_forward_sqd_event = 0;
1140 qp->qp_sqd_still_draining = 0;
1141 qp->qp_hdlrarg = (void *)ibt_qphdl;
1142 qp->qp_mcg_refcnt = 0;
1143 qp->qp_srqhdl = NULL;
1144
1145 /* All special QPs are UD QP service type */
1146 qp->qp_type = IBT_UD_RQP;
1147 qp->qp_serv_type = HERMON_QP_UD;
1148
1149 /*
1150 * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed
1151 */
1152
1153 /*
1154 * Initialize the SQ WQEs - all that needs to be done is every 64 bytes
1155 * set the quadword to all F's - high-order bit is owner (init to one)
1156 * and the rest for the headroom definition of prefetching
1157 *
1158 */
1159
1160 wqesz_shift = qp->qp_sq_log_wqesz;
1161 thewqesz = 1 << wqesz_shift;
1162 thewqe = (uint64_t *)(void *)(qp->qp_sq_buf);
1163 for (i = 0; i < sq_depth; i++) {
1164 /*
1165 * for each stride, go through and every 64 bytes write the
1166 * init value - having set the address once, just keep
1167 * incrementing it
1168 */
1169 for (j = 0; j < thewqesz; j += 64, thewqe += 8) {
1170 *(uint32_t *)thewqe = 0xFFFFFFFF;
1171 }
1172 }
1173
1174
1175 /* Zero out the QP context */
1176 bzero(&qp->qpc, sizeof (hermon_hw_qpc_t));
1177
1178 /*
1179 * Put QP handle in Hermon QPNum-to-QPHdl list. Then fill in the
1180 * "qphdl" and return success
1181 */
1182 hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx + port, qp);
1183
1184 mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER,
1185 DDI_INTR_PRI(state->hs_intrmsi_pri));
1186
1187 *qphdl = qp;
1188
1189 return (DDI_SUCCESS);
1190
1191 /*
1192 * The following is cleanup for all possible failure cases in this routine
1193 */
1194 spec_qpalloc_fail6:
1195 hermon_queue_free(&qp->qp_wqinfo);
1196 if (qp->qp_sq_wqhdr)
1197 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
1198 if (qp->qp_rq_wqhdr)
1199 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
1200 spec_qpalloc_fail5a:
1201 hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr);
1202 spec_qpalloc_fail5:
1203 hermon_rsrc_free(state, &rsrc);
1204 spec_qpalloc_fail4:
1205 if (hermon_special_qp_rsrc_free(state, type, port) != DDI_SUCCESS) {
1206 HERMON_WARNING(state, "failed to free special QP rsrc");
1207 }
1208 spec_qpalloc_fail3:
1209 hermon_cq_refcnt_dec(rq_cq);
1210 spec_qpalloc_fail2:
1211 hermon_cq_refcnt_dec(sq_cq);
1212 spec_qpalloc_fail1:
1213 hermon_pd_refcnt_dec(pd);
1214 spec_qpalloc_fail:
1215 return (status);
1216 }
1217
1218
1219 /*
1220 * hermon_qp_alloc_range()
1221 * Context: Can be called only from user or kernel context.
1222 */
1223 int
1224 hermon_qp_alloc_range(hermon_state_t *state, uint_t log2,
1225 hermon_qp_info_t *qpinfo, ibtl_qp_hdl_t *ibt_qphdl,
1226 ibc_cq_hdl_t *send_cq, ibc_cq_hdl_t *recv_cq,
1227 hermon_qphdl_t *qphdl, uint_t sleepflag)
1228 {
1229 hermon_rsrc_t *qpc, *rsrc;
1230 hermon_rsrc_type_t rsrc_type;
1231 hermon_qphdl_t qp;
1232 hermon_qp_range_t *qp_range_p;
1233 ibt_qp_alloc_attr_t *attr_p;
1234 ibt_qp_type_t type;
1235 hermon_qp_wq_type_t swq_type;
1236 ibt_chan_sizes_t *queuesz_p;
1237 ibt_mr_attr_t mr_attr;
1238 hermon_mr_options_t mr_op;
1239 hermon_srqhdl_t srq;
1240 hermon_pdhdl_t pd;
1241 hermon_cqhdl_t sq_cq, rq_cq;
1242 hermon_mrhdl_t mr;
1243 uint64_t qp_desc_off;
1244 uint64_t *thewqe, thewqesz;
1245 uint32_t *sq_buf, *rq_buf;
1246 uint32_t log_qp_sq_size, log_qp_rq_size;
1247 uint32_t sq_size, rq_size;
1248 uint32_t sq_depth, rq_depth;
1249 uint32_t sq_wqe_size, rq_wqe_size, wqesz_shift;
1250 uint32_t max_sgl, max_recv_sgl, uarpg;
1251 uint_t qp_srq_en, i, j;
1252 int ii; /* loop counter for range */
1253 int status, flag;
1254 uint_t serv_type;
1255
1256 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p, *queuesz_p))
1257
1258 /*
1259 * Extract the necessary info from the hermon_qp_info_t structure
1260 */
1261 attr_p = qpinfo->qpi_attrp;
1262 type = qpinfo->qpi_type;
1263 queuesz_p = qpinfo->qpi_queueszp;
1264
1265 if (attr_p->qp_alloc_flags & IBT_QP_USES_RSS) {
1266 if (log2 > state->hs_ibtfinfo.hca_attr->hca_rss_max_log2_table)
1267 return (IBT_INSUFF_RESOURCE);
1268 rsrc_type = HERMON_QPC;
1269 serv_type = HERMON_QP_UD;
1270 } else if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) {
1271 if (log2 > state->hs_ibtfinfo.hca_attr->hca_fexch_max_log2_qp)
1272 return (IBT_INSUFF_RESOURCE);
1273 switch (attr_p->qp_fc.fc_hca_port) {
1274 case 1:
1275 rsrc_type = HERMON_QPC_FEXCH_PORT1;
1276 break;
1277 case 2:
1278 rsrc_type = HERMON_QPC_FEXCH_PORT2;
1279 break;
1280 default:
1281 return (IBT_INVALID_PARAM);
1282 }
1283 serv_type = HERMON_QP_FEXCH;
1284 } else
1285 return (IBT_INVALID_PARAM);
1286
1287 /*
1288 * Determine whether QP is being allocated for userland access or
1289 * whether it is being allocated for kernel access. If the QP is
1290 * being allocated for userland access, fail (too complex for now).
1291 */
1292 if (attr_p->qp_alloc_flags & IBT_QP_USER_MAP) {
1293 return (IBT_NOT_SUPPORTED);
1294 } else {
1295 uarpg = state->hs_kernel_uar_index;
1296 }
1297
1298 /*
1299 * Determine whether QP is being associated with an SRQ
1300 */
1301 qp_srq_en = (attr_p->qp_alloc_flags & IBT_QP_USES_SRQ) ? 1 : 0;
1302 if (qp_srq_en) {
1303 /*
1304 * Check for valid SRQ handle pointers
1305 */
1306 if (attr_p->qp_ibc_srq_hdl == NULL) {
1307 return (IBT_SRQ_HDL_INVALID);
1308 }
1309 srq = (hermon_srqhdl_t)attr_p->qp_ibc_srq_hdl;
1310 }
1311
1312 /*
1313 * Check for valid QP service type (only UD supported)
1314 */
1315 if (type != IBT_UD_RQP) {
1316 return (IBT_QP_SRV_TYPE_INVALID);
1317 }
1318
1319 /*
1320 * Check for valid PD handle pointer
1321 */
1322 if (attr_p->qp_pd_hdl == NULL) {
1323 return (IBT_PD_HDL_INVALID);
1324 }
1325 pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl;
1326
1327 /*
1328 * If on an SRQ, check to make sure the PD is the same
1329 */
1330 if (qp_srq_en && (pd->pd_pdnum != srq->srq_pdhdl->pd_pdnum)) {
1331 return (IBT_PD_HDL_INVALID);
1332 }
1333
1334 /* set loop variable here, for freeing resources on error */
1335 ii = 0;
1336
1337 /*
1338 * Allocate 2^log2 contiguous/aligned QP context entries. This will
1339 * be filled in with all the necessary parameters to define the
1340 * Queue Pairs. Unlike other Hermon hardware resources, ownership
1341 * is not immediately given to hardware in the final step here.
1342 * Instead, we must wait until the QP is later transitioned to the
1343 * "Init" state before passing the QP to hardware. If we fail here,
1344 * we must undo all the reference count (CQ and PD).
1345 */
1346 status = hermon_rsrc_alloc(state, rsrc_type, 1 << log2, sleepflag,
1347 &qpc);
1348 if (status != DDI_SUCCESS) {
1349 return (IBT_INSUFF_RESOURCE);
1350 }
1351
1352 if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH)
1353 /*
1354 * Need to init the MKEYs for the FEXCH QPs.
1355 *
1356 * For FEXCH QP subranges, we return the QPN base as
1357 * "relative" to the full FEXCH QP range for the port.
1358 */
1359 *(qpinfo->qpi_qpn) = hermon_fcoib_fexch_relative_qpn(state,
1360 attr_p->qp_fc.fc_hca_port, qpc->hr_indx);
1361 else
1362 *(qpinfo->qpi_qpn) = (ib_qpn_t)qpc->hr_indx;
1363
1364 qp_range_p = kmem_alloc(sizeof (*qp_range_p),
1365 (sleepflag == HERMON_SLEEP) ? KM_SLEEP : KM_NOSLEEP);
1366 if (qp_range_p == NULL) {
1367 status = IBT_INSUFF_RESOURCE;
1368 goto qpalloc_fail0;
1369 }
1370 mutex_init(&qp_range_p->hqpr_lock, NULL, MUTEX_DRIVER,
1371 DDI_INTR_PRI(state->hs_intrmsi_pri));
1372 mutex_enter(&qp_range_p->hqpr_lock);
1373 qp_range_p->hqpr_refcnt = 1 << log2;
1374 qp_range_p->hqpr_qpcrsrc = qpc;
1375 mutex_exit(&qp_range_p->hqpr_lock);
1376
1377 for_each_qp:
1378
1379 /* Increment the reference count on the protection domain (PD) */
1380 hermon_pd_refcnt_inc(pd);
1381
1382 rq_cq = (hermon_cqhdl_t)recv_cq[ii];
1383 sq_cq = (hermon_cqhdl_t)send_cq[ii];
1384 if (sq_cq == NULL) {
1385 if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) {
1386 /* if no send completions, just use rq_cq */
1387 sq_cq = rq_cq;
1388 } else {
1389 status = IBT_CQ_HDL_INVALID;
1390 goto qpalloc_fail1;
1391 }
1392 }
1393
1394 /*
1395 * Increment the reference count on the CQs. One or both of these
1396 * could return error if we determine that the given CQ is already
1397 * being used with a special (SMI/GSI) QP.
1398 */
1399 status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_NORMAL);
1400 if (status != DDI_SUCCESS) {
1401 status = IBT_CQ_HDL_INVALID;
1402 goto qpalloc_fail1;
1403 }
1404 status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_NORMAL);
1405 if (status != DDI_SUCCESS) {
1406 status = IBT_CQ_HDL_INVALID;
1407 goto qpalloc_fail2;
1408 }
1409
1410 /*
1411 * Allocate the software structure for tracking the queue pair
1412 * (i.e. the Hermon Queue Pair handle). If we fail here, we must
1413 * undo the reference counts and the previous resource allocation.
1414 */
1415 status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc);
1416 if (status != DDI_SUCCESS) {
1417 status = IBT_INSUFF_RESOURCE;
1418 goto qpalloc_fail4;
1419 }
1420 qp = (hermon_qphdl_t)rsrc->hr_addr;
1421 bzero(qp, sizeof (struct hermon_sw_qp_s));
1422 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
1423 qp->qp_alloc_flags = attr_p->qp_alloc_flags;
1424
1425 /*
1426 * Calculate the QP number from QPC index. This routine handles
1427 * all of the operations necessary to keep track of used, unused,
1428 * and released QP numbers.
1429 */
1430 qp->qp_qpnum = qpc->hr_indx + ii;
1431 qp->qp_ring = qp->qp_qpnum << 8;
1432 qp->qp_qpn_hdl = NULL;
1433
1434 /*
1435 * Allocate the doorbell record. Hermon just needs one for the RQ,
1436 * if the QP is not associated with an SRQ, and use uarpg (above) as
1437 * the uar index
1438 */
1439
1440 if (!qp_srq_en) {
1441 status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl,
1442 &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset);
1443 if (status != DDI_SUCCESS) {
1444 status = IBT_INSUFF_RESOURCE;
1445 goto qpalloc_fail6;
1446 }
1447 }
1448
1449 qp->qp_uses_lso = (attr_p->qp_flags & IBT_USES_LSO);
1450
1451 /*
1452 * We verify that the requested number of SGL is valid (i.e.
1453 * consistent with the device limits and/or software-configured
1454 * limits). If not, then obviously the same cleanup needs to be done.
1455 */
1456 max_sgl = state->hs_ibtfinfo.hca_attr->hca_ud_send_sgl_sz;
1457 swq_type = HERMON_QP_WQ_TYPE_SENDQ_UD;
1458 max_recv_sgl = state->hs_ibtfinfo.hca_attr->hca_recv_sgl_sz;
1459 if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) ||
1460 (!qp_srq_en && (attr_p->qp_sizes.cs_rq_sgl > max_recv_sgl))) {
1461 status = IBT_HCA_SGL_EXCEEDED;
1462 goto qpalloc_fail7;
1463 }
1464
1465 /*
1466 * Determine this QP's WQE stride (for both the Send and Recv WQEs).
1467 * This will depend on the requested number of SGLs. Note: this
1468 * has the side-effect of also calculating the real number of SGLs
1469 * (for the calculated WQE size).
1470 *
1471 * For QP's on an SRQ, we set these to 0.
1472 */
1473 if (qp_srq_en) {
1474 qp->qp_rq_log_wqesz = 0;
1475 qp->qp_rq_sgl = 0;
1476 } else {
1477 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl,
1478 max_recv_sgl, HERMON_QP_WQ_TYPE_RECVQ,
1479 &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl);
1480 }
1481 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
1482 max_sgl, swq_type, &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
1483
1484 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
1485
1486 /* NOTE: currently policy in driver, later maybe IBTF interface */
1487 qp->qp_no_prefetch = 0;
1488
1489 /*
1490 * for prefetching, we need to add the number of wqes in
1491 * the 2k area plus one to the number requested, but
1492 * ONLY for send queue. If no_prefetch == 1 (prefetch off)
1493 * it's exactly TWO wqes for the headroom
1494 */
1495 if (qp->qp_no_prefetch)
1496 qp->qp_sq_headroom = 2 * sq_wqe_size;
1497 else
1498 qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE;
1499 /*
1500 * hdrm wqes must be integral since both sq_wqe_size &
1501 * HERMON_QP_OH_SIZE are power of 2
1502 */
1503 qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size);
1504
1505
1506 /*
1507 * Calculate the appropriate size for the work queues.
1508 * For send queue, add in the headroom wqes to the calculation.
1509 * Note: All Hermon QP work queues must be a power-of-2 in size. Also
1510 * they may not be any smaller than HERMON_QP_MIN_SIZE. This step is
1511 * to round the requested size up to the next highest power-of-2
1512 */
1513 /* first, adjust to a minimum and tell the caller the change */
1514 attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq,
1515 HERMON_QP_MIN_SIZE);
1516 attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq,
1517 HERMON_QP_MIN_SIZE);
1518 /*
1519 * now, calculate the alloc size, taking into account
1520 * the headroom for the sq
1521 */
1522 log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes);
1523 /* if the total is a power of two, reduce it */
1524 if (((attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes) &
1525 (attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes - 1)) == 0) {
1526 log_qp_sq_size = log_qp_sq_size - 1;
1527 }
1528
1529 log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq);
1530 if ((attr_p->qp_sizes.cs_rq & (attr_p->qp_sizes.cs_rq - 1)) == 0) {
1531 log_qp_rq_size = log_qp_rq_size - 1;
1532 }
1533
1534 /*
1535 * Next we verify that the rounded-up size is valid (i.e. consistent
1536 * with the device limits and/or software-configured limits). If not,
1537 * then obviously we have a lot of cleanup to do before returning.
1538 *
1539 * NOTE: the first condition deals with the (test) case of cs_sq
1540 * being just less than 2^32. In this case, the headroom addition
1541 * to the requested cs_sq will pass the test when it should not.
1542 * This test no longer lets that case slip through the check.
1543 */
1544 if ((attr_p->qp_sizes.cs_sq >
1545 (1 << state->hs_cfg_profile->cp_log_max_qp_sz)) ||
1546 (log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) ||
1547 (!qp_srq_en && (log_qp_rq_size >
1548 state->hs_cfg_profile->cp_log_max_qp_sz))) {
1549 status = IBT_HCA_WR_EXCEEDED;
1550 goto qpalloc_fail7;
1551 }
1552
1553 /*
1554 * Allocate the memory for QP work queues. Since Hermon work queues
1555 * are not allowed to cross a 32-bit (4GB) boundary, the alignment of
1556 * the work queue memory is very important. We used to allocate
1557 * work queues (the combined receive and send queues) so that they
1558 * would be aligned on their combined size. That alignment guaranteed
1559 * that they would never cross the 4GB boundary (Hermon work queues
1560 * are on the order of MBs at maximum). Now we are able to relax
1561 * this alignment constraint by ensuring that the IB address assigned
1562 * to the queue memory (as a result of the hermon_mr_register() call)
1563 * is offset from zero.
1564 * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
1565 * guarantee the alignment, but when attempting to use IOMMU bypass
1566 * mode we found that we were not allowed to specify any alignment
1567 * that was more restrictive than the system page size.
1568 * So we avoided this constraint by passing two alignment values,
1569 * one for the memory allocation itself and the other for the DMA
1570 * handle (for later bind). This used to cause more memory than
1571 * necessary to be allocated (in order to guarantee the more
1572 * restrictive alignment contraint). But by guaranteeing the
1573 * zero-based IB virtual address for the queue, we are able to
1574 * conserve this memory.
1575 */
1576 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
1577 sq_depth = 1 << log_qp_sq_size;
1578 sq_size = sq_depth * sq_wqe_size;
1579
1580 /* QP on SRQ sets these to 0 */
1581 if (qp_srq_en) {
1582 rq_wqe_size = 0;
1583 rq_size = 0;
1584 } else {
1585 rq_wqe_size = 1 << qp->qp_rq_log_wqesz;
1586 rq_depth = 1 << log_qp_rq_size;
1587 rq_size = rq_depth * rq_wqe_size;
1588 }
1589
1590 qp->qp_wqinfo.qa_size = sq_size + rq_size;
1591 qp->qp_wqinfo.qa_alloc_align = PAGESIZE;
1592 qp->qp_wqinfo.qa_bind_align = PAGESIZE;
1593 qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
1594 status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag);
1595 if (status != DDI_SUCCESS) {
1596 status = IBT_INSUFF_RESOURCE;
1597 goto qpalloc_fail7;
1598 }
1599
1600 /*
1601 * Sort WQs in memory according to stride (*q_wqe_size), largest first
1602 * If they are equal, still put the SQ first
1603 */
1604 qp->qp_sq_baseaddr = 0;
1605 qp->qp_rq_baseaddr = 0;
1606 if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) {
1607 sq_buf = qp->qp_wqinfo.qa_buf_aligned;
1608
1609 /* if this QP is on an SRQ, set the rq_buf to NULL */
1610 if (qp_srq_en) {
1611 rq_buf = NULL;
1612 } else {
1613 rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size);
1614 qp->qp_rq_baseaddr = sq_size;
1615 }
1616 } else {
1617 rq_buf = qp->qp_wqinfo.qa_buf_aligned;
1618 sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size);
1619 qp->qp_sq_baseaddr = rq_size;
1620 }
1621
1622 qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth);
1623 if (qp->qp_sq_wqhdr == NULL) {
1624 status = IBT_INSUFF_RESOURCE;
1625 goto qpalloc_fail8;
1626 }
1627 if (qp_srq_en) {
1628 qp->qp_rq_wqavl.wqa_wq = srq->srq_wq_wqhdr;
1629 qp->qp_rq_wqavl.wqa_srq_en = 1;
1630 qp->qp_rq_wqavl.wqa_srq = srq;
1631 } else {
1632 qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(rq_depth);
1633 if (qp->qp_rq_wqhdr == NULL) {
1634 status = IBT_INSUFF_RESOURCE;
1635 goto qpalloc_fail8;
1636 }
1637 qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr;
1638 }
1639 qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum;
1640 qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND;
1641 qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr;
1642 qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum;
1643 qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV;
1644
1645 /*
1646 * Register the memory for the QP work queues. The memory for the
1647 * QP must be registered in the Hermon cMPT tables. This gives us the
1648 * LKey to specify in the QP context later. Note: The memory for
1649 * Hermon work queues (both Send and Recv) must be contiguous and
1650 * registered as a single memory region. Note: If the QP memory is
1651 * user-mappable, force DDI_DMA_CONSISTENT mapping. Also, in order to
1652 * meet the alignment restriction, we pass the "mro_bind_override_addr"
1653 * flag in the call to hermon_mr_register(). This guarantees that the
1654 * resulting IB vaddr will be zero-based (modulo the offset into the
1655 * first page). If we fail here, we still have the bunch of resource
1656 * and reference count cleanup to do.
1657 */
1658 flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
1659 IBT_MR_NOSLEEP;
1660 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
1661 mr_attr.mr_len = qp->qp_wqinfo.qa_size;
1662 mr_attr.mr_as = NULL;
1663 mr_attr.mr_flags = flag;
1664 /* HERMON_QUEUE_LOCATION_NORMAL */
1665 mr_op.mro_bind_type =
1666 state->hs_cfg_profile->cp_iommu_bypass;
1667 mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl;
1668 mr_op.mro_bind_override_addr = 1;
1669 status = hermon_mr_register(state, pd, &mr_attr, &mr,
1670 &mr_op, HERMON_QP_CMPT);
1671 if (status != DDI_SUCCESS) {
1672 status = IBT_INSUFF_RESOURCE;
1673 goto qpalloc_fail9;
1674 }
1675
1676 /*
1677 * Calculate the offset between the kernel virtual address space
1678 * and the IB virtual address space. This will be used when
1679 * posting work requests to properly initialize each WQE.
1680 */
1681 qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
1682 (uint64_t)mr->mr_bindinfo.bi_addr;
1683
1684 /*
1685 * Fill in all the return arguments (if necessary). This includes
1686 * real work queue sizes (in wqes), real SGLs, and QP number
1687 */
1688 if (queuesz_p != NULL) {
1689 queuesz_p->cs_sq =
1690 (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes;
1691 queuesz_p->cs_sq_sgl = qp->qp_sq_sgl;
1692
1693 /* if this QP is on an SRQ, set these to 0 */
1694 if (qp_srq_en) {
1695 queuesz_p->cs_rq = 0;
1696 queuesz_p->cs_rq_sgl = 0;
1697 } else {
1698 queuesz_p->cs_rq = (1 << log_qp_rq_size);
1699 queuesz_p->cs_rq_sgl = qp->qp_rq_sgl;
1700 }
1701 }
1702
1703 /*
1704 * Fill in the rest of the Hermon Queue Pair handle.
1705 */
1706 qp->qp_qpcrsrcp = NULL;
1707 qp->qp_rsrcp = rsrc;
1708 qp->qp_state = HERMON_QP_RESET;
1709 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
1710 qp->qp_pdhdl = pd;
1711 qp->qp_mrhdl = mr;
1712 qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ?
1713 HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED;
1714 qp->qp_is_special = 0;
1715 qp->qp_uarpg = uarpg;
1716 qp->qp_umap_dhp = (devmap_cookie_t)NULL;
1717 qp->qp_sq_cqhdl = sq_cq;
1718 qp->qp_sq_bufsz = (1 << log_qp_sq_size);
1719 qp->qp_sq_logqsz = log_qp_sq_size;
1720 qp->qp_sq_buf = sq_buf;
1721 qp->qp_desc_off = qp_desc_off;
1722 qp->qp_rq_cqhdl = rq_cq;
1723 qp->qp_rq_buf = rq_buf;
1724 qp->qp_rlky = (attr_p->qp_flags & IBT_FAST_REG_RES_LKEY) !=
1725 0;
1726
1727 /* if this QP is on an SRQ, set rq_bufsz to 0 */
1728 if (qp_srq_en) {
1729 qp->qp_rq_bufsz = 0;
1730 qp->qp_rq_logqsz = 0;
1731 } else {
1732 qp->qp_rq_bufsz = (1 << log_qp_rq_size);
1733 qp->qp_rq_logqsz = log_qp_rq_size;
1734 }
1735
1736 qp->qp_forward_sqd_event = 0;
1737 qp->qp_sqd_still_draining = 0;
1738 qp->qp_hdlrarg = (void *)ibt_qphdl[ii];
1739 qp->qp_mcg_refcnt = 0;
1740
1741 /*
1742 * If this QP is to be associated with an SRQ, set the SRQ handle
1743 */
1744 if (qp_srq_en) {
1745 qp->qp_srqhdl = srq;
1746 hermon_srq_refcnt_inc(qp->qp_srqhdl);
1747 } else {
1748 qp->qp_srqhdl = NULL;
1749 }
1750
1751 qp->qp_type = IBT_UD_RQP;
1752 qp->qp_serv_type = serv_type;
1753
1754 /*
1755 * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed
1756 */
1757
1758 /*
1759 * Initialize the SQ WQEs - all that needs to be done is every 64 bytes
1760 * set the quadword to all F's - high-order bit is owner (init to one)
1761 * and the rest for the headroom definition of prefetching.
1762 */
1763 if ((attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) == 0) {
1764 wqesz_shift = qp->qp_sq_log_wqesz;
1765 thewqesz = 1 << wqesz_shift;
1766 thewqe = (uint64_t *)(void *)(qp->qp_sq_buf);
1767 for (i = 0; i < sq_depth; i++) {
1768 /*
1769 * for each stride, go through and every 64 bytes
1770 * write the init value - having set the address
1771 * once, just keep incrementing it
1772 */
1773 for (j = 0; j < thewqesz; j += 64, thewqe += 8) {
1774 *(uint32_t *)thewqe = 0xFFFFFFFF;
1775 }
1776 }
1777 }
1778
1779 /* Zero out the QP context */
1780 bzero(&qp->qpc, sizeof (hermon_hw_qpc_t));
1781
1782 /*
1783 * Put QP handle in Hermon QPNum-to-QPHdl list. Then fill in the
1784 * "qphdl" and return success
1785 */
1786 hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx + ii, qp);
1787
1788 mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER,
1789 DDI_INTR_PRI(state->hs_intrmsi_pri));
1790
1791 qp->qp_rangep = qp_range_p;
1792
1793 qphdl[ii] = qp;
1794
1795 if (++ii < (1 << log2))
1796 goto for_each_qp;
1797
1798 return (DDI_SUCCESS);
1799
1800 /*
1801 * The following is cleanup for all possible failure cases in this routine
1802 */
1803 qpalloc_fail9:
1804 hermon_queue_free(&qp->qp_wqinfo);
1805 qpalloc_fail8:
1806 if (qp->qp_sq_wqhdr)
1807 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
1808 if (qp->qp_rq_wqhdr)
1809 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
1810 qpalloc_fail7:
1811 if (!qp_srq_en) {
1812 hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr);
1813 }
1814
1815 qpalloc_fail6:
1816 hermon_rsrc_free(state, &rsrc);
1817 qpalloc_fail4:
1818 hermon_cq_refcnt_dec(rq_cq);
1819 qpalloc_fail2:
1820 hermon_cq_refcnt_dec(sq_cq);
1821 qpalloc_fail1:
1822 hermon_pd_refcnt_dec(pd);
1823 qpalloc_fail0:
1824 if (ii == 0) {
1825 if (qp_range_p)
1826 kmem_free(qp_range_p, sizeof (*qp_range_p));
1827 hermon_rsrc_free(state, &qpc);
1828 } else {
1829 /* qp_range_p and qpc rsrc will be freed in hermon_qp_free */
1830
1831 mutex_enter(&qp->qp_rangep->hqpr_lock);
1832 qp_range_p->hqpr_refcnt = ii;
1833 mutex_exit(&qp->qp_rangep->hqpr_lock);
1834 while (--ii >= 0) {
1835 ibc_qpn_hdl_t qpn_hdl;
1836 int free_status;
1837
1838 free_status = hermon_qp_free(state, &qphdl[ii],
1839 IBC_FREE_QP_AND_QPN, &qpn_hdl, sleepflag);
1840 if (free_status != DDI_SUCCESS)
1841 cmn_err(CE_CONT, "!qp_range: status 0x%x: "
1842 "error status %x during free",
1843 status, free_status);
1844 }
1845 }
1846
1847 return (status);
1848 }
1849
1850
1851 /*
1852 * hermon_qp_free()
1853 * This function frees up the QP resources. Depending on the value
1854 * of the "free_qp_flags", the QP number may not be released until
1855 * a subsequent call to hermon_qp_release_qpn().
1856 *
1857 * Context: Can be called only from user or kernel context.
1858 */
1859 /* ARGSUSED */
1860 int
1861 hermon_qp_free(hermon_state_t *state, hermon_qphdl_t *qphdl,
1862 ibc_free_qp_flags_t free_qp_flags, ibc_qpn_hdl_t *qpnh,
1863 uint_t sleepflag)
1864 {
1865 hermon_rsrc_t *qpc, *rsrc;
1866 hermon_umap_db_entry_t *umapdb;
1867 hermon_qpn_entry_t *entry;
1868 hermon_pdhdl_t pd;
1869 hermon_mrhdl_t mr;
1870 hermon_cqhdl_t sq_cq, rq_cq;
1871 hermon_srqhdl_t srq;
1872 hermon_qphdl_t qp;
1873 uint64_t value;
1874 uint_t type, port;
1875 uint_t maxprot;
1876 uint_t qp_srq_en;
1877 int status;
1878
1879 /*
1880 * Pull all the necessary information from the Hermon Queue Pair
1881 * handle. This is necessary here because the resource for the
1882 * QP handle is going to be freed up as part of this operation.
1883 */
1884 qp = *qphdl;
1885 mutex_enter(&qp->qp_lock);
1886 qpc = qp->qp_qpcrsrcp; /* NULL if part of a "range" */
1887 rsrc = qp->qp_rsrcp;
1888 pd = qp->qp_pdhdl;
1889 srq = qp->qp_srqhdl;
1890 mr = qp->qp_mrhdl;
1891 rq_cq = qp->qp_rq_cqhdl;
1892 sq_cq = qp->qp_sq_cqhdl;
1893 port = qp->qp_portnum;
1894 qp_srq_en = qp->qp_alloc_flags & IBT_QP_USES_SRQ;
1895
1896 /*
1897 * If the QP is part of an MCG, then we fail the qp_free
1898 */
1899 if (qp->qp_mcg_refcnt != 0) {
1900 mutex_exit(&qp->qp_lock);
1901 status = ibc_get_ci_failure(0);
1902 goto qpfree_fail;
1903 }
1904
1905 /*
1906 * If the QP is not already in "Reset" state, then transition to
1907 * "Reset". This is necessary because software does not reclaim
1908 * ownership of the QP context until the QP is in the "Reset" state.
1909 * If the ownership transfer fails for any reason, then it is an
1910 * indication that something (either in HW or SW) has gone seriously
1911 * wrong. So we print a warning message and return.
1912 */
1913 if (qp->qp_state != HERMON_QP_RESET) {
1914 if (hermon_qp_to_reset(state, qp) != DDI_SUCCESS) {
1915 mutex_exit(&qp->qp_lock);
1916 HERMON_WARNING(state, "failed to reset QP context");
1917 status = ibc_get_ci_failure(0);
1918 goto qpfree_fail;
1919 }
1920 qp->qp_state = HERMON_QP_RESET;
1921 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
1922
1923 /*
1924 * Do any additional handling necessary for the transition
1925 * to the "Reset" state (e.g. update the WRID lists)
1926 */
1927 if (hermon_wrid_to_reset_handling(state, qp) != DDI_SUCCESS) {
1928 mutex_exit(&qp->qp_lock);
1929 HERMON_WARNING(state, "failed to reset QP WRID list");
1930 status = ibc_get_ci_failure(0);
1931 goto qpfree_fail;
1932 }
1933 }
1934
1935 /*
1936 * If this was a user-mappable QP, then we need to remove its entry
1937 * from the "userland resources database". If it is also currently
1938 * mmap()'d out to a user process, then we need to call
1939 * devmap_devmem_remap() to remap the QP memory to an invalid mapping.
1940 * We also need to invalidate the QP tracking information for the
1941 * user mapping.
1942 */
1943 if (qp->qp_alloc_flags & IBT_QP_USER_MAP) {
1944 status = hermon_umap_db_find(state->hs_instance, qp->qp_qpnum,
1945 MLNX_UMAP_QPMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE,
1946 &umapdb);
1947 if (status != DDI_SUCCESS) {
1948 mutex_exit(&qp->qp_lock);
1949 HERMON_WARNING(state, "failed to find in database");
1950 return (ibc_get_ci_failure(0));
1951 }
1952 hermon_umap_db_free(umapdb);
1953 if (qp->qp_umap_dhp != NULL) {
1954 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
1955 status = devmap_devmem_remap(qp->qp_umap_dhp,
1956 state->hs_dip, 0, 0, qp->qp_wqinfo.qa_size,
1957 maxprot, DEVMAP_MAPPING_INVALID, NULL);
1958 if (status != DDI_SUCCESS) {
1959 mutex_exit(&qp->qp_lock);
1960 HERMON_WARNING(state, "failed in QP memory "
1961 "devmap_devmem_remap()");
1962 return (ibc_get_ci_failure(0));
1963 }
1964 qp->qp_umap_dhp = (devmap_cookie_t)NULL;
1965 }
1966 }
1967
1968
1969 /*
1970 * Put NULL into the Hermon QPNum-to-QPHdl list. This will allow any
1971 * in-progress events to detect that the QP corresponding to this
1972 * number has been freed. Note: it does depend in whether we are
1973 * freeing a special QP or not.
1974 */
1975 if (qpc == NULL) {
1976 hermon_icm_set_num_to_hdl(state, HERMON_QPC,
1977 qp->qp_qpnum, NULL);
1978 } else if (qp->qp_is_special) {
1979 hermon_icm_set_num_to_hdl(state, HERMON_QPC,
1980 qpc->hr_indx + port, NULL);
1981 } else {
1982 hermon_icm_set_num_to_hdl(state, HERMON_QPC,
1983 qpc->hr_indx, NULL);
1984 }
1985
1986 /*
1987 * Drop the QP lock
1988 * At this point the lock is no longer necessary. We cannot
1989 * protect from multiple simultaneous calls to free the same QP.
1990 * In addition, since the QP lock is contained in the QP "software
1991 * handle" resource, which we will free (see below), it is
1992 * important that we have no further references to that memory.
1993 */
1994 mutex_exit(&qp->qp_lock);
1995 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
1996
1997 /*
1998 * Free the QP resources
1999 * Start by deregistering and freeing the memory for work queues.
2000 * Next free any previously allocated context information
2001 * (depending on QP type)
2002 * Finally, decrement the necessary reference counts.
2003 * If this fails for any reason, then it is an indication that
2004 * something (either in HW or SW) has gone seriously wrong. So we
2005 * print a warning message and return.
2006 */
2007 status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
2008 sleepflag);
2009 if (status != DDI_SUCCESS) {
2010 HERMON_WARNING(state, "failed to deregister QP memory");
2011 status = ibc_get_ci_failure(0);
2012 goto qpfree_fail;
2013 }
2014
2015 /* Free the memory for the QP */
2016 hermon_queue_free(&qp->qp_wqinfo);
2017
2018 if (qp->qp_sq_wqhdr)
2019 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
2020 if (qp->qp_rq_wqhdr)
2021 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
2022
2023 /* Free the dbr */
2024 if (!qp_srq_en) {
2025 hermon_dbr_free(state, qp->qp_uarpg, qp->qp_rq_vdbr);
2026 }
2027
2028 /*
2029 * Free up the remainder of the QP resources. Note: we have a few
2030 * different resources to free up depending on whether the QP is a
2031 * special QP or not. As described above, if any of these fail for
2032 * any reason it is an indication that something (either in HW or SW)
2033 * has gone seriously wrong. So we print a warning message and
2034 * return.
2035 */
2036 if (qp->qp_is_special) {
2037 type = (qp->qp_is_special == HERMON_QP_SMI) ?
2038 IBT_SMI_SQP : IBT_GSI_SQP;
2039
2040 /* Free up resources for the special QP */
2041 status = hermon_special_qp_rsrc_free(state, type, port);
2042 if (status != DDI_SUCCESS) {
2043 HERMON_WARNING(state, "failed to free special QP rsrc");
2044 status = ibc_get_ci_failure(0);
2045 goto qpfree_fail;
2046 }
2047
2048 } else if (qp->qp_rangep) {
2049 int refcnt;
2050 mutex_enter(&qp->qp_rangep->hqpr_lock);
2051 refcnt = --qp->qp_rangep->hqpr_refcnt;
2052 mutex_exit(&qp->qp_rangep->hqpr_lock);
2053 if (refcnt == 0) {
2054 mutex_destroy(&qp->qp_rangep->hqpr_lock);
2055 hermon_rsrc_free(state, &qp->qp_rangep->hqpr_qpcrsrc);
2056 kmem_free(qp->qp_rangep, sizeof (*qp->qp_rangep));
2057 }
2058 qp->qp_rangep = NULL;
2059 } else if (qp->qp_qpn_hdl == NULL) {
2060 hermon_rsrc_free(state, &qpc);
2061 } else {
2062 /*
2063 * Check the flags and determine whether to release the
2064 * QPN or not, based on their value.
2065 */
2066 if (free_qp_flags == IBC_FREE_QP_ONLY) {
2067 entry = qp->qp_qpn_hdl;
2068 hermon_qp_release_qpn(state, qp->qp_qpn_hdl,
2069 HERMON_QPN_FREE_ONLY);
2070 *qpnh = (ibc_qpn_hdl_t)entry;
2071 } else {
2072 hermon_qp_release_qpn(state, qp->qp_qpn_hdl,
2073 HERMON_QPN_RELEASE);
2074 }
2075 }
2076
2077 mutex_destroy(&qp->qp_sq_lock);
2078
2079 /* Free the Hermon Queue Pair handle */
2080 hermon_rsrc_free(state, &rsrc);
2081
2082 /* Decrement the reference counts on CQs, PD and SRQ (if needed) */
2083 hermon_cq_refcnt_dec(rq_cq);
2084 hermon_cq_refcnt_dec(sq_cq);
2085 hermon_pd_refcnt_dec(pd);
2086 if (qp_srq_en == HERMON_QP_SRQ_ENABLED) {
2087 hermon_srq_refcnt_dec(srq);
2088 }
2089
2090 /* Set the qphdl pointer to NULL and return success */
2091 *qphdl = NULL;
2092
2093 return (DDI_SUCCESS);
2094
2095 qpfree_fail:
2096 return (status);
2097 }
2098
2099
2100 /*
2101 * hermon_qp_query()
2102 * Context: Can be called from interrupt or base context.
2103 */
2104 int
2105 hermon_qp_query(hermon_state_t *state, hermon_qphdl_t qp,
2106 ibt_qp_query_attr_t *attr_p)
2107 {
2108 ibt_cep_state_t qp_state;
2109 ibt_qp_ud_attr_t *ud;
2110 ibt_qp_rc_attr_t *rc;
2111 ibt_qp_uc_attr_t *uc;
2112 ibt_cep_flags_t enable_flags;
2113 hermon_hw_addr_path_t *qpc_path, *qpc_alt_path;
2114 ibt_cep_path_t *path_ptr, *alt_path_ptr;
2115 hermon_hw_qpc_t *qpc;
2116 int status;
2117 uint_t tmp_sched_q, tmp_alt_sched_q;
2118
2119 mutex_enter(&qp->qp_lock);
2120
2121 /*
2122 * Grab the temporary QPC entry from QP software state
2123 */
2124 qpc = &qp->qpc;
2125
2126 /* Convert the current Hermon QP state to IBTF QP state */
2127 switch (qp->qp_state) {
2128 case HERMON_QP_RESET:
2129 qp_state = IBT_STATE_RESET; /* "Reset" */
2130 break;
2131 case HERMON_QP_INIT:
2132 qp_state = IBT_STATE_INIT; /* Initialized */
2133 break;
2134 case HERMON_QP_RTR:
2135 qp_state = IBT_STATE_RTR; /* Ready to Receive */
2136 break;
2137 case HERMON_QP_RTS:
2138 qp_state = IBT_STATE_RTS; /* Ready to Send */
2139 break;
2140 case HERMON_QP_SQERR:
2141 qp_state = IBT_STATE_SQE; /* Send Queue Error */
2142 break;
2143 case HERMON_QP_SQD:
2144 if (qp->qp_sqd_still_draining) {
2145 qp_state = IBT_STATE_SQDRAIN; /* SQ Draining */
2146 } else {
2147 qp_state = IBT_STATE_SQD; /* SQ Drained */
2148 }
2149 break;
2150 case HERMON_QP_ERR:
2151 qp_state = IBT_STATE_ERROR; /* Error */
2152 break;
2153 default:
2154 mutex_exit(&qp->qp_lock);
2155 return (ibc_get_ci_failure(0));
2156 }
2157 attr_p->qp_info.qp_state = qp_state;
2158
2159 /* SRQ Hook. */
2160 attr_p->qp_srq = NULL;
2161
2162 /*
2163 * The following QP information is always returned, regardless of
2164 * the current QP state. Note: Some special handling is necessary
2165 * for calculating the QP number on special QP (QP0 and QP1).
2166 */
2167 attr_p->qp_sq_cq =
2168 (qp->qp_sq_cqhdl == NULL) ? NULL : qp->qp_sq_cqhdl->cq_hdlrarg;
2169 attr_p->qp_rq_cq =
2170 (qp->qp_rq_cqhdl == NULL) ? NULL : qp->qp_rq_cqhdl->cq_hdlrarg;
2171 if (qp->qp_is_special) {
2172 attr_p->qp_qpn = (qp->qp_is_special == HERMON_QP_SMI) ? 0 : 1;
2173 } else {
2174 attr_p->qp_qpn = (ib_qpn_t)qp->qp_qpnum;
2175 }
2176 attr_p->qp_sq_sgl = qp->qp_sq_sgl;
2177 attr_p->qp_rq_sgl = qp->qp_rq_sgl;
2178 attr_p->qp_info.qp_sq_sz = qp->qp_sq_bufsz - qp->qp_sq_hdrmwqes;
2179 attr_p->qp_info.qp_rq_sz = qp->qp_rq_bufsz;
2180
2181 /*
2182 * If QP is currently in the "Reset" state, then only the above are
2183 * returned
2184 */
2185 if (qp_state == IBT_STATE_RESET) {
2186 mutex_exit(&qp->qp_lock);
2187 return (DDI_SUCCESS);
2188 }
2189
2190 /*
2191 * Post QUERY_QP command to firmware
2192 *
2193 * We do a HERMON_NOSLEEP here because we are holding the "qp_lock".
2194 * Since we may be in the interrupt context (or subsequently raised
2195 * to interrupt level by priority inversion), we do not want to block
2196 * in this routine waiting for success.
2197 */
2198 tmp_sched_q = qpc->pri_addr_path.sched_q;
2199 tmp_alt_sched_q = qpc->alt_addr_path.sched_q;
2200 status = hermon_cmn_query_cmd_post(state, QUERY_QP, 0, qp->qp_qpnum,
2201 qpc, sizeof (hermon_hw_qpc_t), HERMON_CMD_NOSLEEP_SPIN);
2202 if (status != HERMON_CMD_SUCCESS) {
2203 mutex_exit(&qp->qp_lock);
2204 cmn_err(CE_WARN, "hermon%d: hermon_qp_query: QUERY_QP "
2205 "command failed: %08x\n", state->hs_instance, status);
2206 if (status == HERMON_CMD_INVALID_STATUS) {
2207 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
2208 }
2209 return (ibc_get_ci_failure(0));
2210 }
2211 qpc->pri_addr_path.sched_q = tmp_sched_q;
2212 qpc->alt_addr_path.sched_q = tmp_alt_sched_q;
2213
2214 /*
2215 * Fill in the additional QP info based on the QP's transport type.
2216 */
2217 if (qp->qp_type == IBT_UD_RQP) {
2218
2219 /* Fill in the UD-specific info */
2220 ud = &attr_p->qp_info.qp_transport.ud;
2221 ud->ud_qkey = (ib_qkey_t)qpc->qkey;
2222 ud->ud_sq_psn = qpc->next_snd_psn;
2223 ud->ud_pkey_ix = qpc->pri_addr_path.pkey_indx;
2224 /* port+1 for port 1/2 */
2225 ud->ud_port =
2226 (uint8_t)(((qpc->pri_addr_path.sched_q >> 6) & 0x01) + 1);
2227
2228 attr_p->qp_info.qp_trans = IBT_UD_SRV;
2229
2230 if (qp->qp_serv_type == HERMON_QP_FEXCH) {
2231 ibt_pmr_desc_t *pmr;
2232 uint64_t heart_beat;
2233
2234 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pmr))
2235 pmr = &attr_p->qp_query_fexch.fq_uni_mem_desc;
2236 pmr->pmd_iova = 0;
2237 pmr->pmd_lkey = pmr->pmd_rkey =
2238 hermon_fcoib_qpn_to_mkey(state, qp->qp_qpnum);
2239 pmr->pmd_phys_buf_list_sz =
2240 state->hs_fcoib.hfc_mtts_per_mpt;
2241 pmr->pmd_sync_required = 0;
2242
2243 pmr = &attr_p->qp_query_fexch.fq_bi_mem_desc;
2244 pmr->pmd_iova = 0;
2245 pmr->pmd_lkey = 0;
2246 pmr->pmd_rkey = 0;
2247 pmr->pmd_phys_buf_list_sz = 0;
2248 pmr->pmd_sync_required = 0;
2249
2250 attr_p->qp_query_fexch.fq_flags =
2251 ((hermon_get_heart_beat_rq_cmd_post(state,
2252 qp->qp_qpnum, &heart_beat) == HERMON_CMD_SUCCESS) &&
2253 (heart_beat == 0)) ? IBT_FEXCH_HEART_BEAT_OK :
2254 IBT_FEXCH_NO_FLAGS;
2255
2256 ud->ud_fc = qp->qp_fc_attr;
2257 } else if (qp->qp_serv_type == HERMON_QP_FCMND ||
2258 qp->qp_serv_type == HERMON_QP_RFCI) {
2259 ud->ud_fc = qp->qp_fc_attr;
2260 }
2261
2262 } else if (qp->qp_serv_type == HERMON_QP_RC) {
2263
2264 /* Fill in the RC-specific info */
2265 rc = &attr_p->qp_info.qp_transport.rc;
2266 rc->rc_sq_psn = qpc->next_snd_psn;
2267 rc->rc_rq_psn = qpc->next_rcv_psn;
2268 rc->rc_dst_qpn = qpc->rem_qpn;
2269
2270 /* Grab the path migration state information */
2271 if (qpc->pm_state == HERMON_QP_PMSTATE_MIGRATED) {
2272 rc->rc_mig_state = IBT_STATE_MIGRATED;
2273 } else if (qpc->pm_state == HERMON_QP_PMSTATE_REARM) {
2274 rc->rc_mig_state = IBT_STATE_REARMED;
2275 } else {
2276 rc->rc_mig_state = IBT_STATE_ARMED;
2277 }
2278 rc->rc_rdma_ra_out = (1 << qpc->sra_max);
2279 rc->rc_rdma_ra_in = (1 << qpc->rra_max);
2280 rc->rc_min_rnr_nak = qpc->min_rnr_nak;
2281 rc->rc_path_mtu = qpc->mtu;
2282 rc->rc_retry_cnt = qpc->retry_cnt;
2283
2284 /* Get the common primary address path fields */
2285 qpc_path = &qpc->pri_addr_path;
2286 path_ptr = &rc->rc_path;
2287 hermon_get_addr_path(state, qpc_path, &path_ptr->cep_adds_vect,
2288 HERMON_ADDRPATH_QP);
2289
2290 /* Fill in the additional primary address path fields */
2291 path_ptr->cep_pkey_ix = qpc_path->pkey_indx;
2292 path_ptr->cep_hca_port_num =
2293 path_ptr->cep_adds_vect.av_port_num =
2294 (uint8_t)(((qpc_path->sched_q >> 6) & 0x01) + 1);
2295 path_ptr->cep_timeout = qpc_path->ack_timeout;
2296
2297 /* Get the common alternate address path fields */
2298 qpc_alt_path = &qpc->alt_addr_path;
2299 alt_path_ptr = &rc->rc_alt_path;
2300 hermon_get_addr_path(state, qpc_alt_path,
2301 &alt_path_ptr->cep_adds_vect, HERMON_ADDRPATH_QP);
2302
2303 /* Fill in the additional alternate address path fields */
2304 alt_path_ptr->cep_pkey_ix = qpc_alt_path->pkey_indx;
2305 alt_path_ptr->cep_hca_port_num =
2306 alt_path_ptr->cep_adds_vect.av_port_num =
2307 (uint8_t)(((qpc_alt_path->sched_q >> 6) & 0x01) + 1);
2308 alt_path_ptr->cep_timeout = qpc_alt_path->ack_timeout;
2309
2310 /* Get the RNR retry time from primary path */
2311 rc->rc_rnr_retry_cnt = qpc->rnr_retry;
2312
2313 /* Set the enable flags based on RDMA/Atomic enable bits */
2314 enable_flags = IBT_CEP_NO_FLAGS;
2315 enable_flags |= ((qpc->rre == 0) ? 0 : IBT_CEP_RDMA_RD);
2316 enable_flags |= ((qpc->rwe == 0) ? 0 : IBT_CEP_RDMA_WR);
2317 enable_flags |= ((qpc->rae == 0) ? 0 : IBT_CEP_ATOMIC);
2318 attr_p->qp_info.qp_flags = enable_flags;
2319
2320 attr_p->qp_info.qp_trans = IBT_RC_SRV;
2321
2322 } else if (qp->qp_serv_type == HERMON_QP_UC) {
2323
2324 /* Fill in the UC-specific info */
2325 uc = &attr_p->qp_info.qp_transport.uc;
2326 uc->uc_sq_psn = qpc->next_snd_psn;
2327 uc->uc_rq_psn = qpc->next_rcv_psn;
2328 uc->uc_dst_qpn = qpc->rem_qpn;
2329
2330 /* Grab the path migration state information */
2331 if (qpc->pm_state == HERMON_QP_PMSTATE_MIGRATED) {
2332 uc->uc_mig_state = IBT_STATE_MIGRATED;
2333 } else if (qpc->pm_state == HERMON_QP_PMSTATE_REARM) {
2334 uc->uc_mig_state = IBT_STATE_REARMED;
2335 } else {
2336 uc->uc_mig_state = IBT_STATE_ARMED;
2337 }
2338 uc->uc_path_mtu = qpc->mtu;
2339
2340 /* Get the common primary address path fields */
2341 qpc_path = &qpc->pri_addr_path;
2342 path_ptr = &uc->uc_path;
2343 hermon_get_addr_path(state, qpc_path, &path_ptr->cep_adds_vect,
2344 HERMON_ADDRPATH_QP);
2345
2346 /* Fill in the additional primary address path fields */
2347 path_ptr->cep_pkey_ix = qpc_path->pkey_indx;
2348 path_ptr->cep_hca_port_num =
2349 path_ptr->cep_adds_vect.av_port_num =
2350 (uint8_t)(((qpc_path->sched_q >> 6) & 0x01) + 1);
2351
2352 /* Get the common alternate address path fields */
2353 qpc_alt_path = &qpc->alt_addr_path;
2354 alt_path_ptr = &uc->uc_alt_path;
2355 hermon_get_addr_path(state, qpc_alt_path,
2356 &alt_path_ptr->cep_adds_vect, HERMON_ADDRPATH_QP);
2357
2358 /* Fill in the additional alternate address path fields */
2359 alt_path_ptr->cep_pkey_ix = qpc_alt_path->pkey_indx;
2360 alt_path_ptr->cep_hca_port_num =
2361 alt_path_ptr->cep_adds_vect.av_port_num =
2362 (uint8_t)(((qpc_alt_path->sched_q >> 6) & 0x01) + 1);
2363
2364 /*
2365 * Set the enable flags based on RDMA enable bits (by
2366 * definition UC doesn't support Atomic or RDMA Read)
2367 */
2368 enable_flags = ((qpc->rwe == 0) ? 0 : IBT_CEP_RDMA_WR);
2369 attr_p->qp_info.qp_flags = enable_flags;
2370
2371 attr_p->qp_info.qp_trans = IBT_UC_SRV;
2372
2373 } else {
2374 HERMON_WARNING(state, "unexpected QP transport type");
2375 mutex_exit(&qp->qp_lock);
2376 return (ibc_get_ci_failure(0));
2377 }
2378
2379 /*
2380 * Under certain circumstances it is possible for the Hermon hardware
2381 * to transition to one of the error states without software directly
2382 * knowing about it. The QueryQP() call is the one place where we
2383 * have an opportunity to sample and update our view of the QP state.
2384 */
2385 if (qpc->state == HERMON_QP_SQERR) {
2386 attr_p->qp_info.qp_state = IBT_STATE_SQE;
2387 qp->qp_state = HERMON_QP_SQERR;
2388 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_SQERR);
2389 }
2390 if (qpc->state == HERMON_QP_ERR) {
2391 attr_p->qp_info.qp_state = IBT_STATE_ERROR;
2392 qp->qp_state = HERMON_QP_ERR;
2393 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_ERR);
2394 }
2395 mutex_exit(&qp->qp_lock);
2396
2397 return (DDI_SUCCESS);
2398 }
2399
2400
2401 /*
2402 * hermon_qp_create_qpn()
2403 * Context: Can be called from interrupt or base context.
2404 */
2405 static int
2406 hermon_qp_create_qpn(hermon_state_t *state, hermon_qphdl_t qp,
2407 hermon_rsrc_t *qpc)
2408 {
2409 hermon_qpn_entry_t query;
2410 hermon_qpn_entry_t *entry;
2411 avl_index_t where;
2412
2413 /*
2414 * Build a query (for the AVL tree lookup) and attempt to find
2415 * a previously added entry that has a matching QPC index. If
2416 * no matching entry is found, then allocate, initialize, and
2417 * add an entry to the AVL tree.
2418 * If a matching entry is found, then increment its QPN counter
2419 * and reference counter.
2420 */
2421 query.qpn_indx = qpc->hr_indx;
2422 mutex_enter(&state->hs_qpn_avl_lock);
2423 entry = (hermon_qpn_entry_t *)avl_find(&state->hs_qpn_avl,
2424 &query, &where);
2425 if (entry == NULL) {
2426 /*
2427 * Allocate and initialize a QPN entry, then insert
2428 * it into the AVL tree.
2429 */
2430 entry = (hermon_qpn_entry_t *)kmem_zalloc(
2431 sizeof (hermon_qpn_entry_t), KM_NOSLEEP);
2432 if (entry == NULL) {
2433 mutex_exit(&state->hs_qpn_avl_lock);
2434 return (DDI_FAILURE);
2435 }
2436 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*entry))
2437
2438 entry->qpn_indx = qpc->hr_indx;
2439 entry->qpn_refcnt = 0;
2440 entry->qpn_counter = 0;
2441
2442 avl_insert(&state->hs_qpn_avl, entry, where);
2443 }
2444
2445 /*
2446 * Make the AVL tree entry point to the QP context resource that
2447 * it will be responsible for tracking
2448 */
2449 entry->qpn_qpc = qpc;
2450
2451 /*
2452 * Setup the QP handle to point to the AVL tree entry. Then
2453 * generate the new QP number from the entry's QPN counter value
2454 * and the hardware's QP context table index.
2455 */
2456 qp->qp_qpn_hdl = entry;
2457 qp->qp_qpnum = ((entry->qpn_counter <<
2458 state->hs_cfg_profile->cp_log_num_qp) | qpc->hr_indx) &
2459 HERMON_QP_MAXNUMBER_MSK;
2460 qp->qp_ring = qp->qp_qpnum << 8;
2461
2462 /*
2463 * Increment the reference counter and QPN counter. The QPN
2464 * counter always indicates the next available number for use.
2465 */
2466 entry->qpn_counter++;
2467 entry->qpn_refcnt++;
2468
2469 mutex_exit(&state->hs_qpn_avl_lock);
2470
2471 return (DDI_SUCCESS);
2472 }
2473
2474
2475 /*
2476 * hermon_qp_release_qpn()
2477 * Context: Can be called only from user or kernel context.
2478 */
2479 void
2480 hermon_qp_release_qpn(hermon_state_t *state, hermon_qpn_entry_t *entry,
2481 int flags)
2482 {
2483 ASSERT(entry != NULL);
2484
2485 mutex_enter(&state->hs_qpn_avl_lock);
2486
2487 /*
2488 * If we are releasing the QP number here, then we decrement the
2489 * reference count and check for zero references. If there are
2490 * zero references, then we free the QPC context (if it hadn't
2491 * already been freed during a HERMON_QPN_FREE_ONLY free, i.e. for
2492 * reuse with another similar QP number) and remove the tracking
2493 * structure from the QP number AVL tree and free the structure.
2494 * If we are not releasing the QP number here, then, as long as we
2495 * have not exhausted the usefulness of the QPC context (that is,
2496 * re-used it too many times without the reference count having
2497 * gone to zero), we free up the QPC context for use by another
2498 * thread (which will use it to construct a different QP number
2499 * from the same QPC table index).
2500 */
2501 if (flags == HERMON_QPN_RELEASE) {
2502 entry->qpn_refcnt--;
2503
2504 /*
2505 * If the reference count is zero, then we free the QPC
2506 * context (if it hadn't already been freed in an early
2507 * step, e.g. HERMON_QPN_FREE_ONLY) and remove/free the
2508 * tracking structure from the QP number AVL tree.
2509 */
2510 if (entry->qpn_refcnt == 0) {
2511 if (entry->qpn_qpc != NULL) {
2512 hermon_rsrc_free(state, &entry->qpn_qpc);
2513 }
2514
2515 /*
2516 * If the current entry has served it's useful
2517 * purpose (i.e. been reused the maximum allowable
2518 * number of times), then remove it from QP number
2519 * AVL tree and free it up.
2520 */
2521 if (entry->qpn_counter >= (1 <<
2522 (24 - state->hs_cfg_profile->cp_log_num_qp))) {
2523 avl_remove(&state->hs_qpn_avl, entry);
2524 kmem_free(entry, sizeof (hermon_qpn_entry_t));
2525 }
2526 }
2527
2528 } else if (flags == HERMON_QPN_FREE_ONLY) {
2529 /*
2530 * Even if we are not freeing the QP number, that will not
2531 * always prevent us from releasing the QPC context. In fact,
2532 * since the QPC context only forms part of the whole QPN,
2533 * we want to free it up for use by other consumers. But
2534 * if the reference count is non-zero (which it will always
2535 * be when we are doing HERMON_QPN_FREE_ONLY) and the counter
2536 * has reached its maximum value, then we cannot reuse the
2537 * QPC context until the reference count eventually reaches
2538 * zero (in HERMON_QPN_RELEASE, above).
2539 */
2540 if (entry->qpn_counter < (1 <<
2541 (24 - state->hs_cfg_profile->cp_log_num_qp))) {
2542 hermon_rsrc_free(state, &entry->qpn_qpc);
2543 }
2544 }
2545 mutex_exit(&state->hs_qpn_avl_lock);
2546 }
2547
2548
2549 /*
2550 * hermon_qpn_avl_compare()
2551 * Context: Can be called from user or kernel context.
2552 */
2553 static int
2554 hermon_qpn_avl_compare(const void *q, const void *e)
2555 {
2556 hermon_qpn_entry_t *entry, *query;
2557
2558 entry = (hermon_qpn_entry_t *)e;
2559 query = (hermon_qpn_entry_t *)q;
2560
2561 if (query->qpn_indx < entry->qpn_indx) {
2562 return (-1);
2563 } else if (query->qpn_indx > entry->qpn_indx) {
2564 return (+1);
2565 } else {
2566 return (0);
2567 }
2568 }
2569
2570
2571 /*
2572 * hermon_qpn_avl_init()
2573 * Context: Only called from attach() path context
2574 */
2575 void
2576 hermon_qpn_avl_init(hermon_state_t *state)
2577 {
2578 /* Initialize the lock used for QP number (QPN) AVL tree access */
2579 mutex_init(&state->hs_qpn_avl_lock, NULL, MUTEX_DRIVER,
2580 DDI_INTR_PRI(state->hs_intrmsi_pri));
2581
2582 /* Initialize the AVL tree for the QP number (QPN) storage */
2583 avl_create(&state->hs_qpn_avl, hermon_qpn_avl_compare,
2584 sizeof (hermon_qpn_entry_t),
2585 offsetof(hermon_qpn_entry_t, qpn_avlnode));
2586 }
2587
2588
2589 /*
2590 * hermon_qpn_avl_fini()
2591 * Context: Only called from attach() and/or detach() path contexts
2592 */
2593 void
2594 hermon_qpn_avl_fini(hermon_state_t *state)
2595 {
2596 hermon_qpn_entry_t *entry;
2597 void *cookie;
2598
2599 /*
2600 * Empty all entries (if necessary) and destroy the AVL tree
2601 * that was used for QP number (QPN) tracking.
2602 */
2603 cookie = NULL;
2604 while ((entry = (hermon_qpn_entry_t *)avl_destroy_nodes(
2605 &state->hs_qpn_avl, &cookie)) != NULL) {
2606 kmem_free(entry, sizeof (hermon_qpn_entry_t));
2607 }
2608 avl_destroy(&state->hs_qpn_avl);
2609
2610 /* Destroy the lock used for QP number (QPN) AVL tree access */
2611 mutex_destroy(&state->hs_qpn_avl_lock);
2612 }
2613
2614
2615 /*
2616 * hermon_qphdl_from_qpnum()
2617 * Context: Can be called from interrupt or base context.
2618 *
2619 * This routine is important because changing the unconstrained
2620 * portion of the QP number is critical to the detection of a
2621 * potential race condition in the QP event handler code (i.e. the case
2622 * where a QP is freed and alloc'd again before an event for the
2623 * "old" QP can be handled).
2624 *
2625 * While this is not a perfect solution (not sure that one exists)
2626 * it does help to mitigate the chance that this race condition will
2627 * cause us to deliver a "stale" event to the new QP owner. Note:
2628 * this solution does not scale well because the number of constrained
2629 * bits increases (and, hence, the number of unconstrained bits
2630 * decreases) as the number of supported QPs grows. For small and
2631 * intermediate values, it should hopefully provide sufficient
2632 * protection.
2633 */
2634 hermon_qphdl_t
2635 hermon_qphdl_from_qpnum(hermon_state_t *state, uint_t qpnum)
2636 {
2637 uint_t qpindx, qpmask;
2638
2639 /* Calculate the QP table index from the qpnum */
2640 qpmask = (1 << state->hs_cfg_profile->cp_log_num_qp) - 1;
2641 qpindx = qpnum & qpmask;
2642 return (hermon_icm_num_to_hdl(state, HERMON_QPC, qpindx));
2643 }
2644
2645
2646 /*
2647 * hermon_special_qp_rsrc_alloc
2648 * Context: Can be called from interrupt or base context.
2649 */
2650 static int
2651 hermon_special_qp_rsrc_alloc(hermon_state_t *state, ibt_sqp_type_t type,
2652 uint_t port, hermon_rsrc_t **qp_rsrc)
2653 {
2654 uint_t mask, flags;
2655 int status;
2656
2657 mutex_enter(&state->hs_spec_qplock);
2658 flags = state->hs_spec_qpflags;
2659 if (type == IBT_SMI_SQP) {
2660 /*
2661 * Check here to see if the driver has been configured
2662 * to instruct the Hermon firmware to handle all incoming
2663 * SMP messages (i.e. messages sent to SMA). If so,
2664 * then we will treat QP0 as if it has already been
2665 * allocated (for internal use). Otherwise, if we allow
2666 * the allocation to happen, it will cause unexpected
2667 * behaviors (e.g. Hermon SMA becomes unresponsive).
2668 */
2669 if (state->hs_cfg_profile->cp_qp0_agents_in_fw != 0) {
2670 mutex_exit(&state->hs_spec_qplock);
2671 return (IBT_QP_IN_USE);
2672 }
2673
2674 /*
2675 * If this is the first QP0 allocation, then post
2676 * a CONF_SPECIAL_QP firmware command
2677 */
2678 if ((flags & HERMON_SPECIAL_QP0_RSRC_MASK) == 0) {
2679 status = hermon_conf_special_qp_cmd_post(state,
2680 state->hs_spec_qp0->hr_indx, HERMON_CMD_QP_SMI,
2681 HERMON_CMD_NOSLEEP_SPIN,
2682 HERMON_CMD_SPEC_QP_OPMOD(
2683 state->hs_cfg_profile->cp_qp0_agents_in_fw,
2684 state->hs_cfg_profile->cp_qp1_agents_in_fw));
2685 if (status != HERMON_CMD_SUCCESS) {
2686 mutex_exit(&state->hs_spec_qplock);
2687 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2688 "command failed: %08x\n",
2689 state->hs_instance, status);
2690 return (IBT_INSUFF_RESOURCE);
2691 }
2692 }
2693
2694 /*
2695 * Now check (and, if necessary, modify) the flags to indicate
2696 * whether the allocation was successful
2697 */
2698 mask = (1 << (HERMON_SPECIAL_QP0_RSRC + port));
2699 if (flags & mask) {
2700 mutex_exit(&state->hs_spec_qplock);
2701 return (IBT_QP_IN_USE);
2702 }
2703 state->hs_spec_qpflags |= mask;
2704 *qp_rsrc = state->hs_spec_qp0;
2705
2706 } else {
2707 /*
2708 * If this is the first QP1 allocation, then post
2709 * a CONF_SPECIAL_QP firmware command
2710 */
2711 if ((flags & HERMON_SPECIAL_QP1_RSRC_MASK) == 0) {
2712 status = hermon_conf_special_qp_cmd_post(state,
2713 state->hs_spec_qp1->hr_indx, HERMON_CMD_QP_GSI,
2714 HERMON_CMD_NOSLEEP_SPIN,
2715 HERMON_CMD_SPEC_QP_OPMOD(
2716 state->hs_cfg_profile->cp_qp0_agents_in_fw,
2717 state->hs_cfg_profile->cp_qp1_agents_in_fw));
2718 if (status != HERMON_CMD_SUCCESS) {
2719 mutex_exit(&state->hs_spec_qplock);
2720 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2721 "command failed: %08x\n",
2722 state->hs_instance, status);
2723 return (IBT_INSUFF_RESOURCE);
2724 }
2725 }
2726
2727 /*
2728 * Now check (and, if necessary, modify) the flags to indicate
2729 * whether the allocation was successful
2730 */
2731 mask = (1 << (HERMON_SPECIAL_QP1_RSRC + port));
2732 if (flags & mask) {
2733 mutex_exit(&state->hs_spec_qplock);
2734 return (IBT_QP_IN_USE);
2735 }
2736 state->hs_spec_qpflags |= mask;
2737 *qp_rsrc = state->hs_spec_qp1;
2738 }
2739
2740 mutex_exit(&state->hs_spec_qplock);
2741 return (DDI_SUCCESS);
2742 }
2743
2744
2745 /*
2746 * hermon_special_qp_rsrc_free
2747 * Context: Can be called from interrupt or base context.
2748 */
2749 static int
2750 hermon_special_qp_rsrc_free(hermon_state_t *state, ibt_sqp_type_t type,
2751 uint_t port)
2752 {
2753 uint_t mask, flags;
2754 int status;
2755
2756 mutex_enter(&state->hs_spec_qplock);
2757 if (type == IBT_SMI_SQP) {
2758 mask = (1 << (HERMON_SPECIAL_QP0_RSRC + port));
2759 state->hs_spec_qpflags &= ~mask;
2760 flags = state->hs_spec_qpflags;
2761
2762 /*
2763 * If this is the last QP0 free, then post a CONF_SPECIAL_QP
2764 * NOW, If this is the last Special QP free, then post a
2765 * CONF_SPECIAL_QP firmware command - it'll stop them all
2766 */
2767 if (flags) {
2768 status = hermon_conf_special_qp_cmd_post(state, 0,
2769 HERMON_CMD_QP_SMI, HERMON_CMD_NOSLEEP_SPIN, 0);
2770 if (status != HERMON_CMD_SUCCESS) {
2771 mutex_exit(&state->hs_spec_qplock);
2772 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2773 "command failed: %08x\n",
2774 state->hs_instance, status);
2775 if (status == HERMON_CMD_INVALID_STATUS) {
2776 hermon_fm_ereport(state, HCA_SYS_ERR,
2777 HCA_ERR_SRV_LOST);
2778 }
2779 return (ibc_get_ci_failure(0));
2780 }
2781 }
2782 } else {
2783 mask = (1 << (HERMON_SPECIAL_QP1_RSRC + port));
2784 state->hs_spec_qpflags &= ~mask;
2785 flags = state->hs_spec_qpflags;
2786
2787 /*
2788 * If this is the last QP1 free, then post a CONF_SPECIAL_QP
2789 * NOW, if this is the last special QP free, then post a
2790 * CONF_SPECIAL_QP firmware command - it'll stop them all
2791 */
2792 if (flags) {
2793 status = hermon_conf_special_qp_cmd_post(state, 0,
2794 HERMON_CMD_QP_GSI, HERMON_CMD_NOSLEEP_SPIN, 0);
2795 if (status != HERMON_CMD_SUCCESS) {
2796 mutex_exit(&state->hs_spec_qplock);
2797 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2798 "command failed: %08x\n",
2799 state->hs_instance, status);
2800 if (status == HERMON_CMD_INVALID_STATUS) {
2801 hermon_fm_ereport(state, HCA_SYS_ERR,
2802 HCA_ERR_SRV_LOST);
2803 }
2804 return (ibc_get_ci_failure(0));
2805 }
2806 }
2807 }
2808
2809 mutex_exit(&state->hs_spec_qplock);
2810 return (DDI_SUCCESS);
2811 }
2812
2813
2814 /*
2815 * hermon_qp_sgl_to_logwqesz()
2816 * Context: Can be called from interrupt or base context.
2817 */
2818 static void
2819 hermon_qp_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl,
2820 uint_t real_max_sgl, hermon_qp_wq_type_t wq_type,
2821 uint_t *logwqesz, uint_t *max_sgl)
2822 {
2823 uint_t max_size, log2, actual_sgl;
2824
2825 switch (wq_type) {
2826 case HERMON_QP_WQ_TYPE_SENDQ_UD:
2827 /*
2828 * Use requested maximum SGL to calculate max descriptor size
2829 * (while guaranteeing that the descriptor size is a
2830 * power-of-2 cachelines).
2831 */
2832 max_size = (HERMON_QP_WQE_MLX_SND_HDRS + (num_sgl << 4));
2833 log2 = highbit(max_size);
2834 if ((max_size & (max_size - 1)) == 0) {
2835 log2 = log2 - 1;
2836 }
2837
2838 /* Make sure descriptor is at least the minimum size */
2839 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2840
2841 /* Calculate actual number of SGL (given WQE size) */
2842 actual_sgl = ((1 << log2) -
2843 sizeof (hermon_hw_snd_wqe_ctrl_t)) >> 4;
2844 break;
2845
2846 case HERMON_QP_WQ_TYPE_SENDQ_CONN:
2847 /*
2848 * Use requested maximum SGL to calculate max descriptor size
2849 * (while guaranteeing that the descriptor size is a
2850 * power-of-2 cachelines).
2851 */
2852 max_size = (HERMON_QP_WQE_MLX_SND_HDRS + (num_sgl << 4));
2853 log2 = highbit(max_size);
2854 if ((max_size & (max_size - 1)) == 0) {
2855 log2 = log2 - 1;
2856 }
2857
2858 /* Make sure descriptor is at least the minimum size */
2859 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2860
2861 /* Calculate actual number of SGL (given WQE size) */
2862 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_SND_HDRS) >> 4;
2863 break;
2864
2865 case HERMON_QP_WQ_TYPE_RECVQ:
2866 /*
2867 * Same as above (except for Recv WQEs)
2868 */
2869 max_size = (HERMON_QP_WQE_MLX_RCV_HDRS + (num_sgl << 4));
2870 log2 = highbit(max_size);
2871 if ((max_size & (max_size - 1)) == 0) {
2872 log2 = log2 - 1;
2873 }
2874
2875 /* Make sure descriptor is at least the minimum size */
2876 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2877
2878 /* Calculate actual number of SGL (given WQE size) */
2879 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_RCV_HDRS) >> 4;
2880 break;
2881
2882 case HERMON_QP_WQ_TYPE_SENDMLX_QP0:
2883 /*
2884 * Same as above (except for MLX transport WQEs). For these
2885 * WQEs we have to account for the space consumed by the
2886 * "inline" packet headers. (This is smaller than for QP1
2887 * below because QP0 is not allowed to send packets with a GRH.
2888 */
2889 max_size = (HERMON_QP_WQE_MLX_QP0_HDRS + (num_sgl << 4));
2890 log2 = highbit(max_size);
2891 if ((max_size & (max_size - 1)) == 0) {
2892 log2 = log2 - 1;
2893 }
2894
2895 /* Make sure descriptor is at least the minimum size */
2896 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2897
2898 /* Calculate actual number of SGL (given WQE size) */
2899 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_QP0_HDRS) >> 4;
2900 break;
2901
2902 case HERMON_QP_WQ_TYPE_SENDMLX_QP1:
2903 /*
2904 * Same as above. For these WQEs we again have to account for
2905 * the space consumed by the "inline" packet headers. (This
2906 * is larger than for QP0 above because we have to account for
2907 * the possibility of a GRH in each packet - and this
2908 * introduces an alignment issue that causes us to consume
2909 * an additional 8 bytes).
2910 */
2911 max_size = (HERMON_QP_WQE_MLX_QP1_HDRS + (num_sgl << 4));
2912 log2 = highbit(max_size);
2913 if ((max_size & (max_size - 1)) == 0) {
2914 log2 = log2 - 1;
2915 }
2916
2917 /* Make sure descriptor is at least the minimum size */
2918 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2919
2920 /* Calculate actual number of SGL (given WQE size) */
2921 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_QP1_HDRS) >> 4;
2922 break;
2923
2924 default:
2925 HERMON_WARNING(state, "unexpected work queue type");
2926 break;
2927 }
2928
2929 /* Fill in the return values */
2930 *logwqesz = log2;
2931 *max_sgl = min(real_max_sgl, actual_sgl);
2932 }
--- EOF ---