Print this page
5255 uts shouldn't open-code ISP2
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/common/io/ib/adapters/hermon/hermon_qp.c
+++ new/usr/src/uts/common/io/ib/adapters/hermon/hermon_qp.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 */
25 25
26 26 /*
27 27 * hermon_qp.c
28 28 * Hermon Queue Pair Processing Routines
29 29 *
30 30 * Implements all the routines necessary for allocating, freeing, and
31 31 * querying the Hermon queue pairs.
32 32 */
33 33
34 34 #include <sys/types.h>
35 35 #include <sys/conf.h>
36 36 #include <sys/ddi.h>
37 37 #include <sys/sunddi.h>
38 38 #include <sys/modctl.h>
39 39 #include <sys/bitmap.h>
40 40 #include <sys/sysmacros.h>
41 41
42 42 #include <sys/ib/adapters/hermon/hermon.h>
43 43 #include <sys/ib/ib_pkt_hdrs.h>
44 44
45 45 static int hermon_qp_create_qpn(hermon_state_t *state, hermon_qphdl_t qp,
46 46 hermon_rsrc_t *qpc);
47 47 static int hermon_qpn_avl_compare(const void *q, const void *e);
48 48 static int hermon_special_qp_rsrc_alloc(hermon_state_t *state,
49 49 ibt_sqp_type_t type, uint_t port, hermon_rsrc_t **qp_rsrc);
50 50 static int hermon_special_qp_rsrc_free(hermon_state_t *state,
51 51 ibt_sqp_type_t type, uint_t port);
52 52 static void hermon_qp_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl,
53 53 uint_t real_max_sgl, hermon_qp_wq_type_t wq_type,
54 54 uint_t *logwqesz, uint_t *max_sgl);
55 55
56 56 /*
57 57 * hermon_qp_alloc()
58 58 * Context: Can be called only from user or kernel context.
59 59 */
60 60 int
61 61 hermon_qp_alloc(hermon_state_t *state, hermon_qp_info_t *qpinfo,
62 62 uint_t sleepflag)
63 63 {
64 64 hermon_rsrc_t *qpc, *rsrc;
65 65 hermon_rsrc_type_t rsrc_type;
66 66 hermon_umap_db_entry_t *umapdb;
67 67 hermon_qphdl_t qp;
68 68 ibt_qp_alloc_attr_t *attr_p;
69 69 ibt_qp_alloc_flags_t alloc_flags;
70 70 ibt_qp_type_t type;
71 71 hermon_qp_wq_type_t swq_type;
72 72 ibtl_qp_hdl_t ibt_qphdl;
73 73 ibt_chan_sizes_t *queuesz_p;
74 74 ib_qpn_t *qpn;
75 75 hermon_qphdl_t *qphdl;
76 76 ibt_mr_attr_t mr_attr;
77 77 hermon_mr_options_t mr_op;
78 78 hermon_srqhdl_t srq;
79 79 hermon_pdhdl_t pd;
80 80 hermon_cqhdl_t sq_cq, rq_cq;
81 81 hermon_mrhdl_t mr;
82 82 uint64_t value, qp_desc_off;
83 83 uint64_t *thewqe, thewqesz;
84 84 uint32_t *sq_buf, *rq_buf;
85 85 uint32_t log_qp_sq_size, log_qp_rq_size;
86 86 uint32_t sq_size, rq_size;
87 87 uint32_t sq_depth, rq_depth;
88 88 uint32_t sq_wqe_size, rq_wqe_size, wqesz_shift;
89 89 uint32_t max_sgl, max_recv_sgl, uarpg;
90 90 uint_t qp_is_umap;
91 91 uint_t qp_srq_en, i, j;
92 92 int status, flag;
93 93
94 94 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p, *queuesz_p))
95 95
96 96 /*
97 97 * Extract the necessary info from the hermon_qp_info_t structure
98 98 */
99 99 attr_p = qpinfo->qpi_attrp;
100 100 type = qpinfo->qpi_type;
101 101 ibt_qphdl = qpinfo->qpi_ibt_qphdl;
102 102 queuesz_p = qpinfo->qpi_queueszp;
103 103 qpn = qpinfo->qpi_qpn;
104 104 qphdl = &qpinfo->qpi_qphdl;
105 105 alloc_flags = attr_p->qp_alloc_flags;
106 106
107 107 /*
108 108 * Verify correctness of alloc_flags.
109 109 *
110 110 * 1. FEXCH and RSS are only allocated via qp_range.
111 111 */
112 112 if (alloc_flags & (IBT_QP_USES_FEXCH | IBT_QP_USES_RSS)) {
113 113 return (IBT_INVALID_PARAM);
114 114 }
115 115 rsrc_type = HERMON_QPC;
116 116 qp_is_umap = 0;
117 117
118 118 /* 2. Make sure only one of these flags is set. */
119 119 switch (alloc_flags &
120 120 (IBT_QP_USER_MAP | IBT_QP_USES_RFCI | IBT_QP_USES_FCMD)) {
121 121 case IBT_QP_USER_MAP:
122 122 qp_is_umap = 1;
123 123 break;
124 124 case IBT_QP_USES_RFCI:
125 125 if (type != IBT_UD_RQP)
126 126 return (IBT_INVALID_PARAM);
127 127
128 128 switch (attr_p->qp_fc.fc_hca_port) {
129 129 case 1:
130 130 rsrc_type = HERMON_QPC_RFCI_PORT1;
131 131 break;
132 132 case 2:
133 133 rsrc_type = HERMON_QPC_RFCI_PORT2;
134 134 break;
135 135 default:
136 136 return (IBT_INVALID_PARAM);
137 137 }
138 138 break;
139 139 case IBT_QP_USES_FCMD:
140 140 if (type != IBT_UD_RQP)
141 141 return (IBT_INVALID_PARAM);
142 142 break;
143 143 case 0:
144 144 break;
145 145 default:
146 146 return (IBT_INVALID_PARAM); /* conflicting flags set */
147 147 }
148 148
149 149 /*
150 150 * Determine whether QP is being allocated for userland access or
151 151 * whether it is being allocated for kernel access. If the QP is
152 152 * being allocated for userland access, then lookup the UAR
153 153 * page number for the current process. Note: If this is not found
154 154 * (e.g. if the process has not previously open()'d the Hermon driver),
155 155 * then an error is returned.
156 156 */
157 157 if (qp_is_umap) {
158 158 status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(),
159 159 MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
160 160 if (status != DDI_SUCCESS) {
161 161 return (IBT_INVALID_PARAM);
162 162 }
163 163 uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx;
164 164 } else {
165 165 uarpg = state->hs_kernel_uar_index;
166 166 }
167 167
168 168 /*
169 169 * Determine whether QP is being associated with an SRQ
170 170 */
171 171 qp_srq_en = (alloc_flags & IBT_QP_USES_SRQ) ? 1 : 0;
172 172 if (qp_srq_en) {
173 173 /*
174 174 * Check for valid SRQ handle pointers
175 175 */
176 176 if (attr_p->qp_ibc_srq_hdl == NULL) {
177 177 status = IBT_SRQ_HDL_INVALID;
178 178 goto qpalloc_fail;
179 179 }
180 180 srq = (hermon_srqhdl_t)attr_p->qp_ibc_srq_hdl;
181 181 }
182 182
183 183 /*
184 184 * Check for valid QP service type (only UD/RC/UC supported)
185 185 */
186 186 if (((type != IBT_UD_RQP) && (type != IBT_RC_RQP) &&
187 187 (type != IBT_UC_RQP))) {
188 188 status = IBT_QP_SRV_TYPE_INVALID;
189 189 goto qpalloc_fail;
190 190 }
191 191
192 192
193 193 /*
194 194 * Check for valid PD handle pointer
195 195 */
196 196 if (attr_p->qp_pd_hdl == NULL) {
197 197 status = IBT_PD_HDL_INVALID;
198 198 goto qpalloc_fail;
199 199 }
200 200 pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl;
201 201
202 202 /*
203 203 * If on an SRQ, check to make sure the PD is the same
204 204 */
205 205 if (qp_srq_en && (pd->pd_pdnum != srq->srq_pdhdl->pd_pdnum)) {
206 206 status = IBT_PD_HDL_INVALID;
207 207 goto qpalloc_fail;
208 208 }
209 209
210 210 /* Increment the reference count on the protection domain (PD) */
211 211 hermon_pd_refcnt_inc(pd);
212 212
213 213 /*
214 214 * Check for valid CQ handle pointers
215 215 *
216 216 * FCMD QPs do not require a receive cq handle.
217 217 */
218 218 if (attr_p->qp_ibc_scq_hdl == NULL) {
219 219 status = IBT_CQ_HDL_INVALID;
220 220 goto qpalloc_fail1;
221 221 }
222 222 sq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_scq_hdl;
223 223 if ((attr_p->qp_ibc_rcq_hdl == NULL)) {
224 224 if ((alloc_flags & IBT_QP_USES_FCMD) == 0) {
225 225 status = IBT_CQ_HDL_INVALID;
226 226 goto qpalloc_fail1;
227 227 }
228 228 rq_cq = sq_cq; /* just use the send cq */
229 229 } else
230 230 rq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_rcq_hdl;
231 231
232 232 /*
233 233 * Increment the reference count on the CQs. One or both of these
234 234 * could return error if we determine that the given CQ is already
235 235 * being used with a special (SMI/GSI) QP.
236 236 */
237 237 status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_NORMAL);
238 238 if (status != DDI_SUCCESS) {
239 239 status = IBT_CQ_HDL_INVALID;
240 240 goto qpalloc_fail1;
241 241 }
242 242 status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_NORMAL);
243 243 if (status != DDI_SUCCESS) {
244 244 status = IBT_CQ_HDL_INVALID;
245 245 goto qpalloc_fail2;
246 246 }
247 247
248 248 /*
249 249 * Allocate an QP context entry. This will be filled in with all
250 250 * the necessary parameters to define the Queue Pair. Unlike
251 251 * other Hermon hardware resources, ownership is not immediately
252 252 * given to hardware in the final step here. Instead, we must
253 253 * wait until the QP is later transitioned to the "Init" state before
254 254 * passing the QP to hardware. If we fail here, we must undo all
255 255 * the reference count (CQ and PD).
256 256 */
257 257 status = hermon_rsrc_alloc(state, rsrc_type, 1, sleepflag, &qpc);
258 258 if (status != DDI_SUCCESS) {
259 259 status = IBT_INSUFF_RESOURCE;
260 260 goto qpalloc_fail3;
261 261 }
262 262
263 263 /*
264 264 * Allocate the software structure for tracking the queue pair
265 265 * (i.e. the Hermon Queue Pair handle). If we fail here, we must
266 266 * undo the reference counts and the previous resource allocation.
267 267 */
268 268 status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc);
269 269 if (status != DDI_SUCCESS) {
270 270 status = IBT_INSUFF_RESOURCE;
271 271 goto qpalloc_fail4;
272 272 }
273 273 qp = (hermon_qphdl_t)rsrc->hr_addr;
274 274 bzero(qp, sizeof (struct hermon_sw_qp_s));
275 275 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
276 276
277 277 qp->qp_alloc_flags = alloc_flags;
278 278
279 279 /*
280 280 * Calculate the QP number from QPC index. This routine handles
281 281 * all of the operations necessary to keep track of used, unused,
282 282 * and released QP numbers.
283 283 */
284 284 if (type == IBT_UD_RQP) {
285 285 qp->qp_qpnum = qpc->hr_indx;
286 286 qp->qp_ring = qp->qp_qpnum << 8;
287 287 qp->qp_qpn_hdl = NULL;
288 288 } else {
289 289 status = hermon_qp_create_qpn(state, qp, qpc);
290 290 if (status != DDI_SUCCESS) {
291 291 status = IBT_INSUFF_RESOURCE;
292 292 goto qpalloc_fail5;
293 293 }
294 294 }
295 295
296 296 /*
297 297 * If this will be a user-mappable QP, then allocate an entry for
298 298 * the "userland resources database". This will later be added to
299 299 * the database (after all further QP operations are successful).
300 300 * If we fail here, we must undo the reference counts and the
301 301 * previous resource allocation.
302 302 */
303 303 if (qp_is_umap) {
304 304 umapdb = hermon_umap_db_alloc(state->hs_instance, qp->qp_qpnum,
305 305 MLNX_UMAP_QPMEM_RSRC, (uint64_t)(uintptr_t)rsrc);
306 306 if (umapdb == NULL) {
307 307 status = IBT_INSUFF_RESOURCE;
308 308 goto qpalloc_fail6;
309 309 }
310 310 }
311 311
312 312 /*
313 313 * Allocate the doorbell record. Hermon just needs one for the RQ,
314 314 * if the QP is not associated with an SRQ, and use uarpg (above) as
315 315 * the uar index
316 316 */
317 317
318 318 if (!qp_srq_en) {
319 319 status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl,
320 320 &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset);
321 321 if (status != DDI_SUCCESS) {
322 322 status = IBT_INSUFF_RESOURCE;
323 323 goto qpalloc_fail6;
324 324 }
325 325 }
326 326
327 327 qp->qp_uses_lso = (attr_p->qp_flags & IBT_USES_LSO);
328 328
329 329 /*
330 330 * We verify that the requested number of SGL is valid (i.e.
331 331 * consistent with the device limits and/or software-configured
332 332 * limits). If not, then obviously the same cleanup needs to be done.
333 333 */
334 334 if (type == IBT_UD_RQP) {
335 335 max_sgl = state->hs_ibtfinfo.hca_attr->hca_ud_send_sgl_sz;
336 336 swq_type = HERMON_QP_WQ_TYPE_SENDQ_UD;
337 337 } else {
338 338 max_sgl = state->hs_ibtfinfo.hca_attr->hca_conn_send_sgl_sz;
339 339 swq_type = HERMON_QP_WQ_TYPE_SENDQ_CONN;
340 340 }
341 341 max_recv_sgl = state->hs_ibtfinfo.hca_attr->hca_recv_sgl_sz;
342 342 if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) ||
343 343 (!qp_srq_en && (attr_p->qp_sizes.cs_rq_sgl > max_recv_sgl))) {
344 344 status = IBT_HCA_SGL_EXCEEDED;
345 345 goto qpalloc_fail7;
346 346 }
347 347
348 348 /*
349 349 * Determine this QP's WQE stride (for both the Send and Recv WQEs).
350 350 * This will depend on the requested number of SGLs. Note: this
351 351 * has the side-effect of also calculating the real number of SGLs
352 352 * (for the calculated WQE size).
353 353 *
354 354 * For QP's on an SRQ, we set these to 0.
355 355 */
356 356 if (qp_srq_en) {
357 357 qp->qp_rq_log_wqesz = 0;
358 358 qp->qp_rq_sgl = 0;
359 359 } else {
360 360 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl,
361 361 max_recv_sgl, HERMON_QP_WQ_TYPE_RECVQ,
362 362 &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl);
363 363 }
364 364 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
365 365 max_sgl, swq_type, &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
366 366
367 367 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
368 368
369 369 /* NOTE: currently policy in driver, later maybe IBTF interface */
370 370 qp->qp_no_prefetch = 0;
371 371
372 372 /*
373 373 * for prefetching, we need to add the number of wqes in
374 374 * the 2k area plus one to the number requested, but
375 375 * ONLY for send queue. If no_prefetch == 1 (prefetch off)
376 376 * it's exactly TWO wqes for the headroom
377 377 */
378 378 if (qp->qp_no_prefetch)
379 379 qp->qp_sq_headroom = 2 * sq_wqe_size;
380 380 else
381 381 qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE;
382 382 /*
383 383 * hdrm wqes must be integral since both sq_wqe_size &
384 384 * HERMON_QP_OH_SIZE are power of 2
385 385 */
386 386 qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size);
387 387
388 388
389 389 /*
390 390 * Calculate the appropriate size for the work queues.
391 391 * For send queue, add in the headroom wqes to the calculation.
392 392 * Note: All Hermon QP work queues must be a power-of-2 in size. Also
393 393 * they may not be any smaller than HERMON_QP_MIN_SIZE. This step is
394 394 * to round the requested size up to the next highest power-of-2
395 395 */
396 396 /* first, adjust to a minimum and tell the caller the change */
↓ open down ↓ |
396 lines elided |
↑ open up ↑ |
397 397 attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq,
398 398 HERMON_QP_MIN_SIZE);
399 399 attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq,
400 400 HERMON_QP_MIN_SIZE);
401 401 /*
402 402 * now, calculate the alloc size, taking into account
403 403 * the headroom for the sq
404 404 */
405 405 log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes);
406 406 /* if the total is a power of two, reduce it */
407 - if (((attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes) &
408 - (attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes - 1)) == 0) {
407 + if (ISP2(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes)) {
409 408 log_qp_sq_size = log_qp_sq_size - 1;
410 409 }
411 410
412 411 log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq);
413 - if ((attr_p->qp_sizes.cs_rq & (attr_p->qp_sizes.cs_rq - 1)) == 0) {
412 + if (ISP2(attr_p->qp_sizes.cs_rq)) {
414 413 log_qp_rq_size = log_qp_rq_size - 1;
415 414 }
416 415
417 416 /*
418 417 * Next we verify that the rounded-up size is valid (i.e. consistent
419 418 * with the device limits and/or software-configured limits). If not,
420 419 * then obviously we have a lot of cleanup to do before returning.
421 420 *
422 421 * NOTE: the first condition deals with the (test) case of cs_sq
423 422 * being just less than 2^32. In this case, the headroom addition
424 423 * to the requested cs_sq will pass the test when it should not.
425 424 * This test no longer lets that case slip through the check.
426 425 */
427 426 if ((attr_p->qp_sizes.cs_sq >
428 427 (1 << state->hs_cfg_profile->cp_log_max_qp_sz)) ||
429 428 (log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) ||
430 429 (!qp_srq_en && (log_qp_rq_size >
431 430 state->hs_cfg_profile->cp_log_max_qp_sz))) {
432 431 status = IBT_HCA_WR_EXCEEDED;
433 432 goto qpalloc_fail7;
434 433 }
435 434
436 435 /*
437 436 * Allocate the memory for QP work queues. Since Hermon work queues
438 437 * are not allowed to cross a 32-bit (4GB) boundary, the alignment of
439 438 * the work queue memory is very important. We used to allocate
440 439 * work queues (the combined receive and send queues) so that they
441 440 * would be aligned on their combined size. That alignment guaranteed
442 441 * that they would never cross the 4GB boundary (Hermon work queues
443 442 * are on the order of MBs at maximum). Now we are able to relax
444 443 * this alignment constraint by ensuring that the IB address assigned
445 444 * to the queue memory (as a result of the hermon_mr_register() call)
446 445 * is offset from zero.
447 446 * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
448 447 * guarantee the alignment, but when attempting to use IOMMU bypass
449 448 * mode we found that we were not allowed to specify any alignment
450 449 * that was more restrictive than the system page size.
451 450 * So we avoided this constraint by passing two alignment values,
452 451 * one for the memory allocation itself and the other for the DMA
453 452 * handle (for later bind). This used to cause more memory than
454 453 * necessary to be allocated (in order to guarantee the more
455 454 * restrictive alignment contraint). But by guaranteeing the
456 455 * zero-based IB virtual address for the queue, we are able to
457 456 * conserve this memory.
458 457 */
459 458 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
460 459 sq_depth = 1 << log_qp_sq_size;
461 460 sq_size = sq_depth * sq_wqe_size;
462 461
463 462 /* QP on SRQ sets these to 0 */
464 463 if (qp_srq_en) {
465 464 rq_wqe_size = 0;
466 465 rq_size = 0;
467 466 } else {
468 467 rq_wqe_size = 1 << qp->qp_rq_log_wqesz;
469 468 rq_depth = 1 << log_qp_rq_size;
470 469 rq_size = rq_depth * rq_wqe_size;
471 470 }
472 471
473 472 qp->qp_wqinfo.qa_size = sq_size + rq_size;
474 473
475 474 qp->qp_wqinfo.qa_alloc_align = PAGESIZE;
476 475 qp->qp_wqinfo.qa_bind_align = PAGESIZE;
477 476
478 477 if (qp_is_umap) {
479 478 qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
480 479 } else {
481 480 qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
482 481 }
483 482 status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag);
484 483 if (status != DDI_SUCCESS) {
485 484 status = IBT_INSUFF_RESOURCE;
486 485 goto qpalloc_fail7;
487 486 }
488 487
489 488 /*
490 489 * Sort WQs in memory according to stride (*q_wqe_size), largest first
491 490 * If they are equal, still put the SQ first
492 491 */
493 492 qp->qp_sq_baseaddr = 0;
494 493 qp->qp_rq_baseaddr = 0;
495 494 if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) {
496 495 sq_buf = qp->qp_wqinfo.qa_buf_aligned;
497 496
498 497 /* if this QP is on an SRQ, set the rq_buf to NULL */
499 498 if (qp_srq_en) {
500 499 rq_buf = NULL;
501 500 } else {
502 501 rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size);
503 502 qp->qp_rq_baseaddr = sq_size;
504 503 }
505 504 } else {
506 505 rq_buf = qp->qp_wqinfo.qa_buf_aligned;
507 506 sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size);
508 507 qp->qp_sq_baseaddr = rq_size;
509 508 }
510 509
511 510 if (qp_is_umap == 0) {
512 511 qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth);
513 512 if (qp->qp_sq_wqhdr == NULL) {
514 513 status = IBT_INSUFF_RESOURCE;
515 514 goto qpalloc_fail8;
516 515 }
517 516 if (qp_srq_en) {
518 517 qp->qp_rq_wqavl.wqa_wq = srq->srq_wq_wqhdr;
519 518 qp->qp_rq_wqavl.wqa_srq_en = 1;
520 519 qp->qp_rq_wqavl.wqa_srq = srq;
521 520 } else {
522 521 qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(rq_depth);
523 522 if (qp->qp_rq_wqhdr == NULL) {
524 523 status = IBT_INSUFF_RESOURCE;
525 524 goto qpalloc_fail8;
526 525 }
527 526 qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr;
528 527 }
529 528 qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum;
530 529 qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND;
531 530 qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr;
532 531 qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum;
533 532 qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV;
534 533 }
535 534
536 535 /*
537 536 * Register the memory for the QP work queues. The memory for the
538 537 * QP must be registered in the Hermon cMPT tables. This gives us the
539 538 * LKey to specify in the QP context later. Note: The memory for
540 539 * Hermon work queues (both Send and Recv) must be contiguous and
541 540 * registered as a single memory region. Note: If the QP memory is
542 541 * user-mappable, force DDI_DMA_CONSISTENT mapping. Also, in order to
543 542 * meet the alignment restriction, we pass the "mro_bind_override_addr"
544 543 * flag in the call to hermon_mr_register(). This guarantees that the
545 544 * resulting IB vaddr will be zero-based (modulo the offset into the
546 545 * first page). If we fail here, we still have the bunch of resource
547 546 * and reference count cleanup to do.
548 547 */
549 548 flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
550 549 IBT_MR_NOSLEEP;
551 550 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
552 551 mr_attr.mr_len = qp->qp_wqinfo.qa_size;
553 552 mr_attr.mr_as = NULL;
554 553 mr_attr.mr_flags = flag;
555 554 if (qp_is_umap) {
556 555 mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
557 556 } else {
558 557 /* HERMON_QUEUE_LOCATION_NORMAL */
559 558 mr_op.mro_bind_type =
560 559 state->hs_cfg_profile->cp_iommu_bypass;
561 560 }
562 561 mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl;
563 562 mr_op.mro_bind_override_addr = 1;
564 563 status = hermon_mr_register(state, pd, &mr_attr, &mr,
565 564 &mr_op, HERMON_QP_CMPT);
566 565 if (status != DDI_SUCCESS) {
567 566 status = IBT_INSUFF_RESOURCE;
568 567 goto qpalloc_fail9;
569 568 }
570 569
571 570 /*
572 571 * Calculate the offset between the kernel virtual address space
573 572 * and the IB virtual address space. This will be used when
574 573 * posting work requests to properly initialize each WQE.
575 574 */
576 575 qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
577 576 (uint64_t)mr->mr_bindinfo.bi_addr;
578 577
579 578 /*
580 579 * Fill in all the return arguments (if necessary). This includes
581 580 * real work queue sizes (in wqes), real SGLs, and QP number
582 581 */
583 582 if (queuesz_p != NULL) {
584 583 queuesz_p->cs_sq =
585 584 (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes;
586 585 queuesz_p->cs_sq_sgl = qp->qp_sq_sgl;
587 586
588 587 /* if this QP is on an SRQ, set these to 0 */
589 588 if (qp_srq_en) {
590 589 queuesz_p->cs_rq = 0;
591 590 queuesz_p->cs_rq_sgl = 0;
592 591 } else {
593 592 queuesz_p->cs_rq = (1 << log_qp_rq_size);
594 593 queuesz_p->cs_rq_sgl = qp->qp_rq_sgl;
595 594 }
596 595 }
597 596 if (qpn != NULL) {
598 597 *qpn = (ib_qpn_t)qp->qp_qpnum;
599 598 }
600 599
601 600 /*
602 601 * Fill in the rest of the Hermon Queue Pair handle.
603 602 */
604 603 qp->qp_qpcrsrcp = qpc;
605 604 qp->qp_rsrcp = rsrc;
606 605 qp->qp_state = HERMON_QP_RESET;
607 606 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
608 607 qp->qp_pdhdl = pd;
609 608 qp->qp_mrhdl = mr;
610 609 qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ?
611 610 HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED;
612 611 qp->qp_is_special = 0;
613 612 qp->qp_uarpg = uarpg;
614 613 qp->qp_umap_dhp = (devmap_cookie_t)NULL;
615 614 qp->qp_sq_cqhdl = sq_cq;
616 615 qp->qp_sq_bufsz = (1 << log_qp_sq_size);
617 616 qp->qp_sq_logqsz = log_qp_sq_size;
618 617 qp->qp_sq_buf = sq_buf;
619 618 qp->qp_desc_off = qp_desc_off;
620 619 qp->qp_rq_cqhdl = rq_cq;
621 620 qp->qp_rq_buf = rq_buf;
622 621 qp->qp_rlky = (attr_p->qp_flags & IBT_FAST_REG_RES_LKEY) !=
623 622 0;
624 623
625 624 /* if this QP is on an SRQ, set rq_bufsz to 0 */
626 625 if (qp_srq_en) {
627 626 qp->qp_rq_bufsz = 0;
628 627 qp->qp_rq_logqsz = 0;
629 628 } else {
630 629 qp->qp_rq_bufsz = (1 << log_qp_rq_size);
631 630 qp->qp_rq_logqsz = log_qp_rq_size;
632 631 }
633 632
634 633 qp->qp_forward_sqd_event = 0;
635 634 qp->qp_sqd_still_draining = 0;
636 635 qp->qp_hdlrarg = (void *)ibt_qphdl;
637 636 qp->qp_mcg_refcnt = 0;
638 637
639 638 /*
640 639 * If this QP is to be associated with an SRQ, set the SRQ handle
641 640 */
642 641 if (qp_srq_en) {
643 642 qp->qp_srqhdl = srq;
644 643 hermon_srq_refcnt_inc(qp->qp_srqhdl);
645 644 } else {
646 645 qp->qp_srqhdl = NULL;
647 646 }
648 647
649 648 /* Determine the QP service type */
650 649 qp->qp_type = type;
651 650 if (type == IBT_RC_RQP) {
652 651 qp->qp_serv_type = HERMON_QP_RC;
653 652 } else if (type == IBT_UD_RQP) {
654 653 if (alloc_flags & IBT_QP_USES_RFCI)
655 654 qp->qp_serv_type = HERMON_QP_RFCI;
656 655 else if (alloc_flags & IBT_QP_USES_FCMD)
657 656 qp->qp_serv_type = HERMON_QP_FCMND;
658 657 else
659 658 qp->qp_serv_type = HERMON_QP_UD;
660 659 } else {
661 660 qp->qp_serv_type = HERMON_QP_UC;
662 661 }
663 662
664 663 /*
665 664 * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed
666 665 */
667 666
668 667 /*
669 668 * Initialize the SQ WQEs - all that needs to be done is every 64 bytes
670 669 * set the quadword to all F's - high-order bit is owner (init to one)
671 670 * and the rest for the headroom definition of prefetching
672 671 *
673 672 */
674 673 wqesz_shift = qp->qp_sq_log_wqesz;
675 674 thewqesz = 1 << wqesz_shift;
676 675 thewqe = (uint64_t *)(void *)(qp->qp_sq_buf);
677 676 if (qp_is_umap == 0) {
678 677 for (i = 0; i < sq_depth; i++) {
679 678 /*
680 679 * for each stride, go through and every 64 bytes
681 680 * write the init value - having set the address
682 681 * once, just keep incrementing it
683 682 */
684 683 for (j = 0; j < thewqesz; j += 64, thewqe += 8) {
685 684 *(uint32_t *)thewqe = 0xFFFFFFFF;
686 685 }
687 686 }
688 687 }
689 688
690 689 /* Zero out the QP context */
691 690 bzero(&qp->qpc, sizeof (hermon_hw_qpc_t));
692 691
693 692 /*
694 693 * Put QP handle in Hermon QPNum-to-QPHdl list. Then fill in the
695 694 * "qphdl" and return success
696 695 */
697 696 hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx, qp);
698 697
699 698 /*
700 699 * If this is a user-mappable QP, then we need to insert the previously
701 700 * allocated entry into the "userland resources database". This will
702 701 * allow for later lookup during devmap() (i.e. mmap()) calls.
703 702 */
704 703 if (qp_is_umap) {
705 704 hermon_umap_db_add(umapdb);
706 705 }
707 706 mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER,
708 707 DDI_INTR_PRI(state->hs_intrmsi_pri));
709 708
710 709 *qphdl = qp;
711 710
712 711 return (DDI_SUCCESS);
713 712
714 713 /*
715 714 * The following is cleanup for all possible failure cases in this routine
716 715 */
717 716 qpalloc_fail9:
718 717 hermon_queue_free(&qp->qp_wqinfo);
719 718 qpalloc_fail8:
720 719 if (qp->qp_sq_wqhdr)
721 720 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
722 721 if (qp->qp_rq_wqhdr)
723 722 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
724 723 qpalloc_fail7:
725 724 if (qp_is_umap) {
726 725 hermon_umap_db_free(umapdb);
727 726 }
728 727 if (!qp_srq_en) {
729 728 hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr);
730 729 }
731 730
732 731 qpalloc_fail6:
733 732 /*
734 733 * Releasing the QPN will also free up the QPC context. Update
735 734 * the QPC context pointer to indicate this.
736 735 */
737 736 if (qp->qp_qpn_hdl) {
738 737 hermon_qp_release_qpn(state, qp->qp_qpn_hdl,
739 738 HERMON_QPN_RELEASE);
740 739 } else {
741 740 hermon_rsrc_free(state, &qpc);
742 741 }
743 742 qpc = NULL;
744 743 qpalloc_fail5:
745 744 hermon_rsrc_free(state, &rsrc);
746 745 qpalloc_fail4:
747 746 if (qpc) {
748 747 hermon_rsrc_free(state, &qpc);
749 748 }
750 749 qpalloc_fail3:
751 750 hermon_cq_refcnt_dec(rq_cq);
752 751 qpalloc_fail2:
753 752 hermon_cq_refcnt_dec(sq_cq);
754 753 qpalloc_fail1:
755 754 hermon_pd_refcnt_dec(pd);
756 755 qpalloc_fail:
757 756 return (status);
758 757 }
759 758
760 759
761 760
762 761 /*
763 762 * hermon_special_qp_alloc()
764 763 * Context: Can be called only from user or kernel context.
765 764 */
766 765 int
767 766 hermon_special_qp_alloc(hermon_state_t *state, hermon_qp_info_t *qpinfo,
768 767 uint_t sleepflag)
769 768 {
770 769 hermon_rsrc_t *qpc, *rsrc;
771 770 hermon_qphdl_t qp;
772 771 ibt_qp_alloc_attr_t *attr_p;
773 772 ibt_sqp_type_t type;
774 773 uint8_t port;
775 774 ibtl_qp_hdl_t ibt_qphdl;
776 775 ibt_chan_sizes_t *queuesz_p;
777 776 hermon_qphdl_t *qphdl;
778 777 ibt_mr_attr_t mr_attr;
779 778 hermon_mr_options_t mr_op;
780 779 hermon_pdhdl_t pd;
781 780 hermon_cqhdl_t sq_cq, rq_cq;
782 781 hermon_mrhdl_t mr;
783 782 uint64_t qp_desc_off;
784 783 uint64_t *thewqe, thewqesz;
785 784 uint32_t *sq_buf, *rq_buf;
786 785 uint32_t log_qp_sq_size, log_qp_rq_size;
787 786 uint32_t sq_size, rq_size, max_sgl;
788 787 uint32_t uarpg;
789 788 uint32_t sq_depth;
790 789 uint32_t sq_wqe_size, rq_wqe_size, wqesz_shift;
791 790 int status, flag, i, j;
792 791
793 792 /*
794 793 * Extract the necessary info from the hermon_qp_info_t structure
795 794 */
796 795 attr_p = qpinfo->qpi_attrp;
797 796 type = qpinfo->qpi_type;
798 797 port = qpinfo->qpi_port;
799 798 ibt_qphdl = qpinfo->qpi_ibt_qphdl;
800 799 queuesz_p = qpinfo->qpi_queueszp;
801 800 qphdl = &qpinfo->qpi_qphdl;
802 801
803 802 /*
804 803 * Check for valid special QP type (only SMI & GSI supported)
805 804 */
806 805 if ((type != IBT_SMI_SQP) && (type != IBT_GSI_SQP)) {
807 806 status = IBT_QP_SPECIAL_TYPE_INVALID;
808 807 goto spec_qpalloc_fail;
809 808 }
810 809
811 810 /*
812 811 * Check for valid port number
813 812 */
814 813 if (!hermon_portnum_is_valid(state, port)) {
815 814 status = IBT_HCA_PORT_INVALID;
816 815 goto spec_qpalloc_fail;
817 816 }
818 817 port = port - 1;
819 818
820 819 /*
821 820 * Check for valid PD handle pointer
822 821 */
823 822 if (attr_p->qp_pd_hdl == NULL) {
824 823 status = IBT_PD_HDL_INVALID;
825 824 goto spec_qpalloc_fail;
826 825 }
827 826 pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl;
828 827
829 828 /* Increment the reference count on the PD */
830 829 hermon_pd_refcnt_inc(pd);
831 830
832 831 /*
833 832 * Check for valid CQ handle pointers
834 833 */
835 834 if ((attr_p->qp_ibc_scq_hdl == NULL) ||
836 835 (attr_p->qp_ibc_rcq_hdl == NULL)) {
837 836 status = IBT_CQ_HDL_INVALID;
838 837 goto spec_qpalloc_fail1;
839 838 }
840 839 sq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_scq_hdl;
841 840 rq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_rcq_hdl;
842 841
843 842 /*
844 843 * Increment the reference count on the CQs. One or both of these
845 844 * could return error if we determine that the given CQ is already
846 845 * being used with a non-special QP (i.e. a normal QP).
847 846 */
848 847 status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_SPECIAL);
849 848 if (status != DDI_SUCCESS) {
850 849 status = IBT_CQ_HDL_INVALID;
851 850 goto spec_qpalloc_fail1;
852 851 }
853 852 status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_SPECIAL);
854 853 if (status != DDI_SUCCESS) {
855 854 status = IBT_CQ_HDL_INVALID;
856 855 goto spec_qpalloc_fail2;
857 856 }
858 857
859 858 /*
860 859 * Allocate the special QP resources. Essentially, this allocation
861 860 * amounts to checking if the request special QP has already been
862 861 * allocated. If successful, the QP context return is an actual
863 862 * QP context that has been "aliased" to act as a special QP of the
864 863 * appropriate type (and for the appropriate port). Just as in
865 864 * hermon_qp_alloc() above, ownership for this QP context is not
866 865 * immediately given to hardware in the final step here. Instead, we
867 866 * wait until the QP is later transitioned to the "Init" state before
868 867 * passing the QP to hardware. If we fail here, we must undo all
869 868 * the reference count (CQ and PD).
870 869 */
871 870 status = hermon_special_qp_rsrc_alloc(state, type, port, &qpc);
872 871 if (status != DDI_SUCCESS) {
873 872 goto spec_qpalloc_fail3;
874 873 }
875 874
876 875 /*
877 876 * Allocate the software structure for tracking the special queue
878 877 * pair (i.e. the Hermon Queue Pair handle). If we fail here, we
879 878 * must undo the reference counts and the previous resource allocation.
880 879 */
881 880 status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc);
882 881 if (status != DDI_SUCCESS) {
883 882 status = IBT_INSUFF_RESOURCE;
884 883 goto spec_qpalloc_fail4;
885 884 }
886 885 qp = (hermon_qphdl_t)rsrc->hr_addr;
887 886
888 887 bzero(qp, sizeof (struct hermon_sw_qp_s));
889 888
890 889 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
891 890 qp->qp_alloc_flags = attr_p->qp_alloc_flags;
892 891
893 892 /*
894 893 * Actual QP number is a combination of the index of the QPC and
895 894 * the port number. This is because the special QP contexts must
896 895 * be allocated two-at-a-time.
897 896 */
898 897 qp->qp_qpnum = qpc->hr_indx + port;
899 898 qp->qp_ring = qp->qp_qpnum << 8;
900 899
901 900 uarpg = state->hs_kernel_uar_index; /* must be for spec qp */
902 901 /*
903 902 * Allocate the doorbell record. Hermon uses only one for the RQ so
904 903 * alloc a qp doorbell, using uarpg (above) as the uar index
905 904 */
906 905
907 906 status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl,
908 907 &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset);
909 908 if (status != DDI_SUCCESS) {
910 909 status = IBT_INSUFF_RESOURCE;
911 910 goto spec_qpalloc_fail5;
912 911 }
913 912 /*
↓ open down ↓ |
490 lines elided |
↑ open up ↑ |
914 913 * Calculate the appropriate size for the work queues.
915 914 * Note: All Hermon QP work queues must be a power-of-2 in size. Also
916 915 * they may not be any smaller than HERMON_QP_MIN_SIZE. This step is
917 916 * to round the requested size up to the next highest power-of-2
918 917 */
919 918 attr_p->qp_sizes.cs_sq =
920 919 max(attr_p->qp_sizes.cs_sq, HERMON_QP_MIN_SIZE);
921 920 attr_p->qp_sizes.cs_rq =
922 921 max(attr_p->qp_sizes.cs_rq, HERMON_QP_MIN_SIZE);
923 922 log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq);
924 - if ((attr_p->qp_sizes.cs_sq & (attr_p->qp_sizes.cs_sq - 1)) == 0) {
923 + if (ISP2(attr_p->qp_sizes.cs_sq)) {
925 924 log_qp_sq_size = log_qp_sq_size - 1;
926 925 }
927 926 log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq);
928 - if ((attr_p->qp_sizes.cs_rq & (attr_p->qp_sizes.cs_rq - 1)) == 0) {
927 + if (ISP2(attr_p->qp_sizes.cs_rq)) {
929 928 log_qp_rq_size = log_qp_rq_size - 1;
930 929 }
931 930
932 931 /*
933 932 * Next we verify that the rounded-up size is valid (i.e. consistent
934 933 * with the device limits and/or software-configured limits). If not,
935 934 * then obviously we have a bit of cleanup to do before returning.
936 935 */
937 936 if ((log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) ||
938 937 (log_qp_rq_size > state->hs_cfg_profile->cp_log_max_qp_sz)) {
939 938 status = IBT_HCA_WR_EXCEEDED;
940 939 goto spec_qpalloc_fail5a;
941 940 }
942 941
943 942 /*
944 943 * Next we verify that the requested number of SGL is valid (i.e.
945 944 * consistent with the device limits and/or software-configured
946 945 * limits). If not, then obviously the same cleanup needs to be done.
947 946 */
948 947 max_sgl = state->hs_cfg_profile->cp_wqe_real_max_sgl;
949 948 if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) ||
950 949 (attr_p->qp_sizes.cs_rq_sgl > max_sgl)) {
951 950 status = IBT_HCA_SGL_EXCEEDED;
952 951 goto spec_qpalloc_fail5a;
953 952 }
954 953
955 954 /*
956 955 * Determine this QP's WQE stride (for both the Send and Recv WQEs).
957 956 * This will depend on the requested number of SGLs. Note: this
958 957 * has the side-effect of also calculating the real number of SGLs
959 958 * (for the calculated WQE size).
960 959 */
961 960 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl,
962 961 max_sgl, HERMON_QP_WQ_TYPE_RECVQ,
963 962 &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl);
964 963 if (type == IBT_SMI_SQP) {
965 964 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
966 965 max_sgl, HERMON_QP_WQ_TYPE_SENDMLX_QP0,
967 966 &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
968 967 } else {
969 968 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
970 969 max_sgl, HERMON_QP_WQ_TYPE_SENDMLX_QP1,
971 970 &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
972 971 }
973 972
974 973 /*
975 974 * Allocate the memory for QP work queues. Since Hermon work queues
976 975 * are not allowed to cross a 32-bit (4GB) boundary, the alignment of
977 976 * the work queue memory is very important. We used to allocate
978 977 * work queues (the combined receive and send queues) so that they
979 978 * would be aligned on their combined size. That alignment guaranteed
980 979 * that they would never cross the 4GB boundary (Hermon work queues
981 980 * are on the order of MBs at maximum). Now we are able to relax
982 981 * this alignment constraint by ensuring that the IB address assigned
983 982 * to the queue memory (as a result of the hermon_mr_register() call)
984 983 * is offset from zero.
985 984 * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
986 985 * guarantee the alignment, but when attempting to use IOMMU bypass
987 986 * mode we found that we were not allowed to specify any alignment
988 987 * that was more restrictive than the system page size.
989 988 * So we avoided this constraint by passing two alignment values,
990 989 * one for the memory allocation itself and the other for the DMA
991 990 * handle (for later bind). This used to cause more memory than
992 991 * necessary to be allocated (in order to guarantee the more
993 992 * restrictive alignment contraint). But by guaranteeing the
994 993 * zero-based IB virtual address for the queue, we are able to
995 994 * conserve this memory.
996 995 */
997 996 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
998 997 sq_depth = 1 << log_qp_sq_size;
999 998 sq_size = (1 << log_qp_sq_size) * sq_wqe_size;
1000 999
1001 1000 rq_wqe_size = 1 << qp->qp_rq_log_wqesz;
1002 1001 rq_size = (1 << log_qp_rq_size) * rq_wqe_size;
1003 1002
1004 1003 qp->qp_wqinfo.qa_size = sq_size + rq_size;
1005 1004
1006 1005 qp->qp_wqinfo.qa_alloc_align = PAGESIZE;
1007 1006 qp->qp_wqinfo.qa_bind_align = PAGESIZE;
1008 1007 qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
1009 1008
1010 1009 status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag);
1011 1010 if (status != NULL) {
1012 1011 status = IBT_INSUFF_RESOURCE;
1013 1012 goto spec_qpalloc_fail5a;
1014 1013 }
1015 1014
1016 1015 /*
1017 1016 * Sort WQs in memory according to depth, stride (*q_wqe_size),
1018 1017 * biggest first. If equal, the Send Queue still goes first
1019 1018 */
1020 1019 qp->qp_sq_baseaddr = 0;
1021 1020 qp->qp_rq_baseaddr = 0;
1022 1021 if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) {
1023 1022 sq_buf = qp->qp_wqinfo.qa_buf_aligned;
1024 1023 rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size);
1025 1024 qp->qp_rq_baseaddr = sq_size;
1026 1025 } else {
1027 1026 rq_buf = qp->qp_wqinfo.qa_buf_aligned;
1028 1027 sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size);
1029 1028 qp->qp_sq_baseaddr = rq_size;
1030 1029 }
1031 1030
1032 1031 qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth);
1033 1032 if (qp->qp_sq_wqhdr == NULL) {
1034 1033 status = IBT_INSUFF_RESOURCE;
1035 1034 goto spec_qpalloc_fail6;
1036 1035 }
1037 1036 qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(1 << log_qp_rq_size);
1038 1037 if (qp->qp_rq_wqhdr == NULL) {
1039 1038 status = IBT_INSUFF_RESOURCE;
1040 1039 goto spec_qpalloc_fail6;
1041 1040 }
1042 1041 qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum;
1043 1042 qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND;
1044 1043 qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr;
1045 1044 qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum;
1046 1045 qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV;
1047 1046 qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr;
1048 1047
1049 1048 /*
1050 1049 * Register the memory for the special QP work queues. The memory for
1051 1050 * the special QP must be registered in the Hermon cMPT tables. This
1052 1051 * gives us the LKey to specify in the QP context later. Note: The
1053 1052 * memory for Hermon work queues (both Send and Recv) must be contiguous
1054 1053 * and registered as a single memory region. Also, in order to meet the
1055 1054 * alignment restriction, we pass the "mro_bind_override_addr" flag in
1056 1055 * the call to hermon_mr_register(). This guarantees that the resulting
1057 1056 * IB vaddr will be zero-based (modulo the offset into the first page).
1058 1057 * If we fail here, we have a bunch of resource and reference count
1059 1058 * cleanup to do.
1060 1059 */
1061 1060 flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
1062 1061 IBT_MR_NOSLEEP;
1063 1062 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
1064 1063 mr_attr.mr_len = qp->qp_wqinfo.qa_size;
1065 1064 mr_attr.mr_as = NULL;
1066 1065 mr_attr.mr_flags = flag;
1067 1066
1068 1067 mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
1069 1068 mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl;
1070 1069 mr_op.mro_bind_override_addr = 1;
1071 1070
1072 1071 status = hermon_mr_register(state, pd, &mr_attr, &mr, &mr_op,
1073 1072 HERMON_QP_CMPT);
1074 1073 if (status != DDI_SUCCESS) {
1075 1074 status = IBT_INSUFF_RESOURCE;
1076 1075 goto spec_qpalloc_fail6;
1077 1076 }
1078 1077
1079 1078 /*
1080 1079 * Calculate the offset between the kernel virtual address space
1081 1080 * and the IB virtual address space. This will be used when
1082 1081 * posting work requests to properly initialize each WQE.
1083 1082 */
1084 1083 qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
1085 1084 (uint64_t)mr->mr_bindinfo.bi_addr;
1086 1085
1087 1086 /* set the prefetch - initially, not prefetching */
1088 1087 qp->qp_no_prefetch = 1;
1089 1088
1090 1089 if (qp->qp_no_prefetch)
1091 1090 qp->qp_sq_headroom = 2 * sq_wqe_size;
1092 1091 else
1093 1092 qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE;
1094 1093 /*
1095 1094 * hdrm wqes must be integral since both sq_wqe_size &
1096 1095 * HERMON_QP_OH_SIZE are power of 2
1097 1096 */
1098 1097 qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size);
1099 1098 /*
1100 1099 * Fill in all the return arguments (if necessary). This includes
1101 1100 * real work queue sizes, real SGLs, and QP number (which will be
1102 1101 * either zero or one, depending on the special QP type)
1103 1102 */
1104 1103 if (queuesz_p != NULL) {
1105 1104 queuesz_p->cs_sq =
1106 1105 (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes;
1107 1106 queuesz_p->cs_sq_sgl = qp->qp_sq_sgl;
1108 1107 queuesz_p->cs_rq = (1 << log_qp_rq_size);
1109 1108 queuesz_p->cs_rq_sgl = qp->qp_rq_sgl;
1110 1109 }
1111 1110
1112 1111 /*
1113 1112 * Fill in the rest of the Hermon Queue Pair handle. We can update
1114 1113 * the following fields for use in further operations on the QP.
1115 1114 */
1116 1115 qp->qp_qpcrsrcp = qpc;
1117 1116 qp->qp_rsrcp = rsrc;
1118 1117 qp->qp_state = HERMON_QP_RESET;
1119 1118 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
1120 1119 qp->qp_pdhdl = pd;
1121 1120 qp->qp_mrhdl = mr;
1122 1121 qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ?
1123 1122 HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED;
1124 1123 qp->qp_is_special = (type == IBT_SMI_SQP) ?
1125 1124 HERMON_QP_SMI : HERMON_QP_GSI;
1126 1125 qp->qp_uarpg = uarpg;
1127 1126 qp->qp_umap_dhp = (devmap_cookie_t)NULL;
1128 1127 qp->qp_sq_cqhdl = sq_cq;
1129 1128 qp->qp_sq_bufsz = (1 << log_qp_sq_size);
1130 1129 qp->qp_sq_buf = sq_buf;
1131 1130 qp->qp_sq_logqsz = log_qp_sq_size;
1132 1131 qp->qp_desc_off = qp_desc_off;
1133 1132 qp->qp_rq_cqhdl = rq_cq;
1134 1133 qp->qp_rq_bufsz = (1 << log_qp_rq_size);
1135 1134 qp->qp_rq_buf = rq_buf;
1136 1135 qp->qp_rq_logqsz = log_qp_rq_size;
1137 1136 qp->qp_portnum = port;
1138 1137 qp->qp_pkeyindx = 0;
1139 1138 qp->qp_forward_sqd_event = 0;
1140 1139 qp->qp_sqd_still_draining = 0;
1141 1140 qp->qp_hdlrarg = (void *)ibt_qphdl;
1142 1141 qp->qp_mcg_refcnt = 0;
1143 1142 qp->qp_srqhdl = NULL;
1144 1143
1145 1144 /* All special QPs are UD QP service type */
1146 1145 qp->qp_type = IBT_UD_RQP;
1147 1146 qp->qp_serv_type = HERMON_QP_UD;
1148 1147
1149 1148 /*
1150 1149 * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed
1151 1150 */
1152 1151
1153 1152 /*
1154 1153 * Initialize the SQ WQEs - all that needs to be done is every 64 bytes
1155 1154 * set the quadword to all F's - high-order bit is owner (init to one)
1156 1155 * and the rest for the headroom definition of prefetching
1157 1156 *
1158 1157 */
1159 1158
1160 1159 wqesz_shift = qp->qp_sq_log_wqesz;
1161 1160 thewqesz = 1 << wqesz_shift;
1162 1161 thewqe = (uint64_t *)(void *)(qp->qp_sq_buf);
1163 1162 for (i = 0; i < sq_depth; i++) {
1164 1163 /*
1165 1164 * for each stride, go through and every 64 bytes write the
1166 1165 * init value - having set the address once, just keep
1167 1166 * incrementing it
1168 1167 */
1169 1168 for (j = 0; j < thewqesz; j += 64, thewqe += 8) {
1170 1169 *(uint32_t *)thewqe = 0xFFFFFFFF;
1171 1170 }
1172 1171 }
1173 1172
1174 1173
1175 1174 /* Zero out the QP context */
1176 1175 bzero(&qp->qpc, sizeof (hermon_hw_qpc_t));
1177 1176
1178 1177 /*
1179 1178 * Put QP handle in Hermon QPNum-to-QPHdl list. Then fill in the
1180 1179 * "qphdl" and return success
1181 1180 */
1182 1181 hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx + port, qp);
1183 1182
1184 1183 mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER,
1185 1184 DDI_INTR_PRI(state->hs_intrmsi_pri));
1186 1185
1187 1186 *qphdl = qp;
1188 1187
1189 1188 return (DDI_SUCCESS);
1190 1189
1191 1190 /*
1192 1191 * The following is cleanup for all possible failure cases in this routine
1193 1192 */
1194 1193 spec_qpalloc_fail6:
1195 1194 hermon_queue_free(&qp->qp_wqinfo);
1196 1195 if (qp->qp_sq_wqhdr)
1197 1196 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
1198 1197 if (qp->qp_rq_wqhdr)
1199 1198 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
1200 1199 spec_qpalloc_fail5a:
1201 1200 hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr);
1202 1201 spec_qpalloc_fail5:
1203 1202 hermon_rsrc_free(state, &rsrc);
1204 1203 spec_qpalloc_fail4:
1205 1204 if (hermon_special_qp_rsrc_free(state, type, port) != DDI_SUCCESS) {
1206 1205 HERMON_WARNING(state, "failed to free special QP rsrc");
1207 1206 }
1208 1207 spec_qpalloc_fail3:
1209 1208 hermon_cq_refcnt_dec(rq_cq);
1210 1209 spec_qpalloc_fail2:
1211 1210 hermon_cq_refcnt_dec(sq_cq);
1212 1211 spec_qpalloc_fail1:
1213 1212 hermon_pd_refcnt_dec(pd);
1214 1213 spec_qpalloc_fail:
1215 1214 return (status);
1216 1215 }
1217 1216
1218 1217
1219 1218 /*
1220 1219 * hermon_qp_alloc_range()
1221 1220 * Context: Can be called only from user or kernel context.
1222 1221 */
1223 1222 int
1224 1223 hermon_qp_alloc_range(hermon_state_t *state, uint_t log2,
1225 1224 hermon_qp_info_t *qpinfo, ibtl_qp_hdl_t *ibt_qphdl,
1226 1225 ibc_cq_hdl_t *send_cq, ibc_cq_hdl_t *recv_cq,
1227 1226 hermon_qphdl_t *qphdl, uint_t sleepflag)
1228 1227 {
1229 1228 hermon_rsrc_t *qpc, *rsrc;
1230 1229 hermon_rsrc_type_t rsrc_type;
1231 1230 hermon_qphdl_t qp;
1232 1231 hermon_qp_range_t *qp_range_p;
1233 1232 ibt_qp_alloc_attr_t *attr_p;
1234 1233 ibt_qp_type_t type;
1235 1234 hermon_qp_wq_type_t swq_type;
1236 1235 ibt_chan_sizes_t *queuesz_p;
1237 1236 ibt_mr_attr_t mr_attr;
1238 1237 hermon_mr_options_t mr_op;
1239 1238 hermon_srqhdl_t srq;
1240 1239 hermon_pdhdl_t pd;
1241 1240 hermon_cqhdl_t sq_cq, rq_cq;
1242 1241 hermon_mrhdl_t mr;
1243 1242 uint64_t qp_desc_off;
1244 1243 uint64_t *thewqe, thewqesz;
1245 1244 uint32_t *sq_buf, *rq_buf;
1246 1245 uint32_t log_qp_sq_size, log_qp_rq_size;
1247 1246 uint32_t sq_size, rq_size;
1248 1247 uint32_t sq_depth, rq_depth;
1249 1248 uint32_t sq_wqe_size, rq_wqe_size, wqesz_shift;
1250 1249 uint32_t max_sgl, max_recv_sgl, uarpg;
1251 1250 uint_t qp_srq_en, i, j;
1252 1251 int ii; /* loop counter for range */
1253 1252 int status, flag;
1254 1253 uint_t serv_type;
1255 1254
1256 1255 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p, *queuesz_p))
1257 1256
1258 1257 /*
1259 1258 * Extract the necessary info from the hermon_qp_info_t structure
1260 1259 */
1261 1260 attr_p = qpinfo->qpi_attrp;
1262 1261 type = qpinfo->qpi_type;
1263 1262 queuesz_p = qpinfo->qpi_queueszp;
1264 1263
1265 1264 if (attr_p->qp_alloc_flags & IBT_QP_USES_RSS) {
1266 1265 if (log2 > state->hs_ibtfinfo.hca_attr->hca_rss_max_log2_table)
1267 1266 return (IBT_INSUFF_RESOURCE);
1268 1267 rsrc_type = HERMON_QPC;
1269 1268 serv_type = HERMON_QP_UD;
1270 1269 } else if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) {
1271 1270 if (log2 > state->hs_ibtfinfo.hca_attr->hca_fexch_max_log2_qp)
1272 1271 return (IBT_INSUFF_RESOURCE);
1273 1272 switch (attr_p->qp_fc.fc_hca_port) {
1274 1273 case 1:
1275 1274 rsrc_type = HERMON_QPC_FEXCH_PORT1;
1276 1275 break;
1277 1276 case 2:
1278 1277 rsrc_type = HERMON_QPC_FEXCH_PORT2;
1279 1278 break;
1280 1279 default:
1281 1280 return (IBT_INVALID_PARAM);
1282 1281 }
1283 1282 serv_type = HERMON_QP_FEXCH;
1284 1283 } else
1285 1284 return (IBT_INVALID_PARAM);
1286 1285
1287 1286 /*
1288 1287 * Determine whether QP is being allocated for userland access or
1289 1288 * whether it is being allocated for kernel access. If the QP is
1290 1289 * being allocated for userland access, fail (too complex for now).
1291 1290 */
1292 1291 if (attr_p->qp_alloc_flags & IBT_QP_USER_MAP) {
1293 1292 return (IBT_NOT_SUPPORTED);
1294 1293 } else {
1295 1294 uarpg = state->hs_kernel_uar_index;
1296 1295 }
1297 1296
1298 1297 /*
1299 1298 * Determine whether QP is being associated with an SRQ
1300 1299 */
1301 1300 qp_srq_en = (attr_p->qp_alloc_flags & IBT_QP_USES_SRQ) ? 1 : 0;
1302 1301 if (qp_srq_en) {
1303 1302 /*
1304 1303 * Check for valid SRQ handle pointers
1305 1304 */
1306 1305 if (attr_p->qp_ibc_srq_hdl == NULL) {
1307 1306 return (IBT_SRQ_HDL_INVALID);
1308 1307 }
1309 1308 srq = (hermon_srqhdl_t)attr_p->qp_ibc_srq_hdl;
1310 1309 }
1311 1310
1312 1311 /*
1313 1312 * Check for valid QP service type (only UD supported)
1314 1313 */
1315 1314 if (type != IBT_UD_RQP) {
1316 1315 return (IBT_QP_SRV_TYPE_INVALID);
1317 1316 }
1318 1317
1319 1318 /*
1320 1319 * Check for valid PD handle pointer
1321 1320 */
1322 1321 if (attr_p->qp_pd_hdl == NULL) {
1323 1322 return (IBT_PD_HDL_INVALID);
1324 1323 }
1325 1324 pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl;
1326 1325
1327 1326 /*
1328 1327 * If on an SRQ, check to make sure the PD is the same
1329 1328 */
1330 1329 if (qp_srq_en && (pd->pd_pdnum != srq->srq_pdhdl->pd_pdnum)) {
1331 1330 return (IBT_PD_HDL_INVALID);
1332 1331 }
1333 1332
1334 1333 /* set loop variable here, for freeing resources on error */
1335 1334 ii = 0;
1336 1335
1337 1336 /*
1338 1337 * Allocate 2^log2 contiguous/aligned QP context entries. This will
1339 1338 * be filled in with all the necessary parameters to define the
1340 1339 * Queue Pairs. Unlike other Hermon hardware resources, ownership
1341 1340 * is not immediately given to hardware in the final step here.
1342 1341 * Instead, we must wait until the QP is later transitioned to the
1343 1342 * "Init" state before passing the QP to hardware. If we fail here,
1344 1343 * we must undo all the reference count (CQ and PD).
1345 1344 */
1346 1345 status = hermon_rsrc_alloc(state, rsrc_type, 1 << log2, sleepflag,
1347 1346 &qpc);
1348 1347 if (status != DDI_SUCCESS) {
1349 1348 return (IBT_INSUFF_RESOURCE);
1350 1349 }
1351 1350
1352 1351 if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH)
1353 1352 /*
1354 1353 * Need to init the MKEYs for the FEXCH QPs.
1355 1354 *
1356 1355 * For FEXCH QP subranges, we return the QPN base as
1357 1356 * "relative" to the full FEXCH QP range for the port.
1358 1357 */
1359 1358 *(qpinfo->qpi_qpn) = hermon_fcoib_fexch_relative_qpn(state,
1360 1359 attr_p->qp_fc.fc_hca_port, qpc->hr_indx);
1361 1360 else
1362 1361 *(qpinfo->qpi_qpn) = (ib_qpn_t)qpc->hr_indx;
1363 1362
1364 1363 qp_range_p = kmem_alloc(sizeof (*qp_range_p),
1365 1364 (sleepflag == HERMON_SLEEP) ? KM_SLEEP : KM_NOSLEEP);
1366 1365 if (qp_range_p == NULL) {
1367 1366 status = IBT_INSUFF_RESOURCE;
1368 1367 goto qpalloc_fail0;
1369 1368 }
1370 1369 mutex_init(&qp_range_p->hqpr_lock, NULL, MUTEX_DRIVER,
1371 1370 DDI_INTR_PRI(state->hs_intrmsi_pri));
1372 1371 mutex_enter(&qp_range_p->hqpr_lock);
1373 1372 qp_range_p->hqpr_refcnt = 1 << log2;
1374 1373 qp_range_p->hqpr_qpcrsrc = qpc;
1375 1374 mutex_exit(&qp_range_p->hqpr_lock);
1376 1375
1377 1376 for_each_qp:
1378 1377
1379 1378 /* Increment the reference count on the protection domain (PD) */
1380 1379 hermon_pd_refcnt_inc(pd);
1381 1380
1382 1381 rq_cq = (hermon_cqhdl_t)recv_cq[ii];
1383 1382 sq_cq = (hermon_cqhdl_t)send_cq[ii];
1384 1383 if (sq_cq == NULL) {
1385 1384 if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) {
1386 1385 /* if no send completions, just use rq_cq */
1387 1386 sq_cq = rq_cq;
1388 1387 } else {
1389 1388 status = IBT_CQ_HDL_INVALID;
1390 1389 goto qpalloc_fail1;
1391 1390 }
1392 1391 }
1393 1392
1394 1393 /*
1395 1394 * Increment the reference count on the CQs. One or both of these
1396 1395 * could return error if we determine that the given CQ is already
1397 1396 * being used with a special (SMI/GSI) QP.
1398 1397 */
1399 1398 status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_NORMAL);
1400 1399 if (status != DDI_SUCCESS) {
1401 1400 status = IBT_CQ_HDL_INVALID;
1402 1401 goto qpalloc_fail1;
1403 1402 }
1404 1403 status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_NORMAL);
1405 1404 if (status != DDI_SUCCESS) {
1406 1405 status = IBT_CQ_HDL_INVALID;
1407 1406 goto qpalloc_fail2;
1408 1407 }
1409 1408
1410 1409 /*
1411 1410 * Allocate the software structure for tracking the queue pair
1412 1411 * (i.e. the Hermon Queue Pair handle). If we fail here, we must
1413 1412 * undo the reference counts and the previous resource allocation.
1414 1413 */
1415 1414 status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc);
1416 1415 if (status != DDI_SUCCESS) {
1417 1416 status = IBT_INSUFF_RESOURCE;
1418 1417 goto qpalloc_fail4;
1419 1418 }
1420 1419 qp = (hermon_qphdl_t)rsrc->hr_addr;
1421 1420 bzero(qp, sizeof (struct hermon_sw_qp_s));
1422 1421 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
1423 1422 qp->qp_alloc_flags = attr_p->qp_alloc_flags;
1424 1423
1425 1424 /*
1426 1425 * Calculate the QP number from QPC index. This routine handles
1427 1426 * all of the operations necessary to keep track of used, unused,
1428 1427 * and released QP numbers.
1429 1428 */
1430 1429 qp->qp_qpnum = qpc->hr_indx + ii;
1431 1430 qp->qp_ring = qp->qp_qpnum << 8;
1432 1431 qp->qp_qpn_hdl = NULL;
1433 1432
1434 1433 /*
1435 1434 * Allocate the doorbell record. Hermon just needs one for the RQ,
1436 1435 * if the QP is not associated with an SRQ, and use uarpg (above) as
1437 1436 * the uar index
1438 1437 */
1439 1438
1440 1439 if (!qp_srq_en) {
1441 1440 status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl,
1442 1441 &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset);
1443 1442 if (status != DDI_SUCCESS) {
1444 1443 status = IBT_INSUFF_RESOURCE;
1445 1444 goto qpalloc_fail6;
1446 1445 }
1447 1446 }
1448 1447
1449 1448 qp->qp_uses_lso = (attr_p->qp_flags & IBT_USES_LSO);
1450 1449
1451 1450 /*
1452 1451 * We verify that the requested number of SGL is valid (i.e.
1453 1452 * consistent with the device limits and/or software-configured
1454 1453 * limits). If not, then obviously the same cleanup needs to be done.
1455 1454 */
1456 1455 max_sgl = state->hs_ibtfinfo.hca_attr->hca_ud_send_sgl_sz;
1457 1456 swq_type = HERMON_QP_WQ_TYPE_SENDQ_UD;
1458 1457 max_recv_sgl = state->hs_ibtfinfo.hca_attr->hca_recv_sgl_sz;
1459 1458 if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) ||
1460 1459 (!qp_srq_en && (attr_p->qp_sizes.cs_rq_sgl > max_recv_sgl))) {
1461 1460 status = IBT_HCA_SGL_EXCEEDED;
1462 1461 goto qpalloc_fail7;
1463 1462 }
1464 1463
1465 1464 /*
1466 1465 * Determine this QP's WQE stride (for both the Send and Recv WQEs).
1467 1466 * This will depend on the requested number of SGLs. Note: this
1468 1467 * has the side-effect of also calculating the real number of SGLs
1469 1468 * (for the calculated WQE size).
1470 1469 *
1471 1470 * For QP's on an SRQ, we set these to 0.
1472 1471 */
1473 1472 if (qp_srq_en) {
1474 1473 qp->qp_rq_log_wqesz = 0;
1475 1474 qp->qp_rq_sgl = 0;
1476 1475 } else {
1477 1476 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl,
1478 1477 max_recv_sgl, HERMON_QP_WQ_TYPE_RECVQ,
1479 1478 &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl);
1480 1479 }
1481 1480 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
1482 1481 max_sgl, swq_type, &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
1483 1482
1484 1483 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
1485 1484
1486 1485 /* NOTE: currently policy in driver, later maybe IBTF interface */
1487 1486 qp->qp_no_prefetch = 0;
1488 1487
1489 1488 /*
1490 1489 * for prefetching, we need to add the number of wqes in
1491 1490 * the 2k area plus one to the number requested, but
1492 1491 * ONLY for send queue. If no_prefetch == 1 (prefetch off)
1493 1492 * it's exactly TWO wqes for the headroom
1494 1493 */
1495 1494 if (qp->qp_no_prefetch)
1496 1495 qp->qp_sq_headroom = 2 * sq_wqe_size;
1497 1496 else
1498 1497 qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE;
1499 1498 /*
1500 1499 * hdrm wqes must be integral since both sq_wqe_size &
1501 1500 * HERMON_QP_OH_SIZE are power of 2
1502 1501 */
1503 1502 qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size);
1504 1503
1505 1504
1506 1505 /*
1507 1506 * Calculate the appropriate size for the work queues.
1508 1507 * For send queue, add in the headroom wqes to the calculation.
1509 1508 * Note: All Hermon QP work queues must be a power-of-2 in size. Also
1510 1509 * they may not be any smaller than HERMON_QP_MIN_SIZE. This step is
1511 1510 * to round the requested size up to the next highest power-of-2
1512 1511 */
1513 1512 /* first, adjust to a minimum and tell the caller the change */
↓ open down ↓ |
575 lines elided |
↑ open up ↑ |
1514 1513 attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq,
1515 1514 HERMON_QP_MIN_SIZE);
1516 1515 attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq,
1517 1516 HERMON_QP_MIN_SIZE);
1518 1517 /*
1519 1518 * now, calculate the alloc size, taking into account
1520 1519 * the headroom for the sq
1521 1520 */
1522 1521 log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes);
1523 1522 /* if the total is a power of two, reduce it */
1524 - if (((attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes) &
1525 - (attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes - 1)) == 0) {
1523 + if (ISP2(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes)) {
1526 1524 log_qp_sq_size = log_qp_sq_size - 1;
1527 1525 }
1528 1526
1529 1527 log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq);
1530 - if ((attr_p->qp_sizes.cs_rq & (attr_p->qp_sizes.cs_rq - 1)) == 0) {
1528 + if (ISP2(attr_p->qp_sizes.cs_rq)) {
1531 1529 log_qp_rq_size = log_qp_rq_size - 1;
1532 1530 }
1533 1531
1534 1532 /*
1535 1533 * Next we verify that the rounded-up size is valid (i.e. consistent
1536 1534 * with the device limits and/or software-configured limits). If not,
1537 1535 * then obviously we have a lot of cleanup to do before returning.
1538 1536 *
1539 1537 * NOTE: the first condition deals with the (test) case of cs_sq
1540 1538 * being just less than 2^32. In this case, the headroom addition
1541 1539 * to the requested cs_sq will pass the test when it should not.
1542 1540 * This test no longer lets that case slip through the check.
1543 1541 */
1544 1542 if ((attr_p->qp_sizes.cs_sq >
1545 1543 (1 << state->hs_cfg_profile->cp_log_max_qp_sz)) ||
1546 1544 (log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) ||
1547 1545 (!qp_srq_en && (log_qp_rq_size >
1548 1546 state->hs_cfg_profile->cp_log_max_qp_sz))) {
1549 1547 status = IBT_HCA_WR_EXCEEDED;
1550 1548 goto qpalloc_fail7;
1551 1549 }
1552 1550
1553 1551 /*
1554 1552 * Allocate the memory for QP work queues. Since Hermon work queues
1555 1553 * are not allowed to cross a 32-bit (4GB) boundary, the alignment of
1556 1554 * the work queue memory is very important. We used to allocate
1557 1555 * work queues (the combined receive and send queues) so that they
1558 1556 * would be aligned on their combined size. That alignment guaranteed
1559 1557 * that they would never cross the 4GB boundary (Hermon work queues
1560 1558 * are on the order of MBs at maximum). Now we are able to relax
1561 1559 * this alignment constraint by ensuring that the IB address assigned
1562 1560 * to the queue memory (as a result of the hermon_mr_register() call)
1563 1561 * is offset from zero.
1564 1562 * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
1565 1563 * guarantee the alignment, but when attempting to use IOMMU bypass
1566 1564 * mode we found that we were not allowed to specify any alignment
1567 1565 * that was more restrictive than the system page size.
1568 1566 * So we avoided this constraint by passing two alignment values,
1569 1567 * one for the memory allocation itself and the other for the DMA
1570 1568 * handle (for later bind). This used to cause more memory than
1571 1569 * necessary to be allocated (in order to guarantee the more
1572 1570 * restrictive alignment contraint). But by guaranteeing the
1573 1571 * zero-based IB virtual address for the queue, we are able to
1574 1572 * conserve this memory.
1575 1573 */
1576 1574 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
1577 1575 sq_depth = 1 << log_qp_sq_size;
1578 1576 sq_size = sq_depth * sq_wqe_size;
1579 1577
1580 1578 /* QP on SRQ sets these to 0 */
1581 1579 if (qp_srq_en) {
1582 1580 rq_wqe_size = 0;
1583 1581 rq_size = 0;
1584 1582 } else {
1585 1583 rq_wqe_size = 1 << qp->qp_rq_log_wqesz;
1586 1584 rq_depth = 1 << log_qp_rq_size;
1587 1585 rq_size = rq_depth * rq_wqe_size;
1588 1586 }
1589 1587
1590 1588 qp->qp_wqinfo.qa_size = sq_size + rq_size;
1591 1589 qp->qp_wqinfo.qa_alloc_align = PAGESIZE;
1592 1590 qp->qp_wqinfo.qa_bind_align = PAGESIZE;
1593 1591 qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
1594 1592 status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag);
1595 1593 if (status != DDI_SUCCESS) {
1596 1594 status = IBT_INSUFF_RESOURCE;
1597 1595 goto qpalloc_fail7;
1598 1596 }
1599 1597
1600 1598 /*
1601 1599 * Sort WQs in memory according to stride (*q_wqe_size), largest first
1602 1600 * If they are equal, still put the SQ first
1603 1601 */
1604 1602 qp->qp_sq_baseaddr = 0;
1605 1603 qp->qp_rq_baseaddr = 0;
1606 1604 if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) {
1607 1605 sq_buf = qp->qp_wqinfo.qa_buf_aligned;
1608 1606
1609 1607 /* if this QP is on an SRQ, set the rq_buf to NULL */
1610 1608 if (qp_srq_en) {
1611 1609 rq_buf = NULL;
1612 1610 } else {
1613 1611 rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size);
1614 1612 qp->qp_rq_baseaddr = sq_size;
1615 1613 }
1616 1614 } else {
1617 1615 rq_buf = qp->qp_wqinfo.qa_buf_aligned;
1618 1616 sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size);
1619 1617 qp->qp_sq_baseaddr = rq_size;
1620 1618 }
1621 1619
1622 1620 qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth);
1623 1621 if (qp->qp_sq_wqhdr == NULL) {
1624 1622 status = IBT_INSUFF_RESOURCE;
1625 1623 goto qpalloc_fail8;
1626 1624 }
1627 1625 if (qp_srq_en) {
1628 1626 qp->qp_rq_wqavl.wqa_wq = srq->srq_wq_wqhdr;
1629 1627 qp->qp_rq_wqavl.wqa_srq_en = 1;
1630 1628 qp->qp_rq_wqavl.wqa_srq = srq;
1631 1629 } else {
1632 1630 qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(rq_depth);
1633 1631 if (qp->qp_rq_wqhdr == NULL) {
1634 1632 status = IBT_INSUFF_RESOURCE;
1635 1633 goto qpalloc_fail8;
1636 1634 }
1637 1635 qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr;
1638 1636 }
1639 1637 qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum;
1640 1638 qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND;
1641 1639 qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr;
1642 1640 qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum;
1643 1641 qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV;
1644 1642
1645 1643 /*
1646 1644 * Register the memory for the QP work queues. The memory for the
1647 1645 * QP must be registered in the Hermon cMPT tables. This gives us the
1648 1646 * LKey to specify in the QP context later. Note: The memory for
1649 1647 * Hermon work queues (both Send and Recv) must be contiguous and
1650 1648 * registered as a single memory region. Note: If the QP memory is
1651 1649 * user-mappable, force DDI_DMA_CONSISTENT mapping. Also, in order to
1652 1650 * meet the alignment restriction, we pass the "mro_bind_override_addr"
1653 1651 * flag in the call to hermon_mr_register(). This guarantees that the
1654 1652 * resulting IB vaddr will be zero-based (modulo the offset into the
1655 1653 * first page). If we fail here, we still have the bunch of resource
1656 1654 * and reference count cleanup to do.
1657 1655 */
1658 1656 flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
1659 1657 IBT_MR_NOSLEEP;
1660 1658 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
1661 1659 mr_attr.mr_len = qp->qp_wqinfo.qa_size;
1662 1660 mr_attr.mr_as = NULL;
1663 1661 mr_attr.mr_flags = flag;
1664 1662 /* HERMON_QUEUE_LOCATION_NORMAL */
1665 1663 mr_op.mro_bind_type =
1666 1664 state->hs_cfg_profile->cp_iommu_bypass;
1667 1665 mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl;
1668 1666 mr_op.mro_bind_override_addr = 1;
1669 1667 status = hermon_mr_register(state, pd, &mr_attr, &mr,
1670 1668 &mr_op, HERMON_QP_CMPT);
1671 1669 if (status != DDI_SUCCESS) {
1672 1670 status = IBT_INSUFF_RESOURCE;
1673 1671 goto qpalloc_fail9;
1674 1672 }
1675 1673
1676 1674 /*
1677 1675 * Calculate the offset between the kernel virtual address space
1678 1676 * and the IB virtual address space. This will be used when
1679 1677 * posting work requests to properly initialize each WQE.
1680 1678 */
1681 1679 qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
1682 1680 (uint64_t)mr->mr_bindinfo.bi_addr;
1683 1681
1684 1682 /*
1685 1683 * Fill in all the return arguments (if necessary). This includes
1686 1684 * real work queue sizes (in wqes), real SGLs, and QP number
1687 1685 */
1688 1686 if (queuesz_p != NULL) {
1689 1687 queuesz_p->cs_sq =
1690 1688 (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes;
1691 1689 queuesz_p->cs_sq_sgl = qp->qp_sq_sgl;
1692 1690
1693 1691 /* if this QP is on an SRQ, set these to 0 */
1694 1692 if (qp_srq_en) {
1695 1693 queuesz_p->cs_rq = 0;
1696 1694 queuesz_p->cs_rq_sgl = 0;
1697 1695 } else {
1698 1696 queuesz_p->cs_rq = (1 << log_qp_rq_size);
1699 1697 queuesz_p->cs_rq_sgl = qp->qp_rq_sgl;
1700 1698 }
1701 1699 }
1702 1700
1703 1701 /*
1704 1702 * Fill in the rest of the Hermon Queue Pair handle.
1705 1703 */
1706 1704 qp->qp_qpcrsrcp = NULL;
1707 1705 qp->qp_rsrcp = rsrc;
1708 1706 qp->qp_state = HERMON_QP_RESET;
1709 1707 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
1710 1708 qp->qp_pdhdl = pd;
1711 1709 qp->qp_mrhdl = mr;
1712 1710 qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ?
1713 1711 HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED;
1714 1712 qp->qp_is_special = 0;
1715 1713 qp->qp_uarpg = uarpg;
1716 1714 qp->qp_umap_dhp = (devmap_cookie_t)NULL;
1717 1715 qp->qp_sq_cqhdl = sq_cq;
1718 1716 qp->qp_sq_bufsz = (1 << log_qp_sq_size);
1719 1717 qp->qp_sq_logqsz = log_qp_sq_size;
1720 1718 qp->qp_sq_buf = sq_buf;
1721 1719 qp->qp_desc_off = qp_desc_off;
1722 1720 qp->qp_rq_cqhdl = rq_cq;
1723 1721 qp->qp_rq_buf = rq_buf;
1724 1722 qp->qp_rlky = (attr_p->qp_flags & IBT_FAST_REG_RES_LKEY) !=
1725 1723 0;
1726 1724
1727 1725 /* if this QP is on an SRQ, set rq_bufsz to 0 */
1728 1726 if (qp_srq_en) {
1729 1727 qp->qp_rq_bufsz = 0;
1730 1728 qp->qp_rq_logqsz = 0;
1731 1729 } else {
1732 1730 qp->qp_rq_bufsz = (1 << log_qp_rq_size);
1733 1731 qp->qp_rq_logqsz = log_qp_rq_size;
1734 1732 }
1735 1733
1736 1734 qp->qp_forward_sqd_event = 0;
1737 1735 qp->qp_sqd_still_draining = 0;
1738 1736 qp->qp_hdlrarg = (void *)ibt_qphdl[ii];
1739 1737 qp->qp_mcg_refcnt = 0;
1740 1738
1741 1739 /*
1742 1740 * If this QP is to be associated with an SRQ, set the SRQ handle
1743 1741 */
1744 1742 if (qp_srq_en) {
1745 1743 qp->qp_srqhdl = srq;
1746 1744 hermon_srq_refcnt_inc(qp->qp_srqhdl);
1747 1745 } else {
1748 1746 qp->qp_srqhdl = NULL;
1749 1747 }
1750 1748
1751 1749 qp->qp_type = IBT_UD_RQP;
1752 1750 qp->qp_serv_type = serv_type;
1753 1751
1754 1752 /*
1755 1753 * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed
1756 1754 */
1757 1755
1758 1756 /*
1759 1757 * Initialize the SQ WQEs - all that needs to be done is every 64 bytes
1760 1758 * set the quadword to all F's - high-order bit is owner (init to one)
1761 1759 * and the rest for the headroom definition of prefetching.
1762 1760 */
1763 1761 if ((attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) == 0) {
1764 1762 wqesz_shift = qp->qp_sq_log_wqesz;
1765 1763 thewqesz = 1 << wqesz_shift;
1766 1764 thewqe = (uint64_t *)(void *)(qp->qp_sq_buf);
1767 1765 for (i = 0; i < sq_depth; i++) {
1768 1766 /*
1769 1767 * for each stride, go through and every 64 bytes
1770 1768 * write the init value - having set the address
1771 1769 * once, just keep incrementing it
1772 1770 */
1773 1771 for (j = 0; j < thewqesz; j += 64, thewqe += 8) {
1774 1772 *(uint32_t *)thewqe = 0xFFFFFFFF;
1775 1773 }
1776 1774 }
1777 1775 }
1778 1776
1779 1777 /* Zero out the QP context */
1780 1778 bzero(&qp->qpc, sizeof (hermon_hw_qpc_t));
1781 1779
1782 1780 /*
1783 1781 * Put QP handle in Hermon QPNum-to-QPHdl list. Then fill in the
1784 1782 * "qphdl" and return success
1785 1783 */
1786 1784 hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx + ii, qp);
1787 1785
1788 1786 mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER,
1789 1787 DDI_INTR_PRI(state->hs_intrmsi_pri));
1790 1788
1791 1789 qp->qp_rangep = qp_range_p;
1792 1790
1793 1791 qphdl[ii] = qp;
1794 1792
1795 1793 if (++ii < (1 << log2))
1796 1794 goto for_each_qp;
1797 1795
1798 1796 return (DDI_SUCCESS);
1799 1797
1800 1798 /*
1801 1799 * The following is cleanup for all possible failure cases in this routine
1802 1800 */
1803 1801 qpalloc_fail9:
1804 1802 hermon_queue_free(&qp->qp_wqinfo);
1805 1803 qpalloc_fail8:
1806 1804 if (qp->qp_sq_wqhdr)
1807 1805 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
1808 1806 if (qp->qp_rq_wqhdr)
1809 1807 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
1810 1808 qpalloc_fail7:
1811 1809 if (!qp_srq_en) {
1812 1810 hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr);
1813 1811 }
1814 1812
1815 1813 qpalloc_fail6:
1816 1814 hermon_rsrc_free(state, &rsrc);
1817 1815 qpalloc_fail4:
1818 1816 hermon_cq_refcnt_dec(rq_cq);
1819 1817 qpalloc_fail2:
1820 1818 hermon_cq_refcnt_dec(sq_cq);
1821 1819 qpalloc_fail1:
1822 1820 hermon_pd_refcnt_dec(pd);
1823 1821 qpalloc_fail0:
1824 1822 if (ii == 0) {
1825 1823 if (qp_range_p)
1826 1824 kmem_free(qp_range_p, sizeof (*qp_range_p));
1827 1825 hermon_rsrc_free(state, &qpc);
1828 1826 } else {
1829 1827 /* qp_range_p and qpc rsrc will be freed in hermon_qp_free */
1830 1828
1831 1829 mutex_enter(&qp->qp_rangep->hqpr_lock);
1832 1830 qp_range_p->hqpr_refcnt = ii;
1833 1831 mutex_exit(&qp->qp_rangep->hqpr_lock);
1834 1832 while (--ii >= 0) {
1835 1833 ibc_qpn_hdl_t qpn_hdl;
1836 1834 int free_status;
1837 1835
1838 1836 free_status = hermon_qp_free(state, &qphdl[ii],
1839 1837 IBC_FREE_QP_AND_QPN, &qpn_hdl, sleepflag);
1840 1838 if (free_status != DDI_SUCCESS)
1841 1839 cmn_err(CE_CONT, "!qp_range: status 0x%x: "
1842 1840 "error status %x during free",
1843 1841 status, free_status);
1844 1842 }
1845 1843 }
1846 1844
1847 1845 return (status);
1848 1846 }
1849 1847
1850 1848
1851 1849 /*
1852 1850 * hermon_qp_free()
1853 1851 * This function frees up the QP resources. Depending on the value
1854 1852 * of the "free_qp_flags", the QP number may not be released until
1855 1853 * a subsequent call to hermon_qp_release_qpn().
1856 1854 *
1857 1855 * Context: Can be called only from user or kernel context.
1858 1856 */
1859 1857 /* ARGSUSED */
1860 1858 int
1861 1859 hermon_qp_free(hermon_state_t *state, hermon_qphdl_t *qphdl,
1862 1860 ibc_free_qp_flags_t free_qp_flags, ibc_qpn_hdl_t *qpnh,
1863 1861 uint_t sleepflag)
1864 1862 {
1865 1863 hermon_rsrc_t *qpc, *rsrc;
1866 1864 hermon_umap_db_entry_t *umapdb;
1867 1865 hermon_qpn_entry_t *entry;
1868 1866 hermon_pdhdl_t pd;
1869 1867 hermon_mrhdl_t mr;
1870 1868 hermon_cqhdl_t sq_cq, rq_cq;
1871 1869 hermon_srqhdl_t srq;
1872 1870 hermon_qphdl_t qp;
1873 1871 uint64_t value;
1874 1872 uint_t type, port;
1875 1873 uint_t maxprot;
1876 1874 uint_t qp_srq_en;
1877 1875 int status;
1878 1876
1879 1877 /*
1880 1878 * Pull all the necessary information from the Hermon Queue Pair
1881 1879 * handle. This is necessary here because the resource for the
1882 1880 * QP handle is going to be freed up as part of this operation.
1883 1881 */
1884 1882 qp = *qphdl;
1885 1883 mutex_enter(&qp->qp_lock);
1886 1884 qpc = qp->qp_qpcrsrcp; /* NULL if part of a "range" */
1887 1885 rsrc = qp->qp_rsrcp;
1888 1886 pd = qp->qp_pdhdl;
1889 1887 srq = qp->qp_srqhdl;
1890 1888 mr = qp->qp_mrhdl;
1891 1889 rq_cq = qp->qp_rq_cqhdl;
1892 1890 sq_cq = qp->qp_sq_cqhdl;
1893 1891 port = qp->qp_portnum;
1894 1892 qp_srq_en = qp->qp_alloc_flags & IBT_QP_USES_SRQ;
1895 1893
1896 1894 /*
1897 1895 * If the QP is part of an MCG, then we fail the qp_free
1898 1896 */
1899 1897 if (qp->qp_mcg_refcnt != 0) {
1900 1898 mutex_exit(&qp->qp_lock);
1901 1899 status = ibc_get_ci_failure(0);
1902 1900 goto qpfree_fail;
1903 1901 }
1904 1902
1905 1903 /*
1906 1904 * If the QP is not already in "Reset" state, then transition to
1907 1905 * "Reset". This is necessary because software does not reclaim
1908 1906 * ownership of the QP context until the QP is in the "Reset" state.
1909 1907 * If the ownership transfer fails for any reason, then it is an
1910 1908 * indication that something (either in HW or SW) has gone seriously
1911 1909 * wrong. So we print a warning message and return.
1912 1910 */
1913 1911 if (qp->qp_state != HERMON_QP_RESET) {
1914 1912 if (hermon_qp_to_reset(state, qp) != DDI_SUCCESS) {
1915 1913 mutex_exit(&qp->qp_lock);
1916 1914 HERMON_WARNING(state, "failed to reset QP context");
1917 1915 status = ibc_get_ci_failure(0);
1918 1916 goto qpfree_fail;
1919 1917 }
1920 1918 qp->qp_state = HERMON_QP_RESET;
1921 1919 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
1922 1920
1923 1921 /*
1924 1922 * Do any additional handling necessary for the transition
1925 1923 * to the "Reset" state (e.g. update the WRID lists)
1926 1924 */
1927 1925 if (hermon_wrid_to_reset_handling(state, qp) != DDI_SUCCESS) {
1928 1926 mutex_exit(&qp->qp_lock);
1929 1927 HERMON_WARNING(state, "failed to reset QP WRID list");
1930 1928 status = ibc_get_ci_failure(0);
1931 1929 goto qpfree_fail;
1932 1930 }
1933 1931 }
1934 1932
1935 1933 /*
1936 1934 * If this was a user-mappable QP, then we need to remove its entry
1937 1935 * from the "userland resources database". If it is also currently
1938 1936 * mmap()'d out to a user process, then we need to call
1939 1937 * devmap_devmem_remap() to remap the QP memory to an invalid mapping.
1940 1938 * We also need to invalidate the QP tracking information for the
1941 1939 * user mapping.
1942 1940 */
1943 1941 if (qp->qp_alloc_flags & IBT_QP_USER_MAP) {
1944 1942 status = hermon_umap_db_find(state->hs_instance, qp->qp_qpnum,
1945 1943 MLNX_UMAP_QPMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE,
1946 1944 &umapdb);
1947 1945 if (status != DDI_SUCCESS) {
1948 1946 mutex_exit(&qp->qp_lock);
1949 1947 HERMON_WARNING(state, "failed to find in database");
1950 1948 return (ibc_get_ci_failure(0));
1951 1949 }
1952 1950 hermon_umap_db_free(umapdb);
1953 1951 if (qp->qp_umap_dhp != NULL) {
1954 1952 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
1955 1953 status = devmap_devmem_remap(qp->qp_umap_dhp,
1956 1954 state->hs_dip, 0, 0, qp->qp_wqinfo.qa_size,
1957 1955 maxprot, DEVMAP_MAPPING_INVALID, NULL);
1958 1956 if (status != DDI_SUCCESS) {
1959 1957 mutex_exit(&qp->qp_lock);
1960 1958 HERMON_WARNING(state, "failed in QP memory "
1961 1959 "devmap_devmem_remap()");
1962 1960 return (ibc_get_ci_failure(0));
1963 1961 }
1964 1962 qp->qp_umap_dhp = (devmap_cookie_t)NULL;
1965 1963 }
1966 1964 }
1967 1965
1968 1966
1969 1967 /*
1970 1968 * Put NULL into the Hermon QPNum-to-QPHdl list. This will allow any
1971 1969 * in-progress events to detect that the QP corresponding to this
1972 1970 * number has been freed. Note: it does depend in whether we are
1973 1971 * freeing a special QP or not.
1974 1972 */
1975 1973 if (qpc == NULL) {
1976 1974 hermon_icm_set_num_to_hdl(state, HERMON_QPC,
1977 1975 qp->qp_qpnum, NULL);
1978 1976 } else if (qp->qp_is_special) {
1979 1977 hermon_icm_set_num_to_hdl(state, HERMON_QPC,
1980 1978 qpc->hr_indx + port, NULL);
1981 1979 } else {
1982 1980 hermon_icm_set_num_to_hdl(state, HERMON_QPC,
1983 1981 qpc->hr_indx, NULL);
1984 1982 }
1985 1983
1986 1984 /*
1987 1985 * Drop the QP lock
1988 1986 * At this point the lock is no longer necessary. We cannot
1989 1987 * protect from multiple simultaneous calls to free the same QP.
1990 1988 * In addition, since the QP lock is contained in the QP "software
1991 1989 * handle" resource, which we will free (see below), it is
1992 1990 * important that we have no further references to that memory.
1993 1991 */
1994 1992 mutex_exit(&qp->qp_lock);
1995 1993 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
1996 1994
1997 1995 /*
1998 1996 * Free the QP resources
1999 1997 * Start by deregistering and freeing the memory for work queues.
2000 1998 * Next free any previously allocated context information
2001 1999 * (depending on QP type)
2002 2000 * Finally, decrement the necessary reference counts.
2003 2001 * If this fails for any reason, then it is an indication that
2004 2002 * something (either in HW or SW) has gone seriously wrong. So we
2005 2003 * print a warning message and return.
2006 2004 */
2007 2005 status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
2008 2006 sleepflag);
2009 2007 if (status != DDI_SUCCESS) {
2010 2008 HERMON_WARNING(state, "failed to deregister QP memory");
2011 2009 status = ibc_get_ci_failure(0);
2012 2010 goto qpfree_fail;
2013 2011 }
2014 2012
2015 2013 /* Free the memory for the QP */
2016 2014 hermon_queue_free(&qp->qp_wqinfo);
2017 2015
2018 2016 if (qp->qp_sq_wqhdr)
2019 2017 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
2020 2018 if (qp->qp_rq_wqhdr)
2021 2019 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
2022 2020
2023 2021 /* Free the dbr */
2024 2022 if (!qp_srq_en) {
2025 2023 hermon_dbr_free(state, qp->qp_uarpg, qp->qp_rq_vdbr);
2026 2024 }
2027 2025
2028 2026 /*
2029 2027 * Free up the remainder of the QP resources. Note: we have a few
2030 2028 * different resources to free up depending on whether the QP is a
2031 2029 * special QP or not. As described above, if any of these fail for
2032 2030 * any reason it is an indication that something (either in HW or SW)
2033 2031 * has gone seriously wrong. So we print a warning message and
2034 2032 * return.
2035 2033 */
2036 2034 if (qp->qp_is_special) {
2037 2035 type = (qp->qp_is_special == HERMON_QP_SMI) ?
2038 2036 IBT_SMI_SQP : IBT_GSI_SQP;
2039 2037
2040 2038 /* Free up resources for the special QP */
2041 2039 status = hermon_special_qp_rsrc_free(state, type, port);
2042 2040 if (status != DDI_SUCCESS) {
2043 2041 HERMON_WARNING(state, "failed to free special QP rsrc");
2044 2042 status = ibc_get_ci_failure(0);
2045 2043 goto qpfree_fail;
2046 2044 }
2047 2045
2048 2046 } else if (qp->qp_rangep) {
2049 2047 int refcnt;
2050 2048 mutex_enter(&qp->qp_rangep->hqpr_lock);
2051 2049 refcnt = --qp->qp_rangep->hqpr_refcnt;
2052 2050 mutex_exit(&qp->qp_rangep->hqpr_lock);
2053 2051 if (refcnt == 0) {
2054 2052 mutex_destroy(&qp->qp_rangep->hqpr_lock);
2055 2053 hermon_rsrc_free(state, &qp->qp_rangep->hqpr_qpcrsrc);
2056 2054 kmem_free(qp->qp_rangep, sizeof (*qp->qp_rangep));
2057 2055 }
2058 2056 qp->qp_rangep = NULL;
2059 2057 } else if (qp->qp_qpn_hdl == NULL) {
2060 2058 hermon_rsrc_free(state, &qpc);
2061 2059 } else {
2062 2060 /*
2063 2061 * Check the flags and determine whether to release the
2064 2062 * QPN or not, based on their value.
2065 2063 */
2066 2064 if (free_qp_flags == IBC_FREE_QP_ONLY) {
2067 2065 entry = qp->qp_qpn_hdl;
2068 2066 hermon_qp_release_qpn(state, qp->qp_qpn_hdl,
2069 2067 HERMON_QPN_FREE_ONLY);
2070 2068 *qpnh = (ibc_qpn_hdl_t)entry;
2071 2069 } else {
2072 2070 hermon_qp_release_qpn(state, qp->qp_qpn_hdl,
2073 2071 HERMON_QPN_RELEASE);
2074 2072 }
2075 2073 }
2076 2074
2077 2075 mutex_destroy(&qp->qp_sq_lock);
2078 2076
2079 2077 /* Free the Hermon Queue Pair handle */
2080 2078 hermon_rsrc_free(state, &rsrc);
2081 2079
2082 2080 /* Decrement the reference counts on CQs, PD and SRQ (if needed) */
2083 2081 hermon_cq_refcnt_dec(rq_cq);
2084 2082 hermon_cq_refcnt_dec(sq_cq);
2085 2083 hermon_pd_refcnt_dec(pd);
2086 2084 if (qp_srq_en == HERMON_QP_SRQ_ENABLED) {
2087 2085 hermon_srq_refcnt_dec(srq);
2088 2086 }
2089 2087
2090 2088 /* Set the qphdl pointer to NULL and return success */
2091 2089 *qphdl = NULL;
2092 2090
2093 2091 return (DDI_SUCCESS);
2094 2092
2095 2093 qpfree_fail:
2096 2094 return (status);
2097 2095 }
2098 2096
2099 2097
2100 2098 /*
2101 2099 * hermon_qp_query()
2102 2100 * Context: Can be called from interrupt or base context.
2103 2101 */
2104 2102 int
2105 2103 hermon_qp_query(hermon_state_t *state, hermon_qphdl_t qp,
2106 2104 ibt_qp_query_attr_t *attr_p)
2107 2105 {
2108 2106 ibt_cep_state_t qp_state;
2109 2107 ibt_qp_ud_attr_t *ud;
2110 2108 ibt_qp_rc_attr_t *rc;
2111 2109 ibt_qp_uc_attr_t *uc;
2112 2110 ibt_cep_flags_t enable_flags;
2113 2111 hermon_hw_addr_path_t *qpc_path, *qpc_alt_path;
2114 2112 ibt_cep_path_t *path_ptr, *alt_path_ptr;
2115 2113 hermon_hw_qpc_t *qpc;
2116 2114 int status;
2117 2115 uint_t tmp_sched_q, tmp_alt_sched_q;
2118 2116
2119 2117 mutex_enter(&qp->qp_lock);
2120 2118
2121 2119 /*
2122 2120 * Grab the temporary QPC entry from QP software state
2123 2121 */
2124 2122 qpc = &qp->qpc;
2125 2123
2126 2124 /* Convert the current Hermon QP state to IBTF QP state */
2127 2125 switch (qp->qp_state) {
2128 2126 case HERMON_QP_RESET:
2129 2127 qp_state = IBT_STATE_RESET; /* "Reset" */
2130 2128 break;
2131 2129 case HERMON_QP_INIT:
2132 2130 qp_state = IBT_STATE_INIT; /* Initialized */
2133 2131 break;
2134 2132 case HERMON_QP_RTR:
2135 2133 qp_state = IBT_STATE_RTR; /* Ready to Receive */
2136 2134 break;
2137 2135 case HERMON_QP_RTS:
2138 2136 qp_state = IBT_STATE_RTS; /* Ready to Send */
2139 2137 break;
2140 2138 case HERMON_QP_SQERR:
2141 2139 qp_state = IBT_STATE_SQE; /* Send Queue Error */
2142 2140 break;
2143 2141 case HERMON_QP_SQD:
2144 2142 if (qp->qp_sqd_still_draining) {
2145 2143 qp_state = IBT_STATE_SQDRAIN; /* SQ Draining */
2146 2144 } else {
2147 2145 qp_state = IBT_STATE_SQD; /* SQ Drained */
2148 2146 }
2149 2147 break;
2150 2148 case HERMON_QP_ERR:
2151 2149 qp_state = IBT_STATE_ERROR; /* Error */
2152 2150 break;
2153 2151 default:
2154 2152 mutex_exit(&qp->qp_lock);
2155 2153 return (ibc_get_ci_failure(0));
2156 2154 }
2157 2155 attr_p->qp_info.qp_state = qp_state;
2158 2156
2159 2157 /* SRQ Hook. */
2160 2158 attr_p->qp_srq = NULL;
2161 2159
2162 2160 /*
2163 2161 * The following QP information is always returned, regardless of
2164 2162 * the current QP state. Note: Some special handling is necessary
2165 2163 * for calculating the QP number on special QP (QP0 and QP1).
2166 2164 */
2167 2165 attr_p->qp_sq_cq =
2168 2166 (qp->qp_sq_cqhdl == NULL) ? NULL : qp->qp_sq_cqhdl->cq_hdlrarg;
2169 2167 attr_p->qp_rq_cq =
2170 2168 (qp->qp_rq_cqhdl == NULL) ? NULL : qp->qp_rq_cqhdl->cq_hdlrarg;
2171 2169 if (qp->qp_is_special) {
2172 2170 attr_p->qp_qpn = (qp->qp_is_special == HERMON_QP_SMI) ? 0 : 1;
2173 2171 } else {
2174 2172 attr_p->qp_qpn = (ib_qpn_t)qp->qp_qpnum;
2175 2173 }
2176 2174 attr_p->qp_sq_sgl = qp->qp_sq_sgl;
2177 2175 attr_p->qp_rq_sgl = qp->qp_rq_sgl;
2178 2176 attr_p->qp_info.qp_sq_sz = qp->qp_sq_bufsz - qp->qp_sq_hdrmwqes;
2179 2177 attr_p->qp_info.qp_rq_sz = qp->qp_rq_bufsz;
2180 2178
2181 2179 /*
2182 2180 * If QP is currently in the "Reset" state, then only the above are
2183 2181 * returned
2184 2182 */
2185 2183 if (qp_state == IBT_STATE_RESET) {
2186 2184 mutex_exit(&qp->qp_lock);
2187 2185 return (DDI_SUCCESS);
2188 2186 }
2189 2187
2190 2188 /*
2191 2189 * Post QUERY_QP command to firmware
2192 2190 *
2193 2191 * We do a HERMON_NOSLEEP here because we are holding the "qp_lock".
2194 2192 * Since we may be in the interrupt context (or subsequently raised
2195 2193 * to interrupt level by priority inversion), we do not want to block
2196 2194 * in this routine waiting for success.
2197 2195 */
2198 2196 tmp_sched_q = qpc->pri_addr_path.sched_q;
2199 2197 tmp_alt_sched_q = qpc->alt_addr_path.sched_q;
2200 2198 status = hermon_cmn_query_cmd_post(state, QUERY_QP, 0, qp->qp_qpnum,
2201 2199 qpc, sizeof (hermon_hw_qpc_t), HERMON_CMD_NOSLEEP_SPIN);
2202 2200 if (status != HERMON_CMD_SUCCESS) {
2203 2201 mutex_exit(&qp->qp_lock);
2204 2202 cmn_err(CE_WARN, "hermon%d: hermon_qp_query: QUERY_QP "
2205 2203 "command failed: %08x\n", state->hs_instance, status);
2206 2204 if (status == HERMON_CMD_INVALID_STATUS) {
2207 2205 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
2208 2206 }
2209 2207 return (ibc_get_ci_failure(0));
2210 2208 }
2211 2209 qpc->pri_addr_path.sched_q = tmp_sched_q;
2212 2210 qpc->alt_addr_path.sched_q = tmp_alt_sched_q;
2213 2211
2214 2212 /*
2215 2213 * Fill in the additional QP info based on the QP's transport type.
2216 2214 */
2217 2215 if (qp->qp_type == IBT_UD_RQP) {
2218 2216
2219 2217 /* Fill in the UD-specific info */
2220 2218 ud = &attr_p->qp_info.qp_transport.ud;
2221 2219 ud->ud_qkey = (ib_qkey_t)qpc->qkey;
2222 2220 ud->ud_sq_psn = qpc->next_snd_psn;
2223 2221 ud->ud_pkey_ix = qpc->pri_addr_path.pkey_indx;
2224 2222 /* port+1 for port 1/2 */
2225 2223 ud->ud_port =
2226 2224 (uint8_t)(((qpc->pri_addr_path.sched_q >> 6) & 0x01) + 1);
2227 2225
2228 2226 attr_p->qp_info.qp_trans = IBT_UD_SRV;
2229 2227
2230 2228 if (qp->qp_serv_type == HERMON_QP_FEXCH) {
2231 2229 ibt_pmr_desc_t *pmr;
2232 2230 uint64_t heart_beat;
2233 2231
2234 2232 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pmr))
2235 2233 pmr = &attr_p->qp_query_fexch.fq_uni_mem_desc;
2236 2234 pmr->pmd_iova = 0;
2237 2235 pmr->pmd_lkey = pmr->pmd_rkey =
2238 2236 hermon_fcoib_qpn_to_mkey(state, qp->qp_qpnum);
2239 2237 pmr->pmd_phys_buf_list_sz =
2240 2238 state->hs_fcoib.hfc_mtts_per_mpt;
2241 2239 pmr->pmd_sync_required = 0;
2242 2240
2243 2241 pmr = &attr_p->qp_query_fexch.fq_bi_mem_desc;
2244 2242 pmr->pmd_iova = 0;
2245 2243 pmr->pmd_lkey = 0;
2246 2244 pmr->pmd_rkey = 0;
2247 2245 pmr->pmd_phys_buf_list_sz = 0;
2248 2246 pmr->pmd_sync_required = 0;
2249 2247
2250 2248 attr_p->qp_query_fexch.fq_flags =
2251 2249 ((hermon_get_heart_beat_rq_cmd_post(state,
2252 2250 qp->qp_qpnum, &heart_beat) == HERMON_CMD_SUCCESS) &&
2253 2251 (heart_beat == 0)) ? IBT_FEXCH_HEART_BEAT_OK :
2254 2252 IBT_FEXCH_NO_FLAGS;
2255 2253
2256 2254 ud->ud_fc = qp->qp_fc_attr;
2257 2255 } else if (qp->qp_serv_type == HERMON_QP_FCMND ||
2258 2256 qp->qp_serv_type == HERMON_QP_RFCI) {
2259 2257 ud->ud_fc = qp->qp_fc_attr;
2260 2258 }
2261 2259
2262 2260 } else if (qp->qp_serv_type == HERMON_QP_RC) {
2263 2261
2264 2262 /* Fill in the RC-specific info */
2265 2263 rc = &attr_p->qp_info.qp_transport.rc;
2266 2264 rc->rc_sq_psn = qpc->next_snd_psn;
2267 2265 rc->rc_rq_psn = qpc->next_rcv_psn;
2268 2266 rc->rc_dst_qpn = qpc->rem_qpn;
2269 2267
2270 2268 /* Grab the path migration state information */
2271 2269 if (qpc->pm_state == HERMON_QP_PMSTATE_MIGRATED) {
2272 2270 rc->rc_mig_state = IBT_STATE_MIGRATED;
2273 2271 } else if (qpc->pm_state == HERMON_QP_PMSTATE_REARM) {
2274 2272 rc->rc_mig_state = IBT_STATE_REARMED;
2275 2273 } else {
2276 2274 rc->rc_mig_state = IBT_STATE_ARMED;
2277 2275 }
2278 2276 rc->rc_rdma_ra_out = (1 << qpc->sra_max);
2279 2277 rc->rc_rdma_ra_in = (1 << qpc->rra_max);
2280 2278 rc->rc_min_rnr_nak = qpc->min_rnr_nak;
2281 2279 rc->rc_path_mtu = qpc->mtu;
2282 2280 rc->rc_retry_cnt = qpc->retry_cnt;
2283 2281
2284 2282 /* Get the common primary address path fields */
2285 2283 qpc_path = &qpc->pri_addr_path;
2286 2284 path_ptr = &rc->rc_path;
2287 2285 hermon_get_addr_path(state, qpc_path, &path_ptr->cep_adds_vect,
2288 2286 HERMON_ADDRPATH_QP);
2289 2287
2290 2288 /* Fill in the additional primary address path fields */
2291 2289 path_ptr->cep_pkey_ix = qpc_path->pkey_indx;
2292 2290 path_ptr->cep_hca_port_num =
2293 2291 path_ptr->cep_adds_vect.av_port_num =
2294 2292 (uint8_t)(((qpc_path->sched_q >> 6) & 0x01) + 1);
2295 2293 path_ptr->cep_timeout = qpc_path->ack_timeout;
2296 2294
2297 2295 /* Get the common alternate address path fields */
2298 2296 qpc_alt_path = &qpc->alt_addr_path;
2299 2297 alt_path_ptr = &rc->rc_alt_path;
2300 2298 hermon_get_addr_path(state, qpc_alt_path,
2301 2299 &alt_path_ptr->cep_adds_vect, HERMON_ADDRPATH_QP);
2302 2300
2303 2301 /* Fill in the additional alternate address path fields */
2304 2302 alt_path_ptr->cep_pkey_ix = qpc_alt_path->pkey_indx;
2305 2303 alt_path_ptr->cep_hca_port_num =
2306 2304 alt_path_ptr->cep_adds_vect.av_port_num =
2307 2305 (uint8_t)(((qpc_alt_path->sched_q >> 6) & 0x01) + 1);
2308 2306 alt_path_ptr->cep_timeout = qpc_alt_path->ack_timeout;
2309 2307
2310 2308 /* Get the RNR retry time from primary path */
2311 2309 rc->rc_rnr_retry_cnt = qpc->rnr_retry;
2312 2310
2313 2311 /* Set the enable flags based on RDMA/Atomic enable bits */
2314 2312 enable_flags = IBT_CEP_NO_FLAGS;
2315 2313 enable_flags |= ((qpc->rre == 0) ? 0 : IBT_CEP_RDMA_RD);
2316 2314 enable_flags |= ((qpc->rwe == 0) ? 0 : IBT_CEP_RDMA_WR);
2317 2315 enable_flags |= ((qpc->rae == 0) ? 0 : IBT_CEP_ATOMIC);
2318 2316 attr_p->qp_info.qp_flags = enable_flags;
2319 2317
2320 2318 attr_p->qp_info.qp_trans = IBT_RC_SRV;
2321 2319
2322 2320 } else if (qp->qp_serv_type == HERMON_QP_UC) {
2323 2321
2324 2322 /* Fill in the UC-specific info */
2325 2323 uc = &attr_p->qp_info.qp_transport.uc;
2326 2324 uc->uc_sq_psn = qpc->next_snd_psn;
2327 2325 uc->uc_rq_psn = qpc->next_rcv_psn;
2328 2326 uc->uc_dst_qpn = qpc->rem_qpn;
2329 2327
2330 2328 /* Grab the path migration state information */
2331 2329 if (qpc->pm_state == HERMON_QP_PMSTATE_MIGRATED) {
2332 2330 uc->uc_mig_state = IBT_STATE_MIGRATED;
2333 2331 } else if (qpc->pm_state == HERMON_QP_PMSTATE_REARM) {
2334 2332 uc->uc_mig_state = IBT_STATE_REARMED;
2335 2333 } else {
2336 2334 uc->uc_mig_state = IBT_STATE_ARMED;
2337 2335 }
2338 2336 uc->uc_path_mtu = qpc->mtu;
2339 2337
2340 2338 /* Get the common primary address path fields */
2341 2339 qpc_path = &qpc->pri_addr_path;
2342 2340 path_ptr = &uc->uc_path;
2343 2341 hermon_get_addr_path(state, qpc_path, &path_ptr->cep_adds_vect,
2344 2342 HERMON_ADDRPATH_QP);
2345 2343
2346 2344 /* Fill in the additional primary address path fields */
2347 2345 path_ptr->cep_pkey_ix = qpc_path->pkey_indx;
2348 2346 path_ptr->cep_hca_port_num =
2349 2347 path_ptr->cep_adds_vect.av_port_num =
2350 2348 (uint8_t)(((qpc_path->sched_q >> 6) & 0x01) + 1);
2351 2349
2352 2350 /* Get the common alternate address path fields */
2353 2351 qpc_alt_path = &qpc->alt_addr_path;
2354 2352 alt_path_ptr = &uc->uc_alt_path;
2355 2353 hermon_get_addr_path(state, qpc_alt_path,
2356 2354 &alt_path_ptr->cep_adds_vect, HERMON_ADDRPATH_QP);
2357 2355
2358 2356 /* Fill in the additional alternate address path fields */
2359 2357 alt_path_ptr->cep_pkey_ix = qpc_alt_path->pkey_indx;
2360 2358 alt_path_ptr->cep_hca_port_num =
2361 2359 alt_path_ptr->cep_adds_vect.av_port_num =
2362 2360 (uint8_t)(((qpc_alt_path->sched_q >> 6) & 0x01) + 1);
2363 2361
2364 2362 /*
2365 2363 * Set the enable flags based on RDMA enable bits (by
2366 2364 * definition UC doesn't support Atomic or RDMA Read)
2367 2365 */
2368 2366 enable_flags = ((qpc->rwe == 0) ? 0 : IBT_CEP_RDMA_WR);
2369 2367 attr_p->qp_info.qp_flags = enable_flags;
2370 2368
2371 2369 attr_p->qp_info.qp_trans = IBT_UC_SRV;
2372 2370
2373 2371 } else {
2374 2372 HERMON_WARNING(state, "unexpected QP transport type");
2375 2373 mutex_exit(&qp->qp_lock);
2376 2374 return (ibc_get_ci_failure(0));
2377 2375 }
2378 2376
2379 2377 /*
2380 2378 * Under certain circumstances it is possible for the Hermon hardware
2381 2379 * to transition to one of the error states without software directly
2382 2380 * knowing about it. The QueryQP() call is the one place where we
2383 2381 * have an opportunity to sample and update our view of the QP state.
2384 2382 */
2385 2383 if (qpc->state == HERMON_QP_SQERR) {
2386 2384 attr_p->qp_info.qp_state = IBT_STATE_SQE;
2387 2385 qp->qp_state = HERMON_QP_SQERR;
2388 2386 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_SQERR);
2389 2387 }
2390 2388 if (qpc->state == HERMON_QP_ERR) {
2391 2389 attr_p->qp_info.qp_state = IBT_STATE_ERROR;
2392 2390 qp->qp_state = HERMON_QP_ERR;
2393 2391 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_ERR);
2394 2392 }
2395 2393 mutex_exit(&qp->qp_lock);
2396 2394
2397 2395 return (DDI_SUCCESS);
2398 2396 }
2399 2397
2400 2398
2401 2399 /*
2402 2400 * hermon_qp_create_qpn()
2403 2401 * Context: Can be called from interrupt or base context.
2404 2402 */
2405 2403 static int
2406 2404 hermon_qp_create_qpn(hermon_state_t *state, hermon_qphdl_t qp,
2407 2405 hermon_rsrc_t *qpc)
2408 2406 {
2409 2407 hermon_qpn_entry_t query;
2410 2408 hermon_qpn_entry_t *entry;
2411 2409 avl_index_t where;
2412 2410
2413 2411 /*
2414 2412 * Build a query (for the AVL tree lookup) and attempt to find
2415 2413 * a previously added entry that has a matching QPC index. If
2416 2414 * no matching entry is found, then allocate, initialize, and
2417 2415 * add an entry to the AVL tree.
2418 2416 * If a matching entry is found, then increment its QPN counter
2419 2417 * and reference counter.
2420 2418 */
2421 2419 query.qpn_indx = qpc->hr_indx;
2422 2420 mutex_enter(&state->hs_qpn_avl_lock);
2423 2421 entry = (hermon_qpn_entry_t *)avl_find(&state->hs_qpn_avl,
2424 2422 &query, &where);
2425 2423 if (entry == NULL) {
2426 2424 /*
2427 2425 * Allocate and initialize a QPN entry, then insert
2428 2426 * it into the AVL tree.
2429 2427 */
2430 2428 entry = (hermon_qpn_entry_t *)kmem_zalloc(
2431 2429 sizeof (hermon_qpn_entry_t), KM_NOSLEEP);
2432 2430 if (entry == NULL) {
2433 2431 mutex_exit(&state->hs_qpn_avl_lock);
2434 2432 return (DDI_FAILURE);
2435 2433 }
2436 2434 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*entry))
2437 2435
2438 2436 entry->qpn_indx = qpc->hr_indx;
2439 2437 entry->qpn_refcnt = 0;
2440 2438 entry->qpn_counter = 0;
2441 2439
2442 2440 avl_insert(&state->hs_qpn_avl, entry, where);
2443 2441 }
2444 2442
2445 2443 /*
2446 2444 * Make the AVL tree entry point to the QP context resource that
2447 2445 * it will be responsible for tracking
2448 2446 */
2449 2447 entry->qpn_qpc = qpc;
2450 2448
2451 2449 /*
2452 2450 * Setup the QP handle to point to the AVL tree entry. Then
2453 2451 * generate the new QP number from the entry's QPN counter value
2454 2452 * and the hardware's QP context table index.
2455 2453 */
2456 2454 qp->qp_qpn_hdl = entry;
2457 2455 qp->qp_qpnum = ((entry->qpn_counter <<
2458 2456 state->hs_cfg_profile->cp_log_num_qp) | qpc->hr_indx) &
2459 2457 HERMON_QP_MAXNUMBER_MSK;
2460 2458 qp->qp_ring = qp->qp_qpnum << 8;
2461 2459
2462 2460 /*
2463 2461 * Increment the reference counter and QPN counter. The QPN
2464 2462 * counter always indicates the next available number for use.
2465 2463 */
2466 2464 entry->qpn_counter++;
2467 2465 entry->qpn_refcnt++;
2468 2466
2469 2467 mutex_exit(&state->hs_qpn_avl_lock);
2470 2468
2471 2469 return (DDI_SUCCESS);
2472 2470 }
2473 2471
2474 2472
2475 2473 /*
2476 2474 * hermon_qp_release_qpn()
2477 2475 * Context: Can be called only from user or kernel context.
2478 2476 */
2479 2477 void
2480 2478 hermon_qp_release_qpn(hermon_state_t *state, hermon_qpn_entry_t *entry,
2481 2479 int flags)
2482 2480 {
2483 2481 ASSERT(entry != NULL);
2484 2482
2485 2483 mutex_enter(&state->hs_qpn_avl_lock);
2486 2484
2487 2485 /*
2488 2486 * If we are releasing the QP number here, then we decrement the
2489 2487 * reference count and check for zero references. If there are
2490 2488 * zero references, then we free the QPC context (if it hadn't
2491 2489 * already been freed during a HERMON_QPN_FREE_ONLY free, i.e. for
2492 2490 * reuse with another similar QP number) and remove the tracking
2493 2491 * structure from the QP number AVL tree and free the structure.
2494 2492 * If we are not releasing the QP number here, then, as long as we
2495 2493 * have not exhausted the usefulness of the QPC context (that is,
2496 2494 * re-used it too many times without the reference count having
2497 2495 * gone to zero), we free up the QPC context for use by another
2498 2496 * thread (which will use it to construct a different QP number
2499 2497 * from the same QPC table index).
2500 2498 */
2501 2499 if (flags == HERMON_QPN_RELEASE) {
2502 2500 entry->qpn_refcnt--;
2503 2501
2504 2502 /*
2505 2503 * If the reference count is zero, then we free the QPC
2506 2504 * context (if it hadn't already been freed in an early
2507 2505 * step, e.g. HERMON_QPN_FREE_ONLY) and remove/free the
2508 2506 * tracking structure from the QP number AVL tree.
2509 2507 */
2510 2508 if (entry->qpn_refcnt == 0) {
2511 2509 if (entry->qpn_qpc != NULL) {
2512 2510 hermon_rsrc_free(state, &entry->qpn_qpc);
2513 2511 }
2514 2512
2515 2513 /*
2516 2514 * If the current entry has served it's useful
2517 2515 * purpose (i.e. been reused the maximum allowable
2518 2516 * number of times), then remove it from QP number
2519 2517 * AVL tree and free it up.
2520 2518 */
2521 2519 if (entry->qpn_counter >= (1 <<
2522 2520 (24 - state->hs_cfg_profile->cp_log_num_qp))) {
2523 2521 avl_remove(&state->hs_qpn_avl, entry);
2524 2522 kmem_free(entry, sizeof (hermon_qpn_entry_t));
2525 2523 }
2526 2524 }
2527 2525
2528 2526 } else if (flags == HERMON_QPN_FREE_ONLY) {
2529 2527 /*
2530 2528 * Even if we are not freeing the QP number, that will not
2531 2529 * always prevent us from releasing the QPC context. In fact,
2532 2530 * since the QPC context only forms part of the whole QPN,
2533 2531 * we want to free it up for use by other consumers. But
2534 2532 * if the reference count is non-zero (which it will always
2535 2533 * be when we are doing HERMON_QPN_FREE_ONLY) and the counter
2536 2534 * has reached its maximum value, then we cannot reuse the
2537 2535 * QPC context until the reference count eventually reaches
2538 2536 * zero (in HERMON_QPN_RELEASE, above).
2539 2537 */
2540 2538 if (entry->qpn_counter < (1 <<
2541 2539 (24 - state->hs_cfg_profile->cp_log_num_qp))) {
2542 2540 hermon_rsrc_free(state, &entry->qpn_qpc);
2543 2541 }
2544 2542 }
2545 2543 mutex_exit(&state->hs_qpn_avl_lock);
2546 2544 }
2547 2545
2548 2546
2549 2547 /*
2550 2548 * hermon_qpn_avl_compare()
2551 2549 * Context: Can be called from user or kernel context.
2552 2550 */
2553 2551 static int
2554 2552 hermon_qpn_avl_compare(const void *q, const void *e)
2555 2553 {
2556 2554 hermon_qpn_entry_t *entry, *query;
2557 2555
2558 2556 entry = (hermon_qpn_entry_t *)e;
2559 2557 query = (hermon_qpn_entry_t *)q;
2560 2558
2561 2559 if (query->qpn_indx < entry->qpn_indx) {
2562 2560 return (-1);
2563 2561 } else if (query->qpn_indx > entry->qpn_indx) {
2564 2562 return (+1);
2565 2563 } else {
2566 2564 return (0);
2567 2565 }
2568 2566 }
2569 2567
2570 2568
2571 2569 /*
2572 2570 * hermon_qpn_avl_init()
2573 2571 * Context: Only called from attach() path context
2574 2572 */
2575 2573 void
2576 2574 hermon_qpn_avl_init(hermon_state_t *state)
2577 2575 {
2578 2576 /* Initialize the lock used for QP number (QPN) AVL tree access */
2579 2577 mutex_init(&state->hs_qpn_avl_lock, NULL, MUTEX_DRIVER,
2580 2578 DDI_INTR_PRI(state->hs_intrmsi_pri));
2581 2579
2582 2580 /* Initialize the AVL tree for the QP number (QPN) storage */
2583 2581 avl_create(&state->hs_qpn_avl, hermon_qpn_avl_compare,
2584 2582 sizeof (hermon_qpn_entry_t),
2585 2583 offsetof(hermon_qpn_entry_t, qpn_avlnode));
2586 2584 }
2587 2585
2588 2586
2589 2587 /*
2590 2588 * hermon_qpn_avl_fini()
2591 2589 * Context: Only called from attach() and/or detach() path contexts
2592 2590 */
2593 2591 void
2594 2592 hermon_qpn_avl_fini(hermon_state_t *state)
2595 2593 {
2596 2594 hermon_qpn_entry_t *entry;
2597 2595 void *cookie;
2598 2596
2599 2597 /*
2600 2598 * Empty all entries (if necessary) and destroy the AVL tree
2601 2599 * that was used for QP number (QPN) tracking.
2602 2600 */
2603 2601 cookie = NULL;
2604 2602 while ((entry = (hermon_qpn_entry_t *)avl_destroy_nodes(
2605 2603 &state->hs_qpn_avl, &cookie)) != NULL) {
2606 2604 kmem_free(entry, sizeof (hermon_qpn_entry_t));
2607 2605 }
2608 2606 avl_destroy(&state->hs_qpn_avl);
2609 2607
2610 2608 /* Destroy the lock used for QP number (QPN) AVL tree access */
2611 2609 mutex_destroy(&state->hs_qpn_avl_lock);
2612 2610 }
2613 2611
2614 2612
2615 2613 /*
2616 2614 * hermon_qphdl_from_qpnum()
2617 2615 * Context: Can be called from interrupt or base context.
2618 2616 *
2619 2617 * This routine is important because changing the unconstrained
2620 2618 * portion of the QP number is critical to the detection of a
2621 2619 * potential race condition in the QP event handler code (i.e. the case
2622 2620 * where a QP is freed and alloc'd again before an event for the
2623 2621 * "old" QP can be handled).
2624 2622 *
2625 2623 * While this is not a perfect solution (not sure that one exists)
2626 2624 * it does help to mitigate the chance that this race condition will
2627 2625 * cause us to deliver a "stale" event to the new QP owner. Note:
2628 2626 * this solution does not scale well because the number of constrained
2629 2627 * bits increases (and, hence, the number of unconstrained bits
2630 2628 * decreases) as the number of supported QPs grows. For small and
2631 2629 * intermediate values, it should hopefully provide sufficient
2632 2630 * protection.
2633 2631 */
2634 2632 hermon_qphdl_t
2635 2633 hermon_qphdl_from_qpnum(hermon_state_t *state, uint_t qpnum)
2636 2634 {
2637 2635 uint_t qpindx, qpmask;
2638 2636
2639 2637 /* Calculate the QP table index from the qpnum */
2640 2638 qpmask = (1 << state->hs_cfg_profile->cp_log_num_qp) - 1;
2641 2639 qpindx = qpnum & qpmask;
2642 2640 return (hermon_icm_num_to_hdl(state, HERMON_QPC, qpindx));
2643 2641 }
2644 2642
2645 2643
2646 2644 /*
2647 2645 * hermon_special_qp_rsrc_alloc
2648 2646 * Context: Can be called from interrupt or base context.
2649 2647 */
2650 2648 static int
2651 2649 hermon_special_qp_rsrc_alloc(hermon_state_t *state, ibt_sqp_type_t type,
2652 2650 uint_t port, hermon_rsrc_t **qp_rsrc)
2653 2651 {
2654 2652 uint_t mask, flags;
2655 2653 int status;
2656 2654
2657 2655 mutex_enter(&state->hs_spec_qplock);
2658 2656 flags = state->hs_spec_qpflags;
2659 2657 if (type == IBT_SMI_SQP) {
2660 2658 /*
2661 2659 * Check here to see if the driver has been configured
2662 2660 * to instruct the Hermon firmware to handle all incoming
2663 2661 * SMP messages (i.e. messages sent to SMA). If so,
2664 2662 * then we will treat QP0 as if it has already been
2665 2663 * allocated (for internal use). Otherwise, if we allow
2666 2664 * the allocation to happen, it will cause unexpected
2667 2665 * behaviors (e.g. Hermon SMA becomes unresponsive).
2668 2666 */
2669 2667 if (state->hs_cfg_profile->cp_qp0_agents_in_fw != 0) {
2670 2668 mutex_exit(&state->hs_spec_qplock);
2671 2669 return (IBT_QP_IN_USE);
2672 2670 }
2673 2671
2674 2672 /*
2675 2673 * If this is the first QP0 allocation, then post
2676 2674 * a CONF_SPECIAL_QP firmware command
2677 2675 */
2678 2676 if ((flags & HERMON_SPECIAL_QP0_RSRC_MASK) == 0) {
2679 2677 status = hermon_conf_special_qp_cmd_post(state,
2680 2678 state->hs_spec_qp0->hr_indx, HERMON_CMD_QP_SMI,
2681 2679 HERMON_CMD_NOSLEEP_SPIN,
2682 2680 HERMON_CMD_SPEC_QP_OPMOD(
2683 2681 state->hs_cfg_profile->cp_qp0_agents_in_fw,
2684 2682 state->hs_cfg_profile->cp_qp1_agents_in_fw));
2685 2683 if (status != HERMON_CMD_SUCCESS) {
2686 2684 mutex_exit(&state->hs_spec_qplock);
2687 2685 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2688 2686 "command failed: %08x\n",
2689 2687 state->hs_instance, status);
2690 2688 return (IBT_INSUFF_RESOURCE);
2691 2689 }
2692 2690 }
2693 2691
2694 2692 /*
2695 2693 * Now check (and, if necessary, modify) the flags to indicate
2696 2694 * whether the allocation was successful
2697 2695 */
2698 2696 mask = (1 << (HERMON_SPECIAL_QP0_RSRC + port));
2699 2697 if (flags & mask) {
2700 2698 mutex_exit(&state->hs_spec_qplock);
2701 2699 return (IBT_QP_IN_USE);
2702 2700 }
2703 2701 state->hs_spec_qpflags |= mask;
2704 2702 *qp_rsrc = state->hs_spec_qp0;
2705 2703
2706 2704 } else {
2707 2705 /*
2708 2706 * If this is the first QP1 allocation, then post
2709 2707 * a CONF_SPECIAL_QP firmware command
2710 2708 */
2711 2709 if ((flags & HERMON_SPECIAL_QP1_RSRC_MASK) == 0) {
2712 2710 status = hermon_conf_special_qp_cmd_post(state,
2713 2711 state->hs_spec_qp1->hr_indx, HERMON_CMD_QP_GSI,
2714 2712 HERMON_CMD_NOSLEEP_SPIN,
2715 2713 HERMON_CMD_SPEC_QP_OPMOD(
2716 2714 state->hs_cfg_profile->cp_qp0_agents_in_fw,
2717 2715 state->hs_cfg_profile->cp_qp1_agents_in_fw));
2718 2716 if (status != HERMON_CMD_SUCCESS) {
2719 2717 mutex_exit(&state->hs_spec_qplock);
2720 2718 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2721 2719 "command failed: %08x\n",
2722 2720 state->hs_instance, status);
2723 2721 return (IBT_INSUFF_RESOURCE);
2724 2722 }
2725 2723 }
2726 2724
2727 2725 /*
2728 2726 * Now check (and, if necessary, modify) the flags to indicate
2729 2727 * whether the allocation was successful
2730 2728 */
2731 2729 mask = (1 << (HERMON_SPECIAL_QP1_RSRC + port));
2732 2730 if (flags & mask) {
2733 2731 mutex_exit(&state->hs_spec_qplock);
2734 2732 return (IBT_QP_IN_USE);
2735 2733 }
2736 2734 state->hs_spec_qpflags |= mask;
2737 2735 *qp_rsrc = state->hs_spec_qp1;
2738 2736 }
2739 2737
2740 2738 mutex_exit(&state->hs_spec_qplock);
2741 2739 return (DDI_SUCCESS);
2742 2740 }
2743 2741
2744 2742
2745 2743 /*
2746 2744 * hermon_special_qp_rsrc_free
2747 2745 * Context: Can be called from interrupt or base context.
2748 2746 */
2749 2747 static int
2750 2748 hermon_special_qp_rsrc_free(hermon_state_t *state, ibt_sqp_type_t type,
2751 2749 uint_t port)
2752 2750 {
2753 2751 uint_t mask, flags;
2754 2752 int status;
2755 2753
2756 2754 mutex_enter(&state->hs_spec_qplock);
2757 2755 if (type == IBT_SMI_SQP) {
2758 2756 mask = (1 << (HERMON_SPECIAL_QP0_RSRC + port));
2759 2757 state->hs_spec_qpflags &= ~mask;
2760 2758 flags = state->hs_spec_qpflags;
2761 2759
2762 2760 /*
2763 2761 * If this is the last QP0 free, then post a CONF_SPECIAL_QP
2764 2762 * NOW, If this is the last Special QP free, then post a
2765 2763 * CONF_SPECIAL_QP firmware command - it'll stop them all
2766 2764 */
2767 2765 if (flags) {
2768 2766 status = hermon_conf_special_qp_cmd_post(state, 0,
2769 2767 HERMON_CMD_QP_SMI, HERMON_CMD_NOSLEEP_SPIN, 0);
2770 2768 if (status != HERMON_CMD_SUCCESS) {
2771 2769 mutex_exit(&state->hs_spec_qplock);
2772 2770 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2773 2771 "command failed: %08x\n",
2774 2772 state->hs_instance, status);
2775 2773 if (status == HERMON_CMD_INVALID_STATUS) {
2776 2774 hermon_fm_ereport(state, HCA_SYS_ERR,
2777 2775 HCA_ERR_SRV_LOST);
2778 2776 }
2779 2777 return (ibc_get_ci_failure(0));
2780 2778 }
2781 2779 }
2782 2780 } else {
2783 2781 mask = (1 << (HERMON_SPECIAL_QP1_RSRC + port));
2784 2782 state->hs_spec_qpflags &= ~mask;
2785 2783 flags = state->hs_spec_qpflags;
2786 2784
2787 2785 /*
2788 2786 * If this is the last QP1 free, then post a CONF_SPECIAL_QP
2789 2787 * NOW, if this is the last special QP free, then post a
2790 2788 * CONF_SPECIAL_QP firmware command - it'll stop them all
2791 2789 */
2792 2790 if (flags) {
2793 2791 status = hermon_conf_special_qp_cmd_post(state, 0,
2794 2792 HERMON_CMD_QP_GSI, HERMON_CMD_NOSLEEP_SPIN, 0);
2795 2793 if (status != HERMON_CMD_SUCCESS) {
2796 2794 mutex_exit(&state->hs_spec_qplock);
2797 2795 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2798 2796 "command failed: %08x\n",
2799 2797 state->hs_instance, status);
2800 2798 if (status == HERMON_CMD_INVALID_STATUS) {
2801 2799 hermon_fm_ereport(state, HCA_SYS_ERR,
2802 2800 HCA_ERR_SRV_LOST);
2803 2801 }
2804 2802 return (ibc_get_ci_failure(0));
2805 2803 }
2806 2804 }
2807 2805 }
2808 2806
2809 2807 mutex_exit(&state->hs_spec_qplock);
2810 2808 return (DDI_SUCCESS);
2811 2809 }
2812 2810
2813 2811
2814 2812 /*
2815 2813 * hermon_qp_sgl_to_logwqesz()
2816 2814 * Context: Can be called from interrupt or base context.
2817 2815 */
2818 2816 static void
2819 2817 hermon_qp_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl,
2820 2818 uint_t real_max_sgl, hermon_qp_wq_type_t wq_type,
2821 2819 uint_t *logwqesz, uint_t *max_sgl)
2822 2820 {
2823 2821 uint_t max_size, log2, actual_sgl;
↓ open down ↓ |
1283 lines elided |
↑ open up ↑ |
2824 2822
2825 2823 switch (wq_type) {
2826 2824 case HERMON_QP_WQ_TYPE_SENDQ_UD:
2827 2825 /*
2828 2826 * Use requested maximum SGL to calculate max descriptor size
2829 2827 * (while guaranteeing that the descriptor size is a
2830 2828 * power-of-2 cachelines).
2831 2829 */
2832 2830 max_size = (HERMON_QP_WQE_MLX_SND_HDRS + (num_sgl << 4));
2833 2831 log2 = highbit(max_size);
2834 - if ((max_size & (max_size - 1)) == 0) {
2832 + if (ISP2(max_size)) {
2835 2833 log2 = log2 - 1;
2836 2834 }
2837 2835
2838 2836 /* Make sure descriptor is at least the minimum size */
2839 2837 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2840 2838
2841 2839 /* Calculate actual number of SGL (given WQE size) */
2842 2840 actual_sgl = ((1 << log2) -
2843 2841 sizeof (hermon_hw_snd_wqe_ctrl_t)) >> 4;
2844 2842 break;
2845 2843
2846 2844 case HERMON_QP_WQ_TYPE_SENDQ_CONN:
2847 2845 /*
2848 2846 * Use requested maximum SGL to calculate max descriptor size
2849 2847 * (while guaranteeing that the descriptor size is a
2850 2848 * power-of-2 cachelines).
2851 2849 */
2852 2850 max_size = (HERMON_QP_WQE_MLX_SND_HDRS + (num_sgl << 4));
2853 2851 log2 = highbit(max_size);
2854 - if ((max_size & (max_size - 1)) == 0) {
2852 + if (ISP2(max_size)) {
2855 2853 log2 = log2 - 1;
2856 2854 }
2857 2855
2858 2856 /* Make sure descriptor is at least the minimum size */
2859 2857 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2860 2858
2861 2859 /* Calculate actual number of SGL (given WQE size) */
2862 2860 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_SND_HDRS) >> 4;
2863 2861 break;
2864 2862
2865 2863 case HERMON_QP_WQ_TYPE_RECVQ:
2866 2864 /*
2867 2865 * Same as above (except for Recv WQEs)
2868 2866 */
2869 2867 max_size = (HERMON_QP_WQE_MLX_RCV_HDRS + (num_sgl << 4));
2870 2868 log2 = highbit(max_size);
2871 - if ((max_size & (max_size - 1)) == 0) {
2869 + if (ISP2(max_size)) {
2872 2870 log2 = log2 - 1;
2873 2871 }
2874 2872
2875 2873 /* Make sure descriptor is at least the minimum size */
2876 2874 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2877 2875
2878 2876 /* Calculate actual number of SGL (given WQE size) */
2879 2877 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_RCV_HDRS) >> 4;
2880 2878 break;
2881 2879
2882 2880 case HERMON_QP_WQ_TYPE_SENDMLX_QP0:
2883 2881 /*
2884 2882 * Same as above (except for MLX transport WQEs). For these
2885 2883 * WQEs we have to account for the space consumed by the
2886 2884 * "inline" packet headers. (This is smaller than for QP1
2887 2885 * below because QP0 is not allowed to send packets with a GRH.
2888 2886 */
2889 2887 max_size = (HERMON_QP_WQE_MLX_QP0_HDRS + (num_sgl << 4));
2890 2888 log2 = highbit(max_size);
2891 - if ((max_size & (max_size - 1)) == 0) {
2889 + if (ISP2(max_size)) {
2892 2890 log2 = log2 - 1;
2893 2891 }
2894 2892
2895 2893 /* Make sure descriptor is at least the minimum size */
2896 2894 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2897 2895
2898 2896 /* Calculate actual number of SGL (given WQE size) */
2899 2897 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_QP0_HDRS) >> 4;
2900 2898 break;
2901 2899
2902 2900 case HERMON_QP_WQ_TYPE_SENDMLX_QP1:
↓ open down ↓ |
1 lines elided |
↑ open up ↑ |
2903 2901 /*
2904 2902 * Same as above. For these WQEs we again have to account for
2905 2903 * the space consumed by the "inline" packet headers. (This
2906 2904 * is larger than for QP0 above because we have to account for
2907 2905 * the possibility of a GRH in each packet - and this
2908 2906 * introduces an alignment issue that causes us to consume
2909 2907 * an additional 8 bytes).
2910 2908 */
2911 2909 max_size = (HERMON_QP_WQE_MLX_QP1_HDRS + (num_sgl << 4));
2912 2910 log2 = highbit(max_size);
2913 - if ((max_size & (max_size - 1)) == 0) {
2911 + if (ISP2(max_size)) {
2914 2912 log2 = log2 - 1;
2915 2913 }
2916 2914
2917 2915 /* Make sure descriptor is at least the minimum size */
2918 2916 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2919 2917
2920 2918 /* Calculate actual number of SGL (given WQE size) */
2921 2919 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_QP1_HDRS) >> 4;
2922 2920 break;
2923 2921
2924 2922 default:
2925 2923 HERMON_WARNING(state, "unexpected work queue type");
2926 2924 break;
2927 2925 }
2928 2926
2929 2927 /* Fill in the return values */
2930 2928 *logwqesz = log2;
2931 2929 *max_sgl = min(real_max_sgl, actual_sgl);
2932 2930 }
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX