Print this page
XXXX introduce drv_sectohz
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/common/io/ib/clients/rds/rdsib_buf.c
+++ new/usr/src/uts/common/io/ib/clients/rds/rdsib_buf.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25 /*
26 26 * Copyright (c) 2005 SilverStorm Technologies, Inc. All rights reserved.
27 27 *
28 28 * This software is available to you under a choice of one of two
29 29 * licenses. You may choose to be licensed under the terms of the GNU
30 30 * General Public License (GPL) Version 2, available from the file
31 31 * COPYING in the main directory of this source tree, or the
32 32 * OpenIB.org BSD license below:
33 33 *
34 34 * Redistribution and use in source and binary forms, with or
35 35 * without modification, are permitted provided that the following
36 36 * conditions are met:
37 37 *
38 38 * - Redistributions of source code must retain the above
39 39 * copyright notice, this list of conditions and the following
40 40 * disclaimer.
41 41 *
42 42 * - Redistributions in binary form must reproduce the above
43 43 * copyright notice, this list of conditions and the following
44 44 * disclaimer in the documentation and/or other materials
45 45 * provided with the distribution.
46 46 *
47 47 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
48 48 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
49 49 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
50 50 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
51 51 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
52 52 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
53 53 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
54 54 * SOFTWARE.
55 55 *
56 56 */
57 57 /*
58 58 * Sun elects to include this software in Sun product
59 59 * under the OpenIB BSD license.
60 60 *
61 61 *
62 62 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
63 63 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
64 64 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
65 65 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
66 66 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
67 67 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
68 68 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
69 69 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
70 70 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
71 71 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
72 72 * POSSIBILITY OF SUCH DAMAGE.
73 73 */
74 74
75 75 #include <sys/ib/clients/rds/rdsib_cm.h>
76 76 #include <sys/ib/clients/rds/rdsib_ib.h>
77 77 #include <sys/ib/clients/rds/rdsib_buf.h>
78 78 #include <sys/ib/clients/rds/rdsib_ep.h>
79 79 #include <sys/ib/clients/rds/rds_kstat.h>
80 80
81 81 /*
82 82 * This File contains the buffer management code
83 83 */
84 84
85 85 #define DUMP_USER_PARAMS() \
86 86 RDS_DPRINTF3(LABEL, "MaxNodes = %d", MaxNodes); \
87 87 RDS_DPRINTF3(LABEL, "UserBufferSize = %d", UserBufferSize); \
88 88 RDS_DPRINTF3(LABEL, "RdsPktSize = %d", RdsPktSize); \
89 89 RDS_DPRINTF3(LABEL, "MaxDataSendBuffers = %d", MaxDataSendBuffers); \
90 90 RDS_DPRINTF3(LABEL, "MaxDataRecvBuffers = %d", MaxDataRecvBuffers); \
91 91 RDS_DPRINTF3(LABEL, "MaxCtrlSendBuffers = %d", MaxCtrlSendBuffers); \
92 92 RDS_DPRINTF3(LABEL, "MaxCtrlRecvBuffers = %d", MaxCtrlRecvBuffers); \
93 93 RDS_DPRINTF3(LABEL, "DataRecvBufferLWM = %d", DataRecvBufferLWM); \
94 94 RDS_DPRINTF3(LABEL, "PendingRxPktsHWM = %d", PendingRxPktsHWM); \
95 95 RDS_DPRINTF3(LABEL, "MinRnrRetry = %d", MinRnrRetry)
96 96
97 97 uint_t rds_nbuffers_to_putback;
98 98
99 99 static void
100 100 rds_free_mblk(char *arg)
101 101 {
102 102 rds_buf_t *bp = (rds_buf_t *)(uintptr_t)arg;
103 103
104 104 /* Free the recv buffer */
105 105 RDS_DPRINTF4("rds_free_mblk", "Enter: BP(%p)", bp);
106 106 ASSERT(bp->buf_state == RDS_RCVBUF_ONSOCKQ);
107 107 rds_free_recv_buf(bp, 1);
108 108 RDS_DECR_RXPKTS_PEND(1);
109 109 RDS_DPRINTF4("rds_free_mblk", "Return: BP(%p)", bp);
110 110 }
111 111
112 112 void
113 113 rds_free_recv_caches(rds_state_t *statep)
114 114 {
115 115 rds_hca_t *hcap;
116 116 int ret;
117 117
118 118 RDS_DPRINTF4("rds_free_recv_caches", "Enter");
119 119
120 120 mutex_enter(&rds_dpool.pool_lock);
121 121 if (rds_dpool.pool_memp == NULL) {
122 122 RDS_DPRINTF2("rds_free_recv_caches", "Caches are empty");
123 123 mutex_exit(&rds_dpool.pool_lock);
124 124 return;
125 125 }
↓ open down ↓ |
125 lines elided |
↑ open up ↑ |
126 126
127 127 /*
128 128 * All buffers must have been freed as all sessions are closed
129 129 * and destroyed
130 130 */
131 131 ASSERT(rds_dpool.pool_nbusy == 0);
132 132 RDS_DPRINTF2("rds_free_recv_caches", "Data Pool has "
133 133 "pending buffers: %d", rds_dpool.pool_nbusy);
134 134 while (rds_dpool.pool_nbusy != 0) {
135 135 mutex_exit(&rds_dpool.pool_lock);
136 - delay(drv_usectohz(1000000));
136 + delay(drv_sectohz(1));
137 137 mutex_enter(&rds_dpool.pool_lock);
138 138 }
139 139
140 140 hcap = statep->rds_hcalistp;
141 141 while (hcap != NULL) {
142 142 if (hcap->hca_mrhdl != NULL) {
143 143 ret = ibt_deregister_mr(hcap->hca_hdl,
144 144 hcap->hca_mrhdl);
145 145 if (ret == IBT_SUCCESS) {
146 146 hcap->hca_mrhdl = NULL;
147 147 hcap->hca_lkey = 0;
148 148 hcap->hca_rkey = 0;
149 149 } else {
150 150 RDS_DPRINTF2(LABEL, "ibt_deregister_mr "
151 151 "failed: %d, mrhdl: 0x%p", ret,
152 152 hcap->hca_mrhdl);
153 153 }
154 154 }
155 155 hcap = hcap->hca_nextp;
156 156 }
157 157
158 158 kmem_free(rds_dpool.pool_bufmemp, (rds_dpool.pool_nbuffers +
159 159 rds_cpool.pool_nbuffers) * sizeof (rds_buf_t));
160 160 rds_dpool.pool_bufmemp = NULL;
161 161
162 162 kmem_free(rds_dpool.pool_memp, rds_dpool.pool_memsize);
163 163 rds_dpool.pool_memp = NULL;
164 164
165 165 mutex_exit(&rds_dpool.pool_lock);
166 166
167 167 RDS_DPRINTF4("rds_free_recv_caches", "Return");
168 168 }
169 169
170 170 int
171 171 rds_init_recv_caches(rds_state_t *statep)
172 172 {
173 173 uint8_t *mp;
174 174 rds_buf_t *bp;
175 175 rds_hca_t *hcap;
176 176 uint32_t nsessions;
177 177 uint_t ix;
178 178 uint_t nctrlrx;
179 179 uint8_t *memp;
180 180 uint_t memsize, nbuf;
181 181 rds_buf_t *bufmemp;
182 182 ibt_mr_attr_t mem_attr;
183 183 ibt_mr_desc_t mem_desc;
184 184 int ret;
185 185
186 186 RDS_DPRINTF4("rds_init_recv_caches", "Enter");
187 187
188 188 DUMP_USER_PARAMS();
189 189
190 190 mutex_enter(&rds_dpool.pool_lock);
191 191 if (rds_dpool.pool_memp != NULL) {
192 192 RDS_DPRINTF2("rds_init_recv_caches", "Pools are already "
193 193 "initialized");
194 194 mutex_exit(&rds_dpool.pool_lock);
195 195 return (0);
196 196 }
197 197
198 198 /*
199 199 * High water mark for the receive buffers in the system. If the
200 200 * number of buffers used crosses this mark then all sockets in
201 201 * would be stalled. The port quota for the sockets is set based
202 202 * on this limit.
203 203 */
204 204 rds_rx_pkts_pending_hwm = (PendingRxPktsHWM * NDataRX)/100;
205 205
206 206 rds_nbuffers_to_putback = min(MaxCtrlRecvBuffers, MaxDataRecvBuffers);
207 207
208 208 /* nsessions can never be less than 1 */
209 209 nsessions = MaxNodes - 1;
210 210 nctrlrx = (nsessions + 1) * MaxCtrlRecvBuffers * 2;
211 211
212 212 RDS_DPRINTF3(LABEL, "Number of Possible Sessions: %d", nsessions);
213 213
214 214 /* Add the hdr */
215 215 RdsPktSize = UserBufferSize + RDS_DATA_HDR_SZ;
216 216
217 217 memsize = (NDataRX * RdsPktSize) + (nctrlrx * RDS_CTRLPKT_SIZE);
218 218 nbuf = NDataRX + nctrlrx;
219 219 RDS_DPRINTF3(LABEL, "RDS Buffer Pool Memory: %lld", memsize);
220 220 RDS_DPRINTF3(LABEL, "Total Buffers: %d", nbuf);
221 221
222 222 memp = (uint8_t *)kmem_zalloc(memsize, KM_NOSLEEP);
223 223 if (memp == NULL) {
224 224 RDS_DPRINTF1(LABEL, "RDS Memory allocation failed");
225 225 mutex_exit(&rds_dpool.pool_lock);
226 226 return (-1);
227 227 }
228 228
229 229 RDS_DPRINTF3(LABEL, "RDS Buffer Entries Memory: %lld",
230 230 nbuf * sizeof (rds_buf_t));
231 231
232 232 /* allocate memory for buffer entries */
233 233 bufmemp = (rds_buf_t *)kmem_zalloc(nbuf * sizeof (rds_buf_t),
234 234 KM_SLEEP);
235 235
236 236 /* register the memory with all HCAs */
237 237 mem_attr.mr_vaddr = (ib_vaddr_t)(uintptr_t)memp;
238 238 mem_attr.mr_len = memsize;
239 239 mem_attr.mr_as = NULL;
240 240 mem_attr.mr_flags = IBT_MR_ENABLE_LOCAL_WRITE;
241 241
242 242 rw_enter(&statep->rds_hca_lock, RW_WRITER);
243 243
244 244 hcap = statep->rds_hcalistp;
245 245 while (hcap != NULL) {
246 246 if (hcap->hca_state != RDS_HCA_STATE_OPEN) {
247 247 hcap = hcap->hca_nextp;
248 248 continue;
249 249 }
250 250
251 251 ret = ibt_register_mr(hcap->hca_hdl, hcap->hca_pdhdl,
252 252 &mem_attr, &hcap->hca_mrhdl, &mem_desc);
253 253 if (ret != IBT_SUCCESS) {
254 254 RDS_DPRINTF2(LABEL, "ibt_register_mr failed: %d", ret);
255 255 hcap = statep->rds_hcalistp;
256 256 while ((hcap) && (hcap->hca_mrhdl != NULL)) {
257 257 ret = ibt_deregister_mr(hcap->hca_hdl,
258 258 hcap->hca_mrhdl);
259 259 if (ret == IBT_SUCCESS) {
260 260 hcap->hca_mrhdl = NULL;
261 261 hcap->hca_lkey = 0;
262 262 hcap->hca_rkey = 0;
263 263 } else {
264 264 RDS_DPRINTF2(LABEL, "ibt_deregister_mr "
265 265 "failed: %d, mrhdl: 0x%p", ret,
266 266 hcap->hca_mrhdl);
267 267 }
268 268 hcap = hcap->hca_nextp;
269 269 }
270 270 kmem_free(bufmemp, nbuf * sizeof (rds_buf_t));
271 271 kmem_free(memp, memsize);
272 272 rw_exit(&statep->rds_hca_lock);
273 273 mutex_exit(&rds_dpool.pool_lock);
274 274 return (-1);
275 275 }
276 276
277 277 hcap->hca_state = RDS_HCA_STATE_MEM_REGISTERED;
278 278 hcap->hca_lkey = mem_desc.md_lkey;
279 279 hcap->hca_rkey = mem_desc.md_rkey;
280 280
281 281 hcap = hcap->hca_nextp;
282 282 }
283 283 rw_exit(&statep->rds_hca_lock);
284 284
285 285 /* Initialize data pool */
286 286 rds_dpool.pool_memp = memp;
287 287 rds_dpool.pool_memsize = memsize;
288 288 rds_dpool.pool_bufmemp = bufmemp;
289 289 rds_dpool.pool_nbuffers = NDataRX;
290 290 rds_dpool.pool_nbusy = 0;
291 291 rds_dpool.pool_nfree = NDataRX;
292 292
293 293 /* chain the buffers */
294 294 mp = memp;
295 295 bp = bufmemp;
296 296 for (ix = 0; ix < NDataRX; ix++) {
297 297 bp[ix].buf_nextp = &bp[ix + 1];
298 298 bp[ix].buf_ds.ds_va = (ib_vaddr_t)(uintptr_t)mp;
299 299 bp[ix].buf_state = RDS_RCVBUF_FREE;
300 300 bp[ix].buf_frtn.free_func = rds_free_mblk;
301 301 bp[ix].buf_frtn.free_arg = (char *)&bp[ix];
302 302 mp = mp + RdsPktSize;
303 303 }
304 304 bp[NDataRX - 1].buf_nextp = NULL;
305 305 rds_dpool.pool_headp = &bp[0];
306 306 rds_dpool.pool_tailp = &bp[NDataRX - 1];
307 307
308 308 /* Initialize ctrl pool */
309 309 rds_cpool.pool_nbuffers = nctrlrx;
310 310 rds_cpool.pool_nbusy = 0;
311 311 rds_cpool.pool_nfree = nctrlrx;
312 312
313 313 /* chain the buffers */
314 314 for (ix = NDataRX; ix < nbuf - 1; ix++) {
315 315 bp[ix].buf_nextp = &bp[ix + 1];
316 316 bp[ix].buf_ds.ds_va = (ib_vaddr_t)(uintptr_t)mp;
317 317 mp = mp + RDS_CTRLPKT_SIZE;
318 318 }
319 319 bp[nbuf - 1].buf_ds.ds_va = (ib_vaddr_t)(uintptr_t)mp;
320 320 bp[nbuf - 1].buf_nextp = NULL;
321 321 rds_cpool.pool_headp = &bp[NDataRX];
322 322 rds_cpool.pool_tailp = &bp[nbuf - 1];
323 323
324 324 mutex_exit(&rds_dpool.pool_lock);
325 325
326 326 RDS_DPRINTF3(LABEL, "rdsmemp start: %p end: %p", memp, mp);
327 327 RDS_DPRINTF4("rds_init_recv_caches", "Return");
328 328 return (0);
329 329 }
330 330
331 331 rds_hca_t *rds_lkup_hca(ib_guid_t hca_guid);
332 332
333 333 void
334 334 rds_free_send_pool(rds_ep_t *ep)
335 335 {
336 336 rds_bufpool_t *pool;
337 337 rds_hca_t *hcap;
338 338 int ret;
339 339
340 340 pool = &ep->ep_sndpool;
341 341
342 342 mutex_enter(&pool->pool_lock);
343 343 if (pool->pool_memp == NULL) {
344 344 mutex_exit(&pool->pool_lock);
345 345 RDS_DPRINTF2("rds_free_send_pool",
346 346 "EP(%p) DOUBLE Free on Send Pool", ep);
347 347 return;
348 348 }
349 349
350 350 /* get the hcap for the HCA hosting this channel */
351 351 hcap = rds_lkup_hca(ep->ep_hca_guid);
352 352 if (hcap == NULL) {
353 353 RDS_DPRINTF2("rds_free_send_pool", "HCA (0x%llx) not found",
354 354 ep->ep_hca_guid);
355 355 } else {
356 356 ret = ibt_deregister_mr(hcap->hca_hdl, ep->ep_snd_mrhdl);
357 357 if (ret != IBT_SUCCESS) {
358 358 RDS_DPRINTF2(LABEL,
359 359 "ibt_deregister_mr failed: %d, mrhdl: 0x%p",
360 360 ret, ep->ep_snd_mrhdl);
361 361 }
362 362
363 363 if (ep->ep_ack_addr) {
364 364 ret = ibt_deregister_mr(hcap->hca_hdl, ep->ep_ackhdl);
365 365 if (ret != IBT_SUCCESS) {
366 366 RDS_DPRINTF2(LABEL,
367 367 "ibt_deregister_mr ackhdl failed: %d, "
368 368 "mrhdl: 0x%p", ret, ep->ep_ackhdl);
369 369 }
370 370
371 371 kmem_free((void *)ep->ep_ack_addr, sizeof (uintptr_t));
372 372 ep->ep_ack_addr = NULL;
373 373 }
374 374 }
375 375
376 376 kmem_free(pool->pool_memp, pool->pool_memsize);
377 377 kmem_free(pool->pool_bufmemp,
378 378 pool->pool_nbuffers * sizeof (rds_buf_t));
379 379 pool->pool_memp = NULL;
380 380 pool->pool_bufmemp = NULL;
381 381 mutex_exit(&pool->pool_lock);
382 382 }
383 383
384 384 int
385 385 rds_init_send_pool(rds_ep_t *ep, ib_guid_t hca_guid)
386 386 {
387 387 uint8_t *mp;
388 388 rds_buf_t *bp;
389 389 rds_hca_t *hcap;
390 390 uint_t ix, rcv_len;
391 391 ibt_mr_attr_t mem_attr;
392 392 ibt_mr_desc_t mem_desc;
393 393 uint8_t *memp;
394 394 rds_buf_t *bufmemp;
395 395 uintptr_t ack_addr = NULL;
396 396 uint_t memsize;
397 397 uint_t nbuf;
398 398 rds_bufpool_t *spool;
399 399 rds_data_hdr_t *pktp;
400 400 int ret;
401 401
402 402 RDS_DPRINTF2("rds_init_send_pool", "Enter");
403 403
404 404 spool = &ep->ep_sndpool;
405 405
406 406 ASSERT(spool->pool_memp == NULL);
407 407 ASSERT(ep->ep_hca_guid == 0);
408 408
409 409 /* get the hcap for the HCA hosting this channel */
410 410 hcap = rds_get_hcap(rdsib_statep, hca_guid);
411 411 if (hcap == NULL) {
412 412 RDS_DPRINTF2("rds_init_send_pool", "HCA (0x%llx) not found",
413 413 hca_guid);
414 414 return (-1);
415 415 }
416 416
417 417 if (ep->ep_type == RDS_EP_TYPE_DATA) {
418 418 spool->pool_nbuffers = MaxDataSendBuffers;
419 419 spool->pool_nbusy = 0;
420 420 spool->pool_nfree = MaxDataSendBuffers;
421 421 memsize = (MaxDataSendBuffers * RdsPktSize) +
422 422 sizeof (uintptr_t);
423 423 rcv_len = RdsPktSize;
424 424 } else {
425 425 spool->pool_nbuffers = MaxCtrlSendBuffers;
426 426 spool->pool_nbusy = 0;
427 427 spool->pool_nfree = MaxCtrlSendBuffers;
428 428 memsize = MaxCtrlSendBuffers * RDS_CTRLPKT_SIZE;
429 429 rcv_len = RDS_CTRLPKT_SIZE;
430 430 }
431 431 nbuf = spool->pool_nbuffers;
432 432
433 433 RDS_DPRINTF3(LABEL, "RDS Send Pool Memory: %lld", memsize);
434 434
435 435 memp = (uint8_t *)kmem_zalloc(memsize, KM_NOSLEEP);
436 436 if (memp == NULL) {
437 437 RDS_DPRINTF1(LABEL, "RDS Send Memory allocation failed");
438 438 return (-1);
439 439 }
440 440
441 441 RDS_DPRINTF3(LABEL, "RDS Buffer Entries Memory: %lld",
442 442 nbuf * sizeof (rds_buf_t));
443 443
444 444 /* allocate memory for buffer entries */
445 445 bufmemp = (rds_buf_t *)kmem_zalloc(nbuf * sizeof (rds_buf_t),
446 446 KM_SLEEP);
447 447
448 448 if (ep->ep_type == RDS_EP_TYPE_DATA) {
449 449 ack_addr = (uintptr_t)kmem_zalloc(sizeof (uintptr_t), KM_SLEEP);
450 450
451 451 /* register the memory with the HCA for this channel */
452 452 mem_attr.mr_vaddr = (ib_vaddr_t)ack_addr;
453 453 mem_attr.mr_len = sizeof (uintptr_t);
454 454 mem_attr.mr_as = NULL;
455 455 mem_attr.mr_flags = IBT_MR_SLEEP | IBT_MR_ENABLE_LOCAL_WRITE |
456 456 IBT_MR_ENABLE_REMOTE_WRITE;
457 457
458 458 ret = ibt_register_mr(hcap->hca_hdl, hcap->hca_pdhdl,
459 459 &mem_attr, &ep->ep_ackhdl, &mem_desc);
460 460 if (ret != IBT_SUCCESS) {
461 461 RDS_DPRINTF2("rds_init_send_pool",
462 462 "EP(%p): ibt_register_mr for ack failed: %d",
463 463 ep, ret);
464 464 kmem_free(memp, memsize);
465 465 kmem_free(bufmemp, nbuf * sizeof (rds_buf_t));
466 466 kmem_free((void *)ack_addr, sizeof (uintptr_t));
467 467 return (-1);
468 468 }
469 469 ep->ep_ack_rkey = mem_desc.md_rkey;
470 470 ep->ep_ack_addr = ack_addr;
471 471 }
472 472
473 473 /* register the memory with the HCA for this channel */
474 474 mem_attr.mr_vaddr = (ib_vaddr_t)(uintptr_t)memp;
475 475 mem_attr.mr_len = memsize;
476 476 mem_attr.mr_as = NULL;
477 477 mem_attr.mr_flags = IBT_MR_SLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
478 478
479 479 ret = ibt_register_mr(hcap->hca_hdl, hcap->hca_pdhdl,
480 480 &mem_attr, &ep->ep_snd_mrhdl, &mem_desc);
481 481 if (ret != IBT_SUCCESS) {
482 482 RDS_DPRINTF2("rds_init_send_pool", "EP(%p): ibt_register_mr "
483 483 "failed: %d", ep, ret);
484 484 kmem_free(memp, memsize);
485 485 kmem_free(bufmemp, nbuf * sizeof (rds_buf_t));
486 486 if (ack_addr != NULL)
487 487 kmem_free((void *)ack_addr, sizeof (uintptr_t));
488 488 return (-1);
489 489 }
490 490 ep->ep_snd_lkey = mem_desc.md_lkey;
491 491
492 492
493 493 /* Initialize the pool */
494 494 spool->pool_memp = memp;
495 495 spool->pool_memsize = memsize;
496 496 spool->pool_bufmemp = bufmemp;
497 497 spool->pool_sqpoll_pending = B_FALSE;
498 498
499 499 /* chain the buffers and initialize them */
500 500 mp = memp;
501 501 bp = bufmemp;
502 502
503 503 if (ep->ep_type == RDS_EP_TYPE_DATA) {
504 504 for (ix = 0; ix < nbuf - 1; ix++) {
505 505 bp[ix].buf_nextp = &bp[ix + 1];
506 506 bp[ix].buf_ep = ep;
507 507 bp[ix].buf_ds.ds_va = (ib_vaddr_t)(uintptr_t)mp;
508 508 bp[ix].buf_ds.ds_key = ep->ep_snd_lkey;
509 509 bp[ix].buf_state = RDS_SNDBUF_FREE;
510 510 pktp = (rds_data_hdr_t *)(uintptr_t)mp;
511 511 pktp->dh_bufid = (uintptr_t)&bp[ix];
512 512 mp = mp + rcv_len;
513 513 }
514 514 bp[nbuf - 1].buf_nextp = NULL;
515 515 bp[nbuf - 1].buf_ep = ep;
516 516 bp[nbuf - 1].buf_ds.ds_va = (ib_vaddr_t)(uintptr_t)mp;
517 517 bp[nbuf - 1].buf_ds.ds_key = ep->ep_snd_lkey;
518 518 bp[nbuf - 1].buf_state = RDS_SNDBUF_FREE;
519 519 pktp = (rds_data_hdr_t *)(uintptr_t)mp;
520 520 pktp->dh_bufid = (uintptr_t)&bp[nbuf - 1];
521 521
522 522 spool->pool_headp = &bp[0];
523 523 spool->pool_tailp = &bp[nbuf - 1];
524 524
525 525 mp = mp + rcv_len;
526 526 ep->ep_ackds.ds_va = (ib_vaddr_t)(uintptr_t)mp;
527 527 ep->ep_ackds.ds_key = ep->ep_snd_lkey;
528 528 ep->ep_ackds.ds_len = sizeof (uintptr_t);
529 529
530 530 *(uintptr_t *)ep->ep_ack_addr = (uintptr_t)spool->pool_tailp;
531 531 } else {
532 532 /* control send pool */
533 533 for (ix = 0; ix < nbuf - 1; ix++) {
534 534 bp[ix].buf_nextp = &bp[ix + 1];
535 535 bp[ix].buf_ep = ep;
536 536 bp[ix].buf_ds.ds_va = (ib_vaddr_t)(uintptr_t)mp;
537 537 bp[ix].buf_ds.ds_key = ep->ep_snd_lkey;
538 538 bp[ix].buf_state = RDS_SNDBUF_FREE;
539 539 mp = mp + rcv_len;
540 540 }
541 541 bp[nbuf - 1].buf_nextp = NULL;
542 542 bp[nbuf - 1].buf_ep = ep;
543 543 bp[nbuf - 1].buf_ds.ds_va = (ib_vaddr_t)(uintptr_t)mp;
544 544 bp[nbuf - 1].buf_ds.ds_key = ep->ep_snd_lkey;
545 545 bp[nbuf - 1].buf_state = RDS_SNDBUF_FREE;
546 546 spool->pool_headp = &bp[0];
547 547 spool->pool_tailp = &bp[nbuf - 1];
548 548 }
549 549
550 550 RDS_DPRINTF3(LABEL, "rdsmemp start: %p end: %p", memp, mp);
551 551 RDS_DPRINTF2("rds_init_send_pool", "Return");
552 552
553 553 return (0);
554 554 }
555 555
556 556 int
557 557 rds_reinit_send_pool(rds_ep_t *ep, ib_guid_t hca_guid)
558 558 {
559 559 rds_buf_t *bp;
560 560 rds_hca_t *hcap;
561 561 ibt_mr_attr_t mem_attr;
562 562 ibt_mr_desc_t mem_desc;
563 563 rds_bufpool_t *spool;
564 564 int ret;
565 565
566 566 RDS_DPRINTF2("rds_reinit_send_pool", "Enter: EP(%p)", ep);
567 567
568 568 spool = &ep->ep_sndpool;
569 569 ASSERT(spool->pool_memp != NULL);
570 570
571 571 /* deregister the send pool memory from the previous HCA */
572 572 hcap = rds_get_hcap(rdsib_statep, ep->ep_hca_guid);
573 573 if (hcap == NULL) {
574 574 RDS_DPRINTF2("rds_reinit_send_pool", "HCA (0x%llx) not found",
575 575 ep->ep_hca_guid);
576 576 } else {
577 577 if (ep->ep_snd_mrhdl != NULL) {
578 578 (void) ibt_deregister_mr(hcap->hca_hdl,
579 579 ep->ep_snd_mrhdl);
580 580 ep->ep_snd_mrhdl = NULL;
581 581 ep->ep_snd_lkey = 0;
582 582 }
583 583
584 584 if ((ep->ep_type == RDS_EP_TYPE_DATA) &&
585 585 (ep->ep_ackhdl != NULL)) {
586 586 (void) ibt_deregister_mr(hcap->hca_hdl, ep->ep_ackhdl);
587 587 ep->ep_ackhdl = NULL;
588 588 ep->ep_ack_rkey = 0;
589 589 }
590 590
591 591 ep->ep_hca_guid = NULL;
592 592 }
593 593
594 594 /* get the hcap for the new HCA */
595 595 hcap = rds_get_hcap(rdsib_statep, hca_guid);
596 596 if (hcap == NULL) {
597 597 RDS_DPRINTF2("rds_reinit_send_pool", "HCA (0x%llx) not found",
598 598 hca_guid);
599 599 return (-1);
600 600 }
601 601
602 602 /* register the send memory */
603 603 mem_attr.mr_vaddr = (ib_vaddr_t)(uintptr_t)spool->pool_memp;
604 604 mem_attr.mr_len = spool->pool_memsize;
605 605 mem_attr.mr_as = NULL;
606 606 mem_attr.mr_flags = IBT_MR_SLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
607 607
608 608 ret = ibt_register_mr(hcap->hca_hdl, hcap->hca_pdhdl,
609 609 &mem_attr, &ep->ep_snd_mrhdl, &mem_desc);
610 610 if (ret != IBT_SUCCESS) {
611 611 RDS_DPRINTF2("rds_reinit_send_pool",
612 612 "EP(%p): ibt_register_mr failed: %d", ep, ret);
613 613 return (-1);
614 614 }
615 615 ep->ep_snd_lkey = mem_desc.md_lkey;
616 616
617 617 /* register the acknowledgement space */
618 618 if (ep->ep_type == RDS_EP_TYPE_DATA) {
619 619 mem_attr.mr_vaddr = (ib_vaddr_t)ep->ep_ack_addr;
620 620 mem_attr.mr_len = sizeof (uintptr_t);
621 621 mem_attr.mr_as = NULL;
622 622 mem_attr.mr_flags = IBT_MR_SLEEP | IBT_MR_ENABLE_LOCAL_WRITE |
623 623 IBT_MR_ENABLE_REMOTE_WRITE;
624 624
625 625 ret = ibt_register_mr(hcap->hca_hdl, hcap->hca_pdhdl,
626 626 &mem_attr, &ep->ep_ackhdl, &mem_desc);
627 627 if (ret != IBT_SUCCESS) {
628 628 RDS_DPRINTF2("rds_reinit_send_pool",
629 629 "EP(%p): ibt_register_mr for ack failed: %d",
630 630 ep, ret);
631 631 (void) ibt_deregister_mr(hcap->hca_hdl,
632 632 ep->ep_snd_mrhdl);
633 633 ep->ep_snd_mrhdl = NULL;
634 634 ep->ep_snd_lkey = 0;
635 635 return (-1);
636 636 }
637 637 ep->ep_ack_rkey = mem_desc.md_rkey;
638 638
639 639 /* update the LKEY in the acknowledgement WR */
640 640 ep->ep_ackds.ds_key = ep->ep_snd_lkey;
641 641 }
642 642
643 643 /* update the LKEY in each buffer */
644 644 bp = spool->pool_headp;
645 645 while (bp) {
646 646 bp->buf_ds.ds_key = ep->ep_snd_lkey;
647 647 bp = bp->buf_nextp;
648 648 }
649 649
650 650 ep->ep_hca_guid = hca_guid;
651 651
652 652 RDS_DPRINTF2("rds_reinit_send_pool", "Return: EP(%p)", ep);
653 653
654 654 return (0);
655 655 }
656 656
657 657 void
658 658 rds_free_recv_pool(rds_ep_t *ep)
659 659 {
660 660 rds_bufpool_t *pool;
661 661
662 662 if (ep->ep_type == RDS_EP_TYPE_DATA) {
663 663 pool = &rds_dpool;
664 664 } else {
665 665 pool = &rds_cpool;
666 666 }
667 667
668 668 mutex_enter(&ep->ep_rcvpool.pool_lock);
669 669 if (ep->ep_rcvpool.pool_nfree != 0) {
670 670 rds_free_buf(pool, ep->ep_rcvpool.pool_headp,
671 671 ep->ep_rcvpool.pool_nfree);
672 672 ep->ep_rcvpool.pool_nfree = 0;
673 673 ep->ep_rcvpool.pool_headp = NULL;
674 674 ep->ep_rcvpool.pool_tailp = NULL;
675 675 }
676 676 mutex_exit(&ep->ep_rcvpool.pool_lock);
677 677 }
678 678
679 679 int
680 680 rds_init_recv_pool(rds_ep_t *ep)
681 681 {
682 682 rds_bufpool_t *rpool;
683 683 rds_qp_t *recvqp;
684 684
685 685 recvqp = &ep->ep_recvqp;
686 686 rpool = &ep->ep_rcvpool;
687 687 if (ep->ep_type == RDS_EP_TYPE_DATA) {
688 688 recvqp->qp_depth = MaxDataRecvBuffers;
689 689 recvqp->qp_level = 0;
690 690 recvqp->qp_lwm = (DataRecvBufferLWM * MaxDataRecvBuffers)/100;
691 691 recvqp->qp_taskqpending = B_FALSE;
692 692
693 693 rpool->pool_nbuffers = MaxDataRecvBuffers;
694 694 rpool->pool_nbusy = 0;
695 695 rpool->pool_nfree = 0;
696 696 } else {
697 697 recvqp->qp_depth = MaxCtrlRecvBuffers;
698 698 recvqp->qp_level = 0;
699 699 recvqp->qp_lwm = (CtrlRecvBufferLWM * MaxCtrlRecvBuffers)/100;
700 700 recvqp->qp_taskqpending = B_FALSE;
701 701
702 702 rpool->pool_nbuffers = MaxCtrlRecvBuffers;
703 703 rpool->pool_nbusy = 0;
704 704 rpool->pool_nfree = 0;
705 705 }
706 706
707 707 return (0);
708 708 }
709 709
710 710 /* Free buffers to the global pool, either cpool or dpool */
711 711 void
712 712 rds_free_buf(rds_bufpool_t *pool, rds_buf_t *bp, uint_t nbuf)
713 713 {
714 714 uint_t ix;
715 715
716 716 RDS_DPRINTF4("rds_free_buf", "Enter");
717 717
718 718 ASSERT(nbuf != 0);
719 719
720 720 mutex_enter(&pool->pool_lock);
721 721
722 722 if (pool->pool_nfree != 0) {
723 723 pool->pool_tailp->buf_nextp = bp;
724 724 } else {
725 725 pool->pool_headp = bp;
726 726 }
727 727
728 728 if (nbuf == 1) {
729 729 ASSERT(bp->buf_state == RDS_RCVBUF_FREE);
730 730 bp->buf_ep = NULL;
731 731 bp->buf_nextp = NULL;
732 732 pool->pool_tailp = bp;
733 733 } else {
734 734 for (ix = 1; ix < nbuf; ix++) {
735 735 ASSERT(bp->buf_state == RDS_RCVBUF_FREE);
736 736 bp->buf_ep = NULL;
737 737 bp = bp->buf_nextp;
738 738 }
739 739 ASSERT(bp->buf_state == RDS_RCVBUF_FREE);
740 740 bp->buf_ep = NULL;
741 741 bp->buf_nextp = NULL;
742 742 pool->pool_tailp = bp;
743 743 }
744 744 /* tail is always the last buffer */
745 745 pool->pool_tailp->buf_nextp = NULL;
746 746
747 747 pool->pool_nfree += nbuf;
748 748 pool->pool_nbusy -= nbuf;
749 749
750 750 mutex_exit(&pool->pool_lock);
751 751
752 752 RDS_DPRINTF4("rds_free_buf", "Return");
753 753 }
754 754
755 755 /* Get buffers from the global pools, either cpool or dpool */
756 756 rds_buf_t *
757 757 rds_get_buf(rds_bufpool_t *pool, uint_t nbuf, uint_t *nret)
758 758 {
759 759 rds_buf_t *bp = NULL, *bp1;
760 760 uint_t ix;
761 761
762 762 RDS_DPRINTF4("rds_get_buf", "Enter");
763 763
764 764 mutex_enter(&pool->pool_lock);
765 765
766 766 RDS_DPRINTF3("rds_get_buf", "Available: %d Needed: %d",
767 767 pool->pool_nfree, nbuf);
768 768
769 769 if (nbuf < pool->pool_nfree) {
770 770 *nret = nbuf;
771 771
772 772 bp1 = pool->pool_headp;
773 773 for (ix = 1; ix < nbuf; ix++) {
774 774 bp1 = bp1->buf_nextp;
775 775 }
776 776
777 777 bp = pool->pool_headp;
778 778 pool->pool_headp = bp1->buf_nextp;
779 779 bp1->buf_nextp = NULL;
780 780
781 781 pool->pool_nfree -= nbuf;
782 782 pool->pool_nbusy += nbuf;
783 783 } else if (nbuf >= pool->pool_nfree) {
784 784 *nret = pool->pool_nfree;
785 785
786 786 bp = pool->pool_headp;
787 787
788 788 pool->pool_headp = NULL;
789 789 pool->pool_tailp = NULL;
790 790
791 791 pool->pool_nbusy += pool->pool_nfree;
792 792 pool->pool_nfree = 0;
793 793 }
794 794
795 795 mutex_exit(&pool->pool_lock);
796 796
797 797 RDS_DPRINTF4("rds_get_buf", "Return");
798 798
799 799 return (bp);
800 800 }
801 801
802 802 boolean_t
803 803 rds_is_recvq_empty(rds_ep_t *ep, boolean_t wait)
804 804 {
805 805 rds_qp_t *recvqp;
806 806 rds_bufpool_t *rpool;
807 807 boolean_t ret = B_TRUE;
↓ open down ↓ |
661 lines elided |
↑ open up ↑ |
808 808
809 809 recvqp = &ep->ep_recvqp;
810 810 mutex_enter(&recvqp->qp_lock);
811 811 RDS_DPRINTF2("rds_is_recvq_empty", "EP(%p): QP has %d WRs",
812 812 ep, recvqp->qp_level);
813 813 if (wait) {
814 814 /* wait until the RQ is empty */
815 815 while (recvqp->qp_level != 0) {
816 816 /* wait one second and try again */
817 817 mutex_exit(&recvqp->qp_lock);
818 - delay(drv_usectohz(1000000));
818 + delay(drv_sectohz(1));
819 819 mutex_enter(&recvqp->qp_lock);
820 820 }
821 821 } else if (recvqp->qp_level != 0) {
822 822 ret = B_FALSE;
823 823 }
824 824 mutex_exit(&recvqp->qp_lock);
825 825
826 826 rpool = &ep->ep_rcvpool;
827 827 mutex_enter(&rpool->pool_lock);
828 828
829 829 /*
830 830 * During failovers/reconnects, the app may still have some buffers
831 831 * on thier socket queues. Waiting here for those buffers may
832 832 * cause a hang. It seems ok for those buffers to get freed later.
833 833 */
834 834 if (rpool->pool_nbusy != 0) {
835 835 RDS_DPRINTF2("rds_is_recvq_empty", "EP(%p): "
836 836 "There are %d pending buffers on sockqs", ep,
837 837 rpool->pool_nbusy);
838 838 ret = B_FALSE;
839 839 }
840 840 mutex_exit(&rpool->pool_lock);
841 841
842 842 return (ret);
843 843 }
844 844
845 845 boolean_t
846 846 rds_is_sendq_empty(rds_ep_t *ep, uint_t wait)
847 847 {
848 848 rds_bufpool_t *spool;
849 849 rds_buf_t *bp;
850 850 boolean_t ret1 = B_TRUE;
↓ open down ↓ |
22 lines elided |
↑ open up ↑ |
851 851
852 852 /* check if all the sends completed */
853 853 spool = &ep->ep_sndpool;
854 854 mutex_enter(&spool->pool_lock);
855 855 RDS_DPRINTF2("rds_is_sendq_empty", "EP(%p): "
856 856 "Send Pool contains: %d", ep, spool->pool_nbusy);
857 857 if (wait) {
858 858 while (spool->pool_nbusy != 0) {
859 859 if (rds_no_interrupts) {
860 860 /* wait one second and try again */
861 - delay(drv_usectohz(1000000));
861 + delay(drv_sectohz(1));
862 862 rds_poll_send_completions(ep->ep_sendcq, ep,
863 863 B_TRUE);
864 864 } else {
865 865 /* wait one second and try again */
866 866 mutex_exit(&spool->pool_lock);
867 - delay(drv_usectohz(1000000));
867 + delay(drv_sectohz(1));
868 868 mutex_enter(&spool->pool_lock);
869 869 }
870 870 }
871 871
872 872 if ((wait == 2) && (ep->ep_type == RDS_EP_TYPE_DATA)) {
873 873 rds_buf_t *ackbp;
874 874 rds_buf_t *prev_ackbp;
875 875
876 876 /*
877 877 * If the last one is acknowledged then everything
878 878 * is acknowledged
879 879 */
↓ open down ↓ |
2 lines elided |
↑ open up ↑ |
880 880 bp = spool->pool_tailp;
881 881 ackbp = *(rds_buf_t **)ep->ep_ack_addr;
882 882 prev_ackbp = ackbp;
883 883 RDS_DPRINTF2("rds_is_sendq_empty", "EP(%p): "
884 884 "Checking for acknowledgements", ep);
885 885 while (bp != ackbp) {
886 886 RDS_DPRINTF2("rds_is_sendq_empty",
887 887 "EP(%p) BP(0x%p/0x%p) last "
888 888 "sent/acknowledged", ep, bp, ackbp);
889 889 mutex_exit(&spool->pool_lock);
890 - delay(drv_usectohz(1000000));
890 + delay(drv_sectohz(1));
891 891 mutex_enter(&spool->pool_lock);
892 892
893 893 bp = spool->pool_tailp;
894 894 ackbp = *(rds_buf_t **)ep->ep_ack_addr;
895 895 if (ackbp == prev_ackbp) {
896 896 RDS_DPRINTF2("rds_is_sendq_empty",
897 897 "There has been no progress,"
898 898 "give up and proceed");
899 899 break;
900 900 }
901 901 prev_ackbp = ackbp;
902 902 }
903 903 }
904 904 } else if (spool->pool_nbusy != 0) {
905 905 ret1 = B_FALSE;
906 906 }
↓ open down ↓ |
6 lines elided |
↑ open up ↑ |
907 907 mutex_exit(&spool->pool_lock);
908 908
909 909 /* check if all the rdma acks completed */
910 910 mutex_enter(&ep->ep_lock);
911 911 RDS_DPRINTF2("rds_is_sendq_empty", "EP(%p): "
912 912 "Outstanding RDMA Acks: %d", ep, ep->ep_rdmacnt);
913 913 if (wait) {
914 914 while (ep->ep_rdmacnt != 0) {
915 915 if (rds_no_interrupts) {
916 916 /* wait one second and try again */
917 - delay(drv_usectohz(1000000));
917 + delay(drv_sectohz(1));
918 918 rds_poll_send_completions(ep->ep_sendcq, ep,
919 919 B_FALSE);
920 920 } else {
921 921 /* wait one second and try again */
922 922 mutex_exit(&ep->ep_lock);
923 - delay(drv_usectohz(1000000));
923 + delay(drv_sectohz(1));
924 924 mutex_enter(&ep->ep_lock);
925 925 }
926 926 }
927 927 } else if (ep->ep_rdmacnt != 0) {
928 928 ret1 = B_FALSE;
929 929 }
930 930 mutex_exit(&ep->ep_lock);
931 931
932 932 return (ret1);
933 933 }
934 934
935 935 /* Get buffers from the send pool */
936 936 rds_buf_t *
937 937 rds_get_send_buf(rds_ep_t *ep, uint_t nbuf)
938 938 {
939 939 rds_buf_t *bp = NULL, *bp1;
940 940 rds_bufpool_t *spool;
941 941 uint_t waittime = rds_waittime_ms * 1000;
942 942 uint_t ix;
943 943 int ret;
944 944
945 945 RDS_DPRINTF4("rds_get_send_buf", "Enter: EP(%p) Buffers requested: %d",
946 946 ep, nbuf);
947 947
948 948 spool = &ep->ep_sndpool;
949 949 mutex_enter(&spool->pool_lock);
950 950
951 951 if (rds_no_interrupts) {
952 952 if ((spool->pool_sqpoll_pending == B_FALSE) &&
953 953 (spool->pool_nbusy >
954 954 (spool->pool_nbuffers * rds_poll_percent_full)/100)) {
955 955 spool->pool_sqpoll_pending = B_TRUE;
956 956 mutex_exit(&spool->pool_lock);
957 957 rds_poll_send_completions(ep->ep_sendcq, ep, B_FALSE);
958 958 mutex_enter(&spool->pool_lock);
959 959 spool->pool_sqpoll_pending = B_FALSE;
960 960 }
961 961 }
962 962
963 963 if (spool->pool_nfree < nbuf) {
964 964 /* wait for buffers to become available */
965 965 spool->pool_cv_count += nbuf;
966 966 ret = cv_reltimedwait_sig(&spool->pool_cv, &spool->pool_lock,
967 967 drv_usectohz(waittime), TR_CLOCK_TICK);
968 968 /* ret = cv_wait_sig(&spool->pool_cv, &spool->pool_lock); */
969 969 if (ret == 0) {
970 970 /* signal pending */
971 971 spool->pool_cv_count -= nbuf;
972 972 mutex_exit(&spool->pool_lock);
973 973 return (NULL);
974 974 }
975 975
976 976 spool->pool_cv_count -= nbuf;
977 977 }
978 978
979 979 /* Have the number of buffers needed */
980 980 if (spool->pool_nfree > nbuf) {
981 981 bp = spool->pool_headp;
982 982
983 983 if (ep->ep_type == RDS_EP_TYPE_DATA) {
984 984 rds_buf_t *ackbp;
985 985 ackbp = *(rds_buf_t **)ep->ep_ack_addr;
986 986
987 987 /* check if all the needed buffers are acknowledged */
988 988 bp1 = bp;
989 989 for (ix = 0; ix < nbuf; ix++) {
990 990 if ((bp1 == ackbp) ||
991 991 (bp1->buf_state != RDS_SNDBUF_FREE)) {
992 992 /*
993 993 * The buffer is not yet signalled or
994 994 * is not yet acknowledged
995 995 */
996 996 RDS_DPRINTF5("rds_get_send_buf",
997 997 "EP(%p) Buffer (%p) not yet "
998 998 "acked/completed", ep, bp1);
999 999 mutex_exit(&spool->pool_lock);
1000 1000 return (NULL);
1001 1001 }
1002 1002
1003 1003 bp1 = bp1->buf_nextp;
1004 1004 }
1005 1005 }
1006 1006
1007 1007 /* mark the buffers as pending */
1008 1008 bp1 = bp;
1009 1009 for (ix = 1; ix < nbuf; ix++) {
1010 1010 ASSERT(bp1->buf_state == RDS_SNDBUF_FREE);
1011 1011 bp1->buf_state = RDS_SNDBUF_PENDING;
1012 1012 bp1 = bp1->buf_nextp;
1013 1013 }
1014 1014 ASSERT(bp1->buf_state == RDS_SNDBUF_FREE);
1015 1015 bp1->buf_state = RDS_SNDBUF_PENDING;
1016 1016
1017 1017 spool->pool_headp = bp1->buf_nextp;
1018 1018 bp1->buf_nextp = NULL;
1019 1019 if (spool->pool_headp == NULL)
1020 1020 spool->pool_tailp = NULL;
1021 1021 spool->pool_nfree -= nbuf;
1022 1022 spool->pool_nbusy += nbuf;
1023 1023 }
1024 1024 mutex_exit(&spool->pool_lock);
1025 1025
1026 1026 RDS_DPRINTF4("rds_get_send_buf", "Return: EP(%p) Buffers requested: %d",
1027 1027 ep, nbuf);
1028 1028
1029 1029 return (bp);
1030 1030 }
1031 1031
1032 1032 #define RDS_MIN_BUF_TO_WAKE_THREADS 10
1033 1033
1034 1034 void
1035 1035 rds_free_send_buf(rds_ep_t *ep, rds_buf_t *headp, rds_buf_t *tailp, uint_t nbuf,
1036 1036 boolean_t lock)
1037 1037 {
1038 1038 rds_bufpool_t *spool;
1039 1039 rds_buf_t *tmp;
1040 1040
1041 1041 RDS_DPRINTF4("rds_free_send_buf", "Enter");
1042 1042
1043 1043 ASSERT(nbuf != 0);
1044 1044
1045 1045 if (tailp == NULL) {
1046 1046 if (nbuf > 1) {
1047 1047 tmp = headp;
1048 1048 while (tmp->buf_nextp) {
1049 1049 tmp = tmp->buf_nextp;
1050 1050 }
1051 1051 tailp = tmp;
1052 1052 } else {
1053 1053 tailp = headp;
1054 1054 }
1055 1055 }
1056 1056
1057 1057 spool = &ep->ep_sndpool;
1058 1058
1059 1059 if (lock == B_FALSE) {
1060 1060 /* lock is not held outside */
1061 1061 mutex_enter(&spool->pool_lock);
1062 1062 }
1063 1063
1064 1064 if (spool->pool_nfree) {
1065 1065 spool->pool_tailp->buf_nextp = headp;
1066 1066 } else {
1067 1067 spool->pool_headp = headp;
1068 1068 }
1069 1069 spool->pool_tailp = tailp;
1070 1070
1071 1071 spool->pool_nfree += nbuf;
1072 1072 spool->pool_nbusy -= nbuf;
1073 1073
1074 1074 if ((spool->pool_cv_count > 0) &&
1075 1075 (spool->pool_nfree > RDS_MIN_BUF_TO_WAKE_THREADS)) {
1076 1076 if (spool->pool_nfree >= spool->pool_cv_count)
1077 1077 cv_broadcast(&spool->pool_cv);
1078 1078 else
1079 1079 cv_signal(&spool->pool_cv);
1080 1080 }
1081 1081
1082 1082 if (lock == B_FALSE) {
1083 1083 mutex_exit(&spool->pool_lock);
1084 1084 }
1085 1085
1086 1086 RDS_DPRINTF4("rds_free_send_buf", "Return");
1087 1087 }
1088 1088
1089 1089 void
1090 1090 rds_free_recv_buf(rds_buf_t *bp, uint_t nbuf)
1091 1091 {
1092 1092 rds_ep_t *ep;
1093 1093 rds_bufpool_t *rpool;
1094 1094 rds_buf_t *bp1;
1095 1095 uint_t ix;
1096 1096
1097 1097 RDS_DPRINTF4("rds_free_recv_buf", "Enter");
1098 1098
1099 1099 ASSERT(nbuf != 0);
1100 1100
1101 1101 ep = bp->buf_ep;
1102 1102 rpool = &ep->ep_rcvpool;
1103 1103
1104 1104 mutex_enter(&rpool->pool_lock);
1105 1105
1106 1106 /* Add the buffers to the local pool */
1107 1107 if (rpool->pool_tailp == NULL) {
1108 1108 ASSERT(rpool->pool_headp == NULL);
1109 1109 ASSERT(rpool->pool_nfree == 0);
1110 1110 rpool->pool_headp = bp;
1111 1111 bp1 = bp;
1112 1112 for (ix = 1; ix < nbuf; ix++) {
1113 1113 if (bp1->buf_state == RDS_RCVBUF_ONSOCKQ) {
1114 1114 rpool->pool_nbusy--;
1115 1115 }
1116 1116 bp1->buf_state = RDS_RCVBUF_FREE;
1117 1117 bp1 = bp1->buf_nextp;
1118 1118 }
1119 1119 bp1->buf_nextp = NULL;
1120 1120 if (bp->buf_state == RDS_RCVBUF_ONSOCKQ) {
1121 1121 rpool->pool_nbusy--;
1122 1122 }
1123 1123 bp->buf_state = RDS_RCVBUF_FREE;
1124 1124 rpool->pool_tailp = bp1;
1125 1125 rpool->pool_nfree += nbuf;
1126 1126 } else {
1127 1127 bp1 = bp;
1128 1128 for (ix = 1; ix < nbuf; ix++) {
1129 1129 if (bp1->buf_state == RDS_RCVBUF_ONSOCKQ) {
1130 1130 rpool->pool_nbusy--;
1131 1131 }
1132 1132 bp1->buf_state = RDS_RCVBUF_FREE;
1133 1133 bp1 = bp1->buf_nextp;
1134 1134 }
1135 1135 bp1->buf_nextp = NULL;
1136 1136 if (bp->buf_state == RDS_RCVBUF_ONSOCKQ) {
1137 1137 rpool->pool_nbusy--;
1138 1138 }
1139 1139 bp->buf_state = RDS_RCVBUF_FREE;
1140 1140 rpool->pool_tailp->buf_nextp = bp;
1141 1141 rpool->pool_tailp = bp1;
1142 1142 rpool->pool_nfree += nbuf;
1143 1143 }
1144 1144
1145 1145 if (rpool->pool_nfree >= rds_nbuffers_to_putback) {
1146 1146 bp = rpool->pool_headp;
1147 1147 nbuf = rpool->pool_nfree;
1148 1148 rpool->pool_headp = NULL;
1149 1149 rpool->pool_tailp = NULL;
1150 1150 rpool->pool_nfree = 0;
1151 1151 mutex_exit(&rpool->pool_lock);
1152 1152
1153 1153 /* Free the buffers to the global pool */
1154 1154 if (ep->ep_type == RDS_EP_TYPE_DATA) {
1155 1155 rds_free_buf(&rds_dpool, bp, nbuf);
1156 1156 } else {
1157 1157 rds_free_buf(&rds_cpool, bp, nbuf);
1158 1158 }
1159 1159
1160 1160 return;
1161 1161 }
1162 1162 mutex_exit(&rpool->pool_lock);
1163 1163
1164 1164 RDS_DPRINTF4("rds_free_recv_buf", "Return");
1165 1165 }
↓ open down ↓ |
232 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX