1219 if (ibd_rc_post_srq(state, rwqe) == DDI_FAILURE) {
1220 ibd_print_warn(state, "ibd_rc_srq_freemsg_cb: ibd_rc_post_srq"
1221 " failed");
1222 ibd_rc_srq_free_rwqe(state, rwqe);
1223 return;
1224 }
1225 }
1226
1227 /*
1228 * Post a rwqe to the hardware and add it to the Rx list.
1229 */
1230 static int
1231 ibd_rc_post_srq(ibd_state_t *state, ibd_rwqe_t *rwqe)
1232 {
1233 /*
1234 * Here we should add dl_cnt before post recv, because
1235 * we would have to make sure dl_cnt is updated before
1236 * the corresponding ibd_rc_process_rx() is called.
1237 */
1238 ASSERT(state->rc_srq_rwqe_list.dl_cnt < state->rc_srq_size);
1239 atomic_add_32(&state->rc_srq_rwqe_list.dl_cnt, 1);
1240 if (ibt_post_srq(state->rc_srq_hdl, &rwqe->w_rwr, 1, NULL) !=
1241 IBT_SUCCESS) {
1242 atomic_dec_32(&state->rc_srq_rwqe_list.dl_cnt);
1243 DPRINT(40, "ibd_rc_post_srq : ibt_post_srq() failed");
1244 return (DDI_FAILURE);
1245 }
1246
1247 return (DDI_SUCCESS);
1248 }
1249
1250 /*
1251 * Post a rwqe to the hardware and add it to the Rx list.
1252 */
1253 static int
1254 ibd_rc_post_rwqe(ibd_rc_chan_t *chan, ibd_rwqe_t *rwqe)
1255 {
1256 /*
1257 * Here we should add dl_cnt before post recv, because we would
1258 * have to make sure dl_cnt has already updated before
1259 * corresponding ibd_rc_process_rx() is called.
1260 */
1261 atomic_add_32(&chan->rx_wqe_list.dl_cnt, 1);
1262 if (ibt_post_recv(chan->chan_hdl, &rwqe->w_rwr, 1, NULL) !=
1263 IBT_SUCCESS) {
1264 atomic_dec_32(&chan->rx_wqe_list.dl_cnt);
1265 DPRINT(40, "ibd_rc_post_rwqe : failed in ibt_post_recv()");
1266 return (DDI_FAILURE);
1267 }
1268 return (DDI_SUCCESS);
1269 }
1270
1271 static int
1272 ibd_rc_alloc_rx_copybufs(ibd_rc_chan_t *chan)
1273 {
1274 ibd_state_t *state = chan->state;
1275 ibt_mr_attr_t mem_attr;
1276 uint_t rc_rx_bufs_sz;
1277
1278 /*
1279 * Allocate one big chunk for all regular rx copy bufs
1280 */
1281 rc_rx_bufs_sz = (state->rc_mtu + IPOIB_GRH_SIZE) * chan->rcq_size;
1482
1483 #ifdef DEBUG
1484 if (rxcnt < state->id_rc_rx_rwqe_thresh) {
1485 state->rc_rwqe_short++;
1486 }
1487 #endif
1488
1489 /*
1490 * Possibly replenish the Rx pool if needed.
1491 */
1492 if ((rxcnt >= state->id_rc_rx_rwqe_thresh) &&
1493 (wc->wc_bytes_xfer > state->id_rc_rx_copy_thresh)) {
1494 atomic_add_64(&state->rc_rcv_trans_byte, wc->wc_bytes_xfer);
1495 atomic_inc_64(&state->rc_rcv_trans_pkt);
1496
1497 /*
1498 * Record how many rwqe has been occupied by upper
1499 * network layer
1500 */
1501 if (state->rc_enable_srq) {
1502 atomic_add_32(&state->rc_srq_rwqe_list.
1503 dl_bufs_outstanding, 1);
1504 } else {
1505 atomic_add_32(&chan->rx_wqe_list.
1506 dl_bufs_outstanding, 1);
1507 }
1508 mp = rwqe->rwqe_im_mblk;
1509 } else {
1510 atomic_add_64(&state->rc_rcv_copy_byte, wc->wc_bytes_xfer);
1511 atomic_inc_64(&state->rc_rcv_copy_pkt);
1512
1513 if ((mp = allocb(wc->wc_bytes_xfer + IPOIB_GRH_SIZE,
1514 BPRI_HI)) == NULL) { /* no memory */
1515 DPRINT(40, "ibd_rc_process_rx: allocb() failed");
1516 state->rc_rcv_alloc_fail++;
1517 if (state->rc_enable_srq) {
1518 if (ibd_rc_post_srq(state, rwqe) ==
1519 DDI_FAILURE) {
1520 ibd_rc_srq_free_rwqe(state, rwqe);
1521 }
1522 } else {
1523 if (ibd_rc_post_rwqe(chan, rwqe) ==
1524 DDI_FAILURE) {
1525 ibd_rc_free_rwqe(chan, rwqe);
1526 }
1652
1653 rwqe->rwqe_im_mblk = desballoc(rwqe->rwqe_copybuf.ic_bufaddr,
1654 state->rc_mtu + IPOIB_GRH_SIZE, 0, &rwqe->w_freemsg_cb);
1655 if (rwqe->rwqe_im_mblk == NULL) {
1656 DPRINT(40, "ibd_rc_freemsg_cb: desballoc() failed");
1657 ibd_rc_free_rwqe(chan, rwqe);
1658 return;
1659 }
1660
1661 /*
1662 * Post back to h/w. We could actually have more than
1663 * id_num_rwqe WQEs on the list if there were multiple
1664 * ibd_freemsg_cb() calls outstanding (since the lock is
1665 * not held the entire time). This will start getting
1666 * corrected over subsequent ibd_freemsg_cb() calls.
1667 */
1668 if (ibd_rc_post_rwqe(chan, rwqe) == DDI_FAILURE) {
1669 ibd_rc_free_rwqe(chan, rwqe);
1670 return;
1671 }
1672 atomic_add_32(&chan->rx_wqe_list.dl_bufs_outstanding, -1);
1673 }
1674
1675 /*
1676 * Common code for interrupt handling as well as for polling
1677 * for all completed wqe's while detaching.
1678 */
1679 static void
1680 ibd_rc_poll_rcq(ibd_rc_chan_t *chan, ibt_cq_hdl_t cq_hdl)
1681 {
1682 ibd_wqe_t *wqe;
1683 ibt_wc_t *wc, *wcs;
1684 uint_t numwcs, real_numwcs;
1685 int i;
1686
1687 wcs = chan->rx_wc;
1688 numwcs = IBD_RC_MAX_CQ_WC;
1689
1690 while (ibt_poll_cq(cq_hdl, wcs, numwcs, &real_numwcs) == IBT_SUCCESS) {
1691 for (i = 0, wc = wcs; i < real_numwcs; i++, wc++) {
1692 wqe = (ibd_wqe_t *)(uintptr_t)wc->wc_id;
|
1219 if (ibd_rc_post_srq(state, rwqe) == DDI_FAILURE) {
1220 ibd_print_warn(state, "ibd_rc_srq_freemsg_cb: ibd_rc_post_srq"
1221 " failed");
1222 ibd_rc_srq_free_rwqe(state, rwqe);
1223 return;
1224 }
1225 }
1226
1227 /*
1228 * Post a rwqe to the hardware and add it to the Rx list.
1229 */
1230 static int
1231 ibd_rc_post_srq(ibd_state_t *state, ibd_rwqe_t *rwqe)
1232 {
1233 /*
1234 * Here we should add dl_cnt before post recv, because
1235 * we would have to make sure dl_cnt is updated before
1236 * the corresponding ibd_rc_process_rx() is called.
1237 */
1238 ASSERT(state->rc_srq_rwqe_list.dl_cnt < state->rc_srq_size);
1239 atomic_inc_32(&state->rc_srq_rwqe_list.dl_cnt);
1240 if (ibt_post_srq(state->rc_srq_hdl, &rwqe->w_rwr, 1, NULL) !=
1241 IBT_SUCCESS) {
1242 atomic_dec_32(&state->rc_srq_rwqe_list.dl_cnt);
1243 DPRINT(40, "ibd_rc_post_srq : ibt_post_srq() failed");
1244 return (DDI_FAILURE);
1245 }
1246
1247 return (DDI_SUCCESS);
1248 }
1249
1250 /*
1251 * Post a rwqe to the hardware and add it to the Rx list.
1252 */
1253 static int
1254 ibd_rc_post_rwqe(ibd_rc_chan_t *chan, ibd_rwqe_t *rwqe)
1255 {
1256 /*
1257 * Here we should add dl_cnt before post recv, because we would
1258 * have to make sure dl_cnt has already updated before
1259 * corresponding ibd_rc_process_rx() is called.
1260 */
1261 atomic_inc_32(&chan->rx_wqe_list.dl_cnt);
1262 if (ibt_post_recv(chan->chan_hdl, &rwqe->w_rwr, 1, NULL) !=
1263 IBT_SUCCESS) {
1264 atomic_dec_32(&chan->rx_wqe_list.dl_cnt);
1265 DPRINT(40, "ibd_rc_post_rwqe : failed in ibt_post_recv()");
1266 return (DDI_FAILURE);
1267 }
1268 return (DDI_SUCCESS);
1269 }
1270
1271 static int
1272 ibd_rc_alloc_rx_copybufs(ibd_rc_chan_t *chan)
1273 {
1274 ibd_state_t *state = chan->state;
1275 ibt_mr_attr_t mem_attr;
1276 uint_t rc_rx_bufs_sz;
1277
1278 /*
1279 * Allocate one big chunk for all regular rx copy bufs
1280 */
1281 rc_rx_bufs_sz = (state->rc_mtu + IPOIB_GRH_SIZE) * chan->rcq_size;
1482
1483 #ifdef DEBUG
1484 if (rxcnt < state->id_rc_rx_rwqe_thresh) {
1485 state->rc_rwqe_short++;
1486 }
1487 #endif
1488
1489 /*
1490 * Possibly replenish the Rx pool if needed.
1491 */
1492 if ((rxcnt >= state->id_rc_rx_rwqe_thresh) &&
1493 (wc->wc_bytes_xfer > state->id_rc_rx_copy_thresh)) {
1494 atomic_add_64(&state->rc_rcv_trans_byte, wc->wc_bytes_xfer);
1495 atomic_inc_64(&state->rc_rcv_trans_pkt);
1496
1497 /*
1498 * Record how many rwqe has been occupied by upper
1499 * network layer
1500 */
1501 if (state->rc_enable_srq) {
1502 atomic_inc_32(
1503 &state->rc_srq_rwqe_list.dl_bufs_outstanding);
1504 } else {
1505 atomic_inc_32(&chan->rx_wqe_list.dl_bufs_outstanding);
1506 }
1507 mp = rwqe->rwqe_im_mblk;
1508 } else {
1509 atomic_add_64(&state->rc_rcv_copy_byte, wc->wc_bytes_xfer);
1510 atomic_inc_64(&state->rc_rcv_copy_pkt);
1511
1512 if ((mp = allocb(wc->wc_bytes_xfer + IPOIB_GRH_SIZE,
1513 BPRI_HI)) == NULL) { /* no memory */
1514 DPRINT(40, "ibd_rc_process_rx: allocb() failed");
1515 state->rc_rcv_alloc_fail++;
1516 if (state->rc_enable_srq) {
1517 if (ibd_rc_post_srq(state, rwqe) ==
1518 DDI_FAILURE) {
1519 ibd_rc_srq_free_rwqe(state, rwqe);
1520 }
1521 } else {
1522 if (ibd_rc_post_rwqe(chan, rwqe) ==
1523 DDI_FAILURE) {
1524 ibd_rc_free_rwqe(chan, rwqe);
1525 }
1651
1652 rwqe->rwqe_im_mblk = desballoc(rwqe->rwqe_copybuf.ic_bufaddr,
1653 state->rc_mtu + IPOIB_GRH_SIZE, 0, &rwqe->w_freemsg_cb);
1654 if (rwqe->rwqe_im_mblk == NULL) {
1655 DPRINT(40, "ibd_rc_freemsg_cb: desballoc() failed");
1656 ibd_rc_free_rwqe(chan, rwqe);
1657 return;
1658 }
1659
1660 /*
1661 * Post back to h/w. We could actually have more than
1662 * id_num_rwqe WQEs on the list if there were multiple
1663 * ibd_freemsg_cb() calls outstanding (since the lock is
1664 * not held the entire time). This will start getting
1665 * corrected over subsequent ibd_freemsg_cb() calls.
1666 */
1667 if (ibd_rc_post_rwqe(chan, rwqe) == DDI_FAILURE) {
1668 ibd_rc_free_rwqe(chan, rwqe);
1669 return;
1670 }
1671 atomic_dec_32(&chan->rx_wqe_list.dl_bufs_outstanding);
1672 }
1673
1674 /*
1675 * Common code for interrupt handling as well as for polling
1676 * for all completed wqe's while detaching.
1677 */
1678 static void
1679 ibd_rc_poll_rcq(ibd_rc_chan_t *chan, ibt_cq_hdl_t cq_hdl)
1680 {
1681 ibd_wqe_t *wqe;
1682 ibt_wc_t *wc, *wcs;
1683 uint_t numwcs, real_numwcs;
1684 int i;
1685
1686 wcs = chan->rx_wc;
1687 numwcs = IBD_RC_MAX_CQ_WC;
1688
1689 while (ibt_poll_cq(cq_hdl, wcs, numwcs, &real_numwcs) == IBT_SUCCESS) {
1690 for (i = 0, wc = wcs; i < real_numwcs; i++, wc++) {
1691 wqe = (ibd_wqe_t *)(uintptr_t)wc->wc_id;
|