Print this page
5045 use atomic_{inc,dec}_* instead of atomic_add_*
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/hxge/hxge_rxdma.c
+++ new/usr/src/uts/common/io/hxge/hxge_rxdma.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 #include <hxge_impl.h>
27 27 #include <hxge_rxdma.h>
28 28 #include <hpi.h>
29 29 #include <hpi_vir.h>
30 30
31 31 /*
32 32 * Number of blocks to accumulate before re-enabling DMA
33 33 * when we get RBR empty.
34 34 */
35 35 #define HXGE_RBR_EMPTY_THRESHOLD 64
36 36
37 37 /*
38 38 * Globals: tunable parameters (/etc/system or adb)
39 39 *
40 40 */
41 41 extern uint32_t hxge_rbr_size;
42 42 extern uint32_t hxge_rcr_size;
43 43 extern uint32_t hxge_rbr_spare_size;
44 44 extern uint32_t hxge_mblks_pending;
45 45
46 46 /*
47 47 * Tunables to manage the receive buffer blocks.
48 48 *
49 49 * hxge_rx_threshold_hi: copy all buffers.
50 50 * hxge_rx_bcopy_size_type: receive buffer block size type.
51 51 * hxge_rx_threshold_lo: copy only up to tunable block size type.
52 52 */
53 53 extern hxge_rxbuf_threshold_t hxge_rx_threshold_hi;
54 54 extern hxge_rxbuf_type_t hxge_rx_buf_size_type;
55 55 extern hxge_rxbuf_threshold_t hxge_rx_threshold_lo;
56 56
57 57 /*
58 58 * Static local functions.
59 59 */
60 60 static hxge_status_t hxge_map_rxdma(p_hxge_t hxgep);
61 61 static void hxge_unmap_rxdma(p_hxge_t hxgep);
62 62 static hxge_status_t hxge_rxdma_hw_start_common(p_hxge_t hxgep);
63 63 static hxge_status_t hxge_rxdma_hw_start(p_hxge_t hxgep);
64 64 static void hxge_rxdma_hw_stop(p_hxge_t hxgep);
65 65 static hxge_status_t hxge_map_rxdma_channel(p_hxge_t hxgep, uint16_t channel,
66 66 p_hxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p,
67 67 uint32_t num_chunks, p_hxge_dma_common_t *dma_rbr_cntl_p,
68 68 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p,
69 69 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p);
70 70 static void hxge_unmap_rxdma_channel(p_hxge_t hxgep, uint16_t channel,
71 71 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p);
72 72 static hxge_status_t hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep,
73 73 uint16_t dma_channel, p_hxge_dma_common_t *dma_rbr_cntl_p,
74 74 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p,
75 75 p_rx_rbr_ring_t *rbr_p, p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p);
76 76 static void hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep,
77 77 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p);
78 78 static hxge_status_t hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep,
79 79 uint16_t channel, p_hxge_dma_common_t *dma_buf_p,
80 80 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks);
81 81 static void hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep,
82 82 p_rx_rbr_ring_t rbr_p);
83 83 static hxge_status_t hxge_rxdma_start_channel(p_hxge_t hxgep, uint16_t channel,
84 84 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p,
85 85 int n_init_kick);
86 86 static hxge_status_t hxge_rxdma_stop_channel(p_hxge_t hxgep, uint16_t channel);
87 87 static mblk_t *hxge_rx_pkts(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp,
88 88 p_rx_rcr_ring_t rcr_p, rdc_stat_t cs, int bytes_to_read);
89 89 static uint32_t hxge_scan_for_last_eop(p_rx_rcr_ring_t rcr_p,
90 90 p_rcr_entry_t rcr_desc_rd_head_p, uint32_t num_rcrs);
91 91 static void hxge_receive_packet(p_hxge_t hxgep, p_rx_rcr_ring_t rcr_p,
92 92 p_rcr_entry_t rcr_desc_rd_head_p, boolean_t *multi_p,
93 93 mblk_t ** mp, mblk_t ** mp_cont, uint32_t *invalid_rcr_entry);
94 94 static hxge_status_t hxge_disable_rxdma_channel(p_hxge_t hxgep,
95 95 uint16_t channel);
96 96 static p_rx_msg_t hxge_allocb(size_t, uint32_t, p_hxge_dma_common_t);
97 97 static void hxge_freeb(p_rx_msg_t);
98 98 static hxge_status_t hxge_rx_err_evnts(p_hxge_t hxgep, uint_t index,
99 99 p_hxge_ldv_t ldvp, rdc_stat_t cs);
100 100 static hxge_status_t hxge_rxbuf_index_info_init(p_hxge_t hxgep,
101 101 p_rx_rbr_ring_t rx_dmap);
102 102 static hxge_status_t hxge_rxdma_fatal_err_recover(p_hxge_t hxgep,
103 103 uint16_t channel);
104 104 static hxge_status_t hxge_rx_port_fatal_err_recover(p_hxge_t hxgep);
105 105 static void hxge_rbr_empty_restore(p_hxge_t hxgep,
106 106 p_rx_rbr_ring_t rx_rbr_p);
107 107
108 108 hxge_status_t
109 109 hxge_init_rxdma_channels(p_hxge_t hxgep)
110 110 {
111 111 hxge_status_t status = HXGE_OK;
112 112 block_reset_t reset_reg;
113 113 int i;
114 114
115 115 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_init_rxdma_channels"));
116 116
117 117 for (i = 0; i < HXGE_MAX_RDCS; i++)
118 118 hxgep->rdc_first_intr[i] = B_TRUE;
119 119
120 120 /* Reset RDC block from PEU to clear any previous state */
121 121 reset_reg.value = 0;
122 122 reset_reg.bits.rdc_rst = 1;
123 123 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value);
124 124 HXGE_DELAY(1000);
125 125
126 126 status = hxge_map_rxdma(hxgep);
127 127 if (status != HXGE_OK) {
128 128 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
129 129 "<== hxge_init_rxdma: status 0x%x", status));
130 130 return (status);
131 131 }
132 132
133 133 status = hxge_rxdma_hw_start_common(hxgep);
134 134 if (status != HXGE_OK) {
135 135 hxge_unmap_rxdma(hxgep);
136 136 }
137 137
138 138 status = hxge_rxdma_hw_start(hxgep);
139 139 if (status != HXGE_OK) {
140 140 hxge_unmap_rxdma(hxgep);
141 141 }
142 142
143 143 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
144 144 "<== hxge_init_rxdma_channels: status 0x%x", status));
145 145 return (status);
146 146 }
147 147
148 148 void
149 149 hxge_uninit_rxdma_channels(p_hxge_t hxgep)
150 150 {
151 151 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_uninit_rxdma_channels"));
152 152
153 153 hxge_rxdma_hw_stop(hxgep);
154 154 hxge_unmap_rxdma(hxgep);
155 155
156 156 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_uinit_rxdma_channels"));
157 157 }
158 158
159 159 hxge_status_t
160 160 hxge_init_rxdma_channel_cntl_stat(p_hxge_t hxgep, uint16_t channel,
161 161 rdc_stat_t *cs_p)
162 162 {
163 163 hpi_handle_t handle;
164 164 hpi_status_t rs = HPI_SUCCESS;
165 165 hxge_status_t status = HXGE_OK;
166 166
167 167 HXGE_DEBUG_MSG((hxgep, DMA_CTL,
168 168 "<== hxge_init_rxdma_channel_cntl_stat"));
169 169
170 170 handle = HXGE_DEV_HPI_HANDLE(hxgep);
171 171 rs = hpi_rxdma_control_status(handle, OP_SET, channel, cs_p);
172 172
173 173 if (rs != HPI_SUCCESS) {
174 174 status = HXGE_ERROR | rs;
175 175 }
176 176 return (status);
177 177 }
178 178
179 179
180 180 hxge_status_t
181 181 hxge_enable_rxdma_channel(p_hxge_t hxgep, uint16_t channel,
182 182 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p,
183 183 int n_init_kick)
184 184 {
185 185 hpi_handle_t handle;
186 186 rdc_desc_cfg_t rdc_desc;
187 187 rdc_rcr_cfg_b_t *cfgb_p;
188 188 hpi_status_t rs = HPI_SUCCESS;
189 189
190 190 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel"));
191 191 handle = HXGE_DEV_HPI_HANDLE(hxgep);
192 192
193 193 /*
194 194 * Use configuration data composed at init time. Write to hardware the
195 195 * receive ring configurations.
196 196 */
197 197 rdc_desc.mbox_enable = 1;
198 198 rdc_desc.mbox_addr = mbox_p->mbox_addr;
199 199 HXGE_DEBUG_MSG((hxgep, RX_CTL,
200 200 "==> hxge_enable_rxdma_channel: mboxp $%p($%p)",
201 201 mbox_p->mbox_addr, rdc_desc.mbox_addr));
202 202
203 203 rdc_desc.rbr_len = rbr_p->rbb_max;
204 204 rdc_desc.rbr_addr = rbr_p->rbr_addr;
205 205
206 206 switch (hxgep->rx_bksize_code) {
207 207 case RBR_BKSIZE_4K:
208 208 rdc_desc.page_size = SIZE_4KB;
209 209 break;
210 210 case RBR_BKSIZE_8K:
211 211 rdc_desc.page_size = SIZE_8KB;
212 212 break;
213 213 }
214 214
215 215 rdc_desc.size0 = rbr_p->hpi_pkt_buf_size0;
216 216 rdc_desc.valid0 = 1;
217 217
218 218 rdc_desc.size1 = rbr_p->hpi_pkt_buf_size1;
219 219 rdc_desc.valid1 = 1;
220 220
221 221 rdc_desc.size2 = rbr_p->hpi_pkt_buf_size2;
222 222 rdc_desc.valid2 = 1;
223 223
224 224 rdc_desc.full_hdr = rcr_p->full_hdr_flag;
225 225 rdc_desc.offset = rcr_p->sw_priv_hdr_len;
226 226
227 227 rdc_desc.rcr_len = rcr_p->comp_size;
228 228 rdc_desc.rcr_addr = rcr_p->rcr_addr;
229 229
230 230 cfgb_p = &(rcr_p->rcr_cfgb);
231 231 rdc_desc.rcr_threshold = cfgb_p->bits.pthres;
232 232 rdc_desc.rcr_timeout = cfgb_p->bits.timeout;
233 233 rdc_desc.rcr_timeout_enable = cfgb_p->bits.entout;
234 234
235 235 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel: "
236 236 "rbr_len qlen %d pagesize code %d rcr_len %d",
237 237 rdc_desc.rbr_len, rdc_desc.page_size, rdc_desc.rcr_len));
238 238 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel: "
239 239 "size 0 %d size 1 %d size 2 %d",
240 240 rbr_p->hpi_pkt_buf_size0, rbr_p->hpi_pkt_buf_size1,
241 241 rbr_p->hpi_pkt_buf_size2));
242 242
243 243 rs = hpi_rxdma_cfg_rdc_ring(handle, rbr_p->rdc, &rdc_desc);
244 244 if (rs != HPI_SUCCESS) {
245 245 return (HXGE_ERROR | rs);
246 246 }
247 247
248 248 /*
249 249 * Enable the timeout and threshold.
250 250 */
251 251 rs = hpi_rxdma_cfg_rdc_rcr_threshold(handle, channel,
252 252 rdc_desc.rcr_threshold);
253 253 if (rs != HPI_SUCCESS) {
254 254 return (HXGE_ERROR | rs);
255 255 }
256 256
257 257 rs = hpi_rxdma_cfg_rdc_rcr_timeout(handle, channel,
258 258 rdc_desc.rcr_timeout);
259 259 if (rs != HPI_SUCCESS) {
260 260 return (HXGE_ERROR | rs);
261 261 }
262 262
263 263 /* Kick the DMA engine */
264 264 hpi_rxdma_rdc_rbr_kick(handle, channel, n_init_kick);
265 265
266 266 /* Clear the rbr empty bit */
267 267 (void) hpi_rxdma_channel_rbr_empty_clear(handle, channel);
268 268
269 269 /*
270 270 * Enable the DMA
271 271 */
272 272 rs = hpi_rxdma_cfg_rdc_enable(handle, channel);
273 273 if (rs != HPI_SUCCESS) {
274 274 return (HXGE_ERROR | rs);
275 275 }
276 276
277 277 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_enable_rxdma_channel"));
278 278
279 279 return (HXGE_OK);
280 280 }
281 281
282 282 static hxge_status_t
283 283 hxge_disable_rxdma_channel(p_hxge_t hxgep, uint16_t channel)
284 284 {
285 285 hpi_handle_t handle;
286 286 hpi_status_t rs = HPI_SUCCESS;
287 287
288 288 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_disable_rxdma_channel"));
289 289
290 290 handle = HXGE_DEV_HPI_HANDLE(hxgep);
291 291
292 292 /* disable the DMA */
293 293 rs = hpi_rxdma_cfg_rdc_disable(handle, channel);
294 294 if (rs != HPI_SUCCESS) {
295 295 HXGE_DEBUG_MSG((hxgep, RX_CTL,
296 296 "<== hxge_disable_rxdma_channel:failed (0x%x)", rs));
297 297 return (HXGE_ERROR | rs);
298 298 }
299 299 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_disable_rxdma_channel"));
300 300 return (HXGE_OK);
301 301 }
302 302
303 303 hxge_status_t
304 304 hxge_rxdma_channel_rcrflush(p_hxge_t hxgep, uint8_t channel)
305 305 {
306 306 hpi_handle_t handle;
307 307 hxge_status_t status = HXGE_OK;
308 308
309 309 HXGE_DEBUG_MSG((hxgep, DMA_CTL,
310 310 "==> hxge_rxdma_channel_rcrflush"));
311 311
312 312 handle = HXGE_DEV_HPI_HANDLE(hxgep);
313 313 hpi_rxdma_rdc_rcr_flush(handle, channel);
314 314
315 315 HXGE_DEBUG_MSG((hxgep, DMA_CTL,
316 316 "<== hxge_rxdma_channel_rcrflush"));
317 317 return (status);
318 318
319 319 }
320 320
321 321 #define MID_INDEX(l, r) ((r + l + 1) >> 1)
322 322
323 323 #define TO_LEFT -1
324 324 #define TO_RIGHT 1
325 325 #define BOTH_RIGHT (TO_RIGHT + TO_RIGHT)
326 326 #define BOTH_LEFT (TO_LEFT + TO_LEFT)
327 327 #define IN_MIDDLE (TO_RIGHT + TO_LEFT)
328 328 #define NO_HINT 0xffffffff
329 329
330 330 /*ARGSUSED*/
331 331 hxge_status_t
332 332 hxge_rxbuf_pp_to_vp(p_hxge_t hxgep, p_rx_rbr_ring_t rbr_p,
333 333 uint8_t pktbufsz_type, uint64_t *pkt_buf_addr_pp,
334 334 uint64_t **pkt_buf_addr_p, uint32_t *bufoffset, uint32_t *msg_index)
335 335 {
336 336 int bufsize;
337 337 uint64_t pktbuf_pp;
338 338 uint64_t dvma_addr;
339 339 rxring_info_t *ring_info;
340 340 int base_side, end_side;
341 341 int r_index, l_index, anchor_index;
342 342 int found, search_done;
343 343 uint32_t offset, chunk_size, block_size, page_size_mask;
344 344 uint32_t chunk_index, block_index, total_index;
345 345 int max_iterations, iteration;
346 346 rxbuf_index_info_t *bufinfo;
347 347
348 348 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> hxge_rxbuf_pp_to_vp"));
349 349
350 350 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
351 351 "==> hxge_rxbuf_pp_to_vp: buf_pp $%p btype %d",
352 352 pkt_buf_addr_pp, pktbufsz_type));
353 353
354 354 #if defined(__i386)
355 355 pktbuf_pp = (uint64_t)(uint32_t)pkt_buf_addr_pp;
356 356 #else
357 357 pktbuf_pp = (uint64_t)pkt_buf_addr_pp;
358 358 #endif
359 359
360 360 switch (pktbufsz_type) {
361 361 case 0:
362 362 bufsize = rbr_p->pkt_buf_size0;
363 363 break;
364 364 case 1:
365 365 bufsize = rbr_p->pkt_buf_size1;
366 366 break;
367 367 case 2:
368 368 bufsize = rbr_p->pkt_buf_size2;
369 369 break;
370 370 case RCR_SINGLE_BLOCK:
371 371 bufsize = 0;
372 372 anchor_index = 0;
373 373 break;
374 374 default:
375 375 return (HXGE_ERROR);
376 376 }
377 377
378 378 if (rbr_p->num_blocks == 1) {
379 379 anchor_index = 0;
380 380 ring_info = rbr_p->ring_info;
381 381 bufinfo = (rxbuf_index_info_t *)ring_info->buffer;
382 382
383 383 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
384 384 "==> hxge_rxbuf_pp_to_vp: (found, 1 block) "
385 385 "buf_pp $%p btype %d anchor_index %d bufinfo $%p",
386 386 pkt_buf_addr_pp, pktbufsz_type, anchor_index, bufinfo));
387 387
388 388 goto found_index;
389 389 }
390 390
391 391 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
392 392 "==> hxge_rxbuf_pp_to_vp: buf_pp $%p btype %d anchor_index %d",
393 393 pkt_buf_addr_pp, pktbufsz_type, anchor_index));
394 394
395 395 ring_info = rbr_p->ring_info;
396 396 found = B_FALSE;
397 397 bufinfo = (rxbuf_index_info_t *)ring_info->buffer;
398 398 iteration = 0;
399 399 max_iterations = ring_info->max_iterations;
400 400
401 401 /*
402 402 * First check if this block have been seen recently. This is indicated
403 403 * by a hint which is initialized when the first buffer of the block is
404 404 * seen. The hint is reset when the last buffer of the block has been
405 405 * processed. As three block sizes are supported, three hints are kept.
406 406 * The idea behind the hints is that once the hardware uses a block
407 407 * for a buffer of that size, it will use it exclusively for that size
408 408 * and will use it until it is exhausted. It is assumed that there
409 409 * would a single block being used for the same buffer sizes at any
410 410 * given time.
411 411 */
412 412 if (ring_info->hint[pktbufsz_type] != NO_HINT) {
413 413 anchor_index = ring_info->hint[pktbufsz_type];
414 414 dvma_addr = bufinfo[anchor_index].dvma_addr;
415 415 chunk_size = bufinfo[anchor_index].buf_size;
416 416 if ((pktbuf_pp >= dvma_addr) &&
417 417 (pktbuf_pp < (dvma_addr + chunk_size))) {
418 418 found = B_TRUE;
419 419 /*
420 420 * check if this is the last buffer in the block If so,
421 421 * then reset the hint for the size;
422 422 */
423 423
424 424 if ((pktbuf_pp + bufsize) >= (dvma_addr + chunk_size))
425 425 ring_info->hint[pktbufsz_type] = NO_HINT;
426 426 }
427 427 }
428 428
429 429 if (found == B_FALSE) {
430 430 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
431 431 "==> hxge_rxbuf_pp_to_vp: (!found)"
432 432 "buf_pp $%p btype %d anchor_index %d",
433 433 pkt_buf_addr_pp, pktbufsz_type, anchor_index));
434 434
435 435 /*
436 436 * This is the first buffer of the block of this size. Need to
437 437 * search the whole information array. the search algorithm
438 438 * uses a binary tree search algorithm. It assumes that the
439 439 * information is already sorted with increasing order info[0]
440 440 * < info[1] < info[2] .... < info[n-1] where n is the size of
441 441 * the information array
442 442 */
443 443 r_index = rbr_p->num_blocks - 1;
444 444 l_index = 0;
445 445 search_done = B_FALSE;
446 446 anchor_index = MID_INDEX(r_index, l_index);
447 447 while (search_done == B_FALSE) {
448 448 if ((r_index == l_index) ||
449 449 (iteration >= max_iterations))
450 450 search_done = B_TRUE;
451 451
452 452 end_side = TO_RIGHT; /* to the right */
453 453 base_side = TO_LEFT; /* to the left */
454 454 /* read the DVMA address information and sort it */
455 455 dvma_addr = bufinfo[anchor_index].dvma_addr;
456 456 chunk_size = bufinfo[anchor_index].buf_size;
457 457
458 458 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
459 459 "==> hxge_rxbuf_pp_to_vp: (searching)"
460 460 "buf_pp $%p btype %d "
461 461 "anchor_index %d chunk_size %d dvmaaddr $%p",
462 462 pkt_buf_addr_pp, pktbufsz_type, anchor_index,
463 463 chunk_size, dvma_addr));
464 464
465 465 if (pktbuf_pp >= dvma_addr)
466 466 base_side = TO_RIGHT; /* to the right */
467 467 if (pktbuf_pp < (dvma_addr + chunk_size))
468 468 end_side = TO_LEFT; /* to the left */
469 469
470 470 switch (base_side + end_side) {
471 471 case IN_MIDDLE:
472 472 /* found */
473 473 found = B_TRUE;
474 474 search_done = B_TRUE;
475 475 if ((pktbuf_pp + bufsize) <
476 476 (dvma_addr + chunk_size))
477 477 ring_info->hint[pktbufsz_type] =
478 478 bufinfo[anchor_index].buf_index;
479 479 break;
480 480 case BOTH_RIGHT:
481 481 /* not found: go to the right */
482 482 l_index = anchor_index + 1;
483 483 anchor_index = MID_INDEX(r_index, l_index);
484 484 break;
485 485
486 486 case BOTH_LEFT:
487 487 /* not found: go to the left */
488 488 r_index = anchor_index - 1;
489 489 anchor_index = MID_INDEX(r_index, l_index);
490 490 break;
491 491 default: /* should not come here */
492 492 return (HXGE_ERROR);
493 493 }
494 494 iteration++;
495 495 }
496 496
497 497 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
498 498 "==> hxge_rxbuf_pp_to_vp: (search done)"
499 499 "buf_pp $%p btype %d anchor_index %d",
500 500 pkt_buf_addr_pp, pktbufsz_type, anchor_index));
501 501 }
502 502
503 503 if (found == B_FALSE) {
504 504 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
505 505 "==> hxge_rxbuf_pp_to_vp: (search failed)"
506 506 "buf_pp $%p btype %d anchor_index %d",
507 507 pkt_buf_addr_pp, pktbufsz_type, anchor_index));
508 508 return (HXGE_ERROR);
509 509 }
510 510
511 511 found_index:
512 512 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
513 513 "==> hxge_rxbuf_pp_to_vp: (FOUND1)"
514 514 "buf_pp $%p btype %d bufsize %d anchor_index %d",
515 515 pkt_buf_addr_pp, pktbufsz_type, bufsize, anchor_index));
516 516
517 517 /* index of the first block in this chunk */
518 518 chunk_index = bufinfo[anchor_index].start_index;
519 519 dvma_addr = bufinfo[anchor_index].dvma_addr;
520 520 page_size_mask = ring_info->block_size_mask;
521 521
522 522 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
523 523 "==> hxge_rxbuf_pp_to_vp: (FOUND3), get chunk)"
524 524 "buf_pp $%p btype %d bufsize %d "
525 525 "anchor_index %d chunk_index %d dvma $%p",
526 526 pkt_buf_addr_pp, pktbufsz_type, bufsize,
527 527 anchor_index, chunk_index, dvma_addr));
528 528
529 529 offset = pktbuf_pp - dvma_addr; /* offset within the chunk */
530 530 block_size = rbr_p->block_size; /* System block(page) size */
531 531
532 532 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
533 533 "==> hxge_rxbuf_pp_to_vp: (FOUND4), get chunk)"
534 534 "buf_pp $%p btype %d bufsize %d "
535 535 "anchor_index %d chunk_index %d dvma $%p "
536 536 "offset %d block_size %d",
537 537 pkt_buf_addr_pp, pktbufsz_type, bufsize, anchor_index,
538 538 chunk_index, dvma_addr, offset, block_size));
539 539 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> getting total index"));
540 540
541 541 block_index = (offset / block_size); /* index within chunk */
542 542 total_index = chunk_index + block_index;
543 543
544 544 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
545 545 "==> hxge_rxbuf_pp_to_vp: "
546 546 "total_index %d dvma_addr $%p "
547 547 "offset %d block_size %d "
548 548 "block_index %d ",
549 549 total_index, dvma_addr, offset, block_size, block_index));
550 550
551 551 #if defined(__i386)
552 552 *pkt_buf_addr_p = (uint64_t *)((uint32_t)bufinfo[anchor_index].kaddr +
553 553 (uint32_t)offset);
554 554 #else
555 555 *pkt_buf_addr_p = (uint64_t *)((uint64_t)bufinfo[anchor_index].kaddr +
556 556 offset);
557 557 #endif
558 558
559 559 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
560 560 "==> hxge_rxbuf_pp_to_vp: "
561 561 "total_index %d dvma_addr $%p "
562 562 "offset %d block_size %d "
563 563 "block_index %d "
564 564 "*pkt_buf_addr_p $%p",
565 565 total_index, dvma_addr, offset, block_size,
566 566 block_index, *pkt_buf_addr_p));
567 567
568 568 *msg_index = total_index;
569 569 *bufoffset = (offset & page_size_mask);
570 570
571 571 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
572 572 "==> hxge_rxbuf_pp_to_vp: get msg index: "
573 573 "msg_index %d bufoffset_index %d",
574 574 *msg_index, *bufoffset));
575 575 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "<== hxge_rxbuf_pp_to_vp"));
576 576
577 577 return (HXGE_OK);
578 578 }
579 579
580 580
581 581 /*
582 582 * used by quick sort (qsort) function
583 583 * to perform comparison
584 584 */
585 585 static int
586 586 hxge_sort_compare(const void *p1, const void *p2)
587 587 {
588 588
589 589 rxbuf_index_info_t *a, *b;
590 590
591 591 a = (rxbuf_index_info_t *)p1;
592 592 b = (rxbuf_index_info_t *)p2;
593 593
594 594 if (a->dvma_addr > b->dvma_addr)
595 595 return (1);
596 596 if (a->dvma_addr < b->dvma_addr)
597 597 return (-1);
598 598 return (0);
599 599 }
600 600
601 601 /*
602 602 * Grabbed this sort implementation from common/syscall/avl.c
603 603 *
604 604 * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified.
605 605 * v = Ptr to array/vector of objs
606 606 * n = # objs in the array
607 607 * s = size of each obj (must be multiples of a word size)
608 608 * f = ptr to function to compare two objs
609 609 * returns (-1 = less than, 0 = equal, 1 = greater than
610 610 */
611 611 void
612 612 hxge_ksort(caddr_t v, int n, int s, int (*f) ())
613 613 {
614 614 int g, i, j, ii;
615 615 unsigned int *p1, *p2;
616 616 unsigned int tmp;
617 617
618 618 /* No work to do */
619 619 if (v == NULL || n <= 1)
620 620 return;
621 621 /* Sanity check on arguments */
622 622 ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0);
623 623 ASSERT(s > 0);
624 624
625 625 for (g = n / 2; g > 0; g /= 2) {
626 626 for (i = g; i < n; i++) {
627 627 for (j = i - g; j >= 0 &&
628 628 (*f) (v + j * s, v + (j + g) * s) == 1; j -= g) {
629 629 p1 = (unsigned *)(v + j * s);
630 630 p2 = (unsigned *)(v + (j + g) * s);
631 631 for (ii = 0; ii < s / 4; ii++) {
632 632 tmp = *p1;
633 633 *p1++ = *p2;
634 634 *p2++ = tmp;
635 635 }
636 636 }
637 637 }
638 638 }
639 639 }
640 640
641 641 /*
642 642 * Initialize data structures required for rxdma
643 643 * buffer dvma->vmem address lookup
644 644 */
645 645 /*ARGSUSED*/
646 646 static hxge_status_t
647 647 hxge_rxbuf_index_info_init(p_hxge_t hxgep, p_rx_rbr_ring_t rbrp)
648 648 {
649 649 int index;
650 650 rxring_info_t *ring_info;
651 651 int max_iteration = 0, max_index = 0;
652 652
653 653 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_rxbuf_index_info_init"));
654 654
655 655 ring_info = rbrp->ring_info;
656 656 ring_info->hint[0] = NO_HINT;
657 657 ring_info->hint[1] = NO_HINT;
658 658 ring_info->hint[2] = NO_HINT;
659 659 ring_info->hint[3] = NO_HINT;
660 660 max_index = rbrp->num_blocks;
661 661
662 662 /* read the DVMA address information and sort it */
663 663 /* do init of the information array */
664 664
665 665 HXGE_DEBUG_MSG((hxgep, DMA2_CTL,
666 666 " hxge_rxbuf_index_info_init Sort ptrs"));
667 667
668 668 /* sort the array */
669 669 hxge_ksort((void *) ring_info->buffer, max_index,
670 670 sizeof (rxbuf_index_info_t), hxge_sort_compare);
671 671
672 672 for (index = 0; index < max_index; index++) {
673 673 HXGE_DEBUG_MSG((hxgep, DMA2_CTL,
674 674 " hxge_rxbuf_index_info_init: sorted chunk %d "
675 675 " ioaddr $%p kaddr $%p size %x",
676 676 index, ring_info->buffer[index].dvma_addr,
677 677 ring_info->buffer[index].kaddr,
678 678 ring_info->buffer[index].buf_size));
679 679 }
680 680
681 681 max_iteration = 0;
682 682 while (max_index >= (1ULL << max_iteration))
683 683 max_iteration++;
684 684 ring_info->max_iterations = max_iteration + 1;
685 685
686 686 HXGE_DEBUG_MSG((hxgep, DMA2_CTL,
687 687 " hxge_rxbuf_index_info_init Find max iter %d",
688 688 ring_info->max_iterations));
689 689 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_rxbuf_index_info_init"));
690 690
691 691 return (HXGE_OK);
692 692 }
693 693
694 694 /*ARGSUSED*/
695 695 void
696 696 hxge_dump_rcr_entry(p_hxge_t hxgep, p_rcr_entry_t entry_p)
697 697 {
698 698 #ifdef HXGE_DEBUG
699 699
700 700 uint32_t bptr;
701 701 uint64_t pp;
702 702
703 703 bptr = entry_p->bits.pkt_buf_addr;
704 704
705 705 HXGE_DEBUG_MSG((hxgep, RX_CTL,
706 706 "\trcr entry $%p "
707 707 "\trcr entry 0x%0llx "
708 708 "\trcr entry 0x%08x "
709 709 "\trcr entry 0x%08x "
710 710 "\tvalue 0x%0llx\n"
711 711 "\tmulti = %d\n"
712 712 "\tpkt_type = 0x%x\n"
713 713 "\terror = 0x%04x\n"
714 714 "\tl2_len = %d\n"
715 715 "\tpktbufsize = %d\n"
716 716 "\tpkt_buf_addr = $%p\n"
717 717 "\tpkt_buf_addr (<< 6) = $%p\n",
718 718 entry_p,
719 719 *(int64_t *)entry_p,
720 720 *(int32_t *)entry_p,
721 721 *(int32_t *)((char *)entry_p + 32),
722 722 entry_p->value,
723 723 entry_p->bits.multi,
724 724 entry_p->bits.pkt_type,
725 725 entry_p->bits.error,
726 726 entry_p->bits.l2_len,
727 727 entry_p->bits.pktbufsz,
728 728 bptr,
729 729 entry_p->bits.pkt_buf_addr_l));
730 730
731 731 pp = (entry_p->value & RCR_PKT_BUF_ADDR_MASK) <<
732 732 RCR_PKT_BUF_ADDR_SHIFT;
733 733
734 734 HXGE_DEBUG_MSG((hxgep, RX_CTL, "rcr pp 0x%llx l2 len %d",
735 735 pp, (*(int64_t *)entry_p >> 40) & 0x3fff));
736 736 #endif
737 737 }
738 738
739 739 /*ARGSUSED*/
740 740 void
741 741 hxge_rxdma_stop(p_hxge_t hxgep)
742 742 {
743 743 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop"));
744 744
745 745 MUTEX_ENTER(&hxgep->vmac_lock);
746 746 (void) hxge_rx_vmac_disable(hxgep);
747 747 (void) hxge_rxdma_hw_mode(hxgep, HXGE_DMA_STOP);
748 748 MUTEX_EXIT(&hxgep->vmac_lock);
749 749
750 750 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop"));
751 751 }
752 752
753 753 void
754 754 hxge_rxdma_stop_reinit(p_hxge_t hxgep)
755 755 {
756 756 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_reinit"));
757 757
758 758 (void) hxge_rxdma_stop(hxgep);
759 759 (void) hxge_uninit_rxdma_channels(hxgep);
760 760 (void) hxge_init_rxdma_channels(hxgep);
761 761
762 762 MUTEX_ENTER(&hxgep->vmac_lock);
763 763 (void) hxge_rx_vmac_enable(hxgep);
764 764 MUTEX_EXIT(&hxgep->vmac_lock);
765 765
766 766 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop_reinit"));
767 767 }
768 768
769 769 hxge_status_t
770 770 hxge_rxdma_hw_mode(p_hxge_t hxgep, boolean_t enable)
771 771 {
772 772 int i, ndmas;
773 773 uint16_t channel;
774 774 p_rx_rbr_rings_t rx_rbr_rings;
775 775 p_rx_rbr_ring_t *rbr_rings;
776 776 hpi_handle_t handle;
777 777 hpi_status_t rs = HPI_SUCCESS;
778 778 hxge_status_t status = HXGE_OK;
779 779
780 780 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
781 781 "==> hxge_rxdma_hw_mode: mode %d", enable));
782 782
783 783 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) {
784 784 HXGE_DEBUG_MSG((hxgep, RX_CTL,
785 785 "<== hxge_rxdma_mode: not initialized"));
786 786 return (HXGE_ERROR);
787 787 }
788 788
789 789 rx_rbr_rings = hxgep->rx_rbr_rings;
790 790 if (rx_rbr_rings == NULL) {
791 791 HXGE_DEBUG_MSG((hxgep, RX_CTL,
792 792 "<== hxge_rxdma_mode: NULL ring pointer"));
793 793 return (HXGE_ERROR);
794 794 }
795 795
796 796 if (rx_rbr_rings->rbr_rings == NULL) {
797 797 HXGE_DEBUG_MSG((hxgep, RX_CTL,
798 798 "<== hxge_rxdma_mode: NULL rbr rings pointer"));
799 799 return (HXGE_ERROR);
800 800 }
801 801
802 802 ndmas = rx_rbr_rings->ndmas;
803 803 if (!ndmas) {
804 804 HXGE_DEBUG_MSG((hxgep, RX_CTL,
805 805 "<== hxge_rxdma_mode: no channel"));
806 806 return (HXGE_ERROR);
807 807 }
808 808
809 809 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
810 810 "==> hxge_rxdma_mode (ndmas %d)", ndmas));
811 811
812 812 rbr_rings = rx_rbr_rings->rbr_rings;
813 813
814 814 handle = HXGE_DEV_HPI_HANDLE(hxgep);
815 815
816 816 for (i = 0; i < ndmas; i++) {
817 817 if (rbr_rings == NULL || rbr_rings[i] == NULL) {
818 818 continue;
819 819 }
820 820 channel = rbr_rings[i]->rdc;
821 821 if (enable) {
822 822 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
823 823 "==> hxge_rxdma_hw_mode: channel %d (enable)",
824 824 channel));
825 825 rs = hpi_rxdma_cfg_rdc_enable(handle, channel);
826 826 } else {
827 827 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
828 828 "==> hxge_rxdma_hw_mode: channel %d (disable)",
829 829 channel));
830 830 rs = hpi_rxdma_cfg_rdc_disable(handle, channel);
831 831 }
832 832 }
833 833
834 834 status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs);
835 835 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
836 836 "<== hxge_rxdma_hw_mode: status 0x%x", status));
837 837
838 838 return (status);
839 839 }
840 840
841 841 /*
842 842 * Static functions start here.
843 843 */
844 844 static p_rx_msg_t
845 845 hxge_allocb(size_t size, uint32_t pri, p_hxge_dma_common_t dmabuf_p)
846 846 {
847 847 p_rx_msg_t hxge_mp = NULL;
848 848 p_hxge_dma_common_t dmamsg_p;
849 849 uchar_t *buffer;
850 850
851 851 hxge_mp = KMEM_ZALLOC(sizeof (rx_msg_t), KM_NOSLEEP);
852 852 if (hxge_mp == NULL) {
853 853 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL,
854 854 "Allocation of a rx msg failed."));
855 855 goto hxge_allocb_exit;
856 856 }
857 857
858 858 hxge_mp->use_buf_pool = B_FALSE;
859 859 if (dmabuf_p) {
860 860 hxge_mp->use_buf_pool = B_TRUE;
861 861
862 862 dmamsg_p = (p_hxge_dma_common_t)&hxge_mp->buf_dma;
863 863 *dmamsg_p = *dmabuf_p;
864 864 dmamsg_p->nblocks = 1;
865 865 dmamsg_p->block_size = size;
866 866 dmamsg_p->alength = size;
867 867 buffer = (uchar_t *)dmabuf_p->kaddrp;
868 868
869 869 dmabuf_p->kaddrp = (void *)((char *)dmabuf_p->kaddrp + size);
870 870 dmabuf_p->ioaddr_pp = (void *)
871 871 ((char *)dmabuf_p->ioaddr_pp + size);
872 872
873 873 dmabuf_p->alength -= size;
874 874 dmabuf_p->offset += size;
875 875 dmabuf_p->dma_cookie.dmac_laddress += size;
876 876 dmabuf_p->dma_cookie.dmac_size -= size;
877 877 } else {
878 878 buffer = KMEM_ALLOC(size, KM_NOSLEEP);
879 879 if (buffer == NULL) {
880 880 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL,
881 881 "Allocation of a receive page failed."));
882 882 goto hxge_allocb_fail1;
883 883 }
884 884 }
885 885
886 886 hxge_mp->rx_mblk_p = desballoc(buffer, size, pri, &hxge_mp->freeb);
887 887 if (hxge_mp->rx_mblk_p == NULL) {
888 888 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, "desballoc failed."));
889 889 goto hxge_allocb_fail2;
890 890 }
891 891 hxge_mp->buffer = buffer;
892 892 hxge_mp->block_size = size;
893 893 hxge_mp->freeb.free_func = (void (*) ()) hxge_freeb;
894 894 hxge_mp->freeb.free_arg = (caddr_t)hxge_mp;
895 895 hxge_mp->ref_cnt = 1;
896 896 hxge_mp->free = B_TRUE;
897 897 hxge_mp->rx_use_bcopy = B_FALSE;
898 898
899 899 atomic_inc_32(&hxge_mblks_pending);
900 900
901 901 goto hxge_allocb_exit;
902 902
903 903 hxge_allocb_fail2:
904 904 if (!hxge_mp->use_buf_pool) {
905 905 KMEM_FREE(buffer, size);
906 906 }
907 907 hxge_allocb_fail1:
908 908 KMEM_FREE(hxge_mp, sizeof (rx_msg_t));
909 909 hxge_mp = NULL;
910 910
911 911 hxge_allocb_exit:
912 912 return (hxge_mp);
913 913 }
914 914
915 915 p_mblk_t
916 916 hxge_dupb(p_rx_msg_t hxge_mp, uint_t offset, size_t size)
917 917 {
918 918 p_mblk_t mp;
919 919
920 920 HXGE_DEBUG_MSG((NULL, MEM_CTL, "==> hxge_dupb"));
921 921 HXGE_DEBUG_MSG((NULL, MEM_CTL, "hxge_mp = $%p "
922 922 "offset = 0x%08X " "size = 0x%08X", hxge_mp, offset, size));
923 923
924 924 mp = desballoc(&hxge_mp->buffer[offset], size, 0, &hxge_mp->freeb);
925 925 if (mp == NULL) {
926 926 HXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed"));
927 927 goto hxge_dupb_exit;
928 928 }
929 929
930 930 atomic_inc_32(&hxge_mp->ref_cnt);
931 931
932 932 hxge_dupb_exit:
933 933 HXGE_DEBUG_MSG((NULL, MEM_CTL, "<== hxge_dupb mp = $%p", hxge_mp));
934 934 return (mp);
935 935 }
936 936
937 937 p_mblk_t
938 938 hxge_dupb_bcopy(p_rx_msg_t hxge_mp, uint_t offset, size_t size)
939 939 {
940 940 p_mblk_t mp;
941 941 uchar_t *dp;
942 942
943 943 mp = allocb(size + HXGE_RXBUF_EXTRA, 0);
944 944 if (mp == NULL) {
945 945 HXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed"));
946 946 goto hxge_dupb_bcopy_exit;
947 947 }
948 948 dp = mp->b_rptr = mp->b_rptr + HXGE_RXBUF_EXTRA;
949 949 bcopy((void *) &hxge_mp->buffer[offset], dp, size);
950 950 mp->b_wptr = dp + size;
951 951
952 952 hxge_dupb_bcopy_exit:
953 953
954 954 HXGE_DEBUG_MSG((NULL, MEM_CTL, "<== hxge_dupb mp = $%p", hxge_mp));
955 955
956 956 return (mp);
957 957 }
958 958
959 959 void hxge_post_page(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p,
960 960 p_rx_msg_t rx_msg_p);
961 961
962 962 void
963 963 hxge_post_page(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p, p_rx_msg_t rx_msg_p)
964 964 {
965 965 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_post_page"));
966 966
967 967 /* Reuse this buffer */
968 968 rx_msg_p->free = B_FALSE;
969 969 rx_msg_p->cur_usage_cnt = 0;
970 970 rx_msg_p->max_usage_cnt = 0;
971 971 rx_msg_p->pkt_buf_size = 0;
972 972
973 973 if (rx_rbr_p->rbr_use_bcopy) {
974 974 rx_msg_p->rx_use_bcopy = B_FALSE;
975 975 atomic_dec_32(&rx_rbr_p->rbr_consumed);
976 976 }
977 977 atomic_dec_32(&rx_rbr_p->rbr_used);
978 978
979 979 /*
980 980 * Get the rbr header pointer and its offset index.
981 981 */
982 982 rx_rbr_p->rbr_wr_index = ((rx_rbr_p->rbr_wr_index + 1) &
983 983 rx_rbr_p->rbr_wrap_mask);
984 984 rx_rbr_p->rbr_desc_vp[rx_rbr_p->rbr_wr_index] = rx_msg_p->shifted_addr;
985 985
986 986 /*
987 987 * Accumulate some buffers in the ring before re-enabling the
988 988 * DMA channel, if rbr empty was signaled.
989 989 */
990 990 hpi_rxdma_rdc_rbr_kick(HXGE_DEV_HPI_HANDLE(hxgep), rx_rbr_p->rdc, 1);
991 991 if (rx_rbr_p->rbr_is_empty && (rx_rbr_p->rbb_max -
992 992 rx_rbr_p->rbr_used) >= HXGE_RBR_EMPTY_THRESHOLD) {
993 993 hxge_rbr_empty_restore(hxgep, rx_rbr_p);
994 994 }
995 995
996 996 HXGE_DEBUG_MSG((hxgep, RX_CTL,
997 997 "<== hxge_post_page (channel %d post_next_index %d)",
998 998 rx_rbr_p->rdc, rx_rbr_p->rbr_wr_index));
999 999 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_post_page"));
1000 1000 }
1001 1001
1002 1002 void
1003 1003 hxge_freeb(p_rx_msg_t rx_msg_p)
1004 1004 {
1005 1005 size_t size;
1006 1006 uchar_t *buffer = NULL;
1007 1007 int ref_cnt;
1008 1008 boolean_t free_state = B_FALSE;
1009 1009 rx_rbr_ring_t *ring = rx_msg_p->rx_rbr_p;
1010 1010
1011 1011 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "==> hxge_freeb"));
1012 1012 HXGE_DEBUG_MSG((NULL, MEM2_CTL,
1013 1013 "hxge_freeb:rx_msg_p = $%p (block pending %d)",
1014 1014 rx_msg_p, hxge_mblks_pending));
1015 1015
1016 1016 if (ring == NULL)
1017 1017 return;
1018 1018
1019 1019 /*
1020 1020 * This is to prevent posting activities while we are recovering
1021 1021 * from fatal errors. This should not be a performance drag since
1022 1022 * ref_cnt != 0 most times.
1023 1023 */
|
↓ open down ↓ |
1023 lines elided |
↑ open up ↑ |
1024 1024 if (ring->rbr_state == RBR_POSTING)
1025 1025 MUTEX_ENTER(&ring->post_lock);
1026 1026
1027 1027 /*
1028 1028 * First we need to get the free state, then
1029 1029 * atomic decrement the reference count to prevent
1030 1030 * the race condition with the interrupt thread that
1031 1031 * is processing a loaned up buffer block.
1032 1032 */
1033 1033 free_state = rx_msg_p->free;
1034 - ref_cnt = atomic_add_32_nv(&rx_msg_p->ref_cnt, -1);
1034 + ref_cnt = atomic_dec_32_nv(&rx_msg_p->ref_cnt);
1035 1035 if (!ref_cnt) {
1036 1036 atomic_dec_32(&hxge_mblks_pending);
1037 1037
1038 1038 buffer = rx_msg_p->buffer;
1039 1039 size = rx_msg_p->block_size;
1040 1040
1041 1041 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "hxge_freeb: "
1042 1042 "will free: rx_msg_p = $%p (block pending %d)",
1043 1043 rx_msg_p, hxge_mblks_pending));
1044 1044
1045 1045 if (!rx_msg_p->use_buf_pool) {
1046 1046 KMEM_FREE(buffer, size);
1047 1047 }
1048 1048
1049 1049 KMEM_FREE(rx_msg_p, sizeof (rx_msg_t));
1050 1050 /*
1051 1051 * Decrement the receive buffer ring's reference
1052 1052 * count, too.
1053 1053 */
1054 1054 atomic_dec_32(&ring->rbr_ref_cnt);
1055 1055
1056 1056 /*
1057 1057 * Free the receive buffer ring, iff
1058 1058 * 1. all the receive buffers have been freed
1059 1059 * 2. and we are in the proper state (that is,
1060 1060 * we are not UNMAPPING).
1061 1061 */
1062 1062 if (ring->rbr_ref_cnt == 0 &&
1063 1063 ring->rbr_state == RBR_UNMAPPED) {
1064 1064 KMEM_FREE(ring, sizeof (*ring));
1065 1065 /* post_lock has been destroyed already */
1066 1066 return;
1067 1067 }
1068 1068 }
1069 1069
1070 1070 /*
1071 1071 * Repost buffer.
1072 1072 */
1073 1073 if (free_state && (ref_cnt == 1)) {
1074 1074 HXGE_DEBUG_MSG((NULL, RX_CTL,
1075 1075 "hxge_freeb: post page $%p:", rx_msg_p));
1076 1076 if (ring->rbr_state == RBR_POSTING)
1077 1077 hxge_post_page(rx_msg_p->hxgep, ring, rx_msg_p);
1078 1078 }
1079 1079
1080 1080 if (ring->rbr_state == RBR_POSTING)
1081 1081 MUTEX_EXIT(&ring->post_lock);
1082 1082
1083 1083 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "<== hxge_freeb"));
1084 1084 }
1085 1085
1086 1086 uint_t
1087 1087 hxge_rx_intr(caddr_t arg1, caddr_t arg2)
1088 1088 {
1089 1089 p_hxge_ring_handle_t rhp;
1090 1090 p_hxge_ldv_t ldvp = (p_hxge_ldv_t)arg1;
1091 1091 p_hxge_t hxgep = (p_hxge_t)arg2;
1092 1092 p_hxge_ldg_t ldgp;
1093 1093 uint8_t channel;
1094 1094 hpi_handle_t handle;
1095 1095 rdc_stat_t cs;
1096 1096 p_rx_rcr_ring_t ring;
1097 1097 p_rx_rbr_ring_t rbrp;
1098 1098 mblk_t *mp = NULL;
1099 1099
1100 1100 if (ldvp == NULL) {
1101 1101 HXGE_DEBUG_MSG((NULL, RX_INT_CTL,
1102 1102 "<== hxge_rx_intr: arg2 $%p arg1 $%p", hxgep, ldvp));
1103 1103 return (DDI_INTR_UNCLAIMED);
1104 1104 }
1105 1105
1106 1106 if (arg2 == NULL || (void *) ldvp->hxgep != arg2) {
1107 1107 hxgep = ldvp->hxgep;
1108 1108 }
1109 1109
1110 1110 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1111 1111 "==> hxge_rx_intr: arg2 $%p arg1 $%p", hxgep, ldvp));
1112 1112
1113 1113 /*
1114 1114 * This interrupt handler is for a specific receive dma channel.
1115 1115 */
1116 1116 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1117 1117
1118 1118 /*
1119 1119 * Get the control and status for this channel.
1120 1120 */
1121 1121 channel = ldvp->vdma_index;
1122 1122 ring = hxgep->rx_rcr_rings->rcr_rings[channel];
1123 1123 rhp = &hxgep->rx_ring_handles[channel];
1124 1124 ldgp = ldvp->ldgp;
1125 1125
1126 1126 ASSERT(ring != NULL);
1127 1127 #if defined(DEBUG)
1128 1128 if (rhp->started) {
1129 1129 ASSERT(ring->ldgp == ldgp);
1130 1130 ASSERT(ring->ldvp == ldvp);
1131 1131 }
1132 1132 #endif
1133 1133
1134 1134 MUTEX_ENTER(&ring->lock);
1135 1135
1136 1136 if (!ring->poll_flag) {
1137 1137 RXDMA_REG_READ64(handle, RDC_STAT, channel, &cs.value);
1138 1138 cs.bits.ptrread = 0;
1139 1139 cs.bits.pktread = 0;
1140 1140 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value);
1141 1141
1142 1142 /*
1143 1143 * Process packets, if we are not in polling mode, the ring is
1144 1144 * started and the interface is started. The MAC layer under
1145 1145 * load will be operating in polling mode for RX traffic.
1146 1146 */
1147 1147 if ((rhp->started) &&
1148 1148 (hxgep->hxge_mac_state == HXGE_MAC_STARTED)) {
1149 1149 mp = hxge_rx_pkts(hxgep, ldvp->vdma_index,
1150 1150 ldvp, ring, cs, -1);
1151 1151 }
1152 1152
1153 1153 /* Process error events. */
1154 1154 if (cs.value & RDC_STAT_ERROR) {
1155 1155 MUTEX_EXIT(&ring->lock);
1156 1156 (void) hxge_rx_err_evnts(hxgep, channel, ldvp, cs);
1157 1157 MUTEX_ENTER(&ring->lock);
1158 1158 }
1159 1159
1160 1160 /*
1161 1161 * Enable the mailbox update interrupt if we want to use
1162 1162 * mailbox. We probably don't need to use mailbox as it only
1163 1163 * saves us one pio read. Also write 1 to rcrthres and
1164 1164 * rcrto to clear these two edge triggered bits.
1165 1165 */
1166 1166 rbrp = hxgep->rx_rbr_rings->rbr_rings[channel];
1167 1167 MUTEX_ENTER(&rbrp->post_lock);
1168 1168 if (!rbrp->rbr_is_empty) {
1169 1169 cs.value = 0;
1170 1170 cs.bits.mex = 1;
1171 1171 cs.bits.ptrread = 0;
1172 1172 cs.bits.pktread = 0;
1173 1173 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value);
1174 1174 }
1175 1175 MUTEX_EXIT(&rbrp->post_lock);
1176 1176
1177 1177 if (ldgp->nldvs == 1) {
1178 1178 /*
1179 1179 * Re-arm the group.
1180 1180 */
1181 1181 (void) hpi_intr_ldg_mgmt_set(handle, ldgp->ldg, B_TRUE,
1182 1182 ldgp->ldg_timer);
1183 1183 }
1184 1184 } else if ((ldgp->nldvs == 1) && (ring->poll_flag)) {
1185 1185 /*
1186 1186 * Disarm the group, if we are not a shared interrupt.
1187 1187 */
1188 1188 (void) hpi_intr_ldg_mgmt_set(handle, ldgp->ldg, B_FALSE, 0);
1189 1189 } else if (ring->poll_flag) {
1190 1190 /*
1191 1191 * Mask-off this device from the group.
1192 1192 */
1193 1193 (void) hpi_intr_mask_set(handle, ldvp->ldv, 1);
1194 1194 }
1195 1195
1196 1196 MUTEX_EXIT(&ring->lock);
1197 1197
1198 1198 /*
1199 1199 * Send the packets up the stack.
1200 1200 */
1201 1201 if (mp != NULL) {
1202 1202 mac_rx_ring(hxgep->mach, ring->rcr_mac_handle, mp,
1203 1203 ring->rcr_gen_num);
1204 1204 }
1205 1205
1206 1206 HXGE_DEBUG_MSG((NULL, RX_INT_CTL, "<== hxge_rx_intr"));
1207 1207 return (DDI_INTR_CLAIMED);
1208 1208 }
1209 1209
1210 1210 /*
1211 1211 * Enable polling for a ring. Interrupt for the ring is disabled when
1212 1212 * the hxge interrupt comes (see hxge_rx_intr).
1213 1213 */
1214 1214 int
1215 1215 hxge_enable_poll(void *arg)
1216 1216 {
1217 1217 p_hxge_ring_handle_t ring_handle = (p_hxge_ring_handle_t)arg;
1218 1218 p_rx_rcr_ring_t ringp;
1219 1219 p_hxge_t hxgep;
1220 1220 p_hxge_ldg_t ldgp;
1221 1221
1222 1222 if (ring_handle == NULL) {
1223 1223 ASSERT(ring_handle != NULL);
1224 1224 return (1);
1225 1225 }
1226 1226
1227 1227
1228 1228 hxgep = ring_handle->hxgep;
1229 1229 ringp = hxgep->rx_rcr_rings->rcr_rings[ring_handle->index];
1230 1230
1231 1231 MUTEX_ENTER(&ringp->lock);
1232 1232
1233 1233 /*
1234 1234 * Are we already polling ?
1235 1235 */
1236 1236 if (ringp->poll_flag) {
1237 1237 MUTEX_EXIT(&ringp->lock);
1238 1238 return (1);
1239 1239 }
1240 1240
1241 1241 ldgp = ringp->ldgp;
1242 1242 if (ldgp == NULL) {
1243 1243 MUTEX_EXIT(&ringp->lock);
1244 1244 return (1);
1245 1245 }
1246 1246
1247 1247 /*
1248 1248 * Enable polling
1249 1249 */
1250 1250 ringp->poll_flag = B_TRUE;
1251 1251
1252 1252 MUTEX_EXIT(&ringp->lock);
1253 1253 return (0);
1254 1254 }
1255 1255
1256 1256 /*
1257 1257 * Disable polling for a ring and enable its interrupt.
1258 1258 */
1259 1259 int
1260 1260 hxge_disable_poll(void *arg)
1261 1261 {
1262 1262 p_hxge_ring_handle_t ring_handle = (p_hxge_ring_handle_t)arg;
1263 1263 p_rx_rcr_ring_t ringp;
1264 1264 p_hxge_t hxgep;
1265 1265
1266 1266 if (ring_handle == NULL) {
1267 1267 ASSERT(ring_handle != NULL);
1268 1268 return (0);
1269 1269 }
1270 1270
1271 1271 hxgep = ring_handle->hxgep;
1272 1272 ringp = hxgep->rx_rcr_rings->rcr_rings[ring_handle->index];
1273 1273
1274 1274 MUTEX_ENTER(&ringp->lock);
1275 1275
1276 1276 /*
1277 1277 * Disable polling: enable interrupt
1278 1278 */
1279 1279 if (ringp->poll_flag) {
1280 1280 hpi_handle_t handle;
1281 1281 rdc_stat_t cs;
1282 1282 p_hxge_ldg_t ldgp;
1283 1283
1284 1284 /*
1285 1285 * Get the control and status for this channel.
1286 1286 */
1287 1287 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1288 1288
1289 1289 /*
1290 1290 * Rearm this logical group if this is a single device
1291 1291 * group.
1292 1292 */
1293 1293 ldgp = ringp->ldgp;
1294 1294 if (ldgp == NULL) {
1295 1295 MUTEX_EXIT(&ringp->lock);
1296 1296 return (1);
1297 1297 }
1298 1298
1299 1299 ringp->poll_flag = B_FALSE;
1300 1300
1301 1301 /*
1302 1302 * Enable mailbox update, to start interrupts again.
1303 1303 */
1304 1304 cs.value = 0ULL;
1305 1305 cs.bits.mex = 1;
1306 1306 cs.bits.pktread = 0;
1307 1307 cs.bits.ptrread = 0;
1308 1308 RXDMA_REG_WRITE64(handle, RDC_STAT, ringp->rdc, cs.value);
1309 1309
1310 1310 if (ldgp->nldvs == 1) {
1311 1311 /*
1312 1312 * Re-arm the group, since it is the only member
1313 1313 * of the group.
1314 1314 */
1315 1315 (void) hpi_intr_ldg_mgmt_set(handle, ldgp->ldg, B_TRUE,
1316 1316 ldgp->ldg_timer);
1317 1317 } else {
1318 1318 /*
1319 1319 * Mask-on interrupts for the device and re-arm
1320 1320 * the group.
1321 1321 */
1322 1322 (void) hpi_intr_mask_set(handle, ringp->ldvp->ldv, 0);
1323 1323 (void) hpi_intr_ldg_mgmt_set(handle, ldgp->ldg, B_TRUE,
1324 1324 ldgp->ldg_timer);
1325 1325 }
1326 1326 }
1327 1327 MUTEX_EXIT(&ringp->lock);
1328 1328 return (0);
1329 1329 }
1330 1330
1331 1331 /*
1332 1332 * Poll 'bytes_to_pickup' bytes of message from the rx ring.
1333 1333 */
1334 1334 mblk_t *
1335 1335 hxge_rx_poll(void *arg, int bytes_to_pickup)
1336 1336 {
1337 1337 p_hxge_ring_handle_t rhp = (p_hxge_ring_handle_t)arg;
1338 1338 p_rx_rcr_ring_t ring;
1339 1339 p_hxge_t hxgep;
1340 1340 hpi_handle_t handle;
1341 1341 rdc_stat_t cs;
1342 1342 mblk_t *mblk;
1343 1343 p_hxge_ldv_t ldvp;
1344 1344
1345 1345 hxgep = rhp->hxgep;
1346 1346
1347 1347 /*
1348 1348 * Get the control and status for this channel.
1349 1349 */
1350 1350 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1351 1351 ring = hxgep->rx_rcr_rings->rcr_rings[rhp->index];
1352 1352
1353 1353 MUTEX_ENTER(&ring->lock);
1354 1354 ASSERT(ring->poll_flag == B_TRUE);
1355 1355 ASSERT(rhp->started);
1356 1356
1357 1357 if (!ring->poll_flag) {
1358 1358 MUTEX_EXIT(&ring->lock);
1359 1359 return ((mblk_t *)NULL);
1360 1360 }
1361 1361
1362 1362 /*
1363 1363 * Get the control and status bits for the ring.
1364 1364 */
1365 1365 RXDMA_REG_READ64(handle, RDC_STAT, rhp->index, &cs.value);
1366 1366 cs.bits.ptrread = 0;
1367 1367 cs.bits.pktread = 0;
1368 1368 RXDMA_REG_WRITE64(handle, RDC_STAT, rhp->index, cs.value);
1369 1369
1370 1370 /*
1371 1371 * Process packets.
1372 1372 */
1373 1373 mblk = hxge_rx_pkts(hxgep, ring->ldvp->vdma_index,
1374 1374 ring->ldvp, ring, cs, bytes_to_pickup);
1375 1375 ldvp = ring->ldvp;
1376 1376
1377 1377 /*
1378 1378 * Process Error Events.
1379 1379 */
1380 1380 if (ldvp && (cs.value & RDC_STAT_ERROR)) {
1381 1381 /*
1382 1382 * Recovery routines will grab the RCR ring lock.
1383 1383 */
1384 1384 MUTEX_EXIT(&ring->lock);
1385 1385 (void) hxge_rx_err_evnts(hxgep, ldvp->vdma_index, ldvp, cs);
1386 1386 MUTEX_ENTER(&ring->lock);
1387 1387 }
1388 1388
1389 1389 MUTEX_EXIT(&ring->lock);
1390 1390 return (mblk);
1391 1391 }
1392 1392
1393 1393 /*ARGSUSED*/
1394 1394 mblk_t *
1395 1395 hxge_rx_pkts(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp,
1396 1396 p_rx_rcr_ring_t rcrp, rdc_stat_t cs, int bytes_to_read)
1397 1397 {
1398 1398 hpi_handle_t handle;
1399 1399 uint8_t channel;
1400 1400 uint32_t comp_rd_index;
1401 1401 p_rcr_entry_t rcr_desc_rd_head_p;
1402 1402 p_rcr_entry_t rcr_desc_rd_head_pp;
1403 1403 p_mblk_t nmp, mp_cont, head_mp, *tail_mp;
1404 1404 uint16_t qlen, nrcr_read, npkt_read;
1405 1405 uint32_t qlen_hw, npkts, num_rcrs;
1406 1406 uint32_t invalid_rcr_entry;
1407 1407 boolean_t multi;
1408 1408 rdc_stat_t pktcs;
1409 1409 rdc_rcr_cfg_b_t rcr_cfg_b;
1410 1410 uint64_t rcr_head_index, rcr_tail_index;
1411 1411 uint64_t rcr_tail;
1412 1412 rdc_rcr_tail_t rcr_tail_reg;
1413 1413 p_hxge_rx_ring_stats_t rdc_stats;
1414 1414 int totallen = 0;
1415 1415
1416 1416 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts:vindex %d "
1417 1417 "channel %d", vindex, ldvp->channel));
1418 1418
1419 1419 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1420 1420 channel = rcrp->rdc;
1421 1421 if (channel != ldvp->channel) {
1422 1422 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts:index %d "
1423 1423 "channel %d, and rcr channel %d not matched.",
1424 1424 vindex, ldvp->channel, channel));
1425 1425 return (NULL);
1426 1426 }
1427 1427
1428 1428 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1429 1429 "==> hxge_rx_pkts: START: rcr channel %d "
1430 1430 "head_p $%p head_pp $%p index %d ",
1431 1431 channel, rcrp->rcr_desc_rd_head_p,
1432 1432 rcrp->rcr_desc_rd_head_pp, rcrp->comp_rd_index));
1433 1433
1434 1434 (void) hpi_rxdma_rdc_rcr_qlen_get(handle, channel, &qlen);
1435 1435 RXDMA_REG_READ64(handle, RDC_RCR_TAIL, channel, &rcr_tail_reg.value);
1436 1436 rcr_tail = rcr_tail_reg.bits.tail;
1437 1437
1438 1438 if (!qlen) {
1439 1439 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1440 1440 "<== hxge_rx_pkts:rcr channel %d qlen %d (no pkts)",
1441 1441 channel, qlen));
1442 1442 return (NULL);
1443 1443 }
1444 1444
1445 1445 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_pkts:rcr channel %d "
1446 1446 "qlen %d", channel, qlen));
1447 1447
1448 1448 comp_rd_index = rcrp->comp_rd_index;
1449 1449
1450 1450 rcr_desc_rd_head_p = rcrp->rcr_desc_rd_head_p;
1451 1451 rcr_desc_rd_head_pp = rcrp->rcr_desc_rd_head_pp;
1452 1452 nrcr_read = npkt_read = 0;
1453 1453
1454 1454 if (hxgep->rdc_first_intr[channel])
1455 1455 qlen_hw = qlen;
1456 1456 else
1457 1457 qlen_hw = qlen - 1;
1458 1458
1459 1459 head_mp = NULL;
1460 1460 tail_mp = &head_mp;
1461 1461 nmp = mp_cont = NULL;
1462 1462 multi = B_FALSE;
1463 1463
1464 1464 rcr_head_index = rcrp->rcr_desc_rd_head_p - rcrp->rcr_desc_first_p;
1465 1465 rcr_tail_index = rcr_tail - rcrp->rcr_tail_begin;
1466 1466
1467 1467 if (rcr_tail_index >= rcr_head_index) {
1468 1468 num_rcrs = rcr_tail_index - rcr_head_index;
1469 1469 } else {
1470 1470 /* rcr_tail has wrapped around */
1471 1471 num_rcrs = (rcrp->comp_size - rcr_head_index) + rcr_tail_index;
1472 1472 }
1473 1473
1474 1474 npkts = hxge_scan_for_last_eop(rcrp, rcr_desc_rd_head_p, num_rcrs);
1475 1475 if (!npkts)
1476 1476 return (NULL);
1477 1477
1478 1478 if (qlen_hw > npkts) {
1479 1479 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1480 1480 "Channel %d, rcr_qlen from reg %d and from rcr_tail %d\n",
1481 1481 channel, qlen_hw, qlen_sw));
1482 1482 qlen_hw = npkts;
1483 1483 }
1484 1484
1485 1485 while (qlen_hw) {
1486 1486 #ifdef HXGE_DEBUG
1487 1487 hxge_dump_rcr_entry(hxgep, rcr_desc_rd_head_p);
1488 1488 #endif
1489 1489 /*
1490 1490 * Process one completion ring entry.
1491 1491 */
1492 1492 invalid_rcr_entry = 0;
1493 1493 hxge_receive_packet(hxgep,
1494 1494 rcrp, rcr_desc_rd_head_p, &multi, &nmp, &mp_cont,
1495 1495 &invalid_rcr_entry);
1496 1496 if (invalid_rcr_entry != 0) {
1497 1497 rdc_stats = rcrp->rdc_stats;
1498 1498 rdc_stats->rcr_invalids++;
1499 1499 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1500 1500 "Channel %d could only read 0x%x packets, "
1501 1501 "but 0x%x pending\n", channel, npkt_read, qlen_hw));
1502 1502 break;
1503 1503 }
1504 1504
1505 1505 /*
1506 1506 * message chaining modes (nemo msg chaining)
1507 1507 */
1508 1508 if (nmp) {
1509 1509 nmp->b_next = NULL;
1510 1510 if (!multi && !mp_cont) { /* frame fits a partition */
1511 1511 *tail_mp = nmp;
1512 1512 tail_mp = &nmp->b_next;
1513 1513 nmp = NULL;
1514 1514 } else if (multi && !mp_cont) { /* first segment */
1515 1515 *tail_mp = nmp;
1516 1516 tail_mp = &nmp->b_cont;
1517 1517 } else if (multi && mp_cont) { /* mid of multi segs */
1518 1518 *tail_mp = mp_cont;
1519 1519 tail_mp = &mp_cont->b_cont;
1520 1520 } else if (!multi && mp_cont) { /* last segment */
1521 1521 *tail_mp = mp_cont;
1522 1522 tail_mp = &nmp->b_next;
1523 1523 totallen += MBLKL(mp_cont);
1524 1524 nmp = NULL;
1525 1525 }
1526 1526 }
1527 1527
1528 1528 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1529 1529 "==> hxge_rx_pkts: loop: rcr channel %d "
1530 1530 "before updating: multi %d "
1531 1531 "nrcr_read %d "
1532 1532 "npk read %d "
1533 1533 "head_pp $%p index %d ",
1534 1534 channel, multi,
1535 1535 nrcr_read, npkt_read, rcr_desc_rd_head_pp, comp_rd_index));
1536 1536
1537 1537 if (!multi) {
1538 1538 qlen_hw--;
1539 1539 npkt_read++;
1540 1540 }
1541 1541
1542 1542 /*
1543 1543 * Update the next read entry.
1544 1544 */
1545 1545 comp_rd_index = NEXT_ENTRY(comp_rd_index,
1546 1546 rcrp->comp_wrap_mask);
1547 1547
1548 1548 rcr_desc_rd_head_p = NEXT_ENTRY_PTR(rcr_desc_rd_head_p,
1549 1549 rcrp->rcr_desc_first_p, rcrp->rcr_desc_last_p);
1550 1550
1551 1551 nrcr_read++;
1552 1552
1553 1553 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1554 1554 "<== hxge_rx_pkts: (SAM, process one packet) "
1555 1555 "nrcr_read %d", nrcr_read));
1556 1556 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1557 1557 "==> hxge_rx_pkts: loop: rcr channel %d "
1558 1558 "multi %d nrcr_read %d npk read %d head_pp $%p index %d ",
1559 1559 channel, multi, nrcr_read, npkt_read, rcr_desc_rd_head_pp,
1560 1560 comp_rd_index));
1561 1561
1562 1562 if ((bytes_to_read != -1) &&
1563 1563 (totallen >= bytes_to_read)) {
1564 1564 break;
1565 1565 }
1566 1566 }
1567 1567
1568 1568 rcrp->rcr_desc_rd_head_pp = rcr_desc_rd_head_pp;
1569 1569 rcrp->comp_rd_index = comp_rd_index;
1570 1570 rcrp->rcr_desc_rd_head_p = rcr_desc_rd_head_p;
1571 1571
1572 1572 if ((hxgep->intr_timeout != rcrp->intr_timeout) ||
1573 1573 (hxgep->intr_threshold != rcrp->intr_threshold)) {
1574 1574 rcrp->intr_timeout = hxgep->intr_timeout;
1575 1575 rcrp->intr_threshold = hxgep->intr_threshold;
1576 1576 rcr_cfg_b.value = 0x0ULL;
1577 1577 if (rcrp->intr_timeout)
1578 1578 rcr_cfg_b.bits.entout = 1;
1579 1579 rcr_cfg_b.bits.timeout = rcrp->intr_timeout;
1580 1580 rcr_cfg_b.bits.pthres = rcrp->intr_threshold;
1581 1581 RXDMA_REG_WRITE64(handle, RDC_RCR_CFG_B,
1582 1582 channel, rcr_cfg_b.value);
1583 1583 }
1584 1584
1585 1585 pktcs.value = 0;
1586 1586 if (hxgep->rdc_first_intr[channel] && (npkt_read > 0)) {
1587 1587 hxgep->rdc_first_intr[channel] = B_FALSE;
1588 1588 pktcs.bits.pktread = npkt_read - 1;
1589 1589 } else
1590 1590 pktcs.bits.pktread = npkt_read;
1591 1591 pktcs.bits.ptrread = nrcr_read;
1592 1592 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, pktcs.value);
1593 1593
1594 1594 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1595 1595 "==> hxge_rx_pkts: EXIT: rcr channel %d "
1596 1596 "head_pp $%p index %016llx ",
1597 1597 channel, rcrp->rcr_desc_rd_head_pp, rcrp->comp_rd_index));
1598 1598
1599 1599 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "<== hxge_rx_pkts"));
1600 1600 return (head_mp);
1601 1601 }
1602 1602
1603 1603 #define RCR_ENTRY_PATTERN 0x5a5a6b6b7c7c8d8dULL
1604 1604 #define NO_PORT_BIT 0x20
1605 1605 #define L4_CS_EQ_BIT 0x40
1606 1606
1607 1607 static uint32_t hxge_scan_for_last_eop(p_rx_rcr_ring_t rcrp,
1608 1608 p_rcr_entry_t rcr_desc_rd_head_p, uint32_t num_rcrs)
1609 1609 {
1610 1610 uint64_t rcr_entry;
1611 1611 uint32_t rcrs = 0;
1612 1612 uint32_t pkts = 0;
1613 1613
1614 1614 while (rcrs < num_rcrs) {
1615 1615 rcr_entry = *((uint64_t *)rcr_desc_rd_head_p);
1616 1616
1617 1617 if ((rcr_entry == 0x0) || (rcr_entry == RCR_ENTRY_PATTERN))
1618 1618 break;
1619 1619
1620 1620 if (!(rcr_entry & RCR_MULTI_MASK))
1621 1621 pkts++;
1622 1622
1623 1623 rcr_desc_rd_head_p = NEXT_ENTRY_PTR(rcr_desc_rd_head_p,
1624 1624 rcrp->rcr_desc_first_p, rcrp->rcr_desc_last_p);
1625 1625
1626 1626 rcrs++;
1627 1627 }
1628 1628
1629 1629 return (pkts);
1630 1630 }
1631 1631
1632 1632 /*ARGSUSED*/
1633 1633 void
1634 1634 hxge_receive_packet(p_hxge_t hxgep, p_rx_rcr_ring_t rcr_p,
1635 1635 p_rcr_entry_t rcr_desc_rd_head_p, boolean_t *multi_p, mblk_t **mp,
1636 1636 mblk_t **mp_cont, uint32_t *invalid_rcr_entry)
1637 1637 {
1638 1638 p_mblk_t nmp = NULL;
1639 1639 uint64_t multi;
1640 1640 uint8_t channel;
1641 1641 boolean_t first_entry = B_TRUE;
1642 1642 boolean_t is_tcp_udp = B_FALSE;
1643 1643 boolean_t buffer_free = B_FALSE;
1644 1644 boolean_t error_send_up = B_FALSE;
1645 1645 uint8_t error_type;
1646 1646 uint16_t l2_len;
1647 1647 uint16_t skip_len;
1648 1648 uint8_t pktbufsz_type;
1649 1649 uint64_t rcr_entry;
1650 1650 uint64_t *pkt_buf_addr_pp;
1651 1651 uint64_t *pkt_buf_addr_p;
1652 1652 uint32_t buf_offset;
1653 1653 uint32_t bsize;
1654 1654 uint32_t msg_index;
1655 1655 p_rx_rbr_ring_t rx_rbr_p;
1656 1656 p_rx_msg_t *rx_msg_ring_p;
1657 1657 p_rx_msg_t rx_msg_p;
1658 1658 uint16_t sw_offset_bytes = 0, hdr_size = 0;
1659 1659 hxge_status_t status = HXGE_OK;
1660 1660 boolean_t is_valid = B_FALSE;
1661 1661 p_hxge_rx_ring_stats_t rdc_stats;
1662 1662 uint32_t bytes_read;
1663 1663 uint8_t header0 = 0;
1664 1664 uint8_t header1 = 0;
1665 1665 uint64_t pkt_type;
1666 1666 uint8_t no_port_bit = 0;
1667 1667 uint8_t l4_cs_eq_bit = 0;
1668 1668
1669 1669 channel = rcr_p->rdc;
1670 1670
1671 1671 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> hxge_receive_packet"));
1672 1672
1673 1673 first_entry = (*mp == NULL) ? B_TRUE : B_FALSE;
1674 1674 rcr_entry = *((uint64_t *)rcr_desc_rd_head_p);
1675 1675
1676 1676 /* Verify the content of the rcr_entry for a hardware bug workaround */
1677 1677 if ((rcr_entry == 0x0) || (rcr_entry == RCR_ENTRY_PATTERN)) {
1678 1678 *invalid_rcr_entry = 1;
1679 1679 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "hxge_receive_packet "
1680 1680 "Channel %d invalid RCR entry 0x%llx found, returning\n",
1681 1681 channel, (long long) rcr_entry));
1682 1682 return;
1683 1683 }
1684 1684 *((uint64_t *)rcr_desc_rd_head_p) = RCR_ENTRY_PATTERN;
1685 1685
1686 1686 multi = (rcr_entry & RCR_MULTI_MASK);
1687 1687 pkt_type = (rcr_entry & RCR_PKT_TYPE_MASK);
1688 1688
1689 1689 error_type = ((rcr_entry & RCR_ERROR_MASK) >> RCR_ERROR_SHIFT);
1690 1690 l2_len = ((rcr_entry & RCR_L2_LEN_MASK) >> RCR_L2_LEN_SHIFT);
1691 1691
1692 1692 /*
1693 1693 * Hardware does not strip the CRC due bug ID 11451 where
1694 1694 * the hardware mis handles minimum size packets.
1695 1695 */
1696 1696 l2_len -= ETHERFCSL;
1697 1697
1698 1698 pktbufsz_type = ((rcr_entry & RCR_PKTBUFSZ_MASK) >>
1699 1699 RCR_PKTBUFSZ_SHIFT);
1700 1700 #if defined(__i386)
1701 1701 pkt_buf_addr_pp = (uint64_t *)(uint32_t)((rcr_entry &
1702 1702 RCR_PKT_BUF_ADDR_MASK) << RCR_PKT_BUF_ADDR_SHIFT);
1703 1703 #else
1704 1704 pkt_buf_addr_pp = (uint64_t *)((rcr_entry & RCR_PKT_BUF_ADDR_MASK) <<
1705 1705 RCR_PKT_BUF_ADDR_SHIFT);
1706 1706 #endif
1707 1707
1708 1708 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1709 1709 "==> hxge_receive_packet: entryp $%p entry 0x%0llx "
1710 1710 "pkt_buf_addr_pp $%p l2_len %d multi %d "
1711 1711 "error_type 0x%x pktbufsz_type %d ",
1712 1712 rcr_desc_rd_head_p, rcr_entry, pkt_buf_addr_pp, l2_len,
1713 1713 multi, error_type, pktbufsz_type));
1714 1714
1715 1715 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1716 1716 "==> hxge_receive_packet: entryp $%p entry 0x%0llx "
1717 1717 "pkt_buf_addr_pp $%p l2_len %d multi %d "
1718 1718 "error_type 0x%x ", rcr_desc_rd_head_p,
1719 1719 rcr_entry, pkt_buf_addr_pp, l2_len, multi, error_type));
1720 1720
1721 1721 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1722 1722 "==> (rbr) hxge_receive_packet: entry 0x%0llx "
1723 1723 "full pkt_buf_addr_pp $%p l2_len %d",
1724 1724 rcr_entry, pkt_buf_addr_pp, l2_len));
1725 1725
1726 1726 /* get the stats ptr */
1727 1727 rdc_stats = rcr_p->rdc_stats;
1728 1728
1729 1729 if (!l2_len) {
1730 1730 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1731 1731 "<== hxge_receive_packet: failed: l2 length is 0."));
1732 1732 return;
1733 1733 }
1734 1734
1735 1735 /* shift 6 bits to get the full io address */
1736 1736 #if defined(__i386)
1737 1737 pkt_buf_addr_pp = (uint64_t *)((uint32_t)pkt_buf_addr_pp <<
1738 1738 RCR_PKT_BUF_ADDR_SHIFT_FULL);
1739 1739 #else
1740 1740 pkt_buf_addr_pp = (uint64_t *)((uint64_t)pkt_buf_addr_pp <<
1741 1741 RCR_PKT_BUF_ADDR_SHIFT_FULL);
1742 1742 #endif
1743 1743 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1744 1744 "==> (rbr) hxge_receive_packet: entry 0x%0llx "
1745 1745 "full pkt_buf_addr_pp $%p l2_len %d",
1746 1746 rcr_entry, pkt_buf_addr_pp, l2_len));
1747 1747
1748 1748 rx_rbr_p = rcr_p->rx_rbr_p;
1749 1749 rx_msg_ring_p = rx_rbr_p->rx_msg_ring;
1750 1750
1751 1751 if (first_entry) {
1752 1752 hdr_size = (rcr_p->full_hdr_flag ? RXDMA_HDR_SIZE_FULL :
1753 1753 RXDMA_HDR_SIZE_DEFAULT);
1754 1754
1755 1755 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1756 1756 "==> hxge_receive_packet: first entry 0x%016llx "
1757 1757 "pkt_buf_addr_pp $%p l2_len %d hdr %d",
1758 1758 rcr_entry, pkt_buf_addr_pp, l2_len, hdr_size));
1759 1759 }
1760 1760
1761 1761 MUTEX_ENTER(&rx_rbr_p->lock);
1762 1762
1763 1763 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1764 1764 "==> (rbr 1) hxge_receive_packet: entry 0x%0llx "
1765 1765 "full pkt_buf_addr_pp $%p l2_len %d",
1766 1766 rcr_entry, pkt_buf_addr_pp, l2_len));
1767 1767
1768 1768 /*
1769 1769 * Packet buffer address in the completion entry points to the starting
1770 1770 * buffer address (offset 0). Use the starting buffer address to locate
1771 1771 * the corresponding kernel address.
1772 1772 */
1773 1773 status = hxge_rxbuf_pp_to_vp(hxgep, rx_rbr_p,
1774 1774 pktbufsz_type, pkt_buf_addr_pp, &pkt_buf_addr_p,
1775 1775 &buf_offset, &msg_index);
1776 1776
1777 1777 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1778 1778 "==> (rbr 2) hxge_receive_packet: entry 0x%0llx "
1779 1779 "full pkt_buf_addr_pp $%p l2_len %d",
1780 1780 rcr_entry, pkt_buf_addr_pp, l2_len));
1781 1781
1782 1782 if (status != HXGE_OK) {
1783 1783 MUTEX_EXIT(&rx_rbr_p->lock);
1784 1784 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1785 1785 "<== hxge_receive_packet: found vaddr failed %d", status));
1786 1786 return;
1787 1787 }
1788 1788
1789 1789 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1790 1790 "==> (rbr 3) hxge_receive_packet: entry 0x%0llx "
1791 1791 "full pkt_buf_addr_pp $%p l2_len %d",
1792 1792 rcr_entry, pkt_buf_addr_pp, l2_len));
1793 1793 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1794 1794 "==> (rbr 4 msgindex %d) hxge_receive_packet: entry 0x%0llx "
1795 1795 "full pkt_buf_addr_pp $%p l2_len %d",
1796 1796 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len));
1797 1797
1798 1798 if (msg_index >= rx_rbr_p->tnblocks) {
1799 1799 MUTEX_EXIT(&rx_rbr_p->lock);
1800 1800 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1801 1801 "==> hxge_receive_packet: FATAL msg_index (%d) "
1802 1802 "should be smaller than tnblocks (%d)\n",
1803 1803 msg_index, rx_rbr_p->tnblocks));
1804 1804 return;
1805 1805 }
1806 1806
1807 1807 rx_msg_p = rx_msg_ring_p[msg_index];
1808 1808
1809 1809 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1810 1810 "==> (rbr 4 msgindex %d) hxge_receive_packet: entry 0x%0llx "
1811 1811 "full pkt_buf_addr_pp $%p l2_len %d",
1812 1812 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len));
1813 1813
1814 1814 switch (pktbufsz_type) {
1815 1815 case RCR_PKTBUFSZ_0:
1816 1816 bsize = rx_rbr_p->pkt_buf_size0_bytes;
1817 1817 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1818 1818 "==> hxge_receive_packet: 0 buf %d", bsize));
1819 1819 break;
1820 1820 case RCR_PKTBUFSZ_1:
1821 1821 bsize = rx_rbr_p->pkt_buf_size1_bytes;
1822 1822 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1823 1823 "==> hxge_receive_packet: 1 buf %d", bsize));
1824 1824 break;
1825 1825 case RCR_PKTBUFSZ_2:
1826 1826 bsize = rx_rbr_p->pkt_buf_size2_bytes;
1827 1827 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1828 1828 "==> hxge_receive_packet: 2 buf %d", bsize));
1829 1829 break;
1830 1830 case RCR_SINGLE_BLOCK:
1831 1831 bsize = rx_msg_p->block_size;
1832 1832 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1833 1833 "==> hxge_receive_packet: single %d", bsize));
1834 1834
1835 1835 break;
1836 1836 default:
1837 1837 MUTEX_EXIT(&rx_rbr_p->lock);
1838 1838 return;
1839 1839 }
1840 1840
1841 1841 DMA_COMMON_SYNC_OFFSET(rx_msg_p->buf_dma,
1842 1842 (buf_offset + sw_offset_bytes), (hdr_size + l2_len),
1843 1843 DDI_DMA_SYNC_FORCPU);
1844 1844
1845 1845 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1846 1846 "==> hxge_receive_packet: after first dump:usage count"));
1847 1847
1848 1848 if (rx_msg_p->cur_usage_cnt == 0) {
1849 1849 atomic_inc_32(&rx_rbr_p->rbr_used);
1850 1850 if (rx_rbr_p->rbr_use_bcopy) {
1851 1851 atomic_inc_32(&rx_rbr_p->rbr_consumed);
1852 1852 if (rx_rbr_p->rbr_consumed <
1853 1853 rx_rbr_p->rbr_threshold_hi) {
1854 1854 if (rx_rbr_p->rbr_threshold_lo == 0 ||
1855 1855 ((rx_rbr_p->rbr_consumed >=
1856 1856 rx_rbr_p->rbr_threshold_lo) &&
1857 1857 (rx_rbr_p->rbr_bufsize_type >=
1858 1858 pktbufsz_type))) {
1859 1859 rx_msg_p->rx_use_bcopy = B_TRUE;
1860 1860 }
1861 1861 } else {
1862 1862 rx_msg_p->rx_use_bcopy = B_TRUE;
1863 1863 }
1864 1864 }
1865 1865 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1866 1866 "==> hxge_receive_packet: buf %d (new block) ", bsize));
1867 1867
1868 1868 rx_msg_p->pkt_buf_size_code = pktbufsz_type;
1869 1869 rx_msg_p->pkt_buf_size = bsize;
1870 1870 rx_msg_p->cur_usage_cnt = 1;
1871 1871 if (pktbufsz_type == RCR_SINGLE_BLOCK) {
1872 1872 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1873 1873 "==> hxge_receive_packet: buf %d (single block) ",
1874 1874 bsize));
1875 1875 /*
1876 1876 * Buffer can be reused once the free function is
1877 1877 * called.
1878 1878 */
1879 1879 rx_msg_p->max_usage_cnt = 1;
1880 1880 buffer_free = B_TRUE;
1881 1881 } else {
1882 1882 rx_msg_p->max_usage_cnt = rx_msg_p->block_size / bsize;
1883 1883 if (rx_msg_p->max_usage_cnt == 1) {
1884 1884 buffer_free = B_TRUE;
1885 1885 }
1886 1886 }
1887 1887 } else {
1888 1888 rx_msg_p->cur_usage_cnt++;
1889 1889 if (rx_msg_p->cur_usage_cnt == rx_msg_p->max_usage_cnt) {
1890 1890 buffer_free = B_TRUE;
1891 1891 }
1892 1892 }
1893 1893
1894 1894 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1895 1895 "msgbuf index = %d l2len %d bytes usage %d max_usage %d ",
1896 1896 msg_index, l2_len,
1897 1897 rx_msg_p->cur_usage_cnt, rx_msg_p->max_usage_cnt));
1898 1898
1899 1899 if (error_type) {
1900 1900 rdc_stats->ierrors++;
1901 1901 /* Update error stats */
1902 1902 rdc_stats->errlog.compl_err_type = error_type;
1903 1903 HXGE_FM_REPORT_ERROR(hxgep, NULL, HXGE_FM_EREPORT_RDMC_RCR_ERR);
1904 1904
1905 1905 if (error_type & RCR_CTRL_FIFO_DED) {
1906 1906 rdc_stats->ctrl_fifo_ecc_err++;
1907 1907 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
1908 1908 " hxge_receive_packet: "
1909 1909 " channel %d RCR ctrl_fifo_ded error", channel));
1910 1910 } else if (error_type & RCR_DATA_FIFO_DED) {
1911 1911 rdc_stats->data_fifo_ecc_err++;
1912 1912 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
1913 1913 " hxge_receive_packet: channel %d"
1914 1914 " RCR data_fifo_ded error", channel));
1915 1915 }
1916 1916
1917 1917 /*
1918 1918 * Update and repost buffer block if max usage count is
1919 1919 * reached.
1920 1920 */
1921 1921 if (error_send_up == B_FALSE) {
1922 1922 atomic_inc_32(&rx_msg_p->ref_cnt);
1923 1923 if (buffer_free == B_TRUE) {
1924 1924 rx_msg_p->free = B_TRUE;
1925 1925 }
1926 1926
1927 1927 MUTEX_EXIT(&rx_rbr_p->lock);
1928 1928 hxge_freeb(rx_msg_p);
1929 1929 return;
1930 1930 }
1931 1931 }
1932 1932
1933 1933 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1934 1934 "==> hxge_receive_packet: DMA sync second "));
1935 1935
1936 1936 bytes_read = rcr_p->rcvd_pkt_bytes;
1937 1937 skip_len = sw_offset_bytes + hdr_size;
1938 1938
1939 1939 if (first_entry) {
1940 1940 header0 = rx_msg_p->buffer[buf_offset];
1941 1941 no_port_bit = header0 & NO_PORT_BIT;
1942 1942 header1 = rx_msg_p->buffer[buf_offset + 1];
1943 1943 l4_cs_eq_bit = header1 & L4_CS_EQ_BIT;
1944 1944 }
1945 1945
1946 1946 if (!rx_msg_p->rx_use_bcopy) {
1947 1947 /*
1948 1948 * For loaned up buffers, the driver reference count
1949 1949 * will be incremented first and then the free state.
1950 1950 */
1951 1951 if ((nmp = hxge_dupb(rx_msg_p, buf_offset, bsize)) != NULL) {
1952 1952 if (first_entry) {
1953 1953 nmp->b_rptr = &nmp->b_rptr[skip_len];
1954 1954 if (l2_len < bsize - skip_len) {
1955 1955 nmp->b_wptr = &nmp->b_rptr[l2_len];
1956 1956 } else {
1957 1957 nmp->b_wptr = &nmp->b_rptr[bsize
1958 1958 - skip_len];
1959 1959 }
1960 1960 } else {
1961 1961 if (l2_len - bytes_read < bsize) {
1962 1962 nmp->b_wptr =
1963 1963 &nmp->b_rptr[l2_len - bytes_read];
1964 1964 } else {
1965 1965 nmp->b_wptr = &nmp->b_rptr[bsize];
1966 1966 }
1967 1967 }
1968 1968 }
1969 1969 } else {
1970 1970 if (first_entry) {
1971 1971 nmp = hxge_dupb_bcopy(rx_msg_p, buf_offset + skip_len,
1972 1972 l2_len < bsize - skip_len ?
1973 1973 l2_len : bsize - skip_len);
1974 1974 } else {
1975 1975 nmp = hxge_dupb_bcopy(rx_msg_p, buf_offset,
1976 1976 l2_len - bytes_read < bsize ?
1977 1977 l2_len - bytes_read : bsize);
1978 1978 }
1979 1979 }
1980 1980
1981 1981 if (nmp != NULL) {
1982 1982 if (first_entry)
1983 1983 bytes_read = nmp->b_wptr - nmp->b_rptr;
1984 1984 else
1985 1985 bytes_read += nmp->b_wptr - nmp->b_rptr;
1986 1986
1987 1987 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1988 1988 "==> hxge_receive_packet after dupb: "
1989 1989 "rbr consumed %d "
1990 1990 "pktbufsz_type %d "
1991 1991 "nmp $%p rptr $%p wptr $%p "
1992 1992 "buf_offset %d bzise %d l2_len %d skip_len %d",
1993 1993 rx_rbr_p->rbr_consumed,
1994 1994 pktbufsz_type,
1995 1995 nmp, nmp->b_rptr, nmp->b_wptr,
1996 1996 buf_offset, bsize, l2_len, skip_len));
1997 1997 } else {
1998 1998 cmn_err(CE_WARN, "!hxge_receive_packet: update stats (error)");
1999 1999
2000 2000 atomic_inc_32(&rx_msg_p->ref_cnt);
2001 2001 if (buffer_free == B_TRUE) {
2002 2002 rx_msg_p->free = B_TRUE;
2003 2003 }
2004 2004
2005 2005 MUTEX_EXIT(&rx_rbr_p->lock);
2006 2006 hxge_freeb(rx_msg_p);
2007 2007 return;
2008 2008 }
2009 2009
2010 2010 if (buffer_free == B_TRUE) {
2011 2011 rx_msg_p->free = B_TRUE;
2012 2012 }
2013 2013
2014 2014 /*
2015 2015 * ERROR, FRAG and PKT_TYPE are only reported in the first entry. If a
2016 2016 * packet is not fragmented and no error bit is set, then L4 checksum
2017 2017 * is OK.
2018 2018 */
2019 2019 is_valid = (nmp != NULL);
2020 2020 if (first_entry) {
2021 2021 rdc_stats->ipackets++; /* count only 1st seg for jumbo */
2022 2022 if (l2_len > (STD_FRAME_SIZE - ETHERFCSL))
2023 2023 rdc_stats->jumbo_pkts++;
2024 2024 rdc_stats->ibytes += skip_len + l2_len < bsize ?
2025 2025 l2_len : bsize;
2026 2026 } else {
2027 2027 /*
2028 2028 * Add the current portion of the packet to the kstats.
2029 2029 * The current portion of the packet is calculated by using
2030 2030 * length of the packet and the previously received portion.
2031 2031 */
2032 2032 rdc_stats->ibytes += l2_len - rcr_p->rcvd_pkt_bytes < bsize ?
2033 2033 l2_len - rcr_p->rcvd_pkt_bytes : bsize;
2034 2034 }
2035 2035
2036 2036 rcr_p->rcvd_pkt_bytes = bytes_read;
2037 2037
2038 2038 if (rx_msg_p->free && rx_msg_p->rx_use_bcopy) {
2039 2039 atomic_inc_32(&rx_msg_p->ref_cnt);
2040 2040 MUTEX_EXIT(&rx_rbr_p->lock);
2041 2041 hxge_freeb(rx_msg_p);
2042 2042 } else
2043 2043 MUTEX_EXIT(&rx_rbr_p->lock);
2044 2044
2045 2045 if (is_valid) {
2046 2046 nmp->b_cont = NULL;
2047 2047 if (first_entry) {
2048 2048 *mp = nmp;
2049 2049 *mp_cont = NULL;
2050 2050 } else {
2051 2051 *mp_cont = nmp;
2052 2052 }
2053 2053 }
2054 2054
2055 2055 /*
2056 2056 * Update stats and hardware checksuming.
2057 2057 */
2058 2058 if (is_valid && !multi) {
2059 2059 is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP ||
2060 2060 pkt_type == RCR_PKT_IS_UDP) ? B_TRUE : B_FALSE);
2061 2061
2062 2062 if (!no_port_bit && l4_cs_eq_bit && is_tcp_udp && !error_type) {
2063 2063 mac_hcksum_set(nmp, 0, 0, 0, 0, HCK_FULLCKSUM_OK);
2064 2064
2065 2065 HXGE_DEBUG_MSG((hxgep, RX_CTL,
2066 2066 "==> hxge_receive_packet: Full tcp/udp cksum "
2067 2067 "is_valid 0x%x multi %d error %d",
2068 2068 is_valid, multi, error_type));
2069 2069 }
2070 2070 }
2071 2071
2072 2072 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
2073 2073 "==> hxge_receive_packet: *mp 0x%016llx", *mp));
2074 2074
2075 2075 *multi_p = (multi == RCR_MULTI_MASK);
2076 2076
2077 2077 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_receive_packet: "
2078 2078 "multi %d nmp 0x%016llx *mp 0x%016llx *mp_cont 0x%016llx",
2079 2079 *multi_p, nmp, *mp, *mp_cont));
2080 2080 }
2081 2081
2082 2082 static void
2083 2083 hxge_rx_rbr_empty_recover(p_hxge_t hxgep, uint8_t channel)
2084 2084 {
2085 2085 hpi_handle_t handle;
2086 2086 p_rx_rcr_ring_t rcrp;
2087 2087 p_rx_rbr_ring_t rbrp;
2088 2088
2089 2089 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel];
2090 2090 rbrp = rcrp->rx_rbr_p;
2091 2091 handle = HXGE_DEV_HPI_HANDLE(hxgep);
2092 2092
2093 2093 /*
2094 2094 * Wait for the channel to be quiet
2095 2095 */
2096 2096 (void) hpi_rxdma_cfg_rdc_wait_for_qst(handle, channel);
2097 2097
2098 2098 /*
2099 2099 * Post page will accumulate some buffers before re-enabling
2100 2100 * the DMA channel.
2101 2101 */
2102 2102
2103 2103 MUTEX_ENTER(&rbrp->post_lock);
2104 2104 if ((rbrp->rbb_max - rbrp->rbr_used) >= HXGE_RBR_EMPTY_THRESHOLD) {
2105 2105 hxge_rbr_empty_restore(hxgep, rbrp);
2106 2106 } else {
2107 2107 rbrp->rbr_is_empty = B_TRUE;
2108 2108 }
2109 2109 MUTEX_EXIT(&rbrp->post_lock);
2110 2110 }
2111 2111
2112 2112
2113 2113 /*ARGSUSED*/
2114 2114 static hxge_status_t
2115 2115 hxge_rx_err_evnts(p_hxge_t hxgep, uint_t index, p_hxge_ldv_t ldvp,
2116 2116 rdc_stat_t cs)
2117 2117 {
2118 2118 p_hxge_rx_ring_stats_t rdc_stats;
2119 2119 hpi_handle_t handle;
2120 2120 boolean_t rxchan_fatal = B_FALSE;
2121 2121 uint8_t channel;
2122 2122 hxge_status_t status = HXGE_OK;
2123 2123
2124 2124 HXGE_DEBUG_MSG((hxgep, INT_CTL, "==> hxge_rx_err_evnts"));
2125 2125
2126 2126 handle = HXGE_DEV_HPI_HANDLE(hxgep);
2127 2127 channel = ldvp->channel;
2128 2128
2129 2129 rdc_stats = &hxgep->statsp->rdc_stats[ldvp->vdma_index];
2130 2130
2131 2131 if (cs.bits.rbr_cpl_to) {
2132 2132 rdc_stats->rbr_tmout++;
2133 2133 HXGE_FM_REPORT_ERROR(hxgep, channel,
2134 2134 HXGE_FM_EREPORT_RDMC_RBR_CPL_TO);
2135 2135 rxchan_fatal = B_TRUE;
2136 2136 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2137 2137 "==> hxge_rx_err_evnts(channel %d): "
2138 2138 "fatal error: rx_rbr_timeout", channel));
2139 2139 }
2140 2140
2141 2141 if ((cs.bits.rcr_shadow_par_err) || (cs.bits.rbr_prefetch_par_err)) {
2142 2142 (void) hpi_rxdma_ring_perr_stat_get(handle,
2143 2143 &rdc_stats->errlog.pre_par, &rdc_stats->errlog.sha_par);
2144 2144 }
2145 2145
2146 2146 if (cs.bits.rcr_shadow_par_err) {
2147 2147 rdc_stats->rcr_sha_par++;
2148 2148 HXGE_FM_REPORT_ERROR(hxgep, channel,
2149 2149 HXGE_FM_EREPORT_RDMC_RCR_SHA_PAR);
2150 2150 rxchan_fatal = B_TRUE;
2151 2151 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2152 2152 "==> hxge_rx_err_evnts(channel %d): "
2153 2153 "fatal error: rcr_shadow_par_err", channel));
2154 2154 }
2155 2155
2156 2156 if (cs.bits.rbr_prefetch_par_err) {
2157 2157 rdc_stats->rbr_pre_par++;
2158 2158 HXGE_FM_REPORT_ERROR(hxgep, channel,
2159 2159 HXGE_FM_EREPORT_RDMC_RBR_PRE_PAR);
2160 2160 rxchan_fatal = B_TRUE;
2161 2161 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2162 2162 "==> hxge_rx_err_evnts(channel %d): "
2163 2163 "fatal error: rbr_prefetch_par_err", channel));
2164 2164 }
2165 2165
2166 2166 if (cs.bits.rbr_pre_empty) {
2167 2167 rdc_stats->rbr_pre_empty++;
2168 2168 HXGE_FM_REPORT_ERROR(hxgep, channel,
2169 2169 HXGE_FM_EREPORT_RDMC_RBR_PRE_EMPTY);
2170 2170 rxchan_fatal = B_TRUE;
2171 2171 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2172 2172 "==> hxge_rx_err_evnts(channel %d): "
2173 2173 "fatal error: rbr_pre_empty", channel));
2174 2174 }
2175 2175
2176 2176 if (cs.bits.peu_resp_err) {
2177 2177 rdc_stats->peu_resp_err++;
2178 2178 HXGE_FM_REPORT_ERROR(hxgep, channel,
2179 2179 HXGE_FM_EREPORT_RDMC_PEU_RESP_ERR);
2180 2180 rxchan_fatal = B_TRUE;
2181 2181 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2182 2182 "==> hxge_rx_err_evnts(channel %d): "
2183 2183 "fatal error: peu_resp_err", channel));
2184 2184 }
2185 2185
2186 2186 if (cs.bits.rcr_thres) {
2187 2187 rdc_stats->rcr_thres++;
2188 2188 }
2189 2189
2190 2190 if (cs.bits.rcr_to) {
2191 2191 rdc_stats->rcr_to++;
2192 2192 }
2193 2193
2194 2194 if (cs.bits.rcr_shadow_full) {
2195 2195 rdc_stats->rcr_shadow_full++;
2196 2196 HXGE_FM_REPORT_ERROR(hxgep, channel,
2197 2197 HXGE_FM_EREPORT_RDMC_RCR_SHA_FULL);
2198 2198 rxchan_fatal = B_TRUE;
2199 2199 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2200 2200 "==> hxge_rx_err_evnts(channel %d): "
2201 2201 "fatal error: rcr_shadow_full", channel));
2202 2202 }
2203 2203
2204 2204 if (cs.bits.rcr_full) {
2205 2205 rdc_stats->rcrfull++;
2206 2206 HXGE_FM_REPORT_ERROR(hxgep, channel,
2207 2207 HXGE_FM_EREPORT_RDMC_RCRFULL);
2208 2208 rxchan_fatal = B_TRUE;
2209 2209 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2210 2210 "==> hxge_rx_err_evnts(channel %d): "
2211 2211 "fatal error: rcrfull error", channel));
2212 2212 }
2213 2213
2214 2214 if (cs.bits.rbr_empty) {
2215 2215 rdc_stats->rbr_empty++;
2216 2216 hxge_rx_rbr_empty_recover(hxgep, channel);
2217 2217 }
2218 2218
2219 2219 if (cs.bits.rbr_full) {
2220 2220 rdc_stats->rbrfull++;
2221 2221 HXGE_FM_REPORT_ERROR(hxgep, channel,
2222 2222 HXGE_FM_EREPORT_RDMC_RBRFULL);
2223 2223 rxchan_fatal = B_TRUE;
2224 2224 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2225 2225 "==> hxge_rx_err_evnts(channel %d): "
2226 2226 "fatal error: rbr_full error", channel));
2227 2227 }
2228 2228
2229 2229 if (rxchan_fatal) {
2230 2230 p_rx_rcr_ring_t rcrp;
2231 2231 p_rx_rbr_ring_t rbrp;
2232 2232
2233 2233 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel];
2234 2234 rbrp = rcrp->rx_rbr_p;
2235 2235
2236 2236 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2237 2237 " hxge_rx_err_evnts: fatal error on Channel #%d\n",
2238 2238 channel));
2239 2239
2240 2240 MUTEX_ENTER(&rbrp->post_lock);
2241 2241 /* This function needs to be inside the post_lock */
2242 2242 status = hxge_rxdma_fatal_err_recover(hxgep, channel);
2243 2243 MUTEX_EXIT(&rbrp->post_lock);
2244 2244 if (status == HXGE_OK) {
2245 2245 FM_SERVICE_RESTORED(hxgep);
2246 2246 }
2247 2247 }
2248 2248
2249 2249 HXGE_DEBUG_MSG((hxgep, INT_CTL, "<== hxge_rx_err_evnts"));
2250 2250 return (status);
2251 2251 }
2252 2252
2253 2253 static hxge_status_t
2254 2254 hxge_map_rxdma(p_hxge_t hxgep)
2255 2255 {
2256 2256 int i, ndmas;
2257 2257 uint16_t channel;
2258 2258 p_rx_rbr_rings_t rx_rbr_rings;
2259 2259 p_rx_rbr_ring_t *rbr_rings;
2260 2260 p_rx_rcr_rings_t rx_rcr_rings;
2261 2261 p_rx_rcr_ring_t *rcr_rings;
2262 2262 p_rx_mbox_areas_t rx_mbox_areas_p;
2263 2263 p_rx_mbox_t *rx_mbox_p;
2264 2264 p_hxge_dma_pool_t dma_buf_poolp;
2265 2265 p_hxge_dma_common_t *dma_buf_p;
2266 2266 p_hxge_dma_pool_t dma_rbr_cntl_poolp;
2267 2267 p_hxge_dma_common_t *dma_rbr_cntl_p;
2268 2268 p_hxge_dma_pool_t dma_rcr_cntl_poolp;
2269 2269 p_hxge_dma_common_t *dma_rcr_cntl_p;
2270 2270 p_hxge_dma_pool_t dma_mbox_cntl_poolp;
2271 2271 p_hxge_dma_common_t *dma_mbox_cntl_p;
2272 2272 uint32_t *num_chunks;
2273 2273 hxge_status_t status = HXGE_OK;
2274 2274
2275 2275 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_map_rxdma"));
2276 2276
2277 2277 dma_buf_poolp = hxgep->rx_buf_pool_p;
2278 2278 dma_rbr_cntl_poolp = hxgep->rx_rbr_cntl_pool_p;
2279 2279 dma_rcr_cntl_poolp = hxgep->rx_rcr_cntl_pool_p;
2280 2280 dma_mbox_cntl_poolp = hxgep->rx_mbox_cntl_pool_p;
2281 2281
2282 2282 if (!dma_buf_poolp->buf_allocated ||
2283 2283 !dma_rbr_cntl_poolp->buf_allocated ||
2284 2284 !dma_rcr_cntl_poolp->buf_allocated ||
2285 2285 !dma_mbox_cntl_poolp->buf_allocated) {
2286 2286 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2287 2287 "<== hxge_map_rxdma: buf not allocated"));
2288 2288 return (HXGE_ERROR);
2289 2289 }
2290 2290
2291 2291 ndmas = dma_buf_poolp->ndmas;
2292 2292 if (!ndmas) {
2293 2293 HXGE_DEBUG_MSG((hxgep, RX_CTL,
2294 2294 "<== hxge_map_rxdma: no dma allocated"));
2295 2295 return (HXGE_ERROR);
2296 2296 }
2297 2297
2298 2298 num_chunks = dma_buf_poolp->num_chunks;
2299 2299 dma_buf_p = dma_buf_poolp->dma_buf_pool_p;
2300 2300 dma_rbr_cntl_p = dma_rbr_cntl_poolp->dma_buf_pool_p;
2301 2301 dma_rcr_cntl_p = dma_rcr_cntl_poolp->dma_buf_pool_p;
2302 2302 dma_mbox_cntl_p = dma_mbox_cntl_poolp->dma_buf_pool_p;
2303 2303
2304 2304 rx_rbr_rings = (p_rx_rbr_rings_t)
2305 2305 KMEM_ZALLOC(sizeof (rx_rbr_rings_t), KM_SLEEP);
2306 2306 rbr_rings = (p_rx_rbr_ring_t *)KMEM_ZALLOC(
2307 2307 sizeof (p_rx_rbr_ring_t) * ndmas, KM_SLEEP);
2308 2308
2309 2309 rx_rcr_rings = (p_rx_rcr_rings_t)
2310 2310 KMEM_ZALLOC(sizeof (rx_rcr_rings_t), KM_SLEEP);
2311 2311 rcr_rings = (p_rx_rcr_ring_t *)KMEM_ZALLOC(
2312 2312 sizeof (p_rx_rcr_ring_t) * ndmas, KM_SLEEP);
2313 2313
2314 2314 rx_mbox_areas_p = (p_rx_mbox_areas_t)
2315 2315 KMEM_ZALLOC(sizeof (rx_mbox_areas_t), KM_SLEEP);
2316 2316 rx_mbox_p = (p_rx_mbox_t *)KMEM_ZALLOC(
2317 2317 sizeof (p_rx_mbox_t) * ndmas, KM_SLEEP);
2318 2318
2319 2319 /*
2320 2320 * Timeout should be set based on the system clock divider.
2321 2321 * The following timeout value of 1 assumes that the
2322 2322 * granularity (1000) is 3 microseconds running at 300MHz.
2323 2323 */
2324 2324
2325 2325 hxgep->intr_threshold = RXDMA_RCR_PTHRES_DEFAULT;
2326 2326 hxgep->intr_timeout = RXDMA_RCR_TO_DEFAULT;
2327 2327
2328 2328 /*
2329 2329 * Map descriptors from the buffer polls for each dam channel.
2330 2330 */
2331 2331 for (i = 0; i < ndmas; i++) {
2332 2332 if (((p_hxge_dma_common_t)dma_buf_p[i]) == NULL) {
2333 2333 status = HXGE_ERROR;
2334 2334 goto hxge_map_rxdma_fail1;
2335 2335 }
2336 2336
2337 2337 /*
2338 2338 * Set up and prepare buffer blocks, descriptors and mailbox.
2339 2339 */
2340 2340 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
2341 2341 status = hxge_map_rxdma_channel(hxgep, channel,
2342 2342 (p_hxge_dma_common_t *)&dma_buf_p[i],
2343 2343 (p_rx_rbr_ring_t *)&rbr_rings[i],
2344 2344 num_chunks[i],
2345 2345 (p_hxge_dma_common_t *)&dma_rbr_cntl_p[i],
2346 2346 (p_hxge_dma_common_t *)&dma_rcr_cntl_p[i],
2347 2347 (p_hxge_dma_common_t *)&dma_mbox_cntl_p[i],
2348 2348 (p_rx_rcr_ring_t *)&rcr_rings[i],
2349 2349 (p_rx_mbox_t *)&rx_mbox_p[i]);
2350 2350 if (status != HXGE_OK) {
2351 2351 goto hxge_map_rxdma_fail1;
2352 2352 }
2353 2353 rbr_rings[i]->index = (uint16_t)i;
2354 2354 rcr_rings[i]->index = (uint16_t)i;
2355 2355 rcr_rings[i]->rdc_stats = &hxgep->statsp->rdc_stats[i];
2356 2356 }
2357 2357
2358 2358 rx_rbr_rings->ndmas = rx_rcr_rings->ndmas = ndmas;
2359 2359 rx_rbr_rings->rbr_rings = rbr_rings;
2360 2360 hxgep->rx_rbr_rings = rx_rbr_rings;
2361 2361 rx_rcr_rings->rcr_rings = rcr_rings;
2362 2362 hxgep->rx_rcr_rings = rx_rcr_rings;
2363 2363
2364 2364 rx_mbox_areas_p->rxmbox_areas = rx_mbox_p;
2365 2365 hxgep->rx_mbox_areas_p = rx_mbox_areas_p;
2366 2366
2367 2367 goto hxge_map_rxdma_exit;
2368 2368
2369 2369 hxge_map_rxdma_fail1:
2370 2370 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2371 2371 "==> hxge_map_rxdma: unmap rbr,rcr (status 0x%x channel %d i %d)",
2372 2372 status, channel, i));
2373 2373 i--;
2374 2374 for (; i >= 0; i--) {
2375 2375 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
2376 2376 hxge_unmap_rxdma_channel(hxgep, channel,
2377 2377 rbr_rings[i], rcr_rings[i], rx_mbox_p[i]);
2378 2378 }
2379 2379
2380 2380 KMEM_FREE(rbr_rings, sizeof (p_rx_rbr_ring_t) * ndmas);
2381 2381 KMEM_FREE(rx_rbr_rings, sizeof (rx_rbr_rings_t));
2382 2382 KMEM_FREE(rcr_rings, sizeof (p_rx_rcr_ring_t) * ndmas);
2383 2383 KMEM_FREE(rx_rcr_rings, sizeof (rx_rcr_rings_t));
2384 2384 KMEM_FREE(rx_mbox_p, sizeof (p_rx_mbox_t) * ndmas);
2385 2385 KMEM_FREE(rx_mbox_areas_p, sizeof (rx_mbox_areas_t));
2386 2386
2387 2387 hxge_map_rxdma_exit:
2388 2388 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2389 2389 "<== hxge_map_rxdma: (status 0x%x channel %d)", status, channel));
2390 2390
2391 2391 return (status);
2392 2392 }
2393 2393
2394 2394 static void
2395 2395 hxge_unmap_rxdma(p_hxge_t hxgep)
2396 2396 {
2397 2397 int i, ndmas;
2398 2398 uint16_t channel;
2399 2399 p_rx_rbr_rings_t rx_rbr_rings;
2400 2400 p_rx_rbr_ring_t *rbr_rings;
2401 2401 p_rx_rcr_rings_t rx_rcr_rings;
2402 2402 p_rx_rcr_ring_t *rcr_rings;
2403 2403 p_rx_mbox_areas_t rx_mbox_areas_p;
2404 2404 p_rx_mbox_t *rx_mbox_p;
2405 2405 p_hxge_dma_pool_t dma_buf_poolp;
2406 2406 p_hxge_dma_pool_t dma_rbr_cntl_poolp;
2407 2407 p_hxge_dma_pool_t dma_rcr_cntl_poolp;
2408 2408 p_hxge_dma_pool_t dma_mbox_cntl_poolp;
2409 2409 p_hxge_dma_common_t *dma_buf_p;
2410 2410
2411 2411 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_unmap_rxdma"));
2412 2412
2413 2413 dma_buf_poolp = hxgep->rx_buf_pool_p;
2414 2414 dma_rbr_cntl_poolp = hxgep->rx_rbr_cntl_pool_p;
2415 2415 dma_rcr_cntl_poolp = hxgep->rx_rcr_cntl_pool_p;
2416 2416 dma_mbox_cntl_poolp = hxgep->rx_mbox_cntl_pool_p;
2417 2417
2418 2418 if (!dma_buf_poolp->buf_allocated ||
2419 2419 !dma_rbr_cntl_poolp->buf_allocated ||
2420 2420 !dma_rcr_cntl_poolp->buf_allocated ||
2421 2421 !dma_mbox_cntl_poolp->buf_allocated) {
2422 2422 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2423 2423 "<== hxge_unmap_rxdma: NULL buf pointers"));
2424 2424 return;
2425 2425 }
2426 2426
2427 2427 rx_rbr_rings = hxgep->rx_rbr_rings;
2428 2428 rx_rcr_rings = hxgep->rx_rcr_rings;
2429 2429 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) {
2430 2430 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2431 2431 "<== hxge_unmap_rxdma: NULL pointers"));
2432 2432 return;
2433 2433 }
2434 2434
2435 2435 ndmas = rx_rbr_rings->ndmas;
2436 2436 if (!ndmas) {
2437 2437 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2438 2438 "<== hxge_unmap_rxdma: no channel"));
2439 2439 return;
2440 2440 }
2441 2441
2442 2442 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2443 2443 "==> hxge_unmap_rxdma (ndmas %d)", ndmas));
2444 2444
2445 2445 rbr_rings = rx_rbr_rings->rbr_rings;
2446 2446 rcr_rings = rx_rcr_rings->rcr_rings;
2447 2447 rx_mbox_areas_p = hxgep->rx_mbox_areas_p;
2448 2448 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas;
2449 2449 dma_buf_p = dma_buf_poolp->dma_buf_pool_p;
2450 2450
2451 2451 for (i = 0; i < ndmas; i++) {
2452 2452 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
2453 2453 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2454 2454 "==> hxge_unmap_rxdma (ndmas %d) channel %d",
2455 2455 ndmas, channel));
2456 2456 (void) hxge_unmap_rxdma_channel(hxgep, channel,
2457 2457 (p_rx_rbr_ring_t)rbr_rings[i],
2458 2458 (p_rx_rcr_ring_t)rcr_rings[i],
2459 2459 (p_rx_mbox_t)rx_mbox_p[i]);
2460 2460 }
2461 2461
2462 2462 KMEM_FREE(rx_rbr_rings, sizeof (rx_rbr_rings_t));
2463 2463 KMEM_FREE(rbr_rings, sizeof (p_rx_rbr_ring_t) * ndmas);
2464 2464 KMEM_FREE(rx_rcr_rings, sizeof (rx_rcr_rings_t));
2465 2465 KMEM_FREE(rcr_rings, sizeof (p_rx_rcr_ring_t) * ndmas);
2466 2466 KMEM_FREE(rx_mbox_areas_p, sizeof (rx_mbox_areas_t));
2467 2467 KMEM_FREE(rx_mbox_p, sizeof (p_rx_mbox_t) * ndmas);
2468 2468
2469 2469 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_unmap_rxdma"));
2470 2470 }
2471 2471
2472 2472 hxge_status_t
2473 2473 hxge_map_rxdma_channel(p_hxge_t hxgep, uint16_t channel,
2474 2474 p_hxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p,
2475 2475 uint32_t num_chunks, p_hxge_dma_common_t *dma_rbr_cntl_p,
2476 2476 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p,
2477 2477 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p)
2478 2478 {
2479 2479 int status = HXGE_OK;
2480 2480
2481 2481 /*
2482 2482 * Set up and prepare buffer blocks, descriptors and mailbox.
2483 2483 */
2484 2484 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2485 2485 "==> hxge_map_rxdma_channel (channel %d)", channel));
2486 2486
2487 2487 /*
2488 2488 * Receive buffer blocks
2489 2489 */
2490 2490 status = hxge_map_rxdma_channel_buf_ring(hxgep, channel,
2491 2491 dma_buf_p, rbr_p, num_chunks);
2492 2492 if (status != HXGE_OK) {
2493 2493 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2494 2494 "==> hxge_map_rxdma_channel (channel %d): "
2495 2495 "map buffer failed 0x%x", channel, status));
2496 2496 goto hxge_map_rxdma_channel_exit;
2497 2497 }
2498 2498
2499 2499 /*
2500 2500 * Receive block ring, completion ring and mailbox.
2501 2501 */
2502 2502 status = hxge_map_rxdma_channel_cfg_ring(hxgep, channel,
2503 2503 dma_rbr_cntl_p, dma_rcr_cntl_p, dma_mbox_cntl_p,
2504 2504 rbr_p, rcr_p, rx_mbox_p);
2505 2505 if (status != HXGE_OK) {
2506 2506 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2507 2507 "==> hxge_map_rxdma_channel (channel %d): "
2508 2508 "map config failed 0x%x", channel, status));
2509 2509 goto hxge_map_rxdma_channel_fail2;
2510 2510 }
2511 2511 goto hxge_map_rxdma_channel_exit;
2512 2512
2513 2513 hxge_map_rxdma_channel_fail3:
2514 2514 /* Free rbr, rcr */
2515 2515 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2516 2516 "==> hxge_map_rxdma_channel: free rbr/rcr (status 0x%x channel %d)",
2517 2517 status, channel));
2518 2518 hxge_unmap_rxdma_channel_cfg_ring(hxgep, *rcr_p, *rx_mbox_p);
2519 2519
2520 2520 hxge_map_rxdma_channel_fail2:
2521 2521 /* Free buffer blocks */
2522 2522 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2523 2523 "==> hxge_map_rxdma_channel: free rx buffers"
2524 2524 "(hxgep 0x%x status 0x%x channel %d)",
2525 2525 hxgep, status, channel));
2526 2526 hxge_unmap_rxdma_channel_buf_ring(hxgep, *rbr_p);
2527 2527
2528 2528 status = HXGE_ERROR;
2529 2529
2530 2530 hxge_map_rxdma_channel_exit:
2531 2531 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2532 2532 "<== hxge_map_rxdma_channel: (hxgep 0x%x status 0x%x channel %d)",
2533 2533 hxgep, status, channel));
2534 2534
2535 2535 return (status);
2536 2536 }
2537 2537
2538 2538 /*ARGSUSED*/
2539 2539 static void
2540 2540 hxge_unmap_rxdma_channel(p_hxge_t hxgep, uint16_t channel,
2541 2541 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p)
2542 2542 {
2543 2543 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2544 2544 "==> hxge_unmap_rxdma_channel (channel %d)", channel));
2545 2545
2546 2546 /*
2547 2547 * unmap receive block ring, completion ring and mailbox.
2548 2548 */
2549 2549 (void) hxge_unmap_rxdma_channel_cfg_ring(hxgep, rcr_p, rx_mbox_p);
2550 2550
2551 2551 /* unmap buffer blocks */
2552 2552 (void) hxge_unmap_rxdma_channel_buf_ring(hxgep, rbr_p);
2553 2553
2554 2554 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_unmap_rxdma_channel"));
2555 2555 }
2556 2556
2557 2557 /*ARGSUSED*/
2558 2558 static hxge_status_t
2559 2559 hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep, uint16_t dma_channel,
2560 2560 p_hxge_dma_common_t *dma_rbr_cntl_p, p_hxge_dma_common_t *dma_rcr_cntl_p,
2561 2561 p_hxge_dma_common_t *dma_mbox_cntl_p, p_rx_rbr_ring_t *rbr_p,
2562 2562 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p)
2563 2563 {
2564 2564 p_rx_rbr_ring_t rbrp;
2565 2565 p_rx_rcr_ring_t rcrp;
2566 2566 p_rx_mbox_t mboxp;
2567 2567 p_hxge_dma_common_t cntl_dmap;
2568 2568 p_hxge_dma_common_t dmap;
2569 2569 p_rx_msg_t *rx_msg_ring;
2570 2570 p_rx_msg_t rx_msg_p;
2571 2571 rdc_rbr_cfg_a_t *rcfga_p;
2572 2572 rdc_rbr_cfg_b_t *rcfgb_p;
2573 2573 rdc_rcr_cfg_a_t *cfga_p;
2574 2574 rdc_rcr_cfg_b_t *cfgb_p;
2575 2575 rdc_rx_cfg1_t *cfig1_p;
2576 2576 rdc_rx_cfg2_t *cfig2_p;
2577 2577 rdc_rbr_kick_t *kick_p;
2578 2578 uint32_t dmaaddrp;
2579 2579 uint32_t *rbr_vaddrp;
2580 2580 uint32_t bkaddr;
2581 2581 hxge_status_t status = HXGE_OK;
2582 2582 int i;
2583 2583 uint32_t hxge_port_rcr_size;
2584 2584
2585 2585 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2586 2586 "==> hxge_map_rxdma_channel_cfg_ring"));
2587 2587
2588 2588 cntl_dmap = *dma_rbr_cntl_p;
2589 2589
2590 2590 /*
2591 2591 * Map in the receive block ring
2592 2592 */
2593 2593 rbrp = *rbr_p;
2594 2594 dmap = (p_hxge_dma_common_t)&rbrp->rbr_desc;
2595 2595 hxge_setup_dma_common(dmap, cntl_dmap, rbrp->rbb_max, 4);
2596 2596
2597 2597 /*
2598 2598 * Zero out buffer block ring descriptors.
2599 2599 */
2600 2600 bzero((caddr_t)dmap->kaddrp, dmap->alength);
2601 2601
2602 2602 rcfga_p = &(rbrp->rbr_cfga);
2603 2603 rcfgb_p = &(rbrp->rbr_cfgb);
2604 2604 kick_p = &(rbrp->rbr_kick);
2605 2605 rcfga_p->value = 0;
2606 2606 rcfgb_p->value = 0;
2607 2607 kick_p->value = 0;
2608 2608 rbrp->rbr_addr = dmap->dma_cookie.dmac_laddress;
2609 2609 rcfga_p->value = (rbrp->rbr_addr &
2610 2610 (RBR_CFIG_A_STDADDR_MASK | RBR_CFIG_A_STDADDR_BASE_MASK));
2611 2611 rcfga_p->value |= ((uint64_t)rbrp->rbb_max << RBR_CFIG_A_LEN_SHIFT);
2612 2612
2613 2613 /* XXXX: how to choose packet buffer sizes */
2614 2614 rcfgb_p->bits.bufsz0 = rbrp->pkt_buf_size0;
2615 2615 rcfgb_p->bits.vld0 = 1;
2616 2616 rcfgb_p->bits.bufsz1 = rbrp->pkt_buf_size1;
2617 2617 rcfgb_p->bits.vld1 = 1;
2618 2618 rcfgb_p->bits.bufsz2 = rbrp->pkt_buf_size2;
2619 2619 rcfgb_p->bits.vld2 = 1;
2620 2620 rcfgb_p->bits.bksize = hxgep->rx_bksize_code;
2621 2621
2622 2622 /*
2623 2623 * For each buffer block, enter receive block address to the ring.
2624 2624 */
2625 2625 rbr_vaddrp = (uint32_t *)dmap->kaddrp;
2626 2626 rbrp->rbr_desc_vp = (uint32_t *)dmap->kaddrp;
2627 2627 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2628 2628 "==> hxge_map_rxdma_channel_cfg_ring: channel %d "
2629 2629 "rbr_vaddrp $%p", dma_channel, rbr_vaddrp));
2630 2630
2631 2631 rx_msg_ring = rbrp->rx_msg_ring;
2632 2632 for (i = 0; i < rbrp->tnblocks; i++) {
2633 2633 rx_msg_p = rx_msg_ring[i];
2634 2634 rx_msg_p->hxgep = hxgep;
2635 2635 rx_msg_p->rx_rbr_p = rbrp;
2636 2636 bkaddr = (uint32_t)
2637 2637 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >>
2638 2638 RBR_BKADDR_SHIFT));
2639 2639 rx_msg_p->free = B_FALSE;
2640 2640 rx_msg_p->max_usage_cnt = 0xbaddcafe;
2641 2641
2642 2642 *rbr_vaddrp++ = bkaddr;
2643 2643 }
2644 2644
2645 2645 kick_p->bits.bkadd = rbrp->rbb_max;
2646 2646 rbrp->rbr_wr_index = (rbrp->rbb_max - 1);
2647 2647
2648 2648 rbrp->rbr_rd_index = 0;
2649 2649
2650 2650 rbrp->rbr_consumed = 0;
2651 2651 rbrp->rbr_used = 0;
2652 2652 rbrp->rbr_use_bcopy = B_TRUE;
2653 2653 rbrp->rbr_bufsize_type = RCR_PKTBUFSZ_0;
2654 2654
2655 2655 /*
2656 2656 * Do bcopy on packets greater than bcopy size once the lo threshold is
2657 2657 * reached. This lo threshold should be less than the hi threshold.
2658 2658 *
2659 2659 * Do bcopy on every packet once the hi threshold is reached.
2660 2660 */
2661 2661 if (hxge_rx_threshold_lo >= hxge_rx_threshold_hi) {
2662 2662 /* default it to use hi */
2663 2663 hxge_rx_threshold_lo = hxge_rx_threshold_hi;
2664 2664 }
2665 2665 if (hxge_rx_buf_size_type > HXGE_RBR_TYPE2) {
2666 2666 hxge_rx_buf_size_type = HXGE_RBR_TYPE2;
2667 2667 }
2668 2668 rbrp->rbr_bufsize_type = hxge_rx_buf_size_type;
2669 2669
2670 2670 switch (hxge_rx_threshold_hi) {
2671 2671 default:
2672 2672 case HXGE_RX_COPY_NONE:
2673 2673 /* Do not do bcopy at all */
2674 2674 rbrp->rbr_use_bcopy = B_FALSE;
2675 2675 rbrp->rbr_threshold_hi = rbrp->rbb_max;
2676 2676 break;
2677 2677
2678 2678 case HXGE_RX_COPY_1:
2679 2679 case HXGE_RX_COPY_2:
2680 2680 case HXGE_RX_COPY_3:
2681 2681 case HXGE_RX_COPY_4:
2682 2682 case HXGE_RX_COPY_5:
2683 2683 case HXGE_RX_COPY_6:
2684 2684 case HXGE_RX_COPY_7:
2685 2685 rbrp->rbr_threshold_hi =
2686 2686 rbrp->rbb_max * (hxge_rx_threshold_hi) /
2687 2687 HXGE_RX_BCOPY_SCALE;
2688 2688 break;
2689 2689
2690 2690 case HXGE_RX_COPY_ALL:
2691 2691 rbrp->rbr_threshold_hi = 0;
2692 2692 break;
2693 2693 }
2694 2694
2695 2695 switch (hxge_rx_threshold_lo) {
2696 2696 default:
2697 2697 case HXGE_RX_COPY_NONE:
2698 2698 /* Do not do bcopy at all */
2699 2699 if (rbrp->rbr_use_bcopy) {
2700 2700 rbrp->rbr_use_bcopy = B_FALSE;
2701 2701 }
2702 2702 rbrp->rbr_threshold_lo = rbrp->rbb_max;
2703 2703 break;
2704 2704
2705 2705 case HXGE_RX_COPY_1:
2706 2706 case HXGE_RX_COPY_2:
2707 2707 case HXGE_RX_COPY_3:
2708 2708 case HXGE_RX_COPY_4:
2709 2709 case HXGE_RX_COPY_5:
2710 2710 case HXGE_RX_COPY_6:
2711 2711 case HXGE_RX_COPY_7:
2712 2712 rbrp->rbr_threshold_lo =
2713 2713 rbrp->rbb_max * (hxge_rx_threshold_lo) /
2714 2714 HXGE_RX_BCOPY_SCALE;
2715 2715 break;
2716 2716
2717 2717 case HXGE_RX_COPY_ALL:
2718 2718 rbrp->rbr_threshold_lo = 0;
2719 2719 break;
2720 2720 }
2721 2721
2722 2722 HXGE_DEBUG_MSG((hxgep, RX_CTL,
2723 2723 "hxge_map_rxdma_channel_cfg_ring: channel %d rbb_max %d "
2724 2724 "rbrp->rbr_bufsize_type %d rbb_threshold_hi %d "
2725 2725 "rbb_threshold_lo %d",
2726 2726 dma_channel, rbrp->rbb_max, rbrp->rbr_bufsize_type,
2727 2727 rbrp->rbr_threshold_hi, rbrp->rbr_threshold_lo));
2728 2728
2729 2729 /* Map in the receive completion ring */
2730 2730 rcrp = (p_rx_rcr_ring_t)KMEM_ZALLOC(sizeof (rx_rcr_ring_t), KM_SLEEP);
2731 2731 MUTEX_INIT(&rcrp->lock, NULL, MUTEX_DRIVER,
2732 2732 (void *) hxgep->interrupt_cookie);
2733 2733 rcrp->rdc = dma_channel;
2734 2734 rcrp->hxgep = hxgep;
2735 2735
2736 2736 hxge_port_rcr_size = hxgep->hxge_port_rcr_size;
2737 2737 rcrp->comp_size = hxge_port_rcr_size;
2738 2738 rcrp->comp_wrap_mask = hxge_port_rcr_size - 1;
2739 2739
2740 2740 cntl_dmap = *dma_rcr_cntl_p;
2741 2741
2742 2742 dmap = (p_hxge_dma_common_t)&rcrp->rcr_desc;
2743 2743 hxge_setup_dma_common(dmap, cntl_dmap, rcrp->comp_size,
2744 2744 sizeof (rcr_entry_t));
2745 2745 rcrp->comp_rd_index = 0;
2746 2746 rcrp->comp_wt_index = 0;
2747 2747 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p =
2748 2748 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc);
2749 2749 #if defined(__i386)
2750 2750 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
2751 2751 (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
2752 2752 #else
2753 2753 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
2754 2754 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
2755 2755 #endif
2756 2756 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p +
2757 2757 (hxge_port_rcr_size - 1);
2758 2758 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp +
2759 2759 (hxge_port_rcr_size - 1);
2760 2760
2761 2761 rcrp->rcr_tail_begin = DMA_COMMON_IOADDR(rcrp->rcr_desc);
2762 2762 rcrp->rcr_tail_begin = (rcrp->rcr_tail_begin & 0x7ffffULL) >> 3;
2763 2763
2764 2764 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2765 2765 "==> hxge_map_rxdma_channel_cfg_ring: channel %d "
2766 2766 "rbr_vaddrp $%p rcr_desc_rd_head_p $%p "
2767 2767 "rcr_desc_rd_head_pp $%p rcr_desc_rd_last_p $%p "
2768 2768 "rcr_desc_rd_last_pp $%p ",
2769 2769 dma_channel, rbr_vaddrp, rcrp->rcr_desc_rd_head_p,
2770 2770 rcrp->rcr_desc_rd_head_pp, rcrp->rcr_desc_last_p,
2771 2771 rcrp->rcr_desc_last_pp));
2772 2772
2773 2773 /*
2774 2774 * Zero out buffer block ring descriptors.
2775 2775 */
2776 2776 bzero((caddr_t)dmap->kaddrp, dmap->alength);
2777 2777 rcrp->intr_timeout = hxgep->intr_timeout;
2778 2778 rcrp->intr_threshold = hxgep->intr_threshold;
2779 2779 rcrp->full_hdr_flag = B_FALSE;
2780 2780 rcrp->sw_priv_hdr_len = 0;
2781 2781
2782 2782 cfga_p = &(rcrp->rcr_cfga);
2783 2783 cfgb_p = &(rcrp->rcr_cfgb);
2784 2784 cfga_p->value = 0;
2785 2785 cfgb_p->value = 0;
2786 2786 rcrp->rcr_addr = dmap->dma_cookie.dmac_laddress;
2787 2787
2788 2788 cfga_p->value = (rcrp->rcr_addr &
2789 2789 (RCRCFIG_A_STADDR_MASK | RCRCFIG_A_STADDR_BASE_MASK));
2790 2790
2791 2791 cfga_p->value |= ((uint64_t)rcrp->comp_size << RCRCFIG_A_LEN_SHIF);
2792 2792
2793 2793 /*
2794 2794 * Timeout should be set based on the system clock divider. The
2795 2795 * following timeout value of 1 assumes that the granularity (1000) is
2796 2796 * 3 microseconds running at 300MHz.
2797 2797 */
2798 2798 cfgb_p->bits.pthres = rcrp->intr_threshold;
2799 2799 cfgb_p->bits.timeout = rcrp->intr_timeout;
2800 2800 cfgb_p->bits.entout = 1;
2801 2801
2802 2802 /* Map in the mailbox */
2803 2803 cntl_dmap = *dma_mbox_cntl_p;
2804 2804 mboxp = (p_rx_mbox_t)KMEM_ZALLOC(sizeof (rx_mbox_t), KM_SLEEP);
2805 2805 dmap = (p_hxge_dma_common_t)&mboxp->rx_mbox;
2806 2806 hxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (rxdma_mailbox_t));
2807 2807 cfig1_p = (rdc_rx_cfg1_t *)&mboxp->rx_cfg1;
2808 2808 cfig2_p = (rdc_rx_cfg2_t *)&mboxp->rx_cfg2;
2809 2809 cfig1_p->value = cfig2_p->value = 0;
2810 2810
2811 2811 mboxp->mbox_addr = dmap->dma_cookie.dmac_laddress;
2812 2812 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2813 2813 "==> hxge_map_rxdma_channel_cfg_ring: "
2814 2814 "channel %d cfg1 0x%016llx cfig2 0x%016llx cookie 0x%016llx",
2815 2815 dma_channel, cfig1_p->value, cfig2_p->value,
2816 2816 mboxp->mbox_addr));
2817 2817
2818 2818 dmaaddrp = (uint32_t)((dmap->dma_cookie.dmac_laddress >> 32) & 0xfff);
2819 2819 cfig1_p->bits.mbaddr_h = dmaaddrp;
2820 2820
2821 2821 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 0xffffffff);
2822 2822 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress &
2823 2823 RXDMA_CFIG2_MBADDR_L_MASK);
2824 2824
2825 2825 cfig2_p->bits.mbaddr_l = (dmaaddrp >> RXDMA_CFIG2_MBADDR_L_SHIFT);
2826 2826
2827 2827 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2828 2828 "==> hxge_map_rxdma_channel_cfg_ring: channel %d damaddrp $%p "
2829 2829 "cfg1 0x%016llx cfig2 0x%016llx",
2830 2830 dma_channel, dmaaddrp, cfig1_p->value, cfig2_p->value));
2831 2831
2832 2832 cfig2_p->bits.full_hdr = rcrp->full_hdr_flag;
2833 2833 cfig2_p->bits.offset = rcrp->sw_priv_hdr_len;
2834 2834
2835 2835 rbrp->rx_rcr_p = rcrp;
2836 2836 rcrp->rx_rbr_p = rbrp;
2837 2837 *rcr_p = rcrp;
2838 2838 *rx_mbox_p = mboxp;
2839 2839
2840 2840 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2841 2841 "<== hxge_map_rxdma_channel_cfg_ring status 0x%08x", status));
2842 2842 return (status);
2843 2843 }
2844 2844
2845 2845 /*ARGSUSED*/
2846 2846 static void
2847 2847 hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep,
2848 2848 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p)
2849 2849 {
2850 2850 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2851 2851 "==> hxge_unmap_rxdma_channel_cfg_ring: channel %d", rcr_p->rdc));
2852 2852
2853 2853 MUTEX_DESTROY(&rcr_p->lock);
2854 2854 KMEM_FREE(rcr_p, sizeof (rx_rcr_ring_t));
2855 2855 KMEM_FREE(rx_mbox_p, sizeof (rx_mbox_t));
2856 2856
2857 2857 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2858 2858 "<== hxge_unmap_rxdma_channel_cfg_ring"));
2859 2859 }
2860 2860
2861 2861 static hxge_status_t
2862 2862 hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep, uint16_t channel,
2863 2863 p_hxge_dma_common_t *dma_buf_p,
2864 2864 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks)
2865 2865 {
2866 2866 p_rx_rbr_ring_t rbrp;
2867 2867 p_hxge_dma_common_t dma_bufp, tmp_bufp;
2868 2868 p_rx_msg_t *rx_msg_ring;
2869 2869 p_rx_msg_t rx_msg_p;
2870 2870 p_mblk_t mblk_p;
2871 2871
2872 2872 rxring_info_t *ring_info;
2873 2873 hxge_status_t status = HXGE_OK;
2874 2874 int i, j, index;
2875 2875 uint32_t size, bsize, nblocks, nmsgs;
2876 2876
2877 2877 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2878 2878 "==> hxge_map_rxdma_channel_buf_ring: channel %d", channel));
2879 2879
2880 2880 dma_bufp = tmp_bufp = *dma_buf_p;
2881 2881 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2882 2882 " hxge_map_rxdma_channel_buf_ring: channel %d to map %d "
2883 2883 "chunks bufp 0x%016llx", channel, num_chunks, dma_bufp));
2884 2884
2885 2885 nmsgs = 0;
2886 2886 for (i = 0; i < num_chunks; i++, tmp_bufp++) {
2887 2887 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2888 2888 "==> hxge_map_rxdma_channel_buf_ring: channel %d "
2889 2889 "bufp 0x%016llx nblocks %d nmsgs %d",
2890 2890 channel, tmp_bufp, tmp_bufp->nblocks, nmsgs));
2891 2891 nmsgs += tmp_bufp->nblocks;
2892 2892 }
2893 2893 if (!nmsgs) {
2894 2894 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2895 2895 "<== hxge_map_rxdma_channel_buf_ring: channel %d "
2896 2896 "no msg blocks", channel));
2897 2897 status = HXGE_ERROR;
2898 2898 goto hxge_map_rxdma_channel_buf_ring_exit;
2899 2899 }
2900 2900 rbrp = (p_rx_rbr_ring_t)KMEM_ZALLOC(sizeof (rx_rbr_ring_t), KM_SLEEP);
2901 2901
2902 2902 size = nmsgs * sizeof (p_rx_msg_t);
2903 2903 rx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP);
2904 2904 ring_info = (rxring_info_t *)KMEM_ZALLOC(sizeof (rxring_info_t),
2905 2905 KM_SLEEP);
2906 2906
2907 2907 MUTEX_INIT(&rbrp->lock, NULL, MUTEX_DRIVER,
2908 2908 (void *) hxgep->interrupt_cookie);
2909 2909 MUTEX_INIT(&rbrp->post_lock, NULL, MUTEX_DRIVER,
2910 2910 (void *) hxgep->interrupt_cookie);
2911 2911
2912 2912 rbrp->rdc = channel;
2913 2913 rbrp->num_blocks = num_chunks;
2914 2914 rbrp->tnblocks = nmsgs;
2915 2915 rbrp->rbb_max = nmsgs;
2916 2916 rbrp->rbr_max_size = nmsgs;
2917 2917 rbrp->rbr_wrap_mask = (rbrp->rbb_max - 1);
2918 2918
2919 2919 /*
2920 2920 * Buffer sizes: 256, 1K, and 2K.
2921 2921 *
2922 2922 * Blk 0 size.
2923 2923 */
2924 2924 rbrp->pkt_buf_size0 = RBR_BUFSZ0_256B;
2925 2925 rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_256_BYTES;
2926 2926 rbrp->hpi_pkt_buf_size0 = SIZE_256B;
2927 2927
2928 2928 /*
2929 2929 * Blk 1 size.
2930 2930 */
2931 2931 rbrp->pkt_buf_size1 = RBR_BUFSZ1_1K;
2932 2932 rbrp->pkt_buf_size1_bytes = RBR_BUFSZ1_1K_BYTES;
2933 2933 rbrp->hpi_pkt_buf_size1 = SIZE_1KB;
2934 2934
2935 2935 /*
2936 2936 * Blk 2 size.
2937 2937 */
2938 2938 rbrp->pkt_buf_size2 = RBR_BUFSZ2_2K;
2939 2939 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_2K_BYTES;
2940 2940 rbrp->hpi_pkt_buf_size2 = SIZE_2KB;
2941 2941
2942 2942 rbrp->block_size = hxgep->rx_default_block_size;
2943 2943
2944 2944 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2945 2945 "==> hxge_map_rxdma_channel_buf_ring: channel %d "
2946 2946 "actual rbr max %d rbb_max %d nmsgs %d "
2947 2947 "rbrp->block_size %d default_block_size %d "
2948 2948 "(config hxge_rbr_size %d hxge_rbr_spare_size %d)",
2949 2949 channel, rbrp->rbr_max_size, rbrp->rbb_max, nmsgs,
2950 2950 rbrp->block_size, hxgep->rx_default_block_size,
2951 2951 hxge_rbr_size, hxge_rbr_spare_size));
2952 2952
2953 2953 /*
2954 2954 * Map in buffers from the buffer pool.
2955 2955 * Note that num_blocks is the num_chunks. For Sparc, there is likely
2956 2956 * only one chunk. For x86, there will be many chunks.
2957 2957 * Loop over chunks.
2958 2958 */
2959 2959 index = 0;
2960 2960 for (i = 0; i < rbrp->num_blocks; i++, dma_bufp++) {
2961 2961 bsize = dma_bufp->block_size;
2962 2962 nblocks = dma_bufp->nblocks;
2963 2963 #if defined(__i386)
2964 2964 ring_info->buffer[i].dvma_addr = (uint32_t)dma_bufp->ioaddr_pp;
2965 2965 #else
2966 2966 ring_info->buffer[i].dvma_addr = (uint64_t)dma_bufp->ioaddr_pp;
2967 2967 #endif
2968 2968 ring_info->buffer[i].buf_index = i;
2969 2969 ring_info->buffer[i].buf_size = dma_bufp->alength;
2970 2970 ring_info->buffer[i].start_index = index;
2971 2971 #if defined(__i386)
2972 2972 ring_info->buffer[i].kaddr = (uint32_t)dma_bufp->kaddrp;
2973 2973 #else
2974 2974 ring_info->buffer[i].kaddr = (uint64_t)dma_bufp->kaddrp;
2975 2975 #endif
2976 2976
2977 2977 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2978 2978 " hxge_map_rxdma_channel_buf_ring: map channel %d "
2979 2979 "chunk %d nblocks %d chunk_size %x block_size 0x%x "
2980 2980 "dma_bufp $%p dvma_addr $%p", channel, i,
2981 2981 dma_bufp->nblocks,
2982 2982 ring_info->buffer[i].buf_size, bsize, dma_bufp,
2983 2983 ring_info->buffer[i].dvma_addr));
2984 2984
2985 2985 /* loop over blocks within a chunk */
2986 2986 for (j = 0; j < nblocks; j++) {
2987 2987 if ((rx_msg_p = hxge_allocb(bsize, BPRI_LO,
2988 2988 dma_bufp)) == NULL) {
2989 2989 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2990 2990 "allocb failed (index %d i %d j %d)",
2991 2991 index, i, j));
2992 2992 goto hxge_map_rxdma_channel_buf_ring_fail1;
2993 2993 }
2994 2994 rx_msg_ring[index] = rx_msg_p;
2995 2995 rx_msg_p->block_index = index;
2996 2996 rx_msg_p->shifted_addr = (uint32_t)
2997 2997 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >>
2998 2998 RBR_BKADDR_SHIFT));
2999 2999 /*
3000 3000 * Too much output
3001 3001 * HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3002 3002 * "index %d j %d rx_msg_p $%p mblk %p",
3003 3003 * index, j, rx_msg_p, rx_msg_p->rx_mblk_p));
3004 3004 */
3005 3005 mblk_p = rx_msg_p->rx_mblk_p;
3006 3006 mblk_p->b_wptr = mblk_p->b_rptr + bsize;
3007 3007
3008 3008 rbrp->rbr_ref_cnt++;
3009 3009 index++;
3010 3010 rx_msg_p->buf_dma.dma_channel = channel;
3011 3011 }
3012 3012 }
3013 3013 if (i < rbrp->num_blocks) {
3014 3014 goto hxge_map_rxdma_channel_buf_ring_fail1;
3015 3015 }
3016 3016 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3017 3017 "hxge_map_rxdma_channel_buf_ring: done buf init "
3018 3018 "channel %d msg block entries %d", channel, index));
3019 3019 ring_info->block_size_mask = bsize - 1;
3020 3020 rbrp->rx_msg_ring = rx_msg_ring;
3021 3021 rbrp->dma_bufp = dma_buf_p;
3022 3022 rbrp->ring_info = ring_info;
3023 3023
3024 3024 status = hxge_rxbuf_index_info_init(hxgep, rbrp);
3025 3025 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, " hxge_map_rxdma_channel_buf_ring: "
3026 3026 "channel %d done buf info init", channel));
3027 3027
3028 3028 /*
3029 3029 * Finally, permit hxge_freeb() to call hxge_post_page().
3030 3030 */
3031 3031 rbrp->rbr_state = RBR_POSTING;
3032 3032
3033 3033 *rbr_p = rbrp;
3034 3034
3035 3035 goto hxge_map_rxdma_channel_buf_ring_exit;
3036 3036
3037 3037 hxge_map_rxdma_channel_buf_ring_fail1:
3038 3038 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3039 3039 " hxge_map_rxdma_channel_buf_ring: failed channel (0x%x)",
3040 3040 channel, status));
3041 3041
3042 3042 index--;
3043 3043 for (; index >= 0; index--) {
3044 3044 rx_msg_p = rx_msg_ring[index];
3045 3045 if (rx_msg_p != NULL) {
3046 3046 freeb(rx_msg_p->rx_mblk_p);
3047 3047 rx_msg_ring[index] = NULL;
3048 3048 }
3049 3049 }
3050 3050
3051 3051 hxge_map_rxdma_channel_buf_ring_fail:
3052 3052 MUTEX_DESTROY(&rbrp->post_lock);
3053 3053 MUTEX_DESTROY(&rbrp->lock);
3054 3054 KMEM_FREE(ring_info, sizeof (rxring_info_t));
3055 3055 KMEM_FREE(rx_msg_ring, size);
3056 3056 KMEM_FREE(rbrp, sizeof (rx_rbr_ring_t));
3057 3057
3058 3058 status = HXGE_ERROR;
3059 3059
3060 3060 hxge_map_rxdma_channel_buf_ring_exit:
3061 3061 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3062 3062 "<== hxge_map_rxdma_channel_buf_ring status 0x%08x", status));
3063 3063
3064 3064 return (status);
3065 3065 }
3066 3066
3067 3067 /*ARGSUSED*/
3068 3068 static void
3069 3069 hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep,
3070 3070 p_rx_rbr_ring_t rbr_p)
3071 3071 {
3072 3072 p_rx_msg_t *rx_msg_ring;
3073 3073 p_rx_msg_t rx_msg_p;
3074 3074 rxring_info_t *ring_info;
3075 3075 int i;
3076 3076 uint32_t size;
3077 3077
3078 3078 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3079 3079 "==> hxge_unmap_rxdma_channel_buf_ring"));
3080 3080 if (rbr_p == NULL) {
3081 3081 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3082 3082 "<== hxge_unmap_rxdma_channel_buf_ring: NULL rbrp"));
3083 3083 return;
3084 3084 }
3085 3085 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3086 3086 "==> hxge_unmap_rxdma_channel_buf_ring: channel %d", rbr_p->rdc));
3087 3087
3088 3088 rx_msg_ring = rbr_p->rx_msg_ring;
3089 3089 ring_info = rbr_p->ring_info;
3090 3090
3091 3091 if (rx_msg_ring == NULL || ring_info == NULL) {
3092 3092 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3093 3093 "<== hxge_unmap_rxdma_channel_buf_ring: "
3094 3094 "rx_msg_ring $%p ring_info $%p", rx_msg_p, ring_info));
3095 3095 return;
3096 3096 }
3097 3097
3098 3098 size = rbr_p->tnblocks * sizeof (p_rx_msg_t);
3099 3099 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3100 3100 " hxge_unmap_rxdma_channel_buf_ring: channel %d chunks %d "
3101 3101 "tnblocks %d (max %d) size ptrs %d ", rbr_p->rdc, rbr_p->num_blocks,
3102 3102 rbr_p->tnblocks, rbr_p->rbr_max_size, size));
3103 3103
3104 3104 for (i = 0; i < rbr_p->tnblocks; i++) {
3105 3105 rx_msg_p = rx_msg_ring[i];
3106 3106 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3107 3107 " hxge_unmap_rxdma_channel_buf_ring: "
3108 3108 "rx_msg_p $%p", rx_msg_p));
3109 3109 if (rx_msg_p != NULL) {
3110 3110 freeb(rx_msg_p->rx_mblk_p);
3111 3111 rx_msg_ring[i] = NULL;
3112 3112 }
3113 3113 }
3114 3114
3115 3115 /*
3116 3116 * We no longer may use the mutex <post_lock>. By setting
3117 3117 * <rbr_state> to anything but POSTING, we prevent
3118 3118 * hxge_post_page() from accessing a dead mutex.
3119 3119 */
3120 3120 rbr_p->rbr_state = RBR_UNMAPPING;
3121 3121 MUTEX_DESTROY(&rbr_p->post_lock);
3122 3122
3123 3123 MUTEX_DESTROY(&rbr_p->lock);
3124 3124 KMEM_FREE(ring_info, sizeof (rxring_info_t));
3125 3125 KMEM_FREE(rx_msg_ring, size);
3126 3126
3127 3127 if (rbr_p->rbr_ref_cnt == 0) {
3128 3128 /* This is the normal state of affairs. */
3129 3129 KMEM_FREE(rbr_p, sizeof (*rbr_p));
3130 3130 } else {
3131 3131 /*
3132 3132 * Some of our buffers are still being used.
3133 3133 * Therefore, tell hxge_freeb() this ring is
3134 3134 * unmapped, so it may free <rbr_p> for us.
3135 3135 */
3136 3136 rbr_p->rbr_state = RBR_UNMAPPED;
3137 3137 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3138 3138 "unmap_rxdma_buf_ring: %d %s outstanding.",
3139 3139 rbr_p->rbr_ref_cnt,
3140 3140 rbr_p->rbr_ref_cnt == 1 ? "msg" : "msgs"));
3141 3141 }
3142 3142
3143 3143 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3144 3144 "<== hxge_unmap_rxdma_channel_buf_ring"));
3145 3145 }
3146 3146
3147 3147 static hxge_status_t
3148 3148 hxge_rxdma_hw_start_common(p_hxge_t hxgep)
3149 3149 {
3150 3150 hxge_status_t status = HXGE_OK;
3151 3151
3152 3152 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start_common"));
3153 3153
3154 3154 /*
3155 3155 * Load the sharable parameters by writing to the function zero control
3156 3156 * registers. These FZC registers should be initialized only once for
3157 3157 * the entire chip.
3158 3158 */
3159 3159 (void) hxge_init_fzc_rx_common(hxgep);
3160 3160
3161 3161 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start_common"));
3162 3162
3163 3163 return (status);
3164 3164 }
3165 3165
3166 3166 static hxge_status_t
3167 3167 hxge_rxdma_hw_start(p_hxge_t hxgep)
3168 3168 {
3169 3169 int i, ndmas;
3170 3170 uint16_t channel;
3171 3171 p_rx_rbr_rings_t rx_rbr_rings;
3172 3172 p_rx_rbr_ring_t *rbr_rings;
3173 3173 p_rx_rcr_rings_t rx_rcr_rings;
3174 3174 p_rx_rcr_ring_t *rcr_rings;
3175 3175 p_rx_mbox_areas_t rx_mbox_areas_p;
3176 3176 p_rx_mbox_t *rx_mbox_p;
3177 3177 hxge_status_t status = HXGE_OK;
3178 3178
3179 3179 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start"));
3180 3180
3181 3181 rx_rbr_rings = hxgep->rx_rbr_rings;
3182 3182 rx_rcr_rings = hxgep->rx_rcr_rings;
3183 3183 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) {
3184 3184 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3185 3185 "<== hxge_rxdma_hw_start: NULL ring pointers"));
3186 3186 return (HXGE_ERROR);
3187 3187 }
3188 3188
3189 3189 ndmas = rx_rbr_rings->ndmas;
3190 3190 if (ndmas == 0) {
3191 3191 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3192 3192 "<== hxge_rxdma_hw_start: no dma channel allocated"));
3193 3193 return (HXGE_ERROR);
3194 3194 }
3195 3195 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3196 3196 "==> hxge_rxdma_hw_start (ndmas %d)", ndmas));
3197 3197
3198 3198 /*
3199 3199 * Scrub the RDC Rx DMA Prefetch Buffer Command.
3200 3200 */
3201 3201 for (i = 0; i < 128; i++) {
3202 3202 HXGE_REG_WR64(hxgep->hpi_handle, RDC_PREF_CMD, i);
3203 3203 }
3204 3204
3205 3205 /*
3206 3206 * Scrub Rx DMA Shadow Tail Command.
3207 3207 */
3208 3208 for (i = 0; i < 64; i++) {
3209 3209 HXGE_REG_WR64(hxgep->hpi_handle, RDC_SHADOW_CMD, i);
3210 3210 }
3211 3211
3212 3212 /*
3213 3213 * Scrub Rx DMA Control Fifo Command.
3214 3214 */
3215 3215 for (i = 0; i < 512; i++) {
3216 3216 HXGE_REG_WR64(hxgep->hpi_handle, RDC_CTRL_FIFO_CMD, i);
3217 3217 }
3218 3218
3219 3219 /*
3220 3220 * Scrub Rx DMA Data Fifo Command.
3221 3221 */
3222 3222 for (i = 0; i < 1536; i++) {
3223 3223 HXGE_REG_WR64(hxgep->hpi_handle, RDC_DATA_FIFO_CMD, i);
3224 3224 }
3225 3225
3226 3226 /*
3227 3227 * Reset the FIFO Error Stat.
3228 3228 */
3229 3229 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_STAT, 0xFF);
3230 3230
3231 3231 /* Set the error mask to receive interrupts */
3232 3232 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_INT_MASK, 0x0);
3233 3233
3234 3234 rbr_rings = rx_rbr_rings->rbr_rings;
3235 3235 rcr_rings = rx_rcr_rings->rcr_rings;
3236 3236 rx_mbox_areas_p = hxgep->rx_mbox_areas_p;
3237 3237 if (rx_mbox_areas_p) {
3238 3238 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas;
3239 3239 }
3240 3240
3241 3241 for (i = 0; i < ndmas; i++) {
3242 3242 channel = rbr_rings[i]->rdc;
3243 3243 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3244 3244 "==> hxge_rxdma_hw_start (ndmas %d) channel %d",
3245 3245 ndmas, channel));
3246 3246 status = hxge_rxdma_start_channel(hxgep, channel,
3247 3247 (p_rx_rbr_ring_t)rbr_rings[i],
3248 3248 (p_rx_rcr_ring_t)rcr_rings[i],
3249 3249 (p_rx_mbox_t)rx_mbox_p[i], rbr_rings[i]->rbb_max);
3250 3250 if (status != HXGE_OK) {
3251 3251 goto hxge_rxdma_hw_start_fail1;
3252 3252 }
3253 3253 }
3254 3254
3255 3255 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start: "
3256 3256 "rx_rbr_rings 0x%016llx rings 0x%016llx",
3257 3257 rx_rbr_rings, rx_rcr_rings));
3258 3258 goto hxge_rxdma_hw_start_exit;
3259 3259
3260 3260 hxge_rxdma_hw_start_fail1:
3261 3261 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3262 3262 "==> hxge_rxdma_hw_start: disable "
3263 3263 "(status 0x%x channel %d i %d)", status, channel, i));
3264 3264 for (; i >= 0; i--) {
3265 3265 channel = rbr_rings[i]->rdc;
3266 3266 (void) hxge_rxdma_stop_channel(hxgep, channel);
3267 3267 }
3268 3268
3269 3269 hxge_rxdma_hw_start_exit:
3270 3270 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3271 3271 "==> hxge_rxdma_hw_start: (status 0x%x)", status));
3272 3272 return (status);
3273 3273 }
3274 3274
3275 3275 static void
3276 3276 hxge_rxdma_hw_stop(p_hxge_t hxgep)
3277 3277 {
3278 3278 int i, ndmas;
3279 3279 uint16_t channel;
3280 3280 p_rx_rbr_rings_t rx_rbr_rings;
3281 3281 p_rx_rbr_ring_t *rbr_rings;
3282 3282 p_rx_rcr_rings_t rx_rcr_rings;
3283 3283
3284 3284 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_stop"));
3285 3285
3286 3286 rx_rbr_rings = hxgep->rx_rbr_rings;
3287 3287 rx_rcr_rings = hxgep->rx_rcr_rings;
3288 3288
3289 3289 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) {
3290 3290 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3291 3291 "<== hxge_rxdma_hw_stop: NULL ring pointers"));
3292 3292 return;
3293 3293 }
3294 3294
3295 3295 ndmas = rx_rbr_rings->ndmas;
3296 3296 if (!ndmas) {
3297 3297 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3298 3298 "<== hxge_rxdma_hw_stop: no dma channel allocated"));
3299 3299 return;
3300 3300 }
3301 3301
3302 3302 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3303 3303 "==> hxge_rxdma_hw_stop (ndmas %d)", ndmas));
3304 3304
3305 3305 rbr_rings = rx_rbr_rings->rbr_rings;
3306 3306 for (i = 0; i < ndmas; i++) {
3307 3307 channel = rbr_rings[i]->rdc;
3308 3308 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3309 3309 "==> hxge_rxdma_hw_stop (ndmas %d) channel %d",
3310 3310 ndmas, channel));
3311 3311 (void) hxge_rxdma_stop_channel(hxgep, channel);
3312 3312 }
3313 3313
3314 3314 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_stop: "
3315 3315 "rx_rbr_rings 0x%016llx rings 0x%016llx",
3316 3316 rx_rbr_rings, rx_rcr_rings));
3317 3317
3318 3318 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_rxdma_hw_stop"));
3319 3319 }
3320 3320
3321 3321 static hxge_status_t
3322 3322 hxge_rxdma_start_channel(p_hxge_t hxgep, uint16_t channel,
3323 3323 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p,
3324 3324 int n_init_kick)
3325 3325 {
3326 3326 hpi_handle_t handle;
3327 3327 hpi_status_t rs = HPI_SUCCESS;
3328 3328 rdc_stat_t cs;
3329 3329 rdc_int_mask_t ent_mask;
3330 3330 hxge_status_t status = HXGE_OK;
3331 3331
3332 3332 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel"));
3333 3333
3334 3334 handle = HXGE_DEV_HPI_HANDLE(hxgep);
3335 3335
3336 3336 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "hxge_rxdma_start_channel: "
3337 3337 "hpi handle addr $%p acc $%p",
3338 3338 hxgep->hpi_handle.regp, hxgep->hpi_handle.regh));
3339 3339
3340 3340 /* Reset RXDMA channel */
3341 3341 rs = hpi_rxdma_cfg_rdc_reset(handle, channel);
3342 3342 if (rs != HPI_SUCCESS) {
3343 3343 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3344 3344 "==> hxge_rxdma_start_channel: "
3345 3345 "reset rxdma failed (0x%08x channel %d)",
3346 3346 status, channel));
3347 3347 return (HXGE_ERROR | rs);
3348 3348 }
3349 3349 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3350 3350 "==> hxge_rxdma_start_channel: reset done: channel %d", channel));
3351 3351
3352 3352 /*
3353 3353 * Initialize the RXDMA channel specific FZC control configurations.
3354 3354 * These FZC registers are pertaining to each RX channel (logical
3355 3355 * pages).
3356 3356 */
3357 3357 status = hxge_init_fzc_rxdma_channel(hxgep,
3358 3358 channel, rbr_p, rcr_p, mbox_p);
3359 3359 if (status != HXGE_OK) {
3360 3360 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3361 3361 "==> hxge_rxdma_start_channel: "
3362 3362 "init fzc rxdma failed (0x%08x channel %d)",
3363 3363 status, channel));
3364 3364 return (status);
3365 3365 }
3366 3366 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3367 3367 "==> hxge_rxdma_start_channel: fzc done"));
3368 3368
3369 3369 /*
3370 3370 * Zero out the shadow and prefetch ram.
3371 3371 */
3372 3372 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3373 3373 "==> hxge_rxdma_start_channel: ram done"));
3374 3374
3375 3375 /* Set up the interrupt event masks. */
3376 3376 ent_mask.value = 0;
3377 3377 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask);
3378 3378 if (rs != HPI_SUCCESS) {
3379 3379 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3380 3380 "==> hxge_rxdma_start_channel: "
3381 3381 "init rxdma event masks failed (0x%08x channel %d)",
3382 3382 status, channel));
3383 3383 return (HXGE_ERROR | rs);
3384 3384 }
3385 3385 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: "
3386 3386 "event done: channel %d (mask 0x%016llx)",
3387 3387 channel, ent_mask.value));
3388 3388
3389 3389 /*
3390 3390 * Load RXDMA descriptors, buffers, mailbox, initialise the receive DMA
3391 3391 * channels and enable each DMA channel.
3392 3392 */
3393 3393 status = hxge_enable_rxdma_channel(hxgep,
3394 3394 channel, rbr_p, rcr_p, mbox_p, n_init_kick);
3395 3395 if (status != HXGE_OK) {
3396 3396 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3397 3397 " hxge_rxdma_start_channel: "
3398 3398 " init enable rxdma failed (0x%08x channel %d)",
3399 3399 status, channel));
3400 3400 return (status);
3401 3401 }
3402 3402
3403 3403 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: "
3404 3404 "control done - channel %d cs 0x%016llx", channel, cs.value));
3405 3405
3406 3406 /*
3407 3407 * Initialize the receive DMA control and status register
3408 3408 * Note that rdc_stat HAS to be set after RBR and RCR rings are set
3409 3409 */
3410 3410 cs.value = 0;
3411 3411 cs.bits.mex = 1;
3412 3412 cs.bits.rcr_thres = 1;
3413 3413 cs.bits.rcr_to = 1;
3414 3414 cs.bits.rbr_empty = 1;
3415 3415 status = hxge_init_rxdma_channel_cntl_stat(hxgep, channel, &cs);
3416 3416 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: "
3417 3417 "channel %d rx_dma_cntl_stat 0x%0016llx", channel, cs.value));
3418 3418 if (status != HXGE_OK) {
3419 3419 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3420 3420 "==> hxge_rxdma_start_channel: "
3421 3421 "init rxdma control register failed (0x%08x channel %d",
3422 3422 status, channel));
3423 3423 return (status);
3424 3424 }
3425 3425
3426 3426 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: "
3427 3427 "control done - channel %d cs 0x%016llx", channel, cs.value));
3428 3428 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3429 3429 "==> hxge_rxdma_start_channel: enable done"));
3430 3430 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_rxdma_start_channel"));
3431 3431 return (HXGE_OK);
3432 3432 }
3433 3433
3434 3434 static hxge_status_t
3435 3435 hxge_rxdma_stop_channel(p_hxge_t hxgep, uint16_t channel)
3436 3436 {
3437 3437 hpi_handle_t handle;
3438 3438 hpi_status_t rs = HPI_SUCCESS;
3439 3439 rdc_stat_t cs;
3440 3440 rdc_int_mask_t ent_mask;
3441 3441 hxge_status_t status = HXGE_OK;
3442 3442
3443 3443 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_channel"));
3444 3444
3445 3445 handle = HXGE_DEV_HPI_HANDLE(hxgep);
3446 3446
3447 3447 HXGE_DEBUG_MSG((hxgep, RX_CTL, "hxge_rxdma_stop_channel: "
3448 3448 "hpi handle addr $%p acc $%p",
3449 3449 hxgep->hpi_handle.regp, hxgep->hpi_handle.regh));
3450 3450
3451 3451 /* Reset RXDMA channel */
3452 3452 rs = hpi_rxdma_cfg_rdc_reset(handle, channel);
3453 3453 if (rs != HPI_SUCCESS) {
3454 3454 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3455 3455 " hxge_rxdma_stop_channel: "
3456 3456 " reset rxdma failed (0x%08x channel %d)",
3457 3457 rs, channel));
3458 3458 return (HXGE_ERROR | rs);
3459 3459 }
3460 3460 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3461 3461 "==> hxge_rxdma_stop_channel: reset done"));
3462 3462
3463 3463 /* Set up the interrupt event masks. */
3464 3464 ent_mask.value = RDC_INT_MASK_ALL;
3465 3465 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask);
3466 3466 if (rs != HPI_SUCCESS) {
3467 3467 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3468 3468 "==> hxge_rxdma_stop_channel: "
3469 3469 "set rxdma event masks failed (0x%08x channel %d)",
3470 3470 rs, channel));
3471 3471 return (HXGE_ERROR | rs);
3472 3472 }
3473 3473 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3474 3474 "==> hxge_rxdma_stop_channel: event done"));
3475 3475
3476 3476 /* Initialize the receive DMA control and status register */
3477 3477 cs.value = 0;
3478 3478 status = hxge_init_rxdma_channel_cntl_stat(hxgep, channel, &cs);
3479 3479
3480 3480 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_channel: control "
3481 3481 " to default (all 0s) 0x%08x", cs.value));
3482 3482
3483 3483 if (status != HXGE_OK) {
3484 3484 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3485 3485 " hxge_rxdma_stop_channel: init rxdma"
3486 3486 " control register failed (0x%08x channel %d",
3487 3487 status, channel));
3488 3488 return (status);
3489 3489 }
3490 3490
3491 3491 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3492 3492 "==> hxge_rxdma_stop_channel: control done"));
3493 3493
3494 3494 /* disable dma channel */
3495 3495 status = hxge_disable_rxdma_channel(hxgep, channel);
3496 3496
3497 3497 if (status != HXGE_OK) {
3498 3498 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3499 3499 " hxge_rxdma_stop_channel: "
3500 3500 " init enable rxdma failed (0x%08x channel %d)",
3501 3501 status, channel));
3502 3502 return (status);
3503 3503 }
3504 3504
3505 3505 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3506 3506 "==> hxge_rxdma_stop_channel: disable done"));
3507 3507 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop_channel"));
3508 3508
3509 3509 return (HXGE_OK);
3510 3510 }
3511 3511
3512 3512 hxge_status_t
3513 3513 hxge_rxdma_handle_sys_errors(p_hxge_t hxgep)
3514 3514 {
3515 3515 hpi_handle_t handle;
3516 3516 p_hxge_rdc_sys_stats_t statsp;
3517 3517 rdc_fifo_err_stat_t stat;
3518 3518 hxge_status_t status = HXGE_OK;
3519 3519
3520 3520 handle = hxgep->hpi_handle;
3521 3521 statsp = (p_hxge_rdc_sys_stats_t)&hxgep->statsp->rdc_sys_stats;
3522 3522
3523 3523 /* Get the error status and clear the register */
3524 3524 HXGE_REG_RD64(handle, RDC_FIFO_ERR_STAT, &stat.value);
3525 3525 HXGE_REG_WR64(handle, RDC_FIFO_ERR_STAT, stat.value);
3526 3526
3527 3527 if (stat.bits.rx_ctrl_fifo_sec) {
3528 3528 statsp->ctrl_fifo_sec++;
3529 3529 if (statsp->ctrl_fifo_sec == 1)
3530 3530 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3531 3531 "==> hxge_rxdma_handle_sys_errors: "
3532 3532 "rx_ctrl_fifo_sec"));
3533 3533 }
3534 3534
3535 3535 if (stat.bits.rx_ctrl_fifo_ded) {
3536 3536 /* Global fatal error encountered */
3537 3537 statsp->ctrl_fifo_ded++;
3538 3538 HXGE_FM_REPORT_ERROR(hxgep, NULL,
3539 3539 HXGE_FM_EREPORT_RDMC_CTRL_FIFO_DED);
3540 3540 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3541 3541 "==> hxge_rxdma_handle_sys_errors: "
3542 3542 "fatal error: rx_ctrl_fifo_ded error"));
3543 3543 }
3544 3544
3545 3545 if (stat.bits.rx_data_fifo_sec) {
3546 3546 statsp->data_fifo_sec++;
3547 3547 if (statsp->data_fifo_sec == 1)
3548 3548 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3549 3549 "==> hxge_rxdma_handle_sys_errors: "
3550 3550 "rx_data_fifo_sec"));
3551 3551 }
3552 3552
3553 3553 if (stat.bits.rx_data_fifo_ded) {
3554 3554 /* Global fatal error encountered */
3555 3555 statsp->data_fifo_ded++;
3556 3556 HXGE_FM_REPORT_ERROR(hxgep, NULL,
3557 3557 HXGE_FM_EREPORT_RDMC_DATA_FIFO_DED);
3558 3558 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3559 3559 "==> hxge_rxdma_handle_sys_errors: "
3560 3560 "fatal error: rx_data_fifo_ded error"));
3561 3561 }
3562 3562
3563 3563 if (stat.bits.rx_ctrl_fifo_ded || stat.bits.rx_data_fifo_ded) {
3564 3564 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3565 3565 " hxge_rxdma_handle_sys_errors: fatal error\n"));
3566 3566 status = hxge_rx_port_fatal_err_recover(hxgep);
3567 3567 if (status == HXGE_OK) {
3568 3568 FM_SERVICE_RESTORED(hxgep);
3569 3569 }
3570 3570 }
3571 3571
3572 3572 return (HXGE_OK);
3573 3573 }
3574 3574
3575 3575 static hxge_status_t
3576 3576 hxge_rxdma_fatal_err_recover(p_hxge_t hxgep, uint16_t channel)
3577 3577 {
3578 3578 hpi_handle_t handle;
3579 3579 hpi_status_t rs = HPI_SUCCESS;
3580 3580 p_rx_rbr_ring_t rbrp;
3581 3581 p_rx_rcr_ring_t rcrp;
3582 3582 p_rx_mbox_t mboxp;
3583 3583 rdc_int_mask_t ent_mask;
3584 3584 p_hxge_dma_common_t dmap;
3585 3585 p_rx_msg_t rx_msg_p;
3586 3586 int i;
3587 3587 uint32_t hxge_port_rcr_size;
3588 3588 uint64_t tmp;
3589 3589 int n_init_kick = 0;
3590 3590
3591 3591 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_fatal_err_recover"));
3592 3592
3593 3593 /*
3594 3594 * Stop the dma channel waits for the stop done. If the stop done bit
3595 3595 * is not set, then create an error.
3596 3596 */
3597 3597
3598 3598 handle = HXGE_DEV_HPI_HANDLE(hxgep);
3599 3599
3600 3600 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Rx DMA stop..."));
3601 3601
3602 3602 rbrp = (p_rx_rbr_ring_t)hxgep->rx_rbr_rings->rbr_rings[channel];
3603 3603 rcrp = (p_rx_rcr_ring_t)hxgep->rx_rcr_rings->rcr_rings[channel];
3604 3604
3605 3605 MUTEX_ENTER(&rcrp->lock);
3606 3606 MUTEX_ENTER(&rbrp->lock);
3607 3607
3608 3608 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxDMA channel..."));
3609 3609
3610 3610 rs = hpi_rxdma_cfg_rdc_disable(handle, channel);
3611 3611 if (rs != HPI_SUCCESS) {
3612 3612 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3613 3613 "hxge_disable_rxdma_channel:failed"));
3614 3614 goto fail;
3615 3615 }
3616 3616 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxDMA interrupt..."));
3617 3617
3618 3618 /* Disable interrupt */
3619 3619 ent_mask.value = RDC_INT_MASK_ALL;
3620 3620 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask);
3621 3621 if (rs != HPI_SUCCESS) {
3622 3622 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3623 3623 "Set rxdma event masks failed (channel %d)", channel));
3624 3624 }
3625 3625 HXGE_DEBUG_MSG((hxgep, RX_CTL, "RxDMA channel reset..."));
3626 3626
3627 3627 /* Reset RXDMA channel */
3628 3628 rs = hpi_rxdma_cfg_rdc_reset(handle, channel);
3629 3629 if (rs != HPI_SUCCESS) {
3630 3630 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3631 3631 "Reset rxdma failed (channel %d)", channel));
3632 3632 goto fail;
3633 3633 }
3634 3634 hxge_port_rcr_size = hxgep->hxge_port_rcr_size;
3635 3635 mboxp = (p_rx_mbox_t)hxgep->rx_mbox_areas_p->rxmbox_areas[channel];
3636 3636
3637 3637 rbrp->rbr_wr_index = (rbrp->rbb_max - 1);
3638 3638 rbrp->rbr_rd_index = 0;
3639 3639
3640 3640 rcrp->comp_rd_index = 0;
3641 3641 rcrp->comp_wt_index = 0;
3642 3642 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p =
3643 3643 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc);
3644 3644 #if defined(__i386)
3645 3645 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
3646 3646 (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
3647 3647 #else
3648 3648 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
3649 3649 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
3650 3650 #endif
3651 3651
3652 3652 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p +
3653 3653 (hxge_port_rcr_size - 1);
3654 3654 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp +
3655 3655 (hxge_port_rcr_size - 1);
3656 3656
3657 3657 rcrp->rcr_tail_begin = DMA_COMMON_IOADDR(rcrp->rcr_desc);
3658 3658 rcrp->rcr_tail_begin = (rcrp->rcr_tail_begin & 0x7ffffULL) >> 3;
3659 3659
3660 3660 dmap = (p_hxge_dma_common_t)&rcrp->rcr_desc;
3661 3661 bzero((caddr_t)dmap->kaddrp, dmap->alength);
3662 3662
3663 3663 HXGE_DEBUG_MSG((hxgep, RX_CTL, "rbr entries = %d\n",
3664 3664 rbrp->rbr_max_size));
3665 3665
3666 3666 /* Count the number of buffers owned by the hardware at this moment */
3667 3667 for (i = 0; i < rbrp->rbr_max_size; i++) {
3668 3668 rx_msg_p = rbrp->rx_msg_ring[i];
3669 3669 if (rx_msg_p->ref_cnt == 1) {
3670 3670 n_init_kick++;
3671 3671 }
3672 3672 }
3673 3673
3674 3674 HXGE_DEBUG_MSG((hxgep, RX_CTL, "RxDMA channel re-start..."));
3675 3675
3676 3676 /*
3677 3677 * This is error recover! Some buffers are owned by the hardware and
3678 3678 * the rest are owned by the apps. We should only kick in those
3679 3679 * owned by the hardware initially. The apps will post theirs
3680 3680 * eventually.
3681 3681 */
3682 3682 (void) hxge_rxdma_start_channel(hxgep, channel, rbrp, rcrp, mboxp,
3683 3683 n_init_kick);
3684 3684
3685 3685 /*
3686 3686 * The DMA channel may disable itself automatically.
3687 3687 * The following is a work-around.
3688 3688 */
3689 3689 HXGE_REG_RD64(handle, RDC_RX_CFG1, &tmp);
3690 3690 rs = hpi_rxdma_cfg_rdc_enable(handle, channel);
3691 3691 if (rs != HPI_SUCCESS) {
3692 3692 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3693 3693 "hpi_rxdma_cfg_rdc_enable (channel %d)", channel));
3694 3694 }
3695 3695
3696 3696 /*
3697 3697 * Delay a bit of time by doing reads.
3698 3698 */
3699 3699 for (i = 0; i < 1024; i++) {
3700 3700 uint64_t value;
3701 3701 RXDMA_REG_READ64(HXGE_DEV_HPI_HANDLE(hxgep),
3702 3702 RDC_INT_MASK, i & 3, &value);
3703 3703 }
3704 3704
3705 3705 MUTEX_EXIT(&rbrp->lock);
3706 3706 MUTEX_EXIT(&rcrp->lock);
3707 3707
3708 3708 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_fatal_err_recover"));
3709 3709 return (HXGE_OK);
3710 3710
3711 3711 fail:
3712 3712 MUTEX_EXIT(&rbrp->lock);
3713 3713 MUTEX_EXIT(&rcrp->lock);
3714 3714 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3715 3715 "Error Recovery failed for channel(%d)", channel));
3716 3716 return (HXGE_ERROR | rs);
3717 3717 }
3718 3718
3719 3719 static hxge_status_t
3720 3720 hxge_rx_port_fatal_err_recover(p_hxge_t hxgep)
3721 3721 {
3722 3722 hxge_status_t status = HXGE_OK;
3723 3723 p_hxge_dma_common_t *dma_buf_p;
3724 3724 uint16_t channel;
3725 3725 int ndmas;
3726 3726 int i;
3727 3727 block_reset_t reset_reg;
3728 3728 p_rx_rcr_ring_t rcrp;
3729 3729 p_rx_rbr_ring_t rbrp;
3730 3730
3731 3731 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_port_fatal_err_recover"));
3732 3732 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovering from RDC error ..."));
3733 3733
3734 3734 /* Disable RxMAC */
3735 3735 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxMAC...\n"));
3736 3736 MUTEX_ENTER(&hxgep->vmac_lock);
3737 3737 if (hxge_rx_vmac_disable(hxgep) != HXGE_OK)
3738 3738 goto fail;
3739 3739
3740 3740 HXGE_DELAY(1000);
3741 3741
3742 3742 /*
3743 3743 * Reset RDC block from PEU for this fatal error
3744 3744 */
3745 3745 reset_reg.value = 0;
3746 3746 reset_reg.bits.rdc_rst = 1;
3747 3747 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value);
3748 3748
3749 3749 HXGE_DELAY(1000);
3750 3750
3751 3751 /* Restore any common settings after PEU reset */
3752 3752 if (hxge_rxdma_hw_start_common(hxgep) != HXGE_OK)
3753 3753 goto fail;
3754 3754
3755 3755 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Stop all RxDMA channels..."));
3756 3756
3757 3757 ndmas = hxgep->rx_buf_pool_p->ndmas;
3758 3758 dma_buf_p = hxgep->rx_buf_pool_p->dma_buf_pool_p;
3759 3759
3760 3760 for (i = 0; i < ndmas; i++) {
3761 3761 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
3762 3762 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel];
3763 3763 rbrp = rcrp->rx_rbr_p;
3764 3764
3765 3765 MUTEX_ENTER(&rbrp->post_lock);
3766 3766
3767 3767 /*
3768 3768 * This function needs to be inside the post_lock
3769 3769 */
3770 3770 if (hxge_rxdma_fatal_err_recover(hxgep, channel) != HXGE_OK) {
3771 3771 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3772 3772 "Could not recover channel %d", channel));
3773 3773 }
3774 3774 MUTEX_EXIT(&rbrp->post_lock);
3775 3775 }
3776 3776
3777 3777 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Reset RxMAC..."));
3778 3778
3779 3779 /* Reset RxMAC */
3780 3780 if (hxge_rx_vmac_reset(hxgep) != HXGE_OK) {
3781 3781 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3782 3782 "hxge_rx_port_fatal_err_recover: Failed to reset RxMAC"));
3783 3783 goto fail;
3784 3784 }
3785 3785
3786 3786 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Re-initialize RxMAC..."));
3787 3787
3788 3788 /* Re-Initialize RxMAC */
3789 3789 if ((status = hxge_rx_vmac_init(hxgep)) != HXGE_OK) {
3790 3790 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3791 3791 "hxge_rx_port_fatal_err_recover: Failed to reset RxMAC"));
3792 3792 goto fail;
3793 3793 }
3794 3794 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Re-enable RxMAC..."));
3795 3795
3796 3796 /* Re-enable RxMAC */
3797 3797 if ((status = hxge_rx_vmac_enable(hxgep)) != HXGE_OK) {
3798 3798 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3799 3799 "hxge_rx_port_fatal_err_recover: Failed to enable RxMAC"));
3800 3800 goto fail;
3801 3801 }
3802 3802 MUTEX_EXIT(&hxgep->vmac_lock);
3803 3803
3804 3804 /* Reset the error mask since PEU reset cleared it */
3805 3805 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_INT_MASK, 0x0);
3806 3806
3807 3807 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3808 3808 "Recovery Successful, RxPort Restored"));
3809 3809 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rx_port_fatal_err_recover"));
3810 3810 return (HXGE_OK);
3811 3811
3812 3812 fail:
3813 3813 MUTEX_EXIT(&hxgep->vmac_lock);
3814 3814 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3815 3815 "Error Recovery failed for hxge(%d)", hxgep->instance));
3816 3816 return (status);
3817 3817 }
3818 3818
3819 3819 static void
3820 3820 hxge_rbr_empty_restore(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p)
3821 3821 {
3822 3822 hpi_status_t hpi_status;
3823 3823 hxge_status_t status;
3824 3824 rdc_stat_t cs;
3825 3825 p_hxge_rx_ring_stats_t rdc_stats;
3826 3826
3827 3827 rdc_stats = &hxgep->statsp->rdc_stats[rx_rbr_p->rdc];
3828 3828
3829 3829 /*
3830 3830 * Complete the processing for the RBR Empty by:
3831 3831 * 0) kicking back HXGE_RBR_EMPTY_THRESHOLD
3832 3832 * packets.
3833 3833 * 1) Disable the RX vmac.
3834 3834 * 2) Re-enable the affected DMA channel.
3835 3835 * 3) Re-enable the RX vmac.
3836 3836 */
3837 3837
3838 3838 /*
3839 3839 * Disable the RX VMAC, but setting the framelength
3840 3840 * to 0, since there is a hardware bug when disabling
3841 3841 * the vmac.
3842 3842 */
3843 3843 MUTEX_ENTER(&hxgep->vmac_lock);
3844 3844 (void) hxge_rx_vmac_disable(hxgep);
3845 3845
3846 3846 /*
3847 3847 * Re-arm the mex bit for interrupts to be enabled.
3848 3848 */
3849 3849 cs.value = 0;
3850 3850 cs.bits.mex = 1;
3851 3851 RXDMA_REG_WRITE64(HXGE_DEV_HPI_HANDLE(hxgep), RDC_STAT,
3852 3852 rx_rbr_p->rdc, cs.value);
3853 3853
3854 3854 hpi_status = hpi_rxdma_cfg_rdc_enable(
3855 3855 HXGE_DEV_HPI_HANDLE(hxgep), rx_rbr_p->rdc);
3856 3856 if (hpi_status != HPI_SUCCESS) {
3857 3857 rdc_stats->rbr_empty_fail++;
3858 3858
3859 3859 /* Assume we are already inside the post_lock */
3860 3860 status = hxge_rxdma_fatal_err_recover(hxgep, rx_rbr_p->rdc);
3861 3861 if (status != HXGE_OK) {
3862 3862 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3863 3863 "hxge(%d): channel(%d) is empty.",
3864 3864 hxgep->instance, rx_rbr_p->rdc));
3865 3865 }
3866 3866 }
3867 3867
3868 3868 /*
3869 3869 * Re-enable the RX VMAC.
3870 3870 */
3871 3871 (void) hxge_rx_vmac_enable(hxgep);
3872 3872 MUTEX_EXIT(&hxgep->vmac_lock);
3873 3873
3874 3874 rdc_stats->rbr_empty_restore++;
3875 3875 rx_rbr_p->rbr_is_empty = B_FALSE;
3876 3876 }
|
↓ open down ↓ |
2832 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX