1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include <sys/nxge/nxge_impl.h>
28 #include <sys/nxge/nxge_txdma.h>
29 #include <sys/nxge/nxge_hio.h>
30 #include <npi_tx_rd64.h>
31 #include <npi_tx_wr64.h>
32 #include <sys/llc1.h>
33
34 uint32_t nxge_reclaim_pending = TXDMA_RECLAIM_PENDING_DEFAULT;
35 uint32_t nxge_tx_minfree = 64;
36 uint32_t nxge_tx_intr_thres = 0;
37 uint32_t nxge_tx_max_gathers = TX_MAX_GATHER_POINTERS;
38 uint32_t nxge_tx_tiny_pack = 1;
39 uint32_t nxge_tx_use_bcopy = 1;
40
41 extern uint32_t nxge_tx_ring_size;
42 extern uint32_t nxge_bcopy_thresh;
43 extern uint32_t nxge_dvma_thresh;
44 extern uint32_t nxge_dma_stream_thresh;
45 extern dma_method_t nxge_force_dma;
46 extern uint32_t nxge_cksum_offload;
47
48 /* Device register access attributes for PIO. */
49 extern ddi_device_acc_attr_t nxge_dev_reg_acc_attr;
50 /* Device descriptor access attributes for DMA. */
51 extern ddi_device_acc_attr_t nxge_dev_desc_dma_acc_attr;
52 /* Device buffer access attributes for DMA. */
53 extern ddi_device_acc_attr_t nxge_dev_buf_dma_acc_attr;
54 extern ddi_dma_attr_t nxge_desc_dma_attr;
55 extern ddi_dma_attr_t nxge_tx_dma_attr;
56
57 extern void nxge_tx_ring_task(void *arg);
58
59 static nxge_status_t nxge_map_txdma(p_nxge_t, int);
60
61 static nxge_status_t nxge_txdma_hw_start(p_nxge_t, int);
62
63 static nxge_status_t nxge_map_txdma_channel(p_nxge_t, uint16_t,
64 p_nxge_dma_common_t *, p_tx_ring_t *,
65 uint32_t, p_nxge_dma_common_t *,
66 p_tx_mbox_t *);
67 static void nxge_unmap_txdma_channel(p_nxge_t, uint16_t);
68
69 static nxge_status_t nxge_map_txdma_channel_buf_ring(p_nxge_t, uint16_t,
70 p_nxge_dma_common_t *, p_tx_ring_t *, uint32_t);
71 static void nxge_unmap_txdma_channel_buf_ring(p_nxge_t, p_tx_ring_t);
72
73 static void nxge_map_txdma_channel_cfg_ring(p_nxge_t, uint16_t,
74 p_nxge_dma_common_t *, p_tx_ring_t,
75 p_tx_mbox_t *);
76 static void nxge_unmap_txdma_channel_cfg_ring(p_nxge_t,
77 p_tx_ring_t, p_tx_mbox_t);
78
79 static nxge_status_t nxge_txdma_start_channel(p_nxge_t, uint16_t,
80 p_tx_ring_t, p_tx_mbox_t);
81 static nxge_status_t nxge_txdma_stop_channel(p_nxge_t, uint16_t);
82
83 static p_tx_ring_t nxge_txdma_get_ring(p_nxge_t, uint16_t);
84 static nxge_status_t nxge_tx_err_evnts(p_nxge_t, uint_t,
85 p_nxge_ldv_t, tx_cs_t);
86 static p_tx_mbox_t nxge_txdma_get_mbox(p_nxge_t, uint16_t);
87 static nxge_status_t nxge_txdma_fatal_err_recover(p_nxge_t,
88 uint16_t, p_tx_ring_t);
89
90 static void nxge_txdma_fixup_hung_channel(p_nxge_t nxgep,
91 p_tx_ring_t ring_p, uint16_t channel);
92
93 nxge_status_t
94 nxge_init_txdma_channels(p_nxge_t nxgep)
95 {
96 nxge_grp_set_t *set = &nxgep->tx_set;
97 int i, tdc, count;
98 nxge_grp_t *group;
99 dc_map_t map;
100 int dev_gindex;
101
102 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_init_txdma_channels"));
103
104 for (i = 0, count = 0; i < NXGE_LOGICAL_GROUP_MAX; i++) {
105 if ((1 << i) & set->lg.map) {
106 group = set->group[i];
107 dev_gindex =
108 nxgep->pt_config.hw_config.def_mac_txdma_grpid + i;
109 map = nxgep->pt_config.tdc_grps[dev_gindex].map;
110 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
111 if ((1 << tdc) & map) {
112 if ((nxge_grp_dc_add(nxgep,
113 group, VP_BOUND_TX, tdc)))
114 goto init_txdma_channels_exit;
115 }
116 }
117 }
118 if (++count == set->lg.count)
119 break;
120 }
121
122 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_init_txdma_channels"));
123 return (NXGE_OK);
124
125 init_txdma_channels_exit:
126 for (i = 0, count = 0; i < NXGE_LOGICAL_GROUP_MAX; i++) {
127 if ((1 << i) & set->lg.map) {
128 group = set->group[i];
129 dev_gindex =
130 nxgep->pt_config.hw_config.def_mac_txdma_grpid + i;
131 map = nxgep->pt_config.tdc_grps[dev_gindex].map;
132 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
133 if ((1 << tdc) & map) {
134 nxge_grp_dc_remove(nxgep,
135 VP_BOUND_TX, tdc);
136 }
137 }
138 }
139 if (++count == set->lg.count)
140 break;
141 }
142
143 return (NXGE_ERROR);
144
145 }
146
147 nxge_status_t
148 nxge_init_txdma_channel(
149 p_nxge_t nxge,
150 int channel)
151 {
152 nxge_status_t status;
153
154 NXGE_DEBUG_MSG((nxge, MEM2_CTL, "==> nxge_init_txdma_channel"));
155
156 status = nxge_map_txdma(nxge, channel);
157 if (status != NXGE_OK) {
158 NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL,
159 "<== nxge_init_txdma_channel: status 0x%x", status));
160 (void) npi_txdma_dump_tdc_regs(nxge->npi_handle, channel);
161 return (status);
162 }
163
164 status = nxge_txdma_hw_start(nxge, channel);
165 if (status != NXGE_OK) {
166 (void) nxge_unmap_txdma_channel(nxge, channel);
167 (void) npi_txdma_dump_tdc_regs(nxge->npi_handle, channel);
168 return (status);
169 }
170
171 if (!nxge->statsp->tdc_ksp[channel])
172 nxge_setup_tdc_kstats(nxge, channel);
173
174 NXGE_DEBUG_MSG((nxge, MEM2_CTL, "<== nxge_init_txdma_channel"));
175
176 return (status);
177 }
178
179 void
180 nxge_uninit_txdma_channels(p_nxge_t nxgep)
181 {
182 nxge_grp_set_t *set = &nxgep->tx_set;
183 int tdc;
184
185 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_uninit_txdma_channels"));
186
187 if (set->owned.map == 0) {
188 NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
189 "nxge_uninit_txdma_channels: no channels"));
190 return;
191 }
192
193 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
194 if ((1 << tdc) & set->owned.map) {
195 nxge_grp_dc_remove(nxgep, VP_BOUND_TX, tdc);
196 }
197 }
198
199 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_uninit_txdma_channels"));
200 }
201
202 void
203 nxge_uninit_txdma_channel(p_nxge_t nxgep, int channel)
204 {
205 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_uninit_txdma_channel"));
206
207 if (nxgep->statsp->tdc_ksp[channel]) {
208 kstat_delete(nxgep->statsp->tdc_ksp[channel]);
209 nxgep->statsp->tdc_ksp[channel] = 0;
210 }
211
212 if (nxge_txdma_stop_channel(nxgep, channel) != NXGE_OK)
213 goto nxge_uninit_txdma_channel_exit;
214
215 nxge_unmap_txdma_channel(nxgep, channel);
216
217 nxge_uninit_txdma_channel_exit:
218 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_uninit_txdma_channel"));
219 }
220
221 void
222 nxge_setup_dma_common(p_nxge_dma_common_t dest_p, p_nxge_dma_common_t src_p,
223 uint32_t entries, uint32_t size)
224 {
225 size_t tsize;
226 *dest_p = *src_p;
227 tsize = size * entries;
228 dest_p->alength = tsize;
229 dest_p->nblocks = entries;
230 dest_p->block_size = size;
231 dest_p->offset += tsize;
232
233 src_p->kaddrp = (caddr_t)dest_p->kaddrp + tsize;
234 src_p->alength -= tsize;
235 src_p->dma_cookie.dmac_laddress += tsize;
236 src_p->dma_cookie.dmac_size -= tsize;
237 }
238
239 /*
240 * nxge_reset_txdma_channel
241 *
242 * Reset a TDC.
243 *
244 * Arguments:
245 * nxgep
246 * channel The channel to reset.
247 * reg_data The current TX_CS.
248 *
249 * Notes:
250 *
251 * NPI/NXGE function calls:
252 * npi_txdma_channel_reset()
253 * npi_txdma_channel_control()
254 *
255 * Registers accessed:
256 * TX_CS DMC+0x40028 Transmit Control And Status
257 * TX_RING_KICK DMC+0x40018 Transmit Ring Kick
258 *
259 * Context:
260 * Any domain
261 */
262 nxge_status_t
263 nxge_reset_txdma_channel(p_nxge_t nxgep, uint16_t channel, uint64_t reg_data)
264 {
265 npi_status_t rs = NPI_SUCCESS;
266 nxge_status_t status = NXGE_OK;
267 npi_handle_t handle;
268
269 NXGE_DEBUG_MSG((nxgep, TX_CTL, " ==> nxge_reset_txdma_channel"));
270
271 handle = NXGE_DEV_NPI_HANDLE(nxgep);
272 if ((reg_data & TX_CS_RST_MASK) == TX_CS_RST_MASK) {
273 rs = npi_txdma_channel_reset(handle, channel);
274 } else {
275 rs = npi_txdma_channel_control(handle, TXDMA_RESET,
276 channel);
277 }
278
279 if (rs != NPI_SUCCESS) {
280 status = NXGE_ERROR | rs;
281 }
282
283 /*
284 * Reset the tail (kick) register to 0.
285 * (Hardware will not reset it. Tx overflow fatal
286 * error if tail is not set to 0 after reset!
287 */
288 TXDMA_REG_WRITE64(handle, TX_RING_KICK_REG, channel, 0);
289
290 NXGE_DEBUG_MSG((nxgep, TX_CTL, " <== nxge_reset_txdma_channel"));
291 return (status);
292 }
293
294 /*
295 * nxge_init_txdma_channel_event_mask
296 *
297 * Enable interrupts for a set of events.
298 *
299 * Arguments:
300 * nxgep
301 * channel The channel to map.
302 * mask_p The events to enable.
303 *
304 * Notes:
305 *
306 * NPI/NXGE function calls:
307 * npi_txdma_event_mask()
308 *
309 * Registers accessed:
310 * TX_ENT_MSK DMC+0x40020 Transmit Event Mask
311 *
312 * Context:
313 * Any domain
314 */
315 nxge_status_t
316 nxge_init_txdma_channel_event_mask(p_nxge_t nxgep, uint16_t channel,
317 p_tx_dma_ent_msk_t mask_p)
318 {
319 npi_handle_t handle;
320 npi_status_t rs = NPI_SUCCESS;
321 nxge_status_t status = NXGE_OK;
322
323 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
324 "<== nxge_init_txdma_channel_event_mask"));
325
326 handle = NXGE_DEV_NPI_HANDLE(nxgep);
327 rs = npi_txdma_event_mask(handle, OP_SET, channel, mask_p);
328 if (rs != NPI_SUCCESS) {
329 status = NXGE_ERROR | rs;
330 }
331
332 return (status);
333 }
334
335 /*
336 * nxge_init_txdma_channel_cntl_stat
337 *
338 * Stop a TDC. If at first we don't succeed, inject an error.
339 *
340 * Arguments:
341 * nxgep
342 * channel The channel to stop.
343 *
344 * Notes:
345 *
346 * NPI/NXGE function calls:
347 * npi_txdma_control_status()
348 *
349 * Registers accessed:
350 * TX_CS DMC+0x40028 Transmit Control And Status
351 *
352 * Context:
353 * Any domain
354 */
355 nxge_status_t
356 nxge_init_txdma_channel_cntl_stat(p_nxge_t nxgep, uint16_t channel,
357 uint64_t reg_data)
358 {
359 npi_handle_t handle;
360 npi_status_t rs = NPI_SUCCESS;
361 nxge_status_t status = NXGE_OK;
362
363 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
364 "<== nxge_init_txdma_channel_cntl_stat"));
365
366 handle = NXGE_DEV_NPI_HANDLE(nxgep);
367 rs = npi_txdma_control_status(handle, OP_SET, channel,
368 (p_tx_cs_t)®_data);
369
370 if (rs != NPI_SUCCESS) {
371 status = NXGE_ERROR | rs;
372 }
373
374 return (status);
375 }
376
377 /*
378 * nxge_enable_txdma_channel
379 *
380 * Enable a TDC.
381 *
382 * Arguments:
383 * nxgep
384 * channel The channel to enable.
385 * tx_desc_p channel's transmit descriptor ring.
386 * mbox_p channel's mailbox,
387 *
388 * Notes:
389 *
390 * NPI/NXGE function calls:
391 * npi_txdma_ring_config()
392 * npi_txdma_mbox_config()
393 * npi_txdma_channel_init_enable()
394 *
395 * Registers accessed:
396 * TX_RNG_CFIG DMC+0x40000 Transmit Ring Configuration
397 * TXDMA_MBH DMC+0x40030 TXDMA Mailbox High
398 * TXDMA_MBL DMC+0x40038 TXDMA Mailbox Low
399 * TX_CS DMC+0x40028 Transmit Control And Status
400 *
401 * Context:
402 * Any domain
403 */
404 nxge_status_t
405 nxge_enable_txdma_channel(p_nxge_t nxgep,
406 uint16_t channel, p_tx_ring_t tx_desc_p, p_tx_mbox_t mbox_p)
407 {
408 npi_handle_t handle;
409 npi_status_t rs = NPI_SUCCESS;
410 nxge_status_t status = NXGE_OK;
411
412 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_enable_txdma_channel"));
413
414 handle = NXGE_DEV_NPI_HANDLE(nxgep);
415 /*
416 * Use configuration data composed at init time.
417 * Write to hardware the transmit ring configurations.
418 */
419 rs = npi_txdma_ring_config(handle, OP_SET, channel,
420 (uint64_t *)&(tx_desc_p->tx_ring_cfig.value));
421
422 if (rs != NPI_SUCCESS) {
423 return (NXGE_ERROR | rs);
424 }
425
426 if (isLDOMguest(nxgep)) {
427 /* Add interrupt handler for this channel. */
428 if (nxge_hio_intr_add(nxgep, VP_BOUND_TX, channel) != NXGE_OK)
429 return (NXGE_ERROR);
430 }
431
432 /* Write to hardware the mailbox */
433 rs = npi_txdma_mbox_config(handle, OP_SET, channel,
434 (uint64_t *)&mbox_p->tx_mbox.dma_cookie.dmac_laddress);
435
436 if (rs != NPI_SUCCESS) {
437 return (NXGE_ERROR | rs);
438 }
439
440 /* Start the DMA engine. */
441 rs = npi_txdma_channel_init_enable(handle, channel);
442
443 if (rs != NPI_SUCCESS) {
444 return (NXGE_ERROR | rs);
445 }
446
447 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_enable_txdma_channel"));
448
449 return (status);
450 }
451
452 void
453 nxge_fill_tx_hdr(p_mblk_t mp, boolean_t fill_len,
454 boolean_t l4_cksum, int pkt_len, uint8_t npads,
455 p_tx_pkt_hdr_all_t pkthdrp,
456 t_uscalar_t start_offset,
457 t_uscalar_t stuff_offset)
458 {
459 p_tx_pkt_header_t hdrp;
460 p_mblk_t nmp;
461 uint64_t tmp;
462 size_t mblk_len;
463 size_t iph_len;
464 size_t hdrs_size;
465 uint8_t hdrs_buf[sizeof (struct ether_header) +
466 64 + sizeof (uint32_t)];
467 uint8_t *cursor;
468 uint8_t *ip_buf;
469 uint16_t eth_type;
470 uint8_t ipproto;
471 boolean_t is_vlan = B_FALSE;
472 size_t eth_hdr_size;
473
474 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: mp $%p", mp));
475
476 /*
477 * Caller should zero out the headers first.
478 */
479 hdrp = (p_tx_pkt_header_t)&pkthdrp->pkthdr;
480
481 if (fill_len) {
482 NXGE_DEBUG_MSG((NULL, TX_CTL,
483 "==> nxge_fill_tx_hdr: pkt_len %d "
484 "npads %d", pkt_len, npads));
485 tmp = (uint64_t)pkt_len;
486 hdrp->value |= (tmp << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);
487 goto fill_tx_header_done;
488 }
489
490 hdrp->value |= (((uint64_t)npads) << TX_PKT_HEADER_PAD_SHIFT);
491
492 /*
493 * mp is the original data packet (does not include the
494 * Neptune transmit header).
495 */
496 nmp = mp;
497 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: "
498 "mp $%p b_rptr $%p len %d",
499 mp, nmp->b_rptr, MBLKL(nmp)));
500 /* copy ether_header from mblk to hdrs_buf */
501 cursor = &hdrs_buf[0];
502 tmp = sizeof (struct ether_vlan_header);
503 while ((nmp != NULL) && (tmp > 0)) {
504 size_t buflen;
505 mblk_len = MBLKL(nmp);
506 buflen = min((size_t)tmp, mblk_len);
507 bcopy(nmp->b_rptr, cursor, buflen);
508 cursor += buflen;
509 tmp -= buflen;
510 nmp = nmp->b_cont;
511 }
512
513 nmp = mp;
514 mblk_len = MBLKL(nmp);
515 ip_buf = NULL;
516 eth_type = ntohs(((p_ether_header_t)hdrs_buf)->ether_type);
517 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> : nxge_fill_tx_hdr: (value 0x%llx) "
518 "ether type 0x%x", eth_type, hdrp->value));
519
520 if (eth_type < ETHERMTU) {
521 tmp = 1ull;
522 hdrp->value |= (tmp << TX_PKT_HEADER_LLC_SHIFT);
523 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: LLC "
524 "value 0x%llx", hdrp->value));
525 if (*(hdrs_buf + sizeof (struct ether_header))
526 == LLC_SNAP_SAP) {
527 eth_type = ntohs(*((uint16_t *)(hdrs_buf +
528 sizeof (struct ether_header) + 6)));
529 NXGE_DEBUG_MSG((NULL, TX_CTL,
530 "==> nxge_tx_pkt_hdr_init: LLC ether type 0x%x",
531 eth_type));
532 } else {
533 goto fill_tx_header_done;
534 }
535 } else if (eth_type == VLAN_ETHERTYPE) {
536 tmp = 1ull;
537 hdrp->value |= (tmp << TX_PKT_HEADER_VLAN__SHIFT);
538
539 eth_type = ntohs(((struct ether_vlan_header *)
540 hdrs_buf)->ether_type);
541 is_vlan = B_TRUE;
542 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: VLAN "
543 "value 0x%llx", hdrp->value));
544 }
545
546 if (!is_vlan) {
547 eth_hdr_size = sizeof (struct ether_header);
548 } else {
549 eth_hdr_size = sizeof (struct ether_vlan_header);
550 }
551
552 switch (eth_type) {
553 case ETHERTYPE_IP:
554 if (mblk_len > eth_hdr_size + sizeof (uint8_t)) {
555 ip_buf = nmp->b_rptr + eth_hdr_size;
556 mblk_len -= eth_hdr_size;
557 iph_len = ((*ip_buf) & 0x0f);
558 if (mblk_len > (iph_len + sizeof (uint32_t))) {
559 ip_buf = nmp->b_rptr;
560 ip_buf += eth_hdr_size;
561 } else {
562 ip_buf = NULL;
563 }
564
565 }
566 if (ip_buf == NULL) {
567 hdrs_size = 0;
568 ((p_ether_header_t)hdrs_buf)->ether_type = 0;
569 while ((nmp) && (hdrs_size <
570 sizeof (hdrs_buf))) {
571 mblk_len = (size_t)nmp->b_wptr -
572 (size_t)nmp->b_rptr;
573 if (mblk_len >=
574 (sizeof (hdrs_buf) - hdrs_size))
575 mblk_len = sizeof (hdrs_buf) -
576 hdrs_size;
577 bcopy(nmp->b_rptr,
578 &hdrs_buf[hdrs_size], mblk_len);
579 hdrs_size += mblk_len;
580 nmp = nmp->b_cont;
581 }
582 ip_buf = hdrs_buf;
583 ip_buf += eth_hdr_size;
584 iph_len = ((*ip_buf) & 0x0f);
585 }
586
587 ipproto = ip_buf[9];
588
589 tmp = (uint64_t)iph_len;
590 hdrp->value |= (tmp << TX_PKT_HEADER_IHL_SHIFT);
591 tmp = (uint64_t)(eth_hdr_size >> 1);
592 hdrp->value |= (tmp << TX_PKT_HEADER_L3START_SHIFT);
593
594 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: IPv4 "
595 " iph_len %d l3start %d eth_hdr_size %d proto 0x%x"
596 "tmp 0x%x",
597 iph_len, hdrp->bits.hdw.l3start, eth_hdr_size,
598 ipproto, tmp));
599 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: IP "
600 "value 0x%llx", hdrp->value));
601
602 break;
603
604 case ETHERTYPE_IPV6:
605 hdrs_size = 0;
606 ((p_ether_header_t)hdrs_buf)->ether_type = 0;
607 while ((nmp) && (hdrs_size <
608 sizeof (hdrs_buf))) {
609 mblk_len = (size_t)nmp->b_wptr - (size_t)nmp->b_rptr;
610 if (mblk_len >=
611 (sizeof (hdrs_buf) - hdrs_size))
612 mblk_len = sizeof (hdrs_buf) -
613 hdrs_size;
614 bcopy(nmp->b_rptr,
615 &hdrs_buf[hdrs_size], mblk_len);
616 hdrs_size += mblk_len;
617 nmp = nmp->b_cont;
618 }
619 ip_buf = hdrs_buf;
620 ip_buf += eth_hdr_size;
621
622 tmp = 1ull;
623 hdrp->value |= (tmp << TX_PKT_HEADER_IP_VER_SHIFT);
624
625 tmp = (eth_hdr_size >> 1);
626 hdrp->value |= (tmp << TX_PKT_HEADER_L3START_SHIFT);
627
628 /* byte 6 is the next header protocol */
629 ipproto = ip_buf[6];
630
631 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: IPv6 "
632 " iph_len %d l3start %d eth_hdr_size %d proto 0x%x",
633 iph_len, hdrp->bits.hdw.l3start, eth_hdr_size,
634 ipproto));
635 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: IPv6 "
636 "value 0x%llx", hdrp->value));
637
638 break;
639
640 default:
641 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: non-IP"));
642 goto fill_tx_header_done;
643 }
644
645 switch (ipproto) {
646 case IPPROTO_TCP:
647 NXGE_DEBUG_MSG((NULL, TX_CTL,
648 "==> nxge_fill_tx_hdr: TCP (cksum flag %d)", l4_cksum));
649 if (l4_cksum) {
650 hdrp->value |= TX_CKSUM_EN_PKT_TYPE_TCP;
651 hdrp->value |=
652 (((uint64_t)(start_offset >> 1)) <<
653 TX_PKT_HEADER_L4START_SHIFT);
654 hdrp->value |=
655 (((uint64_t)(stuff_offset >> 1)) <<
656 TX_PKT_HEADER_L4STUFF_SHIFT);
657
658 NXGE_DEBUG_MSG((NULL, TX_CTL,
659 "==> nxge_tx_pkt_hdr_init: TCP CKSUM "
660 "value 0x%llx", hdrp->value));
661 }
662
663 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: TCP "
664 "value 0x%llx", hdrp->value));
665 break;
666
667 case IPPROTO_UDP:
668 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: UDP"));
669 if (l4_cksum) {
670 if (!nxge_cksum_offload) {
671 uint16_t *up;
672 uint16_t cksum;
673 t_uscalar_t stuff_len;
674
675 /*
676 * The checksum field has the
677 * partial checksum.
678 * IP_CSUM() macro calls ip_cksum() which
679 * can add in the partial checksum.
680 */
681 cksum = IP_CSUM(mp, start_offset, 0);
682 stuff_len = stuff_offset;
683 nmp = mp;
684 mblk_len = MBLKL(nmp);
685 while ((nmp != NULL) &&
686 (mblk_len < stuff_len)) {
687 stuff_len -= mblk_len;
688 nmp = nmp->b_cont;
689 if (nmp)
690 mblk_len = MBLKL(nmp);
691 }
692 ASSERT(nmp);
693 up = (uint16_t *)(nmp->b_rptr + stuff_len);
694
695 *up = cksum;
696 hdrp->value &= ~TX_CKSUM_EN_PKT_TYPE_UDP;
697 NXGE_DEBUG_MSG((NULL, TX_CTL,
698 "==> nxge_tx_pkt_hdr_init: UDP offset %d "
699 "use sw cksum "
700 "write to $%p cksum 0x%x content up 0x%x",
701 stuff_len,
702 up,
703 cksum,
704 *up));
705 } else {
706 /* Hardware will compute the full checksum */
707 hdrp->value |= TX_CKSUM_EN_PKT_TYPE_UDP;
708 hdrp->value |=
709 (((uint64_t)(start_offset >> 1)) <<
710 TX_PKT_HEADER_L4START_SHIFT);
711 hdrp->value |=
712 (((uint64_t)(stuff_offset >> 1)) <<
713 TX_PKT_HEADER_L4STUFF_SHIFT);
714
715 NXGE_DEBUG_MSG((NULL, TX_CTL,
716 "==> nxge_tx_pkt_hdr_init: UDP offset %d "
717 " use partial checksum "
718 "cksum 0x%x ",
719 "value 0x%llx",
720 stuff_offset,
721 IP_CSUM(mp, start_offset, 0),
722 hdrp->value));
723 }
724 }
725
726 NXGE_DEBUG_MSG((NULL, TX_CTL,
727 "==> nxge_tx_pkt_hdr_init: UDP"
728 "value 0x%llx", hdrp->value));
729 break;
730
731 default:
732 goto fill_tx_header_done;
733 }
734
735 fill_tx_header_done:
736 NXGE_DEBUG_MSG((NULL, TX_CTL,
737 "==> nxge_fill_tx_hdr: pkt_len %d "
738 "npads %d value 0x%llx", pkt_len, npads, hdrp->value));
739
740 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_fill_tx_hdr"));
741 }
742
743 /*ARGSUSED*/
744 p_mblk_t
745 nxge_tx_pkt_header_reserve(p_mblk_t mp, uint8_t *npads)
746 {
747 p_mblk_t newmp = NULL;
748
749 if ((newmp = allocb(TX_PKT_HEADER_SIZE, BPRI_MED)) == NULL) {
750 NXGE_DEBUG_MSG((NULL, TX_CTL,
751 "<== nxge_tx_pkt_header_reserve: allocb failed"));
752 return (NULL);
753 }
754
755 NXGE_DEBUG_MSG((NULL, TX_CTL,
756 "==> nxge_tx_pkt_header_reserve: get new mp"));
757 DB_TYPE(newmp) = M_DATA;
758 newmp->b_rptr = newmp->b_wptr = DB_LIM(newmp);
759 linkb(newmp, mp);
760 newmp->b_rptr -= TX_PKT_HEADER_SIZE;
761
762 NXGE_DEBUG_MSG((NULL, TX_CTL, "==>nxge_tx_pkt_header_reserve: "
763 "b_rptr $%p b_wptr $%p",
764 newmp->b_rptr, newmp->b_wptr));
765
766 NXGE_DEBUG_MSG((NULL, TX_CTL,
767 "<== nxge_tx_pkt_header_reserve: use new mp"));
768
769 return (newmp);
770 }
771
772 int
773 nxge_tx_pkt_nmblocks(p_mblk_t mp, int *tot_xfer_len_p)
774 {
775 uint_t nmblks;
776 ssize_t len;
777 uint_t pkt_len;
778 p_mblk_t nmp, bmp, tmp;
779 uint8_t *b_wptr;
780
781 NXGE_DEBUG_MSG((NULL, TX_CTL,
782 "==> nxge_tx_pkt_nmblocks: mp $%p rptr $%p wptr $%p "
783 "len %d", mp, mp->b_rptr, mp->b_wptr, MBLKL(mp)));
784
785 nmp = mp;
786 bmp = mp;
787 nmblks = 0;
788 pkt_len = 0;
789 *tot_xfer_len_p = 0;
790
791 while (nmp) {
792 len = MBLKL(nmp);
793 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_nmblocks: "
794 "len %d pkt_len %d nmblks %d tot_xfer_len %d",
795 len, pkt_len, nmblks,
796 *tot_xfer_len_p));
797
798 if (len <= 0) {
799 bmp = nmp;
800 nmp = nmp->b_cont;
801 NXGE_DEBUG_MSG((NULL, TX_CTL,
802 "==> nxge_tx_pkt_nmblocks: "
803 "len (0) pkt_len %d nmblks %d",
804 pkt_len, nmblks));
805 continue;
806 }
807
808 *tot_xfer_len_p += len;
809 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_nmblocks: "
810 "len %d pkt_len %d nmblks %d tot_xfer_len %d",
811 len, pkt_len, nmblks,
812 *tot_xfer_len_p));
813
814 if (len < nxge_bcopy_thresh) {
815 NXGE_DEBUG_MSG((NULL, TX_CTL,
816 "==> nxge_tx_pkt_nmblocks: "
817 "len %d (< thresh) pkt_len %d nmblks %d",
818 len, pkt_len, nmblks));
819 if (pkt_len == 0)
820 nmblks++;
821 pkt_len += len;
822 if (pkt_len >= nxge_bcopy_thresh) {
823 pkt_len = 0;
824 len = 0;
825 nmp = bmp;
826 }
827 } else {
828 NXGE_DEBUG_MSG((NULL, TX_CTL,
829 "==> nxge_tx_pkt_nmblocks: "
830 "len %d (> thresh) pkt_len %d nmblks %d",
831 len, pkt_len, nmblks));
832 pkt_len = 0;
833 nmblks++;
834 /*
835 * Hardware limits the transfer length to 4K.
836 * If len is more than 4K, we need to break
837 * it up to at most 2 more blocks.
838 */
839 if (len > TX_MAX_TRANSFER_LENGTH) {
840 uint32_t nsegs;
841
842 nsegs = 1;
843 NXGE_DEBUG_MSG((NULL, TX_CTL,
844 "==> nxge_tx_pkt_nmblocks: "
845 "len %d pkt_len %d nmblks %d nsegs %d",
846 len, pkt_len, nmblks, nsegs));
847 if (len % (TX_MAX_TRANSFER_LENGTH * 2)) {
848 ++nsegs;
849 }
850 do {
851 b_wptr = nmp->b_rptr +
852 TX_MAX_TRANSFER_LENGTH;
853 nmp->b_wptr = b_wptr;
854 if ((tmp = dupb(nmp)) == NULL) {
855 return (0);
856 }
857 tmp->b_rptr = b_wptr;
858 tmp->b_wptr = nmp->b_wptr;
859 tmp->b_cont = nmp->b_cont;
860 nmp->b_cont = tmp;
861 nmblks++;
862 if (--nsegs) {
863 nmp = tmp;
864 }
865 } while (nsegs);
866 nmp = tmp;
867 }
868 }
869
870 /*
871 * Hardware limits the transmit gather pointers to 15.
872 */
873 if (nmp->b_cont && (nmblks + TX_GATHER_POINTERS_THRESHOLD) >
874 TX_MAX_GATHER_POINTERS) {
875 NXGE_DEBUG_MSG((NULL, TX_CTL,
876 "==> nxge_tx_pkt_nmblocks: pull msg - "
877 "len %d pkt_len %d nmblks %d",
878 len, pkt_len, nmblks));
879 /* Pull all message blocks from b_cont */
880 if ((tmp = msgpullup(nmp->b_cont, -1)) == NULL) {
881 return (0);
882 }
883 freemsg(nmp->b_cont);
884 nmp->b_cont = tmp;
885 pkt_len = 0;
886 }
887 bmp = nmp;
888 nmp = nmp->b_cont;
889 }
890
891 NXGE_DEBUG_MSG((NULL, TX_CTL,
892 "<== nxge_tx_pkt_nmblocks: rptr $%p wptr $%p "
893 "nmblks %d len %d tot_xfer_len %d",
894 mp->b_rptr, mp->b_wptr, nmblks,
895 MBLKL(mp), *tot_xfer_len_p));
896
897 return (nmblks);
898 }
899
900 boolean_t
901 nxge_txdma_reclaim(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, int nmblks)
902 {
903 boolean_t status = B_TRUE;
904 p_nxge_dma_common_t tx_desc_dma_p;
905 nxge_dma_common_t desc_area;
906 p_tx_desc_t tx_desc_ring_vp;
907 p_tx_desc_t tx_desc_p;
908 p_tx_desc_t tx_desc_pp;
909 tx_desc_t r_tx_desc;
910 p_tx_msg_t tx_msg_ring;
911 p_tx_msg_t tx_msg_p;
912 npi_handle_t handle;
913 tx_ring_hdl_t tx_head;
914 uint32_t pkt_len;
915 uint_t tx_rd_index;
916 uint16_t head_index, tail_index;
917 uint8_t tdc;
918 boolean_t head_wrap, tail_wrap;
919 p_nxge_tx_ring_stats_t tdc_stats;
920 int rc;
921
922 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_reclaim"));
923
924 status = ((tx_ring_p->descs_pending < nxge_reclaim_pending) &&
925 (nmblks != 0));
926 NXGE_DEBUG_MSG((nxgep, TX_CTL,
927 "==> nxge_txdma_reclaim: pending %d reclaim %d nmblks %d",
928 tx_ring_p->descs_pending, nxge_reclaim_pending,
929 nmblks));
930 if (!status) {
931 tx_desc_dma_p = &tx_ring_p->tdc_desc;
932 desc_area = tx_ring_p->tdc_desc;
933 handle = NXGE_DEV_NPI_HANDLE(nxgep);
934 tx_desc_ring_vp = tx_desc_dma_p->kaddrp;
935 tx_desc_ring_vp =
936 (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
937 tx_rd_index = tx_ring_p->rd_index;
938 tx_desc_p = &tx_desc_ring_vp[tx_rd_index];
939 tx_msg_ring = tx_ring_p->tx_msg_ring;
940 tx_msg_p = &tx_msg_ring[tx_rd_index];
941 tdc = tx_ring_p->tdc;
942 tdc_stats = tx_ring_p->tdc_stats;
943 if (tx_ring_p->descs_pending > tdc_stats->tx_max_pend) {
944 tdc_stats->tx_max_pend = tx_ring_p->descs_pending;
945 }
946
947 tail_index = tx_ring_p->wr_index;
948 tail_wrap = tx_ring_p->wr_index_wrap;
949
950 NXGE_DEBUG_MSG((nxgep, TX_CTL,
951 "==> nxge_txdma_reclaim: tdc %d tx_rd_index %d "
952 "tail_index %d tail_wrap %d "
953 "tx_desc_p $%p ($%p) ",
954 tdc, tx_rd_index, tail_index, tail_wrap,
955 tx_desc_p, (*(uint64_t *)tx_desc_p)));
956 /*
957 * Read the hardware maintained transmit head
958 * and wrap around bit.
959 */
960 TXDMA_REG_READ64(handle, TX_RING_HDL_REG, tdc, &tx_head.value);
961 head_index = tx_head.bits.ldw.head;
962 head_wrap = tx_head.bits.ldw.wrap;
963 NXGE_DEBUG_MSG((nxgep, TX_CTL,
964 "==> nxge_txdma_reclaim: "
965 "tx_rd_index %d tail %d tail_wrap %d "
966 "head %d wrap %d",
967 tx_rd_index, tail_index, tail_wrap,
968 head_index, head_wrap));
969
970 if (head_index == tail_index) {
971 if (TXDMA_RING_EMPTY(head_index, head_wrap,
972 tail_index, tail_wrap) &&
973 (head_index == tx_rd_index)) {
974 NXGE_DEBUG_MSG((nxgep, TX_CTL,
975 "==> nxge_txdma_reclaim: EMPTY"));
976 return (B_TRUE);
977 }
978
979 NXGE_DEBUG_MSG((nxgep, TX_CTL,
980 "==> nxge_txdma_reclaim: Checking "
981 "if ring full"));
982 if (TXDMA_RING_FULL(head_index, head_wrap, tail_index,
983 tail_wrap)) {
984 NXGE_DEBUG_MSG((nxgep, TX_CTL,
985 "==> nxge_txdma_reclaim: full"));
986 return (B_FALSE);
987 }
988 }
989
990 NXGE_DEBUG_MSG((nxgep, TX_CTL,
991 "==> nxge_txdma_reclaim: tx_rd_index and head_index"));
992
993 tx_desc_pp = &r_tx_desc;
994 while ((tx_rd_index != head_index) &&
995 (tx_ring_p->descs_pending != 0)) {
996
997 NXGE_DEBUG_MSG((nxgep, TX_CTL,
998 "==> nxge_txdma_reclaim: Checking if pending"));
999
1000 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1001 "==> nxge_txdma_reclaim: "
1002 "descs_pending %d ",
1003 tx_ring_p->descs_pending));
1004
1005 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1006 "==> nxge_txdma_reclaim: "
1007 "(tx_rd_index %d head_index %d "
1008 "(tx_desc_p $%p)",
1009 tx_rd_index, head_index,
1010 tx_desc_p));
1011
1012 tx_desc_pp->value = tx_desc_p->value;
1013 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1014 "==> nxge_txdma_reclaim: "
1015 "(tx_rd_index %d head_index %d "
1016 "tx_desc_p $%p (desc value 0x%llx) ",
1017 tx_rd_index, head_index,
1018 tx_desc_pp, (*(uint64_t *)tx_desc_pp)));
1019
1020 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1021 "==> nxge_txdma_reclaim: dump desc:"));
1022
1023 pkt_len = tx_desc_pp->bits.hdw.tr_len;
1024 tdc_stats->obytes += (pkt_len - TX_PKT_HEADER_SIZE);
1025 tdc_stats->opackets += tx_desc_pp->bits.hdw.sop;
1026 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1027 "==> nxge_txdma_reclaim: pkt_len %d "
1028 "tdc channel %d opackets %d",
1029 pkt_len,
1030 tdc,
1031 tdc_stats->opackets));
1032
1033 if (tx_msg_p->flags.dma_type == USE_DVMA) {
1034 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1035 "tx_desc_p = $%p "
1036 "tx_desc_pp = $%p "
1037 "index = %d",
1038 tx_desc_p,
1039 tx_desc_pp,
1040 tx_ring_p->rd_index));
1041 (void) dvma_unload(tx_msg_p->dvma_handle,
1042 0, -1);
1043 tx_msg_p->dvma_handle = NULL;
1044 if (tx_ring_p->dvma_wr_index ==
1045 tx_ring_p->dvma_wrap_mask) {
1046 tx_ring_p->dvma_wr_index = 0;
1047 } else {
1048 tx_ring_p->dvma_wr_index++;
1049 }
1050 tx_ring_p->dvma_pending--;
1051 } else if (tx_msg_p->flags.dma_type ==
1052 USE_DMA) {
1053 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1054 "==> nxge_txdma_reclaim: "
1055 "USE DMA"));
1056 if (rc = ddi_dma_unbind_handle
1057 (tx_msg_p->dma_handle)) {
1058 cmn_err(CE_WARN, "!nxge_reclaim: "
1059 "ddi_dma_unbind_handle "
1060 "failed. status %d", rc);
1061 }
1062 }
1063 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1064 "==> nxge_txdma_reclaim: count packets"));
1065 /*
1066 * count a chained packet only once.
1067 */
1068 if (tx_msg_p->tx_message != NULL) {
1069 freemsg(tx_msg_p->tx_message);
1070 tx_msg_p->tx_message = NULL;
1071 }
1072
1073 tx_msg_p->flags.dma_type = USE_NONE;
1074 tx_rd_index = tx_ring_p->rd_index;
1075 tx_rd_index = (tx_rd_index + 1) &
1076 tx_ring_p->tx_wrap_mask;
1077 tx_ring_p->rd_index = tx_rd_index;
1078 tx_ring_p->descs_pending--;
1079 tx_desc_p = &tx_desc_ring_vp[tx_rd_index];
1080 tx_msg_p = &tx_msg_ring[tx_rd_index];
1081 }
1082
1083 status = (nmblks <= ((int)tx_ring_p->tx_ring_size -
1084 (int)tx_ring_p->descs_pending - TX_FULL_MARK));
1085 if (status) {
1086 (void) cas32((uint32_t *)&tx_ring_p->queueing, 1, 0);
1087 }
1088 } else {
1089 status = (nmblks <= ((int)tx_ring_p->tx_ring_size -
1090 (int)tx_ring_p->descs_pending - TX_FULL_MARK));
1091 }
1092
1093 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1094 "<== nxge_txdma_reclaim status = 0x%08x", status));
1095
1096 return (status);
1097 }
1098
1099 /*
1100 * nxge_tx_intr
1101 *
1102 * Process a TDC interrupt
1103 *
1104 * Arguments:
1105 * arg1 A Logical Device state Vector (LSV) data structure.
1106 * arg2 nxge_t *
1107 *
1108 * Notes:
1109 *
1110 * NPI/NXGE function calls:
1111 * npi_txdma_control_status()
1112 * npi_intr_ldg_mgmt_set()
1113 *
1114 * nxge_tx_err_evnts()
1115 * nxge_txdma_reclaim()
1116 *
1117 * Registers accessed:
1118 * TX_CS DMC+0x40028 Transmit Control And Status
1119 * PIO_LDSV
1120 *
1121 * Context:
1122 * Any domain
1123 */
1124 uint_t
1125 nxge_tx_intr(void *arg1, void *arg2)
1126 {
1127 p_nxge_ldv_t ldvp = (p_nxge_ldv_t)arg1;
1128 p_nxge_t nxgep = (p_nxge_t)arg2;
1129 p_nxge_ldg_t ldgp;
1130 uint8_t channel;
1131 uint32_t vindex;
1132 npi_handle_t handle;
1133 tx_cs_t cs;
1134 p_tx_ring_t *tx_rings;
1135 p_tx_ring_t tx_ring_p;
1136 npi_status_t rs = NPI_SUCCESS;
1137 uint_t serviced = DDI_INTR_UNCLAIMED;
1138 nxge_status_t status = NXGE_OK;
1139
1140 if (ldvp == NULL) {
1141 NXGE_DEBUG_MSG((NULL, INT_CTL,
1142 "<== nxge_tx_intr: nxgep $%p ldvp $%p",
1143 nxgep, ldvp));
1144 return (DDI_INTR_UNCLAIMED);
1145 }
1146
1147 if (arg2 == NULL || (void *)ldvp->nxgep != arg2) {
1148 nxgep = ldvp->nxgep;
1149 }
1150 NXGE_DEBUG_MSG((nxgep, INT_CTL,
1151 "==> nxge_tx_intr: nxgep(arg2) $%p ldvp(arg1) $%p",
1152 nxgep, ldvp));
1153
1154 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) ||
1155 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) {
1156 NXGE_DEBUG_MSG((nxgep, INT_CTL,
1157 "<== nxge_tx_intr: interface not started or intialized"));
1158 return (DDI_INTR_CLAIMED);
1159 }
1160
1161 /*
1162 * This interrupt handler is for a specific
1163 * transmit dma channel.
1164 */
1165 handle = NXGE_DEV_NPI_HANDLE(nxgep);
1166 /* Get the control and status for this channel. */
1167 channel = ldvp->channel;
1168 ldgp = ldvp->ldgp;
1169 NXGE_DEBUG_MSG((nxgep, INT_CTL,
1170 "==> nxge_tx_intr: nxgep $%p ldvp (ldvp) $%p "
1171 "channel %d",
1172 nxgep, ldvp, channel));
1173
1174 rs = npi_txdma_control_status(handle, OP_GET, channel, &cs);
1175 vindex = ldvp->vdma_index;
1176 NXGE_DEBUG_MSG((nxgep, INT_CTL,
1177 "==> nxge_tx_intr:channel %d ring index %d status 0x%08x",
1178 channel, vindex, rs));
1179 if (!rs && cs.bits.ldw.mk) {
1180 NXGE_DEBUG_MSG((nxgep, INT_CTL,
1181 "==> nxge_tx_intr:channel %d ring index %d "
1182 "status 0x%08x (mk bit set)",
1183 channel, vindex, rs));
1184 tx_rings = nxgep->tx_rings->rings;
1185 tx_ring_p = tx_rings[vindex];
1186 NXGE_DEBUG_MSG((nxgep, INT_CTL,
1187 "==> nxge_tx_intr:channel %d ring index %d "
1188 "status 0x%08x (mk bit set, calling reclaim)",
1189 channel, vindex, rs));
1190
1191 nxge_tx_ring_task((void *)tx_ring_p);
1192 }
1193
1194 /*
1195 * Process other transmit control and status.
1196 * Check the ldv state.
1197 */
1198 status = nxge_tx_err_evnts(nxgep, ldvp->vdma_index, ldvp, cs);
1199 /*
1200 * Rearm this logical group if this is a single device
1201 * group.
1202 */
1203 if (ldgp->nldvs == 1) {
1204 NXGE_DEBUG_MSG((nxgep, INT_CTL,
1205 "==> nxge_tx_intr: rearm"));
1206 if (status == NXGE_OK) {
1207 if (isLDOMguest(nxgep)) {
1208 nxge_hio_ldgimgn(nxgep, ldgp);
1209 } else {
1210 (void) npi_intr_ldg_mgmt_set(handle, ldgp->ldg,
1211 B_TRUE, ldgp->ldg_timer);
1212 }
1213 }
1214 }
1215
1216 NXGE_DEBUG_MSG((nxgep, INT_CTL, "<== nxge_tx_intr"));
1217 serviced = DDI_INTR_CLAIMED;
1218 return (serviced);
1219 }
1220
1221 void
1222 nxge_txdma_stop(p_nxge_t nxgep) /* Dead */
1223 {
1224 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_stop"));
1225
1226 (void) nxge_link_monitor(nxgep, LINK_MONITOR_STOP);
1227
1228 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_stop"));
1229 }
1230
1231 void
1232 nxge_txdma_stop_start(p_nxge_t nxgep) /* Dead */
1233 {
1234 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_stop_start"));
1235
1236 (void) nxge_txdma_stop(nxgep);
1237
1238 (void) nxge_fixup_txdma_rings(nxgep);
1239 (void) nxge_txdma_hw_mode(nxgep, NXGE_DMA_START);
1240 (void) nxge_tx_mac_enable(nxgep);
1241 (void) nxge_txdma_hw_kick(nxgep);
1242
1243 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_stop_start"));
1244 }
1245
1246 npi_status_t
1247 nxge_txdma_channel_disable(
1248 nxge_t *nxge,
1249 int channel)
1250 {
1251 npi_handle_t handle = NXGE_DEV_NPI_HANDLE(nxge);
1252 npi_status_t rs;
1253 tdmc_intr_dbg_t intr_dbg;
1254
1255 /*
1256 * Stop the dma channel and wait for the stop-done.
1257 * If the stop-done bit is not present, then force
1258 * an error so TXC will stop.
1259 * All channels bound to this port need to be stopped
1260 * and reset after injecting an interrupt error.
1261 */
1262 rs = npi_txdma_channel_disable(handle, channel);
1263 NXGE_DEBUG_MSG((nxge, MEM3_CTL,
1264 "==> nxge_txdma_channel_disable(%d) "
1265 "rs 0x%x", channel, rs));
1266 if (rs != NPI_SUCCESS) {
1267 /* Inject any error */
1268 intr_dbg.value = 0;
1269 intr_dbg.bits.ldw.nack_pref = 1;
1270 NXGE_DEBUG_MSG((nxge, MEM3_CTL,
1271 "==> nxge_txdma_hw_mode: "
1272 "channel %d (stop failed 0x%x) "
1273 "(inject err)", rs, channel));
1274 (void) npi_txdma_inj_int_error_set(
1275 handle, channel, &intr_dbg);
1276 rs = npi_txdma_channel_disable(handle, channel);
1277 NXGE_DEBUG_MSG((nxge, MEM3_CTL,
1278 "==> nxge_txdma_hw_mode: "
1279 "channel %d (stop again 0x%x) "
1280 "(after inject err)",
1281 rs, channel));
1282 }
1283
1284 return (rs);
1285 }
1286
1287 /*
1288 * nxge_txdma_hw_mode
1289 *
1290 * Toggle all TDCs on (enable) or off (disable).
1291 *
1292 * Arguments:
1293 * nxgep
1294 * enable Enable or disable a TDC.
1295 *
1296 * Notes:
1297 *
1298 * NPI/NXGE function calls:
1299 * npi_txdma_channel_enable(TX_CS)
1300 * npi_txdma_channel_disable(TX_CS)
1301 * npi_txdma_inj_int_error_set(TDMC_INTR_DBG)
1302 *
1303 * Registers accessed:
1304 * TX_CS DMC+0x40028 Transmit Control And Status
1305 * TDMC_INTR_DBG DMC + 0x40060 Transmit DMA Interrupt Debug
1306 *
1307 * Context:
1308 * Any domain
1309 */
1310 nxge_status_t
1311 nxge_txdma_hw_mode(p_nxge_t nxgep, boolean_t enable)
1312 {
1313 nxge_grp_set_t *set = &nxgep->tx_set;
1314
1315 npi_handle_t handle;
1316 nxge_status_t status;
1317 npi_status_t rs;
1318 int tdc;
1319
1320 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
1321 "==> nxge_txdma_hw_mode: enable mode %d", enable));
1322
1323 if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) {
1324 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1325 "<== nxge_txdma_mode: not initialized"));
1326 return (NXGE_ERROR);
1327 }
1328
1329 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
1330 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1331 "<== nxge_txdma_hw_mode: NULL ring pointer(s)"));
1332 return (NXGE_ERROR);
1333 }
1334
1335 /* Enable or disable all of the TDCs owned by us. */
1336 handle = NXGE_DEV_NPI_HANDLE(nxgep);
1337 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
1338 if ((1 << tdc) & set->owned.map) {
1339 tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
1340 if (ring) {
1341 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
1342 "==> nxge_txdma_hw_mode: channel %d", tdc));
1343 if (enable) {
1344 rs = npi_txdma_channel_enable
1345 (handle, tdc);
1346 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
1347 "==> nxge_txdma_hw_mode: "
1348 "channel %d (enable) rs 0x%x",
1349 tdc, rs));
1350 } else {
1351 rs = nxge_txdma_channel_disable
1352 (nxgep, tdc);
1353 }
1354 }
1355 }
1356 }
1357
1358 status = ((rs == NPI_SUCCESS) ? NXGE_OK : NXGE_ERROR | rs);
1359
1360 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
1361 "<== nxge_txdma_hw_mode: status 0x%x", status));
1362
1363 return (status);
1364 }
1365
1366 void
1367 nxge_txdma_enable_channel(p_nxge_t nxgep, uint16_t channel)
1368 {
1369 npi_handle_t handle;
1370
1371 NXGE_DEBUG_MSG((nxgep, DMA_CTL,
1372 "==> nxge_txdma_enable_channel: channel %d", channel));
1373
1374 handle = NXGE_DEV_NPI_HANDLE(nxgep);
1375 /* enable the transmit dma channels */
1376 (void) npi_txdma_channel_enable(handle, channel);
1377
1378 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_txdma_enable_channel"));
1379 }
1380
1381 void
1382 nxge_txdma_disable_channel(p_nxge_t nxgep, uint16_t channel)
1383 {
1384 npi_handle_t handle;
1385
1386 NXGE_DEBUG_MSG((nxgep, DMA_CTL,
1387 "==> nxge_txdma_disable_channel: channel %d", channel));
1388
1389 handle = NXGE_DEV_NPI_HANDLE(nxgep);
1390 /* stop the transmit dma channels */
1391 (void) npi_txdma_channel_disable(handle, channel);
1392
1393 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_disable_channel"));
1394 }
1395
1396 /*
1397 * nxge_txdma_stop_inj_err
1398 *
1399 * Stop a TDC. If at first we don't succeed, inject an error.
1400 *
1401 * Arguments:
1402 * nxgep
1403 * channel The channel to stop.
1404 *
1405 * Notes:
1406 *
1407 * NPI/NXGE function calls:
1408 * npi_txdma_channel_disable()
1409 * npi_txdma_inj_int_error_set()
1410 * #if defined(NXGE_DEBUG)
1411 * nxge_txdma_regs_dump_channels(nxgep);
1412 * #endif
1413 *
1414 * Registers accessed:
1415 * TX_CS DMC+0x40028 Transmit Control And Status
1416 * TDMC_INTR_DBG DMC + 0x40060 Transmit DMA Interrupt Debug
1417 *
1418 * Context:
1419 * Any domain
1420 */
1421 int
1422 nxge_txdma_stop_inj_err(p_nxge_t nxgep, int channel)
1423 {
1424 npi_handle_t handle;
1425 tdmc_intr_dbg_t intr_dbg;
1426 int status;
1427 npi_status_t rs = NPI_SUCCESS;
1428
1429 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_stop_inj_err"));
1430 /*
1431 * Stop the dma channel waits for the stop done.
1432 * If the stop done bit is not set, then create
1433 * an error.
1434 */
1435 handle = NXGE_DEV_NPI_HANDLE(nxgep);
1436 rs = npi_txdma_channel_disable(handle, channel);
1437 status = ((rs == NPI_SUCCESS) ? NXGE_OK : NXGE_ERROR | rs);
1438 if (status == NXGE_OK) {
1439 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1440 "<== nxge_txdma_stop_inj_err (channel %d): "
1441 "stopped OK", channel));
1442 return (status);
1443 }
1444
1445 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
1446 "==> nxge_txdma_stop_inj_err (channel %d): stop failed (0x%x) "
1447 "injecting error", channel, rs));
1448 /* Inject any error */
1449 intr_dbg.value = 0;
1450 intr_dbg.bits.ldw.nack_pref = 1;
1451 (void) npi_txdma_inj_int_error_set(handle, channel, &intr_dbg);
1452
1453 /* Stop done bit will be set as a result of error injection */
1454 rs = npi_txdma_channel_disable(handle, channel);
1455 status = ((rs == NPI_SUCCESS) ? NXGE_OK : NXGE_ERROR | rs);
1456 if (!(rs & NPI_TXDMA_STOP_FAILED)) {
1457 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1458 "<== nxge_txdma_stop_inj_err (channel %d): "
1459 "stopped OK ", channel));
1460 return (status);
1461 }
1462
1463 #if defined(NXGE_DEBUG)
1464 nxge_txdma_regs_dump_channels(nxgep);
1465 #endif
1466 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
1467 "==> nxge_txdma_stop_inj_err (channel): stop failed (0x%x) "
1468 " (injected error but still not stopped)", channel, rs));
1469
1470 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_stop_inj_err"));
1471 return (status);
1472 }
1473
1474 /*ARGSUSED*/
1475 void
1476 nxge_fixup_txdma_rings(p_nxge_t nxgep)
1477 {
1478 nxge_grp_set_t *set = &nxgep->tx_set;
1479 int tdc;
1480
1481 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_fixup_txdma_rings"));
1482
1483 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
1484 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1485 "<== nxge_fixup_txdma_rings: NULL ring pointer(s)"));
1486 return;
1487 }
1488
1489 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
1490 if ((1 << tdc) & set->owned.map) {
1491 tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
1492 if (ring) {
1493 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
1494 "==> nxge_fixup_txdma_rings: channel %d",
1495 tdc));
1496 nxge_txdma_fixup_channel(nxgep, ring, tdc);
1497 }
1498 }
1499 }
1500
1501 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_fixup_txdma_rings"));
1502 }
1503
1504 /*ARGSUSED*/
1505 void
1506 nxge_txdma_fix_channel(p_nxge_t nxgep, uint16_t channel)
1507 {
1508 p_tx_ring_t ring_p;
1509
1510 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fix_channel"));
1511 ring_p = nxge_txdma_get_ring(nxgep, channel);
1512 if (ring_p == NULL) {
1513 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fix_channel"));
1514 return;
1515 }
1516
1517 if (ring_p->tdc != channel) {
1518 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1519 "<== nxge_txdma_fix_channel: channel not matched "
1520 "ring tdc %d passed channel",
1521 ring_p->tdc, channel));
1522 return;
1523 }
1524
1525 nxge_txdma_fixup_channel(nxgep, ring_p, channel);
1526
1527 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fix_channel"));
1528 }
1529
1530 /*ARGSUSED*/
1531 void
1532 nxge_txdma_fixup_channel(p_nxge_t nxgep, p_tx_ring_t ring_p, uint16_t channel)
1533 {
1534 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fixup_channel"));
1535
1536 if (ring_p == NULL) {
1537 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1538 "<== nxge_txdma_fixup_channel: NULL ring pointer"));
1539 return;
1540 }
1541
1542 if (ring_p->tdc != channel) {
1543 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1544 "<== nxge_txdma_fixup_channel: channel not matched "
1545 "ring tdc %d passed channel",
1546 ring_p->tdc, channel));
1547 return;
1548 }
1549
1550 MUTEX_ENTER(&ring_p->lock);
1551 (void) nxge_txdma_reclaim(nxgep, ring_p, 0);
1552 ring_p->rd_index = 0;
1553 ring_p->wr_index = 0;
1554 ring_p->ring_head.value = 0;
1555 ring_p->ring_kick_tail.value = 0;
1556 ring_p->descs_pending = 0;
1557 MUTEX_EXIT(&ring_p->lock);
1558
1559 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fixup_channel"));
1560 }
1561
1562 /*ARGSUSED*/
1563 void
1564 nxge_txdma_hw_kick(p_nxge_t nxgep)
1565 {
1566 nxge_grp_set_t *set = &nxgep->tx_set;
1567 int tdc;
1568
1569 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_hw_kick"));
1570
1571 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
1572 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1573 "<== nxge_txdma_hw_kick: NULL ring pointer(s)"));
1574 return;
1575 }
1576
1577 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
1578 if ((1 << tdc) & set->owned.map) {
1579 tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
1580 if (ring) {
1581 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
1582 "==> nxge_txdma_hw_kick: channel %d", tdc));
1583 nxge_txdma_hw_kick_channel(nxgep, ring, tdc);
1584 }
1585 }
1586 }
1587
1588 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_hw_kick"));
1589 }
1590
1591 /*ARGSUSED*/
1592 void
1593 nxge_txdma_kick_channel(p_nxge_t nxgep, uint16_t channel)
1594 {
1595 p_tx_ring_t ring_p;
1596
1597 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_kick_channel"));
1598
1599 ring_p = nxge_txdma_get_ring(nxgep, channel);
1600 if (ring_p == NULL) {
1601 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1602 " nxge_txdma_kick_channel"));
1603 return;
1604 }
1605
1606 if (ring_p->tdc != channel) {
1607 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1608 "<== nxge_txdma_kick_channel: channel not matched "
1609 "ring tdc %d passed channel",
1610 ring_p->tdc, channel));
1611 return;
1612 }
1613
1614 nxge_txdma_hw_kick_channel(nxgep, ring_p, channel);
1615
1616 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_kick_channel"));
1617 }
1618
1619 /*ARGSUSED*/
1620 void
1621 nxge_txdma_hw_kick_channel(p_nxge_t nxgep, p_tx_ring_t ring_p, uint16_t channel)
1622 {
1623
1624 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_hw_kick_channel"));
1625
1626 if (ring_p == NULL) {
1627 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1628 "<== nxge_txdma_hw_kick_channel: NULL ring pointer"));
1629 return;
1630 }
1631
1632 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_hw_kick_channel"));
1633 }
1634
1635 /*
1636 * nxge_check_tx_hang
1637 *
1638 * Check the state of all TDCs belonging to nxgep.
1639 *
1640 * Arguments:
1641 * nxgep
1642 *
1643 * Notes:
1644 * Called by nxge_hw.c:nxge_check_hw_state().
1645 *
1646 * NPI/NXGE function calls:
1647 *
1648 * Registers accessed:
1649 *
1650 * Context:
1651 * Any domain
1652 */
1653 /*ARGSUSED*/
1654 void
1655 nxge_check_tx_hang(p_nxge_t nxgep)
1656 {
1657 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_check_tx_hang"));
1658
1659 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) ||
1660 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) {
1661 goto nxge_check_tx_hang_exit;
1662 }
1663
1664 /*
1665 * Needs inputs from hardware for regs:
1666 * head index had not moved since last timeout.
1667 * packets not transmitted or stuffed registers.
1668 */
1669 if (nxge_txdma_hung(nxgep)) {
1670 nxge_fixup_hung_txdma_rings(nxgep);
1671 }
1672
1673 nxge_check_tx_hang_exit:
1674 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_check_tx_hang"));
1675 }
1676
1677 /*
1678 * nxge_txdma_hung
1679 *
1680 * Reset a TDC.
1681 *
1682 * Arguments:
1683 * nxgep
1684 * channel The channel to reset.
1685 * reg_data The current TX_CS.
1686 *
1687 * Notes:
1688 * Called by nxge_check_tx_hang()
1689 *
1690 * NPI/NXGE function calls:
1691 * nxge_txdma_channel_hung()
1692 *
1693 * Registers accessed:
1694 *
1695 * Context:
1696 * Any domain
1697 */
1698 int
1699 nxge_txdma_hung(p_nxge_t nxgep)
1700 {
1701 nxge_grp_set_t *set = &nxgep->tx_set;
1702 int tdc;
1703 boolean_t shared;
1704
1705 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_hung"));
1706
1707 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
1708 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1709 "<== nxge_txdma_hung: NULL ring pointer(s)"));
1710 return (B_FALSE);
1711 }
1712
1713 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
1714 /*
1715 * Grab the shared state of the TDC.
1716 */
1717 if (isLDOMservice(nxgep)) {
1718 nxge_hio_data_t *nhd =
1719 (nxge_hio_data_t *)nxgep->nxge_hw_p->hio;
1720
1721 MUTEX_ENTER(&nhd->lock);
1722 shared = nxgep->tdc_is_shared[tdc];
1723 MUTEX_EXIT(&nhd->lock);
1724 } else {
1725 shared = B_FALSE;
1726 }
1727
1728 /*
1729 * Now, process continue to process.
1730 */
1731 if (((1 << tdc) & set->owned.map) && !shared) {
1732 tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
1733 if (ring) {
1734 if (nxge_txdma_channel_hung(nxgep, ring, tdc)) {
1735 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1736 "==> nxge_txdma_hung: TDC %d hung",
1737 tdc));
1738 return (B_TRUE);
1739 }
1740 }
1741 }
1742 }
1743
1744 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_hung"));
1745
1746 return (B_FALSE);
1747 }
1748
1749 /*
1750 * nxge_txdma_channel_hung
1751 *
1752 * Reset a TDC.
1753 *
1754 * Arguments:
1755 * nxgep
1756 * ring <channel>'s ring.
1757 * channel The channel to reset.
1758 *
1759 * Notes:
1760 * Called by nxge_txdma.c:nxge_txdma_hung()
1761 *
1762 * NPI/NXGE function calls:
1763 * npi_txdma_ring_head_get()
1764 *
1765 * Registers accessed:
1766 * TX_RING_HDL DMC+0x40010 Transmit Ring Head Low
1767 *
1768 * Context:
1769 * Any domain
1770 */
1771 int
1772 nxge_txdma_channel_hung(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, uint16_t channel)
1773 {
1774 uint16_t head_index, tail_index;
1775 boolean_t head_wrap, tail_wrap;
1776 npi_handle_t handle;
1777 tx_ring_hdl_t tx_head;
1778 uint_t tx_rd_index;
1779
1780 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_channel_hung"));
1781
1782 handle = NXGE_DEV_NPI_HANDLE(nxgep);
1783 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1784 "==> nxge_txdma_channel_hung: channel %d", channel));
1785 MUTEX_ENTER(&tx_ring_p->lock);
1786 (void) nxge_txdma_reclaim(nxgep, tx_ring_p, 0);
1787
1788 tail_index = tx_ring_p->wr_index;
1789 tail_wrap = tx_ring_p->wr_index_wrap;
1790 tx_rd_index = tx_ring_p->rd_index;
1791 MUTEX_EXIT(&tx_ring_p->lock);
1792
1793 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1794 "==> nxge_txdma_channel_hung: tdc %d tx_rd_index %d "
1795 "tail_index %d tail_wrap %d ",
1796 channel, tx_rd_index, tail_index, tail_wrap));
1797 /*
1798 * Read the hardware maintained transmit head
1799 * and wrap around bit.
1800 */
1801 (void) npi_txdma_ring_head_get(handle, channel, &tx_head);
1802 head_index = tx_head.bits.ldw.head;
1803 head_wrap = tx_head.bits.ldw.wrap;
1804 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1805 "==> nxge_txdma_channel_hung: "
1806 "tx_rd_index %d tail %d tail_wrap %d "
1807 "head %d wrap %d",
1808 tx_rd_index, tail_index, tail_wrap,
1809 head_index, head_wrap));
1810
1811 if (TXDMA_RING_EMPTY(head_index, head_wrap,
1812 tail_index, tail_wrap) &&
1813 (head_index == tx_rd_index)) {
1814 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1815 "==> nxge_txdma_channel_hung: EMPTY"));
1816 return (B_FALSE);
1817 }
1818
1819 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1820 "==> nxge_txdma_channel_hung: Checking if ring full"));
1821 if (TXDMA_RING_FULL(head_index, head_wrap, tail_index,
1822 tail_wrap)) {
1823 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1824 "==> nxge_txdma_channel_hung: full"));
1825 return (B_TRUE);
1826 }
1827
1828 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_channel_hung"));
1829
1830 return (B_FALSE);
1831 }
1832
1833 /*
1834 * nxge_fixup_hung_txdma_rings
1835 *
1836 * Disable a TDC.
1837 *
1838 * Arguments:
1839 * nxgep
1840 * channel The channel to reset.
1841 * reg_data The current TX_CS.
1842 *
1843 * Notes:
1844 * Called by nxge_check_tx_hang()
1845 *
1846 * NPI/NXGE function calls:
1847 * npi_txdma_ring_head_get()
1848 *
1849 * Registers accessed:
1850 * TX_RING_HDL DMC+0x40010 Transmit Ring Head Low
1851 *
1852 * Context:
1853 * Any domain
1854 */
1855 /*ARGSUSED*/
1856 void
1857 nxge_fixup_hung_txdma_rings(p_nxge_t nxgep)
1858 {
1859 nxge_grp_set_t *set = &nxgep->tx_set;
1860 int tdc;
1861
1862 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_fixup_hung_txdma_rings"));
1863
1864 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
1865 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1866 "<== nxge_fixup_hung_txdma_rings: NULL ring pointer(s)"));
1867 return;
1868 }
1869
1870 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
1871 if ((1 << tdc) & set->owned.map) {
1872 tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
1873 if (ring) {
1874 nxge_txdma_fixup_hung_channel(nxgep, ring, tdc);
1875 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1876 "==> nxge_fixup_hung_txdma_rings: TDC %d",
1877 tdc));
1878 }
1879 }
1880 }
1881
1882 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_fixup_hung_txdma_rings"));
1883 }
1884
1885 /*
1886 * nxge_txdma_fixup_hung_channel
1887 *
1888 * 'Fix' a hung TDC.
1889 *
1890 * Arguments:
1891 * nxgep
1892 * channel The channel to fix.
1893 *
1894 * Notes:
1895 * Called by nxge_fixup_hung_txdma_rings()
1896 *
1897 * 1. Reclaim the TDC.
1898 * 2. Disable the TDC.
1899 *
1900 * NPI/NXGE function calls:
1901 * nxge_txdma_reclaim()
1902 * npi_txdma_channel_disable(TX_CS)
1903 * npi_txdma_inj_int_error_set(TDMC_INTR_DBG)
1904 *
1905 * Registers accessed:
1906 * TX_CS DMC+0x40028 Transmit Control And Status
1907 * TDMC_INTR_DBG DMC + 0x40060 Transmit DMA Interrupt Debug
1908 *
1909 * Context:
1910 * Any domain
1911 */
1912 /*ARGSUSED*/
1913 void
1914 nxge_txdma_fix_hung_channel(p_nxge_t nxgep, uint16_t channel)
1915 {
1916 p_tx_ring_t ring_p;
1917
1918 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fix_hung_channel"));
1919 ring_p = nxge_txdma_get_ring(nxgep, channel);
1920 if (ring_p == NULL) {
1921 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1922 "<== nxge_txdma_fix_hung_channel"));
1923 return;
1924 }
1925
1926 if (ring_p->tdc != channel) {
1927 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1928 "<== nxge_txdma_fix_hung_channel: channel not matched "
1929 "ring tdc %d passed channel",
1930 ring_p->tdc, channel));
1931 return;
1932 }
1933
1934 nxge_txdma_fixup_channel(nxgep, ring_p, channel);
1935
1936 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fix_hung_channel"));
1937 }
1938
1939 /*ARGSUSED*/
1940 void
1941 nxge_txdma_fixup_hung_channel(p_nxge_t nxgep, p_tx_ring_t ring_p,
1942 uint16_t channel)
1943 {
1944 npi_handle_t handle;
1945 tdmc_intr_dbg_t intr_dbg;
1946 int status = NXGE_OK;
1947
1948 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fixup_hung_channel"));
1949
1950 if (ring_p == NULL) {
1951 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1952 "<== nxge_txdma_fixup_channel: NULL ring pointer"));
1953 return;
1954 }
1955
1956 if (ring_p->tdc != channel) {
1957 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1958 "<== nxge_txdma_fixup_hung_channel: channel "
1959 "not matched "
1960 "ring tdc %d passed channel",
1961 ring_p->tdc, channel));
1962 return;
1963 }
1964
1965 /* Reclaim descriptors */
1966 MUTEX_ENTER(&ring_p->lock);
1967 (void) nxge_txdma_reclaim(nxgep, ring_p, 0);
1968 MUTEX_EXIT(&ring_p->lock);
1969
1970 handle = NXGE_DEV_NPI_HANDLE(nxgep);
1971 /*
1972 * Stop the dma channel waits for the stop done.
1973 * If the stop done bit is not set, then force
1974 * an error.
1975 */
1976 status = npi_txdma_channel_disable(handle, channel);
1977 if (!(status & NPI_TXDMA_STOP_FAILED)) {
1978 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1979 "<== nxge_txdma_fixup_hung_channel: stopped OK "
1980 "ring tdc %d passed channel %d",
1981 ring_p->tdc, channel));
1982 return;
1983 }
1984
1985 /* Inject any error */
1986 intr_dbg.value = 0;
1987 intr_dbg.bits.ldw.nack_pref = 1;
1988 (void) npi_txdma_inj_int_error_set(handle, channel, &intr_dbg);
1989
1990 /* Stop done bit will be set as a result of error injection */
1991 status = npi_txdma_channel_disable(handle, channel);
1992 if (!(status & NPI_TXDMA_STOP_FAILED)) {
1993 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1994 "<== nxge_txdma_fixup_hung_channel: stopped again"
1995 "ring tdc %d passed channel",
1996 ring_p->tdc, channel));
1997 return;
1998 }
1999
2000 NXGE_DEBUG_MSG((nxgep, TX_CTL,
2001 "<== nxge_txdma_fixup_hung_channel: stop done still not set!! "
2002 "ring tdc %d passed channel",
2003 ring_p->tdc, channel));
2004
2005 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fixup_hung_channel"));
2006 }
2007
2008 /*ARGSUSED*/
2009 void
2010 nxge_reclaim_rings(p_nxge_t nxgep)
2011 {
2012 nxge_grp_set_t *set = &nxgep->tx_set;
2013 int tdc;
2014
2015 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reclaim_rings"));
2016
2017 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
2018 NXGE_DEBUG_MSG((nxgep, TX_CTL,
2019 "<== nxge_fixup_hung_txdma_rings: NULL ring pointer(s)"));
2020 return;
2021 }
2022
2023 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
2024 if ((1 << tdc) & set->owned.map) {
2025 tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
2026 if (ring) {
2027 NXGE_DEBUG_MSG((nxgep, TX_CTL,
2028 "==> nxge_reclaim_rings: TDC %d", tdc));
2029 MUTEX_ENTER(&ring->lock);
2030 (void) nxge_txdma_reclaim(nxgep, ring, 0);
2031 MUTEX_EXIT(&ring->lock);
2032 }
2033 }
2034 }
2035
2036 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_reclaim_rings"));
2037 }
2038
2039 void
2040 nxge_txdma_regs_dump_channels(p_nxge_t nxgep)
2041 {
2042 nxge_grp_set_t *set = &nxgep->tx_set;
2043 npi_handle_t handle;
2044 int tdc;
2045
2046 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_regs_dump_channels"));
2047
2048 handle = NXGE_DEV_NPI_HANDLE(nxgep);
2049
2050 if (!isLDOMguest(nxgep)) {
2051 (void) npi_txdma_dump_fzc_regs(handle);
2052
2053 /* Dump TXC registers. */
2054 (void) npi_txc_dump_fzc_regs(handle);
2055 (void) npi_txc_dump_port_fzc_regs(handle, nxgep->function_num);
2056 }
2057
2058 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
2059 NXGE_DEBUG_MSG((nxgep, TX_CTL,
2060 "<== nxge_fixup_hung_txdma_rings: NULL ring pointer(s)"));
2061 return;
2062 }
2063
2064 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
2065 if ((1 << tdc) & set->owned.map) {
2066 tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
2067 if (ring) {
2068 NXGE_DEBUG_MSG((nxgep, TX_CTL,
2069 "==> nxge_txdma_regs_dump_channels: "
2070 "TDC %d", tdc));
2071 (void) npi_txdma_dump_tdc_regs(handle, tdc);
2072
2073 /* Dump TXC registers, if able to. */
2074 if (!isLDOMguest(nxgep)) {
2075 NXGE_DEBUG_MSG((nxgep, TX_CTL,
2076 "==> nxge_txdma_regs_dump_channels:"
2077 " FZC TDC %d", tdc));
2078 (void) npi_txc_dump_tdc_fzc_regs
2079 (handle, tdc);
2080 }
2081 nxge_txdma_regs_dump(nxgep, tdc);
2082 }
2083 }
2084 }
2085
2086 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_regs_dump"));
2087 }
2088
2089 void
2090 nxge_txdma_regs_dump(p_nxge_t nxgep, int channel)
2091 {
2092 npi_handle_t handle;
2093 tx_ring_hdl_t hdl;
2094 tx_ring_kick_t kick;
2095 tx_cs_t cs;
2096 txc_control_t control;
2097 uint32_t bitmap = 0;
2098 uint32_t burst = 0;
2099 uint32_t bytes = 0;
2100 dma_log_page_t cfg;
2101
2102 printf("\n\tfunc # %d tdc %d ",
2103 nxgep->function_num, channel);
2104 cfg.page_num = 0;
2105 handle = NXGE_DEV_NPI_HANDLE(nxgep);
2106 (void) npi_txdma_log_page_get(handle, channel, &cfg);
2107 printf("\n\tlog page func %d valid page 0 %d",
2108 cfg.func_num, cfg.valid);
2109 cfg.page_num = 1;
2110 (void) npi_txdma_log_page_get(handle, channel, &cfg);
2111 printf("\n\tlog page func %d valid page 1 %d",
2112 cfg.func_num, cfg.valid);
2113
2114 (void) npi_txdma_ring_head_get(handle, channel, &hdl);
2115 (void) npi_txdma_desc_kick_reg_get(handle, channel, &kick);
2116 printf("\n\thead value is 0x%0llx",
2117 (long long)hdl.value);
2118 printf("\n\thead index %d", hdl.bits.ldw.head);
2119 printf("\n\tkick value is 0x%0llx",
2120 (long long)kick.value);
2121 printf("\n\ttail index %d\n", kick.bits.ldw.tail);
2122
2123 (void) npi_txdma_control_status(handle, OP_GET, channel, &cs);
2124 printf("\n\tControl statue is 0x%0llx", (long long)cs.value);
2125 printf("\n\tControl status RST state %d", cs.bits.ldw.rst);
2126
2127 (void) npi_txc_control(handle, OP_GET, &control);
2128 (void) npi_txc_port_dma_list_get(handle, nxgep->function_num, &bitmap);
2129 (void) npi_txc_dma_max_burst(handle, OP_GET, channel, &burst);
2130 (void) npi_txc_dma_bytes_transmitted(handle, channel, &bytes);
2131
2132 printf("\n\tTXC port control 0x%0llx",
2133 (long long)control.value);
2134 printf("\n\tTXC port bitmap 0x%x", bitmap);
2135 printf("\n\tTXC max burst %d", burst);
2136 printf("\n\tTXC bytes xmt %d\n", bytes);
2137
2138 {
2139 ipp_status_t status;
2140
2141 (void) npi_ipp_get_status(handle, nxgep->function_num, &status);
2142 #if defined(__i386)
2143 printf("\n\tIPP status 0x%llux\n", (uint64_t)status.value);
2144 #else
2145 printf("\n\tIPP status 0x%lux\n", (uint64_t)status.value);
2146 #endif
2147 }
2148 }
2149
2150 /*
2151 * nxge_tdc_hvio_setup
2152 *
2153 * I'm not exactly sure what this code does.
2154 *
2155 * Arguments:
2156 * nxgep
2157 * channel The channel to map.
2158 *
2159 * Notes:
2160 *
2161 * NPI/NXGE function calls:
2162 * na
2163 *
2164 * Context:
2165 * Service domain?
2166 */
2167 #if defined(sun4v) && defined(NIU_LP_WORKAROUND)
2168 static void
2169 nxge_tdc_hvio_setup(
2170 nxge_t *nxgep, int channel)
2171 {
2172 nxge_dma_common_t *data;
2173 nxge_dma_common_t *control;
2174 tx_ring_t *ring;
2175
2176 ring = nxgep->tx_rings->rings[channel];
2177 data = nxgep->tx_buf_pool_p->dma_buf_pool_p[channel];
2178
2179 ring->hv_set = B_FALSE;
2180
2181 ring->hv_tx_buf_base_ioaddr_pp =
2182 (uint64_t)data->orig_ioaddr_pp;
2183 ring->hv_tx_buf_ioaddr_size =
2184 (uint64_t)data->orig_alength;
2185
2186 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma_channel: "
2187 "hv data buf base io $%p size 0x%llx (%d) buf base io $%p "
2188 "orig vatopa base io $%p orig_len 0x%llx (%d)",
2189 ring->hv_tx_buf_base_ioaddr_pp,
2190 ring->hv_tx_buf_ioaddr_size, ring->hv_tx_buf_ioaddr_size,
2191 data->ioaddr_pp, data->orig_vatopa,
2192 data->orig_alength, data->orig_alength));
2193
2194 control = nxgep->tx_cntl_pool_p->dma_buf_pool_p[channel];
2195
2196 ring->hv_tx_cntl_base_ioaddr_pp =
2197 (uint64_t)control->orig_ioaddr_pp;
2198 ring->hv_tx_cntl_ioaddr_size =
2199 (uint64_t)control->orig_alength;
2200
2201 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma_channel: "
2202 "hv cntl base io $%p orig ioaddr_pp ($%p) "
2203 "orig vatopa ($%p) size 0x%llx (%d 0x%x)",
2204 ring->hv_tx_cntl_base_ioaddr_pp,
2205 control->orig_ioaddr_pp, control->orig_vatopa,
2206 ring->hv_tx_cntl_ioaddr_size,
2207 control->orig_alength, control->orig_alength));
2208 }
2209 #endif
2210
2211 static nxge_status_t
2212 nxge_map_txdma(p_nxge_t nxgep, int channel)
2213 {
2214 nxge_dma_common_t **pData;
2215 nxge_dma_common_t **pControl;
2216 tx_ring_t **pRing, *ring;
2217 tx_mbox_t **mailbox;
2218 uint32_t num_chunks;
2219
2220 nxge_status_t status = NXGE_OK;
2221
2222 NXGE_ERROR_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma"));
2223
2224 if (!nxgep->tx_cntl_pool_p->buf_allocated) {
2225 if (nxge_alloc_tx_mem_pool(nxgep) != NXGE_OK) {
2226 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2227 "<== nxge_map_txdma: buf not allocated"));
2228 return (NXGE_ERROR);
2229 }
2230 }
2231
2232 if (nxge_alloc_txb(nxgep, channel) != NXGE_OK)
2233 return (NXGE_ERROR);
2234
2235 num_chunks = nxgep->tx_buf_pool_p->num_chunks[channel];
2236 pData = &nxgep->tx_buf_pool_p->dma_buf_pool_p[channel];
2237 pControl = &nxgep->tx_cntl_pool_p->dma_buf_pool_p[channel];
2238 pRing = &nxgep->tx_rings->rings[channel];
2239 mailbox = &nxgep->tx_mbox_areas_p->txmbox_areas_p[channel];
2240
2241 NXGE_ERROR_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma: "
2242 "tx_rings $%p tx_desc_rings $%p",
2243 nxgep->tx_rings, nxgep->tx_rings->rings));
2244
2245 /*
2246 * Map descriptors from the buffer pools for <channel>.
2247 */
2248
2249 /*
2250 * Set up and prepare buffer blocks, descriptors
2251 * and mailbox.
2252 */
2253 status = nxge_map_txdma_channel(nxgep, channel,
2254 pData, pRing, num_chunks, pControl, mailbox);
2255 if (status != NXGE_OK) {
2256 NXGE_ERROR_MSG((nxgep, MEM3_CTL,
2257 "==> nxge_map_txdma(%d): nxge_map_txdma_channel() "
2258 "returned 0x%x",
2259 nxgep, channel, status));
2260 return (status);
2261 }
2262
2263 ring = *pRing;
2264
2265 ring->index = (uint16_t)channel;
2266 ring->tdc_stats = &nxgep->statsp->tdc_stats[channel];
2267
2268 #if defined(sun4v) && defined(NIU_LP_WORKAROUND)
2269 if (isLDOMguest(nxgep)) {
2270 (void) nxge_tdc_lp_conf(nxgep, channel);
2271 } else {
2272 nxge_tdc_hvio_setup(nxgep, channel);
2273 }
2274 #endif
2275
2276 NXGE_ERROR_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma: "
2277 "(status 0x%x channel %d)", status, channel));
2278
2279 return (status);
2280 }
2281
2282 static nxge_status_t
2283 nxge_map_txdma_channel(p_nxge_t nxgep, uint16_t channel,
2284 p_nxge_dma_common_t *dma_buf_p,
2285 p_tx_ring_t *tx_desc_p,
2286 uint32_t num_chunks,
2287 p_nxge_dma_common_t *dma_cntl_p,
2288 p_tx_mbox_t *tx_mbox_p)
2289 {
2290 int status = NXGE_OK;
2291
2292 /*
2293 * Set up and prepare buffer blocks, descriptors
2294 * and mailbox.
2295 */
2296 NXGE_ERROR_MSG((nxgep, MEM3_CTL,
2297 "==> nxge_map_txdma_channel (channel %d)", channel));
2298 /*
2299 * Transmit buffer blocks
2300 */
2301 status = nxge_map_txdma_channel_buf_ring(nxgep, channel,
2302 dma_buf_p, tx_desc_p, num_chunks);
2303 if (status != NXGE_OK) {
2304 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2305 "==> nxge_map_txdma_channel (channel %d): "
2306 "map buffer failed 0x%x", channel, status));
2307 goto nxge_map_txdma_channel_exit;
2308 }
2309
2310 /*
2311 * Transmit block ring, and mailbox.
2312 */
2313 nxge_map_txdma_channel_cfg_ring(nxgep, channel, dma_cntl_p, *tx_desc_p,
2314 tx_mbox_p);
2315
2316 goto nxge_map_txdma_channel_exit;
2317
2318 nxge_map_txdma_channel_fail1:
2319 NXGE_ERROR_MSG((nxgep, MEM3_CTL,
2320 "==> nxge_map_txdma_channel: unmap buf"
2321 "(status 0x%x channel %d)",
2322 status, channel));
2323 nxge_unmap_txdma_channel_buf_ring(nxgep, *tx_desc_p);
2324
2325 nxge_map_txdma_channel_exit:
2326 NXGE_ERROR_MSG((nxgep, MEM3_CTL,
2327 "<== nxge_map_txdma_channel: "
2328 "(status 0x%x channel %d)",
2329 status, channel));
2330
2331 return (status);
2332 }
2333
2334 /*ARGSUSED*/
2335 static void
2336 nxge_unmap_txdma_channel(p_nxge_t nxgep, uint16_t channel)
2337 {
2338 tx_ring_t *ring;
2339 tx_mbox_t *mailbox;
2340
2341 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2342 "==> nxge_unmap_txdma_channel (channel %d)", channel));
2343 /*
2344 * unmap tx block ring, and mailbox.
2345 */
2346 ring = nxgep->tx_rings->rings[channel];
2347 mailbox = nxgep->tx_mbox_areas_p->txmbox_areas_p[channel];
2348
2349 (void) nxge_unmap_txdma_channel_cfg_ring(nxgep, ring, mailbox);
2350
2351 /* unmap buffer blocks */
2352 (void) nxge_unmap_txdma_channel_buf_ring(nxgep, ring);
2353
2354 nxge_free_txb(nxgep, channel);
2355
2356 /*
2357 * Cleanup the reference to the ring now that it does not exist.
2358 */
2359 nxgep->tx_rings->rings[channel] = NULL;
2360
2361 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_unmap_txdma_channel"));
2362 }
2363
2364 /*
2365 * nxge_map_txdma_channel_cfg_ring
2366 *
2367 * Map a TDC into our kernel space.
2368 * This function allocates all of the per-channel data structures.
2369 *
2370 * Arguments:
2371 * nxgep
2372 * dma_channel The channel to map.
2373 * dma_cntl_p
2374 * tx_ring_p dma_channel's transmit ring
2375 * tx_mbox_p dma_channel's mailbox
2376 *
2377 * Notes:
2378 *
2379 * NPI/NXGE function calls:
2380 * nxge_setup_dma_common()
2381 *
2382 * Registers accessed:
2383 * none.
2384 *
2385 * Context:
2386 * Any domain
2387 */
2388 /*ARGSUSED*/
2389 static void
2390 nxge_map_txdma_channel_cfg_ring(p_nxge_t nxgep, uint16_t dma_channel,
2391 p_nxge_dma_common_t *dma_cntl_p,
2392 p_tx_ring_t tx_ring_p,
2393 p_tx_mbox_t *tx_mbox_p)
2394 {
2395 p_tx_mbox_t mboxp;
2396 p_nxge_dma_common_t cntl_dmap;
2397 p_nxge_dma_common_t dmap;
2398 p_tx_rng_cfig_t tx_ring_cfig_p;
2399 p_tx_ring_kick_t tx_ring_kick_p;
2400 p_tx_cs_t tx_cs_p;
2401 p_tx_dma_ent_msk_t tx_evmask_p;
2402 p_txdma_mbh_t mboxh_p;
2403 p_txdma_mbl_t mboxl_p;
2404 uint64_t tx_desc_len;
2405
2406 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2407 "==> nxge_map_txdma_channel_cfg_ring"));
2408
2409 cntl_dmap = *dma_cntl_p;
2410
2411 dmap = (p_nxge_dma_common_t)&tx_ring_p->tdc_desc;
2412 nxge_setup_dma_common(dmap, cntl_dmap, tx_ring_p->tx_ring_size,
2413 sizeof (tx_desc_t));
2414 /*
2415 * Zero out transmit ring descriptors.
2416 */
2417 bzero((caddr_t)dmap->kaddrp, dmap->alength);
2418 tx_ring_cfig_p = &(tx_ring_p->tx_ring_cfig);
2419 tx_ring_kick_p = &(tx_ring_p->tx_ring_kick);
2420 tx_cs_p = &(tx_ring_p->tx_cs);
2421 tx_evmask_p = &(tx_ring_p->tx_evmask);
2422 tx_ring_cfig_p->value = 0;
2423 tx_ring_kick_p->value = 0;
2424 tx_cs_p->value = 0;
2425 tx_evmask_p->value = 0;
2426
2427 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2428 "==> nxge_map_txdma_channel_cfg_ring: channel %d des $%p",
2429 dma_channel,
2430 dmap->dma_cookie.dmac_laddress));
2431
2432 tx_ring_cfig_p->value = 0;
2433 tx_desc_len = (uint64_t)(tx_ring_p->tx_ring_size >> 3);
2434 tx_ring_cfig_p->value =
2435 (dmap->dma_cookie.dmac_laddress & TX_RNG_CFIG_ADDR_MASK) |
2436 (tx_desc_len << TX_RNG_CFIG_LEN_SHIFT);
2437
2438 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2439 "==> nxge_map_txdma_channel_cfg_ring: channel %d cfg 0x%llx",
2440 dma_channel,
2441 tx_ring_cfig_p->value));
2442
2443 tx_cs_p->bits.ldw.rst = 1;
2444
2445 /* Map in mailbox */
2446 mboxp = (p_tx_mbox_t)
2447 KMEM_ZALLOC(sizeof (tx_mbox_t), KM_SLEEP);
2448 dmap = (p_nxge_dma_common_t)&mboxp->tx_mbox;
2449 nxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (txdma_mailbox_t));
2450 mboxh_p = (p_txdma_mbh_t)&tx_ring_p->tx_mbox_mbh;
2451 mboxl_p = (p_txdma_mbl_t)&tx_ring_p->tx_mbox_mbl;
2452 mboxh_p->value = mboxl_p->value = 0;
2453
2454 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2455 "==> nxge_map_txdma_channel_cfg_ring: mbox 0x%lx",
2456 dmap->dma_cookie.dmac_laddress));
2457
2458 mboxh_p->bits.ldw.mbaddr = ((dmap->dma_cookie.dmac_laddress >>
2459 TXDMA_MBH_ADDR_SHIFT) & TXDMA_MBH_MASK);
2460
2461 mboxl_p->bits.ldw.mbaddr = ((dmap->dma_cookie.dmac_laddress &
2462 TXDMA_MBL_MASK) >> TXDMA_MBL_SHIFT);
2463
2464 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2465 "==> nxge_map_txdma_channel_cfg_ring: mbox 0x%lx",
2466 dmap->dma_cookie.dmac_laddress));
2467 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2468 "==> nxge_map_txdma_channel_cfg_ring: hmbox $%p "
2469 "mbox $%p",
2470 mboxh_p->bits.ldw.mbaddr, mboxl_p->bits.ldw.mbaddr));
2471 tx_ring_p->page_valid.value = 0;
2472 tx_ring_p->page_mask_1.value = tx_ring_p->page_mask_2.value = 0;
2473 tx_ring_p->page_value_1.value = tx_ring_p->page_value_2.value = 0;
2474 tx_ring_p->page_reloc_1.value = tx_ring_p->page_reloc_2.value = 0;
2475 tx_ring_p->page_hdl.value = 0;
2476
2477 tx_ring_p->page_valid.bits.ldw.page0 = 1;
2478 tx_ring_p->page_valid.bits.ldw.page1 = 1;
2479
2480 tx_ring_p->max_burst.value = 0;
2481 tx_ring_p->max_burst.bits.ldw.dma_max_burst = TXC_DMA_MAX_BURST_DEFAULT;
2482
2483 *tx_mbox_p = mboxp;
2484
2485 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2486 "<== nxge_map_txdma_channel_cfg_ring"));
2487 }
2488
2489 /*ARGSUSED*/
2490 static void
2491 nxge_unmap_txdma_channel_cfg_ring(p_nxge_t nxgep,
2492 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
2493 {
2494 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2495 "==> nxge_unmap_txdma_channel_cfg_ring: channel %d",
2496 tx_ring_p->tdc));
2497
2498 KMEM_FREE(tx_mbox_p, sizeof (tx_mbox_t));
2499
2500 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2501 "<== nxge_unmap_txdma_channel_cfg_ring"));
2502 }
2503
2504 /*
2505 * nxge_map_txdma_channel_buf_ring
2506 *
2507 *
2508 * Arguments:
2509 * nxgep
2510 * channel The channel to map.
2511 * dma_buf_p
2512 * tx_desc_p channel's descriptor ring
2513 * num_chunks
2514 *
2515 * Notes:
2516 *
2517 * NPI/NXGE function calls:
2518 * nxge_setup_dma_common()
2519 *
2520 * Registers accessed:
2521 * none.
2522 *
2523 * Context:
2524 * Any domain
2525 */
2526 static nxge_status_t
2527 nxge_map_txdma_channel_buf_ring(p_nxge_t nxgep, uint16_t channel,
2528 p_nxge_dma_common_t *dma_buf_p,
2529 p_tx_ring_t *tx_desc_p, uint32_t num_chunks)
2530 {
2531 p_nxge_dma_common_t dma_bufp, tmp_bufp;
2532 p_nxge_dma_common_t dmap;
2533 nxge_os_dma_handle_t tx_buf_dma_handle;
2534 p_tx_ring_t tx_ring_p;
2535 p_tx_msg_t tx_msg_ring;
2536 nxge_status_t status = NXGE_OK;
2537 int ddi_status = DDI_SUCCESS;
2538 int i, j, index;
2539 uint32_t size, bsize;
2540 uint32_t nblocks, nmsgs;
2541 char qname[TASKQ_NAMELEN];
2542
2543 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2544 "==> nxge_map_txdma_channel_buf_ring"));
2545
2546 dma_bufp = tmp_bufp = *dma_buf_p;
2547 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2548 " nxge_map_txdma_channel_buf_ring: channel %d to map %d "
2549 "chunks bufp $%p",
2550 channel, num_chunks, dma_bufp));
2551
2552 nmsgs = 0;
2553 for (i = 0; i < num_chunks; i++, tmp_bufp++) {
2554 nmsgs += tmp_bufp->nblocks;
2555 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2556 "==> nxge_map_txdma_channel_buf_ring: channel %d "
2557 "bufp $%p nblocks %d nmsgs %d",
2558 channel, tmp_bufp, tmp_bufp->nblocks, nmsgs));
2559 }
2560 if (!nmsgs) {
2561 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2562 "<== nxge_map_txdma_channel_buf_ring: channel %d "
2563 "no msg blocks",
2564 channel));
2565 status = NXGE_ERROR;
2566 goto nxge_map_txdma_channel_buf_ring_exit;
2567 }
2568
2569 tx_ring_p = (p_tx_ring_t)
2570 KMEM_ZALLOC(sizeof (tx_ring_t), KM_SLEEP);
2571 MUTEX_INIT(&tx_ring_p->lock, NULL, MUTEX_DRIVER,
2572 (void *)nxgep->interrupt_cookie);
2573
2574 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, NXGE_TX_RING_ONLINE);
2575 tx_ring_p->tx_ring_busy = B_FALSE;
2576 tx_ring_p->nxgep = nxgep;
2577 tx_ring_p->tx_ring_handle = (mac_ring_handle_t)NULL;
2578 (void) snprintf(qname, TASKQ_NAMELEN, "tx_%d_%d",
2579 nxgep->instance, channel);
2580 tx_ring_p->taskq = ddi_taskq_create(nxgep->dip, qname, 1,
2581 TASKQ_DEFAULTPRI, 0);
2582 if (tx_ring_p->taskq == NULL) {
2583 goto nxge_map_txdma_channel_buf_ring_fail1;
2584 }
2585
2586 /*
2587 * Allocate transmit message rings and handles for packets
2588 * not to be copied to premapped buffers.
2589 */
2590 size = nmsgs * sizeof (tx_msg_t);
2591 tx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP);
2592 for (i = 0; i < nmsgs; i++) {
2593 ddi_status = ddi_dma_alloc_handle(nxgep->dip, &nxge_tx_dma_attr,
2594 DDI_DMA_DONTWAIT, 0,
2595 &tx_msg_ring[i].dma_handle);
2596 if (ddi_status != DDI_SUCCESS) {
2597 status |= NXGE_DDI_FAILED;
2598 break;
2599 }
2600 }
2601 if (i < nmsgs) {
2602 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2603 "Allocate handles failed."));
2604 goto nxge_map_txdma_channel_buf_ring_fail1;
2605 }
2606
2607 tx_ring_p->tdc = channel;
2608 tx_ring_p->tx_msg_ring = tx_msg_ring;
2609 tx_ring_p->tx_ring_size = nmsgs;
2610 tx_ring_p->num_chunks = num_chunks;
2611 if (!nxge_tx_intr_thres) {
2612 nxge_tx_intr_thres = tx_ring_p->tx_ring_size/4;
2613 }
2614 tx_ring_p->tx_wrap_mask = tx_ring_p->tx_ring_size - 1;
2615 tx_ring_p->rd_index = 0;
2616 tx_ring_p->wr_index = 0;
2617 tx_ring_p->ring_head.value = 0;
2618 tx_ring_p->ring_kick_tail.value = 0;
2619 tx_ring_p->descs_pending = 0;
2620
2621 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2622 "==> nxge_map_txdma_channel_buf_ring: channel %d "
2623 "actual tx desc max %d nmsgs %d "
2624 "(config nxge_tx_ring_size %d)",
2625 channel, tx_ring_p->tx_ring_size, nmsgs,
2626 nxge_tx_ring_size));
2627
2628 /*
2629 * Map in buffers from the buffer pool.
2630 */
2631 index = 0;
2632 bsize = dma_bufp->block_size;
2633
2634 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma_channel_buf_ring: "
2635 "dma_bufp $%p tx_rng_p $%p "
2636 "tx_msg_rng_p $%p bsize %d",
2637 dma_bufp, tx_ring_p, tx_msg_ring, bsize));
2638
2639 tx_buf_dma_handle = dma_bufp->dma_handle;
2640 for (i = 0; i < num_chunks; i++, dma_bufp++) {
2641 bsize = dma_bufp->block_size;
2642 nblocks = dma_bufp->nblocks;
2643 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2644 "==> nxge_map_txdma_channel_buf_ring: dma chunk %d "
2645 "size %d dma_bufp $%p",
2646 i, sizeof (nxge_dma_common_t), dma_bufp));
2647
2648 for (j = 0; j < nblocks; j++) {
2649 tx_msg_ring[index].buf_dma_handle = tx_buf_dma_handle;
2650 dmap = &tx_msg_ring[index++].buf_dma;
2651 #ifdef TX_MEM_DEBUG
2652 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2653 "==> nxge_map_txdma_channel_buf_ring: j %d"
2654 "dmap $%p", i, dmap));
2655 #endif
2656 nxge_setup_dma_common(dmap, dma_bufp, 1,
2657 bsize);
2658 }
2659 }
2660
2661 if (i < num_chunks) {
2662 status = NXGE_ERROR;
2663 goto nxge_map_txdma_channel_buf_ring_fail1;
2664 }
2665
2666 *tx_desc_p = tx_ring_p;
2667
2668 goto nxge_map_txdma_channel_buf_ring_exit;
2669
2670 nxge_map_txdma_channel_buf_ring_fail1:
2671 if (tx_ring_p->taskq) {
2672 ddi_taskq_destroy(tx_ring_p->taskq);
2673 tx_ring_p->taskq = NULL;
2674 }
2675
2676 index--;
2677 for (; index >= 0; index--) {
2678 if (tx_msg_ring[index].dma_handle != NULL) {
2679 ddi_dma_free_handle(&tx_msg_ring[index].dma_handle);
2680 }
2681 }
2682 MUTEX_DESTROY(&tx_ring_p->lock);
2683 KMEM_FREE(tx_msg_ring, size);
2684 KMEM_FREE(tx_ring_p, sizeof (tx_ring_t));
2685
2686 status = NXGE_ERROR;
2687
2688 nxge_map_txdma_channel_buf_ring_exit:
2689 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2690 "<== nxge_map_txdma_channel_buf_ring status 0x%x", status));
2691
2692 return (status);
2693 }
2694
2695 /*ARGSUSED*/
2696 static void
2697 nxge_unmap_txdma_channel_buf_ring(p_nxge_t nxgep, p_tx_ring_t tx_ring_p)
2698 {
2699 p_tx_msg_t tx_msg_ring;
2700 p_tx_msg_t tx_msg_p;
2701 int i;
2702
2703 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2704 "==> nxge_unmap_txdma_channel_buf_ring"));
2705 if (tx_ring_p == NULL) {
2706 NXGE_DEBUG_MSG((nxgep, TX_CTL,
2707 "<== nxge_unmap_txdma_channel_buf_ring: NULL ringp"));
2708 return;
2709 }
2710 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2711 "==> nxge_unmap_txdma_channel_buf_ring: channel %d",
2712 tx_ring_p->tdc));
2713
2714 tx_msg_ring = tx_ring_p->tx_msg_ring;
2715
2716 /*
2717 * Since the serialization thread, timer thread and
2718 * interrupt thread can all call the transmit reclaim,
2719 * the unmapping function needs to acquire the lock
2720 * to free those buffers which were transmitted
2721 * by the hardware already.
2722 */
2723 MUTEX_ENTER(&tx_ring_p->lock);
2724 NXGE_DEBUG_MSG((nxgep, TX_CTL,
2725 "==> nxge_unmap_txdma_channel_buf_ring (reclaim): "
2726 "channel %d",
2727 tx_ring_p->tdc));
2728 (void) nxge_txdma_reclaim(nxgep, tx_ring_p, 0);
2729
2730 for (i = 0; i < tx_ring_p->tx_ring_size; i++) {
2731 tx_msg_p = &tx_msg_ring[i];
2732 if (tx_msg_p->tx_message != NULL) {
2733 freemsg(tx_msg_p->tx_message);
2734 tx_msg_p->tx_message = NULL;
2735 }
2736 }
2737
2738 for (i = 0; i < tx_ring_p->tx_ring_size; i++) {
2739 if (tx_msg_ring[i].dma_handle != NULL) {
2740 ddi_dma_free_handle(&tx_msg_ring[i].dma_handle);
2741 }
2742 tx_msg_ring[i].dma_handle = NULL;
2743 }
2744
2745 MUTEX_EXIT(&tx_ring_p->lock);
2746
2747 if (tx_ring_p->taskq) {
2748 ddi_taskq_destroy(tx_ring_p->taskq);
2749 tx_ring_p->taskq = NULL;
2750 }
2751
2752 MUTEX_DESTROY(&tx_ring_p->lock);
2753 KMEM_FREE(tx_msg_ring, sizeof (tx_msg_t) * tx_ring_p->tx_ring_size);
2754 KMEM_FREE(tx_ring_p, sizeof (tx_ring_t));
2755
2756 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2757 "<== nxge_unmap_txdma_channel_buf_ring"));
2758 }
2759
2760 static nxge_status_t
2761 nxge_txdma_hw_start(p_nxge_t nxgep, int channel)
2762 {
2763 p_tx_rings_t tx_rings;
2764 p_tx_ring_t *tx_desc_rings;
2765 p_tx_mbox_areas_t tx_mbox_areas_p;
2766 p_tx_mbox_t *tx_mbox_p;
2767 nxge_status_t status = NXGE_OK;
2768
2769 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_txdma_hw_start"));
2770
2771 tx_rings = nxgep->tx_rings;
2772 if (tx_rings == NULL) {
2773 NXGE_DEBUG_MSG((nxgep, TX_CTL,
2774 "<== nxge_txdma_hw_start: NULL ring pointer"));
2775 return (NXGE_ERROR);
2776 }
2777 tx_desc_rings = tx_rings->rings;
2778 if (tx_desc_rings == NULL) {
2779 NXGE_DEBUG_MSG((nxgep, TX_CTL,
2780 "<== nxge_txdma_hw_start: NULL ring pointers"));
2781 return (NXGE_ERROR);
2782 }
2783
2784 NXGE_ERROR_MSG((nxgep, MEM3_CTL, "==> nxge_txdma_hw_start: "
2785 "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings));
2786
2787 tx_mbox_areas_p = nxgep->tx_mbox_areas_p;
2788 tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
2789
2790 status = nxge_txdma_start_channel(nxgep, channel,
2791 (p_tx_ring_t)tx_desc_rings[channel],
2792 (p_tx_mbox_t)tx_mbox_p[channel]);
2793 if (status != NXGE_OK) {
2794 goto nxge_txdma_hw_start_fail1;
2795 }
2796
2797 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_txdma_hw_start: "
2798 "tx_rings $%p rings $%p",
2799 nxgep->tx_rings, nxgep->tx_rings->rings));
2800 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_txdma_hw_start: "
2801 "tx_rings $%p tx_desc_rings $%p",
2802 nxgep->tx_rings, tx_desc_rings));
2803
2804 goto nxge_txdma_hw_start_exit;
2805
2806 nxge_txdma_hw_start_fail1:
2807 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2808 "==> nxge_txdma_hw_start: disable "
2809 "(status 0x%x channel %d)", status, channel));
2810
2811 nxge_txdma_hw_start_exit:
2812 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2813 "==> nxge_txdma_hw_start: (status 0x%x)", status));
2814
2815 return (status);
2816 }
2817
2818 /*
2819 * nxge_txdma_start_channel
2820 *
2821 * Start a TDC.
2822 *
2823 * Arguments:
2824 * nxgep
2825 * channel The channel to start.
2826 * tx_ring_p channel's transmit descriptor ring.
2827 * tx_mbox_p channel' smailbox.
2828 *
2829 * Notes:
2830 *
2831 * NPI/NXGE function calls:
2832 * nxge_reset_txdma_channel()
2833 * nxge_init_txdma_channel_event_mask()
2834 * nxge_enable_txdma_channel()
2835 *
2836 * Registers accessed:
2837 * none directly (see functions above).
2838 *
2839 * Context:
2840 * Any domain
2841 */
2842 static nxge_status_t
2843 nxge_txdma_start_channel(p_nxge_t nxgep, uint16_t channel,
2844 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
2845
2846 {
2847 nxge_status_t status = NXGE_OK;
2848
2849 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2850 "==> nxge_txdma_start_channel (channel %d)", channel));
2851 /*
2852 * TXDMA/TXC must be in stopped state.
2853 */
2854 (void) nxge_txdma_stop_inj_err(nxgep, channel);
2855
2856 /*
2857 * Reset TXDMA channel
2858 */
2859 tx_ring_p->tx_cs.value = 0;
2860 tx_ring_p->tx_cs.bits.ldw.rst = 1;
2861 status = nxge_reset_txdma_channel(nxgep, channel,
2862 tx_ring_p->tx_cs.value);
2863 if (status != NXGE_OK) {
2864 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2865 "==> nxge_txdma_start_channel (channel %d)"
2866 " reset channel failed 0x%x", channel, status));
2867 goto nxge_txdma_start_channel_exit;
2868 }
2869
2870 /*
2871 * Initialize the TXDMA channel specific FZC control
2872 * configurations. These FZC registers are pertaining
2873 * to each TX channel (i.e. logical pages).
2874 */
2875 if (!isLDOMguest(nxgep)) {
2876 status = nxge_init_fzc_txdma_channel(nxgep, channel,
2877 tx_ring_p, tx_mbox_p);
2878 if (status != NXGE_OK) {
2879 goto nxge_txdma_start_channel_exit;
2880 }
2881 }
2882
2883 /*
2884 * Initialize the event masks.
2885 */
2886 tx_ring_p->tx_evmask.value = 0;
2887 status = nxge_init_txdma_channel_event_mask(nxgep,
2888 channel, &tx_ring_p->tx_evmask);
2889 if (status != NXGE_OK) {
2890 goto nxge_txdma_start_channel_exit;
2891 }
2892
2893 /*
2894 * Load TXDMA descriptors, buffers, mailbox,
2895 * initialise the DMA channels and
2896 * enable each DMA channel.
2897 */
2898 status = nxge_enable_txdma_channel(nxgep, channel,
2899 tx_ring_p, tx_mbox_p);
2900 if (status != NXGE_OK) {
2901 goto nxge_txdma_start_channel_exit;
2902 }
2903
2904 nxge_txdma_start_channel_exit:
2905 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_txdma_start_channel"));
2906
2907 return (status);
2908 }
2909
2910 /*
2911 * nxge_txdma_stop_channel
2912 *
2913 * Stop a TDC.
2914 *
2915 * Arguments:
2916 * nxgep
2917 * channel The channel to stop.
2918 * tx_ring_p channel's transmit descriptor ring.
2919 * tx_mbox_p channel' smailbox.
2920 *
2921 * Notes:
2922 *
2923 * NPI/NXGE function calls:
2924 * nxge_txdma_stop_inj_err()
2925 * nxge_reset_txdma_channel()
2926 * nxge_init_txdma_channel_event_mask()
2927 * nxge_init_txdma_channel_cntl_stat()
2928 * nxge_disable_txdma_channel()
2929 *
2930 * Registers accessed:
2931 * none directly (see functions above).
2932 *
2933 * Context:
2934 * Any domain
2935 */
2936 /*ARGSUSED*/
2937 static nxge_status_t
2938 nxge_txdma_stop_channel(p_nxge_t nxgep, uint16_t channel)
2939 {
2940 p_tx_ring_t tx_ring_p;
2941 int status = NXGE_OK;
2942
2943 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2944 "==> nxge_txdma_stop_channel: channel %d", channel));
2945
2946 /*
2947 * Stop (disable) TXDMA and TXC (if stop bit is set
2948 * and STOP_N_GO bit not set, the TXDMA reset state will
2949 * not be set if reset TXDMA.
2950 */
2951 (void) nxge_txdma_stop_inj_err(nxgep, channel);
2952
2953 if (nxgep->tx_rings == NULL) {
2954 status = NXGE_ERROR;
2955 goto nxge_txdma_stop_channel_exit;
2956 }
2957
2958 tx_ring_p = nxgep->tx_rings->rings[channel];
2959 if (tx_ring_p == NULL) {
2960 status = NXGE_ERROR;
2961 goto nxge_txdma_stop_channel_exit;
2962 }
2963
2964 /*
2965 * Reset TXDMA channel
2966 */
2967 tx_ring_p->tx_cs.value = 0;
2968 tx_ring_p->tx_cs.bits.ldw.rst = 1;
2969 status = nxge_reset_txdma_channel(nxgep, channel,
2970 tx_ring_p->tx_cs.value);
2971 if (status != NXGE_OK) {
2972 goto nxge_txdma_stop_channel_exit;
2973 }
2974
2975 #ifdef HARDWARE_REQUIRED
2976 /* Set up the interrupt event masks. */
2977 tx_ring_p->tx_evmask.value = 0;
2978 status = nxge_init_txdma_channel_event_mask(nxgep,
2979 channel, &tx_ring_p->tx_evmask);
2980 if (status != NXGE_OK) {
2981 goto nxge_txdma_stop_channel_exit;
2982 }
2983
2984 /* Initialize the DMA control and status register */
2985 tx_ring_p->tx_cs.value = TX_ENT_MSK_MK_ALL;
2986 status = nxge_init_txdma_channel_cntl_stat(nxgep, channel,
2987 tx_ring_p->tx_cs.value);
2988 if (status != NXGE_OK) {
2989 goto nxge_txdma_stop_channel_exit;
2990 }
2991
2992 tx_mbox_p = nxgep->tx_mbox_areas_p->txmbox_areas_p[channel];
2993
2994 /* Disable channel */
2995 status = nxge_disable_txdma_channel(nxgep, channel,
2996 tx_ring_p, tx_mbox_p);
2997 if (status != NXGE_OK) {
2998 goto nxge_txdma_start_channel_exit;
2999 }
3000
3001 NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
3002 "==> nxge_txdma_stop_channel: event done"));
3003
3004 #endif
3005
3006 nxge_txdma_stop_channel_exit:
3007 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_txdma_stop_channel"));
3008 return (status);
3009 }
3010
3011 /*
3012 * nxge_txdma_get_ring
3013 *
3014 * Get the ring for a TDC.
3015 *
3016 * Arguments:
3017 * nxgep
3018 * channel
3019 *
3020 * Notes:
3021 *
3022 * NPI/NXGE function calls:
3023 *
3024 * Registers accessed:
3025 *
3026 * Context:
3027 * Any domain
3028 */
3029 static p_tx_ring_t
3030 nxge_txdma_get_ring(p_nxge_t nxgep, uint16_t channel)
3031 {
3032 nxge_grp_set_t *set = &nxgep->tx_set;
3033 int tdc;
3034
3035 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_get_ring"));
3036
3037 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
3038 NXGE_DEBUG_MSG((nxgep, TX_CTL,
3039 "<== nxge_txdma_get_ring: NULL ring pointer(s)"));
3040 goto return_null;
3041 }
3042
3043 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3044 if ((1 << tdc) & set->owned.map) {
3045 tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
3046 if (ring) {
3047 if (channel == ring->tdc) {
3048 NXGE_DEBUG_MSG((nxgep, TX_CTL,
3049 "<== nxge_txdma_get_ring: "
3050 "tdc %d ring $%p", tdc, ring));
3051 return (ring);
3052 }
3053 }
3054 }
3055 }
3056
3057 return_null:
3058 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_get_ring: "
3059 "ring not found"));
3060
3061 return (NULL);
3062 }
3063
3064 /*
3065 * nxge_txdma_get_mbox
3066 *
3067 * Get the mailbox for a TDC.
3068 *
3069 * Arguments:
3070 * nxgep
3071 * channel
3072 *
3073 * Notes:
3074 *
3075 * NPI/NXGE function calls:
3076 *
3077 * Registers accessed:
3078 *
3079 * Context:
3080 * Any domain
3081 */
3082 static p_tx_mbox_t
3083 nxge_txdma_get_mbox(p_nxge_t nxgep, uint16_t channel)
3084 {
3085 nxge_grp_set_t *set = &nxgep->tx_set;
3086 int tdc;
3087
3088 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_get_mbox"));
3089
3090 if (nxgep->tx_mbox_areas_p == 0 ||
3091 nxgep->tx_mbox_areas_p->txmbox_areas_p == 0) {
3092 NXGE_DEBUG_MSG((nxgep, TX_CTL,
3093 "<== nxge_txdma_get_mbox: NULL mailbox pointer(s)"));
3094 goto return_null;
3095 }
3096
3097 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
3098 NXGE_DEBUG_MSG((nxgep, TX_CTL,
3099 "<== nxge_txdma_get_mbox: NULL ring pointer(s)"));
3100 goto return_null;
3101 }
3102
3103 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3104 if ((1 << tdc) & set->owned.map) {
3105 tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
3106 if (ring) {
3107 if (channel == ring->tdc) {
3108 tx_mbox_t *mailbox = nxgep->
3109 tx_mbox_areas_p->
3110 txmbox_areas_p[tdc];
3111 NXGE_DEBUG_MSG((nxgep, TX_CTL,
3112 "<== nxge_txdma_get_mbox: tdc %d "
3113 "ring $%p", tdc, mailbox));
3114 return (mailbox);
3115 }
3116 }
3117 }
3118 }
3119
3120 return_null:
3121 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_get_mbox: "
3122 "mailbox not found"));
3123
3124 return (NULL);
3125 }
3126
3127 /*
3128 * nxge_tx_err_evnts
3129 *
3130 * Recover a TDC.
3131 *
3132 * Arguments:
3133 * nxgep
3134 * index The index to the TDC ring.
3135 * ldvp Used to get the channel number ONLY.
3136 * cs A copy of the bits from TX_CS.
3137 *
3138 * Notes:
3139 * Calling tree:
3140 * nxge_tx_intr()
3141 *
3142 * NPI/NXGE function calls:
3143 * npi_txdma_ring_error_get()
3144 * npi_txdma_inj_par_error_get()
3145 * nxge_txdma_fatal_err_recover()
3146 *
3147 * Registers accessed:
3148 * TX_RNG_ERR_LOGH DMC+0x40048 Transmit Ring Error Log High
3149 * TX_RNG_ERR_LOGL DMC+0x40050 Transmit Ring Error Log Low
3150 * TDMC_INJ_PAR_ERR (FZC_DMC + 0x45040) TDMC Inject Parity Error
3151 *
3152 * Context:
3153 * Any domain XXX Remove code which accesses TDMC_INJ_PAR_ERR.
3154 */
3155 /*ARGSUSED*/
3156 static nxge_status_t
3157 nxge_tx_err_evnts(p_nxge_t nxgep, uint_t index, p_nxge_ldv_t ldvp, tx_cs_t cs)
3158 {
3159 npi_handle_t handle;
3160 npi_status_t rs;
3161 uint8_t channel;
3162 p_tx_ring_t *tx_rings;
3163 p_tx_ring_t tx_ring_p;
3164 p_nxge_tx_ring_stats_t tdc_stats;
3165 boolean_t txchan_fatal = B_FALSE;
3166 nxge_status_t status = NXGE_OK;
3167 tdmc_inj_par_err_t par_err;
3168 uint32_t value;
3169
3170 NXGE_DEBUG_MSG((nxgep, TX2_CTL, "==> nxge_tx_err_evnts"));
3171 handle = NXGE_DEV_NPI_HANDLE(nxgep);
3172 channel = ldvp->channel;
3173
3174 tx_rings = nxgep->tx_rings->rings;
3175 tx_ring_p = tx_rings[index];
3176 tdc_stats = tx_ring_p->tdc_stats;
3177 if ((cs.bits.ldw.pkt_size_err) || (cs.bits.ldw.pref_buf_par_err) ||
3178 (cs.bits.ldw.nack_pref) || (cs.bits.ldw.nack_pkt_rd) ||
3179 (cs.bits.ldw.conf_part_err) || (cs.bits.ldw.pkt_prt_err)) {
3180 if ((rs = npi_txdma_ring_error_get(handle, channel,
3181 &tdc_stats->errlog)) != NPI_SUCCESS)
3182 return (NXGE_ERROR | rs);
3183 }
3184
3185 if (cs.bits.ldw.mbox_err) {
3186 tdc_stats->mbox_err++;
3187 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
3188 NXGE_FM_EREPORT_TDMC_MBOX_ERR);
3189 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3190 "==> nxge_tx_err_evnts(channel %d): "
3191 "fatal error: mailbox", channel));
3192 txchan_fatal = B_TRUE;
3193 }
3194 if (cs.bits.ldw.pkt_size_err) {
3195 tdc_stats->pkt_size_err++;
3196 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
3197 NXGE_FM_EREPORT_TDMC_PKT_SIZE_ERR);
3198 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3199 "==> nxge_tx_err_evnts(channel %d): "
3200 "fatal error: pkt_size_err", channel));
3201 txchan_fatal = B_TRUE;
3202 }
3203 if (cs.bits.ldw.tx_ring_oflow) {
3204 tdc_stats->tx_ring_oflow++;
3205 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
3206 NXGE_FM_EREPORT_TDMC_TX_RING_OFLOW);
3207 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3208 "==> nxge_tx_err_evnts(channel %d): "
3209 "fatal error: tx_ring_oflow", channel));
3210 txchan_fatal = B_TRUE;
3211 }
3212 if (cs.bits.ldw.pref_buf_par_err) {
3213 tdc_stats->pre_buf_par_err++;
3214 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
3215 NXGE_FM_EREPORT_TDMC_PREF_BUF_PAR_ERR);
3216 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3217 "==> nxge_tx_err_evnts(channel %d): "
3218 "fatal error: pre_buf_par_err", channel));
3219 /* Clear error injection source for parity error */
3220 (void) npi_txdma_inj_par_error_get(handle, &value);
3221 par_err.value = value;
3222 par_err.bits.ldw.inject_parity_error &= ~(1 << channel);
3223 (void) npi_txdma_inj_par_error_set(handle, par_err.value);
3224 txchan_fatal = B_TRUE;
3225 }
3226 if (cs.bits.ldw.nack_pref) {
3227 tdc_stats->nack_pref++;
3228 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
3229 NXGE_FM_EREPORT_TDMC_NACK_PREF);
3230 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3231 "==> nxge_tx_err_evnts(channel %d): "
3232 "fatal error: nack_pref", channel));
3233 txchan_fatal = B_TRUE;
3234 }
3235 if (cs.bits.ldw.nack_pkt_rd) {
3236 tdc_stats->nack_pkt_rd++;
3237 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
3238 NXGE_FM_EREPORT_TDMC_NACK_PKT_RD);
3239 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3240 "==> nxge_tx_err_evnts(channel %d): "
3241 "fatal error: nack_pkt_rd", channel));
3242 txchan_fatal = B_TRUE;
3243 }
3244 if (cs.bits.ldw.conf_part_err) {
3245 tdc_stats->conf_part_err++;
3246 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
3247 NXGE_FM_EREPORT_TDMC_CONF_PART_ERR);
3248 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3249 "==> nxge_tx_err_evnts(channel %d): "
3250 "fatal error: config_partition_err", channel));
3251 txchan_fatal = B_TRUE;
3252 }
3253 if (cs.bits.ldw.pkt_prt_err) {
3254 tdc_stats->pkt_part_err++;
3255 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
3256 NXGE_FM_EREPORT_TDMC_PKT_PRT_ERR);
3257 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3258 "==> nxge_tx_err_evnts(channel %d): "
3259 "fatal error: pkt_prt_err", channel));
3260 txchan_fatal = B_TRUE;
3261 }
3262
3263 /* Clear error injection source in case this is an injected error */
3264 TXDMA_REG_WRITE64(nxgep->npi_handle, TDMC_INTR_DBG_REG, channel, 0);
3265
3266 if (txchan_fatal) {
3267 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3268 " nxge_tx_err_evnts: "
3269 " fatal error on channel %d cs 0x%llx\n",
3270 channel, cs.value));
3271 status = nxge_txdma_fatal_err_recover(nxgep, channel,
3272 tx_ring_p);
3273 if (status == NXGE_OK) {
3274 FM_SERVICE_RESTORED(nxgep);
3275 }
3276 }
3277
3278 NXGE_DEBUG_MSG((nxgep, TX2_CTL, "<== nxge_tx_err_evnts"));
3279
3280 return (status);
3281 }
3282
3283 static nxge_status_t
3284 nxge_txdma_fatal_err_recover(
3285 p_nxge_t nxgep,
3286 uint16_t channel,
3287 p_tx_ring_t tx_ring_p)
3288 {
3289 npi_handle_t handle;
3290 npi_status_t rs = NPI_SUCCESS;
3291 p_tx_mbox_t tx_mbox_p;
3292 nxge_status_t status = NXGE_OK;
3293
3294 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fatal_err_recover"));
3295 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3296 "Recovering from TxDMAChannel#%d error...", channel));
3297
3298 /*
3299 * Stop the dma channel waits for the stop done.
3300 * If the stop done bit is not set, then create
3301 * an error.
3302 */
3303
3304 handle = NXGE_DEV_NPI_HANDLE(nxgep);
3305 NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel stop..."));
3306 MUTEX_ENTER(&tx_ring_p->lock);
3307 rs = npi_txdma_channel_control(handle, TXDMA_STOP, channel);
3308 if (rs != NPI_SUCCESS) {
3309 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3310 "==> nxge_txdma_fatal_err_recover (channel %d): "
3311 "stop failed ", channel));
3312 goto fail;
3313 }
3314
3315 NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel reclaim..."));
3316 (void) nxge_txdma_reclaim(nxgep, tx_ring_p, 0);
3317
3318 /*
3319 * Reset TXDMA channel
3320 */
3321 NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel reset..."));
3322 if ((rs = npi_txdma_channel_control(handle, TXDMA_RESET, channel)) !=
3323 NPI_SUCCESS) {
3324 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3325 "==> nxge_txdma_fatal_err_recover (channel %d)"
3326 " reset channel failed 0x%x", channel, rs));
3327 goto fail;
3328 }
3329
3330 /*
3331 * Reset the tail (kick) register to 0.
3332 * (Hardware will not reset it. Tx overflow fatal
3333 * error if tail is not set to 0 after reset!
3334 */
3335 TXDMA_REG_WRITE64(handle, TX_RING_KICK_REG, channel, 0);
3336
3337 /* Restart TXDMA channel */
3338
3339 if (!isLDOMguest(nxgep)) {
3340 tx_mbox_p = nxge_txdma_get_mbox(nxgep, channel);
3341
3342 // XXX This is a problem in HIO!
3343 /*
3344 * Initialize the TXDMA channel specific FZC control
3345 * configurations. These FZC registers are pertaining
3346 * to each TX channel (i.e. logical pages).
3347 */
3348 NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel restart..."));
3349 status = nxge_init_fzc_txdma_channel(nxgep, channel,
3350 tx_ring_p, tx_mbox_p);
3351 if (status != NXGE_OK)
3352 goto fail;
3353 }
3354
3355 /*
3356 * Initialize the event masks.
3357 */
3358 tx_ring_p->tx_evmask.value = 0;
3359 status = nxge_init_txdma_channel_event_mask(nxgep, channel,
3360 &tx_ring_p->tx_evmask);
3361 if (status != NXGE_OK)
3362 goto fail;
3363
3364 tx_ring_p->wr_index_wrap = B_FALSE;
3365 tx_ring_p->wr_index = 0;
3366 tx_ring_p->rd_index = 0;
3367
3368 /*
3369 * Load TXDMA descriptors, buffers, mailbox,
3370 * initialise the DMA channels and
3371 * enable each DMA channel.
3372 */
3373 NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel enable..."));
3374 status = nxge_enable_txdma_channel(nxgep, channel,
3375 tx_ring_p, tx_mbox_p);
3376 MUTEX_EXIT(&tx_ring_p->lock);
3377 if (status != NXGE_OK)
3378 goto fail;
3379
3380 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3381 "Recovery Successful, TxDMAChannel#%d Restored",
3382 channel));
3383 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fatal_err_recover"));
3384
3385 return (NXGE_OK);
3386
3387 fail:
3388 MUTEX_EXIT(&tx_ring_p->lock);
3389
3390 NXGE_DEBUG_MSG((nxgep, TX_CTL,
3391 "nxge_txdma_fatal_err_recover (channel %d): "
3392 "failed to recover this txdma channel", channel));
3393 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed"));
3394
3395 return (status);
3396 }
3397
3398 /*
3399 * nxge_tx_port_fatal_err_recover
3400 *
3401 * Attempt to recover from a fatal port error.
3402 *
3403 * Arguments:
3404 * nxgep
3405 *
3406 * Notes:
3407 * How would a guest do this?
3408 *
3409 * NPI/NXGE function calls:
3410 *
3411 * Registers accessed:
3412 *
3413 * Context:
3414 * Service domain
3415 */
3416 nxge_status_t
3417 nxge_tx_port_fatal_err_recover(p_nxge_t nxgep)
3418 {
3419 nxge_grp_set_t *set = &nxgep->tx_set;
3420 nxge_channel_t tdc;
3421
3422 tx_ring_t *ring;
3423 tx_mbox_t *mailbox;
3424
3425 npi_handle_t handle;
3426 nxge_status_t status;
3427 npi_status_t rs;
3428
3429 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_tx_port_fatal_err_recover"));
3430 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3431 "Recovering from TxPort error..."));
3432
3433 if (isLDOMguest(nxgep)) {
3434 return (NXGE_OK);
3435 }
3436
3437 if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) {
3438 NXGE_DEBUG_MSG((nxgep, TX_CTL,
3439 "<== nxge_tx_port_fatal_err_recover: not initialized"));
3440 return (NXGE_ERROR);
3441 }
3442
3443 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
3444 NXGE_DEBUG_MSG((nxgep, TX_CTL,
3445 "<== nxge_tx_port_fatal_err_recover: "
3446 "NULL ring pointer(s)"));
3447 return (NXGE_ERROR);
3448 }
3449
3450 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3451 if ((1 << tdc) & set->owned.map) {
3452 tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
3453 if (ring)
3454 MUTEX_ENTER(&ring->lock);
3455 }
3456 }
3457
3458 handle = NXGE_DEV_NPI_HANDLE(nxgep);
3459
3460 /*
3461 * Stop all the TDCs owned by us.
3462 * (The shared TDCs will have been stopped by their owners.)
3463 */
3464 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3465 if ((1 << tdc) & set->owned.map) {
3466 ring = nxgep->tx_rings->rings[tdc];
3467 if (ring) {
3468 rs = npi_txdma_channel_control
3469 (handle, TXDMA_STOP, tdc);
3470 if (rs != NPI_SUCCESS) {
3471 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3472 "nxge_tx_port_fatal_err_recover "
3473 "(channel %d): stop failed ", tdc));
3474 goto fail;
3475 }
3476 }
3477 }
3478 }
3479
3480 NXGE_DEBUG_MSG((nxgep, TX_CTL, "Reclaiming all TDCs..."));
3481
3482 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3483 if ((1 << tdc) & set->owned.map) {
3484 tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
3485 if (ring) {
3486 (void) nxge_txdma_reclaim(nxgep, ring, 0);
3487 }
3488 }
3489 }
3490
3491 /*
3492 * Reset all the TDCs.
3493 */
3494 NXGE_DEBUG_MSG((nxgep, TX_CTL, "Resetting all TDCs..."));
3495
3496 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3497 if ((1 << tdc) & set->owned.map) {
3498 tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
3499 if (ring) {
3500 if ((rs = npi_txdma_channel_control
3501 (handle, TXDMA_RESET, tdc))
3502 != NPI_SUCCESS) {
3503 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3504 "nxge_tx_port_fatal_err_recover "
3505 "(channel %d) reset channel "
3506 "failed 0x%x", tdc, rs));
3507 goto fail;
3508 }
3509 }
3510 /*
3511 * Reset the tail (kick) register to 0.
3512 * (Hardware will not reset it. Tx overflow fatal
3513 * error if tail is not set to 0 after reset!
3514 */
3515 TXDMA_REG_WRITE64(handle, TX_RING_KICK_REG, tdc, 0);
3516 }
3517 }
3518
3519 NXGE_DEBUG_MSG((nxgep, TX_CTL, "Restarting all TDCs..."));
3520
3521 /* Restart all the TDCs */
3522 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3523 if ((1 << tdc) & set->owned.map) {
3524 ring = nxgep->tx_rings->rings[tdc];
3525 if (ring) {
3526 mailbox = nxge_txdma_get_mbox(nxgep, tdc);
3527 status = nxge_init_fzc_txdma_channel(nxgep, tdc,
3528 ring, mailbox);
3529 ring->tx_evmask.value = 0;
3530 /*
3531 * Initialize the event masks.
3532 */
3533 status = nxge_init_txdma_channel_event_mask
3534 (nxgep, tdc, &ring->tx_evmask);
3535
3536 ring->wr_index_wrap = B_FALSE;
3537 ring->wr_index = 0;
3538 ring->rd_index = 0;
3539
3540 if (status != NXGE_OK)
3541 goto fail;
3542 if (status != NXGE_OK)
3543 goto fail;
3544 }
3545 }
3546 }
3547
3548 NXGE_DEBUG_MSG((nxgep, TX_CTL, "Re-enabling all TDCs..."));
3549
3550 /* Re-enable all the TDCs */
3551 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3552 if ((1 << tdc) & set->owned.map) {
3553 ring = nxgep->tx_rings->rings[tdc];
3554 if (ring) {
3555 mailbox = nxge_txdma_get_mbox(nxgep, tdc);
3556 status = nxge_enable_txdma_channel(nxgep, tdc,
3557 ring, mailbox);
3558 if (status != NXGE_OK)
3559 goto fail;
3560 }
3561 }
3562 }
3563
3564 /*
3565 * Unlock all the TDCs.
3566 */
3567 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3568 if ((1 << tdc) & set->owned.map) {
3569 tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
3570 if (ring)
3571 MUTEX_EXIT(&ring->lock);
3572 }
3573 }
3574
3575 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Tx port recovery succeeded"));
3576 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_port_fatal_err_recover"));
3577
3578 return (NXGE_OK);
3579
3580 fail:
3581 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3582 if ((1 << tdc) & set->owned.map) {
3583 ring = nxgep->tx_rings->rings[tdc];
3584 if (ring)
3585 MUTEX_EXIT(&ring->lock);
3586 }
3587 }
3588
3589 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Tx port recovery failed"));
3590 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_port_fatal_err_recover"));
3591
3592 return (status);
3593 }
3594
3595 /*
3596 * nxge_txdma_inject_err
3597 *
3598 * Inject an error into a TDC.
3599 *
3600 * Arguments:
3601 * nxgep
3602 * err_id The error to inject.
3603 * chan The channel to inject into.
3604 *
3605 * Notes:
3606 * This is called from nxge_main.c:nxge_err_inject()
3607 * Has this ioctl ever been used?
3608 *
3609 * NPI/NXGE function calls:
3610 * npi_txdma_inj_par_error_get()
3611 * npi_txdma_inj_par_error_set()
3612 *
3613 * Registers accessed:
3614 * TDMC_INJ_PAR_ERR (FZC_DMC + 0x45040) TDMC Inject Parity Error
3615 * TDMC_INTR_DBG DMC + 0x40060 Transmit DMA Interrupt Debug
3616 * TDMC_INTR_DBG DMC + 0x40060 Transmit DMA Interrupt Debug
3617 *
3618 * Context:
3619 * Service domain
3620 */
3621 void
3622 nxge_txdma_inject_err(p_nxge_t nxgep, uint32_t err_id, uint8_t chan)
3623 {
3624 tdmc_intr_dbg_t tdi;
3625 tdmc_inj_par_err_t par_err;
3626 uint32_t value;
3627 npi_handle_t handle;
3628
3629 switch (err_id) {
3630
3631 case NXGE_FM_EREPORT_TDMC_PREF_BUF_PAR_ERR:
3632 handle = NXGE_DEV_NPI_HANDLE(nxgep);
3633 /* Clear error injection source for parity error */
3634 (void) npi_txdma_inj_par_error_get(handle, &value);
3635 par_err.value = value;
3636 par_err.bits.ldw.inject_parity_error &= ~(1 << chan);
3637 (void) npi_txdma_inj_par_error_set(handle, par_err.value);
3638
3639 par_err.bits.ldw.inject_parity_error = (1 << chan);
3640 (void) npi_txdma_inj_par_error_get(handle, &value);
3641 par_err.value = value;
3642 par_err.bits.ldw.inject_parity_error |= (1 << chan);
3643 cmn_err(CE_NOTE, "!Write 0x%llx to TDMC_INJ_PAR_ERR_REG\n",
3644 (unsigned long long)par_err.value);
3645 (void) npi_txdma_inj_par_error_set(handle, par_err.value);
3646 break;
3647
3648 case NXGE_FM_EREPORT_TDMC_MBOX_ERR:
3649 case NXGE_FM_EREPORT_TDMC_NACK_PREF:
3650 case NXGE_FM_EREPORT_TDMC_NACK_PKT_RD:
3651 case NXGE_FM_EREPORT_TDMC_PKT_SIZE_ERR:
3652 case NXGE_FM_EREPORT_TDMC_TX_RING_OFLOW:
3653 case NXGE_FM_EREPORT_TDMC_CONF_PART_ERR:
3654 case NXGE_FM_EREPORT_TDMC_PKT_PRT_ERR:
3655 TXDMA_REG_READ64(nxgep->npi_handle, TDMC_INTR_DBG_REG,
3656 chan, &tdi.value);
3657 if (err_id == NXGE_FM_EREPORT_TDMC_PREF_BUF_PAR_ERR)
3658 tdi.bits.ldw.pref_buf_par_err = 1;
3659 else if (err_id == NXGE_FM_EREPORT_TDMC_MBOX_ERR)
3660 tdi.bits.ldw.mbox_err = 1;
3661 else if (err_id == NXGE_FM_EREPORT_TDMC_NACK_PREF)
3662 tdi.bits.ldw.nack_pref = 1;
3663 else if (err_id == NXGE_FM_EREPORT_TDMC_NACK_PKT_RD)
3664 tdi.bits.ldw.nack_pkt_rd = 1;
3665 else if (err_id == NXGE_FM_EREPORT_TDMC_PKT_SIZE_ERR)
3666 tdi.bits.ldw.pkt_size_err = 1;
3667 else if (err_id == NXGE_FM_EREPORT_TDMC_TX_RING_OFLOW)
3668 tdi.bits.ldw.tx_ring_oflow = 1;
3669 else if (err_id == NXGE_FM_EREPORT_TDMC_CONF_PART_ERR)
3670 tdi.bits.ldw.conf_part_err = 1;
3671 else if (err_id == NXGE_FM_EREPORT_TDMC_PKT_PRT_ERR)
3672 tdi.bits.ldw.pkt_part_err = 1;
3673 #if defined(__i386)
3674 cmn_err(CE_NOTE, "!Write 0x%llx to TDMC_INTR_DBG_REG\n",
3675 tdi.value);
3676 #else
3677 cmn_err(CE_NOTE, "!Write 0x%lx to TDMC_INTR_DBG_REG\n",
3678 tdi.value);
3679 #endif
3680 TXDMA_REG_WRITE64(nxgep->npi_handle, TDMC_INTR_DBG_REG,
3681 chan, tdi.value);
3682
3683 break;
3684 }
3685 }