Print this page
5045 use atomic_{inc,dec}_* instead of atomic_add_*
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/cxgbe/t4nex/t4_l2t.c
+++ new/usr/src/uts/common/io/cxgbe/t4nex/t4_l2t.c
1 1 /*
2 2 * This file and its contents are supplied under the terms of the
3 3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 4 * You may only use this file in accordance with the terms of version
5 5 * 1.0 of the CDDL.
6 6 *
7 7 * A full copy of the text of the CDDL should have accompanied this
8 8 * source. A copy of the CDDL is also available via the Internet at
9 9 * http://www.illumos.org/license/CDDL.
10 10 */
11 11
12 12 /*
13 13 * This file is part of the Chelsio T4 support code.
14 14 *
15 15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved.
16 16 *
17 17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
20 20 * release for licensing terms and conditions.
21 21 */
22 22
23 23 #include <sys/ddi.h>
24 24 #include <sys/sunddi.h>
25 25 #include <sys/sunndi.h>
26 26 #include <sys/atomic.h>
27 27 #include <sys/dlpi.h>
28 28 #include <sys/pattr.h>
29 29 #include <sys/strsubr.h>
30 30 #include <sys/stream.h>
31 31 #include <sys/strsun.h>
32 32 #include <sys/ethernet.h>
33 33 #include <inet/ip.h>
34 34 #include <inet/ipclassifier.h>
35 35 #include <inet/tcp.h>
36 36
37 37 #include "common/common.h"
38 38 #include "common/t4_msg.h"
39 39 #include "common/t4_regs.h"
40 40 #include "common/t4_regs_values.h"
41 41 #include "t4_l2t.h"
42 42
43 43 /* identifies sync vs async L2T_WRITE_REQs */
44 44 #define S_SYNC_WR 12
45 45 #define V_SYNC_WR(x) ((x) << S_SYNC_WR)
46 46 #define F_SYNC_WR V_SYNC_WR(1)
47 47 #define VLAN_NONE 0xfff
48 48
49 49 /*
50 50 * jhash.h: Jenkins hash support.
51 51 *
52 52 * Copyright (C) 1996 Bob Jenkins (bob_jenkins@burtleburtle.net)
53 53 *
54 54 * http://burtleburtle.net/bob/hash/
55 55 *
56 56 * These are the credits from Bob's sources:
57 57 *
58 58 * lookup2.c, by Bob Jenkins, December 1996, Public Domain.
59 59 * hash(), hash2(), hash3, and mix() are externally useful functions.
60 60 * Routines to test the hash are included if SELF_TEST is defined.
61 61 * You can use this free for any purpose. It has no warranty.
62 62 */
63 63
64 64 /* NOTE: Arguments are modified. */
65 65 #define __jhash_mix(a, b, c) \
66 66 { \
67 67 a -= b; a -= c; a ^= (c>>13); \
68 68 b -= c; b -= a; b ^= (a<<8); \
69 69 c -= a; c -= b; c ^= (b>>13); \
70 70 a -= b; a -= c; a ^= (c>>12); \
71 71 b -= c; b -= a; b ^= (a<<16); \
72 72 c -= a; c -= b; c ^= (b>>5); \
73 73 a -= b; a -= c; a ^= (c>>3); \
74 74 b -= c; b -= a; b ^= (a<<10); \
75 75 c -= a; c -= b; c ^= (b>>15); \
76 76 }
77 77
78 78 /* The golden ration: an arbitrary value */
79 79 #define JHASH_GOLDEN_RATIO 0x9e3779b9
80 80
81 81 /*
82 82 * A special ultra-optimized versions that knows they are hashing exactly
83 83 * 3, 2 or 1 word(s).
84 84 *
85 85 * NOTE: In partilar the "c += length; __jhash_mix(a,b,c);" normally
86 86 * done at the end is not done here.
87 87 */
88 88 static inline u32
89 89 jhash_3words(u32 a, u32 b, u32 c, u32 initval)
90 90 {
91 91 a += JHASH_GOLDEN_RATIO;
92 92 b += JHASH_GOLDEN_RATIO;
93 93 c += initval;
94 94
95 95 __jhash_mix(a, b, c);
96 96
97 97 return (c);
98 98 }
99 99
100 100 static inline u32
101 101 jhash_2words(u32 a, u32 b, u32 initval)
102 102 {
103 103 return (jhash_3words(a, b, 0, initval));
104 104 }
105 105
106 106 #ifndef container_of
107 107 #define container_of(p, s, f) ((s *)(((uint8_t *)(p)) - offsetof(s, f)))
108 108 #endif
109 109
110 110 #if defined(__GNUC__)
111 111 #define likely(x) __builtin_expect((x), 1)
112 112 #define unlikely(x) __builtin_expect((x), 0)
113 113 #else
114 114 #define likely(x) (x)
115 115 #define unlikely(x) (x)
116 116 #endif /* defined(__GNUC__) */
117 117
118 118 enum {
119 119 L2T_STATE_VALID, /* entry is up to date */
120 120 L2T_STATE_STALE, /* entry may be used but needs revalidation */
121 121 L2T_STATE_RESOLVING, /* entry needs address resolution */
122 122 L2T_STATE_SYNC_WRITE, /* synchronous write of entry underway */
123 123
124 124 /* when state is one of the below the entry is not hashed */
125 125 L2T_STATE_SWITCHING, /* entry is being used by a switching filter */
126 126 L2T_STATE_UNUSED /* entry not in use */
127 127 };
128 128
129 129 struct l2t_data {
130 130 krwlock_t lock;
131 131 volatile uint_t nfree; /* number of free entries */
132 132 struct l2t_entry *rover; /* starting point for next allocation */
133 133 struct l2t_entry l2tab[L2T_SIZE];
134 134 };
135 135
136 136 #define VLAN_NONE 0xfff
137 137 #define SA(x) ((struct sockaddr *)(x))
138 138 #define SIN(x) ((struct sockaddr_in *)(x))
139 139 #define SINADDR(x) (SIN(x)->sin_addr.s_addr)
140 140 #define atomic_read(x) atomic_add_int_nv(x, 0)
141 141 /*
142 142 * Allocate a free L2T entry.
143 143 * Must be called with l2t_data.lockatomic_load_acq_int held.
144 144 */
145 145 static struct l2t_entry *
146 146 alloc_l2e(struct l2t_data *d)
147 147 {
148 148 struct l2t_entry *end, *e, **p;
149 149
150 150 ASSERT(rw_write_held(&d->lock));
151 151
152 152 if (!atomic_read(&d->nfree))
153 153 return (NULL);
154 154
155 155 /* there's definitely a free entry */
156 156 for (e = d->rover, end = &d->l2tab[L2T_SIZE]; e != end; ++e)
157 157 if (atomic_read(&e->refcnt) == 0)
158 158 goto found;
159 159
160 160 for (e = d->l2tab; atomic_read(&e->refcnt); ++e)
161 161 /* */;
162 162 found:
163 163 d->rover = e + 1;
164 164 atomic_dec_uint(&d->nfree);
165 165
166 166 /*
167 167 * The entry we found may be an inactive entry that is
168 168 * presently in the hash table. We need to remove it.
169 169 */
170 170 if (e->state < L2T_STATE_SWITCHING) {
171 171 for (p = &d->l2tab[e->hash].first; *p; p = &(*p)->next) {
172 172 if (*p == e) {
173 173 *p = e->next;
174 174 e->next = NULL;
175 175 break;
176 176 }
177 177 }
178 178 }
179 179
180 180 e->state = L2T_STATE_UNUSED;
181 181 return (e);
182 182 }
183 183
184 184 /*
185 185 * Write an L2T entry. Must be called with the entry locked.
186 186 * The write may be synchronous or asynchronous.
187 187 */
188 188 static int
189 189 write_l2e(adapter_t *sc, struct l2t_entry *e, int sync)
190 190 {
191 191 mblk_t *m;
192 192 struct cpl_l2t_write_req *req;
193 193
194 194 ASSERT(MUTEX_HELD(&e->lock));
195 195
196 196 if ((m = allocb(sizeof (*req), BPRI_HI)) == NULL)
197 197 return (ENOMEM);
198 198
199 199 /* LINTED: E_BAD_PTR_CAST_ALIGN */
200 200 req = (struct cpl_l2t_write_req *)m->b_wptr;
201 201
202 202 /* LINTED: E_CONSTANT_CONDITION */
203 203 INIT_TP_WR(req, 0);
204 204 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, e->idx |
205 205 V_SYNC_WR(sync) | V_TID_QID(sc->sge.fwq.abs_id)));
206 206 req->params = htons(V_L2T_W_PORT(e->lport) | V_L2T_W_NOREPLY(!sync));
207 207 req->l2t_idx = htons(e->idx);
208 208 req->vlan = htons(e->vlan);
209 209 (void) memcpy(req->dst_mac, e->dmac, sizeof (req->dst_mac));
210 210
211 211 m->b_wptr += sizeof (*req);
212 212
213 213 (void) t4_mgmt_tx(sc, m);
214 214
215 215 if (sync && e->state != L2T_STATE_SWITCHING)
216 216 e->state = L2T_STATE_SYNC_WRITE;
217 217
218 218 return (0);
219 219 }
220 220
221 221 struct l2t_data *
222 222 t4_init_l2t(struct adapter *sc)
223 223 {
224 224 int i;
225 225 struct l2t_data *d;
226 226
227 227 d = kmem_zalloc(sizeof (*d), KM_SLEEP);
228 228
229 229 d->rover = d->l2tab;
230 230 (void) atomic_swap_uint(&d->nfree, L2T_SIZE);
231 231 rw_init(&d->lock, NULL, RW_DRIVER, NULL);
232 232
233 233 for (i = 0; i < L2T_SIZE; i++) {
234 234 /* LINTED: E_ASSIGN_NARROW_CONV */
235 235 d->l2tab[i].idx = i;
236 236 d->l2tab[i].state = L2T_STATE_UNUSED;
237 237 mutex_init(&d->l2tab[i].lock, NULL, MUTEX_DRIVER, NULL);
238 238 (void) atomic_swap_uint(&d->l2tab[i].refcnt, 0);
239 239 }
240 240
241 241 (void) t4_register_cpl_handler(sc, CPL_L2T_WRITE_RPL, do_l2t_write_rpl);
242 242
243 243 return (d);
244 244 }
245 245
246 246 int
247 247 t4_free_l2t(struct l2t_data *d)
248 248 {
249 249 int i;
250 250
251 251 for (i = 0; i < L2T_SIZE; i++)
252 252 mutex_destroy(&d->l2tab[i].lock);
253 253 rw_destroy(&d->lock);
254 254 kmem_free(d, sizeof (*d));
255 255
256 256 return (0);
257 257 }
258 258
259 259 #ifndef TCP_OFFLOAD_DISABLE
260 260 static inline void
261 261 l2t_hold(struct l2t_data *d, struct l2t_entry *e)
262 262 {
263 263 if (atomic_inc_uint_nv(&e->refcnt) == 1) /* 0 -> 1 transition */
264 264 atomic_dec_uint(&d->nfree);
265 265 }
266 266
267 267 /*
268 268 * To avoid having to check address families we do not allow v4 and v6
269 269 * neighbors to be on the same hash chain. We keep v4 entries in the first
270 270 * half of available hash buckets and v6 in the second.
271 271 */
272 272 enum {
273 273 L2T_SZ_HALF = L2T_SIZE / 2,
274 274 L2T_HASH_MASK = L2T_SZ_HALF - 1
275 275 };
276 276
277 277 static inline unsigned int
278 278 arp_hash(const uint32_t *key, int ifindex)
279 279 {
280 280 return (jhash_2words(*key, ifindex, 0) & L2T_HASH_MASK);
281 281 }
282 282
283 283 static inline unsigned int
284 284 ipv6_hash(const uint32_t *key, int ifindex)
285 285 {
286 286 uint32_t xor = key[0] ^ key[1] ^ key[2] ^ key[3];
287 287
288 288 return (L2T_SZ_HALF + (jhash_2words(xor, ifindex, 0) & L2T_HASH_MASK));
289 289 }
290 290
291 291 static inline unsigned int
292 292 addr_hash(const uint32_t *addr, int addr_len, int ifindex)
293 293 {
294 294 return (addr_len == 4 ? arp_hash(addr, ifindex) :
295 295 ipv6_hash(addr, ifindex));
296 296 }
297 297
298 298 /*
299 299 * Checks if an L2T entry is for the given IP/IPv6 address. It does not check
300 300 * whether the L2T entry and the address are of the same address family.
301 301 * Callers ensure an address is only checked against L2T entries of the same
302 302 * family, something made trivial by the separation of IP and IPv6 hash chains
303 303 * mentioned above. Returns 0 if there's a match,
304 304 */
305 305 static inline int
306 306 addreq(const struct l2t_entry *e, const uint32_t *addr)
307 307 {
308 308 if (e->v6 != 0)
309 309 return ((e->addr[0] ^ addr[0]) | (e->addr[1] ^ addr[1]) |
310 310 (e->addr[2] ^ addr[2]) | (e->addr[3] ^ addr[3]));
311 311 return (e->addr[0] ^ addr[0]);
312 312 }
313 313
314 314 /*
315 315 * Add a packet to an L2T entry's queue of packets awaiting resolution.
316 316 * Must be called with the entry's lock held.
317 317 */
318 318 static inline void
319 319 arpq_enqueue(struct l2t_entry *e, mblk_t *m)
320 320 {
321 321 ASSERT(MUTEX_HELD(&e->lock));
322 322
323 323 ASSERT(m->b_next == NULL);
324 324 if (e->arpq_head != NULL)
325 325 e->arpq_tail->b_next = m;
326 326 else
327 327 e->arpq_head = m;
328 328 e->arpq_tail = m;
329 329 }
330 330
331 331 static inline void
332 332 send_pending(struct adapter *sc, struct l2t_entry *e)
333 333 {
334 334 mblk_t *m, *next;
335 335
336 336 ASSERT(MUTEX_HELD(&e->lock));
337 337
338 338 for (m = e->arpq_head; m; m = next) {
339 339 next = m->b_next;
340 340 m->b_next = NULL;
341 341 (void) t4_wrq_tx(sc, MBUF_EQ(m), m);
342 342 }
343 343 e->arpq_head = e->arpq_tail = NULL;
344 344 }
345 345
346 346 int
347 347 t4_l2t_send(struct adapter *sc, mblk_t *m, struct l2t_entry *e)
348 348 {
349 349 sin_t *sin;
350 350 ip2mac_t ip2m;
351 351
352 352 if (e->v6 != 0)
353 353 ASSERT(0);
354 354 again:
355 355 switch (e->state) {
356 356 case L2T_STATE_STALE: /* entry is stale, kick off revalidation */
357 357
358 358 /* Fall through */
359 359 case L2T_STATE_VALID: /* fast-path, send the packet on */
360 360 (void) t4_wrq_tx(sc, MBUF_EQ(m), m);
361 361 return (0);
362 362
363 363 case L2T_STATE_RESOLVING:
364 364 case L2T_STATE_SYNC_WRITE:
365 365 mutex_enter(&e->lock);
366 366 if (e->state != L2T_STATE_SYNC_WRITE &&
367 367 e->state != L2T_STATE_RESOLVING) {
368 368 /* state changed by the time we got here */
369 369 mutex_exit(&e->lock);
370 370 goto again;
371 371 }
372 372 arpq_enqueue(e, m);
373 373 mutex_exit(&e->lock);
374 374
375 375 bzero(&ip2m, sizeof (ip2m));
376 376 sin = (sin_t *)&ip2m.ip2mac_pa;
377 377 sin->sin_family = AF_INET;
378 378 sin->sin_addr.s_addr = e->in_addr;
379 379 ip2m.ip2mac_ifindex = e->ifindex;
380 380
381 381 if (e->state == L2T_STATE_RESOLVING) {
382 382 (void) ip2mac(IP2MAC_RESOLVE, &ip2m, t4_l2t_update, e,
383 383 0);
384 384 if (ip2m.ip2mac_err == EINPROGRESS)
385 385 ASSERT(0);
386 386 else if (ip2m.ip2mac_err == 0)
387 387 t4_l2t_update(&ip2m, e);
388 388 else
389 389 ASSERT(0);
390 390 }
391 391 }
392 392
393 393 return (0);
394 394 }
395 395
396 396 /*
397 397 * Called when an L2T entry has no more users. The entry is left in the hash
398 398 * table since it is likely to be reused but we also bump nfree to indicate
399 399 * that the entry can be reallocated for a different neighbor. We also drop
400 400 * the existing neighbor reference in case the neighbor is going away and is
401 401 * waiting on our reference.
402 402 *
403 403 * Because entries can be reallocated to other neighbors once their ref count
404 404 * drops to 0 we need to take the entry's lock to avoid races with a new
405 405 * incarnation.
406 406 */
407 407 static void
408 408 t4_l2e_free(struct l2t_entry *e)
409 409 {
410 410 struct l2t_data *d;
411 411
412 412 mutex_enter(&e->lock);
413 413 /* LINTED: E_NOP_IF_STMT */
414 414 if (atomic_read(&e->refcnt) == 0) { /* hasn't been recycled */
|
↓ open down ↓ |
414 lines elided |
↑ open up ↑ |
415 415 /*
416 416 * Don't need to worry about the arpq, an L2T entry can't be
417 417 * released if any packets are waiting for resolution as we
418 418 * need to be able to communicate with the device to close a
419 419 * connection.
420 420 */
421 421 }
422 422 mutex_exit(&e->lock);
423 423
424 424 d = container_of(e, struct l2t_data, l2tab[e->idx]);
425 - atomic_add_int(&d->nfree, 1);
425 + atomic_inc_uint(&d->nfree);
426 426
427 427 }
428 428
429 429 void
430 430 t4_l2t_release(struct l2t_entry *e)
431 431 {
432 432 if (atomic_dec_uint_nv(&e->refcnt) == 0)
433 433 t4_l2e_free(e);
434 434 }
435 435
436 436 /* ARGSUSED */
437 437 int
438 438 do_l2t_write_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
439 439 {
440 440 struct adapter *sc = iq->adapter;
441 441 const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
442 442 unsigned int tid = GET_TID(rpl);
443 443 unsigned int idx = tid & (L2T_SIZE - 1);
444 444
445 445 if (likely(rpl->status != CPL_ERR_NONE)) {
446 446 cxgb_printf(sc->dip, CE_WARN,
447 447 "Unexpected L2T_WRITE_RPL status %u for entry %u",
448 448 rpl->status, idx);
449 449 return (-EINVAL);
450 450 }
451 451
452 452 if (tid & F_SYNC_WR) {
453 453 struct l2t_entry *e = &sc->l2t->l2tab[idx];
454 454
455 455 mutex_enter(&e->lock);
456 456 if (e->state != L2T_STATE_SWITCHING) {
457 457 send_pending(sc, e);
458 458 e->state = L2T_STATE_VALID;
459 459 }
460 460 mutex_exit(&e->lock);
461 461 }
462 462
463 463 return (0);
464 464 }
465 465
466 466 /*
467 467 * The TOE wants an L2 table entry that it can use to reach the next hop over
468 468 * the specified port. Produce such an entry - create one if needed.
469 469 *
470 470 * Note that the ifnet could be a pseudo-device like if_vlan, if_lagg, etc. on
471 471 * top of the real cxgbe interface.
472 472 */
473 473 struct l2t_entry *
474 474 t4_l2t_get(struct port_info *pi, conn_t *connp)
475 475 {
476 476 struct l2t_entry *e;
477 477 struct l2t_data *d = pi->adapter->l2t;
478 478 int addr_len;
479 479 uint32_t *addr;
480 480 int hash;
481 481 int index = \
482 482 connp->conn_ixa->ixa_ire->ire_ill->ill_phyint->phyint_ifindex;
483 483 unsigned int smt_idx = pi->port_id;
484 484 addr = (uint32_t *)&connp->conn_faddr_v4;
485 485 addr_len = sizeof (connp->conn_faddr_v4);
486 486
487 487 hash = addr_hash(addr, addr_len, index);
488 488
489 489 rw_enter(&d->lock, RW_WRITER);
490 490 for (e = d->l2tab[hash].first; e; e = e->next) {
491 491 if (!addreq(e, addr) && e->smt_idx == smt_idx) {
492 492 l2t_hold(d, e);
493 493 goto done;
494 494 }
495 495 }
496 496
497 497 /* Need to allocate a new entry */
498 498 e = alloc_l2e(d);
499 499 if (e != NULL) {
500 500 mutex_enter(&e->lock); /* avoid race with t4_l2t_free */
501 501 e->state = L2T_STATE_RESOLVING;
502 502 (void) memcpy(e->addr, addr, addr_len);
503 503 e->in_addr = connp->conn_faddr_v4;
504 504 e->ifindex = index;
505 505 /* LINTED: E_ASSIGN_NARROW_CONV */
506 506 e->smt_idx = smt_idx;
507 507 /* LINTED: E_ASSIGN_NARROW_CONV */
508 508 e->hash = hash;
509 509 e->lport = pi->lport;
510 510 e->arpq_head = e->arpq_tail = NULL;
511 511 e->v6 = (addr_len == 16);
512 512 e->sc = pi->adapter;
513 513 (void) atomic_swap_uint(&e->refcnt, 1);
514 514 e->vlan = VLAN_NONE;
515 515 e->next = d->l2tab[hash].first;
516 516 d->l2tab[hash].first = e;
517 517 mutex_exit(&e->lock);
518 518 } else {
519 519 ASSERT(0);
520 520 }
521 521
522 522 done:
523 523 rw_exit(&d->lock);
524 524 return (e);
525 525 }
526 526
527 527 /*
528 528 * Called when the host's neighbor layer makes a change to some entry that is
529 529 * loaded into the HW L2 table.
530 530 */
531 531 void
532 532 t4_l2t_update(ip2mac_t *ip2macp, void *arg)
533 533 {
534 534 struct l2t_entry *e = (struct l2t_entry *)arg;
535 535 struct adapter *sc = e->sc;
536 536 uchar_t *cp;
537 537
538 538 if (ip2macp->ip2mac_err != 0) {
539 539 ASSERT(0); /* Don't know what to do. Needs to be investigated */
540 540 }
541 541
542 542 mutex_enter(&e->lock);
543 543 if (atomic_read(&e->refcnt) != 0)
544 544 goto found;
545 545 e->state = L2T_STATE_STALE;
546 546 mutex_exit(&e->lock);
547 547
548 548 /* The TOE has no interest in this LLE */
549 549 return;
550 550
551 551 found:
552 552 if (atomic_read(&e->refcnt) != 0) {
553 553
554 554 /* Entry is referenced by at least 1 offloaded connection. */
555 555
556 556 cp = (uchar_t *)LLADDR(&ip2macp->ip2mac_ha);
557 557 bcopy(cp, e->dmac, 6);
558 558 (void) write_l2e(sc, e, 1);
559 559 e->state = L2T_STATE_VALID;
560 560
561 561 }
562 562 mutex_exit(&e->lock);
563 563 }
564 564 #endif
|
↓ open down ↓ |
129 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX