Print this page
4804 apix & pcplusmp are nearly warning free already
Tentatively Reviewed by: Robert Mustacchi <rm@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/io/mp_platform_misc.c
+++ new/usr/src/uts/i86pc/io/mp_platform_misc.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23 23 */
24 24 /*
25 25 * Copyright (c) 2010, Intel Corporation.
26 26 * All rights reserved.
27 27 */
28 28
29 29 /*
30 30 * PSMI 1.1 extensions are supported only in 2.6 and later versions.
31 31 * PSMI 1.2 extensions are supported only in 2.7 and later versions.
32 32 * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
33 33 * PSMI 1.5 extensions are supported in Solaris Nevada.
34 34 * PSMI 1.6 extensions are supported in Solaris Nevada.
35 35 * PSMI 1.7 extensions are supported in Solaris Nevada.
36 36 */
37 37 #define PSMI_1_7
38 38
39 39 #include <sys/processor.h>
40 40 #include <sys/time.h>
41 41 #include <sys/psm.h>
42 42 #include <sys/smp_impldefs.h>
43 43 #include <sys/inttypes.h>
44 44 #include <sys/cram.h>
45 45 #include <sys/acpi/acpi.h>
46 46 #include <sys/acpica.h>
47 47 #include <sys/psm_common.h>
48 48 #include <sys/apic.h>
49 49 #include <sys/apic_common.h>
50 50 #include <sys/pit.h>
51 51 #include <sys/ddi.h>
52 52 #include <sys/sunddi.h>
53 53 #include <sys/ddi_impldefs.h>
54 54 #include <sys/pci.h>
55 55 #include <sys/promif.h>
56 56 #include <sys/x86_archext.h>
57 57 #include <sys/cpc_impl.h>
58 58 #include <sys/uadmin.h>
59 59 #include <sys/panic.h>
60 60 #include <sys/debug.h>
61 61 #include <sys/archsystm.h>
62 62 #include <sys/trap.h>
63 63 #include <sys/machsystm.h>
64 64 #include <sys/cpuvar.h>
65 65 #include <sys/rm_platter.h>
66 66 #include <sys/privregs.h>
67 67 #include <sys/cyclic.h>
68 68 #include <sys/note.h>
69 69 #include <sys/pci_intr_lib.h>
70 70 #include <sys/sunndi.h>
71 71 #include <sys/hpet.h>
72 72 #include <sys/clock.h>
73 73
74 74 /*
75 75 * Part of mp_platfrom_common.c that's used only by pcplusmp & xpv_psm
76 76 * but not apix.
77 77 * These functions may be moved to xpv_psm later when apix and pcplusmp
78 78 * are merged together
79 79 */
80 80
81 81 /*
82 82 * Local Function Prototypes
83 83 */
84 84 static void apic_mark_vector(uchar_t oldvector, uchar_t newvector);
85 85 static void apic_xlate_vector_free_timeout_handler(void *arg);
86 86 static int apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu,
87 87 int new_bind_cpu, int apicindex, int intin_no, int which_irq,
88 88 struct ioapic_reprogram_data *drep);
89 89 static int apic_setup_irq_table(dev_info_t *dip, int irqno,
90 90 struct apic_io_intr *intrp, struct intrspec *ispec, iflag_t *intr_flagp,
91 91 int type);
92 92 static void apic_try_deferred_reprogram(int ipl, int vect);
93 93 static void delete_defer_repro_ent(int which_irq);
94 94 static void apic_ioapic_wait_pending_clear(int ioapicindex,
95 95 int intin_no);
96 96
97 97 extern int apic_acpi_translate_pci_irq(dev_info_t *dip, int busid, int devid,
98 98 int ipin, int *pci_irqp, iflag_t *intr_flagp);
99 99 extern int apic_handle_pci_pci_bridge(dev_info_t *idip, int child_devno,
100 100 int child_ipin, struct apic_io_intr **intrp);
101 101 extern uchar_t acpi_find_ioapic(int irq);
102 102 extern struct apic_io_intr *apic_find_io_intr_w_busid(int irqno, int busid);
103 103 extern int apic_find_bus_id(int bustype);
104 104 extern int apic_find_intin(uchar_t ioapic, uchar_t intin);
105 105 extern void apic_record_rdt_entry(apic_irq_t *irqptr, int irq);
106 106
107 107 extern int apic_sci_vect;
108 108 extern iflag_t apic_sci_flags;
109 109 /* ACPI HPET interrupt configuration; -1 if HPET not used */
110 110 extern int apic_hpet_vect;
111 111 extern iflag_t apic_hpet_flags;
112 112 extern int apic_intr_policy;
113 113 extern char *psm_name;
114 114
115 115 /*
116 116 * number of bits per byte, from <sys/param.h>
117 117 */
118 118 #define UCHAR_MAX UINT8_MAX
119 119
120 120 /* Max wait time (in repetitions) for flags to clear in an RDT entry. */
121 121 extern int apic_max_reps_clear_pending;
122 122
123 123 /* The irq # is implicit in the array index: */
124 124 struct ioapic_reprogram_data apic_reprogram_info[APIC_MAX_VECTOR+1];
125 125 /*
126 126 * APIC_MAX_VECTOR + 1 is the maximum # of IRQs as well. ioapic_reprogram_info
127 127 * is indexed by IRQ number, NOT by vector number.
128 128 */
129 129
130 130 extern int apic_int_busy_mark;
131 131 extern int apic_int_free_mark;
132 132 extern int apic_diff_for_redistribution;
133 133 extern int apic_sample_factor_redistribution;
134 134 extern int apic_redist_cpu_skip;
135 135 extern int apic_num_imbalance;
136 136 extern int apic_num_rebind;
137 137
138 138 /* timeout for xlate_vector, mark_vector */
139 139 int apic_revector_timeout = 16 * 10000; /* 160 millisec */
140 140
141 141 extern int apic_defconf;
142 142 extern int apic_irq_translate;
143 143
144 144 extern int apic_use_acpi_madt_only; /* 1=ONLY use MADT from ACPI */
145 145
146 146 extern uchar_t apic_io_vectbase[MAX_IO_APIC];
147 147
148 148 extern boolean_t ioapic_mask_workaround[MAX_IO_APIC];
149 149
150 150 /*
151 151 * First available slot to be used as IRQ index into the apic_irq_table
152 152 * for those interrupts (like MSI/X) that don't have a physical IRQ.
153 153 */
154 154 extern int apic_first_avail_irq;
155 155
156 156 /*
157 157 * apic_defer_reprogram_lock ensures that only one processor is handling
158 158 * deferred interrupt programming at *_intr_exit time.
159 159 */
160 160 static lock_t apic_defer_reprogram_lock;
161 161
162 162 /*
163 163 * The current number of deferred reprogrammings outstanding
164 164 */
165 165 uint_t apic_reprogram_outstanding = 0;
166 166
167 167 #ifdef DEBUG
168 168 /*
169 169 * Counters that keep track of deferred reprogramming stats
170 170 */
171 171 uint_t apic_intr_deferrals = 0;
172 172 uint_t apic_intr_deliver_timeouts = 0;
173 173 uint_t apic_last_ditch_reprogram_failures = 0;
174 174 uint_t apic_deferred_setup_failures = 0;
175 175 uint_t apic_defer_repro_total_retries = 0;
176 176 uint_t apic_defer_repro_successes = 0;
177 177 uint_t apic_deferred_spurious_enters = 0;
178 178 #endif
179 179
180 180 extern int apic_io_max;
181 181 extern struct apic_io_intr *apic_io_intrp;
182 182
183 183 uchar_t apic_vector_to_irq[APIC_MAX_VECTOR+1];
184 184
185 185 extern uint32_t eisa_level_intr_mask;
186 186 /* At least MSB will be set if EISA bus */
187 187
188 188 extern int apic_pci_bus_total;
189 189 extern uchar_t apic_single_pci_busid;
190 190
191 191 /*
192 192 * Following declarations are for revectoring; used when ISRs at different
193 193 * IPLs share an irq.
194 194 */
195 195 static lock_t apic_revector_lock;
196 196 int apic_revector_pending = 0;
197 197 static uchar_t *apic_oldvec_to_newvec;
198 198 static uchar_t *apic_newvec_to_oldvec;
199 199
200 200 /* ACPI Interrupt Source Override Structure ptr */
201 201 extern ACPI_MADT_INTERRUPT_OVERRIDE *acpi_isop;
202 202 extern int acpi_iso_cnt;
203 203
204 204 /*
205 205 * Auto-configuration routines
206 206 */
207 207
208 208 /*
209 209 * Initialise vector->ipl and ipl->pri arrays. level_intr and irqtable
210 210 * are also set to NULL. vector->irq is set to a value which cannot map
211 211 * to a real irq to show that it is free.
212 212 */
213 213 void
214 214 apic_init_common(void)
215 215 {
216 216 int i, j, indx;
217 217 int *iptr;
218 218
219 219 /*
220 220 * Initialize apic_ipls from apic_vectortoipl. This array is
221 221 * used in apic_intr_enter to determine the IPL to use for the
222 222 * corresponding vector. On some systems, due to hardware errata
223 223 * and interrupt sharing, the IPL may not correspond to the IPL listed
224 224 * in apic_vectortoipl (see apic_addspl and apic_delspl).
225 225 */
226 226 for (i = 0; i < (APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL); i++) {
227 227 indx = i * APIC_VECTOR_PER_IPL;
228 228
229 229 for (j = 0; j < APIC_VECTOR_PER_IPL; j++, indx++)
230 230 apic_ipls[indx] = apic_vectortoipl[i];
231 231 }
232 232
233 233 /* cpu 0 is always up (for now) */
234 234 apic_cpus[0].aci_status = APIC_CPU_ONLINE | APIC_CPU_INTR_ENABLE;
235 235
236 236 iptr = (int *)&apic_irq_table[0];
237 237 for (i = 0; i <= APIC_MAX_VECTOR; i++) {
238 238 apic_level_intr[i] = 0;
239 239 *iptr++ = NULL;
240 240 apic_vector_to_irq[i] = APIC_RESV_IRQ;
241 241
242 242 /* These *must* be initted to B_TRUE! */
243 243 apic_reprogram_info[i].done = B_TRUE;
244 244 apic_reprogram_info[i].irqp = NULL;
245 245 apic_reprogram_info[i].tries = 0;
246 246 apic_reprogram_info[i].bindcpu = 0;
247 247 }
248 248
249 249 /*
250 250 * Allocate a dummy irq table entry for the reserved entry.
251 251 * This takes care of the race between removing an irq and
252 252 * clock detecting a CPU in that irq during interrupt load
253 253 * sampling.
254 254 */
255 255 apic_irq_table[APIC_RESV_IRQ] =
256 256 kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
257 257
258 258 mutex_init(&airq_mutex, NULL, MUTEX_DEFAULT, NULL);
259 259 }
260 260
261 261 void
262 262 ioapic_init_intr(int mask_apic)
263 263 {
264 264 int ioapic_ix;
265 265 struct intrspec ispec;
266 266 apic_irq_t *irqptr;
267 267 int i, j;
268 268 ulong_t iflag;
269 269
270 270 LOCK_INIT_CLEAR(&apic_revector_lock);
271 271 LOCK_INIT_CLEAR(&apic_defer_reprogram_lock);
272 272
273 273 /* mask interrupt vectors */
274 274 for (j = 0; j < apic_io_max && mask_apic; j++) {
275 275 int intin_max;
276 276
277 277 ioapic_ix = j;
278 278 /* Bits 23-16 define the maximum redirection entries */
279 279 intin_max = (ioapic_read(ioapic_ix, APIC_VERS_CMD) >> 16)
280 280 & 0xff;
281 281 for (i = 0; i <= intin_max; i++)
282 282 ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * i, AV_MASK);
283 283 }
284 284
285 285 /*
286 286 * Hack alert: deal with ACPI SCI interrupt chicken/egg here
287 287 */
288 288 if (apic_sci_vect > 0) {
289 289 /*
290 290 * acpica has already done add_avintr(); we just
291 291 * to finish the job by mimicing translate_irq()
292 292 *
293 293 * Fake up an intrspec and setup the tables
294 294 */
295 295 ispec.intrspec_vec = apic_sci_vect;
296 296 ispec.intrspec_pri = SCI_IPL;
297 297
298 298 if (apic_setup_irq_table(NULL, apic_sci_vect, NULL,
299 299 &ispec, &apic_sci_flags, DDI_INTR_TYPE_FIXED) < 0) {
300 300 cmn_err(CE_WARN, "!apic: SCI setup failed");
301 301 return;
302 302 }
303 303 irqptr = apic_irq_table[apic_sci_vect];
304 304
305 305 iflag = intr_clear();
306 306 lock_set(&apic_ioapic_lock);
307 307
308 308 /* Program I/O APIC */
309 309 (void) apic_setup_io_intr(irqptr, apic_sci_vect, B_FALSE);
310 310
311 311 lock_clear(&apic_ioapic_lock);
312 312 intr_restore(iflag);
313 313
314 314 irqptr->airq_share++;
315 315 }
316 316
317 317 /*
318 318 * Hack alert: deal with ACPI HPET interrupt chicken/egg here.
319 319 */
320 320 if (apic_hpet_vect > 0) {
321 321 /*
322 322 * hpet has already done add_avintr(); we just need
323 323 * to finish the job by mimicing translate_irq()
324 324 *
325 325 * Fake up an intrspec and setup the tables
326 326 */
327 327 ispec.intrspec_vec = apic_hpet_vect;
328 328 ispec.intrspec_pri = CBE_HIGH_PIL;
329 329
330 330 if (apic_setup_irq_table(NULL, apic_hpet_vect, NULL,
331 331 &ispec, &apic_hpet_flags, DDI_INTR_TYPE_FIXED) < 0) {
332 332 cmn_err(CE_WARN, "!apic: HPET setup failed");
333 333 return;
334 334 }
335 335 irqptr = apic_irq_table[apic_hpet_vect];
336 336
337 337 iflag = intr_clear();
338 338 lock_set(&apic_ioapic_lock);
339 339
340 340 /* Program I/O APIC */
341 341 (void) apic_setup_io_intr(irqptr, apic_hpet_vect, B_FALSE);
342 342
343 343 lock_clear(&apic_ioapic_lock);
344 344 intr_restore(iflag);
345 345
346 346 irqptr->airq_share++;
347 347 }
348 348 }
349 349
350 350 /*
351 351 * Add mask bits to disable interrupt vector from happening
352 352 * at or above IPL. In addition, it should remove mask bits
353 353 * to enable interrupt vectors below the given IPL.
354 354 *
355 355 * Both add and delspl are complicated by the fact that different interrupts
356 356 * may share IRQs. This can happen in two ways.
357 357 * 1. The same H/W line is shared by more than 1 device
358 358 * 1a. with interrupts at different IPLs
359 359 * 1b. with interrupts at same IPL
360 360 * 2. We ran out of vectors at a given IPL and started sharing vectors.
361 361 * 1b and 2 should be handled gracefully, except for the fact some ISRs
362 362 * will get called often when no interrupt is pending for the device.
363 363 * For 1a, we handle it at the higher IPL.
364 364 */
365 365 /*ARGSUSED*/
366 366 int
367 367 apic_addspl_common(int irqno, int ipl, int min_ipl, int max_ipl)
368 368 {
369 369 uchar_t vector;
370 370 ulong_t iflag;
371 371 apic_irq_t *irqptr, *irqheadptr;
372 372 int irqindex;
373 373
374 374 ASSERT(max_ipl <= UCHAR_MAX);
375 375 irqindex = IRQINDEX(irqno);
376 376
377 377 if ((irqindex == -1) || (!apic_irq_table[irqindex]))
378 378 return (PSM_FAILURE);
379 379
380 380 mutex_enter(&airq_mutex);
381 381 irqptr = irqheadptr = apic_irq_table[irqindex];
382 382
383 383 DDI_INTR_IMPLDBG((CE_CONT, "apic_addspl: dip=0x%p type=%d irqno=0x%x "
384 384 "vector=0x%x\n", (void *)irqptr->airq_dip,
385 385 irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector));
386 386
387 387 while (irqptr) {
388 388 if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno)
389 389 break;
390 390 irqptr = irqptr->airq_next;
391 391 }
392 392 irqptr->airq_share++;
393 393
394 394 mutex_exit(&airq_mutex);
395 395
396 396 /* return if it is not hardware interrupt */
397 397 if (irqptr->airq_mps_intr_index == RESERVE_INDEX)
398 398 return (PSM_SUCCESS);
399 399
400 400 /* Or if there are more interupts at a higher IPL */
401 401 if (ipl != max_ipl)
402 402 return (PSM_SUCCESS);
403 403
404 404 /*
405 405 * if apic_picinit() has not been called yet, just return.
406 406 * At the end of apic_picinit(), we will call setup_io_intr().
407 407 */
408 408
409 409 if (!apic_picinit_called)
410 410 return (PSM_SUCCESS);
411 411
412 412 /*
413 413 * Upgrade vector if max_ipl is not earlier ipl. If we cannot allocate,
414 414 * return failure.
415 415 */
416 416 if (irqptr->airq_ipl != max_ipl &&
417 417 !ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
418 418
419 419 vector = apic_allocate_vector(max_ipl, irqindex, 1);
420 420 if (vector == 0) {
421 421 irqptr->airq_share--;
422 422 return (PSM_FAILURE);
423 423 }
424 424 irqptr = irqheadptr;
425 425 apic_mark_vector(irqptr->airq_vector, vector);
426 426 while (irqptr) {
427 427 irqptr->airq_vector = vector;
428 428 irqptr->airq_ipl = (uchar_t)max_ipl;
429 429 /*
430 430 * reprogram irq being added and every one else
431 431 * who is not in the UNINIT state
432 432 */
433 433 if ((VIRTIRQ(irqindex, irqptr->airq_share_id) ==
434 434 irqno) || (irqptr->airq_temp_cpu != IRQ_UNINIT)) {
435 435 apic_record_rdt_entry(irqptr, irqindex);
436 436
437 437 iflag = intr_clear();
438 438 lock_set(&apic_ioapic_lock);
439 439
440 440 (void) apic_setup_io_intr(irqptr, irqindex,
441 441 B_FALSE);
442 442
443 443 lock_clear(&apic_ioapic_lock);
444 444 intr_restore(iflag);
445 445 }
446 446 irqptr = irqptr->airq_next;
447 447 }
448 448 return (PSM_SUCCESS);
449 449
450 450 } else if (irqptr->airq_ipl != max_ipl &&
451 451 ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
452 452 /*
453 453 * We cannot upgrade the vector, but we can change
454 454 * the IPL that this vector induces.
455 455 *
456 456 * Note that we subtract APIC_BASE_VECT from the vector
457 457 * here because this array is used in apic_intr_enter
458 458 * (no need to add APIC_BASE_VECT in that hot code
459 459 * path since we can do it in the rarely-executed path
460 460 * here).
461 461 */
462 462 apic_ipls[irqptr->airq_vector - APIC_BASE_VECT] =
463 463 (uchar_t)max_ipl;
464 464
465 465 irqptr = irqheadptr;
466 466 while (irqptr) {
467 467 irqptr->airq_ipl = (uchar_t)max_ipl;
468 468 irqptr = irqptr->airq_next;
469 469 }
470 470
471 471 return (PSM_SUCCESS);
472 472 }
473 473
474 474 ASSERT(irqptr);
475 475
476 476 iflag = intr_clear();
477 477 lock_set(&apic_ioapic_lock);
478 478
479 479 (void) apic_setup_io_intr(irqptr, irqindex, B_FALSE);
480 480
481 481 lock_clear(&apic_ioapic_lock);
482 482 intr_restore(iflag);
483 483
484 484 return (PSM_SUCCESS);
485 485 }
486 486
487 487 /*
488 488 * Recompute mask bits for the given interrupt vector.
489 489 * If there is no interrupt servicing routine for this
490 490 * vector, this function should disable interrupt vector
491 491 * from happening at all IPLs. If there are still
492 492 * handlers using the given vector, this function should
493 493 * disable the given vector from happening below the lowest
494 494 * IPL of the remaining hadlers.
495 495 */
496 496 /*ARGSUSED*/
497 497 int
498 498 apic_delspl_common(int irqno, int ipl, int min_ipl, int max_ipl)
499 499 {
500 500 uchar_t vector;
501 501 uint32_t bind_cpu;
502 502 int intin, irqindex;
503 503 int ioapic_ix;
504 504 apic_irq_t *irqptr, *preirqptr, *irqheadptr, *irqp;
505 505 ulong_t iflag;
506 506
507 507 mutex_enter(&airq_mutex);
508 508 irqindex = IRQINDEX(irqno);
509 509 irqptr = preirqptr = irqheadptr = apic_irq_table[irqindex];
510 510
511 511 DDI_INTR_IMPLDBG((CE_CONT, "apic_delspl: dip=0x%p type=%d irqno=0x%x "
512 512 "vector=0x%x\n", (void *)irqptr->airq_dip,
513 513 irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector));
514 514
515 515 while (irqptr) {
516 516 if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno)
517 517 break;
518 518 preirqptr = irqptr;
519 519 irqptr = irqptr->airq_next;
520 520 }
521 521 ASSERT(irqptr);
522 522
523 523 irqptr->airq_share--;
524 524
525 525 mutex_exit(&airq_mutex);
526 526
527 527 /*
528 528 * If there are more interrupts at a higher IPL, we don't need
529 529 * to disable anything.
530 530 */
531 531 if (ipl < max_ipl)
532 532 return (PSM_SUCCESS);
533 533
534 534 /* return if it is not hardware interrupt */
535 535 if (irqptr->airq_mps_intr_index == RESERVE_INDEX)
536 536 return (PSM_SUCCESS);
537 537
538 538 if (!apic_picinit_called) {
539 539 /*
540 540 * Clear irq_struct. If two devices shared an intpt
541 541 * line & 1 unloaded before picinit, we are hosed. But, then
542 542 * we hope the machine survive.
543 543 */
544 544 irqptr->airq_mps_intr_index = FREE_INDEX;
545 545 irqptr->airq_temp_cpu = IRQ_UNINIT;
|
↓ open down ↓ |
545 lines elided |
↑ open up ↑ |
546 546 apic_free_vector(irqptr->airq_vector);
547 547 return (PSM_SUCCESS);
548 548 }
549 549 /*
550 550 * Downgrade vector to new max_ipl if needed. If we cannot allocate,
551 551 * use old IPL. Not very elegant, but it should work.
552 552 */
553 553 if ((irqptr->airq_ipl != max_ipl) && (max_ipl != PSM_INVALID_IPL) &&
554 554 !ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
555 555 apic_irq_t *irqp;
556 - if (vector = apic_allocate_vector(max_ipl, irqno, 1)) {
556 + if ((vector = apic_allocate_vector(max_ipl, irqno, 1))) {
557 557 apic_mark_vector(irqheadptr->airq_vector, vector);
558 558 irqp = irqheadptr;
559 559 while (irqp) {
560 560 irqp->airq_vector = vector;
561 561 irqp->airq_ipl = (uchar_t)max_ipl;
562 562 if (irqp->airq_temp_cpu != IRQ_UNINIT) {
563 563 apic_record_rdt_entry(irqp, irqindex);
564 564
565 565 iflag = intr_clear();
566 566 lock_set(&apic_ioapic_lock);
567 567
568 568 (void) apic_setup_io_intr(irqp,
569 569 irqindex, B_FALSE);
570 570
571 571 lock_clear(&apic_ioapic_lock);
572 572 intr_restore(iflag);
573 573 }
574 574 irqp = irqp->airq_next;
575 575 }
576 576 }
577 577
578 578 } else if (irqptr->airq_ipl != max_ipl &&
579 579 max_ipl != PSM_INVALID_IPL &&
580 580 ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
581 581
582 582 /*
583 583 * We cannot downgrade the IPL of the vector below the vector's
584 584 * hardware priority. If we did, it would be possible for a
585 585 * higher-priority hardware vector to interrupt a CPU running at an IPL
586 586 * lower than the hardware priority of the interrupting vector (but
587 587 * higher than the soft IPL of this IRQ). When this happens, we would
588 588 * then try to drop the IPL BELOW what it was (effectively dropping
589 589 * below base_spl) which would be potentially catastrophic.
590 590 *
591 591 * (e.g. Suppose the hardware vector associated with this IRQ is 0x40
592 592 * (hardware IPL of 4). Further assume that the old IPL of this IRQ
593 593 * was 4, but the new IPL is 1. If we forced vector 0x40 to result in
594 594 * an IPL of 1, it would be possible for the processor to be executing
595 595 * at IPL 3 and for an interrupt to come in on vector 0x40, interrupting
596 596 * the currently-executing ISR. When apic_intr_enter consults
597 597 * apic_irqs[], it will return 1, bringing the IPL of the CPU down to 1
598 598 * so even though the processor was running at IPL 4, an IPL 1
599 599 * interrupt will have interrupted it, which must not happen)).
600 600 *
601 601 * Effectively, this means that the hardware priority corresponding to
602 602 * the IRQ's IPL (in apic_ipls[]) cannot be lower than the vector's
603 603 * hardware priority.
604 604 *
605 605 * (In the above example, then, after removal of the IPL 4 device's
606 606 * interrupt handler, the new IPL will continue to be 4 because the
607 607 * hardware priority that IPL 1 implies is lower than the hardware
608 608 * priority of the vector used.)
609 609 */
610 610 /* apic_ipls is indexed by vector, starting at APIC_BASE_VECT */
611 611 const int apic_ipls_index = irqptr->airq_vector -
612 612 APIC_BASE_VECT;
613 613 const int vect_inherent_hwpri = irqptr->airq_vector >>
614 614 APIC_IPL_SHIFT;
615 615
616 616 /*
617 617 * If there are still devices using this IRQ, determine the
618 618 * new ipl to use.
619 619 */
620 620 if (irqptr->airq_share) {
621 621 int vect_desired_hwpri, hwpri;
622 622
623 623 ASSERT(max_ipl < MAXIPL);
624 624 vect_desired_hwpri = apic_ipltopri[max_ipl] >>
625 625 APIC_IPL_SHIFT;
626 626
627 627 /*
628 628 * If the desired IPL's hardware priority is lower
629 629 * than that of the vector, use the hardware priority
630 630 * of the vector to determine the new IPL.
631 631 */
632 632 hwpri = (vect_desired_hwpri < vect_inherent_hwpri) ?
633 633 vect_inherent_hwpri : vect_desired_hwpri;
634 634
635 635 /*
636 636 * Now, to get the right index for apic_vectortoipl,
637 637 * we need to subtract APIC_BASE_VECT from the
638 638 * hardware-vector-equivalent (in hwpri). Since hwpri
639 639 * is already shifted, we shift APIC_BASE_VECT before
640 640 * doing the subtraction.
641 641 */
642 642 hwpri -= (APIC_BASE_VECT >> APIC_IPL_SHIFT);
643 643
644 644 ASSERT(hwpri >= 0);
645 645 ASSERT(hwpri < MAXIPL);
646 646 max_ipl = apic_vectortoipl[hwpri];
647 647 apic_ipls[apic_ipls_index] = max_ipl;
648 648
649 649 irqp = irqheadptr;
650 650 while (irqp) {
651 651 irqp->airq_ipl = (uchar_t)max_ipl;
652 652 irqp = irqp->airq_next;
653 653 }
654 654 } else {
655 655 /*
656 656 * No more devices on this IRQ, so reset this vector's
657 657 * element in apic_ipls to the original IPL for this
658 658 * vector
659 659 */
660 660 apic_ipls[apic_ipls_index] =
661 661 apic_vectortoipl[vect_inherent_hwpri];
662 662 }
663 663 }
664 664
665 665 /*
666 666 * If there are still active interrupts, we are done.
667 667 */
668 668 if (irqptr->airq_share)
669 669 return (PSM_SUCCESS);
670 670
671 671 iflag = intr_clear();
672 672 lock_set(&apic_ioapic_lock);
673 673
674 674 if (irqptr->airq_mps_intr_index == MSI_INDEX) {
675 675 /*
676 676 * Disable the MSI vector
677 677 * Make sure we only disable on the last
678 678 * of the multi-MSI support
679 679 */
680 680 if (i_ddi_intr_get_current_nenables(irqptr->airq_dip) == 1) {
681 681 apic_pci_msi_disable_mode(irqptr->airq_dip,
682 682 DDI_INTR_TYPE_MSI);
683 683 }
684 684 } else if (irqptr->airq_mps_intr_index == MSIX_INDEX) {
685 685 /*
686 686 * Disable the MSI-X vector
687 687 * needs to clear its mask and addr/data for each MSI-X
688 688 */
689 689 apic_pci_msi_unconfigure(irqptr->airq_dip, DDI_INTR_TYPE_MSIX,
690 690 irqptr->airq_origirq);
691 691 /*
692 692 * Make sure we only disable on the last MSI-X
693 693 */
694 694 if (i_ddi_intr_get_current_nenables(irqptr->airq_dip) == 1) {
695 695 apic_pci_msi_disable_mode(irqptr->airq_dip,
696 696 DDI_INTR_TYPE_MSIX);
697 697 }
698 698 } else {
699 699 /*
700 700 * The assumption here is that this is safe, even for
701 701 * systems with IOAPICs that suffer from the hardware
702 702 * erratum because all devices have been quiesced before
703 703 * they unregister their interrupt handlers. If that
704 704 * assumption turns out to be false, this mask operation
705 705 * can induce the same erratum result we're trying to
706 706 * avoid.
707 707 */
708 708 ioapic_ix = irqptr->airq_ioapicindex;
709 709 intin = irqptr->airq_intin_no;
710 710 ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * intin, AV_MASK);
711 711 }
712 712
713 713 apic_vt_ops->apic_intrmap_free_entry(&irqptr->airq_intrmap_private);
714 714
715 715 /*
716 716 * This irq entry is the only one in the chain.
717 717 */
718 718 if (irqheadptr->airq_next == NULL) {
719 719 ASSERT(irqheadptr == irqptr);
720 720 bind_cpu = irqptr->airq_temp_cpu;
721 721 if (((uint32_t)bind_cpu != IRQ_UNBOUND) &&
722 722 ((uint32_t)bind_cpu != IRQ_UNINIT)) {
723 723 ASSERT(apic_cpu_in_range(bind_cpu));
724 724 if (bind_cpu & IRQ_USER_BOUND) {
725 725 /* If hardbound, temp_cpu == cpu */
726 726 bind_cpu &= ~IRQ_USER_BOUND;
727 727 apic_cpus[bind_cpu].aci_bound--;
728 728 } else
729 729 apic_cpus[bind_cpu].aci_temp_bound--;
730 730 }
731 731 irqptr->airq_temp_cpu = IRQ_UNINIT;
732 732 irqptr->airq_mps_intr_index = FREE_INDEX;
733 733 lock_clear(&apic_ioapic_lock);
734 734 intr_restore(iflag);
735 735 apic_free_vector(irqptr->airq_vector);
736 736 return (PSM_SUCCESS);
737 737 }
738 738
739 739 /*
740 740 * If we get here, we are sharing the vector and there are more than
741 741 * one active irq entries in the chain.
742 742 */
743 743 lock_clear(&apic_ioapic_lock);
744 744 intr_restore(iflag);
745 745
746 746 mutex_enter(&airq_mutex);
747 747 /* Remove the irq entry from the chain */
748 748 if (irqptr == irqheadptr) { /* The irq entry is at the head */
749 749 apic_irq_table[irqindex] = irqptr->airq_next;
750 750 } else {
751 751 preirqptr->airq_next = irqptr->airq_next;
752 752 }
753 753 /* Free the irq entry */
754 754 kmem_free(irqptr, sizeof (apic_irq_t));
755 755 mutex_exit(&airq_mutex);
756 756
757 757 return (PSM_SUCCESS);
758 758 }
759 759
760 760 /*
761 761 * apic_introp_xlate() replaces apic_translate_irq() and is
762 762 * called only from apic_intr_ops(). With the new ADII framework,
763 763 * the priority can no longer be retrieved through i_ddi_get_intrspec().
764 764 * It has to be passed in from the caller.
765 765 *
766 766 * Return value:
767 767 * Success: irqno for the given device
768 768 * Failure: -1
769 769 */
770 770 int
771 771 apic_introp_xlate(dev_info_t *dip, struct intrspec *ispec, int type)
772 772 {
773 773 char dev_type[16];
774 774 int dev_len, pci_irq, newirq, bustype, devid, busid, i;
775 775 int irqno = ispec->intrspec_vec;
776 776 ddi_acc_handle_t cfg_handle;
777 777 uchar_t ipin;
778 778 struct apic_io_intr *intrp;
779 779 iflag_t intr_flag;
780 780 ACPI_SUBTABLE_HEADER *hp;
781 781 ACPI_MADT_INTERRUPT_OVERRIDE *isop;
782 782 apic_irq_t *airqp;
783 783 int parent_is_pci_or_pciex = 0;
784 784 int child_is_pciex = 0;
785 785
786 786 DDI_INTR_IMPLDBG((CE_CONT, "apic_introp_xlate: dip=0x%p name=%s "
787 787 "type=%d irqno=0x%x\n", (void *)dip, ddi_get_name(dip), type,
788 788 irqno));
789 789
790 790 dev_len = sizeof (dev_type);
791 791 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, ddi_get_parent(dip),
792 792 DDI_PROP_DONTPASS, "device_type", (caddr_t)dev_type,
793 793 &dev_len) == DDI_PROP_SUCCESS) {
794 794 if ((strcmp(dev_type, "pci") == 0) ||
795 795 (strcmp(dev_type, "pciex") == 0))
796 796 parent_is_pci_or_pciex = 1;
797 797 }
798 798
799 799 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
800 800 DDI_PROP_DONTPASS, "compatible", (caddr_t)dev_type,
801 801 &dev_len) == DDI_PROP_SUCCESS) {
802 802 if (strstr(dev_type, "pciex"))
803 803 child_is_pciex = 1;
804 804 }
805 805
806 806 if (DDI_INTR_IS_MSI_OR_MSIX(type)) {
807 807 if ((airqp = apic_find_irq(dip, ispec, type)) != NULL) {
808 808 airqp->airq_iflag.bustype =
809 809 child_is_pciex ? BUS_PCIE : BUS_PCI;
810 810 return (apic_vector_to_irq[airqp->airq_vector]);
811 811 }
812 812 return (apic_setup_irq_table(dip, irqno, NULL, ispec,
813 813 NULL, type));
814 814 }
815 815
816 816 bustype = 0;
817 817
818 818 /* check if we have already translated this irq */
819 819 mutex_enter(&airq_mutex);
820 820 newirq = apic_min_device_irq;
821 821 for (; newirq <= apic_max_device_irq; newirq++) {
822 822 airqp = apic_irq_table[newirq];
823 823 while (airqp) {
824 824 if ((airqp->airq_dip == dip) &&
825 825 (airqp->airq_origirq == irqno) &&
826 826 (airqp->airq_mps_intr_index != FREE_INDEX)) {
827 827
828 828 mutex_exit(&airq_mutex);
829 829 return (VIRTIRQ(newirq, airqp->airq_share_id));
830 830 }
831 831 airqp = airqp->airq_next;
832 832 }
833 833 }
834 834 mutex_exit(&airq_mutex);
835 835
836 836 if (apic_defconf)
837 837 goto defconf;
838 838
839 839 if ((dip == NULL) || (!apic_irq_translate && !apic_enable_acpi))
840 840 goto nonpci;
841 841
842 842 if (parent_is_pci_or_pciex) {
843 843 /* pci device */
844 844 if (acpica_get_bdf(dip, &busid, &devid, NULL) != 0)
845 845 goto nonpci;
846 846 if (busid == 0 && apic_pci_bus_total == 1)
847 847 busid = (int)apic_single_pci_busid;
848 848
849 849 if (pci_config_setup(dip, &cfg_handle) != DDI_SUCCESS)
850 850 return (-1);
851 851 ipin = pci_config_get8(cfg_handle, PCI_CONF_IPIN) - PCI_INTA;
852 852 pci_config_teardown(&cfg_handle);
853 853 if (apic_enable_acpi && !apic_use_acpi_madt_only) {
854 854 if (apic_acpi_translate_pci_irq(dip, busid, devid,
855 855 ipin, &pci_irq, &intr_flag) != ACPI_PSM_SUCCESS)
856 856 return (-1);
857 857
858 858 intr_flag.bustype = child_is_pciex ? BUS_PCIE : BUS_PCI;
859 859 return (apic_setup_irq_table(dip, pci_irq, NULL, ispec,
860 860 &intr_flag, type));
861 861 } else {
862 862 pci_irq = ((devid & 0x1f) << 2) | (ipin & 0x3);
863 863 if ((intrp = apic_find_io_intr_w_busid(pci_irq, busid))
864 864 == NULL) {
865 865 if ((pci_irq = apic_handle_pci_pci_bridge(dip,
866 866 devid, ipin, &intrp)) == -1)
867 867 return (-1);
868 868 }
869 869 return (apic_setup_irq_table(dip, pci_irq, intrp, ispec,
870 870 NULL, type));
871 871 }
872 872 } else if (strcmp(dev_type, "isa") == 0)
873 873 bustype = BUS_ISA;
874 874 else if (strcmp(dev_type, "eisa") == 0)
875 875 bustype = BUS_EISA;
876 876
877 877 nonpci:
878 878 if (apic_enable_acpi && !apic_use_acpi_madt_only) {
879 879 /* search iso entries first */
880 880 if (acpi_iso_cnt != 0) {
881 881 hp = (ACPI_SUBTABLE_HEADER *)acpi_isop;
882 882 i = 0;
883 883 while (i < acpi_iso_cnt) {
884 884 if (hp->Type ==
885 885 ACPI_MADT_TYPE_INTERRUPT_OVERRIDE) {
886 886 isop =
887 887 (ACPI_MADT_INTERRUPT_OVERRIDE *) hp;
888 888 if (isop->Bus == 0 &&
889 889 isop->SourceIrq == irqno) {
890 890 newirq = isop->GlobalIrq;
891 891 intr_flag.intr_po =
892 892 isop->IntiFlags &
893 893 ACPI_MADT_POLARITY_MASK;
894 894 intr_flag.intr_el =
895 895 (isop->IntiFlags &
896 896 ACPI_MADT_TRIGGER_MASK)
897 897 >> 2;
898 898 intr_flag.bustype = BUS_ISA;
899 899
900 900 return (apic_setup_irq_table(
901 901 dip, newirq, NULL, ispec,
902 902 &intr_flag, type));
903 903
904 904 }
905 905 i++;
906 906 }
907 907 hp = (ACPI_SUBTABLE_HEADER *)(((char *)hp) +
908 908 hp->Length);
909 909 }
910 910 }
911 911 intr_flag.intr_po = INTR_PO_ACTIVE_HIGH;
912 912 intr_flag.intr_el = INTR_EL_EDGE;
913 913 intr_flag.bustype = BUS_ISA;
914 914 return (apic_setup_irq_table(dip, irqno, NULL, ispec,
915 915 &intr_flag, type));
916 916 } else {
917 917 if (bustype == 0) /* not initialized */
918 918 bustype = eisa_level_intr_mask ? BUS_EISA : BUS_ISA;
919 919 for (i = 0; i < 2; i++) {
920 920 if (((busid = apic_find_bus_id(bustype)) != -1) &&
921 921 ((intrp = apic_find_io_intr_w_busid(irqno, busid))
922 922 != NULL)) {
923 923 if ((newirq = apic_setup_irq_table(dip, irqno,
924 924 intrp, ispec, NULL, type)) != -1) {
925 925 return (newirq);
926 926 }
927 927 goto defconf;
928 928 }
929 929 bustype = (bustype == BUS_EISA) ? BUS_ISA : BUS_EISA;
930 930 }
931 931 }
932 932
933 933 /* MPS default configuration */
934 934 defconf:
935 935 newirq = apic_setup_irq_table(dip, irqno, NULL, ispec, NULL, type);
936 936 if (newirq == -1)
937 937 return (-1);
938 938 ASSERT(IRQINDEX(newirq) == irqno);
939 939 ASSERT(apic_irq_table[irqno]);
940 940 return (newirq);
941 941 }
942 942
943 943 /*
944 944 * Attempt to share vector with someone else
945 945 */
946 946 static int
947 947 apic_share_vector(int irqno, iflag_t *intr_flagp, short intr_index, int ipl,
948 948 uchar_t ioapicindex, uchar_t ipin, apic_irq_t **irqptrp)
949 949 {
950 950 #ifdef DEBUG
951 951 apic_irq_t *tmpirqp = NULL;
952 952 #endif /* DEBUG */
953 953 apic_irq_t *irqptr, dummyirq;
954 954 int newirq, chosen_irq = -1, share = 127;
955 955 int lowest, highest, i;
956 956 uchar_t share_id;
957 957
958 958 DDI_INTR_IMPLDBG((CE_CONT, "apic_share_vector: irqno=0x%x "
959 959 "intr_index=0x%x ipl=0x%x\n", irqno, intr_index, ipl));
960 960
961 961 highest = apic_ipltopri[ipl] + APIC_VECTOR_MASK;
962 962 lowest = apic_ipltopri[ipl-1] + APIC_VECTOR_PER_IPL;
963 963
964 964 if (highest < lowest) /* Both ipl and ipl-1 map to same pri */
965 965 lowest -= APIC_VECTOR_PER_IPL;
966 966 dummyirq.airq_mps_intr_index = intr_index;
967 967 dummyirq.airq_ioapicindex = ioapicindex;
968 968 dummyirq.airq_intin_no = ipin;
969 969 if (intr_flagp)
970 970 dummyirq.airq_iflag = *intr_flagp;
971 971 apic_record_rdt_entry(&dummyirq, irqno);
972 972 for (i = lowest; i <= highest; i++) {
973 973 newirq = apic_vector_to_irq[i];
974 974 if (newirq == APIC_RESV_IRQ)
975 975 continue;
976 976 irqptr = apic_irq_table[newirq];
977 977
978 978 if ((dummyirq.airq_rdt_entry & 0xFF00) !=
979 979 (irqptr->airq_rdt_entry & 0xFF00))
980 980 /* not compatible */
981 981 continue;
982 982
983 983 if (irqptr->airq_share < share) {
984 984 share = irqptr->airq_share;
985 985 chosen_irq = newirq;
986 986 }
987 987 }
988 988 if (chosen_irq != -1) {
989 989 /*
990 990 * Assign a share id which is free or which is larger
991 991 * than the largest one.
992 992 */
993 993 share_id = 1;
994 994 mutex_enter(&airq_mutex);
995 995 irqptr = apic_irq_table[chosen_irq];
996 996 while (irqptr) {
997 997 if (irqptr->airq_mps_intr_index == FREE_INDEX) {
998 998 share_id = irqptr->airq_share_id;
999 999 break;
1000 1000 }
1001 1001 if (share_id <= irqptr->airq_share_id)
1002 1002 share_id = irqptr->airq_share_id + 1;
1003 1003 #ifdef DEBUG
1004 1004 tmpirqp = irqptr;
1005 1005 #endif /* DEBUG */
1006 1006 irqptr = irqptr->airq_next;
1007 1007 }
1008 1008 if (!irqptr) {
1009 1009 irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
1010 1010 irqptr->airq_temp_cpu = IRQ_UNINIT;
1011 1011 irqptr->airq_next =
1012 1012 apic_irq_table[chosen_irq]->airq_next;
1013 1013 apic_irq_table[chosen_irq]->airq_next = irqptr;
1014 1014 #ifdef DEBUG
1015 1015 tmpirqp = apic_irq_table[chosen_irq];
1016 1016 #endif /* DEBUG */
1017 1017 }
1018 1018 irqptr->airq_mps_intr_index = intr_index;
1019 1019 irqptr->airq_ioapicindex = ioapicindex;
1020 1020 irqptr->airq_intin_no = ipin;
1021 1021 if (intr_flagp)
1022 1022 irqptr->airq_iflag = *intr_flagp;
1023 1023 irqptr->airq_vector = apic_irq_table[chosen_irq]->airq_vector;
1024 1024 irqptr->airq_share_id = share_id;
1025 1025 apic_record_rdt_entry(irqptr, irqno);
1026 1026 *irqptrp = irqptr;
1027 1027 #ifdef DEBUG
1028 1028 /* shuffle the pointers to test apic_delspl path */
1029 1029 if (tmpirqp) {
1030 1030 tmpirqp->airq_next = irqptr->airq_next;
1031 1031 irqptr->airq_next = apic_irq_table[chosen_irq];
1032 1032 apic_irq_table[chosen_irq] = irqptr;
1033 1033 }
1034 1034 #endif /* DEBUG */
1035 1035 mutex_exit(&airq_mutex);
1036 1036 return (VIRTIRQ(chosen_irq, share_id));
1037 1037 }
1038 1038 return (-1);
1039 1039 }
1040 1040
1041 1041 /*
1042 1042 * Allocate/Initialize the apic_irq_table[] entry for given irqno. If the entry
1043 1043 * is used already, we will try to allocate a new irqno.
1044 1044 *
1045 1045 * Return value:
1046 1046 * Success: irqno
1047 1047 * Failure: -1
1048 1048 */
1049 1049 static int
1050 1050 apic_setup_irq_table(dev_info_t *dip, int irqno, struct apic_io_intr *intrp,
1051 1051 struct intrspec *ispec, iflag_t *intr_flagp, int type)
1052 1052 {
1053 1053 int origirq = ispec->intrspec_vec;
1054 1054 uchar_t ipl = ispec->intrspec_pri;
1055 1055 int newirq, intr_index;
1056 1056 uchar_t ipin, ioapic, ioapicindex, vector;
1057 1057 apic_irq_t *irqptr;
1058 1058 major_t major;
1059 1059 dev_info_t *sdip;
1060 1060
1061 1061 DDI_INTR_IMPLDBG((CE_CONT, "apic_setup_irq_table: dip=0x%p type=%d "
1062 1062 "irqno=0x%x origirq=0x%x\n", (void *)dip, type, irqno, origirq));
1063 1063
1064 1064 ASSERT(ispec != NULL);
1065 1065
1066 1066 major = (dip != NULL) ? ddi_driver_major(dip) : 0;
1067 1067
1068 1068 if (DDI_INTR_IS_MSI_OR_MSIX(type)) {
1069 1069 /* MSI/X doesn't need to setup ioapic stuffs */
1070 1070 ioapicindex = 0xff;
1071 1071 ioapic = 0xff;
1072 1072 ipin = (uchar_t)0xff;
1073 1073 intr_index = (type == DDI_INTR_TYPE_MSI) ? MSI_INDEX :
1074 1074 MSIX_INDEX;
1075 1075 mutex_enter(&airq_mutex);
1076 1076 if ((irqno = apic_allocate_irq(apic_first_avail_irq)) == -1) {
1077 1077 mutex_exit(&airq_mutex);
1078 1078 /* need an irq for MSI/X to index into autovect[] */
1079 1079 cmn_err(CE_WARN, "No interrupt irq: %s instance %d",
1080 1080 ddi_get_name(dip), ddi_get_instance(dip));
1081 1081 return (-1);
1082 1082 }
1083 1083 mutex_exit(&airq_mutex);
1084 1084
1085 1085 } else if (intrp != NULL) {
1086 1086 intr_index = (int)(intrp - apic_io_intrp);
1087 1087 ioapic = intrp->intr_destid;
1088 1088 ipin = intrp->intr_destintin;
1089 1089 /* Find ioapicindex. If destid was ALL, we will exit with 0. */
1090 1090 for (ioapicindex = apic_io_max - 1; ioapicindex; ioapicindex--)
1091 1091 if (apic_io_id[ioapicindex] == ioapic)
1092 1092 break;
1093 1093 ASSERT((ioapic == apic_io_id[ioapicindex]) ||
1094 1094 (ioapic == INTR_ALL_APIC));
1095 1095
1096 1096 /* check whether this intin# has been used by another irqno */
1097 1097 if ((newirq = apic_find_intin(ioapicindex, ipin)) != -1) {
1098 1098 return (newirq);
1099 1099 }
1100 1100
1101 1101 } else if (intr_flagp != NULL) {
1102 1102 /* ACPI case */
1103 1103 intr_index = ACPI_INDEX;
1104 1104 ioapicindex = acpi_find_ioapic(irqno);
1105 1105 ASSERT(ioapicindex != 0xFF);
1106 1106 ioapic = apic_io_id[ioapicindex];
1107 1107 ipin = irqno - apic_io_vectbase[ioapicindex];
1108 1108 if (apic_irq_table[irqno] &&
1109 1109 apic_irq_table[irqno]->airq_mps_intr_index == ACPI_INDEX) {
1110 1110 ASSERT(apic_irq_table[irqno]->airq_intin_no == ipin &&
1111 1111 apic_irq_table[irqno]->airq_ioapicindex ==
1112 1112 ioapicindex);
1113 1113 return (irqno);
1114 1114 }
1115 1115
1116 1116 } else {
1117 1117 /* default configuration */
1118 1118 ioapicindex = 0;
1119 1119 ioapic = apic_io_id[ioapicindex];
1120 1120 ipin = (uchar_t)irqno;
1121 1121 intr_index = DEFAULT_INDEX;
1122 1122 }
1123 1123
1124 1124 if (ispec == NULL) {
1125 1125 APIC_VERBOSE_IOAPIC((CE_WARN, "No intrspec for irqno = %x\n",
1126 1126 irqno));
1127 1127 } else if ((vector = apic_allocate_vector(ipl, irqno, 0)) == 0) {
1128 1128 if ((newirq = apic_share_vector(irqno, intr_flagp, intr_index,
1129 1129 ipl, ioapicindex, ipin, &irqptr)) != -1) {
1130 1130 irqptr->airq_ipl = ipl;
1131 1131 irqptr->airq_origirq = (uchar_t)origirq;
1132 1132 irqptr->airq_dip = dip;
1133 1133 irqptr->airq_major = major;
1134 1134 sdip = apic_irq_table[IRQINDEX(newirq)]->airq_dip;
1135 1135 /* This is OK to do really */
1136 1136 if (sdip == NULL) {
1137 1137 cmn_err(CE_WARN, "Sharing vectors: %s"
1138 1138 " instance %d and SCI",
1139 1139 ddi_get_name(dip), ddi_get_instance(dip));
1140 1140 } else {
1141 1141 cmn_err(CE_WARN, "Sharing vectors: %s"
1142 1142 " instance %d and %s instance %d",
1143 1143 ddi_get_name(sdip), ddi_get_instance(sdip),
1144 1144 ddi_get_name(dip), ddi_get_instance(dip));
1145 1145 }
1146 1146 return (newirq);
1147 1147 }
1148 1148 /* try high priority allocation now that share has failed */
1149 1149 if ((vector = apic_allocate_vector(ipl, irqno, 1)) == 0) {
1150 1150 cmn_err(CE_WARN, "No interrupt vector: %s instance %d",
1151 1151 ddi_get_name(dip), ddi_get_instance(dip));
1152 1152 return (-1);
1153 1153 }
1154 1154 }
1155 1155
1156 1156 mutex_enter(&airq_mutex);
1157 1157 if (apic_irq_table[irqno] == NULL) {
1158 1158 irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
1159 1159 irqptr->airq_temp_cpu = IRQ_UNINIT;
1160 1160 apic_irq_table[irqno] = irqptr;
1161 1161 } else {
1162 1162 irqptr = apic_irq_table[irqno];
1163 1163 if (irqptr->airq_mps_intr_index != FREE_INDEX) {
1164 1164 /*
1165 1165 * The slot is used by another irqno, so allocate
1166 1166 * a free irqno for this interrupt
1167 1167 */
1168 1168 newirq = apic_allocate_irq(apic_first_avail_irq);
1169 1169 if (newirq == -1) {
1170 1170 mutex_exit(&airq_mutex);
1171 1171 return (-1);
1172 1172 }
1173 1173 irqno = newirq;
1174 1174 irqptr = apic_irq_table[irqno];
1175 1175 if (irqptr == NULL) {
1176 1176 irqptr = kmem_zalloc(sizeof (apic_irq_t),
1177 1177 KM_SLEEP);
1178 1178 irqptr->airq_temp_cpu = IRQ_UNINIT;
1179 1179 apic_irq_table[irqno] = irqptr;
1180 1180 }
1181 1181 vector = apic_modify_vector(vector, newirq);
1182 1182 }
1183 1183 }
1184 1184 apic_max_device_irq = max(irqno, apic_max_device_irq);
1185 1185 apic_min_device_irq = min(irqno, apic_min_device_irq);
1186 1186 mutex_exit(&airq_mutex);
1187 1187 irqptr->airq_ioapicindex = ioapicindex;
1188 1188 irqptr->airq_intin_no = ipin;
1189 1189 irqptr->airq_ipl = ipl;
1190 1190 irqptr->airq_vector = vector;
1191 1191 irqptr->airq_origirq = (uchar_t)origirq;
1192 1192 irqptr->airq_share_id = 0;
1193 1193 irqptr->airq_mps_intr_index = (short)intr_index;
1194 1194 irqptr->airq_dip = dip;
1195 1195 irqptr->airq_major = major;
1196 1196 irqptr->airq_cpu = apic_bind_intr(dip, irqno, ioapic, ipin);
1197 1197 if (intr_flagp)
1198 1198 irqptr->airq_iflag = *intr_flagp;
1199 1199
1200 1200 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
1201 1201 /* setup I/O APIC entry for non-MSI/X interrupts */
1202 1202 apic_record_rdt_entry(irqptr, irqno);
1203 1203 }
1204 1204 return (irqno);
1205 1205 }
1206 1206
1207 1207 /*
1208 1208 * return the cpu to which this intr should be bound.
1209 1209 * Check properties or any other mechanism to see if user wants it
1210 1210 * bound to a specific CPU. If so, return the cpu id with high bit set.
1211 1211 * If not, use the policy to choose a cpu and return the id.
1212 1212 */
1213 1213 uint32_t
1214 1214 apic_bind_intr(dev_info_t *dip, int irq, uchar_t ioapicid, uchar_t intin)
1215 1215 {
1216 1216 int instance, instno, prop_len, bind_cpu, count;
1217 1217 uint_t i, rc;
1218 1218 uint32_t cpu;
1219 1219 major_t major;
1220 1220 char *name, *drv_name, *prop_val, *cptr;
1221 1221 char prop_name[32];
1222 1222 ulong_t iflag;
1223 1223
1224 1224
1225 1225 if (apic_intr_policy == INTR_LOWEST_PRIORITY)
1226 1226 return (IRQ_UNBOUND);
1227 1227
1228 1228 if (apic_nproc == 1)
1229 1229 return (0);
1230 1230
1231 1231 drv_name = NULL;
1232 1232 rc = DDI_PROP_NOT_FOUND;
1233 1233 major = (major_t)-1;
1234 1234 if (dip != NULL) {
1235 1235 name = ddi_get_name(dip);
1236 1236 major = ddi_name_to_major(name);
1237 1237 drv_name = ddi_major_to_name(major);
1238 1238 instance = ddi_get_instance(dip);
1239 1239 if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) {
1240 1240 i = apic_min_device_irq;
1241 1241 for (; i <= apic_max_device_irq; i++) {
1242 1242
1243 1243 if ((i == irq) || (apic_irq_table[i] == NULL) ||
1244 1244 (apic_irq_table[i]->airq_mps_intr_index
1245 1245 == FREE_INDEX))
1246 1246 continue;
1247 1247
1248 1248 if ((apic_irq_table[i]->airq_major == major) &&
1249 1249 (!(apic_irq_table[i]->airq_cpu &
1250 1250 IRQ_USER_BOUND))) {
1251 1251
1252 1252 cpu = apic_irq_table[i]->airq_cpu;
1253 1253
1254 1254 cmn_err(CE_CONT,
1255 1255 "!%s: %s (%s) instance #%d "
1256 1256 "irq 0x%x vector 0x%x ioapic 0x%x "
1257 1257 "intin 0x%x is bound to cpu %d\n",
1258 1258 psm_name,
1259 1259 name, drv_name, instance, irq,
1260 1260 apic_irq_table[irq]->airq_vector,
1261 1261 ioapicid, intin, cpu);
1262 1262 return (cpu);
1263 1263 }
1264 1264 }
1265 1265 }
1266 1266 /*
1267 1267 * search for "drvname"_intpt_bind_cpus property first, the
1268 1268 * syntax of the property should be "a[,b,c,...]" where
1269 1269 * instance 0 binds to cpu a, instance 1 binds to cpu b,
1270 1270 * instance 3 binds to cpu c...
1271 1271 * ddi_getlongprop() will search /option first, then /
1272 1272 * if "drvname"_intpt_bind_cpus doesn't exist, then find
1273 1273 * intpt_bind_cpus property. The syntax is the same, and
1274 1274 * it applies to all the devices if its "drvname" specific
1275 1275 * property doesn't exist
1276 1276 */
1277 1277 (void) strcpy(prop_name, drv_name);
1278 1278 (void) strcat(prop_name, "_intpt_bind_cpus");
1279 1279 rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, prop_name,
1280 1280 (caddr_t)&prop_val, &prop_len);
1281 1281 if (rc != DDI_PROP_SUCCESS) {
1282 1282 rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0,
1283 1283 "intpt_bind_cpus", (caddr_t)&prop_val, &prop_len);
1284 1284 }
1285 1285 }
1286 1286 if (rc == DDI_PROP_SUCCESS) {
1287 1287 for (i = count = 0; i < (prop_len - 1); i++)
1288 1288 if (prop_val[i] == ',')
1289 1289 count++;
1290 1290 if (prop_val[i-1] != ',')
1291 1291 count++;
1292 1292 /*
1293 1293 * if somehow the binding instances defined in the
1294 1294 * property are not enough for this instno., then
1295 1295 * reuse the pattern for the next instance until
1296 1296 * it reaches the requested instno
1297 1297 */
1298 1298 instno = instance % count;
1299 1299 i = 0;
1300 1300 cptr = prop_val;
1301 1301 while (i < instno)
1302 1302 if (*cptr++ == ',')
1303 1303 i++;
1304 1304 bind_cpu = stoi(&cptr);
1305 1305 kmem_free(prop_val, prop_len);
1306 1306 /* if specific CPU is bogus, then default to next cpu */
1307 1307 if (!apic_cpu_in_range(bind_cpu)) {
1308 1308 cmn_err(CE_WARN, "%s: %s=%s: CPU %d not present",
1309 1309 psm_name, prop_name, prop_val, bind_cpu);
1310 1310 rc = DDI_PROP_NOT_FOUND;
1311 1311 } else {
1312 1312 /* indicate that we are bound at user request */
1313 1313 bind_cpu |= IRQ_USER_BOUND;
1314 1314 }
1315 1315 /*
1316 1316 * no need to check apic_cpus[].aci_status, if specific CPU is
1317 1317 * not up, then post_cpu_start will handle it.
1318 1318 */
1319 1319 }
1320 1320 if (rc != DDI_PROP_SUCCESS) {
1321 1321 iflag = intr_clear();
1322 1322 lock_set(&apic_ioapic_lock);
1323 1323 bind_cpu = apic_get_next_bind_cpu();
1324 1324 lock_clear(&apic_ioapic_lock);
1325 1325 intr_restore(iflag);
1326 1326 }
1327 1327
1328 1328 if (drv_name != NULL)
1329 1329 cmn_err(CE_CONT, "!%s: %s (%s) instance %d irq 0x%x "
1330 1330 "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
1331 1331 psm_name, name, drv_name, instance, irq,
1332 1332 apic_irq_table[irq]->airq_vector, ioapicid, intin,
1333 1333 bind_cpu & ~IRQ_USER_BOUND);
1334 1334 else
1335 1335 cmn_err(CE_CONT, "!%s: irq 0x%x "
1336 1336 "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
1337 1337 psm_name, irq, apic_irq_table[irq]->airq_vector, ioapicid,
1338 1338 intin, bind_cpu & ~IRQ_USER_BOUND);
1339 1339
1340 1340 return ((uint32_t)bind_cpu);
1341 1341 }
1342 1342
1343 1343 /*
1344 1344 * Mark vector as being in the process of being deleted. Interrupts
1345 1345 * may still come in on some CPU. The moment an interrupt comes with
1346 1346 * the new vector, we know we can free the old one. Called only from
1347 1347 * addspl and delspl with interrupts disabled. Because an interrupt
1348 1348 * can be shared, but no interrupt from either device may come in,
1349 1349 * we also use a timeout mechanism, which we arbitrarily set to
1350 1350 * apic_revector_timeout microseconds.
1351 1351 */
1352 1352 static void
1353 1353 apic_mark_vector(uchar_t oldvector, uchar_t newvector)
1354 1354 {
1355 1355 ulong_t iflag;
1356 1356
1357 1357 iflag = intr_clear();
1358 1358 lock_set(&apic_revector_lock);
1359 1359 if (!apic_oldvec_to_newvec) {
1360 1360 apic_oldvec_to_newvec =
1361 1361 kmem_zalloc(sizeof (newvector) * APIC_MAX_VECTOR * 2,
1362 1362 KM_NOSLEEP);
1363 1363
1364 1364 if (!apic_oldvec_to_newvec) {
1365 1365 /*
1366 1366 * This failure is not catastrophic.
1367 1367 * But, the oldvec will never be freed.
1368 1368 */
1369 1369 apic_error |= APIC_ERR_MARK_VECTOR_FAIL;
1370 1370 lock_clear(&apic_revector_lock);
1371 1371 intr_restore(iflag);
1372 1372 return;
1373 1373 }
1374 1374 apic_newvec_to_oldvec = &apic_oldvec_to_newvec[APIC_MAX_VECTOR];
1375 1375 }
1376 1376
1377 1377 /* See if we already did this for drivers which do double addintrs */
1378 1378 if (apic_oldvec_to_newvec[oldvector] != newvector) {
1379 1379 apic_oldvec_to_newvec[oldvector] = newvector;
1380 1380 apic_newvec_to_oldvec[newvector] = oldvector;
1381 1381 apic_revector_pending++;
1382 1382 }
1383 1383 lock_clear(&apic_revector_lock);
1384 1384 intr_restore(iflag);
1385 1385 (void) timeout(apic_xlate_vector_free_timeout_handler,
1386 1386 (void *)(uintptr_t)oldvector, drv_usectohz(apic_revector_timeout));
1387 1387 }
1388 1388
1389 1389 /*
1390 1390 * xlate_vector is called from intr_enter if revector_pending is set.
1391 1391 * It will xlate it if needed and mark the old vector as free.
1392 1392 */
1393 1393 uchar_t
1394 1394 apic_xlate_vector(uchar_t vector)
1395 1395 {
1396 1396 uchar_t newvector, oldvector = 0;
1397 1397
1398 1398 lock_set(&apic_revector_lock);
1399 1399 /* Do we really need to do this ? */
1400 1400 if (!apic_revector_pending) {
1401 1401 lock_clear(&apic_revector_lock);
1402 1402 return (vector);
1403 1403 }
1404 1404 if ((newvector = apic_oldvec_to_newvec[vector]) != 0)
1405 1405 oldvector = vector;
1406 1406 else {
1407 1407 /*
1408 1408 * The incoming vector is new . See if a stale entry is
1409 1409 * remaining
1410 1410 */
1411 1411 if ((oldvector = apic_newvec_to_oldvec[vector]) != 0)
1412 1412 newvector = vector;
1413 1413 }
1414 1414
1415 1415 if (oldvector) {
1416 1416 apic_revector_pending--;
1417 1417 apic_oldvec_to_newvec[oldvector] = 0;
1418 1418 apic_newvec_to_oldvec[newvector] = 0;
1419 1419 apic_free_vector(oldvector);
1420 1420 lock_clear(&apic_revector_lock);
1421 1421 /* There could have been more than one reprogramming! */
1422 1422 return (apic_xlate_vector(newvector));
1423 1423 }
1424 1424 lock_clear(&apic_revector_lock);
1425 1425 return (vector);
1426 1426 }
1427 1427
1428 1428 void
1429 1429 apic_xlate_vector_free_timeout_handler(void *arg)
1430 1430 {
1431 1431 ulong_t iflag;
1432 1432 uchar_t oldvector, newvector;
1433 1433
1434 1434 oldvector = (uchar_t)(uintptr_t)arg;
1435 1435 iflag = intr_clear();
1436 1436 lock_set(&apic_revector_lock);
1437 1437 if ((newvector = apic_oldvec_to_newvec[oldvector]) != 0) {
1438 1438 apic_free_vector(oldvector);
1439 1439 apic_oldvec_to_newvec[oldvector] = 0;
1440 1440 apic_newvec_to_oldvec[newvector] = 0;
1441 1441 apic_revector_pending--;
1442 1442 }
1443 1443
1444 1444 lock_clear(&apic_revector_lock);
1445 1445 intr_restore(iflag);
1446 1446 }
1447 1447
1448 1448 /*
1449 1449 * Bind interrupt corresponding to irq_ptr to bind_cpu.
1450 1450 * Must be called with interrupts disabled and apic_ioapic_lock held
1451 1451 */
1452 1452 int
1453 1453 apic_rebind(apic_irq_t *irq_ptr, int bind_cpu,
1454 1454 struct ioapic_reprogram_data *drep)
1455 1455 {
1456 1456 int ioapicindex, intin_no;
1457 1457 uint32_t airq_temp_cpu;
1458 1458 apic_cpus_info_t *cpu_infop;
1459 1459 uint32_t rdt_entry;
1460 1460 int which_irq;
1461 1461 ioapic_rdt_t irdt;
1462 1462
1463 1463 which_irq = apic_vector_to_irq[irq_ptr->airq_vector];
1464 1464
1465 1465 intin_no = irq_ptr->airq_intin_no;
1466 1466 ioapicindex = irq_ptr->airq_ioapicindex;
1467 1467 airq_temp_cpu = irq_ptr->airq_temp_cpu;
1468 1468 if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu != IRQ_UNBOUND) {
1469 1469 if (airq_temp_cpu & IRQ_USER_BOUND)
1470 1470 /* Mask off high bit so it can be used as array index */
1471 1471 airq_temp_cpu &= ~IRQ_USER_BOUND;
1472 1472
1473 1473 ASSERT(apic_cpu_in_range(airq_temp_cpu));
1474 1474 }
1475 1475
1476 1476 /*
1477 1477 * Can't bind to a CPU that's not accepting interrupts:
1478 1478 */
1479 1479 cpu_infop = &apic_cpus[bind_cpu & ~IRQ_USER_BOUND];
1480 1480 if (!(cpu_infop->aci_status & APIC_CPU_INTR_ENABLE))
1481 1481 return (1);
1482 1482
1483 1483 /*
1484 1484 * If we are about to change the interrupt vector for this interrupt,
1485 1485 * and this interrupt is level-triggered, attached to an IOAPIC,
1486 1486 * has been delivered to a CPU and that CPU has not handled it
1487 1487 * yet, we cannot reprogram the IOAPIC now.
1488 1488 */
1489 1489 if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) {
1490 1490
1491 1491 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex,
1492 1492 intin_no);
1493 1493
1494 1494 if ((irq_ptr->airq_vector != RDT_VECTOR(rdt_entry)) &&
1495 1495 apic_check_stuck_interrupt(irq_ptr, airq_temp_cpu,
1496 1496 bind_cpu, ioapicindex, intin_no, which_irq, drep) != 0) {
1497 1497
1498 1498 return (0);
1499 1499 }
1500 1500
1501 1501 /*
1502 1502 * NOTE: We do not unmask the RDT here, as an interrupt MAY
1503 1503 * still come in before we have a chance to reprogram it below.
1504 1504 * The reprogramming below will simultaneously change and
1505 1505 * unmask the RDT entry.
1506 1506 */
1507 1507
1508 1508 if ((uint32_t)bind_cpu == IRQ_UNBOUND) {
1509 1509 irdt.ir_lo = AV_LDEST | AV_LOPRI |
1510 1510 irq_ptr->airq_rdt_entry;
1511 1511
1512 1512 irdt.ir_hi = AV_TOALL >> APIC_ID_BIT_OFFSET;
1513 1513
1514 1514 apic_vt_ops->apic_intrmap_alloc_entry(
1515 1515 &irq_ptr->airq_intrmap_private, NULL,
1516 1516 DDI_INTR_TYPE_FIXED, 1, ioapicindex);
1517 1517 apic_vt_ops->apic_intrmap_map_entry(
1518 1518 irq_ptr->airq_intrmap_private, (void *)&irdt,
1519 1519 DDI_INTR_TYPE_FIXED, 1);
1520 1520 apic_vt_ops->apic_intrmap_record_rdt(
1521 1521 irq_ptr->airq_intrmap_private, &irdt);
1522 1522
1523 1523 /* Write the RDT entry -- no specific CPU binding */
1524 1524 WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no,
1525 1525 irdt.ir_hi | AV_TOALL);
1526 1526
1527 1527 if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu !=
1528 1528 IRQ_UNBOUND)
1529 1529 apic_cpus[airq_temp_cpu].aci_temp_bound--;
1530 1530
1531 1531 /*
1532 1532 * Write the vector, trigger, and polarity portion of
1533 1533 * the RDT
1534 1534 */
1535 1535 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin_no,
1536 1536 irdt.ir_lo);
1537 1537
1538 1538 irq_ptr->airq_temp_cpu = IRQ_UNBOUND;
1539 1539 return (0);
1540 1540 }
1541 1541 }
1542 1542
1543 1543 if (bind_cpu & IRQ_USER_BOUND) {
1544 1544 cpu_infop->aci_bound++;
1545 1545 } else {
1546 1546 cpu_infop->aci_temp_bound++;
1547 1547 }
1548 1548 ASSERT(apic_cpu_in_range(bind_cpu));
1549 1549
1550 1550 if ((airq_temp_cpu != IRQ_UNBOUND) && (airq_temp_cpu != IRQ_UNINIT)) {
1551 1551 apic_cpus[airq_temp_cpu].aci_temp_bound--;
1552 1552 }
1553 1553 if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) {
1554 1554
1555 1555 irdt.ir_lo = AV_PDEST | AV_FIXED | irq_ptr->airq_rdt_entry;
1556 1556 irdt.ir_hi = cpu_infop->aci_local_id;
1557 1557
1558 1558 apic_vt_ops->apic_intrmap_alloc_entry(
1559 1559 &irq_ptr->airq_intrmap_private, NULL, DDI_INTR_TYPE_FIXED,
1560 1560 1, ioapicindex);
1561 1561 apic_vt_ops->apic_intrmap_map_entry(
1562 1562 irq_ptr->airq_intrmap_private,
1563 1563 (void *)&irdt, DDI_INTR_TYPE_FIXED, 1);
1564 1564 apic_vt_ops->apic_intrmap_record_rdt(
1565 1565 irq_ptr->airq_intrmap_private, &irdt);
1566 1566
1567 1567 /* Write the RDT entry -- bind to a specific CPU: */
1568 1568 WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no,
1569 1569 irdt.ir_hi);
1570 1570
1571 1571 /* Write the vector, trigger, and polarity portion of the RDT */
1572 1572 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin_no,
1573 1573 irdt.ir_lo);
1574 1574
1575 1575 } else {
1576 1576 int type = (irq_ptr->airq_mps_intr_index == MSI_INDEX) ?
1577 1577 DDI_INTR_TYPE_MSI : DDI_INTR_TYPE_MSIX;
1578 1578 if (type == DDI_INTR_TYPE_MSI) {
1579 1579 if (irq_ptr->airq_ioapicindex ==
1580 1580 irq_ptr->airq_origirq) {
1581 1581 /* first one */
1582 1582 DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call "
1583 1583 "apic_pci_msi_enable_vector\n"));
1584 1584 apic_pci_msi_enable_vector(irq_ptr,
1585 1585 type, which_irq, irq_ptr->airq_vector,
1586 1586 irq_ptr->airq_intin_no,
1587 1587 cpu_infop->aci_local_id);
1588 1588 }
1589 1589 if ((irq_ptr->airq_ioapicindex +
1590 1590 irq_ptr->airq_intin_no - 1) ==
1591 1591 irq_ptr->airq_origirq) { /* last one */
1592 1592 DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call "
1593 1593 "apic_pci_msi_enable_mode\n"));
1594 1594 apic_pci_msi_enable_mode(irq_ptr->airq_dip,
1595 1595 type, which_irq);
1596 1596 }
1597 1597 } else { /* MSI-X */
1598 1598 apic_pci_msi_enable_vector(irq_ptr, type,
1599 1599 irq_ptr->airq_origirq, irq_ptr->airq_vector, 1,
1600 1600 cpu_infop->aci_local_id);
1601 1601 apic_pci_msi_enable_mode(irq_ptr->airq_dip, type,
1602 1602 irq_ptr->airq_origirq);
1603 1603 }
1604 1604 }
1605 1605 irq_ptr->airq_temp_cpu = (uint32_t)bind_cpu;
1606 1606 apic_redist_cpu_skip &= ~(1 << (bind_cpu & ~IRQ_USER_BOUND));
1607 1607 return (0);
1608 1608 }
1609 1609
1610 1610 static void
1611 1611 apic_last_ditch_clear_remote_irr(int ioapic_ix, int intin_no)
1612 1612 {
1613 1613 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no)
1614 1614 & AV_REMOTE_IRR) != 0) {
1615 1615 /*
1616 1616 * Trying to clear the bit through normal
1617 1617 * channels has failed. So as a last-ditch
1618 1618 * effort, try to set the trigger mode to
1619 1619 * edge, then to level. This has been
1620 1620 * observed to work on many systems.
1621 1621 */
1622 1622 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1623 1623 intin_no,
1624 1624 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1625 1625 intin_no) & ~AV_LEVEL);
1626 1626
1627 1627 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1628 1628 intin_no,
1629 1629 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1630 1630 intin_no) | AV_LEVEL);
1631 1631
1632 1632 /*
1633 1633 * If the bit's STILL set, this interrupt may
1634 1634 * be hosed.
1635 1635 */
1636 1636 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1637 1637 intin_no) & AV_REMOTE_IRR) != 0) {
1638 1638
1639 1639 prom_printf("%s: Remote IRR still "
1640 1640 "not clear for IOAPIC %d intin %d.\n"
1641 1641 "\tInterrupts to this pin may cease "
1642 1642 "functioning.\n", psm_name, ioapic_ix,
1643 1643 intin_no);
1644 1644 #ifdef DEBUG
1645 1645 apic_last_ditch_reprogram_failures++;
1646 1646 #endif
1647 1647 }
1648 1648 }
1649 1649 }
1650 1650
1651 1651 /*
1652 1652 * This function is protected by apic_ioapic_lock coupled with the
1653 1653 * fact that interrupts are disabled.
1654 1654 */
1655 1655 static void
1656 1656 delete_defer_repro_ent(int which_irq)
1657 1657 {
1658 1658 ASSERT(which_irq >= 0);
1659 1659 ASSERT(which_irq <= 255);
1660 1660 ASSERT(LOCK_HELD(&apic_ioapic_lock));
1661 1661
1662 1662 if (apic_reprogram_info[which_irq].done)
1663 1663 return;
1664 1664
1665 1665 apic_reprogram_info[which_irq].done = B_TRUE;
1666 1666
1667 1667 #ifdef DEBUG
1668 1668 apic_defer_repro_total_retries +=
1669 1669 apic_reprogram_info[which_irq].tries;
1670 1670
1671 1671 apic_defer_repro_successes++;
1672 1672 #endif
1673 1673
1674 1674 if (--apic_reprogram_outstanding == 0) {
1675 1675
1676 1676 setlvlx = psm_intr_exit_fn();
1677 1677 }
1678 1678 }
1679 1679
1680 1680
1681 1681 /*
1682 1682 * Interrupts must be disabled during this function to prevent
1683 1683 * self-deadlock. Interrupts are disabled because this function
1684 1684 * is called from apic_check_stuck_interrupt(), which is called
1685 1685 * from apic_rebind(), which requires its caller to disable interrupts.
1686 1686 */
1687 1687 static void
1688 1688 add_defer_repro_ent(apic_irq_t *irq_ptr, int which_irq, int new_bind_cpu)
1689 1689 {
1690 1690 ASSERT(which_irq >= 0);
1691 1691 ASSERT(which_irq <= 255);
1692 1692 ASSERT(!interrupts_enabled());
1693 1693
1694 1694 /*
1695 1695 * On the off-chance that there's already a deferred
1696 1696 * reprogramming on this irq, check, and if so, just update the
1697 1697 * CPU and irq pointer to which the interrupt is targeted, then return.
1698 1698 */
1699 1699 if (!apic_reprogram_info[which_irq].done) {
1700 1700 apic_reprogram_info[which_irq].bindcpu = new_bind_cpu;
1701 1701 apic_reprogram_info[which_irq].irqp = irq_ptr;
1702 1702 return;
1703 1703 }
1704 1704
1705 1705 apic_reprogram_info[which_irq].irqp = irq_ptr;
1706 1706 apic_reprogram_info[which_irq].bindcpu = new_bind_cpu;
1707 1707 apic_reprogram_info[which_irq].tries = 0;
1708 1708 /*
1709 1709 * This must be the last thing set, since we're not
1710 1710 * grabbing any locks, apic_try_deferred_reprogram() will
1711 1711 * make its decision about using this entry iff done
1712 1712 * is false.
1713 1713 */
1714 1714 apic_reprogram_info[which_irq].done = B_FALSE;
1715 1715
1716 1716 /*
1717 1717 * If there were previously no deferred reprogrammings, change
1718 1718 * setlvlx to call apic_try_deferred_reprogram()
1719 1719 */
1720 1720 if (++apic_reprogram_outstanding == 1) {
1721 1721
1722 1722 setlvlx = apic_try_deferred_reprogram;
1723 1723 }
1724 1724 }
1725 1725
1726 1726 static void
1727 1727 apic_try_deferred_reprogram(int prev_ipl, int irq)
1728 1728 {
1729 1729 int reproirq;
1730 1730 ulong_t iflag;
1731 1731 struct ioapic_reprogram_data *drep;
1732 1732
1733 1733 (*psm_intr_exit_fn())(prev_ipl, irq);
1734 1734
1735 1735 if (!lock_try(&apic_defer_reprogram_lock)) {
1736 1736 return;
1737 1737 }
1738 1738
1739 1739 /*
1740 1740 * Acquire the apic_ioapic_lock so that any other operations that
1741 1741 * may affect the apic_reprogram_info state are serialized.
1742 1742 * It's still possible for the last deferred reprogramming to clear
1743 1743 * between the time we entered this function and the time we get to
1744 1744 * the for loop below. In that case, *setlvlx will have been set
1745 1745 * back to *_intr_exit and drep will be NULL. (There's no way to
1746 1746 * stop that from happening -- we would need to grab a lock before
1747 1747 * calling *setlvlx, which is neither realistic nor prudent).
1748 1748 */
1749 1749 iflag = intr_clear();
1750 1750 lock_set(&apic_ioapic_lock);
1751 1751
1752 1752 /*
1753 1753 * For each deferred RDT entry, try to reprogram it now. Note that
1754 1754 * there is no lock acquisition to read apic_reprogram_info because
1755 1755 * '.done' is set only after the other fields in the structure are set.
1756 1756 */
1757 1757
1758 1758 drep = NULL;
1759 1759 for (reproirq = 0; reproirq <= APIC_MAX_VECTOR; reproirq++) {
1760 1760 if (apic_reprogram_info[reproirq].done == B_FALSE) {
1761 1761 drep = &apic_reprogram_info[reproirq];
1762 1762 break;
1763 1763 }
1764 1764 }
1765 1765
1766 1766 /*
1767 1767 * Either we found a deferred action to perform, or
1768 1768 * we entered this function spuriously, after *setlvlx
1769 1769 * was restored to point to *_intr_exit. Any other
1770 1770 * permutation is invalid.
1771 1771 */
1772 1772 ASSERT(drep != NULL || *setlvlx == psm_intr_exit_fn());
1773 1773
1774 1774 /*
1775 1775 * Though we can't really do anything about errors
1776 1776 * at this point, keep track of them for reporting.
1777 1777 * Note that it is very possible for apic_setup_io_intr
1778 1778 * to re-register this very timeout if the Remote IRR bit
1779 1779 * has not yet cleared.
1780 1780 */
1781 1781
1782 1782 #ifdef DEBUG
1783 1783 if (drep != NULL) {
1784 1784 if (apic_setup_io_intr(drep, reproirq, B_TRUE) != 0) {
1785 1785 apic_deferred_setup_failures++;
1786 1786 }
1787 1787 } else {
1788 1788 apic_deferred_spurious_enters++;
1789 1789 }
1790 1790 #else
1791 1791 if (drep != NULL)
1792 1792 (void) apic_setup_io_intr(drep, reproirq, B_TRUE);
1793 1793 #endif
1794 1794
1795 1795 lock_clear(&apic_ioapic_lock);
1796 1796 intr_restore(iflag);
1797 1797
1798 1798 lock_clear(&apic_defer_reprogram_lock);
1799 1799 }
1800 1800
1801 1801 static void
1802 1802 apic_ioapic_wait_pending_clear(int ioapic_ix, int intin_no)
1803 1803 {
1804 1804 int waited;
1805 1805
1806 1806 /*
1807 1807 * Wait for the delivery pending bit to clear.
1808 1808 */
1809 1809 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) &
1810 1810 (AV_LEVEL|AV_PENDING)) == (AV_LEVEL|AV_PENDING)) {
1811 1811
1812 1812 /*
1813 1813 * If we're still waiting on the delivery of this interrupt,
1814 1814 * continue to wait here until it is delivered (this should be
1815 1815 * a very small amount of time, but include a timeout just in
1816 1816 * case).
1817 1817 */
1818 1818 for (waited = 0; waited < apic_max_reps_clear_pending;
1819 1819 waited++) {
1820 1820 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1821 1821 intin_no) & AV_PENDING) == 0) {
1822 1822 break;
1823 1823 }
1824 1824 }
1825 1825 }
1826 1826 }
1827 1827
1828 1828
1829 1829 /*
1830 1830 * Checks to see if the IOAPIC interrupt entry specified has its Remote IRR
1831 1831 * bit set. Calls functions that modify the function that setlvlx points to,
1832 1832 * so that the reprogramming can be retried very shortly.
1833 1833 *
1834 1834 * This function will mask the RDT entry if the interrupt is level-triggered.
1835 1835 * (The caller is responsible for unmasking the RDT entry.)
1836 1836 *
1837 1837 * Returns non-zero if the caller should defer IOAPIC reprogramming.
1838 1838 */
1839 1839 static int
1840 1840 apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu,
1841 1841 int new_bind_cpu, int ioapic_ix, int intin_no, int which_irq,
1842 1842 struct ioapic_reprogram_data *drep)
1843 1843 {
1844 1844 int32_t rdt_entry;
1845 1845 int waited;
1846 1846 int reps = 0;
1847 1847
1848 1848 /*
1849 1849 * Wait for the delivery pending bit to clear.
1850 1850 */
1851 1851 do {
1852 1852 ++reps;
1853 1853
1854 1854 apic_ioapic_wait_pending_clear(ioapic_ix, intin_no);
1855 1855
1856 1856 /*
1857 1857 * Mask the RDT entry, but only if it's a level-triggered
1858 1858 * interrupt
1859 1859 */
1860 1860 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1861 1861 intin_no);
1862 1862 if ((rdt_entry & (AV_LEVEL|AV_MASK)) == AV_LEVEL) {
1863 1863
1864 1864 /* Mask it */
1865 1865 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no,
1866 1866 AV_MASK | rdt_entry);
1867 1867 }
1868 1868
1869 1869 if ((rdt_entry & AV_LEVEL) == AV_LEVEL) {
1870 1870 /*
1871 1871 * If there was a race and an interrupt was injected
1872 1872 * just before we masked, check for that case here.
1873 1873 * Then, unmask the RDT entry and try again. If we're
1874 1874 * on our last try, don't unmask (because we want the
1875 1875 * RDT entry to remain masked for the rest of the
1876 1876 * function).
1877 1877 */
1878 1878 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1879 1879 intin_no);
1880 1880 if ((rdt_entry & AV_PENDING) &&
1881 1881 (reps < apic_max_reps_clear_pending)) {
1882 1882 /* Unmask it */
1883 1883 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1884 1884 intin_no, rdt_entry & ~AV_MASK);
1885 1885 }
1886 1886 }
1887 1887
1888 1888 } while ((rdt_entry & AV_PENDING) &&
1889 1889 (reps < apic_max_reps_clear_pending));
1890 1890
1891 1891 #ifdef DEBUG
1892 1892 if (rdt_entry & AV_PENDING)
1893 1893 apic_intr_deliver_timeouts++;
1894 1894 #endif
1895 1895
1896 1896 /*
1897 1897 * If the remote IRR bit is set, then the interrupt has been sent
1898 1898 * to a CPU for processing. We have no choice but to wait for
1899 1899 * that CPU to process the interrupt, at which point the remote IRR
1900 1900 * bit will be cleared.
1901 1901 */
1902 1902 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) &
1903 1903 (AV_LEVEL|AV_REMOTE_IRR)) == (AV_LEVEL|AV_REMOTE_IRR)) {
1904 1904
1905 1905 /*
1906 1906 * If the CPU that this RDT is bound to is NOT the current
1907 1907 * CPU, wait until that CPU handles the interrupt and ACKs
1908 1908 * it. If this interrupt is not bound to any CPU (that is,
1909 1909 * if it's bound to the logical destination of "anyone"), it
1910 1910 * may have been delivered to the current CPU so handle that
1911 1911 * case by deferring the reprogramming (below).
1912 1912 */
1913 1913 if ((old_bind_cpu != IRQ_UNBOUND) &&
1914 1914 (old_bind_cpu != IRQ_UNINIT) &&
1915 1915 (old_bind_cpu != psm_get_cpu_id())) {
1916 1916 for (waited = 0; waited < apic_max_reps_clear_pending;
1917 1917 waited++) {
1918 1918 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1919 1919 intin_no) & AV_REMOTE_IRR) == 0) {
1920 1920
1921 1921 delete_defer_repro_ent(which_irq);
1922 1922
1923 1923 /* Remote IRR has cleared! */
1924 1924 return (0);
1925 1925 }
1926 1926 }
1927 1927 }
1928 1928
1929 1929 /*
1930 1930 * If we waited and the Remote IRR bit is still not cleared,
1931 1931 * AND if we've invoked the timeout APIC_REPROGRAM_MAX_TIMEOUTS
1932 1932 * times for this interrupt, try the last-ditch workaround:
1933 1933 */
1934 1934 if (drep && drep->tries >= APIC_REPROGRAM_MAX_TRIES) {
1935 1935
1936 1936 apic_last_ditch_clear_remote_irr(ioapic_ix, intin_no);
1937 1937
1938 1938 /* Mark this one as reprogrammed: */
1939 1939 delete_defer_repro_ent(which_irq);
1940 1940
1941 1941 return (0);
1942 1942 } else {
1943 1943 #ifdef DEBUG
1944 1944 apic_intr_deferrals++;
1945 1945 #endif
1946 1946
1947 1947 /*
1948 1948 * If waiting for the Remote IRR bit (above) didn't
1949 1949 * allow it to clear, defer the reprogramming.
1950 1950 * Add a new deferred-programming entry if the
1951 1951 * caller passed a NULL one (and update the existing one
1952 1952 * in case anything changed).
1953 1953 */
1954 1954 add_defer_repro_ent(irq_ptr, which_irq, new_bind_cpu);
1955 1955 if (drep)
1956 1956 drep->tries++;
1957 1957
1958 1958 /* Inform caller to defer IOAPIC programming: */
1959 1959 return (1);
1960 1960 }
1961 1961
1962 1962 }
1963 1963
1964 1964 /* Remote IRR is clear */
1965 1965 delete_defer_repro_ent(which_irq);
1966 1966
1967 1967 return (0);
1968 1968 }
1969 1969
1970 1970 /*
1971 1971 * Called to migrate all interrupts at an irq to another cpu.
1972 1972 * Must be called with interrupts disabled and apic_ioapic_lock held
1973 1973 */
1974 1974 int
1975 1975 apic_rebind_all(apic_irq_t *irq_ptr, int bind_cpu)
1976 1976 {
1977 1977 apic_irq_t *irqptr = irq_ptr;
1978 1978 int retval = 0;
1979 1979
1980 1980 while (irqptr) {
1981 1981 if (irqptr->airq_temp_cpu != IRQ_UNINIT)
1982 1982 retval |= apic_rebind(irqptr, bind_cpu, NULL);
1983 1983 irqptr = irqptr->airq_next;
1984 1984 }
1985 1985
1986 1986 return (retval);
1987 1987 }
1988 1988
1989 1989 /*
1990 1990 * apic_intr_redistribute does all the messy computations for identifying
1991 1991 * which interrupt to move to which CPU. Currently we do just one interrupt
1992 1992 * at a time. This reduces the time we spent doing all this within clock
1993 1993 * interrupt. When it is done in idle, we could do more than 1.
1994 1994 * First we find the most busy and the most free CPU (time in ISR only)
1995 1995 * skipping those CPUs that has been identified as being ineligible (cpu_skip)
1996 1996 * Then we look for IRQs which are closest to the difference between the
1997 1997 * most busy CPU and the average ISR load. We try to find one whose load
1998 1998 * is less than difference.If none exists, then we chose one larger than the
1999 1999 * difference, provided it does not make the most idle CPU worse than the
2000 2000 * most busy one. In the end, we clear all the busy fields for CPUs. For
2001 2001 * IRQs, they are cleared as they are scanned.
2002 2002 */
2003 2003 void
2004 2004 apic_intr_redistribute(void)
2005 2005 {
2006 2006 int busiest_cpu, most_free_cpu;
2007 2007 int cpu_free, cpu_busy, max_busy, min_busy;
2008 2008 int min_free, diff;
2009 2009 int average_busy, cpus_online;
2010 2010 int i, busy;
2011 2011 ulong_t iflag;
2012 2012 apic_cpus_info_t *cpu_infop;
2013 2013 apic_irq_t *min_busy_irq = NULL;
2014 2014 apic_irq_t *max_busy_irq = NULL;
2015 2015
2016 2016 busiest_cpu = most_free_cpu = -1;
2017 2017 cpu_free = cpu_busy = max_busy = average_busy = 0;
2018 2018 min_free = apic_sample_factor_redistribution;
2019 2019 cpus_online = 0;
2020 2020 /*
2021 2021 * Below we will check for CPU_INTR_ENABLE, bound, temp_bound, temp_cpu
2022 2022 * without ioapic_lock. That is OK as we are just doing statistical
2023 2023 * sampling anyway and any inaccuracy now will get corrected next time
2024 2024 * The call to rebind which actually changes things will make sure
2025 2025 * we are consistent.
2026 2026 */
2027 2027 for (i = 0; i < apic_nproc; i++) {
2028 2028 if (apic_cpu_in_range(i) &&
2029 2029 !(apic_redist_cpu_skip & (1 << i)) &&
2030 2030 (apic_cpus[i].aci_status & APIC_CPU_INTR_ENABLE)) {
2031 2031
2032 2032 cpu_infop = &apic_cpus[i];
2033 2033 /*
2034 2034 * If no unbound interrupts or only 1 total on this
2035 2035 * CPU, skip
2036 2036 */
2037 2037 if (!cpu_infop->aci_temp_bound ||
2038 2038 (cpu_infop->aci_bound + cpu_infop->aci_temp_bound)
2039 2039 == 1) {
2040 2040 apic_redist_cpu_skip |= 1 << i;
2041 2041 continue;
2042 2042 }
2043 2043
2044 2044 busy = cpu_infop->aci_busy;
2045 2045 average_busy += busy;
2046 2046 cpus_online++;
2047 2047 if (max_busy < busy) {
2048 2048 max_busy = busy;
2049 2049 busiest_cpu = i;
2050 2050 }
2051 2051 if (min_free > busy) {
2052 2052 min_free = busy;
2053 2053 most_free_cpu = i;
2054 2054 }
2055 2055 if (busy > apic_int_busy_mark) {
2056 2056 cpu_busy |= 1 << i;
2057 2057 } else {
2058 2058 if (busy < apic_int_free_mark)
2059 2059 cpu_free |= 1 << i;
2060 2060 }
2061 2061 }
2062 2062 }
2063 2063 if ((cpu_busy && cpu_free) ||
2064 2064 (max_busy >= (min_free + apic_diff_for_redistribution))) {
2065 2065
2066 2066 apic_num_imbalance++;
2067 2067 #ifdef DEBUG
2068 2068 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
2069 2069 prom_printf(
2070 2070 "redistribute busy=%x free=%x max=%x min=%x",
2071 2071 cpu_busy, cpu_free, max_busy, min_free);
2072 2072 }
2073 2073 #endif /* DEBUG */
2074 2074
2075 2075
2076 2076 average_busy /= cpus_online;
2077 2077
2078 2078 diff = max_busy - average_busy;
2079 2079 min_busy = max_busy; /* start with the max possible value */
2080 2080 max_busy = 0;
2081 2081 min_busy_irq = max_busy_irq = NULL;
2082 2082 i = apic_min_device_irq;
2083 2083 for (; i <= apic_max_device_irq; i++) {
2084 2084 apic_irq_t *irq_ptr;
2085 2085 /* Change to linked list per CPU ? */
2086 2086 if ((irq_ptr = apic_irq_table[i]) == NULL)
2087 2087 continue;
2088 2088 /* Check for irq_busy & decide which one to move */
2089 2089 /* Also zero them for next round */
2090 2090 if ((irq_ptr->airq_temp_cpu == busiest_cpu) &&
2091 2091 irq_ptr->airq_busy) {
2092 2092 if (irq_ptr->airq_busy < diff) {
2093 2093 /*
2094 2094 * Check for least busy CPU,
2095 2095 * best fit or what ?
2096 2096 */
2097 2097 if (max_busy < irq_ptr->airq_busy) {
2098 2098 /*
2099 2099 * Most busy within the
2100 2100 * required differential
2101 2101 */
2102 2102 max_busy = irq_ptr->airq_busy;
2103 2103 max_busy_irq = irq_ptr;
2104 2104 }
2105 2105 } else {
2106 2106 if (min_busy > irq_ptr->airq_busy) {
2107 2107 /*
2108 2108 * least busy, but more than
2109 2109 * the reqd diff
2110 2110 */
2111 2111 if (min_busy <
2112 2112 (diff + average_busy -
2113 2113 min_free)) {
2114 2114 /*
2115 2115 * Making sure new cpu
2116 2116 * will not end up
2117 2117 * worse
2118 2118 */
2119 2119 min_busy =
2120 2120 irq_ptr->airq_busy;
2121 2121
2122 2122 min_busy_irq = irq_ptr;
2123 2123 }
2124 2124 }
2125 2125 }
2126 2126 }
2127 2127 irq_ptr->airq_busy = 0;
2128 2128 }
2129 2129
2130 2130 if (max_busy_irq != NULL) {
2131 2131 #ifdef DEBUG
2132 2132 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
2133 2133 prom_printf("rebinding %x to %x",
2134 2134 max_busy_irq->airq_vector, most_free_cpu);
2135 2135 }
2136 2136 #endif /* DEBUG */
2137 2137 iflag = intr_clear();
2138 2138 if (lock_try(&apic_ioapic_lock)) {
2139 2139 if (apic_rebind_all(max_busy_irq,
2140 2140 most_free_cpu) == 0) {
2141 2141 /* Make change permenant */
2142 2142 max_busy_irq->airq_cpu =
2143 2143 (uint32_t)most_free_cpu;
2144 2144 }
2145 2145 lock_clear(&apic_ioapic_lock);
2146 2146 }
2147 2147 intr_restore(iflag);
2148 2148
2149 2149 } else if (min_busy_irq != NULL) {
2150 2150 #ifdef DEBUG
2151 2151 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
2152 2152 prom_printf("rebinding %x to %x",
2153 2153 min_busy_irq->airq_vector, most_free_cpu);
2154 2154 }
2155 2155 #endif /* DEBUG */
2156 2156
2157 2157 iflag = intr_clear();
2158 2158 if (lock_try(&apic_ioapic_lock)) {
2159 2159 if (apic_rebind_all(min_busy_irq,
2160 2160 most_free_cpu) == 0) {
2161 2161 /* Make change permenant */
2162 2162 min_busy_irq->airq_cpu =
2163 2163 (uint32_t)most_free_cpu;
2164 2164 }
2165 2165 lock_clear(&apic_ioapic_lock);
2166 2166 }
2167 2167 intr_restore(iflag);
2168 2168
2169 2169 } else {
2170 2170 if (cpu_busy != (1 << busiest_cpu)) {
2171 2171 apic_redist_cpu_skip |= 1 << busiest_cpu;
2172 2172 /*
2173 2173 * We leave cpu_skip set so that next time we
2174 2174 * can choose another cpu
2175 2175 */
2176 2176 }
2177 2177 }
2178 2178 apic_num_rebind++;
2179 2179 } else {
2180 2180 /*
2181 2181 * found nothing. Could be that we skipped over valid CPUs
2182 2182 * or we have balanced everything. If we had a variable
2183 2183 * ticks_for_redistribution, it could be increased here.
2184 2184 * apic_int_busy, int_free etc would also need to be
2185 2185 * changed.
2186 2186 */
2187 2187 if (apic_redist_cpu_skip)
2188 2188 apic_redist_cpu_skip = 0;
2189 2189 }
2190 2190 for (i = 0; i < apic_nproc; i++) {
2191 2191 if (apic_cpu_in_range(i)) {
2192 2192 apic_cpus[i].aci_busy = 0;
2193 2193 }
2194 2194 }
2195 2195 }
2196 2196
2197 2197 void
2198 2198 apic_cleanup_busy(void)
2199 2199 {
2200 2200 int i;
2201 2201 apic_irq_t *irq_ptr;
2202 2202
2203 2203 for (i = 0; i < apic_nproc; i++) {
2204 2204 if (apic_cpu_in_range(i)) {
2205 2205 apic_cpus[i].aci_busy = 0;
2206 2206 }
2207 2207 }
2208 2208
2209 2209 for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) {
2210 2210 if ((irq_ptr = apic_irq_table[i]) != NULL)
2211 2211 irq_ptr->airq_busy = 0;
2212 2212 }
2213 2213 }
|
↓ open down ↓ |
1647 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX