Print this page
patch use-x2apic-feature
patch apic-simplify
patch apic-task-reg-write-dup
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/i86pc/io/pcplusmp/apic_regops.c
+++ new/usr/src/uts/i86pc/io/pcplusmp/apic_regops.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 #include <sys/cpuvar.h>
27 27 #include <sys/psm.h>
28 28 #include <sys/archsystm.h>
29 29 #include <sys/apic.h>
30 30 #include <sys/sunddi.h>
31 31 #include <sys/ddi_impldefs.h>
32 32 #include <sys/mach_intr.h>
33 33 #include <sys/sysmacros.h>
34 34 #include <sys/trap.h>
35 35 #include <sys/x86_archext.h>
36 36 #include <sys/privregs.h>
37 37 #include <sys/psm_common.h>
38 38
39 39 /* Function prototypes of local apic and X2APIC */
40 40 static uint64_t local_apic_read(uint32_t reg);
41 41 static void local_apic_write(uint32_t reg, uint64_t value);
42 42 static int get_local_apic_pri(void);
43 43 static void local_apic_write_task_reg(uint64_t value);
44 44 static void local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
45 45 static uint64_t local_x2apic_read(uint32_t msr);
46 46 static void local_x2apic_write(uint32_t msr, uint64_t value);
47 47 static int get_local_x2apic_pri(void);
48 48 static void local_x2apic_write_task_reg(uint64_t value);
49 49 static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
50 50
51 51 /*
52 52 * According to the X2APIC specification:
53 53 *
54 54 * xAPIC global enable X2APIC enable Description
55 55 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10])
56 56 * -----------------------------------------------------------
57 57 * 0 0 APIC is disabled
58 58 * 0 1 Invalid
59 59 * 1 0 APIC is enabled in xAPIC mode
60 60 * 1 1 APIC is enabled in X2APIC mode
61 61 * -----------------------------------------------------------
62 62 */
63 63 int x2apic_enable = 1;
64 64 int apic_mode = LOCAL_APIC; /* Default mode is Local APIC */
65 65
66 66 /* Uses MMIO (Memory Mapped IO) */
67 67 static apic_reg_ops_t local_apic_regs_ops = {
68 68 local_apic_read,
69 69 local_apic_write,
70 70 get_local_apic_pri,
71 71 local_apic_write_task_reg,
72 72 local_apic_write_int_cmd,
73 73 apic_send_EOI,
74 74 };
75 75
76 76 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */
77 77 static apic_reg_ops_t x2apic_regs_ops = {
78 78 local_x2apic_read,
79 79 local_x2apic_write,
80 80 get_local_x2apic_pri,
81 81 local_x2apic_write_task_reg,
82 82 local_x2apic_write_int_cmd,
83 83 apic_send_EOI,
84 84 };
85 85
86 86 int apic_have_32bit_cr8 = 0;
↓ open down ↓ |
86 lines elided |
↑ open up ↑ |
87 87
88 88 /* The default ops is local APIC (Memory Mapped IO) */
89 89 apic_reg_ops_t *apic_reg_ops = &local_apic_regs_ops;
90 90
91 91 /*
92 92 * APIC register ops related data sturctures and functions.
93 93 */
94 94 void apic_send_EOI();
95 95 void apic_send_directed_EOI(uint32_t irq);
96 96
97 -#define X2APIC_CPUID_BIT 21
98 97 #define X2APIC_ENABLE_BIT 10
99 98
100 99 /*
101 100 * Local APIC Implementation
102 101 */
103 102 static uint64_t
104 103 local_apic_read(uint32_t reg)
105 104 {
106 105 return ((uint32_t)apicadr[reg]);
107 106 }
108 107
109 108 static void
110 109 local_apic_write(uint32_t reg, uint64_t value)
111 110 {
112 - apicadr[reg] = (uint32_t)value;
111 + LOCAL_APIC_WRITE_REG(reg, (uint32_t)value);
113 112 }
114 113
115 114 static int
116 115 get_local_apic_pri(void)
117 116 {
118 117 #if defined(__amd64)
119 118 return ((int)getcr8());
120 119 #else
121 120 if (apic_have_32bit_cr8)
122 121 return ((int)getcr8());
123 122 return (apicadr[APIC_TASK_REG]);
124 123 #endif
125 124 }
↓ open down ↓ |
3 lines elided |
↑ open up ↑ |
126 125
127 126 static void
128 127 local_apic_write_task_reg(uint64_t value)
129 128 {
130 129 #if defined(__amd64)
131 130 setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
132 131 #else
133 132 if (apic_have_32bit_cr8)
134 133 setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
135 134 else
136 - apicadr[APIC_TASK_REG] = (uint32_t)value;
135 + LOCAL_APIC_WRITE_REG(APIC_TASK_REG, (uint32_t)value);
137 136 #endif
138 137 }
139 138
140 139 static void
141 140 local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
142 141 {
143 142 apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET;
144 143 apicadr[APIC_INT_CMD1] = cmd1;
145 144 }
146 145
147 146 /*
148 147 * X2APIC Implementation.
149 148 */
150 149 static uint64_t
151 150 local_x2apic_read(uint32_t msr)
152 151 {
153 152 uint64_t i;
154 153
155 154 i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff);
156 155 return (i);
157 156 }
158 157
159 158 static void
160 159 local_x2apic_write(uint32_t msr, uint64_t value)
161 160 {
162 161 uint64_t tmp;
163 162
164 163 if (msr != APIC_EOI_REG) {
165 164 tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2));
166 165 tmp = (tmp & 0xffffffff00000000) | value;
167 166 } else {
168 167 tmp = 0;
169 168 }
170 169
171 170 wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp);
172 171 }
173 172
174 173 static int
175 174 get_local_x2apic_pri(void)
176 175 {
177 176 return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2)));
178 177 }
179 178
180 179 static void
181 180 local_x2apic_write_task_reg(uint64_t value)
182 181 {
183 182 X2APIC_WRITE(APIC_TASK_REG, value);
184 183 }
185 184
186 185 static void
187 186 local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
188 187 {
189 188 wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)),
190 189 (((uint64_t)cpu_id << 32) | cmd1));
191 190 }
192 191
193 192 /*ARGSUSED*/
194 193 void
195 194 apic_send_EOI(uint32_t irq)
196 195 {
197 196 apic_reg_ops->apic_write(APIC_EOI_REG, 0);
198 197 }
199 198
200 199 /*
201 200 * Support for Directed EOI capability is available in both the xAPIC
202 201 * and x2APIC mode.
203 202 */
204 203 void
205 204 apic_send_directed_EOI(uint32_t irq)
206 205 {
207 206 uchar_t ioapicindex;
208 207 uchar_t vector;
209 208 apic_irq_t *apic_irq;
210 209 short intr_index;
211 210
212 211 /*
213 212 * Following the EOI to the local APIC unit, perform a directed
214 213 * EOI to the IOxAPIC generating the interrupt by writing to its
215 214 * EOI register.
216 215 *
217 216 * A broadcast EOI is not generated.
218 217 */
219 218 apic_reg_ops->apic_write(APIC_EOI_REG, 0);
220 219
221 220 apic_irq = apic_irq_table[irq];
222 221 while (apic_irq) {
223 222 intr_index = apic_irq->airq_mps_intr_index;
224 223 if (intr_index == ACPI_INDEX || intr_index >= 0) {
225 224 ioapicindex = apic_irq->airq_ioapicindex;
↓ open down ↓ |
79 lines elided |
↑ open up ↑ |
226 225 vector = apic_irq->airq_vector;
227 226 ioapic_write_eoi(ioapicindex, vector);
228 227 }
229 228 apic_irq = apic_irq->airq_next;
230 229 }
231 230 }
232 231
233 232 int
234 233 apic_detect_x2apic(void)
235 234 {
236 - struct cpuid_regs cp;
237 -
238 235 if (x2apic_enable == 0)
239 236 return (0);
240 237
241 - cp.cp_eax = 1;
242 - (void) __cpuid_insn(&cp);
243 -
244 - return ((cp.cp_ecx & (0x1 << X2APIC_CPUID_BIT)) ? 1 : 0);
238 + return is_x86_feature(x86_featureset, X86FSET_X2APIC);
245 239 }
246 240
247 241 void
248 242 apic_enable_x2apic(void)
249 243 {
250 244 uint64_t apic_base_msr;
251 245
252 246 if (apic_local_mode() == LOCAL_X2APIC) {
253 247 /* BIOS apparently has enabled X2APIC */
254 248 if (apic_mode != LOCAL_X2APIC)
255 249 x2apic_update_psm();
256 250 return;
257 251 }
258 252
259 253 /*
260 254 * This is the first time we are enabling X2APIC on this CPU
261 255 */
262 256 apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
263 257 apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT);
264 258 wrmsr(REG_APIC_BASE_MSR, apic_base_msr);
265 259
266 260 if (apic_mode != LOCAL_X2APIC)
267 261 x2apic_update_psm();
268 262 }
269 263
270 264 /*
271 265 * Determine which mode the current CPU is in. See the table above.
272 266 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10])
273 267 */
274 268 int
275 269 apic_local_mode(void)
276 270 {
277 271 uint64_t apic_base_msr;
278 272 int bit = ((0x1 << (X2APIC_ENABLE_BIT + 1)) |
279 273 (0x1 << X2APIC_ENABLE_BIT));
280 274
281 275 apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
282 276
283 277 if ((apic_base_msr & bit) == bit)
284 278 return (LOCAL_X2APIC);
285 279 else
286 280 return (LOCAL_APIC);
287 281 }
288 282
289 283 void
290 284 apic_set_directed_EOI_handler()
291 285 {
292 286 apic_reg_ops->apic_send_eoi = apic_send_directed_EOI;
293 287 }
294 288
295 289 int
296 290 apic_directed_EOI_supported()
297 291 {
298 292 uint32_t ver;
299 293
300 294 ver = apic_reg_ops->apic_read(APIC_VERS_REG);
301 295 if (ver & APIC_DIRECTED_EOI_BIT)
302 296 return (1);
303 297
304 298 return (0);
305 299 }
306 300
307 301 /*
308 302 * Change apic_reg_ops depending upon the apic_mode.
309 303 */
310 304 void
311 305 apic_change_ops()
312 306 {
313 307 if (apic_mode == LOCAL_APIC)
314 308 apic_reg_ops = &local_apic_regs_ops;
315 309 else if (apic_mode == LOCAL_X2APIC)
316 310 apic_reg_ops = &x2apic_regs_ops;
317 311 }
318 312
319 313 /*
320 314 * Generates an interprocessor interrupt to another CPU when X2APIC mode is
321 315 * enabled.
322 316 */
323 317 void
324 318 x2apic_send_ipi(int cpun, int ipl)
325 319 {
326 320 int vector;
327 321 ulong_t flag;
328 322
329 323 ASSERT(apic_mode == LOCAL_X2APIC);
330 324
331 325 /*
332 326 * With X2APIC, Intel relaxed the semantics of the
333 327 * WRMSR instruction such that references to the X2APIC
334 328 * MSR registers are no longer serializing instructions.
335 329 * The code that initiates IPIs assumes that some sort
336 330 * of memory serialization occurs. The old APIC code
337 331 * did a write to uncachable memory mapped registers.
338 332 * Any reference to uncached memory is a serializing
339 333 * operation. To mimic those semantics here, we do an
340 334 * atomic operation, which translates to a LOCK OR instruction,
341 335 * which is serializing.
342 336 */
343 337 atomic_or_ulong(&flag, 1);
344 338
345 339 vector = apic_resv_vector[ipl];
346 340
347 341 flag = intr_clear();
348 342
349 343 /*
350 344 * According to X2APIC specification in section '2.3.5.1' of
351 345 * Interrupt Command Register Semantics, the semantics of
352 346 * programming Interrupt Command Register to dispatch an interrupt
353 347 * is simplified. A single MSR write to the 64-bit ICR is required
354 348 * for dispatching an interrupt. Specifically with the 64-bit MSR
355 349 * interface to ICR, system software is not required to check the
356 350 * status of the delivery status bit prior to writing to the ICR
357 351 * to send an IPI. With the removal of the Delivery Status bit,
358 352 * system software no longer has a reason to read the ICR. It remains
359 353 * readable only to aid in debugging.
360 354 */
361 355 #ifdef DEBUG
362 356 APIC_AV_PENDING_SET();
363 357 #endif /* DEBUG */
364 358
365 359 if ((cpun == psm_get_cpu_id())) {
366 360 X2APIC_WRITE(X2APIC_SELF_IPI, vector);
367 361 } else {
368 362 apic_reg_ops->apic_write_int_cmd(
369 363 apic_cpus[cpun].aci_local_id, vector);
370 364 }
371 365
372 366 intr_restore(flag);
373 367 }
374 368
375 369 /*
376 370 * Generates IPI to another CPU depending on the local APIC mode.
377 371 * apic_send_ipi() and x2apic_send_ipi() depends on the configured
378 372 * mode of the local APIC, but that may not match the actual mode
379 373 * early in CPU startup.
380 374 *
381 375 * Any changes made to this routine must be accompanied by similar
382 376 * changes to apic_send_ipi().
383 377 */
384 378 void
385 379 apic_common_send_ipi(int cpun, int ipl)
386 380 {
387 381 int vector;
388 382 ulong_t flag;
389 383 int mode = apic_local_mode();
390 384
391 385 if (mode == LOCAL_X2APIC) {
392 386 x2apic_send_ipi(cpun, ipl);
393 387 return;
394 388 }
395 389
396 390 ASSERT(mode == LOCAL_APIC);
397 391
398 392 vector = apic_resv_vector[ipl];
399 393 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
400 394 flag = intr_clear();
401 395 while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING)
402 396 apic_ret();
403 397 local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
404 398 vector);
405 399 intr_restore(flag);
406 400 }
↓ open down ↓ |
152 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX