Print this page
4804 apix & pcplusmp are nearly warning free already
Tentatively Reviewed by: Robert Mustacchi <rm@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/io/hpet_acpi.c
+++ new/usr/src/uts/i86pc/io/hpet_acpi.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 */
24 24
25 25 #include <sys/hpet_acpi.h>
26 26 #include <sys/hpet.h>
27 27 #include <sys/bitmap.h>
28 28 #include <sys/inttypes.h>
29 29 #include <sys/time.h>
30 30 #include <sys/sunddi.h>
31 31 #include <sys/ksynch.h>
32 32 #include <sys/apic.h>
33 33 #include <sys/callb.h>
34 34 #include <sys/clock.h>
35 35 #include <sys/archsystm.h>
36 36 #include <sys/cpupart.h>
37 37
38 38 static int hpet_init_proxy(int *hpet_vect, iflag_t *hpet_flags);
39 39 static boolean_t hpet_install_proxy(void);
40 40 static boolean_t hpet_callback(int code);
41 41 static boolean_t hpet_cpr(int code);
42 42 static boolean_t hpet_resume(void);
43 43 static void hpet_cst_callback(uint32_t code);
44 44 static boolean_t hpet_deep_idle_config(int code);
45 45 static int hpet_validate_table(ACPI_TABLE_HPET *hpet_table);
46 46 static boolean_t hpet_checksum_table(unsigned char *table, unsigned int len);
|
↓ open down ↓ |
46 lines elided |
↑ open up ↑ |
47 47 static void *hpet_memory_map(ACPI_TABLE_HPET *hpet_table);
48 48 static int hpet_start_main_counter(hpet_info_t *hip);
49 49 static int hpet_stop_main_counter(hpet_info_t *hip);
50 50 static uint64_t hpet_read_main_counter_value(hpet_info_t *hip);
51 51 static uint64_t hpet_set_leg_rt_cnf(hpet_info_t *hip, uint32_t new_value);
52 52 static uint64_t hpet_read_gen_cap(hpet_info_t *hip);
53 53 static uint64_t hpet_read_gen_config(hpet_info_t *hip);
54 54 static uint64_t hpet_read_gen_intrpt_stat(hpet_info_t *hip);
55 55 static uint64_t hpet_read_timer_N_config(hpet_info_t *hip, uint_t n);
56 56 static hpet_TN_conf_cap_t hpet_convert_timer_N_config(uint64_t conf);
57 -/* LINTED E_STATIC_UNUSED */
58 -static uint64_t hpet_read_timer_N_comp(hpet_info_t *hip, uint_t n);
59 -/* LINTED E_STATIC_UNUSED */
60 -static void hpet_write_gen_cap(hpet_info_t *hip, uint64_t l);
61 57 static void hpet_write_gen_config(hpet_info_t *hip, uint64_t l);
62 58 static void hpet_write_gen_intrpt_stat(hpet_info_t *hip, uint64_t l);
63 59 static void hpet_write_timer_N_config(hpet_info_t *hip, uint_t n, uint64_t l);
64 60 static void hpet_write_timer_N_comp(hpet_info_t *hip, uint_t n, uint64_t l);
65 61 static void hpet_disable_timer(hpet_info_t *hip, uint32_t timer_n);
66 62 static void hpet_enable_timer(hpet_info_t *hip, uint32_t timer_n);
67 -/* LINTED E_STATIC_UNUSED */
68 -static void hpet_write_main_counter_value(hpet_info_t *hip, uint64_t l);
69 63 static int hpet_get_IOAPIC_intr_capable_timer(hpet_info_t *hip);
70 64 static int hpet_timer_available(uint32_t allocated_timers, uint32_t n);
71 65 static void hpet_timer_alloc(uint32_t *allocated_timers, uint32_t n);
72 66 static void hpet_timer_set_up(hpet_info_t *hip, uint32_t timer_n,
73 67 uint32_t interrupt);
74 68 static uint_t hpet_isr(char *arg);
75 69 static uint32_t hpet_install_interrupt_handler(uint_t (*func)(char *),
76 70 int vector);
77 71 static void hpet_uninstall_interrupt_handler(void);
78 72 static void hpet_expire_all(void);
79 73 static boolean_t hpet_guaranteed_schedule(hrtime_t required_wakeup_time);
80 74 static boolean_t hpet_use_hpet_timer(hrtime_t *expire);
81 75 static void hpet_use_lapic_timer(hrtime_t expire);
82 76 static void hpet_init_proxy_data(void);
83 77
84 78 /*
85 79 * hpet_state_lock is used to synchronize disabling/enabling deep c-states
86 80 * and to synchronize suspend/resume.
87 81 */
88 82 static kmutex_t hpet_state_lock;
89 83 static struct hpet_state {
90 84 boolean_t proxy_installed; /* CBE proxy interrupt setup */
91 85 boolean_t cpr; /* currently in CPR */
92 86 boolean_t cpu_deep_idle; /* user enable/disable */
93 87 boolean_t uni_cstate; /* disable if only one cstate */
94 88 } hpet_state = { B_FALSE, B_FALSE, B_TRUE, B_TRUE};
95 89
96 90 uint64_t hpet_spin_check = HPET_SPIN_CHECK;
97 91 uint64_t hpet_spin_timeout = HPET_SPIN_TIMEOUT;
98 92 uint64_t hpet_idle_spin_timeout = HPET_SPIN_TIMEOUT;
99 93 uint64_t hpet_isr_spin_timeout = HPET_SPIN_TIMEOUT;
100 94
101 95 static kmutex_t hpet_proxy_lock; /* lock for lAPIC proxy data */
102 96 /*
103 97 * hpet_proxy_users is a per-cpu array.
104 98 */
105 99 static hpet_proxy_t *hpet_proxy_users; /* one per CPU */
106 100
107 101
108 102 ACPI_TABLE_HPET *hpet_table; /* ACPI HPET table */
109 103 hpet_info_t hpet_info; /* Human readable Information */
110 104
111 105 /*
112 106 * Provide HPET access from unix.so.
113 107 * Set up pointers to access symbols in pcplusmp.
114 108 */
115 109 static void
116 110 hpet_establish_hooks(void)
117 111 {
118 112 hpet.install_proxy = &hpet_install_proxy;
119 113 hpet.callback = &hpet_callback;
120 114 hpet.use_hpet_timer = &hpet_use_hpet_timer;
121 115 hpet.use_lapic_timer = &hpet_use_lapic_timer;
122 116 }
123 117
124 118 /*
125 119 * Get the ACPI "HPET" table.
126 120 * acpi_probe() calls this function from mp_startup before drivers are loaded.
127 121 * acpi_probe() verified the system is using ACPI before calling this.
128 122 *
129 123 * There may be more than one ACPI HPET table (Itanium only?).
130 124 * Intel's HPET spec defines each timer block to have up to 32 counters and
131 125 * be 1024 bytes long. There can be more than one timer block of 32 counters.
132 126 * Each timer block would have an additional ACPI HPET table.
133 127 * Typical x86 systems today only have 1 HPET with 3 counters.
134 128 * On x86 we only consume HPET table "1" for now.
135 129 */
136 130 int
137 131 hpet_acpi_init(int *hpet_vect, iflag_t *hpet_flags)
138 132 {
139 133 extern hrtime_t tsc_read(void);
140 134 extern int idle_cpu_no_deep_c;
141 135 extern int cpuid_deep_cstates_supported(void);
142 136 void *la;
143 137 uint64_t ret;
144 138 uint_t num_timers;
145 139 uint_t ti;
146 140
147 141 (void) memset(&hpet_info, 0, sizeof (hpet_info));
148 142 hpet.supported = HPET_NO_SUPPORT;
149 143
150 144 if (idle_cpu_no_deep_c)
151 145 return (DDI_FAILURE);
152 146
153 147 if (!cpuid_deep_cstates_supported())
154 148 return (DDI_FAILURE);
155 149
156 150 hpet_establish_hooks();
157 151
158 152 /*
159 153 * Get HPET ACPI table 1.
160 154 */
161 155 if (ACPI_FAILURE(AcpiGetTable(ACPI_SIG_HPET, HPET_TABLE_1,
162 156 (ACPI_TABLE_HEADER **)&hpet_table))) {
163 157 cmn_err(CE_NOTE, "!hpet_acpi: unable to get ACPI HPET table");
164 158 return (DDI_FAILURE);
165 159 }
166 160
167 161 if (hpet_validate_table(hpet_table) != AE_OK) {
168 162 cmn_err(CE_NOTE, "!hpet_acpi: invalid HPET table");
169 163 return (DDI_FAILURE);
170 164 }
171 165
172 166 la = hpet_memory_map(hpet_table);
173 167 if (la == NULL) {
174 168 cmn_err(CE_NOTE, "!hpet_acpi: memory map HPET failed");
175 169 return (DDI_FAILURE);
176 170 }
177 171 hpet_info.logical_address = la;
178 172
179 173 ret = hpet_read_gen_cap(&hpet_info);
180 174 hpet_info.gen_cap.counter_clk_period = HPET_GCAP_CNTR_CLK_PERIOD(ret);
181 175 hpet_info.gen_cap.vendor_id = HPET_GCAP_VENDOR_ID(ret);
182 176 hpet_info.gen_cap.leg_route_cap = HPET_GCAP_LEG_ROUTE_CAP(ret);
183 177 hpet_info.gen_cap.count_size_cap = HPET_GCAP_CNT_SIZE_CAP(ret);
184 178 /*
185 179 * Hardware contains the last timer's number.
186 180 * Add 1 to get the number of timers.
187 181 */
188 182 hpet_info.gen_cap.num_tim_cap = HPET_GCAP_NUM_TIM_CAP(ret) + 1;
189 183 hpet_info.gen_cap.rev_id = HPET_GCAP_REV_ID(ret);
190 184
191 185 if (hpet_info.gen_cap.counter_clk_period > HPET_MAX_CLK_PERIOD) {
192 186 cmn_err(CE_NOTE, "!hpet_acpi: COUNTER_CLK_PERIOD 0x%lx > 0x%lx",
193 187 (long)hpet_info.gen_cap.counter_clk_period,
194 188 (long)HPET_MAX_CLK_PERIOD);
195 189 return (DDI_FAILURE);
196 190 }
197 191
198 192 num_timers = (uint_t)hpet_info.gen_cap.num_tim_cap;
199 193 if ((num_timers < 3) || (num_timers > 32)) {
200 194 cmn_err(CE_NOTE, "!hpet_acpi: invalid number of HPET timers "
201 195 "%lx", (long)num_timers);
202 196 return (DDI_FAILURE);
203 197 }
204 198 hpet_info.timer_n_config = (hpet_TN_conf_cap_t *)kmem_zalloc(
205 199 num_timers * sizeof (uint64_t), KM_SLEEP);
206 200
207 201 ret = hpet_read_gen_config(&hpet_info);
208 202 hpet_info.gen_config.leg_rt_cnf = HPET_GCFR_LEG_RT_CNF_BITX(ret);
209 203 hpet_info.gen_config.enable_cnf = HPET_GCFR_ENABLE_CNF_BITX(ret);
210 204
211 205 /*
212 206 * Solaris does not use the HPET Legacy Replacement Route capabilities.
213 207 * This feature has been off by default on test systems.
214 208 * The HPET spec does not specify if Legacy Replacement Route is
215 209 * on or off by default, so we explicitely set it off here.
216 210 * It should not matter which mode the HPET is in since we use
217 211 * the first available non-legacy replacement timer: timer 2.
218 212 */
219 213 (void) hpet_set_leg_rt_cnf(&hpet_info, 0);
220 214
221 215 ret = hpet_read_gen_config(&hpet_info);
222 216 hpet_info.gen_config.leg_rt_cnf = HPET_GCFR_LEG_RT_CNF_BITX(ret);
223 217 hpet_info.gen_config.enable_cnf = HPET_GCFR_ENABLE_CNF_BITX(ret);
224 218
225 219 hpet_info.gen_intrpt_stat = hpet_read_gen_intrpt_stat(&hpet_info);
226 220 hpet_info.main_counter_value = hpet_read_main_counter_value(&hpet_info);
227 221
228 222 for (ti = 0; ti < num_timers; ++ti) {
229 223 ret = hpet_read_timer_N_config(&hpet_info, ti);
230 224 /*
231 225 * Make sure no timers are enabled (think fast reboot or
232 226 * virtual hardware).
233 227 */
234 228 if (ret & HPET_TIMER_N_INT_ENB_CNF_BIT) {
235 229 hpet_disable_timer(&hpet_info, ti);
236 230 ret &= ~HPET_TIMER_N_INT_ENB_CNF_BIT;
237 231 }
238 232
239 233 hpet_info.timer_n_config[ti] = hpet_convert_timer_N_config(ret);
240 234 }
241 235
242 236 /*
243 237 * Be aware the Main Counter may need to be initialized in the future
244 238 * if it is used for more than just Deep C-State support.
245 239 * The HPET's Main Counter does not need to be initialize to a specific
246 240 * value before starting it for use to wake up CPUs from Deep C-States.
247 241 */
248 242 if (hpet_start_main_counter(&hpet_info) != AE_OK) {
249 243 cmn_err(CE_NOTE, "!hpet_acpi: hpet_start_main_counter failed");
250 244 return (DDI_FAILURE);
251 245 }
252 246
253 247 hpet_info.period = hpet_info.gen_cap.counter_clk_period;
254 248 /*
255 249 * Read main counter twice to record HPET latency for debugging.
256 250 */
257 251 hpet_info.tsc[0] = tsc_read();
258 252 hpet_info.hpet_main_counter_reads[0] =
259 253 hpet_read_main_counter_value(&hpet_info);
260 254 hpet_info.tsc[1] = tsc_read();
261 255 hpet_info.hpet_main_counter_reads[1] =
262 256 hpet_read_main_counter_value(&hpet_info);
263 257 hpet_info.tsc[2] = tsc_read();
264 258
265 259 ret = hpet_read_gen_config(&hpet_info);
266 260 hpet_info.gen_config.leg_rt_cnf = HPET_GCFR_LEG_RT_CNF_BITX(ret);
267 261 hpet_info.gen_config.enable_cnf = HPET_GCFR_ENABLE_CNF_BITX(ret);
268 262
269 263 /*
270 264 * HPET main counter reads are supported now.
271 265 */
272 266 hpet.supported = HPET_TIMER_SUPPORT;
273 267
274 268 return (hpet_init_proxy(hpet_vect, hpet_flags));
275 269 }
276 270
277 271 void
278 272 hpet_acpi_fini(void)
279 273 {
280 274 if (hpet.supported == HPET_NO_SUPPORT)
281 275 return;
282 276 if (hpet.supported >= HPET_TIMER_SUPPORT)
283 277 (void) hpet_stop_main_counter(&hpet_info);
284 278 if (hpet.supported > HPET_TIMER_SUPPORT)
285 279 hpet_disable_timer(&hpet_info, hpet_info.cstate_timer.timer);
286 280 }
287 281
288 282 /*
289 283 * Do initial setup to use a HPET timer as a proxy for Deep C-state stalled
290 284 * LAPIC Timers. Get a free HPET timer that supports I/O APIC routed interrupt.
291 285 * Setup data to handle the timer's ISR, and add the timer's interrupt.
292 286 *
293 287 * The ddi cannot be use to allocate the HPET timer's interrupt.
294 288 * ioapic_init_intr() in mp_platform_common() later sets up the I/O APIC
295 289 * to handle the HPET timer's interrupt.
296 290 *
297 291 * Note: FSB (MSI) interrupts are not currently supported by Intel HPETs as of
298 292 * ICH9. The HPET spec allows for MSI. In the future MSI may be prefered.
299 293 */
300 294 static int
301 295 hpet_init_proxy(int *hpet_vect, iflag_t *hpet_flags)
302 296 {
303 297 if (hpet_get_IOAPIC_intr_capable_timer(&hpet_info) == -1) {
304 298 cmn_err(CE_WARN, "!hpet_acpi: get ioapic intr failed.");
305 299 return (DDI_FAILURE);
306 300 }
307 301
308 302 hpet_init_proxy_data();
309 303
310 304 if (hpet_install_interrupt_handler(&hpet_isr,
311 305 hpet_info.cstate_timer.intr) != AE_OK) {
312 306 cmn_err(CE_WARN, "!hpet_acpi: install interrupt failed.");
313 307 return (DDI_FAILURE);
314 308 }
315 309 *hpet_vect = hpet_info.cstate_timer.intr;
316 310 hpet_flags->intr_el = INTR_EL_LEVEL;
317 311 hpet_flags->intr_po = INTR_PO_ACTIVE_HIGH;
318 312 hpet_flags->bustype = BUS_PCI; /* we *do* conform to PCI */
319 313
320 314 /*
321 315 * Avoid a possibly stuck interrupt by programing the HPET's timer here
322 316 * before the I/O APIC is programmed to handle this interrupt.
323 317 */
324 318 hpet_timer_set_up(&hpet_info, hpet_info.cstate_timer.timer,
325 319 hpet_info.cstate_timer.intr);
326 320
327 321 /*
328 322 * All HPET functionality is supported.
329 323 */
330 324 hpet.supported = HPET_FULL_SUPPORT;
331 325 return (DDI_SUCCESS);
332 326 }
333 327
334 328 /*
335 329 * Called by kernel if it can support Deep C-States.
336 330 */
337 331 static boolean_t
338 332 hpet_install_proxy(void)
339 333 {
340 334 if (hpet_state.proxy_installed == B_TRUE)
341 335 return (B_TRUE);
342 336
343 337 if (hpet.supported != HPET_FULL_SUPPORT)
344 338 return (B_FALSE);
345 339
346 340 hpet_enable_timer(&hpet_info, hpet_info.cstate_timer.timer);
347 341 hpet_state.proxy_installed = B_TRUE;
348 342
349 343 return (B_TRUE);
350 344 }
351 345
352 346 /*
353 347 * Remove the interrupt that was added with add_avintr() in
354 348 * hpet_install_interrupt_handler().
355 349 */
356 350 static void
357 351 hpet_uninstall_interrupt_handler(void)
358 352 {
359 353 rem_avintr(NULL, CBE_HIGH_PIL, (avfunc)&hpet_isr,
360 354 hpet_info.cstate_timer.intr);
361 355 }
362 356
363 357 static int
364 358 hpet_validate_table(ACPI_TABLE_HPET *hpet_table)
365 359 {
366 360 ACPI_TABLE_HEADER *table_header = (ACPI_TABLE_HEADER *)hpet_table;
367 361
368 362 if (table_header->Length != sizeof (ACPI_TABLE_HPET)) {
369 363 cmn_err(CE_WARN, "!hpet_validate_table: Length %lx != sizeof ("
370 364 "ACPI_TABLE_HPET) %lx.",
371 365 (unsigned long)((ACPI_TABLE_HEADER *)hpet_table)->Length,
372 366 (unsigned long)sizeof (ACPI_TABLE_HPET));
373 367 return (AE_ERROR);
374 368 }
375 369
376 370 if (!ACPI_COMPARE_NAME(table_header->Signature, ACPI_SIG_HPET)) {
377 371 cmn_err(CE_WARN, "!hpet_validate_table: Invalid HPET table "
378 372 "signature");
379 373 return (AE_ERROR);
380 374 }
381 375
382 376 if (!hpet_checksum_table((unsigned char *)hpet_table,
383 377 (unsigned int)table_header->Length)) {
384 378 cmn_err(CE_WARN, "!hpet_validate_table: Invalid HPET checksum");
385 379 return (AE_ERROR);
386 380 }
387 381
388 382 /*
389 383 * Sequence should be table number - 1. We are using table 1.
390 384 */
391 385 if (hpet_table->Sequence != HPET_TABLE_1 - 1) {
392 386 cmn_err(CE_WARN, "!hpet_validate_table: Invalid Sequence %lx",
393 387 (long)hpet_table->Sequence);
394 388 return (AE_ERROR);
395 389 }
396 390
397 391 return (AE_OK);
398 392 }
399 393
400 394 static boolean_t
401 395 hpet_checksum_table(unsigned char *table, unsigned int length)
402 396 {
403 397 unsigned char checksum = 0;
404 398 int i;
405 399
406 400 for (i = 0; i < length; ++i, ++table)
407 401 checksum += *table;
408 402
409 403 return (checksum == 0);
410 404 }
411 405
412 406 static void *
413 407 hpet_memory_map(ACPI_TABLE_HPET *hpet_table)
414 408 {
415 409 return (AcpiOsMapMemory(hpet_table->Address.Address, HPET_SIZE));
416 410 }
417 411
418 412 static int
419 413 hpet_start_main_counter(hpet_info_t *hip)
420 414 {
421 415 uint64_t *gcr_ptr;
422 416 uint64_t gcr;
423 417
424 418 gcr_ptr = (uint64_t *)HPET_GEN_CONFIG_ADDRESS(hip->logical_address);
425 419 gcr = *gcr_ptr;
426 420
427 421 gcr |= HPET_GCFR_ENABLE_CNF;
428 422 *gcr_ptr = gcr;
429 423 gcr = *gcr_ptr;
430 424
431 425 return (gcr & HPET_GCFR_ENABLE_CNF ? AE_OK : ~AE_OK);
432 426 }
433 427
434 428 static int
435 429 hpet_stop_main_counter(hpet_info_t *hip)
436 430 {
437 431 uint64_t *gcr_ptr;
438 432 uint64_t gcr;
439 433
440 434 gcr_ptr = (uint64_t *)HPET_GEN_CONFIG_ADDRESS(hip->logical_address);
441 435 gcr = *gcr_ptr;
442 436
443 437 gcr &= ~HPET_GCFR_ENABLE_CNF;
444 438 *gcr_ptr = gcr;
445 439 gcr = *gcr_ptr;
446 440
447 441 return (gcr & HPET_GCFR_ENABLE_CNF ? ~AE_OK : AE_OK);
448 442 }
449 443
450 444 /*
451 445 * Set the Legacy Replacement Route bit.
452 446 * This should be called before setting up timers.
453 447 * The HPET specification is silent regarding setting this after timers are
454 448 * programmed.
455 449 */
456 450 static uint64_t
457 451 hpet_set_leg_rt_cnf(hpet_info_t *hip, uint32_t new_value)
458 452 {
459 453 uint64_t gen_conf = hpet_read_gen_config(hip);
460 454
461 455 switch (new_value) {
462 456 case 0:
463 457 gen_conf &= ~HPET_GCFR_LEG_RT_CNF;
464 458 break;
465 459
466 460 case HPET_GCFR_LEG_RT_CNF:
467 461 gen_conf |= HPET_GCFR_LEG_RT_CNF;
468 462 break;
469 463
470 464 default:
471 465 ASSERT(new_value == 0 || new_value == HPET_GCFR_LEG_RT_CNF);
472 466 break;
473 467 }
474 468 hpet_write_gen_config(hip, gen_conf);
475 469 return (gen_conf);
476 470 }
477 471
478 472 static uint64_t
479 473 hpet_read_gen_cap(hpet_info_t *hip)
480 474 {
481 475 return (*(uint64_t *)HPET_GEN_CAP_ADDRESS(hip->logical_address));
482 476 }
483 477
484 478 static uint64_t
485 479 hpet_read_gen_config(hpet_info_t *hip)
486 480 {
487 481 return (*(uint64_t *)
488 482 HPET_GEN_CONFIG_ADDRESS(hip->logical_address));
489 483 }
490 484
491 485 static uint64_t
492 486 hpet_read_gen_intrpt_stat(hpet_info_t *hip)
493 487 {
494 488 hip->gen_intrpt_stat = *(uint64_t *)HPET_GEN_INTR_STAT_ADDRESS(
495 489 hip->logical_address);
496 490 return (hip->gen_intrpt_stat);
497 491 }
498 492
499 493 static uint64_t
500 494 hpet_read_timer_N_config(hpet_info_t *hip, uint_t n)
501 495 {
502 496 uint64_t conf = *(uint64_t *)HPET_TIMER_N_CONF_ADDRESS(
503 497 hip->logical_address, n);
504 498 hip->timer_n_config[n] = hpet_convert_timer_N_config(conf);
505 499 return (conf);
506 500 }
507 501
508 502 static hpet_TN_conf_cap_t
509 503 hpet_convert_timer_N_config(uint64_t conf)
510 504 {
511 505 hpet_TN_conf_cap_t cc = { 0 };
512 506
513 507 cc.int_route_cap = HPET_TIMER_N_INT_ROUTE_CAP(conf);
514 508 cc.fsb_int_del_cap = HPET_TIMER_N_FSB_INT_DEL_CAP(conf);
515 509 cc.fsb_int_en_cnf = HPET_TIMER_N_FSB_EN_CNF(conf);
516 510 cc.int_route_cnf = HPET_TIMER_N_INT_ROUTE_CNF(conf);
517 511 cc.mode32_cnf = HPET_TIMER_N_MODE32_CNF(conf);
518 512 cc.val_set_cnf = HPET_TIMER_N_VAL_SET_CNF(conf);
|
↓ open down ↓ |
440 lines elided |
↑ open up ↑ |
519 513 cc.size_cap = HPET_TIMER_N_SIZE_CAP(conf);
520 514 cc.per_int_cap = HPET_TIMER_N_PER_INT_CAP(conf);
521 515 cc.type_cnf = HPET_TIMER_N_TYPE_CNF(conf);
522 516 cc.int_enb_cnf = HPET_TIMER_N_INT_ENB_CNF(conf);
523 517 cc.int_type_cnf = HPET_TIMER_N_INT_TYPE_CNF(conf);
524 518
525 519 return (cc);
526 520 }
527 521
528 522 static uint64_t
529 -hpet_read_timer_N_comp(hpet_info_t *hip, uint_t n)
530 -{
531 - if (hip->timer_n_config[n].size_cap == 1)
532 - return (*(uint64_t *)
533 - HPET_TIMER_N_COMP_ADDRESS(hip->logical_address, n));
534 - else
535 - return (*(uint32_t *)
536 - HPET_TIMER_N_COMP_ADDRESS(hip->logical_address, n));
537 -}
538 -
539 -static uint64_t
540 523 hpet_read_main_counter_value(hpet_info_t *hip)
541 524 {
542 525 uint64_t value;
543 526 uint32_t *counter;
544 527 uint32_t high1, high2, low;
545 528
546 529 counter = (uint32_t *)HPET_MAIN_COUNTER_ADDRESS(hip->logical_address);
547 530
548 531 /*
549 532 * 32-bit main counters
550 533 */
551 534 if (hip->gen_cap.count_size_cap == 0) {
552 535 value = (uint64_t)*counter;
553 536 hip->main_counter_value = value;
554 537 return (value);
555 538 }
556 539
557 540 /*
558 541 * HPET spec claims a 64-bit read can be split into two 32-bit reads
559 542 * by the hardware connection to the HPET.
560 543 */
561 544 high2 = counter[1];
562 545 do {
563 546 high1 = high2;
|
↓ open down ↓ |
14 lines elided |
↑ open up ↑ |
564 547 low = counter[0];
565 548 high2 = counter[1];
566 549 } while (high2 != high1);
567 550
568 551 value = ((uint64_t)high1 << 32) | low;
569 552 hip->main_counter_value = value;
570 553 return (value);
571 554 }
572 555
573 556 static void
574 -hpet_write_gen_cap(hpet_info_t *hip, uint64_t l)
575 -{
576 - *(uint64_t *)HPET_GEN_CAP_ADDRESS(hip->logical_address) = l;
577 -}
578 -
579 -static void
580 557 hpet_write_gen_config(hpet_info_t *hip, uint64_t l)
581 558 {
582 559 *(uint64_t *)HPET_GEN_CONFIG_ADDRESS(hip->logical_address) = l;
583 560 }
584 561
585 562 static void
586 563 hpet_write_gen_intrpt_stat(hpet_info_t *hip, uint64_t l)
587 564 {
588 565 *(uint64_t *)HPET_GEN_INTR_STAT_ADDRESS(hip->logical_address) = l;
589 566 }
590 567
591 568 static void
592 569 hpet_write_timer_N_config(hpet_info_t *hip, uint_t n, uint64_t l)
593 570 {
594 571 if (hip->timer_n_config[n].size_cap == 1)
595 572 *(uint64_t *)HPET_TIMER_N_CONF_ADDRESS(
596 573 hip->logical_address, n) = l;
597 574 else
598 575 *(uint32_t *)HPET_TIMER_N_CONF_ADDRESS(
599 576 hip->logical_address, n) = (uint32_t)(0xFFFFFFFF & l);
600 577 }
601 578
602 579 static void
603 580 hpet_write_timer_N_comp(hpet_info_t *hip, uint_t n, uint64_t l)
604 581 {
605 582 *(uint64_t *)HPET_TIMER_N_COMP_ADDRESS(hip->logical_address, n) = l;
606 583 }
607 584
608 585 static void
609 586 hpet_disable_timer(hpet_info_t *hip, uint32_t timer_n)
610 587 {
611 588 uint64_t l;
612 589
613 590 l = hpet_read_timer_N_config(hip, timer_n);
614 591 l &= ~HPET_TIMER_N_INT_ENB_CNF_BIT;
615 592 hpet_write_timer_N_config(hip, timer_n, l);
616 593 }
617 594
|
↓ open down ↓ |
28 lines elided |
↑ open up ↑ |
618 595 static void
619 596 hpet_enable_timer(hpet_info_t *hip, uint32_t timer_n)
620 597 {
621 598 uint64_t l;
622 599
623 600 l = hpet_read_timer_N_config(hip, timer_n);
624 601 l |= HPET_TIMER_N_INT_ENB_CNF_BIT;
625 602 hpet_write_timer_N_config(hip, timer_n, l);
626 603 }
627 604
628 -static void
629 -hpet_write_main_counter_value(hpet_info_t *hip, uint64_t l)
630 -{
631 - uint32_t *address;
632 -
633 - /*
634 - * HPET spec 1.0a states main counter register should be halted before
635 - * it is written to.
636 - */
637 - ASSERT(!(hpet_read_gen_config(hip) & HPET_GCFR_ENABLE_CNF));
638 -
639 - if (hip->gen_cap.count_size_cap == 1) {
640 - *(uint64_t *)HPET_MAIN_COUNTER_ADDRESS(hip->logical_address)
641 - = l;
642 - } else {
643 - address = (uint32_t *)HPET_MAIN_COUNTER_ADDRESS(
644 - hip->logical_address);
645 -
646 - address[0] = (uint32_t)(l & 0xFFFFFFFF);
647 - }
648 -}
649 -
650 605 /*
651 606 * Add the interrupt handler for I/O APIC interrupt number (interrupt line).
652 607 *
653 608 * The I/O APIC line (vector) is programmed in ioapic_init_intr() called
654 609 * from apic_picinit() psm_ops apic_ops entry point after we return from
655 610 * apic_init() psm_ops entry point.
656 611 */
657 612 static uint32_t
658 613 hpet_install_interrupt_handler(uint_t (*func)(char *), int vector)
659 614 {
660 615 uint32_t retval;
661 616
662 617 retval = add_avintr(NULL, CBE_HIGH_PIL, (avfunc)func, "HPET Timer",
663 618 vector, NULL, NULL, NULL, NULL);
664 619 if (retval == 0) {
665 620 cmn_err(CE_WARN, "!hpet_acpi: add_avintr() failed");
666 621 return (AE_BAD_PARAMETER);
667 622 }
668 623 return (AE_OK);
669 624 }
670 625
671 626 /*
672 627 * The HPET timers specify which I/O APIC interrupts they can be routed to.
673 628 * Find the first available non-legacy-replacement timer and its I/O APIC irq.
674 629 * Supported I/O APIC IRQs are specified in the int_route_cap bitmap in each
675 630 * timer's timer_n_config register.
676 631 */
677 632 static int
678 633 hpet_get_IOAPIC_intr_capable_timer(hpet_info_t *hip)
679 634 {
680 635 int timer;
681 636 int intr;
682 637
683 638 for (timer = HPET_FIRST_NON_LEGACY_TIMER;
684 639 timer < hip->gen_cap.num_tim_cap; ++timer) {
685 640
686 641 if (!hpet_timer_available(hip->allocated_timers, timer))
687 642 continue;
688 643
689 644 intr = lowbit(hip->timer_n_config[timer].int_route_cap) - 1;
690 645 if (intr >= 0) {
691 646 hpet_timer_alloc(&hip->allocated_timers, timer);
692 647 hip->cstate_timer.timer = timer;
693 648 hip->cstate_timer.intr = intr;
694 649 return (timer);
695 650 }
696 651 }
697 652
698 653 return (-1);
699 654 }
700 655
701 656 /*
702 657 * Mark this timer as used.
703 658 */
704 659 static void
705 660 hpet_timer_alloc(uint32_t *allocated_timers, uint32_t n)
706 661 {
707 662 *allocated_timers |= 1 << n;
708 663 }
709 664
710 665 /*
711 666 * Check if this timer is available.
712 667 * No mutual exclusion because only one thread uses this.
713 668 */
714 669 static int
715 670 hpet_timer_available(uint32_t allocated_timers, uint32_t n)
716 671 {
717 672 return ((allocated_timers & (1 << n)) == 0);
718 673 }
719 674
720 675 /*
721 676 * Setup timer N to route its interrupt to I/O APIC.
722 677 */
723 678 static void
724 679 hpet_timer_set_up(hpet_info_t *hip, uint32_t timer_n, uint32_t interrupt)
725 680 {
726 681 uint64_t conf;
727 682
728 683 conf = hpet_read_timer_N_config(hip, timer_n);
729 684
730 685 /*
731 686 * Caller is required to verify this interrupt route is supported.
732 687 */
733 688 ASSERT(HPET_TIMER_N_INT_ROUTE_CAP(conf) & (1 << interrupt));
734 689
735 690 conf &= ~HPET_TIMER_N_FSB_EN_CNF_BIT; /* use IOAPIC */
736 691 conf |= HPET_TIMER_N_INT_ROUTE_SHIFT(interrupt);
737 692 conf &= ~HPET_TIMER_N_TYPE_CNF_BIT; /* non periodic */
738 693 conf &= ~HPET_TIMER_N_INT_ENB_CNF_BIT; /* disabled */
739 694 conf |= HPET_TIMER_N_INT_TYPE_CNF_BIT; /* Level Triggered */
740 695
741 696 hpet_write_timer_N_config(hip, timer_n, conf);
742 697 }
743 698
744 699 /*
745 700 * The HPET's Main Counter is not stopped before programming an HPET timer.
746 701 * This will allow the HPET to be used as a time source.
747 702 * The programmed timer interrupt may occur before this function returns.
748 703 * Callers must block interrupts before calling this function if they must
749 704 * guarantee the interrupt is handled after this function returns.
750 705 *
751 706 * Return 0 if main counter is less than timer after enabling timer.
752 707 * The interrupt was programmed, but it may fire before this returns.
753 708 * Return !0 if main counter is greater than timer after enabling timer.
754 709 * In other words: the timer will not fire, and we do not know if it did fire.
755 710 *
756 711 * delta is in HPET ticks.
757 712 *
758 713 * Writing a 64-bit value to a 32-bit register will "wrap around".
759 714 * A 32-bit HPET timer will wrap around in a little over 5 minutes.
760 715 */
761 716 int
762 717 hpet_timer_program(hpet_info_t *hip, uint32_t timer, uint64_t delta)
763 718 {
764 719 uint64_t time, program;
765 720
766 721 program = hpet_read_main_counter_value(hip);
767 722 program += delta;
768 723 hpet_write_timer_N_comp(hip, timer, program);
769 724
770 725 time = hpet_read_main_counter_value(hip);
771 726 if (time < program)
772 727 return (AE_OK);
773 728
774 729 return (AE_TIME);
775 730 }
776 731
777 732 /*
778 733 * CPR and power policy-change callback entry point.
779 734 */
780 735 boolean_t
781 736 hpet_callback(int code)
782 737 {
783 738 switch (code) {
784 739 case PM_DEFAULT_CPU_DEEP_IDLE:
785 740 /*FALLTHROUGH*/
786 741 case PM_ENABLE_CPU_DEEP_IDLE:
787 742 /*FALLTHROUGH*/
788 743 case PM_DISABLE_CPU_DEEP_IDLE:
789 744 return (hpet_deep_idle_config(code));
790 745
791 746 case CB_CODE_CPR_RESUME:
792 747 /*FALLTHROUGH*/
793 748 case CB_CODE_CPR_CHKPT:
794 749 return (hpet_cpr(code));
795 750
796 751 case CST_EVENT_MULTIPLE_CSTATES:
797 752 hpet_cst_callback(CST_EVENT_MULTIPLE_CSTATES);
798 753 return (B_TRUE);
799 754
800 755 case CST_EVENT_ONE_CSTATE:
801 756 hpet_cst_callback(CST_EVENT_ONE_CSTATE);
802 757 return (B_TRUE);
803 758
804 759 default:
805 760 cmn_err(CE_NOTE, "!hpet_callback: invalid code %d\n", code);
806 761 return (B_FALSE);
807 762 }
808 763 }
809 764
810 765 /*
811 766 * According to the HPET spec 1.0a: the Operating System must save and restore
812 767 * HPET event timer hardware context through ACPI sleep state transitions.
813 768 * Timer registers (including the main counter) may not be preserved through
814 769 * ACPI S3, S4, or S5 sleep states. This code does not not support S1 nor S2.
815 770 *
816 771 * Current HPET state is already in hpet.supported and
817 772 * hpet_state.proxy_installed. hpet_info contains the proxy interrupt HPET
818 773 * Timer state.
819 774 *
820 775 * Future projects beware: the HPET Main Counter is undefined after ACPI S3 or
821 776 * S4, and it is not saved/restored here. Future projects cannot expect the
822 777 * Main Counter to be monotomically (or accurately) increasing across CPR.
823 778 *
824 779 * Note: the CPR Checkpoint path later calls pause_cpus() which ensures all
825 780 * CPUs are awake and in a spin loop before the system suspends. The HPET is
826 781 * not needed for Deep C-state wakeup when CPUs are in cpu_pause().
827 782 * It is safe to leave the HPET running as the system suspends; we just
828 783 * disable the timer from generating interrupts here.
829 784 */
830 785 static boolean_t
831 786 hpet_cpr(int code)
832 787 {
833 788 ulong_t intr, dead_count = 0;
834 789 hrtime_t dead = gethrtime() + hpet_spin_timeout;
835 790 boolean_t ret = B_TRUE;
836 791
837 792 mutex_enter(&hpet_state_lock);
838 793 switch (code) {
839 794 case CB_CODE_CPR_CHKPT:
840 795 if (hpet_state.proxy_installed == B_FALSE)
841 796 break;
842 797
843 798 hpet_state.cpr = B_TRUE;
844 799
845 800 intr = intr_clear();
846 801 while (!mutex_tryenter(&hpet_proxy_lock)) {
847 802 /*
848 803 * spin
849 804 */
850 805 intr_restore(intr);
851 806 if (dead_count++ > hpet_spin_check) {
852 807 dead_count = 0;
853 808 if (gethrtime() > dead) {
854 809 hpet_state.cpr = B_FALSE;
855 810 mutex_exit(&hpet_state_lock);
856 811 cmn_err(CE_NOTE, "!hpet_cpr: deadman");
857 812 return (B_FALSE);
858 813 }
859 814 }
860 815 intr = intr_clear();
861 816 }
862 817 hpet_expire_all();
863 818 mutex_exit(&hpet_proxy_lock);
864 819 intr_restore(intr);
865 820
866 821 hpet_disable_timer(&hpet_info, hpet_info.cstate_timer.timer);
867 822 break;
868 823
869 824 case CB_CODE_CPR_RESUME:
870 825 if (hpet_resume() == B_TRUE)
871 826 hpet_state.cpr = B_FALSE;
872 827 else
873 828 cmn_err(CE_NOTE, "!hpet_resume failed.");
874 829 break;
875 830
876 831 default:
877 832 cmn_err(CE_NOTE, "!hpet_cpr: invalid code %d\n", code);
878 833 ret = B_FALSE;
879 834 break;
880 835 }
881 836 mutex_exit(&hpet_state_lock);
882 837 return (ret);
883 838 }
884 839
885 840 /*
886 841 * Assume the HPET stopped in Suspend state and timer state was lost.
887 842 */
888 843 static boolean_t
889 844 hpet_resume(void)
890 845 {
891 846 if (hpet.supported != HPET_TIMER_SUPPORT)
892 847 return (B_TRUE);
893 848
894 849 /*
895 850 * The HPET spec does not specify if Legacy Replacement Route is
896 851 * on or off by default, so we set it off here.
897 852 */
898 853 (void) hpet_set_leg_rt_cnf(&hpet_info, 0);
899 854
900 855 if (hpet_start_main_counter(&hpet_info) != AE_OK) {
901 856 cmn_err(CE_NOTE, "!hpet_resume: start main counter failed");
902 857 hpet.supported = HPET_NO_SUPPORT;
903 858 if (hpet_state.proxy_installed == B_TRUE) {
904 859 hpet_state.proxy_installed = B_FALSE;
905 860 hpet_uninstall_interrupt_handler();
906 861 }
907 862 return (B_FALSE);
908 863 }
909 864
910 865 if (hpet_state.proxy_installed == B_FALSE)
911 866 return (B_TRUE);
912 867
913 868 hpet_timer_set_up(&hpet_info, hpet_info.cstate_timer.timer,
914 869 hpet_info.cstate_timer.intr);
915 870 if (hpet_state.cpu_deep_idle == B_TRUE)
916 871 hpet_enable_timer(&hpet_info, hpet_info.cstate_timer.timer);
917 872
918 873 return (B_TRUE);
919 874 }
920 875
921 876 /*
922 877 * Callback to enable/disable Deep C-States based on power.conf setting.
923 878 */
924 879 static boolean_t
925 880 hpet_deep_idle_config(int code)
926 881 {
927 882 ulong_t intr, dead_count = 0;
928 883 hrtime_t dead = gethrtime() + hpet_spin_timeout;
929 884 boolean_t ret = B_TRUE;
930 885
931 886 mutex_enter(&hpet_state_lock);
932 887 switch (code) {
933 888 case PM_DEFAULT_CPU_DEEP_IDLE:
934 889 /*FALLTHROUGH*/
935 890 case PM_ENABLE_CPU_DEEP_IDLE:
936 891
937 892 if (hpet_state.cpu_deep_idle == B_TRUE)
938 893 break;
939 894
940 895 if (hpet_state.proxy_installed == B_FALSE) {
941 896 ret = B_FALSE; /* Deep C-States not supported */
942 897 break;
943 898 }
944 899
945 900 hpet_enable_timer(&hpet_info, hpet_info.cstate_timer.timer);
946 901 hpet_state.cpu_deep_idle = B_TRUE;
947 902 break;
948 903
949 904 case PM_DISABLE_CPU_DEEP_IDLE:
950 905
951 906 if ((hpet_state.cpu_deep_idle == B_FALSE) ||
952 907 (hpet_state.proxy_installed == B_FALSE))
953 908 break;
954 909
955 910 /*
956 911 * The order of these operations is important to avoid
957 912 * lost wakeups: Set a flag to refuse all future LAPIC Timer
958 913 * proxy requests, then wake up all CPUs from deep C-state,
959 914 * and finally disable the HPET interrupt-generating timer.
960 915 */
961 916 hpet_state.cpu_deep_idle = B_FALSE;
962 917
963 918 intr = intr_clear();
964 919 while (!mutex_tryenter(&hpet_proxy_lock)) {
965 920 /*
966 921 * spin
967 922 */
968 923 intr_restore(intr);
969 924 if (dead_count++ > hpet_spin_check) {
970 925 dead_count = 0;
971 926 if (gethrtime() > dead) {
972 927 hpet_state.cpu_deep_idle = B_TRUE;
973 928 mutex_exit(&hpet_state_lock);
974 929 cmn_err(CE_NOTE,
975 930 "!hpet_deep_idle_config: deadman");
976 931 return (B_FALSE);
977 932 }
978 933 }
979 934 intr = intr_clear();
980 935 }
981 936 hpet_expire_all();
982 937 mutex_exit(&hpet_proxy_lock);
983 938 intr_restore(intr);
984 939
985 940 hpet_disable_timer(&hpet_info, hpet_info.cstate_timer.timer);
986 941 break;
987 942
988 943 default:
989 944 cmn_err(CE_NOTE, "!hpet_deep_idle_config: invalid code %d\n",
990 945 code);
991 946 ret = B_FALSE;
992 947 break;
993 948 }
994 949 mutex_exit(&hpet_state_lock);
995 950
996 951 return (ret);
997 952 }
998 953
999 954 /*
1000 955 * Callback for _CST c-state change notifications.
1001 956 */
1002 957 static void
1003 958 hpet_cst_callback(uint32_t code)
1004 959 {
1005 960 ulong_t intr, dead_count = 0;
1006 961 hrtime_t dead = gethrtime() + hpet_spin_timeout;
1007 962
1008 963 switch (code) {
1009 964 case CST_EVENT_ONE_CSTATE:
1010 965 hpet_state.uni_cstate = B_TRUE;
1011 966 intr = intr_clear();
1012 967 while (!mutex_tryenter(&hpet_proxy_lock)) {
1013 968 /*
1014 969 * spin
1015 970 */
1016 971 intr_restore(intr);
1017 972 if (dead_count++ > hpet_spin_check) {
1018 973 dead_count = 0;
1019 974 if (gethrtime() > dead) {
1020 975 hpet_expire_all();
1021 976 cmn_err(CE_NOTE,
1022 977 "!hpet_cst_callback: deadman");
1023 978 return;
1024 979 }
1025 980 }
1026 981 intr = intr_clear();
1027 982 }
1028 983 hpet_expire_all();
1029 984 mutex_exit(&hpet_proxy_lock);
1030 985 intr_restore(intr);
1031 986 break;
1032 987
1033 988 case CST_EVENT_MULTIPLE_CSTATES:
1034 989 hpet_state.uni_cstate = B_FALSE;
1035 990 break;
1036 991
1037 992 default:
1038 993 cmn_err(CE_NOTE, "!hpet_cst_callback: invalid code %d\n", code);
1039 994 break;
1040 995 }
1041 996 }
1042 997
1043 998 /*
1044 999 * Interrupt Service Routine for HPET I/O-APIC-generated interrupts.
1045 1000 * Used to wakeup CPUs from Deep C-state when their Local APIC Timer stops.
1046 1001 * This ISR runs on one CPU which pokes other CPUs out of Deep C-state as
1047 1002 * needed.
1048 1003 */
1049 1004 /* ARGSUSED */
1050 1005 static uint_t
1051 1006 hpet_isr(char *arg)
1052 1007 {
1053 1008 uint64_t timer_status;
1054 1009 uint64_t timer_mask;
1055 1010 ulong_t intr, dead_count = 0;
1056 1011 hrtime_t dead = gethrtime() + hpet_isr_spin_timeout;
1057 1012
1058 1013 timer_mask = HPET_INTR_STATUS_MASK(hpet_info.cstate_timer.timer);
1059 1014
1060 1015 /*
1061 1016 * We are using a level-triggered interrupt.
1062 1017 * HPET sets timer's General Interrupt Status Register bit N.
1063 1018 * ISR checks this bit to see if it needs servicing.
1064 1019 * ISR then clears this bit by writing 1 to that bit.
1065 1020 */
1066 1021 timer_status = hpet_read_gen_intrpt_stat(&hpet_info);
1067 1022 if (!(timer_status & timer_mask))
1068 1023 return (DDI_INTR_UNCLAIMED);
1069 1024 hpet_write_gen_intrpt_stat(&hpet_info, timer_mask);
1070 1025
1071 1026 /*
1072 1027 * Do not touch ISR data structures before checking the HPET's General
1073 1028 * Interrupt Status register. The General Interrupt Status register
1074 1029 * will not be set by hardware until after timer interrupt generation
1075 1030 * is enabled by software. Software allocates necessary data
1076 1031 * structures before enabling timer interrupts. ASSERT the software
1077 1032 * data structures required to handle this interrupt are initialized.
1078 1033 */
1079 1034 ASSERT(hpet_proxy_users != NULL);
1080 1035
1081 1036 /*
1082 1037 * CPUs in deep c-states do not enable interrupts until after
1083 1038 * performing idle cleanup which includes descheduling themselves from
1084 1039 * the HPET. The CPU running this ISR will NEVER find itself in the
1085 1040 * proxy list. A lost wakeup may occur if this is false.
1086 1041 */
1087 1042 ASSERT(hpet_proxy_users[CPU->cpu_id] == HPET_INFINITY);
1088 1043
1089 1044 /*
1090 1045 * Higher level interrupts may deadlock with CPUs going idle if this
1091 1046 * ISR is prempted while holding hpet_proxy_lock.
1092 1047 */
1093 1048 intr = intr_clear();
1094 1049 while (!mutex_tryenter(&hpet_proxy_lock)) {
1095 1050 /*
1096 1051 * spin
1097 1052 */
1098 1053 intr_restore(intr);
1099 1054 if (dead_count++ > hpet_spin_check) {
1100 1055 dead_count = 0;
1101 1056 if (gethrtime() > dead) {
1102 1057 hpet_expire_all();
1103 1058 return (DDI_INTR_CLAIMED);
1104 1059 }
1105 1060 }
1106 1061 intr = intr_clear();
1107 1062 }
1108 1063 (void) hpet_guaranteed_schedule(HPET_INFINITY);
1109 1064 mutex_exit(&hpet_proxy_lock);
1110 1065 intr_restore(intr);
1111 1066
1112 1067 return (DDI_INTR_CLAIMED);
1113 1068 }
1114 1069
1115 1070 /*
1116 1071 * Used when disabling the HPET Timer interrupt. CPUs in Deep C-state must be
1117 1072 * woken up because they can no longer rely on the HPET's Timer to wake them.
1118 1073 * We do not need to wait for CPUs to wakeup.
1119 1074 */
1120 1075 static void
1121 1076 hpet_expire_all(void)
1122 1077 {
1123 1078 processorid_t id;
1124 1079
1125 1080 for (id = 0; id < max_ncpus; ++id) {
1126 1081 if (hpet_proxy_users[id] != HPET_INFINITY) {
1127 1082 hpet_proxy_users[id] = HPET_INFINITY;
1128 1083 if (id != CPU->cpu_id)
1129 1084 poke_cpu(id);
1130 1085 }
1131 1086 }
1132 1087 }
1133 1088
1134 1089 /*
1135 1090 * To avoid missed wakeups this function must guarantee either the HPET timer
1136 1091 * was successfully programmed to the next expire time or there are no waiting
1137 1092 * CPUs.
1138 1093 *
1139 1094 * Callers cannot enter C2 or deeper if the HPET could not be programmed to
1140 1095 * generate its next interrupt to happen at required_wakeup_time or sooner.
1141 1096 * Returns B_TRUE if the HPET was programmed to interrupt by
1142 1097 * required_wakeup_time, B_FALSE if not.
1143 1098 */
1144 1099 static boolean_t
1145 1100 hpet_guaranteed_schedule(hrtime_t required_wakeup_time)
1146 1101 {
1147 1102 hrtime_t now, next_proxy_time;
1148 1103 processorid_t id, next_proxy_id;
1149 1104 int proxy_timer = hpet_info.cstate_timer.timer;
1150 1105 boolean_t done = B_FALSE;
1151 1106
1152 1107 ASSERT(mutex_owned(&hpet_proxy_lock));
1153 1108
1154 1109 /*
1155 1110 * Loop until we successfully program the HPET,
1156 1111 * or no CPUs are scheduled to use the HPET as a proxy.
1157 1112 */
1158 1113 do {
1159 1114 /*
1160 1115 * Wake all CPUs that expired before now.
1161 1116 * Find the next CPU to wake up and next HPET program time.
1162 1117 */
1163 1118 now = gethrtime();
1164 1119 next_proxy_time = HPET_INFINITY;
1165 1120 next_proxy_id = CPU->cpu_id;
1166 1121 for (id = 0; id < max_ncpus; ++id) {
1167 1122 if (hpet_proxy_users[id] < now) {
1168 1123 hpet_proxy_users[id] = HPET_INFINITY;
1169 1124 if (id != CPU->cpu_id)
1170 1125 poke_cpu(id);
1171 1126 } else if (hpet_proxy_users[id] < next_proxy_time) {
1172 1127 next_proxy_time = hpet_proxy_users[id];
1173 1128 next_proxy_id = id;
1174 1129 }
1175 1130 }
1176 1131
1177 1132 if (next_proxy_time == HPET_INFINITY) {
1178 1133 done = B_TRUE;
1179 1134 /*
1180 1135 * There are currently no CPUs using the HPET's Timer
1181 1136 * as a proxy for their LAPIC Timer. The HPET's Timer
1182 1137 * does not need to be programmed.
1183 1138 *
1184 1139 * Letting the HPET timer wrap around to the current
1185 1140 * time is the longest possible timeout.
1186 1141 * A 64-bit timer will wrap around in ~ 2^44 seconds.
1187 1142 * A 32-bit timer will wrap around in ~ 2^12 seconds.
1188 1143 *
1189 1144 * Disabling the HPET's timer interrupt requires a
1190 1145 * (relatively expensive) write to the HPET.
1191 1146 * Instead we do nothing.
1192 1147 *
1193 1148 * We are gambling some CPU will attempt to enter a
1194 1149 * deep c-state before the timer wraps around.
1195 1150 * We assume one spurious interrupt in a little over an
1196 1151 * hour has less performance impact than writing to the
1197 1152 * HPET's timer disable bit every time all CPUs wakeup
1198 1153 * from deep c-state.
1199 1154 */
1200 1155
1201 1156 } else {
1202 1157 /*
1203 1158 * Idle CPUs disable interrupts before programming the
1204 1159 * HPET to prevent a lost wakeup if the HPET
1205 1160 * interrupts the idle cpu before it can enter a
1206 1161 * Deep C-State.
1207 1162 */
1208 1163 if (hpet_timer_program(&hpet_info, proxy_timer,
1209 1164 HRTIME_TO_HPET_TICKS(next_proxy_time - gethrtime()))
1210 1165 != AE_OK) {
1211 1166 /*
1212 1167 * We could not program the HPET to wakeup the
1213 1168 * next CPU. We must wake the CPU ourself to
1214 1169 * avoid a lost wakeup.
1215 1170 */
1216 1171 hpet_proxy_users[next_proxy_id] = HPET_INFINITY;
1217 1172 if (next_proxy_id != CPU->cpu_id)
1218 1173 poke_cpu(next_proxy_id);
1219 1174 } else {
1220 1175 done = B_TRUE;
1221 1176 }
1222 1177 }
1223 1178
1224 1179 } while (!done);
1225 1180
1226 1181 return (next_proxy_time <= required_wakeup_time);
1227 1182 }
1228 1183
1229 1184 /*
1230 1185 * Use an HPET timer to act as this CPU's proxy local APIC timer.
1231 1186 * Used in deep c-states C2 and above while the CPU's local APIC timer stalls.
1232 1187 * Called by the idle thread with interrupts enabled.
1233 1188 * Always returns with interrupts disabled.
1234 1189 *
1235 1190 * There are 3 possible outcomes from this function:
1236 1191 * 1. The Local APIC Timer was already disabled before this function was called.
1237 1192 * LAPIC TIMER : disabled
1238 1193 * HPET : not scheduled to wake this CPU
1239 1194 * *lapic_expire : (hrtime_t)HPET_INFINITY
1240 1195 * Returns : B_TRUE
1241 1196 * 2. Successfully programmed the HPET to act as a LAPIC Timer proxy.
1242 1197 * LAPIC TIMER : disabled
1243 1198 * HPET : scheduled to wake this CPU
1244 1199 * *lapic_expire : hrtime_t when LAPIC timer would have expired
1245 1200 * Returns : B_TRUE
1246 1201 * 3. Failed to programmed the HPET to act as a LAPIC Timer proxy.
1247 1202 * LAPIC TIMER : enabled
1248 1203 * HPET : not scheduled to wake this CPU
1249 1204 * *lapic_expire : (hrtime_t)HPET_INFINITY
1250 1205 * Returns : B_FALSE
1251 1206 *
1252 1207 * The idle thread cannot enter Deep C-State in case 3.
1253 1208 * The idle thread must re-enable & re-program the LAPIC_TIMER in case 2.
1254 1209 */
1255 1210 static boolean_t
1256 1211 hpet_use_hpet_timer(hrtime_t *lapic_expire)
1257 1212 {
1258 1213 extern hrtime_t apic_timer_stop_count(void);
1259 1214 extern void apic_timer_restart(hrtime_t);
1260 1215 hrtime_t now, expire, dead;
1261 1216 uint64_t lapic_count, dead_count;
1262 1217 cpupart_t *cpu_part;
1263 1218 processorid_t cpu_sid;
1264 1219 processorid_t cpu_id = CPU->cpu_id;
1265 1220 processorid_t id;
1266 1221 boolean_t rslt;
1267 1222 boolean_t hset_update;
1268 1223
1269 1224 cpu_part = CPU->cpu_part;
1270 1225 cpu_sid = CPU->cpu_seqid;
1271 1226
1272 1227 ASSERT(CPU->cpu_thread == CPU->cpu_idle_thread);
1273 1228
1274 1229 /*
1275 1230 * A critical section exists between when the HPET is programmed
1276 1231 * to interrupt the CPU and when this CPU enters an idle state.
1277 1232 * Interrupts must be blocked during that time to prevent lost
1278 1233 * CBE wakeup interrupts from either LAPIC or HPET.
1279 1234 *
1280 1235 * Must block interrupts before acquiring hpet_proxy_lock to prevent
1281 1236 * a deadlock with the ISR if the ISR runs on this CPU after the
1282 1237 * idle thread acquires the mutex but before it clears interrupts.
1283 1238 */
1284 1239 ASSERT(!interrupts_enabled());
1285 1240 lapic_count = apic_timer_stop_count();
1286 1241 now = gethrtime();
1287 1242 dead = now + hpet_idle_spin_timeout;
1288 1243 *lapic_expire = expire = now + lapic_count;
1289 1244 if (lapic_count == (hrtime_t)-1) {
1290 1245 /*
1291 1246 * LAPIC timer is currently disabled.
1292 1247 * Will not use the HPET as a LAPIC Timer proxy.
1293 1248 */
1294 1249 *lapic_expire = (hrtime_t)HPET_INFINITY;
1295 1250 return (B_TRUE);
1296 1251 }
1297 1252
1298 1253 /*
1299 1254 * Serialize hpet_proxy data structure manipulation.
1300 1255 */
1301 1256 dead_count = 0;
1302 1257 while (!mutex_tryenter(&hpet_proxy_lock)) {
1303 1258 /*
1304 1259 * spin
1305 1260 */
1306 1261 apic_timer_restart(expire);
1307 1262 sti();
1308 1263 cli();
1309 1264
1310 1265 if (dead_count++ > hpet_spin_check) {
1311 1266 dead_count = 0;
1312 1267 hset_update = (((CPU->cpu_flags & CPU_OFFLINE) == 0) &&
1313 1268 (ncpus > 1));
1314 1269 if (hset_update &&
1315 1270 !bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) {
1316 1271 *lapic_expire = (hrtime_t)HPET_INFINITY;
1317 1272 return (B_FALSE);
1318 1273 }
1319 1274 }
1320 1275
1321 1276 lapic_count = apic_timer_stop_count();
1322 1277 now = gethrtime();
1323 1278 *lapic_expire = expire = now + lapic_count;
1324 1279 if (lapic_count == (hrtime_t)-1) {
1325 1280 /*
1326 1281 * LAPIC timer is currently disabled.
1327 1282 * Will not use the HPET as a LAPIC Timer proxy.
1328 1283 */
1329 1284 *lapic_expire = (hrtime_t)HPET_INFINITY;
1330 1285 return (B_TRUE);
1331 1286 }
1332 1287 if (now > dead) {
1333 1288 apic_timer_restart(expire);
1334 1289 *lapic_expire = (hrtime_t)HPET_INFINITY;
1335 1290 return (B_FALSE);
1336 1291 }
1337 1292 }
1338 1293
1339 1294 if ((hpet_state.cpr == B_TRUE) ||
1340 1295 (hpet_state.cpu_deep_idle == B_FALSE) ||
1341 1296 (hpet_state.proxy_installed == B_FALSE) ||
1342 1297 (hpet_state.uni_cstate == B_TRUE)) {
1343 1298 mutex_exit(&hpet_proxy_lock);
1344 1299 apic_timer_restart(expire);
1345 1300 *lapic_expire = (hrtime_t)HPET_INFINITY;
1346 1301 return (B_FALSE);
1347 1302 }
1348 1303
1349 1304 hpet_proxy_users[cpu_id] = expire;
1350 1305
1351 1306 /*
1352 1307 * We are done if another cpu is scheduled on the HPET with an
1353 1308 * expire time before us. The next HPET interrupt has been programmed
1354 1309 * to fire before our expire time.
1355 1310 */
1356 1311 for (id = 0; id < max_ncpus; ++id) {
1357 1312 if ((hpet_proxy_users[id] <= expire) && (id != cpu_id)) {
1358 1313 mutex_exit(&hpet_proxy_lock);
1359 1314 return (B_TRUE);
1360 1315 }
1361 1316 }
1362 1317
1363 1318 /*
1364 1319 * We are the next lAPIC to expire.
1365 1320 * Program the HPET with our expire time.
1366 1321 */
1367 1322 rslt = hpet_guaranteed_schedule(expire);
1368 1323 mutex_exit(&hpet_proxy_lock);
1369 1324
1370 1325 if (rslt == B_FALSE) {
1371 1326 apic_timer_restart(expire);
1372 1327 *lapic_expire = (hrtime_t)HPET_INFINITY;
1373 1328 }
1374 1329
1375 1330 return (rslt);
1376 1331 }
1377 1332
1378 1333 /*
1379 1334 * Called by the idle thread when waking up from Deep C-state before enabling
1380 1335 * interrupts. With an array data structure it is faster to always remove
1381 1336 * ourself from the array without checking if the HPET ISR already removed.
1382 1337 *
1383 1338 * We use a lazy algorithm for removing CPUs from the HPET's schedule.
1384 1339 * We do not reprogram the HPET here because this CPU has real work to do.
1385 1340 * On a idle system the CPU was probably woken up by the HPET's ISR.
1386 1341 * On a heavily loaded system CPUs are not going into Deep C-state.
1387 1342 * On a moderately loaded system another CPU will usually enter Deep C-state
1388 1343 * and reprogram the HPET before the HPET fires with our wakeup.
1389 1344 */
1390 1345 static void
1391 1346 hpet_use_lapic_timer(hrtime_t expire)
1392 1347 {
1393 1348 extern void apic_timer_restart(hrtime_t);
1394 1349 processorid_t cpu_id = CPU->cpu_id;
1395 1350
1396 1351 ASSERT(CPU->cpu_thread == CPU->cpu_idle_thread);
1397 1352 ASSERT(!interrupts_enabled());
1398 1353
1399 1354 hpet_proxy_users[cpu_id] = HPET_INFINITY;
1400 1355
1401 1356 /*
1402 1357 * Do not enable a LAPIC Timer that was initially disabled.
1403 1358 */
1404 1359 if (expire != HPET_INFINITY)
1405 1360 apic_timer_restart(expire);
1406 1361 }
1407 1362
1408 1363 /*
1409 1364 * Initialize data structure to keep track of CPUs using HPET as a proxy for
1410 1365 * their stalled local APIC timer. For now this is just an array.
1411 1366 */
1412 1367 static void
1413 1368 hpet_init_proxy_data(void)
1414 1369 {
1415 1370 processorid_t id;
1416 1371
1417 1372 /*
1418 1373 * Use max_ncpus for hot plug compliance.
1419 1374 */
1420 1375 hpet_proxy_users = kmem_zalloc(max_ncpus * sizeof (*hpet_proxy_users),
1421 1376 KM_SLEEP);
1422 1377
1423 1378 /*
1424 1379 * Unused entries always contain HPET_INFINITY.
1425 1380 */
1426 1381 for (id = 0; id < max_ncpus; ++id)
1427 1382 hpet_proxy_users[id] = HPET_INFINITY;
1428 1383 }
|
↓ open down ↓ |
769 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX