Print this page
5045 use atomic_{inc,dec}_* instead of atomic_add_*
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/os/clock.c
+++ new/usr/src/uts/common/os/clock.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
22 22 /* All Rights Reserved */
23 23
24 24 /*
25 25 * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
26 26 * Copyright (c) 2013, Joyent, Inc. All rights reserved.
27 27 */
28 28
29 29 #include <sys/param.h>
30 30 #include <sys/t_lock.h>
31 31 #include <sys/types.h>
32 32 #include <sys/tuneable.h>
33 33 #include <sys/sysmacros.h>
34 34 #include <sys/systm.h>
35 35 #include <sys/cpuvar.h>
36 36 #include <sys/lgrp.h>
37 37 #include <sys/user.h>
38 38 #include <sys/proc.h>
39 39 #include <sys/callo.h>
40 40 #include <sys/kmem.h>
41 41 #include <sys/var.h>
42 42 #include <sys/cmn_err.h>
43 43 #include <sys/swap.h>
44 44 #include <sys/vmsystm.h>
45 45 #include <sys/class.h>
46 46 #include <sys/time.h>
47 47 #include <sys/debug.h>
48 48 #include <sys/vtrace.h>
49 49 #include <sys/spl.h>
50 50 #include <sys/atomic.h>
51 51 #include <sys/dumphdr.h>
52 52 #include <sys/archsystm.h>
53 53 #include <sys/fs/swapnode.h>
54 54 #include <sys/panic.h>
55 55 #include <sys/disp.h>
56 56 #include <sys/msacct.h>
57 57 #include <sys/mem_cage.h>
58 58
59 59 #include <vm/page.h>
60 60 #include <vm/anon.h>
61 61 #include <vm/rm.h>
62 62 #include <sys/cyclic.h>
63 63 #include <sys/cpupart.h>
64 64 #include <sys/rctl.h>
65 65 #include <sys/task.h>
66 66 #include <sys/sdt.h>
67 67 #include <sys/ddi_periodic.h>
68 68 #include <sys/random.h>
69 69 #include <sys/modctl.h>
70 70
71 71 /*
72 72 * for NTP support
73 73 */
74 74 #include <sys/timex.h>
75 75 #include <sys/inttypes.h>
76 76
77 77 #include <sys/sunddi.h>
78 78 #include <sys/clock_impl.h>
79 79
80 80 /*
81 81 * clock() is called straight from the clock cyclic; see clock_init().
82 82 *
83 83 * Functions:
84 84 * reprime clock
85 85 * maintain date
86 86 * jab the scheduler
87 87 */
88 88
89 89 extern kcondvar_t fsflush_cv;
90 90 extern sysinfo_t sysinfo;
91 91 extern vminfo_t vminfo;
92 92 extern int idleswtch; /* flag set while idle in pswtch() */
93 93 extern hrtime_t volatile devinfo_freeze;
94 94
95 95 /*
96 96 * high-precision avenrun values. These are needed to make the
97 97 * regular avenrun values accurate.
98 98 */
99 99 static uint64_t hp_avenrun[3];
100 100 int avenrun[3]; /* FSCALED average run queue lengths */
101 101 time_t time; /* time in seconds since 1970 - for compatibility only */
102 102
103 103 static struct loadavg_s loadavg;
104 104 /*
105 105 * Phase/frequency-lock loop (PLL/FLL) definitions
106 106 *
107 107 * The following variables are read and set by the ntp_adjtime() system
108 108 * call.
109 109 *
110 110 * time_state shows the state of the system clock, with values defined
111 111 * in the timex.h header file.
112 112 *
113 113 * time_status shows the status of the system clock, with bits defined
114 114 * in the timex.h header file.
115 115 *
116 116 * time_offset is used by the PLL/FLL to adjust the system time in small
117 117 * increments.
118 118 *
119 119 * time_constant determines the bandwidth or "stiffness" of the PLL.
120 120 *
121 121 * time_tolerance determines maximum frequency error or tolerance of the
122 122 * CPU clock oscillator and is a property of the architecture; however,
123 123 * in principle it could change as result of the presence of external
124 124 * discipline signals, for instance.
125 125 *
126 126 * time_precision is usually equal to the kernel tick variable; however,
127 127 * in cases where a precision clock counter or external clock is
128 128 * available, the resolution can be much less than this and depend on
129 129 * whether the external clock is working or not.
130 130 *
131 131 * time_maxerror is initialized by a ntp_adjtime() call and increased by
132 132 * the kernel once each second to reflect the maximum error bound
133 133 * growth.
134 134 *
135 135 * time_esterror is set and read by the ntp_adjtime() call, but
136 136 * otherwise not used by the kernel.
137 137 */
138 138 int32_t time_state = TIME_OK; /* clock state */
139 139 int32_t time_status = STA_UNSYNC; /* clock status bits */
140 140 int32_t time_offset = 0; /* time offset (us) */
141 141 int32_t time_constant = 0; /* pll time constant */
142 142 int32_t time_tolerance = MAXFREQ; /* frequency tolerance (scaled ppm) */
143 143 int32_t time_precision = 1; /* clock precision (us) */
144 144 int32_t time_maxerror = MAXPHASE; /* maximum error (us) */
145 145 int32_t time_esterror = MAXPHASE; /* estimated error (us) */
146 146
147 147 /*
148 148 * The following variables establish the state of the PLL/FLL and the
149 149 * residual time and frequency offset of the local clock. The scale
150 150 * factors are defined in the timex.h header file.
151 151 *
152 152 * time_phase and time_freq are the phase increment and the frequency
153 153 * increment, respectively, of the kernel time variable.
154 154 *
155 155 * time_freq is set via ntp_adjtime() from a value stored in a file when
156 156 * the synchronization daemon is first started. Its value is retrieved
157 157 * via ntp_adjtime() and written to the file about once per hour by the
158 158 * daemon.
159 159 *
160 160 * time_adj is the adjustment added to the value of tick at each timer
161 161 * interrupt and is recomputed from time_phase and time_freq at each
162 162 * seconds rollover.
163 163 *
164 164 * time_reftime is the second's portion of the system time at the last
165 165 * call to ntp_adjtime(). It is used to adjust the time_freq variable
166 166 * and to increase the time_maxerror as the time since last update
167 167 * increases.
168 168 */
169 169 int32_t time_phase = 0; /* phase offset (scaled us) */
170 170 int32_t time_freq = 0; /* frequency offset (scaled ppm) */
171 171 int32_t time_adj = 0; /* tick adjust (scaled 1 / hz) */
172 172 int32_t time_reftime = 0; /* time at last adjustment (s) */
173 173
174 174 /*
175 175 * The scale factors of the following variables are defined in the
176 176 * timex.h header file.
177 177 *
178 178 * pps_time contains the time at each calibration interval, as read by
179 179 * microtime(). pps_count counts the seconds of the calibration
180 180 * interval, the duration of which is nominally pps_shift in powers of
181 181 * two.
182 182 *
183 183 * pps_offset is the time offset produced by the time median filter
184 184 * pps_tf[], while pps_jitter is the dispersion (jitter) measured by
185 185 * this filter.
186 186 *
187 187 * pps_freq is the frequency offset produced by the frequency median
188 188 * filter pps_ff[], while pps_stabil is the dispersion (wander) measured
189 189 * by this filter.
190 190 *
191 191 * pps_usec is latched from a high resolution counter or external clock
192 192 * at pps_time. Here we want the hardware counter contents only, not the
193 193 * contents plus the time_tv.usec as usual.
194 194 *
195 195 * pps_valid counts the number of seconds since the last PPS update. It
196 196 * is used as a watchdog timer to disable the PPS discipline should the
197 197 * PPS signal be lost.
198 198 *
199 199 * pps_glitch counts the number of seconds since the beginning of an
200 200 * offset burst more than tick/2 from current nominal offset. It is used
201 201 * mainly to suppress error bursts due to priority conflicts between the
202 202 * PPS interrupt and timer interrupt.
203 203 *
204 204 * pps_intcnt counts the calibration intervals for use in the interval-
205 205 * adaptation algorithm. It's just too complicated for words.
206 206 */
207 207 struct timeval pps_time; /* kernel time at last interval */
208 208 int32_t pps_tf[] = {0, 0, 0}; /* pps time offset median filter (us) */
209 209 int32_t pps_offset = 0; /* pps time offset (us) */
210 210 int32_t pps_jitter = MAXTIME; /* time dispersion (jitter) (us) */
211 211 int32_t pps_ff[] = {0, 0, 0}; /* pps frequency offset median filter */
212 212 int32_t pps_freq = 0; /* frequency offset (scaled ppm) */
213 213 int32_t pps_stabil = MAXFREQ; /* frequency dispersion (scaled ppm) */
214 214 int32_t pps_usec = 0; /* microsec counter at last interval */
215 215 int32_t pps_valid = PPS_VALID; /* pps signal watchdog counter */
216 216 int32_t pps_glitch = 0; /* pps signal glitch counter */
217 217 int32_t pps_count = 0; /* calibration interval counter (s) */
218 218 int32_t pps_shift = PPS_SHIFT; /* interval duration (s) (shift) */
219 219 int32_t pps_intcnt = 0; /* intervals at current duration */
220 220
221 221 /*
222 222 * PPS signal quality monitors
223 223 *
224 224 * pps_jitcnt counts the seconds that have been discarded because the
225 225 * jitter measured by the time median filter exceeds the limit MAXTIME
226 226 * (100 us).
227 227 *
228 228 * pps_calcnt counts the frequency calibration intervals, which are
229 229 * variable from 4 s to 256 s.
230 230 *
231 231 * pps_errcnt counts the calibration intervals which have been discarded
232 232 * because the wander exceeds the limit MAXFREQ (100 ppm) or where the
233 233 * calibration interval jitter exceeds two ticks.
234 234 *
235 235 * pps_stbcnt counts the calibration intervals that have been discarded
236 236 * because the frequency wander exceeds the limit MAXFREQ / 4 (25 us).
237 237 */
238 238 int32_t pps_jitcnt = 0; /* jitter limit exceeded */
239 239 int32_t pps_calcnt = 0; /* calibration intervals */
240 240 int32_t pps_errcnt = 0; /* calibration errors */
241 241 int32_t pps_stbcnt = 0; /* stability limit exceeded */
242 242
243 243 kcondvar_t lbolt_cv;
244 244
245 245 /*
246 246 * Hybrid lbolt implementation:
247 247 *
248 248 * The service historically provided by the lbolt and lbolt64 variables has
249 249 * been replaced by the ddi_get_lbolt() and ddi_get_lbolt64() routines, and the
250 250 * original symbols removed from the system. The once clock driven variables are
251 251 * now implemented in an event driven fashion, backed by gethrtime() coarsed to
252 252 * the appropriate clock resolution. The default event driven implementation is
253 253 * complemented by a cyclic driven one, active only during periods of intense
254 254 * activity around the DDI lbolt routines, when a lbolt specific cyclic is
255 255 * reprogramed to fire at a clock tick interval to serve consumers of lbolt who
256 256 * rely on the original low cost of consulting a memory position.
257 257 *
258 258 * The implementation uses the number of calls to these routines and the
259 259 * frequency of these to determine when to transition from event to cyclic
260 260 * driven and vice-versa. These values are kept on a per CPU basis for
261 261 * scalability reasons and to prevent CPUs from constantly invalidating a single
262 262 * cache line when modifying a global variable. The transition from event to
263 263 * cyclic mode happens once the thresholds are crossed, and activity on any CPU
264 264 * can cause such transition.
265 265 *
266 266 * The lbolt_hybrid function pointer is called by ddi_get_lbolt() and
267 267 * ddi_get_lbolt64(), and will point to lbolt_event_driven() or
268 268 * lbolt_cyclic_driven() according to the current mode. When the thresholds
269 269 * are exceeded, lbolt_event_driven() will reprogram the lbolt cyclic to
270 270 * fire at a nsec_per_tick interval and increment an internal variable at
271 271 * each firing. lbolt_hybrid will then point to lbolt_cyclic_driven(), which
272 272 * will simply return the value of such variable. lbolt_cyclic() will attempt
273 273 * to shut itself off at each threshold interval (sampling period for calls
274 274 * to the DDI lbolt routines), and return to the event driven mode, but will
275 275 * be prevented from doing so if lbolt_cyclic_driven() is being heavily used.
276 276 *
277 277 * lbolt_bootstrap is used during boot to serve lbolt consumers who don't wait
278 278 * for the cyclic subsystem to be intialized.
279 279 *
280 280 */
281 281 int64_t lbolt_bootstrap(void);
282 282 int64_t lbolt_event_driven(void);
283 283 int64_t lbolt_cyclic_driven(void);
284 284 int64_t (*lbolt_hybrid)(void) = lbolt_bootstrap;
285 285 uint_t lbolt_ev_to_cyclic(caddr_t, caddr_t);
286 286
287 287 /*
288 288 * lbolt's cyclic, installed by clock_init().
289 289 */
290 290 static void lbolt_cyclic(void);
291 291
292 292 /*
293 293 * Tunable to keep lbolt in cyclic driven mode. This will prevent the system
294 294 * from switching back to event driven, once it reaches cyclic mode.
295 295 */
296 296 static boolean_t lbolt_cyc_only = B_FALSE;
297 297
298 298 /*
299 299 * Cache aligned, per CPU structure with lbolt usage statistics.
300 300 */
301 301 static lbolt_cpu_t *lb_cpu;
302 302
303 303 /*
304 304 * Single, cache aligned, structure with all the information required by
305 305 * the lbolt implementation.
306 306 */
307 307 lbolt_info_t *lb_info;
308 308
309 309
310 310 int one_sec = 1; /* turned on once every second */
311 311 static int fsflushcnt; /* counter for t_fsflushr */
312 312 int dosynctodr = 1; /* patchable; enable/disable sync to TOD chip */
313 313 int tod_needsync = 0; /* need to sync tod chip with software time */
314 314 static int tod_broken = 0; /* clock chip doesn't work */
315 315 time_t boot_time = 0; /* Boot time in seconds since 1970 */
316 316 cyclic_id_t clock_cyclic; /* clock()'s cyclic_id */
317 317 cyclic_id_t deadman_cyclic; /* deadman()'s cyclic_id */
318 318
319 319 extern void clock_tick_schedule(int);
320 320
321 321 static int lgrp_ticks; /* counter to schedule lgrp load calcs */
322 322
323 323 /*
324 324 * for tod fault detection
325 325 */
326 326 #define TOD_REF_FREQ ((longlong_t)(NANOSEC))
327 327 #define TOD_STALL_THRESHOLD (TOD_REF_FREQ * 3 / 2)
328 328 #define TOD_JUMP_THRESHOLD (TOD_REF_FREQ / 2)
329 329 #define TOD_FILTER_N 4
330 330 #define TOD_FILTER_SETTLE (4 * TOD_FILTER_N)
331 331 static enum tod_fault_type tod_faulted = TOD_NOFAULT;
332 332
333 333 static int tod_status_flag = 0; /* used by tod_validate() */
334 334
335 335 static hrtime_t prev_set_tick = 0; /* gethrtime() prior to tod_set() */
336 336 static time_t prev_set_tod = 0; /* tv_sec value passed to tod_set() */
337 337
338 338 /* patchable via /etc/system */
339 339 int tod_validate_enable = 1;
340 340
341 341 /* Diagnose/Limit messages about delay(9F) called from interrupt context */
342 342 int delay_from_interrupt_diagnose = 0;
343 343 volatile uint32_t delay_from_interrupt_msg = 20;
344 344
345 345 /*
346 346 * On non-SPARC systems, TOD validation must be deferred until gethrtime
347 347 * returns non-zero values (after mach_clkinit's execution).
348 348 * On SPARC systems, it must be deferred until after hrtime_base
349 349 * and hres_last_tick are set (in the first invocation of hres_tick).
350 350 * Since in both cases the prerequisites occur before the invocation of
351 351 * tod_get() in clock(), the deferment is lifted there.
352 352 */
353 353 static boolean_t tod_validate_deferred = B_TRUE;
354 354
355 355 /*
356 356 * tod_fault_table[] must be aligned with
357 357 * enum tod_fault_type in systm.h
358 358 */
359 359 static char *tod_fault_table[] = {
360 360 "Reversed", /* TOD_REVERSED */
361 361 "Stalled", /* TOD_STALLED */
362 362 "Jumped", /* TOD_JUMPED */
363 363 "Changed in Clock Rate", /* TOD_RATECHANGED */
364 364 "Is Read-Only" /* TOD_RDONLY */
365 365 /*
366 366 * no strings needed for TOD_NOFAULT
367 367 */
368 368 };
369 369
370 370 /*
371 371 * test hook for tod broken detection in tod_validate
372 372 */
373 373 int tod_unit_test = 0;
374 374 time_t tod_test_injector;
375 375
376 376 #define CLOCK_ADJ_HIST_SIZE 4
377 377
378 378 static int adj_hist_entry;
379 379
380 380 int64_t clock_adj_hist[CLOCK_ADJ_HIST_SIZE];
381 381
382 382 static void calcloadavg(int, uint64_t *);
383 383 static int genloadavg(struct loadavg_s *);
384 384 static void loadavg_update();
385 385
386 386 void (*cmm_clock_callout)() = NULL;
387 387 void (*cpucaps_clock_callout)() = NULL;
388 388
389 389 extern clock_t clock_tick_proc_max;
390 390
391 391 static int64_t deadman_counter = 0;
392 392
393 393 static void
394 394 clock(void)
395 395 {
396 396 kthread_t *t;
397 397 uint_t nrunnable;
398 398 uint_t w_io;
399 399 cpu_t *cp;
400 400 cpupart_t *cpupart;
401 401 extern void set_freemem();
402 402 void (*funcp)();
403 403 int32_t ltemp;
404 404 int64_t lltemp;
405 405 int s;
406 406 int do_lgrp_load;
407 407 int i;
408 408 clock_t now = LBOLT_NO_ACCOUNT; /* current tick */
409 409
410 410 if (panicstr)
411 411 return;
412 412
413 413 /*
414 414 * Make sure that 'freemem' do not drift too far from the truth
415 415 */
416 416 set_freemem();
417 417
418 418
419 419 /*
420 420 * Before the section which is repeated is executed, we do
421 421 * the time delta processing which occurs every clock tick
422 422 *
423 423 * There is additional processing which happens every time
424 424 * the nanosecond counter rolls over which is described
425 425 * below - see the section which begins with : if (one_sec)
426 426 *
427 427 * This section marks the beginning of the precision-kernel
428 428 * code fragment.
429 429 *
430 430 * First, compute the phase adjustment. If the low-order bits
431 431 * (time_phase) of the update overflow, bump the higher order
432 432 * bits (time_update).
433 433 */
434 434 time_phase += time_adj;
435 435 if (time_phase <= -FINEUSEC) {
436 436 ltemp = -time_phase / SCALE_PHASE;
437 437 time_phase += ltemp * SCALE_PHASE;
438 438 s = hr_clock_lock();
439 439 timedelta -= ltemp * (NANOSEC/MICROSEC);
440 440 hr_clock_unlock(s);
441 441 } else if (time_phase >= FINEUSEC) {
442 442 ltemp = time_phase / SCALE_PHASE;
443 443 time_phase -= ltemp * SCALE_PHASE;
444 444 s = hr_clock_lock();
445 445 timedelta += ltemp * (NANOSEC/MICROSEC);
446 446 hr_clock_unlock(s);
447 447 }
448 448
449 449 /*
450 450 * End of precision-kernel code fragment which is processed
451 451 * every timer interrupt.
452 452 *
453 453 * Continue with the interrupt processing as scheduled.
454 454 */
455 455 /*
456 456 * Count the number of runnable threads and the number waiting
457 457 * for some form of I/O to complete -- gets added to
458 458 * sysinfo.waiting. To know the state of the system, must add
459 459 * wait counts from all CPUs. Also add up the per-partition
460 460 * statistics.
461 461 */
462 462 w_io = 0;
463 463 nrunnable = 0;
464 464
465 465 /*
466 466 * keep track of when to update lgrp/part loads
467 467 */
468 468
469 469 do_lgrp_load = 0;
470 470 if (lgrp_ticks++ >= hz / 10) {
471 471 lgrp_ticks = 0;
472 472 do_lgrp_load = 1;
473 473 }
474 474
475 475 if (one_sec) {
476 476 loadavg_update();
477 477 deadman_counter++;
478 478 }
479 479
480 480 /*
481 481 * First count the threads waiting on kpreempt queues in each
482 482 * CPU partition.
483 483 */
484 484
485 485 cpupart = cp_list_head;
486 486 do {
487 487 uint_t cpupart_nrunnable = cpupart->cp_kp_queue.disp_nrunnable;
488 488
489 489 cpupart->cp_updates++;
490 490 nrunnable += cpupart_nrunnable;
491 491 cpupart->cp_nrunnable_cum += cpupart_nrunnable;
492 492 if (one_sec) {
493 493 cpupart->cp_nrunning = 0;
494 494 cpupart->cp_nrunnable = cpupart_nrunnable;
495 495 }
496 496 } while ((cpupart = cpupart->cp_next) != cp_list_head);
497 497
498 498
499 499 /* Now count the per-CPU statistics. */
500 500 cp = cpu_list;
501 501 do {
502 502 uint_t cpu_nrunnable = cp->cpu_disp->disp_nrunnable;
503 503
504 504 nrunnable += cpu_nrunnable;
505 505 cpupart = cp->cpu_part;
506 506 cpupart->cp_nrunnable_cum += cpu_nrunnable;
507 507 if (one_sec) {
508 508 cpupart->cp_nrunnable += cpu_nrunnable;
509 509 /*
510 510 * Update user, system, and idle cpu times.
511 511 */
512 512 cpupart->cp_nrunning++;
513 513 /*
514 514 * w_io is used to update sysinfo.waiting during
515 515 * one_second processing below. Only gather w_io
516 516 * information when we walk the list of cpus if we're
517 517 * going to perform one_second processing.
518 518 */
519 519 w_io += CPU_STATS(cp, sys.iowait);
520 520 }
521 521
522 522 if (one_sec && (cp->cpu_flags & CPU_EXISTS)) {
523 523 int i, load, change;
524 524 hrtime_t intracct, intrused;
525 525 const hrtime_t maxnsec = 1000000000;
526 526 const int precision = 100;
527 527
528 528 /*
529 529 * Estimate interrupt load on this cpu each second.
530 530 * Computes cpu_intrload as %utilization (0-99).
531 531 */
532 532
533 533 /* add up interrupt time from all micro states */
534 534 for (intracct = 0, i = 0; i < NCMSTATES; i++)
535 535 intracct += cp->cpu_intracct[i];
536 536 scalehrtime(&intracct);
537 537
538 538 /* compute nsec used in the past second */
539 539 intrused = intracct - cp->cpu_intrlast;
540 540 cp->cpu_intrlast = intracct;
541 541
542 542 /* limit the value for safety (and the first pass) */
543 543 if (intrused >= maxnsec)
544 544 intrused = maxnsec - 1;
545 545
546 546 /* calculate %time in interrupt */
547 547 load = (precision * intrused) / maxnsec;
548 548 ASSERT(load >= 0 && load < precision);
549 549 change = cp->cpu_intrload - load;
550 550
551 551 /* jump to new max, or decay the old max */
552 552 if (change < 0)
553 553 cp->cpu_intrload = load;
554 554 else if (change > 0)
555 555 cp->cpu_intrload -= (change + 3) / 4;
556 556
557 557 DTRACE_PROBE3(cpu_intrload,
558 558 cpu_t *, cp,
559 559 hrtime_t, intracct,
560 560 hrtime_t, intrused);
561 561 }
562 562
563 563 if (do_lgrp_load &&
564 564 (cp->cpu_flags & CPU_EXISTS)) {
565 565 /*
566 566 * When updating the lgroup's load average,
567 567 * account for the thread running on the CPU.
568 568 * If the CPU is the current one, then we need
569 569 * to account for the underlying thread which
570 570 * got the clock interrupt not the thread that is
571 571 * handling the interrupt and caculating the load
572 572 * average
573 573 */
574 574 t = cp->cpu_thread;
575 575 if (CPU == cp)
576 576 t = t->t_intr;
577 577
578 578 /*
579 579 * Account for the load average for this thread if
580 580 * it isn't the idle thread or it is on the interrupt
581 581 * stack and not the current CPU handling the clock
582 582 * interrupt
583 583 */
584 584 if ((t && t != cp->cpu_idle_thread) || (CPU != cp &&
585 585 CPU_ON_INTR(cp))) {
586 586 if (t->t_lpl == cp->cpu_lpl) {
587 587 /* local thread */
588 588 cpu_nrunnable++;
589 589 } else {
590 590 /*
591 591 * This is a remote thread, charge it
592 592 * against its home lgroup. Note that
593 593 * we notice that a thread is remote
594 594 * only if it's currently executing.
595 595 * This is a reasonable approximation,
596 596 * since queued remote threads are rare.
597 597 * Note also that if we didn't charge
598 598 * it to its home lgroup, remote
599 599 * execution would often make a system
600 600 * appear balanced even though it was
601 601 * not, and thread placement/migration
602 602 * would often not be done correctly.
603 603 */
604 604 lgrp_loadavg(t->t_lpl,
605 605 LGRP_LOADAVG_IN_THREAD_MAX, 0);
606 606 }
607 607 }
608 608 lgrp_loadavg(cp->cpu_lpl,
609 609 cpu_nrunnable * LGRP_LOADAVG_IN_THREAD_MAX, 1);
610 610 }
611 611 } while ((cp = cp->cpu_next) != cpu_list);
612 612
613 613 clock_tick_schedule(one_sec);
614 614
615 615 /*
616 616 * Check for a callout that needs be called from the clock
617 617 * thread to support the membership protocol in a clustered
618 618 * system. Copy the function pointer so that we can reset
619 619 * this to NULL if needed.
620 620 */
621 621 if ((funcp = cmm_clock_callout) != NULL)
622 622 (*funcp)();
623 623
624 624 if ((funcp = cpucaps_clock_callout) != NULL)
625 625 (*funcp)();
626 626
627 627 /*
628 628 * Wakeup the cageout thread waiters once per second.
629 629 */
630 630 if (one_sec)
631 631 kcage_tick();
632 632
633 633 if (one_sec) {
634 634
635 635 int drift, absdrift;
636 636 timestruc_t tod;
637 637 int s;
638 638
639 639 /*
640 640 * Beginning of precision-kernel code fragment executed
641 641 * every second.
642 642 *
643 643 * On rollover of the second the phase adjustment to be
644 644 * used for the next second is calculated. Also, the
645 645 * maximum error is increased by the tolerance. If the
646 646 * PPS frequency discipline code is present, the phase is
647 647 * increased to compensate for the CPU clock oscillator
648 648 * frequency error.
649 649 *
650 650 * On a 32-bit machine and given parameters in the timex.h
651 651 * header file, the maximum phase adjustment is +-512 ms
652 652 * and maximum frequency offset is (a tad less than)
653 653 * +-512 ppm. On a 64-bit machine, you shouldn't need to ask.
654 654 */
655 655 time_maxerror += time_tolerance / SCALE_USEC;
656 656
657 657 /*
658 658 * Leap second processing. If in leap-insert state at
659 659 * the end of the day, the system clock is set back one
660 660 * second; if in leap-delete state, the system clock is
661 661 * set ahead one second. The microtime() routine or
662 662 * external clock driver will insure that reported time
663 663 * is always monotonic. The ugly divides should be
664 664 * replaced.
665 665 */
666 666 switch (time_state) {
667 667
668 668 case TIME_OK:
669 669 if (time_status & STA_INS)
670 670 time_state = TIME_INS;
671 671 else if (time_status & STA_DEL)
672 672 time_state = TIME_DEL;
673 673 break;
674 674
675 675 case TIME_INS:
676 676 if (hrestime.tv_sec % 86400 == 0) {
677 677 s = hr_clock_lock();
678 678 hrestime.tv_sec--;
679 679 hr_clock_unlock(s);
680 680 time_state = TIME_OOP;
681 681 }
682 682 break;
683 683
684 684 case TIME_DEL:
685 685 if ((hrestime.tv_sec + 1) % 86400 == 0) {
686 686 s = hr_clock_lock();
687 687 hrestime.tv_sec++;
688 688 hr_clock_unlock(s);
689 689 time_state = TIME_WAIT;
690 690 }
691 691 break;
692 692
693 693 case TIME_OOP:
694 694 time_state = TIME_WAIT;
695 695 break;
696 696
697 697 case TIME_WAIT:
698 698 if (!(time_status & (STA_INS | STA_DEL)))
699 699 time_state = TIME_OK;
700 700 default:
701 701 break;
702 702 }
703 703
704 704 /*
705 705 * Compute the phase adjustment for the next second. In
706 706 * PLL mode, the offset is reduced by a fixed factor
707 707 * times the time constant. In FLL mode the offset is
708 708 * used directly. In either mode, the maximum phase
709 709 * adjustment for each second is clamped so as to spread
710 710 * the adjustment over not more than the number of
711 711 * seconds between updates.
712 712 */
713 713 if (time_offset == 0)
714 714 time_adj = 0;
715 715 else if (time_offset < 0) {
716 716 lltemp = -time_offset;
717 717 if (!(time_status & STA_FLL)) {
718 718 if ((1 << time_constant) >= SCALE_KG)
719 719 lltemp *= (1 << time_constant) /
720 720 SCALE_KG;
721 721 else
722 722 lltemp = (lltemp / SCALE_KG) >>
723 723 time_constant;
724 724 }
725 725 if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
726 726 lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
727 727 time_offset += lltemp;
728 728 time_adj = -(lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
729 729 } else {
730 730 lltemp = time_offset;
731 731 if (!(time_status & STA_FLL)) {
732 732 if ((1 << time_constant) >= SCALE_KG)
733 733 lltemp *= (1 << time_constant) /
734 734 SCALE_KG;
735 735 else
736 736 lltemp = (lltemp / SCALE_KG) >>
737 737 time_constant;
738 738 }
739 739 if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
740 740 lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
741 741 time_offset -= lltemp;
742 742 time_adj = (lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
743 743 }
744 744
745 745 /*
746 746 * Compute the frequency estimate and additional phase
747 747 * adjustment due to frequency error for the next
748 748 * second. When the PPS signal is engaged, gnaw on the
749 749 * watchdog counter and update the frequency computed by
750 750 * the pll and the PPS signal.
751 751 */
752 752 pps_valid++;
753 753 if (pps_valid == PPS_VALID) {
754 754 pps_jitter = MAXTIME;
755 755 pps_stabil = MAXFREQ;
756 756 time_status &= ~(STA_PPSSIGNAL | STA_PPSJITTER |
757 757 STA_PPSWANDER | STA_PPSERROR);
758 758 }
759 759 lltemp = time_freq + pps_freq;
760 760
761 761 if (lltemp)
762 762 time_adj += (lltemp * SCALE_PHASE) / (SCALE_USEC * hz);
763 763
764 764 /*
765 765 * End of precision kernel-code fragment
766 766 *
767 767 * The section below should be modified if we are planning
768 768 * to use NTP for synchronization.
769 769 *
770 770 * Note: the clock synchronization code now assumes
771 771 * the following:
772 772 * - if dosynctodr is 1, then compute the drift between
773 773 * the tod chip and software time and adjust one or
774 774 * the other depending on the circumstances
775 775 *
776 776 * - if dosynctodr is 0, then the tod chip is independent
777 777 * of the software clock and should not be adjusted,
778 778 * but allowed to free run. this allows NTP to sync.
779 779 * hrestime without any interference from the tod chip.
780 780 */
781 781
782 782 tod_validate_deferred = B_FALSE;
783 783 mutex_enter(&tod_lock);
784 784 tod = tod_get();
785 785 drift = tod.tv_sec - hrestime.tv_sec;
786 786 absdrift = (drift >= 0) ? drift : -drift;
787 787 if (tod_needsync || absdrift > 1) {
788 788 int s;
789 789 if (absdrift > 2) {
790 790 if (!tod_broken && tod_faulted == TOD_NOFAULT) {
791 791 s = hr_clock_lock();
792 792 hrestime = tod;
793 793 membar_enter(); /* hrestime visible */
794 794 timedelta = 0;
795 795 timechanged++;
796 796 tod_needsync = 0;
797 797 hr_clock_unlock(s);
798 798 callout_hrestime();
799 799
800 800 }
801 801 } else {
802 802 if (tod_needsync || !dosynctodr) {
803 803 gethrestime(&tod);
804 804 tod_set(tod);
805 805 s = hr_clock_lock();
806 806 if (timedelta == 0)
807 807 tod_needsync = 0;
808 808 hr_clock_unlock(s);
809 809 } else {
810 810 /*
811 811 * If the drift is 2 seconds on the
812 812 * money, then the TOD is adjusting
813 813 * the clock; record that.
814 814 */
815 815 clock_adj_hist[adj_hist_entry++ %
816 816 CLOCK_ADJ_HIST_SIZE] = now;
817 817 s = hr_clock_lock();
818 818 timedelta = (int64_t)drift*NANOSEC;
819 819 hr_clock_unlock(s);
820 820 }
821 821 }
822 822 }
823 823 one_sec = 0;
824 824 time = gethrestime_sec(); /* for crusty old kmem readers */
825 825 mutex_exit(&tod_lock);
826 826
827 827 /*
828 828 * Some drivers still depend on this... XXX
829 829 */
830 830 cv_broadcast(&lbolt_cv);
831 831
832 832 vminfo.freemem += freemem;
833 833 {
834 834 pgcnt_t maxswap, resv, free;
835 835 pgcnt_t avail =
836 836 MAX((spgcnt_t)(availrmem - swapfs_minfree), 0);
837 837
838 838 maxswap = k_anoninfo.ani_mem_resv +
839 839 k_anoninfo.ani_max +avail;
840 840 /* Update ani_free */
841 841 set_anoninfo();
842 842 free = k_anoninfo.ani_free + avail;
843 843 resv = k_anoninfo.ani_phys_resv +
844 844 k_anoninfo.ani_mem_resv;
845 845
846 846 vminfo.swap_resv += resv;
847 847 /* number of reserved and allocated pages */
848 848 #ifdef DEBUG
849 849 if (maxswap < free)
850 850 cmn_err(CE_WARN, "clock: maxswap < free");
851 851 if (maxswap < resv)
852 852 cmn_err(CE_WARN, "clock: maxswap < resv");
853 853 #endif
854 854 vminfo.swap_alloc += maxswap - free;
855 855 vminfo.swap_avail += maxswap - resv;
856 856 vminfo.swap_free += free;
857 857 }
858 858 vminfo.updates++;
859 859 if (nrunnable) {
860 860 sysinfo.runque += nrunnable;
861 861 sysinfo.runocc++;
862 862 }
863 863 if (nswapped) {
864 864 sysinfo.swpque += nswapped;
865 865 sysinfo.swpocc++;
866 866 }
867 867 sysinfo.waiting += w_io;
868 868 sysinfo.updates++;
869 869
870 870 /*
871 871 * Wake up fsflush to write out DELWRI
872 872 * buffers, dirty pages and other cached
873 873 * administrative data, e.g. inodes.
874 874 */
875 875 if (--fsflushcnt <= 0) {
876 876 fsflushcnt = tune.t_fsflushr;
877 877 cv_signal(&fsflush_cv);
878 878 }
879 879
880 880 vmmeter();
881 881 calcloadavg(genloadavg(&loadavg), hp_avenrun);
882 882 for (i = 0; i < 3; i++)
883 883 /*
884 884 * At the moment avenrun[] can only hold 31
885 885 * bits of load average as it is a signed
886 886 * int in the API. We need to ensure that
887 887 * hp_avenrun[i] >> (16 - FSHIFT) will not be
888 888 * too large. If it is, we put the largest value
889 889 * that we can use into avenrun[i]. This is
890 890 * kludgey, but about all we can do until we
891 891 * avenrun[] is declared as an array of uint64[]
892 892 */
893 893 if (hp_avenrun[i] < ((uint64_t)1<<(31+16-FSHIFT)))
894 894 avenrun[i] = (int32_t)(hp_avenrun[i] >>
895 895 (16 - FSHIFT));
896 896 else
897 897 avenrun[i] = 0x7fffffff;
898 898
899 899 cpupart = cp_list_head;
900 900 do {
901 901 calcloadavg(genloadavg(&cpupart->cp_loadavg),
902 902 cpupart->cp_hp_avenrun);
903 903 } while ((cpupart = cpupart->cp_next) != cp_list_head);
904 904
905 905 /*
906 906 * Wake up the swapper thread if necessary.
907 907 */
908 908 if (runin ||
909 909 (runout && (avefree < desfree || wake_sched_sec))) {
910 910 t = &t0;
911 911 thread_lock(t);
912 912 if (t->t_state == TS_STOPPED) {
913 913 runin = runout = 0;
914 914 wake_sched_sec = 0;
915 915 t->t_whystop = 0;
916 916 t->t_whatstop = 0;
917 917 t->t_schedflag &= ~TS_ALLSTART;
918 918 THREAD_TRANSITION(t);
919 919 setfrontdq(t);
920 920 }
921 921 thread_unlock(t);
922 922 }
923 923 }
924 924
925 925 /*
926 926 * Wake up the swapper if any high priority swapped-out threads
927 927 * became runable during the last tick.
928 928 */
929 929 if (wake_sched) {
930 930 t = &t0;
931 931 thread_lock(t);
932 932 if (t->t_state == TS_STOPPED) {
933 933 runin = runout = 0;
934 934 wake_sched = 0;
935 935 t->t_whystop = 0;
936 936 t->t_whatstop = 0;
937 937 t->t_schedflag &= ~TS_ALLSTART;
938 938 THREAD_TRANSITION(t);
939 939 setfrontdq(t);
940 940 }
941 941 thread_unlock(t);
942 942 }
943 943 }
944 944
945 945 void
946 946 clock_init(void)
947 947 {
948 948 cyc_handler_t clk_hdlr, lbolt_hdlr;
949 949 cyc_time_t clk_when, lbolt_when;
950 950 int i, sz;
951 951 intptr_t buf;
952 952
953 953 /*
954 954 * Setup handler and timer for the clock cyclic.
955 955 */
956 956 clk_hdlr.cyh_func = (cyc_func_t)clock;
957 957 clk_hdlr.cyh_level = CY_LOCK_LEVEL;
958 958 clk_hdlr.cyh_arg = NULL;
959 959
960 960 clk_when.cyt_when = 0;
961 961 clk_when.cyt_interval = nsec_per_tick;
962 962
963 963 /*
964 964 * The lbolt cyclic will be reprogramed to fire at a nsec_per_tick
965 965 * interval to satisfy performance needs of the DDI lbolt consumers.
966 966 * It is off by default.
967 967 */
968 968 lbolt_hdlr.cyh_func = (cyc_func_t)lbolt_cyclic;
969 969 lbolt_hdlr.cyh_level = CY_LOCK_LEVEL;
970 970 lbolt_hdlr.cyh_arg = NULL;
971 971
972 972 lbolt_when.cyt_interval = nsec_per_tick;
973 973
974 974 /*
975 975 * Allocate cache line aligned space for the per CPU lbolt data and
976 976 * lbolt info structures, and initialize them with their default
977 977 * values. Note that these structures are also cache line sized.
978 978 */
979 979 sz = sizeof (lbolt_info_t) + CPU_CACHE_COHERENCE_SIZE;
980 980 buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
981 981 lb_info = (lbolt_info_t *)P2ROUNDUP(buf, CPU_CACHE_COHERENCE_SIZE);
982 982
983 983 if (hz != HZ_DEFAULT)
984 984 lb_info->lbi_thresh_interval = LBOLT_THRESH_INTERVAL *
985 985 hz/HZ_DEFAULT;
986 986 else
987 987 lb_info->lbi_thresh_interval = LBOLT_THRESH_INTERVAL;
988 988
989 989 lb_info->lbi_thresh_calls = LBOLT_THRESH_CALLS;
990 990
991 991 sz = (sizeof (lbolt_cpu_t) * max_ncpus) + CPU_CACHE_COHERENCE_SIZE;
992 992 buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
993 993 lb_cpu = (lbolt_cpu_t *)P2ROUNDUP(buf, CPU_CACHE_COHERENCE_SIZE);
994 994
995 995 for (i = 0; i < max_ncpus; i++)
996 996 lb_cpu[i].lbc_counter = lb_info->lbi_thresh_calls;
997 997
998 998 /*
999 999 * Install the softint used to switch between event and cyclic driven
1000 1000 * lbolt. We use a soft interrupt to make sure the context of the
1001 1001 * cyclic reprogram call is safe.
1002 1002 */
1003 1003 lbolt_softint_add();
1004 1004
1005 1005 /*
1006 1006 * Since the hybrid lbolt implementation is based on a hardware counter
1007 1007 * that is reset at every hardware reboot and that we'd like to have
1008 1008 * the lbolt value starting at zero after both a hardware and a fast
1009 1009 * reboot, we calculate the number of clock ticks the system's been up
1010 1010 * and store it in the lbi_debug_time field of the lbolt info structure.
1011 1011 * The value of this field will be subtracted from lbolt before
1012 1012 * returning it.
1013 1013 */
1014 1014 lb_info->lbi_internal = lb_info->lbi_debug_time =
1015 1015 (gethrtime()/nsec_per_tick);
1016 1016
1017 1017 /*
1018 1018 * lbolt_hybrid points at lbolt_bootstrap until now. The LBOLT_* macros
1019 1019 * and lbolt_debug_{enter,return} use this value as an indication that
1020 1020 * the initializaion above hasn't been completed. Setting lbolt_hybrid
1021 1021 * to either lbolt_{cyclic,event}_driven here signals those code paths
1022 1022 * that the lbolt related structures can be used.
1023 1023 */
1024 1024 if (lbolt_cyc_only) {
1025 1025 lbolt_when.cyt_when = 0;
1026 1026 lbolt_hybrid = lbolt_cyclic_driven;
1027 1027 } else {
1028 1028 lbolt_when.cyt_when = CY_INFINITY;
1029 1029 lbolt_hybrid = lbolt_event_driven;
1030 1030 }
1031 1031
1032 1032 /*
1033 1033 * Grab cpu_lock and install all three cyclics.
1034 1034 */
1035 1035 mutex_enter(&cpu_lock);
1036 1036
1037 1037 clock_cyclic = cyclic_add(&clk_hdlr, &clk_when);
1038 1038 lb_info->id.lbi_cyclic_id = cyclic_add(&lbolt_hdlr, &lbolt_when);
1039 1039
1040 1040 mutex_exit(&cpu_lock);
1041 1041 }
1042 1042
1043 1043 /*
1044 1044 * Called before calcloadavg to get 10-sec moving loadavg together
1045 1045 */
1046 1046
1047 1047 static int
1048 1048 genloadavg(struct loadavg_s *avgs)
1049 1049 {
1050 1050 int avg;
1051 1051 int spos; /* starting position */
1052 1052 int cpos; /* moving current position */
1053 1053 int i;
1054 1054 int slen;
1055 1055 hrtime_t hr_avg;
1056 1056
1057 1057 /* 10-second snapshot, calculate first positon */
1058 1058 if (avgs->lg_len == 0) {
1059 1059 return (0);
1060 1060 }
1061 1061 slen = avgs->lg_len < S_MOVAVG_SZ ? avgs->lg_len : S_MOVAVG_SZ;
1062 1062
1063 1063 spos = (avgs->lg_cur - 1) >= 0 ? avgs->lg_cur - 1 :
1064 1064 S_LOADAVG_SZ + (avgs->lg_cur - 1);
1065 1065 for (i = hr_avg = 0; i < slen; i++) {
1066 1066 cpos = (spos - i) >= 0 ? spos - i : S_LOADAVG_SZ + (spos - i);
1067 1067 hr_avg += avgs->lg_loads[cpos];
1068 1068 }
1069 1069
1070 1070 hr_avg = hr_avg / slen;
1071 1071 avg = hr_avg / (NANOSEC / LGRP_LOADAVG_IN_THREAD_MAX);
1072 1072
1073 1073 return (avg);
1074 1074 }
1075 1075
1076 1076 /*
1077 1077 * Run every second from clock () to update the loadavg count available to the
1078 1078 * system and cpu-partitions.
1079 1079 *
1080 1080 * This works by sampling the previous usr, sys, wait time elapsed,
1081 1081 * computing a delta, and adding that delta to the elapsed usr, sys,
1082 1082 * wait increase.
1083 1083 */
1084 1084
1085 1085 static void
1086 1086 loadavg_update()
1087 1087 {
1088 1088 cpu_t *cp;
1089 1089 cpupart_t *cpupart;
1090 1090 hrtime_t cpu_total;
1091 1091 int prev;
1092 1092
1093 1093 cp = cpu_list;
1094 1094 loadavg.lg_total = 0;
1095 1095
1096 1096 /*
1097 1097 * first pass totals up per-cpu statistics for system and cpu
1098 1098 * partitions
1099 1099 */
1100 1100
1101 1101 do {
1102 1102 struct loadavg_s *lavg;
1103 1103
1104 1104 lavg = &cp->cpu_loadavg;
1105 1105
1106 1106 cpu_total = cp->cpu_acct[CMS_USER] +
1107 1107 cp->cpu_acct[CMS_SYSTEM] + cp->cpu_waitrq;
1108 1108 /* compute delta against last total */
1109 1109 scalehrtime(&cpu_total);
1110 1110 prev = (lavg->lg_cur - 1) >= 0 ? lavg->lg_cur - 1 :
1111 1111 S_LOADAVG_SZ + (lavg->lg_cur - 1);
1112 1112 if (lavg->lg_loads[prev] <= 0) {
1113 1113 lavg->lg_loads[lavg->lg_cur] = cpu_total;
1114 1114 cpu_total = 0;
1115 1115 } else {
1116 1116 lavg->lg_loads[lavg->lg_cur] = cpu_total;
1117 1117 cpu_total = cpu_total - lavg->lg_loads[prev];
1118 1118 if (cpu_total < 0)
1119 1119 cpu_total = 0;
1120 1120 }
1121 1121
1122 1122 lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
1123 1123 lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
1124 1124 lavg->lg_len + 1 : S_LOADAVG_SZ;
1125 1125
1126 1126 loadavg.lg_total += cpu_total;
1127 1127 cp->cpu_part->cp_loadavg.lg_total += cpu_total;
1128 1128
1129 1129 } while ((cp = cp->cpu_next) != cpu_list);
1130 1130
1131 1131 loadavg.lg_loads[loadavg.lg_cur] = loadavg.lg_total;
1132 1132 loadavg.lg_cur = (loadavg.lg_cur + 1) % S_LOADAVG_SZ;
1133 1133 loadavg.lg_len = (loadavg.lg_len + 1) < S_LOADAVG_SZ ?
1134 1134 loadavg.lg_len + 1 : S_LOADAVG_SZ;
1135 1135 /*
1136 1136 * Second pass updates counts
1137 1137 */
1138 1138 cpupart = cp_list_head;
1139 1139
1140 1140 do {
1141 1141 struct loadavg_s *lavg;
1142 1142
1143 1143 lavg = &cpupart->cp_loadavg;
1144 1144 lavg->lg_loads[lavg->lg_cur] = lavg->lg_total;
1145 1145 lavg->lg_total = 0;
1146 1146 lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
1147 1147 lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
1148 1148 lavg->lg_len + 1 : S_LOADAVG_SZ;
1149 1149
1150 1150 } while ((cpupart = cpupart->cp_next) != cp_list_head);
1151 1151
1152 1152 }
1153 1153
1154 1154 /*
1155 1155 * clock_update() - local clock update
1156 1156 *
1157 1157 * This routine is called by ntp_adjtime() to update the local clock
1158 1158 * phase and frequency. The implementation is of an
1159 1159 * adaptive-parameter, hybrid phase/frequency-lock loop (PLL/FLL). The
1160 1160 * routine computes new time and frequency offset estimates for each
1161 1161 * call. The PPS signal itself determines the new time offset,
1162 1162 * instead of the calling argument. Presumably, calls to
1163 1163 * ntp_adjtime() occur only when the caller believes the local clock
1164 1164 * is valid within some bound (+-128 ms with NTP). If the caller's
1165 1165 * time is far different than the PPS time, an argument will ensue,
1166 1166 * and it's not clear who will lose.
1167 1167 *
1168 1168 * For uncompensated quartz crystal oscillatores and nominal update
1169 1169 * intervals less than 1024 s, operation should be in phase-lock mode
1170 1170 * (STA_FLL = 0), where the loop is disciplined to phase. For update
1171 1171 * intervals greater than this, operation should be in frequency-lock
1172 1172 * mode (STA_FLL = 1), where the loop is disciplined to frequency.
1173 1173 *
1174 1174 * Note: mutex(&tod_lock) is in effect.
1175 1175 */
1176 1176 void
1177 1177 clock_update(int offset)
1178 1178 {
1179 1179 int ltemp, mtemp, s;
1180 1180
1181 1181 ASSERT(MUTEX_HELD(&tod_lock));
1182 1182
1183 1183 if (!(time_status & STA_PLL) && !(time_status & STA_PPSTIME))
1184 1184 return;
1185 1185 ltemp = offset;
1186 1186 if ((time_status & STA_PPSTIME) && (time_status & STA_PPSSIGNAL))
1187 1187 ltemp = pps_offset;
1188 1188
1189 1189 /*
1190 1190 * Scale the phase adjustment and clamp to the operating range.
1191 1191 */
1192 1192 if (ltemp > MAXPHASE)
1193 1193 time_offset = MAXPHASE * SCALE_UPDATE;
1194 1194 else if (ltemp < -MAXPHASE)
1195 1195 time_offset = -(MAXPHASE * SCALE_UPDATE);
1196 1196 else
1197 1197 time_offset = ltemp * SCALE_UPDATE;
1198 1198
1199 1199 /*
1200 1200 * Select whether the frequency is to be controlled and in which
1201 1201 * mode (PLL or FLL). Clamp to the operating range. Ugly
1202 1202 * multiply/divide should be replaced someday.
1203 1203 */
1204 1204 if (time_status & STA_FREQHOLD || time_reftime == 0)
1205 1205 time_reftime = hrestime.tv_sec;
1206 1206
1207 1207 mtemp = hrestime.tv_sec - time_reftime;
1208 1208 time_reftime = hrestime.tv_sec;
1209 1209
1210 1210 if (time_status & STA_FLL) {
1211 1211 if (mtemp >= MINSEC) {
1212 1212 ltemp = ((time_offset / mtemp) * (SCALE_USEC /
1213 1213 SCALE_UPDATE));
1214 1214 if (ltemp)
1215 1215 time_freq += ltemp / SCALE_KH;
1216 1216 }
1217 1217 } else {
1218 1218 if (mtemp < MAXSEC) {
1219 1219 ltemp *= mtemp;
1220 1220 if (ltemp)
1221 1221 time_freq += (int)(((int64_t)ltemp *
1222 1222 SCALE_USEC) / SCALE_KF)
1223 1223 / (1 << (time_constant * 2));
1224 1224 }
1225 1225 }
1226 1226 if (time_freq > time_tolerance)
1227 1227 time_freq = time_tolerance;
1228 1228 else if (time_freq < -time_tolerance)
1229 1229 time_freq = -time_tolerance;
1230 1230
1231 1231 s = hr_clock_lock();
1232 1232 tod_needsync = 1;
1233 1233 hr_clock_unlock(s);
1234 1234 }
1235 1235
1236 1236 /*
1237 1237 * ddi_hardpps() - discipline CPU clock oscillator to external PPS signal
1238 1238 *
1239 1239 * This routine is called at each PPS interrupt in order to discipline
1240 1240 * the CPU clock oscillator to the PPS signal. It measures the PPS phase
1241 1241 * and leaves it in a handy spot for the clock() routine. It
1242 1242 * integrates successive PPS phase differences and calculates the
1243 1243 * frequency offset. This is used in clock() to discipline the CPU
1244 1244 * clock oscillator so that intrinsic frequency error is cancelled out.
1245 1245 * The code requires the caller to capture the time and hardware counter
1246 1246 * value at the on-time PPS signal transition.
1247 1247 *
1248 1248 * Note that, on some Unix systems, this routine runs at an interrupt
1249 1249 * priority level higher than the timer interrupt routine clock().
1250 1250 * Therefore, the variables used are distinct from the clock()
1251 1251 * variables, except for certain exceptions: The PPS frequency pps_freq
1252 1252 * and phase pps_offset variables are determined by this routine and
1253 1253 * updated atomically. The time_tolerance variable can be considered a
1254 1254 * constant, since it is infrequently changed, and then only when the
1255 1255 * PPS signal is disabled. The watchdog counter pps_valid is updated
1256 1256 * once per second by clock() and is atomically cleared in this
1257 1257 * routine.
1258 1258 *
1259 1259 * tvp is the time of the last tick; usec is a microsecond count since the
1260 1260 * last tick.
1261 1261 *
1262 1262 * Note: In Solaris systems, the tick value is actually given by
1263 1263 * usec_per_tick. This is called from the serial driver cdintr(),
1264 1264 * or equivalent, at a high PIL. Because the kernel keeps a
1265 1265 * highresolution time, the following code can accept either
1266 1266 * the traditional argument pair, or the current highres timestamp
1267 1267 * in tvp and zero in usec.
1268 1268 */
1269 1269 void
1270 1270 ddi_hardpps(struct timeval *tvp, int usec)
1271 1271 {
1272 1272 int u_usec, v_usec, bigtick;
1273 1273 time_t cal_sec;
1274 1274 int cal_usec;
1275 1275
1276 1276 /*
1277 1277 * An occasional glitch can be produced when the PPS interrupt
1278 1278 * occurs in the clock() routine before the time variable is
1279 1279 * updated. Here the offset is discarded when the difference
1280 1280 * between it and the last one is greater than tick/2, but not
1281 1281 * if the interval since the first discard exceeds 30 s.
1282 1282 */
1283 1283 time_status |= STA_PPSSIGNAL;
1284 1284 time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR);
1285 1285 pps_valid = 0;
1286 1286 u_usec = -tvp->tv_usec;
1287 1287 if (u_usec < -(MICROSEC/2))
1288 1288 u_usec += MICROSEC;
1289 1289 v_usec = pps_offset - u_usec;
1290 1290 if (v_usec < 0)
1291 1291 v_usec = -v_usec;
1292 1292 if (v_usec > (usec_per_tick >> 1)) {
1293 1293 if (pps_glitch > MAXGLITCH) {
1294 1294 pps_glitch = 0;
1295 1295 pps_tf[2] = u_usec;
1296 1296 pps_tf[1] = u_usec;
1297 1297 } else {
1298 1298 pps_glitch++;
1299 1299 u_usec = pps_offset;
1300 1300 }
1301 1301 } else
1302 1302 pps_glitch = 0;
1303 1303
1304 1304 /*
1305 1305 * A three-stage median filter is used to help deglitch the pps
1306 1306 * time. The median sample becomes the time offset estimate; the
1307 1307 * difference between the other two samples becomes the time
1308 1308 * dispersion (jitter) estimate.
1309 1309 */
1310 1310 pps_tf[2] = pps_tf[1];
1311 1311 pps_tf[1] = pps_tf[0];
1312 1312 pps_tf[0] = u_usec;
1313 1313 if (pps_tf[0] > pps_tf[1]) {
1314 1314 if (pps_tf[1] > pps_tf[2]) {
1315 1315 pps_offset = pps_tf[1]; /* 0 1 2 */
1316 1316 v_usec = pps_tf[0] - pps_tf[2];
1317 1317 } else if (pps_tf[2] > pps_tf[0]) {
1318 1318 pps_offset = pps_tf[0]; /* 2 0 1 */
1319 1319 v_usec = pps_tf[2] - pps_tf[1];
1320 1320 } else {
1321 1321 pps_offset = pps_tf[2]; /* 0 2 1 */
1322 1322 v_usec = pps_tf[0] - pps_tf[1];
1323 1323 }
1324 1324 } else {
1325 1325 if (pps_tf[1] < pps_tf[2]) {
1326 1326 pps_offset = pps_tf[1]; /* 2 1 0 */
1327 1327 v_usec = pps_tf[2] - pps_tf[0];
1328 1328 } else if (pps_tf[2] < pps_tf[0]) {
1329 1329 pps_offset = pps_tf[0]; /* 1 0 2 */
1330 1330 v_usec = pps_tf[1] - pps_tf[2];
1331 1331 } else {
1332 1332 pps_offset = pps_tf[2]; /* 1 2 0 */
1333 1333 v_usec = pps_tf[1] - pps_tf[0];
1334 1334 }
1335 1335 }
1336 1336 if (v_usec > MAXTIME)
1337 1337 pps_jitcnt++;
1338 1338 v_usec = (v_usec << PPS_AVG) - pps_jitter;
1339 1339 pps_jitter += v_usec / (1 << PPS_AVG);
1340 1340 if (pps_jitter > (MAXTIME >> 1))
1341 1341 time_status |= STA_PPSJITTER;
1342 1342
1343 1343 /*
1344 1344 * During the calibration interval adjust the starting time when
1345 1345 * the tick overflows. At the end of the interval compute the
1346 1346 * duration of the interval and the difference of the hardware
1347 1347 * counters at the beginning and end of the interval. This code
1348 1348 * is deliciously complicated by the fact valid differences may
1349 1349 * exceed the value of tick when using long calibration
1350 1350 * intervals and small ticks. Note that the counter can be
1351 1351 * greater than tick if caught at just the wrong instant, but
1352 1352 * the values returned and used here are correct.
1353 1353 */
1354 1354 bigtick = (int)usec_per_tick * SCALE_USEC;
1355 1355 pps_usec -= pps_freq;
1356 1356 if (pps_usec >= bigtick)
1357 1357 pps_usec -= bigtick;
1358 1358 if (pps_usec < 0)
1359 1359 pps_usec += bigtick;
1360 1360 pps_time.tv_sec++;
1361 1361 pps_count++;
1362 1362 if (pps_count < (1 << pps_shift))
1363 1363 return;
1364 1364 pps_count = 0;
1365 1365 pps_calcnt++;
1366 1366 u_usec = usec * SCALE_USEC;
1367 1367 v_usec = pps_usec - u_usec;
1368 1368 if (v_usec >= bigtick >> 1)
1369 1369 v_usec -= bigtick;
1370 1370 if (v_usec < -(bigtick >> 1))
1371 1371 v_usec += bigtick;
1372 1372 if (v_usec < 0)
1373 1373 v_usec = -(-v_usec >> pps_shift);
1374 1374 else
1375 1375 v_usec = v_usec >> pps_shift;
1376 1376 pps_usec = u_usec;
1377 1377 cal_sec = tvp->tv_sec;
1378 1378 cal_usec = tvp->tv_usec;
1379 1379 cal_sec -= pps_time.tv_sec;
1380 1380 cal_usec -= pps_time.tv_usec;
1381 1381 if (cal_usec < 0) {
1382 1382 cal_usec += MICROSEC;
1383 1383 cal_sec--;
1384 1384 }
1385 1385 pps_time = *tvp;
1386 1386
1387 1387 /*
1388 1388 * Check for lost interrupts, noise, excessive jitter and
1389 1389 * excessive frequency error. The number of timer ticks during
1390 1390 * the interval may vary +-1 tick. Add to this a margin of one
1391 1391 * tick for the PPS signal jitter and maximum frequency
1392 1392 * deviation. If the limits are exceeded, the calibration
1393 1393 * interval is reset to the minimum and we start over.
1394 1394 */
1395 1395 u_usec = (int)usec_per_tick << 1;
1396 1396 if (!((cal_sec == -1 && cal_usec > (MICROSEC - u_usec)) ||
1397 1397 (cal_sec == 0 && cal_usec < u_usec)) ||
1398 1398 v_usec > time_tolerance || v_usec < -time_tolerance) {
1399 1399 pps_errcnt++;
1400 1400 pps_shift = PPS_SHIFT;
1401 1401 pps_intcnt = 0;
1402 1402 time_status |= STA_PPSERROR;
1403 1403 return;
1404 1404 }
1405 1405
1406 1406 /*
1407 1407 * A three-stage median filter is used to help deglitch the pps
1408 1408 * frequency. The median sample becomes the frequency offset
1409 1409 * estimate; the difference between the other two samples
1410 1410 * becomes the frequency dispersion (stability) estimate.
1411 1411 */
1412 1412 pps_ff[2] = pps_ff[1];
1413 1413 pps_ff[1] = pps_ff[0];
1414 1414 pps_ff[0] = v_usec;
1415 1415 if (pps_ff[0] > pps_ff[1]) {
1416 1416 if (pps_ff[1] > pps_ff[2]) {
1417 1417 u_usec = pps_ff[1]; /* 0 1 2 */
1418 1418 v_usec = pps_ff[0] - pps_ff[2];
1419 1419 } else if (pps_ff[2] > pps_ff[0]) {
1420 1420 u_usec = pps_ff[0]; /* 2 0 1 */
1421 1421 v_usec = pps_ff[2] - pps_ff[1];
1422 1422 } else {
1423 1423 u_usec = pps_ff[2]; /* 0 2 1 */
1424 1424 v_usec = pps_ff[0] - pps_ff[1];
1425 1425 }
1426 1426 } else {
1427 1427 if (pps_ff[1] < pps_ff[2]) {
1428 1428 u_usec = pps_ff[1]; /* 2 1 0 */
1429 1429 v_usec = pps_ff[2] - pps_ff[0];
1430 1430 } else if (pps_ff[2] < pps_ff[0]) {
1431 1431 u_usec = pps_ff[0]; /* 1 0 2 */
1432 1432 v_usec = pps_ff[1] - pps_ff[2];
1433 1433 } else {
1434 1434 u_usec = pps_ff[2]; /* 1 2 0 */
1435 1435 v_usec = pps_ff[1] - pps_ff[0];
1436 1436 }
1437 1437 }
1438 1438
1439 1439 /*
1440 1440 * Here the frequency dispersion (stability) is updated. If it
1441 1441 * is less than one-fourth the maximum (MAXFREQ), the frequency
1442 1442 * offset is updated as well, but clamped to the tolerance. It
1443 1443 * will be processed later by the clock() routine.
1444 1444 */
1445 1445 v_usec = (v_usec >> 1) - pps_stabil;
1446 1446 if (v_usec < 0)
1447 1447 pps_stabil -= -v_usec >> PPS_AVG;
1448 1448 else
1449 1449 pps_stabil += v_usec >> PPS_AVG;
1450 1450 if (pps_stabil > MAXFREQ >> 2) {
1451 1451 pps_stbcnt++;
1452 1452 time_status |= STA_PPSWANDER;
1453 1453 return;
1454 1454 }
1455 1455 if (time_status & STA_PPSFREQ) {
1456 1456 if (u_usec < 0) {
1457 1457 pps_freq -= -u_usec >> PPS_AVG;
1458 1458 if (pps_freq < -time_tolerance)
1459 1459 pps_freq = -time_tolerance;
1460 1460 u_usec = -u_usec;
1461 1461 } else {
1462 1462 pps_freq += u_usec >> PPS_AVG;
1463 1463 if (pps_freq > time_tolerance)
1464 1464 pps_freq = time_tolerance;
1465 1465 }
1466 1466 }
1467 1467
1468 1468 /*
1469 1469 * Here the calibration interval is adjusted. If the maximum
1470 1470 * time difference is greater than tick / 4, reduce the interval
1471 1471 * by half. If this is not the case for four consecutive
1472 1472 * intervals, double the interval.
1473 1473 */
1474 1474 if (u_usec << pps_shift > bigtick >> 2) {
1475 1475 pps_intcnt = 0;
1476 1476 if (pps_shift > PPS_SHIFT)
1477 1477 pps_shift--;
1478 1478 } else if (pps_intcnt >= 4) {
1479 1479 pps_intcnt = 0;
1480 1480 if (pps_shift < PPS_SHIFTMAX)
1481 1481 pps_shift++;
1482 1482 } else
1483 1483 pps_intcnt++;
1484 1484
1485 1485 /*
1486 1486 * If recovering from kmdb, then make sure the tod chip gets resynced.
1487 1487 * If we took an early exit above, then we don't yet have a stable
1488 1488 * calibration signal to lock onto, so don't mark the tod for sync
1489 1489 * until we get all the way here.
1490 1490 */
1491 1491 {
1492 1492 int s = hr_clock_lock();
1493 1493
1494 1494 tod_needsync = 1;
1495 1495 hr_clock_unlock(s);
1496 1496 }
1497 1497 }
1498 1498
1499 1499 /*
1500 1500 * Handle clock tick processing for a thread.
1501 1501 * Check for timer action, enforce CPU rlimit, do profiling etc.
1502 1502 */
1503 1503 void
1504 1504 clock_tick(kthread_t *t, int pending)
1505 1505 {
1506 1506 struct proc *pp;
1507 1507 klwp_id_t lwp;
1508 1508 struct as *as;
1509 1509 clock_t ticks;
1510 1510 int poke = 0; /* notify another CPU */
1511 1511 int user_mode;
1512 1512 size_t rss;
1513 1513 int i, total_usec, usec;
1514 1514 rctl_qty_t secs;
1515 1515
1516 1516 ASSERT(pending > 0);
1517 1517
1518 1518 /* Must be operating on a lwp/thread */
1519 1519 if ((lwp = ttolwp(t)) == NULL) {
1520 1520 panic("clock_tick: no lwp");
1521 1521 /*NOTREACHED*/
1522 1522 }
1523 1523
1524 1524 for (i = 0; i < pending; i++) {
1525 1525 CL_TICK(t); /* Class specific tick processing */
1526 1526 DTRACE_SCHED1(tick, kthread_t *, t);
1527 1527 }
1528 1528
1529 1529 pp = ttoproc(t);
1530 1530
1531 1531 /* pp->p_lock makes sure that the thread does not exit */
1532 1532 ASSERT(MUTEX_HELD(&pp->p_lock));
1533 1533
1534 1534 user_mode = (lwp->lwp_state == LWP_USER);
1535 1535
1536 1536 ticks = (pp->p_utime + pp->p_stime) % hz;
1537 1537 /*
1538 1538 * Update process times. Should use high res clock and state
1539 1539 * changes instead of statistical sampling method. XXX
1540 1540 */
1541 1541 if (user_mode) {
1542 1542 pp->p_utime += pending;
1543 1543 } else {
1544 1544 pp->p_stime += pending;
1545 1545 }
1546 1546
1547 1547 pp->p_ttime += pending;
1548 1548 as = pp->p_as;
1549 1549
1550 1550 /*
1551 1551 * Update user profiling statistics. Get the pc from the
1552 1552 * lwp when the AST happens.
1553 1553 */
1554 1554 if (pp->p_prof.pr_scale) {
1555 1555 atomic_add_32(&lwp->lwp_oweupc, (int32_t)pending);
1556 1556 if (user_mode) {
1557 1557 poke = 1;
1558 1558 aston(t);
1559 1559 }
1560 1560 }
1561 1561
1562 1562 /*
1563 1563 * If CPU was in user state, process lwp-virtual time
1564 1564 * interval timer. The value passed to itimerdecr() has to be
1565 1565 * in microseconds and has to be less than one second. Hence
1566 1566 * this loop.
1567 1567 */
1568 1568 total_usec = usec_per_tick * pending;
1569 1569 while (total_usec > 0) {
1570 1570 usec = MIN(total_usec, (MICROSEC - 1));
1571 1571 if (user_mode &&
1572 1572 timerisset(&lwp->lwp_timer[ITIMER_VIRTUAL].it_value) &&
1573 1573 itimerdecr(&lwp->lwp_timer[ITIMER_VIRTUAL], usec) == 0) {
1574 1574 poke = 1;
1575 1575 sigtoproc(pp, t, SIGVTALRM);
1576 1576 }
1577 1577 total_usec -= usec;
1578 1578 }
1579 1579
1580 1580 /*
1581 1581 * If CPU was in user state, process lwp-profile
1582 1582 * interval timer.
1583 1583 */
1584 1584 total_usec = usec_per_tick * pending;
1585 1585 while (total_usec > 0) {
1586 1586 usec = MIN(total_usec, (MICROSEC - 1));
1587 1587 if (timerisset(&lwp->lwp_timer[ITIMER_PROF].it_value) &&
1588 1588 itimerdecr(&lwp->lwp_timer[ITIMER_PROF], usec) == 0) {
1589 1589 poke = 1;
1590 1590 sigtoproc(pp, t, SIGPROF);
1591 1591 }
1592 1592 total_usec -= usec;
1593 1593 }
1594 1594
1595 1595 /*
1596 1596 * Enforce CPU resource controls:
1597 1597 * (a) process.max-cpu-time resource control
1598 1598 *
1599 1599 * Perform the check only if we have accumulated more a second.
1600 1600 */
1601 1601 if ((ticks + pending) >= hz) {
1602 1602 (void) rctl_test(rctlproc_legacy[RLIMIT_CPU], pp->p_rctls, pp,
1603 1603 (pp->p_utime + pp->p_stime)/hz, RCA_UNSAFE_SIGINFO);
1604 1604 }
1605 1605
1606 1606 /*
1607 1607 * (b) task.max-cpu-time resource control
1608 1608 *
1609 1609 * If we have accumulated enough ticks, increment the task CPU
1610 1610 * time usage and test for the resource limit. This minimizes the
1611 1611 * number of calls to the rct_test(). The task CPU time mutex
1612 1612 * is highly contentious as many processes can be sharing a task.
1613 1613 */
1614 1614 if (pp->p_ttime >= clock_tick_proc_max) {
1615 1615 secs = task_cpu_time_incr(pp->p_task, pp->p_ttime);
1616 1616 pp->p_ttime = 0;
1617 1617 if (secs) {
1618 1618 (void) rctl_test(rc_task_cpu_time, pp->p_task->tk_rctls,
1619 1619 pp, secs, RCA_UNSAFE_SIGINFO);
1620 1620 }
1621 1621 }
1622 1622
1623 1623 /*
1624 1624 * Update memory usage for the currently running process.
1625 1625 */
1626 1626 rss = rm_asrss(as);
1627 1627 PTOU(pp)->u_mem += rss;
1628 1628 if (rss > PTOU(pp)->u_mem_max)
1629 1629 PTOU(pp)->u_mem_max = rss;
1630 1630
1631 1631 /*
1632 1632 * Notify the CPU the thread is running on.
1633 1633 */
1634 1634 if (poke && t->t_cpu != CPU)
1635 1635 poke_cpu(t->t_cpu->cpu_id);
1636 1636 }
1637 1637
1638 1638 void
1639 1639 profil_tick(uintptr_t upc)
1640 1640 {
1641 1641 int ticks;
1642 1642 proc_t *p = ttoproc(curthread);
1643 1643 klwp_t *lwp = ttolwp(curthread);
1644 1644 struct prof *pr = &p->p_prof;
1645 1645
1646 1646 do {
1647 1647 ticks = lwp->lwp_oweupc;
1648 1648 } while (atomic_cas_32(&lwp->lwp_oweupc, ticks, 0) != ticks);
1649 1649
1650 1650 mutex_enter(&p->p_pflock);
1651 1651 if (pr->pr_scale >= 2 && upc >= pr->pr_off) {
1652 1652 /*
1653 1653 * Old-style profiling
1654 1654 */
1655 1655 uint16_t *slot = pr->pr_base;
1656 1656 uint16_t old, new;
1657 1657 if (pr->pr_scale != 2) {
1658 1658 uintptr_t delta = upc - pr->pr_off;
1659 1659 uintptr_t byteoff = ((delta >> 16) * pr->pr_scale) +
1660 1660 (((delta & 0xffff) * pr->pr_scale) >> 16);
1661 1661 if (byteoff >= (uintptr_t)pr->pr_size) {
1662 1662 mutex_exit(&p->p_pflock);
1663 1663 return;
1664 1664 }
1665 1665 slot += byteoff / sizeof (uint16_t);
1666 1666 }
1667 1667 if (fuword16(slot, &old) < 0 ||
1668 1668 (new = old + ticks) > SHRT_MAX ||
1669 1669 suword16(slot, new) < 0) {
1670 1670 pr->pr_scale = 0;
1671 1671 }
1672 1672 } else if (pr->pr_scale == 1) {
1673 1673 /*
1674 1674 * PC Sampling
1675 1675 */
1676 1676 model_t model = lwp_getdatamodel(lwp);
1677 1677 int result;
1678 1678 #ifdef __lint
1679 1679 model = model;
1680 1680 #endif
1681 1681 while (ticks-- > 0) {
1682 1682 if (pr->pr_samples == pr->pr_size) {
1683 1683 /* buffer full, turn off sampling */
1684 1684 pr->pr_scale = 0;
1685 1685 break;
1686 1686 }
1687 1687 switch (SIZEOF_PTR(model)) {
1688 1688 case sizeof (uint32_t):
1689 1689 result = suword32(pr->pr_base, (uint32_t)upc);
1690 1690 break;
1691 1691 #ifdef _LP64
1692 1692 case sizeof (uint64_t):
1693 1693 result = suword64(pr->pr_base, (uint64_t)upc);
1694 1694 break;
1695 1695 #endif
1696 1696 default:
1697 1697 cmn_err(CE_WARN, "profil_tick: unexpected "
1698 1698 "data model");
1699 1699 result = -1;
1700 1700 break;
1701 1701 }
1702 1702 if (result != 0) {
1703 1703 pr->pr_scale = 0;
1704 1704 break;
1705 1705 }
1706 1706 pr->pr_base = (caddr_t)pr->pr_base + SIZEOF_PTR(model);
1707 1707 pr->pr_samples++;
1708 1708 }
1709 1709 }
1710 1710 mutex_exit(&p->p_pflock);
1711 1711 }
1712 1712
1713 1713 static void
1714 1714 delay_wakeup(void *arg)
1715 1715 {
1716 1716 kthread_t *t = arg;
1717 1717
1718 1718 mutex_enter(&t->t_delay_lock);
1719 1719 cv_signal(&t->t_delay_cv);
1720 1720 mutex_exit(&t->t_delay_lock);
1721 1721 }
1722 1722
1723 1723 /*
1724 1724 * The delay(9F) man page indicates that it can only be called from user or
1725 1725 * kernel context - detect and diagnose bad calls. The following macro will
1726 1726 * produce a limited number of messages identifying bad callers. This is done
1727 1727 * in a macro so that caller() is meaningful. When a bad caller is identified,
1728 1728 * switching to 'drv_usecwait(TICK_TO_USEC(ticks));' may be appropriate.
1729 1729 */
1730 1730 #define DELAY_CONTEXT_CHECK() { \
1731 1731 uint32_t m; \
1732 1732 char *f; \
1733 1733 ulong_t off; \
1734 1734 \
1735 1735 m = delay_from_interrupt_msg; \
1736 1736 if (delay_from_interrupt_diagnose && servicing_interrupt() && \
1737 1737 !panicstr && !devinfo_freeze && \
1738 1738 atomic_cas_32(&delay_from_interrupt_msg, m ? m : 1, m-1)) { \
1739 1739 f = modgetsymname((uintptr_t)caller(), &off); \
1740 1740 cmn_err(CE_WARN, "delay(9F) called from " \
1741 1741 "interrupt context: %s`%s", \
1742 1742 mod_containing_pc(caller()), f ? f : "..."); \
1743 1743 } \
1744 1744 }
1745 1745
1746 1746 /*
1747 1747 * delay_common: common delay code.
1748 1748 */
1749 1749 static void
1750 1750 delay_common(clock_t ticks)
1751 1751 {
1752 1752 kthread_t *t = curthread;
1753 1753 clock_t deadline;
1754 1754 clock_t timeleft;
1755 1755 callout_id_t id;
1756 1756
1757 1757 /* If timeouts aren't running all we can do is spin. */
1758 1758 if (panicstr || devinfo_freeze) {
1759 1759 /* Convert delay(9F) call into drv_usecwait(9F) call. */
1760 1760 if (ticks > 0)
1761 1761 drv_usecwait(TICK_TO_USEC(ticks));
1762 1762 return;
1763 1763 }
1764 1764
1765 1765 deadline = ddi_get_lbolt() + ticks;
1766 1766 while ((timeleft = deadline - ddi_get_lbolt()) > 0) {
1767 1767 mutex_enter(&t->t_delay_lock);
1768 1768 id = timeout_default(delay_wakeup, t, timeleft);
1769 1769 cv_wait(&t->t_delay_cv, &t->t_delay_lock);
1770 1770 mutex_exit(&t->t_delay_lock);
1771 1771 (void) untimeout_default(id, 0);
1772 1772 }
1773 1773 }
1774 1774
1775 1775 /*
1776 1776 * Delay specified number of clock ticks.
1777 1777 */
1778 1778 void
1779 1779 delay(clock_t ticks)
1780 1780 {
1781 1781 DELAY_CONTEXT_CHECK();
1782 1782
1783 1783 delay_common(ticks);
1784 1784 }
1785 1785
1786 1786 /*
1787 1787 * Delay a random number of clock ticks between 1 and ticks.
1788 1788 */
1789 1789 void
1790 1790 delay_random(clock_t ticks)
1791 1791 {
1792 1792 int r;
1793 1793
1794 1794 DELAY_CONTEXT_CHECK();
1795 1795
1796 1796 (void) random_get_pseudo_bytes((void *)&r, sizeof (r));
1797 1797 if (ticks == 0)
1798 1798 ticks = 1;
1799 1799 ticks = (r % ticks) + 1;
1800 1800 delay_common(ticks);
1801 1801 }
1802 1802
1803 1803 /*
1804 1804 * Like delay, but interruptible by a signal.
1805 1805 */
1806 1806 int
1807 1807 delay_sig(clock_t ticks)
1808 1808 {
1809 1809 kthread_t *t = curthread;
1810 1810 clock_t deadline;
1811 1811 clock_t rc;
1812 1812
1813 1813 /* If timeouts aren't running all we can do is spin. */
1814 1814 if (panicstr || devinfo_freeze) {
1815 1815 if (ticks > 0)
1816 1816 drv_usecwait(TICK_TO_USEC(ticks));
1817 1817 return (0);
1818 1818 }
1819 1819
1820 1820 deadline = ddi_get_lbolt() + ticks;
1821 1821 mutex_enter(&t->t_delay_lock);
1822 1822 do {
1823 1823 rc = cv_timedwait_sig(&t->t_delay_cv,
1824 1824 &t->t_delay_lock, deadline);
1825 1825 /* loop until past deadline or signaled */
1826 1826 } while (rc > 0);
1827 1827 mutex_exit(&t->t_delay_lock);
1828 1828 if (rc == 0)
1829 1829 return (EINTR);
1830 1830 return (0);
1831 1831 }
1832 1832
1833 1833
1834 1834 #define SECONDS_PER_DAY 86400
1835 1835
1836 1836 /*
1837 1837 * Initialize the system time based on the TOD chip. approx is used as
1838 1838 * an approximation of time (e.g. from the filesystem) in the event that
1839 1839 * the TOD chip has been cleared or is unresponsive. An approx of -1
1840 1840 * means the filesystem doesn't keep time.
1841 1841 */
1842 1842 void
1843 1843 clkset(time_t approx)
1844 1844 {
1845 1845 timestruc_t ts;
1846 1846 int spl;
1847 1847 int set_clock = 0;
1848 1848
1849 1849 mutex_enter(&tod_lock);
1850 1850 ts = tod_get();
1851 1851
1852 1852 if (ts.tv_sec > 365 * SECONDS_PER_DAY) {
1853 1853 /*
1854 1854 * If the TOD chip is reporting some time after 1971,
1855 1855 * then it probably didn't lose power or become otherwise
1856 1856 * cleared in the recent past; check to assure that
1857 1857 * the time coming from the filesystem isn't in the future
1858 1858 * according to the TOD chip.
1859 1859 */
1860 1860 if (approx != -1 && approx > ts.tv_sec) {
1861 1861 cmn_err(CE_WARN, "Last shutdown is later "
1862 1862 "than time on time-of-day chip; check date.");
1863 1863 }
1864 1864 } else {
1865 1865 /*
1866 1866 * If the TOD chip isn't giving correct time, set it to the
1867 1867 * greater of i) approx and ii) 1987. That way if approx
1868 1868 * is negative or is earlier than 1987, we set the clock
1869 1869 * back to a time when Oliver North, ALF and Dire Straits
1870 1870 * were all on the collective brain: 1987.
1871 1871 */
1872 1872 timestruc_t tmp;
1873 1873 time_t diagnose_date = (1987 - 1970) * 365 * SECONDS_PER_DAY;
1874 1874 ts.tv_sec = (approx > diagnose_date ? approx : diagnose_date);
1875 1875 ts.tv_nsec = 0;
1876 1876
1877 1877 /*
1878 1878 * Attempt to write the new time to the TOD chip. Set spl high
1879 1879 * to avoid getting preempted between the tod_set and tod_get.
1880 1880 */
1881 1881 spl = splhi();
1882 1882 tod_set(ts);
1883 1883 tmp = tod_get();
1884 1884 splx(spl);
1885 1885
1886 1886 if (tmp.tv_sec != ts.tv_sec && tmp.tv_sec != ts.tv_sec + 1) {
1887 1887 tod_broken = 1;
1888 1888 dosynctodr = 0;
1889 1889 cmn_err(CE_WARN, "Time-of-day chip unresponsive.");
1890 1890 } else {
1891 1891 cmn_err(CE_WARN, "Time-of-day chip had "
1892 1892 "incorrect date; check and reset.");
1893 1893 }
1894 1894 set_clock = 1;
1895 1895 }
1896 1896
1897 1897 if (!boot_time) {
1898 1898 boot_time = ts.tv_sec;
1899 1899 set_clock = 1;
1900 1900 }
1901 1901
1902 1902 if (set_clock)
1903 1903 set_hrestime(&ts);
1904 1904
1905 1905 mutex_exit(&tod_lock);
1906 1906 }
1907 1907
1908 1908 int timechanged; /* for testing if the system time has been reset */
1909 1909
1910 1910 void
1911 1911 set_hrestime(timestruc_t *ts)
1912 1912 {
1913 1913 int spl = hr_clock_lock();
1914 1914 hrestime = *ts;
1915 1915 membar_enter(); /* hrestime must be visible before timechanged++ */
1916 1916 timedelta = 0;
1917 1917 timechanged++;
1918 1918 hr_clock_unlock(spl);
1919 1919 callout_hrestime();
1920 1920 }
1921 1921
1922 1922 static uint_t deadman_seconds;
1923 1923 static uint32_t deadman_panics;
1924 1924 static int deadman_enabled = 0;
1925 1925 static int deadman_panic_timers = 1;
1926 1926
1927 1927 static void
1928 1928 deadman(void)
1929 1929 {
1930 1930 if (panicstr) {
1931 1931 /*
1932 1932 * During panic, other CPUs besides the panic
1933 1933 * master continue to handle cyclics and some other
1934 1934 * interrupts. The code below is intended to be
1935 1935 * single threaded, so any CPU other than the master
1936 1936 * must keep out.
1937 1937 */
1938 1938 if (CPU->cpu_id != panic_cpu.cpu_id)
1939 1939 return;
1940 1940
1941 1941 if (!deadman_panic_timers)
1942 1942 return; /* allow all timers to be manually disabled */
1943 1943
1944 1944 /*
1945 1945 * If we are generating a crash dump or syncing filesystems and
1946 1946 * the corresponding timer is set, decrement it and re-enter
1947 1947 * the panic code to abort it and advance to the next state.
1948 1948 * The panic states and triggers are explained in panic.c.
1949 1949 */
1950 1950 if (panic_dump) {
1951 1951 if (dump_timeleft && (--dump_timeleft == 0)) {
1952 1952 panic("panic dump timeout");
1953 1953 /*NOTREACHED*/
1954 1954 }
1955 1955 } else if (panic_sync) {
1956 1956 if (sync_timeleft && (--sync_timeleft == 0)) {
1957 1957 panic("panic sync timeout");
1958 1958 /*NOTREACHED*/
1959 1959 }
1960 1960 }
1961 1961
1962 1962 return;
1963 1963 }
1964 1964
1965 1965 if (deadman_counter != CPU->cpu_deadman_counter) {
1966 1966 CPU->cpu_deadman_counter = deadman_counter;
1967 1967 CPU->cpu_deadman_countdown = deadman_seconds;
1968 1968 return;
1969 1969 }
1970 1970
1971 1971 if (--CPU->cpu_deadman_countdown > 0)
1972 1972 return;
1973 1973
1974 1974 /*
|
↓ open down ↓ |
1974 lines elided |
↑ open up ↑ |
1975 1975 * Regardless of whether or not we actually bring the system down,
1976 1976 * bump the deadman_panics variable.
1977 1977 *
1978 1978 * N.B. deadman_panics is incremented once for each CPU that
1979 1979 * passes through here. It's expected that all the CPUs will
1980 1980 * detect this condition within one second of each other, so
1981 1981 * when deadman_enabled is off, deadman_panics will
1982 1982 * typically be a multiple of the total number of CPUs in
1983 1983 * the system.
1984 1984 */
1985 - atomic_add_32(&deadman_panics, 1);
1985 + atomic_inc_32(&deadman_panics);
1986 1986
1987 1987 if (!deadman_enabled) {
1988 1988 CPU->cpu_deadman_countdown = deadman_seconds;
1989 1989 return;
1990 1990 }
1991 1991
1992 1992 /*
1993 1993 * If we're here, we want to bring the system down.
1994 1994 */
1995 1995 panic("deadman: timed out after %d seconds of clock "
1996 1996 "inactivity", deadman_seconds);
1997 1997 /*NOTREACHED*/
1998 1998 }
1999 1999
2000 2000 /*ARGSUSED*/
2001 2001 static void
2002 2002 deadman_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
2003 2003 {
2004 2004 cpu->cpu_deadman_counter = 0;
2005 2005 cpu->cpu_deadman_countdown = deadman_seconds;
2006 2006
2007 2007 hdlr->cyh_func = (cyc_func_t)deadman;
2008 2008 hdlr->cyh_level = CY_HIGH_LEVEL;
2009 2009 hdlr->cyh_arg = NULL;
2010 2010
2011 2011 /*
2012 2012 * Stagger the CPUs so that they don't all run deadman() at
2013 2013 * the same time. Simplest reason to do this is to make it
2014 2014 * more likely that only one CPU will panic in case of a
2015 2015 * timeout. This is (strictly speaking) an aesthetic, not a
2016 2016 * technical consideration.
2017 2017 */
2018 2018 when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU);
2019 2019 when->cyt_interval = NANOSEC;
2020 2020 }
2021 2021
2022 2022
2023 2023 void
2024 2024 deadman_init(void)
2025 2025 {
2026 2026 cyc_omni_handler_t hdlr;
2027 2027
2028 2028 if (deadman_seconds == 0)
2029 2029 deadman_seconds = snoop_interval / MICROSEC;
2030 2030
2031 2031 if (snooping)
2032 2032 deadman_enabled = 1;
2033 2033
2034 2034 hdlr.cyo_online = deadman_online;
2035 2035 hdlr.cyo_offline = NULL;
2036 2036 hdlr.cyo_arg = NULL;
2037 2037
2038 2038 mutex_enter(&cpu_lock);
2039 2039 deadman_cyclic = cyclic_add_omni(&hdlr);
2040 2040 mutex_exit(&cpu_lock);
2041 2041 }
2042 2042
2043 2043 /*
2044 2044 * tod_fault() is for updating tod validate mechanism state:
2045 2045 * (1) TOD_NOFAULT: for resetting the state to 'normal'.
2046 2046 * currently used for debugging only
2047 2047 * (2) The following four cases detected by tod validate mechanism:
2048 2048 * TOD_REVERSED: current tod value is less than previous value.
2049 2049 * TOD_STALLED: current tod value hasn't advanced.
2050 2050 * TOD_JUMPED: current tod value advanced too far from previous value.
2051 2051 * TOD_RATECHANGED: the ratio between average tod delta and
2052 2052 * average tick delta has changed.
2053 2053 * (3) TOD_RDONLY: when the TOD clock is not writeable e.g. because it is
2054 2054 * a virtual TOD provided by a hypervisor.
2055 2055 */
2056 2056 enum tod_fault_type
2057 2057 tod_fault(enum tod_fault_type ftype, int off)
2058 2058 {
2059 2059 ASSERT(MUTEX_HELD(&tod_lock));
2060 2060
2061 2061 if (tod_faulted != ftype) {
2062 2062 switch (ftype) {
2063 2063 case TOD_NOFAULT:
2064 2064 plat_tod_fault(TOD_NOFAULT);
2065 2065 cmn_err(CE_NOTE, "Restarted tracking "
2066 2066 "Time of Day clock.");
2067 2067 tod_faulted = ftype;
2068 2068 break;
2069 2069 case TOD_REVERSED:
2070 2070 case TOD_JUMPED:
2071 2071 if (tod_faulted == TOD_NOFAULT) {
2072 2072 plat_tod_fault(ftype);
2073 2073 cmn_err(CE_WARN, "Time of Day clock error: "
2074 2074 "reason [%s by 0x%x]. -- "
2075 2075 " Stopped tracking Time Of Day clock.",
2076 2076 tod_fault_table[ftype], off);
2077 2077 tod_faulted = ftype;
2078 2078 }
2079 2079 break;
2080 2080 case TOD_STALLED:
2081 2081 case TOD_RATECHANGED:
2082 2082 if (tod_faulted == TOD_NOFAULT) {
2083 2083 plat_tod_fault(ftype);
2084 2084 cmn_err(CE_WARN, "Time of Day clock error: "
2085 2085 "reason [%s]. -- "
2086 2086 " Stopped tracking Time Of Day clock.",
2087 2087 tod_fault_table[ftype]);
2088 2088 tod_faulted = ftype;
2089 2089 }
2090 2090 break;
2091 2091 case TOD_RDONLY:
2092 2092 if (tod_faulted == TOD_NOFAULT) {
2093 2093 plat_tod_fault(ftype);
2094 2094 cmn_err(CE_NOTE, "!Time of Day clock is "
2095 2095 "Read-Only; set of Date/Time will not "
2096 2096 "persist across reboot.");
2097 2097 tod_faulted = ftype;
2098 2098 }
2099 2099 break;
2100 2100 default:
2101 2101 break;
2102 2102 }
2103 2103 }
2104 2104 return (tod_faulted);
2105 2105 }
2106 2106
2107 2107 /*
2108 2108 * Two functions that allow tod_status_flag to be manipulated by functions
2109 2109 * external to this file.
2110 2110 */
2111 2111
2112 2112 void
2113 2113 tod_status_set(int tod_flag)
2114 2114 {
2115 2115 tod_status_flag |= tod_flag;
2116 2116 }
2117 2117
2118 2118 void
2119 2119 tod_status_clear(int tod_flag)
2120 2120 {
2121 2121 tod_status_flag &= ~tod_flag;
2122 2122 }
2123 2123
2124 2124 /*
2125 2125 * Record a timestamp and the value passed to tod_set(). The next call to
2126 2126 * tod_validate() can use these values, prev_set_tick and prev_set_tod,
2127 2127 * when checking the timestruc_t returned by tod_get(). Ordinarily,
2128 2128 * tod_validate() will use prev_tick and prev_tod for this task but these
2129 2129 * become obsolete, and will be re-assigned with the prev_set_* values,
2130 2130 * in the case when the TOD is re-written.
2131 2131 */
2132 2132 void
2133 2133 tod_set_prev(timestruc_t ts)
2134 2134 {
2135 2135 if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
2136 2136 tod_validate_deferred) {
2137 2137 return;
2138 2138 }
2139 2139 prev_set_tick = gethrtime();
2140 2140 /*
2141 2141 * A negative value will be set to zero in utc_to_tod() so we fake
2142 2142 * a zero here in such a case. This would need to change if the
2143 2143 * behavior of utc_to_tod() changes.
2144 2144 */
2145 2145 prev_set_tod = ts.tv_sec < 0 ? 0 : ts.tv_sec;
2146 2146 }
2147 2147
2148 2148 /*
2149 2149 * tod_validate() is used for checking values returned by tod_get().
2150 2150 * Four error cases can be detected by this routine:
2151 2151 * TOD_REVERSED: current tod value is less than previous.
2152 2152 * TOD_STALLED: current tod value hasn't advanced.
2153 2153 * TOD_JUMPED: current tod value advanced too far from previous value.
2154 2154 * TOD_RATECHANGED: the ratio between average tod delta and
2155 2155 * average tick delta has changed.
2156 2156 */
2157 2157 time_t
2158 2158 tod_validate(time_t tod)
2159 2159 {
2160 2160 time_t diff_tod;
2161 2161 hrtime_t diff_tick;
2162 2162
2163 2163 long dtick;
2164 2164 int dtick_delta;
2165 2165
2166 2166 int off = 0;
2167 2167 enum tod_fault_type tod_bad = TOD_NOFAULT;
2168 2168
2169 2169 static int firsttime = 1;
2170 2170
2171 2171 static time_t prev_tod = 0;
2172 2172 static hrtime_t prev_tick = 0;
2173 2173 static long dtick_avg = TOD_REF_FREQ;
2174 2174
2175 2175 int cpr_resume_done = 0;
2176 2176 int dr_resume_done = 0;
2177 2177
2178 2178 hrtime_t tick = gethrtime();
2179 2179
2180 2180 ASSERT(MUTEX_HELD(&tod_lock));
2181 2181
2182 2182 /*
2183 2183 * tod_validate_enable is patchable via /etc/system.
2184 2184 * If TOD is already faulted, or if TOD validation is deferred,
2185 2185 * there is nothing to do.
2186 2186 */
2187 2187 if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
2188 2188 tod_validate_deferred) {
2189 2189 return (tod);
2190 2190 }
2191 2191
2192 2192 /*
2193 2193 * If this is the first time through, we just need to save the tod
2194 2194 * we were called with and hrtime so we can use them next time to
2195 2195 * validate tod_get().
2196 2196 */
2197 2197 if (firsttime) {
2198 2198 firsttime = 0;
2199 2199 prev_tod = tod;
2200 2200 prev_tick = tick;
2201 2201 return (tod);
2202 2202 }
2203 2203
2204 2204 /*
2205 2205 * Handle any flags that have been turned on by tod_status_set().
2206 2206 * In the case where a tod_set() is done and then a subsequent
2207 2207 * tod_get() fails (ie, both TOD_SET_DONE and TOD_GET_FAILED are
2208 2208 * true), we treat the TOD_GET_FAILED with precedence by switching
2209 2209 * off the flag, returning tod and leaving TOD_SET_DONE asserted
2210 2210 * until such time as tod_get() completes successfully.
2211 2211 */
2212 2212 if (tod_status_flag & TOD_GET_FAILED) {
2213 2213 /*
2214 2214 * tod_get() has encountered an issue, possibly transitory,
2215 2215 * when reading TOD. We'll just return the incoming tod
2216 2216 * value (which is actually hrestime.tv_sec in this case)
2217 2217 * and when we get a genuine tod, following a successful
2218 2218 * tod_get(), we can validate using prev_tod and prev_tick.
2219 2219 */
2220 2220 tod_status_flag &= ~TOD_GET_FAILED;
2221 2221 return (tod);
2222 2222 } else if (tod_status_flag & TOD_SET_DONE) {
2223 2223 /*
2224 2224 * TOD has been modified. Just before the TOD was written,
2225 2225 * tod_set_prev() saved tod and hrtime; we can now use
2226 2226 * those values, prev_set_tod and prev_set_tick, to validate
2227 2227 * the incoming tod that's just been read.
2228 2228 */
2229 2229 prev_tod = prev_set_tod;
2230 2230 prev_tick = prev_set_tick;
2231 2231 dtick_avg = TOD_REF_FREQ;
2232 2232 tod_status_flag &= ~TOD_SET_DONE;
2233 2233 /*
2234 2234 * If a tod_set() preceded a cpr_suspend() without an
2235 2235 * intervening tod_validate(), we need to ensure that a
2236 2236 * TOD_JUMPED condition is ignored.
2237 2237 * Note this isn't a concern in the case of DR as we've
2238 2238 * just reassigned dtick_avg, above.
2239 2239 */
2240 2240 if (tod_status_flag & TOD_CPR_RESUME_DONE) {
2241 2241 cpr_resume_done = 1;
2242 2242 tod_status_flag &= ~TOD_CPR_RESUME_DONE;
2243 2243 }
2244 2244 } else if (tod_status_flag & TOD_CPR_RESUME_DONE) {
2245 2245 /*
2246 2246 * The system's coming back from a checkpoint resume.
2247 2247 */
2248 2248 cpr_resume_done = 1;
2249 2249 tod_status_flag &= ~TOD_CPR_RESUME_DONE;
2250 2250 /*
2251 2251 * We need to handle the possibility of a CPR suspend
2252 2252 * operation having been initiated whilst a DR event was
2253 2253 * in-flight.
2254 2254 */
2255 2255 if (tod_status_flag & TOD_DR_RESUME_DONE) {
2256 2256 dr_resume_done = 1;
2257 2257 tod_status_flag &= ~TOD_DR_RESUME_DONE;
2258 2258 }
2259 2259 } else if (tod_status_flag & TOD_DR_RESUME_DONE) {
2260 2260 /*
2261 2261 * A Dynamic Reconfiguration event has taken place.
2262 2262 */
2263 2263 dr_resume_done = 1;
2264 2264 tod_status_flag &= ~TOD_DR_RESUME_DONE;
2265 2265 }
2266 2266
2267 2267 /* test hook */
2268 2268 switch (tod_unit_test) {
2269 2269 case 1: /* for testing jumping tod */
2270 2270 tod += tod_test_injector;
2271 2271 tod_unit_test = 0;
2272 2272 break;
2273 2273 case 2: /* for testing stuck tod bit */
2274 2274 tod |= 1 << tod_test_injector;
2275 2275 tod_unit_test = 0;
2276 2276 break;
2277 2277 case 3: /* for testing stalled tod */
2278 2278 tod = prev_tod;
2279 2279 tod_unit_test = 0;
2280 2280 break;
2281 2281 case 4: /* reset tod fault status */
2282 2282 (void) tod_fault(TOD_NOFAULT, 0);
2283 2283 tod_unit_test = 0;
2284 2284 break;
2285 2285 default:
2286 2286 break;
2287 2287 }
2288 2288
2289 2289 diff_tod = tod - prev_tod;
2290 2290 diff_tick = tick - prev_tick;
2291 2291
2292 2292 ASSERT(diff_tick >= 0);
2293 2293
2294 2294 if (diff_tod < 0) {
2295 2295 /* ERROR - tod reversed */
2296 2296 tod_bad = TOD_REVERSED;
2297 2297 off = (int)(prev_tod - tod);
2298 2298 } else if (diff_tod == 0) {
2299 2299 /* tod did not advance */
2300 2300 if (diff_tick > TOD_STALL_THRESHOLD) {
2301 2301 /* ERROR - tod stalled */
2302 2302 tod_bad = TOD_STALLED;
2303 2303 } else {
2304 2304 /*
2305 2305 * Make sure we don't update prev_tick
2306 2306 * so that diff_tick is calculated since
2307 2307 * the first diff_tod == 0
2308 2308 */
2309 2309 return (tod);
2310 2310 }
2311 2311 } else {
2312 2312 /* calculate dtick */
2313 2313 dtick = diff_tick / diff_tod;
2314 2314
2315 2315 /* update dtick averages */
2316 2316 dtick_avg += ((dtick - dtick_avg) / TOD_FILTER_N);
2317 2317
2318 2318 /*
2319 2319 * Calculate dtick_delta as
2320 2320 * variation from reference freq in quartiles
2321 2321 */
2322 2322 dtick_delta = (dtick_avg - TOD_REF_FREQ) /
2323 2323 (TOD_REF_FREQ >> 2);
2324 2324
2325 2325 /*
2326 2326 * Even with a perfectly functioning TOD device,
2327 2327 * when the number of elapsed seconds is low the
2328 2328 * algorithm can calculate a rate that is beyond
2329 2329 * tolerance, causing an error. The algorithm is
2330 2330 * inaccurate when elapsed time is low (less than
2331 2331 * 5 seconds).
2332 2332 */
2333 2333 if (diff_tod > 4) {
2334 2334 if (dtick < TOD_JUMP_THRESHOLD) {
2335 2335 /*
2336 2336 * If we've just done a CPR resume, we detect
2337 2337 * a jump in the TOD but, actually, what's
2338 2338 * happened is that the TOD has been increasing
2339 2339 * whilst the system was suspended and the tick
2340 2340 * count hasn't kept up. We consider the first
2341 2341 * occurrence of this after a resume as normal
2342 2342 * and ignore it; otherwise, in a non-resume
2343 2343 * case, we regard it as a TOD problem.
2344 2344 */
2345 2345 if (!cpr_resume_done) {
2346 2346 /* ERROR - tod jumped */
2347 2347 tod_bad = TOD_JUMPED;
2348 2348 off = (int)diff_tod;
2349 2349 }
2350 2350 }
2351 2351 if (dtick_delta) {
2352 2352 /*
2353 2353 * If we've just done a DR resume, dtick_avg
2354 2354 * can go a bit askew so we reset it and carry
2355 2355 * on; otherwise, the TOD is in error.
2356 2356 */
2357 2357 if (dr_resume_done) {
2358 2358 dtick_avg = TOD_REF_FREQ;
2359 2359 } else {
2360 2360 /* ERROR - change in clock rate */
2361 2361 tod_bad = TOD_RATECHANGED;
2362 2362 }
2363 2363 }
2364 2364 }
2365 2365 }
2366 2366
2367 2367 if (tod_bad != TOD_NOFAULT) {
2368 2368 (void) tod_fault(tod_bad, off);
2369 2369
2370 2370 /*
2371 2371 * Disable dosynctodr since we are going to fault
2372 2372 * the TOD chip anyway here
2373 2373 */
2374 2374 dosynctodr = 0;
2375 2375
2376 2376 /*
2377 2377 * Set tod to the correct value from hrestime
2378 2378 */
2379 2379 tod = hrestime.tv_sec;
2380 2380 }
2381 2381
2382 2382 prev_tod = tod;
2383 2383 prev_tick = tick;
2384 2384 return (tod);
2385 2385 }
2386 2386
2387 2387 static void
2388 2388 calcloadavg(int nrun, uint64_t *hp_ave)
2389 2389 {
2390 2390 static int64_t f[3] = { 135, 27, 9 };
2391 2391 uint_t i;
2392 2392 int64_t q, r;
2393 2393
2394 2394 /*
2395 2395 * Compute load average over the last 1, 5, and 15 minutes
2396 2396 * (60, 300, and 900 seconds). The constants in f[3] are for
2397 2397 * exponential decay:
2398 2398 * (1 - exp(-1/60)) << 13 = 135,
2399 2399 * (1 - exp(-1/300)) << 13 = 27,
2400 2400 * (1 - exp(-1/900)) << 13 = 9.
2401 2401 */
2402 2402
2403 2403 /*
2404 2404 * a little hoop-jumping to avoid integer overflow
2405 2405 */
2406 2406 for (i = 0; i < 3; i++) {
2407 2407 q = (hp_ave[i] >> 16) << 7;
2408 2408 r = (hp_ave[i] & 0xffff) << 7;
2409 2409 hp_ave[i] += ((nrun - q) * f[i] - ((r * f[i]) >> 16)) >> 4;
2410 2410 }
2411 2411 }
2412 2412
2413 2413 /*
2414 2414 * lbolt_hybrid() is used by ddi_get_lbolt() and ddi_get_lbolt64() to
2415 2415 * calculate the value of lbolt according to the current mode. In the event
2416 2416 * driven mode (the default), lbolt is calculated by dividing the current hires
2417 2417 * time by the number of nanoseconds per clock tick. In the cyclic driven mode
2418 2418 * an internal variable is incremented at each firing of the lbolt cyclic
2419 2419 * and returned by lbolt_cyclic_driven().
2420 2420 *
2421 2421 * The system will transition from event to cyclic driven mode when the number
2422 2422 * of calls to lbolt_event_driven() exceeds the (per CPU) threshold within a
2423 2423 * window of time. It does so by reprograming lbolt_cyclic from CY_INFINITY to
2424 2424 * nsec_per_tick. The lbolt cyclic will remain ON while at least one CPU is
2425 2425 * causing enough activity to cross the thresholds.
2426 2426 */
2427 2427 int64_t
2428 2428 lbolt_bootstrap(void)
2429 2429 {
2430 2430 return (0);
2431 2431 }
2432 2432
2433 2433 /* ARGSUSED */
2434 2434 uint_t
2435 2435 lbolt_ev_to_cyclic(caddr_t arg1, caddr_t arg2)
2436 2436 {
2437 2437 hrtime_t ts, exp;
2438 2438 int ret;
2439 2439
2440 2440 ASSERT(lbolt_hybrid != lbolt_cyclic_driven);
2441 2441
2442 2442 kpreempt_disable();
2443 2443
2444 2444 ts = gethrtime();
2445 2445 lb_info->lbi_internal = (ts/nsec_per_tick);
2446 2446
2447 2447 /*
2448 2448 * Align the next expiration to a clock tick boundary.
2449 2449 */
2450 2450 exp = ts + nsec_per_tick - 1;
2451 2451 exp = (exp/nsec_per_tick) * nsec_per_tick;
2452 2452
2453 2453 ret = cyclic_reprogram(lb_info->id.lbi_cyclic_id, exp);
2454 2454 ASSERT(ret);
2455 2455
2456 2456 lbolt_hybrid = lbolt_cyclic_driven;
2457 2457 lb_info->lbi_cyc_deactivate = B_FALSE;
2458 2458 lb_info->lbi_cyc_deac_start = lb_info->lbi_internal;
2459 2459
2460 2460 kpreempt_enable();
2461 2461
2462 2462 ret = atomic_dec_32_nv(&lb_info->lbi_token);
2463 2463 ASSERT(ret == 0);
2464 2464
2465 2465 return (1);
2466 2466 }
2467 2467
2468 2468 int64_t
2469 2469 lbolt_event_driven(void)
2470 2470 {
2471 2471 hrtime_t ts;
2472 2472 int64_t lb;
2473 2473 int ret, cpu = CPU->cpu_seqid;
2474 2474
2475 2475 ts = gethrtime();
2476 2476 ASSERT(ts > 0);
2477 2477
2478 2478 ASSERT(nsec_per_tick > 0);
2479 2479 lb = (ts/nsec_per_tick);
2480 2480
2481 2481 /*
2482 2482 * Switch to cyclic mode if the number of calls to this routine
2483 2483 * has reached the threshold within the interval.
2484 2484 */
2485 2485 if ((lb - lb_cpu[cpu].lbc_cnt_start) < lb_info->lbi_thresh_interval) {
2486 2486
2487 2487 if (--lb_cpu[cpu].lbc_counter == 0) {
2488 2488 /*
2489 2489 * Reached the threshold within the interval, reset
2490 2490 * the usage statistics.
2491 2491 */
2492 2492 lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2493 2493 lb_cpu[cpu].lbc_cnt_start = lb;
2494 2494
2495 2495 /*
2496 2496 * Make sure only one thread reprograms the
2497 2497 * lbolt cyclic and changes the mode.
2498 2498 */
2499 2499 if (panicstr == NULL &&
2500 2500 atomic_cas_32(&lb_info->lbi_token, 0, 1) == 0) {
2501 2501
2502 2502 if (lbolt_hybrid == lbolt_cyclic_driven) {
2503 2503 ret = atomic_dec_32_nv(
2504 2504 &lb_info->lbi_token);
2505 2505 ASSERT(ret == 0);
2506 2506 } else {
2507 2507 lbolt_softint_post();
2508 2508 }
2509 2509 }
2510 2510 }
2511 2511 } else {
2512 2512 /*
2513 2513 * Exceeded the interval, reset the usage statistics.
2514 2514 */
2515 2515 lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2516 2516 lb_cpu[cpu].lbc_cnt_start = lb;
2517 2517 }
2518 2518
2519 2519 ASSERT(lb >= lb_info->lbi_debug_time);
2520 2520
2521 2521 return (lb - lb_info->lbi_debug_time);
2522 2522 }
2523 2523
2524 2524 int64_t
2525 2525 lbolt_cyclic_driven(void)
2526 2526 {
2527 2527 int64_t lb = lb_info->lbi_internal;
2528 2528 int cpu;
2529 2529
2530 2530 /*
2531 2531 * If a CPU has already prevented the lbolt cyclic from deactivating
2532 2532 * itself, don't bother tracking the usage. Otherwise check if we're
2533 2533 * within the interval and how the per CPU counter is doing.
2534 2534 */
2535 2535 if (lb_info->lbi_cyc_deactivate) {
2536 2536 cpu = CPU->cpu_seqid;
2537 2537 if ((lb - lb_cpu[cpu].lbc_cnt_start) <
2538 2538 lb_info->lbi_thresh_interval) {
2539 2539
2540 2540 if (lb_cpu[cpu].lbc_counter == 0)
2541 2541 /*
2542 2542 * Reached the threshold within the interval,
2543 2543 * prevent the lbolt cyclic from turning itself
2544 2544 * off.
2545 2545 */
2546 2546 lb_info->lbi_cyc_deactivate = B_FALSE;
2547 2547 else
2548 2548 lb_cpu[cpu].lbc_counter--;
2549 2549 } else {
2550 2550 /*
2551 2551 * Only reset the usage statistics when we have
2552 2552 * exceeded the interval.
2553 2553 */
2554 2554 lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2555 2555 lb_cpu[cpu].lbc_cnt_start = lb;
2556 2556 }
2557 2557 }
2558 2558
2559 2559 ASSERT(lb >= lb_info->lbi_debug_time);
2560 2560
2561 2561 return (lb - lb_info->lbi_debug_time);
2562 2562 }
2563 2563
2564 2564 /*
2565 2565 * The lbolt_cyclic() routine will fire at a nsec_per_tick interval to satisfy
2566 2566 * performance needs of ddi_get_lbolt() and ddi_get_lbolt64() consumers.
2567 2567 * It is inactive by default, and will be activated when switching from event
2568 2568 * to cyclic driven lbolt. The cyclic will turn itself off unless signaled
2569 2569 * by lbolt_cyclic_driven().
2570 2570 */
2571 2571 static void
2572 2572 lbolt_cyclic(void)
2573 2573 {
2574 2574 int ret;
2575 2575
2576 2576 lb_info->lbi_internal++;
2577 2577
2578 2578 if (!lbolt_cyc_only) {
2579 2579
2580 2580 if (lb_info->lbi_cyc_deactivate) {
2581 2581 /*
2582 2582 * Switching from cyclic to event driven mode.
2583 2583 */
2584 2584 if (panicstr == NULL &&
2585 2585 atomic_cas_32(&lb_info->lbi_token, 0, 1) == 0) {
2586 2586
2587 2587 if (lbolt_hybrid == lbolt_event_driven) {
2588 2588 ret = atomic_dec_32_nv(
2589 2589 &lb_info->lbi_token);
2590 2590 ASSERT(ret == 0);
2591 2591 return;
2592 2592 }
2593 2593
2594 2594 kpreempt_disable();
2595 2595
2596 2596 lbolt_hybrid = lbolt_event_driven;
2597 2597 ret = cyclic_reprogram(
2598 2598 lb_info->id.lbi_cyclic_id,
2599 2599 CY_INFINITY);
2600 2600 ASSERT(ret);
2601 2601
2602 2602 kpreempt_enable();
2603 2603
2604 2604 ret = atomic_dec_32_nv(&lb_info->lbi_token);
2605 2605 ASSERT(ret == 0);
2606 2606 }
2607 2607 }
2608 2608
2609 2609 /*
2610 2610 * The lbolt cyclic should not try to deactivate itself before
2611 2611 * the sampling period has elapsed.
2612 2612 */
2613 2613 if (lb_info->lbi_internal - lb_info->lbi_cyc_deac_start >=
2614 2614 lb_info->lbi_thresh_interval) {
2615 2615 lb_info->lbi_cyc_deactivate = B_TRUE;
2616 2616 lb_info->lbi_cyc_deac_start = lb_info->lbi_internal;
2617 2617 }
2618 2618 }
2619 2619 }
2620 2620
2621 2621 /*
2622 2622 * Since the lbolt service was historically cyclic driven, it must be 'stopped'
2623 2623 * when the system drops into the kernel debugger. lbolt_debug_entry() is
2624 2624 * called by the KDI system claim callbacks to record a hires timestamp at
2625 2625 * debug enter time. lbolt_debug_return() is called by the sistem release
2626 2626 * callbacks to account for the time spent in the debugger. The value is then
2627 2627 * accumulated in the lb_info structure and used by lbolt_event_driven() and
2628 2628 * lbolt_cyclic_driven(), as well as the mdb_get_lbolt() routine.
2629 2629 */
2630 2630 void
2631 2631 lbolt_debug_entry(void)
2632 2632 {
2633 2633 if (lbolt_hybrid != lbolt_bootstrap) {
2634 2634 ASSERT(lb_info != NULL);
2635 2635 lb_info->lbi_debug_ts = gethrtime();
2636 2636 }
2637 2637 }
2638 2638
2639 2639 /*
2640 2640 * Calculate the time spent in the debugger and add it to the lbolt info
2641 2641 * structure. We also update the internal lbolt value in case we were in
2642 2642 * cyclic driven mode going in.
2643 2643 */
2644 2644 void
2645 2645 lbolt_debug_return(void)
2646 2646 {
2647 2647 hrtime_t ts;
2648 2648
2649 2649 if (lbolt_hybrid != lbolt_bootstrap) {
2650 2650 ASSERT(lb_info != NULL);
2651 2651 ASSERT(nsec_per_tick > 0);
2652 2652
2653 2653 ts = gethrtime();
2654 2654 lb_info->lbi_internal = (ts/nsec_per_tick);
2655 2655 lb_info->lbi_debug_time +=
2656 2656 ((ts - lb_info->lbi_debug_ts)/nsec_per_tick);
2657 2657
2658 2658 lb_info->lbi_debug_ts = 0;
2659 2659 }
2660 2660 }
|
↓ open down ↓ |
665 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX