1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T     */
  22 /*        All Rights Reserved   */
  23 
  24 /*
  25  * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
  26  * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
  27  */
  28 
  29 #include <sys/param.h>
  30 #include <sys/t_lock.h>
  31 #include <sys/types.h>
  32 #include <sys/tuneable.h>
  33 #include <sys/sysmacros.h>
  34 #include <sys/systm.h>
  35 #include <sys/cpuvar.h>
  36 #include <sys/lgrp.h>
  37 #include <sys/user.h>
  38 #include <sys/proc.h>
  39 #include <sys/callo.h>
  40 #include <sys/kmem.h>
  41 #include <sys/var.h>
  42 #include <sys/cmn_err.h>
  43 #include <sys/swap.h>
  44 #include <sys/vmsystm.h>
  45 #include <sys/class.h>
  46 #include <sys/time.h>
  47 #include <sys/debug.h>
  48 #include <sys/vtrace.h>
  49 #include <sys/spl.h>
  50 #include <sys/atomic.h>
  51 #include <sys/dumphdr.h>
  52 #include <sys/archsystm.h>
  53 #include <sys/fs/swapnode.h>
  54 #include <sys/panic.h>
  55 #include <sys/disp.h>
  56 #include <sys/msacct.h>
  57 #include <sys/mem_cage.h>
  58 
  59 #include <vm/page.h>
  60 #include <vm/anon.h>
  61 #include <vm/rm.h>
  62 #include <sys/cyclic.h>
  63 #include <sys/cpupart.h>
  64 #include <sys/rctl.h>
  65 #include <sys/task.h>
  66 #include <sys/sdt.h>
  67 #include <sys/ddi_periodic.h>
  68 #include <sys/random.h>
  69 #include <sys/modctl.h>
  70 
  71 /*
  72  * for NTP support
  73  */
  74 #include <sys/timex.h>
  75 #include <sys/inttypes.h>
  76 
  77 #include <sys/sunddi.h>
  78 #include <sys/clock_impl.h>
  79 
  80 /*
  81  * clock() is called straight from the clock cyclic; see clock_init().
  82  *
  83  * Functions:
  84  *      reprime clock
  85  *      maintain date
  86  *      jab the scheduler
  87  */
  88 
  89 extern kcondvar_t       fsflush_cv;
  90 extern sysinfo_t        sysinfo;
  91 extern vminfo_t vminfo;
  92 extern int      idleswtch;      /* flag set while idle in pswtch() */
  93 extern hrtime_t volatile devinfo_freeze;
  94 
  95 /*
  96  * high-precision avenrun values.  These are needed to make the
  97  * regular avenrun values accurate.
  98  */
  99 static uint64_t hp_avenrun[3];
 100 int     avenrun[3];             /* FSCALED average run queue lengths */
 101 time_t  time;   /* time in seconds since 1970 - for compatibility only */
 102 
 103 static struct loadavg_s loadavg;
 104 /*
 105  * Phase/frequency-lock loop (PLL/FLL) definitions
 106  *
 107  * The following variables are read and set by the ntp_adjtime() system
 108  * call.
 109  *
 110  * time_state shows the state of the system clock, with values defined
 111  * in the timex.h header file.
 112  *
 113  * time_status shows the status of the system clock, with bits defined
 114  * in the timex.h header file.
 115  *
 116  * time_offset is used by the PLL/FLL to adjust the system time in small
 117  * increments.
 118  *
 119  * time_constant determines the bandwidth or "stiffness" of the PLL.
 120  *
 121  * time_tolerance determines maximum frequency error or tolerance of the
 122  * CPU clock oscillator and is a property of the architecture; however,
 123  * in principle it could change as result of the presence of external
 124  * discipline signals, for instance.
 125  *
 126  * time_precision is usually equal to the kernel tick variable; however,
 127  * in cases where a precision clock counter or external clock is
 128  * available, the resolution can be much less than this and depend on
 129  * whether the external clock is working or not.
 130  *
 131  * time_maxerror is initialized by a ntp_adjtime() call and increased by
 132  * the kernel once each second to reflect the maximum error bound
 133  * growth.
 134  *
 135  * time_esterror is set and read by the ntp_adjtime() call, but
 136  * otherwise not used by the kernel.
 137  */
 138 int32_t time_state = TIME_OK;   /* clock state */
 139 int32_t time_status = STA_UNSYNC;       /* clock status bits */
 140 int32_t time_offset = 0;                /* time offset (us) */
 141 int32_t time_constant = 0;              /* pll time constant */
 142 int32_t time_tolerance = MAXFREQ;       /* frequency tolerance (scaled ppm) */
 143 int32_t time_precision = 1;     /* clock precision (us) */
 144 int32_t time_maxerror = MAXPHASE;       /* maximum error (us) */
 145 int32_t time_esterror = MAXPHASE;       /* estimated error (us) */
 146 
 147 /*
 148  * The following variables establish the state of the PLL/FLL and the
 149  * residual time and frequency offset of the local clock. The scale
 150  * factors are defined in the timex.h header file.
 151  *
 152  * time_phase and time_freq are the phase increment and the frequency
 153  * increment, respectively, of the kernel time variable.
 154  *
 155  * time_freq is set via ntp_adjtime() from a value stored in a file when
 156  * the synchronization daemon is first started. Its value is retrieved
 157  * via ntp_adjtime() and written to the file about once per hour by the
 158  * daemon.
 159  *
 160  * time_adj is the adjustment added to the value of tick at each timer
 161  * interrupt and is recomputed from time_phase and time_freq at each
 162  * seconds rollover.
 163  *
 164  * time_reftime is the second's portion of the system time at the last
 165  * call to ntp_adjtime(). It is used to adjust the time_freq variable
 166  * and to increase the time_maxerror as the time since last update
 167  * increases.
 168  */
 169 int32_t time_phase = 0;         /* phase offset (scaled us) */
 170 int32_t time_freq = 0;          /* frequency offset (scaled ppm) */
 171 int32_t time_adj = 0;           /* tick adjust (scaled 1 / hz) */
 172 int32_t time_reftime = 0;               /* time at last adjustment (s) */
 173 
 174 /*
 175  * The scale factors of the following variables are defined in the
 176  * timex.h header file.
 177  *
 178  * pps_time contains the time at each calibration interval, as read by
 179  * microtime(). pps_count counts the seconds of the calibration
 180  * interval, the duration of which is nominally pps_shift in powers of
 181  * two.
 182  *
 183  * pps_offset is the time offset produced by the time median filter
 184  * pps_tf[], while pps_jitter is the dispersion (jitter) measured by
 185  * this filter.
 186  *
 187  * pps_freq is the frequency offset produced by the frequency median
 188  * filter pps_ff[], while pps_stabil is the dispersion (wander) measured
 189  * by this filter.
 190  *
 191  * pps_usec is latched from a high resolution counter or external clock
 192  * at pps_time. Here we want the hardware counter contents only, not the
 193  * contents plus the time_tv.usec as usual.
 194  *
 195  * pps_valid counts the number of seconds since the last PPS update. It
 196  * is used as a watchdog timer to disable the PPS discipline should the
 197  * PPS signal be lost.
 198  *
 199  * pps_glitch counts the number of seconds since the beginning of an
 200  * offset burst more than tick/2 from current nominal offset. It is used
 201  * mainly to suppress error bursts due to priority conflicts between the
 202  * PPS interrupt and timer interrupt.
 203  *
 204  * pps_intcnt counts the calibration intervals for use in the interval-
 205  * adaptation algorithm. It's just too complicated for words.
 206  */
 207 struct timeval pps_time;        /* kernel time at last interval */
 208 int32_t pps_tf[] = {0, 0, 0};   /* pps time offset median filter (us) */
 209 int32_t pps_offset = 0;         /* pps time offset (us) */
 210 int32_t pps_jitter = MAXTIME;   /* time dispersion (jitter) (us) */
 211 int32_t pps_ff[] = {0, 0, 0};   /* pps frequency offset median filter */
 212 int32_t pps_freq = 0;           /* frequency offset (scaled ppm) */
 213 int32_t pps_stabil = MAXFREQ;   /* frequency dispersion (scaled ppm) */
 214 int32_t pps_usec = 0;           /* microsec counter at last interval */
 215 int32_t pps_valid = PPS_VALID;  /* pps signal watchdog counter */
 216 int32_t pps_glitch = 0;         /* pps signal glitch counter */
 217 int32_t pps_count = 0;          /* calibration interval counter (s) */
 218 int32_t pps_shift = PPS_SHIFT;  /* interval duration (s) (shift) */
 219 int32_t pps_intcnt = 0;         /* intervals at current duration */
 220 
 221 /*
 222  * PPS signal quality monitors
 223  *
 224  * pps_jitcnt counts the seconds that have been discarded because the
 225  * jitter measured by the time median filter exceeds the limit MAXTIME
 226  * (100 us).
 227  *
 228  * pps_calcnt counts the frequency calibration intervals, which are
 229  * variable from 4 s to 256 s.
 230  *
 231  * pps_errcnt counts the calibration intervals which have been discarded
 232  * because the wander exceeds the limit MAXFREQ (100 ppm) or where the
 233  * calibration interval jitter exceeds two ticks.
 234  *
 235  * pps_stbcnt counts the calibration intervals that have been discarded
 236  * because the frequency wander exceeds the limit MAXFREQ / 4 (25 us).
 237  */
 238 int32_t pps_jitcnt = 0;         /* jitter limit exceeded */
 239 int32_t pps_calcnt = 0;         /* calibration intervals */
 240 int32_t pps_errcnt = 0;         /* calibration errors */
 241 int32_t pps_stbcnt = 0;         /* stability limit exceeded */
 242 
 243 kcondvar_t lbolt_cv;
 244 
 245 /*
 246  * Hybrid lbolt implementation:
 247  *
 248  * The service historically provided by the lbolt and lbolt64 variables has
 249  * been replaced by the ddi_get_lbolt() and ddi_get_lbolt64() routines, and the
 250  * original symbols removed from the system. The once clock driven variables are
 251  * now implemented in an event driven fashion, backed by gethrtime() coarsed to
 252  * the appropriate clock resolution. The default event driven implementation is
 253  * complemented by a cyclic driven one, active only during periods of intense
 254  * activity around the DDI lbolt routines, when a lbolt specific cyclic is
 255  * reprogramed to fire at a clock tick interval to serve consumers of lbolt who
 256  * rely on the original low cost of consulting a memory position.
 257  *
 258  * The implementation uses the number of calls to these routines and the
 259  * frequency of these to determine when to transition from event to cyclic
 260  * driven and vice-versa. These values are kept on a per CPU basis for
 261  * scalability reasons and to prevent CPUs from constantly invalidating a single
 262  * cache line when modifying a global variable. The transition from event to
 263  * cyclic mode happens once the thresholds are crossed, and activity on any CPU
 264  * can cause such transition.
 265  *
 266  * The lbolt_hybrid function pointer is called by ddi_get_lbolt() and
 267  * ddi_get_lbolt64(), and will point to lbolt_event_driven() or
 268  * lbolt_cyclic_driven() according to the current mode. When the thresholds
 269  * are exceeded, lbolt_event_driven() will reprogram the lbolt cyclic to
 270  * fire at a nsec_per_tick interval and increment an internal variable at
 271  * each firing. lbolt_hybrid will then point to lbolt_cyclic_driven(), which
 272  * will simply return the value of such variable. lbolt_cyclic() will attempt
 273  * to shut itself off at each threshold interval (sampling period for calls
 274  * to the DDI lbolt routines), and return to the event driven mode, but will
 275  * be prevented from doing so if lbolt_cyclic_driven() is being heavily used.
 276  *
 277  * lbolt_bootstrap is used during boot to serve lbolt consumers who don't wait
 278  * for the cyclic subsystem to be intialized.
 279  *
 280  */
 281 int64_t lbolt_bootstrap(void);
 282 int64_t lbolt_event_driven(void);
 283 int64_t lbolt_cyclic_driven(void);
 284 int64_t (*lbolt_hybrid)(void) = lbolt_bootstrap;
 285 uint_t lbolt_ev_to_cyclic(caddr_t, caddr_t);
 286 
 287 /*
 288  * lbolt's cyclic, installed by clock_init().
 289  */
 290 static void lbolt_cyclic(void);
 291 
 292 /*
 293  * Tunable to keep lbolt in cyclic driven mode. This will prevent the system
 294  * from switching back to event driven, once it reaches cyclic mode.
 295  */
 296 static boolean_t lbolt_cyc_only = B_FALSE;
 297 
 298 /*
 299  * Cache aligned, per CPU structure with lbolt usage statistics.
 300  */
 301 static lbolt_cpu_t *lb_cpu;
 302 
 303 /*
 304  * Single, cache aligned, structure with all the information required by
 305  * the lbolt implementation.
 306  */
 307 lbolt_info_t *lb_info;
 308 
 309 
 310 int one_sec = 1; /* turned on once every second */
 311 static int fsflushcnt;  /* counter for t_fsflushr */
 312 int     dosynctodr = 1; /* patchable; enable/disable sync to TOD chip */
 313 int     tod_needsync = 0;       /* need to sync tod chip with software time */
 314 static int tod_broken = 0;      /* clock chip doesn't work */
 315 time_t  boot_time = 0;          /* Boot time in seconds since 1970 */
 316 cyclic_id_t clock_cyclic;       /* clock()'s cyclic_id */
 317 cyclic_id_t deadman_cyclic;     /* deadman()'s cyclic_id */
 318 
 319 extern void     clock_tick_schedule(int);
 320 
 321 static int lgrp_ticks;          /* counter to schedule lgrp load calcs */
 322 
 323 /*
 324  * for tod fault detection
 325  */
 326 #define TOD_REF_FREQ            ((longlong_t)(NANOSEC))
 327 #define TOD_STALL_THRESHOLD     (TOD_REF_FREQ * 3 / 2)
 328 #define TOD_JUMP_THRESHOLD      (TOD_REF_FREQ / 2)
 329 #define TOD_FILTER_N            4
 330 #define TOD_FILTER_SETTLE       (4 * TOD_FILTER_N)
 331 static int tod_faulted = TOD_NOFAULT;
 332 
 333 static int tod_status_flag = 0;         /* used by tod_validate() */
 334 
 335 static hrtime_t prev_set_tick = 0;      /* gethrtime() prior to tod_set() */
 336 static time_t prev_set_tod = 0;         /* tv_sec value passed to tod_set() */
 337 
 338 /* patchable via /etc/system */
 339 int tod_validate_enable = 1;
 340 
 341 /* Diagnose/Limit messages about delay(9F) called from interrupt context */
 342 int                     delay_from_interrupt_diagnose = 0;
 343 volatile uint32_t       delay_from_interrupt_msg = 20;
 344 
 345 /*
 346  * On non-SPARC systems, TOD validation must be deferred until gethrtime
 347  * returns non-zero values (after mach_clkinit's execution).
 348  * On SPARC systems, it must be deferred until after hrtime_base
 349  * and hres_last_tick are set (in the first invocation of hres_tick).
 350  * Since in both cases the prerequisites occur before the invocation of
 351  * tod_get() in clock(), the deferment is lifted there.
 352  */
 353 static boolean_t tod_validate_deferred = B_TRUE;
 354 
 355 /*
 356  * tod_fault_table[] must be aligned with
 357  * enum tod_fault_type in systm.h
 358  */
 359 static char *tod_fault_table[] = {
 360         "Reversed",                     /* TOD_REVERSED */
 361         "Stalled",                      /* TOD_STALLED */
 362         "Jumped",                       /* TOD_JUMPED */
 363         "Changed in Clock Rate",        /* TOD_RATECHANGED */
 364         "Is Read-Only"                  /* TOD_RDONLY */
 365         /*
 366          * no strings needed for TOD_NOFAULT
 367          */
 368 };
 369 
 370 /*
 371  * test hook for tod broken detection in tod_validate
 372  */
 373 int tod_unit_test = 0;
 374 time_t tod_test_injector;
 375 
 376 #define CLOCK_ADJ_HIST_SIZE     4
 377 
 378 static int      adj_hist_entry;
 379 
 380 int64_t clock_adj_hist[CLOCK_ADJ_HIST_SIZE];
 381 
 382 static void calcloadavg(int, uint64_t *);
 383 static int genloadavg(struct loadavg_s *);
 384 static void loadavg_update();
 385 
 386 void (*cmm_clock_callout)() = NULL;
 387 void (*cpucaps_clock_callout)() = NULL;
 388 
 389 extern clock_t clock_tick_proc_max;
 390 
 391 static int64_t deadman_counter = 0;
 392 
 393 static void
 394 clock(void)
 395 {
 396         kthread_t       *t;
 397         uint_t  nrunnable;
 398         uint_t  w_io;
 399         cpu_t   *cp;
 400         cpupart_t *cpupart;
 401         extern  void    set_freemem();
 402         void    (*funcp)();
 403         int32_t ltemp;
 404         int64_t lltemp;
 405         int s;
 406         int do_lgrp_load;
 407         int i;
 408         clock_t now = LBOLT_NO_ACCOUNT; /* current tick */
 409 
 410         if (panicstr)
 411                 return;
 412 
 413         /*
 414          * Make sure that 'freemem' do not drift too far from the truth
 415          */
 416         set_freemem();
 417 
 418 
 419         /*
 420          * Before the section which is repeated is executed, we do
 421          * the time delta processing which occurs every clock tick
 422          *
 423          * There is additional processing which happens every time
 424          * the nanosecond counter rolls over which is described
 425          * below - see the section which begins with : if (one_sec)
 426          *
 427          * This section marks the beginning of the precision-kernel
 428          * code fragment.
 429          *
 430          * First, compute the phase adjustment. If the low-order bits
 431          * (time_phase) of the update overflow, bump the higher order
 432          * bits (time_update).
 433          */
 434         time_phase += time_adj;
 435         if (time_phase <= -FINEUSEC) {
 436                 ltemp = -time_phase / SCALE_PHASE;
 437                 time_phase += ltemp * SCALE_PHASE;
 438                 s = hr_clock_lock();
 439                 timedelta -= ltemp * (NANOSEC/MICROSEC);
 440                 hr_clock_unlock(s);
 441         } else if (time_phase >= FINEUSEC) {
 442                 ltemp = time_phase / SCALE_PHASE;
 443                 time_phase -= ltemp * SCALE_PHASE;
 444                 s = hr_clock_lock();
 445                 timedelta += ltemp * (NANOSEC/MICROSEC);
 446                 hr_clock_unlock(s);
 447         }
 448 
 449         /*
 450          * End of precision-kernel code fragment which is processed
 451          * every timer interrupt.
 452          *
 453          * Continue with the interrupt processing as scheduled.
 454          */
 455         /*
 456          * Count the number of runnable threads and the number waiting
 457          * for some form of I/O to complete -- gets added to
 458          * sysinfo.waiting.  To know the state of the system, must add
 459          * wait counts from all CPUs.  Also add up the per-partition
 460          * statistics.
 461          */
 462         w_io = 0;
 463         nrunnable = 0;
 464 
 465         /*
 466          * keep track of when to update lgrp/part loads
 467          */
 468 
 469         do_lgrp_load = 0;
 470         if (lgrp_ticks++ >= hz / 10) {
 471                 lgrp_ticks = 0;
 472                 do_lgrp_load = 1;
 473         }
 474 
 475         if (one_sec) {
 476                 loadavg_update();
 477                 deadman_counter++;
 478         }
 479 
 480         /*
 481          * First count the threads waiting on kpreempt queues in each
 482          * CPU partition.
 483          */
 484 
 485         cpupart = cp_list_head;
 486         do {
 487                 uint_t cpupart_nrunnable = cpupart->cp_kp_queue.disp_nrunnable;
 488 
 489                 cpupart->cp_updates++;
 490                 nrunnable += cpupart_nrunnable;
 491                 cpupart->cp_nrunnable_cum += cpupart_nrunnable;
 492                 if (one_sec) {
 493                         cpupart->cp_nrunning = 0;
 494                         cpupart->cp_nrunnable = cpupart_nrunnable;
 495                 }
 496         } while ((cpupart = cpupart->cp_next) != cp_list_head);
 497 
 498 
 499         /* Now count the per-CPU statistics. */
 500         cp = cpu_list;
 501         do {
 502                 uint_t cpu_nrunnable = cp->cpu_disp->disp_nrunnable;
 503 
 504                 nrunnable += cpu_nrunnable;
 505                 cpupart = cp->cpu_part;
 506                 cpupart->cp_nrunnable_cum += cpu_nrunnable;
 507                 if (one_sec) {
 508                         cpupart->cp_nrunnable += cpu_nrunnable;
 509                         /*
 510                          * Update user, system, and idle cpu times.
 511                          */
 512                         cpupart->cp_nrunning++;
 513                         /*
 514                          * w_io is used to update sysinfo.waiting during
 515                          * one_second processing below.  Only gather w_io
 516                          * information when we walk the list of cpus if we're
 517                          * going to perform one_second processing.
 518                          */
 519                         w_io += CPU_STATS(cp, sys.iowait);
 520                 }
 521 
 522                 if (one_sec && (cp->cpu_flags & CPU_EXISTS)) {
 523                         int i, load, change;
 524                         hrtime_t intracct, intrused;
 525                         const hrtime_t maxnsec = 1000000000;
 526                         const int precision = 100;
 527 
 528                         /*
 529                          * Estimate interrupt load on this cpu each second.
 530                          * Computes cpu_intrload as %utilization (0-99).
 531                          */
 532 
 533                         /* add up interrupt time from all micro states */
 534                         for (intracct = 0, i = 0; i < NCMSTATES; i++)
 535                                 intracct += cp->cpu_intracct[i];
 536                         scalehrtime(&intracct);
 537 
 538                         /* compute nsec used in the past second */
 539                         intrused = intracct - cp->cpu_intrlast;
 540                         cp->cpu_intrlast = intracct;
 541 
 542                         /* limit the value for safety (and the first pass) */
 543                         if (intrused >= maxnsec)
 544                                 intrused = maxnsec - 1;
 545 
 546                         /* calculate %time in interrupt */
 547                         load = (precision * intrused) / maxnsec;
 548                         ASSERT(load >= 0 && load < precision);
 549                         change = cp->cpu_intrload - load;
 550 
 551                         /* jump to new max, or decay the old max */
 552                         if (change < 0)
 553                                 cp->cpu_intrload = load;
 554                         else if (change > 0)
 555                                 cp->cpu_intrload -= (change + 3) / 4;
 556 
 557                         DTRACE_PROBE3(cpu_intrload,
 558                             cpu_t *, cp,
 559                             hrtime_t, intracct,
 560                             hrtime_t, intrused);
 561                 }
 562 
 563                 if (do_lgrp_load &&
 564                     (cp->cpu_flags & CPU_EXISTS)) {
 565                         /*
 566                          * When updating the lgroup's load average,
 567                          * account for the thread running on the CPU.
 568                          * If the CPU is the current one, then we need
 569                          * to account for the underlying thread which
 570                          * got the clock interrupt not the thread that is
 571                          * handling the interrupt and caculating the load
 572                          * average
 573                          */
 574                         t = cp->cpu_thread;
 575                         if (CPU == cp)
 576                                 t = t->t_intr;
 577 
 578                         /*
 579                          * Account for the load average for this thread if
 580                          * it isn't the idle thread or it is on the interrupt
 581                          * stack and not the current CPU handling the clock
 582                          * interrupt
 583                          */
 584                         if ((t && t != cp->cpu_idle_thread) || (CPU != cp &&
 585                             CPU_ON_INTR(cp))) {
 586                                 if (t->t_lpl == cp->cpu_lpl) {
 587                                         /* local thread */
 588                                         cpu_nrunnable++;
 589                                 } else {
 590                                         /*
 591                                          * This is a remote thread, charge it
 592                                          * against its home lgroup.  Note that
 593                                          * we notice that a thread is remote
 594                                          * only if it's currently executing.
 595                                          * This is a reasonable approximation,
 596                                          * since queued remote threads are rare.
 597                                          * Note also that if we didn't charge
 598                                          * it to its home lgroup, remote
 599                                          * execution would often make a system
 600                                          * appear balanced even though it was
 601                                          * not, and thread placement/migration
 602                                          * would often not be done correctly.
 603                                          */
 604                                         lgrp_loadavg(t->t_lpl,
 605                                             LGRP_LOADAVG_IN_THREAD_MAX, 0);
 606                                 }
 607                         }
 608                         lgrp_loadavg(cp->cpu_lpl,
 609                             cpu_nrunnable * LGRP_LOADAVG_IN_THREAD_MAX, 1);
 610                 }
 611         } while ((cp = cp->cpu_next) != cpu_list);
 612 
 613         clock_tick_schedule(one_sec);
 614 
 615         /*
 616          * Check for a callout that needs be called from the clock
 617          * thread to support the membership protocol in a clustered
 618          * system.  Copy the function pointer so that we can reset
 619          * this to NULL if needed.
 620          */
 621         if ((funcp = cmm_clock_callout) != NULL)
 622                 (*funcp)();
 623 
 624         if ((funcp = cpucaps_clock_callout) != NULL)
 625                 (*funcp)();
 626 
 627         /*
 628          * Wakeup the cageout thread waiters once per second.
 629          */
 630         if (one_sec)
 631                 kcage_tick();
 632 
 633         if (one_sec) {
 634 
 635                 int drift, absdrift;
 636                 timestruc_t tod;
 637                 int s;
 638 
 639                 /*
 640                  * Beginning of precision-kernel code fragment executed
 641                  * every second.
 642                  *
 643                  * On rollover of the second the phase adjustment to be
 644                  * used for the next second is calculated.  Also, the
 645                  * maximum error is increased by the tolerance.  If the
 646                  * PPS frequency discipline code is present, the phase is
 647                  * increased to compensate for the CPU clock oscillator
 648                  * frequency error.
 649                  *
 650                  * On a 32-bit machine and given parameters in the timex.h
 651                  * header file, the maximum phase adjustment is +-512 ms
 652                  * and maximum frequency offset is (a tad less than)
 653                  * +-512 ppm. On a 64-bit machine, you shouldn't need to ask.
 654                  */
 655                 time_maxerror += time_tolerance / SCALE_USEC;
 656 
 657                 /*
 658                  * Leap second processing. If in leap-insert state at
 659                  * the end of the day, the system clock is set back one
 660                  * second; if in leap-delete state, the system clock is
 661                  * set ahead one second. The microtime() routine or
 662                  * external clock driver will insure that reported time
 663                  * is always monotonic. The ugly divides should be
 664                  * replaced.
 665                  */
 666                 switch (time_state) {
 667 
 668                 case TIME_OK:
 669                         if (time_status & STA_INS)
 670                                 time_state = TIME_INS;
 671                         else if (time_status & STA_DEL)
 672                                 time_state = TIME_DEL;
 673                         break;
 674 
 675                 case TIME_INS:
 676                         if (hrestime.tv_sec % 86400 == 0) {
 677                                 s = hr_clock_lock();
 678                                 hrestime.tv_sec--;
 679                                 hr_clock_unlock(s);
 680                                 time_state = TIME_OOP;
 681                         }
 682                         break;
 683 
 684                 case TIME_DEL:
 685                         if ((hrestime.tv_sec + 1) % 86400 == 0) {
 686                                 s = hr_clock_lock();
 687                                 hrestime.tv_sec++;
 688                                 hr_clock_unlock(s);
 689                                 time_state = TIME_WAIT;
 690                         }
 691                         break;
 692 
 693                 case TIME_OOP:
 694                         time_state = TIME_WAIT;
 695                         break;
 696 
 697                 case TIME_WAIT:
 698                         if (!(time_status & (STA_INS | STA_DEL)))
 699                                 time_state = TIME_OK;
 700                 default:
 701                         break;
 702                 }
 703 
 704                 /*
 705                  * Compute the phase adjustment for the next second. In
 706                  * PLL mode, the offset is reduced by a fixed factor
 707                  * times the time constant. In FLL mode the offset is
 708                  * used directly. In either mode, the maximum phase
 709                  * adjustment for each second is clamped so as to spread
 710                  * the adjustment over not more than the number of
 711                  * seconds between updates.
 712                  */
 713                 if (time_offset == 0)
 714                         time_adj = 0;
 715                 else if (time_offset < 0) {
 716                         lltemp = -time_offset;
 717                         if (!(time_status & STA_FLL)) {
 718                                 if ((1 << time_constant) >= SCALE_KG)
 719                                         lltemp *= (1 << time_constant) /
 720                                             SCALE_KG;
 721                                 else
 722                                         lltemp = (lltemp / SCALE_KG) >>
 723                                             time_constant;
 724                         }
 725                         if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
 726                                 lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
 727                         time_offset += lltemp;
 728                         time_adj = -(lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
 729                 } else {
 730                         lltemp = time_offset;
 731                         if (!(time_status & STA_FLL)) {
 732                                 if ((1 << time_constant) >= SCALE_KG)
 733                                         lltemp *= (1 << time_constant) /
 734                                             SCALE_KG;
 735                                 else
 736                                         lltemp = (lltemp / SCALE_KG) >>
 737                                             time_constant;
 738                         }
 739                         if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
 740                                 lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
 741                         time_offset -= lltemp;
 742                         time_adj = (lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
 743                 }
 744 
 745                 /*
 746                  * Compute the frequency estimate and additional phase
 747                  * adjustment due to frequency error for the next
 748                  * second. When the PPS signal is engaged, gnaw on the
 749                  * watchdog counter and update the frequency computed by
 750                  * the pll and the PPS signal.
 751                  */
 752                 pps_valid++;
 753                 if (pps_valid == PPS_VALID) {
 754                         pps_jitter = MAXTIME;
 755                         pps_stabil = MAXFREQ;
 756                         time_status &= ~(STA_PPSSIGNAL | STA_PPSJITTER |
 757                             STA_PPSWANDER | STA_PPSERROR);
 758                 }
 759                 lltemp = time_freq + pps_freq;
 760 
 761                 if (lltemp)
 762                         time_adj += (lltemp * SCALE_PHASE) / (SCALE_USEC * hz);
 763 
 764                 /*
 765                  * End of precision kernel-code fragment
 766                  *
 767                  * The section below should be modified if we are planning
 768                  * to use NTP for synchronization.
 769                  *
 770                  * Note: the clock synchronization code now assumes
 771                  * the following:
 772                  *   - if dosynctodr is 1, then compute the drift between
 773                  *      the tod chip and software time and adjust one or
 774                  *      the other depending on the circumstances
 775                  *
 776                  *   - if dosynctodr is 0, then the tod chip is independent
 777                  *      of the software clock and should not be adjusted,
 778                  *      but allowed to free run.  this allows NTP to sync.
 779                  *      hrestime without any interference from the tod chip.
 780                  */
 781 
 782                 tod_validate_deferred = B_FALSE;
 783                 mutex_enter(&tod_lock);
 784                 tod = tod_get();
 785                 drift = tod.tv_sec - hrestime.tv_sec;
 786                 absdrift = (drift >= 0) ? drift : -drift;
 787                 if (tod_needsync || absdrift > 1) {
 788                         int s;
 789                         if (absdrift > 2) {
 790                                 if (!tod_broken && tod_faulted == TOD_NOFAULT) {
 791                                         s = hr_clock_lock();
 792                                         hrestime = tod;
 793                                         membar_enter(); /* hrestime visible */
 794                                         timedelta = 0;
 795                                         timechanged++;
 796                                         tod_needsync = 0;
 797                                         hr_clock_unlock(s);
 798                                         callout_hrestime();
 799 
 800                                 }
 801                         } else {
 802                                 if (tod_needsync || !dosynctodr) {
 803                                         gethrestime(&tod);
 804                                         tod_set(tod);
 805                                         s = hr_clock_lock();
 806                                         if (timedelta == 0)
 807                                                 tod_needsync = 0;
 808                                         hr_clock_unlock(s);
 809                                 } else {
 810                                         /*
 811                                          * If the drift is 2 seconds on the
 812                                          * money, then the TOD is adjusting
 813                                          * the clock;  record that.
 814                                          */
 815                                         clock_adj_hist[adj_hist_entry++ %
 816                                             CLOCK_ADJ_HIST_SIZE] = now;
 817                                         s = hr_clock_lock();
 818                                         timedelta = (int64_t)drift*NANOSEC;
 819                                         hr_clock_unlock(s);
 820                                 }
 821                         }
 822                 }
 823                 one_sec = 0;
 824                 time = gethrestime_sec();  /* for crusty old kmem readers */
 825                 mutex_exit(&tod_lock);
 826 
 827                 /*
 828                  * Some drivers still depend on this... XXX
 829                  */
 830                 cv_broadcast(&lbolt_cv);
 831 
 832                 vminfo.freemem += freemem;
 833                 {
 834                         pgcnt_t maxswap, resv, free;
 835                         pgcnt_t avail =
 836                             MAX((spgcnt_t)(availrmem - swapfs_minfree), 0);
 837 
 838                         maxswap = k_anoninfo.ani_mem_resv +
 839                             k_anoninfo.ani_max +avail;
 840                         /* Update ani_free */
 841                         set_anoninfo();
 842                         free = k_anoninfo.ani_free + avail;
 843                         resv = k_anoninfo.ani_phys_resv +
 844                             k_anoninfo.ani_mem_resv;
 845 
 846                         vminfo.swap_resv += resv;
 847                         /* number of reserved and allocated pages */
 848 #ifdef  DEBUG
 849                         if (maxswap < free)
 850                                 cmn_err(CE_WARN, "clock: maxswap < free");
 851                         if (maxswap < resv)
 852                                 cmn_err(CE_WARN, "clock: maxswap < resv");
 853 #endif
 854                         vminfo.swap_alloc += maxswap - free;
 855                         vminfo.swap_avail += maxswap - resv;
 856                         vminfo.swap_free += free;
 857                 }
 858                 vminfo.updates++;
 859                 if (nrunnable) {
 860                         sysinfo.runque += nrunnable;
 861                         sysinfo.runocc++;
 862                 }
 863                 if (nswapped) {
 864                         sysinfo.swpque += nswapped;
 865                         sysinfo.swpocc++;
 866                 }
 867                 sysinfo.waiting += w_io;
 868                 sysinfo.updates++;
 869 
 870                 /*
 871                  * Wake up fsflush to write out DELWRI
 872                  * buffers, dirty pages and other cached
 873                  * administrative data, e.g. inodes.
 874                  */
 875                 if (--fsflushcnt <= 0) {
 876                         fsflushcnt = tune.t_fsflushr;
 877                         cv_signal(&fsflush_cv);
 878                 }
 879 
 880                 vmmeter();
 881                 calcloadavg(genloadavg(&loadavg), hp_avenrun);
 882                 for (i = 0; i < 3; i++)
 883                         /*
 884                          * At the moment avenrun[] can only hold 31
 885                          * bits of load average as it is a signed
 886                          * int in the API. We need to ensure that
 887                          * hp_avenrun[i] >> (16 - FSHIFT) will not be
 888                          * too large. If it is, we put the largest value
 889                          * that we can use into avenrun[i]. This is
 890                          * kludgey, but about all we can do until we
 891                          * avenrun[] is declared as an array of uint64[]
 892                          */
 893                         if (hp_avenrun[i] < ((uint64_t)1<<(31+16-FSHIFT)))
 894                                 avenrun[i] = (int32_t)(hp_avenrun[i] >>
 895                                     (16 - FSHIFT));
 896                         else
 897                                 avenrun[i] = 0x7fffffff;
 898 
 899                 cpupart = cp_list_head;
 900                 do {
 901                         calcloadavg(genloadavg(&cpupart->cp_loadavg),
 902                             cpupart->cp_hp_avenrun);
 903                 } while ((cpupart = cpupart->cp_next) != cp_list_head);
 904         }
 905 }
 906 
 907 void
 908 clock_init(void)
 909 {
 910         cyc_handler_t clk_hdlr, lbolt_hdlr;
 911         cyc_time_t clk_when, lbolt_when;
 912         int i, sz;
 913         intptr_t buf;
 914 
 915         /*
 916          * Setup handler and timer for the clock cyclic.
 917          */
 918         clk_hdlr.cyh_func = (cyc_func_t)clock;
 919         clk_hdlr.cyh_level = CY_LOCK_LEVEL;
 920         clk_hdlr.cyh_arg = NULL;
 921 
 922         clk_when.cyt_when = 0;
 923         clk_when.cyt_interval = nsec_per_tick;
 924 
 925         /*
 926          * The lbolt cyclic will be reprogramed to fire at a nsec_per_tick
 927          * interval to satisfy performance needs of the DDI lbolt consumers.
 928          * It is off by default.
 929          */
 930         lbolt_hdlr.cyh_func = (cyc_func_t)lbolt_cyclic;
 931         lbolt_hdlr.cyh_level = CY_LOCK_LEVEL;
 932         lbolt_hdlr.cyh_arg = NULL;
 933 
 934         lbolt_when.cyt_interval = nsec_per_tick;
 935 
 936         /*
 937          * Allocate cache line aligned space for the per CPU lbolt data and
 938          * lbolt info structures, and initialize them with their default
 939          * values. Note that these structures are also cache line sized.
 940          */
 941         sz = sizeof (lbolt_info_t) + CPU_CACHE_COHERENCE_SIZE;
 942         buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
 943         lb_info = (lbolt_info_t *)P2ROUNDUP(buf, CPU_CACHE_COHERENCE_SIZE);
 944 
 945         if (hz != HZ_DEFAULT)
 946                 lb_info->lbi_thresh_interval = LBOLT_THRESH_INTERVAL *
 947                     hz/HZ_DEFAULT;
 948         else
 949                 lb_info->lbi_thresh_interval = LBOLT_THRESH_INTERVAL;
 950 
 951         lb_info->lbi_thresh_calls = LBOLT_THRESH_CALLS;
 952 
 953         sz = (sizeof (lbolt_cpu_t) * max_ncpus) + CPU_CACHE_COHERENCE_SIZE;
 954         buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
 955         lb_cpu = (lbolt_cpu_t *)P2ROUNDUP(buf, CPU_CACHE_COHERENCE_SIZE);
 956 
 957         for (i = 0; i < max_ncpus; i++)
 958                 lb_cpu[i].lbc_counter = lb_info->lbi_thresh_calls;
 959 
 960         /*
 961          * Install the softint used to switch between event and cyclic driven
 962          * lbolt. We use a soft interrupt to make sure the context of the
 963          * cyclic reprogram call is safe.
 964          */
 965         lbolt_softint_add();
 966 
 967         /*
 968          * Since the hybrid lbolt implementation is based on a hardware counter
 969          * that is reset at every hardware reboot and that we'd like to have
 970          * the lbolt value starting at zero after both a hardware and a fast
 971          * reboot, we calculate the number of clock ticks the system's been up
 972          * and store it in the lbi_debug_time field of the lbolt info structure.
 973          * The value of this field will be subtracted from lbolt before
 974          * returning it.
 975          */
 976         lb_info->lbi_internal = lb_info->lbi_debug_time =
 977             (gethrtime()/nsec_per_tick);
 978 
 979         /*
 980          * lbolt_hybrid points at lbolt_bootstrap until now. The LBOLT_* macros
 981          * and lbolt_debug_{enter,return} use this value as an indication that
 982          * the initializaion above hasn't been completed. Setting lbolt_hybrid
 983          * to either lbolt_{cyclic,event}_driven here signals those code paths
 984          * that the lbolt related structures can be used.
 985          */
 986         if (lbolt_cyc_only) {
 987                 lbolt_when.cyt_when = 0;
 988                 lbolt_hybrid = lbolt_cyclic_driven;
 989         } else {
 990                 lbolt_when.cyt_when = CY_INFINITY;
 991                 lbolt_hybrid = lbolt_event_driven;
 992         }
 993 
 994         /*
 995          * Grab cpu_lock and install all three cyclics.
 996          */
 997         mutex_enter(&cpu_lock);
 998 
 999         clock_cyclic = cyclic_add(&clk_hdlr, &clk_when);
1000         lb_info->id.lbi_cyclic_id = cyclic_add(&lbolt_hdlr, &lbolt_when);
1001 
1002         mutex_exit(&cpu_lock);
1003 }
1004 
1005 /*
1006  * Called before calcloadavg to get 10-sec moving loadavg together
1007  */
1008 
1009 static int
1010 genloadavg(struct loadavg_s *avgs)
1011 {
1012         int avg;
1013         int spos; /* starting position */
1014         int cpos; /* moving current position */
1015         int i;
1016         int slen;
1017         hrtime_t hr_avg;
1018 
1019         /* 10-second snapshot, calculate first positon */
1020         if (avgs->lg_len == 0) {
1021                 return (0);
1022         }
1023         slen = avgs->lg_len < S_MOVAVG_SZ ? avgs->lg_len : S_MOVAVG_SZ;
1024 
1025         spos = (avgs->lg_cur - 1) >= 0 ? avgs->lg_cur - 1 :
1026             S_LOADAVG_SZ + (avgs->lg_cur - 1);
1027         for (i = hr_avg = 0; i < slen; i++) {
1028                 cpos = (spos - i) >= 0 ? spos - i : S_LOADAVG_SZ + (spos - i);
1029                 hr_avg += avgs->lg_loads[cpos];
1030         }
1031 
1032         hr_avg = hr_avg / slen;
1033         avg = hr_avg / (NANOSEC / LGRP_LOADAVG_IN_THREAD_MAX);
1034 
1035         return (avg);
1036 }
1037 
1038 /*
1039  * Run every second from clock () to update the loadavg count available to the
1040  * system and cpu-partitions.
1041  *
1042  * This works by sampling the previous usr, sys, wait time elapsed,
1043  * computing a delta, and adding that delta to the elapsed usr, sys,
1044  * wait increase.
1045  */
1046 
1047 static void
1048 loadavg_update()
1049 {
1050         cpu_t *cp;
1051         cpupart_t *cpupart;
1052         hrtime_t cpu_total;
1053         int prev;
1054 
1055         cp = cpu_list;
1056         loadavg.lg_total = 0;
1057 
1058         /*
1059          * first pass totals up per-cpu statistics for system and cpu
1060          * partitions
1061          */
1062 
1063         do {
1064                 struct loadavg_s *lavg;
1065 
1066                 lavg = &cp->cpu_loadavg;
1067 
1068                 cpu_total = cp->cpu_acct[CMS_USER] +
1069                     cp->cpu_acct[CMS_SYSTEM] + cp->cpu_waitrq;
1070                 /* compute delta against last total */
1071                 scalehrtime(&cpu_total);
1072                 prev = (lavg->lg_cur - 1) >= 0 ? lavg->lg_cur - 1 :
1073                     S_LOADAVG_SZ + (lavg->lg_cur - 1);
1074                 if (lavg->lg_loads[prev] <= 0) {
1075                         lavg->lg_loads[lavg->lg_cur] = cpu_total;
1076                         cpu_total = 0;
1077                 } else {
1078                         lavg->lg_loads[lavg->lg_cur] = cpu_total;
1079                         cpu_total = cpu_total - lavg->lg_loads[prev];
1080                         if (cpu_total < 0)
1081                                 cpu_total = 0;
1082                 }
1083 
1084                 lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
1085                 lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
1086                     lavg->lg_len + 1 : S_LOADAVG_SZ;
1087 
1088                 loadavg.lg_total += cpu_total;
1089                 cp->cpu_part->cp_loadavg.lg_total += cpu_total;
1090 
1091         } while ((cp = cp->cpu_next) != cpu_list);
1092 
1093         loadavg.lg_loads[loadavg.lg_cur] = loadavg.lg_total;
1094         loadavg.lg_cur = (loadavg.lg_cur + 1) % S_LOADAVG_SZ;
1095         loadavg.lg_len = (loadavg.lg_len + 1) < S_LOADAVG_SZ ?
1096             loadavg.lg_len + 1 : S_LOADAVG_SZ;
1097         /*
1098          * Second pass updates counts
1099          */
1100         cpupart = cp_list_head;
1101 
1102         do {
1103                 struct loadavg_s *lavg;
1104 
1105                 lavg = &cpupart->cp_loadavg;
1106                 lavg->lg_loads[lavg->lg_cur] = lavg->lg_total;
1107                 lavg->lg_total = 0;
1108                 lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
1109                 lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
1110                     lavg->lg_len + 1 : S_LOADAVG_SZ;
1111 
1112         } while ((cpupart = cpupart->cp_next) != cp_list_head);
1113 
1114 }
1115 
1116 /*
1117  * clock_update() - local clock update
1118  *
1119  * This routine is called by ntp_adjtime() to update the local clock
1120  * phase and frequency. The implementation is of an
1121  * adaptive-parameter, hybrid phase/frequency-lock loop (PLL/FLL). The
1122  * routine computes new time and frequency offset estimates for each
1123  * call.  The PPS signal itself determines the new time offset,
1124  * instead of the calling argument.  Presumably, calls to
1125  * ntp_adjtime() occur only when the caller believes the local clock
1126  * is valid within some bound (+-128 ms with NTP). If the caller's
1127  * time is far different than the PPS time, an argument will ensue,
1128  * and it's not clear who will lose.
1129  *
1130  * For uncompensated quartz crystal oscillatores and nominal update
1131  * intervals less than 1024 s, operation should be in phase-lock mode
1132  * (STA_FLL = 0), where the loop is disciplined to phase. For update
1133  * intervals greater than this, operation should be in frequency-lock
1134  * mode (STA_FLL = 1), where the loop is disciplined to frequency.
1135  *
1136  * Note: mutex(&tod_lock) is in effect.
1137  */
1138 void
1139 clock_update(int offset)
1140 {
1141         int ltemp, mtemp, s;
1142 
1143         ASSERT(MUTEX_HELD(&tod_lock));
1144 
1145         if (!(time_status & STA_PLL) && !(time_status & STA_PPSTIME))
1146                 return;
1147         ltemp = offset;
1148         if ((time_status & STA_PPSTIME) && (time_status & STA_PPSSIGNAL))
1149                 ltemp = pps_offset;
1150 
1151         /*
1152          * Scale the phase adjustment and clamp to the operating range.
1153          */
1154         if (ltemp > MAXPHASE)
1155                 time_offset = MAXPHASE * SCALE_UPDATE;
1156         else if (ltemp < -MAXPHASE)
1157                 time_offset = -(MAXPHASE * SCALE_UPDATE);
1158         else
1159                 time_offset = ltemp * SCALE_UPDATE;
1160 
1161         /*
1162          * Select whether the frequency is to be controlled and in which
1163          * mode (PLL or FLL). Clamp to the operating range. Ugly
1164          * multiply/divide should be replaced someday.
1165          */
1166         if (time_status & STA_FREQHOLD || time_reftime == 0)
1167                 time_reftime = hrestime.tv_sec;
1168 
1169         mtemp = hrestime.tv_sec - time_reftime;
1170         time_reftime = hrestime.tv_sec;
1171 
1172         if (time_status & STA_FLL) {
1173                 if (mtemp >= MINSEC) {
1174                         ltemp = ((time_offset / mtemp) * (SCALE_USEC /
1175                             SCALE_UPDATE));
1176                         if (ltemp)
1177                                 time_freq += ltemp / SCALE_KH;
1178                 }
1179         } else {
1180                 if (mtemp < MAXSEC) {
1181                         ltemp *= mtemp;
1182                         if (ltemp)
1183                                 time_freq += (int)(((int64_t)ltemp *
1184                                     SCALE_USEC) / SCALE_KF)
1185                                     / (1 << (time_constant * 2));
1186                 }
1187         }
1188         if (time_freq > time_tolerance)
1189                 time_freq = time_tolerance;
1190         else if (time_freq < -time_tolerance)
1191                 time_freq = -time_tolerance;
1192 
1193         s = hr_clock_lock();
1194         tod_needsync = 1;
1195         hr_clock_unlock(s);
1196 }
1197 
1198 /*
1199  * ddi_hardpps() - discipline CPU clock oscillator to external PPS signal
1200  *
1201  * This routine is called at each PPS interrupt in order to discipline
1202  * the CPU clock oscillator to the PPS signal. It measures the PPS phase
1203  * and leaves it in a handy spot for the clock() routine. It
1204  * integrates successive PPS phase differences and calculates the
1205  * frequency offset. This is used in clock() to discipline the CPU
1206  * clock oscillator so that intrinsic frequency error is cancelled out.
1207  * The code requires the caller to capture the time and hardware counter
1208  * value at the on-time PPS signal transition.
1209  *
1210  * Note that, on some Unix systems, this routine runs at an interrupt
1211  * priority level higher than the timer interrupt routine clock().
1212  * Therefore, the variables used are distinct from the clock()
1213  * variables, except for certain exceptions: The PPS frequency pps_freq
1214  * and phase pps_offset variables are determined by this routine and
1215  * updated atomically. The time_tolerance variable can be considered a
1216  * constant, since it is infrequently changed, and then only when the
1217  * PPS signal is disabled. The watchdog counter pps_valid is updated
1218  * once per second by clock() and is atomically cleared in this
1219  * routine.
1220  *
1221  * tvp is the time of the last tick; usec is a microsecond count since the
1222  * last tick.
1223  *
1224  * Note: In Solaris systems, the tick value is actually given by
1225  *       usec_per_tick.  This is called from the serial driver cdintr(),
1226  *       or equivalent, at a high PIL.  Because the kernel keeps a
1227  *       highresolution time, the following code can accept either
1228  *       the traditional argument pair, or the current highres timestamp
1229  *       in tvp and zero in usec.
1230  */
1231 void
1232 ddi_hardpps(struct timeval *tvp, int usec)
1233 {
1234         int u_usec, v_usec, bigtick;
1235         time_t cal_sec;
1236         int cal_usec;
1237 
1238         /*
1239          * An occasional glitch can be produced when the PPS interrupt
1240          * occurs in the clock() routine before the time variable is
1241          * updated. Here the offset is discarded when the difference
1242          * between it and the last one is greater than tick/2, but not
1243          * if the interval since the first discard exceeds 30 s.
1244          */
1245         time_status |= STA_PPSSIGNAL;
1246         time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR);
1247         pps_valid = 0;
1248         u_usec = -tvp->tv_usec;
1249         if (u_usec < -(MICROSEC/2))
1250                 u_usec += MICROSEC;
1251         v_usec = pps_offset - u_usec;
1252         if (v_usec < 0)
1253                 v_usec = -v_usec;
1254         if (v_usec > (usec_per_tick >> 1)) {
1255                 if (pps_glitch > MAXGLITCH) {
1256                         pps_glitch = 0;
1257                         pps_tf[2] = u_usec;
1258                         pps_tf[1] = u_usec;
1259                 } else {
1260                         pps_glitch++;
1261                         u_usec = pps_offset;
1262                 }
1263         } else
1264                 pps_glitch = 0;
1265 
1266         /*
1267          * A three-stage median filter is used to help deglitch the pps
1268          * time. The median sample becomes the time offset estimate; the
1269          * difference between the other two samples becomes the time
1270          * dispersion (jitter) estimate.
1271          */
1272         pps_tf[2] = pps_tf[1];
1273         pps_tf[1] = pps_tf[0];
1274         pps_tf[0] = u_usec;
1275         if (pps_tf[0] > pps_tf[1]) {
1276                 if (pps_tf[1] > pps_tf[2]) {
1277                         pps_offset = pps_tf[1];         /* 0 1 2 */
1278                         v_usec = pps_tf[0] - pps_tf[2];
1279                 } else if (pps_tf[2] > pps_tf[0]) {
1280                         pps_offset = pps_tf[0];         /* 2 0 1 */
1281                         v_usec = pps_tf[2] - pps_tf[1];
1282                 } else {
1283                         pps_offset = pps_tf[2];         /* 0 2 1 */
1284                         v_usec = pps_tf[0] - pps_tf[1];
1285                 }
1286         } else {
1287                 if (pps_tf[1] < pps_tf[2]) {
1288                         pps_offset = pps_tf[1];         /* 2 1 0 */
1289                         v_usec = pps_tf[2] - pps_tf[0];
1290                 } else  if (pps_tf[2] < pps_tf[0]) {
1291                         pps_offset = pps_tf[0];         /* 1 0 2 */
1292                         v_usec = pps_tf[1] - pps_tf[2];
1293                 } else {
1294                         pps_offset = pps_tf[2];         /* 1 2 0 */
1295                         v_usec = pps_tf[1] - pps_tf[0];
1296                 }
1297         }
1298         if (v_usec > MAXTIME)
1299                 pps_jitcnt++;
1300         v_usec = (v_usec << PPS_AVG) - pps_jitter;
1301         pps_jitter += v_usec / (1 << PPS_AVG);
1302         if (pps_jitter > (MAXTIME >> 1))
1303                 time_status |= STA_PPSJITTER;
1304 
1305         /*
1306          * During the calibration interval adjust the starting time when
1307          * the tick overflows. At the end of the interval compute the
1308          * duration of the interval and the difference of the hardware
1309          * counters at the beginning and end of the interval. This code
1310          * is deliciously complicated by the fact valid differences may
1311          * exceed the value of tick when using long calibration
1312          * intervals and small ticks. Note that the counter can be
1313          * greater than tick if caught at just the wrong instant, but
1314          * the values returned and used here are correct.
1315          */
1316         bigtick = (int)usec_per_tick * SCALE_USEC;
1317         pps_usec -= pps_freq;
1318         if (pps_usec >= bigtick)
1319                 pps_usec -= bigtick;
1320         if (pps_usec < 0)
1321                 pps_usec += bigtick;
1322         pps_time.tv_sec++;
1323         pps_count++;
1324         if (pps_count < (1 << pps_shift))
1325                 return;
1326         pps_count = 0;
1327         pps_calcnt++;
1328         u_usec = usec * SCALE_USEC;
1329         v_usec = pps_usec - u_usec;
1330         if (v_usec >= bigtick >> 1)
1331                 v_usec -= bigtick;
1332         if (v_usec < -(bigtick >> 1))
1333                 v_usec += bigtick;
1334         if (v_usec < 0)
1335                 v_usec = -(-v_usec >> pps_shift);
1336         else
1337                 v_usec = v_usec >> pps_shift;
1338         pps_usec = u_usec;
1339         cal_sec = tvp->tv_sec;
1340         cal_usec = tvp->tv_usec;
1341         cal_sec -= pps_time.tv_sec;
1342         cal_usec -= pps_time.tv_usec;
1343         if (cal_usec < 0) {
1344                 cal_usec += MICROSEC;
1345                 cal_sec--;
1346         }
1347         pps_time = *tvp;
1348 
1349         /*
1350          * Check for lost interrupts, noise, excessive jitter and
1351          * excessive frequency error. The number of timer ticks during
1352          * the interval may vary +-1 tick. Add to this a margin of one
1353          * tick for the PPS signal jitter and maximum frequency
1354          * deviation. If the limits are exceeded, the calibration
1355          * interval is reset to the minimum and we start over.
1356          */
1357         u_usec = (int)usec_per_tick << 1;
1358         if (!((cal_sec == -1 && cal_usec > (MICROSEC - u_usec)) ||
1359             (cal_sec == 0 && cal_usec < u_usec)) ||
1360             v_usec > time_tolerance || v_usec < -time_tolerance) {
1361                 pps_errcnt++;
1362                 pps_shift = PPS_SHIFT;
1363                 pps_intcnt = 0;
1364                 time_status |= STA_PPSERROR;
1365                 return;
1366         }
1367 
1368         /*
1369          * A three-stage median filter is used to help deglitch the pps
1370          * frequency. The median sample becomes the frequency offset
1371          * estimate; the difference between the other two samples
1372          * becomes the frequency dispersion (stability) estimate.
1373          */
1374         pps_ff[2] = pps_ff[1];
1375         pps_ff[1] = pps_ff[0];
1376         pps_ff[0] = v_usec;
1377         if (pps_ff[0] > pps_ff[1]) {
1378                 if (pps_ff[1] > pps_ff[2]) {
1379                         u_usec = pps_ff[1];             /* 0 1 2 */
1380                         v_usec = pps_ff[0] - pps_ff[2];
1381                 } else if (pps_ff[2] > pps_ff[0]) {
1382                         u_usec = pps_ff[0];             /* 2 0 1 */
1383                         v_usec = pps_ff[2] - pps_ff[1];
1384                 } else {
1385                         u_usec = pps_ff[2];             /* 0 2 1 */
1386                         v_usec = pps_ff[0] - pps_ff[1];
1387                 }
1388         } else {
1389                 if (pps_ff[1] < pps_ff[2]) {
1390                         u_usec = pps_ff[1];             /* 2 1 0 */
1391                         v_usec = pps_ff[2] - pps_ff[0];
1392                 } else  if (pps_ff[2] < pps_ff[0]) {
1393                         u_usec = pps_ff[0];             /* 1 0 2 */
1394                         v_usec = pps_ff[1] - pps_ff[2];
1395                 } else {
1396                         u_usec = pps_ff[2];             /* 1 2 0 */
1397                         v_usec = pps_ff[1] - pps_ff[0];
1398                 }
1399         }
1400 
1401         /*
1402          * Here the frequency dispersion (stability) is updated. If it
1403          * is less than one-fourth the maximum (MAXFREQ), the frequency
1404          * offset is updated as well, but clamped to the tolerance. It
1405          * will be processed later by the clock() routine.
1406          */
1407         v_usec = (v_usec >> 1) - pps_stabil;
1408         if (v_usec < 0)
1409                 pps_stabil -= -v_usec >> PPS_AVG;
1410         else
1411                 pps_stabil += v_usec >> PPS_AVG;
1412         if (pps_stabil > MAXFREQ >> 2) {
1413                 pps_stbcnt++;
1414                 time_status |= STA_PPSWANDER;
1415                 return;
1416         }
1417         if (time_status & STA_PPSFREQ) {
1418                 if (u_usec < 0) {
1419                         pps_freq -= -u_usec >> PPS_AVG;
1420                         if (pps_freq < -time_tolerance)
1421                                 pps_freq = -time_tolerance;
1422                         u_usec = -u_usec;
1423                 } else {
1424                         pps_freq += u_usec >> PPS_AVG;
1425                         if (pps_freq > time_tolerance)
1426                                 pps_freq = time_tolerance;
1427                 }
1428         }
1429 
1430         /*
1431          * Here the calibration interval is adjusted. If the maximum
1432          * time difference is greater than tick / 4, reduce the interval
1433          * by half. If this is not the case for four consecutive
1434          * intervals, double the interval.
1435          */
1436         if (u_usec << pps_shift > bigtick >> 2) {
1437                 pps_intcnt = 0;
1438                 if (pps_shift > PPS_SHIFT)
1439                         pps_shift--;
1440         } else if (pps_intcnt >= 4) {
1441                 pps_intcnt = 0;
1442                 if (pps_shift < PPS_SHIFTMAX)
1443                         pps_shift++;
1444         } else
1445                 pps_intcnt++;
1446 
1447         /*
1448          * If recovering from kmdb, then make sure the tod chip gets resynced.
1449          * If we took an early exit above, then we don't yet have a stable
1450          * calibration signal to lock onto, so don't mark the tod for sync
1451          * until we get all the way here.
1452          */
1453         {
1454                 int s = hr_clock_lock();
1455 
1456                 tod_needsync = 1;
1457                 hr_clock_unlock(s);
1458         }
1459 }
1460 
1461 /*
1462  * Handle clock tick processing for a thread.
1463  * Check for timer action, enforce CPU rlimit, do profiling etc.
1464  */
1465 void
1466 clock_tick(kthread_t *t, int pending)
1467 {
1468         struct proc *pp;
1469         klwp_id_t    lwp;
1470         struct as *as;
1471         clock_t ticks;
1472         int     poke = 0;               /* notify another CPU */
1473         int     user_mode;
1474         size_t   rss;
1475         int i, total_usec, usec;
1476         rctl_qty_t secs;
1477 
1478         ASSERT(pending > 0);
1479 
1480         /* Must be operating on a lwp/thread */
1481         if ((lwp = ttolwp(t)) == NULL) {
1482                 panic("clock_tick: no lwp");
1483                 /*NOTREACHED*/
1484         }
1485 
1486         for (i = 0; i < pending; i++) {
1487                 CL_TICK(t);     /* Class specific tick processing */
1488                 DTRACE_SCHED1(tick, kthread_t *, t);
1489         }
1490 
1491         pp = ttoproc(t);
1492 
1493         /* pp->p_lock makes sure that the thread does not exit */
1494         ASSERT(MUTEX_HELD(&pp->p_lock));
1495 
1496         user_mode = (lwp->lwp_state == LWP_USER);
1497 
1498         ticks = (pp->p_utime + pp->p_stime) % hz;
1499         /*
1500          * Update process times. Should use high res clock and state
1501          * changes instead of statistical sampling method. XXX
1502          */
1503         if (user_mode) {
1504                 pp->p_utime += pending;
1505         } else {
1506                 pp->p_stime += pending;
1507         }
1508 
1509         pp->p_ttime += pending;
1510         as = pp->p_as;
1511 
1512         /*
1513          * Update user profiling statistics. Get the pc from the
1514          * lwp when the AST happens.
1515          */
1516         if (pp->p_prof.pr_scale) {
1517                 atomic_add_32(&lwp->lwp_oweupc, (int32_t)pending);
1518                 if (user_mode) {
1519                         poke = 1;
1520                         aston(t);
1521                 }
1522         }
1523 
1524         /*
1525          * If CPU was in user state, process lwp-virtual time
1526          * interval timer. The value passed to itimerdecr() has to be
1527          * in microseconds and has to be less than one second. Hence
1528          * this loop.
1529          */
1530         total_usec = usec_per_tick * pending;
1531         while (total_usec > 0) {
1532                 usec = MIN(total_usec, (MICROSEC - 1));
1533                 if (user_mode &&
1534                     timerisset(&lwp->lwp_timer[ITIMER_VIRTUAL].it_value) &&
1535                     itimerdecr(&lwp->lwp_timer[ITIMER_VIRTUAL], usec) == 0) {
1536                         poke = 1;
1537                         sigtoproc(pp, t, SIGVTALRM);
1538                 }
1539                 total_usec -= usec;
1540         }
1541 
1542         /*
1543          * If CPU was in user state, process lwp-profile
1544          * interval timer.
1545          */
1546         total_usec = usec_per_tick * pending;
1547         while (total_usec > 0) {
1548                 usec = MIN(total_usec, (MICROSEC - 1));
1549                 if (timerisset(&lwp->lwp_timer[ITIMER_PROF].it_value) &&
1550                     itimerdecr(&lwp->lwp_timer[ITIMER_PROF], usec) == 0) {
1551                         poke = 1;
1552                         sigtoproc(pp, t, SIGPROF);
1553                 }
1554                 total_usec -= usec;
1555         }
1556 
1557         /*
1558          * Enforce CPU resource controls:
1559          *   (a) process.max-cpu-time resource control
1560          *
1561          * Perform the check only if we have accumulated more a second.
1562          */
1563         if ((ticks + pending) >= hz) {
1564                 (void) rctl_test(rctlproc_legacy[RLIMIT_CPU], pp->p_rctls, pp,
1565                     (pp->p_utime + pp->p_stime)/hz, RCA_UNSAFE_SIGINFO);
1566         }
1567 
1568         /*
1569          *   (b) task.max-cpu-time resource control
1570          *
1571          * If we have accumulated enough ticks, increment the task CPU
1572          * time usage and test for the resource limit. This minimizes the
1573          * number of calls to the rct_test(). The task CPU time mutex
1574          * is highly contentious as many processes can be sharing a task.
1575          */
1576         if (pp->p_ttime >= clock_tick_proc_max) {
1577                 secs = task_cpu_time_incr(pp->p_task, pp->p_ttime);
1578                 pp->p_ttime = 0;
1579                 if (secs) {
1580                         (void) rctl_test(rc_task_cpu_time, pp->p_task->tk_rctls,
1581                             pp, secs, RCA_UNSAFE_SIGINFO);
1582                 }
1583         }
1584 
1585         /*
1586          * Update memory usage for the currently running process.
1587          */
1588         rss = rm_asrss(as);
1589         PTOU(pp)->u_mem += rss;
1590         if (rss > PTOU(pp)->u_mem_max)
1591                 PTOU(pp)->u_mem_max = rss;
1592 
1593         /*
1594          * Notify the CPU the thread is running on.
1595          */
1596         if (poke && t->t_cpu != CPU)
1597                 poke_cpu(t->t_cpu->cpu_id);
1598 }
1599 
1600 void
1601 profil_tick(uintptr_t upc)
1602 {
1603         int ticks;
1604         proc_t *p = ttoproc(curthread);
1605         klwp_t *lwp = ttolwp(curthread);
1606         struct prof *pr = &p->p_prof;
1607 
1608         do {
1609                 ticks = lwp->lwp_oweupc;
1610         } while (cas32(&lwp->lwp_oweupc, ticks, 0) != ticks);
1611 
1612         mutex_enter(&p->p_pflock);
1613         if (pr->pr_scale >= 2 && upc >= pr->pr_off) {
1614                 /*
1615                  * Old-style profiling
1616                  */
1617                 uint16_t *slot = pr->pr_base;
1618                 uint16_t old, new;
1619                 if (pr->pr_scale != 2) {
1620                         uintptr_t delta = upc - pr->pr_off;
1621                         uintptr_t byteoff = ((delta >> 16) * pr->pr_scale) +
1622                             (((delta & 0xffff) * pr->pr_scale) >> 16);
1623                         if (byteoff >= (uintptr_t)pr->pr_size) {
1624                                 mutex_exit(&p->p_pflock);
1625                                 return;
1626                         }
1627                         slot += byteoff / sizeof (uint16_t);
1628                 }
1629                 if (fuword16(slot, &old) < 0 ||
1630                     (new = old + ticks) > SHRT_MAX ||
1631                     suword16(slot, new) < 0) {
1632                         pr->pr_scale = 0;
1633                 }
1634         } else if (pr->pr_scale == 1) {
1635                 /*
1636                  * PC Sampling
1637                  */
1638                 model_t model = lwp_getdatamodel(lwp);
1639                 int result;
1640 #ifdef __lint
1641                 model = model;
1642 #endif
1643                 while (ticks-- > 0) {
1644                         if (pr->pr_samples == pr->pr_size) {
1645                                 /* buffer full, turn off sampling */
1646                                 pr->pr_scale = 0;
1647                                 break;
1648                         }
1649                         switch (SIZEOF_PTR(model)) {
1650                         case sizeof (uint32_t):
1651                                 result = suword32(pr->pr_base, (uint32_t)upc);
1652                                 break;
1653 #ifdef _LP64
1654                         case sizeof (uint64_t):
1655                                 result = suword64(pr->pr_base, (uint64_t)upc);
1656                                 break;
1657 #endif
1658                         default:
1659                                 cmn_err(CE_WARN, "profil_tick: unexpected "
1660                                     "data model");
1661                                 result = -1;
1662                                 break;
1663                         }
1664                         if (result != 0) {
1665                                 pr->pr_scale = 0;
1666                                 break;
1667                         }
1668                         pr->pr_base = (caddr_t)pr->pr_base + SIZEOF_PTR(model);
1669                         pr->pr_samples++;
1670                 }
1671         }
1672         mutex_exit(&p->p_pflock);
1673 }
1674 
1675 static void
1676 delay_wakeup(void *arg)
1677 {
1678         kthread_t       *t = arg;
1679 
1680         mutex_enter(&t->t_delay_lock);
1681         cv_signal(&t->t_delay_cv);
1682         mutex_exit(&t->t_delay_lock);
1683 }
1684 
1685 /*
1686  * The delay(9F) man page indicates that it can only be called from user or
1687  * kernel context - detect and diagnose bad calls. The following macro will
1688  * produce a limited number of messages identifying bad callers.  This is done
1689  * in a macro so that caller() is meaningful. When a bad caller is identified,
1690  * switching to 'drv_usecwait(TICK_TO_USEC(ticks));' may be appropriate.
1691  */
1692 #define DELAY_CONTEXT_CHECK()   {                                       \
1693         uint32_t        m;                                              \
1694         char            *f;                                             \
1695         ulong_t         off;                                            \
1696                                                                         \
1697         m = delay_from_interrupt_msg;                                   \
1698         if (delay_from_interrupt_diagnose && servicing_interrupt() &&   \
1699             !panicstr && !devinfo_freeze &&                             \
1700             atomic_cas_32(&delay_from_interrupt_msg, m ? m : 1, m-1)) {     \
1701                 f = modgetsymname((uintptr_t)caller(), &off);               \
1702                 cmn_err(CE_WARN, "delay(9F) called from "               \
1703                     "interrupt context: %s`%s",                         \
1704                     mod_containing_pc(caller()), f ? f : "...");        \
1705         }                                                               \
1706 }
1707 
1708 /*
1709  * delay_common: common delay code.
1710  */
1711 static void
1712 delay_common(clock_t ticks)
1713 {
1714         kthread_t       *t = curthread;
1715         clock_t         deadline;
1716         clock_t         timeleft;
1717         callout_id_t    id;
1718 
1719         /* If timeouts aren't running all we can do is spin. */
1720         if (panicstr || devinfo_freeze) {
1721                 /* Convert delay(9F) call into drv_usecwait(9F) call. */
1722                 if (ticks > 0)
1723                         drv_usecwait(TICK_TO_USEC(ticks));
1724                 return;
1725         }
1726 
1727         deadline = ddi_get_lbolt() + ticks;
1728         while ((timeleft = deadline - ddi_get_lbolt()) > 0) {
1729                 mutex_enter(&t->t_delay_lock);
1730                 id = timeout_default(delay_wakeup, t, timeleft);
1731                 cv_wait(&t->t_delay_cv, &t->t_delay_lock);
1732                 mutex_exit(&t->t_delay_lock);
1733                 (void) untimeout_default(id, 0);
1734         }
1735 }
1736 
1737 /*
1738  * Delay specified number of clock ticks.
1739  */
1740 void
1741 delay(clock_t ticks)
1742 {
1743         DELAY_CONTEXT_CHECK();
1744 
1745         delay_common(ticks);
1746 }
1747 
1748 /*
1749  * Delay a random number of clock ticks between 1 and ticks.
1750  */
1751 void
1752 delay_random(clock_t ticks)
1753 {
1754         int     r;
1755 
1756         DELAY_CONTEXT_CHECK();
1757 
1758         (void) random_get_pseudo_bytes((void *)&r, sizeof (r));
1759         if (ticks == 0)
1760                 ticks = 1;
1761         ticks = (r % ticks) + 1;
1762         delay_common(ticks);
1763 }
1764 
1765 /*
1766  * Like delay, but interruptible by a signal.
1767  */
1768 int
1769 delay_sig(clock_t ticks)
1770 {
1771         kthread_t       *t = curthread;
1772         clock_t         deadline;
1773         clock_t         rc;
1774 
1775         /* If timeouts aren't running all we can do is spin. */
1776         if (panicstr || devinfo_freeze) {
1777                 if (ticks > 0)
1778                         drv_usecwait(TICK_TO_USEC(ticks));
1779                 return (0);
1780         }
1781 
1782         deadline = ddi_get_lbolt() + ticks;
1783         mutex_enter(&t->t_delay_lock);
1784         do {
1785                 rc = cv_timedwait_sig(&t->t_delay_cv,
1786                     &t->t_delay_lock, deadline);
1787                 /* loop until past deadline or signaled */
1788         } while (rc > 0);
1789         mutex_exit(&t->t_delay_lock);
1790         if (rc == 0)
1791                 return (EINTR);
1792         return (0);
1793 }
1794 
1795 
1796 #define SECONDS_PER_DAY 86400
1797 
1798 /*
1799  * Initialize the system time based on the TOD chip.  approx is used as
1800  * an approximation of time (e.g. from the filesystem) in the event that
1801  * the TOD chip has been cleared or is unresponsive.  An approx of -1
1802  * means the filesystem doesn't keep time.
1803  */
1804 void
1805 clkset(time_t approx)
1806 {
1807         timestruc_t ts;
1808         int spl;
1809         int set_clock = 0;
1810 
1811         mutex_enter(&tod_lock);
1812         ts = tod_get();
1813 
1814         if (ts.tv_sec > 365 * SECONDS_PER_DAY) {
1815                 /*
1816                  * If the TOD chip is reporting some time after 1971,
1817                  * then it probably didn't lose power or become otherwise
1818                  * cleared in the recent past;  check to assure that
1819                  * the time coming from the filesystem isn't in the future
1820                  * according to the TOD chip.
1821                  */
1822                 if (approx != -1 && approx > ts.tv_sec) {
1823                         cmn_err(CE_WARN, "Last shutdown is later "
1824                             "than time on time-of-day chip; check date.");
1825                 }
1826         } else {
1827                 /*
1828                  * If the TOD chip isn't giving correct time, set it to the
1829                  * greater of i) approx and ii) 1987. That way if approx
1830                  * is negative or is earlier than 1987, we set the clock
1831                  * back to a time when Oliver North, ALF and Dire Straits
1832                  * were all on the collective brain:  1987.
1833                  */
1834                 timestruc_t tmp;
1835                 time_t diagnose_date = (1987 - 1970) * 365 * SECONDS_PER_DAY;
1836                 ts.tv_sec = (approx > diagnose_date ? approx : diagnose_date);
1837                 ts.tv_nsec = 0;
1838 
1839                 /*
1840                  * Attempt to write the new time to the TOD chip.  Set spl high
1841                  * to avoid getting preempted between the tod_set and tod_get.
1842                  */
1843                 spl = splhi();
1844                 tod_set(ts);
1845                 tmp = tod_get();
1846                 splx(spl);
1847 
1848                 if (tmp.tv_sec != ts.tv_sec && tmp.tv_sec != ts.tv_sec + 1) {
1849                         tod_broken = 1;
1850                         dosynctodr = 0;
1851                         cmn_err(CE_WARN, "Time-of-day chip unresponsive.");
1852                 } else {
1853                         cmn_err(CE_WARN, "Time-of-day chip had "
1854                             "incorrect date; check and reset.");
1855                 }
1856                 set_clock = 1;
1857         }
1858 
1859         if (!boot_time) {
1860                 boot_time = ts.tv_sec;
1861                 set_clock = 1;
1862         }
1863 
1864         if (set_clock)
1865                 set_hrestime(&ts);
1866 
1867         mutex_exit(&tod_lock);
1868 }
1869 
1870 int     timechanged;    /* for testing if the system time has been reset */
1871 
1872 void
1873 set_hrestime(timestruc_t *ts)
1874 {
1875         int spl = hr_clock_lock();
1876         hrestime = *ts;
1877         membar_enter(); /* hrestime must be visible before timechanged++ */
1878         timedelta = 0;
1879         timechanged++;
1880         hr_clock_unlock(spl);
1881         callout_hrestime();
1882 }
1883 
1884 static uint_t deadman_seconds;
1885 static uint32_t deadman_panics;
1886 static int deadman_enabled = 0;
1887 static int deadman_panic_timers = 1;
1888 
1889 static void
1890 deadman(void)
1891 {
1892         if (panicstr) {
1893                 /*
1894                  * During panic, other CPUs besides the panic
1895                  * master continue to handle cyclics and some other
1896                  * interrupts.  The code below is intended to be
1897                  * single threaded, so any CPU other than the master
1898                  * must keep out.
1899                  */
1900                 if (CPU->cpu_id != panic_cpu.cpu_id)
1901                         return;
1902 
1903                 if (!deadman_panic_timers)
1904                         return; /* allow all timers to be manually disabled */
1905 
1906                 /*
1907                  * If we are generating a crash dump or syncing filesystems and
1908                  * the corresponding timer is set, decrement it and re-enter
1909                  * the panic code to abort it and advance to the next state.
1910                  * The panic states and triggers are explained in panic.c.
1911                  */
1912                 if (panic_dump) {
1913                         if (dump_timeleft && (--dump_timeleft == 0)) {
1914                                 panic("panic dump timeout");
1915                                 /*NOTREACHED*/
1916                         }
1917                 } else if (panic_sync) {
1918                         if (sync_timeleft && (--sync_timeleft == 0)) {
1919                                 panic("panic sync timeout");
1920                                 /*NOTREACHED*/
1921                         }
1922                 }
1923 
1924                 return;
1925         }
1926 
1927         if (deadman_counter != CPU->cpu_deadman_counter) {
1928                 CPU->cpu_deadman_counter = deadman_counter;
1929                 CPU->cpu_deadman_countdown = deadman_seconds;
1930                 return;
1931         }
1932 
1933         if (--CPU->cpu_deadman_countdown > 0)
1934                 return;
1935 
1936         /*
1937          * Regardless of whether or not we actually bring the system down,
1938          * bump the deadman_panics variable.
1939          *
1940          * N.B. deadman_panics is incremented once for each CPU that
1941          * passes through here.  It's expected that all the CPUs will
1942          * detect this condition within one second of each other, so
1943          * when deadman_enabled is off, deadman_panics will
1944          * typically be a multiple of the total number of CPUs in
1945          * the system.
1946          */
1947         atomic_add_32(&deadman_panics, 1);
1948 
1949         if (!deadman_enabled) {
1950                 CPU->cpu_deadman_countdown = deadman_seconds;
1951                 return;
1952         }
1953 
1954         /*
1955          * If we're here, we want to bring the system down.
1956          */
1957         panic("deadman: timed out after %d seconds of clock "
1958             "inactivity", deadman_seconds);
1959         /*NOTREACHED*/
1960 }
1961 
1962 /*ARGSUSED*/
1963 static void
1964 deadman_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
1965 {
1966         cpu->cpu_deadman_counter = 0;
1967         cpu->cpu_deadman_countdown = deadman_seconds;
1968 
1969         hdlr->cyh_func = (cyc_func_t)deadman;
1970         hdlr->cyh_level = CY_HIGH_LEVEL;
1971         hdlr->cyh_arg = NULL;
1972 
1973         /*
1974          * Stagger the CPUs so that they don't all run deadman() at
1975          * the same time.  Simplest reason to do this is to make it
1976          * more likely that only one CPU will panic in case of a
1977          * timeout.  This is (strictly speaking) an aesthetic, not a
1978          * technical consideration.
1979          */
1980         when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU);
1981         when->cyt_interval = NANOSEC;
1982 }
1983 
1984 
1985 void
1986 deadman_init(void)
1987 {
1988         cyc_omni_handler_t hdlr;
1989 
1990         if (deadman_seconds == 0)
1991                 deadman_seconds = snoop_interval / MICROSEC;
1992 
1993         if (snooping)
1994                 deadman_enabled = 1;
1995 
1996         hdlr.cyo_online = deadman_online;
1997         hdlr.cyo_offline = NULL;
1998         hdlr.cyo_arg = NULL;
1999 
2000         mutex_enter(&cpu_lock);
2001         deadman_cyclic = cyclic_add_omni(&hdlr);
2002         mutex_exit(&cpu_lock);
2003 }
2004 
2005 /*
2006  * tod_fault() is for updating tod validate mechanism state:
2007  * (1) TOD_NOFAULT: for resetting the state to 'normal'.
2008  *     currently used for debugging only
2009  * (2) The following four cases detected by tod validate mechanism:
2010  *       TOD_REVERSED: current tod value is less than previous value.
2011  *       TOD_STALLED: current tod value hasn't advanced.
2012  *       TOD_JUMPED: current tod value advanced too far from previous value.
2013  *       TOD_RATECHANGED: the ratio between average tod delta and
2014  *       average tick delta has changed.
2015  * (3) TOD_RDONLY: when the TOD clock is not writeable e.g. because it is
2016  *     a virtual TOD provided by a hypervisor.
2017  */
2018 enum tod_fault_type
2019 tod_fault(enum tod_fault_type ftype, int off)
2020 {
2021         ASSERT(MUTEX_HELD(&tod_lock));
2022 
2023         if (tod_faulted != ftype) {
2024                 switch (ftype) {
2025                 case TOD_NOFAULT:
2026                         plat_tod_fault(TOD_NOFAULT);
2027                         cmn_err(CE_NOTE, "Restarted tracking "
2028                             "Time of Day clock.");
2029                         tod_faulted = ftype;
2030                         break;
2031                 case TOD_REVERSED:
2032                 case TOD_JUMPED:
2033                         if (tod_faulted == TOD_NOFAULT) {
2034                                 plat_tod_fault(ftype);
2035                                 cmn_err(CE_WARN, "Time of Day clock error: "
2036                                     "reason [%s by 0x%x]. -- "
2037                                     " Stopped tracking Time Of Day clock.",
2038                                     tod_fault_table[ftype], off);
2039                                 tod_faulted = ftype;
2040                         }
2041                         break;
2042                 case TOD_STALLED:
2043                 case TOD_RATECHANGED:
2044                         if (tod_faulted == TOD_NOFAULT) {
2045                                 plat_tod_fault(ftype);
2046                                 cmn_err(CE_WARN, "Time of Day clock error: "
2047                                     "reason [%s]. -- "
2048                                     " Stopped tracking Time Of Day clock.",
2049                                     tod_fault_table[ftype]);
2050                                 tod_faulted = ftype;
2051                         }
2052                         break;
2053                 case TOD_RDONLY:
2054                         if (tod_faulted == TOD_NOFAULT) {
2055                                 plat_tod_fault(ftype);
2056                                 cmn_err(CE_NOTE, "!Time of Day clock is "
2057                                     "Read-Only; set of Date/Time will not "
2058                                     "persist across reboot.");
2059                                 tod_faulted = ftype;
2060                         }
2061                         break;
2062                 default:
2063                         break;
2064                 }
2065         }
2066         return (tod_faulted);
2067 }
2068 
2069 /*
2070  * Two functions that allow tod_status_flag to be manipulated by functions
2071  * external to this file.
2072  */
2073 
2074 void
2075 tod_status_set(int tod_flag)
2076 {
2077         tod_status_flag |= tod_flag;
2078 }
2079 
2080 void
2081 tod_status_clear(int tod_flag)
2082 {
2083         tod_status_flag &= ~tod_flag;
2084 }
2085 
2086 /*
2087  * Record a timestamp and the value passed to tod_set().  The next call to
2088  * tod_validate() can use these values, prev_set_tick and prev_set_tod,
2089  * when checking the timestruc_t returned by tod_get().  Ordinarily,
2090  * tod_validate() will use prev_tick and prev_tod for this task but these
2091  * become obsolete, and will be re-assigned with the prev_set_* values,
2092  * in the case when the TOD is re-written.
2093  */
2094 void
2095 tod_set_prev(timestruc_t ts)
2096 {
2097         if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
2098             tod_validate_deferred) {
2099                 return;
2100         }
2101         prev_set_tick = gethrtime();
2102         /*
2103          * A negative value will be set to zero in utc_to_tod() so we fake
2104          * a zero here in such a case.  This would need to change if the
2105          * behavior of utc_to_tod() changes.
2106          */
2107         prev_set_tod = ts.tv_sec < 0 ? 0 : ts.tv_sec;
2108 }
2109 
2110 /*
2111  * tod_validate() is used for checking values returned by tod_get().
2112  * Four error cases can be detected by this routine:
2113  *   TOD_REVERSED: current tod value is less than previous.
2114  *   TOD_STALLED: current tod value hasn't advanced.
2115  *   TOD_JUMPED: current tod value advanced too far from previous value.
2116  *   TOD_RATECHANGED: the ratio between average tod delta and
2117  *   average tick delta has changed.
2118  */
2119 time_t
2120 tod_validate(time_t tod)
2121 {
2122         time_t diff_tod;
2123         hrtime_t diff_tick;
2124 
2125         long dtick;
2126         int dtick_delta;
2127 
2128         int off = 0;
2129         enum tod_fault_type tod_bad = TOD_NOFAULT;
2130 
2131         static int firsttime = 1;
2132 
2133         static time_t prev_tod = 0;
2134         static hrtime_t prev_tick = 0;
2135         static long dtick_avg = TOD_REF_FREQ;
2136 
2137         int cpr_resume_done = 0;
2138         int dr_resume_done = 0;
2139 
2140         hrtime_t tick = gethrtime();
2141 
2142         ASSERT(MUTEX_HELD(&tod_lock));
2143 
2144         /*
2145          * tod_validate_enable is patchable via /etc/system.
2146          * If TOD is already faulted, or if TOD validation is deferred,
2147          * there is nothing to do.
2148          */
2149         if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
2150             tod_validate_deferred) {
2151                 return (tod);
2152         }
2153 
2154         /*
2155          * If this is the first time through, we just need to save the tod
2156          * we were called with and hrtime so we can use them next time to
2157          * validate tod_get().
2158          */
2159         if (firsttime) {
2160                 firsttime = 0;
2161                 prev_tod = tod;
2162                 prev_tick = tick;
2163                 return (tod);
2164         }
2165 
2166         /*
2167          * Handle any flags that have been turned on by tod_status_set().
2168          * In the case where a tod_set() is done and then a subsequent
2169          * tod_get() fails (ie, both TOD_SET_DONE and TOD_GET_FAILED are
2170          * true), we treat the TOD_GET_FAILED with precedence by switching
2171          * off the flag, returning tod and leaving TOD_SET_DONE asserted
2172          * until such time as tod_get() completes successfully.
2173          */
2174         if (tod_status_flag & TOD_GET_FAILED) {
2175                 /*
2176                  * tod_get() has encountered an issue, possibly transitory,
2177                  * when reading TOD.  We'll just return the incoming tod
2178                  * value (which is actually hrestime.tv_sec in this case)
2179                  * and when we get a genuine tod, following a successful
2180                  * tod_get(), we can validate using prev_tod and prev_tick.
2181                  */
2182                 tod_status_flag &= ~TOD_GET_FAILED;
2183                 return (tod);
2184         } else if (tod_status_flag & TOD_SET_DONE) {
2185                 /*
2186                  * TOD has been modified.  Just before the TOD was written,
2187                  * tod_set_prev() saved tod and hrtime; we can now use
2188                  * those values, prev_set_tod and prev_set_tick, to validate
2189                  * the incoming tod that's just been read.
2190                  */
2191                 prev_tod = prev_set_tod;
2192                 prev_tick = prev_set_tick;
2193                 dtick_avg = TOD_REF_FREQ;
2194                 tod_status_flag &= ~TOD_SET_DONE;
2195                 /*
2196                  * If a tod_set() preceded a cpr_suspend() without an
2197                  * intervening tod_validate(), we need to ensure that a
2198                  * TOD_JUMPED condition is ignored.
2199                  * Note this isn't a concern in the case of DR as we've
2200                  * just reassigned dtick_avg, above.
2201                  */
2202                 if (tod_status_flag & TOD_CPR_RESUME_DONE) {
2203                         cpr_resume_done = 1;
2204                         tod_status_flag &= ~TOD_CPR_RESUME_DONE;
2205                 }
2206         } else if (tod_status_flag & TOD_CPR_RESUME_DONE) {
2207                 /*
2208                  * The system's coming back from a checkpoint resume.
2209                  */
2210                 cpr_resume_done = 1;
2211                 tod_status_flag &= ~TOD_CPR_RESUME_DONE;
2212                 /*
2213                  * We need to handle the possibility of a CPR suspend
2214                  * operation having been initiated whilst a DR event was
2215                  * in-flight.
2216                  */
2217                 if (tod_status_flag & TOD_DR_RESUME_DONE) {
2218                         dr_resume_done = 1;
2219                         tod_status_flag &= ~TOD_DR_RESUME_DONE;
2220                 }
2221         } else if (tod_status_flag & TOD_DR_RESUME_DONE) {
2222                 /*
2223                  * A Dynamic Reconfiguration event has taken place.
2224                  */
2225                 dr_resume_done = 1;
2226                 tod_status_flag &= ~TOD_DR_RESUME_DONE;
2227         }
2228 
2229         /* test hook */
2230         switch (tod_unit_test) {
2231         case 1: /* for testing jumping tod */
2232                 tod += tod_test_injector;
2233                 tod_unit_test = 0;
2234                 break;
2235         case 2: /* for testing stuck tod bit */
2236                 tod |= 1 << tod_test_injector;
2237                 tod_unit_test = 0;
2238                 break;
2239         case 3: /* for testing stalled tod */
2240                 tod = prev_tod;
2241                 tod_unit_test = 0;
2242                 break;
2243         case 4: /* reset tod fault status */
2244                 (void) tod_fault(TOD_NOFAULT, 0);
2245                 tod_unit_test = 0;
2246                 break;
2247         default:
2248                 break;
2249         }
2250 
2251         diff_tod = tod - prev_tod;
2252         diff_tick = tick - prev_tick;
2253 
2254         ASSERT(diff_tick >= 0);
2255 
2256         if (diff_tod < 0) {
2257                 /* ERROR - tod reversed */
2258                 tod_bad = TOD_REVERSED;
2259                 off = (int)(prev_tod - tod);
2260         } else if (diff_tod == 0) {
2261                 /* tod did not advance */
2262                 if (diff_tick > TOD_STALL_THRESHOLD) {
2263                         /* ERROR - tod stalled */
2264                         tod_bad = TOD_STALLED;
2265                 } else {
2266                         /*
2267                          * Make sure we don't update prev_tick
2268                          * so that diff_tick is calculated since
2269                          * the first diff_tod == 0
2270                          */
2271                         return (tod);
2272                 }
2273         } else {
2274                 /* calculate dtick */
2275                 dtick = diff_tick / diff_tod;
2276 
2277                 /* update dtick averages */
2278                 dtick_avg += ((dtick - dtick_avg) / TOD_FILTER_N);
2279 
2280                 /*
2281                  * Calculate dtick_delta as
2282                  * variation from reference freq in quartiles
2283                  */
2284                 dtick_delta = (dtick_avg - TOD_REF_FREQ) /
2285                     (TOD_REF_FREQ >> 2);
2286 
2287                 /*
2288                  * Even with a perfectly functioning TOD device,
2289                  * when the number of elapsed seconds is low the
2290                  * algorithm can calculate a rate that is beyond
2291                  * tolerance, causing an error.  The algorithm is
2292                  * inaccurate when elapsed time is low (less than
2293                  * 5 seconds).
2294                  */
2295                 if (diff_tod > 4) {
2296                         if (dtick < TOD_JUMP_THRESHOLD) {
2297                                 /*
2298                                  * If we've just done a CPR resume, we detect
2299                                  * a jump in the TOD but, actually, what's
2300                                  * happened is that the TOD has been increasing
2301                                  * whilst the system was suspended and the tick
2302                                  * count hasn't kept up.  We consider the first
2303                                  * occurrence of this after a resume as normal
2304                                  * and ignore it; otherwise, in a non-resume
2305                                  * case, we regard it as a TOD problem.
2306                                  */
2307                                 if (!cpr_resume_done) {
2308                                         /* ERROR - tod jumped */
2309                                         tod_bad = TOD_JUMPED;
2310                                         off = (int)diff_tod;
2311                                 }
2312                         }
2313                         if (dtick_delta) {
2314                                 /*
2315                                  * If we've just done a DR resume, dtick_avg
2316                                  * can go a bit askew so we reset it and carry
2317                                  * on; otherwise, the TOD is in error.
2318                                  */
2319                                 if (dr_resume_done) {
2320                                         dtick_avg = TOD_REF_FREQ;
2321                                 } else {
2322                                         /* ERROR - change in clock rate */
2323                                         tod_bad = TOD_RATECHANGED;
2324                                 }
2325                         }
2326                 }
2327         }
2328 
2329         if (tod_bad != TOD_NOFAULT) {
2330                 (void) tod_fault(tod_bad, off);
2331 
2332                 /*
2333                  * Disable dosynctodr since we are going to fault
2334                  * the TOD chip anyway here
2335                  */
2336                 dosynctodr = 0;
2337 
2338                 /*
2339                  * Set tod to the correct value from hrestime
2340                  */
2341                 tod = hrestime.tv_sec;
2342         }
2343 
2344         prev_tod = tod;
2345         prev_tick = tick;
2346         return (tod);
2347 }
2348 
2349 static void
2350 calcloadavg(int nrun, uint64_t *hp_ave)
2351 {
2352         static int64_t f[3] = { 135, 27, 9 };
2353         uint_t i;
2354         int64_t q, r;
2355 
2356         /*
2357          * Compute load average over the last 1, 5, and 15 minutes
2358          * (60, 300, and 900 seconds).  The constants in f[3] are for
2359          * exponential decay:
2360          * (1 - exp(-1/60)) << 13 = 135,
2361          * (1 - exp(-1/300)) << 13 = 27,
2362          * (1 - exp(-1/900)) << 13 = 9.
2363          */
2364 
2365         /*
2366          * a little hoop-jumping to avoid integer overflow
2367          */
2368         for (i = 0; i < 3; i++) {
2369                 q = (hp_ave[i]  >> 16) << 7;
2370                 r = (hp_ave[i]  & 0xffff) << 7;
2371                 hp_ave[i] += ((nrun - q) * f[i] - ((r * f[i]) >> 16)) >> 4;
2372         }
2373 }
2374 
2375 /*
2376  * lbolt_hybrid() is used by ddi_get_lbolt() and ddi_get_lbolt64() to
2377  * calculate the value of lbolt according to the current mode. In the event
2378  * driven mode (the default), lbolt is calculated by dividing the current hires
2379  * time by the number of nanoseconds per clock tick. In the cyclic driven mode
2380  * an internal variable is incremented at each firing of the lbolt cyclic
2381  * and returned by lbolt_cyclic_driven().
2382  *
2383  * The system will transition from event to cyclic driven mode when the number
2384  * of calls to lbolt_event_driven() exceeds the (per CPU) threshold within a
2385  * window of time. It does so by reprograming lbolt_cyclic from CY_INFINITY to
2386  * nsec_per_tick. The lbolt cyclic will remain ON while at least one CPU is
2387  * causing enough activity to cross the thresholds.
2388  */
2389 int64_t
2390 lbolt_bootstrap(void)
2391 {
2392         return (0);
2393 }
2394 
2395 /* ARGSUSED */
2396 uint_t
2397 lbolt_ev_to_cyclic(caddr_t arg1, caddr_t arg2)
2398 {
2399         hrtime_t ts, exp;
2400         int ret;
2401 
2402         ASSERT(lbolt_hybrid != lbolt_cyclic_driven);
2403 
2404         kpreempt_disable();
2405 
2406         ts = gethrtime();
2407         lb_info->lbi_internal = (ts/nsec_per_tick);
2408 
2409         /*
2410          * Align the next expiration to a clock tick boundary.
2411          */
2412         exp = ts + nsec_per_tick - 1;
2413         exp = (exp/nsec_per_tick) * nsec_per_tick;
2414 
2415         ret = cyclic_reprogram(lb_info->id.lbi_cyclic_id, exp);
2416         ASSERT(ret);
2417 
2418         lbolt_hybrid = lbolt_cyclic_driven;
2419         lb_info->lbi_cyc_deactivate = B_FALSE;
2420         lb_info->lbi_cyc_deac_start = lb_info->lbi_internal;
2421 
2422         kpreempt_enable();
2423 
2424         ret = atomic_dec_32_nv(&lb_info->lbi_token);
2425         ASSERT(ret == 0);
2426 
2427         return (1);
2428 }
2429 
2430 int64_t
2431 lbolt_event_driven(void)
2432 {
2433         hrtime_t ts;
2434         int64_t lb;
2435         int ret, cpu = CPU->cpu_seqid;
2436 
2437         ts = gethrtime();
2438         ASSERT(ts > 0);
2439 
2440         ASSERT(nsec_per_tick > 0);
2441         lb = (ts/nsec_per_tick);
2442 
2443         /*
2444          * Switch to cyclic mode if the number of calls to this routine
2445          * has reached the threshold within the interval.
2446          */
2447         if ((lb - lb_cpu[cpu].lbc_cnt_start) < lb_info->lbi_thresh_interval) {
2448 
2449                 if (--lb_cpu[cpu].lbc_counter == 0) {
2450                         /*
2451                          * Reached the threshold within the interval, reset
2452                          * the usage statistics.
2453                          */
2454                         lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2455                         lb_cpu[cpu].lbc_cnt_start = lb;
2456 
2457                         /*
2458                          * Make sure only one thread reprograms the
2459                          * lbolt cyclic and changes the mode.
2460                          */
2461                         if (panicstr == NULL &&
2462                             atomic_cas_32(&lb_info->lbi_token, 0, 1) == 0) {
2463 
2464                                 if (lbolt_hybrid == lbolt_cyclic_driven) {
2465                                         ret = atomic_dec_32_nv(
2466                                             &lb_info->lbi_token);
2467                                         ASSERT(ret == 0);
2468                                 } else {
2469                                         lbolt_softint_post();
2470                                 }
2471                         }
2472                 }
2473         } else {
2474                 /*
2475                  * Exceeded the interval, reset the usage statistics.
2476                  */
2477                 lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2478                 lb_cpu[cpu].lbc_cnt_start = lb;
2479         }
2480 
2481         ASSERT(lb >= lb_info->lbi_debug_time);
2482 
2483         return (lb - lb_info->lbi_debug_time);
2484 }
2485 
2486 int64_t
2487 lbolt_cyclic_driven(void)
2488 {
2489         int64_t lb = lb_info->lbi_internal;
2490         int cpu;
2491 
2492         /*
2493          * If a CPU has already prevented the lbolt cyclic from deactivating
2494          * itself, don't bother tracking the usage. Otherwise check if we're
2495          * within the interval and how the per CPU counter is doing.
2496          */
2497         if (lb_info->lbi_cyc_deactivate) {
2498                 cpu = CPU->cpu_seqid;
2499                 if ((lb - lb_cpu[cpu].lbc_cnt_start) <
2500                     lb_info->lbi_thresh_interval) {
2501 
2502                         if (lb_cpu[cpu].lbc_counter == 0)
2503                                 /*
2504                                  * Reached the threshold within the interval,
2505                                  * prevent the lbolt cyclic from turning itself
2506                                  * off.
2507                                  */
2508                                 lb_info->lbi_cyc_deactivate = B_FALSE;
2509                         else
2510                                 lb_cpu[cpu].lbc_counter--;
2511                 } else {
2512                         /*
2513                          * Only reset the usage statistics when we have
2514                          * exceeded the interval.
2515                          */
2516                         lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2517                         lb_cpu[cpu].lbc_cnt_start = lb;
2518                 }
2519         }
2520 
2521         ASSERT(lb >= lb_info->lbi_debug_time);
2522 
2523         return (lb - lb_info->lbi_debug_time);
2524 }
2525 
2526 /*
2527  * The lbolt_cyclic() routine will fire at a nsec_per_tick interval to satisfy
2528  * performance needs of ddi_get_lbolt() and ddi_get_lbolt64() consumers.
2529  * It is inactive by default, and will be activated when switching from event
2530  * to cyclic driven lbolt. The cyclic will turn itself off unless signaled
2531  * by lbolt_cyclic_driven().
2532  */
2533 static void
2534 lbolt_cyclic(void)
2535 {
2536         int ret;
2537 
2538         lb_info->lbi_internal++;
2539 
2540         if (!lbolt_cyc_only) {
2541 
2542                 if (lb_info->lbi_cyc_deactivate) {
2543                         /*
2544                          * Switching from cyclic to event driven mode.
2545                          */
2546                         if (panicstr == NULL &&
2547                             atomic_cas_32(&lb_info->lbi_token, 0, 1) == 0) {
2548 
2549                                 if (lbolt_hybrid == lbolt_event_driven) {
2550                                         ret = atomic_dec_32_nv(
2551                                             &lb_info->lbi_token);
2552                                         ASSERT(ret == 0);
2553                                         return;
2554                                 }
2555 
2556                                 kpreempt_disable();
2557 
2558                                 lbolt_hybrid = lbolt_event_driven;
2559                                 ret = cyclic_reprogram(
2560                                     lb_info->id.lbi_cyclic_id,
2561                                     CY_INFINITY);
2562                                 ASSERT(ret);
2563 
2564                                 kpreempt_enable();
2565 
2566                                 ret = atomic_dec_32_nv(&lb_info->lbi_token);
2567                                 ASSERT(ret == 0);
2568                         }
2569                 }
2570 
2571                 /*
2572                  * The lbolt cyclic should not try to deactivate itself before
2573                  * the sampling period has elapsed.
2574                  */
2575                 if (lb_info->lbi_internal - lb_info->lbi_cyc_deac_start >=
2576                     lb_info->lbi_thresh_interval) {
2577                         lb_info->lbi_cyc_deactivate = B_TRUE;
2578                         lb_info->lbi_cyc_deac_start = lb_info->lbi_internal;
2579                 }
2580         }
2581 }
2582 
2583 /*
2584  * Since the lbolt service was historically cyclic driven, it must be 'stopped'
2585  * when the system drops into the kernel debugger. lbolt_debug_entry() is
2586  * called by the KDI system claim callbacks to record a hires timestamp at
2587  * debug enter time. lbolt_debug_return() is called by the sistem release
2588  * callbacks to account for the time spent in the debugger. The value is then
2589  * accumulated in the lb_info structure and used by lbolt_event_driven() and
2590  * lbolt_cyclic_driven(), as well as the mdb_get_lbolt() routine.
2591  */
2592 void
2593 lbolt_debug_entry(void)
2594 {
2595         if (lbolt_hybrid != lbolt_bootstrap) {
2596                 ASSERT(lb_info != NULL);
2597                 lb_info->lbi_debug_ts = gethrtime();
2598         }
2599 }
2600 
2601 /*
2602  * Calculate the time spent in the debugger and add it to the lbolt info
2603  * structure. We also update the internal lbolt value in case we were in
2604  * cyclic driven mode going in.
2605  */
2606 void
2607 lbolt_debug_return(void)
2608 {
2609         hrtime_t ts;
2610 
2611         if (lbolt_hybrid != lbolt_bootstrap) {
2612                 ASSERT(lb_info != NULL);
2613                 ASSERT(nsec_per_tick > 0);
2614 
2615                 ts = gethrtime();
2616                 lb_info->lbi_internal = (ts/nsec_per_tick);
2617                 lb_info->lbi_debug_time +=
2618                     ((ts - lb_info->lbi_debug_ts)/nsec_per_tick);
2619 
2620                 lb_info->lbi_debug_ts = 0;
2621         }
2622 }