1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 /*
  26  * Copyright (c) 2010, Intel Corporation.
  27  * All rights reserved.
  28  */
  29 
  30 #include <sys/types.h>
  31 #include <sys/t_lock.h>
  32 #include <sys/param.h>
  33 #include <sys/segments.h>
  34 #include <sys/sysmacros.h>
  35 #include <sys/signal.h>
  36 #include <sys/systm.h>
  37 #include <sys/user.h>
  38 #include <sys/mman.h>
  39 #include <sys/vm.h>
  40 
  41 #include <sys/disp.h>
  42 #include <sys/class.h>
  43 
  44 #include <sys/proc.h>
  45 #include <sys/buf.h>
  46 #include <sys/kmem.h>
  47 
  48 #include <sys/reboot.h>
  49 #include <sys/uadmin.h>
  50 #include <sys/callb.h>
  51 
  52 #include <sys/cred.h>
  53 #include <sys/vnode.h>
  54 #include <sys/file.h>
  55 
  56 #include <sys/procfs.h>
  57 #include <sys/acct.h>
  58 
  59 #include <sys/vfs.h>
  60 #include <sys/dnlc.h>
  61 #include <sys/var.h>
  62 #include <sys/cmn_err.h>
  63 #include <sys/utsname.h>
  64 #include <sys/debug.h>
  65 
  66 #include <sys/dumphdr.h>
  67 #include <sys/bootconf.h>
  68 #include <sys/varargs.h>
  69 #include <sys/promif.h>
  70 #include <sys/modctl.h>
  71 
  72 #include <sys/consdev.h>
  73 #include <sys/frame.h>
  74 
  75 #include <sys/sunddi.h>
  76 #include <sys/ddidmareq.h>
  77 #include <sys/psw.h>
  78 #include <sys/regset.h>
  79 #include <sys/privregs.h>
  80 #include <sys/clock.h>
  81 #include <sys/tss.h>
  82 #include <sys/cpu.h>
  83 #include <sys/stack.h>
  84 #include <sys/trap.h>
  85 #include <sys/pic.h>
  86 #include <vm/hat.h>
  87 #include <vm/anon.h>
  88 #include <vm/as.h>
  89 #include <vm/page.h>
  90 #include <vm/seg.h>
  91 #include <vm/seg_kmem.h>
  92 #include <vm/seg_map.h>
  93 #include <vm/seg_vn.h>
  94 #include <vm/seg_kp.h>
  95 #include <vm/hat_i86.h>
  96 #include <sys/swap.h>
  97 #include <sys/thread.h>
  98 #include <sys/sysconf.h>
  99 #include <sys/vm_machparam.h>
 100 #include <sys/archsystm.h>
 101 #include <sys/machsystm.h>
 102 #include <sys/machlock.h>
 103 #include <sys/x_call.h>
 104 #include <sys/instance.h>
 105 
 106 #include <sys/time.h>
 107 #include <sys/smp_impldefs.h>
 108 #include <sys/psm_types.h>
 109 #include <sys/atomic.h>
 110 #include <sys/panic.h>
 111 #include <sys/cpuvar.h>
 112 #include <sys/dtrace.h>
 113 #include <sys/bl.h>
 114 #include <sys/nvpair.h>
 115 #include <sys/x86_archext.h>
 116 #include <sys/pool_pset.h>
 117 #include <sys/autoconf.h>
 118 #include <sys/mem.h>
 119 #include <sys/dumphdr.h>
 120 #include <sys/compress.h>
 121 #include <sys/cpu_module.h>
 122 #if defined(__xpv)
 123 #include <sys/hypervisor.h>
 124 #include <sys/xpv_panic.h>
 125 #endif
 126 
 127 #include <sys/fastboot.h>
 128 #include <sys/machelf.h>
 129 #include <sys/kobj.h>
 130 #include <sys/multiboot.h>
 131 
 132 #ifdef  TRAPTRACE
 133 #include <sys/traptrace.h>
 134 #endif  /* TRAPTRACE */
 135 
 136 #include <c2/audit.h>
 137 #include <sys/clock_impl.h>
 138 
 139 extern void audit_enterprom(int);
 140 extern void audit_exitprom(int);
 141 
 142 /*
 143  * Tunable to enable apix PSM; if set to 0, pcplusmp PSM will be used.
 144  */
 145 int     apix_enable = 1;
 146 
 147 int     apic_nvidia_io_max = 0; /* no. of NVIDIA i/o apics */
 148 
 149 /*
 150  * Occassionally the kernel knows better whether to power-off or reboot.
 151  */
 152 int force_shutdown_method = AD_UNKNOWN;
 153 
 154 /*
 155  * The panicbuf array is used to record messages and state:
 156  */
 157 char panicbuf[PANICBUFSIZE];
 158 
 159 /*
 160  * Flags to control Dynamic Reconfiguration features.
 161  */
 162 uint64_t plat_dr_options;
 163 
 164 /*
 165  * Maximum physical address for memory DR operations.
 166  */
 167 uint64_t plat_dr_physmax;
 168 
 169 /*
 170  * maxphys - used during physio
 171  * klustsize - used for klustering by swapfs and specfs
 172  */
 173 int maxphys = 56 * 1024;    /* XXX See vm_subr.c - max b_count in physio */
 174 int klustsize = 56 * 1024;
 175 
 176 caddr_t p0_va;          /* Virtual address for accessing physical page 0 */
 177 
 178 /*
 179  * defined here, though unused on x86,
 180  * to make kstat_fr.c happy.
 181  */
 182 int vac;
 183 
 184 void debug_enter(char *);
 185 
 186 extern void pm_cfb_check_and_powerup(void);
 187 extern void pm_cfb_rele(void);
 188 
 189 extern fastboot_info_t newkernel;
 190 
 191 /*
 192  * Machine dependent code to reboot.
 193  * "mdep" is interpreted as a character pointer; if non-null, it is a pointer
 194  * to a string to be used as the argument string when rebooting.
 195  *
 196  * "invoke_cb" is a boolean. It is set to true when mdboot() can safely
 197  * invoke CB_CL_MDBOOT callbacks before shutting the system down, i.e. when
 198  * we are in a normal shutdown sequence (interrupts are not blocked, the
 199  * system is not panic'ing or being suspended).
 200  */
 201 /*ARGSUSED*/
 202 void
 203 mdboot(int cmd, int fcn, char *mdep, boolean_t invoke_cb)
 204 {
 205         processorid_t bootcpuid = 0;
 206         static int is_first_quiesce = 1;
 207         static int is_first_reset = 1;
 208         int reset_status = 0;
 209         static char fallback_str[] = "Falling back to regular reboot.\n";
 210 
 211         if (fcn == AD_FASTREBOOT && !newkernel.fi_valid)
 212                 fcn = AD_BOOT;
 213 
 214         if (!panicstr) {
 215                 kpreempt_disable();
 216                 if (fcn == AD_FASTREBOOT) {
 217                         mutex_enter(&cpu_lock);
 218                         if (CPU_ACTIVE(cpu_get(bootcpuid))) {
 219                                 affinity_set(bootcpuid);
 220                         }
 221                         mutex_exit(&cpu_lock);
 222                 } else {
 223                         affinity_set(CPU_CURRENT);
 224                 }
 225         }
 226 
 227         if (force_shutdown_method != AD_UNKNOWN)
 228                 fcn = force_shutdown_method;
 229 
 230         /*
 231          * XXX - rconsvp is set to NULL to ensure that output messages
 232          * are sent to the underlying "hardware" device using the
 233          * monitor's printf routine since we are in the process of
 234          * either rebooting or halting the machine.
 235          */
 236         rconsvp = NULL;
 237 
 238         /*
 239          * Print the reboot message now, before pausing other cpus.
 240          * There is a race condition in the printing support that
 241          * can deadlock multiprocessor machines.
 242          */
 243         if (!(fcn == AD_HALT || fcn == AD_POWEROFF))
 244                 prom_printf("rebooting...\n");
 245 
 246         if (IN_XPV_PANIC())
 247                 reset();
 248 
 249         /*
 250          * We can't bring up the console from above lock level, so do it now
 251          */
 252         pm_cfb_check_and_powerup();
 253 
 254         /* make sure there are no more changes to the device tree */
 255         devtree_freeze();
 256 
 257         if (invoke_cb)
 258                 (void) callb_execute_class(CB_CL_MDBOOT, NULL);
 259 
 260         /*
 261          * Clear any unresolved UEs from memory.
 262          */
 263         page_retire_mdboot();
 264 
 265 #if defined(__xpv)
 266         /*
 267          * XXPV Should probably think some more about how we deal
 268          *      with panicing before it's really safe to panic.
 269          *      On hypervisors, we reboot very quickly..  Perhaps panic
 270          *      should only attempt to recover by rebooting if,
 271          *      say, we were able to mount the root filesystem,
 272          *      or if we successfully launched init(1m).
 273          */
 274         if (panicstr && proc_init == NULL)
 275                 (void) HYPERVISOR_shutdown(SHUTDOWN_poweroff);
 276 #endif
 277         /*
 278          * stop other cpus and raise our priority.  since there is only
 279          * one active cpu after this, and our priority will be too high
 280          * for us to be preempted, we're essentially single threaded
 281          * from here on out.
 282          */
 283         (void) spl6();
 284         if (!panicstr) {
 285                 mutex_enter(&cpu_lock);
 286                 pause_cpus(NULL);
 287                 mutex_exit(&cpu_lock);
 288         }
 289 
 290         /*
 291          * If the system is panicking, the preloaded kernel is valid, and
 292          * fastreboot_onpanic has been set, and the system has been up for
 293          * longer than fastreboot_onpanic_uptime (default to 10 minutes),
 294          * choose Fast Reboot.
 295          */
 296         if (fcn == AD_BOOT && panicstr && newkernel.fi_valid &&
 297             fastreboot_onpanic &&
 298             (panic_lbolt - lbolt_at_boot) > fastreboot_onpanic_uptime) {
 299                 fcn = AD_FASTREBOOT;
 300         }
 301 
 302         /*
 303          * Try to quiesce devices.
 304          */
 305         if (is_first_quiesce) {
 306                 /*
 307                  * Clear is_first_quiesce before calling quiesce_devices()
 308                  * so that if quiesce_devices() causes panics, it will not
 309                  * be invoked again.
 310                  */
 311                 is_first_quiesce = 0;
 312 
 313                 quiesce_active = 1;
 314                 quiesce_devices(ddi_root_node(), &reset_status);
 315                 if (reset_status == -1) {
 316                         if (fcn == AD_FASTREBOOT && !force_fastreboot) {
 317                                 prom_printf("Driver(s) not capable of fast "
 318                                     "reboot.\n");
 319                                 prom_printf(fallback_str);
 320                                 fastreboot_capable = 0;
 321                                 fcn = AD_BOOT;
 322                         } else if (fcn != AD_FASTREBOOT)
 323                                 fastreboot_capable = 0;
 324                 }
 325                 quiesce_active = 0;
 326         }
 327 
 328         /*
 329          * Try to reset devices. reset_leaves() should only be called
 330          * a) when there are no other threads that could be accessing devices,
 331          *    and
 332          * b) on a system that's not capable of fast reboot (fastreboot_capable
 333          *    being 0), or on a system where quiesce_devices() failed to
 334          *    complete (quiesce_active being 1).
 335          */
 336         if (is_first_reset && (!fastreboot_capable || quiesce_active)) {
 337                 /*
 338                  * Clear is_first_reset before calling reset_devices()
 339                  * so that if reset_devices() causes panics, it will not
 340                  * be invoked again.
 341                  */
 342                 is_first_reset = 0;
 343                 reset_leaves();
 344         }
 345 
 346         /* Verify newkernel checksum */
 347         if (fastreboot_capable && fcn == AD_FASTREBOOT &&
 348             fastboot_cksum_verify(&newkernel) != 0) {
 349                 fastreboot_capable = 0;
 350                 prom_printf("Fast reboot: checksum failed for the new "
 351                     "kernel.\n");
 352                 prom_printf(fallback_str);
 353         }
 354 
 355         (void) spl8();
 356 
 357         if (fastreboot_capable && fcn == AD_FASTREBOOT) {
 358                 /*
 359                  * psm_shutdown is called within fast_reboot()
 360                  */
 361                 fast_reboot();
 362         } else {
 363                 (*psm_shutdownf)(cmd, fcn);
 364 
 365                 if (fcn == AD_HALT || fcn == AD_POWEROFF)
 366                         halt((char *)NULL);
 367                 else
 368                         prom_reboot("");
 369         }
 370         /*NOTREACHED*/
 371 }
 372 
 373 /* mdpreboot - may be called prior to mdboot while root fs still mounted */
 374 /*ARGSUSED*/
 375 void
 376 mdpreboot(int cmd, int fcn, char *mdep)
 377 {
 378         if (fcn == AD_FASTREBOOT && !fastreboot_capable) {
 379                 fcn = AD_BOOT;
 380 #ifdef  __xpv
 381                 cmn_err(CE_WARN, "Fast reboot is not supported on xVM");
 382 #else
 383                 cmn_err(CE_WARN,
 384                     "Fast reboot is not supported on this platform%s",
 385                     fastreboot_nosup_message());
 386 #endif
 387         }
 388 
 389         if (fcn == AD_FASTREBOOT) {
 390                 fastboot_load_kernel(mdep);
 391                 if (!newkernel.fi_valid)
 392                         fcn = AD_BOOT;
 393         }
 394 
 395         (*psm_preshutdownf)(cmd, fcn);
 396 }
 397 
 398 static void
 399 stop_other_cpus(void)
 400 {
 401         ulong_t s = clear_int_flag(); /* fast way to keep CPU from changing */
 402         cpuset_t xcset;
 403 
 404         CPUSET_ALL_BUT(xcset, CPU->cpu_id);
 405         xc_priority(0, 0, 0, CPUSET2BV(xcset), (xc_func_t)mach_cpu_halt);
 406         restore_int_flag(s);
 407 }
 408 
 409 /*
 410  *      Machine dependent abort sequence handling
 411  */
 412 void
 413 abort_sequence_enter(char *msg)
 414 {
 415         if (abort_enable == 0) {
 416                 if (AU_ZONE_AUDITING(GET_KCTX_GZ))
 417                         audit_enterprom(0);
 418                 return;
 419         }
 420         if (AU_ZONE_AUDITING(GET_KCTX_GZ))
 421                 audit_enterprom(1);
 422         debug_enter(msg);
 423         if (AU_ZONE_AUDITING(GET_KCTX_GZ))
 424                 audit_exitprom(1);
 425 }
 426 
 427 /*
 428  * Enter debugger.  Called when the user types ctrl-alt-d or whenever
 429  * code wants to enter the debugger and possibly resume later.
 430  */
 431 void
 432 debug_enter(
 433         char    *msg)           /* message to print, possibly NULL */
 434 {
 435         if (dtrace_debugger_init != NULL)
 436                 (*dtrace_debugger_init)();
 437 
 438         if (msg)
 439                 prom_printf("%s\n", msg);
 440 
 441         if (boothowto & RB_DEBUG)
 442                 kmdb_enter();
 443 
 444         if (dtrace_debugger_fini != NULL)
 445                 (*dtrace_debugger_fini)();
 446 }
 447 
 448 void
 449 reset(void)
 450 {
 451         extern  void acpi_reset_system();
 452 #if !defined(__xpv)
 453         ushort_t *bios_memchk;
 454 
 455         /*
 456          * Can't use psm_map_phys or acpi_reset_system before the hat is
 457          * initialized.
 458          */
 459         if (khat_running) {
 460                 bios_memchk = (ushort_t *)psm_map_phys(0x472,
 461                     sizeof (ushort_t), PROT_READ | PROT_WRITE);
 462                 if (bios_memchk)
 463                         *bios_memchk = 0x1234;  /* bios memory check disable */
 464 
 465                 if (options_dip != NULL &&
 466                     ddi_prop_exists(DDI_DEV_T_ANY, ddi_root_node(), 0,
 467                     "efi-systab")) {
 468                         efi_reset();
 469                 }
 470 
 471                 /*
 472                  * The problem with using stubs is that we can call
 473                  * acpi_reset_system only after the kernel is up and running.
 474                  *
 475                  * We should create a global state to keep track of how far
 476                  * up the kernel is but for the time being we will depend on
 477                  * bootops. bootops cleared in startup_end().
 478                  */
 479                 if (bootops == NULL)
 480                         acpi_reset_system();
 481         }
 482 
 483         pc_reset();
 484 #else
 485         if (IN_XPV_PANIC()) {
 486                 if (khat_running && bootops == NULL) {
 487                         acpi_reset_system();
 488                 }
 489 
 490                 pc_reset();
 491         }
 492 
 493         (void) HYPERVISOR_shutdown(SHUTDOWN_reboot);
 494         panic("HYPERVISOR_shutdown() failed");
 495 #endif
 496         /*NOTREACHED*/
 497 }
 498 
 499 /*
 500  * Halt the machine and return to the monitor
 501  */
 502 void
 503 halt(char *s)
 504 {
 505         stop_other_cpus();      /* send stop signal to other CPUs */
 506         if (s)
 507                 prom_printf("(%s) \n", s);
 508         prom_exit_to_mon();
 509         /*NOTREACHED*/
 510 }
 511 
 512 /*
 513  * Initiate interrupt redistribution.
 514  */
 515 void
 516 i_ddi_intr_redist_all_cpus()
 517 {
 518 }
 519 
 520 /*
 521  * XXX These probably ought to live somewhere else
 522  * XXX They are called from mem.c
 523  */
 524 
 525 /*
 526  * Convert page frame number to an OBMEM page frame number
 527  * (i.e. put in the type bits -- zero for this implementation)
 528  */
 529 pfn_t
 530 impl_obmem_pfnum(pfn_t pf)
 531 {
 532         return (pf);
 533 }
 534 
 535 #ifdef  NM_DEBUG
 536 int nmi_test = 0;       /* checked in intentry.s during clock int */
 537 int nmtest = -1;
 538 nmfunc1(arg, rp)
 539 int     arg;
 540 struct regs *rp;
 541 {
 542         printf("nmi called with arg = %x, regs = %x\n", arg, rp);
 543         nmtest += 50;
 544         if (arg == nmtest) {
 545                 printf("ip = %x\n", rp->r_pc);
 546                 return (1);
 547         }
 548         return (0);
 549 }
 550 
 551 #endif
 552 
 553 #include <sys/bootsvcs.h>
 554 
 555 /* Hacked up initialization for initial kernel check out is HERE. */
 556 /* The basic steps are: */
 557 /*      kernel bootfuncs definition/initialization for KADB */
 558 /*      kadb bootfuncs pointer initialization */
 559 /*      putchar/getchar (interrupts disabled) */
 560 
 561 /* kadb bootfuncs pointer initialization */
 562 
 563 int
 564 sysp_getchar()
 565 {
 566         int i;
 567         ulong_t s;
 568 
 569         if (cons_polledio == NULL) {
 570                 /* Uh oh */
 571                 prom_printf("getchar called with no console\n");
 572                 for (;;)
 573                         /* LOOP FOREVER */;
 574         }
 575 
 576         s = clear_int_flag();
 577         i = cons_polledio->cons_polledio_getchar(
 578             cons_polledio->cons_polledio_argument);
 579         restore_int_flag(s);
 580         return (i);
 581 }
 582 
 583 void
 584 sysp_putchar(int c)
 585 {
 586         ulong_t s;
 587 
 588         /*
 589          * We have no alternative but to drop the output on the floor.
 590          */
 591         if (cons_polledio == NULL ||
 592             cons_polledio->cons_polledio_putchar == NULL)
 593                 return;
 594 
 595         s = clear_int_flag();
 596         cons_polledio->cons_polledio_putchar(
 597             cons_polledio->cons_polledio_argument, c);
 598         restore_int_flag(s);
 599 }
 600 
 601 int
 602 sysp_ischar()
 603 {
 604         int i;
 605         ulong_t s;
 606 
 607         if (cons_polledio == NULL ||
 608             cons_polledio->cons_polledio_ischar == NULL)
 609                 return (0);
 610 
 611         s = clear_int_flag();
 612         i = cons_polledio->cons_polledio_ischar(
 613             cons_polledio->cons_polledio_argument);
 614         restore_int_flag(s);
 615         return (i);
 616 }
 617 
 618 int
 619 goany(void)
 620 {
 621         prom_printf("Type any key to continue ");
 622         (void) prom_getchar();
 623         prom_printf("\n");
 624         return (1);
 625 }
 626 
 627 static struct boot_syscalls kern_sysp = {
 628         sysp_getchar,   /*      unchar  (*getchar)();   7  */
 629         sysp_putchar,   /*      int     (*putchar)();   8  */
 630         sysp_ischar,    /*      int     (*ischar)();    9  */
 631 };
 632 
 633 #if defined(__xpv)
 634 int using_kern_polledio;
 635 #endif
 636 
 637 void
 638 kadb_uses_kernel()
 639 {
 640         /*
 641          * This routine is now totally misnamed, since it does not in fact
 642          * control kadb's I/O; it only controls the kernel's prom_* I/O.
 643          */
 644         sysp = &kern_sysp;
 645 #if defined(__xpv)
 646         using_kern_polledio = 1;
 647 #endif
 648 }
 649 
 650 /*
 651  *      the interface to the outside world
 652  */
 653 
 654 /*
 655  * poll_port -- wait for a register to achieve a
 656  *              specific state.  Arguments are a mask of bits we care about,
 657  *              and two sub-masks.  To return normally, all the bits in the
 658  *              first sub-mask must be ON, all the bits in the second sub-
 659  *              mask must be OFF.  If about seconds pass without the register
 660  *              achieving the desired bit configuration, we return 1, else
 661  *              0.
 662  */
 663 int
 664 poll_port(ushort_t port, ushort_t mask, ushort_t onbits, ushort_t offbits)
 665 {
 666         int i;
 667         ushort_t maskval;
 668 
 669         for (i = 500000; i; i--) {
 670                 maskval = inb(port) & mask;
 671                 if (((maskval & onbits) == onbits) &&
 672                     ((maskval & offbits) == 0))
 673                         return (0);
 674                 drv_usecwait(10);
 675         }
 676         return (1);
 677 }
 678 
 679 /*
 680  * set_idle_cpu is called from idle() when a CPU becomes idle.
 681  */
 682 /*LINTED: static unused */
 683 static uint_t last_idle_cpu;
 684 
 685 /*ARGSUSED*/
 686 void
 687 set_idle_cpu(int cpun)
 688 {
 689         last_idle_cpu = cpun;
 690         (*psm_set_idle_cpuf)(cpun);
 691 }
 692 
 693 /*
 694  * unset_idle_cpu is called from idle() when a CPU is no longer idle.
 695  */
 696 /*ARGSUSED*/
 697 void
 698 unset_idle_cpu(int cpun)
 699 {
 700         (*psm_unset_idle_cpuf)(cpun);
 701 }
 702 
 703 /*
 704  * This routine is almost correct now, but not quite.  It still needs the
 705  * equivalent concept of "hres_last_tick", just like on the sparc side.
 706  * The idea is to take a snapshot of the hi-res timer while doing the
 707  * hrestime_adj updates under hres_lock in locore, so that the small
 708  * interval between interrupt assertion and interrupt processing is
 709  * accounted for correctly.  Once we have this, the code below should
 710  * be modified to subtract off hres_last_tick rather than hrtime_base.
 711  *
 712  * I'd have done this myself, but I don't have source to all of the
 713  * vendor-specific hi-res timer routines (grrr...).  The generic hook I
 714  * need is something like "gethrtime_unlocked()", which would be just like
 715  * gethrtime() but would assume that you're already holding CLOCK_LOCK().
 716  * This is what the GET_HRTIME() macro is for on sparc (although it also
 717  * serves the function of making time available without a function call
 718  * so you don't take a register window overflow while traps are disabled).
 719  */
 720 void
 721 pc_gethrestime(timestruc_t *tp)
 722 {
 723         int lock_prev;
 724         timestruc_t now;
 725         int nslt;               /* nsec since last tick */
 726         int adj;                /* amount of adjustment to apply */
 727 
 728 loop:
 729         lock_prev = hres_lock;
 730         now = hrestime;
 731         nslt = (int)(gethrtime() - hres_last_tick);
 732         if (nslt < 0) {
 733                 /*
 734                  * nslt < 0 means a tick came between sampling
 735                  * gethrtime() and hres_last_tick; restart the loop
 736                  */
 737 
 738                 goto loop;
 739         }
 740         now.tv_nsec += nslt;
 741         if (hrestime_adj != 0) {
 742                 if (hrestime_adj > 0) {
 743                         adj = (nslt >> ADJ_SHIFT);
 744                         if (adj > hrestime_adj)
 745                                 adj = (int)hrestime_adj;
 746                 } else {
 747                         adj = -(nslt >> ADJ_SHIFT);
 748                         if (adj < hrestime_adj)
 749                                 adj = (int)hrestime_adj;
 750                 }
 751                 now.tv_nsec += adj;
 752         }
 753         while ((unsigned long)now.tv_nsec >= NANOSEC) {
 754 
 755                 /*
 756                  * We might have a large adjustment or have been in the
 757                  * debugger for a long time; take care of (at most) four
 758                  * of those missed seconds (tv_nsec is 32 bits, so
 759                  * anything >4s will be wrapping around).  However,
 760                  * anything more than 2 seconds out of sync will trigger
 761                  * timedelta from clock() to go correct the time anyway,
 762                  * so do what we can, and let the big crowbar do the
 763                  * rest.  A similar correction while loop exists inside
 764                  * hres_tick(); in all cases we'd like tv_nsec to
 765                  * satisfy 0 <= tv_nsec < NANOSEC to avoid confusing
 766                  * user processes, but if tv_sec's a little behind for a
 767                  * little while, that's OK; time still monotonically
 768                  * increases.
 769                  */
 770 
 771                 now.tv_nsec -= NANOSEC;
 772                 now.tv_sec++;
 773         }
 774         if ((hres_lock & ~1) != lock_prev)
 775                 goto loop;
 776 
 777         *tp = now;
 778 }
 779 
 780 void
 781 gethrestime_lasttick(timespec_t *tp)
 782 {
 783         int s;
 784 
 785         s = hr_clock_lock();
 786         *tp = hrestime;
 787         hr_clock_unlock(s);
 788 }
 789 
 790 time_t
 791 gethrestime_sec(void)
 792 {
 793         timestruc_t now;
 794 
 795         gethrestime(&now);
 796         return (now.tv_sec);
 797 }
 798 
 799 /*
 800  * Initialize a kernel thread's stack
 801  */
 802 
 803 caddr_t
 804 thread_stk_init(caddr_t stk)
 805 {
 806         ASSERT(((uintptr_t)stk & (STACK_ALIGN - 1)) == 0);
 807         return (stk - SA(MINFRAME));
 808 }
 809 
 810 /*
 811  * Initialize lwp's kernel stack.
 812  */
 813 
 814 #ifdef TRAPTRACE
 815 /*
 816  * There's a tricky interdependency here between use of sysenter and
 817  * TRAPTRACE which needs recording to avoid future confusion (this is
 818  * about the third time I've re-figured this out ..)
 819  *
 820  * Here's how debugging lcall works with TRAPTRACE.
 821  *
 822  * 1 We're in userland with a breakpoint on the lcall instruction.
 823  * 2 We execute the instruction - the instruction pushes the userland
 824  *   %ss, %esp, %efl, %cs, %eip on the stack and zips into the kernel
 825  *   via the call gate.
 826  * 3 The hardware raises a debug trap in kernel mode, the hardware
 827  *   pushes %efl, %cs, %eip and gets to dbgtrap via the idt.
 828  * 4 dbgtrap pushes the error code and trapno and calls cmntrap
 829  * 5 cmntrap finishes building a trap frame
 830  * 6 The TRACE_REGS macros in cmntrap copy a REGSIZE worth chunk
 831  *   off the stack into the traptrace buffer.
 832  *
 833  * This means that the traptrace buffer contains the wrong values in
 834  * %esp and %ss, but everything else in there is correct.
 835  *
 836  * Here's how debugging sysenter works with TRAPTRACE.
 837  *
 838  * a We're in userland with a breakpoint on the sysenter instruction.
 839  * b We execute the instruction - the instruction pushes -nothing-
 840  *   on the stack, but sets %cs, %eip, %ss, %esp to prearranged
 841  *   values to take us to sys_sysenter, at the top of the lwp's
 842  *   stack.
 843  * c goto 3
 844  *
 845  * At this point, because we got into the kernel without the requisite
 846  * five pushes on the stack, if we didn't make extra room, we'd
 847  * end up with the TRACE_REGS macro fetching the saved %ss and %esp
 848  * values from negative (unmapped) stack addresses -- which really bites.
 849  * That's why we do the '-= 8' below.
 850  *
 851  * XXX  Note that reading "up" lwp0's stack works because t0 is declared
 852  *      right next to t0stack in locore.s
 853  */
 854 #endif
 855 
 856 caddr_t
 857 lwp_stk_init(klwp_t *lwp, caddr_t stk)
 858 {
 859         caddr_t oldstk;
 860         struct pcb *pcb = &lwp->lwp_pcb;
 861 
 862         oldstk = stk;
 863         stk -= SA(sizeof (struct regs) + SA(MINFRAME));
 864 #ifdef TRAPTRACE
 865         stk -= 2 * sizeof (greg_t); /* space for phony %ss:%sp (see above) */
 866 #endif
 867         stk = (caddr_t)((uintptr_t)stk & ~(STACK_ALIGN - 1ul));
 868         bzero(stk, oldstk - stk);
 869         lwp->lwp_regs = (void *)(stk + SA(MINFRAME));
 870 
 871         /*
 872          * Arrange that the virtualized %fs and %gs GDT descriptors
 873          * have a well-defined initial state (present, ring 3
 874          * and of type data).
 875          */
 876 #if defined(__amd64)
 877         if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE)
 878                 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
 879         else
 880                 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc;
 881 #elif defined(__i386)
 882         pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
 883 #endif  /* __i386 */
 884         lwp_installctx(lwp);
 885         return (stk);
 886 }
 887 
 888 /*ARGSUSED*/
 889 void
 890 lwp_stk_fini(klwp_t *lwp)
 891 {}
 892 
 893 /*
 894  * If we're not the panic CPU, we wait in panic_idle for reboot.
 895  */
 896 void
 897 panic_idle(void)
 898 {
 899         splx(ipltospl(CLOCK_LEVEL));
 900         (void) setjmp(&curthread->t_pcb);
 901 
 902         dumpsys_helper();
 903 
 904 #ifndef __xpv
 905         for (;;)
 906                 i86_halt();
 907 #else
 908         for (;;)
 909                 ;
 910 #endif
 911 }
 912 
 913 /*
 914  * Stop the other CPUs by cross-calling them and forcing them to enter
 915  * the panic_idle() loop above.
 916  */
 917 /*ARGSUSED*/
 918 void
 919 panic_stopcpus(cpu_t *cp, kthread_t *t, int spl)
 920 {
 921         processorid_t i;
 922         cpuset_t xcset;
 923 
 924         /*
 925          * In the case of a Xen panic, the hypervisor has already stopped
 926          * all of the CPUs.
 927          */
 928         if (!IN_XPV_PANIC()) {
 929                 (void) splzs();
 930 
 931                 CPUSET_ALL_BUT(xcset, cp->cpu_id);
 932                 xc_priority(0, 0, 0, CPUSET2BV(xcset), (xc_func_t)panic_idle);
 933         }
 934 
 935         for (i = 0; i < NCPU; i++) {
 936                 if (i != cp->cpu_id && cpu[i] != NULL &&
 937                     (cpu[i]->cpu_flags & CPU_EXISTS))
 938                         cpu[i]->cpu_flags |= CPU_QUIESCED;
 939         }
 940 }
 941 
 942 /*
 943  * Platform callback following each entry to panicsys().
 944  */
 945 /*ARGSUSED*/
 946 void
 947 panic_enter_hw(int spl)
 948 {
 949         /* Nothing to do here */
 950 }
 951 
 952 /*
 953  * Platform-specific code to execute after panicstr is set: we invoke
 954  * the PSM entry point to indicate that a panic has occurred.
 955  */
 956 /*ARGSUSED*/
 957 void
 958 panic_quiesce_hw(panic_data_t *pdp)
 959 {
 960         psm_notifyf(PSM_PANIC_ENTER);
 961 
 962         cmi_panic_callback();
 963 
 964 #ifdef  TRAPTRACE
 965         /*
 966          * Turn off TRAPTRACE
 967          */
 968         TRAPTRACE_FREEZE;
 969 #endif  /* TRAPTRACE */
 970 }
 971 
 972 /*
 973  * Platform callback prior to writing crash dump.
 974  */
 975 /*ARGSUSED*/
 976 void
 977 panic_dump_hw(int spl)
 978 {
 979         /* Nothing to do here */
 980 }
 981 
 982 void *
 983 plat_traceback(void *fpreg)
 984 {
 985 #ifdef __xpv
 986         if (IN_XPV_PANIC())
 987                 return (xpv_traceback(fpreg));
 988 #endif
 989         return (fpreg);
 990 }
 991 
 992 /*ARGSUSED*/
 993 void
 994 plat_tod_fault(enum tod_fault_type tod_bad)
 995 {}
 996 
 997 /*ARGSUSED*/
 998 int
 999 blacklist(int cmd, const char *scheme, nvlist_t *fmri, const char *class)
1000 {
1001         return (ENOTSUP);
1002 }
1003 
1004 /*
1005  * The underlying console output routines are protected by raising IPL in case
1006  * we are still calling into the early boot services.  Once we start calling
1007  * the kernel console emulator, it will disable interrupts completely during
1008  * character rendering (see sysp_putchar, for example).  Refer to the comments
1009  * and code in common/os/console.c for more information on these callbacks.
1010  */
1011 /*ARGSUSED*/
1012 int
1013 console_enter(int busy)
1014 {
1015         return (splzs());
1016 }
1017 
1018 /*ARGSUSED*/
1019 void
1020 console_exit(int busy, int spl)
1021 {
1022         splx(spl);
1023 }
1024 
1025 /*
1026  * Allocate a region of virtual address space, unmapped.
1027  * Stubbed out except on sparc, at least for now.
1028  */
1029 /*ARGSUSED*/
1030 void *
1031 boot_virt_alloc(void *addr, size_t size)
1032 {
1033         return (addr);
1034 }
1035 
1036 volatile unsigned long  tenmicrodata;
1037 
1038 void
1039 tenmicrosec(void)
1040 {
1041         extern int gethrtime_hires;
1042 
1043         if (gethrtime_hires) {
1044                 hrtime_t start, end;
1045                 start = end =  gethrtime();
1046                 while ((end - start) < (10 * (NANOSEC / MICROSEC))) {
1047                         SMT_PAUSE();
1048                         end = gethrtime();
1049                 }
1050         } else {
1051 #if defined(__xpv)
1052                 hrtime_t newtime;
1053 
1054                 newtime = xpv_gethrtime() + 10000; /* now + 10 us */
1055                 while (xpv_gethrtime() < newtime)
1056                         SMT_PAUSE();
1057 #else   /* __xpv */
1058                 int i;
1059 
1060                 /*
1061                  * Artificial loop to induce delay.
1062                  */
1063                 for (i = 0; i < microdata; i++)
1064                         tenmicrodata = microdata;
1065 #endif  /* __xpv */
1066         }
1067 }
1068 
1069 /*
1070  * get_cpu_mstate() is passed an array of timestamps, NCMSTATES
1071  * long, and it fills in the array with the time spent on cpu in
1072  * each of the mstates, where time is returned in nsec.
1073  *
1074  * No guarantee is made that the returned values in times[] will
1075  * monotonically increase on sequential calls, although this will
1076  * be true in the long run. Any such guarantee must be handled by
1077  * the caller, if needed. This can happen if we fail to account
1078  * for elapsed time due to a generation counter conflict, yet we
1079  * did account for it on a prior call (see below).
1080  *
1081  * The complication is that the cpu in question may be updating
1082  * its microstate at the same time that we are reading it.
1083  * Because the microstate is only updated when the CPU's state
1084  * changes, the values in cpu_intracct[] can be indefinitely out
1085  * of date. To determine true current values, it is necessary to
1086  * compare the current time with cpu_mstate_start, and add the
1087  * difference to times[cpu_mstate].
1088  *
1089  * This can be a problem if those values are changing out from
1090  * under us. Because the code path in new_cpu_mstate() is
1091  * performance critical, we have not added a lock to it. Instead,
1092  * we have added a generation counter. Before beginning
1093  * modifications, the counter is set to 0. After modifications,
1094  * it is set to the old value plus one.
1095  *
1096  * get_cpu_mstate() will not consider the values of cpu_mstate
1097  * and cpu_mstate_start to be usable unless the value of
1098  * cpu_mstate_gen is both non-zero and unchanged, both before and
1099  * after reading the mstate information. Note that we must
1100  * protect against out-of-order loads around accesses to the
1101  * generation counter. Also, this is a best effort approach in
1102  * that we do not retry should the counter be found to have
1103  * changed.
1104  *
1105  * cpu_intracct[] is used to identify time spent in each CPU
1106  * mstate while handling interrupts. Such time should be reported
1107  * against system time, and so is subtracted out from its
1108  * corresponding cpu_acct[] time and added to
1109  * cpu_acct[CMS_SYSTEM].
1110  */
1111 
1112 void
1113 get_cpu_mstate(cpu_t *cpu, hrtime_t *times)
1114 {
1115         int i;
1116         hrtime_t now, start;
1117         uint16_t gen;
1118         uint16_t state;
1119         hrtime_t intracct[NCMSTATES];
1120 
1121         /*
1122          * Load all volatile state under the protection of membar.
1123          * cpu_acct[cpu_mstate] must be loaded to avoid double counting
1124          * of (now - cpu_mstate_start) by a change in CPU mstate that
1125          * arrives after we make our last check of cpu_mstate_gen.
1126          */
1127 
1128         now = gethrtime_unscaled();
1129         gen = cpu->cpu_mstate_gen;
1130 
1131         membar_consumer();      /* guarantee load ordering */
1132         start = cpu->cpu_mstate_start;
1133         state = cpu->cpu_mstate;
1134         for (i = 0; i < NCMSTATES; i++) {
1135                 intracct[i] = cpu->cpu_intracct[i];
1136                 times[i] = cpu->cpu_acct[i];
1137         }
1138         membar_consumer();      /* guarantee load ordering */
1139 
1140         if (gen != 0 && gen == cpu->cpu_mstate_gen && now > start)
1141                 times[state] += now - start;
1142 
1143         for (i = 0; i < NCMSTATES; i++) {
1144                 if (i == CMS_SYSTEM)
1145                         continue;
1146                 times[i] -= intracct[i];
1147                 if (times[i] < 0) {
1148                         intracct[i] += times[i];
1149                         times[i] = 0;
1150                 }
1151                 times[CMS_SYSTEM] += intracct[i];
1152                 scalehrtime(&times[i]);
1153         }
1154         scalehrtime(&times[CMS_SYSTEM]);
1155 }
1156 
1157 /*
1158  * This is a version of the rdmsr instruction that allows
1159  * an error code to be returned in the case of failure.
1160  */
1161 int
1162 checked_rdmsr(uint_t msr, uint64_t *value)
1163 {
1164         if (!is_x86_feature(x86_featureset, X86FSET_MSR))
1165                 return (ENOTSUP);
1166         *value = rdmsr(msr);
1167         return (0);
1168 }
1169 
1170 /*
1171  * This is a version of the wrmsr instruction that allows
1172  * an error code to be returned in the case of failure.
1173  */
1174 int
1175 checked_wrmsr(uint_t msr, uint64_t value)
1176 {
1177         if (!is_x86_feature(x86_featureset, X86FSET_MSR))
1178                 return (ENOTSUP);
1179         wrmsr(msr, value);
1180         return (0);
1181 }
1182 
1183 /*
1184  * The mem driver's usual method of using hat_devload() to establish a
1185  * temporary mapping will not work for foreign pages mapped into this
1186  * domain or for the special hypervisor-provided pages.  For the foreign
1187  * pages, we often don't know which domain owns them, so we can't ask the
1188  * hypervisor to set up a new mapping.  For the other pages, we don't have
1189  * a pfn, so we can't create a new PTE.  For these special cases, we do a
1190  * direct uiomove() from the existing kernel virtual address.
1191  */
1192 /*ARGSUSED*/
1193 int
1194 plat_mem_do_mmio(struct uio *uio, enum uio_rw rw)
1195 {
1196 #if defined(__xpv)
1197         void *va = (void *)(uintptr_t)uio->uio_loffset;
1198         off_t pageoff = uio->uio_loffset & PAGEOFFSET;
1199         size_t nbytes = MIN((size_t)(PAGESIZE - pageoff),
1200             (size_t)uio->uio_iov->iov_len);
1201 
1202         if ((rw == UIO_READ &&
1203             (va == HYPERVISOR_shared_info || va == xen_info)) ||
1204             (pfn_is_foreign(hat_getpfnum(kas.a_hat, va))))
1205                 return (uiomove(va, nbytes, rw, uio));
1206 #endif
1207         return (ENOTSUP);
1208 }
1209 
1210 pgcnt_t
1211 num_phys_pages()
1212 {
1213         pgcnt_t npages = 0;
1214         struct memlist *mp;
1215 
1216 #if defined(__xpv)
1217         if (DOMAIN_IS_INITDOMAIN(xen_info))
1218                 return (xpv_nr_phys_pages());
1219 #endif /* __xpv */
1220 
1221         for (mp = phys_install; mp != NULL; mp = mp->ml_next)
1222                 npages += mp->ml_size >> PAGESHIFT;
1223 
1224         return (npages);
1225 }
1226 
1227 /* cpu threshold for compressed dumps */
1228 #ifdef _LP64
1229 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_64_MINCPU;
1230 #else
1231 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_32_MINCPU;
1232 #endif
1233 
1234 int
1235 dump_plat_addr()
1236 {
1237 #ifdef __xpv
1238         pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN;
1239         mem_vtop_t mem_vtop;
1240         int cnt;
1241 
1242         /*
1243          * On the hypervisor, we want to dump the page with shared_info on it.
1244          */
1245         if (!IN_XPV_PANIC()) {
1246                 mem_vtop.m_as = &kas;
1247                 mem_vtop.m_va = HYPERVISOR_shared_info;
1248                 mem_vtop.m_pfn = pfn;
1249                 dumpvp_write(&mem_vtop, sizeof (mem_vtop_t));
1250                 cnt = 1;
1251         } else {
1252                 cnt = dump_xpv_addr();
1253         }
1254         return (cnt);
1255 #else
1256         return (0);
1257 #endif
1258 }
1259 
1260 void
1261 dump_plat_pfn()
1262 {
1263 #ifdef __xpv
1264         pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN;
1265 
1266         if (!IN_XPV_PANIC())
1267                 dumpvp_write(&pfn, sizeof (pfn));
1268         else
1269                 dump_xpv_pfn();
1270 #endif
1271 }
1272 
1273 /*ARGSUSED*/
1274 int
1275 dump_plat_data(void *dump_cbuf)
1276 {
1277 #ifdef __xpv
1278         uint32_t csize;
1279         int cnt;
1280 
1281         if (!IN_XPV_PANIC()) {
1282                 csize = (uint32_t)compress(HYPERVISOR_shared_info, dump_cbuf,
1283                     PAGESIZE);
1284                 dumpvp_write(&csize, sizeof (uint32_t));
1285                 dumpvp_write(dump_cbuf, csize);
1286                 cnt = 1;
1287         } else {
1288                 cnt = dump_xpv_data(dump_cbuf);
1289         }
1290         return (cnt);
1291 #else
1292         return (0);
1293 #endif
1294 }
1295 
1296 /*
1297  * Calculates a linear address, given the CS selector and PC values
1298  * by looking up the %cs selector process's LDT or the CPU's GDT.
1299  * proc->p_ldtlock must be held across this call.
1300  */
1301 int
1302 linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp)
1303 {
1304         user_desc_t     *descrp;
1305         caddr_t         baseaddr;
1306         uint16_t        idx = SELTOIDX(rp->r_cs);
1307 
1308         ASSERT(rp->r_cs <= 0xFFFF);
1309         ASSERT(MUTEX_HELD(&p->p_ldtlock));
1310 
1311         if (SELISLDT(rp->r_cs)) {
1312                 /*
1313                  * Currently 64 bit processes cannot have private LDTs.
1314                  */
1315                 ASSERT(p->p_model != DATAMODEL_LP64);
1316 
1317                 if (p->p_ldt == NULL)
1318                         return (-1);
1319 
1320                 descrp = &p->p_ldt[idx];
1321                 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1322 
1323                 /*
1324                  * Calculate the linear address (wraparound is not only ok,
1325                  * it's expected behavior).  The cast to uint32_t is because
1326                  * LDT selectors are only allowed in 32-bit processes.
1327                  */
1328                 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr +
1329                     rp->r_pc);
1330         } else {
1331 #ifdef DEBUG
1332                 descrp = &CPU->cpu_gdt[idx];
1333                 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1334                 /* GDT-based descriptors' base addresses should always be 0 */
1335                 ASSERT(baseaddr == 0);
1336 #endif
1337                 *linearp = (caddr_t)(uintptr_t)rp->r_pc;
1338         }
1339 
1340         return (0);
1341 }
1342 
1343 /*
1344  * The implementation of dtrace_linear_pc is similar to the that of
1345  * linear_pc, above, but here we acquire p_ldtlock before accessing
1346  * p_ldt.  This implementation is used by the pid provider; we prefix
1347  * it with "dtrace_" to avoid inducing spurious tracing events.
1348  */
1349 int
1350 dtrace_linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp)
1351 {
1352         user_desc_t     *descrp;
1353         caddr_t         baseaddr;
1354         uint16_t        idx = SELTOIDX(rp->r_cs);
1355 
1356         ASSERT(rp->r_cs <= 0xFFFF);
1357 
1358         if (SELISLDT(rp->r_cs)) {
1359                 /*
1360                  * Currently 64 bit processes cannot have private LDTs.
1361                  */
1362                 ASSERT(p->p_model != DATAMODEL_LP64);
1363 
1364                 mutex_enter(&p->p_ldtlock);
1365                 if (p->p_ldt == NULL) {
1366                         mutex_exit(&p->p_ldtlock);
1367                         return (-1);
1368                 }
1369                 descrp = &p->p_ldt[idx];
1370                 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1371                 mutex_exit(&p->p_ldtlock);
1372 
1373                 /*
1374                  * Calculate the linear address (wraparound is not only ok,
1375                  * it's expected behavior).  The cast to uint32_t is because
1376                  * LDT selectors are only allowed in 32-bit processes.
1377                  */
1378                 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr +
1379                     rp->r_pc);
1380         } else {
1381 #ifdef DEBUG
1382                 descrp = &CPU->cpu_gdt[idx];
1383                 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1384                 /* GDT-based descriptors' base addresses should always be 0 */
1385                 ASSERT(baseaddr == 0);
1386 #endif
1387                 *linearp = (caddr_t)(uintptr_t)rp->r_pc;
1388         }
1389 
1390         return (0);
1391 }
1392 
1393 /*
1394  * We need to post a soft interrupt to reprogram the lbolt cyclic when
1395  * switching from event to cyclic driven lbolt. The following code adds
1396  * and posts the softint for x86.
1397  */
1398 static ddi_softint_hdl_impl_t lbolt_softint_hdl =
1399         {0, NULL, NULL, NULL, 0, NULL, NULL, NULL};
1400 
1401 void
1402 lbolt_softint_add(void)
1403 {
1404         (void) add_avsoftintr((void *)&lbolt_softint_hdl, LOCK_LEVEL,
1405             (avfunc)lbolt_ev_to_cyclic, "lbolt_ev_to_cyclic", NULL, NULL);
1406 }
1407 
1408 void
1409 lbolt_softint_post(void)
1410 {
1411         (*setsoftint)(CBE_LOCK_PIL, lbolt_softint_hdl.ih_pending);
1412 }
1413 
1414 boolean_t
1415 plat_dr_check_capability(uint64_t features)
1416 {
1417         return ((plat_dr_options & features) == features);
1418 }
1419 
1420 boolean_t
1421 plat_dr_support_cpu(void)
1422 {
1423         return (plat_dr_options & PLAT_DR_FEATURE_CPU);
1424 }
1425 
1426 boolean_t
1427 plat_dr_support_memory(void)
1428 {
1429         return (plat_dr_options & PLAT_DR_FEATURE_MEMORY);
1430 }
1431 
1432 void
1433 plat_dr_enable_capability(uint64_t features)
1434 {
1435         atomic_or_64(&plat_dr_options, features);
1436 }
1437 
1438 void
1439 plat_dr_disable_capability(uint64_t features)
1440 {
1441         atomic_and_64(&plat_dr_options, ~features);
1442 }