1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1987, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 
  25 /*
  26  * VM - Hardware Address Translation management.
  27  *
  28  * This file describes the contents of the sun-reference-mmu(sfmmu)-
  29  * specific hat data structures and the sfmmu-specific hat procedures.
  30  * The machine-independent interface is described in <vm/hat.h>.
  31  */
  32 
  33 #ifndef _VM_HAT_SFMMU_H
  34 #define _VM_HAT_SFMMU_H
  35 
  36 #ifdef  __cplusplus
  37 extern "C" {
  38 #endif
  39 
  40 #ifndef _ASM
  41 
  42 #include <sys/types.h>
  43 
  44 #endif /* _ASM */
  45 
  46 #ifdef  _KERNEL
  47 
  48 #include <sys/pte.h>
  49 #include <vm/mach_sfmmu.h>
  50 #include <sys/mmu.h>
  51 
  52 /*
  53  * Don't alter these without considering changes to ism_map_t.
  54  */
  55 #define DEFAULT_ISM_PAGESIZE            MMU_PAGESIZE4M
  56 #define DEFAULT_ISM_PAGESZC             TTE4M
  57 #define ISM_PG_SIZE(ism_vbshift)        (1 << ism_vbshift)
  58 #define ISM_SZ_MASK(ism_vbshift)        (ISM_PG_SIZE(ism_vbshift) - 1)
  59 #define ISM_MAP_SLOTS   8       /* Change this carefully. */
  60 
  61 #ifndef _ASM
  62 
  63 #include <sys/t_lock.h>
  64 #include <vm/hat.h>
  65 #include <vm/seg.h>
  66 #include <sys/machparam.h>
  67 #include <sys/systm.h>
  68 #include <sys/x_call.h>
  69 #include <vm/page.h>
  70 #include <sys/ksynch.h>
  71 
  72 typedef struct hat sfmmu_t;
  73 typedef struct sf_scd sf_scd_t;
  74 
  75 /*
  76  * SFMMU attributes for hat_memload/hat_devload
  77  */
  78 #define SFMMU_UNCACHEPTTE       0x01000000      /* unencache in physical $ */
  79 #define SFMMU_UNCACHEVTTE       0x02000000      /* unencache in virtual $ */
  80 #define SFMMU_SIDEFFECT         0x04000000      /* set side effect bit */
  81 #define SFMMU_LOAD_ALLATTR      (HAT_PROT_MASK | HAT_ORDER_MASK |       \
  82                 HAT_ENDIAN_MASK | HAT_NOFAULT | HAT_NOSYNC |            \
  83                 SFMMU_UNCACHEPTTE | SFMMU_UNCACHEVTTE | SFMMU_SIDEFFECT)
  84 
  85 
  86 /*
  87  * sfmmu flags for hat_memload/hat_devload
  88  */
  89 #define SFMMU_NO_TSBLOAD        0x08000000      /* do not preload tsb */
  90 #define SFMMU_LOAD_ALLFLAG      (HAT_LOAD | HAT_LOAD_LOCK |             \
  91                 HAT_LOAD_ADV | HAT_LOAD_CONTIG | HAT_LOAD_NOCONSIST |   \
  92                 HAT_LOAD_SHARE | HAT_LOAD_REMAP | SFMMU_NO_TSBLOAD |    \
  93                 HAT_RELOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_TEXT)
  94 
  95 /*
  96  * sfmmu internal flag to hat_pageunload that spares locked mappings
  97  */
  98 #define SFMMU_KERNEL_RELOC      0x8000
  99 
 100 /*
 101  * mode for sfmmu_chgattr
 102  */
 103 #define SFMMU_SETATTR   0x0
 104 #define SFMMU_CLRATTR   0x1
 105 #define SFMMU_CHGATTR   0x2
 106 
 107 /*
 108  * sfmmu specific flags for page_t
 109  */
 110 #define P_PNC   0x8             /* non-caching is permanent bit */
 111 #define P_TNC   0x10            /* non-caching is temporary bit */
 112 #define P_KPMS  0x20            /* kpm mapped small (vac alias prevention) */
 113 #define P_KPMC  0x40            /* kpm conflict page (vac alias prevention) */
 114 
 115 #define PP_GENERIC_ATTR(pp)     ((pp)->p_nrm & (P_MOD | P_REF | P_RO))
 116 #define PP_ISMOD(pp)            ((pp)->p_nrm & P_MOD)
 117 #define PP_ISREF(pp)            ((pp)->p_nrm & P_REF)
 118 #define PP_ISRO(pp)             ((pp)->p_nrm & P_RO)
 119 #define PP_ISNC(pp)             ((pp)->p_nrm & (P_PNC|P_TNC))
 120 #define PP_ISPNC(pp)            ((pp)->p_nrm & P_PNC)
 121 #ifdef VAC
 122 #define PP_ISTNC(pp)            ((pp)->p_nrm & P_TNC)
 123 #endif
 124 #define PP_ISKPMS(pp)           ((pp)->p_nrm & P_KPMS)
 125 #define PP_ISKPMC(pp)           ((pp)->p_nrm & P_KPMC)
 126 
 127 #define PP_SETMOD(pp)           ((pp)->p_nrm |= P_MOD)
 128 #define PP_SETREF(pp)           ((pp)->p_nrm |= P_REF)
 129 #define PP_SETREFMOD(pp)        ((pp)->p_nrm |= (P_REF|P_MOD))
 130 #define PP_SETRO(pp)            ((pp)->p_nrm |= P_RO)
 131 #define PP_SETREFRO(pp)         ((pp)->p_nrm |= (P_REF|P_RO))
 132 #define PP_SETPNC(pp)           ((pp)->p_nrm |= P_PNC)
 133 #ifdef VAC
 134 #define PP_SETTNC(pp)           ((pp)->p_nrm |= P_TNC)
 135 #endif
 136 #define PP_SETKPMS(pp)          ((pp)->p_nrm |= P_KPMS)
 137 #define PP_SETKPMC(pp)          ((pp)->p_nrm |= P_KPMC)
 138 
 139 #define PP_CLRMOD(pp)           ((pp)->p_nrm &= ~P_MOD)
 140 #define PP_CLRREF(pp)           ((pp)->p_nrm &= ~P_REF)
 141 #define PP_CLRREFMOD(pp)        ((pp)->p_nrm &= ~(P_REF|P_MOD))
 142 #define PP_CLRRO(pp)            ((pp)->p_nrm &= ~P_RO)
 143 #define PP_CLRPNC(pp)           ((pp)->p_nrm &= ~P_PNC)
 144 #ifdef VAC
 145 #define PP_CLRTNC(pp)           ((pp)->p_nrm &= ~P_TNC)
 146 #endif
 147 #define PP_CLRKPMS(pp)          ((pp)->p_nrm &= ~P_KPMS)
 148 #define PP_CLRKPMC(pp)          ((pp)->p_nrm &= ~P_KPMC)
 149 
 150 /*
 151  * All shared memory segments attached with the SHM_SHARE_MMU flag (ISM)
 152  * will be constrained to a 4M, 32M or 256M alignment. Also since every newly-
 153  * created ISM segment is created out of a new address space at base va
 154  * of 0 we don't need to store it.
 155  */
 156 #define ISM_ALIGN(shift)        (1 << shift)      /* base va aligned to <n>M  */
 157 #define ISM_ALIGNED(shift, va)  (((uintptr_t)va & (ISM_ALIGN(shift) - 1)) == 0)
 158 #define ISM_SHIFT(shift, x)     ((uintptr_t)x >> (shift))
 159 
 160 /*
 161  * Pad locks out to cache sub-block boundaries to prevent
 162  * false sharing, so several processes don't contend for
 163  * the same line if they aren't using the same lock.  Since
 164  * this is a typedef we also have a bit of freedom in
 165  * changing lock implementations later if we decide it
 166  * is necessary.
 167  */
 168 typedef struct hat_lock {
 169         kmutex_t hl_mutex;
 170         uchar_t hl_pad[64 - sizeof (kmutex_t)];
 171 } hatlock_t;
 172 
 173 #define HATLOCK_MUTEXP(hatlockp)        (&((hatlockp)->hl_mutex))
 174 
 175 /*
 176  * All segments mapped with ISM are guaranteed to be 4M, 32M or 256M aligned.
 177  * Also size is guaranteed to be in 4M, 32M or 256M chunks.
 178  * ism_seg consists of the following members:
 179  * [XX..22] base address of ism segment. XX is 63 or 31 depending whether
 180  *      caddr_t is 64 bits or 32 bits.
 181  * [21..0] size of segment.
 182  *
 183  * NOTE: Don't alter this structure without changing defines above and
 184  * the tsb_miss and protection handlers.
 185  */
 186 typedef struct ism_map {
 187         uintptr_t       imap_seg;       /* base va + sz of ISM segment */
 188         uchar_t         imap_vb_shift;  /* mmu_pageshift for ism page size */
 189         uchar_t         imap_rid;       /* region id for ism */
 190         ushort_t        imap_hatflags;  /* primary ism page size */
 191         uint_t          imap_sz_mask;   /* mmu_pagemask for ism page size */
 192         sfmmu_t         *imap_ismhat;   /* hat id of dummy ISM as */
 193         struct ism_ment *imap_ment;     /* pointer to mapping list entry */
 194 } ism_map_t;
 195 
 196 #define ism_start(map)  ((caddr_t)((map).imap_seg & \
 197                                 ~ISM_SZ_MASK((map).imap_vb_shift)))
 198 #define ism_size(map)   ((map).imap_seg & ISM_SZ_MASK((map).imap_vb_shift))
 199 #define ism_end(map)    ((caddr_t)(ism_start(map) + (ism_size(map) * \
 200                                 ISM_PG_SIZE((map).imap_vb_shift))))
 201 /*
 202  * ISM mapping entry. Used to link all hat's sharing a ism_hat.
 203  * Same function as the p_mapping list for a page.
 204  */
 205 typedef struct ism_ment {
 206         sfmmu_t         *iment_hat;     /* back pointer to hat_share() hat */
 207         caddr_t         iment_base_va;  /* hat's va base for this ism seg */
 208         struct ism_ment *iment_next;    /* next ism map entry */
 209         struct ism_ment *iment_prev;    /* prev ism map entry */
 210 } ism_ment_t;
 211 
 212 /*
 213  * ISM segment block. One will be hung off the sfmmu structure if a
 214  * a process uses ISM.  More will be linked using ismblk_next if more
 215  * than ISM_MAP_SLOTS segments are attached to this proc.
 216  *
 217  * All modifications to fields in this structure will be protected
 218  * by the hat mutex.  In order to avoid grabbing this lock in low level
 219  * routines (tsb miss/protection handlers and vatopfn) while not
 220  * introducing any race conditions with hat_unshare, we will set
 221  * CTX_ISM_BUSY bit in the ctx struct. Any mmu traps that occur
 222  * for this ctx while this bit is set will be handled in sfmmu_tsb_excption
 223  * where it will synchronize behind the hat mutex.
 224  */
 225 typedef struct ism_blk {
 226         ism_map_t               iblk_maps[ISM_MAP_SLOTS];
 227         struct ism_blk          *iblk_next;
 228         uint64_t                iblk_nextpa;
 229 } ism_blk_t;
 230 
 231 /*
 232  * TSB access information.  All fields are protected by the process's
 233  * hat lock.
 234  */
 235 
 236 struct tsb_info {
 237         caddr_t         tsb_va;         /* tsb base virtual address */
 238         uint64_t        tsb_pa;         /* tsb base physical address */
 239         struct tsb_info *tsb_next;      /* next tsb used by this process */
 240         uint16_t        tsb_szc;        /* tsb size code */
 241         uint16_t        tsb_flags;      /* flags for this tsb; see below */
 242         uint_t          tsb_ttesz_mask; /* page size masks; see below */
 243 
 244         tte_t           tsb_tte;        /* tte to lock into DTLB */
 245         sfmmu_t         *tsb_sfmmu;     /* sfmmu */
 246         kmem_cache_t    *tsb_cache;     /* cache from which mem allocated */
 247         vmem_t          *tsb_vmp;       /* vmem arena from which mem alloc'd */
 248 };
 249 
 250 /*
 251  * Values for "tsb_ttesz_mask" bitmask.
 252  */
 253 #define TSB8K   (1 << TTE8K)
 254 #define TSB64K  (1 << TTE64K)
 255 #define TSB512K (1 << TTE512K)
 256 #define TSB4M   (1 << TTE4M)
 257 #define TSB32M  (1 << TTE32M)
 258 #define TSB256M (1 << TTE256M)
 259 
 260 /*
 261  * Values for "tsb_flags" field.
 262  */
 263 #define TSB_RELOC_FLAG          0x1
 264 #define TSB_FLUSH_NEEDED        0x2
 265 #define TSB_SWAPPED     0x4
 266 #define TSB_SHAREDCTX           0x8
 267 
 268 #endif  /* !_ASM */
 269 
 270 /*
 271  * Data structures for shared hmeblk support.
 272  */
 273 
 274 /*
 275  * Do not increase the maximum number of ism/hme regions without checking first
 276  * the impact on ism_map_t, TSB miss area, hblk tag and region id type in
 277  * sf_region structure.
 278  * Initially, shared hmes will only be used for the main text segment
 279  * therefore this value will be set to 64, it will be increased when shared
 280  * libraries are included.
 281  */
 282 
 283 #define SFMMU_MAX_HME_REGIONS           (64)
 284 #define SFMMU_HMERGNMAP_WORDS           BT_BITOUL(SFMMU_MAX_HME_REGIONS)
 285 
 286 #define SFMMU_PRIVATE   0
 287 #define SFMMU_SHARED    1
 288 
 289 #define HMEBLK_ENDPA    1
 290 
 291 #ifndef _ASM
 292 
 293 #define SFMMU_MAX_ISM_REGIONS           (64)
 294 #define SFMMU_ISMRGNMAP_WORDS           BT_BITOUL(SFMMU_MAX_ISM_REGIONS)
 295 
 296 #define SFMMU_RGNMAP_WORDS      (SFMMU_HMERGNMAP_WORDS + SFMMU_ISMRGNMAP_WORDS)
 297 
 298 #define SFMMU_MAX_REGION_BUCKETS        (128)
 299 #define SFMMU_MAX_SRD_BUCKETS           (2048)
 300 
 301 typedef struct sf_hmeregion_map {
 302         ulong_t bitmap[SFMMU_HMERGNMAP_WORDS];
 303 } sf_hmeregion_map_t;
 304 
 305 typedef struct sf_ismregion_map {
 306         ulong_t bitmap[SFMMU_ISMRGNMAP_WORDS];
 307 } sf_ismregion_map_t;
 308 
 309 typedef union sf_region_map_u {
 310         struct _h_rmap_s {
 311                 sf_hmeregion_map_t hmeregion_map;
 312                 sf_ismregion_map_t ismregion_map;
 313         } h_rmap_s;
 314         ulong_t bitmap[SFMMU_RGNMAP_WORDS];
 315 } sf_region_map_t;
 316 
 317 #define SF_RGNMAP_ZERO(map) {                           \
 318         int _i;                                         \
 319         for (_i = 0; _i < SFMMU_RGNMAP_WORDS; _i++) {        \
 320                 (map).bitmap[_i] = 0;                   \
 321         }                                               \
 322 }
 323 
 324 /*
 325  * Returns 1 if map1 and map2 are equal.
 326  */
 327 #define SF_RGNMAP_EQUAL(map1, map2, rval)       {               \
 328         int _i;                                                 \
 329         for (_i = 0; _i < SFMMU_RGNMAP_WORDS; _i++) {                \
 330                 if ((map1)->bitmap[_i] != (map2)->bitmap[_i])     \
 331                         break;                                  \
 332         }                                                       \
 333         if (_i < SFMMU_RGNMAP_WORDS)                         \
 334                 rval = 0;                                       \
 335         else                                                    \
 336                 rval = 1;                                       \
 337 }
 338 
 339 #define SF_RGNMAP_ADD(map, r)           BT_SET((map).bitmap, r)
 340 #define SF_RGNMAP_DEL(map, r)           BT_CLEAR((map).bitmap, r)
 341 #define SF_RGNMAP_TEST(map, r)          BT_TEST((map).bitmap, r)
 342 
 343 /*
 344  * Tests whether map2 is a subset of map1, returns 1 if
 345  * this assertion is true.
 346  */
 347 #define SF_RGNMAP_IS_SUBSET(map1, map2, rval)   {               \
 348         int _i;                                                 \
 349         for (_i = 0; _i < SFMMU_RGNMAP_WORDS; _i++) {                \
 350                 if (((map1)->bitmap[_i]      & (map2)->bitmap[_i])    \
 351                     != (map2)->bitmap[_i])  {                        \
 352                         break;                                  \
 353                 }                                               \
 354         }                                                       \
 355         if (_i < SFMMU_RGNMAP_WORDS)                         \
 356                 rval = 0;                                       \
 357         else                                                    \
 358                 rval = 1;                                       \
 359 }
 360 
 361 #define SF_SCD_INCR_REF(scdp) {                                         \
 362         atomic_inc_32((volatile uint32_t *)&(scdp)->scd_refcnt); \
 363 }
 364 
 365 #define SF_SCD_DECR_REF(srdp, scdp) {                           \
 366         sf_region_map_t _scd_rmap = (scdp)->scd_region_map;  \
 367         if (!atomic_dec_32_nv((volatile uint32_t *)&(scdp)->scd_refcnt)) {\
 368                 sfmmu_destroy_scd((srdp), (scdp), &_scd_rmap);      \
 369         }                                                       \
 370 }
 371 
 372 /*
 373  * A sfmmup link in the link list of sfmmups that share the same region.
 374  */
 375 typedef struct sf_rgn_link {
 376         sfmmu_t *next;
 377         sfmmu_t *prev;
 378 } sf_rgn_link_t;
 379 
 380 /*
 381  * rgn_flags values.
 382  */
 383 #define SFMMU_REGION_HME        0x1
 384 #define SFMMU_REGION_ISM        0x2
 385 #define SFMMU_REGION_FREE       0x8
 386 
 387 #define SFMMU_REGION_TYPE_MASK  (0x3)
 388 
 389 /*
 390  * sf_region defines a text or (D)ISM segment which map
 391  * the same underlying physical object.
 392  */
 393 typedef struct sf_region {
 394         caddr_t                 rgn_saddr;   /* base addr of attached seg */
 395         size_t                  rgn_size;    /* size of attached seg */
 396         void                    *rgn_obj;    /* the underlying object id */
 397         u_offset_t              rgn_objoff;  /* offset in the object mapped */
 398         uchar_t                 rgn_perm;    /* PROT_READ/WRITE/EXEC */
 399         uchar_t                 rgn_pgszc;   /* page size of the region */
 400         uchar_t                 rgn_flags;   /* region type, free flag */
 401         uchar_t                 rgn_id;
 402         int                     rgn_refcnt;  /* # of hats sharing the region */
 403         /* callback function for hat_unload_callback */
 404         hat_rgn_cb_func_t       rgn_cb_function;
 405         struct sf_region        *rgn_hash;   /* hash chain linking the rgns */
 406         kmutex_t                rgn_mutex;   /* protect region sfmmu list */
 407         /* A link list of processes attached to this region */
 408         sfmmu_t                 *rgn_sfmmu_head;
 409         ulong_t                 rgn_ttecnt[MMU_PAGE_SIZES];
 410         uint16_t                rgn_hmeflags; /* rgn tte size flags */
 411 } sf_region_t;
 412 
 413 #define rgn_next        rgn_hash
 414 
 415 /* srd */
 416 typedef struct sf_shared_region_domain {
 417         vnode_t                 *srd_evp;       /* executable vnode */
 418         /* hme region table */
 419         sf_region_t             *srd_hmergnp[SFMMU_MAX_HME_REGIONS];
 420         /* ism region table */
 421         sf_region_t             *srd_ismrgnp[SFMMU_MAX_ISM_REGIONS];
 422         /* hash chain linking srds */
 423         struct sf_shared_region_domain *srd_hash;
 424         /* pointer to the next free hme region */
 425         sf_region_t             *srd_hmergnfree;
 426         /* pointer to the next free ism region */
 427         sf_region_t             *srd_ismrgnfree;
 428         /* id of next ism region created */
 429         uint16_t                srd_next_ismrid;
 430         /* id of next hme region created */
 431         uint16_t                srd_next_hmerid;
 432         uint16_t                srd_ismbusyrgns; /* # of ism rgns in use */
 433         uint16_t                srd_hmebusyrgns; /* # of hme rgns in use */
 434         int                     srd_refcnt;      /* # of procs in the srd */
 435         kmutex_t                srd_mutex;       /* sync add/remove rgns */
 436         kmutex_t                srd_scd_mutex;
 437         sf_scd_t                *srd_scdp;       /* list of scds in srd */
 438         /* hash of regions associated with the same executable */
 439         sf_region_t             *srd_rgnhash[SFMMU_MAX_REGION_BUCKETS];
 440 } sf_srd_t;
 441 
 442 typedef struct sf_srd_bucket {
 443         kmutex_t        srdb_lock;
 444         sf_srd_t        *srdb_srdp;
 445 } sf_srd_bucket_t;
 446 
 447 /*
 448  * The value of SFMMU_L1_HMERLINKS and SFMMU_L2_HMERLINKS will be increased
 449  * to 16 when the use of shared hmes for shared libraries is enabled.
 450  */
 451 
 452 #define SFMMU_L1_HMERLINKS              (8)
 453 #define SFMMU_L2_HMERLINKS              (8)
 454 #define SFMMU_L1_HMERLINKS_SHIFT        (3)
 455 #define SFMMU_L1_HMERLINKS_MASK         (SFMMU_L1_HMERLINKS - 1)
 456 #define SFMMU_L2_HMERLINKS_MASK         (SFMMU_L2_HMERLINKS - 1)
 457 #define SFMMU_L1_HMERLINKS_SIZE         \
 458         (SFMMU_L1_HMERLINKS * sizeof (sf_rgn_link_t *))
 459 #define SFMMU_L2_HMERLINKS_SIZE         \
 460         (SFMMU_L2_HMERLINKS * sizeof (sf_rgn_link_t))
 461 
 462 #if (SFMMU_L1_HMERLINKS * SFMMU_L2_HMERLINKS < SFMMU_MAX_HME_REGIONS)
 463 #error Not Enough HMERLINKS
 464 #endif
 465 
 466 /*
 467  * This macro grabs hat lock and allocates level 2 hat chain
 468  * associated with a shme rgn. In the majority of cases, the macro
 469  * is called with alloc = 0, and lock = 0.
 470  * A pointer to the level 2 sf_rgn_link_t structure is returned in the lnkp
 471  * parameter.
 472  */
 473 #define SFMMU_HMERID2RLINKP(sfmmup, rid, lnkp, alloc, lock)             \
 474 {                                                                       \
 475         int _l1ix = ((rid) >> SFMMU_L1_HMERLINKS_SHIFT) &             \
 476             SFMMU_L1_HMERLINKS_MASK;                                    \
 477         int _l2ix = ((rid) & SFMMU_L2_HMERLINKS_MASK);                      \
 478         hatlock_t *_hatlockp;                                           \
 479         lnkp = (sfmmup)->sfmmu_hmeregion_links[_l1ix];                       \
 480         if (lnkp != NULL) {                                             \
 481                 lnkp = &lnkp[_l2ix];                                        \
 482         } else if (alloc && lock) {                                     \
 483                 lnkp = kmem_zalloc(SFMMU_L2_HMERLINKS_SIZE, KM_SLEEP);  \
 484                 _hatlockp = sfmmu_hat_enter(sfmmup);                    \
 485                 if ((sfmmup)->sfmmu_hmeregion_links[_l1ix] != NULL) {        \
 486                         sfmmu_hat_exit(_hatlockp);                      \
 487                         kmem_free(lnkp, SFMMU_L2_HMERLINKS_SIZE);       \
 488                         lnkp = (sfmmup)->sfmmu_hmeregion_links[_l1ix];       \
 489                         ASSERT(lnkp != NULL);                           \
 490                 } else {                                                \
 491                         (sfmmup)->sfmmu_hmeregion_links[_l1ix] = lnkp;       \
 492                         sfmmu_hat_exit(_hatlockp);                      \
 493                 }                                                       \
 494                 lnkp = &lnkp[_l2ix];                                        \
 495         } else if (alloc) {                                             \
 496                 lnkp = kmem_zalloc(SFMMU_L2_HMERLINKS_SIZE, KM_SLEEP);  \
 497                 ASSERT((sfmmup)->sfmmu_hmeregion_links[_l1ix] == NULL);      \
 498                 (sfmmup)->sfmmu_hmeregion_links[_l1ix] = lnkp;               \
 499                 lnkp = &lnkp[_l2ix];                                        \
 500         }                                                               \
 501 }
 502 
 503 /*
 504  *  Per cpu pending freelist of hmeblks.
 505  */
 506 typedef struct cpu_hme_pend {
 507         struct   hme_blk *chp_listp;
 508         kmutex_t chp_mutex;
 509         time_t   chp_timestamp;
 510         uint_t   chp_count;
 511         uint8_t  chp_pad[36];           /* pad to 64 bytes */
 512 } cpu_hme_pend_t;
 513 
 514 /*
 515  * The default value of the threshold for the per cpu pending queues of hmeblks.
 516  * The queues are flushed if either the number of hmeblks on the queue is above
 517  * the threshold, or one second has elapsed since the last flush.
 518  */
 519 #define CPU_HME_PEND_THRESH 1000
 520 
 521 /*
 522  * Per-MMU context domain kstats.
 523  *
 524  * TSB Miss Exceptions
 525  *      Number of times a TSB miss exception is handled in an MMU. See
 526  *      sfmmu_tsbmiss_exception() for more details.
 527  * TSB Raise Exception
 528  *      Number of times the CPUs within an MMU are cross-called
 529  *      to invalidate either a specific process context (when the process
 530  *      switches MMU contexts) or the context of any process that is
 531  *      running on those CPUs (as part of the MMU context wrap-around).
 532  * Wrap Around
 533  *      The number of times a wrap-around of MMU context happens.
 534  */
 535 typedef enum mmu_ctx_stat_types {
 536         MMU_CTX_TSB_EXCEPTIONS,         /* TSB miss exceptions handled */
 537         MMU_CTX_TSB_RAISE_EXCEPTION,    /* ctx invalidation cross calls */
 538         MMU_CTX_WRAP_AROUND,            /* wraparounds */
 539         MMU_CTX_NUM_STATS
 540 } mmu_ctx_stat_t;
 541 
 542 /*
 543  * Per-MMU context domain structure. This is instantiated the first time a CPU
 544  * belonging to the MMU context domain is configured into the system, at boot
 545  * time or at DR time.
 546  *
 547  * mmu_gnum
 548  *      The current generation number for the context IDs on this MMU context
 549  *      domain. It is protected by mmu_lock.
 550  * mmu_cnum
 551  *      The current cnum to be allocated on this MMU context domain. It
 552  *      is protected via CAS.
 553  * mmu_nctxs
 554  *      The max number of context IDs supported on every CPU in this
 555  *      MMU context domain. This is needed here in case the system supports
 556  *      mixed type of processors/MMUs. It also helps to make ctx switch code
 557  *      access fewer cache lines i.e. no need to retrieve it from some global
 558  *      nctxs.
 559  * mmu_lock
 560  *      The mutex spin lock used to serialize context ID wrap around
 561  * mmu_idx
 562  *      The index for this MMU context domain structure in the global array
 563  *      mmu_ctxdoms.
 564  * mmu_ncpus
 565  *      The actual number of CPUs that have been configured in this
 566  *      MMU context domain. This also acts as a reference count for the
 567  *      structure. When the last CPU in an MMU context domain is unconfigured,
 568  *      the structure is freed. It is protected by mmu_lock.
 569  * mmu_cpuset
 570  *      The CPU set of configured CPUs for this MMU context domain. Used
 571  *      to cross-call all the CPUs in the MMU context domain to invalidate
 572  *      context IDs during a wraparound operation. It is protected by mmu_lock.
 573  */
 574 
 575 typedef struct mmu_ctx {
 576         uint64_t        mmu_gnum;
 577         uint_t          mmu_cnum;
 578         uint_t          mmu_nctxs;
 579         kmutex_t        mmu_lock;
 580         uint_t          mmu_idx;
 581         uint_t          mmu_ncpus;
 582         cpuset_t        mmu_cpuset;
 583         kstat_t         *mmu_kstat;
 584         kstat_named_t   mmu_kstat_data[MMU_CTX_NUM_STATS];
 585 } mmu_ctx_t;
 586 
 587 #define mmu_tsb_exceptions      \
 588                 mmu_kstat_data[MMU_CTX_TSB_EXCEPTIONS].value.ui64
 589 #define mmu_tsb_raise_exception \
 590                 mmu_kstat_data[MMU_CTX_TSB_RAISE_EXCEPTION].value.ui64
 591 #define mmu_wrap_around         \
 592                 mmu_kstat_data[MMU_CTX_WRAP_AROUND].value.ui64
 593 
 594 extern uint_t           max_mmu_ctxdoms;
 595 extern mmu_ctx_t        **mmu_ctxs_tbl;
 596 
 597 extern void     sfmmu_cpu_init(cpu_t *);
 598 extern void     sfmmu_cpu_cleanup(cpu_t *);
 599 
 600 extern uint_t   sfmmu_ctxdom_nctxs(int);
 601 
 602 #ifdef sun4v
 603 extern void     sfmmu_ctxdoms_remove(void);
 604 extern void     sfmmu_ctxdoms_lock(void);
 605 extern void     sfmmu_ctxdoms_unlock(void);
 606 extern void     sfmmu_ctxdoms_update(void);
 607 #endif
 608 
 609 /*
 610  * The following structure is used to get MMU context domain information for
 611  * a CPU from the platform.
 612  *
 613  * mmu_idx
 614  *      The MMU context domain index within the global array mmu_ctxs
 615  * mmu_nctxs
 616  *      The number of context IDs supported in the MMU context domain
 617  */
 618 typedef struct mmu_ctx_info {
 619         uint_t          mmu_idx;
 620         uint_t          mmu_nctxs;
 621 } mmu_ctx_info_t;
 622 
 623 #pragma weak plat_cpuid_to_mmu_ctx_info
 624 
 625 extern void     plat_cpuid_to_mmu_ctx_info(processorid_t, mmu_ctx_info_t *);
 626 
 627 /*
 628  * Each address space has an array of sfmmu_ctx_t structures, one structure
 629  * per MMU context domain.
 630  *
 631  * cnum
 632  *      The context ID allocated for an address space on an MMU context domain
 633  * gnum
 634  *      The generation number for the context ID in the MMU context domain.
 635  *
 636  * This structure needs to be a power-of-two in size.
 637  */
 638 typedef struct sfmmu_ctx {
 639         uint64_t        gnum:48;
 640         uint64_t        cnum:16;
 641 } sfmmu_ctx_t;
 642 
 643 
 644 /*
 645  * The platform dependent hat structure.
 646  * tte counts should be protected by cas.
 647  * cpuset is protected by cas.
 648  *
 649  * ttecnt accounting for mappings which do not use shared hme is carried out
 650  * during pagefault handling. In the shared hme case, only the first process
 651  * to access a mapping generates a pagefault, subsequent processes simply
 652  * find the shared hme entry during trap handling and therefore there is no
 653  * corresponding event to initiate ttecnt accounting. Currently, as shared
 654  * hmes are only used for text segments, when joining a region we assume the
 655  * worst case and add the the number of ttes required to map the entire region
 656  * to the ttecnt corresponding to the region pagesize. However, if the region
 657  * has a 4M pagesize, and memory is low, the allocation of 4M pages may fail
 658  * then 8K pages will be allocated instead and the first TSB which stores 8K
 659  * mappings will potentially be undersized. To compensate for the potential
 660  * underaccounting in this case we always add 1/4 of the region size to the 8K
 661  * ttecnt.
 662  *
 663  * Note that sfmmu_xhat_provider MUST be the first element.
 664  */
 665 
 666 struct hat {
 667         void            *sfmmu_xhat_provider;   /* NULL for CPU hat */
 668         cpuset_t        sfmmu_cpusran;  /* cpu bit mask for efficient xcalls */
 669         struct  as      *sfmmu_as;      /* as this hat provides mapping for */
 670         /* per pgsz private ttecnt + shme rgns ttecnt for rgns not in SCD */
 671         ulong_t         sfmmu_ttecnt[MMU_PAGE_SIZES];
 672         /* shme rgns ttecnt for rgns in SCD */
 673         ulong_t         sfmmu_scdrttecnt[MMU_PAGE_SIZES];
 674         /* est. ism ttes that are NOT in a SCD */
 675         ulong_t         sfmmu_ismttecnt[MMU_PAGE_SIZES];
 676         /* ttecnt for isms that are in a SCD */
 677         ulong_t         sfmmu_scdismttecnt[MMU_PAGE_SIZES];
 678         /* inflate tsb0 to allow for large page alloc failure in region */
 679         ulong_t         sfmmu_tsb0_4minflcnt;
 680         union _h_un {
 681                 ism_blk_t       *sfmmu_iblkp;  /* maps to ismhat(s) */
 682                 ism_ment_t      *sfmmu_imentp; /* ism hat's mapping list */
 683         } h_un;
 684         uint_t          sfmmu_free:1;   /* hat to be freed - set on as_free */
 685         uint_t          sfmmu_ismhat:1; /* hat is dummy ism hatid */
 686         uint_t          sfmmu_scdhat:1; /* hat is dummy scd hatid */
 687         uchar_t         sfmmu_rmstat;   /* refmod stats refcnt */
 688         ushort_t        sfmmu_clrstart; /* start color bin for page coloring */
 689         ushort_t        sfmmu_clrbin;   /* per as phys page coloring bin */
 690         ushort_t        sfmmu_flags;    /* flags */
 691         uchar_t         sfmmu_tteflags; /* pgsz flags */
 692         uchar_t         sfmmu_rtteflags; /* pgsz flags for SRD hmes */
 693         struct tsb_info *sfmmu_tsb;     /* list of per as tsbs */
 694         uint64_t        sfmmu_ismblkpa; /* pa of sfmmu_iblkp, or -1 */
 695         lock_t          sfmmu_ctx_lock; /* sync ctx alloc and invalidation */
 696         kcondvar_t      sfmmu_tsb_cv;   /* signals TSB swapin or relocation */
 697         uchar_t         sfmmu_cext;     /* context page size encoding */
 698         uint8_t         sfmmu_pgsz[MMU_PAGE_SIZES];  /* ranking for MMU */
 699         sf_srd_t        *sfmmu_srdp;
 700         sf_scd_t        *sfmmu_scdp;    /* scd this address space belongs to */
 701         sf_region_map_t sfmmu_region_map;
 702         sf_rgn_link_t   *sfmmu_hmeregion_links[SFMMU_L1_HMERLINKS];
 703         sf_rgn_link_t   sfmmu_scd_link; /* link to scd or pending queue */
 704 #ifdef sun4v
 705         struct hv_tsb_block sfmmu_hvblock;
 706 #endif
 707         /*
 708          * sfmmu_ctxs is a variable length array of max_mmu_ctxdoms # of
 709          * elements. max_mmu_ctxdoms is determined at run-time.
 710          * sfmmu_ctxs[1] is just the fist element of an array, it always
 711          * has to be the last field to ensure that the memory allocated
 712          * for sfmmu_ctxs is consecutive with the memory of the rest of
 713          * the hat data structure.
 714          */
 715         sfmmu_ctx_t     sfmmu_ctxs[1];
 716 
 717 };
 718 
 719 #define sfmmu_iblk      h_un.sfmmu_iblkp
 720 #define sfmmu_iment     h_un.sfmmu_imentp
 721 
 722 #define sfmmu_hmeregion_map     sfmmu_region_map.h_rmap_s.hmeregion_map
 723 #define sfmmu_ismregion_map     sfmmu_region_map.h_rmap_s.ismregion_map
 724 
 725 #define SF_RGNMAP_ISNULL(sfmmup)        \
 726         (sfrgnmap_isnull(&(sfmmup)->sfmmu_region_map))
 727 #define SF_HMERGNMAP_ISNULL(sfmmup)     \
 728         (sfhmergnmap_isnull(&(sfmmup)->sfmmu_hmeregion_map))
 729 
 730 struct sf_scd {
 731         sfmmu_t         *scd_sfmmup;    /* shared context hat */
 732         /* per pgsz ttecnt for shme rgns in SCD */
 733         ulong_t         scd_rttecnt[MMU_PAGE_SIZES];
 734         uint_t          scd_refcnt;     /* address spaces attached to scd */
 735         sf_region_map_t scd_region_map; /* bit mask of attached segments */
 736         sf_scd_t        *scd_next;      /* link pointers for srd_scd list */
 737         sf_scd_t        *scd_prev;
 738         sfmmu_t         *scd_sf_list;   /* list of doubly linked hat structs */
 739         kmutex_t        scd_mutex;
 740         /*
 741          * Link used to add an scd to the sfmmu_iment list.
 742          */
 743         ism_ment_t      scd_ism_links[SFMMU_MAX_ISM_REGIONS];
 744 };
 745 
 746 #define scd_hmeregion_map       scd_region_map.h_rmap_s.hmeregion_map
 747 #define scd_ismregion_map       scd_region_map.h_rmap_s.ismregion_map
 748 
 749 extern int disable_shctx;
 750 extern int shctx_on;
 751 
 752 /*
 753  * bit mask for managing vac conflicts on large pages.
 754  * bit 1 is for uncache flag.
 755  * bits 2 through min(num of cache colors + 1,31) are
 756  * for cache colors that have already been flushed.
 757  */
 758 #ifdef VAC
 759 #define CACHE_NUM_COLOR         (shm_alignment >> MMU_PAGESHIFT)
 760 #else
 761 #define CACHE_NUM_COLOR         1
 762 #endif
 763 
 764 #define CACHE_VCOLOR_MASK(vcolor)       (2 << (vcolor & (CACHE_NUM_COLOR - 1)))
 765 
 766 #define CacheColor_IsFlushed(flag, vcolor) \
 767                                         ((flag) & CACHE_VCOLOR_MASK(vcolor))
 768 
 769 #define CacheColor_SetFlushed(flag, vcolor) \
 770                                         ((flag) |= CACHE_VCOLOR_MASK(vcolor))
 771 /*
 772  * Flags passed to sfmmu_page_cache to flush page from vac or not.
 773  */
 774 #define CACHE_FLUSH     0
 775 #define CACHE_NO_FLUSH  1
 776 
 777 /*
 778  * Flags passed to sfmmu_tlbcache_demap
 779  */
 780 #define FLUSH_NECESSARY_CPUS    0
 781 #define FLUSH_ALL_CPUS          1
 782 
 783 #ifdef  DEBUG
 784 /*
 785  * For debugging purpose only. Maybe removed later.
 786  */
 787 struct ctx_trace {
 788         sfmmu_t         *sc_sfmmu_stolen;
 789         sfmmu_t         *sc_sfmmu_stealing;
 790         clock_t         sc_time;
 791         ushort_t        sc_type;
 792         ushort_t        sc_cnum;
 793 };
 794 #define CTX_TRC_STEAL   0x1
 795 #define CTX_TRC_FREE    0x0
 796 #define TRSIZE  0x400
 797 #define NEXT_CTXTR(ptr) (((ptr) >= ctx_trace_last) ? \
 798                 ctx_trace_first : ((ptr) + 1))
 799 #define TRACE_CTXS(mutex, ptr, cnum, stolen_sfmmu, stealing_sfmmu, type) \
 800         mutex_enter(mutex);                                             \
 801         (ptr)->sc_sfmmu_stolen = (stolen_sfmmu);                     \
 802         (ptr)->sc_sfmmu_stealing = (stealing_sfmmu);                 \
 803         (ptr)->sc_cnum = (cnum);                                     \
 804         (ptr)->sc_type = (type);                                     \
 805         (ptr)->sc_time = ddi_get_lbolt();                            \
 806         (ptr) = NEXT_CTXTR(ptr);                                        \
 807         num_ctx_stolen += (type);                                       \
 808         mutex_exit(mutex);
 809 #else
 810 
 811 #define TRACE_CTXS(mutex, ptr, cnum, stolen_sfmmu, stealing_sfmmu, type)
 812 
 813 #endif  /* DEBUG */
 814 
 815 #endif  /* !_ASM */
 816 
 817 /*
 818  * Macros for sfmmup->sfmmu_flags access.  The macros that change the flags
 819  * ASSERT() that we're holding the HAT lock before changing the flags;
 820  * however callers that read the flags may do so without acquiring the lock
 821  * in a fast path, and then recheck the flag after acquiring the lock in
 822  * a slow path.
 823  */
 824 #define SFMMU_FLAGS_ISSET(sfmmup, flags) \
 825         (((sfmmup)->sfmmu_flags & (flags)) == (flags))
 826 
 827 #define SFMMU_FLAGS_CLEAR(sfmmup, flags) \
 828         (ASSERT(sfmmu_hat_lock_held((sfmmup))), \
 829         (sfmmup)->sfmmu_flags &= ~(flags))
 830 
 831 #define SFMMU_FLAGS_SET(sfmmup, flags) \
 832         (ASSERT(sfmmu_hat_lock_held((sfmmup))), \
 833         (sfmmup)->sfmmu_flags |= (flags))
 834 
 835 #define SFMMU_TTEFLAGS_ISSET(sfmmup, flags) \
 836         ((((sfmmup)->sfmmu_tteflags | (sfmmup)->sfmmu_rtteflags) & (flags)) == \
 837             (flags))
 838 
 839 
 840 /*
 841  * sfmmu tte HAT flags, must fit in 8 bits
 842  */
 843 #define HAT_CHKCTX1_FLAG 0x1
 844 #define HAT_64K_FLAG    (0x1 << TTE64K)
 845 #define HAT_512K_FLAG   (0x1 << TTE512K)
 846 #define HAT_4M_FLAG     (0x1 << TTE4M)
 847 #define HAT_32M_FLAG    (0x1 << TTE32M)
 848 #define HAT_256M_FLAG   (0x1 << TTE256M)
 849 
 850 /*
 851  * sfmmu HAT flags, 16 bits at the moment.
 852  */
 853 #define HAT_4MTEXT_FLAG         0x01
 854 #define HAT_32M_ISM             0x02
 855 #define HAT_256M_ISM            0x04
 856 #define HAT_SWAPPED             0x08 /* swapped out */
 857 #define HAT_SWAPIN              0x10 /* swapping in */
 858 #define HAT_BUSY                0x20 /* replacing TSB(s) */
 859 #define HAT_ISMBUSY             0x40 /* adding/removing/traversing ISM maps */
 860 
 861 #define HAT_CTX1_FLAG           0x100 /* ISM imap hatflag for ctx1 */
 862 #define HAT_JOIN_SCD            0x200 /* region is joining scd */
 863 #define HAT_ALLCTX_INVALID      0x400 /* all per-MMU ctxs are invalidated */
 864 
 865 #define SFMMU_LGPGS_INUSE(sfmmup)                                       \
 866         (((sfmmup)->sfmmu_tteflags | (sfmmup)->sfmmu_rtteflags) ||        \
 867             ((sfmmup)->sfmmu_iblk != NULL))
 868 
 869 /*
 870  * Starting with context 0, the first NUM_LOCKED_CTXS contexts
 871  * are locked so that sfmmu_getctx can't steal any of these
 872  * contexts.  At the time this software was being developed, the
 873  * only context that needs to be locked is context 0 (the kernel
 874  * context), and context 1 (reserved for stolen context). So this constant
 875  * was originally defined to be 2.
 876  *
 877  * For sun4v only, USER_CONTEXT_TYPE represents any user context.  Many
 878  * routines only care whether the context is kernel, invalid or user.
 879  */
 880 
 881 #define NUM_LOCKED_CTXS 2
 882 #define INVALID_CONTEXT 1
 883 
 884 #ifdef sun4v
 885 #define USER_CONTEXT_TYPE       NUM_LOCKED_CTXS
 886 #endif
 887 #if defined(sun4v) || defined(UTSB_PHYS)
 888 /*
 889  * Get the location in the 4MB base TSB of the tsbe for this fault.
 890  * Assumes that the second TSB only contains 4M mappings.
 891  *
 892  * In:
 893  *   tagacc = tag access register (not clobbered)
 894  *   tsbe = 2nd TSB base register
 895  *   tmp1, tmp2 = scratch registers
 896  * Out:
 897  *   tsbe = pointer to the tsbe in the 2nd TSB
 898  */
 899 
 900 #define GET_4MBASE_TSBE_PTR(tagacc, tsbe, tmp1, tmp2)                   \
 901         and     tsbe, TSB_SOFTSZ_MASK, tmp2;    /* tmp2=szc */          \
 902         andn    tsbe, TSB_SOFTSZ_MASK, tsbe;    /* tsbbase */           \
 903         mov     TSB_ENTRIES(0), tmp1;   /* nentries in TSB size 0 */    \
 904         sllx    tmp1, tmp2, tmp1;       /* tmp1 = nentries in TSB */    \
 905         sub     tmp1, 1, tmp1;          /* mask = nentries - 1 */       \
 906         srlx    tagacc, MMU_PAGESHIFT4M, tmp2;                          \
 907         and     tmp2, tmp1, tmp1;       /* tsbent = virtpage & mask */      \
 908         sllx    tmp1, TSB_ENTRY_SHIFT, tmp1;    /* entry num --> ptr */      \
 909         add     tsbe, tmp1, tsbe        /* add entry offset to TSB base */
 910 
 911 #define GET_2ND_TSBE_PTR(tagacc, tsbe, tmp1, tmp2)                      \
 912         GET_4MBASE_TSBE_PTR(tagacc, tsbe, tmp1, tmp2)
 913 
 914 /*
 915  * Get the location in the 3rd TSB of the tsbe for this fault.
 916  * The 3rd TSB corresponds to the shared context, and is used
 917  * for 8K - 512k pages.
 918  *
 919  * In:
 920  *   tagacc = tag access register (not clobbered)
 921  *   tsbe, tmp1, tmp2 = scratch registers
 922  * Out:
 923  *   tsbe = pointer to the tsbe in the 3rd TSB
 924  */
 925 
 926 #define GET_3RD_TSBE_PTR(tagacc, tsbe, tmp1, tmp2)                      \
 927         and     tsbe, TSB_SOFTSZ_MASK, tmp2;    /* tmp2=szc */          \
 928         andn    tsbe, TSB_SOFTSZ_MASK, tsbe;    /* tsbbase */           \
 929         mov     TSB_ENTRIES(0), tmp1;   /* nentries in TSB size 0 */    \
 930         sllx    tmp1, tmp2, tmp1;       /* tmp1 = nentries in TSB */    \
 931         sub     tmp1, 1, tmp1;          /* mask = nentries - 1 */       \
 932         srlx    tagacc, MMU_PAGESHIFT, tmp2;                            \
 933         and     tmp2, tmp1, tmp1;       /* tsbent = virtpage & mask */      \
 934         sllx    tmp1, TSB_ENTRY_SHIFT, tmp1;    /* entry num --> ptr */      \
 935         add     tsbe, tmp1, tsbe        /* add entry offset to TSB base */
 936 
 937 #define GET_4TH_TSBE_PTR(tagacc, tsbe, tmp1, tmp2)                      \
 938         GET_4MBASE_TSBE_PTR(tagacc, tsbe, tmp1, tmp2)
 939 /*
 940  * Copy the sfmmu_region_map or scd_region_map to the tsbmiss
 941  * shmermap or scd_shmermap, from sfmmu_load_mmustate.
 942  */
 943 #define SET_REGION_MAP(rgn_map, tsbmiss_map, cnt, tmp, label)           \
 944         /* BEGIN CSTYLED */                                             \
 945 label:                                                                  ;\
 946         ldx     [rgn_map], tmp                                          ;\
 947         dec     cnt                                                     ;\
 948         add     rgn_map, CLONGSIZE, rgn_map                             ;\
 949         stx     tmp, [tsbmiss_map]                                      ;\
 950         brnz,pt cnt, label                                              ;\
 951             add   tsbmiss_map, CLONGSIZE, tsbmiss_map                    \
 952         /* END CSTYLED */
 953 
 954 /*
 955  * If there is no scd, then zero the tsbmiss scd_shmermap,
 956  * from sfmmu_load_mmustate.
 957  */
 958 #define ZERO_REGION_MAP(tsbmiss_map, cnt, label)                        \
 959         /* BEGIN CSTYLED */                                             \
 960 label:                                                                  ;\
 961         dec     cnt                                                     ;\
 962         stx     %g0, [tsbmiss_map]                                      ;\
 963         brnz,pt cnt, label                                              ;\
 964             add   tsbmiss_map, CLONGSIZE, tsbmiss_map                    
 965         /* END CSTYLED */
 966 
 967 /*
 968  * Set hmemisc to 1 if the shared hme is also part of an scd.
 969  * In:
 970  *   tsbarea = tsbmiss area (not clobbered)
 971  *   hmeblkpa  = hmeblkpa +  hmentoff + SFHME_TTE (not clobbered)
 972  *   hmentoff = hmentoff + SFHME_TTE = tte offset(clobbered)
 973  * Out:
 974  *   use_shctx = 1 if shme is in scd and 0 otherwise
 975  */
 976 #define GET_SCDSHMERMAP(tsbarea, hmeblkpa, hmentoff, use_shctx)               \
 977         /* BEGIN CSTYLED */                                                   \
 978         sub     hmeblkpa, hmentoff, hmentoff    /* hmentofff = hmeblkpa */   ;\
 979         add     hmentoff, HMEBLK_TAG, hmentoff                               ;\
 980         ldxa    [hmentoff]ASI_MEM, hmentoff     /* read 1st part of tag */   ;\
 981         and     hmentoff, HTAG_RID_MASK, hmentoff       /* mask off rid */   ;\
 982         and     hmentoff, BT_ULMASK, use_shctx  /* mask bit index */         ;\
 983         srlx    hmentoff, BT_ULSHIFT, hmentoff  /* extract word */           ;\
 984         sllx    hmentoff, CLONGSHIFT, hmentoff  /* index */                  ;\
 985         add     tsbarea, hmentoff, hmentoff             /* add to tsbarea */ ;\
 986         ldx     [hmentoff + TSBMISS_SCDSHMERMAP], hmentoff      /* scdrgn */ ;\
 987         srlx    hmentoff, use_shctx, use_shctx                               ;\
 988         and     use_shctx, 0x1, use_shctx                                     \
 989         /* END CSTYLED */
 990 
 991 /*
 992  * Synthesize a TSB base register contents for a process.
 993  *
 994  * In:
 995  *   tsbinfo = TSB info pointer (ro)
 996  *   tsbreg, tmp1 = scratch registers
 997  * Out:
 998  *   tsbreg = value to program into TSB base register
 999  */
1000 
1001 #define MAKE_UTSBREG(tsbinfo, tsbreg, tmp1)                     \
1002         ldx     [tsbinfo + TSBINFO_PADDR], tsbreg;              \
1003         lduh    [tsbinfo + TSBINFO_SZCODE], tmp1;               \
1004         and     tmp1, TSB_SOFTSZ_MASK, tmp1;                    \
1005         or      tsbreg, tmp1, tsbreg;
1006 
1007 
1008 /*
1009  * Load TSB base register to TSBMISS area for privte contexts.
1010  * This register contains utsb_pabase in bits 63:13, and TSB size
1011  * code in bits 2:0.
1012  *
1013  * For private context
1014  * In:
1015  *   tsbreg = value to load (ro)
1016  *   regnum = constant or register
1017  *   tmp1 = scratch register
1018  * Out:
1019  *   Specified scratchpad register updated
1020  *
1021  */
1022 #define SET_UTSBREG(regnum, tsbreg, tmp1)                               \
1023         mov     regnum, tmp1;                                           \
1024         stxa    tsbreg, [tmp1]ASI_SCRATCHPAD    /* save tsbreg */
1025 /*
1026  * Get TSB base register from the scratchpad for private contexts
1027  *
1028  * In:
1029  *   regnum = constant or register
1030  *   tsbreg = scratch
1031  * Out:
1032  *   tsbreg = tsbreg from the specified scratchpad register
1033  */
1034 #define GET_UTSBREG(regnum, tsbreg)                                     \
1035         mov     regnum, tsbreg;                                         \
1036         ldxa    [tsbreg]ASI_SCRATCHPAD, tsbreg
1037 
1038 /*
1039  * Load TSB base register to TSBMISS area for shared contexts.
1040  * This register contains utsb_pabase in bits 63:13, and TSB size
1041  * code in bits 2:0.
1042  *
1043  * In:
1044  *   tsbmiss = pointer to tsbmiss area
1045  *   tsbmissoffset = offset to right tsb pointer
1046  *   tsbreg = value to load (ro)
1047  * Out:
1048  *   Specified tsbmiss area updated
1049  *
1050  */
1051 #define SET_UTSBREG_SHCTX(tsbmiss, tsbmissoffset, tsbreg)               \
1052         stx     tsbreg, [tsbmiss + tsbmissoffset]       /* save tsbreg */
1053 
1054 /*
1055  * Get TSB base register from the scratchpad for
1056  * shared contexts
1057  *
1058  * In:
1059  *   tsbmiss = pointer to tsbmiss area
1060  *   tsbmissoffset = offset to right tsb pointer
1061  *   tsbreg = scratch
1062  * Out:
1063  *   tsbreg = tsbreg from the specified scratchpad register
1064  */
1065 #define GET_UTSBREG_SHCTX(tsbmiss, tsbmissoffset, tsbreg)               \
1066         ldx     [tsbmiss + tsbmissoffset], tsbreg
1067 
1068 #endif /* defined(sun4v) || defined(UTSB_PHYS) */
1069 
1070 #ifndef _ASM
1071 
1072 /*
1073  * Kernel page relocation stuff.
1074  */
1075 struct sfmmu_callback {
1076         int key;
1077         int (*prehandler)(caddr_t, uint_t, uint_t, void *);
1078         int (*posthandler)(caddr_t, uint_t, uint_t, void *, pfn_t);
1079         int (*errhandler)(caddr_t, uint_t, uint_t, void *);
1080         int capture_cpus;
1081 };
1082 
1083 extern int sfmmu_max_cb_id;
1084 extern struct sfmmu_callback *sfmmu_cb_table;
1085 
1086 struct pa_hment;
1087 
1088 /*
1089  * RFE: With multihat gone we gain back an int.  We could use this to
1090  * keep ref bits on a per cpu basis to eliminate xcalls.
1091  */
1092 struct sf_hment {
1093         tte_t hme_tte;                  /* tte for this hment */
1094 
1095         union {
1096                 struct page *page;      /* what page this maps */
1097                 struct pa_hment *data;  /* pa_hment */
1098         } sf_hment_un;
1099 
1100         struct  sf_hment *hme_next;     /* next hment */
1101         struct  sf_hment *hme_prev;     /* prev hment */
1102 };
1103 
1104 struct pa_hment {
1105         caddr_t         addr;           /* va */
1106         uint_t          len;            /* bytes */
1107         ushort_t        flags;          /* internal flags */
1108         ushort_t        refcnt;         /* reference count */
1109         id_t            cb_id;          /* callback id, table index */
1110         void            *pvt;           /* handler's private data */
1111         struct sf_hment sfment;         /* corresponding dummy sf_hment */
1112 };
1113 
1114 #define hme_page                sf_hment_un.page
1115 #define hme_data                sf_hment_un.data
1116 #define hme_size(sfhmep)        ((int)(TTE_CSZ(&(sfhmep)->hme_tte)))
1117 #define PAHME_SZ                (sizeof (struct pa_hment))
1118 #define SFHME_SZ                (sizeof (struct sf_hment))
1119 
1120 #define IS_PAHME(hme)   ((hme)->hme_tte.ll == 0)
1121 
1122 /*
1123  * hmeblk_tag structure
1124  * structure used to obtain a match on a hme_blk.  Currently consists of
1125  * the address of the sfmmu struct (or hatid), the base page address of the
1126  * hme_blk, and the rehash count.  The rehash count is actually only 2 bits
1127  * and has the following meaning:
1128  * 1 = 8k or 64k hash sequence.
1129  * 2 = 512k hash sequence.
1130  * 3 = 4M hash sequence.
1131  * We require this count because we don't want to get a false hit on a 512K or
1132  * 4M rehash with a base address corresponding to a 8k or 64k hmeblk.
1133  * Note:  The ordering and size of the hmeblk_tag members are implictly known
1134  * by the tsb miss handlers written in assembly.  Do not change this structure
1135  * without checking those routines.  See HTAG_SFMMUPSZ define.
1136  */
1137 
1138 /*
1139  * In private hmeblks hblk_rid field must be SFMMU_INVALID_RID.
1140  */
1141 typedef union {
1142         struct {
1143                 uint64_t        hblk_basepg: 51,        /* hme_blk base pg # */
1144                                 hblk_rehash: 3,         /* rehash number */
1145                                 hblk_rid: 10;           /* hme_blk region id */
1146                 void            *hblk_id;
1147         } hblk_tag_un;
1148         uint64_t                htag_tag[2];
1149 } hmeblk_tag;
1150 
1151 #define htag_id         hblk_tag_un.hblk_id
1152 #define htag_bspage     hblk_tag_un.hblk_basepg
1153 #define htag_rehash     hblk_tag_un.hblk_rehash
1154 #define htag_rid        hblk_tag_un.hblk_rid
1155 
1156 #endif /* !_ASM */
1157 
1158 #define HTAG_REHASH_SHIFT       10
1159 #define HTAG_MAX_RID    (((0x1 << HTAG_REHASH_SHIFT) - 1))
1160 #define HTAG_RID_MASK   HTAG_MAX_RID
1161 
1162 /* used for tagging all per sfmmu (i.e. non SRD) private hmeblks */
1163 #define SFMMU_INVALID_SHMERID   HTAG_MAX_RID
1164 
1165 #if SFMMU_INVALID_SHMERID < SFMMU_MAX_HME_REGIONS
1166 #error SFMMU_INVALID_SHMERID < SFMMU_MAX_HME_REGIONS
1167 #endif
1168 
1169 #define SFMMU_IS_SHMERID_VALID(rid)     ((rid) != SFMMU_INVALID_SHMERID)
1170 
1171 /* ISM regions */
1172 #define SFMMU_INVALID_ISMRID    0xff
1173 
1174 #if SFMMU_INVALID_ISMRID < SFMMU_MAX_ISM_REGIONS
1175 #error SFMMU_INVALID_ISMRID < SFMMU_MAX_ISM_REGIONS
1176 #endif
1177 
1178 #define SFMMU_IS_ISMRID_VALID(rid)      ((rid) != SFMMU_INVALID_ISMRID)
1179 
1180 
1181 #define HTAGS_EQ(tag1, tag2)    (((tag1.htag_tag[0] ^ tag2.htag_tag[0]) | \
1182                                 (tag1.htag_tag[1] ^ tag2.htag_tag[1])) == 0)
1183 
1184 /*
1185  * this macro must only be used for comparing tags in shared hmeblks.
1186  */
1187 #define HTAGS_EQ_SHME(hmetag, tag, hrmap)                               \
1188         (((hmetag).htag_rid != SFMMU_INVALID_SHMERID) &&                \
1189         (((((hmetag).htag_tag[0] ^ (tag).htag_tag[0]) &                     \
1190                 ~HTAG_RID_MASK) |                                       \
1191             ((hmetag).htag_tag[1] ^ (tag).htag_tag[1])) == 0) &&        \
1192         SF_RGNMAP_TEST(hrmap, hmetag.htag_rid))
1193 
1194 #define HME_REHASH(sfmmup)                                              \
1195         ((sfmmup)->sfmmu_ttecnt[TTE512K] != 0 ||                     \
1196         (sfmmup)->sfmmu_ttecnt[TTE4M] != 0 ||                                \
1197         (sfmmup)->sfmmu_ttecnt[TTE32M] != 0 ||                               \
1198         (sfmmup)->sfmmu_ttecnt[TTE256M] != 0)
1199 
1200 #define NHMENTS         8               /* # of hments in an 8k hme_blk */
1201                                         /* needs to be multiple of 2 */
1202 
1203 #ifndef _ASM
1204 
1205 #ifdef  HBLK_TRACE
1206 
1207 #define HBLK_LOCK               1
1208 #define HBLK_UNLOCK             0
1209 #define HBLK_STACK_DEPTH        6
1210 #define HBLK_AUDIT_CACHE_SIZE   16
1211 #define HBLK_LOCK_PATTERN       0xaaaaaaaa
1212 #define HBLK_UNLOCK_PATTERN     0xbbbbbbbb
1213 
1214 struct hblk_lockcnt_audit {
1215         int             flag;           /* lock or unlock */
1216         kthread_id_t    thread;
1217         int             depth;
1218         pc_t            stack[HBLK_STACK_DEPTH];
1219 };
1220 
1221 #endif  /* HBLK_TRACE */
1222 
1223 
1224 /*
1225  * Hment block structure.
1226  * The hme_blk is the node data structure which the hash structure
1227  * mantains. An hme_blk can have 2 different sizes depending on the
1228  * number of hments it implicitly contains.  When dealing with 64K, 512K,
1229  * or 4M hments there is one hment per hme_blk.  When dealing with
1230  * 8k hments we allocate an hme_blk plus an additional 7 hments to
1231  * give us a total of 8 (NHMENTS) hments that can be referenced through a
1232  * hme_blk.
1233  *
1234  * The hmeblk structure contains 2 tte reference counters used to determine if
1235  * it is ok to free up the hmeblk.  Both counters have to be zero in order
1236  * to be able to free up hmeblk.  They are protected by cas.
1237  * hblk_hmecnt is the number of hments present on pp mapping lists.
1238  * hblk_vcnt reflects number of valid ttes in hmeblk.
1239  *
1240  * The hmeblk now also has per tte lock cnts.  This is required because
1241  * the counts can be high and there are not enough bits in the tte. When
1242  * physio is fixed to not lock the translations we should be able to move
1243  * the lock cnt back to the tte.  See bug id 1198554.
1244  *
1245  * Note that xhat_hme_blk's layout follows this structure: hme_blk_misc
1246  * and sf_hment are at the same offsets in both structures. Whenever
1247  * hme_blk is changed, xhat_hme_blk may need to be updated as well.
1248  */
1249 
1250 struct hme_blk_misc {
1251         uint_t  notused:25;
1252         uint_t  shared_bit:1;   /* set for SRD shared hmeblk */
1253         uint_t  xhat_bit:1;     /* set for an xhat hme_blk */
1254         uint_t  shadow_bit:1;   /* set for a shadow hme_blk */
1255         uint_t  nucleus_bit:1;  /* set for a nucleus hme_blk */
1256         uint_t  ttesize:3;      /* contains ttesz of hmeblk */
1257 };
1258 
1259 struct hme_blk {
1260         volatile uint64_t hblk_nextpa;  /* physical address for hash list */
1261 
1262         hmeblk_tag      hblk_tag;       /* tag used to obtain an hmeblk match */
1263 
1264         struct hme_blk  *hblk_next;     /* on free list or on hash list */
1265                                         /* protected by hash lock */
1266 
1267         struct hme_blk  *hblk_shadow;   /* pts to shadow hblk */
1268                                         /* protected by hash lock */
1269         uint_t          hblk_span;      /* span of memory hmeblk maps */
1270 
1271         struct hme_blk_misc     hblk_misc;
1272 
1273         union {
1274                 struct {
1275                         ushort_t hblk_hmecount; /* hment on mlists counter */
1276                         ushort_t hblk_validcnt; /* valid tte reference count */
1277                 } hblk_counts;
1278                 uint_t          hblk_shadow_mask;
1279         } hblk_un;
1280 
1281         uint_t          hblk_lckcnt;
1282 
1283 #ifdef  HBLK_TRACE
1284         kmutex_t        hblk_audit_lock;        /* lock to protect index */
1285         uint_t          hblk_audit_index;       /* index into audit_cache */
1286         struct  hblk_lockcnt_audit hblk_audit_cache[HBLK_AUDIT_CACHE_SIZE];
1287 #endif  /* HBLK_AUDIT */
1288 
1289         struct sf_hment hblk_hme[1];    /* hment array */
1290 };
1291 
1292 #define hblk_shared     hblk_misc.shared_bit
1293 #define hblk_xhat_bit   hblk_misc.xhat_bit
1294 #define hblk_shw_bit    hblk_misc.shadow_bit
1295 #define hblk_nuc_bit    hblk_misc.nucleus_bit
1296 #define hblk_ttesz      hblk_misc.ttesize
1297 #define hblk_hmecnt     hblk_un.hblk_counts.hblk_hmecount
1298 #define hblk_vcnt       hblk_un.hblk_counts.hblk_validcnt
1299 #define hblk_shw_mask   hblk_un.hblk_shadow_mask
1300 
1301 #define MAX_HBLK_LCKCNT 0xFFFFFFFF
1302 #define HMEBLK_ALIGN    0x8             /* hmeblk has to be double aligned */
1303 
1304 #ifdef  HBLK_TRACE
1305 
1306 #define HBLK_STACK_TRACE(hmeblkp, lock)                                 \
1307 {                                                                       \
1308         int flag = lock;        /* to pacify lint */                    \
1309         int audit_index;                                                \
1310                                                                         \
1311         mutex_enter(&hmeblkp->hblk_audit_lock);                          \
1312         audit_index = hmeblkp->hblk_audit_index;                     \
1313         hmeblkp->hblk_audit_index = ((hmeblkp->hblk_audit_index + 1) &        \
1314             (HBLK_AUDIT_CACHE_SIZE - 1));                               \
1315         mutex_exit(&hmeblkp->hblk_audit_lock);                           \
1316                                                                         \
1317         if (flag)                                                       \
1318                 hmeblkp->hblk_audit_cache[audit_index].flag =                \
1319                     HBLK_LOCK_PATTERN;                                  \
1320         else                                                            \
1321                 hmeblkp->hblk_audit_cache[audit_index].flag =                \
1322                     HBLK_UNLOCK_PATTERN;                                \
1323                                                                         \
1324         hmeblkp->hblk_audit_cache[audit_index].thread = curthread;   \
1325         hmeblkp->hblk_audit_cache[audit_index].depth =                       \
1326             getpcstack(hmeblkp->hblk_audit_cache[audit_index].stack, \
1327             HBLK_STACK_DEPTH);                                          \
1328 }
1329 
1330 #else
1331 
1332 #define HBLK_STACK_TRACE(hmeblkp, lock)
1333 
1334 #endif  /* HBLK_TRACE */
1335 
1336 #define HMEHASH_FACTOR  16      /* used to calc # of buckets in hme hash */
1337 
1338 /*
1339  * A maximum number of user hmeblks is defined in order to place an upper
1340  * limit on how much nucleus memory is required and to avoid overflowing the
1341  * tsbmiss uhashsz and khashsz data areas. The number below corresponds to
1342  * the number of buckets required, for an average hash chain length of 4 on
1343  * a 16TB machine.
1344  */
1345 
1346 #define MAX_UHME_BUCKETS        (0x1 << 30)
1347 #define MAX_KHME_BUCKETS        (0x1 << 30)
1348 
1349 /*
1350  * The minimum number of kernel hash buckets.
1351  */
1352 #define MIN_KHME_BUCKETS        0x800
1353 
1354 /*
1355  * The number of hash buckets must be a power of 2. If the initial calculated
1356  * value is less than USER_BUCKETS_THRESHOLD we round up to the next greater
1357  * power of 2, otherwise we round down to avoid huge over allocations.
1358  */
1359 #define USER_BUCKETS_THRESHOLD  (1<<22)
1360 
1361 #define MAX_NUCUHME_BUCKETS     0x4000
1362 #define MAX_NUCKHME_BUCKETS     0x2000
1363 
1364 /*
1365  * There are 2 locks in the hmehash bucket.  The hmehash_mutex is
1366  * a regular mutex used to make sure operations on a hash link are only
1367  * done by one thread.  Any operation which comes into the hat with
1368  * a <vaddr, as> will grab the hmehash_mutex.  Normally one would expect
1369  * the tsb miss handlers to grab the hash lock to make sure the hash list
1370  * is consistent while we traverse it.  Unfortunately this can lead to
1371  * deadlocks or recursive mutex enters since it is possible for
1372  * someone holding the lock to take a tlb/tsb miss.
1373  * To solve this problem we have added the hmehash_listlock.  This lock
1374  * is only grabbed by the tsb miss handlers, vatopfn, and while
1375  * adding/removing a hmeblk from the hash list. The code is written to
1376  * guarantee we won't take a tlb miss while holding this lock.
1377  */
1378 struct hmehash_bucket {
1379         kmutex_t        hmehash_mutex;
1380         volatile uint64_t hmeh_nextpa;  /* physical address for hash list */
1381         struct hme_blk *hmeblkp;
1382         uint_t          hmeh_listlock;
1383 };
1384 
1385 #endif /* !_ASM */
1386 
1387 #define SFMMU_PGCNT_MASK        0x3f
1388 #define SFMMU_PGCNT_SHIFT       6
1389 #define INVALID_MMU_ID          -1
1390 #define SFMMU_MMU_GNUM_RSHIFT   16
1391 #define SFMMU_MMU_CNUM_LSHIFT   (64 - SFMMU_MMU_GNUM_RSHIFT)
1392 #define MAX_SFMMU_CTX_VAL       ((1 << 16) - 1) /* for sanity check */
1393 #define MAX_SFMMU_GNUM_VAL      ((0x1UL << 48) - 1)
1394 
1395 /*
1396  * The tsb miss handlers written in assembly know that sfmmup
1397  * is a 64 bit ptr.
1398  *
1399  * The bspage and re-hash part is 64 bits, with the sfmmup being another 64
1400  * bits.
1401  */
1402 #define HTAG_SFMMUPSZ           0       /* Not really used for LP64 */
1403 #define HTAG_BSPAGE_SHIFT       13
1404 
1405 /*
1406  * Assembly routines need to be able to get to ttesz
1407  */
1408 #define HBLK_SZMASK             0x7
1409 
1410 #ifndef _ASM
1411 
1412 /*
1413  * Returns the number of bytes that an hmeblk spans given its tte size
1414  */
1415 #define get_hblk_span(hmeblkp) ((hmeblkp)->hblk_span)
1416 #define get_hblk_ttesz(hmeblkp) ((hmeblkp)->hblk_ttesz)
1417 #define get_hblk_cache(hmeblkp) (((hmeblkp)->hblk_ttesz == TTE8K) ? \
1418         sfmmu8_cache : sfmmu1_cache)
1419 #define HMEBLK_SPAN(ttesz)                                              \
1420         ((ttesz == TTE8K)? (TTEBYTES(ttesz) * NHMENTS) : TTEBYTES(ttesz))
1421 
1422 #define set_hblk_sz(hmeblkp, ttesz)                             \
1423         (hmeblkp)->hblk_ttesz = (ttesz);                     \
1424         (hmeblkp)->hblk_span = HMEBLK_SPAN(ttesz)
1425 
1426 #define get_hblk_base(hmeblkp)                                  \
1427         ((uintptr_t)(hmeblkp)->hblk_tag.htag_bspage << MMU_PAGESHIFT)
1428 
1429 #define get_hblk_endaddr(hmeblkp)                               \
1430         ((caddr_t)(get_hblk_base(hmeblkp) + get_hblk_span(hmeblkp)))
1431 
1432 #define in_hblk_range(hmeblkp, vaddr)                                   \
1433         (((uintptr_t)(vaddr) >= get_hblk_base(hmeblkp)) &&           \
1434         ((uintptr_t)(vaddr) < (get_hblk_base(hmeblkp) +                      \
1435         get_hblk_span(hmeblkp))))
1436 
1437 #define tte_to_vaddr(hmeblkp, tte)      ((caddr_t)(get_hblk_base(hmeblkp) \
1438         + (TTEBYTES(TTE_CSZ(&tte)) * (tte).tte_hmenum)))
1439 
1440 #define tte_to_evaddr(hmeblkp, ttep)    ((caddr_t)(get_hblk_base(hmeblkp) \
1441         + (TTEBYTES(TTE_CSZ(ttep)) * ((ttep)->tte_hmenum + 1))))
1442 
1443 #define vaddr_to_vshift(hblktag, vaddr, shwsz)                          \
1444         ((((uintptr_t)(vaddr) >> MMU_PAGESHIFT) - (hblktag.htag_bspage)) >>\
1445         TTE_BSZS_SHIFT((shwsz) - 1))
1446 
1447 #define HME8BLK_SZ      (sizeof (struct hme_blk) + \
1448                         (NHMENTS - 1) * sizeof (struct sf_hment))
1449 #define HME1BLK_SZ      (sizeof (struct hme_blk))
1450 #define H1MIN           (2 + MAX_BIGKTSB_TTES)  /* nucleus text+data, ktsb */
1451 
1452 /*
1453  * Hme_blk hash structure
1454  * Active mappings are kept in a hash structure of hme_blks.  The hash
1455  * function is based on (ctx, vaddr) The size of the hash table size is a
1456  * power of 2 such that the average hash chain lenth is HMENT_HASHAVELEN.
1457  * The hash actually consists of 2 separate hashes.  One hash is for the user
1458  * address space and the other hash is for the kernel address space.
1459  * The number of buckets are calculated at boot time and stored in the global
1460  * variables "uhmehash_num" and "khmehash_num".  By making the hash table size
1461  * a power of 2 we can use a simply & function to derive an index instead of
1462  * a divide.
1463  *
1464  * HME_HASH_FUNCTION(hatid, vaddr, shift) returns a pointer to a hme_hash
1465  * bucket.
1466  * An hme hash bucket contains a pointer to an hme_blk and the mutex that
1467  * protects the link list.
1468  * Spitfire supports 4 page sizes.  8k and 64K pages only need one hash.
1469  * 512K pages need 2 hashes and 4M pages need 3 hashes.
1470  * The 'shift' parameter controls how many bits the vaddr will be shifted in
1471  * the hash function. It is calculated in the HME_HASH_SHIFT(ttesz) function
1472  * and it varies depending on the page size as follows:
1473  *      8k pages:       HBLK_RANGE_SHIFT
1474  *      64k pages:      MMU_PAGESHIFT64K
1475  *      512K pages:     MMU_PAGESHIFT512K
1476  *      4M pages:       MMU_PAGESHIFT4M
1477  * An assembly version of the hash function exists in sfmmu_ktsb_miss(). All
1478  * changes should be reflected in both versions.  This function and the TSB
1479  * miss handlers are the only places which know about the two hashes.
1480  *
1481  * HBLK_RANGE_SHIFT controls range of virtual addresses that will fall
1482  * into the same bucket for a particular process.  It is currently set to
1483  * be equivalent to 64K range or one hme_blk.
1484  *
1485  * The hme_blks in the hash are protected by a per hash bucket mutex
1486  * known as SFMMU_HASH_LOCK.
1487  * You need to acquire this lock before traversing the hash bucket link
1488  * list, while adding/removing a hme_blk to the list, and while
1489  * modifying an hme_blk.  A possible optimization is to replace these
1490  * mutexes by readers/writer lock but right now it is not clear whether
1491  * this is a win or not.
1492  *
1493  * The HME_HASH_TABLE_SEARCH will search the hash table for the
1494  * hme_blk that contains the hment that corresponds to the passed
1495  * ctx and vaddr.  It assumed the SFMMU_HASH_LOCK is held.
1496  */
1497 
1498 #endif /* ! _ASM */
1499 
1500 #define KHATID                  ksfmmup
1501 #define UHMEHASH_SZ             uhmehash_num
1502 #define KHMEHASH_SZ             khmehash_num
1503 #define HMENT_HASHAVELEN        4
1504 #define HBLK_RANGE_SHIFT        MMU_PAGESHIFT64K /* shift for HBLK_BS_MASK */
1505 #define HBLK_MIN_TTESZ          1
1506 #define HBLK_MIN_BYTES          MMU_PAGESIZE64K
1507 #define HBLK_MIN_SHIFT          MMU_PAGESHIFT64K
1508 #define MAX_HASHCNT             5
1509 #define DEFAULT_MAX_HASHCNT     3
1510 
1511 #ifndef _ASM
1512 
1513 #define HASHADDR_MASK(hashno)   TTE_PAGEMASK(hashno)
1514 
1515 #define HME_HASH_SHIFT(ttesz)                                           \
1516         ((ttesz == TTE8K)? HBLK_RANGE_SHIFT : TTE_PAGE_SHIFT(ttesz))
1517 
1518 #define HME_HASH_ADDR(vaddr, hmeshift)                                  \
1519         ((caddr_t)(((uintptr_t)(vaddr) >> (hmeshift)) << (hmeshift)))
1520 
1521 #define HME_HASH_BSPAGE(vaddr, hmeshift)                                \
1522         (((uintptr_t)(vaddr) >> (hmeshift)) << ((hmeshift) - MMU_PAGESHIFT))
1523 
1524 #define HME_HASH_REHASH(ttesz)                                          \
1525         (((ttesz) < TTE512K)? 1 : (ttesz))
1526 
1527 #define HME_HASH_FUNCTION(hatid, vaddr, shift)                               \
1528         ((((void *)hatid) != ((void *)KHATID)) ?                             \
1529         (&uhme_hash[ (((uintptr_t)(hatid) ^ ((uintptr_t)vaddr >> (shift))) & \
1530             UHMEHASH_SZ) ]):                                                 \
1531         (&khme_hash[ (((uintptr_t)(hatid) ^ ((uintptr_t)vaddr >> (shift))) & \
1532             KHMEHASH_SZ) ]))
1533 
1534 /*
1535  * This macro will traverse a hmeblk hash link list looking for an hme_blk
1536  * that owns the specified vaddr and hatid.  If if doesn't find one , hmeblkp
1537  * will be set to NULL, otherwise it will point to the correct hme_blk.
1538  * This macro also cleans empty hblks.
1539  */
1540 #define HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, pr_hblk, listp)     \
1541 {                                                                       \
1542         struct hme_blk *nx_hblk;                                        \
1543                                                                         \
1544         ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));                          \
1545         hblkp = hmebp->hmeblkp;                                              \
1546         pr_hblk = NULL;                                                 \
1547         while (hblkp) {                                                 \
1548                 if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) {            \
1549                         /* found hme_blk */                             \
1550                         break;                                          \
1551                 }                                                       \
1552                 nx_hblk = hblkp->hblk_next;                          \
1553                 if (!hblkp->hblk_vcnt && !hblkp->hblk_hmecnt) {           \
1554                         sfmmu_hblk_hash_rm(hmebp, hblkp, pr_hblk,       \
1555                             listp, 0);                                  \
1556                 } else {                                                \
1557                         pr_hblk = hblkp;                                \
1558                 }                                                       \
1559                 hblkp = nx_hblk;                                        \
1560         }                                                               \
1561 }
1562 
1563 #define HME_HASH_SEARCH(hmebp, hblktag, hblkp, listp)                   \
1564 {                                                                       \
1565         struct hme_blk *pr_hblk;                                        \
1566                                                                         \
1567         HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp,  pr_hblk, listp);   \
1568 }
1569 
1570 /*
1571  * This macro will traverse a hmeblk hash link list looking for an hme_blk
1572  * that owns the specified vaddr and hatid.  If if doesn't find one , hmeblkp
1573  * will be set to NULL, otherwise it will point to the correct hme_blk.
1574  * It doesn't remove empty hblks.
1575  */
1576 #define HME_HASH_FAST_SEARCH(hmebp, hblktag, hblkp)                     \
1577         ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));                          \
1578         for (hblkp = hmebp->hmeblkp; hblkp;                          \
1579             hblkp = hblkp->hblk_next) {                                      \
1580                 if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) {            \
1581                         /* found hme_blk */                             \
1582                         break;                                          \
1583                 }                                                       \
1584         }
1585 
1586 #define SFMMU_HASH_LOCK(hmebp)                                          \
1587                 (mutex_enter(&hmebp->hmehash_mutex))
1588 
1589 #define SFMMU_HASH_UNLOCK(hmebp)                                        \
1590                 (mutex_exit(&hmebp->hmehash_mutex))
1591 
1592 #define SFMMU_HASH_LOCK_TRYENTER(hmebp)                                 \
1593                 (mutex_tryenter(&hmebp->hmehash_mutex))
1594 
1595 #define SFMMU_HASH_LOCK_ISHELD(hmebp)                                   \
1596                 (mutex_owned(&hmebp->hmehash_mutex))
1597 
1598 #define SFMMU_XCALL_STATS(sfmmup)                                       \
1599 {                                                                       \
1600         if (sfmmup == ksfmmup) {                                        \
1601                 SFMMU_STAT(sf_kernel_xcalls);                           \
1602         } else {                                                        \
1603                 SFMMU_STAT(sf_user_xcalls);                             \
1604         }                                                               \
1605 }
1606 
1607 #define astosfmmu(as)           ((as)->a_hat)
1608 #define hblktosfmmu(hmeblkp)    ((sfmmu_t *)(hmeblkp)->hblk_tag.htag_id)
1609 #define hblktosrd(hmeblkp)      ((sf_srd_t *)(hmeblkp)->hblk_tag.htag_id)
1610 #define sfmmutoas(sfmmup)       ((sfmmup)->sfmmu_as)
1611 
1612 #define sfmmutohtagid(sfmmup, rid)                         \
1613         (((rid) == SFMMU_INVALID_SHMERID) ? (void *)(sfmmup) : \
1614         (void *)((sfmmup)->sfmmu_srdp))
1615 
1616 /*
1617  * We use the sfmmu data structure to keep the per as page coloring info.
1618  */
1619 #define as_color_bin(as)        (astosfmmu(as)->sfmmu_clrbin)
1620 #define as_color_start(as)      (astosfmmu(as)->sfmmu_clrstart)
1621 
1622 typedef struct {
1623         char    h8[HME8BLK_SZ];
1624 } hblk8_t;
1625 
1626 typedef struct {
1627         char    h1[HME1BLK_SZ];
1628 } hblk1_t;
1629 
1630 typedef struct {
1631         ulong_t         index;
1632         ulong_t         len;
1633         hblk8_t         *list;
1634 } nucleus_hblk8_info_t;
1635 
1636 typedef struct {
1637         ulong_t         index;
1638         ulong_t         len;
1639         hblk1_t         *list;
1640 } nucleus_hblk1_info_t;
1641 
1642 /*
1643  * This struct is used for accumlating information about a range
1644  * of pages that are unloading so that a single xcall can flush
1645  * the entire range from remote tlbs. A function that must demap
1646  * a range of virtual addresses declares one of these structures
1647  * and initializes using DEMP_RANGE_INIT(). It then passes a pointer to this
1648  * struct to the appropriate sfmmu_hblk_* level function which does
1649  * all the bookkeeping using the other macros. When the function has
1650  * finished the virtual address range, it needs to call DEMAP_RANGE_FLUSH()
1651  * macro to take care of any remaining unflushed mappings.
1652  *
1653  * The maximum range this struct can represent is the number of bits
1654  * in the dmr_bitvec field times the pagesize in dmr_pgsz. Currently, only
1655  * MMU_PAGESIZE pages are supported.
1656  *
1657  * Since there are now cases where it's no longer necessary to do
1658  * flushes (e.g. when the process isn't runnable because it's swapping
1659  * out or exiting) we allow these macros to take a NULL dmr input and do
1660  * nothing in that case.
1661  */
1662 typedef struct {
1663         sfmmu_t         *dmr_sfmmup;    /* relevant hat */
1664         caddr_t         dmr_addr;       /* beginning address */
1665         caddr_t         dmr_endaddr;    /* ending  address */
1666         ulong_t         dmr_bitvec;     /* valid pages found */
1667         ulong_t         dmr_bit;        /* next page to examine */
1668         ulong_t         dmr_maxbit;     /* highest page in range */
1669         ulong_t         dmr_pgsz;       /* page size in range */
1670 } demap_range_t;
1671 
1672 #define DMR_MAXBIT ((ulong_t)1<<63) /* dmr_bit high bit */
1673 
1674 #define DEMAP_RANGE_INIT(sfmmup, dmrp) \
1675         (dmrp)->dmr_sfmmup = (sfmmup); \
1676         (dmrp)->dmr_bitvec = 0; \
1677         (dmrp)->dmr_maxbit = sfmmu_dmr_maxbit; \
1678         (dmrp)->dmr_pgsz = MMU_PAGESIZE;
1679 
1680 #define DEMAP_RANGE_PGSZ(dmrp) ((dmrp)? (dmrp)->dmr_pgsz : MMU_PAGESIZE)
1681 
1682 #define DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr) \
1683         if ((dmrp) != NULL) { \
1684         if ((dmrp)->dmr_bitvec != 0 && (dmrp)->dmr_endaddr != (addr)) \
1685                 sfmmu_tlb_range_demap(dmrp); \
1686         (dmrp)->dmr_endaddr = (endaddr); \
1687         }
1688 
1689 #define DEMAP_RANGE_FLUSH(dmrp) \
1690         if ((dmrp)->dmr_bitvec != 0)                 \
1691                 sfmmu_tlb_range_demap(dmrp);
1692 
1693 
1694 #define DEMAP_RANGE_MARKPG(dmrp, addr) \
1695         if ((dmrp) != NULL) { \
1696                 if ((dmrp)->dmr_bitvec == 0) { \
1697                         (dmrp)->dmr_addr = (addr); \
1698                         (dmrp)->dmr_bit = 1; \
1699                 } \
1700                 (dmrp)->dmr_bitvec |= (dmrp)->dmr_bit; \
1701         }
1702 
1703 #define DEMAP_RANGE_NEXTPG(dmrp) \
1704         if ((dmrp) != NULL && (dmrp)->dmr_bitvec != 0) { \
1705                 if ((dmrp)->dmr_bit & (dmrp)->dmr_maxbit) { \
1706                         sfmmu_tlb_range_demap(dmrp); \
1707                 } else { \
1708                         (dmrp)->dmr_bit <<= 1; \
1709                 } \
1710         }
1711 
1712 /*
1713  * TSB related structures
1714  *
1715  * The TSB is made up of tte entries.  Both the tag and data are present
1716  * in the TSB.  The TSB locking is managed as follows:
1717  * A software bit in the tsb tag is used to indicate that entry is locked.
1718  * If a cpu servicing a tsb miss reads a locked entry the tag compare will
1719  * fail forcing the cpu to go to the hat hash for the translation.
1720  * The cpu who holds the lock can then modify the data side, and the tag side.
1721  * The last write should be to the word containing the lock bit which will
1722  * clear the lock and allow the tsb entry to be read.  It is assumed that all
1723  * cpus reading the tsb will do so with atomic 128-bit loads.  An atomic 128
1724  * bit load is required to prevent the following from happening:
1725  *
1726  * cpu 0                        cpu 1                   comments
1727  *
1728  * ldx tag                                              tag unlocked
1729  *                              ldstub lock             set lock
1730  *                              stx data
1731  *                              stx tag                 unlock
1732  * ldx tag                                              incorrect tte!!!
1733  *
1734  * The software also maintains a bit in the tag to indicate an invalid
1735  * tsb entry.  The purpose of this bit is to allow the tsb invalidate code
1736  * to invalidate a tsb entry with a single cas.  See code for details.
1737  */
1738 
1739 union tsb_tag {
1740         struct {
1741                 uint32_t        tag_res0:16;    /* reserved - context area */
1742                 uint32_t        tag_inv:1;      /* sw - invalid tsb entry */
1743                 uint32_t        tag_lock:1;     /* sw - locked tsb entry */
1744                 uint32_t        tag_res1:4;     /* reserved */
1745                 uint32_t        tag_va_hi:10;   /* va[63:54] */
1746                 uint32_t        tag_va_lo;      /* va[53:22] */
1747         } tagbits;
1748         struct tsb_tagints {
1749                 uint32_t        inthi;
1750                 uint32_t        intlo;
1751         } tagints;
1752 };
1753 #define tag_invalid             tagbits.tag_inv
1754 #define tag_locked              tagbits.tag_lock
1755 #define tag_vahi                tagbits.tag_va_hi
1756 #define tag_valo                tagbits.tag_va_lo
1757 #define tag_inthi               tagints.inthi
1758 #define tag_intlo               tagints.intlo
1759 
1760 struct tsbe {
1761         union tsb_tag   tte_tag;
1762         tte_t           tte_data;
1763 };
1764 
1765 /*
1766  * A per cpu struct is kept that duplicates some info
1767  * used by the tl>0 tsb miss handlers plus it provides
1768  * a scratch area.  Its purpose is to minimize cache misses
1769  * in the tsb miss handler and is 128 bytes (2 e$ lines).
1770  *
1771  * There should be one allocated per cpu in nucleus memory
1772  * and should be aligned on an ecache line boundary.
1773  */
1774 struct tsbmiss {
1775         sfmmu_t                 *ksfmmup;       /* kernel hat id */
1776         sfmmu_t                 *usfmmup;       /* user hat id */
1777         sf_srd_t                *usrdp;         /* user's SRD hat id */
1778         struct tsbe             *tsbptr;        /* hardware computed ptr */
1779         struct tsbe             *tsbptr4m;      /* hardware computed ptr */
1780         struct tsbe             *tsbscdptr;     /* hardware computed ptr */
1781         struct tsbe             *tsbscdptr4m;   /* hardware computed ptr */
1782         uint64_t                ismblkpa;
1783         struct hmehash_bucket   *khashstart;
1784         struct hmehash_bucket   *uhashstart;
1785         uint_t                  khashsz;
1786         uint_t                  uhashsz;
1787         uint16_t                dcache_line_mask; /* used to flush dcache */
1788         uchar_t                 uhat_tteflags;  /* private page sizes */
1789         uchar_t                 uhat_rtteflags; /* SHME pagesizes */
1790         uint32_t                utsb_misses;
1791         uint32_t                ktsb_misses;
1792         uint16_t                uprot_traps;
1793         uint16_t                kprot_traps;
1794         /*
1795          * scratch[0] -> TSB_TAGACC
1796          * scratch[1] -> TSBMISS_HMEBP
1797          * scratch[2] -> TSBMISS_HATID
1798          */
1799         uintptr_t               scratch[3];
1800         ulong_t         shmermap[SFMMU_HMERGNMAP_WORDS];        /* 8 bytes */
1801         ulong_t         scd_shmermap[SFMMU_HMERGNMAP_WORDS];    /* 8 bytes */
1802         uint8_t         pad[48];                        /* pad to 64 bytes */
1803 };
1804 
1805 /*
1806  * A per cpu struct is kept for the use within the tl>0 kpm tsb
1807  * miss handler. Some members are duplicates of common data or
1808  * the physical addresses of common data. A few members are also
1809  * written by the tl>0 kpm tsb miss handler. Its purpose is to
1810  * minimize cache misses in the kpm tsb miss handler and occupies
1811  * one ecache line. There should be one allocated per cpu in
1812  * nucleus memory and it should be aligned on an ecache line
1813  * boundary. It is not merged w/ struct tsbmiss since there is
1814  * not much to share and the tsbmiss pathes are different, so
1815  * a kpm tlbmiss/tsbmiss only touches one cacheline, except for
1816  * (DEBUG || SFMMU_STAT_GATHER) where the dtlb_misses counter
1817  * of struct tsbmiss is used on every dtlb miss.
1818  */
1819 struct kpmtsbm {
1820         caddr_t         vbase;          /* start of address kpm range */
1821         caddr_t         vend;           /* end of address kpm range */
1822         uchar_t         flags;          /* flags needed in TL tsbmiss handler */
1823         uchar_t         sz_shift;       /* for single kpm window */
1824         uchar_t         kpmp_shift;     /* hash lock shift */
1825         uchar_t         kpmp2pshft;     /* kpm page to page shift */
1826         uint_t          kpmp_table_sz;  /* size of kpmp_table or kpmp_stable */
1827         uint64_t        kpmp_tablepa;   /* paddr of kpmp_table or kpmp_stable */
1828         uint64_t        msegphashpa;    /* paddr of memseg_phash */
1829         struct tsbe     *tsbptr;        /* saved ktsb pointer */
1830         uint_t          kpm_dtlb_misses; /* kpm tlbmiss counter */
1831         uint_t          kpm_tsb_misses; /* kpm tsbmiss counter */
1832         uintptr_t       pad[1];
1833 };
1834 
1835 extern size_t   tsb_slab_size;
1836 extern uint_t   tsb_slab_shift;
1837 extern size_t   tsb_slab_mask;
1838 
1839 #endif /* !_ASM */
1840 
1841 /*
1842  * Flags for TL kpm tsbmiss handler
1843  */
1844 #define KPMTSBM_ENABLE_FLAG     0x01    /* bit copy of kpm_enable */
1845 #define KPMTSBM_TLTSBM_FLAG     0x02    /* use TL tsbmiss handler */
1846 #define KPMTSBM_TSBPHYS_FLAG    0x04    /* use ASI_MEM for TSB update */
1847 
1848 /*
1849  * The TSB
1850  * All TSB sizes supported by the hardware are now supported (8K - 1M).
1851  * For kernel TSBs we may go beyond the hardware supported sizes and support
1852  * larger TSBs via software.
1853  * All TTE sizes are supported in the TSB; the manner in which this is
1854  * done is cpu dependent.
1855  */
1856 #define TSB_MIN_SZCODE          TSB_8K_SZCODE   /* min. supported TSB size */
1857 #define TSB_MIN_OFFSET_MASK     (TSB_OFFSET_MASK(TSB_MIN_SZCODE))
1858 
1859 #ifdef sun4v
1860 #define UTSB_MAX_SZCODE         TSB_256M_SZCODE /* max. supported TSB size */
1861 #else /* sun4u */
1862 #define UTSB_MAX_SZCODE         TSB_1M_SZCODE   /* max. supported TSB size */
1863 #endif /* sun4v */
1864 
1865 #define UTSB_MAX_OFFSET_MASK    (TSB_OFFSET_MASK(UTSB_MAX_SZCODE))
1866 
1867 #define TSB_FREEMEM_MIN         0x1000          /* 32 mb */
1868 #define TSB_FREEMEM_LARGE       0x10000         /* 512 mb */
1869 #define TSB_8K_SZCODE           0               /* 512 entries */
1870 #define TSB_16K_SZCODE          1               /* 1k entries */
1871 #define TSB_32K_SZCODE          2               /* 2k entries */
1872 #define TSB_64K_SZCODE          3               /* 4k entries */
1873 #define TSB_128K_SZCODE         4               /* 8k entries */
1874 #define TSB_256K_SZCODE         5               /* 16k entries */
1875 #define TSB_512K_SZCODE         6               /* 32k entries */
1876 #define TSB_1M_SZCODE           7               /* 64k entries */
1877 #define TSB_2M_SZCODE           8               /* 128k entries */
1878 #define TSB_4M_SZCODE           9               /* 256k entries */
1879 #define TSB_8M_SZCODE           10              /* 512k entries */
1880 #define TSB_16M_SZCODE          11              /* 1M entries */
1881 #define TSB_32M_SZCODE          12              /* 2M entries */
1882 #define TSB_64M_SZCODE          13              /* 4M entries */
1883 #define TSB_128M_SZCODE         14              /* 8M entries */
1884 #define TSB_256M_SZCODE         15              /* 16M entries */
1885 #define TSB_ENTRY_SHIFT         4       /* each entry = 128 bits = 16 bytes */
1886 #define TSB_ENTRY_SIZE          (1 << 4)
1887 #define TSB_START_SIZE          9
1888 #define TSB_ENTRIES(tsbsz)      (1 << (TSB_START_SIZE + tsbsz))
1889 #define TSB_BYTES(tsbsz)        (TSB_ENTRIES(tsbsz) << TSB_ENTRY_SHIFT)
1890 #define TSB_OFFSET_MASK(tsbsz)  (TSB_ENTRIES(tsbsz) - 1)
1891 #define TSB_BASEADDR_MASK       ((1 << 12) - 1)
1892 
1893 /*
1894  * sun4u platforms
1895  * ---------------
1896  * We now support two user TSBs with one TSB base register.
1897  * Hence the TSB base register is split up as follows:
1898  *
1899  * When only one TSB present:
1900  *   [63  62..42  41..13  12..4  3..0]
1901  *     ^   ^       ^       ^     ^
1902  *     |   |       |       |     |
1903  *     |   |       |       |     |_ TSB size code
1904  *     |   |       |       |
1905  *     |   |       |       |_ Reserved 0
1906  *     |   |       |
1907  *     |   |       |_ TSB VA[41..13]
1908  *     |   |
1909  *     |   |_ VA hole (Spitfire), zeros (Cheetah and beyond)
1910  *     |
1911  *     |_ 0
1912  *
1913  * When second TSB present:
1914  *   [63  62..42  41..33  32..29  28..22  21..13  12..4  3..0]
1915  *     ^   ^       ^       ^       ^       ^       ^     ^
1916  *     |   |       |       |       |       |       |     |
1917  *     |   |       |       |       |       |       |     |_ First TSB size code
1918  *     |   |       |       |       |       |       |
1919  *     |   |       |       |       |       |       |_ Reserved 0
1920  *     |   |       |       |       |       |
1921  *     |   |       |       |       |       |_ First TSB's VA[21..13]
1922  *     |   |       |       |       |
1923  *     |   |       |       |       |_ Reserved for future use
1924  *     |   |       |       |
1925  *     |   |       |       |_ Second TSB's size code
1926  *     |   |       |
1927  *     |   |       |_ Second TSB's VA[21..13]
1928  *     |   |
1929  *     |   |_ VA hole (Spitfire) / ones (Cheetah and beyond)
1930  *     |
1931  *     |_ 1
1932  *
1933  * Note that since we store 21..13 of each TSB's VA, TSBs and their slabs
1934  * may be up to 4M in size.  For now, only hardware supported TSB sizes
1935  * are supported, though the slabs are usually 4M in size.
1936  *
1937  * sun4u platforms that define UTSB_PHYS use physical addressing to access
1938  * the user TSBs at TL>0.  The first user TSB base is in the MMU I/D TSB Base
1939  * registers.  The second TSB base uses a dedicated scratchpad register which
1940  * requires a definition of SCRATCHPAD_UTSBREG2 in mach_sfmmu.h.  The layout for
1941  * both registers is equivalent to sun4v below, except the TSB PA range is
1942  * [46..13] for sun4u.
1943  *
1944  * sun4v platforms
1945  * ---------------
1946  * On sun4v platforms, we use two dedicated scratchpad registers as pseudo
1947  * hardware TSB base registers to hold up to two different user TSBs.
1948  *
1949  * Each register contains TSB's physical base and size code information
1950  * as follows:
1951  *
1952  *   [63..56  55..13  12..4  3..0]
1953  *      ^       ^       ^     ^
1954  *      |       |       |     |
1955  *      |       |       |     |_ TSB size code
1956  *      |       |       |
1957  *      |       |       |_ Reserved 0
1958  *      |       |
1959  *      |       |_ TSB PA[55..13]
1960  *      |
1961  *      |
1962  *      |
1963  *      |_ 0 for valid TSB
1964  *
1965  * Absence of a user TSB (primarily the second user TSB) is indicated by
1966  * storing a negative value in the TSB base register. This allows us to
1967  * check for presence of a user TSB by simply checking bit# 63.
1968  */
1969 #define TSBREG_MSB_SHIFT        32              /* set upper bits */
1970 #define TSBREG_MSB_CONST        0xfffff800      /* set bits 63..43 */
1971 #define TSBREG_FIRTSB_SHIFT     42              /* to clear bits 63:22 */
1972 #define TSBREG_SECTSB_MKSHIFT   20              /* 21:13 --> 41:33 */
1973 #define TSBREG_SECTSB_LSHIFT    22              /* to clear bits 63:42 */
1974 #define TSBREG_SECTSB_RSHIFT    (TSBREG_SECTSB_MKSHIFT + TSBREG_SECTSB_LSHIFT)
1975                                                 /* sectsb va -> bits 21:13 */
1976                                                 /* after clearing upper bits */
1977 #define TSBREG_SECSZ_SHIFT      29              /* to get sectsb szc to 3:0 */
1978 #define TSBREG_VAMASK_SHIFT     13              /* set up VA mask */
1979 
1980 #define BIGKTSB_SZ_MASK         0xf
1981 #define TSB_SOFTSZ_MASK         BIGKTSB_SZ_MASK
1982 #define MIN_BIGKTSB_SZCODE      9       /* 256k entries */
1983 #define MAX_BIGKTSB_SZCODE      11      /* 1024k entries */
1984 #define MAX_BIGKTSB_TTES        (TSB_BYTES(MAX_BIGKTSB_SZCODE) / MMU_PAGESIZE4M)
1985 
1986 #define TAG_VALO_SHIFT          22              /* tag's va are bits 63-22 */
1987 /*
1988  * sw bits used on tsb_tag - bit masks used only in assembly
1989  * use only a sethi for these fields.
1990  */
1991 #define TSBTAG_INVALID  0x00008000              /* tsb_tag.tag_invalid */
1992 #define TSBTAG_LOCKED   0x00004000              /* tsb_tag.tag_locked */
1993 
1994 #ifdef  _ASM
1995 
1996 /*
1997  * Marker to indicate that this instruction will be hot patched at runtime
1998  * to some other value.
1999  * This value must be zero since it fills in the imm bits of the target
2000  * instructions to be patched
2001  */
2002 #define RUNTIME_PATCH   (0)
2003 
2004 /*
2005  * V9 defines nop instruction as the following, which we use
2006  * at runtime to nullify some instructions we don't want to
2007  * execute in the trap handlers on certain platforms.
2008  */
2009 #define MAKE_NOP_INSTR(reg)     \
2010         sethi   %hi(0x1000000), reg
2011 
2012 /*
2013  * This macro constructs a SPARC V9 "jmpl <source reg>, %g0"
2014  * instruction, with the source register specified by the jump_reg_number.
2015  * The jmp opcode [24:19] = 11 1000 and source register is bits [18:14].
2016  * The instruction is returned in reg. The macro is used to patch in a jmpl
2017  * instruction at runtime.
2018  */
2019 #define MAKE_JMP_INSTR(jump_reg_number, reg, tmp)       \
2020         sethi   %hi(0x81c00000), reg;                   \
2021         mov     jump_reg_number, tmp;                   \
2022         sll     tmp, 14, tmp;                           \
2023         or      reg, tmp, reg
2024 
2025 /*
2026  * Macro to get hat per-MMU cnum on this CPU.
2027  * sfmmu - In, pass in "sfmmup" from the caller.
2028  * cnum - Out, return 'cnum' to the caller
2029  * scr  - scratch
2030  */
2031 #define SFMMU_CPU_CNUM(sfmmu, cnum, scr)                                      \
2032         CPU_ADDR(scr, cnum);    /* scr = load CPU struct addr */              \
2033         ld      [scr + CPU_MMU_IDX], cnum;      /* cnum = mmuid */            \
2034         add     sfmmu, SFMMU_CTXS, scr; /* scr = sfmmup->sfmmu_ctxs[] */      \
2035         sllx    cnum, SFMMU_MMU_CTX_SHIFT, cnum;                              \
2036         add     scr, cnum, scr;         /* scr = sfmmup->sfmmu_ctxs[id] */    \
2037         ldx     [scr + SFMMU_MMU_GC_NUM], scr;  /* sfmmu_ctxs[id].gcnum */    \
2038         sllx    scr, SFMMU_MMU_CNUM_LSHIFT, scr;                              \
2039         srlx    scr, SFMMU_MMU_CNUM_LSHIFT, cnum;       /* cnum = sfmmu cnum */
2040 
2041 /*
2042  * Macro to get hat gnum & cnum assocaited with sfmmu_ctx[mmuid] entry
2043  * entry - In,  pass in (&sfmmu_ctxs[mmuid] - SFMMU_CTXS) from the caller.
2044  * gnum - Out, return sfmmu gnum
2045  * cnum - Out, return sfmmu cnum
2046  * reg  - scratch
2047  */
2048 #define SFMMU_MMUID_GNUM_CNUM(entry, gnum, cnum, reg)                        \
2049         ldx     [entry + SFMMU_CTXS], reg;  /* reg = sfmmu (gnum | cnum) */  \
2050         srlx    reg, SFMMU_MMU_GNUM_RSHIFT, gnum;    /* gnum = sfmmu gnum */ \
2051         sllx    reg, SFMMU_MMU_CNUM_LSHIFT, cnum;                            \
2052         srlx    cnum, SFMMU_MMU_CNUM_LSHIFT, cnum;   /* cnum = sfmmu cnum */
2053 
2054 /*
2055  * Macro to get this CPU's tsbmiss area.
2056  */
2057 #define CPU_TSBMISS_AREA(tsbmiss, tmp1)                                 \
2058         CPU_INDEX(tmp1, tsbmiss);               /* tmp1 = cpu idx */    \
2059         sethi   %hi(tsbmiss_area), tsbmiss;     /* tsbmiss base ptr */  \
2060         mulx    tmp1, TSBMISS_SIZE, tmp1;       /* byte offset */       \
2061         or      tsbmiss, %lo(tsbmiss_area), tsbmiss;                    \
2062         add     tsbmiss, tmp1, tsbmiss          /* tsbmiss area of CPU */
2063 
2064 
2065 /*
2066  * Macro to set kernel context + page size codes in DMMU primary context
2067  * register. It is only necessary for sun4u because sun4v does not need
2068  * page size codes
2069  */
2070 #ifdef sun4v
2071 
2072 #define SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3)
2073 
2074 #else
2075 
2076 #define SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3) \
2077         sethi   %hi(kcontextreg), reg0;                                 \
2078         ldx     [reg0 + %lo(kcontextreg)], reg0;                        \
2079         mov     MMU_PCONTEXT, reg1;                                     \
2080         ldxa    [reg1]ASI_MMU_CTX, reg2;                                \
2081         xor     reg0, reg2, reg2;                                       \
2082         brz     reg2, label3;                                           \
2083         srlx    reg2, CTXREG_NEXT_SHIFT, reg2;                          \
2084         rdpr    %pstate, reg3;          /* disable interrupts */        \
2085         btst    PSTATE_IE, reg3;                                        \
2086 /*CSTYLED*/                                                             \
2087         bnz,a,pt %icc, label1;                                          \
2088         wrpr    reg3, PSTATE_IE, %pstate;                               \
2089 /*CSTYLED*/                                                             \
2090 label1:;                                                                \
2091         brz     reg2, label2;      /* need demap if N_pgsz0/1 change */ \
2092         sethi   %hi(FLUSH_ADDR), reg4;                                  \
2093         mov     DEMAP_ALL_TYPE, reg2;                                   \
2094         stxa    %g0, [reg2]ASI_DTLB_DEMAP;                              \
2095         stxa    %g0, [reg2]ASI_ITLB_DEMAP;                              \
2096 /*CSTYLED*/                                                             \
2097 label2:;                                                                \
2098         stxa    reg0, [reg1]ASI_MMU_CTX;                                \
2099         flush   reg4;                                                   \
2100         btst    PSTATE_IE, reg3;                                        \
2101 /*CSTYLED*/                                                             \
2102         bnz,a,pt %icc, label3;                                          \
2103         wrpr    %g0, reg3, %pstate;     /* restore interrupt state */   \
2104 label3:;
2105 
2106 #endif
2107 
2108 /*
2109  * Macro to setup arguments with kernel sfmmup context + page size before
2110  * calling sfmmu_setctx_sec()
2111  */
2112 #ifdef sun4v
2113 #define SET_KAS_CTXSEC_ARGS(sfmmup, arg0, arg1)                 \
2114         set     KCONTEXT, arg0;                                 \
2115         set     0, arg1;
2116 #else
2117 #define SET_KAS_CTXSEC_ARGS(sfmmup, arg0, arg1)                 \
2118         ldub    [sfmmup + SFMMU_CEXT], arg1;                    \
2119         set     KCONTEXT, arg0;                                 \
2120         sll     arg1, CTXREG_EXT_SHIFT, arg1;
2121 #endif
2122 
2123 #define PANIC_IF_INTR_DISABLED_PSTR(pstatereg, label, scr)              \
2124         andcc   pstatereg, PSTATE_IE, %g0;      /* panic if intrs */    \
2125 /*CSTYLED*/                                                             \
2126         bnz,pt  %icc, label;                    /* already disabled */  \
2127         nop;                                                            \
2128                                                                         \
2129         sethi   %hi(panicstr), scr;                                     \
2130         ldx     [scr + %lo(panicstr)], scr;                             \
2131         tst     scr;                                                    \
2132 /*CSTYLED*/                                                             \
2133         bnz,pt  %xcc, label;                                            \
2134         nop;                                                            \
2135                                                                         \
2136         save    %sp, -SA(MINFRAME), %sp;                                \
2137         sethi   %hi(sfmmu_panic1), %o0;                                 \
2138         call    panic;                                                  \
2139         or      %o0, %lo(sfmmu_panic1), %o0;                            \
2140 /*CSTYLED*/                                                             \
2141 label:
2142 
2143 #define PANIC_IF_INTR_ENABLED_PSTR(label, scr)                          \
2144         /*                                                              \
2145          * The caller must have disabled interrupts.                    \
2146          * If interrupts are not disabled, panic                        \
2147          */                                                             \
2148         rdpr    %pstate, scr;                                           \
2149         andcc   scr, PSTATE_IE, %g0;                                    \
2150 /*CSTYLED*/                                                             \
2151         bz,pt   %icc, label;                                            \
2152         nop;                                                            \
2153                                                                         \
2154         sethi   %hi(panicstr), scr;                                     \
2155         ldx     [scr + %lo(panicstr)], scr;                             \
2156         tst     scr;                                                    \
2157 /*CSTYLED*/                                                             \
2158         bnz,pt  %xcc, label;                                            \
2159         nop;                                                            \
2160                                                                         \
2161         sethi   %hi(sfmmu_panic6), %o0;                                 \
2162         call    panic;                                                  \
2163         or      %o0, %lo(sfmmu_panic6), %o0;                            \
2164 /*CSTYLED*/                                                             \
2165 label:
2166 
2167 #endif  /* _ASM */
2168 
2169 #ifndef _ASM
2170 
2171 #ifdef VAC
2172 /*
2173  * Page coloring
2174  * The p_vcolor field of the page struct (1 byte) is used to store the
2175  * virtual page color.  This provides for 255 colors.  The value zero is
2176  * used to mean the page has no color - never been mapped or somehow
2177  * purified.
2178  */
2179 
2180 #define PP_GET_VCOLOR(pp)       (((pp)->p_vcolor) - 1)
2181 #define PP_NEWPAGE(pp)          (!(pp)->p_vcolor)
2182 #define PP_SET_VCOLOR(pp, color)                                          \
2183         ((pp)->p_vcolor = ((color) + 1))
2184 
2185 /*
2186  * As mentioned p_vcolor == 0 means there is no color for this page.
2187  * But PP_SET_VCOLOR(pp, color) expects 'color' to be real color minus
2188  * one so we define this constant.
2189  */
2190 #define NO_VCOLOR       (-1)
2191 
2192 #define addr_to_vcolor(addr) \
2193         (((uint_t)(uintptr_t)(addr) >> MMU_PAGESHIFT) & vac_colors_mask)
2194 #else   /* VAC */
2195 #define addr_to_vcolor(addr)    (0)
2196 #endif  /* VAC */
2197 
2198 /*
2199  * The field p_index in the psm page structure is for large pages support.
2200  * P_index is a bit-vector of the different mapping sizes that a given page
2201  * is part of. An hme structure for a large mapping is only added in the
2202  * group leader page (first page). All pages covered by a given large mapping
2203  * have the corrosponding mapping bit set in their p_index field. This allows
2204  * us to only store an explicit hme structure in the leading page which
2205  * simplifies the mapping link list management. Furthermore, it provides us
2206  * a fast mechanism for determining the largest mapping a page is part of. For
2207  * exmaple, a page with a 64K and a 4M mappings has a p_index value of 0x0A.
2208  *
2209  * Implementation note: even though the first bit in p_index is reserved
2210  * for 8K mappings, it is NOT USED by the code and SHOULD NOT be set.
2211  * In addition, the upper four bits of the p_index field are used by the
2212  * code as temporaries
2213  */
2214 
2215 /*
2216  * Defines for psm page struct fields and large page support
2217  */
2218 #define SFMMU_INDEX_SHIFT               6
2219 #define SFMMU_INDEX_MASK                ((1 << SFMMU_INDEX_SHIFT) - 1)
2220 
2221 /* Return the mapping index */
2222 #define PP_MAPINDEX(pp) ((pp)->p_index & SFMMU_INDEX_MASK)
2223 
2224 /*
2225  * These macros rely on the following property:
2226  * All pages constituting a large page are covered by a virtually
2227  * contiguous set of page_t's.
2228  */
2229 
2230 /* Return the leader for this mapping size */
2231 #define PP_GROUPLEADER(pp, sz) \
2232         (&(pp)[-(int)(pp->p_pagenum & (TTEPAGES(sz)-1))])
2233 
2234 /* Return the root page for this page based on p_szc */
2235 #define PP_PAGEROOT(pp) ((pp)->p_szc == 0 ? (pp) : \
2236         PP_GROUPLEADER((pp), (pp)->p_szc))
2237 
2238 #define PP_PAGENEXT_N(pp, n)    ((pp) + (n))
2239 #define PP_PAGENEXT(pp)         PP_PAGENEXT_N((pp), 1)
2240 
2241 #define PP_PAGEPREV_N(pp, n)    ((pp) - (n))
2242 #define PP_PAGEPREV(pp)         PP_PAGEPREV_N((pp), 1)
2243 
2244 #define PP_ISMAPPED_LARGE(pp)   (PP_MAPINDEX(pp) != 0)
2245 
2246 /* Need function to test the page mappping which takes p_index into account */
2247 #define PP_ISMAPPED(pp) ((pp)->p_mapping || PP_ISMAPPED_LARGE(pp))
2248 
2249 /*
2250  * Don't call this macro with sz equal to zero. 8K mappings SHOULD NOT
2251  * set p_index field.
2252  */
2253 #define PAGESZ_TO_INDEX(sz)     (1 << (sz))
2254 
2255 
2256 /*
2257  * prototypes for hat assembly routines.  Some of these are
2258  * known to machine dependent VM code.
2259  */
2260 extern uint64_t sfmmu_make_tsbtag(caddr_t);
2261 extern struct tsbe *
2262                 sfmmu_get_tsbe(uint64_t, caddr_t, int, int);
2263 extern void     sfmmu_load_tsbe(struct tsbe *, uint64_t, tte_t *, int);
2264 extern void     sfmmu_unload_tsbe(struct tsbe *, uint64_t, int);
2265 extern void     sfmmu_load_mmustate(sfmmu_t *);
2266 extern void     sfmmu_raise_tsb_exception(uint64_t, uint64_t);
2267 #ifndef sun4v
2268 extern void     sfmmu_itlb_ld_kva(caddr_t, tte_t *);
2269 extern void     sfmmu_dtlb_ld_kva(caddr_t, tte_t *);
2270 #endif /* sun4v */
2271 extern void     sfmmu_copytte(tte_t *, tte_t *);
2272 extern int      sfmmu_modifytte(tte_t *, tte_t *, tte_t *);
2273 extern int      sfmmu_modifytte_try(tte_t *, tte_t *, tte_t *);
2274 extern pfn_t    sfmmu_ttetopfn(tte_t *, caddr_t);
2275 extern uint_t   sfmmu_disable_intrs(void);
2276 extern void     sfmmu_enable_intrs(uint_t);
2277 /*
2278  * functions exported to machine dependent VM code
2279  */
2280 extern void     sfmmu_patch_ktsb(void);
2281 #ifndef UTSB_PHYS
2282 extern void     sfmmu_patch_utsb(void);
2283 #endif /* UTSB_PHYS */
2284 extern pfn_t    sfmmu_vatopfn(caddr_t, sfmmu_t *, tte_t *);
2285 extern void     sfmmu_vatopfn_suspended(caddr_t, sfmmu_t *, tte_t *);
2286 extern pfn_t    sfmmu_kvaszc2pfn(caddr_t, int);
2287 #ifdef  DEBUG
2288 extern void     sfmmu_check_kpfn(pfn_t);
2289 #else
2290 #define         sfmmu_check_kpfn(pfn)   /* disabled */
2291 #endif  /* DEBUG */
2292 extern void     sfmmu_memtte(tte_t *, pfn_t, uint_t, int);
2293 extern void     sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *, uint_t);
2294 extern void     sfmmu_tsbmiss_exception(struct regs *, uintptr_t, uint_t);
2295 extern void     sfmmu_init_tsbs(void);
2296 extern caddr_t  sfmmu_ktsb_alloc(caddr_t);
2297 extern int      sfmmu_getctx_pri(void);
2298 extern int      sfmmu_getctx_sec(void);
2299 extern void     sfmmu_setctx_sec(uint_t);
2300 extern void     sfmmu_inv_tsb(caddr_t, uint_t);
2301 extern void     sfmmu_init_ktsbinfo(void);
2302 extern int      sfmmu_setup_4lp(void);
2303 extern void     sfmmu_patch_mmu_asi(int);
2304 extern void     sfmmu_init_nucleus_hblks(caddr_t, size_t, int, int);
2305 extern void     sfmmu_cache_flushall(void);
2306 extern pgcnt_t  sfmmu_tte_cnt(sfmmu_t *, uint_t);
2307 extern void     *sfmmu_tsb_segkmem_alloc(vmem_t *, size_t, int);
2308 extern void     sfmmu_tsb_segkmem_free(vmem_t *, void *, size_t);
2309 extern void     sfmmu_reprog_pgsz_arr(sfmmu_t *, uint8_t *);
2310 
2311 extern void     hat_kern_setup(void);
2312 extern int      hat_page_relocate(page_t **, page_t **, spgcnt_t *);
2313 extern int      sfmmu_get_ppvcolor(struct page *);
2314 extern int      sfmmu_get_addrvcolor(caddr_t);
2315 extern int      sfmmu_hat_lock_held(sfmmu_t *);
2316 extern int      sfmmu_alloc_ctx(sfmmu_t *, int, struct cpu *, int);
2317 
2318 /*
2319  * Functions exported to xhat_sfmmu.c
2320  */
2321 extern kmutex_t *sfmmu_mlist_enter(page_t *);
2322 extern void     sfmmu_mlist_exit(kmutex_t *);
2323 extern int      sfmmu_mlist_held(struct page *);
2324 extern struct hme_blk *sfmmu_hmetohblk(struct sf_hment *);
2325 
2326 /*
2327  * MMU-specific functions optionally imported from the CPU module
2328  */
2329 #pragma weak mmu_init_scd
2330 #pragma weak mmu_large_pages_disabled
2331 #pragma weak mmu_set_ctx_page_sizes
2332 #pragma weak mmu_check_page_sizes
2333 
2334 extern void mmu_init_scd(sf_scd_t *);
2335 extern uint_t mmu_large_pages_disabled(uint_t);
2336 extern void mmu_set_ctx_page_sizes(sfmmu_t *);
2337 extern void mmu_check_page_sizes(sfmmu_t *, uint64_t *);
2338 
2339 extern sfmmu_t          *ksfmmup;
2340 extern caddr_t          ktsb_base;
2341 extern uint64_t         ktsb_pbase;
2342 extern int              ktsb_sz;
2343 extern int              ktsb_szcode;
2344 extern caddr_t          ktsb4m_base;
2345 extern uint64_t         ktsb4m_pbase;
2346 extern int              ktsb4m_sz;
2347 extern int              ktsb4m_szcode;
2348 extern uint64_t         kpm_tsbbase;
2349 extern int              kpm_tsbsz;
2350 extern int              ktsb_phys;
2351 extern int              enable_bigktsb;
2352 #ifndef sun4v
2353 extern int              utsb_dtlb_ttenum;
2354 extern int              utsb4m_dtlb_ttenum;
2355 #endif /* sun4v */
2356 extern int              uhmehash_num;
2357 extern int              khmehash_num;
2358 extern struct hmehash_bucket *uhme_hash;
2359 extern struct hmehash_bucket *khme_hash;
2360 extern uint_t           hblk_alloc_dynamic;
2361 extern struct tsbmiss   tsbmiss_area[NCPU];
2362 extern struct kpmtsbm   kpmtsbm_area[NCPU];
2363 
2364 #ifndef sun4v
2365 extern int              dtlb_resv_ttenum;
2366 extern caddr_t          utsb_vabase;
2367 extern caddr_t          utsb4m_vabase;
2368 #endif /* sun4v */
2369 extern vmem_t           *kmem_tsb_default_arena[];
2370 extern int              tsb_lgrp_affinity;
2371 
2372 extern uint_t           disable_large_pages;
2373 extern uint_t           disable_ism_large_pages;
2374 extern uint_t           disable_auto_data_large_pages;
2375 extern uint_t           disable_auto_text_large_pages;
2376 
2377 /* kpm externals */
2378 extern pfn_t            sfmmu_kpm_vatopfn(caddr_t);
2379 extern void             sfmmu_kpm_patch_tlbm(void);
2380 extern void             sfmmu_kpm_patch_tsbm(void);
2381 extern void             sfmmu_patch_shctx(void);
2382 extern void             sfmmu_kpm_load_tsb(caddr_t, tte_t *, int);
2383 extern void             sfmmu_kpm_unload_tsb(caddr_t, int);
2384 extern void             sfmmu_kpm_tsbmtl(short *, uint_t *, int);
2385 extern int              sfmmu_kpm_stsbmtl(uchar_t *, uint_t *, int);
2386 extern caddr_t          kpm_vbase;
2387 extern size_t           kpm_size;
2388 extern struct memseg    *memseg_hash[];
2389 extern uint64_t         memseg_phash[];
2390 extern kpm_hlk_t        *kpmp_table;
2391 extern kpm_shlk_t       *kpmp_stable;
2392 extern uint_t           kpmp_table_sz;
2393 extern uint_t           kpmp_stable_sz;
2394 extern uchar_t          kpmp_shift;
2395 
2396 #define PP_ISMAPPED_KPM(pp)     ((pp)->p_kpmref > 0)
2397 
2398 #define IS_KPM_ALIAS_RANGE(vaddr)                                       \
2399         (((vaddr) - kpm_vbase) >> (uintptr_t)kpm_size_shift > 0)
2400 
2401 #endif /* !_ASM */
2402 
2403 /* sfmmu_kpm_tsbmtl flags */
2404 #define KPMTSBM_STOP            0
2405 #define KPMTSBM_START           1
2406 
2407 /*
2408  * For kpm_smallpages, the state about how a kpm page is mapped and whether
2409  * it is ready to go is indicated by the two 4-bit fields defined in the
2410  * kpm_spage structure as follows:
2411  * kp_mapped_flag bit[0:3] - the page is mapped cacheable or not
2412  * kp_mapped_flag bit[4:7] - the mapping is ready to go or not
2413  * If the bit KPM_MAPPED_GO is on, it indicates that the assembly tsb miss
2414  * handler can drop the mapping in regardless of the caching state of the
2415  * mapping. Otherwise, we will have C handler resolve the VAC conflict no
2416  * matter the page is currently mapped cacheable or non-cacheable.
2417  */
2418 #define KPM_MAPPEDS             0x1     /* small mapping valid, no conflict */
2419 #define KPM_MAPPEDSC            0x2     /* small mapping valid, conflict */
2420 #define KPM_MAPPED_GO           0x10    /* the mapping is ready to go */
2421 #define KPM_MAPPED_MASK         0xf
2422 
2423 /* Physical memseg address NULL marker */
2424 #define MSEG_NULLPTR_PA         -1
2425 
2426 /*
2427  * Memseg hash defines for kpm trap level tsbmiss handler.
2428  * Must be in sync w/ page.h .
2429  */
2430 #define SFMMU_MEM_HASH_SHIFT            0x9
2431 #define SFMMU_N_MEM_SLOTS               0x200
2432 #define SFMMU_MEM_HASH_ENTRY_SHIFT      3
2433 
2434 #ifndef _ASM
2435 #if (SFMMU_MEM_HASH_SHIFT != MEM_HASH_SHIFT)
2436 #error SFMMU_MEM_HASH_SHIFT != MEM_HASH_SHIFT
2437 #endif
2438 #if (SFMMU_N_MEM_SLOTS != N_MEM_SLOTS)
2439 #error SFMMU_N_MEM_SLOTS != N_MEM_SLOTS
2440 #endif
2441 
2442 /* Physical memseg address NULL marker */
2443 #define SFMMU_MEMSEG_NULLPTR_PA         -1
2444 
2445 /*
2446  * Check KCONTEXT to be zero, asm parts depend on that assumption.
2447  */
2448 #if (KCONTEXT != 0)
2449 #error KCONTEXT != 0
2450 #endif
2451 #endif  /* !_ASM */
2452 
2453 
2454 #endif /* _KERNEL */
2455 
2456 #ifndef _ASM
2457 /*
2458  * ctx, hmeblk, mlistlock and other stats for sfmmu
2459  */
2460 struct sfmmu_global_stat {
2461         int             sf_tsb_exceptions;      /* # of tsb exceptions */
2462         int             sf_tsb_raise_exception; /* # tsb exc. w/o TLB flush */
2463 
2464         int             sf_pagefaults;          /* # of pagefaults */
2465 
2466         int             sf_uhash_searches;      /* # of user hash searches */
2467         int             sf_uhash_links;         /* # of user hash links */
2468         int             sf_khash_searches;      /* # of kernel hash searches */
2469         int             sf_khash_links;         /* # of kernel hash links */
2470 
2471         int             sf_swapout;             /* # times hat swapped out */
2472 
2473         int             sf_tsb_alloc;           /* # TSB allocations */
2474         int             sf_tsb_allocfail;       /* # times TSB alloc fail */
2475         int             sf_tsb_sectsb_create;   /* # times second TSB added */
2476 
2477         int             sf_scd_1sttsb_alloc;    /* # SCD 1st TSB allocations */
2478         int             sf_scd_2ndtsb_alloc;    /* # SCD 2nd TSB allocations */
2479         int             sf_scd_1sttsb_allocfail; /* # SCD 1st TSB alloc fail */
2480         int             sf_scd_2ndtsb_allocfail; /* # SCD 2nd TSB alloc fail */
2481 
2482 
2483         int             sf_tteload8k;           /* calls to sfmmu_tteload */
2484         int             sf_tteload64k;          /* calls to sfmmu_tteload */
2485         int             sf_tteload512k;         /* calls to sfmmu_tteload */
2486         int             sf_tteload4m;           /* calls to sfmmu_tteload */
2487         int             sf_tteload32m;          /* calls to sfmmu_tteload */
2488         int             sf_tteload256m;         /* calls to sfmmu_tteload */
2489 
2490         int             sf_tsb_load8k;          /* # times loaded 8K tsbent */
2491         int             sf_tsb_load4m;          /* # times loaded 4M tsbent */
2492 
2493         int             sf_hblk_hit;            /* found hblk during tteload */
2494         int             sf_hblk8_ncreate;       /* static hblk8's created */
2495         int             sf_hblk8_nalloc;        /* static hblk8's allocated */
2496         int             sf_hblk1_ncreate;       /* static hblk1's created */
2497         int             sf_hblk1_nalloc;        /* static hblk1's allocated */
2498         int             sf_hblk_slab_cnt;       /* sfmmu8_cache slab creates */
2499         int             sf_hblk_reserve_cnt;    /* hblk_reserve usage */
2500         int             sf_hblk_recurse_cnt;    /* hblk_reserve owner reqs */
2501         int             sf_hblk_reserve_hit;    /* hblk_reserve hash hits */
2502         int             sf_get_free_success;    /* reserve list allocs */
2503         int             sf_get_free_throttle;   /* fails due to throttling */
2504         int             sf_get_free_fail;       /* fails due to empty list */
2505         int             sf_put_free_success;    /* reserve list frees */
2506         int             sf_put_free_fail;       /* fails due to full list */
2507 
2508         int             sf_pgcolor_conflict;    /* VAC conflict resolution */
2509         int             sf_uncache_conflict;    /* VAC conflict resolution */
2510         int             sf_unload_conflict;     /* VAC unload resolution */
2511         int             sf_ism_uncache;         /* VAC conflict resolution */
2512         int             sf_ism_recache;         /* VAC conflict resolution */
2513         int             sf_recache;             /* VAC conflict resolution */
2514 
2515         int             sf_steal_count;         /* # of hblks stolen */
2516 
2517         int             sf_pagesync;            /* # of pagesyncs */
2518         int             sf_clrwrt;              /* # of clear write perms */
2519         int             sf_pagesync_invalid;    /* pagesync with inv tte */
2520 
2521         int             sf_kernel_xcalls;       /* # of kernel cross calls */
2522         int             sf_user_xcalls;         /* # of user cross calls */
2523 
2524         int             sf_tsb_grow;            /* # of user tsb grows */
2525         int             sf_tsb_shrink;          /* # of user tsb shrinks */
2526         int             sf_tsb_resize_failures; /* # of user tsb resize */
2527         int             sf_tsb_reloc;           /* # of user tsb relocations */
2528 
2529         int             sf_user_vtop;           /* # of user vatopfn calls */
2530 
2531         int             sf_ctx_inv;             /* #times invalidate MMU ctx */
2532 
2533         int             sf_tlb_reprog_pgsz;     /* # times switch TLB pgsz */
2534 
2535         int             sf_region_remap_demap;  /* # times shme remap demap */
2536 
2537         int             sf_create_scd;          /* # times SCD is created */
2538         int             sf_join_scd;            /* # process joined scd */
2539         int             sf_leave_scd;           /* # process left scd */
2540         int             sf_destroy_scd;         /* # times SCD is destroyed */
2541 };
2542 
2543 struct sfmmu_tsbsize_stat {
2544         int             sf_tsbsz_8k;
2545         int             sf_tsbsz_16k;
2546         int             sf_tsbsz_32k;
2547         int             sf_tsbsz_64k;
2548         int             sf_tsbsz_128k;
2549         int             sf_tsbsz_256k;
2550         int             sf_tsbsz_512k;
2551         int             sf_tsbsz_1m;
2552         int             sf_tsbsz_2m;
2553         int             sf_tsbsz_4m;
2554         int             sf_tsbsz_8m;
2555         int             sf_tsbsz_16m;
2556         int             sf_tsbsz_32m;
2557         int             sf_tsbsz_64m;
2558         int             sf_tsbsz_128m;
2559         int             sf_tsbsz_256m;
2560 };
2561 
2562 struct sfmmu_percpu_stat {
2563         int     sf_itlb_misses;         /* # of itlb misses */
2564         int     sf_dtlb_misses;         /* # of dtlb misses */
2565         int     sf_utsb_misses;         /* # of user tsb misses */
2566         int     sf_ktsb_misses;         /* # of kernel tsb misses */
2567         int     sf_tsb_hits;            /* # of tsb hits */
2568         int     sf_umod_faults;         /* # of mod (prot viol) flts */
2569         int     sf_kmod_faults;         /* # of mod (prot viol) flts */
2570 };
2571 
2572 #define SFMMU_STAT(stat)                sfmmu_global_stat.stat++
2573 #define SFMMU_STAT_ADD(stat, amount)    sfmmu_global_stat.stat += (amount)
2574 #define SFMMU_STAT_SET(stat, count)     sfmmu_global_stat.stat = (count)
2575 
2576 #define SFMMU_MMU_STAT(stat)            {               \
2577         mmu_ctx_t *ctx = CPU->cpu_m.cpu_mmu_ctxp;    \
2578         if (ctx)                                        \
2579                 ctx->stat++;                         \
2580 }
2581 
2582 #endif /* !_ASM */
2583 
2584 #ifdef  __cplusplus
2585 }
2586 #endif
2587 
2588 #endif  /* _VM_HAT_SFMMU_H */